User-agent: *
Allow: /public/
Disallow: /cgi-bin/

User-agent: webcrawler
Allow: /public/
Disallow: /cgi-bin/

User-agent: Googlebot
Allow: /public/
Disallow: /cgi-bin/

User-agent: Slurp
Allow: /public/
Disallow: /cgi-bin/

User-agent: msnbot
Allow: /public/
Disallow: /cgi-bin/

User-agent: webcrawler
Allow: /
Disallow: /cgi-bin/

User-agent: Googlebot
Allow: /
Disallow: /cgi-bin/

User-agent: Slurp
Allow: /
Disallow: /cgi-bin/

User-agent: msnbot
Allow: /
Disallow: /cgi-bin/

User-agent:  ArchitextSpider
allow: /
Disallow: /cgi-bin/

User-agent:  ia_archiver
allow: /
Disallow: /cgi-bin/

User-agent:  Scooter
allow: /
Disallow: /cgi-bin/

User-agent:  Sidewinder
allow: /
Disallow: /cgi-bin/

User-agent:  Slurp
allow: /
Disallow: /cgi-bin/

User-agent:  Gulliver
allow: /
Disallow: /cgi-bin/

User-agent:  fast
allow: /
Disallow: /cgi-bin/

User-agent: *
Allow: /
Disallow: /cgi-bin/