User-agent: *    # applies to all robots


Disallow: /cgi-bin
Disallow: /inc
Disallow: /mysql
Disallow: /newsletter
Disallow: /php
Disallow: /pics
Disallow: /pictures
Disallow: /registration
Disallow: /test
Disallow: /webstats

catalog
User-Agent: sitecheck.internetseer.com
Disallow: /

User-agent: Teleport Pro/1.28
Disallow: *

User-agent: Teleport Pro/1.29
Disallow: *

User-agent: semanticdiscovery
Disallow: *