User-agent: robot
Disallow: /

User-agent: crawl
Disallow: /

User-agent: spider
Disallow: /

User-agent: bot/
Disallow: /

User-agent: bot-
Disallow: /

User-agent: * 
Disallow: /internt/
Disallow: /ip/
Disallow: /css/
Disallow: /images/
Disallow: /cgi-bin/