# Google crawler (see http://www.google.com/support/webmasters/)
# allow to crawl everything apart from the cached page
User-Agent: Googlebot
Disallow: /html-resource.html
Disallow: /xml-resource.html
Disallow: /static
Disallow: /a4j

# Google AdSense crawler (see https://www.google.com/adsense/support/bin/answer.py?answer=10532)
# allow to crawl everything apart from the cached page
User-agent: Mediapartners-Google*
Disallow: /html-resource.html
Disallow: /xml-resource.html
Disallow: /static
Disallow: /a4j

# Yahoo crawler (see http://help.yahoo.com/l/us/yahoo/search/webcrawler/)
# allow to crawl everything apart from the cached page
User-agent: Slurp
Disallow: /html-resource.html
Disallow: /xml-resource.html
Disallow: /static
Disallow: /a4j
Crawl-delay: 120

# Ask crawler (see http://about.ask.com/en/docs/about/webmasters.shtml)
# allow to crawl everything apart from the cached page
User-agent: Teoma
Disallow: /html-resource.html
Disallow: /xml-resource.html
Disallow: /static
Disallow: /a4j
Crawl-delay: 120


# MSN Live crawler (see http://help.live.com/help.aspx?mkt=en-us&project=wl_webmasters)
# allow to crawl everything apart from the cached page
User-agent: msnbot
Disallow: /html-resource.html
Disallow: /xml-resource.html
Disallow: /static
Disallow: /a4j
Crawl-delay: 120

# Entirely disallow all the rest
User-agent: *
Disallow: /