# Define access-restrictions for robots/spiders
# http://www.robotstxt.org/wc/norobots.html



# By default we allow robots to access all areas of our site 
# already accessible to anonymous users

User-agent: *
Disallow: /*sendto_form$
Disallow: /*folder_factories$
Disallow: /*search$
Disallow: /*search_form$
Disallow: /*portal_javascripts
Disallow: /*portal_css



# Add Googlebot-specific syntax extension to exclude forms 
# that are repeated for each piece of content in the site 
# the wildcard is only supported by Googlebot
# http://www.google.com/support/webmasters/bin/answer.py?answer=40367&ctx=sibling

User-Agent: Googlebot
Disallow: /*sendto_form$
Disallow: /*folder_factories$
Disallow: /*search$
Disallow: /*search_form$
Disallow: /*portal_javascripts
Disallow: /*portal_css



# Add PennStateSpider-specific syntax extension to exclude forms 
# that are repeated for each piece of content in the site 
# the wildcard is only supported by Googlebot
# http://www.google.com/support/webmasters/bin/answer.py?answer=40367&ctx=sibling

User-Agent: PennStateSpider
Disallow: /*sendto_form$
Disallow: /*folder_factories$
Disallow: /*search$
Disallow: /*search_form$
Disallow: /*portal_javascripts
Disallow: /*portal_css



# Add Yahoo Slurp! -specific syntax extension to exclude forms 
# that are repeated for each piece of content in the site 
# http://help.yahoo.com/l/us/yahoo/search/webcrawler/

User-Agent: slurp
Disallow: /*sendto_form$
Disallow: /*folder_factories$
Disallow: /*search$
Disallow: /*search_form$
Disallow: /*portal_javascripts
Disallow: /*portal_css



# Bad bot graveyard. These bots are placed here because of various reasons -
# overzealous crawling, executing JS (skewing Analytics), or just for
# principle of not being 'nice'.

User-agent: Yeti/1.0
Disallow: /

User-agent: Yeti/0.01
Disallow: /

User-agent: Yeti
Disallow: /

User-agent: NaverBot
Disallow: /