#robots.txt file for http://test.fda.gov

#Allow Google-Test
User-agent: Google-Test
Allow: /
Disallow: /Management/   # don't crawl healthcheck

#Allow GoogleGSA
User-agent: gsa-crawler
Allow: /
Disallow: /Management/   # don't crawl healthcheck

#Allow GooglePOC
User-agent: gsa-crawler
Allow: /
Disallow: /Management/   # don't crawl healthcheck

#Allow GooglePOC – new http headers
User-agent: gsa-crawler-GooglePOC
Allow: /
Disallow: /Management/   # don't crawl healthcheck

#Disallow all other crawlers (temporarily)
User-agent: * 
Disallow: /


Hit-rate: 30 # wait 30 seconds before starting a new URL request default=30
#Visiting-hours: 23:00EDT-05:00EDT #index this site between 11PM - 5AM EDT
Concurrent-hits: 2 # limit concurrent active URLS to 2 for each index server