# robots.txt for http://math.nist.gov/ # temporarily stop googler.*.nist.gov from crawling every 10 min until they update s/w #User-agent: gsa-crawler #Disallow: / User-agent: * Disallow: /cgi-bin/ Disallow: /acm/ Disallow: /MatrixMarket.old/ Disallow: /DFTdata.old/ Disallow: /gams/serve.cgi/ Disallow: /gams/search.cgi/ Disallow: /nosuchdirectory/