User-agent: * Disallow: /cgi-bin/ Disallow: /images/ Disallow: /inc/ Disallow: /modeldata/ Disallow: /testbed/ Disallow: /stormlist/ # Crawl-delay: 10 # google ignores and others may choke? So leave it out too #Request-rate: 1/5 # maximum rate is one page every 5 seconds (google doesn't understand) #Visit-time: 0000-0745 # only visit between 00:00 and 07:45 UTC (GMT ) (google doesn't understand) # $Revision: 1.2 $