# robots.txt for web site User-agent: * Disallow: /cgi-bin/ Disallow: /weather/ Disallow: /*.*/ #Other examples of things to put in the file #Disallow: /foo.html #Disallow: /tmp/