User-agent: * # All spiders should avoid Disallow: /cgi-bin/ # Image maps Disallow: /logs/ # Huge server statistic logs Disallow: /newlogs/ # Huge server statistic logs Disallow: /images/ # Images Disallow: /ftp/ # Huge server download Disallow: / # Avoid directories