User-Agent: *# Disallow all crawlers access to certain pages.
Disallow: /.htpasswds
Disallow: /access-logs
Disallow: /etc
Disallow: /mail
Disallow: /public_ftp
Disallow: /tmp
Allow: /