User-agent: *   # directed to all spiders

Disallow: /demo/

Disallow: /cgi-bin/

Disallow: /stats/

Disallow: /logs/

Disallow: /pdf/