User-agent: *
Disallow: /admin/ # skip cd images directory
Disallow: /cd/ # skip cd images directory
Disallow: /css/ # skip root css directory
Disallow: /webstats/ # not reliable
Disallow: /espanol/admin/ # skip spanish admin directory
Disallow: /espanol/trotsky/ceip/ # it's not linked to, only goofs up link check
Disallow: /history/canada/socialisthistory/ # mirror
Disallow: /korean/trotsky/ # bad links
Disallow: /cgi-bin/ # scripts
Crawl-delay: 5