User-Agent: *
Disallow: /admin/errors/ # skip the errors directory
Disallow: /admin/new-archives/ # skip news archive directory
Disallow: /admin/intro/history/webstats/ # not reliable
Disallow: /admin/intro/history/marxists/traffic # not reliable
Disallow: /espanol/admin/ # skip spanish admin directory
Disallow: /espanol/trotsky/ceip/ # it's not linked to, only goofs up link check
Disallow: /history/canada/socialisthistory/ # mirror
Disallow: /korean/trotsky/ # bad links
Disallow: /cgi-bin/ # scripts
Disallow: /webstats/ # not reliable
Crawl-delay: 5
User-Agent: W3C-checklink
Disallow:
User-Agent: LinkChecker
Disallow: