# For details see http://web.nexor.co.uk/mak/doc/robots/norobots.html
#
# keep them all out of /pub
#
User-agent: *
#wait 60 seconds between each contact attempt
Crawl-delay: 60
Disallow: /pub
Disallow: /cgi-bin
Disallow: /www
Disallow: /manual
#for surplus, disallow searching of archives, they can be searched on saf
Disallow: /mailarchives
#allow campus indexing of the static lists, external crawlers will be blocked by the server
Allow: /cgi-bin/caltech/cit_surplus.pl