# robots.txt for http://www.slac.stanford.edu/ 
# See documentation in: http://info.webcrawler.com/mak/projects/robots/norobots.html
# more documentation:  http://www.robotstxt.org/
#
# because of aliases all babar stuff needs to be specified THREE times: /exp/babar, /BF, /BFROOT
#
# Revised by Bebo, 27 May 1999
# Revised by Bebo, 11 April 2000 - to disallow FrontPage paths 
# Revised by Bebo, 20 April 2000 - to disallow /archive/* 
# Revised by Bebo, 5 March 2002 - to disallow hepdata entries 
# Revised by Bebo 21 March 2002 - to disallow /BFROOT/detector 
# Revised by Bebo 26 March 2002 - to add SLAC User Agent 
# Revised by Bebo 29 January 2004 - to disallow server info pages 
# Revised by Ruth 20 April 2004 - to disallow /library/slaconly/cok per K.Sutton 
# Revised by Ruth/Travis 4 Jan 2005 sep. entry for google, indexing spires
# Revised by Bartelt 1 June 2005 - set delay for Yahoo Slurp
# Revised by Bartelt 25 Apr 2006 - to disallow /comp/net/wan-mon/tulip/TULIP/ per Les Cottrell
# Revised by Bartelt 19 July 2006 - disallow glast's cgi's:  /www-glast-{dev|pro}/cgi 
# Revised by Bartelt 13 Dec 2006 - add /%7E and /%7e to /~ ; also removed redundant & obsolete /~valeri/
# Revised by Bartelt 30 March 2007 - did above for the last (*) stanza.
#   also consolidated all the frontpage /_xxxx entries as just /_
# Revised by Bartelt 6 Nov 2008 - disallow /grp/SuperB
# Revised by Bartelt 7 Jan 2009 - disallow 
# /exp/glast/ground/obsolete-LATSoft-old/status/documentation  and 
# /exp/glast/ground/software/status/documentation (heavy load from robots)
# Revised by Bartelt 9 Jan 2009 - due to load, disallow
# /comp/net/iepm-bw.slac.stanford.edu and /comp/net/slac-netflow
# and also /BFROOT/www/Detector/Trigger/operations/Monitoring/
# and /exp/glast/ground/monitor/ACD/tools/mips
# Revised by Bartelt 25 Aug 2009 - add delay for Msnbot
#   This means duplicating all disallows again.
# general clean up; make sure babar dirs are excluded all three ways;
#  other corrections  Bartelt 2009 Oct 23
# 2010 Nov 15 Added /grp/ssrl/spear/epics/dblist/ to Disallow list,
#   by request relayed via Jimmy Pham
# block all access to BFROOT/simu/backgrounds (NFS server withhuge files)
# 2011 Apr 29 - Disallow all from comp/scicomp/media
# 2013 Mar 05 - Disallow all from exp/e165 RT 439150 (and move /exp lines together)
# 2013 Sep 11 - Disallow all from /accel/ilc/codes/ (Yandex+Google=trouble)
# 2013 Oct 10 - update Allow block; RT 479795
User-agent: Googlebot
#Google can look into Spires and techpubs docs 
#Allow following links to techpubs fulltext and abstracts
Allow: /cgi-wrap/getdoc
Allow: /cgi-wrap/pubpage
#This helps fulltext indexing of eConf
Allow: /spires/find/conf/wwweconfindex?
#
## Allow full google indexing of SPIRES
Allow: /find/slacarc
Allow: /Find/photoindex
Allow: /find/slacspeak
Allow: /find/books
Allow: /find/libresource
#####
#These two allow progressive search through spires by dateadded 
#one record at a time.  No internal spires links will be followed
### commented out in lieu of new lines above
#Allow: /spires/find/hep/wwwdateindex?
#Allow: /spires/find/hep/www?indexer=1
#Then the standard disallows
Disallow: /cgi-bin/
Disallow: /cgi-wrap/
Disallow: /cgi-hepdata/
Disallow: /spires/hepdata/
Disallow: /www-glast-dev/cgi/
Disallow: /www-glast-pro/cgi/
Disallow: /find/
Disallow: /Find/
Disallow: /FIND/
Disallow: /spires/find/
Disallow: /comp/unix/man/
Disallow: /archive/
Disallow: /grp/techpubs/testground/
Disallow: /grp/SuperB/
Disallow: /usr/local/scs/
Disallow: /_
Disallow: /owner/
Disallow: /contact/
Disallow: /~
Disallow: /%7E
Disallow: /%7e
Disallow: /exp/sld/figure/
Disallow: /exp/e158/internal/
Disallow: /exp/glast/ground/LATSoft/
Disallow: /exp/glast/ground/software/
Disallow: /exp/e165/
Disallow: /server-status
Disallow: /server-info
Disallow: /library/slaconly/cok/
Disallow: /accel/nlc/local/
Disallow: /slac/media-info/
Disallow: /comp/net/wan-mon/tulip/TULIP/
Disallow: /comp/scicomp/media/
Disallow: /exp/glast/ground/obsolete-LATSoft-old/status/documentation
Disallow: /exp/glast/ground/software/status/documentation
Disallow: /comp/net/iepm-bw.slac.stanford.edu
Disallow: /comp/net/slac-netflow
Disallow: /exp/glast/ground/monitor/ACD/tools/mips
Disallow: /grp/ssrl/spear/epics/dblist/
Disallow: /accel/ilc/codes/
##
Disallow: /exp/babar/doc/HyperNews/
Disallow: /BF/doc/HyperNews/
Disallow: /BFROOT/doc/HyperNews/
#
Disallow: /exp/babar/doc/Computing/www/QA
Disallow: /BF/doc/Computing/www/QA
Disallow: /BFROOT/doc/Computing/www/QA
#
Disallow: /exp/babar/doc/Computing/www/QC
Disallow: /BF/doc/Computing/www/QC
Disallow: /BFROOT/doc/Computing/www/QC
#
Disallow: /exp/babar/doc/BaBarNotes/
Disallow: /BF/doc/BaBarNotes/
Disallow: /BFROOT/doc/BaBarNotes/
#
Disallow: /exp/babar/detector/
Disallow: /BF/detector/
Disallow: /BFROOT/detector/
#
Disallow: /exp/babar/prod/log/allruns/
Disallow: /BF/prod/log/allruns/
Disallow: /BFROOT/prod/log/allruns/
#
Disallow: /exp/babar/doc/BaBarNotes/
Disallow: /BF/doc/BaBarNotes/
Disallow: /BFROOT/doc/BaBarNotes/
#
Disallow: /exp/babar/dist/packages/
Disallow: /BF/dist/packages/
Disallow: /BFROOT/dist/packages/
#
Disallow: /exp/babar/bin
Disallow: /BF/bin
Disallow: /BFROOT/bin
#
Disallow: /exp/babar/lib
Disallow: /BF/lib
Disallow: /BFROOT/lib
#
Disallow: /exp/babar/dist/
Disallow: /BF/dist/
Disallow: /BFROOT/dist/
#
Disallow: /exp/babar/man/
Disallow: /BF/man/
Disallow: /BFROOT/man/
#
Disallow: /exp/babar/www/Detector/Trigger/operations/Monitoring/
Disallow: /BF/www/Detector/Trigger/operations/Monitoring/
Disallow: /BFROOT/www/Detector/Trigger/operations/Monitoring/
#
Disallow: /exp/babar/simu/backgrounds/
Disallow: /BF/simu/backgrounds/
Disallow: /BFROOT/simu/backgrounds/
#
Disallow: /grp/ppa/epp/lsst/videos/
#
#
# having added this crawl delay, I think we need to 
# specify everything for Yahoo slurp
User-agent: Slurp
Crawl-delay: 60
Disallow: /cgi-bin/
Disallow: /cgi-wrap/
Disallow: /cgi-hepdata/
Disallow: /spires/hepdata/
Disallow: /www-glast-dev/cgi/
Disallow: /www-glast-pro/cgi/
Disallow: /find/
Disallow: /Find/
Disallow: /FIND/
Disallow: /spires/find/
Disallow: /comp/unix/man/
Disallow: /archive/
Disallow: /grp/techpubs/testground/
Disallow: /grp/SuperB/
Disallow: /usr/local/scs/
Disallow: /_
Disallow: /owner/
Disallow: /contact/
Disallow: /~
Disallow: /%7E
Disallow: /%7e
Disallow: /exp/sld/figure/
Disallow: /exp/e158/internal/
Disallow: /exp/glast/ground/LATSoft/
Disallow: /exp/glast/ground/software/
Disallow: /exp/e165/
Disallow: /exp/glast/ground/obsolete-LATSoft-old/status/documentation
Disallow: /exp/glast/ground/software/status/documentation
Disallow: /exp/glast/ground/monitor/ACD/tools/mips
Disallow: /server-status
Disallow: /server-info
Disallow: /library/slaconly/cok/
Disallow: /accel/nlc/local/
Disallow: /slac/media-info/
Disallow: /comp/net/wan-mon/tulip/TULIP/
Disallow: /comp/scicomp/media/
Disallow: /comp/net/iepm-bw.slac.stanford.edu
Disallow: /comp/net/slac-netflow
Disallow: /grp/ssrl/spear/epics/dblist/
Disallow: /accel/ilc/codes/
##
Disallow: /exp/babar/doc/HyperNews/
Disallow: /BF/doc/HyperNews/
Disallow: /BFROOT/doc/HyperNews/
#
Disallow: /exp/babar/doc/Computing/www/QA
Disallow: /BF/doc/Computing/www/QA
Disallow: /BFROOT/doc/Computing/www/QA
#
Disallow: /exp/babar/doc/Computing/www/QC
Disallow: /BF/doc/Computing/www/QC
Disallow: /BFROOT/doc/Computing/www/QC
#
Disallow: /exp/babar/doc/BaBarNotes/
Disallow: /BF/doc/BaBarNotes/
Disallow: /BFROOT/doc/BaBarNotes/
#
Disallow: /exp/babar/detector/
Disallow: /BF/detector/
Disallow: /BFROOT/detector/
#
Disallow: /exp/babar/prod/log/allruns/
Disallow: /BF/prod/log/allruns/
Disallow: /BFROOT/prod/log/allruns/
#
Disallow: /exp/babar/doc/BaBarNotes/
Disallow: /BF/doc/BaBarNotes/
Disallow: /BFROOT/doc/BaBarNotes/
#
Disallow: /exp/babar/dist/packages/
Disallow: /BF/dist/packages/
Disallow: /BFROOT/dist/packages/
#
Disallow: /exp/babar/bin
Disallow: /BF/bin
Disallow: /BFROOT/bin
#
Disallow: /exp/babar/lib
Disallow: /BF/lib
Disallow: /BFROOT/lib
#
Disallow: /exp/babar/dist/
Disallow: /BF/dist/
Disallow: /BFROOT/dist/
#
Disallow: /exp/babar/man/
Disallow: /BF/man/
Disallow: /BFROOT/man/
#
Disallow: /exp/babar/www/Detector/Trigger/operations/Monitoring/
Disallow: /BF/www/Detector/Trigger/operations/Monitoring/
Disallow: /BFROOT/www/Detector/Trigger/operations/Monitoring/
#
Disallow: /exp/babar/simu/backgrounds/
Disallow: /BF/simu/backgrounds/
Disallow: /BFROOT/simu/backgrounds/
#
Disallow: /grp/ppa/epp/lsst/videos/
#
#
# msnbot is a pain during fileserver outages.  
User-agent: msnbot
Crawl-delay: 60
Disallow: /cgi-bin/
Disallow: /cgi-wrap/
Disallow: /cgi-hepdata/
Disallow: /spires/hepdata/
Disallow: /www-glast-dev/cgi/
Disallow: /www-glast-pro/cgi/
Disallow: /find/
Disallow: /Find/
Disallow: /FIND/
Disallow: /spires/find/
Disallow: /comp/unix/man/
Disallow: /archive/
Disallow: /grp/techpubs/testground/
Disallow: /grp/SuperB/
Disallow: /usr/local/scs/
Disallow: /_
Disallow: /owner/
Disallow: /contact/
Disallow: /~
Disallow: /%7E
Disallow: /%7e
Disallow: /exp/sld/figure/
Disallow: /exp/e158/internal/
Disallow: /exp/glast/ground/LATSoft/
Disallow: /exp/glast/ground/software/
Disallow: /exp/e165/
Disallow: /exp/glast/ground/obsolete-LATSoft-old/status/documentation
Disallow: /exp/glast/ground/software/status/documentation
Disallow: /exp/glast/ground/monitor/ACD/tools/mips
Disallow: /server-status
Disallow: /server-info
Disallow: /library/slaconly/cok/
Disallow: /accel/nlc/local/
Disallow: /slac/media-info/
Disallow: /comp/net/wan-mon/tulip/TULIP/
Disallow: /comp/scicomp/media/
Disallow: /comp/net/iepm-bw.slac.stanford.edu
Disallow: /comp/net/slac-netflow
Disallow: /grp/ssrl/spear/epics/dblist/
Disallow: /accel/ilc/codes/
##
Disallow: /exp/babar/doc/HyperNews/
Disallow: /BF/doc/HyperNews/
Disallow: /BFROOT/doc/HyperNews/
#
Disallow: /exp/babar/doc/Computing/www/QA
Disallow: /BF/doc/Computing/www/QA
Disallow: /BFROOT/doc/Computing/www/QA
#
Disallow: /exp/babar/doc/Computing/www/QC
Disallow: /BF/doc/Computing/www/QC
Disallow: /BFROOT/doc/Computing/www/QC
#
Disallow: /exp/babar/doc/BaBarNotes/
Disallow: /BF/doc/BaBarNotes/
Disallow: /BFROOT/doc/BaBarNotes/
#
Disallow: /exp/babar/detector/
Disallow: /BF/detector/
Disallow: /BFROOT/detector/
#
Disallow: /exp/babar/prod/log/allruns/
Disallow: /BF/prod/log/allruns/
Disallow: /BFROOT/prod/log/allruns/
#
Disallow: /exp/babar/doc/BaBarNotes/
Disallow: /BF/doc/BaBarNotes/
Disallow: /BFROOT/doc/BaBarNotes/
#
Disallow: /exp/babar/dist/packages/
Disallow: /BF/dist/packages/
Disallow: /BFROOT/dist/packages/
#
Disallow: /exp/babar/bin
Disallow: /BF/bin
Disallow: /BFROOT/bin
#
Disallow: /exp/babar/lib
Disallow: /BF/lib
Disallow: /BFROOT/lib
#
Disallow: /exp/babar/dist/
Disallow: /BF/dist/
Disallow: /BFROOT/dist/
#
Disallow: /exp/babar/man/
Disallow: /BF/man/
Disallow: /BFROOT/man/
#
Disallow: /exp/babar/www/Detector/Trigger/operations/Monitoring/
Disallow: /BF/www/Detector/Trigger/operations/Monitoring/
Disallow: /BFROOT/www/Detector/Trigger/operations/Monitoring/
#
Disallow: /exp/babar/simu/backgrounds/
Disallow: /BF/simu/backgrounds/
Disallow: /BFROOT/simu/backgrounds/
#
Disallow: /grp/ppa/epp/lsst/videos/
#
#
User-agent: *
Disallow: /cgi-bin/
Disallow: /cgi-wrap/
Disallow: /cgi-hepdata/
Disallow: /spires/hepdata/
Disallow: /www-glast-dev/cgi/
Disallow: /www-glast-pro/cgi/
Disallow: /find/
Disallow: /Find/
Disallow: /FIND/
Disallow: /spires/find/
Disallow: /comp/unix/man/
Disallow: /archive/
Disallow: /grp/techpubs/testground/
Disallow: /grp/SuperB/
Disallow: /usr/local/scs/
Disallow: /_
Disallow: /owner/
Disallow: /contact/
Disallow: /~
Disallow: /%7E
Disallow: /%7e
Disallow: /exp/sld/figure/
Disallow: /exp/e158/internal/
Disallow: /exp/glast/ground/LATSoft/
Disallow: /exp/glast/ground/software/
Disallow: /exp/e165/
Disallow: /exp/glast/ground/obsolete-LATSoft-old/status/documentation
Disallow: /exp/glast/ground/software/status/documentation
Disallow: /exp/glast/ground/monitor/ACD/tools/mips
Disallow: /server-status
Disallow: /server-info
Disallow: /library/slaconly/cok/
Disallow: /accel/nlc/local/
Disallow: /slac/media-info/
Disallow: /comp/net/wan-mon/tulip/TULIP/
Disallow: /comp/scicomp/media/
Disallow: /comp/net/iepm-bw.slac.stanford.edu
Disallow: /comp/net/slac-netflow
Disallow: /grp/ssrl/spear/epics/dblist/
Disallow: /accel/ilc/codes/
##
Disallow: /exp/babar/doc/HyperNews/
Disallow: /BF/doc/HyperNews/
Disallow: /BFROOT/doc/HyperNews/
#
Disallow: /exp/babar/doc/Computing/www/QA
Disallow: /BF/doc/Computing/www/QA
Disallow: /BFROOT/doc/Computing/www/QA
#
Disallow: /exp/babar/doc/Computing/www/QC
Disallow: /BF/doc/Computing/www/QC
Disallow: /BFROOT/doc/Computing/www/QC
#
Disallow: /exp/babar/doc/BaBarNotes/
Disallow: /BF/doc/BaBarNotes/
Disallow: /BFROOT/doc/BaBarNotes/
#
Disallow: /exp/babar/detector/
Disallow: /BF/detector/
Disallow: /BFROOT/detector/
#
Disallow: /exp/babar/prod/log/allruns/
Disallow: /BF/prod/log/allruns/
Disallow: /BFROOT/prod/log/allruns/
#
Disallow: /exp/babar/doc/BaBarNotes/
Disallow: /BF/doc/BaBarNotes/
Disallow: /BFROOT/doc/BaBarNotes/
#
Disallow: /exp/babar/dist/packages/
Disallow: /BF/dist/packages/
Disallow: /BFROOT/dist/packages/
#
Disallow: /exp/babar/bin
Disallow: /BF/bin
Disallow: /BFROOT/bin
#
Disallow: /exp/babar/lib
Disallow: /BF/lib
Disallow: /BFROOT/lib
#
Disallow: /exp/babar/dist/
Disallow: /BF/dist/
Disallow: /BFROOT/dist/
#
Disallow: /exp/babar/man/
Disallow: /BF/man/
Disallow: /BFROOT/man/
#
Disallow: /exp/babar/www/Detector/Trigger/operations/Monitoring/
Disallow: /BF/www/Detector/Trigger/operations/Monitoring/
Disallow: /BFROOT/www/Detector/Trigger/operations/Monitoring/
#
Disallow: /exp/babar/simu/backgrounds/
Disallow: /BF/simu/backgrounds/
Disallow: /BFROOT/simu/backgrounds/
#
Disallow: /grp/ppa/epp/lsst/videos/