# $Id: robots.txt,v 1.1 1997/10/27 09:00:00 fielding Exp $
# robots.txt for http://www.colby.edu/

# This version of robots.txt is moved to /web/prod/colby each business day
# morning at 8:00 AM. It restricts access to commonspot pages between 8AM
# and 10PM each buisness day, otherwise multiple robots (google, yahoo,
# ask, etc) indexing the site at the same time can swamp coldfusion.

#--- this section is for the Colby search engine:
User-agent: ColbySpider
Disallow: /cgi-bin/
Disallow: /info.tech/stats/

User-agent: Colby-gsa-crawler
Disallow: /cgi-bin/
Disallow: /info.tech/stats/

#--- this section is for all other search engines:
User-agent: *			# All other spiders should avoid 
Disallow: /commonspot/          # Commonspot installation
Disallow: /cgi-bin/		#     Script files
Disallow: /focus/		#     nonextant club
Disallow: /fusetalk/		# fusetalk forums
Disallow: /cpa/			#     unfinished club
Disallow: /security/		#     unfinished dept.
Disallow: /college/		#     internal
Disallow: /i/
Disallow: /info.tech/stats/
Disallow: /classof/2001.summer/
Disallow: /faculty/
Disallow: /colby.mag/issues/2004/spring/alumni/		#	Spring 2004 Magazine | Alumni Section
Disallow: /about_cs/		# Commonspot directories
Disallow: /academics_cs/
Disallow: /administration_cs/
Disallow: /admissions_cs/
Disallow: /alumni_parents_cs/
Disallow: /athletics_cs/
Disallow: /campaign/
Disallow: /campus_cs/
Disallow: /college_cs/
Disallow: /directory_cs/
Disallow: /echo/
Disallow: /faculty_staff_cs/
Disallow: /music/
Disallow: /news_events/
Disallow: /other/
Disallow: /prospective_cs/
Disallow: /search_cs/
Disallow: /sitemap/
Disallow: /slideshow/
Disallow: /students_cs/
Disallow: /test/