#########################################################
#		 -- NETPOINTERS INTL. --		#
#		http://www.netpointers.com		#
#							#
#		    Vestergade 18 B			#
#		  DK-1456 Copenhagen K			#
#							#
# Robot exclusion file. Sets recommended access rights	#
# for known and unknown robots.				#
#							#
# Created: 2001-07-27					#
# Author: Rasmus T. Mohr (rmo@netpointers.com)		#
# Modified: 						#
#							#
# (c) Netpointers Intl. 2001				#
#########################################################

# -- Known bots -- #

# AltaVista -> av
User-agent: Scooter
Disallow: 
Disallow: /search/searchhb
Disallow: /search/searchex
Disallow: /search/searchgo
Disallow: /search/searchly
Disallow: /search/searchfs
Disallow: /search/searchdh

# Inktomi -> hb
User-agent: Slurp
Disallow:
Disallow: /search/searchav
Disallow: /search/searchex
Disallow: /search/searchgo
Disallow: /search/searchly
Disallow: /search/searchfs
Disallow: /search/searchdh

# Excite -> ex
User-agent: ArchitextSpider
Disallow:
Disallow: /search/searchav
Disallow: /search/searchhb
Disallow: /search/searchgo
Disallow: /search/searchly
Disallow: /search/searchfs
Disallow: /search/searchdh

# Google -> go
User-agent: Googlebot
Disallow:
Disallow: /search/searchav
Disallow: /search/searchhb
Disallow: /search/searchex
Disallow: /search/searchly
Disallow: /search/searchfs
Disallow: /search/searchdh

# Lycos -> ly
User-agent: Lycos
Disallow:
Disallow: /search/searchav
Disallow: /search/searchhb
Disallow: /search/searchex
Disallow: /search/searchgo
Disallow: /search/searchfs
Disallow: /search/searchdh

# Fast -> fs
User-agent: Fast
Disallow:
Disallow: /search/searchav
Disallow: /search/searchhb
Disallow: /search/searchex
Disallow: /search/searchgo
Disallow: /search/searchly
Disallow: /search/searchdh

# DirectHit
User-agent: Grabber->dh
Disallow:
Disallow: /search/searchav
Disallow: /search/searchhb
Disallow: /search/searchex
Disallow: /search/searchgo
Disallow: /search/searchly
Disallow: /search/searchfs

# -- Unknown bots (i.e. all the others) -- #

# Uses /search/searchav

User-agent: *
Disallow:
Disallow: /search/searchhb
Disallow: /search/searchex
Disallow: /search/searchgo
Disallow: /search/searchly
Disallow: /search/searchfs
Disallow: /search/searchdh


# -- Email collectors, density analysers --#

User-agent: EmailCollector
Disallow: /

User-agent: EmailSiphon
Disallow: /

User-agent: EmailWolf
Disallow: /

User-agent: ExtractorPro
Disallow: /

User-agent: NetMechanic
Disallow: /

User-agent: WWW-Collector-E
Disallow: /

User-agent: ExtractorPro
Disallow: /

User-agent: Keyword Density/0.9
Disallow: /