# Tells Scanning Robots Where They Are And Are Not Welcome
# User-agent:         can also specify by name "*" is for everyone
# Disallow:           if this matches first part of requested path, forget it
# Tells Scanning Robots Where They Are And Are Not Welcome
#
# Webweaver.net Disallows
User-agent: * 
Disallow: /cgi/
Disallow: /cgi-bin/
Disallow: /private
Disallow: /agent
Disallow: /badrobots/
Disallow: /private/
Disallow: /pass/
Disallow: /test/
Disallow: /ftp/
Disallow: /trace/
Disallow: /browsers/
Disallow: /scripts/
Disallow: /develop
Disallow: /develop/
Disallow: /digits
Disallow: /digits/
Disallow: /RCS/

User-agent: ConveraMultiMediaCrawler*
Disallow: /


## <META NAME="ROBOTS" CONTENT="NOINDEX, NOFOLLOW">

# User-agent 
#     The value of this field is the name of the robot the record is describing access policy for. 
#
#     If more than one User-agent field is present the record describes an identical access policy for more than one robot.
#     At least one field needs to be present per record.
#
#     The robot should be liberal in interpreting this field. A case insensitive substring match of the name without version
#     information is recommended.
#
#     If the value is '*', the record describes the default access policy for any robot that has not not matched any of the other
#     records. It is not allowed to have two such records in the "/robots.txt" file.
#
#Disallow 
#     The value of this field specifies a partial URL that is not to be visited. This can be a full path, or a partial path; any
#     URL that starts with this value will not be retrieved. For example, Disallow: /help disallows both /help.html
#     and /help/index.html, whereas Disallow: /help/ would disallow /help/index.html but allow
#     /help.html. 
#
#     Any empty value, indicates that all URLs can be retrieved. At least one Disallow field needs to be present in a record.
#
# The presence of an empty "/robots.txt" file has no explicit associated semantics, it will be treated as if it was not present,
# i.e. all robots will consider themselves welcome. 
#
#
# Examples
#
# The following example "/robots.txt" file specifies that no robots should visit any URL starting with "/cyberworld/map/"
# or "/tmp/: 
#
#
# robots.txt for http://www.site.com/
#
#User-agent: *
# Disallow: /cyberworld/map/ # This is an infinite virtual URL space
# Disallow: /tmp/ # these will soon disappear
#
#
# This example "/robots.txt" file specifies that no robots should visit any URL starting with "/cyberworld/map/", except
# the robot called "cybermapper": 
#
# robots.txt for http://www.site.com/
#
# User-agent: *
# Disallow: /cyberworld/map/ # This is an infinite virtual URL space
#
# Cybermapper knows where to go.
# User-agent: cybermapper
# Disallow:
#
# This example indicates that no robots should visit this site further: 
# go away
# User-agent: *
# Disallow: /