User-agent: Java/1.4.2_09
Disallow: /


User-agent: Nutch
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: Wget/1.8.2
Disallow: /

User-agent: FunWebProducts
Disallow: /

# Still Beta - Remove when production
User-agent: Pompos/1.3
Disallow: /

User-agent: PEERbot
Disallow: /


# below this line are offline browser agents - they can be configured to by pass

User-agent: Teleport+Pro
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: LinkWalker
Disallow: /

User-agent: GetBot
Disallow: /

User-agent: WebZIP/3.80
Disallow: /

User-agent: WebVulnScan/1.0
Disallow: /

User-agent: libwww-perl/5.803
Disallow: /

User-agent: Java/
Disallow: /

User-agent: Microsoft-WebDAV-MiniRedir/
Disallow: /

User-agent: BDFetch
Disallow: /

User-Agent: OmniExplorer_Bot
Disallow: /


User-Agent: FAST Enterprise Crawler
Disallow: /

# Known Rouge Bot - Email Harvesters - ignores robots.txt

User-agent: grub-client-2.3
Disallow: /




User-agent: *
Disallow: /news/
Disallow: /access/
Disallow: /acce ss/
Disallow: /errors/
Disallow: /newsaccess/
Disallow: /news access/
Disallow: /crawler/