robot-id: jcrawler
robot-name: JCrawler
robot-cover-url: http://www.nihongo.org/jcrawler/
robot-details-url:
robot-owner-name: Benjamin Franz
robot-owner-url: http://www.nihongo.org/snowhare/
robot-owner-email: snowhare@netimages.com
robot-status: active
robot-purpose: indexing
robot-type: standalone
robot-platform: unix
robot-availability: none
robot-exclusion: yes
robot-exclusion-useragent: jcrawler
robot-noindex: yes
robot-host: db.netimages.com
robot-from: yes
robot-useragent: JCrawler/0.2
robot-language: perl5
robot-description: JCrawler is currently used to build the Vietnam topic
                   specific WWW index for VietGATE
                   <URL:http://www.vietgate.net/>. It schedules visits
                   randomly, but will not visit a site more than once
                   every two minutes. It uses a subject matter relevance
                   pruning algorithm to determine what pages to crawl
                   and index and will not generally index pages with
                   no Vietnam related content. Uses Unicode internally,
                   and detects and converts several different Vietnamese
                   character encodings.
robot-history:
robot-environment: service
modified-date: Wed, 08 Oct 1997 00:09:52 GMT
modified-by: Benjamin Franz