# Policies for all agents
User-agent: *
Disallow: /distrib
Disallow: /index.php
# keep robots out of zip files and dynamic stuff
Disallow: /*.tgz
Disallow: /*.gz
Disallow: /*.zip
Disallow: /*.cgi
Disallow: /*.php

# keep some rude engines out of the way
User-agent: grub-client
Disallow: /
User-agent: OmniExplorer_Bot
Disallow: /
User-agent: BecomeBot
Disallow: /
User-agent: psbot
Disallow: /

# RM 20060505 reduce activity
User-agent: Slurp
Crawl-delay: 120
User-agent: msnbot
Crawl-delay: 3600