# Disallow some requests from Slurp (Inktomi) so they dont see cross-links
User-agent: Slurp
Crawl-Delay: 5

User-agent: msnbot
Crawl-Delay: 5

# Allow requests from other robots, except to the disallowed dirs and files
User-agent: *
Disallow: /content