# robots.txt

User-agent: Slurp
User-agent: Googlebot
User-agent: msnbot
Crawl-delay: 10

# Directories
Disallow: /includes/*
Disallow: /misc/*
Disallow: /modules/*
Disallow: /profiles/*
Disallow: /scripts/*
Disallow: /themes/*

# Files
Disallow: /CHANGELOG.txt
Disallow: /cron.php
Disallow: /INSTALL.mysql.txt
Disallow: /INSTALL.pgsql.txt
Disallow: /INSTALL.sqlite.txt
Disallow: /install.php
Disallow: /INSTALL.txt
Disallow: /LICENSE.txt
Disallow: /MAINTAINERS.txt
Disallow: /update.php
Disallow: /UPGRADE.txt
Disallow: /xmlrpc.php

# Paths (clean URLs)
Disallow: /admin/*
Disallow: /comment/reply/*
Disallow: /filter/tips/*
Disallow: /user/logout/*
Disallow: /node/add/*
Disallow: /search/*
Disallow: /user/register/*
Disallow: /user/password/*
Disallow: /user/login/*
Disallow: /sites/all/libraries/*
Disallow: /sites/all/modules/*
Disallow: /sites/all/themes/*

# Paths (no clean URLs)
Disallow: /?q=admin/
Disallow: /?q=comment/reply/
Disallow: /?q=filter/tips/
Disallow: /?q=user/logout/
Disallow: /?q=node/add/
Disallow: /?q=search/
Disallow: /?q=user/password/
Disallow: /?q=user/register/
Disallow: /?q=user/login/
Disallow: /?q=search/

# Extras as on drupal.org
# no access for table sorting paths or any paths that have parameters
Disallow: /*?sort*
Disallow: /*&sort*
Disallow: /*?solrsort*
Disallow: /*&solrsort*

# no access to profile listings, they are costly to generate and don't provide useful information
Disallow: /profile
Disallow: /profile/*
Disallow: /?q=profile
Disallow: /?q=profile/*

# Disallow bogus aggregator pages
Disallow: /aggregator/*

User-agent: *
Disallow: /