User-agent: *

# these generally need logins, javascript or at least humans to do
# something.  It's pointless for the crawlers to crawl them...
Disallow: /login
Disallow: /register?
Disallow: /contact?
Disallow: /update_scene
Disallow: /feedback
Disallow: /forgot_password
Disallow: /logout
Disallow: /post_comment
Disallow: /m/
Disallow: /admin/
Disallow: /biz/
Disallow: /fb/
Disallow: /myspace/
Disallow: /api/
Disallow: /ajax_get_popular_scenes

User-agent: Fasterfox
Disallow: /

User-agent: Nutch
Disallow: /

User-agent: spock
Disallow: /

User-agent: OmniExplorer_Bot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: TurnitinBot
Disallow: /

User-agent: BecomeBot 
Disallow: /

User-agent: genieBot
Disallow: /

User-agent: dotbot 
Disallow: /


Sitemap: http://www.thescene.com/sitemap/sitemap-urls-index.xml.gz