# User-agent: bingbot
# going to lift it's crawl delay restrictions now that they moved to HTTP/1.1
# Crawl-delay: 10
# /sbin/route add -net 65.52.0.0 netmask 255.252.0.0  reject  // block all of Microsoft if it doesn't adhere

# Does not use HTTP/1.1 with compression
# These are the brain dead spiders from major search engines
#
# User-agent: Yahoo! Slurp
# Disallow: /
# Allow: if ($_SERVER["SERVER_PROTOCOL"] === 'HTTP/1.1' || $relevancy > 0)
#
# Learn to use a HTTP protocol standard that's more than a decade old
# http://www.w3.org/Protocols/rfc2068/rfc2068.txt

User-agent: Mediapartners-Google
Allow: /conversations/
Allow: /account/

User-agent: *
Disallow: /account/
Disallow: /conversations/
Disallow: /find-new/
Disallow: /login
Disallow: /posts/*/ip$
Disallow: /posts/*/tweet
Disallow: /members/*/trophies
Disallow: /search/

User-agent: Baiduspider
Disallow: /

User-Agent: AhrefsBot
Crawl-Delay: 5