1
0
Fork 0
mayvaneday/robots.txt

56 lines
725 B
Text
Executable file

User-agent: webproxy-mozz
Disallow:
User-agent: serpstatbot
Disallow: /
User-agent: ltx71
Disallow: /
User-agent: oBot
Disallow: /
User-agent: DuckDuckBot
Disallow:
User-agent: ia_archiver
Disallow: /
User-agent: ltx71 - (http://ltx71.com/)
Disallow: /
User-agent: BLEXBot
Disallow: /
User-agent: Twingly Recon
Disallow: /
User-agent: www.deadlinkchecker.com
Disallow: /rollcall.html
User-agent: Googlebot
Disallow: /identity/
Disallow: /webring.html
Disallow: /blog/
User-agent: Googlebot-Image
Disallow: /
User-agent: PageThing
Disallow: /
User-agent: adsbot
Disallow: /
User-agent: SurdotlyBot
Disallow: /
User-agent: DataForSeoBot
Disallow: /
User-agent: SpiderLing
Disallow: /
User-agent: *
Disallow: /