#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/wc/robots.html
#
# For syntax checking, see:
# http://www.sxw.org.uk/computing/robots/check.html

User-agent: *
Crawl-delay: 10
# Directories
Disallow: /dyn/
Disallow: /objects/
Disallow: /includes/
Disallow: /misc/
Disallow: /modules/
Disallow: /profiles/
Disallow: /scripts/
Disallow: /themes/
# Files
Disallow: /CHANGELOG.txt
Disallow: /cron.php
Disallow: /INSTALL.mysql.txt
Disallow: /INSTALL.pgsql.txt
Disallow: /install.php
Disallow: /INSTALL.txt
Disallow: /LICENSE.txt
Disallow: /MAINTAINERS.txt
Disallow: /update.php
Disallow: /UPGRADE.txt
Disallow: /xmlrpc.php
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /logout/
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register/
Disallow: /user/password/
Disallow: /user/login/
# Paths (no clean URLs)
Disallow: /?q=admin/
Disallow: /?q=comment/reply/
Disallow: /?q=logout/
Disallow: /?q=node/add/
Disallow: /?q=search/
Disallow: /?q=user/password/
Disallow: /?q=user/register/
Disallow: /?q=user/login/


User-Agent: bingbot
Crawl-Delay: 4
Disallow: /install/
Disallow: /customavatars/
Disallow: /admincp/
Disallow: /clientscript/
Disallow: /cpstyles/
Disallow: /customprofilepics/
Disallow: /modcp/
Disallow: /ajax.php
Disallow: /attachment.php
Disallow: /calendar.php
Disallow: /cron.php
Disallow: /editpost.php
Disallow: /global.php
Disallow: /image.php
Disallow: /inlinemod.php
Disallow: /joinrequests.php
Disallow: /login.php
Disallow: /members/
Disallow: /members.php
Disallow: /misc.php
Disallow: /moderator.php
Disallow: /newattachment.php
Disallow: /newreply.php
Disallow: /newthread.php
Disallow: /online.php
Disallow: /postings.php
Disallow: /printthread.php
Disallow: /private.php
Disallow: /register.php
Disallow: /report.php
Disallow: /reputation.php
Disallow: /search.php
Disallow: /sendmessage.php
Disallow: /showgroups.php
Disallow: /showpost.php
Disallow: /subscription.php
Disallow: /threadrate.php
Disallow: /usercp.php
Disallow: /usernote.php
Disallow: /customgroupicons/
Disallow: /signaturepics/
Disallow: /showthread.php
Disallow: /member.php
Disallow: /forumdisplay.php
Disallow: /blog.php
Disallow: /activity.php
Disallow: /index.php
Disallow: /itrader.php
Disallow: /garage.php

User-Agent: Slurp
User-agent: Yahoo! Slurp
User-agent: msnbot
User-agent: msnbot-media
User-agent: twitterbot
User-agent: Baiduspider
User-agent: TweetedTimes Bot
User-agent: TweetmemeBot
User-agent: PaperLiBot
User-agent: NetSeer 
User-agent: BomboraBot
User-agent: WF search
User-agent: heritrix
User-agent: expo9
User-agent: NetSeer
User-agent: AddThis.com
User-agent: Qwantify
User-agent: Feed
User-agent: Sogou
User-agent: ZumBot
User-agent: Feedspot
User-agent: acebookexternalhit
User-agent: facebookexternalhit
User-agent: StorygizeBot
User-agent: Scrapy
User-agent: GermCrawler
User-agent: Netvibes
User-agent: Vagabondo
User-agent: Genieo
User-agent: um-IC
User-agent: Typhoeus
User-agent: newspaper
User-agent: weborama-fetcher
User-agent: ScooperBot
User-agent: BYO-2
User-agent: SimplePie
User-agent: GroupHigh
User-agent: feedparser
User-agent: UniversalFeedParser
User-agent: Pulsepoint
User-agent: Steeler
User-agent: GarlikCrawler
User-agent: Flipboard
User-agent: FlipboardRSS
User-agent: Moreover
User-agent: Mercury
User-agent: FeedBurner
User-agent: Yahoo
User-agent: omgili
User-agent: Bloglovin
User-agent: YahooCacheSystem
User-agent: mTalk
User-agent: EveryoneSocialBot
User-agent: EzineArticlesLinkScanner
User-agent: dlvr.it
User-agent: pfblogs
User-agent: Aol
User-agent: ToutiaoSpider
User-agent: Photon
User-agent: PicoFeed
User-agent: um-FC
User-agent: Digg
User-agent: Synaptive
User-agent: YisouSpider
User-agent: BYO-4
User-agent: Feedbin
User-agent: Twingly
User-agent: Tiny
User-agent: WWW-Mechanize
User-agent: CommaFeed
User-agent: TkBot 
User-agent: TinEye-bot
User-agent: Qwantify
User-agent: Exabot
User-agent: coccoc
User-agent: BLEXBot
User-agent: uMBot-LN
User-agent: QuerySeekerSpider
User-agent: Yandex
User-agent: YandexBot
User-agent: YandexMobileBot
User-agent: YandexImageResizer
User-agent: YandexImages
User-agent: sogou spider
User-agent: Sogou web spider
User-agent: Baiduspider
User-agent: YoudaoBot
User-agent: MJ12bot
User-agent: spbot
User-agent: 360Spider
User-agent: SeznamBot
User-Agent: AhrefsBot
User-Agent: Linkdex
User-agent: linkdexbot
User-agent: SemrushBot
User-agent: SemrushBot-SA
User-agent: CFNetwork
User-agent: SputnikBot
User-agent: TurnitinBot
User-agent: proximic
User-agent: grapeshot
User-agent: VigLink
User-agent: Clickagy
User-agent: YaK
User-agent: DeuSu
User-agent: Taboolabot
User-agent: Pinterestbot
User-agent: psbot
User-agent: Plukkie
User-agent: GetIntent
User-agent: Facebot
User-agent: linkdexbot
User-agent: Vegi
User-agent: istellabot
User-agent: Leikibot
User-agent: Claritybot
User-agent: PiplBot
User-agent: Feedspotbot
User-agent: moatbot
User-agent: Yeti
User-agent: contxbot
User-agent: SEOkicks-Robot
User-agent: rogerbot
User-agent: Discordbot
User-agent: Tapatalk
User-agent: Mail.RU_Bot
User-agent: coccocbot
User-agent: HybridBot
User-agent: Slack-ImgProxy
User-agent: Twitterbot
User-agent: adbeat_bot
User-agent: Kraken
User-agent: MojeekBot
User-agent: calculon
User-agent: mcsbot
User-agent: avkzarabotok
User-agent: SearchmetricsBot
User-agent: Uptimebot
User-agent: yacybot
User-agent: Applebot
User-agent: Gigabot
User-agent: g2reader-bot
User-agent: Seoterritory
User-agent: AhrefsBot
User-agent: feedbot
User-agent: Findxbot
User-agent: Wotbox
User-agent: msnbot
User-agent: SMTBot
User-agent: Ocarinabot
User-agent: Dragonbot
User-agent: Zombiebot
User-agent: metabot.ru
User-agent: RSSMicro
User-agent: _zbot
User-agent: UptimeRobot
User-agent: uipbot
User-agent: ZoominfoBot
User-agent: Flamingo_SearchEngine
User-agent: bot-pge.chlooe.com
User-agent: Girafabot
User-agent: Multiviewbot
User-agent: SafeDNSBot
User-agent: GnowitNewsbot
User-agent: Treato-Bot
User-agent: Pvblcbot
User-agent: Laserlikebot
User-agent: trendiction
User-agent: Mediatoolkitbot
User-agent: CharlieBot
User-agent: BUbiNG
User-agent: Superfeedr
User-agent: focusbot
User-agent: Sottopop
User-agent: MaxPointCrawler
User-agent: Nutch-1.10 
User-agent: wonderbot
User-agent: zarabotok--doma
User-agent: Treato-Bot
User-agent: Pvblcbot
User-agent: Laserlikebot
User-agent: trendiction
User-agent: Mediatoolkitbot
User-agent: CharlieBot
User-agent: BUbiNG
User-agent: Superfeedr
User-agent: focusbot
User-agent: Sottopop
User-agent: MaxPointCrawler
User-agent: Nutch-1.10 
User-agent: wonderbot
User-agent: zarabotok--doma
User-agent: UptimeRobot
User-agent: uipbot
User-agent: ZoominfoBot
User-agent: Flamingo_SearchEngine
User-agent: bot-pge.chlooe.com
User-agent: Girafabot
User-agent: Multiviewbot
User-agent: SafeDNSBot
User-agent: GnowitNewsbot
User-agent: Treato-Bot
User-agent: Pvblcbot
User-agent: Laserlikebot
User-agent: trendiction
User-agent: Mediatoolkitbot
User-agent: CharlieBot
User-agent: BUbiNG
User-agent: Superfeedr
User-agent: focusbot
User-agent: Sottopop
User-agent: MaxPointCrawler
User-agent: Nutch-1.10 
User-agent: wonderbot
User-agent: zarabotok--doma
User-agent: Treato-Bot
User-agent: Pvblcbot
User-agent: Laserlikebot
User-agent: trendiction
User-agent: Mediatoolkitbot
User-agent: CharlieBot
User-agent: BUbiNG
User-agent: Superfeedr
User-agent: focusbot
User-agent: Sottopop
User-agent: MaxPointCrawler
User-agent: Nutch-1.10 
User-agent: wonderbot
User-agent: zarabotok--doma
User-agent: Cliqzbot
User-agent: DotBot
User-agent: Exabot
User-agent: moatbot
User-agent: Tapatalk
Crawl-delay: 30
Disallow: /