#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    https://alleppeyhouseboats.net/robots.txt
# Ignored: https://alleppeyhouseboats.net/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/wc/robots.html
#
# For syntax checking, see:
# http://www.sxw.org.uk/computing/robots/check.html
Sitemap: https://alleppeyhouseboats.net/sitemap.xml

User-agent: *
Crawl-delay: 10
# Directories
Allow: /includes/
Allow: /misc/
Allow: /modules/
Allow: /profiles/
Allow: /scripts/
Allow: /themes/
# Files
Allow: /CHANGELOG.txt
Allow: /cron.php
Allow: /INSTALL.mysql.txt
Allow: /INSTALL.pgsql.txt
Allow: /INSTALL.sqlite.txt
Allow: /install.php
Allow: /INSTALL.txt
Allow: /LICENSE.txt
Allow: /MAINTAINERS.txt
Allow: /update.php
Allow: /UPGRADE.txt
Allow: /xmlrpc.php
# Paths (clean URLs)
Allow: /admin/
Allow: /comment/reply/
Allow: /filter/tips/
Allow: /node/add/
Allow: /search/
Allow: /user/register/
Allow: /user/password/
Allow: /user/login/
Allow: /user/logout/
# Paths (no clean URLs)
Allow: /?q=admin/
Allow: /?q=comment/reply/
Allow: /?q=filter/tips/
Allow: /?q=node/add/
Allow: /?q=search/
Allow: /?q=user/password/
Allow: /?q=user/register/
Allow: /?q=user/login/
Allow: /?q=user/logout/

Allow: *.xlsx$
Allow: *.xls$

# Begin block Bad-Robots from robots.txt
User-agent: asterias
Allow:/
User-agent: BackDoorBot/1.0
Allow:/
User-agent: Black Hole
Allow:/
User-agent: BlowFish/1.0
Allow:/
User-agent: BotALot
Allow:/
User-agent: BuiltBotTough
Allow:/
User-agent: Bullseye/1.0
Allow:/
User-agent: BunnySlippers
Allow:/
User-agent: Cegbfeieh
Allow:/
User-agent: CheeseBot
Allow:/
User-agent: CherryPicker
Allow:/
User-agent: CherryPickerElite/1.0
Allow:/
User-agent: CherryPickerSE/1.0
Allow:/
User-agent: CopyRightCheck
Allow:/
User-agent: cosmos
Allow:/
User-agent: Crescent
Allow:/
User-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0
Allow:/
User-agent: DittoSpyder
Allow:/
User-agent: EmailCollector
Allow:/
User-agent: EmailSiphon
Allow:/
User-agent: EmailWolf
Allow:/
User-agent: EroCrawler
Allow:/
User-agent: ExtractorPro
Allow:/
User-agent: Foobot
Allow:/
User-agent: Harvest/1.5
Allow:/
User-agent: hloader
Allow:/
User-agent: httplib
Allow:/
User-agent: humanlinks
Allow:/
User-agent: ia_archiver
Allow:/
User-agent: InfoNaviRobot
Allow:/
User-agent: JennyBot
Allow:/
User-agent: Kenjin Spider
Allow:/
User-agent: Keyword Density/0.9
Allow:/
User-agent: LexiBot
Allow:/
User-agent: libWeb/clsHTTP
Allow:/
User-agent: LinkextractorPro
Allow:/
User-agent: LinkScan/8.1a Unix
Allow:/
User-agent: LinkWalker
Allow:/
User-agent: LNSpiderguy
Allow:/
User-agent: lwp-trivial
Allow:/
User-agent: lwp-trivial/1.34
Allow:/
User-agent: Mata Hari
Allow:/
User-agent: Microsoft URL Control - 5.01.4511
Allow:/
User-agent: Microsoft URL Control - 6.00.8169
Allow:/
User-agent: MIIxpc
Allow:/
User-agent: MIIxpc/4.2
Allow:/
User-agent: Mister PiX
Allow:/
User-agent: moget
Allow:/
User-agent: moget/2.1
Allow:/
User-agent: mozilla/4
Allow:/
User-agent: Mozilla/4.0 (compatible; BullsEye; Windows 95)
Allow:/
User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows 95)
Allow:/
User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows 98)
Allow:/
User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows NT)
Allow:/
User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows XP)
Allow:/
User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows 2000)
Allow:/
User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows ME)
Allow:/
User-agent: mozilla/5
Allow:/
User-agent: NetAnts
Allow:/
User-agent: NICErsPRO
Allow:/
User-agent: Offline Explorer
Allow:/
User-agent: Openfind
Allow:/
User-agent: Openfind data gathere
Allow:/
User-agent: ProPowerBot/2.14
Allow:/
User-agent: ProWebWalker
Allow:/
User-agent: QueryN Metasearch
Allow:/
User-agent: RepoMonkey
Allow:/
User-agent: RepoMonkey Bait & Tackle/v1.01
Allow:/
User-agent: RMA
Allow:/
User-agent: SiteSnagger
Allow:/
User-agent: SpankBot
Allow:/
User-agent: spanner
Allow:/
User-agent: suzuran
Allow:/
User-agent: Szukacz/1.4
Allow:/
User-agent: Teleport
Allow:/
User-agent: TeleportPro
Allow:/
User-agent: Telesoft
Allow:/
User-agent: The Intraformant
Allow:/
User-agent: TheNomad
Allow:/
User-agent: TightTwatBot
Allow:/
User-agent: Titan
Allow:/
User-agent: toCrawl/UrlDispatcher
Allow:/
User-agent: True_Robot
Allow:/
User-agent: True_Robot/1.0
Allow:/
User-agent: turingos
Allow:/
User-agent: URLy Warning
Allow:/
User-agent: VCI
Allow:/
User-agent: VCI WebViewer VCI WebViewer Win32
Allow:/
User-agent: Web Image Collector
Allow:/
User-agent: WebAuto
Allow:/
User-agent: WebBandit
Allow:/
User-agent: WebBandit/3.50
Allow:/
User-agent: WebCopier
Allow:/
User-agent: WebEnhancer
Allow:/
User-agent: WebmasterWorldForumBot
Allow:/
User-agent: WebSauger
Allow:/
User-agent: Website Quester
Allow:/
User-agent: Webster Pro
Allow:/
User-agent: WebStripper
Allow:/
User-agent: WebZip
Allow:/
User-agent: WebZip/4.0
Allow:/
User-agent: Wget
Allow:/
User-agent: Wget/1.5.3
Allow:/
User-agent: Wget/1.6
Allow:/
User-agent: WWW-Collector-E
Allow:/
User-agent: Xenu's
Allow:/
User-agent: Xenu's Link Sleuth 1.1c
Allow:/
User-agent: Zeus
Allow:/
User-agent: Zeus 32297 Webster Pro V2.9 Win32
Allow:/

# SEO-related bots
User-agent: rogerbot
Allow:/
User-agent: mj12bot
Allow:/
User-agent: dotbot
Allow:/
User-agent: ahrefsbot
Allow:/