################################# CRAWLER WARNING ########################################
#
# - The terms of service and the robots.txt file disallows crawling of this site,
#   please see https://omim.org/help/agreement for more information.
#
# - A number of data files are available for download at https://omim.org/downloads.
#
# - We have an API which you can learn about at https://omim.org/help/api and register
#   for at https://omim.org/api, this provides access to the data in JSON & XML formats.
#
# - You should feel free to contact us at https://omim.org/contact to figure out the best
#   approach to getting the data you need for your work.
#
# - WE WILL AUTOMATICALLY BLOCK YOUR IP ADDRESS IF YOU CRAWL THIS SITE.
#
# - WE WILL ALSO AUTOMATICALLY BLOCK SUB-DOMAINS AND ADDRESS RANGES IMPLICATED IN
#   DISTRIBUTED CRAWLS OF THIS SITE.
#
################################# CRAWLER WARNING ########################################


#
# Robots.txt
#
# Specification for robots.txt
# https://developers.google.com/search/reference/robots_txt
# https://www.robotstxt.org
#


# Global - disallow everything except for home page
User-agent: *
Allow: /$
Disallow: /


# Crawl delay, every two seconds (not in the official spec)
User-agent: *
Crawl-delay: 2


# Google - disallow everything except for specific paths
# https://www.google.com/bot.html
# https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers
User-agent: Googlebot
Allow: /$
Disallow: /entry/$
Allow: /entry/
Disallow: /clinicalSynopsis/$
Allow: /clinicalSynopsis/
Allow: /about
Allow: /help/
Allow: /downloads
Allow: /api
Disallow: /

# Google-Extended - disallow everything
User-agent: Google-Extended
Disallow: /


# Bing - disallow everything except for specific paths
# https://www.bing.com/bingbot.htm
User-agent: Bingbot
Allow: /$
Disallow: /entry/$
Allow: /entry/
Disallow: /clinicalSynopsis/$
Allow: /clinicalSynopsis/
Allow: /about
Allow: /help/
Allow: /downloads
Allow: /api
Disallow: /


# DuckDuckGo - disallow everything except for specific paths
# https://duckduckgo.com/duckduckbot
User-agent: DuckDuckBot
Allow: /$
Disallow: /entry/$
Allow: /entry/
Disallow: /clinicalSynopsis/$
Allow: /clinicalSynopsis/
Allow: /about
Allow: /help/
Allow: /downloads
Allow: /api
Disallow: /


# Applebot - disallow everything except for home page
# https://support.apple.com/en-us/119829
User-agent: Applebot
Allow: /$
Disallow: /


# AI Crawlers - disallow everything
# https://github.com/ai-robots-txt/ai.robots.txt
User-agent: AI2Bot
User-agent: Ai2Bot-Dolma
User-agent: Amazonbot
User-agent: anthropic-ai
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Bytespider
User-agent: CCBot
User-agent: ChatGPT-User
User-agent: Claude-Web
User-agent: ClaudeBot
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: Crawlspace
User-agent: Diffbot
User-agent: DuckAssistBot
User-agent: FacebookBot
User-agent: FriendlyCrawler
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: GoogleOther-Image
User-agent: GoogleOther-Video
User-agent: GPTBot
User-agent: iaskspider/2.0
User-agent: ICC-Crawler
User-agent: ImagesiftBot
User-agent: img2dataset
User-agent: ISSCyberRiskCrawler
User-agent: Kangaroo Bot
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: OAI-SearchBot
User-agent: omgili
User-agent: omgilibot
User-agent: PanguBot
User-agent: PerplexityBot
User-agent: PetalBot
User-agent: Scrapy
User-agent: SemrushBot
User-agent: Sidetrade indexer bot
User-agent: Timpibot
User-agent: VelenPublicWebCrawler
User-agent: Webzio-Extended
User-agent: YouBot
Disallow: /