User-agent: *
Disallow: /wp-content/uploads/wpo/wpo-plugins-tables-list.json

# enforce crawl delay for those who use it
User-agent: *
Crawl-delay: 10

# Block AI crawlers
# Block all known AI crawlers and assistants
# from using content for training AI models.
# Source: https://robotstxt.com/ai
User-Agent: GPTBot
User-Agent: ClaudeBot
User-Agent: Claude-User
User-Agent: Claude-SearchBot
User-Agent: CCBot
User-Agent: Google-Extended
User-Agent: Applebot-Extended
User-Agent: Facebookbot
User-Agent: Meta-ExternalAgent
User-Agent: Meta-ExternalFetcher
User-Agent: diffbot
User-Agent: PerplexityBot
User-Agent: Perplexity‑User
User-Agent: Omgili
User-Agent: Omgilibot
User-Agent: webzio-extended
User-Agent: ImagesiftBot
User-Agent: Bytespider
User-Agent: TikTokSpider
User-Agent: Amazonbot
User-Agent: Youbot
User-Agent: SemrushBot-OCOB
User-Agent: Petalbot
User-Agent: VelenPublicWebCrawler
User-Agent: TurnitinBot
User-Agent: Timpibot
User-Agent: OAI-SearchBot
User-Agent: ICC-Crawler
User-Agent: AI2Bot
User-Agent: AI2Bot-Dolma
User-Agent: DataForSeoBot
User-Agent: AwarioBot
User-Agent: AwarioSmartBot
User-Agent: AwarioRssBot
User-Agent: Google-CloudVertexBot
User-Agent: PanguBot
User-Agent: Kangaroo Bot
User-Agent: Sentibot
User-Agent: img2dataset
User-Agent: Meltwater
User-Agent: Seekr
User-Agent: peer39_crawler
User-Agent: cohere-ai
User-Agent: cohere-training-data-crawler
User-Agent: DuckAssistBot
User-Agent: Scrapy
User-Agent: Cotoyogi
User-Agent: aiHitBot
User-Agent: Factset_spyderbot
User-Agent: FirecrawlAgent
Disallow: /

# START YOAST BLOCK
# ---------------------------
User-agent: *
Disallow: /wp-admin/
Disallow: /ab/*
Disallow: /aa/*
Disallow: /af/*
Disallow: /ak/*
Disallow: /sq/*
Disallow: /am/*
Disallow: /ar/*
Disallow: /an/*
Disallow: /hy/*
Disallow: /as/*
Disallow: /av/*
Disallow: /ae/*
Disallow: /ay/*
Disallow: /az/*
Disallow: /bm/*
Disallow: /ba/*
Disallow: /eu/*
Disallow: /be/*
Disallow: /bn/*
Disallow: /bh/*
Disallow: /bi/*
Disallow: /bs/*
Disallow: /br/*
Disallow: /bg/*
Disallow: /my/*
Disallow: /ca/*
Disallow: /ch/*
Disallow: /ce/*
Disallow: /ny/*
Disallow: /zh/*
Disallow: /zh-Hans/*
Disallow: /zh-Hant/*
Disallow: /cv/*
Disallow: /kw/*
Disallow: /co/*
Disallow: /cr/*
Disallow: /hr/*
Disallow: /cs/*
Disallow: /da/*
Disallow: /dv/*
Disallow: /nl/*
Disallow: /dz/*
Disallow: /en/*
Disallow: /eo/*
Disallow: /et/*
Disallow: /ee/*
Disallow: /fo/*
Disallow: /fj/*
Disallow: /fi/*
Disallow: /fr/*
Disallow: /ff/*
Disallow: /gl/*
Disallow: /gd/*
Disallow: /gv/*
Disallow: /ka/*
Disallow: /de/*
Disallow: /el/*
Disallow: /kl/*
Disallow: /gn/*
Disallow: /gu/*
Disallow: /ht/*
Disallow: /ha/*
Disallow: /he/*
Disallow: /hz/*
Disallow: /hi/*
Disallow: /ho/*
Disallow: /hu/*
Disallow: /is/*
Disallow: /io/*
Disallow: /ig/*
Disallow: /id/* 
Disallow: /in/*
Disallow: /ia/*
Disallow: /ie/*
Disallow: /iu/*
Disallow: /ik/*
Disallow: /ga/*
Disallow: /it/*
Disallow: /ja/*
Disallow: /jv/*
Disallow: /kl/*
Disallow: /kn/*
Disallow: /kr/*
Disallow: /ks/*
Disallow: /kk/*
Disallow: /km/*
Disallow: /ki/*
Disallow: /rw/*
Disallow: /rn/*
Disallow: /ky/*
Disallow: /kv/*
Disallow: /kg/*
Disallow: /ko/*
Disallow: /ku/*
Disallow: /kj/*
Disallow: /lo/*
Disallow: /la/*
Disallow: /lv/*
Disallow: /li/*
Disallow: /ln/*
Disallow: /lt/*
Disallow: /lu/*
Disallow: /lg/*
Disallow: /lb/*
Disallow: /gv/*
Disallow: /mk/*
Disallow: /mg/*
Disallow: /ms/*
Disallow: /ml/*
Disallow: /mt/*
Disallow: /mi/*
Disallow: /mr/*
Disallow: /mh/*
Disallow: /mo/*
Disallow: /mn/*
Disallow: /na/*
Disallow: /nv/*
Disallow: /ng/*
Disallow: /nd/*
Disallow: /ne/*
Disallow: /no/*
Disallow: /nb/*
Disallow: /nn/*
Disallow: /ii/*
Disallow: /oc/*
Disallow: /oj/*
Disallow: /cu/*
Disallow: /or/*
Disallow: /om/*
Disallow: /os/*
Disallow: /pi/*
Disallow: /ps/*
Disallow: /fa/*
Disallow: /pl/*
Disallow: /pt/*
Disallow: /pa/*
Disallow: /qu/*
Disallow: /rm/*
Disallow: /ro/*
Disallow: /ru/*
Disallow: /se/*
Disallow: /sm/*
Disallow: /sg/*
Disallow: /sa/*
Disallow: /sr/*
Disallow: /sh/*
Disallow: /st/*
Disallow: /tn/*
Disallow: /sn/*
Disallow: /ii/*
Disallow: /sd/*
Disallow: /si/*
Disallow: /ss/*
Disallow: /sk/*
Disallow: /sl/*
Disallow: /so/*
Disallow: /nr/*
Disallow: /es/*
Disallow: /su/*
Disallow: /sw/*
Disallow: /ss/*
Disallow: /sv/*
Disallow: /tl/*
Disallow: /ty/*
Disallow: /tg/*
Disallow: /ta/*
Disallow: /tt/*
Disallow: /te/*
Disallow: /th/*
Disallow: /bo/*
Disallow: /ti/*
Disallow: /to/*
Disallow: /ts/*
Disallow: /tr/*
Disallow: /tk/*
Disallow: /tw/*
Disallow: /ug/*
Disallow: /uk/*
Disallow: /ur/*
Disallow: /uz/*
Disallow: /ve/*
Disallow: /vi/*
Disallow: /vo/*
Disallow: /wa/*
Disallow: /cy/*
Disallow: /wo/*
Disallow: /fy/*
Disallow: /xh/*
Disallow: /yi/* 
Disallow: /ji/*
Disallow: /yo/*
Disallow: /za/*
Disallow: /zu/*


Sitemap: https://www.carnegielibrary.org/sitemap_index.xml
# ---------------------------
# END YOAST BLOCK