{"domain":"nytimes.com","summary":"nytimes.com: robots.txt blocks 13 of 14 AI bots (GPTBot, OAI-SearchBot, ChatGPT-User, ClaudeBot, Claude-SearchBot, Claude-User, …) and allows 1. TollBit bot paywall: not detected. Cloudflare: not detected. llms.txt: no.","checked_at":"2026-10-12T23:31:13.887Z","robots_txt":{"found":true,"status":200,"content_signals":null},"bots":[{"bot":"GPTBot","owner":"OpenAI","purpose":"model training","allowed_on_homepage":false,"rule_group":"GPTBot"},{"bot":"OAI-SearchBot","owner":"OpenAI","purpose":"ChatGPT search index","allowed_on_homepage":false,"rule_group":"OAI-SearchBot"},{"bot":"ChatGPT-User","owner":"OpenAI","purpose":"live fetch for a ChatGPT user","allowed_on_homepage":false,"rule_group":"ChatGPT-User"},{"bot":"ClaudeBot","owner":"Anthropic","purpose":"model training","allowed_on_homepage":false,"rule_group":"ClaudeBot"},{"bot":"Claude-SearchBot","owner":"Anthropic","purpose":"Claude search index","allowed_on_homepage":false,"rule_group":"Claude-SearchBot"},{"bot":"Claude-User","owner":"Anthropic","purpose":"live fetch for a Claude user","allowed_on_homepage":false,"rule_group":"Claude-User"},{"bot":"PerplexityBot","owner":"Perplexity","purpose":"Perplexity search index","allowed_on_homepage":false,"rule_group":"PerplexityBot"},{"bot":"Perplexity-User","owner":"Perplexity","purpose":"live fetch for a Perplexity user","allowed_on_homepage":false,"rule_group":"Perplexity-User"},{"bot":"Google-Extended","owner":"Google","purpose":"Gemini training and grounding (not Search)","allowed_on_homepage":false,"rule_group":"Google-Extended"},{"bot":"Applebot-Extended","owner":"Apple","purpose":"Apple AI training","allowed_on_homepage":false,"rule_group":"Applebot-Extended"},{"bot":"meta-externalagent","owner":"Meta","purpose":"Meta AI training and fetches","allowed_on_homepage":false,"rule_group":"meta-externalagent"},{"bot":"CCBot","owner":"Common Crawl","purpose":"open crawl used for training","allowed_on_homepage":false,"rule_group":"CCBot"},{"bot":"Bytespider","owner":"ByteDance","purpose":"model training","allowed_on_homepage":false,"rule_group":"Bytespider"},{"bot":"Amazonbot","owner":"Amazon","purpose":"Alexa / Amazon AI","allowed_on_homepage":true,"rule_group":"Amazonbot"}],"tollbit":{"subdomain":"tollbit.nytimes.com","live":false,"status":null,"referenced_in_robots":false},"cloudflare":false,"llms_txt":false,"notes":["Allowed/blocked is what robots.txt asks for the homepage path; bots may ignore it, and Cloudflare or a CDN can block bots regardless of robots.txt.","A live tollbit.<domain> subdomain means the publisher uses TollBit to charge AI bots per crawl; whether bots are actually redirected there depends on the publisher's CDN rules."],"docs":"https://www.oasy.ai/api/aibots/n4d76vhe"}