{"openapi":"3.1.0","info":{"title":"AI Crawler Access Checker API (robots.txt, llms.txt)","version":"1.0.0","description":"Can ChatGPT, Claude, Perplexity, Gemini, Apple, Meta, Amazon and other AI crawlers read a website? For 26 AI crawlers (training, AI search and user-triggered agents, each checked against its vendor's documentation) the API reports allowed / partial / blocked with the matching robots.txt rule, plus the site's AI policy class and scores, llms.txt and llms-full.txt, Cloudflare Content Signals (search, ai-input, ai-train), noai meta directives, and a ready-to-paste robots.txt fix.\n\nAll endpoints accept GET (query string) or POST (JSON body). Errors use one shape: `{\"success\": false, \"error\": {\"code\", \"message\", \"retryable\"}}`. When a numeric `Retry-After` header is present, `error.retryAfterSeconds` is included.","contact":{"name":"TidyTools","url":"https://apify.com/tidytools"}},"servers":[{"url":"https://webcapture-api.yukailin.workers.dev","description":"Origin (reachable only through the marketplace gateway)"}],"tags":[{"name":"AI crawlers"}],"paths":{"/api/v1/ai-crawlers":{"get":{"operationId":"aiCrawlersGet","summary":"Check which AI crawlers a site allows (GET)","description":"Reads robots.txt (RFC 9309 matching, with vendor-documented fallbacks such as Applebot following Googlebot), llms.txt, llms-full.txt and the home page's robots meta tags.","tags":["AI crawlers"],"parameters":[{"name":"url","schema":{"type":"string"},"description":"Website or page (a bare domain works). A path in the URL is also tested. Alias: `domain`.","required":true,"example":"nytimes.com","in":"query"},{"name":"paths","schema":{"type":"string"},"description":"Up to 5 extra paths to test, comma-separated (e.g. `/blog/,/docs/`).","required":false,"in":"query"},{"name":"includeSnippets","schema":{"type":"boolean","default":true},"description":"Include the robots.txt fix snippet and a suggested AI policy.","required":false,"in":"query"}],"responses":{"200":{"description":"Success.","content":{"application/json":{"schema":{"type":"object","properties":{"success":{"type":"boolean"},"site":{"type":"string"},"verdict":{"type":"string"},"policy":{"type":"string"},"aiSearchScore":{"type":"integer"},"aiAccessScore":{"type":"integer"},"allowedBots":{"type":"array","items":{"type":"string"}},"blockedBots":{"type":"array","items":{"type":"string"}},"partialBots":{"type":"array","items":{"type":"string"}},"unknownBots":{"type":"array","items":{}},"contentSignals":{"type":"string","nullable":true},"robotsTxt":{"type":"object","properties":{"url":{"type":"string"},"status":{"type":"integer"},"state":{"type":"string"},"reason":{"type":"string","nullable":true},"sitemaps":{"type":"array","items":{"type":"string"}}}},"llmsTxt":{"type":"object","properties":{"found":{"type":"boolean"},"url":{"type":"string"},"status":{"type":"integer"},"startsWithH1":{"type":"string","nullable":true}}},"llmsFullTxt":{"type":"object","properties":{"found":{"type":"boolean"},"url":{"type":"string"},"status":{"type":"integer"},"startsWithH1":{"type":"string","nullable":true}}},"homepage":{"type":"object","properties":{"status":{"type":"integer"},"metaRobots":{"type":"string","nullable":true},"xRobotsTag":{"type":"string","nullable":true}}},"noAiDirective":{"type":"boolean"},"testedPaths":{"type":"array","items":{"type":"string"}},"bots":{"type":"array","items":{"type":"object","properties":{"bot":{"type":"string"},"company":{"type":"string"},"purpose":{"type":"string"},"status":{"type":"string"},"allowedAll":{"type":"boolean"},"matchedGroup":{"type":"string"},"paths":{"type":"array","items":{"type":"object","properties":{"path":{"type":"string"},"allowed":{"type":"boolean"},"rule":{"type":"string"}}}}}}},"recommendedRobotsSnippet":{"type":"string"},"suggestedPolicy":{"type":"string"},"crawlerList":{"type":"object","properties":{"count":{"type":"integer"},"updated":{"type":"string"},"version":{"type":"string"}}}}},"example":{"success":true,"site":"https://nytimes.com","verdict":"Some AI search/assistant crawlers are blocked (OAI-SearchBot, Claude-SearchBot, PerplexityBot, meta-webindexer, DuckAssistBot, ChatGPT-User, Claude-User, Perplexity-User, meta-externalfetcher): the site may not appear in those AI answers.","policy":"blocks-search","aiSearchScore":47,"aiAccessScore":35,"allowedBots":["Google-GeminiNotebook","Google-Agent","Applebot","Amazonbot"],"blockedBots":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot"],"partialBots":["Applebot","Amazonbot"],"unknownBots":[],"contentSignals":null,"robotsTxt":{"url":"https://nytimes.com/robots.txt","status":200,"state":"found","reason":null,"sitemaps":["https://www.nytimes.com/sitemaps/new/news.xml.gz","https://www.nytimes.com/sitemaps/new/sitemap.xml.gz","https://www.nytimes.com/sitemaps/new/collections.xml.gz","https://www.nytimes.com/sitemaps/new/video.xml.gz"]},"llmsTxt":{"found":false,"url":"https://nytimes.com/llms.txt","status":404,"startsWithH1":null},"llmsFullTxt":{"found":false,"url":"https://nytimes.com/llms-full.txt","status":404,"startsWithH1":null},"homepage":{"status":403,"metaRobots":null,"xRobotsTag":null},"noAiDirective":false,"testedPaths":["/"],"bots":[{"bot":"GPTBot","company":"OpenAI","purpose":"training","status":"blocked","allowedAll":false,"matchedGroup":"GPTBot","paths":[{"path":"/","allowed":false,"rule":"Disallow: /"}]},{"bot":"OAI-SearchBot","company":"OpenAI","purpose":"search","status":"blocked","allowedAll":false,"matchedGroup":"OAI-SearchBot","paths":[{"path":"/","allowed":false,"rule":"Disallow: /"}]},{"bot":"ChatGPT-User","company":"OpenAI","purpose":"user","status":"blocked","allowedAll":false,"matchedGroup":"ChatGPT-User","ignoresRobots":true,"paths":[{"path":"/","allowed":false,"rule":"Disallow: /"}]},{"bot":"ClaudeBot","company":"Anthropic","purpose":"training","status":"blocked","allowedAll":false,"matchedGroup":"ClaudeBot","paths":[{"path":"/","allowed":false,"rule":"Disallow: /"}]}],"recommendedRobotsSnippet":"# Allow AI search and assistant crawlers (they decide whether the site can appear in AI answers)\n# Also remove \"Disallow: /\" from the existing group(s) for: OAI-SearchBot, ChatGPT-User, Claude-SearchBot, Claude-User, PerplexityBot, Perplexity-User, meta-webindexer, meta-externalfetcher, DuckAssistBo…","suggestedPolicy":"# AI model training crawlers: blocked (does not affect AI search answers)\nUser-agent: GPTBot\nUser-agent: ClaudeBot\nUser-agent: Google-Extended\nUser-agent: Applebot-Extended\nUser-agent: Meta-ExternalAgent\nUser-agent: MistralAI-Training\nUser-agent: CCBot\nUser-agent: Bytespider\nUser-agent: cohere-ai\nDi…","crawlerList":{"count":26,"updated":"2026-09-29","version":"26-5d7f1211"}}}}},"400":{"description":"Invalid parameter (`invalid_request`) or a private/local target (`url_not_allowed`).","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"},"example":{"success":false,"error":{"code":"url_not_allowed","message":"private, local and reserved hosts are not allowed"}}}}},"413":{"description":"Request body or target document too large (`too_large`).","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"422":{"description":"The target site refused (`target_blocked`), could not be reached or its domain does not resolve (`target_unreachable`), the page does not exist (`target_error`, HTTP 404/410), or returned unsupported content.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"},"example":{"success":false,"error":{"code":"target_blocked","message":"The target refused the request: target returned HTTP 403","details":{"targetStatus":403,"finalUrl":"https://www.g2.com/"}}}}}},"502":{"description":"Temporary failure on our side (`upstream_error`); safe to retry.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"504":{"description":"The target did not respond in time (`timeout`).","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}}},"post":{"operationId":"aiCrawlers","summary":"Check which AI crawlers a site allows","description":"Reads robots.txt (RFC 9309 matching, with vendor-documented fallbacks such as Applebot following Googlebot), llms.txt, llms-full.txt and the home page's robots meta tags.\n\nThe same parameters as the GET form, sent as a JSON body.","tags":["AI crawlers"],"requestBody":{"required":true,"content":{"application/json":{"schema":{"type":"object","properties":{"url":{"type":"string","description":"Website or page (a bare domain works). A path in the URL is also tested. Alias: `domain`."},"paths":{"type":"string","description":"Up to 5 extra paths to test, comma-separated (e.g. `/blog/,/docs/`)."},"includeSnippets":{"type":"boolean","default":true,"description":"Include the robots.txt fix snippet and a suggested AI policy."}},"required":["url"]}}}},"responses":{"200":{"description":"Success.","content":{"application/json":{"schema":{"type":"object","properties":{"success":{"type":"boolean"},"site":{"type":"string"},"verdict":{"type":"string"},"policy":{"type":"string"},"aiSearchScore":{"type":"integer"},"aiAccessScore":{"type":"integer"},"allowedBots":{"type":"array","items":{"type":"string"}},"blockedBots":{"type":"array","items":{"type":"string"}},"partialBots":{"type":"array","items":{"type":"string"}},"unknownBots":{"type":"array","items":{}},"contentSignals":{"type":"string","nullable":true},"robotsTxt":{"type":"object","properties":{"url":{"type":"string"},"status":{"type":"integer"},"state":{"type":"string"},"reason":{"type":"string","nullable":true},"sitemaps":{"type":"array","items":{"type":"string"}}}},"llmsTxt":{"type":"object","properties":{"found":{"type":"boolean"},"url":{"type":"string"},"status":{"type":"integer"},"startsWithH1":{"type":"string","nullable":true}}},"llmsFullTxt":{"type":"object","properties":{"found":{"type":"boolean"},"url":{"type":"string"},"status":{"type":"integer"},"startsWithH1":{"type":"string","nullable":true}}},"homepage":{"type":"object","properties":{"status":{"type":"integer"},"metaRobots":{"type":"string","nullable":true},"xRobotsTag":{"type":"string","nullable":true}}},"noAiDirective":{"type":"boolean"},"testedPaths":{"type":"array","items":{"type":"string"}},"bots":{"type":"array","items":{"type":"object","properties":{"bot":{"type":"string"},"company":{"type":"string"},"purpose":{"type":"string"},"status":{"type":"string"},"allowedAll":{"type":"boolean"},"matchedGroup":{"type":"string"},"paths":{"type":"array","items":{"type":"object","properties":{"path":{"type":"string"},"allowed":{"type":"boolean"},"rule":{"type":"string"}}}}}}},"recommendedRobotsSnippet":{"type":"string"},"suggestedPolicy":{"type":"string"},"crawlerList":{"type":"object","properties":{"count":{"type":"integer"},"updated":{"type":"string"},"version":{"type":"string"}}}}},"example":{"success":true,"site":"https://nytimes.com","verdict":"Some AI search/assistant crawlers are blocked (OAI-SearchBot, Claude-SearchBot, PerplexityBot, meta-webindexer, DuckAssistBot, ChatGPT-User, Claude-User, Perplexity-User, meta-externalfetcher): the site may not appear in those AI answers.","policy":"blocks-search","aiSearchScore":47,"aiAccessScore":35,"allowedBots":["Google-GeminiNotebook","Google-Agent","Applebot","Amazonbot"],"blockedBots":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot"],"partialBots":["Applebot","Amazonbot"],"unknownBots":[],"contentSignals":null,"robotsTxt":{"url":"https://nytimes.com/robots.txt","status":200,"state":"found","reason":null,"sitemaps":["https://www.nytimes.com/sitemaps/new/news.xml.gz","https://www.nytimes.com/sitemaps/new/sitemap.xml.gz","https://www.nytimes.com/sitemaps/new/collections.xml.gz","https://www.nytimes.com/sitemaps/new/video.xml.gz"]},"llmsTxt":{"found":false,"url":"https://nytimes.com/llms.txt","status":404,"startsWithH1":null},"llmsFullTxt":{"found":false,"url":"https://nytimes.com/llms-full.txt","status":404,"startsWithH1":null},"homepage":{"status":403,"metaRobots":null,"xRobotsTag":null},"noAiDirective":false,"testedPaths":["/"],"bots":[{"bot":"GPTBot","company":"OpenAI","purpose":"training","status":"blocked","allowedAll":false,"matchedGroup":"GPTBot","paths":[{"path":"/","allowed":false,"rule":"Disallow: /"}]},{"bot":"OAI-SearchBot","company":"OpenAI","purpose":"search","status":"blocked","allowedAll":false,"matchedGroup":"OAI-SearchBot","paths":[{"path":"/","allowed":false,"rule":"Disallow: /"}]},{"bot":"ChatGPT-User","company":"OpenAI","purpose":"user","status":"blocked","allowedAll":false,"matchedGroup":"ChatGPT-User","ignoresRobots":true,"paths":[{"path":"/","allowed":false,"rule":"Disallow: /"}]},{"bot":"ClaudeBot","company":"Anthropic","purpose":"training","status":"blocked","allowedAll":false,"matchedGroup":"ClaudeBot","paths":[{"path":"/","allowed":false,"rule":"Disallow: /"}]}],"recommendedRobotsSnippet":"# Allow AI search and assistant crawlers (they decide whether the site can appear in AI answers)\n# Also remove \"Disallow: /\" from the existing group(s) for: OAI-SearchBot, ChatGPT-User, Claude-SearchBot, Claude-User, PerplexityBot, Perplexity-User, meta-webindexer, meta-externalfetcher, DuckAssistBo…","suggestedPolicy":"# AI model training crawlers: blocked (does not affect AI search answers)\nUser-agent: GPTBot\nUser-agent: ClaudeBot\nUser-agent: Google-Extended\nUser-agent: Applebot-Extended\nUser-agent: Meta-ExternalAgent\nUser-agent: MistralAI-Training\nUser-agent: CCBot\nUser-agent: Bytespider\nUser-agent: cohere-ai\nDi…","crawlerList":{"count":26,"updated":"2026-09-29","version":"26-5d7f1211"}}}}},"400":{"description":"Invalid parameter (`invalid_request`) or a private/local target (`url_not_allowed`).","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"},"example":{"success":false,"error":{"code":"url_not_allowed","message":"private, local and reserved hosts are not allowed"}}}}},"413":{"description":"Request body or target document too large (`too_large`).","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"422":{"description":"The target site refused (`target_blocked`), could not be reached or its domain does not resolve (`target_unreachable`), the page does not exist (`target_error`, HTTP 404/410), or returned unsupported content.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"},"example":{"success":false,"error":{"code":"target_blocked","message":"The target refused the request: target returned HTTP 403","details":{"targetStatus":403,"finalUrl":"https://www.g2.com/"}}}}}},"502":{"description":"Temporary failure on our side (`upstream_error`); safe to retry.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"504":{"description":"The target did not respond in time (`timeout`).","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}}}}},"components":{"schemas":{"Error":{"type":"object","required":["success","error"],"properties":{"success":{"const":false},"error":{"type":"object","required":["code","message"],"properties":{"code":{"type":"string","enum":["not_configured","unauthorized","not_found","method_not_allowed","invalid_request","rate_limited","url_not_allowed","target_blocked","target_unreachable","target_error","unsupported_content","too_large","timeout","upstream_error","board_not_found","app_not_found"]},"message":{"type":"string"},"retryable":{"type":"boolean","description":"true for rate_limited, timeout, upstream_error and browser_busy; false for other errors."},"retryAfterSeconds":{"type":"integer","description":"Seconds to wait before retrying, when a numeric Retry-After header is present."},"details":{"type":"object","additionalProperties":true}}}}}}}}