{"success":true,"site":"https://repubblica.it","verdict":"Some AI search/assistant crawlers are blocked (PerplexityBot, Applebot, DuckAssistBot, meta-externalfetcher): the site may not appear in those AI answers.","policy":"blocks-search","aiSearchScore":76,"aiAccessScore":63,"allowedBots":["OAI-SearchBot","ChatGPT-User","OAI-AdsBot","ClaudeBot","Claude-SearchBot","Claude-User","Perplexity-User","Google-CloudVertexBot","Google-GeminiNotebook","Google-Agent","Meta-ExternalAgent","meta-webindexer","meta-externalads","Amazonbot","Amzn-SearchBot","Amzn-User","MistralAI-User","MistralAI-Index","MistralAI-Training"],"blockedBots":["GPTBot","PerplexityBot","Google-Extended","Applebot","Applebot-Extended","meta-externalfetcher","DuckAssistBot","CCBot","Bytespider","cohere-ai"],"partialBots":[],"unknownBots":[],"contentSignals":null,"robotsTxt":{"url":"https://repubblica.it/robots.txt","status":200,"state":"found","reason":null,"sitemaps":["https://www.repubblica.it/sitemap-n.xml","https://video.repubblica.it/sitemap-v-day.xml","https://video.repubblica.it/sitemap-v.xml","https://www.repubblica.it/sitemap-italian-tech-n.xml","https://www.repubblica.it/sitemap-il-gusto-n.xml","https://www.repubblica.it/sitemap-salute-n.xml","https://www.repubblica.it/sitemap-green-and-blue-n.xml","https://www.repubblica.it/sitemap-dirette-sport-n.xml","https://www.repubblica.it/sitemap.xml","https://www.repubblica.it/sitemap-italian-tech.xml","https://www.repubblica.it/sitemap-il-gusto.xml","https://www.repubblica.it/sitemap-salute.xml","https://www.repubblica.it/sitemap-green-and-blue.xml"]},"llmsTxt":{"found":true,"url":"https://repubblica.it/llms.txt","status":200,"startsWithH1":true},"llmsFullTxt":{"found":false,"url":"https://repubblica.it/llms-full.txt","status":404,"startsWithH1":null},"homepage":{"status":200,"metaRobots":"max-image-preview:large","xRobotsTag":"noarchive, max-image-preview:large"},"noAiDirective":false,"testedPaths":["/"],"bots":[{"bot":"GPTBot","company":"OpenAI","purpose":"training","status":"blocked","allowedAll":false,"matchedGroup":"GPTBot","paths":[{"path":"/","allowed":false,"rule":"Disallow: /"}]},{"bot":"OAI-SearchBot","company":"OpenAI","purpose":"search","status":"allowed","allowedAll":true,"matchedGroup":"*","paths":[{"path":"/","allowed":true,"rule":"Allow: /"}]},{"bot":"ChatGPT-User","company":"OpenAI","purpose":"user","status":"allowed","allowedAll":true,"matchedGroup":"*","ignoresRobots":true,"paths":[{"path":"/","allowed":true,"rule":"Allow: /"}]},{"bot":"OAI-AdsBot","company":"OpenAI","purpose":"ads","status":"allowed","allowedAll":true,"matchedGroup":"*","paths":[{"path":"/","allowed":true,"rule":"Allow: /"}]},{"bot":"ClaudeBot","company":"Anthropic","purpose":"training","status":"allowed","allowedAll":true,"matchedGroup":"*","paths":[{"path":"/","allowed":true,"rule":"Allow: /"}]},{"bot":"Claude-SearchBot","company":"Anthropic","purpose":"search","status":"allowed","allowedAll":true,"matchedGroup":"*","paths":[{"path":"/","allowed":true,"rule":"Allow: /"}]},{"bot":"Claude-User","company":"Anthropic","purpose":"user","status":"allowed","allowedAll":true,"matchedGroup":"*","paths":[{"path":"/","allowed":true,"rule":"Allow: /"}]},{"bot":"PerplexityBot","company":"Perplexity","purpose":"search","status":"blocked","allowedAll":false,"matchedGroup":"PerplexityBot","paths":[{"path":"/","allowed":false,"rule":"Disallow: /"}]},{"bot":"Perplexity-User","company":"Perplexity","purpose":"user","status":"allowed","allowedAll":true,"matchedGroup":"*","ignoresRobots":true,"paths":[{"path":"/","allowed":true,"rule":"Allow: /"}]},{"bot":"Google-Extended","company":"Google","purpose":"training","status":"blocked","allowedAll":false,"matchedGroup":"Google-Extended","paths":[{"path":"/","allowed":false,"rule":"Disallow: /"}]},{"bot":"Google-CloudVertexBot","company":"Google","purpose":"search","status":"allowed","allowedAll":true,"matchedGroup":"*","paths":[{"path":"/","allowed":true,"rule":"Allow: /"}]},{"bot":"Google-GeminiNotebook","company":"Google","purpose":"user","status":"allowed","allowedAll":true,"matchedGroup":"*","ignoresRobots":true,"paths":[{"path":"/","allowed":true,"rule":"Allow: /"}]},{"bot":"Google-Agent","company":"Google","purpose":"user","status":"allowed","allowedAll":true,"matchedGroup":"*","ignoresRobots":true,"paths":[{"path":"/","allowed":true,"rule":"Allow: /"}]},{"bot":"Applebot","company":"Apple","purpose":"search","status":"blocked","allowedAll":false,"matchedGroup":"Applebot","paths":[{"path":"/","allowed":false,"rule":"Disallow: /"}]},{"bot":"Applebot-Extended","company":"Apple","purpose":"training","status":"blocked","allowedAll":false,"matchedGroup":"Applebot-Extended","paths":[{"path":"/","allowed":false,"rule":"Disallow: /"}]},{"bot":"Meta-ExternalAgent","company":"Meta","purpose":"training","status":"allowed","allowedAll":true,"matchedGroup":"*","paths":[{"path":"/","allowed":true,"rule":"Allow: /"}]},{"bot":"meta-webindexer","company":"Meta","purpose":"search","status":"allowed","allowedAll":true,"matchedGroup":"*","paths":[{"path":"/","allowed":true,"rule":"Allow: /"}]},{"bot":"meta-externalfetcher","company":"Meta","purpose":"user","status":"blocked","allowedAll":false,"matchedGroup":"meta-externalfetcher","ignoresRobots":true,"paths":[{"path":"/","allowed":false,"rule":"Disallow: /"}]},{"bot":"meta-externalads","company":"Meta","purpose":"ads","status":"allowed","allowedAll":true,"matchedGroup":"*","paths":[{"path":"/","allowed":true,"rule":"Allow: /"}]},{"bot":"Amazonbot","company":"Amazon","purpose":"training","status":"allowed","allowedAll":true,"matchedGroup":"Amazonbot","paths":[{"path":"/","allowed":true,"rule":"Allow: /"}]},{"bot":"Amzn-SearchBot","company":"Amazon","purpose":"search","status":"allowed","allowedAll":true,"matchedGroup":"*","paths":[{"path":"/","allowed":true,"rule":"Allow: /"}]},{"bot":"Amzn-User","company":"Amazon","purpose":"user","status":"allowed","allowedAll":true,"matchedGroup":"*","ignoresRobots":true,"paths":[{"path":"/","allowed":true,"rule":"Allow: /"}]},{"bot":"DuckAssistBot","company":"DuckDuckGo","purpose":"search","status":"blocked","allowedAll":false,"matchedGroup":"DuckAssistBot","paths":[{"path":"/","allowed":false,"rule":"Disallow: /"}]},{"bot":"MistralAI-User","company":"Mistral","purpose":"user","status":"allowed","allowedAll":true,"matchedGroup":"*","paths":[{"path":"/","allowed":true,"rule":"Allow: /"}]},{"bot":"MistralAI-Index","company":"Mistral","purpose":"search","status":"allowed","allowedAll":true,"matchedGroup":"*","paths":[{"path":"/","allowed":true,"rule":"Allow: /"}]},{"bot":"MistralAI-Training","company":"Mistral","purpose":"training","status":"allowed","allowedAll":true,"matchedGroup":"*","paths":[{"path":"/","allowed":true,"rule":"Allow: /"}]},{"bot":"CCBot","company":"Common Crawl","purpose":"training","status":"blocked","allowedAll":false,"matchedGroup":"CCBot","paths":[{"path":"/","allowed":false,"rule":"Disallow: /"}]},{"bot":"Bytespider","company":"ByteDance","purpose":"training","status":"blocked","allowedAll":false,"matchedGroup":"Bytespider","paths":[{"path":"/","allowed":false,"rule":"Disallow: /"}]},{"bot":"cohere-ai","company":"Cohere","purpose":"training","status":"blocked","allowedAll":false,"matchedGroup":"cohere-ai","paths":[{"path":"/","allowed":false,"rule":"Disallow: /"}]}],"recommendedRobotsSnippet":"# Allow AI search and assistant crawlers (they decide whether the site can appear in AI answers)\n# Also remove \"Disallow: /\" from the existing group(s) for: PerplexityBot, Applebot, meta-externalfetcher, DuckAssistBot\nUser-agent: PerplexityBot\nUser-agent: Applebot\nUser-agent: meta-externalfetcher\nUser-agent: DuckAssistBot\nAllow: /\nDisallow: /ultimora/24ore/nazionale/news-dettaglio/3688417\nDisallow: /ultimora/24ore/AUTO-IN-CAMBIO-DI-DROGA-13-ARRESTI-DELLA-POLIZIA-DI-PESCARA-(2)/news-dettaglio/3746479\nDisallow: /sport/ciclismo/2023/10/24/news/doping_ciclismo_virtuale_azzurro_zanasca_rischia_una_lunga_squalifica-418660589/\nDisallow: /spettacoli/people/2023/09/07/news/kanye_west_e_bianca_censori_in_italia_la_lunga_estate_calda_degli_scandali-413629176/<\nDisallow: /speciale/\nDisallow: /social/sites/repubblica/d/boxes/shares/\nDisallow: /servizi/\nDisallow: /ricerca/\nDisallow: /politica/2017/01/19/news/il_fratello_del_ministro_alfano_rischia_il_posto_e_costera_all_erario_mezzo_milione_di_euro[1]156371739/\nDisallow: /politica/2/10/13/news/da_maroni_a_fontana_c_e_sempre_quel_notaio_nelle_indagini_sulla_lega-301070805/\nDisallow: /persone/\nDisallow: /online/cronaca/veronaecstasy/bologna/bologna.html\nDisallow: /online/\nDisallow: /news/ired/ultimora/cronaca/rep_cronaca_n_3487677.html\nDisallow: /java\nDisallow: /ipad/\nDisallow: /interstitial/\nDisallow: /indici/\nDisallow: /il-gusto/2022/04/27/news/frode_fiscale_bancarotta_fraudolenta_arrestato_giovanni_maspero_ristorante_i_tigli_in_theoria-347110162/\nDisallow: /esteri/2024/07/24/diretta/kamala_harris_nel_mirino_dei_repubblicani_usa_ricorso_contro_il_trasferimento_dei_fondi_dalla_campagna_di_biden-423411481/\nDisallow: /embed/*\nDisallow: /economia/miojob/\nDisallow: /economia/2013/01/31/news/mps_verso_sequestro_conservativo- 51615470/\nDisallow: /dal-quotidiano/\nDisallow: /corporate/privacy/index_static.html?\nDisallow: /cgi-bin/back.cgi\nDisallow: /blaize/datalayer\nDisallow: /audio/i-daily-di-repubblica\nDisallow: /audio-test-app\nDisallow: /api/checkpw-v1/premium/amp/pingback*\nDisallow: /api/*\nDisallow: /annunci/\nDisallow: /2010/\nDisallow: /2009/\nDisallow: /2008/\nDisallow: /2007/\nDisallow: /2006/05/gallerie/politica/offese-da-premier-102/esterne130937331310093801_big.jpg\nDisallow: /2006/05/gallerie/politica/farabutti-foto-lettori-95/esterne191030071909103240_big.jpg\nDisallow: /2006/05/gallerie/politica/farabutti-foto-lettori-95/esterne191030071909103240.jpg\nDisallow: /2006/\nDisallow: /2005/\nDisallow: /2004/\nDisallow: /2003/\nDisallow: /coccodrilli/\nDisallow: /swf\nDisallow: /serietv/schede/cerca/?query=\nDisallow: /online3\nDisallow: /mobile/\nDisallow: /ztest/\nDisallow: /dossier/sapori/guide-espresso*\n","suggestedPolicy":"# AI model training crawlers: blocked (does not affect AI search answers)\nUser-agent: GPTBot\nUser-agent: ClaudeBot\nUser-agent: Google-Extended\nUser-agent: Applebot-Extended\nUser-agent: Meta-ExternalAgent\nUser-agent: Amazonbot\nUser-agent: MistralAI-Training\nUser-agent: CCBot\nUser-agent: Bytespider\nUser-agent: cohere-ai\nDisallow: /\n\n# AI search and assistant crawlers: allowed\nUser-agent: OAI-SearchBot\nUser-agent: ChatGPT-User\nUser-agent: Claude-SearchBot\nUser-agent: Claude-User\nUser-agent: PerplexityBot\nUser-agent: Perplexity-User\nUser-agent: Google-CloudVertexBot\nUser-agent: Google-GeminiNotebook\nUser-agent: Google-Agent\nUser-agent: Applebot\nUser-agent: meta-webindexer\nUser-agent: meta-externalfetcher\nUser-agent: Amzn-SearchBot\nUser-agent: Amzn-User\nUser-agent: DuckAssistBot\nUser-agent: MistralAI-User\nUser-agent: MistralAI-Index\nAllow: /\nDisallow: /ultimora/24ore/nazionale/news-dettaglio/3688417\nDisallow: /ultimora/24ore/AUTO-IN-CAMBIO-DI-DROGA-13-ARRESTI-DELLA-POLIZIA-DI-PESCARA-(2)/news-dettaglio/3746479\nDisallow: /sport/ciclismo/2023/10/24/news/doping_ciclismo_virtuale_azzurro_zanasca_rischia_una_lunga_squalifica-418660589/\nDisallow: /spettacoli/people/2023/09/07/news/kanye_west_e_bianca_censori_in_italia_la_lunga_estate_calda_degli_scandali-413629176/<\nDisallow: /speciale/\nDisallow: /social/sites/repubblica/d/boxes/shares/\nDisallow: /servizi/\nDisallow: /ricerca/\nDisallow: /politica/2017/01/19/news/il_fratello_del_ministro_alfano_rischia_il_posto_e_costera_all_erario_mezzo_milione_di_euro[1]156371739/\nDisallow: /politica/2/10/13/news/da_maroni_a_fontana_c_e_sempre_quel_notaio_nelle_indagini_sulla_lega-301070805/\nDisallow: /persone/\nDisallow: /online/cronaca/veronaecstasy/bologna/bologna.html\nDisallow: /online/\nDisallow: /news/ired/ultimora/cronaca/rep_cronaca_n_3487677.html\nDisallow: /java\nDisallow: /ipad/\nDisallow: /interstitial/\nDisallow: /indici/\nDisallow: /il-gusto/2022/04/27/news/frode_fiscale_bancarotta_fraudolenta_arrestato_giovanni_maspero_ristorante_i_tigli_in_theoria-347110162/\nDisallow: /esteri/2024/07/24/diretta/kamala_harris_nel_mirino_dei_repubblicani_usa_ricorso_contro_il_trasferimento_dei_fondi_dalla_campagna_di_biden-423411481/\nDisallow: /embed/*\nDisallow: /economia/miojob/\nDisallow: /economia/2013/01/31/news/mps_verso_sequestro_conservativo- 51615470/\nDisallow: /dal-quotidiano/\nDisallow: /corporate/privacy/index_static.html?\nDisallow: /cgi-bin/back.cgi\nDisallow: /blaize/datalayer\nDisallow: /audio/i-daily-di-repubblica\nDisallow: /audio-test-app\nDisallow: /api/checkpw-v1/premium/amp/pingback*\nDisallow: /api/*\nDisallow: /annunci/\nDisallow: /2010/\nDisallow: /2009/\nDisallow: /2008/\nDisallow: /2007/\nDisallow: /2006/05/gallerie/politica/offese-da-premier-102/esterne130937331310093801_big.jpg\nDisallow: /2006/05/gallerie/politica/farabutti-foto-lettori-95/esterne191030071909103240_big.jpg\nDisallow: /2006/05/gallerie/politica/farabutti-foto-lettori-95/esterne191030071909103240.jpg\nDisallow: /2006/\nDisallow: /2005/\nDisallow: /2004/\nDisallow: /2003/\nDisallow: /coccodrilli/\nDisallow: /swf\nDisallow: /serietv/schede/cerca/?query=\nDisallow: /online3\nDisallow: /mobile/\nDisallow: /ztest/\nDisallow: /dossier/sapori/guide-espresso*\n\n# Everyone else\nUser-agent: *\nDisallow: /ultimora/24ore/nazionale/news-dettaglio/3688417\nDisallow: /ultimora/24ore/AUTO-IN-CAMBIO-DI-DROGA-13-ARRESTI-DELLA-POLIZIA-DI-PESCARA-(2)/news-dettaglio/3746479\nDisallow: /sport/ciclismo/2023/10/24/news/doping_ciclismo_virtuale_azzurro_zanasca_rischia_una_lunga_squalifica-418660589/\nDisallow: /spettacoli/people/2023/09/07/news/kanye_west_e_bianca_censori_in_italia_la_lunga_estate_calda_degli_scandali-413629176/<\nDisallow: /speciale/\nDisallow: /social/sites/repubblica/d/boxes/shares/\nDisallow: /servizi/\nDisallow: /ricerca/\nDisallow: /politica/2017/01/19/news/il_fratello_del_ministro_alfano_rischia_il_posto_e_costera_all_erario_mezzo_milione_di_euro[1]156371739/\nDisallow: /politica/2/10/13/news/da_maroni_a_fontana_c_e_sempre_quel_notaio_nelle_indagini_sulla_lega-301070805/\nDisallow: /persone/\nDisallow: /online/cronaca/veronaecstasy/bologna/bologna.html\nDisallow: /online/\nDisallow: /news/ired/ultimora/cronaca/rep_cronaca_n_3487677.html\nDisallow: /java\nDisallow: /ipad/\nDisallow: /interstitial/\nDisallow: /indici/\nDisallow: /il-gusto/2022/04/27/news/frode_fiscale_bancarotta_fraudolenta_arrestato_giovanni_maspero_ristorante_i_tigli_in_theoria-347110162/\nDisallow: /esteri/2024/07/24/diretta/kamala_harris_nel_mirino_dei_repubblicani_usa_ricorso_contro_il_trasferimento_dei_fondi_dalla_campagna_di_biden-423411481/\nDisallow: /embed/*\nDisallow: /economia/miojob/\nDisallow: /economia/2013/01/31/news/mps_verso_sequestro_conservativo- 51615470/\nDisallow: /dal-quotidiano/\nDisallow: /corporate/privacy/index_static.html?\nDisallow: /cgi-bin/back.cgi\nDisallow: /blaize/datalayer\nDisallow: /audio/i-daily-di-repubblica\nDisallow: /audio-test-app\nDisallow: /api/checkpw-v1/premium/amp/pingback*\nDisallow: /api/*\nDisallow: /annunci/\nDisallow: /2010/\nDisallow: /2009/\nDisallow: /2008/\nDisallow: /2007/\nDisallow: /2006/05/gallerie/politica/offese-da-premier-102/esterne130937331310093801_big.jpg\nDisallow: /2006/05/gallerie/politica/farabutti-foto-lettori-95/esterne191030071909103240_big.jpg\nDisallow: /2006/05/gallerie/politica/farabutti-foto-lettori-95/esterne191030071909103240.jpg\nDisallow: /2006/\nDisallow: /2005/\nDisallow: /2004/\nDisallow: /2003/\nDisallow: /coccodrilli/\nDisallow: /swf\nDisallow: /serietv/schede/cerca/?query=\nDisallow: /online3\nDisallow: /mobile/\nDisallow: /ztest/\nDisallow: /dossier/sapori/guide-espresso*\n\nSitemap: https://www.repubblica.it/sitemap-n.xml\nSitemap: https://video.repubblica.it/sitemap-v-day.xml\nSitemap: https://video.repubblica.it/sitemap-v.xml\nSitemap: https://www.repubblica.it/sitemap-italian-tech-n.xml\nSitemap: https://www.repubblica.it/sitemap-il-gusto-n.xml\nSitemap: https://www.repubblica.it/sitemap-salute-n.xml\nSitemap: https://www.repubblica.it/sitemap-green-and-blue-n.xml\nSitemap: https://www.repubblica.it/sitemap-dirette-sport-n.xml\nSitemap: https://www.repubblica.it/sitemap.xml\nSitemap: https://www.repubblica.it/sitemap-italian-tech.xml\nSitemap: https://www.repubblica.it/sitemap-il-gusto.xml\nSitemap: https://www.repubblica.it/sitemap-salute.xml\nSitemap: https://www.repubblica.it/sitemap-green-and-blue.xml\n","crawlerList":{"count":29,"updated":"2026-10-01","version":"29-19a6fb23"},"tier":"free","upgrade":{"api":"https://tools.yukai.uk/go/ai-crawlers-api?from=free-ai-crawlers&c=upgrade","bulk":"https://tools.yukai.uk/go/ai-crawlers-apify?from=free-ai-crawlers&c=upgrade","all":"https://tools.yukai.uk/llms.txt","note":"More checks per month: AI Crawler Access Checker API (free plan: 100 requests/month). Many sites per run and change monitoring: AI Crawler Access Checker on Apify ($2 per 1,000 sites)."},"index":"https://tools.yukai.uk/ai-crawler-index"}