{"id":364840,"date":"2026-09-07T17:07:17","date_gmt":"2026-09-07T17:07:17","guid":{"rendered":"https:\/\/wordpress.org\/plugins\/crawlwatch-ai-bot-insights\/"},"modified":"2026-09-22T16:31:48","modified_gmt":"2026-09-22T16:31:48","slug":"crawlwatch-ai-bot-insights","status":"publish","type":"plugin","link":"https:\/\/vec.wordpress.org\/plugins\/crawlwatch-ai-bot-insights\/","author":17924966,"comment_status":"closed","ping_status":"closed","template":"","meta":{"version":"1.1.2","stable_tag":"1.1.2","tested":"7.1.2","requires":"6.2","requires_php":"7.4","requires_plugins":null,"header_name":"CrawlWatch \u2013 AI Bot Insights","header_author":"Monowar Hossain","header_description":"See which AI bots read your site, track AI referrals & get AI-ready with llms.txt \u2013 fast, private, no API key. 100% free.","assets_banners_color":"1a4c7e","last_updated":"2026-09-22 16:31:48","external_support_url":"","external_repository_url":"","donate_link":"","header_plugin_uri":"https:\/\/devmonowar.github.io\/crawlwatch-ai-bot-insights\/","header_author_uri":"https:\/\/devmonowar.github.io\/","rating":5,"author_block_rating":0,"active_installs":0,"downloads":524,"num_ratings":2,"support_threads":0,"support_threads_resolved":0,"author_block_count":0,"sections":["description","installation","faq","changelog"],"tags":{"1.0.0":{"tag":"1.0.0","author":"kstmonowar","date":"2026-09-07 17:06:49","revision":3685313},"1.1.0":{"tag":"1.1.0","author":"kstmonowar","date":"2026-09-10 03:01:46","revision":3689237},"1.1.1":{"tag":"1.1.1","author":"kstmonowar","date":"2026-09-21 14:07:07","revision":3705716},"1.1.2":{"tag":"1.1.2","author":"kstmonowar","date":"2026-09-22 16:31:48","revision":3707765}},"upgrade_notice":{"1.1.2":"<p>Privacy + reliability + performance fixes. No settings change needed; rescan schema once to fill the batched report.<\/p>","1.1.1":"<p>Composer distribution via Packagist plus internal QA tooling. No functional changes for existing installs.<\/p>"},"ratings":{"1":0,"2":0,"3":0,"4":0,"5":2},"assets_icons":{"icon-128x128.png":{"filename":"icon-128x128.png","revision":3685313,"resolution":"128x128","location":"assets","locale":"","width":128,"height":128},"icon-256x256.png":{"filename":"icon-256x256.png","revision":3685313,"resolution":"256x256","location":"assets","locale":"","width":256,"height":256}},"assets_banners":{"banner-1544x500.png":{"filename":"banner-1544x500.png","revision":3685313,"resolution":"1544x500","location":"assets","locale":"","width":1544,"height":500},"banner-772x250.png":{"filename":"banner-772x250.png","revision":3685313,"resolution":"772x250","location":"assets","locale":"","width":772,"height":250}},"assets_blueprints":{},"all_blocks":[],"tagged_versions":["1.0.0","1.1.0","1.1.1","1.1.2"],"block_files":[],"assets_screenshots":{"screenshot-1.png":{"filename":"screenshot-1.png","revision":3689237,"resolution":"1","location":"assets","locale":"","width":1920,"height":1080},"screenshot-2.png":{"filename":"screenshot-2.png","revision":3689237,"resolution":"2","location":"assets","locale":"","width":1280,"height":871},"screenshot-3.png":{"filename":"screenshot-3.png","revision":3685313,"resolution":"3","location":"assets","locale":"","width":1280,"height":665},"screenshot-4.png":{"filename":"screenshot-4.png","revision":3689237,"resolution":"4","location":"assets","locale":"","width":1280,"height":760},"screenshot-5.png":{"filename":"screenshot-5.png","revision":3685313,"resolution":"5","location":"assets","locale":"","width":1280,"height":662},"screenshot-6.png":{"filename":"screenshot-6.png","revision":3685313,"resolution":"6","location":"assets","locale":"","width":1280,"height":670},"screenshot-7.png":{"filename":"screenshot-7.png","revision":3689237,"resolution":"7","location":"assets","locale":"","width":1920,"height":1080}},"screenshots":{"1":"Overview dashboard with AI hits, bots, referrals and readiness score.","2":"Bots table with filter, search and pagination.","3":"llms.txt editor + robots.txt manager.","4":"Settings with retention, privacy notice and danger zone.","5":"AI-ready llms.txt served to crawlers.","6":"llms-full.txt with full post content for deep AI context.","7":"AI Readiness page with missing excerpts and alt text plus edit links."}},"plugin_section":[],"plugin_tags":[264107,232780,274092,247966,244604],"plugin_category":[36,55],"plugin_contributors":[269133],"plugin_business_model":[],"class_list":["post-364840","plugin","type-plugin","status-publish","hentry","plugin_tags-ai-bots","plugin_tags-ai-crawler","plugin_tags-bot-traffic","plugin_tags-gptbot","plugin_tags-llms-txt","plugin_category-analytics","plugin_category-seo-and-marketing","plugin_contributors-kstmonowar","plugin_committers-kstmonowar"],"banners":{"banner":"https:\/\/ps.w.org\/crawlwatch-ai-bot-insights\/assets\/banner-772x250.png?rev=3685313","banner_2x":"https:\/\/ps.w.org\/crawlwatch-ai-bot-insights\/assets\/banner-1544x500.png?rev=3685313","banner_rtl":false,"banner_2x_rtl":false},"icons":{"svg":false,"icon":"https:\/\/ps.w.org\/crawlwatch-ai-bot-insights\/assets\/icon-128x128.png?rev=3685313","icon_2x":"https:\/\/ps.w.org\/crawlwatch-ai-bot-insights\/assets\/icon-256x256.png?rev=3685313","generated":false},"screenshots":[{"src":"https:\/\/ps.w.org\/crawlwatch-ai-bot-insights\/assets\/screenshot-1.png?rev=3689237","caption":"Overview dashboard with AI hits, bots, referrals and readiness score."},{"src":"https:\/\/ps.w.org\/crawlwatch-ai-bot-insights\/assets\/screenshot-2.png?rev=3689237","caption":"Bots table with filter, search and pagination."},{"src":"https:\/\/ps.w.org\/crawlwatch-ai-bot-insights\/assets\/screenshot-3.png?rev=3685313","caption":"llms.txt editor + robots.txt manager."},{"src":"https:\/\/ps.w.org\/crawlwatch-ai-bot-insights\/assets\/screenshot-4.png?rev=3689237","caption":"Settings with retention, privacy notice and danger zone."},{"src":"https:\/\/ps.w.org\/crawlwatch-ai-bot-insights\/assets\/screenshot-5.png?rev=3685313","caption":"AI-ready llms.txt served to crawlers."},{"src":"https:\/\/ps.w.org\/crawlwatch-ai-bot-insights\/assets\/screenshot-6.png?rev=3685313","caption":"llms-full.txt with full post content for deep AI context."},{"src":"https:\/\/ps.w.org\/crawlwatch-ai-bot-insights\/assets\/screenshot-7.png?rev=3689237","caption":"AI Readiness page with missing excerpts and alt text plus edit links."}],"raw_content":"<!--section=description-->\n<p>CrawlWatch shows you the invisible traffic: GPTBot, ClaudeBot, PerplexityBot, Google-Extended and 40+ AI crawlers visiting your WordPress site.<\/p>\n\n<p><strong><a href=\"https:\/\/devmonowar.github.io\/blog\/llms-txt-wordpress\/\">Read the llms.txt guide<\/a><\/strong> \u2014 what llms.txt is, the exact format that works, and the mistakes that make one useless \u00b7 <strong><a href=\"https:\/\/devmonowar.github.io\/blog\/ai-bot-traffic-wordpress\/\">AI bot traffic in WordPress<\/a><\/strong> \u2014 how to see it, measure it, and decide what to do about it \u00b7 <strong><a href=\"https:\/\/devmonowar.github.io\/crawlwatch-ai-bot-insights\/\">Plugin page<\/a><\/strong> \u00b7 <strong><a href=\"https:\/\/github.com\/devmonowar\/crawlwatch-ai-bot-insights\">Development on GitHub<\/a><\/strong> \u2014 report issues or contribute.<\/p>\n\n<ul>\n<li>AI bot hits + which URLs they read<\/li>\n<li>AI referrals (visitors coming from ChatGPT, Perplexity, Gemini, Claude)<\/li>\n<li>One-click llms.txt + ai.txt generator<\/li>\n<li>robots.txt allow\/block per AI bot (preview, no code)<\/li>\n<li>Readiness Score 0-100 with ranked Quick Wins<\/li>\n<li>100% free, no upsell. Data stays on your server. IPs are hashed, never stored raw.<\/li>\n<\/ul>\n\n<p>No API key. No external service in V1. Lightweight: normal visitors cause zero extra queries.<\/p>\n\n<h3>External services<\/h3>\n\n<p>None. CrawlWatch talks to no third-party service \u2014 the only HTTP requests it makes are loopback fetches of your own posts for the on-demand schema scan. Everything is stored and computed on your own server.<\/p>\n\n<h3>Privacy<\/h3>\n\n<p>CrawlWatch logs AI crawler visits locally: time, bot name, visited URL, referrer domain, truncated user-agent, and SHA-256 hashed IP (IPv4 last octet and IPv6 \/64 anonymized before hashing). Raw IPs are never stored. Retention is configurable (default 30 days) and all data is deleted on uninstall if enabled in Settings. No visitor data leaves your site.<\/p>\n\n<!--section=installation-->\n<ol>\n<li>Upload the plugin folder to <code>\/wp-content\/plugins\/<\/code> or install from Plugins &gt; Add New.<\/li>\n<li>Activate through the Plugins screen.<\/li>\n<li>Go to CrawlWatch in the left menu and follow the 3-step setup.<\/li>\n<li>Open <code>yoursite.com\/llms.txt<\/code> to verify (plain permalinks or subdirectory installs: use <code>yoursite.com\/?crawlwatch_file=llms.txt<\/code>).<\/li>\n<\/ol>\n\n<!--section=faq-->\n<dl>\n<dt id=\"does%20this%20send%20data%20outside%20my%20site%3F\"><h3>Does this send data outside my site?<\/h3><\/dt>\n<dd><p>No. Everything is stored locally in one table (<code>wp_crawlwatch_logs<\/code>). IPs are SHA-256 hashed (IPv4 last octet and IPv6 \/64 anonymized first). No third-party calls \u2014 the only HTTP requests CrawlWatch ever makes are loopback fetches of your own posts for the on-demand schema scan.<\/p><\/dd>\n<dt id=\"will%20it%20slow%20my%20site%3F\"><h3>Will it slow my site?<\/h3><\/dt>\n<dd><p>No. Normal human visits do zero extra queries. Only detected AI hits do one dedupe check plus one insert. No CSS\/JS on the frontend.<\/p><\/dd>\n<dt id=\"how%20long%20are%20logs%20kept%3F\"><h3>How long are logs kept?<\/h3><\/dt>\n<dd><p>Default 30 days (7\/14\/30\/90\/180 selectable). Daily WP-Cron auto-deletes older rows.<\/p><\/dd>\n<dt id=\"does%20it%20work%20without%20yoast%2Frankmath%3F\"><h3>Does it work without Yoast\/RankMath?<\/h3><\/dt>\n<dd><p>Yes. It only detects whether an SEO plugin is active (for the schema part of the score). No integration needed.<\/p><\/dd>\n<dt id=\"will%20blocking%20a%20bot%20hurt%20my%20seo%3F\"><h3>Will blocking a bot hurt my SEO?<\/h3><\/dt>\n<dd><p>Blocking AI training bots (GPTBot, ClaudeBot) does not affect Google search. Never block regular Googlebot \u2014 CrawlWatch only manages AI bots, and Google-Extended asks for confirmation first.<\/p><\/dd>\n<dt id=\"why%20do%20i%20see%20few%20or%20zero%20hits%20with%20a%20cache%20plugin%3F\"><h3>Why do I see few or zero hits with a cache plugin?<\/h3><\/dt>\n<dd><p>Tracking runs in PHP. Full-page caches (WP Rocket, LiteSpeed, W3 Total Cache, Cloudflare APO) serve cached pages without running PHP, so those visits are never logged. CrawlWatch shows a notice on its own pages when a known cache is active.<\/p><\/dd>\n<dt id=\"how%20do%20i%20block%20gptbot%20in%20wordpress%3F\"><h3>How do I block GPTBot in WordPress?<\/h3><\/dt>\n<dd><p>Add a <code>Disallow<\/code> rule for GPTBot in robots.txt \u2014 CrawlWatch writes it for you with preview. That keeps your content out of OpenAI training data. It does not remove anything already trained on.<\/p><\/dd>\n<dt id=\"does%20blocking%20gptbot%20stop%20chatgpt%20from%20citing%20my%20site%3F\"><h3>Does blocking GPTBot stop ChatGPT from citing my site?<\/h3><\/dt>\n<dd><p>No \u2014 that is a different bot. GPTBot governs training; OAI-SearchBot governs ChatGPT Search citability. Blocking GPTBot hides you from training while leaving you citable. The same split exists at Anthropic (ClaudeBot vs Claude-SearchBot). Google-Extended only affects Gemini training, never AI Overviews.<\/p><\/dd>\n<dt id=\"where%20does%20llms.txt%20go%20on%20a%20wordpress%20site%3F\"><h3>Where does llms.txt go on a WordPress site?<\/h3><\/dt>\n<dd><p>At the domain root (<code>yoursite.com\/llms.txt<\/code>). CrawlWatch generates and serves it with no file writes. Anything deeper and most consumers never look.<\/p><\/dd>\n<dt id=\"is%20llms.txt%20actually%20read%20by%20ai%20assistants%3F\"><h3>Is llms.txt actually read by AI assistants?<\/h3><\/dt>\n<dd><p>Some read it, some do not \u2014 adoption is limited and Google does not use it for search. It costs nothing to have, and it makes the assistants that do read it describe you accurately.<\/p><\/dd>\n\n<\/dl>\n\n<!--section=changelog-->\n<h4>1.1.2<\/h4>\n\n<ul>\n<li>Privacy: only the referrer host is stored now (path and query never reach the database), matching the Privacy statement.<\/li>\n<li>Reliability: weekly digest is re-scheduled on reactivation; llms.txt rebuild is debounced into one cron run; schema scan runs in batches of 5 with continue button.<\/li>\n<li>Performance: Woo top products cached 12h; AI Readiness and Score queries no longer N+1.<\/li>\n<li>Tooling: PHPCS WordPress-Extra clean, PHPStan 2, CI (lint + stan + phpcs + phpunit) on push\/PR.<\/li>\n<\/ul>\n\n<h4>1.1.1<\/h4>\n\n<ul>\n<li>Packagist distribution (composer.json) + full QA tooling: PHPCS, PHPStan level 5, PHPUnit.<\/li>\n<\/ul>\n\n<h4>1.1.0<\/h4>\n\n<ul>\n<li>Grouped-by-bot tab, daily hits trend chart, score breakdown, Content Gaps renamed to AI Readiness.<\/li>\n<li>Robots preview fallback for subdirectory\/plain-permalink installs, Quick Wins link to AI Readiness, CSV formula guard with site-timezone times, page-cache notice + FAQ, uninstall row-count hint.<\/li>\n<li>Fixes: grouped-stats prepare notice, Reset keeps active tab, compact chart layout, Overview submenu removed.<\/li>\n<\/ul>\n\n<h4>1.0.0<\/h4>\n\n<ul>\n<li>Initial release: AI bot + referral tracker (40+ crawlers), Overview dashboard with score + Quick Wins, Bots filter\/search\/pagination, llms.txt + ai.txt generator (no file writes), robots.txt AI allow\/block manager, Settings (retention\/privacy\/clear logs), 3-step setup wizard. 100% free, local-only, hashed IPs.<\/li>\n<\/ul>","raw_excerpt":"Track AI bot traffic on WordPress \u2013 see GPTBot, ClaudeBot &amp; 40+ bots, block unwanted crawlers, generate llms.txt. Free.","jetpack_sharing_enabled":true,"_links":{"self":[{"href":"https:\/\/vec.wordpress.org\/plugins\/wp-json\/wp\/v2\/plugin\/364840","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/vec.wordpress.org\/plugins\/wp-json\/wp\/v2\/plugin"}],"about":[{"href":"https:\/\/vec.wordpress.org\/plugins\/wp-json\/wp\/v2\/types\/plugin"}],"replies":[{"embeddable":true,"href":"https:\/\/vec.wordpress.org\/plugins\/wp-json\/wp\/v2\/comments?post=364840"}],"author":[{"embeddable":true,"href":"https:\/\/vec.wordpress.org\/plugins\/wp-json\/wporg\/v1\/users\/kstmonowar"}],"wp:attachment":[{"href":"https:\/\/vec.wordpress.org\/plugins\/wp-json\/wp\/v2\/media?parent=364840"}],"wp:term":[{"taxonomy":"plugin_section","embeddable":true,"href":"https:\/\/vec.wordpress.org\/plugins\/wp-json\/wp\/v2\/plugin_section?post=364840"},{"taxonomy":"plugin_tags","embeddable":true,"href":"https:\/\/vec.wordpress.org\/plugins\/wp-json\/wp\/v2\/plugin_tags?post=364840"},{"taxonomy":"plugin_category","embeddable":true,"href":"https:\/\/vec.wordpress.org\/plugins\/wp-json\/wp\/v2\/plugin_category?post=364840"},{"taxonomy":"plugin_contributors","embeddable":true,"href":"https:\/\/vec.wordpress.org\/plugins\/wp-json\/wp\/v2\/plugin_contributors?post=364840"},{"taxonomy":"plugin_business_model","embeddable":true,"href":"https:\/\/vec.wordpress.org\/plugins\/wp-json\/wp\/v2\/plugin_business_model?post=364840"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}