Skip to content

Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.

feat(traffic): report which AI bots read the site, and cited pages no search bot read (#1182)

Cihan9a877a6ff4e5
diff --git a/supabase/migrations/00185_ai_bot_report.sql b/supabase/migrations/00185_ai_bot_report.sql
new file mode 100644
index 000000000..8e45de672
--- /dev/null
+++ b/supabase/migrations/00185_ai_bot_report.sql
@@ -0,0 +1,205 @@
+-- ============================================================================
+-- 00185: the AI bot report (AT-011)
+-- ============================================================================
+-- Spec §3.7, "AI bots reading your site": the verified AI-bot requests a
+-- brand's Cloudflare or Vercel sent (00183), read from the bot × page × day
+-- counters, which hold every request even past the month's cap:
+--
+-- * types — training crawlers, search crawlers and user fetches, reported
+-- separately: hits, pages read and when last seen, with the bots seen;
+-- * bots — the same per bot;
+-- * pages — the most read pages, with when each type last read them (the
+-- per-page last crawl);
+-- * uncrawled — the alert "cited pages not read by any search bot in 30
+-- days": pages tracked AI answers cited in the last 30 days that no
+-- search crawler read in them. It only means something while the bot
+-- data is whole, so it is null, and alert_state says why, until the
+-- first request is 30 days old ('waiting'), and when no live connection
+-- has sent for 2 days or search hits went on '(other)' rows past the
+-- month's cap in those 30 days ('stale': a sender stopped, was revoked
+-- or hit its raw cap, and the 30 days of pages read would drain).
+--
+-- "All time" (p_since null) covers the last 90 days here: a site crawled at
+-- 10k pages a day keeps millions of rows over the counters' 400 days. The
+-- rows are added up once by bot and page, and the first and last day come
+-- from the ends of the (brand_id, day) index.
+--
+-- Pages are keyed as the Pages section keys them (00178). citation_urls has
+-- no member select policy, so this runs as its owner after checking the
+-- caller belongs to the brand's organization; a caller from elsewhere gets
+-- nothing.
+
+create or replace function public.ai_bot_report(
+ p_brand_id uuid,
+ p_since date,
+ p_until date,
+ p_limit integer default 20
+)
+returns jsonb
+language plpgsql
+stable
+security definer
+set search_path to 'public'
+as $$
+declare
+ v_first date;
+ v_last_day date;
+ v_last timestamptz;
+ v_since date;
+ v_alert text;
+ v_result jsonb;
+begin
+ if not exists (
+ select 1 from public.brands b
+ join public.profiles pf on pf.organization_id = b.organization_id
+ where b.id = p_brand_id and pf.id = auth.uid()
+ ) and auth.role() is distinct from 'service_role' then
+ return null;
+ end if;
+
+ select d.day into v_first
+ from public.ai_bot_daily d
+ where d.brand_id = p_brand_id
+ order by d.day
+ limit 1;
+ select d.day into v_last_day
+ from public.ai_bot_daily d
+ where d.brand_id = p_brand_id
+ order by d.day desc
+ limit 1;
+ select max(d.last_seen) into v_last
+ from public.ai_bot_daily d
+ where d.brand_id = p_brand_id and d.day = v_last_day;
+
+ v_since := coalesce(p_since, current_date - 89);
+
+ v_alert := case
+ when v_first is null or v_first > current_date - 29 then 'waiting'
+ when not exists (
+ select 1 from public.ai_bot_connections c
+ where c.brand_id = p_brand_id
+ and c.revoked_at is null
+ and c.last_event_at >= now() - interval '2 days'
+ ) or exists (
+ select 1 from public.ai_bot_daily d
+ where d.brand_id = p_brand_id
+ and d.bot_type = 'search'
+ and d.page = '(other)'
+ and d.day >= current_date - 29
+ ) then 'stale'
+ else 'on'
+ end;
+
+ with g as (
+ -- Each bot's hits on each page in the range, added up once.
+ select
+ d.bot_type,
+ d.bot,
+ d.page,
+ sum(d.hits)::bigint as hits,
+ max(d.last_seen) as last_seen
+ from public.ai_bot_daily d
+ where d.brand_id = p_brand_id
+ and d.day >= v_since
+ and d.day <= p_until
+ group by d.bot_type, d.bot, d.page
+ ),
+ types as (
+ select
+ g.bot_type,
+ sum(g.hits)::bigint as hits,
+ count(distinct g.page) filter (where g.page <> '(other)')::bigint as pages,
+ max(g.last_seen) as last_seen,
+ array_agg(distinct g.bot order by g.bot) as bots
+ from g
+ group by g.bot_type
+ ),
+ bots as (
+ select
+ g.bot,
+ min(g.bot_type) as bot_type,
+ sum(g.hits)::bigint as hits,
+ count(*) filter (where g.page <> '(other)')::bigint as pages,
+ max(g.last_seen) as last_seen
+ from g
+ group by g.bot
+ ),
+ -- Hits folded past the month's cap ('(other)') aren't a page.
+ pages as (
+ select
+ g.page,
+ sum(g.hits)::bigint as hits,
+ max(g.last_seen) filter (where g.bot_type = 'training') as training_last,
+ max(g.last_seen) filter (where g.bot_type = 'search') as search_last,
+ max(g.last_seen) filter (where g.bot_type = 'user') as user_last
+ from g
+ where g.page <> '(other)'
+ group by g.page
+ ),
+ -- The alert reads its own 30 days, whatever the range.
+ uncrawled as (
+ select c.page, c.citations
+ from public.ai_traffic_page_citations(p_brand_id, current_date - 29, current_date) c
+ where v_alert = 'on'
+ and not exists (
+ select 1 from public.ai_bot_daily d
+ where d.brand_id = p_brand_id
+ and d.bot_type = 'search'
+ and d.page = c.page_key
+ and d.day >= current_date - 29
+ )
+ )
+ select jsonb_build_object(
+ 'first_day', v_first,
+ 'last_seen', v_last,
+ 'types', coalesce((
+ select jsonb_agg(
+ jsonb_build_object(
+ 'type', t.bot_type, 'hits', t.hits, 'pages', t.pages,
+ 'last_seen', t.last_seen, 'bots', to_jsonb(t.bots)
+ )
+ order by array_position(array['training', 'search', 'user'], t.bot_type)
+ )
+ from types t
+ ), '[]'::jsonb),
+ 'bots', coalesce((
+ select jsonb_agg(
+ jsonb_build_object(
+ 'bot', b.bot, 'type', b.bot_type, 'hits', b.hits, 'pages', b.pages,
+ 'last_seen', b.last_seen
+ )
+ order by b.hits desc, b.bot
+ )
+ from bots b
+ ), '[]'::jsonb),
+ 'page_count', (select count(*) from pages),
+ 'pages', coalesce((
+ select jsonb_agg(to_jsonb(p) order by p.hits desc, p.page)
+ from (select * from pages order by hits desc, page limit p_limit) p
+ ), '[]'::jsonb),
+ -- Null unless the bot data is whole (alert_state 'on').
+ 'uncrawled_count', case when v_alert = 'on' then (select count(*) from uncrawled) end,
+ 'uncrawled', case
+ when v_alert = 'on' then coalesce((
+ select jsonb_agg(to_jsonb(u) order by u.citations desc, u.page)
+ from (select * from uncrawled order by citations desc, page limit 10) u
+ ), '[]'::jsonb)
+ end,
+ 'alert_state', v_alert,
+ 'alert_from', case when v_first is not null then v_first + 29 end,
+ 'since', v_since
+ )
+ into v_result;
+
+ return v_result;
+end;
+$$;
+
+revoke all on function public.ai_bot_report(uuid, date, date, integer) from public, anon;
+grant execute on function public.ai_bot_report(uuid, date, date, integer)
+ to authenticated, service_role;
+
+comment on function public.ai_bot_report(uuid, date, date, integer) is
+ 'AI Traffic "AI bots reading your site" (AT-011, spec §3.7): verified AI-bot hits between two days by type (training, search, user) and by bot, the most read pages with each type''s last read, and the cited pages no search bot read in the last 30 days (null until 30 days of bot data). Security definer with an organization check.';
+
+notify pgrst, 'reload schema';
diff --git a/supabase/schema.sql b/supabase/schema.sql
index 0daae23f8..8374cfa72 100644
--- a/supabase/schema.sql
+++ b/supabase/schema.sql
@@ -25312,3 +25312,212 @@ comment on function public.ai_bot_count_raw(uuid, bigint, bigint) is
notify pgrst, 'reload schema';
+-- ─────────────────────────────────────────────────────────────────────────
+-- migrations/00185_ai_bot_report.sql
+-- ─────────────────────────────────────────────────────────────────────────
+-- ============================================================================
+-- 00185: the AI bot report (AT-011)
+-- ============================================================================
+-- Spec §3.7, "AI bots reading your site": the verified AI-bot requests a
+-- brand's Cloudflare or Vercel sent (00183), read from the bot × page × day
+-- counters, which hold every request even past the month's cap:
+--
+-- * types — training crawlers, search crawlers and user fetches, reported
+-- separately: hits, pages read and when last seen, with the bots seen;
+-- * bots — the same per bot;
+-- * pages — the most read pages, with when each type last read them (the
+-- per-page last crawl);
+-- * uncrawled — the alert "cited pages not read by any search bot in 30
+-- days": pages tracked AI answers cited in the last 30 days that no
+-- search crawler read in them. It only means something while the bot
+-- data is whole, so it is null, and alert_state says why, until the
+-- first request is 30 days old ('waiting'), and when no live connection
+-- has sent for 2 days or search hits went on '(other)' rows past the
+-- month's cap in those 30 days ('stale': a sender stopped, was revoked
+-- or hit its raw cap, and the 30 days of pages read would drain).
+--
+-- "All time" (p_since null) covers the last 90 days here: a site crawled at
+-- 10k pages a day keeps millions of rows over the counters' 400 days. The
+-- rows are added up once by bot and page, and the first and last day come
+-- from the ends of the (brand_id, day) index.
+--
+-- Pages are keyed as the Pages section keys them (00178). citation_urls has
+-- no member select policy, so this runs as its owner after checking the
+-- caller belongs to the brand's organization; a caller from elsewhere gets
+-- nothing.
+
+create or replace function public.ai_bot_report(
+ p_brand_id uuid,
+ p_since date,
+ p_until date,
+ p_limit integer default 20
+)
+returns jsonb
+language plpgsql
+stable
+security definer
+set search_path to 'public'
+as $$
+declare
+ v_first date;
+ v_last_day date;
+ v_last timestamptz;
+ v_since date;
+ v_alert text;
+ v_result jsonb;
+begin
+ if not exists (
+ select 1 from public.brands b
+ join public.profiles pf on pf.organization_id = b.organization_id
+ where b.id = p_brand_id and pf.id = auth.uid()
+ ) and auth.role() is distinct from 'service_role' then
+ return null;
+ end if;
+
+ select d.day into v_first
+ from public.ai_bot_daily d
+ where d.brand_id = p_brand_id
+ order by d.day
+ limit 1;
+ select d.day into v_last_day
+ from public.ai_bot_daily d
+ where d.brand_id = p_brand_id
+ order by d.day desc
+ limit 1;
+ select max(d.last_seen) into v_last
+ from public.ai_bot_daily d
+ where d.brand_id = p_brand_id and d.day = v_last_day;
+
+ v_since := coalesce(p_since, current_date - 89);
+
+ v_alert := case
+ when v_first is null or v_first > current_date - 29 then 'waiting'
+ when not exists (
+ select 1 from public.ai_bot_connections c
+ where c.brand_id = p_brand_id
+ and c.revoked_at is null
+ and c.last_event_at >= now() - interval '2 days'
+ ) or exists (
+ select 1 from public.ai_bot_daily d
+ where d.brand_id = p_brand_id
+ and d.bot_type = 'search'
+ and d.page = '(other)'
+ and d.day >= current_date - 29
+ ) then 'stale'
+ else 'on'
+ end;
+
+ with g as (
+ -- Each bot's hits on each page in the range, added up once.
+ select
+ d.bot_type,
+ d.bot,
+ d.page,
+ sum(d.hits)::bigint as hits,
+ max(d.last_seen) as last_seen
+ from public.ai_bot_daily d
+ where d.brand_id = p_brand_id
+ and d.day >= v_since
+ and d.day <= p_until
+ group by d.bot_type, d.bot, d.page
+ ),
+ types as (
+ select
+ g.bot_type,
+ sum(g.hits)::bigint as hits,
+ count(distinct g.page) filter (where g.page <> '(other)')::bigint as pages,
+ max(g.last_seen) as last_seen,
+ array_agg(distinct g.bot order by g.bot) as bots
+ from g
+ group by g.bot_type
+ ),
+ bots as (
+ select
+ g.bot,
+ min(g.bot_type) as bot_type,
+ sum(g.hits)::bigint as hits,
+ count(*) filter (where g.page <> '(other)')::bigint as pages,
+ max(g.last_seen) as last_seen
+ from g
+ group by g.bot
+ ),
+ -- Hits folded past the month's cap ('(other)') aren't a page.
+ pages as (
+ select
+ g.page,
+ sum(g.hits)::bigint as hits,
+ max(g.last_seen) filter (where g.bot_type = 'training') as training_last,
+ max(g.last_seen) filter (where g.bot_type = 'search') as search_last,
+ max(g.last_seen) filter (where g.bot_type = 'user') as user_last
+ from g
+ where g.page <> '(other)'
+ group by g.page
+ ),
+ -- The alert reads its own 30 days, whatever the range.
+ uncrawled as (
+ select c.page, c.citations
+ from public.ai_traffic_page_citations(p_brand_id, current_date - 29, current_date) c
+ where v_alert = 'on'
+ and not exists (
+ select 1 from public.ai_bot_daily d
+ where d.brand_id = p_brand_id
+ and d.bot_type = 'search'
+ and d.page = c.page_key
+ and d.day >= current_date - 29
+ )
+ )
+ select jsonb_build_object(
+ 'first_day', v_first,
+ 'last_seen', v_last,
+ 'types', coalesce((
+ select jsonb_agg(
+ jsonb_build_object(
+ 'type', t.bot_type, 'hits', t.hits, 'pages', t.pages,
+ 'last_seen', t.last_seen, 'bots', to_jsonb(t.bots)
+ )
+ order by array_position(array['training', 'search', 'user'], t.bot_type)
+ )
+ from types t
+ ), '[]'::jsonb),
+ 'bots', coalesce((
+ select jsonb_agg(
+ jsonb_build_object(
+ 'bot', b.bot, 'type', b.bot_type, 'hits', b.hits, 'pages', b.pages,
+ 'last_seen', b.last_seen
+ )
+ order by b.hits desc, b.bot
+ )
+ from bots b
+ ), '[]'::jsonb),
+ 'page_count', (select count(*) from pages),
+ 'pages', coalesce((
+ select jsonb_agg(to_jsonb(p) order by p.hits desc, p.page)
+ from (select * from pages order by hits desc, page limit p_limit) p
+ ), '[]'::jsonb),
+ -- Null unless the bot data is whole (alert_state 'on').
+ 'uncrawled_count', case when v_alert = 'on' then (select count(*) from uncrawled) end,
+ 'uncrawled', case
+ when v_alert = 'on' then coalesce((
+ select jsonb_agg(to_jsonb(u) order by u.citations desc, u.page)
+ from (select * from uncrawled order by citations desc, page limit 10) u
+ ), '[]'::jsonb)
+ end,
+ 'alert_state', v_alert,
+ 'alert_from', case when v_first is not null then v_first + 29 end,
+ 'since', v_since
+ )
+ into v_result;
+
+ return v_result;
+end;
+$$;
+
+revoke all on function public.ai_bot_report(uuid, date, date, integer) from public, anon;
+grant execute on function public.ai_bot_report(uuid, date, date, integer)
+ to authenticated, service_role;
+
+comment on function public.ai_bot_report(uuid, date, date, integer) is
+ 'AI Traffic "AI bots reading your site" (AT-011, spec §3.7): verified AI-bot hits between two days by type (training, search, user) and by bot, the most read pages with each type''s last read, and the cited pages no search bot read in the last 30 days (null until 30 days of bot data). Security definer with an organization check.';
+
+notify pgrst, 'reload schema';
+
diff --git a/web/messages/en.json b/web/messages/en.json
index febf6ccb9..836af8ee2 100644
--- a/web/messages/en.json
+++ b/web/messages/en.json
@@ -3136,6 +3136,8 @@
"pageStatus": "Pages by status",
"googlePages": "Is AI taking Google clicks",
"googleQueries": "Untracked Google queries",
+ "bots": "AI bots",
+ "botPages": "Pages AI bots read",
"visits": "Visits"
},
"columns": {
@@ -3183,7 +3185,15 @@
"previousClicks": "Clicks, previous period",
"days": "Days",
"previousDays": "Days, previous period",
- "query": "Query"
+ "query": "Query",
+ "bot": "Bot",
+ "botType": "Bot type",
+ "hits": "Hits",
+ "pagesRead": "Pages read",
+ "lastSeen": "Last seen (UTC)",
+ "trainingLast": "Training, last read (UTC)",
+ "searchLast": "Search index, last read (UTC)",
+ "userLast": "User fetch, last read (UTC)"
},
"metrics": {
"visits": "AI-referred visits",
@@ -3207,7 +3217,13 @@
"googleHistoryShort": "Google Search Console's page history doesn't reach back to the period before this range",
"noGoogleQueries": "No query data from Google Search Console yet",
"noUntrackedQueries": "Every Google query with clicks in this range is tracked",
- "queriesCapped": "Only the 5,000 most clicked queries were checked"
+ "queriesCapped": "Only the 5,000 most clicked queries were checked",
+ "noBots": "No AI-bot requests in this range"
+ },
+ "botTypes": {
+ "training": "Training",
+ "search": "Search index",
+ "user": "User fetch"
},
"sourceStates": {
"active": "Active",
@@ -3420,6 +3436,42 @@
]
}
},
+ "botReport": {
+ "title": "AI bots reading your site",
+ "needs": "Growth and above · needs Cloudflare or Vercel",
+ "typesTitle": "By bot type",
+ "pagesTitle": "Pages AI bots read",
+ "columns": {
+ "type": "Bot type",
+ "hits": "Hits",
+ "pages": "Pages",
+ "lastSeen": "Last seen",
+ "page": "Page"
+ },
+ "types": {
+ "training": "Training",
+ "search": "Search index",
+ "user": "User fetch"
+ },
+ "overlayTitle": "See which AI bots read your pages",
+ "connectBody": "Connect Cloudflare or Vercel in 2 minutes. Verified bots only. No change to your site's speed.",
+ "lockedBody": "The AI bot report comes with the Growth plan. Verified bots only, from your own Cloudflare or Vercel.",
+ "connect": "Connect",
+ "preview": "Preview with sample data",
+ "upgrade": "Upgrade",
+ "sampleNote": "Sample data: connect Cloudflare or Vercel to see yours.",
+ "waiting": "Waiting for the first AI-bot request from your Cloudflare or Vercel. Requests show here as soon as they arrive.",
+ "emptyRange": "No AI bot read your site in this range.",
+ "failed": "Couldn't load the AI bot report.",
+ "morePages": "{shown} of {count} pages · the export has them all",
+ "uncrawled": "{count, plural, one {# page AI answers cited wasn't read by any search bot in the last 30 days} other {# pages AI answers cited weren't read by any search bot in the last 30 days}}",
+ "cited": "{count, plural, one {cited # time} other {cited # times}}",
+ "moreUncrawled": "and {count} more",
+ "allCrawled": "Every page AI answers cited in the last 30 days was read by a search bot.",
+ "alertStarts": "The search-bot alert starts on {day}, 30 days after the first AI-bot request.",
+ "alertStale": "The search-bot alert is paused: no connection has sent AI-bot requests in the last 2 days, or the month's cap was reached, so the pages search bots read aren't complete.",
+ "allTimeCapped": "last 90 days"
+ },
"gate": {
"needs": "Needs {source}",
"notConnectedTitle": "Connect {source} to see this",
diff --git a/web/src/app/[locale]/(dashboard)/dashboard/traffic/_bots-connect.tsx b/web/src/app/[locale]/(dashboard)/dashboard/traffic/_bots-connect.tsx
index d2e1de3f5..273f93af5 100644
--- a/web/src/app/[locale]/(dashboard)/dashboard/traffic/_bots-connect.tsx
+++ b/web/src/app/[locale]/(dashboard)/dashboard/traffic/_bots-connect.tsx
@@ -321,7 +321,19 @@ export function BotsDialog({
* waiting until the first request, then the last request's time; Manage
* opens the same dialog.
*/
-export function BotsSource({ brandId, canConnect }: { brandId: string; canConnect: boolean }) {
+export function BotsSource({
+ brandId,
+ canConnect,
+ revision = 0,
+ onChanged,
+}: {
+ brandId: string;
+ canConnect: boolean;
+ /** Bumped when a connection is made or revoked anywhere on the page. */
+ revision?: number;
+ /** A connection was made or revoked here. */
+ onChanged?: () => void;
+}) {
const t = useTranslations('traffic.sources');
const tCommon = useTranslations('common');
const [info, setInfo] = useState<BotConnections | { error: string } | null>(null);
@@ -335,7 +347,7 @@ export function BotsSource({ brandId, canConnect }: { brandId: string; canConnec
useEffect(() => {
load();
- }, [load]);
+ }, [load, revision]);
const title = t('bots');
if (!info) return <SourceCard tone="off" title={title} line={t('botsLoading')} />;
@@ -407,7 +419,7 @@ export function BotsSource({ brandId, canConnect }: { brandId: string; canConnec
info={info}
canConnect={canConnect}
onOpenChange={setOpen}
- onChanged={load}
+ onChanged={() => (onChanged ? onChanged() : load())}
/>
)}
</>
diff --git a/web/src/app/[locale]/(dashboard)/dashboard/traffic/_bots-report.test.tsx b/web/src/app/[locale]/(dashboard)/dashboard/traffic/_bots-report.test.tsx
new file mode 100644
index 000000000..e8168a81a
--- /dev/null
+++ b/web/src/app/[locale]/(dashboard)/dashboard/traffic/_bots-report.test.tsx
@@ -0,0 +1,244 @@
+import { act } from 'react';
+import { createRoot, type Root } from 'react-dom/client';
+import { NextIntlClientProvider } from 'next-intl';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import messages from '../../../../../../messages/en.json';
+import type { BotConnections } from '@/lib/actions/ai-bots';
+import type { TrafficBots, TrafficDateWindow } from '@/lib/actions/traffic';
+
+/**
+ * AI bots reading your site (AT-011, spec §3.7, §7): Starter locked with
+ * sample data and Upgrade; sample data under Connect with a preview; waiting
+ * for the first request; then bot types, pages and the search-bot alert.
+ */
+
+const getBotConnections = vi.fn();
+const createBotConnection = vi.fn();
+vi.mock('@/lib/actions/ai-bots', () => ({
+ getBotConnections: (...a: unknown[]) => getBotConnections(...a),
+ createBotConnection: (...a: unknown[]) => createBotConnection(...a),
+ revokeBotConnection: vi.fn(),
+}));
+const getTrafficBots = vi.fn();
+vi.mock('@/lib/actions/traffic', () => ({
+ getTrafficBots: (...a: unknown[]) => getTrafficBots(...a),
+}));
+vi.mock('@/i18n/navigation', () => ({
+ Link: ({ href, ...props }: { href: string }) => <a href={href} {...props} />,
+}));
+vi.mock('sonner', () => ({ toast: { success: vi.fn(), error: vi.fn() } }));
+
+const { BotsReportSection } = await import('./_bots-report');
+
+(globalThis as typeof globalThis & { IS_REACT_ACT_ENVIRONMENT: boolean }).IS_REACT_ACT_ENVIRONMENT =
+ true;
+
+const WEEK: TrafficDateWindow = { dateFrom: '2026-10-12T09:00:00.000Z' };
+const hoursAgo = (h: number) => new Date(Date.now() - h * 3_600_000).toISOString();
+
+const info = (over: Partial<BotConnections> = {}): BotConnections => ({
+ connections: [{ id: 'c1', kind: 'cloudflare_worker', createdAt: 't', lastEventAt: hoursAgo(1) }],
+ usage: { month: '2026-10-01', stored: 10, counted: 10, cappedAt: null, cap: 2_000_000 },
+ ...over,
+});
+const report = (over: Partial<TrafficBots> = {}): TrafficBots => ({
+ firstDay: '2026-09-01',
+ lastSeen: hoursAgo(1),
+ types: [
+ {
+ type: 'training',
+ hits: 1200,
+ pages: 300,
+ lastSeen: hoursAgo(2),
+ bots: ['gptbot', 'claudebot'],
+ },
+ { type: 'search', hits: 400, pages: 120, lastSeen: hoursAgo(1), bots: ['oai-searchbot'] },
+ ],
+ bots: [],
+ pageCount: 14,
+ pages: [
+ { page: '/menu', hits: 90, trainingLast: hoursAgo(2), searchLast: null, userLast: hoursAgo(1) },
+ ],
+ uncrawledCount: 3,
+ uncrawled: [{ page: '/gifts', citations: 14 }],
+ alertState: 'on',
+ alertFrom: '2026-09-30',
+ since: '2026-10-12',
+ ...over,
+});
+
+let container: HTMLDivElement;
+let root: Root;
+
+beforeEach(() => {
+ getBotConnections.mockReset().mockResolvedValue(info());
+ getTrafficBots.mockReset().mockResolvedValue(report());
+ container = document.createElement('div');
+ document.body.appendChild(container);
+ root = createRoot(container);
+});
+
+afterEach(() => {
+ act(() => root.unmount());
+ container.remove();
+ document.body.innerHTML = '';
+});
+
+async function render(canConnect = true, revision = 0, onChanged?: () => void) {
+ await act(async () => {
+ root.render(
+ <NextIntlClientProvider locale="en" messages={messages} timeZone="UTC">
+ <BotsReportSection
+ brandId="b1"
+ dateWindow={WEEK}
+ rangeLabel="last 7 days"
+ canConnect={canConnect}
+ revision={revision}
+ onChanged={onChanged}
+ />
+ </NextIntlClientProvider>,
+ );
+ });
+}
+
+/** A fresh section: it reads again. */
+async function remount(canConnect = true) {
+ act(() => root.unmount());
+ root = createRoot(container);
+ await render(canConnect);
+}
+
+const text = () => container.textContent ?? '';
+const rows = () =>
+ [...container.querySelectorAll('tbody tr')].map((r) =>
+ [...r.querySelectorAll('td')].map((td) => td.textContent),
+ );
+
+describe('BotsReportSection', () => {
+ it("keeps the new connection's settings up while the page reads the section again", async () => {
+ // Not connected yet: the sample with Connect.
+ getBotConnections.mockResolvedValue(info({ connections: [] }));
+ getTrafficBots.mockResolvedValue(report({ lastSeen: null }));
+ createBotConnection.mockResolvedValue({
+ connection: { id: 'c9', kind: 'cloudflare_worker', createdAt: 't', lastEventAt: null },
+ secret: 'abot_new_secret',
+ endpoint: 'https://api.example/bots/ingest',
+ workerScript: 'const SECRET = "abot_new_secret";',
+ logpushJob: null,
+ vercelDrain: null,
+ });
+ const onChanged = vi.fn();
+ await render(true, 0, onChanged);
+ const button = (label: string) =>
+ [...document.querySelectorAll('button')].find((b) => b.textContent === label)!;
+ await act(async () => button('Connect').click());
+ await act(async () => button('Create connection').click());
+ expect(onChanged).toHaveBeenCalled();
+ // The page bumps the section's revision: it reads again, and the dialog
+ // still shows the secret, once.
+ await render(true, 1, onChanged);
+ expect(getBotConnections).toHaveBeenCalledTimes(2);
+ expect(document.body.textContent).toContain("the secret in them isn't shown again");
+ expect(document.body.textContent).toContain('abot_new_secret');
+ });
+
+ it('shows the bot types, the pages each type last read, and the search-bot alert', async () => {
+ await render();
+ expect(getTrafficBots).toHaveBeenCalledWith('b1', WEEK);
+ expect(text()).toContain('Growth and above · needs Cloudflare or Vercel');
+ expect(rows()).toEqual([
+ ['Training (GPTBot, ClaudeBot)', '1,200', '300', '2h ago'],
+ ['Search index (OAI-SearchBot)', '400', '120', '1h ago'],
+ ['/menu', '90', '2h ago', '—', '1h ago'],
+ ]);
+ expect(text()).toContain(
+ "3 pages AI answers cited weren't read by any search bot in the last 30 days",
+ );
+ expect(text()).toContain('/giftscited 14 times');
+ expect(text()).toContain('and 2 more');
+ expect(text()).toContain('1 of 14 pages');
+ });
+
+ it('says when the alert starts, or that every cited page was read', async () => {
+ getTrafficBots.mockResolvedValue(report({ uncrawledCount: null, uncrawled: null }));
+ await render();
+ expect(text()).toContain('The search-bot alert starts on Sep 30');
+
+ getTrafficBots.mockResolvedValue(report({ uncrawledCount: 0, uncrawled: [] }));
+ await remount();
+ expect(text()).toContain('Every page AI answers cited in the last 30 days was read');
+ });
+
+ it("pauses the alert while the bot data isn't whole, rather than listing every cited page", async () => {
+ getTrafficBots.mockResolvedValue(
+ report({ alertState: 'stale', uncrawledCount: null, uncrawled: null }),
+ );
+ await render();
+ expect(text()).toContain('The search-bot alert is paused');
+ expect(text()).not.toContain('The search-bot alert starts on');
+ });
+
+ it('says "last 90 days" for All time, which is what it covers', async () => {
+ await act(async () => {
+ root.render(
+ <NextIntlClientProvider locale="en" messages={messages} timeZone="UTC">
+ <BotsReportSection brandId="b1" dateWindow={{}} rangeLabel="all time" canConnect />
+ </NextIntlClientProvider>,
+ );
+ });
+ expect(text()).toContain('last 90 days');
+ expect(text()).not.toContain('all time');
+ });
+
+ it('locks the report with sample data and Upgrade on Starter', async () => {
+ getBotConnections.mockResolvedValue(info({ usage: { ...info().usage, cap: 0 } }));
+ await render();
+ expect(text()).toContain('The AI bot report comes with the Growth plan.');
+ expect(text()).toContain('8,420');
+ expect(container.querySelector('a')?.getAttribute('href')).toBe(
+ '/dashboard/settings?tab=billing',
+ );
+ });
+
+ it('offers Connect over sample data, and a preview of it', async () => {
+ getBotConnections.mockResolvedValue(info({ connections: [] }));
+ getTrafficBots.mockResolvedValue(report({ lastSeen: null, types: [], pages: [] }));
+ await render();
+ expect(text()).toContain('Connect Cloudflare or Vercel in 2 minutes.');
+ const preview = [...container.querySelectorAll('button')].find(
+ (b) => b.textContent === 'Preview with sample data',
+ )!;
+ await act(async () => preview.click());
+ expect(text()).not.toContain('Connect Cloudflare or Vercel in 2 minutes.');
+ expect(text()).toContain('Sample data: connect Cloudflare or Vercel to see yours.');
+
+ await remount(false);
+ const connect = [...container.querySelectorAll('button')].find(
+ (b) => b.textContent === 'Connect',
+ )!;
+ expect(connect.disabled).toBe(true);
+ });
+
+ it('waits for the first request once connected', async () => {
+ getBotConnections.mockResolvedValue(
+ info({
+ connections: [{ id: 'c1', kind: 'vercel_drain', createdAt: 't', lastEventAt: null }],
+ }),
+ );
+ getTrafficBots.mockResolvedValue(report({ lastSeen: null, types: [], pages: [] }));
+ await render();
+ expect(text()).toContain('Waiting for the first AI-bot request');
+ });
+
+ it('says when the range had no bot, and when the report failed', async () => {
+ getTrafficBots.mockResolvedValue(report({ types: [], pages: [] }));
+ await render();
+ expect(text()).toContain('No AI bot read your site in this range.');
+
+ const quiet = vi.spyOn(console, 'error').mockImplementation(() => {});
+ getTrafficBots.mockRejectedValue(new Error('boom'));
+ await remount();
+ expect(text()).toContain("Couldn't load the AI bot report.");
+ quiet.mockRestore();
+ });
+});
diff --git a/web/src/app/[locale]/(dashboard)/dashboard/traffic/_bots-report.tsx b/web/src/app/[locale]/(dashboard)/dashboard/traffic/_bots-report.tsx
new file mode 100644
index 000000000..2879f7837
--- /dev/null
+++ b/web/src/app/[locale]/(dashboard)/dashboard/traffic/_bots-report.tsx
@@ -0,0 +1,390 @@
+'use client';
+
+import { useEffect, useState, type ReactNode } from 'react';
+import { useTranslations } from 'next-intl';
+import { Bot, TriangleAlert } from 'lucide-react';
+import { Link } from '@/i18n/navigation';
+import { Badge } from '@/components/ui/badge';
+import { Button } from '@/components/ui/button';
+import { buttonVariants } from '@/components/ui/button-variants';
+import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card';
+import { Skeleton } from '@/components/ui/skeleton';
+import {
+ Table,
+ TableBody,
+ TableCell,
+ TableHead,
+ TableHeader,
+ TableRow,
+} from '@/components/ui/table';
+import { cn } from '@/lib/utils';
+import { formatNumber } from '@/lib/format';
+import { formatRelative } from '@/lib/format-relative';
+import { botName } from '@/lib/ai-bot-names';
+import { botReportState, rowsPausedUntil, sampleBotTypes } from '@/lib/ai-bots';
+import { getBotConnections, type BotConnections } from '@/lib/actions/ai-bots';
+import {
+ getTrafficBots,
+ type BotTypeRow,
+ type TrafficBots,
+ type TrafficDateWindow,
+} from '@/lib/actions/traffic';
+import { formatTrafficBucketDate } from './_charts';
+import { BotsDialog } from './_bots-connect';
+
+/** Where Upgrade goes, as on the Prompts page's usage cards. */
+const PLAN_PAGE = '/dashboard/settings?tab=billing';
+
+/** Training, search index, user fetch: hits, pages read and when last seen (spec §3.7). */
+function TypesTable({ types }: { types: BotTypeRow[] }) {
+ const t = useTranslations('traffic.botReport');
+ const tCommon = useTranslations('common');
+ return (
+ <Table>
+ <TableHeader>
+ <TableRow>
+ <TableHead className="pl-6">{t('columns.type')}</TableHead>
+ <TableHead className="text-right">{t('columns.hits')}</TableHead>
+ <TableHead className="text-right">{t('columns.pages')}</TableHead>
+ <TableHead className="pr-6 text-right">{t('columns.lastSeen')}</TableHead>
+ </TableRow>
+ </TableHeader>
+ <TableBody>
+ {types.map((row) => (
+ <TableRow key={row.type}>
+ <TableCell className="pl-6 text-sm whitespace-normal">
+ <span className="font-medium">{t(`types.${row.type}`)}</span>
+ {row.bots.length > 0 && (
+ <span className="text-muted-foreground">
+ {` (${row.bots.map(botName).join(', ')})`}
+ </span>
+ )}
+ </TableCell>
+ <TableCell className="text-right text-sm font-semibold tabular-nums">
+ {formatNumber(row.hits)}
+ </TableCell>
+ <TableCell className="text-right text-sm tabular-nums">
+ {formatNumber(row.pages)}
+ </TableCell>
+ <TableCell className="pr-6 text-right text-sm text-muted-foreground">
+ {row.lastSeen ? formatRelative(row.lastSeen, tCommon) : '—'}
+ </TableCell>
+ </TableRow>
+ ))}
+ </TableBody>
+ </Table>
+ );
+}
+
+/** The most read pages, with when each type last read them (the per-page last crawl). */
+function PagesTable({ report }: { report: TrafficBots }) {
+ const t = useTranslations('traffic.botReport');
+ const tCommon = useTranslations('common');
+ const when = (iso: string | null) => (iso ? formatRelative(iso, tCommon) : '—');
+ return (
+ <>
+ <Table>
+ <TableHeader>
+ <TableRow>
+ <TableHead className="pl-6">{t('columns.page')}</TableHead>
+ <TableHead className="text-right">{t('columns.hits')}</TableHead>
+ <TableHead className="text-right">{t('types.training')}</TableHead>
+ <TableHead className="text-right">{t('types.search')}</TableHead>
+ <TableHead className="pr-6 text-right">{t('types.user')}</TableHead>
+ </TableRow>
+ </TableHeader>
+ <TableBody>
+ {report.pages.map((row) => (
+ <TableRow key={row.page}>
+ <TableCell className="pl-6 max-w-[180px] xl:max-w-[320px]">
+ <span
+ className="font-mono text-xs text-muted-foreground line-clamp-1"
+ title={row.page}
+ >
+ {row.page}
+ </span>
+ </TableCell>
+ <TableCell className="text-right text-sm font-semibold tabular-nums">
+ {formatNumber(row.hits)}
+ </TableCell>
+ <TableCell className="text-right text-sm text-muted-foreground">
+ {when(row.trainingLast)}
+ </TableCell>
+ <TableCell className="text-right text-sm text-muted-foreground">
+ {when(row.searchLast)}
+ </TableCell>
+ <TableCell className="pr-6 text-right text-sm text-muted-foreground">
+ {when(row.userLast)}
+ </TableCell>
+ </TableRow>
+ ))}
+ </TableBody>
+ </Table>
+ {report.pageCount > report.pages.length && (
+ <p className="px-6 pt-3 text-xs text-muted-foreground">
+ {t('morePages', {
+ shown: formatNumber(report.pages.length),
+ count: formatNumber(report.pageCount),
+ })}
+ </p>
+ )}
+ </>
+ );
+}
+
+/** "Cited pages not read by any search bot in 30 days" (spec §3.7). */
+function UncrawledAlert({ report }: { report: TrafficBots }) {
+ const t = useTranslations('traffic.botReport');
+ if (report.alertState === 'stale') {
+ return <p className="px-6 py-3 text-xs text-muted-foreground">{t('alertStale')}</p>;
+ }
+ if (report.uncrawledCount === null) {
+ return report.alertFrom ? (
+ <p className="px-6 py-3 text-xs text-muted-foreground">
+ {t('alertStarts', { day: formatTrafficBucketDate(report.alertFrom) })}
+ </p>
+ ) : null;
+ }
+ if (report.uncrawledCount === 0) {
+ return <p className="px-6 py-3 text-xs text-muted-foreground">{t('allCrawled')}</p>;
+ }
+ return (
+ <div className="mx-6 my-3 rounded-md border border-amber-500/30 bg-amber-500/10 px-3 py-2 text-xs text-amber-800 dark:text-amber-300">
+ <p className="flex items-center gap-1.5 font-medium">
+ <TriangleAlert className="h-3.5 w-3.5 shrink-0" />
+ {t('uncrawled', { count: report.uncrawledCount })}
+ </p>
+ <ul className="mt-1 space-y-0.5 pl-5">
+ {(report.uncrawled ?? []).map((row) => (
+ <li key={row.page} className="flex justify-between gap-3">
+ <span className="min-w-0 truncate font-mono" title={row.page}>
+ {row.page}
+ </span>
+ <span className="shrink-0">{t('cited', { count: row.citations })}</span>
+ </li>
+ ))}
+ </ul>
+ {report.uncrawledCount > (report.uncrawled?.length ?? 0) && (
+ <p className="mt-1 pl-5">
+ {t('moreUncrawled', {
+ count: report.uncrawledCount - (report.uncrawled?.length ?? 0),
+ })}
+ </p>
+ )}
+ </div>
+ );
+}
+
+/** Sample data under a note: locked (Upgrade) or not connected (Connect, or preview it). */
+function Overlay({ body, actions }: { body: string; actions: ReactNode }) {
+ const t = useTranslations('traffic.botReport');
+ return (
+ <div className="absolute inset-0 flex flex-col items-center justify-center gap-2 bg-card/70 px-6 text-center backdrop-blur-[1px]">
+ <p className="text-sm font-semibold">{t('overlayTitle')}</p>
+ <p className="max-w-[420px] text-xs text-muted-foreground">{body}</p>
+ <div className="flex flex-wrap justify-center gap-2">{actions}</div>
+ </div>
+ );
+}
+
+/**
+ * AI bots reading your site (AT-011, spec §3.7, §7): the verified AI-bot
+ * requests the brand's Cloudflare or Vercel sends, by type — training,
+ * search index, user fetch — with the most read pages, when each type last
+ * read them, and the cited pages no search bot read in 30 days.
+ *
+ * Starter sees it locked with sample data and Upgrade; a plan with the
+ * report but nothing connected sees sample data under Connect, or can
+ * preview it; once connected it waits for the first request.
+ */
+export function BotsReportSection({
+ brandId,
+ dateWindow,
+ rangeLabel,
+ canConnect,
+ revision = 0,
+ onChanged,
+ typesActions,
+ pagesActions,
+}: {
+ brandId: string;
+ dateWindow: TrafficDateWindow;
+ /** The page's own words for the range, e.g. "last 7 days". */
+ rangeLabel: string;
+ canConnect: boolean;
+ /** Bumped when a connection is made or revoked anywhere on the page. */
+ revision?: number;
+ /** A connection was made or revoked here. */
+ onChanged?: () => void;
+ /** Each card's Excel / CSV buttons (AT-015). */
+ typesActions?: ReactNode;
+ pagesActions?: ReactNode;
+}) {
+ const t = useTranslations('traffic.botReport');
+ const tBots = useTranslations('traffic.bots');
+ const key = `${brandId}|${dateWindow.dateFrom ?? ''}|${dateWindow.dateTo ?? ''}|${revision}`;
+ const [loaded, setLoaded] = useState<{
+ key: string;
+ info: BotConnections | null;
+ report: TrafficBots | null;
+ } | null>(null);
+ const [preview, setPreview] = useState(false);
+ const [connectOpen, setConnectOpen] = useState(false);
+
+ useEffect(() => {
+ let cancelled = false;
+ Promise.all([
+ getBotConnections(brandId).catch(() => ({ error: 'failed' })),
+ getTrafficBots(brandId, dateWindow).catch((err) => {
+ // Best effort: the rest of the page doesn't depend on this section.
+ console.error('Failed to load the AI bot report:', err);
+ return null;
+ }),
+ ]).then(([info, report]) => {
+ if (!cancelled) setLoaded({ key, info: 'error' in info ? null : info, report });
+ });
+ return () => {
+ cancelled = true;
+ };
+ }, [brandId, dateWindow, key]);
+
+ const current = loaded?.key === key ? loaded : null;
+ const info = current?.info ?? null;
+ const report = current?.report ?? null;
+ // The dialog keeps the connections it was opened with while the section
+ // reads again after a change it made (the page bumps `revision`): taken
+ // down, it would lose the new connection's secret and settings.
+ const dialogInfo = info ?? loaded?.info ?? null;
+ const state = info ? botReportState(info, report) : null;
+ const pausedUntil = info ? rowsPausedUntil(info.usage) : null;
+
+ const sample = (
+ <div className="relative min-h-[220px]">
+ <div aria-hidden className="pointer-events-none select-none opacity-50">
+ <TypesTable types={sampleBotTypes()} />
+ </div>
+ {state === 'locked' ? (
+ <Overlay
+ body={t('lockedBody')}
+ actions={
+ <Link href={PLAN_PAGE} className={cn(buttonVariants({ size: 'sm' }), 'h-7 text-xs')}>
+ {t('upgrade')}
+ </Link>
+ }
+ />
+ ) : (
+ !preview && (
+ <Overlay
+ body={t('connectBody')}
+ actions={
+ <>
+ <span title={canConnect ? undefined : tBots('notAllowed')}>
+ <Button
+ size="sm"
+ className="h-7 text-xs"
+ disabled={!canConnect}
+ onClick={() => setConnectOpen(true)}
+ >
+ {t('connect')}
+ </Button>
+ </span>
+ <Button
+ size="sm"
+ variant="outline"
+ className="h-7 text-xs"
+ onClick={() => setPreview(true)}
+ >
+ {t('preview')}
+ </Button>
+ </>
+ }
+ />
+ )
+ )}
+ </div>
+ );
+
+ const body = !current ? (
+ <div className="px-6 py-4">
+ <Skeleton className="h-4 w-56" />
+ </div>
+ ) : !info || (!report && state !== 'locked' && state !== 'off') ? (
+ <p className="px-6 py-4 text-sm text-muted-foreground">{t('failed')}</p>
+ ) : state === 'locked' || state === 'off' ? (
+ sample
+ ) : state === 'waiting' ? (
+ <p className="px-6 py-4 text-sm text-muted-foreground">{t('waiting')}</p>
+ ) : report!.types.length === 0 ? (
+ <p className="px-6 py-4 text-sm text-muted-foreground">{t('emptyRange')}</p>
+ ) : (
+ <TypesTable types={report!.types} />
+ );
+
+ return (
+ <section className="space-y-3" aria-labelledby="ai-bots-title">
+ <div className="flex flex-wrap items-center gap-x-3 gap-y-1">
+ <h2 id="ai-bots-title" className="flex items-center gap-2 text-base font-semibold">
+ <Bot className="h-4 w-4 text-muted-foreground" />
+ {t('title')}
+ </h2>
+ <span className="max-w-full rounded-md border px-1.5 text-[11px] font-medium leading-4 text-muted-foreground sm:whitespace-nowrap">
+ {t('needs')}
+ </span>
+ {preview && state === 'off' && (
+ <span className="text-xs text-muted-foreground">{t('sampleNote')}</span>
+ )}
+ <Badge variant="outline" className="ml-auto text-xs">
+ {/* "All time" covers the last 90 days here (00185). */}
+ {dateWindow.dateFrom ? rangeLabel : t('allTimeCapped')}
+ </Badge>
+ </div>
+
+ <div className={cn('grid grid-cols-1 gap-4', state === 'data' && 'xl:grid-cols-2')}>
+ <Card className={cn(!current && 'opacity-80')} aria-busy={!current}>
+ <CardHeader className="pb-2">
+ <div className="flex items-center justify-between gap-2">
+ <CardTitle className="text-sm font-medium">{t('typesTitle')}</CardTitle>
+ {state === 'data' && typesActions}
+ </div>
+ </CardHeader>
+ <CardContent className="p-0">
+ {body}
+ {state === 'data' && report && <UncrawledAlert report={report} />}
+ {state === 'data' && pausedUntil && (
+ <p className="px-6 pb-3 text-xs text-muted-foreground">
+ {tBots('paused', {
+ day: formatTrafficBucketDate(pausedUntil),
+ })}
+ </p>
+ )}
+ </CardContent>
+ </Card>
+
+ {state === 'data' && report && report.pages.length > 0 && (
+ <Card>
+ <CardHeader className="pb-2">
+ <div className="flex items-center justify-between gap-2">
+ <CardTitle className="text-sm font-medium">{t('pagesTitle')}</CardTitle>
+ {pagesActions}
+ </div>
+ </CardHeader>
+ <CardContent className="p-0 pb-3">
+ <PagesTable report={report} />
+ </CardContent>
+ </Card>
+ )}
+ </div>
+
+ {dialogInfo && botReportState(dialogInfo, null) !== 'locked' && (
+ <BotsDialog
+ brandId={brandId}
+ open={connectOpen}
+ info={dialogInfo}
+ canConnect={canConnect}
+ onOpenChange={setConnectOpen}
+ onChanged={() => onChanged?.()}
+ />
+ )}
+ </section>
+ );
+}
diff --git a/web/src/app/[locale]/(dashboard)/dashboard/traffic/_sources-strip.tsx b/web/src/app/[locale]/(dashboard)/dashboard/traffic/_sources-strip.tsx
index 92abfe8ec..37e7ed510 100644
--- a/web/src/app/[locale]/(dashboard)/dashboard/traffic/_sources-strip.tsx
+++ b/web/src/app/[locale]/(dashboard)/dashboard/traffic/_sources-strip.tsx
@@ -124,6 +124,8 @@ export function SourcesStrip({
canConnect,
onShowCode,
onConnected,
+ botsRevision,
+ onBotsChanged,
}: {
brandId: string;
/** Null while loading. */
@@ -134,6 +136,9 @@ export function SourcesStrip({
canConnect: boolean;
onShowCode: () => void;
onConnected: () => void;
+ /** Bumped when an AI-bot connection is made or revoked on the page. */
+ botsRevision?: number;
+ onBotsChanged?: () => void;
}) {
const t = useTranslations('traffic.sources');
const tCommon = useTranslations('common');
@@ -186,7 +191,12 @@ export function SourcesStrip({
canConnect={canConnect}
onConnected={onConnected}
/>
- <BotsSource brandId={brandId} canConnect={canConnect} />
+ <BotsSource
+ brandId={brandId}
+ canConnect={canConnect}
+ revision={botsRevision}
+ onChanged={onBotsChanged}
+ />
</section>
);
}
diff --git a/web/src/app/[locale]/(dashboard)/dashboard/traffic/page.tsx b/web/src/app/[locale]/(dashboard)/dashboard/traffic/page.tsx
index 0930020dd..50702fa73 100644
--- a/web/src/app/[locale]/(dashboard)/dashboard/traffic/page.tsx
+++ b/web/src/app/[locale]/(dashboard)/dashboard/traffic/page.tsx
@@ -46,6 +46,7 @@ import { SourcesStrip } from './_sources-strip';
import { SnippetBanner, SnippetDialog } from './_snippet-code';
import { GoogleTrendView } from './_google-trend';
import { GoogleSection } from './_google-section';
+import { BotsReportSection } from './_bots-report';
import { NeedsTag } from './_source-gate';
import { googleSourceStatus, snippetDetected } from '@/lib/traffic-sources';
import { useUserRole } from '@/hooks/use-user-role';
@@ -432,6 +433,9 @@ function TrafficPageContent({ brand }: { brand: Brand }) {
const [trendView, setTrendView] = useState<'platforms' | 'google'>('platforms');
// Bumped when Google queries are tracked, so the summary line counts again (AT-009).
const [queriesRevision, setQueriesRevision] = useState(0);
+ // Bumped when an AI-bot connection is made or revoked, so the sources
+ // strip's card and the bot report read it again (AT-010, AT-011).
+ const [botsRevision, setBotsRevision] = useState(0);
const [logs, setLogs] = useState<TrafficLog[]>([]);
const [logsTotal, setLogsTotal] = useState(0);
const [logsPageviews, setLogsPageviews] = useState(0);
@@ -648,6 +652,21 @@ function TrafficPageContent({ brand }: { brand: Brand }) {
/>
);
+ // AI bots reading the site (AT-011, spec §3.7): from the brand's own
+ // Cloudflare or Vercel, so it shows even when the snippet saw no visit.
+ const botsSection = (
+ <BotsReportSection
+ brandId={brand.id}
+ dateWindow={dateWindow}
+ rangeLabel={rangeSubLabel}
+ canConnect={canManage}
+ revision={botsRevision}
+ onChanged={() => setBotsRevision((n) => n + 1)}
+ typesActions={<TableExport exporter={exporter} section="bots" />}
+ pagesActions={<TableExport exporter={exporter} section="botPages" />}
+ />
+ );
+
return (
<div className="space-y-6">
{/* Header */}
@@ -684,6 +703,8 @@ function TrafficPageContent({ brand }: { brand: Brand }) {
canConnect={canManage}
onShowCode={() => setCodeOpen(true)}
onConnected={() => void loadSources()}
+ botsRevision={botsRevision}
+ onBotsChanged={() => setBotsRevision((n) => n + 1)}
/>
)}
{/* The code stays open only while the snippet sends nothing (F3). */}
@@ -712,6 +733,7 @@ function TrafficPageContent({ brand }: { brand: Brand }) {
{isEmpty && conversionsSection}
{isEmpty && pagesCard}
{isEmpty && googleSection}
+ {isEmpty && botsSection}
{!isEmpty && (
<>
{/* KPI cards: two rows of four (AT-017, spec §3.2). */}
@@ -892,6 +914,8 @@ function TrafficPageContent({ brand }: { brand: Brand }) {
{googleSection}
+ {botsSection}
+
{/* Recent Visit Log with filtering and pagination*/}
<Card>
<CardHeader className="pb-3">
diff --git a/web/src/app/api/traffic/export/route.test.ts b/web/src/app/api/traffic/export/route.test.ts
index 610bb0745..26b8b0f70 100644
--- a/web/src/app/api/traffic/export/route.test.ts
+++ b/web/src/app/api/traffic/export/route.test.ts
@@ -35,6 +35,7 @@ const loaders = vi.hoisted(() => ({
getTrafficConversions: vi.fn(),
getTrafficGooglePages: vi.fn(),
getUntrackedGoogleQueries: vi.fn(),
+ getTrafficBots: vi.fn(),
}));
vi.mock('@/lib/actions/traffic', () => loaders);
@@ -112,6 +113,7 @@ beforeEach(() => {
count: 0,
rows: [],
});
+ loaders.getTrafficBots.mockResolvedValue(null);
loaders.getUntrackedGoogleQueries.mockResolvedValue({
lastDay: '2026-10-17',
since: '2026-10-11',
@@ -158,6 +160,8 @@ describe('GET /api/traffic/export', () => {
'Pages by status',
'Is AI taking Google clicks',
'Untracked Google queries',
+ 'AI bots',
+ 'Pages AI bots read',
'Visits',
]);
@@ -196,6 +200,8 @@ describe('GET /api/traffic/export', () => {
'pages-by-status.csv',
'is-ai-taking-google-clicks.csv',
'untracked-google-queries.csv',
+ 'ai-bots.csv',
+ 'pages-ai-bots-read.csv',
'visits.csv',
]);
for (const file of Object.values(files)) expect(startsWithBom(file)).toBe(true);
@@ -282,6 +288,12 @@ describe('GET /api/traffic/export', () => {
expect((await get({ format: 'xlsx' })).status).toBe(404);
});
+ it('exports the rest when the AI bot report times out', async () => {
+ loaders.getTrafficBots.mockRejectedValue(new Error('statement timeout'));
+ const res = await get({ format: 'xlsx' });
+ expect(res.status).toBe(200);
+ });
+
it('reports a failed read as an error, not as an empty file', async () => {
const error = vi.spyOn(console, 'error').mockImplementation(() => {});
loaders.getTrafficSummary.mockRejectedValue(new Error('timeout'));
diff --git a/web/src/app/api/traffic/export/route.ts b/web/src/app/api/traffic/export/route.ts
index a11ee9413..a8a4597ec 100644
--- a/web/src/app/api/traffic/export/route.ts
+++ b/web/src/app/api/traffic/export/route.ts
@@ -9,6 +9,7 @@ import {
getTrafficGooglePages,
getTrafficPages,
getTrafficSources,
+ getTrafficBots,
getTrafficSummary,
getTrafficTrend,
getTrafficVisitsForExport,
@@ -94,39 +95,53 @@ export async function GET(req: Request) {
const wholePage = !q.section;
try {
- const [summary, trend, visits, sources, pageStatus, conversions, googlePages, googleQueries] =
- await Promise.all([
- needs('kpis', 'platforms', 'pages')
- ? getTrafficSummary(q.brandId, window, { topPages: Number.MAX_SAFE_INTEGER })
- : undefined,
- needs('trend') ? getTrafficTrend(q.brandId, window) : undefined,
- needs('visits')
- ? getTrafficVisitsForExport(q.brandId, {
- ...window,
- platform: q.platform,
- search: q.search,
- max: TRAFFIC_EXPORT_MAX_ROWS,
- })
- : undefined,
- // For the Filters sheet.
- wholePage ? getTrafficSources(q.brandId).catch(() => null) : undefined,
- // Every page of the Pages section, with its lists (AT-006).
- needs('pageStatus')
- ? getTrafficPages(q.brandId, window, { tab: 'all', limit: TRAFFIC_EXPORT_MAX_ROWS })
- : undefined,
- // Every row of where AI visitors convert, over the window the page reads (AT-014).
- needs('convertingPages', 'buyerAnswers')
- ? getTrafficConversions(q.brandId, window, { limit: TRAFFIC_EXPORT_MAX_ROWS })
- : undefined,
- // Every page of Is AI taking Google clicks?, over the days it compares (AT-008).
- needs('googlePages')
- ? getTrafficGooglePages(q.brandId, window, { limit: TRAFFIC_EXPORT_MAX_ROWS })
- : undefined,
- // Every Google query no tracked prompt covers, as far as the server checks (AT-009).
- needs('googleQueries')
- ? getUntrackedGoogleQueries(q.brandId, window, { limit: UNTRACKED_QUERY_SCAN })
- : undefined,
- ]);
+ const [
+ summary,
+ trend,
+ visits,
+ sources,
+ pageStatus,
+ conversions,
+ googlePages,
+ googleQueries,
+ bots,
+ ] = await Promise.all([
+ needs('kpis', 'platforms', 'pages')
+ ? getTrafficSummary(q.brandId, window, { topPages: Number.MAX_SAFE_INTEGER })
+ : undefined,
+ needs('trend') ? getTrafficTrend(q.brandId, window) : undefined,
+ needs('visits')
+ ? getTrafficVisitsForExport(q.brandId, {
+ ...window,
+ platform: q.platform,
+ search: q.search,
+ max: TRAFFIC_EXPORT_MAX_ROWS,
+ })
+ : undefined,
+ // For the Filters sheet.
+ wholePage ? getTrafficSources(q.brandId).catch(() => null) : undefined,
+ // Every page of the Pages section, with its lists (AT-006).
+ needs('pageStatus')
+ ? getTrafficPages(q.brandId, window, { tab: 'all', limit: TRAFFIC_EXPORT_MAX_ROWS })
+ : undefined,
+ // Every row of where AI visitors convert, over the window the page reads (AT-014).
+ needs('convertingPages', 'buyerAnswers')
+ ? getTrafficConversions(q.brandId, window, { limit: TRAFFIC_EXPORT_MAX_ROWS })
+ : undefined,
+ // Every page of Is AI taking Google clicks?, over the days it compares (AT-008).
+ needs('googlePages')
+ ? getTrafficGooglePages(q.brandId, window, { limit: TRAFFIC_EXPORT_MAX_ROWS })
+ : undefined,
+ // Every Google query no tracked prompt covers, as far as the server checks (AT-009).
+ needs('googleQueries')
+ ? getUntrackedGoogleQueries(q.brandId, window, { limit: UNTRACKED_QUERY_SCAN })
+ : undefined,
+ // Every page AI bots read, with the bot types (AT-011).
+ // Left out when it fails (a timeout on a large site), so the rest still exports.
+ needs('bots', 'botPages')
+ ? getTrafficBots(q.brandId, window, { limit: TRAFFIC_EXPORT_MAX_ROWS }).catch(() => null)
+ : undefined,
+ ]);
// Larger than an export built on request should be (spec §3.9).
if (visits === null) return Response.json({ error: 'too_large' }, { status: 413 });
@@ -148,6 +163,7 @@ export async function GET(req: Request) {
conversions,
googlePages,
googleQueries,
+ bots,
sources,
};
diff --git a/web/src/lib/actions/traffic-bots.test.ts b/web/src/lib/actions/traffic-bots.test.ts
new file mode 100644
index 000000000..08c8c5601
--- /dev/null
+++ b/web/src/lib/actions/traffic-bots.test.ts
@@ -0,0 +1,107 @@
+import { describe, expect, it, vi } from 'vitest';
+
+/** AT-011 (spec §3.7): the AI bot report reads ai_bot_report (00185) over the range's days. */
+
+const rpc = vi.fn();
+vi.mock('@/lib/supabase/server', () => ({
+ createClient: async () => ({ rpc }),
+}));
+vi.mock('@/config/api', () => ({ API_BASE_URL: 'http://api.test' }));
+
+import { getTrafficBots } from './traffic';
+
+describe('getTrafficBots', () => {
+ it('reads the types, bots, pages and the search-bot alert', async () => {
+ rpc.mockResolvedValue({
+ data: {
+ first_day: '2026-09-01',
+ last_seen: '2026-10-19T09:30:00Z',
+ types: [
+ {
+ type: 'search',
+ hits: '30',
+ pages: 12,
+ last_seen: '2026-10-19T09:30:00Z',
+ bots: ['oai-searchbot'],
+ },
+ ],
+ bots: [{ bot: 'oai-searchbot', type: 'search', hits: 30, pages: 12, last_seen: null }],
+ page_count: 12,
+ pages: [
+ {
+ page: '/menu',
+ hits: 9,
+ training_last: null,
+ search_last: '2026-10-19T09:30:00Z',
+ user_last: null,
+ },
+ ],
+ uncrawled_count: 2,
+ uncrawled: [{ page: '/gifts', citations: '14' }],
+ alert_state: 'on',
+ alert_from: '2026-09-30',
+ since: '2026-10-12',
+ },
+ error: null,
+ });
+ const result = await getTrafficBots('b1', {
+ dateFrom: '2026-10-12T09:00:00.000Z',
+ dateTo: '2026-10-19',
+ });
+ expect(rpc).toHaveBeenCalledWith('ai_bot_report', {
+ p_brand_id: 'b1',
+ p_since: '2026-10-12',
+ p_until: '2026-10-19',
+ p_limit: 10,
+ });
+ expect(result).toEqual({
+ firstDay: '2026-09-01',
+ lastSeen: '2026-10-19T09:30:00Z',
+ types: [
+ {
+ type: 'search',
+ hits: 30,
+ pages: 12,
+ lastSeen: '2026-10-19T09:30:00Z',
+ bots: ['oai-searchbot'],
+ },
+ ],
+ bots: [{ bot: 'oai-searchbot', type: 'search', hits: 30, pages: 12, lastSeen: null }],
+ pageCount: 12,
+ pages: [
+ {
+ page: '/menu',
+ hits: 9,
+ trainingLast: null,
+ searchLast: '2026-10-19T09:30:00Z',
+ userLast: null,
+ },
+ ],
+ uncrawledCount: 2,
+ uncrawled: [{ page: '/gifts', citations: 14 }],
+ alertState: 'on',
+ alertFrom: '2026-09-30',
+ since: '2026-10-12',
+ });
+ });
+
+ it('reads the alert as not started before 30 days of bot data, and throws on a failed read', async () => {
+ rpc.mockResolvedValue({
+ data: {
+ types: [],
+ bots: [],
+ pages: [],
+ page_count: 0,
+ uncrawled_count: null,
+ uncrawled: null,
+ },
+ error: null,
+ });
+ const report = await getTrafficBots('b1', undefined);
+ expect(report).toMatchObject({ uncrawledCount: null, uncrawled: null, lastSeen: null });
+ expect(rpc.mock.calls.at(-1)?.[1].p_since).toBeNull();
+
+ rpc.mockResolvedValue({ data: null, error: { message: 'boom' } });
+ await expect(getTrafficBots('b1', undefined)).rejects.toThrow('boom');
+ });
+});
diff --git a/web/src/lib/actions/traffic.ts b/web/src/lib/actions/traffic.ts
index 51d5cd108..a201378c2 100644
--- a/web/src/lib/actions/traffic.ts
+++ b/web/src/lib/actions/traffic.ts
@@ -1111,3 +1111,112 @@ export async function getUntrackedGoogleQueries(
})),
};
}
+
+/** One type of AI bot in the bot report (AT-011): training, search index or user fetch. */
+export interface BotTypeRow {
+ type: 'training' | 'search' | 'user';
+ hits: number;
+ pages: number;
+ lastSeen: string | null;
+ /** The bots of this type seen, by key. */
+ bots: string[];
+}
+
+export interface BotRow {
+ bot: string;
+ type: BotTypeRow['type'];
+ hits: number;
+ pages: number;
+ lastSeen: string | null;
+}
+
+/** A page AI bots read, with when each type last read it. */
+export interface BotPageRow {
+ page: string;
+ hits: number;
+ trainingLast: string | null;
+ searchLast: string | null;
+ userLast: string | null;
+}
+
+export interface TrafficBots {
+ /** The first day any bot was counted; null with none. */
+ firstDay: string | null;
+ lastSeen: string | null;
+ types: BotTypeRow[];
+ bots: BotRow[];
+ pageCount: number;
+ pages: BotPageRow[];
+ /** Cited pages no search bot read in 30 days; null unless alertState is 'on'. */
+ uncrawledCount: number | null;
+ uncrawled: { page: string; citations: number }[] | null;
+ /**
+ * The search-bot alert: 'waiting' for 30 days of bot data, 'stale' while no
+ * live connection has sent for 2 days or search hits went past the month's
+ * cap (the pages read would drain and list every cited page), else 'on'.
+ */
+ alertState: 'waiting' | 'stale' | 'on';
+ /** The day the search-bot alert starts, 30 days after the first request. */
+ alertFrom: string | null;
+ /** The first day the report covers: "All time" is the last 90 days. */
+ since: string | null;
+}
+
+/**
+ * AI bots reading the site (AT-011, spec §3.7; 00185): verified AI-bot hits
+ * in the range by type and by bot, the most read pages with each type's
+ * last read, and the cited pages no search bot read in the last 30 days.
+ */
+export async function getTrafficBots(
+ brandId: string,
+ window: TrafficDateWindow | undefined,
+ { limit = 10 }: { limit?: number } = {},
+): Promise<TrafficBots> {
+ const supabase = await createClient();
+ const { from, to } = resolveTrafficWindow(window);
+ const { data, error } = await supabase.rpc('ai_bot_report', {
+ p_brand_id: brandId,
+ p_since: utcDay(from) ?? null,
+ p_until: utcDay(to) ?? new Date().toISOString().slice(0, 10),
+ p_limit: limit,
+ });
+ if (error) throw new Error(error.message);
+ const raw = (data ?? {}) as Record<string, unknown>;
+ const n = (v: unknown) => Number(v) || 0;
+ const str = (v: unknown) => (typeof v === 'string' ? v : null);
+ const list = (v: unknown) => (Array.isArray(v) ? (v as Record<string, unknown>[]) : []);
+ return {
+ firstDay: str(raw.first_day),
+ lastSeen: str(raw.last_seen),
+ types: list(raw.types).map((r) => ({
+ type: r.type as BotTypeRow['type'],
+ hits: n(r.hits),
+ pages: n(r.pages),
+ lastSeen: str(r.last_seen),
+ bots: Array.isArray(r.bots) ? (r.bots as string[]) : [],
+ })),
+ bots: list(raw.bots).map((r) => ({
+ bot: String(r.bot),
+ type: r.type as BotTypeRow['type'],
+ hits: n(r.hits),
+ pages: n(r.pages),
+ lastSeen: str(r.last_seen),
+ })),
+ pageCount: n(raw.page_count),
+ pages: list(raw.pages).map((r) => ({
+ page: String(r.page),
+ hits: n(r.hits),
+ trainingLast: str(r.training_last),
+ searchLast: str(r.search_last),
+ userLast: str(r.user_last),
+ })),
+ uncrawledCount: raw.uncrawled_count == null ? null : n(raw.uncrawled_count),
+ uncrawled: Array.isArray(raw.uncrawled)
+ ? list(raw.uncrawled).map((r) => ({ page: String(r.page), citations: n(r.citations) }))
+ : null,
+ alertState:
+ raw.alert_state === 'on' || raw.alert_state === 'stale' ? raw.alert_state : 'waiting',
+ alertFrom: str(raw.alert_from),
+ since: str(raw.since),
+ };
+}
diff --git a/web/src/lib/ai-bot-names.test.ts b/web/src/lib/ai-bot-names.test.ts
new file mode 100644
index 000000000..a930b09e4
--- /dev/null
+++ b/web/src/lib/ai-bot-names.test.ts
@@ -0,0 +1,21 @@
+import { readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { describe, expect, it } from 'vitest';
+import { AI_BOT_NAMES, BOT_TYPES, botName } from './ai-bot-names';
+
+/** Every bot on the server's list (AT-010) has a name in the report (AT-011). */
+
+const list = JSON.parse(
+ readFileSync(join(__dirname, '../../../server/src/lib/ai-bots.json'), 'utf8'),
+) as { bots: { key: string; name: string; type: string }[] };
+
+describe('AI bot names', () => {
+ it('names every bot on the list as the list does, with a known type', () => {
+ for (const bot of list.bots) {
+ expect(AI_BOT_NAMES[bot.key]).toBe(bot.name);
+ expect(BOT_TYPES).toContain(bot.type);
+ }
+ expect(Object.keys(AI_BOT_NAMES)).toHaveLength(list.bots.length);
+ expect(botName('newbot')).toBe('newbot');
+ });
+});
diff --git a/web/src/lib/ai-bot-names.ts b/web/src/lib/ai-bot-names.ts
new file mode 100644
index 000000000..7ac1518f3
--- /dev/null
+++ b/web/src/lib/ai-bot-names.ts
@@ -0,0 +1,24 @@
+/**
+ * How the AI bot report names each bot and type (AT-011). The keys are the
+ * bot list's (server/src/lib/ai-bots.json, AT-010): every bot there needs a
+ * name here (ai-bot-names.test.ts checks).
+ */
+
+export const BOT_TYPES = ['training', 'search', 'user'] as const;
+export type BotType = (typeof BOT_TYPES)[number];
+
+export const AI_BOT_NAMES: Record<string, string> = {
+ gptbot: 'GPTBot',
+ 'oai-searchbot': 'OAI-SearchBot',
+ 'chatgpt-user': 'ChatGPT-User',
+ claudebot: 'ClaudeBot',
+ 'claude-searchbot': 'Claude-SearchBot',
+ 'claude-user': 'Claude-User',
+ perplexitybot: 'PerplexityBot',
+ 'perplexity-user': 'Perplexity-User',
+};
+
+/** A bot's name, or its key when it isn't known. */
+export function botName(key: string): string {
+ return AI_BOT_NAMES[key] ?? key;
+}
diff --git a/web/src/lib/ai-bots.test.ts b/web/src/lib/ai-bots.test.ts
index 7c855b377..bad0c6fcd 100644
--- a/web/src/lib/ai-bots.test.ts
+++ b/web/src/lib/ai-bots.test.ts
@@ -1,5 +1,11 @@
import { describe, expect, it } from 'vitest';
-import { botsCardState, lastBotEvent, rowsPausedUntil } from './ai-bots';
+import {
+ botReportState,
+ botsCardState,
+ lastBotEvent,
+ rowsPausedUntil,
+ sampleBotTypes,
+} from './ai-bots';
import type { BotConnections, BotUsage } from '@/lib/actions/ai-bots';
/** The AI bots source card's state (AT-010). */
@@ -51,3 +57,45 @@ describe('lastBotEvent and rowsPausedUntil', () => {
expect(rowsPausedUntil(usage({ cap: -1, cappedAt: '2026-10-20T00:00:00Z' }))).toBeNull();
});
});
+
+describe('botReportState and sampleBotTypes (AT-011)', () => {
+ const report = (lastSeen: string | null) => ({
+ firstDay: null,
+ lastSeen,
+ types: [],
+ bots: [],
+ pageCount: 0,
+ pages: [],
+ uncrawledCount: null,
+ uncrawled: null,
+ alertState: 'waiting' as const,
+ alertFrom: null,
+ since: null,
+ });
+
+ it("is locked without the report, off, waiting, or the brand's own data", () => {
+ expect(
+ botReportState({ connections: [], usage: usage({ cap: 0 }) }, report('2026-10-19T09:00:00Z')),
+ ).toBe('locked');
+ expect(botReportState({ connections: [], usage: usage() }, report(null))).toBe('off');
+ expect(botReportState({ connections: [conn(null)], usage: usage() }, report(null))).toBe(
+ 'waiting',
+ );
+ expect(
+ botReportState({ connections: [conn(null)], usage: usage() }, report('2026-10-19T09:00:00Z')),
+ ).toBe('data');
+ // Bots counted before the connection was revoked still show.
+ expect(
+ botReportState({ connections: [], usage: usage() }, report('2026-10-19T09:00:00Z')),
+ ).toBe('data');
+ });
+
+ it("samples the mockup's numbers, last seen 2h, 40m and 5m ago", () => {
+ const now = Date.parse('2026-10-19T12:00:00.000Z');
+ expect(sampleBotTypes(now).map((t) => [t.type, t.hits, t.pages, t.lastSeen])).toEqual([
+ ['training', 8420, 612, '2026-10-19T10:00:00.000Z'],
+ ['search', 3115, 488, '2026-10-19T11:20:00.000Z'],
+ ['user', 1302, 214, '2026-10-19T11:55:00.000Z'],
+ ]);
+ });
+});
diff --git a/web/src/lib/ai-bots.ts b/web/src/lib/ai-bots.ts
index 041486f5e..6a22f77c4 100644
--- a/web/src/lib/ai-bots.ts
+++ b/web/src/lib/ai-bots.ts
@@ -4,6 +4,7 @@
*/
import type { BotConnection, BotConnections, BotUsage } from '@/lib/actions/ai-bots';
+import type { TrafficBots } from '@/lib/actions/traffic';
/**
* locked — the plan has no bot report (Starter: Upgrade);
@@ -37,3 +38,42 @@ export function rowsPausedUntil(usage: BotUsage): string | null {
const [y, m] = usage.month.split('-').map(Number);
return new Date(Date.UTC(y, m, 1)).toISOString().slice(0, 10);
}
+
+/**
+ * What the bot report section shows (AT-011, spec §3.7): locked with sample
+ * data and Upgrade (Starter); sample data under Connect while nothing is
+ * connected; waiting for the first request; or the brand's own data.
+ */
+export type BotReportState = 'locked' | 'off' | 'waiting' | 'data';
+
+export function botReportState(info: BotConnections, report: TrafficBots | null): BotReportState {
+ if (info.usage.cap === 0) return 'locked';
+ // Bots counted before a connection was revoked still show.
+ if (report?.lastSeen) return 'data';
+ return info.connections.length === 0 ? 'off' : 'waiting';
+}
+
+/**
+ * The sample the locked and unconnected report shows: the mockup's numbers
+ * (spec §3.7), last seen 2 hours, 40 minutes and 5 minutes before `now`.
+ */
+export function sampleBotTypes(now = Date.now()): TrafficBots['types'] {
+ const ago = (minutes: number) => new Date(now - minutes * 60_000).toISOString();
+ return [
+ { type: 'training', hits: 8420, pages: 612, lastSeen: ago(120), bots: ['gptbot', 'claudebot'] },
+ {
+ type: 'search',
+ hits: 3115,
+ pages: 488,
+ lastSeen: ago(40),
+ bots: ['oai-searchbot', 'perplexitybot'],
+ },
+ {
+ type: 'user',
+ hits: 1302,
+ pages: 214,
+ lastSeen: ago(5),
+ bots: ['chatgpt-user', 'claude-user'],
+ },
+ ];
+}
diff --git a/web/src/lib/traffic-export.test.ts b/web/src/lib/traffic-export.test.ts
index 3e72e1404..8dabbdbdc 100644
--- a/web/src/lib/traffic-export.test.ts
+++ b/web/src/lib/traffic-export.test.ts
@@ -475,6 +475,62 @@ describe('trafficTable: Untracked Google queries (AT-009)', () => {
});
});
+describe('trafficTable: AI bots (AT-011)', () => {
+ const bots = (): NonNullable<TrafficExportData['bots']> => ({
+ firstDay: '2026-09-01',
+ lastSeen: '2026-10-19T09:30:00Z',
+ types: [],
+ bots: [
+ { bot: 'gptbot', type: 'training', hits: 120, pages: 40, lastSeen: '2026-10-19T09:30:00Z' },
+ { bot: 'perplexity-user', type: 'user', hits: 3, pages: 2, lastSeen: null },
+ ],
+ pageCount: 1,
+ pages: [
+ {
+ page: '/menu',
+ hits: 80,
+ trainingLast: '2026-10-19T09:30:00Z',
+ searchLast: null,
+ userLast: '2026-10-18T12:00:00Z',
+ },
+ ],
+ uncrawledCount: 0,
+ uncrawled: [],
+ alertState: 'on',
+ alertFrom: '2026-09-30',
+ since: '2026-10-12',
+ });
+
+ it('holds each bot with its type, and each page with every type last read', () => {
+ expect(trafficTable('bots', data({ bots: bots() }), label)).toEqual({
+ headers: ['Bot', 'Bot type', 'Hits', 'Pages read', 'Last seen (UTC)'],
+ rows: [
+ ['GPTBot', 'Training', 120, 40, '2026-10-19 09:30'],
+ ['Perplexity-User', 'User fetch', 3, 2, null],
+ ],
+ });
+ expect(trafficTable('botPages', data({ bots: bots() }), label)).toEqual({
+ headers: [
+ 'Page',
+ 'Hits',
+ 'Training, last read (UTC)',
+ 'Search index, last read (UTC)',
+ 'User fetch, last read (UTC)',
+ ],
+ rows: [['/menu', 80, '2026-10-19 09:30', null, '2026-10-18 12:00']],
+ });
+ });
+
+ it('says there were no AI-bot requests, never an empty sheet', () => {
+ expect(trafficTable('bots', data({ bots: null }), label).rows).toEqual([
+ ['No AI-bot requests in this range'],
+ ]);
+ expect(trafficTable('botPages', data({}), label).rows).toEqual([
+ ['No AI-bot requests in this range'],
+ ]);
+ });
+});
+
describe('filtersTable', () => {
it('records the range, the filters, the sources and when', () => {
const table = filtersTable(
@@ -510,6 +566,8 @@ describe('trafficWorkbook', () => {
'Pages by status',
'Is AI taking Google clicks',
'Untracked Google queries',
+ 'AI bots',
+ 'Pages AI bots read',
'Visits',
]);
// A bold header row, then the rows with their numbers as numbers.
diff --git a/web/src/lib/traffic-export.ts b/web/src/lib/traffic-export.ts
index e4614734f..cba06da52 100644
--- a/web/src/lib/traffic-export.ts
+++ b/web/src/lib/traffic-export.ts
@@ -11,6 +11,7 @@
import type { Sheet } from 'write-excel-file/node';
import type {
TrafficConversions,
+ TrafficBots,
TrafficGooglePages,
TrafficLog,
TrafficUntrackedQueries,
@@ -24,6 +25,7 @@ import { getPlatformName } from '@/lib/traffic-platforms';
import { pageStatuses } from '@/lib/traffic-pages';
import { conversionRate } from '@/lib/traffic-conversions';
import { ctrPct, impressionsRise } from '@/lib/traffic-google-pages';
+import { botName } from '@/lib/ai-bot-names';
import { visitChange } from '@/lib/traffic-change';
import {
googleSourceStatus,
@@ -42,6 +44,8 @@ export const TRAFFIC_EXPORT_SECTIONS = [
'pageStatus',
'googlePages',
'googleQueries',
+ 'bots',
+ 'botPages',
'visits',
] as const;
@@ -56,6 +60,8 @@ export const TRAFFIC_TABLE_SECTIONS = [
'pageStatus',
'googlePages',
'googleQueries',
+ 'bots',
+ 'botPages',
'visits',
] as const;
@@ -86,6 +92,8 @@ export interface TrafficExportData {
googlePages?: TrafficGooglePages | null;
/** Google queries you don't track yet, every one the server checked (AT-009). */
googleQueries?: TrafficUntrackedQueries | null;
+ /** AI bots reading the site, every page (AT-011). */
+ bots?: TrafficBots | null;
sources?: TrafficSources | null;
}
@@ -400,6 +408,53 @@ function googleQueries(data: TrafficExportData, label: ExportLabel): ExportTable
return { ...table, rows: [...table.rows, [label('notes.queriesCapped'), null, null, null]] };
}
+/** AI bots reading the site (AT-011): each bot with its type, hits, pages and last request. */
+function bots(data: TrafficExportData, label: ExportLabel): ExportTable {
+ const rows = data.bots?.bots ?? [];
+ if (rows.length === 0)
+ return { headers: [label('columns.note')], rows: [[label('notes.noBots')]] };
+ return {
+ headers: [
+ label('columns.bot'),
+ label('columns.botType'),
+ label('columns.hits'),
+ label('columns.pagesRead'),
+ label('columns.lastSeen'),
+ ],
+ rows: rows.map((row) => [
+ botName(row.bot),
+ label(`botTypes.${row.type}`),
+ row.hits,
+ row.pages,
+ row.lastSeen ? utcMinute(row.lastSeen) : null,
+ ]),
+ };
+}
+
+/** The pages AI bots read (AT-011), with when each type last read them. */
+function botPages(data: TrafficExportData, label: ExportLabel): ExportTable {
+ const rows = data.bots?.pages ?? [];
+ if (rows.length === 0)
+ return { headers: [label('columns.note')], rows: [[label('notes.noBots')]] };
+ const at = (iso: string | null) => (iso ? utcMinute(iso) : null);
+ return {
+ headers: [
+ label('columns.page'),
+ label('columns.hits'),
+ label('columns.trainingLast'),
+ label('columns.searchLast'),
+ label('columns.userLast'),
+ ],
+ rows: rows.map((row) => [
+ row.page,
+ row.hits,
+ at(row.trainingLast),
+ at(row.searchLast),
+ at(row.userLast),
+ ]),
+ };
+}
+
function visits(data: TrafficExportData, label: ExportLabel): ExportTable {
return {
headers: [
@@ -448,6 +503,10 @@ export function trafficTable(
return googlePages(data, label);
case 'googleQueries':
return googleQueries(data, label);
+ case 'bots':
+ return bots(data, label);
+ case 'botPages':
+ return botPages(data, label);
case 'visits':
return visits(data, label);
}
diff --git a/web/src/types/supabase.ts b/web/src/types/supabase.ts
index dea68b579..352d72b00 100644
--- a/web/src/types/supabase.ts
+++ b/web/src/types/supabase.ts
@@ -2911,6 +2911,15 @@ export type Database = {
}
Returns: Json
}
+ ai_bot_report: {
+ Args: {
+ p_brand_id: string
+ p_limit?: number
+ p_since: string | null
+ p_until: string
+ }
+ Returns: Json
+ }
ai_traffic_conversions: {
Args: {
p_brand_id: string