Skip to content

Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.

fix(traffic): keep the export going without the untracked queries, and match the AI bots sheets to the report (#1244)

Cihance07273d7b46
diff --git a/web/messages/en.json b/web/messages/en.json
index 3d58607af..82a822dbf 100644
--- a/web/messages/en.json
+++ b/web/messages/en.json
@@ -3459,6 +3459,7 @@
"googleQueries": "Untracked Google queries",
"bots": "AI bots",
"botPages": "Pages AI bots read",
+ "uncrawledPages": "Cited pages no search bot read",
"visits": "Visits"
},
"columns": {
@@ -3509,6 +3510,7 @@
"query": "Query",
"bot": "Bot",
"botType": "Bot type",
+ "bots": "Bots",
"hits": "Hits",
"pagesRead": "Pages read",
"lastSeen": "Last seen (UTC)",
@@ -3546,6 +3548,11 @@
"noUntrackedQueries": "Every Google query with clicks in this range is tracked",
"queriesCapped": "Only the 5,000 most clicked queries were checked",
"noBots": "No AI-bot requests in this range",
+ "uncrawledWaiting": "Listed once the AI bots have sent 30 days of data",
+ "uncrawledStale": "Paused: the bot data doesn't cover the last 30 days, or the month's cap was reached, so the pages search bots read aren't complete",
+ "readFailed": "Couldn't read this section; export again to include it",
+ "noUncrawled": "Every cited page was read by an AI search bot in the last 30 days",
+ "uncrawledMore": "{count, plural, one {# more page not listed} other {# more pages not listed}}",
"noBrokenLinks": "No AI visitor landed on a missing page in this range"
},
"botTypes": {
diff --git a/web/src/app/api/traffic/export/route.test.ts b/web/src/app/api/traffic/export/route.test.ts
index e4bfed518..70c4feb27 100644
--- a/web/src/app/api/traffic/export/route.test.ts
+++ b/web/src/app/api/traffic/export/route.test.ts
@@ -165,6 +165,7 @@ describe('GET /api/traffic/export', () => {
'Untracked Google queries',
'AI bots',
'Pages AI bots read',
+ 'Cited pages no search bot read',
'Visits',
]);
@@ -206,6 +207,7 @@ describe('GET /api/traffic/export', () => {
'untracked-google-queries.csv',
'ai-bots.csv',
'pages-ai-bots-read.csv',
+ 'cited-pages-no-search-bot-read.csv',
'visits.csv',
]);
for (const file of Object.values(files)) expect(startsWithBom(file)).toBe(true);
@@ -324,10 +326,35 @@ describe('GET /api/traffic/export', () => {
expect((await get({ format: 'xlsx' })).status).toBe(404);
});
- it('exports the rest when the AI bot report times out', async () => {
+ it('exports the rest when the AI bot report times out, and says its sheets went unread', async () => {
+ vi.spyOn(console, 'error').mockImplementation(() => {});
loaders.getTrafficBots.mockRejectedValue(new Error('statement timeout'));
- const res = await get({ format: 'xlsx' });
+ const res = await get({ format: 'zip' });
+ expect(res.status).toBe(200);
+ const files = unzipSync(new Uint8Array(await res.arrayBuffer()));
+ for (const file of [
+ 'ai-bots.csv',
+ 'pages-ai-bots-read.csv',
+ 'cited-pages-no-search-bot-read.csv',
+ ]) {
+ expect(strFromU8(files[file]), file).toContain(
+ "Couldn't read this section; export again to include it",
+ );
+ }
+ });
+
+ // gkhngyk's note on #1178: an API 5xx on the untracked queries failed the whole export.
+ it('exports the rest when the untracked Google queries fail', async () => {
+ vi.spyOn(console, 'error').mockImplementation(() => {});
+ loaders.getUntrackedGoogleQueries.mockRejectedValue(new Error('502'));
+ const res = await get({ format: 'zip' });
expect(res.status).toBe(200);
+ const files = unzipSync(new Uint8Array(await res.arrayBuffer()));
+ // Not "No query data from Google Search Console yet": the read failed.
+ expect(strFromU8(files['untracked-google-queries.csv'])).toContain(
+ "Couldn't read this section; export again to include it",
+ );
+ expect(strFromU8(files['visits.csv'])).toContain('/menu');
});
it('reports a failed read as an error, not as an empty file', async () => {
diff --git a/web/src/app/api/traffic/export/route.ts b/web/src/app/api/traffic/export/route.ts
index a4366bf6e..8dd2150f7 100644
--- a/web/src/app/api/traffic/export/route.ts
+++ b/web/src/app/api/traffic/export/route.ts
@@ -30,6 +30,7 @@ import {
type ExportLabel,
type TrafficExportData,
type TrafficExportSection,
+ type UnreadSection,
} from '@/lib/traffic-export';
/**
@@ -95,6 +96,15 @@ export async function GET(req: Request) {
const window = { dateFrom: q.from, dateTo: q.to };
const wholePage = !q.section;
+ // A failed read of these is left out, so the rest still exports; its
+ // sheet says it couldn't be read rather than reading as empty.
+ const unread: UnreadSection[] = [];
+ const leftOut = (section: UnreadSection) => (err: unknown) => {
+ console.error(`[traffic-export] ${section} not read:`, err);
+ unread.push(section);
+ return null;
+ };
+
try {
const [
summary,
@@ -135,13 +145,18 @@ export async function GET(req: Request) {
? getTrafficGooglePages(q.brandId, window, { limit: TRAFFIC_EXPORT_MAX_ROWS })
: undefined,
// Every Google query no tracked prompt covers, as far as the server checks (AT-009).
+ // Left out when the API fails, so the rest still exports.
needs('googleQueries')
- ? getUntrackedGoogleQueries(q.brandId, window, { limit: UNTRACKED_QUERY_SCAN })
+ ? getUntrackedGoogleQueries(q.brandId, window, { limit: UNTRACKED_QUERY_SCAN }).catch(
+ leftOut('googleQueries'),
+ )
: undefined,
// Every page AI bots read, with the bot types (AT-011).
// Left out when it fails (a timeout on a large site), so the rest still exports.
- needs('bots', 'botPages')
- ? getTrafficBots(q.brandId, window, { limit: TRAFFIC_EXPORT_MAX_ROWS }).catch(() => null)
+ needs('bots', 'botPages', 'uncrawledPages')
+ ? getTrafficBots(q.brandId, window, { limit: TRAFFIC_EXPORT_MAX_ROWS }).catch(
+ leftOut('bots'),
+ )
: undefined,
// Every broken AI link, as the redirect list (AT-012).
needs('brokenLinks')
@@ -172,6 +187,7 @@ export async function GET(req: Request) {
bots,
brokenLinks,
sources,
+ unread,
};
const fileName = trafficExportFileName({
diff --git a/web/src/lib/traffic-export.test.ts b/web/src/lib/traffic-export.test.ts
index 98e07601d..bff97abc8 100644
--- a/web/src/lib/traffic-export.test.ts
+++ b/web/src/lib/traffic-export.test.ts
@@ -498,7 +498,16 @@ describe('trafficTable: AI bots (AT-011)', () => {
const bots = (): NonNullable<TrafficExportData['bots']> => ({
firstDay: '2026-09-01',
lastSeen: '2026-10-19T09:30:00Z',
- types: [],
+ types: [
+ {
+ type: 'training',
+ hits: 120,
+ pages: 40,
+ lastSeen: '2026-10-19T09:30:00Z',
+ bots: ['gptbot', 'claudebot'],
+ },
+ { type: 'user', hits: 3, pages: 2, lastSeen: null, bots: ['perplexity-user'] },
+ ],
bots: [
{ bot: 'gptbot', type: 'training', hits: 120, pages: 40, lastSeen: '2026-10-19T09:30:00Z' },
{ bot: 'perplexity-user', type: 'user', hits: 3, pages: 2, lastSeen: null },
@@ -520,12 +529,12 @@ describe('trafficTable: AI bots (AT-011)', () => {
since: '2026-10-12',
});
- it('holds each bot with its type, and each page with every type last read', () => {
+ it('holds each bot type with its bots, as the By bot type card, and each page with every type last read', () => {
expect(trafficTable('bots', data({ bots: bots() }), label)).toEqual({
- headers: ['Bot', 'Bot type', 'Hits', 'Pages read', 'Last seen (UTC)'],
+ headers: ['Bot type', 'Bots', 'Hits', 'Pages read', 'Last seen (UTC)'],
rows: [
- ['GPTBot', 'Training', 120, 40, '2026-10-19 09:30'],
- ['Perplexity-User', 'User fetch', 3, 2, null],
+ ['Training', 'GPTBot, ClaudeBot', 120, 40, '2026-10-19 09:30'],
+ ['User fetch', 'Perplexity-User', 3, 2, null],
],
});
expect(trafficTable('botPages', data({ bots: bots() }), label)).toEqual({
@@ -547,6 +556,65 @@ describe('trafficTable: AI bots (AT-011)', () => {
expect(trafficTable('botPages', data({}), label).rows).toEqual([
['No AI-bot requests in this range'],
]);
+ expect(trafficTable('uncrawledPages', data({}), label).rows).toEqual([
+ ['No AI-bot requests in this range'],
+ ]);
+ });
+
+ it('lists the cited pages no search bot read, as the report does, with how many more', () => {
+ const report = {
+ ...bots(),
+ uncrawledCount: 12,
+ uncrawled: [
+ { page: '/guides/espresso', citations: 14 },
+ { page: '/gifts', citations: 9 },
+ ],
+ };
+ expect(trafficTable('uncrawledPages', data({ bots: report }), label)).toEqual({
+ headers: ['Page', 'Citations'],
+ rows: [
+ ['/guides/espresso', 14],
+ ['/gifts', 9],
+ ['10 more pages not listed', null],
+ ],
+ });
+ });
+
+ it('says why no uncrawled page is listed', () => {
+ expect(trafficTable('uncrawledPages', data({ bots: bots() }), label).rows).toEqual([
+ ['Every cited page was read by an AI search bot in the last 30 days'],
+ ]);
+ // Under 30 days of bot data.
+ const waiting = {
+ ...bots(),
+ uncrawledCount: null,
+ uncrawled: null,
+ alertState: 'waiting' as const,
+ };
+ expect(trafficTable('uncrawledPages', data({ bots: waiting }), label).rows).toEqual([
+ ['Listed once the AI bots have sent 30 days of data'],
+ ]);
+ // gkhngyk on #1244: paused, not short, as the report says it.
+ const stale = { ...waiting, alertState: 'stale' as const };
+ expect(trafficTable('uncrawledPages', data({ bots: stale }), label).rows).toEqual([
+ [
+ "Paused: the bot data doesn't cover the last 30 days, or the month's cap was reached, so the pages search bots read aren't complete",
+ ],
+ ]);
+ });
+
+ // gkhngyk on #1244: a failed read said there was no data.
+ it("says a section couldn't be read rather than that it's empty", () => {
+ const failed = data({ unread: ['bots', 'googleQueries'] });
+ for (const section of ['bots', 'botPages', 'uncrawledPages', 'googleQueries'] as const) {
+ expect(trafficTable(section, failed, label).rows, section).toEqual([
+ ["Couldn't read this section; export again to include it"],
+ ]);
+ }
+ // Only the section that failed.
+ expect(trafficTable('bots', data({ unread: ['googleQueries'] }), label).rows).toEqual([
+ ['No AI-bot requests in this range'],
+ ]);
});
});
@@ -675,6 +743,7 @@ describe('trafficWorkbook', () => {
'Untracked Google queries',
'AI bots',
'Pages AI bots read',
+ 'Cited pages no search bot read',
'Visits',
]);
// A bold header row, then the rows with their numbers as numbers.
diff --git a/web/src/lib/traffic-export.ts b/web/src/lib/traffic-export.ts
index 62def94fd..85842b562 100644
--- a/web/src/lib/traffic-export.ts
+++ b/web/src/lib/traffic-export.ts
@@ -48,6 +48,7 @@ export const TRAFFIC_EXPORT_SECTIONS = [
'googleQueries',
'bots',
'botPages',
+ 'uncrawledPages',
'visits',
] as const;
@@ -100,6 +101,19 @@ export interface TrafficExportData {
/** AI bots reading the site, every page (AT-011). */
bots?: TrafficBots | null;
sources?: TrafficSources | null;
+ /**
+ * Sections whose read failed while the rest exported: their sheets say so
+ * rather than reading as empty (gkhngyk on #1244).
+ */
+ unread?: UnreadSection[];
+}
+
+/** The reads an export leaves out when they fail, so the rest still exports. */
+export type UnreadSection = 'googleQueries' | 'bots';
+
+/** A sheet that is only a note. */
+function noteTable(label: ExportLabel, key: string, values?: Record<string, number>): ExportTable {
+ return { headers: [label('columns.note')], rows: [[label(key, values)]] };
}
type Cell = string | number | null;
@@ -433,6 +447,7 @@ function googlePages(data: TrafficExportData, label: ExportLabel): ExportTable {
function googleQueries(data: TrafficExportData, label: ExportLabel): ExportTable {
const q = data.googleQueries;
const note = (key: string) => ({ headers: [label('columns.note')], rows: [[label(key)]] });
+ if (data.unread?.includes('googleQueries')) return note('notes.readFailed');
if (!q || !q.lastDay) return note('notes.noGoogleQueries');
if (q.rows.length === 0) return note('notes.noUntrackedQueries');
const days = q.since ? `${q.since} – ${q.until}` : `${label('filters.start')} – ${q.until}`;
@@ -449,22 +464,25 @@ function googleQueries(data: TrafficExportData, label: ExportLabel): ExportTable
return { ...table, rows: [...table.rows, [label('notes.queriesCapped'), null, null, null]] };
}
-/** AI bots reading the site (AT-011): each bot with its type, hits, pages and last request. */
+/**
+ * AI bots reading the site (AT-011), as the By bot type card lists them:
+ * each type with its bots, hits, pages and last request.
+ */
function bots(data: TrafficExportData, label: ExportLabel): ExportTable {
- const rows = data.bots?.bots ?? [];
- if (rows.length === 0)
- return { headers: [label('columns.note')], rows: [[label('notes.noBots')]] };
+ const rows = data.bots?.types ?? [];
+ if (data.unread?.includes('bots')) return noteTable(label, 'notes.readFailed');
+ if (rows.length === 0) return noteTable(label, 'notes.noBots');
return {
headers: [
- label('columns.bot'),
label('columns.botType'),
+ label('columns.bots'),
label('columns.hits'),
label('columns.pagesRead'),
label('columns.lastSeen'),
],
rows: rows.map((row) => [
- botName(row.bot),
label(`botTypes.${row.type}`),
+ row.bots.map(botName).join(', '),
row.hits,
row.pages,
row.lastSeen ? utcMinute(row.lastSeen) : null,
@@ -472,11 +490,33 @@ function bots(data: TrafficExportData, label: ExportLabel): ExportTable {
};
}
+/**
+ * The cited pages no AI search bot read in the last 30 days (AT-011), as the
+ * report lists them: the most cited first, then how many more there are.
+ */
+function uncrawledPages(data: TrafficExportData, label: ExportLabel): ExportTable {
+ const report = data.bots;
+ if (data.unread?.includes('bots')) return noteTable(label, 'notes.readFailed');
+ if (!report) return noteTable(label, 'notes.noBots');
+ if (report.uncrawledCount === null || report.uncrawled === null) {
+ // Paused, as the report says it (its data isn't whole), or still under 30 days.
+ return noteTable(
+ label,
+ report.alertState === 'stale' ? 'notes.uncrawledStale' : 'notes.uncrawledWaiting',
+ );
+ }
+ if (report.uncrawledCount === 0) return noteTable(label, 'notes.noUncrawled');
+ const rows: Cell[][] = report.uncrawled.map((row) => [row.page, row.citations]);
+ const more = report.uncrawledCount - report.uncrawled.length;
+ if (more > 0) rows.push([label('notes.uncrawledMore', { count: more }), null]);
+ return { headers: [label('columns.page'), label('columns.citations')], rows };
+}
+
/** The pages AI bots read (AT-011), with when each type last read them. */
function botPages(data: TrafficExportData, label: ExportLabel): ExportTable {
const rows = data.bots?.pages ?? [];
- if (rows.length === 0)
- return { headers: [label('columns.note')], rows: [[label('notes.noBots')]] };
+ if (data.unread?.includes('bots')) return noteTable(label, 'notes.readFailed');
+ if (rows.length === 0) return noteTable(label, 'notes.noBots');
const at = (iso: string | null) => (iso ? utcMinute(iso) : null);
return {
headers: [
@@ -550,6 +590,8 @@ export function trafficTable(
return bots(data, label);
case 'botPages':
return botPages(data, label);
+ case 'uncrawledPages':
+ return uncrawledPages(data, label);
case 'visits':
return visits(data, label);
}