Skip to content

Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.

feat(signals): show which crawlers and checks a technical finding rests on (#1065)

Cihan17b9041dfe64
diff --git a/server/src/lib/audit/signals/trust.js b/server/src/lib/audit/signals/trust.js
index 3c5c3c572..410d00a29 100644
--- a/server/src/lib/audit/signals/trust.js
+++ b/server/src/lib/audit/signals/trust.js
@@ -39,6 +39,41 @@ export const https = {
},
};
+/**
+ * robots.txt as its groups: the user agents a group names and whether its
+ * rules disallow or allow the whole site. Consecutive user-agent lines share
+ * a group; comments and blank lines are skipped. An agent is kept as the
+ * crawler matches it: its name in lower case, up to a version ("GPTBot/1.1").
+ */
+export function robotsGroups(robotsTxt) {
+ const groups = [];
+ let current = null;
+ let agentLine = false;
+ for (const raw of String(robotsTxt ?? '').split(/\r?\n/)) {
+ const line = raw.replace(/#.*/, '').trim();
+ if (!line || !line.includes(':')) continue;
+ const at = line.indexOf(':');
+ const field = line.slice(0, at).trim().toLowerCase();
+ const value = line.slice(at + 1).trim();
+ if (field === 'user-agent') {
+ if (!current || !agentLine) {
+ current = { agents: [], disallowAll: false, allowAll: false };
+ groups.push(current);
+ }
+ const agent = value.toLowerCase();
+ current.agents.push(agent.match(/^[a-z_-]+/)?.[0] ?? agent);
+ agentLine = true;
+ } else {
+ agentLine = false;
+ if (current && value === '/') {
+ if (field === 'disallow') current.disallowAll = true;
+ if (field === 'allow') current.allowAll = true;
+ }
+ }
+ }
+ return groups;
+}
+
export const aiBotAccess = {
key: 'ai-bot-access',
evaluate(ctx) {
@@ -48,12 +83,17 @@ export const aiBotAccess = {
}
const lower = ctx.robotsTxt.toLowerCase();
const mentioned = AI_BOTS.filter((b) => lower.includes(b));
- // Detect an explicit block: a mentioned AI UA followed by "disallow: /".
- const blocked = mentioned.some((b) => {
- const idx = lower.indexOf(b);
- const after = lower.slice(idx, idx + 200);
- return /disallow:\s*\/\s*(\n|$)/.test(after);
+ // An explicit block: the bot's own groups disallow the whole site and
+ // none allows it back ("Allow: /" wins a tie). Read group by group, as
+ // crawlers do; scanning the text after a bot's name also caught the next
+ // group's rules, and blocked a bot that robots.txt allows. The Action
+ // Center names the crawlers a block affects.
+ const groups = robotsGroups(ctx.robotsTxt);
+ const blockedBots = mentioned.filter((b) => {
+ const own = groups.filter((g) => g.agents.includes(b));
+ return own.some((g) => g.disallowAll) && !own.some((g) => g.allowAll);
});
+ const blocked = blockedBots.length > 0;
let status = 'warn';
let score = 0.5;
@@ -64,7 +104,7 @@ export const aiBotAccess = {
status = 'pass';
score = 1;
}
- return { status, score, evidence: { aiBotsMentioned: mentioned, blocked } };
+ return { status, score, evidence: { aiBotsMentioned: mentioned, blocked, blockedBots } };
},
};
diff --git a/server/src/lib/audit/signals/trust.test.js b/server/src/lib/audit/signals/trust.test.js
new file mode 100644
index 000000000..369dda71c
--- /dev/null
+++ b/server/src/lib/audit/signals/trust.test.js
@@ -0,0 +1,74 @@
+import { describe, expect, it } from 'vitest';
+import { aiBotAccess } from './trust.js';
+
+/**
+ * The AI bot access check names the bots robots.txt blocks, so the Action
+ * Center can say which crawlers a block affects (AC-001). Any explicit block
+ * fails the check; robots.txt is read group by group, so a bot it allows is
+ * no longer failed for the rules of the group after it.
+ */
+describe('aiBotAccess', () => {
+ it('names each AI bot robots.txt disallows, and fails on any', () => {
+ const robotsTxt = [
+ 'User-agent: GPTBot',
+ 'Disallow: /',
+ '',
+ 'User-agent: ClaudeBot',
+ 'Allow: /',
+ '',
+ 'User-agent: PerplexityBot',
+ 'Disallow: /',
+ ].join('\n');
+ expect(aiBotAccess.evaluate({ robotsTxt })).toEqual({
+ status: 'fail',
+ score: 0,
+ evidence: {
+ aiBotsMentioned: ['gptbot', 'claudebot', 'perplexitybot'],
+ blocked: true,
+ blockedBots: ['gptbot', 'perplexitybot'],
+ },
+ });
+ });
+
+ it("doesn't take the next group's rule for an allowed bot's", () => {
+ const robotsTxt = 'User-agent: ClaudeBot\nAllow: /\n\nUser-agent: GPTBot\nDisallow: /';
+ expect(aiBotAccess.evaluate({ robotsTxt }).evidence.blockedBots).toEqual(['gptbot']);
+ // A group naming several agents blocks each tracked AI bot among them.
+ expect(
+ aiBotAccess.evaluate({
+ robotsTxt: 'User-agent: GPTBot\nUser-agent: CCBot\nDisallow: / # all',
+ }).evidence.blockedBots,
+ ).toEqual(['gptbot']);
+ });
+
+ it('passes a bot robots.txt allows, though everyone else is blocked', () => {
+ // The old reading took the "*" group's rule for ClaudeBot's and failed.
+ expect(
+ aiBotAccess.evaluate({
+ robotsTxt: 'User-agent: ClaudeBot\nAllow: /\n\nUser-agent: *\nDisallow: /',
+ }),
+ ).toMatchObject({ status: 'pass', evidence: { blocked: false, blockedBots: [] } });
+ });
+
+ it('reads agents as crawlers match them, and lets "Allow: /" win', () => {
+ // A version after the name still names the bot.
+ expect(
+ aiBotAccess.evaluate({ robotsTxt: 'User-agent: GPTBot/1.1\nDisallow: /' }).evidence
+ .blockedBots,
+ ).toEqual(['gptbot']);
+ // A bot's groups count together, and allowing the whole site wins a tie.
+ expect(
+ aiBotAccess.evaluate({
+ robotsTxt: 'User-agent: GPTBot\nDisallow: /\n\nUser-agent: GPTBot\nAllow: /',
+ }).status,
+ ).toBe('pass');
+ });
+
+ it('passes with no bot blocked, and warns without robots.txt', () => {
+ expect(aiBotAccess.evaluate({ robotsTxt: 'User-agent: GPTBot\nAllow: /' })).toMatchObject({
+ status: 'pass',
+ evidence: { blocked: false, blockedBots: [] },
+ });
+ expect(aiBotAccess.evaluate({ robotsTxt: null }).status).toBe('warn');
+ });
+});
diff --git a/server/src/lib/signals/library/audits-family.test.js b/server/src/lib/signals/library/audits-family.test.js
new file mode 100644
index 000000000..9ef4d5efd
--- /dev/null
+++ b/server/src/lib/signals/library/audits-family.test.js
@@ -0,0 +1,71 @@
+import { describe, expect, it, vi } from 'vitest';
+
+/**
+ * The audits family writes, with each technical finding, what every failed
+ * check found on each page (AC-001): the check, its label as the audit names
+ * it, and the check's own evidence.
+ */
+
+const tables = {
+ site_audits: [
+ { id: 'a2', url: 'https://acme.example/pricing', final_url: null, completed_at: '2026-10-10' },
+ // An older audit of the same page: the newer one decides.
+ { id: 'a1', url: 'https://acme.example/pricing', final_url: null, completed_at: '2026-09-01' },
+ ],
+ audit_signal_results: [
+ {
+ audit_id: 'a2',
+ signal_key: 'ai-bot-access',
+ evidence: { aiBotsMentioned: ['gptbot'], blocked: true, blockedBots: ['gptbot'] },
+ },
+ { audit_id: 'a2', signal_key: 'freshness', evidence: { ageDays: 412, newest: '2025-08-30' } },
+ ],
+};
+
+function builder(table) {
+ const chain = {
+ select: () => chain,
+ eq: () => chain,
+ gte: () => chain,
+ in: () => chain,
+ order: () => chain,
+ range: async () => ({ data: tables[table] ?? [], error: null }),
+ };
+ return chain;
+}
+
+vi.mock('../../../config/supabase.js', () => ({
+ default: { from: (table) => builder(table), rpc: vi.fn() },
+}));
+vi.mock('../../logger.js', () => ({ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() } }));
+
+const { LIBRARY_FAMILIES } = await import('./detect.js');
+
+describe('the audits family', () => {
+ it("records each failed check's label and evidence on its page", async () => {
+ const audits = LIBRARY_FAMILIES.find((f) => f.name === 'audits');
+ const found = await audits.run({ brandId: 'b1', now: new Date('2026-10-15T00:00:00Z') });
+ const byKind = Object.fromEntries(found.map((f) => [f.kind, f.payload.targets]));
+ expect(byKind.crawler_blocked).toEqual([
+ {
+ id: 'https://acme.example/pricing',
+ label: 'https://acme.example/pricing',
+ checks: ['ai-bot-access'],
+ issues: [
+ {
+ check: 'ai-bot-access',
+ label: 'AI bot access (robots.txt)',
+ evidence: { aiBotsMentioned: ['gptbot'], blocked: true, blockedBots: ['gptbot'] },
+ },
+ ],
+ },
+ ]);
+ expect(byKind.outdated_content[0].issues).toEqual([
+ {
+ check: 'freshness',
+ label: 'Freshness',
+ evidence: { ageDays: 412, newest: '2025-08-30' },
+ },
+ ]);
+ });
+});
diff --git a/server/src/lib/signals/library/detect.js b/server/src/lib/signals/library/detect.js
index 3613a0329..6cc796625 100644
--- a/server/src/lib/signals/library/detect.js
+++ b/server/src/lib/signals/library/detect.js
@@ -25,6 +25,7 @@ import { resolve } from '../../../config/action-engine.js';
import { resolveBrandSources } from '../../action-center/sources.js';
import * as c from './classify.js';
import { isTransientDbError } from '../../pulse/metrics.js';
+import { describeSignal } from '../../audit/rubric.js';
const { library: t } = resolve();
const DAY_MS = 86_400_000;
@@ -714,7 +715,7 @@ const FAMILIES = [
const failures = await selectAll(() =>
supabaseAdmin
.from('audit_signal_results')
- .select('audit_id, signal_key')
+ .select('audit_id, signal_key, evidence')
.in('audit_id', ids)
.eq('status', 'fail'),
);
@@ -725,8 +726,15 @@ const FAMILIES = [
if (!kind) continue;
const page = pageOf.get(f.audit_id);
const set = (groups[kind] ??= new Map());
- const entry = set.get(page) ?? { page, checks: [] };
+ const entry = set.get(page) ?? { page, checks: [], issues: [] };
entry.checks.push(f.signal_key);
+ // What the check found, named as the audit names it (AC-001): the
+ // drawer shows it as the issue's evidence.
+ entry.issues.push({
+ check: f.signal_key,
+ label: describeSignal(f.signal_key)?.label ?? f.signal_key,
+ evidence: f.evidence ?? {},
+ });
set.set(page, entry);
}
return Object.entries(groups).map(([kind, pages]) =>
@@ -734,7 +742,7 @@ const FAMILIES = [
kind,
'pages',
[...pages.values()],
- (e) => ({ id: e.page, label: e.page, checks: e.checks }),
+ (e) => ({ id: e.page, label: e.page, checks: e.checks, issues: e.issues }),
AUDIT,
),
);
diff --git a/web/messages/en.json b/web/messages/en.json
index 01cda6df1..a3bfec545 100644
--- a/web/messages/en.json
+++ b/web/messages/en.json
@@ -350,6 +350,29 @@
"viewPrompt": "View prompt",
"notAvailable": "Not available",
"allPlatforms": "All tracked platforms",
+ "checkEvidence": {
+ "yes": "yes",
+ "no": "no",
+ "keys": {
+ "aiBotsMentioned": "In robots.txt",
+ "ageDays": "Age in days",
+ "newest": "Newest date",
+ "dateSignals": "Date signals",
+ "blockCount": "JSON-LD blocks",
+ "blocks": "JSON-LD blocks",
+ "parsed": "Parsed",
+ "wellFormedNodes": "Well-formed nodes",
+ "types": "Schema types",
+ "faqPage": "FAQ schema",
+ "questionCount": "Questions",
+ "internalLinks": "Internal links",
+ "matches": "Wikidata matches",
+ "qid": "Wikidata ID",
+ "label": "Wikidata label",
+ "description": "Wikidata description",
+ "note": "Note"
+ }
+ },
"mentions": {
"sentiment": {
"positive": "Positive",
diff --git a/web/src/components/action-center/signal-detail-blocks.test.tsx b/web/src/components/action-center/signal-detail-blocks.test.tsx
index 1742ffad6..b348e2be2 100644
--- a/web/src/components/action-center/signal-detail-blocks.test.tsx
+++ b/web/src/components/action-center/signal-detail-blocks.test.tsx
@@ -299,6 +299,87 @@ describe('an uncited-mentions signal with its evidence', () => {
});
});
+describe("a technical finding with its checks' evidence", () => {
+ it('shows the crawlers it blocks and what each check found, page by page', () => {
+ render(
+ <SignalWhereBlock
+ signal={signal({
+ category: 'technical',
+ kind: 'crawler_blocked',
+ currentValue: 1,
+ payload: {
+ entity: 'pages',
+ targets: [
+ {
+ id: '/pricing',
+ label: '/pricing',
+ issues: [
+ {
+ check: 'ai-bot-access',
+ label: 'AI bot access (robots.txt)',
+ evidence: {
+ aiBotsMentioned: ['gptbot', 'claudebot'],
+ blocked: true,
+ blockedBots: ['gptbot'],
+ },
+ },
+ ],
+ },
+ ],
+ },
+ where: NO_WHERE,
+ })}
+ />,
+ );
+ const rows = [...container.querySelectorAll('dt')].map(
+ (dt) => `${dt.textContent}: ${dt.nextElementSibling?.textContent}`,
+ );
+ expect(rows).toContain('Affected crawlers: GPTBot (OpenAI)');
+ expect(rows).toContain(
+ 'Failed checks: /pricingAI bot access (robots.txt) · In robots.txt: GPTBot (OpenAI), ClaudeBot (Anthropic)',
+ );
+ });
+
+ it('names what a check found, and leaves out the model that judged it', () => {
+ render(
+ <SignalWhereBlock
+ signal={signal({
+ category: 'technical',
+ kind: 'content_gap',
+ currentValue: 1,
+ payload: {
+ entity: 'pages',
+ targets: [
+ {
+ id: '/guide',
+ label: '/guide',
+ issues: [
+ {
+ check: 'query-coverage',
+ label: 'Query coverage',
+ evidence: { reason: 'No pricing answers.', model: 'openai/gpt-4.1-mini' },
+ },
+ {
+ check: 'freshness',
+ label: 'Freshness',
+ evidence: { ageDays: 412, newest: '2025-08-30', pages: [{ url: '/a' }] },
+ },
+ ],
+ },
+ ],
+ },
+ where: NO_WHERE,
+ })}
+ />,
+ );
+ const lines = [...container.querySelectorAll('dd li span.block')].map((el) => el.textContent);
+ expect(lines).toEqual([
+ 'Query coverage · No pricing answers.',
+ 'Freshness · Age in days: 412 · Newest date: 2025-08-30',
+ ]);
+ });
+});
+
describe('SignalEvidenceBlock', () => {
it('links the answers and the action the signal is used by', async () => {
getSignalActions.mockResolvedValue([
diff --git a/web/src/components/action-center/signal-detail-blocks.tsx b/web/src/components/action-center/signal-detail-blocks.tsx
index 7336acf47..8debef68b 100644
--- a/web/src/components/action-center/signal-detail-blocks.tsx
+++ b/web/src/components/action-center/signal-detail-blocks.tsx
@@ -7,7 +7,7 @@ import { Link } from '@/i18n/navigation';
import { AIProviderAvatar } from '@/components/ai-provider-avatar';
import { getSignalActions, type Signal, type SignalAction } from '@/lib/actions/signals';
import { actionTexts } from '@/lib/action-center/display';
-import { locationName, platformFamily, platformName } from '@/lib/signals/names';
+import { crawlerName, locationName, platformFamily, platformName } from '@/lib/signals/names';
import {
METRIC_UNITS,
signalChange,
@@ -42,6 +42,65 @@ function More({ shown, total }: { shown: number; total: number }) {
) : null;
}
+// The keys audit checks write when they fail, named in the messages; the
+// rest are shown as written. A model name, and what the crawlers row
+// already says, are left out.
+const EVIDENCE_KEYS = [
+ 'aiBotsMentioned',
+ 'ageDays',
+ 'newest',
+ 'dateSignals',
+ 'blockCount',
+ 'blocks',
+ 'parsed',
+ 'wellFormedNodes',
+ 'types',
+ 'faqPage',
+ 'questionCount',
+ 'internalLinks',
+ 'matches',
+ 'qid',
+ 'label',
+ 'description',
+ 'note',
+] as const;
+const HIDDEN_EVIDENCE = new Set(['model', 'blocked', 'blockedBots']);
+
+/**
+ * An audit check's evidence as a line: "Age in days: 412 · Newest date:
+ * 2025-08-30". The checks write small flat objects meant to be shown; lists
+ * are joined and empty values left out.
+ */
+function describeEvidence(
+ evidence: Record<string, unknown>,
+ t: ReturnType<typeof useTranslations>,
+): string {
+ const show = (key: string, value: unknown): string | null => {
+ if (value === null || value === undefined || value === '') return null;
+ if (Array.isArray(value)) {
+ const items = value.filter((v) => typeof v === 'string' || typeof v === 'number');
+ if (items.length === 0) return null;
+ return items.map((v) => (key === 'aiBotsMentioned' ? crawlerName(String(v)) : v)).join(', ');
+ }
+ if (typeof value === 'boolean')
+ return value ? t('drawer.checkEvidence.yes') : t('drawer.checkEvidence.no');
+ if (typeof value === 'object') return null;
+ return String(value);
+ };
+ const name = (key: string) =>
+ (EVIDENCE_KEYS as readonly string[]).includes(key)
+ ? t(`drawer.checkEvidence.keys.${key as (typeof EVIDENCE_KEYS)[number]}`)
+ : key;
+ return Object.entries(evidence)
+ .filter(([key]) => !HIDDEN_EVIDENCE.has(key))
+ .map(([key, value]) => {
+ const shown = show(key, value);
+ return shown === null ? null : key === 'reason' ? shown : `${name(key)}: ${shown}`;
+ })
+ .filter(Boolean)
+ .join(' · ');
+}
+
function WhereValueView({ value }: { value: WhereValue }) {
const t = useTranslations('actionCenter.signalsPage');
const format = useFormatter();
@@ -148,6 +207,35 @@ function WhereValueView({ value }: { value: WhereValue }) {
))}
</ul>
);
+ case 'crawlers':
+ return <span>{value.agents.map(crawlerName).join(', ')}</span>;
+ case 'issues': {
+ // Page by page, each failed check with what it found.
+ const byPage = new Map<string, typeof value.items>();
+ for (const issue of value.items) {
+ byPage.set(issue.page, [...(byPage.get(issue.page) ?? []), issue]);
+ }
+ const pages = [...byPage].slice(0, LIST_LIMIT);
+ return (
+ <ul className="space-y-2">
+ {pages.map(([page, issues]) => (
+ <li key={page}>
+ <span className="break-all">{page}</span>
+ {issues.map((issue) => {
+ const found = describeEvidence(issue.evidence, t);
+ return (
+ <span key={issue.check} className="block text-xs text-muted-foreground">
+ <span className="font-medium text-foreground">{issue.label}</span>
+ {found && ` · ${found}`}
+ </span>
+ );
+ })}
+ </li>
+ ))}
+ <More shown={pages.length} total={byPage.size} />
+ </ul>
+ );
+ }
case 'mentions':
return (
<ul className="space-y-2.5">
diff --git a/web/src/lib/signals/detail.test.ts b/web/src/lib/signals/detail.test.ts
index 5ac7374bc..45f1361ed 100644
--- a/web/src/lib/signals/detail.test.ts
+++ b/web/src/lib/signals/detail.test.ts
@@ -208,14 +208,21 @@ describe('signalWhereFields', () => {
},
currentValue: 1,
});
+ // Recorded before the checks' evidence: the failed checks by name, page
+ // by page, in the same shape. A stale page has no crawlers to name.
+ const check = (name: string) => ({ page: '/pricing', check: name, label: name, evidence: {} });
expect(signalWhereFields(audit)).toEqual([
{ key: 'pages', value: { type: 'list', items: ['/pricing'], total: 1 } },
- { key: 'crawlers', value: null },
+ { key: 'issue', value: { type: 'list', items: ['dateModified', 'freshness'], total: 2 } },
{
key: 'evidence',
- value: { type: 'list', items: ['dateModified', 'freshness'], total: 2 },
+ value: { type: 'issues', items: [check('dateModified'), check('freshness')] },
},
]);
+ expect(signalDetailForApi(audit, []).where.evidence).toEqual([
+ check('dateModified'),
+ check('freshness'),
+ ]);
});
});
@@ -457,3 +464,53 @@ describe('an uncited-mentions signal', () => {
]);
});
});
+
+describe("a technical finding with its checks' evidence", () => {
+ const issue = {
+ check: 'ai-bot-access',
+ label: 'AI bot access (robots.txt)',
+ evidence: { aiBotsMentioned: ['gptbot', 'claudebot'], blocked: true, blockedBots: ['gptbot'] },
+ };
+ const blocked = signal({
+ category: 'technical',
+ kind: 'crawler_blocked',
+ payload: {
+ entity: 'pages',
+ targets: [{ id: '/pricing', label: '/pricing', checks: ['ai-bot-access'], issues: [issue] }],
+ },
+ currentValue: 1,
+ });
+
+ it('names the issue, the crawlers it blocks, and what the check found', () => {
+ expect(signalWhereFields(blocked)).toEqual([
+ { key: 'pages', value: { type: 'list', items: ['/pricing'], total: 1 } },
+ { key: 'issue', value: { type: 'list', items: ['AI bot access (robots.txt)'], total: 1 } },
+ { key: 'crawlers', value: { type: 'crawlers', agents: ['gptbot'] } },
+ {
+ key: 'evidence',
+ value: { type: 'issues', items: [{ page: '/pricing', ...issue }] },
+ },
+ ]);
+ expect(signalDetailForApi(blocked, []).where.crawlers).toEqual(['gptbot']);
+ });
+
+ it("doesn't guess the crawlers from an audit that only said one was blocked", () => {
+ // That audit also listed bots robots.txt allows.
+ const older = signal({
+ ...blocked,
+ payload: {
+ entity: 'pages',
+ targets: [
+ {
+ id: '/pricing',
+ issues: [
+ { ...issue, evidence: { aiBotsMentioned: ['gptbot', 'claudebot'], blocked: true } },
+ ],
+ },
+ ],
+ },
+ });
+ const byKey = Object.fromEntries(signalWhereFields(older).map((f) => [f.key, f.value]));
+ expect(byKey.crawlers).toBeNull();
+ });
+});
diff --git a/web/src/lib/signals/detail.ts b/web/src/lib/signals/detail.ts
index 54dea14f6..13610b64b 100644
--- a/web/src/lib/signals/detail.ts
+++ b/web/src/lib/signals/detail.ts
@@ -375,6 +375,10 @@ export type WhereValue =
| { type: 'sources'; items: WinningSource[] }
/** The brand's mentions on each prompt: where, how high, in what tone. */
| { type: 'mentions'; items: PromptMention[] }
+ /** What each failed audit check found, page by page. */
+ | { type: 'issues'; items: PageIssue[] }
+ /** AI crawlers by robots.txt user agent (`gptbot`). */
+ | { type: 'crawlers'; agents: string[] }
| { type: 'locations'; codes: readonly string[] }
| { type: 'number'; value: number };
@@ -447,6 +451,53 @@ function promptMentions(value: unknown): PromptMention[] {
}));
}
+/** A failed audit check on a page, with the check's own evidence. */
+export interface PageIssue {
+ page: string;
+ check: string;
+ label: string;
+ evidence: Record<string, unknown>;
+}
+
+/**
+ * The failed checks on each page. Findings recorded before the checks'
+ * evidence name the checks only: their key stands in for the label.
+ */
+function pageIssues(targets: Target[]): PageIssue[] {
+ return targets.flatMap((t) => {
+ const page = label(t);
+ const issues: Record<string, unknown>[] = Array.isArray(t.issues)
+ ? t.issues
+ : Array.isArray(t.checks)
+ ? t.checks.map((check: unknown) => ({ check }))
+ : [];
+ return issues
+ .filter((i) => typeof i?.check === 'string')
+ .map((i) => ({
+ page,
+ check: i.check as string,
+ label: typeof i.label === 'string' ? i.label : (i.check as string),
+ evidence:
+ i.evidence && typeof i.evidence === 'object'
+ ? (i.evidence as Record<string, unknown>)
+ : {},
+ }));
+ });
+}
+
+/**
+ * The AI crawlers robots.txt blocks, from the bot check's evidence. Audits
+ * from before it named them only say some bot is blocked, among bots it may
+ * allow, so they name none.
+ */
+function blockedCrawlers(issues: PageIssue[]): string[] {
+ const agents = issues
+ .filter((i) => i.check === 'ai-bot-access')
+ .flatMap((i) => (Array.isArray(i.evidence.blockedBots) ? i.evidence.blockedBots : []))
+ .filter((a): a is string => typeof a === 'string');
+ return [...new Set(agents)];
+}
+
export interface WinningSource {
domain: string;
/** Answers it is cited in, of the answers read that the competitor wins. */
@@ -527,7 +578,7 @@ const LAYOUT: Record<SignalCategory, WhereFieldKey[]> = {
'excerpt',
],
traffic: ['scope', 'url', 'pages', 'queries', 'engagementRank', 'aiReferrals', 'citablePrompts'],
- technical: ['url', 'pages', 'issue', 'crawlers', 'evidence'],
+ technical: ['url', 'pages', 'issue', 'evidence'],
competitor: [
'competitor',
'domain',
@@ -540,6 +591,9 @@ const LAYOUT: Record<SignalCategory, WhereFieldKey[]> = {
],
};
+// Only a blocked crawler has crawlers to name; another technical issue has none.
+const CRAWLER_LAYOUT: WhereFieldKey[] = ['pages', 'issue', 'crawlers', 'evidence'];
+
// A first citation is a gain: the page and the prompt, not who replaced it.
const FIRST_CITATION_LAYOUT: WhereFieldKey[] = ['url', 'prompt', 'platforms', 'locations'];
@@ -576,6 +630,7 @@ interface Target {
label?: unknown;
competitor?: unknown;
checks?: unknown;
+ issues?: unknown;
page?: unknown;
}
@@ -732,12 +787,17 @@ function fieldValue(signal: Signal, key: WhereFieldKey): WhereValue | null {
const sources = winningSourcesOf(p.winningSources);
return sources.length ? { type: 'sources', items: sources } : null;
}
+ case 'issue': {
+ const labels = [...new Set(pageIssues(targets).map((i) => i.label))];
+ return labels.length ? { type: 'list', items: labels, total: labels.length } : null;
+ }
+ case 'crawlers': {
+ const agents = blockedCrawlers(pageIssues(targets));
+ return agents.length ? { type: 'crawlers', agents } : null;
+ }
case 'evidence': {
- const checks = targets.flatMap((t) =>
- Array.isArray(t.checks) ? t.checks.filter((c): c is string => typeof c === 'string') : [],
- );
- const unique = [...new Set(checks)];
- return unique.length ? { type: 'list', items: unique, total: unique.length } : null;
+ const issues = pageIssues(targets);
+ return issues.length ? { type: 'issues', items: issues } : null;
}
default:
return null;
@@ -777,9 +837,11 @@ export function signalWhereFields(signal: Signal): WhereField[] {
const layout =
signal.kind === 'first_citation'
? FIRST_CITATION_LAYOUT
- : CITATION_LOSS.includes(signal.kind)
- ? CITATION_LOSS_LAYOUT
- : LAYOUT[signal.category];
+ : signal.kind === 'crawler_blocked'
+ ? CRAWLER_LAYOUT
+ : CITATION_LOSS.includes(signal.kind)
+ ? CITATION_LOSS_LAYOUT
+ : LAYOUT[signal.category];
const named = namedPlaces(signal);
// A mention's position, sentiment and quote are given per prompt when the
// evidence is recorded; asked for on their own only when it is not.
@@ -895,6 +957,15 @@ function apiValue(value: WhereValue): unknown {
excerpt_platform: m.excerptPlatform,
excerpt_at: m.excerptAt,
}));
+ case 'issues':
+ return value.items.map((issue) => ({
+ page: issue.page,
+ check: issue.check,
+ label: issue.label,
+ evidence: issue.evidence,
+ }));
+ case 'crawlers':
+ return value.agents;
case 'sources':
return value.items.map((source) => ({
domain: source.domain,
diff --git a/web/src/lib/signals/names.ts b/web/src/lib/signals/names.ts
index 0e6ff8ba2..fedbabb3f 100644
--- a/web/src/lib/signals/names.ts
+++ b/web/src/lib/signals/names.ts
@@ -83,3 +83,19 @@ export function targetName(entity: string, label: string): string {
if (entity === 'regions') return locationName(label);
return label;
}
+
+// The AI crawlers the site audit checks robots.txt for, by user agent.
+const CRAWLER_NAMES: Record<string, string> = {
+ gptbot: 'GPTBot (OpenAI)',
+ 'oai-searchbot': 'OAI-SearchBot (OpenAI)',
+ 'chatgpt-user': 'ChatGPT-User (OpenAI)',
+ claudebot: 'ClaudeBot (Anthropic)',
+ 'anthropic-ai': 'anthropic-ai (Anthropic)',
+ perplexitybot: 'PerplexityBot (Perplexity)',
+ 'google-extended': 'Google-Extended (Google)',
+};
+
+/** A crawler's name as its operator writes it: `gptbot` → "GPTBot (OpenAI)". */
+export function crawlerName(agent: string): string {
+ return CRAWLER_NAMES[agent.toLowerCase()] ?? agent;
+}