Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.
TypeScript aeoai-agentai-searchanswer-engine-optimizationbrand-monitoringchatgptclaudecopilotdockergeminigeogoogle-ai-overviewsgrokmcpnextjsopen-sourceperplexityseosupabasetypescript
fix(traffic): keep only the brand's own hosts from an AI-bot sender (#1250)
Cihand64b87f4aba8
diff --git a/server/src/lib/bot-ingest.js b/server/src/lib/bot-ingest.jsindex 0decb0263..7297de9c9 100644--- a/server/src/lib/bot-ingest.js+++ b/server/src/lib/bot-ingest.js@@ -3,8 +3,9 @@* §6). Each connection has its own secret: the sender presents it, only its* SHA-256 is stored, and the person sees it once when it is made.*- * A batch is verified request by request (user agent on the list, IP in the- * operator's published ranges, ai-bots.js), then written in one step that+ * A batch is verified request by request (a page on one of the brand's+ * domains, user agent on the list, IP in the operator's published ranges,+ * ai-bots.js), then written in one step that* holds the organization's monthly cap (00183 ai_bot_ingest): rows while* there is room, bot × page × day counters always. A log sender's file is* claimed, counted against the raw cap and written in that same transaction@@ -15,6 +16,7 @@import { createHash, randomBytes } from 'crypto';import supabaseAdmin from '../config/supabase.js';import { verifyBotRequest } from './ai-bots.js';+import { onBrandDomain } from './audit/host.js';import { getBotEventCap, getBotRawCap } from './plan-guard.js';import { logger } from './logger.js';@@ -59,14 +61,17 @@ export function publicIngestEndpoint(env = process.env) {return allowed ? `${url.origin}/bots/ingest` : null;}-/** The live connection a secret belongs to, with its brand's organization; null otherwise. */+/**+ * The live connection a secret belongs to, with its brand's organization and+ * domains (read in the same query); null otherwise.+ */export async function connectionForSecret(secret) {if (typeof secret !== 'string' || !secret.startsWith(SECRET_PREFIX) || secret.length > 200) {return null;}const { data, error } = await supabaseAdmin.from('ai_bot_connections')- .select('id, brand_id, kind, revoked_at, brands!inner(organization_id)')+ .select('id, brand_id, kind, revoked_at, brands!inner(organization_id, brand_domains(domain))').eq('secret_hash', hashSecret(secret)).maybeSingle();if (error) throw new Error(error.message);@@ -76,9 +81,23 @@ export async function connectionForSecret(secret) {brandId: data.brand_id,kind: data.kind,organizationId: data.brands?.organization_id ?? null,+ domains: (data.brands?.brand_domains ?? []).map((d) => d.domain).filter(Boolean),};}+/**+ * Whether a request's page is the brand's site: its host is one of the+ * brand's domains or a subdomain of one. A Logpush zone with several+ * hostnames, or a Vercel drain set for a whole team, also sends other sites'+ * requests, which would fill the report and the event cap. A bare path (a+ * Worker on the brand's own route may send one) passes, and so does every+ * page of a brand with no domain stored, which would otherwise read nothing.+ */+export function onBrandSite(url, domains) {+ if (!domains?.length || !/^https?:\/\//i.test(url)) return true;+ return onBrandDomain(url, domains);+}+/** One request as a sender reports it: user agent, IP, URL, status, time, country. */function readRequest(raw, now) {if (!raw || typeof raw !== 'object') return null;@@ -102,9 +121,9 @@ function readRequest(raw, now) {}/** The bots among a sender's requests, as events to write, and why the rest were skipped. */-async function verifyRequests(list, deps) {+async function verifyRequests(list, domains, deps) {const now = deps.now ?? Date.now();- const skipped = { invalid: 0, unknown_agent: 0, unverified_ip: 0, no_ranges: 0 };+ const skipped = { invalid: 0, other_host: 0, unknown_agent: 0, unverified_ip: 0, no_ranges: 0 };const events = [];for (const raw of list) {const request = readRequest(raw, now);@@ -112,6 +131,11 @@ async function verifyRequests(list, deps) {skipped.invalid++;continue;}+ // Before the ranges are read: another site's request isn't verified.+ if (!onBrandSite(request.url, domains)) {+ skipped.other_host++;+ continue;+ }const verdict = await verifyBotRequest(request, deps);if (!verdict.bot) {skipped[verdict.reason]++;@@ -150,7 +174,7 @@ function alertAt80(connection, data, cap) {*/export async function ingestBotRequests(connection, requests, deps = {}) {const list = Array.isArray(requests) ? requests.slice(0, MAX_BATCH) : [];- const { events, skipped } = await verifyRequests(list, deps);+ const { events, skipped } = await verifyRequests(list, connection.domains, deps);let stored = 0;if (events.length > 0) {@@ -235,7 +259,9 @@ export async function ingestLogFile(connection, hash, parsed, deps = {}) {}const capped = await rawCapReached(connection, rawCap, deps.now ?? Date.now());const list = !capped && Array.isArray(parsed.requests) ? parsed.requests : [];- const { events, skipped } = await verifyRequests(list, deps);+ // Every record counts against the raw cap, other sites' too: the cap is+ // what the server reads, and it reads them to tell them apart.+ const { events, skipped } = await verifyRequests(list, connection.domains, deps);const { data, error } = await supabaseAdmin.rpc('ai_bot_ingest_file', {p_connection_id: connection.id,diff --git a/server/src/lib/bot-ingest.test.js b/server/src/lib/bot-ingest.test.jsindex 7f612c991..757a994b1 100644--- a/server/src/lib/bot-ingest.test.js+++ b/server/src/lib/bot-ingest.test.js@@ -51,6 +51,7 @@ const {logFileHash,logFileRead,newConnectionSecret,+ onBrandSite,pruneBotEvents,publicIngestEndpoint,} = await import('./bot-ingest.js');@@ -62,7 +63,13 @@ const fetchImpl = vi.fn(async (url) => ({prefixes: [{ ipv4Prefix: url.includes('openai') ? '20.171.206.0/24' : '160.79.104.0/23' }],}),}));-const connection = { id: 'c1', brandId: 'b1', kind: 'cloudflare_worker', organizationId: 'o1' };+const connection = {+ id: 'c1',+ brandId: 'b1',+ kind: 'cloudflare_worker',+ organizationId: 'o1',+ domains: ['acme.example'],+};const gpt = (over = {}) => ({userAgent: 'Mozilla/5.0; compatible; GPTBot/1.2; +https://openai.com/gptbot',ip: '20.171.206.9',@@ -108,7 +115,7 @@ describe('ingestBotRequests', () => {received: 6,verified: 2,stored: 2,- skipped: { invalid: 2, unknown_agent: 1, unverified_ip: 1, no_ranges: 0 },+ skipped: { invalid: 2, other_host: 0, unknown_agent: 1, unverified_ip: 1, no_ranges: 0 },});expect(getBotEventCap).toHaveBeenCalledWith('o1');expect(rpc).toHaveBeenCalledWith('ai_bot_ingest', {@@ -175,6 +182,63 @@ describe('ingestBotRequests', () => {});});+// gkhngyk on #1180: a Logpush zone with several hostnames, or a team-wide+// Vercel drain, sent other sites' requests too.+describe("the brand's own site", () => {+ it('skips requests for other sites before verifying them', async () => {+ const result = await ingestBotRequests(+ { ...connection, domains: ['acme.example', 'https://www.acme-shop.example/'] },+ [+ gpt({ url: 'https://acme.example/menu' }),+ gpt({ url: 'https://blog.acme.example/beans' }),+ gpt({ url: 'https://acme-shop.example/cart' }),+ // A bare path comes from a Worker on the brand's own route.+ gpt({ url: '/gifts' }),+ gpt({ url: 'https://other.example/menu' }),+ gpt({ url: 'https://notacme.example/menu' }),+ ],+ { fetchImpl, now: NOW },+ );+ expect(result).toMatchObject({ received: 6, verified: 4 });+ expect(result.skipped).toMatchObject({ other_host: 2 });+ const pages = rpc.mock.calls[0][1].p_events.map((e) => e.page);+ expect(pages).toEqual([+ 'https://acme.example/menu',+ 'https://blog.acme.example/beans',+ 'https://acme-shop.example/cart',+ '/gifts',+ ]);+ });++ it('reads every page of a brand with no domain stored', () => {+ expect(onBrandSite('https://other.example/', [])).toBe(true);+ expect(onBrandSite('https://other.example/', undefined)).toBe(true);+ expect(onBrandSite('https://ACME.example:8443/x', ['acme.example'])).toBe(true);+ expect(onBrandSite('https://acme.example.evil.com/', ['acme.example'])).toBe(false);+ });++ it("still counts other sites' records in a log file against the raw cap", async () => {+ rpc.mockResolvedValue({+ data: { received: 1, stored: 1, used: 1, crossed80: false, raw: 3 },+ error: null,+ });+ const drain = { ...connection, kind: 'vercel_drain' };+ const parsed = {+ records: 3,+ requests: [+ gpt(),+ gpt({ url: 'https://other.example/a' }),+ gpt({ url: 'https://x.example/' }),+ ],+ };+ const result = await ingestLogFile(drain, 'h9', parsed, { fetchImpl, now: NOW });+ expect(result).toMatchObject({ received: 3, verified: 1, skipped: { other_host: 2 } });+ const [, args] = rpc.mock.calls[0];+ expect(args.p_records).toBe(3);+ expect(args.p_events.map((e) => e.page)).toEqual(['https://acme.example/menu?x=1']);+ });+});+describe('connection secrets', () => {it('makes a fresh secret each time and finds a connection by its hash only', async () => {const a = newConnectionSecret();@@ -186,7 +250,7 @@ describe('connection secrets', () => {brand_id: 'b1',kind: 'cloudflare_worker',revoked_at: null,- brands: { organization_id: 'o1' },+ brands: { organization_id: 'o1', brand_domains: [{ domain: 'acme.example' }] },};expect(await connectionForSecret(a)).toEqual(connection);expect(selects).toContainEqual(['secret_hash', hashSecret(a)]);