Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.
TypeScript aeoai-agentai-searchanswer-engine-optimizationbrand-monitoringchatgptclaudecopilotdockergeminigeogoogle-ai-overviewsgrokmcpnextjsopen-sourceperplexityseosupabasetypescript
fix(traffic): keep the AI-bot Worker's secret in a Worker Secret, not its code (#1252)
Cihan126928e46b3e
diff --git a/server/src/lib/bot-worker-script.js b/server/src/lib/bot-worker-script.jsindex 6470c6ece..07aca1601 100644--- a/server/src/lib/bot-worker-script.js+++ b/server/src/lib/bot-worker-script.js@@ -7,6 +7,12 @@** The bot tokens come from ai-bots.json, so the edge filter and the server's* check use the same list; the server still verifies every IP.+ *+ * The connection secret isn't in the code: the person adds it to the Worker+ * as a Secret named WORKER_SECRET_NAME, so it doesn't sit in the source a+ * teammate copies, a version keeps or a screenshot shows (#1179's review).+ * Without it the Worker reports nothing and the site is unaffected. Workers+ * set up before, with the secret in their code, keep working.*/import { AI_BOTS } from './ai-bots.js';@@ -14,13 +20,17 @@ import { AI_BOTS } from './ai-bots.js';/** Requests a Worker isolate holds before it sends them. */export const WORKER_BATCH = 100;-export function buildWorkerScript({ endpoint, secret }) {+/** The Worker Secret (Settings › Variables and Secrets) the script reads the connection secret from. */+export const WORKER_SECRET_NAME = 'ANSVISOR_BOT_SECRET';++export function buildWorkerScript({ endpoint }) {const tokens = JSON.stringify(AI_BOTS.map((b) => b.userAgent.toLowerCase()));return `// Ansvisor AI bot report — Cloudflare Worker (AT-010).// Passes every request through; for AI bots only, sends the request line to-// Ansvisor after the response. Route it on your site's hostname.+// Ansvisor after the response. Route it on your site's hostname, and add the+// connection secret under Settings › Variables and Secrets as a Secret named+// ${WORKER_SECRET_NAME}.const ENDPOINT = ${JSON.stringify(endpoint)};-const SECRET = ${JSON.stringify(secret)};const BOTS = ${tokens};const BATCH = ${WORKER_BATCH};let queue = [];@@ -35,14 +45,14 @@ function isAiBot(ua) {return BOTS.some((t) => s.includes(t));}-async function flush() {+async function flush(secret) {if (queue.length === 0) return;const requests = queue;queue = [];try {await fetch(ENDPOINT, {method: 'POST',- headers: { 'Content-Type': 'application/json', Authorization: 'Bearer ' + SECRET },+ headers: { 'Content-Type': 'application/json', Authorization: 'Bearer ' + secret },body: JSON.stringify({ requests }),});} catch (err) {@@ -55,8 +65,10 @@ export default {// Should this script fail, the request still reaches the site.ctx.passThroughOnException();const response = await fetch(request);+ const secret = env && env.${WORKER_SECRET_NAME};const ua = request.headers.get('user-agent');- if (isAiBot(ua)) {+ // No secret added yet: nothing is reported, and the site is unaffected.+ if (secret && isAiBot(ua)) {queue.push({userAgent: ua,ip: request.headers.get('cf-connecting-ip'),@@ -66,8 +78,8 @@ export default {at: new Date().toISOString(),country: request.cf && request.cf.country,});- if (queue.length >= BATCH) ctx.waitUntil(flush());- else ctx.waitUntil(new Promise((r) => setTimeout(r, 10000)).then(flush));+ if (queue.length >= BATCH) ctx.waitUntil(flush(secret));+ else ctx.waitUntil(new Promise((r) => setTimeout(r, 10000)).then(() => flush(secret)));}return response;},diff --git a/server/src/lib/bot-worker-script.test.js b/server/src/lib/bot-worker-script.test.jsindex 8180a85a0..62bda2b8c 100644--- a/server/src/lib/bot-worker-script.test.js+++ b/server/src/lib/bot-worker-script.test.js@@ -1,6 +1,6 @@import { afterEach, describe, expect, it, vi } from 'vitest';import { AI_BOTS } from './ai-bots.js';-import { WORKER_BATCH, buildWorkerScript } from './bot-worker-script.js';+import { WORKER_BATCH, WORKER_SECRET_NAME, buildWorkerScript } from './bot-worker-script.js';// The Worker runtime's fetch types, which Node also has.const { Request, Response } = globalThis;@@ -11,10 +11,7 @@ const { Request, Response } = globalThis;*/async function loadWorker() {- const script = buildWorkerScript({- endpoint: 'https://api.example/bots/ingest',- secret: 'abot_secret',- });+ const script = buildWorkerScript({ endpoint: 'https://api.example/bots/ingest' });const mod = await import(`data:text/javascript,${encodeURIComponent(script)}`);return { script, worker: mod.default };}@@ -47,9 +44,10 @@ describe('buildWorkerScript', () => {const waits = [];const ctx = { waitUntil: (p) => waits.push(p), passThroughOnException: vi.fn() };+ const env = { [WORKER_SECRET_NAME]: 'abot_secret' };const human = await worker.fetch(new Request('https://acme.example/', { headers: { 'user-agent': 'Mozilla/5.0 Chrome/129' } }),- {},+ env,ctx,);expect(await human.text()).toBe('page');@@ -65,7 +63,7 @@ describe('buildWorkerScript', () => {'cf-connecting-ip': '20.171.206.9',},}),- {},+ env,ctx,);}@@ -82,4 +80,32 @@ describe('buildWorkerScript', () => {status: 200,});});++ // gkhngyk on #1179: the secret sat in the Worker's source.+ it('reads the secret from a Worker Secret, never from its code', async () => {+ const { script } = await loadWorker();+ expect(script).not.toContain('abot_');+ expect(script).toContain(`env.${WORKER_SECRET_NAME}`);+ });++ it('reports nothing until the Secret is added, and still serves the page', async () => {+ const posted = vi.fn();+ vi.stubGlobal(+ 'fetch',+ vi.fn(async (input) => {+ if (typeof input === 'string' && input.endsWith('/bots/ingest')) posted();+ return new Response('page', { status: 200 });+ }),+ );+ const { worker } = await loadWorker();+ const waits = [];+ const ctx = { waitUntil: (p) => waits.push(p), passThroughOnException: vi.fn() };+ const bot = new Request('https://acme.example/menu', {+ headers: { 'user-agent': 'Mozilla/5.0; compatible; GPTBot/1.2' },+ });+ const response = await worker.fetch(bot, {}, ctx);+ expect(await response.text()).toBe('page');+ expect(waits).toHaveLength(0);+ expect(posted).not.toHaveBeenCalled();+ });});diff --git a/server/src/routes/bots.js b/server/src/routes/bots.jsindex f2b61eab1..c0c7f5e8a 100644--- a/server/src/routes/bots.js+++ b/server/src/routes/bots.js@@ -21,7 +21,7 @@ import {newConnectionSecret,publicIngestEndpoint,} from '../lib/bot-ingest.js';-import { buildWorkerScript } from '../lib/bot-worker-script.js';+import { WORKER_SECRET_NAME, buildWorkerScript } from '../lib/bot-worker-script.js';import { logpushJob, vercelDrain } from '../lib/bot-sender-setup.js';const router = Router();@@ -124,7 +124,9 @@ router.post('/connections/:brandId', async (req, res) => {connection,secret,endpoint,- workerScript: kind === 'cloudflare_worker' ? buildWorkerScript({ endpoint, secret }) : null,+ // The Worker reads the secret from a Secret of this name, not from its code.+ workerScript: kind === 'cloudflare_worker' ? buildWorkerScript({ endpoint }) : null,+ workerSecretName: kind === 'cloudflare_worker' ? WORKER_SECRET_NAME : null,logpushJob: kind === 'cloudflare_logpush' ? logpushJob({ endpoint, secret }) : null,vercelDrain: kind === 'vercel_drain' ? vercelDrain({ endpoint, secret }) : null,});diff --git a/server/src/routes/bots.test.js b/server/src/routes/bots.test.jsindex 4a6f97cd3..29ccc12ba 100644--- a/server/src/routes/bots.test.js+++ b/server/src/routes/bots.test.js@@ -125,8 +125,10 @@ describe('AI bot connections', () => {expect(body.secret).toMatch(/^abot_/);// The API's public address, not the Host the web server reached it on.expect(body.endpoint).toBe('https://api.example.com/bots/ingest');- expect(body.workerScript).toContain(body.secret);+ // The Worker reads the secret from a Worker Secret, so its code doesn't hold it.+ expect(body.workerScript).not.toContain(body.secret);expect(body.workerScript).toContain(body.endpoint);+ expect(body.workerSecretName).toBe('ANSVISOR_BOT_SECRET');expect(inserted[0]).toMatchObject({brand_id: 'b1',kind: 'cloudflare_worker',@@ -137,6 +139,7 @@ describe('AI bot connections', () => {const drain = await (await create({ kind: 'vercel_drain' })).json();expect(drain.workerScript).toBeNull();+ expect(drain.workerSecretName).toBeNull();expect(drain.vercelDrain.headers).toEqual([`Authorization: Bearer ${drain.secret}`]);const logpush = await (await create({ kind: 'cloudflare_logpush' })).json();diff --git a/web/messages/en.json b/web/messages/en.jsonindex 83f3ba236..75bba19e6 100644--- a/web/messages/en.json+++ b/web/messages/en.json@@ -3448,6 +3448,7 @@"copy": "Copy","copied": "Copied","copyWorker": "Worker code",+ "copyWorkerSecret": "Secret value (name: {name})","copyDestination": "Destination","copyLogpushJob": "Job settings (Cloudflare API: POST /zones/ZONE_ID/logpush/jobs)","copyEndpoint": "Endpoint URL",@@ -3475,6 +3476,7 @@"cloudflare_worker": ["In Cloudflare, open Workers & Pages and create a Worker.","Replace its code with the Worker code below and deploy it.",+ "In the Worker's Settings › Variables and Secrets, add a variable of type Secret with the name and value below.","Add a route for your site, such as example.com/*, so the Worker sees its requests."],"cloudflare_logpush": [diff --git a/web/src/app/[locale]/(dashboard)/dashboard/traffic/_bots-connect.test.tsx b/web/src/app/[locale]/(dashboard)/dashboard/traffic/_bots-connect.test.tsxindex a77ba352d..b0197737c 100644--- a/web/src/app/[locale]/(dashboard)/dashboard/traffic/_bots-connect.test.tsx+++ b/web/src/app/[locale]/(dashboard)/dashboard/traffic/_bots-connect.test.tsx@@ -95,6 +95,7 @@ describe('BotsSource', () => {secret: 'abot_s3cret',endpoint: 'https://api.example/bots/ingest',workerScript: null,+ workerSecretName: null,logpushJob: null,vercelDrain: {endpoint: 'https://api.example/bots/ingest',@@ -116,6 +117,27 @@ describe('BotsSource', () => {expect(getBotConnections).toHaveBeenCalledTimes(2);});+ // gkhngyk on #1179: the secret sat in the Worker's code.+ it('gives a Worker its code without the secret, and the secret to add as a Worker Secret', async () => {+ await render();+ await act(async () => button('Connect')!.click());+ createBotConnection.mockResolvedValue({+ connection: { id: 'c1', kind: 'cloudflare_worker', createdAt: 't', lastEventAt: null },+ secret: 'abot_s3cret',+ endpoint: 'https://api.example/bots/ingest',+ workerScript:+ 'const ENDPOINT = "https://api.example/bots/ingest"; // env.ANSVISOR_BOT_SECRET',+ workerSecretName: 'ANSVISOR_BOT_SECRET',+ logpushJob: null,+ vercelDrain: null,+ } satisfies NewBotConnection);+ await act(async () => button('Create connection')!.click());+ expect(createBotConnection).toHaveBeenCalledWith('b1', 'cloudflare_worker');+ expect(page()).toContain('add a variable of type Secret with the name and value below');+ expect(page()).toContain('Secret value (name: ANSVISOR_BOT_SECRET)');+ expect(page()).toContain('abot_s3cret');+ });+it('shows a Logpush job to copy: the destination and the settings', async () => {await render();await act(async () => button('Connect')!.click());@@ -124,6 +146,7 @@ describe('BotsSource', () => {secret: 'abot_s3cret',endpoint: 'https://api.example/bots/ingest',workerScript: null,+ workerSecretName: null,logpushJob: {dataset: 'http_requests',destination_conf:diff --git a/web/src/app/[locale]/(dashboard)/dashboard/traffic/_bots-connect.tsx b/web/src/app/[locale]/(dashboard)/dashboard/traffic/_bots-connect.tsxindex 273f93af5..3ddaa7df2 100644--- a/web/src/app/[locale]/(dashboard)/dashboard/traffic/_bots-connect.tsx+++ b/web/src/app/[locale]/(dashboard)/dashboard/traffic/_bots-connect.tsx@@ -89,6 +89,13 @@ function SetupSteps({ created }: { created: NewBotConnection }) {{kind === 'cloudflare_worker' && created.workerScript && (<CopyBlock label={t('copyWorker')} value={created.workerScript} code />)}+ {kind === 'cloudflare_worker' && created.workerSecretName && (+ // Added as a Worker Secret, so it isn't in the code anyone with the Worker can read.+ <CopyBlock+ label={t('copyWorkerSecret', { name: created.workerSecretName })}+ value={created.secret}+ />+ )}{kind === 'cloudflare_logpush' && created.logpushJob && (<><CopyBlockdiff --git a/web/src/lib/actions/ai-bots.test.ts b/web/src/lib/actions/ai-bots.test.tsindex a56167fb3..9955cfc11 100644--- a/web/src/lib/actions/ai-bots.test.ts+++ b/web/src/lib/actions/ai-bots.test.ts@@ -53,6 +53,7 @@ describe('AI bot connection actions', () => {secret: 'abot_x',endpoint: 'https://api.test/bots/ingest',workerScript: '// worker',+ workerSecretName: 'ANSVISOR_BOT_SECRET',logpushJob: null,vercelDrain: null,},@@ -63,6 +64,7 @@ describe('AI bot connection actions', () => {expect(made).toMatchObject({secret: 'abot_x',workerScript: '// worker',+ workerSecretName: 'ANSVISOR_BOT_SECRET',connection: { id: 'c2' },});expect(fetchMock.mock.calls[0][1]).toMatchObject({diff --git a/web/src/lib/actions/ai-bots.ts b/web/src/lib/actions/ai-bots.tsindex 1177c3859..dab8f4807 100644--- a/web/src/lib/actions/ai-bots.ts+++ b/web/src/lib/actions/ai-bots.ts@@ -47,6 +47,8 @@ export interface NewBotConnection {secret: string;endpoint: string;workerScript: string | null;+ /** The Worker Secret the script reads the connection secret from; null for other senders. */+ workerSecretName: string | null;logpushJob: Record<string, unknown> | null;vercelDrain: { endpoint: string; format: string; headers: string[]; sources: string[] } | null;}@@ -122,6 +124,7 @@ export async function createBotConnection(secret: String(b.secret),endpoint: String(b.endpoint),workerScript: typeof b.workerScript === 'string' ? b.workerScript : null,+ workerSecretName: typeof b.workerSecretName === 'string' ? b.workerSecretName : null,logpushJob: (b.logpushJob as Record<string, unknown> | null) ?? null,vercelDrain: (b.vercelDrain as NewBotConnection['vercelDrain']) ?? null,};