Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.
TypeScript aeoai-agentai-searchanswer-engine-optimizationbrand-monitoringchatgptclaudecopilotdockergeminigeogoogle-ai-overviewsgrokmcpnextjsopen-sourceperplexityseosupabasetypescript
feat(traffic): receive verified AI-bot requests from a brand's edge, within a monthly cap (#1179)
Cihanf48a7098ae37
diff --git a/server/.env.example b/server/.env.exampleindex 66248eced..79174fe81 100644--- a/server/.env.example+++ b/server/.env.example@@ -129,6 +129,12 @@ AI_VOLUME_MULTIPLIER=0.45# action_task.sent webhook (Action Center → Send to webhook)# PUBLIC_APP_URL=https://app.example.com+# Public https URL of this API server, as the internet reaches it. Written+# into AI-bot connections (Cloudflare Worker, Logpush, Vercel drain) as their+# /bots/ingest endpoint, and used by content webhooks. Without it, AI-bot+# connections can't be made (http is accepted for localhost in development).+# PUBLIC_SERVER_URL=https://api.example.com+# Composio managed auth (Settings → Integrations, #577). Optional — without# these the Integrations cards show "not configured" and everything else works.diff --git a/server/src/config/plans.js b/server/src/config/plans.jsindex e4a8a8099..90ec6d4fc 100644--- a/server/src/config/plans.js+++ b/server/src/config/plans.js@@ -20,6 +20,7 @@ export const PLANS = {maxOpportunitiesPerMonth: -1,maxOpportunitiesPerRequest: 5,maxSiteAudits: -1,+ maxBotEventsPerMonth: -1,maxDailyOnDemand: -1,onDemandCooldownMinutes: 0,features: [@@ -56,6 +57,10 @@ export const PLANS = {maxOpportunitiesPerMonth: 10,maxOpportunitiesPerRequest: 3,maxSiteAudits: 100,+ // AI bot report (AT-010, spec §6.3): verified bot rows stored a month.+ // Starter sees the report locked; Growth 2M, Enterprise 20M (adjustable+ // through plan_overrides); -1 = no cap.+ maxBotEventsPerMonth: 0,maxDailyOnDemand: 3,onDemandCooldownMinutes: 15,allowedScrapers: ['chatgpt-web', 'copilot-web'],@@ -93,6 +98,7 @@ export const PLANS = {maxOpportunitiesPerMonth: 50,maxOpportunitiesPerRequest: 5,maxSiteAudits: 500,+ maxBotEventsPerMonth: 2_000_000,maxDailyOnDemand: 10,onDemandCooldownMinutes: 5,// API-model tracking (Claude) is not part of Growth — scraper engines@@ -136,6 +142,7 @@ export const PLANS = {maxOpportunitiesPerMonth: -1,maxOpportunitiesPerRequest: 5,maxSiteAudits: -1,+ maxBotEventsPerMonth: 20_000_000,maxDailyOnDemand: -1,onDemandCooldownMinutes: 0,// API-model tracking (Claude) is a per-customer opt-in on Enterprise:diff --git a/server/src/lib/ai-bots.js b/server/src/lib/ai-bots.jsnew file mode 100644index 000000000..434a11f76--- /dev/null+++ b/server/src/lib/ai-bots.js@@ -0,0 +1,140 @@+/**+ * The AI bots Ansvisor counts reading a brand's site (AT-010, spec §3.7): one+ * versioned list, in ai-bots.json, of each bot's user-agent token, its type+ * (training crawler, search crawler, or a fetch a person asked for) and the+ * URL where its operator publishes the IP ranges it crawls from.+ *+ * A request counts only when both hold: its user agent names a bot, and its+ * IP is in that bot's published ranges (§3.7: "the user agent must match and+ * the IP must be in the operator's published ranges"). User agents are easy+ * to fake; the ranges are not. No model is involved anywhere.+ *+ * The ranges are fetched on first use and again once a day, and kept in+ * memory. A failed refresh keeps the last list; with none ever read, the+ * bot's requests can't be verified and aren't counted.+ *+ * The list is versioned like ai-sources.json: each change adds an entry to+ * `changes` and bumps `version`.+ */++import { createRequire } from 'module';+import { logger } from './logger.js';++const require = createRequire(import.meta.url);+const config = require('./ai-bots.json');++export const AI_BOTS_VERSION = config.version;+export const AI_BOT_CHANGES = config.changes;+export const AI_BOTS = config.bots;++/** How long a published range list is used before it is read again. */+export const RANGES_TTL_MS = 24 * 60 * 60 * 1000;++const escape = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');++// A token stands on its own in the user agent: "GPTBot/1.2", "(compatible;+// ClaudeBot/1.0; ...)", never as part of a longer name.+const MATCHERS = AI_BOTS.map((bot) => ({+ bot,+ re: new RegExp(`(^|[^a-z0-9-])${escape(bot.userAgent)}([^a-z0-9-]|$)`, 'i'),+}));++/** The bot a user agent names, or null. */+export function matchBot(userAgent) {+ if (typeof userAgent !== 'string' || !userAgent) return null;+ return MATCHERS.find((m) => m.re.test(userAgent))?.bot ?? null;+}++/** An IPv4 or IPv6 address as a number and its family, or null. */+export function parseIp(raw) {+ if (typeof raw !== 'string') return null;+ const ip = raw.trim();+ if (/^\d{1,3}(\.\d{1,3}){3}$/.test(ip)) {+ const parts = ip.split('.').map(Number);+ if (parts.some((p) => p > 255)) return null;+ return { family: 4, value: parts.reduce((n, p) => (n << 8n) + BigInt(p), 0n) };+ }+ if (!ip.includes(':') || !/^[0-9a-f:.]+$/i.test(ip)) return null;+ // An IPv4-mapped address (::ffff:1.2.3.4) reads as the IPv4 one.+ const mapped = ip.match(/^::ffff:(\d{1,3}(\.\d{1,3}){3})$/i);+ if (mapped) return parseIp(mapped[1]);+ const halves = ip.split('::');+ if (halves.length > 2) return null;+ const head = halves[0] ? halves[0].split(':') : [];+ const tail = halves.length === 2 && halves[1] ? halves[1].split(':') : [];+ const missing = 8 - head.length - tail.length;+ if (missing < 0 || (halves.length === 1 && missing !== 0)) return null;+ const groups = [...head, ...Array(missing).fill('0'), ...tail];+ if (groups.some((g) => !/^[0-9a-f]{1,4}$/i.test(g))) return null;+ return { family: 6, value: groups.reduce((n, g) => (n << 16n) + BigInt(parseInt(g, 16)), 0n) };+}++/** "20.171.206.0/24" as a family, a first address and a prefix length; null when malformed. */+export function parsePrefix(cidr) {+ if (typeof cidr !== 'string') return null;+ const [addr, len] = cidr.split('/');+ const ip = parseIp(addr);+ const bits = Number(len);+ const width = ip?.family === 4 ? 32 : 128;+ if (!ip || !Number.isInteger(bits) || bits < 0 || bits > width) return null;+ const shift = BigInt(width - bits);+ return { family: ip.family, base: (ip.value >> shift) << shift, shift };+}++/** True when the address is in one of the prefixes. */+export function inPrefixes(ip, prefixes) {+ const addr = parseIp(ip);+ if (!addr) return false;+ return prefixes.some(+ (p) => p.family === addr.family && (addr.value >> p.shift) << p.shift === p.base,+ );+}++/** The prefixes in a published range file: {prefixes: [{ipv4Prefix | ipv6Prefix}]}. */+export function readRangeFile(json) {+ const list = Array.isArray(json?.prefixes) ? json.prefixes : [];+ return list.map((p) => parsePrefix(p?.ipv4Prefix ?? p?.ipv6Prefix)).filter(Boolean);+}++const cache = new Map();++/**+ * A range file's prefixes: from memory while fresh, read again after a day.+ * A failed read keeps what was read before; null when nothing ever was.+ */+export async function rangesFor(url, { fetchImpl = fetch, now = Date.now() } = {}) {+ const hit = cache.get(url);+ if (hit && now - hit.at < RANGES_TTL_MS) return hit.prefixes;+ try {+ const res = await fetchImpl(url, { signal: AbortSignal.timeout(10_000) });+ if (!res.ok) throw new Error(`HTTP ${res.status}`);+ const prefixes = readRangeFile(await res.json());+ if (prefixes.length === 0) throw new Error('no prefixes');+ cache.set(url, { at: now, prefixes });+ return prefixes;+ } catch (err) {+ logger.warn({ err, url }, '[ai-bots] range list not refreshed');+ // Keep the last list for another hour before trying again.+ if (hit) cache.set(url, { at: now - RANGES_TTL_MS + 60 * 60 * 1000, prefixes: hit.prefixes });+ return hit?.prefixes ?? null;+ }+}++/** Forgets every range list (tests). */+export function clearRangeCache() {+ cache.clear();+}++/**+ * Whether a request is a verified AI bot: its bot, or why it isn't one —+ * `unknown_agent` (not on the list), `unverified_ip` (not in the operator's+ * ranges) or `no_ranges` (the ranges couldn't be read yet).+ */+export async function verifyBotRequest({ userAgent, ip }, deps) {+ const bot = matchBot(userAgent);+ if (!bot) return { reason: 'unknown_agent' };+ const prefixes = await rangesFor(bot.ranges, deps);+ if (!prefixes) return { reason: 'no_ranges' };+ if (!inPrefixes(ip, prefixes)) return { reason: 'unverified_ip' };+ return { bot };+}diff --git a/server/src/lib/ai-bots.json b/server/src/lib/ai-bots.jsonnew file mode 100644index 000000000..ebc4166e5--- /dev/null+++ b/server/src/lib/ai-bots.json@@ -0,0 +1,76 @@+{+ "version": 1,+ "changes": [+ {+ "version": 1,+ "date": "2026-10-19",+ "summary": "OpenAI, Anthropic and Perplexity: their training crawlers, search crawlers and user fetchers, each verified against the operator's published IP ranges."+ }+ ],+ "bots": [+ {+ "key": "gptbot",+ "name": "GPTBot",+ "operator": "OpenAI",+ "type": "training",+ "userAgent": "GPTBot",+ "ranges": "https://openai.com/gptbot.json"+ },+ {+ "key": "oai-searchbot",+ "name": "OAI-SearchBot",+ "operator": "OpenAI",+ "type": "search",+ "userAgent": "OAI-SearchBot",+ "ranges": "https://openai.com/searchbot.json"+ },+ {+ "key": "chatgpt-user",+ "name": "ChatGPT-User",+ "operator": "OpenAI",+ "type": "user",+ "userAgent": "ChatGPT-User",+ "ranges": "https://openai.com/chatgpt-user.json"+ },+ {+ "key": "claudebot",+ "name": "ClaudeBot",+ "operator": "Anthropic",+ "type": "training",+ "userAgent": "ClaudeBot",+ "ranges": "https://claude.com/crawling/bots.json"+ },+ {+ "key": "claude-searchbot",+ "name": "Claude-SearchBot",+ "operator": "Anthropic",+ "type": "search",+ "userAgent": "Claude-SearchBot",+ "ranges": "https://claude.com/crawling/bots.json"+ },+ {+ "key": "claude-user",+ "name": "Claude-User",+ "operator": "Anthropic",+ "type": "user",+ "userAgent": "Claude-User",+ "ranges": "https://claude.com/crawling/bots.json"+ },+ {+ "key": "perplexitybot",+ "name": "PerplexityBot",+ "operator": "Perplexity",+ "type": "search",+ "userAgent": "PerplexityBot",+ "ranges": "https://www.perplexity.com/perplexitybot.json"+ },+ {+ "key": "perplexity-user",+ "name": "Perplexity-User",+ "operator": "Perplexity",+ "type": "user",+ "userAgent": "Perplexity-User",+ "ranges": "https://www.perplexity.com/perplexity-user.json"+ }+ ]+}diff --git a/server/src/lib/ai-bots.test.js b/server/src/lib/ai-bots.test.jsnew file mode 100644index 000000000..25d67ca86--- /dev/null+++ b/server/src/lib/ai-bots.test.js@@ -0,0 +1,107 @@+import { beforeEach, describe, expect, it, vi } from 'vitest';+import {+ AI_BOTS,+ RANGES_TTL_MS,+ clearRangeCache,+ inPrefixes,+ matchBot,+ parseIp,+ parsePrefix,+ rangesFor,+ readRangeFile,+ verifyBotRequest,+} from './ai-bots.js';++/** AI bots (AT-010, spec §3.7): a user agent on the list and an IP in its operator's ranges. */++const json = (body, ok = true) => ({ ok, status: ok ? 200 : 500, json: async () => body });++beforeEach(() => {+ clearRangeCache();+});++describe('matchBot', () => {+ it('reads each bot from its own token, never from a longer name', () => {+ const ua = (token) =>+ `Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ${token}/1.2; +https://example.com/bot`;+ for (const bot of AI_BOTS) expect(matchBot(ua(bot.userAgent))?.key).toBe(bot.key);+ expect(matchBot('Mozilla/5.0 (compatible; ClaudeBot/1.0; [email protected])')?.key).toBe(+ 'claudebot',+ );+ expect(matchBot('gptbot/1.0')?.key).toBe('gptbot');+ expect(matchBot('Mozilla/5.0 (compatible; NotGPTBotX/1.0)')).toBeNull();+ expect(matchBot('Mozilla/5.0 Chrome/129')).toBeNull();+ expect(matchBot(undefined)).toBeNull();+ });++ it('has a type and a published range list for every bot', () => {+ for (const bot of AI_BOTS) {+ expect(['training', 'search', 'user']).toContain(bot.type);+ expect(bot.ranges).toMatch(/^https:\/\//);+ }+ expect(new Set(AI_BOTS.map((b) => b.key)).size).toBe(AI_BOTS.length);+ });+});++describe('addresses and prefixes', () => {+ it('reads IPv4, IPv6 and IPv4-mapped addresses, and refuses the rest', () => {+ expect(parseIp('20.171.206.7')).toEqual({ family: 4, value: 346803719n });+ expect(parseIp('2001:db8::1')?.family).toBe(6);+ expect(parseIp('::ffff:20.171.206.7')).toEqual(parseIp('20.171.206.7'));+ for (const bad of ['256.1.1.1', '1.2.3', 'abc', '1::2::3', '', null]) {+ expect(parseIp(bad)).toBeNull();+ }+ });++ it('tells an address inside a prefix from one outside', () => {+ const prefixes = readRangeFile({+ prefixes: [{ ipv4Prefix: '20.171.206.0/24' }, { ipv6Prefix: '2001:db8:10::/48' }, {}],+ });+ expect(prefixes).toHaveLength(2);+ expect(inPrefixes('20.171.206.255', prefixes)).toBe(true);+ expect(inPrefixes('20.171.207.1', prefixes)).toBe(false);+ expect(inPrefixes('2001:db8:10:ffff::1', prefixes)).toBe(true);+ expect(inPrefixes('2001:db8:11::1', prefixes)).toBe(false);+ expect(inPrefixes('not-an-ip', prefixes)).toBe(false);+ expect(parsePrefix('20.171.206.0/33')).toBeNull();+ });+});++describe('rangesFor', () => {+ it('reads a list once a day and keeps the last one when a refresh fails', async () => {+ const fetchImpl = vi.fn().mockResolvedValue(json({ prefixes: [{ ipv4Prefix: '1.2.3.0/24' }] }));+ const first = await rangesFor('https://x/a.json', { fetchImpl, now: 0 });+ await rangesFor('https://x/a.json', { fetchImpl, now: RANGES_TTL_MS - 1 });+ expect(fetchImpl).toHaveBeenCalledTimes(1);++ fetchImpl.mockResolvedValue(json({}, false));+ const stale = await rangesFor('https://x/a.json', { fetchImpl, now: RANGES_TTL_MS + 1 });+ expect(stale).toEqual(first);+ expect(fetchImpl).toHaveBeenCalledTimes(2);+ });++ it('is null when a list was never read', async () => {+ const fetchImpl = vi.fn().mockRejectedValue(new Error('offline'));+ expect(await rangesFor('https://x/b.json', { fetchImpl, now: 0 })).toBeNull();+ });+});++describe('verifyBotRequest', () => {+ const fetchImpl = vi.fn(async () => json({ prefixes: [{ ipv4Prefix: '20.171.206.0/24' }] }));++ it('counts a bot only from its operator ranges', async () => {+ const ua = 'Mozilla/5.0; compatible; GPTBot/1.2; +https://openai.com/gptbot';+ expect(+ (await verifyBotRequest({ userAgent: ua, ip: '20.171.206.9' }, { fetchImpl })).bot.key,+ ).toBe('gptbot');+ expect(await verifyBotRequest({ userAgent: ua, ip: '8.8.8.8' }, { fetchImpl })).toEqual({+ reason: 'unverified_ip',+ });+ expect(+ await verifyBotRequest(+ { userAgent: 'Mozilla/5.0 Chrome/129', ip: '20.171.206.9' },+ { fetchImpl },+ ),+ ).toEqual({ reason: 'unknown_agent' });+ });+});diff --git a/server/src/lib/bot-ingest.js b/server/src/lib/bot-ingest.jsnew file mode 100644index 000000000..484b0d838--- /dev/null+++ b/server/src/lib/bot-ingest.js@@ -0,0 +1,179 @@+/**+ * AI-bot requests sent from a brand's own Cloudflare or Vercel (AT-010, spec+ * §6). Each connection has its own secret: the sender presents it, only its+ * SHA-256 is stored, and the person sees it once when it is made.+ *+ * A batch is verified request by request (user agent on the list, IP in the+ * operator's published ranges, ai-bots.js), then written in one step that+ * holds the organization's monthly cap (00183 ai_bot_ingest): rows while+ * there is room, bot × page × day counters always. No IP is kept. Crossing+ * 80% of the cap raises the internal alert once a month (§6.3 guardrails).+ */++import { createHash, randomBytes } from 'crypto';+import supabaseAdmin from '../config/supabase.js';+import { verifyBotRequest } from './ai-bots.js';+import { getBotEventCap } from './plan-guard.js';+import { logger } from './logger.js';++/** The most requests one batch may carry (a Worker sends about 100). */+export const MAX_BATCH = 500;++/** Requests older than this are dropped: a sender catching up after a week isn't the site's reading now. */+export const MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000;++export const CONNECTION_KINDS = ['cloudflare_worker', 'cloudflare_logpush', 'vercel_drain'];++const SECRET_PREFIX = 'abot_';++/** A new connection secret: 32 random bytes, shown once. */+export function newConnectionSecret() {+ return `${SECRET_PREFIX}${randomBytes(32).toString('base64url')}`;+}++export function hashSecret(secret) {+ return createHash('sha256').update(secret).digest('hex');+}++/**+ * Where a connection sends its events: the API's public address+ * (PUBLIC_SERVER_URL), never the Host a request came in on. The create call+ * comes from the web server, which may reach the API at an internal address,+ * and the endpoint is written into the customer's Worker, Logpush job or+ * drain, where it is hard to change later. https only, but for localhost+ * outside production. Null when it isn't set or can't be used.+ */+export function publicIngestEndpoint(env = process.env) {+ let url;+ try {+ url = new URL(env.PUBLIC_SERVER_URL ?? '');+ } catch {+ return null;+ }+ const local = ['localhost', '127.0.0.1', '[::1]'].includes(url.hostname);+ const allowed =+ url.protocol === 'https:' ||+ (url.protocol === 'http:' && local && env.NODE_ENV !== 'production');+ return allowed ? `${url.origin}/bots/ingest` : null;+}++/** The live connection a secret belongs to, with its brand's organization; null otherwise. */+export async function connectionForSecret(secret) {+ if (typeof secret !== 'string' || !secret.startsWith(SECRET_PREFIX) || secret.length > 200) {+ return null;+ }+ const { data, error } = await supabaseAdmin+ .from('ai_bot_connections')+ .select('id, brand_id, kind, revoked_at, brands!inner(organization_id)')+ .eq('secret_hash', hashSecret(secret))+ .maybeSingle();+ if (error) throw new Error(error.message);+ if (!data || data.revoked_at) return null;+ return {+ id: data.id,+ brandId: data.brand_id,+ kind: data.kind,+ organizationId: data.brands?.organization_id ?? null,+ };+}++/** One request as a sender reports it: user agent, IP, URL, status, time, country. */+function readRequest(raw, now) {+ if (!raw || typeof raw !== 'object') return null;+ const userAgent = raw.userAgent ?? raw.ua;+ const url = raw.url ?? raw.path;+ if (typeof userAgent !== 'string' || typeof raw.ip !== 'string' || typeof url !== 'string') {+ return null;+ }+ const at = new Date(raw.at ?? raw.ts ?? now);+ const ms = at.getTime();+ if (Number.isNaN(ms) || ms > now + 60 * 60 * 1000 || ms < now - MAX_AGE_MS) return null;+ const status = Number(raw.status);+ return {+ userAgent: userAgent.slice(0, 512),+ ip: raw.ip,+ url: url.slice(0, 2048),+ status: Number.isInteger(status) && status >= 100 && status < 600 ? status : null,+ country: typeof raw.country === 'string' ? raw.country.slice(0, 2).toUpperCase() : null,+ at: at.toISOString(),+ };+}++/**+ * Verify and store one batch for a connection. Returns what was received,+ * verified and stored, and why the rest wasn't counted.+ */+export async function ingestBotRequests(connection, requests, deps = {}) {+ const now = deps.now ?? Date.now();+ const list = Array.isArray(requests) ? requests.slice(0, MAX_BATCH) : [];+ const skipped = { invalid: 0, unknown_agent: 0, unverified_ip: 0, no_ranges: 0 };+ const events = [];+ for (const raw of list) {+ const request = readRequest(raw, now);+ if (!request) {+ skipped.invalid++;+ continue;+ }+ const verdict = await verifyBotRequest(request, deps);+ if (!verdict.bot) {+ skipped[verdict.reason]++;+ continue;+ }+ events.push({+ bot: verdict.bot.key,+ bot_type: verdict.bot.type,+ page: request.url,+ status: request.status,+ country: request.country,+ at: request.at,+ });+ }++ let stored = 0;+ if (events.length > 0) {+ const cap = await getBotEventCap(connection.organizationId);+ // A plan without the report (Starter) stores nothing, not even counters.+ if (cap === 0)+ return { received: list.length, verified: events.length, stored, skipped, locked: true };+ const { data, error } = await supabaseAdmin.rpc('ai_bot_ingest', {+ p_connection_id: connection.id,+ p_cap: cap,+ p_events: events,+ });+ if (error) throw new Error(error.message);+ if (data?.error) throw new Error(`ai_bot_ingest: ${data.error}`);+ stored = data?.stored ?? 0;+ if (data?.crossed80) {+ logger.warn(+ {+ organizationId: connection.organizationId,+ brandId: connection.brandId,+ used: data.used,+ cap,+ },+ '[ai-bots] 80% of the monthly bot event cap reached',+ );+ }+ }++ return { received: list.length, verified: events.length, stored, skipped };+}++/** Verified rows are kept 90 days (§6.2); the bot × page × day counters 400. */+export const EVENT_RETENTION_DAYS = 90;+export const DAILY_RETENTION_DAYS = 400;++/** The daily prune of rows past their retention (the daily run). */+export async function pruneBotEvents(now = Date.now()) {+ const before = (days) => new Date(now - days * 24 * 60 * 60 * 1000).toISOString();+ const { error } = await supabaseAdmin+ .from('ai_bot_events')+ .delete()+ .lt('occurred_at', before(EVENT_RETENTION_DAYS));+ if (error) logger.warn({ err: error }, '[ai-bots] event prune failed');+ const { error: dailyError } = await supabaseAdmin+ .from('ai_bot_daily')+ .delete()+ .lt('day', before(DAILY_RETENTION_DAYS).slice(0, 10));+ if (dailyError) logger.warn({ err: dailyError }, '[ai-bots] daily prune failed');+}diff --git a/server/src/lib/bot-ingest.test.js b/server/src/lib/bot-ingest.test.jsnew file mode 100644index 000000000..27215c603--- /dev/null+++ b/server/src/lib/bot-ingest.test.js@@ -0,0 +1,206 @@+import { beforeEach, describe, expect, it, vi } from 'vitest';++/**+ * AI-bot requests from a brand's edge (AT-010, spec §6): each verified (user+ * agent + published ranges), then written within the month's cap; the+ * connection secret is stored only as its hash.+ */++const rpc = vi.fn();+let connectionRow = null;+const selects = [];+vi.mock('../config/supabase.js', () => ({+ default: {+ rpc: (...args) => rpc(...args),+ from: () => {+ const chain = {+ select: () => chain,+ eq: (col, val) => {+ selects.push([col, val]);+ return chain;+ },+ maybeSingle: async () => ({ data: connectionRow, error: null }),+ };+ return chain;+ },+ },+}));+const getBotEventCap = vi.fn();+vi.mock('./plan-guard.js', () => ({ getBotEventCap: (...a) => getBotEventCap(...a) }));+const warn = vi.fn();+vi.mock('./logger.js', () => ({ logger: { warn: (...a) => warn(...a) } }));++const { clearRangeCache } = await import('./ai-bots.js');+const {+ MAX_BATCH,+ connectionForSecret,+ hashSecret,+ ingestBotRequests,+ newConnectionSecret,+ publicIngestEndpoint,+} = await import('./bot-ingest.js');++const NOW = Date.parse('2026-10-19T10:00:00.000Z');+const fetchImpl = vi.fn(async (url) => ({+ ok: true,+ json: async () => ({+ prefixes: [{ ipv4Prefix: url.includes('openai') ? '20.171.206.0/24' : '160.79.104.0/23' }],+ }),+}));+const connection = { id: 'c1', brandId: 'b1', kind: 'cloudflare_worker', organizationId: 'o1' };+const gpt = (over = {}) => ({+ userAgent: 'Mozilla/5.0; compatible; GPTBot/1.2; +https://openai.com/gptbot',+ ip: '20.171.206.9',+ url: 'https://acme.example/menu?x=1',+ status: 200,+ at: '2026-10-19T09:58:00.000Z',+ country: 'us',+ ...over,+});++beforeEach(() => {+ clearRangeCache();+ rpc+ .mockReset()+ .mockResolvedValue({ data: { stored: 2, used: 2, crossed80: false }, error: null });+ getBotEventCap.mockReset().mockResolvedValue(2_000_000);+ warn.mockReset();+ selects.length = 0;+});++describe('ingestBotRequests', () => {+ it('verifies each request and writes the bots within the cap', async () => {+ const result = await ingestBotRequests(+ connection,+ [+ gpt(),+ gpt({+ userAgent: 'Mozilla/5.0 (compatible; ClaudeBot/1.0)',+ ip: '160.79.104.10',+ url: '/gifts',+ }),+ gpt({ ip: '8.8.8.8' }),+ gpt({ userAgent: 'Mozilla/5.0 Chrome/129' }),+ gpt({ at: '2026-10-01T00:00:00Z' }),+ { userAgent: 'GPTBot' },+ ],+ { fetchImpl, now: NOW },+ );+ expect(result).toEqual({+ received: 6,+ verified: 2,+ stored: 2,+ skipped: { invalid: 2, unknown_agent: 1, unverified_ip: 1, no_ranges: 0 },+ });+ expect(getBotEventCap).toHaveBeenCalledWith('o1');+ expect(rpc).toHaveBeenCalledWith('ai_bot_ingest', {+ p_connection_id: 'c1',+ p_cap: 2_000_000,+ p_events: [+ {+ bot: 'gptbot',+ bot_type: 'training',+ page: 'https://acme.example/menu?x=1',+ status: 200,+ country: 'US',+ at: '2026-10-19T09:58:00.000Z',+ },+ {+ bot: 'claudebot',+ bot_type: 'training',+ page: '/gifts',+ status: 200,+ country: 'US',+ at: '2026-10-19T09:58:00.000Z',+ },+ ],+ });+ });++ it('writes nothing when no request verifies, and keeps a batch to its limit', async () => {+ const none = await ingestBotRequests(connection, [gpt({ ip: '1.1.1.1' })], {+ fetchImpl,+ now: NOW,+ });+ expect(none.stored).toBe(0);+ expect(rpc).not.toHaveBeenCalled();++ const many = await ingestBotRequests(+ connection,+ Array.from({ length: MAX_BATCH + 50 }, () => gpt({ ip: '9.9.9.9' })),+ { fetchImpl, now: NOW },+ );+ expect(many.received).toBe(MAX_BATCH);+ });++ it('writes nothing on a plan without the report', async () => {+ getBotEventCap.mockResolvedValue(0);+ const result = await ingestBotRequests(connection, [gpt()], { fetchImpl, now: NOW });+ expect(result).toMatchObject({ verified: 1, stored: 0, locked: true });+ expect(rpc).not.toHaveBeenCalled();+ });++ it('raises the internal alert when a batch crosses 80% of the cap', async () => {+ rpc.mockResolvedValue({ data: { stored: 1, used: 1_600_000, crossed80: true }, error: null });+ await ingestBotRequests(connection, [gpt()], { fetchImpl, now: NOW });+ expect(warn).toHaveBeenCalledWith(+ expect.objectContaining({ organizationId: 'o1', used: 1_600_000, cap: 2_000_000 }),+ '[ai-bots] 80% of the monthly bot event cap reached',+ );+ });++ it('throws when the write fails', async () => {+ rpc.mockResolvedValue({ data: null, error: { message: 'boom' } });+ await expect(ingestBotRequests(connection, [gpt()], { fetchImpl, now: NOW })).rejects.toThrow(+ 'boom',+ );+ });+});++describe('connection secrets', () => {+ it('makes a fresh secret each time and finds a connection by its hash only', async () => {+ const a = newConnectionSecret();+ expect(a).toMatch(/^abot_[A-Za-z0-9_-]{43}$/);+ expect(newConnectionSecret()).not.toBe(a);++ connectionRow = {+ id: 'c1',+ brand_id: 'b1',+ kind: 'cloudflare_worker',+ revoked_at: null,+ brands: { organization_id: 'o1' },+ };+ expect(await connectionForSecret(a)).toEqual(connection);+ expect(selects).toContainEqual(['secret_hash', hashSecret(a)]);++ connectionRow = { ...connectionRow, revoked_at: '2026-10-18T00:00:00Z' };+ expect(await connectionForSecret(a)).toBeNull();+ expect(await connectionForSecret('not-a-secret')).toBeNull();+ });+});++describe('publicIngestEndpoint', () => {+ it("builds the endpoint from the API's public address, whatever its path", () => {+ expect(publicIngestEndpoint({ PUBLIC_SERVER_URL: 'https://api.ansvisor.com/' })).toBe(+ 'https://api.ansvisor.com/bots/ingest',+ );+ expect(publicIngestEndpoint({ PUBLIC_SERVER_URL: 'https://api.example.com/v1' })).toBe(+ 'https://api.example.com/bots/ingest',+ );+ });++ it('refuses no address, a plain-http one, and localhost in production', () => {+ expect(publicIngestEndpoint({})).toBeNull();+ expect(publicIngestEndpoint({ PUBLIC_SERVER_URL: 'not a url' })).toBeNull();+ expect(publicIngestEndpoint({ PUBLIC_SERVER_URL: 'http://api.example.com' })).toBeNull();+ expect(+ publicIngestEndpoint({ PUBLIC_SERVER_URL: 'http://localhost:80', NODE_ENV: 'production' }),+ ).toBeNull();+ });++ it('allows http on localhost while developing', () => {+ expect(+ publicIngestEndpoint({ PUBLIC_SERVER_URL: 'http://localhost:8082', NODE_ENV: 'development' }),+ ).toBe('http://localhost:8082/bots/ingest');+ });+});diff --git a/server/src/lib/bot-worker-script.js b/server/src/lib/bot-worker-script.jsnew file mode 100644index 000000000..6470c6ece--- /dev/null+++ b/server/src/lib/bot-worker-script.js@@ -0,0 +1,76 @@+/**+ * The Cloudflare Worker a brand installs to send AI-bot requests (AT-010,+ * spec §6.1 option A, "lowest cost for us"). It passes every request through+ * untouched, and only for requests whose user agent names a bot on the list+ * sends {userAgent, ip, url, status, at, country} to Ansvisor in batches of+ * about 100, after the response, without slowing the page.+ *+ * The bot tokens come from ai-bots.json, so the edge filter and the server's+ * check use the same list; the server still verifies every IP.+ */++import { AI_BOTS } from './ai-bots.js';++/** Requests a Worker isolate holds before it sends them. */+export const WORKER_BATCH = 100;++export function buildWorkerScript({ endpoint, secret }) {+ const tokens = JSON.stringify(AI_BOTS.map((b) => b.userAgent.toLowerCase()));+ return `// Ansvisor AI bot report — Cloudflare Worker (AT-010).+// Passes every request through; for AI bots only, sends the request line to+// Ansvisor after the response. Route it on your site's hostname.+const ENDPOINT = ${JSON.stringify(endpoint)};+const SECRET = ${JSON.stringify(secret)};+const BOTS = ${tokens};+const BATCH = ${WORKER_BATCH};+let queue = [];++function pageUrl(url) {+ const u = new URL(url);+ return u.origin + u.pathname;+}++function isAiBot(ua) {+ const s = (ua || '').toLowerCase();+ return BOTS.some((t) => s.includes(t));+}++async function flush() {+ if (queue.length === 0) return;+ const requests = queue;+ queue = [];+ try {+ await fetch(ENDPOINT, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json', Authorization: 'Bearer ' + SECRET },+ body: JSON.stringify({ requests }),+ });+ } catch (err) {+ // Never let reporting affect the site.+ }+}++export default {+ async fetch(request, env, ctx) {+ // Should this script fail, the request still reaches the site.+ ctx.passThroughOnException();+ const response = await fetch(request);+ const ua = request.headers.get('user-agent');+ if (isAiBot(ua)) {+ queue.push({+ userAgent: ua,+ ip: request.headers.get('cf-connecting-ip'),+ // The page, without its query string.+ url: pageUrl(request.url),+ status: response.status,+ at: new Date().toISOString(),+ country: request.cf && request.cf.country,+ });+ if (queue.length >= BATCH) ctx.waitUntil(flush());+ else ctx.waitUntil(new Promise((r) => setTimeout(r, 10000)).then(flush));+ }+ return response;+ },+};+`;+}diff --git a/server/src/lib/bot-worker-script.test.js b/server/src/lib/bot-worker-script.test.jsnew file mode 100644index 000000000..8180a85a0--- /dev/null+++ b/server/src/lib/bot-worker-script.test.js@@ -0,0 +1,85 @@+import { afterEach, describe, expect, it, vi } from 'vitest';+import { AI_BOTS } from './ai-bots.js';+import { WORKER_BATCH, buildWorkerScript } from './bot-worker-script.js';++// The Worker runtime's fetch types, which Node also has.+const { Request, Response } = globalThis;++/**+ * The Cloudflare Worker (AT-010, spec §6.1 option A): passes every request+ * through and sends only AI-bot requests, in batches, to Ansvisor.+ */++async function loadWorker() {+ const script = buildWorkerScript({+ endpoint: 'https://api.example/bots/ingest',+ secret: 'abot_secret',+ });+ const mod = await import(`data:text/javascript,${encodeURIComponent(script)}`);+ return { script, worker: mod.default };+}++afterEach(() => {+ vi.unstubAllGlobals();+ vi.useRealTimers();+});++describe('buildWorkerScript', () => {+ it('names every bot on the list', async () => {+ const { script } = await loadWorker();+ for (const bot of AI_BOTS)+ expect(script).toContain(JSON.stringify(bot.userAgent.toLowerCase()));+ });++ it('passes requests through and sends the AI-bot ones in a batch', async () => {+ const sent = [];+ vi.stubGlobal(+ 'fetch',+ vi.fn(async (input, init) => {+ if (typeof input === 'string' && input.endsWith('/bots/ingest')) {+ sent.push({ init, body: JSON.parse(init.body) });+ return new Response('{}');+ }+ return new Response('page', { status: 200 });+ }),+ );+ const { worker } = await loadWorker();+ const waits = [];+ const ctx = { waitUntil: (p) => waits.push(p), passThroughOnException: vi.fn() };++ const human = await worker.fetch(+ new Request('https://acme.example/', { headers: { 'user-agent': 'Mozilla/5.0 Chrome/129' } }),+ {},+ ctx,+ );+ expect(await human.text()).toBe('page');+ expect(waits).toHaveLength(0);+ // A failure in the script still lets the request reach the site.+ expect(ctx.passThroughOnException).toHaveBeenCalled();++ for (let i = 0; i < WORKER_BATCH; i++) {+ await worker.fetch(+ new Request(`https://acme.example/menu?i=${i}`, {+ headers: {+ 'user-agent': 'Mozilla/5.0; compatible; GPTBot/1.2',+ 'cf-connecting-ip': '20.171.206.9',+ },+ }),+ {},+ ctx,+ );+ }+ // The 100th request sends the batch at once.+ await waits.at(-1);+ expect(sent).toHaveLength(1);+ expect(sent[0].init.headers.Authorization).toBe('Bearer abot_secret');+ expect(sent[0].body.requests).toHaveLength(WORKER_BATCH);+ expect(sent[0].body.requests[0]).toMatchObject({+ userAgent: 'Mozilla/5.0; compatible; GPTBot/1.2',+ ip: '20.171.206.9',+ // The page without its query string, which can carry personal data.+ url: 'https://acme.example/menu',+ status: 200,+ });+ });+});diff --git a/server/src/lib/plan-guard.js b/server/src/lib/plan-guard.jsindex 1d98ccdec..8bd62fa93 100644--- a/server/src/lib/plan-guard.js+++ b/server/src/lib/plan-guard.js@@ -695,6 +695,17 @@ export async function getSiteAuditQuotaStatus(orgId, { now = new Date() } = {})};}+/**+ * Verified AI-bot rows an organization may store a month (AT-010, spec+ * §6.3): -1 = no cap, 0 = the bot report is locked (Starter). Throws+ * PlanLimitError (402) without an active subscription, like every billed+ * feature.+ */+export async function getBotEventCap(orgId) {+ const plan = await resolveOrgPlanByOrgId(orgId);+ return plan.limits.maxBotEventsPerMonth ?? 0;+}+/*** Express middleware that attaches req.plan from the authenticated user.* Must run after auth middleware (req.user must exist).diff --git a/server/src/middleware/rate-limiter.js b/server/src/middleware/rate-limiter.jsindex 6afc5bfea..90f1d85d8 100644--- a/server/src/middleware/rate-limiter.js+++ b/server/src/middleware/rate-limiter.js@@ -1,3 +1,4 @@+import { createHash } from 'node:crypto';import rateLimit from 'express-rate-limit';import { verifiedUserId } from './verified-tokens.js';@@ -77,6 +78,29 @@ export const trafficLimiter = rateLimit({handler: (_req, res) => res.status(204).end(),});+/**+ * AI-bot ingest key (AT-010): the connection, by its secret's hash, since a+ * Cloudflare Worker's requests leave from addresses Cloudflare shares across+ * its customers. Without a secret, the caller's IP.+ */+export function botIngestKey(req) {+ const secret = (req.headers.authorization || '').replace(/^Bearer\s+/i, '');+ if (!secret) return anonymousIpKey(req.ip);+ return 'bot:' + createHash('sha256').update(secret).digest('hex');+}++// AI-bot ingest: 600 requests a minute per connection. A Worker isolate+// sends a batch about every 10 s, and a Logpush job or drain a file at a+// time, so this leaves room for many of them while stopping a runaway sender.+export const botIngestLimiter = rateLimit({+ windowMs: 60 * 1000,+ max: 600,+ keyGenerator: botIngestKey,+ standardHeaders: true,+ legacyHeaders: false,+ message: { error: 'rate_limited' },+});+// Auth endpoints: 10 requests per 15 minutesexport const authLimiter = rateLimit({windowMs: 15 * 60 * 1000,diff --git a/server/src/middleware/rate-limiter.test.js b/server/src/middleware/rate-limiter.test.jsindex 7b068d054..eac7f84e5 100644--- a/server/src/middleware/rate-limiter.test.js+++ b/server/src/middleware/rate-limiter.test.js@@ -1,5 +1,5 @@import { describe, it, expect, beforeEach } from 'vitest';-import { apiLimiterKey } from './rate-limiter.js';+import { apiLimiterKey, botIngestKey } from './rate-limiter.js';import { markTokenVerified, clearVerifiedTokens } from './verified-tokens.js';function reqWith({ authorization, ip = '203.0.113.7' } = {}) {@@ -60,3 +60,19 @@ describe('rate-limiter – apiLimiterKey', () => {expect(a).not.toBe(b);});});++describe('rate-limiter – botIngestKey', () => {+ it("buckets AI-bot ingest by the connection's secret, whatever the address", () => {+ const a = botIngestKey(reqWith({ authorization: 'Bearer abot_one', ip: '1.1.1.1' }));+ const b = botIngestKey(reqWith({ authorization: 'Bearer abot_one', ip: '2.2.2.2' }));+ const other = botIngestKey(reqWith({ authorization: 'Bearer abot_two', ip: '1.1.1.1' }));+ expect(a).toBe(b);+ expect(a).not.toBe(other);+ // The secret itself isn't kept as the key.+ expect(a).toMatch(/^bot:[0-9a-f]{64}$/);+ });++ it('falls back to the IP without a secret', () => {+ expect(botIngestKey(reqWith({ ip: '1.1.1.1' }))).toBe('ip:1.1.1.1');+ });+});diff --git a/server/src/routes/bot-ingest.js b/server/src/routes/bot-ingest.jsnew file mode 100644index 000000000..5419e9a1e--- /dev/null+++ b/server/src/routes/bot-ingest.js@@ -0,0 +1,33 @@+/**+ * POST /bots/ingest — AI-bot requests from a brand's Cloudflare Worker+ * (AT-010, spec §6.1). Public, outside /api: the sender is the customer's+ * edge, authenticated by its connection secret (Authorization: Bearer).+ * Body: { requests: [{ userAgent, ip, url, status, at, country }] }.+ */++import express, { Router } from 'express';+import { PlanLimitError } from '../lib/plan-guard.js';+import { logger } from '../lib/logger.js';+import { botIngestLimiter } from '../middleware/rate-limiter.js';+import { connectionForSecret, ingestBotRequests } from '../lib/bot-ingest.js';++const router = Router();++router.post('/bots/ingest', botIngestLimiter, express.json({ limit: '1mb' }), async (req, res) => {+ try {+ const secret = (req.headers.authorization || '').replace(/^Bearer\s+/i, '');+ const connection = await connectionForSecret(secret);+ if (!connection) return res.status(401).json({ error: 'unknown_connection' });+ const result = await ingestBotRequests(connection, req.body?.requests);+ return res.json(result);+ } catch (error) {+ if (error instanceof PlanLimitError) {+ return res.status(error.statusCode).json({ error: 'plan_limit', message: error.message });+ }+ // Mounted before the request-id middleware: there is no req.log here.+ logger.error({ err: error }, 'bot ingest error');+ return res.status(500).json({ error: 'ingest_failed' });+ }+});++export default router;diff --git a/server/src/routes/bot-ingest.test.js b/server/src/routes/bot-ingest.test.jsnew file mode 100644index 000000000..79aed88ad--- /dev/null+++ b/server/src/routes/bot-ingest.test.js@@ -0,0 +1,74 @@+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';+import express from 'express';++/** POST /bots/ingest (AT-010): a connection's secret, then the batch. */++const connectionForSecret = vi.fn();+const ingestBotRequests = vi.fn();+vi.mock('../lib/bot-ingest.js', () => ({+ connectionForSecret: (...a) => connectionForSecret(...a),+ ingestBotRequests: (...a) => ingestBotRequests(...a),+}));+vi.mock('../lib/plan-guard.js', () => ({+ PlanLimitError: class PlanLimitError extends Error {+ constructor(message, statusCode = 403) {+ super(message);+ this.statusCode = statusCode;+ }+ },+}));++const { default: router } = await import('./bot-ingest.js');+const { PlanLimitError } = await import('../lib/plan-guard.js');++let server;+let url;++beforeEach(async () => {+ connectionForSecret.mockReset();+ ingestBotRequests.mockReset();+ const app = express();+ app.use(router);+ await new Promise((resolve) => {+ server = app.listen(0, resolve);+ });+ url = `http://127.0.0.1:${server.address().port}/bots/ingest`;+});++afterEach(async () => {+ await new Promise((resolve) => server.close(resolve));+});++const post = (secret, body) =>+ fetch(url, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${secret}` },+ body: JSON.stringify(body),+ });++describe('POST /bots/ingest', () => {+ it('ingests a batch for the connection its secret names', async () => {+ const connection = { id: 'c1', brandId: 'b1', organizationId: 'o1' };+ connectionForSecret.mockResolvedValue(connection);+ ingestBotRequests.mockResolvedValue({ received: 1, verified: 1, stored: 1, skipped: {} });+ const res = await post('abot_x', {+ requests: [{ userAgent: 'GPTBot', ip: '1.2.3.4', url: '/' }],+ });+ expect(res.status).toBe(200);+ expect(await res.json()).toMatchObject({ stored: 1 });+ expect(connectionForSecret).toHaveBeenCalledWith('abot_x');+ expect(ingestBotRequests).toHaveBeenCalledWith(connection, [+ { userAgent: 'GPTBot', ip: '1.2.3.4', url: '/' },+ ]);+ });++ it('refuses an unknown secret, and an organization without a subscription', async () => {+ connectionForSecret.mockResolvedValue(null);+ expect((await post('abot_bad', { requests: [] })).status).toBe(401);+ expect(ingestBotRequests).not.toHaveBeenCalled();++ connectionForSecret.mockResolvedValue({ id: 'c1' });+ ingestBotRequests.mockRejectedValue(new PlanLimitError('choose a plan', 402));+ expect((await post('abot_x', { requests: [] })).status).toBe(402);+ });+});diff --git a/server/src/routes/bots.js b/server/src/routes/bots.jsnew file mode 100644index 000000000..0726ac857--- /dev/null+++ b/server/src/routes/bots.js@@ -0,0 +1,150 @@+/**+ * The AI bot report's connections (AT-010): each way a brand sends AI-bot+ * requests — a Cloudflare Worker, a Logpush job, a Vercel drain — with its+ * own secret, shown once. Admins and managers make and revoke them, on plans+ * with the bot report (Starter has it locked, spec §7).+ *+ * GET /api/bots/connections/:brandId — the brand's connections and this month's use+ * POST /api/bots/connections/:brandId — { kind } → a connection, its secret, the endpoint, the Worker+ * DELETE /api/bots/connections/:brandId/:id — revoke one+ */++import { Router } from 'express';+import { z } from 'zod';+import supabaseAdmin from '../config/supabase.js';+import { assertBrandAccess } from '../lib/access.js';+import { PlanLimitError, getBotEventCap } from '../lib/plan-guard.js';+import {+ CONNECTION_KINDS,+ hashSecret,+ newConnectionSecret,+ publicIngestEndpoint,+} from '../lib/bot-ingest.js';+import { buildWorkerScript } from '../lib/bot-worker-script.js';++const router = Router();++const WRITE_ROLES = ['admin', 'manager'];++const createSchema = z.object({ kind: z.enum(CONNECTION_KINDS) });++async function assertWriter(userId) {+ const { data: profile } = await supabaseAdmin+ .from('profiles')+ .select('role')+ .eq('id', userId)+ .single();+ if (!WRITE_ROLES.includes(profile?.role)) {+ const err = new Error('Only admins and managers can manage AI bot connections.');+ err.status = 403;+ throw err;+ }+}++function fail(req, res, error, what) {+ if (error instanceof PlanLimitError) {+ return res.status(error.statusCode).json({ error: 'plan_limit', message: error.message });+ }+ const status = error.status || 500;+ if (status >= 500) req.log.error({ err: error }, `${what} error`);+ return res.status(status).json({ error: error.message || `Failed to ${what}` });+}++router.get('/connections/:brandId', async (req, res) => {+ try {+ const { brand } = await assertBrandAccess(req.params.brandId, req.user.id);+ const month = new Date().toISOString().slice(0, 7) + '-01';+ const [{ data: connections, error }, { data: usage }, cap] = await Promise.all([+ supabaseAdmin+ .from('ai_bot_connections')+ .select('id, kind, created_at, last_event_at')+ .eq('brand_id', brand.id)+ .is('revoked_at', null)+ .order('created_at'),+ supabaseAdmin+ .from('ai_bot_usage')+ .select('stored_events, counted_events, capped_at')+ .eq('organization_id', brand.organization_id)+ .eq('month', month)+ .maybeSingle(),+ getBotEventCap(brand.organization_id),+ ]);+ if (error) throw new Error(error.message);+ return res.json({+ connections: connections ?? [],+ usage: {+ month,+ stored: usage?.stored_events ?? 0,+ counted: usage?.counted_events ?? 0,+ cappedAt: usage?.capped_at ?? null,+ cap,+ },+ });+ } catch (error) {+ return fail(req, res, error, 'load AI bot connections');+ }+});++router.post('/connections/:brandId', async (req, res) => {+ try {+ const parsed = createSchema.safeParse(req.body ?? {});+ if (!parsed.success) return res.status(400).json({ error: 'Invalid request' });+ const { brand } = await assertBrandAccess(req.params.brandId, req.user.id);+ await assertWriter(req.user.id);+ if ((await getBotEventCap(brand.organization_id)) === 0) {+ throw new PlanLimitError('The AI bot report comes with the Growth plan.', 403);+ }+ // Checked before a connection exists: one the sender can't reach is no use.+ const endpoint = publicIngestEndpoint();+ if (!endpoint) {+ req.log.error('[ai-bots] PUBLIC_SERVER_URL is missing or not https');+ return res+ .status(503)+ .json({ error: 'AI bot connections are not set up on this server yet.' });+ }++ const secret = newConnectionSecret();+ const { data: connection, error } = await supabaseAdmin+ .from('ai_bot_connections')+ .insert({+ brand_id: brand.id,+ kind: parsed.data.kind,+ secret_hash: hashSecret(secret),+ created_by: req.user.id,+ })+ .select('id, kind, created_at, last_event_at')+ .single();+ if (error) throw new Error(error.message);++ return res.status(201).json({+ connection,+ secret,+ endpoint,+ workerScript:+ parsed.data.kind === 'cloudflare_worker' ? buildWorkerScript({ endpoint, secret }) : null,+ });+ } catch (error) {+ return fail(req, res, error, 'create the AI bot connection');+ }+});++router.delete('/connections/:brandId/:id', async (req, res) => {+ try {+ const { brand } = await assertBrandAccess(req.params.brandId, req.user.id);+ await assertWriter(req.user.id);+ const { data, error } = await supabaseAdmin+ .from('ai_bot_connections')+ .update({ revoked_at: new Date().toISOString() })+ .eq('id', req.params.id)+ .eq('brand_id', brand.id)+ .is('revoked_at', null)+ .select('id');+ if (error) throw new Error(error.message);+ if (!data?.length) return res.status(404).json({ error: 'Connection not found' });+ return res.status(204).end();+ } catch (error) {+ return fail(req, res, error, 'revoke the AI bot connection');+ }+});++export default router;diff --git a/server/src/routes/bots.test.js b/server/src/routes/bots.test.jsnew file mode 100644index 000000000..d4539cefa--- /dev/null+++ b/server/src/routes/bots.test.js@@ -0,0 +1,180 @@+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';+import express from 'express';++/**+ * The AI bot report's connections (AT-010): made and revoked by admins and+ * managers on plans with the report; the secret is returned once, with the+ * endpoint and, for a Cloudflare Worker, its script.+ */++let role;+let cap;+const inserted = [];+const updates = [];++function table(name) {+ const state = { name, filters: {} };+ const result = () => {+ if (name === 'brands') return { data: { id: 'b1', organization_id: 'o1' }, error: null };+ if (name === 'profiles') return { data: { organization_id: 'o1', role }, error: null };+ if (name === 'ai_bot_connections') {+ if (state.update)+ return { data: state.filters.id === 'gone' ? [] : [{ id: 'c1' }], error: null };+ if (state.insert)+ return {+ data: { id: 'c1', kind: state.insert.kind, created_at: 'now', last_event_at: null },+ error: null,+ };+ return {+ data: [{ id: 'c1', kind: 'cloudflare_worker', created_at: 'then', last_event_at: null }],+ error: null,+ };+ }+ if (name === 'ai_bot_usage')+ return { data: { stored_events: 12, counted_events: 15, capped_at: null }, error: null };+ return { data: null, error: null };+ };+ const chain = new Proxy(+ {},+ {+ get: (_t, prop) => {+ if (prop === 'then')+ return (resolve, reject) => Promise.resolve(result()).then(resolve, reject);+ if (prop === 'single' || prop === 'maybeSingle') return async () => result();+ if (prop === 'insert')+ return (row) => {+ state.insert = row;+ inserted.push(row);+ return chain;+ };+ if (prop === 'update')+ return (row) => {+ state.update = row;+ updates.push(row);+ return chain;+ };+ if (prop === 'eq')+ return (col, val) => {+ state.filters[col] = val;+ return chain;+ };+ return () => chain;+ },+ },+ );+ return chain;+}++vi.mock('../config/supabase.js', () => ({ default: { from: (name) => table(name) } }));+vi.mock('../lib/plan-guard.js', async () => {+ class PlanLimitError extends Error {+ constructor(message, statusCode = 403) {+ super(message);+ this.statusCode = statusCode;+ }+ }+ return {+ PlanLimitError,+ getOrgIdForUser: async () => 'o1',+ getBotEventCap: async () => cap,+ };+});++const { default: router } = await import('./bots.js');++let server;+let baseUrl;++beforeEach(async () => {+ vi.stubEnv('PUBLIC_SERVER_URL', 'https://api.example.com');+ role = 'admin';+ cap = 2_000_000;+ inserted.length = 0;+ updates.length = 0;+ const app = express();+ app.use(express.json());+ app.use((req, _res, next) => {+ req.user = { id: 'u1' };+ req.log = { error: vi.fn() };+ next();+ });+ app.use('/api/bots', router);+ await new Promise((resolve) => {+ server = app.listen(0, resolve);+ });+ baseUrl = `http://127.0.0.1:${server.address().port}/api/bots`;+});++afterEach(async () => {+ vi.unstubAllEnvs();+ await new Promise((resolve) => server.close(resolve));+});++const create = (body) =>+ fetch(`${baseUrl}/connections/b1`, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json' },+ body: JSON.stringify(body),+ });++describe('AI bot connections', () => {+ it('makes a connection and returns its secret once, with the Worker for Cloudflare', async () => {+ const res = await create({ kind: 'cloudflare_worker' });+ expect(res.status).toBe(201);+ const body = await res.json();+ expect(body.secret).toMatch(/^abot_/);+ // The API's public address, not the Host the web server reached it on.+ expect(body.endpoint).toBe('https://api.example.com/bots/ingest');+ expect(body.workerScript).toContain(body.secret);+ expect(body.workerScript).toContain(body.endpoint);+ expect(inserted[0]).toMatchObject({+ brand_id: 'b1',+ kind: 'cloudflare_worker',+ created_by: 'u1',+ });+ expect(inserted[0].secret_hash).toMatch(/^[0-9a-f]{64}$/);+ expect(JSON.stringify(inserted[0])).not.toContain(body.secret);++ const drain = await (await create({ kind: 'vercel_drain' })).json();+ expect(drain.workerScript).toBeNull();+ });++ it('makes no connection while the server has no public https address', async () => {+ vi.stubEnv('PUBLIC_SERVER_URL', 'http://api.example.com');+ const res = await create({ kind: 'cloudflare_worker' });+ expect(res.status).toBe(503);+ expect(inserted).toHaveLength(0);+ });++ it('refuses Starter, members and unknown kinds', async () => {+ cap = 0;+ const starter = await create({ kind: 'cloudflare_worker' });+ expect(starter.status).toBe(403);+ expect((await starter.json()).error).toBe('plan_limit');++ cap = 2_000_000;+ role = 'member';+ expect((await create({ kind: 'cloudflare_worker' })).status).toBe(403);+ role = 'admin';+ expect((await create({ kind: 'ftp' })).status).toBe(400);+ expect(inserted).toHaveLength(0);+ });++ it("lists the brand's connections with this month's use", async () => {+ const res = await fetch(`${baseUrl}/connections/b1`);+ expect(res.status).toBe(200);+ const body = await res.json();+ expect(body.connections).toEqual([+ { id: 'c1', kind: 'cloudflare_worker', created_at: 'then', last_event_at: null },+ ]);+ expect(body.usage).toMatchObject({ stored: 12, counted: 15, cappedAt: null, cap: 2_000_000 });+ expect(body.usage.month).toMatch(/^\d{4}-\d{2}-01$/);+ });++ it('revokes a connection, and says when there is none', async () => {+ const ok = await fetch(`${baseUrl}/connections/b1/c1`, { method: 'DELETE' });+ expect(ok.status).toBe(204);+ expect(updates[0].revoked_at).toBeTruthy();+ expect((await fetch(`${baseUrl}/connections/b1/gone`, { method: 'DELETE' })).status).toBe(404);+ });+});diff --git a/server/src/routes/index.js b/server/src/routes/index.jsindex 146fbeb8e..a44ce0cc7 100644--- a/server/src/routes/index.js+++ b/server/src/routes/index.js@@ -11,6 +11,7 @@ import integrationRoutes from './integrations.js';import mentionRoutes from './mentions.js';import brandRoutes from './brands.js';import actionCenterRoutes from './action-center.js';+import botRoutes from './bots.js';const router = Router();@@ -26,6 +27,7 @@ router.use('/integrations', integrationRoutes);router.use('/mentions', mentionRoutes);router.use('/brands', brandRoutes);router.use('/action-center', actionCenterRoutes);+router.use('/bots', botRoutes);router.get('/health', (req, res) => {res.json({ status: 'ok', user: req.user?.id });diff --git a/server/src/server.js b/server/src/server.jsindex 17751ae88..09bae4656 100644--- a/server/src/server.js+++ b/server/src/server.js@@ -13,6 +13,8 @@ import { apiLimiter } from './middleware/rate-limiter.js';import requestIdMiddleware from './middleware/request-id.js';import routes from './routes/index.js';import trafficRoutes from './routes/traffic.js';+import botIngestRoutes from './routes/bot-ingest.js';+import { pruneBotEvents } from './lib/bot-ingest.js';import {createJob,cleanupStaleJobs,@@ -60,6 +62,9 @@ app.use(bodyParser.urlencoded({ limit: '50mb', extended: true }));// --- Public traffic tracking (before helmet/cors — needs its own CORS for any origin) ---app.use('/', trafficRoutes);+// AI-bot requests from brands' Cloudflare / Vercel (AT-010): authenticated by+// each connection's secret, server to server.+app.use('/', botIngestRoutes);// --- Request logger (before everything to catch all requests) ---app.use(requestIdMiddleware);@@ -254,6 +259,11 @@ async function runDailyTracking() {logger.error({ err }, '[gsc-sync] daily run failed');});+ // AI-bot rows past their retention (AT-010): 90 days, counters 400.+ pruneBotEvents().catch((err) => {+ logger.error({ err }, '[ai-bots] daily prune failed');+ });+// Google Analytics sync (#704) — same contract as the Search Console sync// above: independent, never blocks or fails the tracking run, no-ops when// Composio or the Analytics auth config isn't set.diff --git a/supabase/migrations/00183_ai_bot_events.sql b/supabase/migrations/00183_ai_bot_events.sqlnew file mode 100644index 000000000..93d313e43--- /dev/null+++ b/supabase/migrations/00183_ai_bot_events.sql@@ -0,0 +1,254 @@+-- ============================================================================+-- 00183: AI bots reading the site — ingestion and its hard caps (AT-010, part A)+-- ============================================================================+-- Spec §3.7 and §6: which AI bots read the brand's pages, sent from the+-- customer's own Cloudflare or Vercel. A request counts only when its user+-- agent names a bot on the list (server/src/lib/ai-bots.json) and its IP is+-- in that operator's published ranges; the server checks both before+-- writing, and keeps no IP.+--+-- * ai_bot_connections — one per way a brand sends events (a Cloudflare+-- Worker, a Logpush job, a Vercel drain), each with its own secret.+-- Only the secret's SHA-256 is kept; the person sees it once.+-- * ai_bot_events — one row per verified request, kept 90 days.+-- * ai_bot_daily — bot × page × day counters, always added to. When the+-- month's cap is reached they are all that is written ("Detailed rows+-- paused until Nov 1, totals still counted", §6.3), and past it a bot's+-- hits of the day go on one '(other)' row instead of a row per page:+-- crawler hits are mostly a page, day and bot of their own, so page rows+-- would grow by about one per event. The cap is then a hard ceiling on+-- what is stored.+-- * ai_bot_usage — per organization (the workspace) and month: rows+-- stored, events counted, when the cap was reached, and when the 80%+-- alert fired (§6.3 guardrails).+--+-- Members read their brands' events, counters and connections (all but the+-- secret's hash); only the service role writes.++create table public.ai_bot_connections (+ id uuid primary key default gen_random_uuid(),+ brand_id uuid not null references public.brands(id) on delete cascade,+ kind text not null check (kind in ('cloudflare_worker', 'cloudflare_logpush', 'vercel_drain')),+ secret_hash text not null unique,+ created_by uuid references public.profiles(id) on delete set null,+ created_at timestamptz not null default now(),+ last_event_at timestamptz,+ revoked_at timestamptz+);++create index ai_bot_connections_brand_idx on public.ai_bot_connections (brand_id);++create table public.ai_bot_events (+ id bigint generated always as identity primary key,+ brand_id uuid not null references public.brands(id) on delete cascade,+ bot text not null,+ bot_type text not null check (bot_type in ('training', 'search', 'user')),+ -- The path, as the Pages section keys pages (lower(normalize_page_path)).+ page text not null,+ status smallint,+ country text,+ occurred_at timestamptz not null,+ received_at timestamptz not null default now()+);++create index ai_bot_events_brand_time_idx on public.ai_bot_events (brand_id, occurred_at desc);+-- The daily prune deletes by time across every brand.+create index ai_bot_events_time_idx on public.ai_bot_events (occurred_at);++create table public.ai_bot_daily (+ brand_id uuid not null references public.brands(id) on delete cascade,+ day date not null,+ bot text not null,+ bot_type text not null check (bot_type in ('training', 'search', 'user')),+ page text not null,+ hits integer not null default 0,+ last_seen timestamptz not null,+ primary key (brand_id, day, bot, page)+);++create index ai_bot_daily_brand_day_idx on public.ai_bot_daily (brand_id, day desc);+create index ai_bot_daily_day_idx on public.ai_bot_daily (day);++create table public.ai_bot_usage (+ organization_id uuid not null references public.organizations(id) on delete cascade,+ month date not null,+ stored_events bigint not null default 0,+ counted_events bigint not null default 0,+ capped_at timestamptz,+ alerted_at timestamptz,+ primary key (organization_id, month)+);++alter table public.ai_bot_connections enable row level security;+alter table public.ai_bot_events enable row level security;+alter table public.ai_bot_daily enable row level security;+alter table public.ai_bot_usage enable row level security;++create policy "ai_bot_connections: member select" on public.ai_bot_connections+ for select using (+ brand_id in (+ select b.id+ from public.brands b+ join public.profiles p on p.organization_id = b.organization_id+ where p.id = auth.uid()+ )+ );++create policy "ai_bot_events: member select" on public.ai_bot_events+ for select using (+ brand_id in (+ select b.id+ from public.brands b+ join public.profiles p on p.organization_id = b.organization_id+ where p.id = auth.uid()+ )+ );++create policy "ai_bot_daily: member select" on public.ai_bot_daily+ for select using (+ brand_id in (+ select b.id+ from public.brands b+ join public.profiles p on p.organization_id = b.organization_id+ where p.id = auth.uid()+ )+ );++create policy "ai_bot_usage: member select" on public.ai_bot_usage+ for select using (+ organization_id in (select p.organization_id from public.profiles p where p.id = auth.uid())+ );++-- The secret's hash stays with the server: members read every other column.+revoke all on public.ai_bot_connections from anon, authenticated;+grant select (id, brand_id, kind, created_by, created_at, last_event_at, revoked_at)+ on public.ai_bot_connections to authenticated;+grant select on public.ai_bot_events to authenticated;+grant select on public.ai_bot_daily to authenticated;+grant select on public.ai_bot_usage to authenticated;++-- ─── One batch of verified events, within the month's cap ──────────────────+--+-- p_events: [{bot, bot_type, page, status, country, at}], already verified;+-- `page` is the request's URL or path, keyed here as the Pages section keys+-- pages (lower(normalize_page_path), 00178).+-- The usage row is locked for the batch, so two batches for one+-- organization can't both take the last of its room. Rows are stored while+-- the month has room (p_cap < 0: no cap; 0: none stored); counters are+-- always added, by page for the events stored and on the bot's '(other)' row+-- of the day for the rest. Returns what was received and stored, the month's use, and+-- whether this batch took it past 80% of the cap (the server alerts once).++create or replace function public.ai_bot_ingest(+ p_connection_id uuid,+ p_cap bigint,+ p_events jsonb+)+returns jsonb+language plpgsql+volatile+security definer+set search_path to 'public'+as $$+declare+ v_brand uuid;+ v_org uuid;+ v_month date := date_trunc('month', now() at time zone 'utc')::date;+ v_used bigint;+ v_received integer := coalesce(jsonb_array_length(p_events), 0);+ v_room bigint;+ v_stored integer := 0;+ v_alerted timestamptz;+ v_crossed boolean := false;+begin+ select c.brand_id, b.organization_id into v_brand, v_org+ from public.ai_bot_connections c+ join public.brands b on b.id = c.brand_id+ where c.id = p_connection_id and c.revoked_at is null;+ if v_brand is null then+ return jsonb_build_object('error', 'connection');+ end if;++ insert into public.ai_bot_usage (organization_id, month)+ values (v_org, v_month)+ on conflict (organization_id, month) do nothing;++ select u.stored_events, u.alerted_at into v_used, v_alerted+ from public.ai_bot_usage u+ where u.organization_id = v_org and u.month = v_month+ for update;++ v_room := case when p_cap < 0 then v_received else greatest(0, least(v_received, p_cap - v_used)) end;++ if v_room > 0 then+ insert into public.ai_bot_events (brand_id, bot, bot_type, page, status, country, occurred_at)+ select+ v_brand,+ x.ev ->> 'bot',+ x.ev ->> 'bot_type',+ lower(public.normalize_page_path(x.ev ->> 'page')),+ (x.ev ->> 'status')::smallint,+ x.ev ->> 'country',+ (x.ev ->> 'at')::timestamptz+ from jsonb_array_elements(p_events) with ordinality as x(ev, n)+ order by x.n+ limit v_room;+ get diagnostics v_stored = row_count;+ end if;++ insert into public.ai_bot_daily (brand_id, day, bot, bot_type, page, hits, last_seen)+ select v_brand, e.day, e.bot, min(e.bot_type), e.page, count(*), max(e.at)+ from (+ select+ (r.at at time zone 'utc')::date as day,+ r.bot,+ r.bot_type,+ case+ when x.n <= v_room then lower(public.normalize_page_path(r.page))+ else '(other)'+ end as page,+ r.at+ from jsonb_array_elements(p_events) with ordinality as x(ev, n)+ cross join lateral jsonb_to_record(x.ev)+ as r(bot text, bot_type text, page text, at timestamptz)+ ) e+ group by e.day, e.bot, e.page+ on conflict (brand_id, day, bot, page) do update+ set hits = public.ai_bot_daily.hits + excluded.hits,+ last_seen = greatest(public.ai_bot_daily.last_seen, excluded.last_seen);++ v_crossed := p_cap > 0 and v_alerted is null and v_used + v_stored >= 0.8 * p_cap;++ update public.ai_bot_usage u+ set stored_events = u.stored_events + v_stored,+ counted_events = u.counted_events + v_received,+ capped_at = case+ when p_cap >= 0 and u.capped_at is null and v_stored < v_received then now()+ else u.capped_at+ end,+ alerted_at = case when v_crossed then now() else u.alerted_at end+ where u.organization_id = v_org and u.month = v_month;++ update public.ai_bot_connections c+ set last_event_at = now()+ where c.id = p_connection_id and v_received > 0;++ return jsonb_build_object(+ 'brand_id', v_brand,+ 'organization_id', v_org,+ 'received', v_received,+ 'stored', v_stored,+ 'used', v_used + v_stored,+ 'cap', p_cap,+ 'crossed80', v_crossed+ );+end;+$$;++revoke all on function public.ai_bot_ingest(uuid, bigint, jsonb) from public, anon, authenticated;+grant execute on function public.ai_bot_ingest(uuid, bigint, jsonb) to service_role;++comment on function public.ai_bot_ingest(uuid, bigint, jsonb) is+ 'AI bots (AT-010, spec §6.3): one batch of verified bot events for a connection. Stores rows while the organization''s month has room under p_cap (<0 = no cap), always adds bot × page × day counters, and reports whether the batch crossed 80% of the cap. Locks the usage row so concurrent batches cannot exceed it. Service role only.';++notify pgrst, 'reload schema';diff --git a/supabase/schema.sql b/supabase/schema.sqlindex c8919d894..fab3c7c22 100644--- a/supabase/schema.sql+++ b/supabase/schema.sql@@ -24955,3 +24955,261 @@ comment on function public.ai_traffic_google_queries(uuid, date, date, boolean,notify pgrst, 'reload schema';+-- ─────────────────────────────────────────────────────────────────────────+-- migrations/00183_ai_bot_events.sql+-- ─────────────────────────────────────────────────────────────────────────+-- ============================================================================+-- 00183: AI bots reading the site — ingestion and its hard caps (AT-010, part A)+-- ============================================================================+-- Spec §3.7 and §6: which AI bots read the brand's pages, sent from the+-- customer's own Cloudflare or Vercel. A request counts only when its user+-- agent names a bot on the list (server/src/lib/ai-bots.json) and its IP is+-- in that operator's published ranges; the server checks both before+-- writing, and keeps no IP.+--+-- * ai_bot_connections — one per way a brand sends events (a Cloudflare+-- Worker, a Logpush job, a Vercel drain), each with its own secret.+-- Only the secret's SHA-256 is kept; the person sees it once.+-- * ai_bot_events — one row per verified request, kept 90 days.+-- * ai_bot_daily — bot × page × day counters, always added to. When the+-- month's cap is reached they are all that is written ("Detailed rows+-- paused until Nov 1, totals still counted", §6.3), and past it a bot's+-- hits of the day go on one '(other)' row instead of a row per page:+-- crawler hits are mostly a page, day and bot of their own, so page rows+-- would grow by about one per event. The cap is then a hard ceiling on+-- what is stored.+-- * ai_bot_usage — per organization (the workspace) and month: rows+-- stored, events counted, when the cap was reached, and when the 80%+-- alert fired (§6.3 guardrails).+--+-- Members read their brands' events, counters and connections (all but the+-- secret's hash); only the service role writes.++create table public.ai_bot_connections (+ id uuid primary key default gen_random_uuid(),+ brand_id uuid not null references public.brands(id) on delete cascade,+ kind text not null check (kind in ('cloudflare_worker', 'cloudflare_logpush', 'vercel_drain')),+ secret_hash text not null unique,+ created_by uuid references public.profiles(id) on delete set null,+ created_at timestamptz not null default now(),+ last_event_at timestamptz,+ revoked_at timestamptz+);++create index ai_bot_connections_brand_idx on public.ai_bot_connections (brand_id);++create table public.ai_bot_events (+ id bigint generated always as identity primary key,+ brand_id uuid not null references public.brands(id) on delete cascade,+ bot text not null,+ bot_type text not null check (bot_type in ('training', 'search', 'user')),+ -- The path, as the Pages section keys pages (lower(normalize_page_path)).+ page text not null,+ status smallint,+ country text,+ occurred_at timestamptz not null,+ received_at timestamptz not null default now()+);++create index ai_bot_events_brand_time_idx on public.ai_bot_events (brand_id, occurred_at desc);+-- The daily prune deletes by time across every brand.+create index ai_bot_events_time_idx on public.ai_bot_events (occurred_at);++create table public.ai_bot_daily (+ brand_id uuid not null references public.brands(id) on delete cascade,+ day date not null,+ bot text not null,+ bot_type text not null check (bot_type in ('training', 'search', 'user')),+ page text not null,+ hits integer not null default 0,+ last_seen timestamptz not null,+ primary key (brand_id, day, bot, page)+);++create index ai_bot_daily_brand_day_idx on public.ai_bot_daily (brand_id, day desc);+create index ai_bot_daily_day_idx on public.ai_bot_daily (day);++create table public.ai_bot_usage (+ organization_id uuid not null references public.organizations(id) on delete cascade,+ month date not null,+ stored_events bigint not null default 0,+ counted_events bigint not null default 0,+ capped_at timestamptz,+ alerted_at timestamptz,+ primary key (organization_id, month)+);++alter table public.ai_bot_connections enable row level security;+alter table public.ai_bot_events enable row level security;+alter table public.ai_bot_daily enable row level security;+alter table public.ai_bot_usage enable row level security;++create policy "ai_bot_connections: member select" on public.ai_bot_connections+ for select using (+ brand_id in (+ select b.id+ from public.brands b+ join public.profiles p on p.organization_id = b.organization_id+ where p.id = auth.uid()+ )+ );++create policy "ai_bot_events: member select" on public.ai_bot_events+ for select using (+ brand_id in (+ select b.id+ from public.brands b+ join public.profiles p on p.organization_id = b.organization_id+ where p.id = auth.uid()+ )+ );++create policy "ai_bot_daily: member select" on public.ai_bot_daily+ for select using (+ brand_id in (+ select b.id+ from public.brands b+ join public.profiles p on p.organization_id = b.organization_id+ where p.id = auth.uid()+ )+ );++create policy "ai_bot_usage: member select" on public.ai_bot_usage+ for select using (+ organization_id in (select p.organization_id from public.profiles p where p.id = auth.uid())+ );++-- The secret's hash stays with the server: members read every other column.+revoke all on public.ai_bot_connections from anon, authenticated;+grant select (id, brand_id, kind, created_by, created_at, last_event_at, revoked_at)+ on public.ai_bot_connections to authenticated;+grant select on public.ai_bot_events to authenticated;+grant select on public.ai_bot_daily to authenticated;+grant select on public.ai_bot_usage to authenticated;++-- ─── One batch of verified events, within the month's cap ──────────────────+--+-- p_events: [{bot, bot_type, page, status, country, at}], already verified;+-- `page` is the request's URL or path, keyed here as the Pages section keys+-- pages (lower(normalize_page_path), 00178).+-- The usage row is locked for the batch, so two batches for one+-- organization can't both take the last of its room. Rows are stored while+-- the month has room (p_cap < 0: no cap; 0: none stored); counters are+-- always added, by page for the events stored and on the bot's '(other)' row+-- of the day for the rest. Returns what was received and stored, the month's use, and+-- whether this batch took it past 80% of the cap (the server alerts once).++create or replace function public.ai_bot_ingest(+ p_connection_id uuid,+ p_cap bigint,+ p_events jsonb+)+returns jsonb+language plpgsql+volatile+security definer+set search_path to 'public'+as $$+declare+ v_brand uuid;+ v_org uuid;+ v_month date := date_trunc('month', now() at time zone 'utc')::date;+ v_used bigint;+ v_received integer := coalesce(jsonb_array_length(p_events), 0);+ v_room bigint;+ v_stored integer := 0;+ v_alerted timestamptz;+ v_crossed boolean := false;+begin+ select c.brand_id, b.organization_id into v_brand, v_org+ from public.ai_bot_connections c+ join public.brands b on b.id = c.brand_id+ where c.id = p_connection_id and c.revoked_at is null;+ if v_brand is null then+ return jsonb_build_object('error', 'connection');+ end if;++ insert into public.ai_bot_usage (organization_id, month)+ values (v_org, v_month)+ on conflict (organization_id, month) do nothing;++ select u.stored_events, u.alerted_at into v_used, v_alerted+ from public.ai_bot_usage u+ where u.organization_id = v_org and u.month = v_month+ for update;++ v_room := case when p_cap < 0 then v_received else greatest(0, least(v_received, p_cap - v_used)) end;++ if v_room > 0 then+ insert into public.ai_bot_events (brand_id, bot, bot_type, page, status, country, occurred_at)+ select+ v_brand,+ x.ev ->> 'bot',+ x.ev ->> 'bot_type',+ lower(public.normalize_page_path(x.ev ->> 'page')),+ (x.ev ->> 'status')::smallint,+ x.ev ->> 'country',+ (x.ev ->> 'at')::timestamptz+ from jsonb_array_elements(p_events) with ordinality as x(ev, n)+ order by x.n+ limit v_room;+ get diagnostics v_stored = row_count;+ end if;++ insert into public.ai_bot_daily (brand_id, day, bot, bot_type, page, hits, last_seen)+ select v_brand, e.day, e.bot, min(e.bot_type), e.page, count(*), max(e.at)+ from (+ select+ (r.at at time zone 'utc')::date as day,+ r.bot,+ r.bot_type,+ case+ when x.n <= v_room then lower(public.normalize_page_path(r.page))+ else '(other)'+ end as page,+ r.at+ from jsonb_array_elements(p_events) with ordinality as x(ev, n)+ cross join lateral jsonb_to_record(x.ev)+ as r(bot text, bot_type text, page text, at timestamptz)+ ) e+ group by e.day, e.bot, e.page+ on conflict (brand_id, day, bot, page) do update+ set hits = public.ai_bot_daily.hits + excluded.hits,+ last_seen = greatest(public.ai_bot_daily.last_seen, excluded.last_seen);++ v_crossed := p_cap > 0 and v_alerted is null and v_used + v_stored >= 0.8 * p_cap;++ update public.ai_bot_usage u+ set stored_events = u.stored_events + v_stored,+ counted_events = u.counted_events + v_received,+ capped_at = case+ when p_cap >= 0 and u.capped_at is null and v_stored < v_received then now()+ else u.capped_at+ end,+ alerted_at = case when v_crossed then now() else u.alerted_at end+ where u.organization_id = v_org and u.month = v_month;++ update public.ai_bot_connections c+ set last_event_at = now()+ where c.id = p_connection_id and v_received > 0;++ return jsonb_build_object(+ 'brand_id', v_brand,+ 'organization_id', v_org,+ 'received', v_received,+ 'stored', v_stored,+ 'used', v_used + v_stored,+ 'cap', p_cap,+ 'crossed80', v_crossed+ );+end;+$$;++revoke all on function public.ai_bot_ingest(uuid, bigint, jsonb) from public, anon, authenticated;+grant execute on function public.ai_bot_ingest(uuid, bigint, jsonb) to service_role;++comment on function public.ai_bot_ingest(uuid, bigint, jsonb) is+ 'AI bots (AT-010, spec §6.3): one batch of verified bot events for a connection. Stores rows while the organization''s month has room under p_cap (<0 = no cap), always adds bot × page × day counters, and reports whether the batch crossed 80% of the cap. Locks the usage row so concurrent batches cannot exceed it. Service role only.';++notify pgrst, 'reload schema';+diff --git a/web/src/config/plans.ts b/web/src/config/plans.tsindex 0236fea0b..6a08622c5 100644--- a/web/src/config/plans.ts+++ b/web/src/config/plans.ts@@ -72,6 +72,12 @@ export interface PlanLimits {* Starter 100, Growth 500.*/maxSiteAudits: number;+ /**+ * Verified AI-bot rows stored a month (AT-010, spec §6.3). 0 = the bot+ * report is locked (Starter), -1 = no cap. Growth 2M, Enterprise 20M,+ * adjustable per customer via organizations.plan_overrides.+ */+ maxBotEventsPerMonth: number;maxDailyOnDemand: number;onDemandCooldownMinutes: number;features: readonly Feature[];@@ -113,6 +119,7 @@ export const PLANS: Record<PlanId, Plan> = {maxOpportunitiesPerMonth: -1,maxOpportunitiesPerRequest: 5,maxSiteAudits: -1,+ maxBotEventsPerMonth: -1,maxDailyOnDemand: -1,onDemandCooldownMinutes: 0,features: [@@ -153,6 +160,7 @@ export const PLANS: Record<PlanId, Plan> = {maxOpportunitiesPerMonth: 10,maxOpportunitiesPerRequest: 3,maxSiteAudits: 100,+ maxBotEventsPerMonth: 0,maxDailyOnDemand: 3,onDemandCooldownMinutes: 15,allowedScrapers: ['chatgpt-web', 'copilot-web'],@@ -197,6 +205,7 @@ export const PLANS: Record<PlanId, Plan> = {maxOpportunitiesPerMonth: 50,maxOpportunitiesPerRequest: 5,maxSiteAudits: 500,+ maxBotEventsPerMonth: 2_000_000,maxDailyOnDemand: 10,onDemandCooldownMinutes: 5,// API-model tracking (Claude) is not part of Growth — scraper engines@@ -242,6 +251,7 @@ export const PLANS: Record<PlanId, Plan> = {maxOpportunitiesPerMonth: -1,maxOpportunitiesPerRequest: 5,maxSiteAudits: -1,+ maxBotEventsPerMonth: 20_000_000,maxDailyOnDemand: -1,onDemandCooldownMinutes: 0,// API-model tracking (Claude) is a per-customer opt-in on Enterprise: