Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.
TypeScript aeoai-agentai-searchanswer-engine-optimizationbrand-monitoringchatgptclaudecopilotdockergeminigeogoogle-ai-overviewsgrokmcpnextjsopen-sourceperplexityseosupabasetypescript
feat(traffic): read Cloudflare Logpush files and Vercel drains of AI-bot requests, within a raw cap (#1180)
Cihan6e2e7f7f6e77
diff --git a/server/src/config/plans.js b/server/src/config/plans.jsindex 90ec6d4fc..7eb3bdc10 100644--- a/server/src/config/plans.js+++ b/server/src/config/plans.js@@ -21,6 +21,7 @@ export const PLANS = {maxOpportunitiesPerRequest: 5,maxSiteAudits: -1,maxBotEventsPerMonth: -1,+ maxBotRawRecordsPerMonth: -1,maxDailyOnDemand: -1,onDemandCooldownMinutes: 0,features: [@@ -61,6 +62,9 @@ export const PLANS = {// Starter sees the report locked; Growth 2M, Enterprise 20M (adjustable// through plan_overrides); -1 = no cap.maxBotEventsPerMonth: 0,+ // Raw log records a month a Vercel drain or Logpush job may send+ // (AT-010, spec §6.3): Growth 20M, Enterprise 200M.+ maxBotRawRecordsPerMonth: 0,maxDailyOnDemand: 3,onDemandCooldownMinutes: 15,allowedScrapers: ['chatgpt-web', 'copilot-web'],@@ -99,6 +103,7 @@ export const PLANS = {maxOpportunitiesPerRequest: 5,maxSiteAudits: 500,maxBotEventsPerMonth: 2_000_000,+ maxBotRawRecordsPerMonth: 20_000_000,maxDailyOnDemand: 10,onDemandCooldownMinutes: 5,// API-model tracking (Claude) is not part of Growth — scraper engines@@ -143,6 +148,7 @@ export const PLANS = {maxOpportunitiesPerRequest: 5,maxSiteAudits: -1,maxBotEventsPerMonth: 20_000_000,+ maxBotRawRecordsPerMonth: 200_000_000,maxDailyOnDemand: -1,onDemandCooldownMinutes: 0,// API-model tracking (Claude) is a per-customer opt-in on Enterprise:diff --git a/server/src/lib/bot-ingest.js b/server/src/lib/bot-ingest.jsindex 484b0d838..7a7c47b1a 100644--- a/server/src/lib/bot-ingest.js+++ b/server/src/lib/bot-ingest.js@@ -13,12 +13,15 @@import { createHash, randomBytes } from 'crypto';import supabaseAdmin from '../config/supabase.js';import { verifyBotRequest } from './ai-bots.js';-import { getBotEventCap } from './plan-guard.js';+import { getBotEventCap, getBotRawCap } from './plan-guard.js';import { logger } from './logger.js';-/** The most requests one batch may carry (a Worker sends about 100). */+/** The most requests one Worker batch may carry (a Worker sends about 100). */export const MAX_BATCH = 500;+/** Verified events are written this many at a time. */+export const WRITE_CHUNK = 500;+/** Requests older than this are dropped: a sender catching up after a week isn't the site's reading now. */export const MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000;@@ -101,11 +104,13 @@ function readRequest(raw, now) {/*** Verify and store one batch for a connection. Returns what was received,- * verified and stored, and why the rest wasn't counted.+ * verified and stored, and why the rest wasn't counted. A Worker batch is+ * kept to MAX_BATCH; a log file (`limit: Infinity`) is read whole and its+ * verified events written WRITE_CHUNK at a time.*/export async function ingestBotRequests(connection, requests, deps = {}) {const now = deps.now ?? Date.now();- const list = Array.isArray(requests) ? requests.slice(0, MAX_BATCH) : [];+ const list = Array.isArray(requests) ? requests.slice(0, deps.limit ?? MAX_BATCH) : [];const skipped = { invalid: 0, unknown_agent: 0, unverified_ip: 0, no_ranges: 0 };const events = [];for (const raw of list) {@@ -135,33 +140,88 @@ export async function ingestBotRequests(connection, requests, deps = {}) {// A plan without the report (Starter) stores nothing, not even counters.if (cap === 0)return { received: list.length, verified: events.length, stored, skipped, locked: true };- const { data, error } = await supabaseAdmin.rpc('ai_bot_ingest', {- p_connection_id: connection.id,- p_cap: cap,- p_events: events,- });- if (error) throw new Error(error.message);- if (data?.error) throw new Error(`ai_bot_ingest: ${data.error}`);- stored = data?.stored ?? 0;- if (data?.crossed80) {- logger.warn(- {- organizationId: connection.organizationId,- brandId: connection.brandId,- used: data.used,- cap,- },- '[ai-bots] 80% of the monthly bot event cap reached',- );+ for (let i = 0; i < events.length; i += WRITE_CHUNK) {+ const { data, error } = await supabaseAdmin.rpc('ai_bot_ingest', {+ p_connection_id: connection.id,+ p_cap: cap,+ p_events: events.slice(i, i + WRITE_CHUNK),+ });+ if (error) throw new Error(error.message);+ if (data?.error) throw new Error(`ai_bot_ingest: ${data.error}`);+ stored += data?.stored ?? 0;+ if (data?.crossed80) {+ logger.warn(+ {+ organizationId: connection.organizationId,+ brandId: connection.brandId,+ used: data.used,+ cap,+ },+ '[ai-bots] 80% of the monthly bot event cap reached',+ );+ }}}return { received: list.length, verified: events.length, stored, skipped };}+/**+ * Claim a log file for reading (a Logpush job's or a Vercel drain's), by+ * the SHA-256 of its body per connection: a sender that times out waiting+ * sends the same file again, and its chunks already written would count+ * twice. Returns the hash, or null when the file was read already. A read+ * that fails gives its claim back (releaseLogFile) so the retry is read.+ */+export async function claimLogFile(connection, body) {+ const hash = createHash('sha256')+ .update(Buffer.isBuffer(body) ? body : Buffer.alloc(0))+ .digest('hex');+ const { data, error } = await supabaseAdmin+ .from('ai_bot_ingested_files')+ .upsert(+ { connection_id: connection.id, body_hash: hash },+ { onConflict: 'connection_id,body_hash', ignoreDuplicates: true },+ )+ .select('body_hash');+ if (error) throw new Error(error.message);+ return (data ?? []).length > 0 ? hash : null;+}++/** Give a claim back after a read that failed. */+export async function releaseLogFile(connection, hash) {+ const { error } = await supabaseAdmin+ .from('ai_bot_ingested_files')+ .delete()+ .eq('connection_id', connection.id)+ .eq('body_hash', hash);+ if (error)+ logger.warn({ err: error, connectionId: connection.id }, '[ai-bots] claim release failed');+}++/**+ * Count a log sender's records against the month's raw cap (a Vercel drain+ * or a Logpush job, §6.3). `allowed` says whether this batch may be read; a+ * plan without the report reads none.+ */+export async function countRawRecords(connection, records) {+ const cap = await getBotRawCap(connection.organizationId);+ if (cap === 0) return { allowed: false, locked: true };+ const { data, error } = await supabaseAdmin.rpc('ai_bot_count_raw', {+ p_connection_id: connection.id,+ p_count: records,+ p_cap: cap,+ });+ if (error) throw new Error(error.message);+ if (data?.error) throw new Error(`ai_bot_count_raw: ${data.error}`);+ return { allowed: data?.allowed === true, raw: data?.raw ?? null, cap };+}+/** Verified rows are kept 90 days (§6.2); the bot × page × day counters 400. */export const EVENT_RETENTION_DAYS = 90;export const DAILY_RETENTION_DAYS = 400;+/** Log file hashes kept for spotting a sender's retry (claimLogFile). */+export const FILE_HASH_RETENTION_DAYS = 7;/** The daily prune of rows past their retention (the daily run). */export async function pruneBotEvents(now = Date.now()) {@@ -176,4 +236,10 @@ export async function pruneBotEvents(now = Date.now()) {.delete().lt('day', before(DAILY_RETENTION_DAYS).slice(0, 10));if (dailyError) logger.warn({ err: dailyError }, '[ai-bots] daily prune failed');+ // A sender retries within minutes; a week of file hashes is plenty.+ const { error: filesError } = await supabaseAdmin+ .from('ai_bot_ingested_files')+ .delete()+ .lt('received_at', before(FILE_HASH_RETENTION_DAYS));+ if (filesError) logger.warn({ err: filesError }, '[ai-bots] file hash prune failed');}diff --git a/server/src/lib/bot-ingest.test.js b/server/src/lib/bot-ingest.test.jsindex 27215c603..dc9fcca1e 100644--- a/server/src/lib/bot-ingest.test.js+++ b/server/src/lib/bot-ingest.test.js@@ -26,14 +26,20 @@ vi.mock('../config/supabase.js', () => ({},}));const getBotEventCap = vi.fn();-vi.mock('./plan-guard.js', () => ({ getBotEventCap: (...a) => getBotEventCap(...a) }));+const getBotRawCap = vi.fn();+vi.mock('./plan-guard.js', () => ({+ getBotEventCap: (...a) => getBotEventCap(...a),+ getBotRawCap: (...a) => getBotRawCap(...a),+}));const warn = vi.fn();vi.mock('./logger.js', () => ({ logger: { warn: (...a) => warn(...a) } }));const { clearRangeCache } = await import('./ai-bots.js');const {MAX_BATCH,+ WRITE_CHUNK,connectionForSecret,+ countRawRecords,hashSecret,ingestBotRequests,newConnectionSecret,@@ -64,6 +70,7 @@ beforeEach(() => {.mockReset().mockResolvedValue({ data: { stored: 2, used: 2, crossed80: false }, error: null });getBotEventCap.mockReset().mockResolvedValue(2_000_000);+ getBotRawCap.mockReset().mockResolvedValue(20_000_000);warn.mockReset();selects.length = 0;});@@ -133,6 +140,20 @@ describe('ingestBotRequests', () => {expect(many.received).toBe(MAX_BATCH);});+ it('reads a whole log file and writes its verified events in chunks', async () => {+ rpc.mockResolvedValue({ data: { stored: 1, used: 1, crossed80: false }, error: null });+ const result = await ingestBotRequests(+ connection,+ Array.from({ length: WRITE_CHUNK + 20 }, () => gpt()),+ { fetchImpl, now: NOW, limit: Infinity },+ );+ expect(result.verified).toBe(WRITE_CHUNK + 20);+ expect(rpc).toHaveBeenCalledTimes(2);+ expect(rpc.mock.calls[0][1].p_events).toHaveLength(WRITE_CHUNK);+ expect(rpc.mock.calls[1][1].p_events).toHaveLength(20);+ expect(result.stored).toBe(2);+ });+it('writes nothing on a plan without the report', async () => {getBotEventCap.mockResolvedValue(0);const result = await ingestBotRequests(connection, [gpt()], { fetchImpl, now: NOW });@@ -179,6 +200,34 @@ describe('connection secrets', () => {});});+describe('countRawRecords', () => {+ it("counts a log sender's records against the plan's raw cap", async () => {+ rpc.mockResolvedValue({ data: { allowed: true, raw: 1200, cap: 20_000_000 }, error: null });+ expect(await countRawRecords(connection, 1200)).toEqual({+ allowed: true,+ raw: 1200,+ cap: 20_000_000,+ });+ expect(rpc).toHaveBeenCalledWith('ai_bot_count_raw', {+ p_connection_id: 'c1',+ p_count: 1200,+ p_cap: 20_000_000,+ });++ rpc.mockResolvedValue({+ data: { allowed: false, raw: 20_000_100, cap: 20_000_000 },+ error: null,+ });+ expect((await countRawRecords(connection, 100)).allowed).toBe(false);+ });++ it('reads nothing on a plan without the report', async () => {+ getBotRawCap.mockResolvedValue(0);+ expect(await countRawRecords(connection, 10)).toEqual({ allowed: false, locked: true });+ expect(rpc).not.toHaveBeenCalled();+ });+});+describe('publicIngestEndpoint', () => {it("builds the endpoint from the API's public address, whatever its path", () => {expect(publicIngestEndpoint({ PUBLIC_SERVER_URL: 'https://api.ansvisor.com/' })).toBe(diff --git a/server/src/lib/bot-log-formats.js b/server/src/lib/bot-log-formats.jsnew file mode 100644index 000000000..bea88f6e3--- /dev/null+++ b/server/src/lib/bot-log-formats.js@@ -0,0 +1,173 @@+/**+ * What each AI-bot sender posts to /bots/ingest (AT-010, spec §6.1), read+ * into one request shape — { userAgent, ip, url, status, at, country } — for+ * bot-ingest.js to verify:+ *+ * * Cloudflare Worker (option A): JSON { requests: [...] } already in that+ * shape (bot-worker-script.js).+ * * Cloudflare Logpush (option B): newline-delimited JSON of the+ * http_requests dataset, gzipped — ClientRequestUserAgent, ClientIP,+ * ClientRequestHost, ClientRequestURI, EdgeResponseStatus,+ * EdgeStartTimestamp (unix ns, unix s or RFC 3339, as the job is set)+ * and ClientCountry. Creating the job uploads a test file,+ * {"content":"tests"}, which reads as no requests.+ * * Vercel drain (option C): a JSON array or NDJSON of log entries; only+ * those with a `proxy` object describe a request (clientIp, userAgent[],+ * host, path, statusCode, timestamp in ms). Build and function output+ * lines are skipped.+ *+ * Log senders report every line they hold; `records` is that count, for the+ * raw-record cap.+ *+ * A file is read without holding up the API's other requests: it's inflated+ * asynchronously, to at most MAX_INFLATED_BYTES, holds at most MAX_LOG_LINES+ * (LogTooLargeError otherwise), and its lines are parsed a slice at a time.+ * A line whose time can't be read is dropped, not the file.+ */++import { setImmediate as nextTurn } from 'timers/promises';+import { promisify } from 'util';+import { gunzip } from 'zlib';++const gunzipAsync = promisify(gunzip);++/**+ * A gzipped body opens to at most this much. A Logpush job set up as+ * bot-sender-setup.js says sends at most 10,000 lines, about 3 MB.+ */+export const MAX_INFLATED_BYTES = 16 * 1024 * 1024;++/** Lines one body may hold: twice what such a Logpush file holds. */+export const MAX_LOG_LINES = 20_000;++/** Lines parsed before the API's other requests get a turn. */+const PARSE_SLICE = 1000;++/** A body past MAX_INFLATED_BYTES or MAX_LOG_LINES; the route answers 413. */+export class LogTooLargeError extends Error {}++/**+ * The body as text. A body sent with Content-Encoding: gzip arrives already+ * inflated by the body parser; a gzipped file sent as is (Logpush's) is told+ * by its magic bytes and inflated here.+ */+export async function bodyText(buffer) {+ if (!Buffer.isBuffer(buffer) || buffer.length === 0) return '';+ const gzipped = buffer[0] === 0x1f && buffer[1] === 0x8b;+ if (!gzipped) return buffer.toString('utf8');+ try {+ const inflated = await gunzipAsync(buffer, { maxOutputLength: MAX_INFLATED_BYTES });+ return inflated.toString('utf8');+ } catch (err) {+ if (err?.code === 'ERR_BUFFER_TOO_LARGE') {+ throw new LogTooLargeError(`The file opens to more than ${MAX_INFLATED_BYTES} bytes`);+ }+ throw err;+ }+}++const tooManyLines = (count) =>+ new LogTooLargeError(`The file holds ${count} lines; at most ${MAX_LOG_LINES} are read`);++/** JSON lines, or one JSON array; malformed lines are left out. */+async function readLines(text) {+ const trimmed = text.trim();+ if (!trimmed) return [];+ if (trimmed.startsWith('[')) {+ let list;+ try {+ list = JSON.parse(trimmed);+ } catch {+ return [];+ }+ if (!Array.isArray(list)) return [];+ if (list.length > MAX_LOG_LINES) throw tooManyLines(list.length);+ return list;+ }+ const lines = trimmed.split('\n');+ if (lines.length > MAX_LOG_LINES) throw tooManyLines(lines.length);+ const read = [];+ for (let i = 0; i < lines.length; i++) {+ if (i > 0 && i % PARSE_SLICE === 0) await nextTurn();+ const line = lines[i].trim();+ if (!line) continue;+ try {+ read.push(JSON.parse(line));+ } catch {+ // A malformed line is left out.+ }+ }+ return read;+}++/** An ISO time, or undefined for a value that isn't one. */+function isoTime(ms) {+ const date = new Date(ms);+ return Number.isNaN(date.getTime()) ? undefined : date.toISOString();+}++/** Logpush's EdgeStartTimestamp in any of its formats, as an ISO time. */+function logpushTime(value) {+ if (typeof value === 'string' && !/^\d+$/.test(value)) return isoTime(value);+ const n = Number(value);+ if (!Number.isFinite(n)) return undefined;+ // ns, ms or s since the epoch, told apart by size.+ return isoTime(n > 1e15 ? n / 1e6 : n > 1e12 ? n : n * 1000);+}++export function readWorkerBody(text) {+ try {+ const body = JSON.parse(text || '{}');+ const requests = Array.isArray(body?.requests) ? body.requests : [];+ return { records: requests.length, requests };+ } catch {+ return { records: 0, requests: [] };+ }+}++export async function readLogpushBody(text) {+ const lines = await readLines(text);+ const requests = lines+ .filter((l) => l && typeof l === 'object' && 'ClientRequestUserAgent' in l)+ .map((l) => ({+ userAgent: l.ClientRequestUserAgent,+ ip: l.ClientIP,+ url: l.ClientRequestHost+ ? `https://${l.ClientRequestHost}${l.ClientRequestURI ?? '/'}`+ : l.ClientRequestURI,+ status: l.EdgeResponseStatus,+ at: logpushTime(l.EdgeStartTimestamp),+ country: l.ClientCountry,+ }))+ .filter((r) => r.at !== undefined);+ return { records: lines.length, requests };+}++export async function readVercelBody(text) {+ const lines = await readLines(text);+ const requests = lines+ .filter((l) => l?.proxy && typeof l.proxy === 'object')+ .map(({ proxy, timestamp }) => ({+ userAgent: Array.isArray(proxy.userAgent) ? proxy.userAgent[0] : proxy.userAgent,+ ip: proxy.clientIp,+ url: proxy.host ? `https://${proxy.host}${proxy.path ?? '/'}` : proxy.path,+ status: proxy.statusCode,+ at: isoTime(proxy.timestamp ?? timestamp ?? Date.now()),+ country: null,+ }))+ .filter((r) => r.at !== undefined);+ return { records: lines.length, requests };+}++const READERS = {+ cloudflare_worker: readWorkerBody,+ cloudflare_logpush: readLogpushBody,+ vercel_drain: readVercelBody,+};++/** A connection's body as requests, and how many records it held. */+export async function readSenderBody(kind, buffer) {+ const read = READERS[kind];+ if (!read) return { records: 0, requests: [] };+ return read(await bodyText(buffer));+}diff --git a/server/src/lib/bot-log-formats.test.js b/server/src/lib/bot-log-formats.test.jsnew file mode 100644index 000000000..6145af52a--- /dev/null+++ b/server/src/lib/bot-log-formats.test.js@@ -0,0 +1,148 @@+import { gzipSync } from 'zlib';+import { describe, expect, it } from 'vitest';+import {+ LogTooLargeError,+ MAX_INFLATED_BYTES,+ MAX_LOG_LINES,+ readLogpushBody,+ readSenderBody,+ readVercelBody,+ readWorkerBody,+} from './bot-log-formats.js';++/** What each AI-bot sender posts (AT-010, spec §6.1), read into one request shape. */++const logpushLine = (over = {}) =>+ JSON.stringify({+ ClientRequestUserAgent: 'Mozilla/5.0; compatible; GPTBot/1.2',+ ClientIP: '20.171.206.9',+ ClientRequestHost: 'acme.example',+ ClientRequestURI: '/menu?x=1',+ EdgeResponseStatus: 200,+ EdgeStartTimestamp: 1792400000000000000,+ ClientCountry: 'us',+ ...over,+ });++describe('readLogpushBody', () => {+ it('reads the http_requests fields, in any timestamp format', async () => {+ const text = [+ logpushLine(),+ logpushLine({ EdgeStartTimestamp: 1792400000 }),+ logpushLine({ EdgeStartTimestamp: '2026-10-19T09:00:00Z' }),+ '{not json',+ '',+ ].join('\n');+ const { records, requests } = await readLogpushBody(text);+ expect(records).toBe(3);+ expect(requests[0]).toEqual({+ userAgent: 'Mozilla/5.0; compatible; GPTBot/1.2',+ ip: '20.171.206.9',+ url: 'https://acme.example/menu?x=1',+ status: 200,+ at: '2026-10-19T08:53:20.000Z',+ country: 'us',+ });+ expect(requests[1].at).toBe('2026-10-19T08:53:20.000Z');+ expect(requests[2].at).toBe('2026-10-19T09:00:00.000Z');+ });++ it("reads Logpush's gzipped test upload as no requests", async () => {+ const parsed = await readSenderBody('cloudflare_logpush', gzipSync('{"content":"tests"}'));+ expect(parsed).toEqual({ records: 1, requests: [] });+ });++ it('opens a gzipped file by its magic bytes', async () => {+ const parsed = await readSenderBody(+ 'cloudflare_logpush',+ gzipSync(`${logpushLine()}\n${logpushLine()}`),+ );+ expect(parsed.requests).toHaveLength(2);+ });+});++describe('readVercelBody', () => {+ const entry = (over = {}) => ({+ id: '1',+ source: 'static',+ host: 'acme.vercel.app',+ timestamp: 1792400000000,+ proxy: {+ timestamp: 1792400000500,+ method: 'GET',+ host: 'acme.example',+ path: '/gifts?ref=a',+ userAgent: ['Mozilla/5.0 (compatible; ClaudeBot/1.0)'],+ statusCode: 200,+ clientIp: '160.79.104.10',+ region: 'fra1',+ },+ ...over,+ });++ it('reads the request logs of a JSON array or NDJSON, and skips the rest', async () => {+ const build = { id: '2', source: 'build', message: 'Build completed' };+ const fromArray = await readVercelBody(JSON.stringify([entry(), build]));+ expect(fromArray.records).toBe(2);+ expect(fromArray.requests).toEqual([+ {+ userAgent: 'Mozilla/5.0 (compatible; ClaudeBot/1.0)',+ ip: '160.79.104.10',+ url: 'https://acme.example/gifts?ref=a',+ status: 200,+ at: '2026-10-19T08:53:20.500Z',+ country: null,+ },+ ]);+ const fromLines = await readVercelBody(`${JSON.stringify(entry())}\n${JSON.stringify(build)}`);+ expect(fromLines).toEqual(fromArray);+ });+});++describe('readWorkerBody and unknown kinds', () => {+ it("reads a Worker's batch, and nothing from a malformed body or an unknown kind", async () => {+ expect(readWorkerBody('{"requests":[{"ip":"1.1.1.1"}]}')).toEqual({+ records: 1,+ requests: [{ ip: '1.1.1.1' }],+ });+ expect(readWorkerBody('{bad')).toEqual({ records: 0, requests: [] });+ expect(await readSenderBody('ftp', Buffer.from('x'))).toEqual({ records: 0, requests: [] });+ expect(await readSenderBody('vercel_drain', Buffer.alloc(0))).toEqual({+ records: 0,+ requests: [],+ });+ });+});++describe('a large or odd file', () => {+ it("doesn't read a file past its line or size limit", async () => {+ const lines = Array.from({ length: MAX_LOG_LINES + 1 }, () => '{}').join('\n');+ await expect(readLogpushBody(lines)).rejects.toBeInstanceOf(LogTooLargeError);+ // A small gzip that opens past the limit.+ const bomb = gzipSync(Buffer.alloc(MAX_INFLATED_BYTES + 1, 0x20));+ await expect(readSenderBody('cloudflare_logpush', bomb)).rejects.toBeInstanceOf(+ LogTooLargeError,+ );+ });++ it('reads a file at the line limit, a slice at a time', async () => {+ const lines = Array.from({ length: MAX_LOG_LINES }, () => logpushLine()).join('\n');+ const parsed = await readLogpushBody(lines);+ expect(parsed.records).toBe(MAX_LOG_LINES);+ expect(parsed.requests).toHaveLength(MAX_LOG_LINES);+ });++ it('drops a line whose time is unreadable, not the file', async () => {+ const parsed = await readLogpushBody(+ [logpushLine(), logpushLine({ EdgeStartTimestamp: 1e30 })].join('\n'),+ );+ expect(parsed.records).toBe(2);+ expect(parsed.requests).toHaveLength(1);+ const vercel = await readVercelBody(+ JSON.stringify([+ { proxy: { clientIp: '1.2.3.4', userAgent: ['GPTBot'], path: '/', timestamp: 'never' } },+ ]),+ );+ expect(vercel).toEqual({ records: 1, requests: [] });+ });+});diff --git a/server/src/lib/bot-sender-setup.js b/server/src/lib/bot-sender-setup.jsnew file mode 100644index 000000000..9b2a28bfa--- /dev/null+++ b/server/src/lib/bot-sender-setup.js@@ -0,0 +1,59 @@+/**+ * What a person copies into Cloudflare or Vercel to send AI-bot requests+ * (AT-010, spec §6.1 options B and C), built when the connection is made so+ * the secret is in it once:+ *+ * * a Cloudflare Logpush job on the http_requests dataset, filtered at the+ * edge to user agents on the bot list (only matching lines are sent),+ * posting to /bots/ingest with the secret as a header;+ * * a Vercel drain (logs, JSON) to /bots/ingest with the secret as a+ * custom header. Vercel sends every request log; the server filters+ * them, under the raw-record cap.+ *+ * The Worker (option A) is bot-worker-script.js.+ */++import { AI_BOTS } from './ai-bots.js';++/** The http_requests fields bot-log-formats.js reads. */+export const LOGPUSH_FIELDS = [+ 'ClientRequestUserAgent',+ 'ClientIP',+ 'ClientRequestHost',+ 'ClientRequestURI',+ 'EdgeResponseStatus',+ 'EdgeStartTimestamp',+ 'ClientCountry',+];++/** A Logpush job's settings for the Cloudflare API or dashboard. */+export function logpushJob({ endpoint, secret }) {+ return {+ dataset: 'http_requests',+ destination_conf: `${endpoint}?header_Authorization=${encodeURIComponent(`Bearer ${secret}`)}`,+ output_options: { field_names: LOGPUSH_FIELDS, timestamp_format: 'rfc3339' },+ filter: JSON.stringify({+ where: {+ or: AI_BOTS.map((bot) => ({+ key: 'ClientRequestUserAgent',+ operator: 'contains',+ value: bot.userAgent,+ })),+ },+ }),+ // Small files: each upload stays well under the endpoint's limits+ // (bot-log-formats.js: 16 MB inflated, 20,000 lines).+ max_upload_bytes: 5_000_000,+ max_upload_records: 10_000,+ };+}++/** A Vercel drain's settings (Team Settings › Drains › Add Drain › Logs). */+export function vercelDrain({ endpoint, secret }) {+ return {+ endpoint,+ format: 'json',+ headers: [`Authorization: Bearer ${secret}`],+ sources: ['static', 'lambda', 'edge', 'external', 'redirect'],+ };+}diff --git a/server/src/lib/bot-sender-setup.test.js b/server/src/lib/bot-sender-setup.test.jsnew file mode 100644index 000000000..6e9d965ee--- /dev/null+++ b/server/src/lib/bot-sender-setup.test.js@@ -0,0 +1,50 @@+import { describe, expect, it } from 'vitest';+import { AI_BOTS } from './ai-bots.js';+import { LOGPUSH_FIELDS, logpushJob, vercelDrain } from './bot-sender-setup.js';+import { readLogpushBody } from './bot-log-formats.js';++/** What a person copies into Cloudflare or Vercel (AT-010, spec §6.1 options B and C). */++const sender = { endpoint: 'https://api.example/bots/ingest', secret: 'abot_s3cret' };++describe('logpushJob', () => {+ it('filters the http_requests dataset to the bot list and sends the secret as a header', () => {+ const job = logpushJob(sender);+ expect(job.dataset).toBe('http_requests');+ expect(job.destination_conf).toBe(+ 'https://api.example/bots/ingest?header_Authorization=Bearer%20abot_s3cret',+ );+ const filter = JSON.parse(job.filter);+ expect(filter.where.or.map((c) => c.value)).toEqual(AI_BOTS.map((b) => b.userAgent));+ expect(+ filter.where.or.every((c) => c.key === 'ClientRequestUserAgent' && c.operator === 'contains'),+ ).toBe(true);+ // Files stay within what the endpoint reads (bot-log-formats.js).+ expect(job.max_upload_records).toBe(10_000);+ expect(job.max_upload_bytes).toBe(5_000_000);+ });++ it('asks for the fields the server reads', async () => {+ const line = Object.fromEntries(+ LOGPUSH_FIELDS.map((f) => [f, f === 'EdgeResponseStatus' ? 200 : 'x']),+ );+ line.EdgeStartTimestamp = '2026-10-19T09:00:00Z';+ line.ClientIP = '1.2.3.4';+ expect((await readLogpushBody(JSON.stringify(line))).requests[0]).toMatchObject({+ ip: '1.2.3.4',+ status: 200,+ at: '2026-10-19T09:00:00.000Z',+ });+ });+});++describe('vercelDrain', () => {+ it('sends JSON logs with the secret as a custom header', () => {+ expect(vercelDrain(sender)).toEqual({+ endpoint: 'https://api.example/bots/ingest',+ format: 'json',+ headers: ['Authorization: Bearer abot_s3cret'],+ sources: ['static', 'lambda', 'edge', 'external', 'redirect'],+ });+ });+});diff --git a/server/src/lib/plan-guard.js b/server/src/lib/plan-guard.jsindex 8bd62fa93..8b4b9c9c9 100644--- a/server/src/lib/plan-guard.js+++ b/server/src/lib/plan-guard.js@@ -706,6 +706,15 @@ export async function getBotEventCap(orgId) {return plan.limits.maxBotEventsPerMonth ?? 0;}+/**+ * Raw log records a month an organization's Vercel drains and Logpush jobs+ * may send before the server stops reading them (AT-010, spec §6.3).+ */+export async function getBotRawCap(orgId) {+ const plan = await resolveOrgPlanByOrgId(orgId);+ return plan.limits.maxBotRawRecordsPerMonth ?? 0;+}+/*** Express middleware that attaches req.plan from the authenticated user.* Must run after auth middleware (req.user must exist).diff --git a/server/src/routes/bot-ingest.js b/server/src/routes/bot-ingest.jsindex 5419e9a1e..ecf268f29 100644--- a/server/src/routes/bot-ingest.js+++ b/server/src/routes/bot-ingest.js@@ -1,25 +1,80 @@/**- * POST /bots/ingest — AI-bot requests from a brand's Cloudflare Worker- * (AT-010, spec §6.1). Public, outside /api: the sender is the customer's- * edge, authenticated by its connection secret (Authorization: Bearer).- * Body: { requests: [{ userAgent, ip, url, status, at, country }] }.+ * POST /bots/ingest — AI-bot requests from a brand's own edge (AT-010, spec+ * §6.1). Public, outside /api: the sender is the customer's Cloudflare or+ * Vercel, authenticated by its connection secret (Authorization: Bearer).+ * The body is read by the connection's kind (bot-log-formats.js): a Worker's+ * JSON batch, a Logpush file, or a Vercel drain's log entries. Log senders'+ * records count against the month's raw cap before they are read, and a+ * file they send again (after timing out, say) is read once.*/import express, { Router } from 'express';import { PlanLimitError } from '../lib/plan-guard.js';+import {+ MAX_BATCH,+ claimLogFile,+ connectionForSecret,+ countRawRecords,+ ingestBotRequests,+ releaseLogFile,+} from '../lib/bot-ingest.js';+import { LogTooLargeError, readSenderBody } from '../lib/bot-log-formats.js';import { logger } from '../lib/logger.js';import { botIngestLimiter } from '../middleware/rate-limiter.js';-import { connectionForSecret, ingestBotRequests } from '../lib/bot-ingest.js';const router = Router();-router.post('/bots/ingest', botIngestLimiter, express.json({ limit: '1mb' }), async (req, res) => {+// Raw: each kind has its own body, and a Logpush file comes gzipped. A body+// sent with Content-Encoding: gzip is inflated here, within the limit.+const body = express.raw({ type: () => true, limit: '20mb' });++router.post('/bots/ingest', botIngestLimiter, body, async (req, res) => {+ let connection = null;+ // A log file's claim, given back unless the file was read.+ let claim = null;+ let read = false;try {const secret = (req.headers.authorization || '').replace(/^Bearer\s+/i, '');- const connection = await connectionForSecret(secret);+ connection = await connectionForSecret(secret);if (!connection) return res.status(401).json({ error: 'unknown_connection' });- const result = await ingestBotRequests(connection, req.body?.requests);- return res.json(result);++ const worker = connection.kind === 'cloudflare_worker';+ if (!worker) {+ claim = await claimLogFile(connection, req.body);+ // Answered with 200 so the sender stops sending what was read.+ if (!claim) return res.json({ received: 0, verified: 0, stored: 0, duplicate: true });+ }++ let parsed;+ try {+ parsed = await readSenderBody(connection.kind, req.body);+ } catch (err) {+ if (err instanceof LogTooLargeError) {+ logger.warn({ connectionId: connection.id, err }, '[ai-bots] log file too large');+ return res.status(413).json({ error: 'too_large', message: err.message });+ }+ return res.status(400).json({ error: 'unreadable_body' });+ }++ if (!worker) {+ const raw = await countRawRecords(connection, parsed.records);+ if (!raw.allowed) {+ // Answered with 200 so the sender doesn't retry what won't be read.+ read = true;+ return res.json({+ received: parsed.records,+ verified: 0,+ stored: 0,+ ...(raw.locked ? { locked: true } : { rawCapped: true }),+ });+ }+ }++ const result = await ingestBotRequests(connection, parsed.requests, {+ limit: worker ? MAX_BATCH : Infinity,+ });+ read = true;+ return res.json({ ...result, records: parsed.records });} catch (error) {if (error instanceof PlanLimitError) {return res.status(error.statusCode).json({ error: 'plan_limit', message: error.message });@@ -27,6 +82,8 @@ router.post('/bots/ingest', botIngestLimiter, express.json({ limit: '1mb' }), as// Mounted before the request-id middleware: there is no req.log here.logger.error({ err: error }, 'bot ingest error');return res.status(500).json({ error: 'ingest_failed' });+ } finally {+ if (claim && !read) await releaseLogFile(connection, claim);}});diff --git a/server/src/routes/bot-ingest.test.js b/server/src/routes/bot-ingest.test.jsindex 79aed88ad..f179fdfcd 100644--- a/server/src/routes/bot-ingest.test.js+++ b/server/src/routes/bot-ingest.test.js@@ -1,14 +1,27 @@+import { gzipSync } from 'zlib';import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';import express from 'express';-/** POST /bots/ingest (AT-010): a connection's secret, then the batch. */+/**+ * POST /bots/ingest (AT-010): a connection's secret, then its body read by+ * kind — a Worker's batch, a Vercel drain's entries or a Logpush file — with+ * log senders' records counted against the raw cap first.+ */const connectionForSecret = vi.fn();const ingestBotRequests = vi.fn();+const countRawRecords = vi.fn();+const claimLogFile = vi.fn();+const releaseLogFile = vi.fn();vi.mock('../lib/bot-ingest.js', () => ({+ MAX_BATCH: 500,connectionForSecret: (...a) => connectionForSecret(...a),ingestBotRequests: (...a) => ingestBotRequests(...a),+ countRawRecords: (...a) => countRawRecords(...a),+ claimLogFile: (...a) => claimLogFile(...a),+ releaseLogFile: (...a) => releaseLogFile(...a),}));+vi.mock('../lib/logger.js', () => ({ logger: { warn: vi.fn(), error: vi.fn() } }));vi.mock('../lib/plan-guard.js', () => ({PlanLimitError: class PlanLimitError extends Error {constructor(message, statusCode = 403) {@@ -26,7 +39,12 @@ let url;beforeEach(async () => {connectionForSecret.mockReset();- ingestBotRequests.mockReset();+ ingestBotRequests+ .mockReset()+ .mockResolvedValue({ received: 0, verified: 0, stored: 0, skipped: {} });+ countRawRecords.mockReset().mockResolvedValue({ allowed: true });+ claimLogFile.mockReset().mockResolvedValue('hash-1');+ releaseLogFile.mockReset().mockResolvedValue(undefined);const app = express();app.use(router);await new Promise((resolve) => {@@ -47,19 +65,129 @@ const post = (secret, body) =>});describe('POST /bots/ingest', () => {- it('ingests a batch for the connection its secret names', async () => {- const connection = { id: 'c1', brandId: 'b1', organizationId: 'o1' };+ it("ingests a Worker's batch for the connection its secret names", async () => {+ const connection = { id: 'c1', organizationId: 'o1', kind: 'cloudflare_worker' };connectionForSecret.mockResolvedValue(connection);ingestBotRequests.mockResolvedValue({ received: 1, verified: 1, stored: 1, skipped: {} });const res = await post('abot_x', {requests: [{ userAgent: 'GPTBot', ip: '1.2.3.4', url: '/' }],});expect(res.status).toBe(200);- expect(await res.json()).toMatchObject({ stored: 1 });+ expect(await res.json()).toMatchObject({ stored: 1, records: 1 });expect(connectionForSecret).toHaveBeenCalledWith('abot_x');- expect(ingestBotRequests).toHaveBeenCalledWith(connection, [- { userAgent: 'GPTBot', ip: '1.2.3.4', url: '/' },- ]);+ expect(ingestBotRequests).toHaveBeenCalledWith(+ connection,+ [{ userAgent: 'GPTBot', ip: '1.2.3.4', url: '/' }],+ { limit: 500 },+ );+ expect(countRawRecords).not.toHaveBeenCalled();+ });++ it("reads a Vercel drain's entries after counting them against the raw cap", async () => {+ const connection = { id: 'c2', organizationId: 'o1', kind: 'vercel_drain' };+ connectionForSecret.mockResolvedValue(connection);+ const entries = [+ {+ proxy: {+ clientIp: '1.2.3.4',+ userAgent: ['GPTBot'],+ host: 'acme.example',+ path: '/',+ statusCode: 200,+ timestamp: 1792400000000,+ },+ },+ { source: 'build', message: 'done' },+ ];+ const res = await post('abot_x', entries);+ expect(res.status).toBe(200);+ expect((await res.json()).records).toBe(2);+ expect(countRawRecords).toHaveBeenCalledWith(connection, 2);+ expect(ingestBotRequests.mock.calls[0][1]).toHaveLength(1);+ expect(ingestBotRequests.mock.calls[0][2]).toEqual({ limit: Infinity });++ countRawRecords.mockResolvedValue({ allowed: false });+ ingestBotRequests.mockClear();+ const capped = await post('abot_x', entries);+ expect(await capped.json()).toMatchObject({ received: 2, stored: 0, rawCapped: true });+ expect(ingestBotRequests).not.toHaveBeenCalled();++ countRawRecords.mockResolvedValue({ allowed: false, locked: true });+ expect(await (await post('abot_x', entries)).json()).toMatchObject({ locked: true });+ });++ it("opens a Logpush job's gzipped file, and refuses an unreadable one", async () => {+ connectionForSecret.mockResolvedValue({+ id: 'c3',+ organizationId: 'o1',+ kind: 'cloudflare_logpush',+ });+ const ok = await fetch(url, {+ method: 'POST',+ headers: { Authorization: 'Bearer abot_x' },+ body: gzipSync('{"content":"tests"}'),+ });+ expect(ok.status).toBe(200);+ expect(countRawRecords).toHaveBeenCalledWith(expect.objectContaining({ id: 'c3' }), 1);++ const bad = await fetch(url, {+ method: 'POST',+ headers: { Authorization: 'Bearer abot_x' },+ body: Buffer.from([0x1f, 0x8b, 0x00, 0x01, 0x02]),+ });+ expect(bad.status).toBe(400);+ });++ it('reads a log file once: a resent one is answered without reading it', async () => {+ const connection = { id: 'c2', organizationId: 'o1', kind: 'vercel_drain' };+ connectionForSecret.mockResolvedValue(connection);+ const entries = [{ source: 'build', message: 'done' }];+ expect((await post('abot_x', entries)).status).toBe(200);+ expect(claimLogFile).toHaveBeenCalledWith(connection, expect.any(Buffer));+ expect(releaseLogFile).not.toHaveBeenCalled();++ claimLogFile.mockResolvedValue(null);+ countRawRecords.mockClear();+ const again = await post('abot_x', entries);+ expect(again.status).toBe(200);+ expect(await again.json()).toMatchObject({ duplicate: true, stored: 0 });+ expect(countRawRecords).not.toHaveBeenCalled();+ });++ it('gives the claim back when the read fails, so the retry is read', async () => {+ const connection = { id: 'c2', organizationId: 'o1', kind: 'vercel_drain' };+ connectionForSecret.mockResolvedValue(connection);+ ingestBotRequests.mockRejectedValue(new Error('database down'));+ const entries = [+ {+ proxy: {+ clientIp: '1.2.3.4',+ userAgent: ['GPTBot'],+ host: 'acme.example',+ path: '/',+ statusCode: 200,+ timestamp: 1792400000000,+ },+ },+ ];+ expect((await post('abot_x', entries)).status).toBe(500);+ expect(releaseLogFile).toHaveBeenCalledWith(connection, 'hash-1');+ });++ it('answers 413 for a file past the line limit, without reading it', async () => {+ connectionForSecret.mockResolvedValue({+ id: 'c3',+ organizationId: 'o1',+ kind: 'cloudflare_logpush',+ });+ const res = await fetch(url, {+ method: 'POST',+ headers: { Authorization: 'Bearer abot_x' },+ body: gzipSync(Array.from({ length: 20_001 }, () => '{}').join('\n')),+ });+ expect(res.status).toBe(413);+ expect(countRawRecords).not.toHaveBeenCalled();+ expect(releaseLogFile).toHaveBeenCalled();});it('refuses an unknown secret, and an organization without a subscription', async () => {@@ -67,7 +195,7 @@ describe('POST /bots/ingest', () => {expect((await post('abot_bad', { requests: [] })).status).toBe(401);expect(ingestBotRequests).not.toHaveBeenCalled();- connectionForSecret.mockResolvedValue({ id: 'c1' });+ connectionForSecret.mockResolvedValue({ id: 'c1', kind: 'cloudflare_worker' });ingestBotRequests.mockRejectedValue(new PlanLimitError('choose a plan', 402));expect((await post('abot_x', { requests: [] })).status).toBe(402);});diff --git a/server/src/routes/bots.js b/server/src/routes/bots.jsindex 0726ac857..f2b61eab1 100644--- a/server/src/routes/bots.js+++ b/server/src/routes/bots.js@@ -5,7 +5,8 @@* with the bot report (Starter has it locked, spec §7).** GET /api/bots/connections/:brandId — the brand's connections and this month's use- * POST /api/bots/connections/:brandId — { kind } → a connection, its secret, the endpoint, the Worker+ * POST /api/bots/connections/:brandId — { kind } → a connection, its secret, the endpoint and+ * what to copy: the Worker, a Logpush job or a Vercel drain* DELETE /api/bots/connections/:brandId/:id — revoke one*/@@ -21,6 +22,7 @@ import {publicIngestEndpoint,} from '../lib/bot-ingest.js';import { buildWorkerScript } from '../lib/bot-worker-script.js';+import { logpushJob, vercelDrain } from '../lib/bot-sender-setup.js';const router = Router();@@ -116,12 +118,15 @@ router.post('/connections/:brandId', async (req, res) => {.single();if (error) throw new Error(error.message);+ const { kind } = parsed.data;+ // What the person copies into Cloudflare or Vercel, with the secret in it once.return res.status(201).json({connection,secret,endpoint,- workerScript:- parsed.data.kind === 'cloudflare_worker' ? buildWorkerScript({ endpoint, secret }) : null,+ workerScript: kind === 'cloudflare_worker' ? buildWorkerScript({ endpoint, secret }) : null,+ logpushJob: kind === 'cloudflare_logpush' ? logpushJob({ endpoint, secret }) : null,+ vercelDrain: kind === 'vercel_drain' ? vercelDrain({ endpoint, secret }) : null,});} catch (error) {return fail(req, res, error, 'create the AI bot connection');diff --git a/server/src/routes/bots.test.js b/server/src/routes/bots.test.jsindex d4539cefa..4a6f97cd3 100644--- a/server/src/routes/bots.test.js+++ b/server/src/routes/bots.test.js@@ -137,6 +137,11 @@ describe('AI bot connections', () => {const drain = await (await create({ kind: 'vercel_drain' })).json();expect(drain.workerScript).toBeNull();+ expect(drain.vercelDrain.headers).toEqual([`Authorization: Bearer ${drain.secret}`]);++ const logpush = await (await create({ kind: 'cloudflare_logpush' })).json();+ expect(logpush.logpushJob.destination_conf).toContain(encodeURIComponent(logpush.secret));+ expect(logpush.vercelDrain).toBeNull();});it('makes no connection while the server has no public https address', async () => {diff --git a/server/src/server.js b/server/src/server.jsindex 09bae4656..e2193a844 100644--- a/server/src/server.js+++ b/server/src/server.js@@ -57,6 +57,9 @@ const server = http.createServer(app);// payload exactly as sent, and express.json()'s parse/re-serialize would break it.// Mounted first so the global JSON parser skips this path (body already consumed).app.use('/cloro/callback', express.raw({ type: 'application/json', limit: '50mb' }));+// AI-bot senders (AT-010) post a Worker's JSON, a gzipped Logpush file or a+// Vercel drain's logs: read raw, before the JSON and form parsers take them.+app.use('/bots/ingest', express.raw({ type: () => true, limit: '20mb' }));app.use(bodyParser.json({ limit: '50mb' }));app.use(bodyParser.urlencoded({ limit: '50mb', extended: true }));diff --git a/supabase/migrations/00184_ai_bot_raw_records.sql b/supabase/migrations/00184_ai_bot_raw_records.sqlnew file mode 100644index 000000000..26498b1b4--- /dev/null+++ b/supabase/migrations/00184_ai_bot_raw_records.sql@@ -0,0 +1,95 @@+-- ============================================================================+-- 00184: the raw-record cap for log senders (AT-010, part B)+-- ============================================================================+-- Spec §6.1 and §6.3: a Vercel drain sends every request log and the server+-- filters them, so it needs its own cap on what it receives (Growth 20M raw+-- records a month, Enterprise 200M). A Cloudflare Logpush job is filtered to+-- AI bots at the edge, but a job set up without the filter would send every+-- line, so the same cap guards it.+--+-- ai_bot_usage gains the month's raw records and when that cap was reached;+-- ai_bot_ingested_files keeps a retried file from being read twice.+-- ai_bot_count_raw adds a batch's records under the usage row's lock and+-- says whether the batch may be read: yes while the month was under the cap+-- when it arrived, so a month overshoots by at most the batch that crosses+-- it (a drain sends at most 1 MB). After that, batches are only counted.++alter table public.ai_bot_usage+ add column raw_records bigint not null default 0,+ add column raw_capped_at timestamptz;++-- A log file a sender sends again (it timed out waiting, say) is read once:+-- each connection's files by the SHA-256 of their body, claimed before the+-- read and given back when it fails, kept 7 days (the daily prune). No+-- policies: only the service role reads or writes.+create table public.ai_bot_ingested_files (+ connection_id uuid not null references public.ai_bot_connections(id) on delete cascade,+ body_hash text not null,+ received_at timestamptz not null default now(),+ primary key (connection_id, body_hash)+);++create index ai_bot_ingested_files_time_idx on public.ai_bot_ingested_files (received_at);++alter table public.ai_bot_ingested_files enable row level security;++create or replace function public.ai_bot_count_raw(+ p_connection_id uuid,+ p_count bigint,+ p_cap bigint+)+returns jsonb+language plpgsql+volatile+security definer+set search_path to 'public'+as $$+declare+ v_org uuid;+ v_month date := date_trunc('month', now() at time zone 'utc')::date;+ v_raw bigint;+ v_allowed boolean;+begin+ select b.organization_id into v_org+ from public.ai_bot_connections c+ join public.brands b on b.id = c.brand_id+ where c.id = p_connection_id and c.revoked_at is null;+ if v_org is null then+ return jsonb_build_object('error', 'connection');+ end if;++ insert into public.ai_bot_usage (organization_id, month)+ values (v_org, v_month)+ on conflict (organization_id, month) do nothing;++ select u.raw_records into v_raw+ from public.ai_bot_usage u+ where u.organization_id = v_org and u.month = v_month+ for update;++ v_allowed := p_cap < 0 or v_raw < p_cap;++ update public.ai_bot_usage u+ set raw_records = u.raw_records + greatest(p_count, 0),+ raw_capped_at = case+ when p_cap >= 0 and u.raw_capped_at is null and u.raw_records + greatest(p_count, 0) >= p_cap+ then now()+ else u.raw_capped_at+ end+ where u.organization_id = v_org and u.month = v_month;++ return jsonb_build_object(+ 'allowed', v_allowed,+ 'raw', v_raw + greatest(p_count, 0),+ 'cap', p_cap+ );+end;+$$;++revoke all on function public.ai_bot_count_raw(uuid, bigint, bigint) from public, anon, authenticated;+grant execute on function public.ai_bot_count_raw(uuid, bigint, bigint) to service_role;++comment on function public.ai_bot_count_raw(uuid, bigint, bigint) is+ 'AI bots (AT-010, spec §6.3): adds a log sender''s batch to the organization''s raw records this month and says whether it may be read (the month was under p_cap when it arrived; <0 = no cap). Locks the usage row. Service role only.';++notify pgrst, 'reload schema';diff --git a/supabase/schema.sql b/supabase/schema.sqlindex fab3c7c22..0daae23f8 100644--- a/supabase/schema.sql+++ b/supabase/schema.sql@@ -25213,3 +25213,102 @@ comment on function public.ai_bot_ingest(uuid, bigint, jsonb) isnotify pgrst, 'reload schema';+-- ─────────────────────────────────────────────────────────────────────────+-- migrations/00184_ai_bot_raw_records.sql+-- ─────────────────────────────────────────────────────────────────────────+-- ============================================================================+-- 00184: the raw-record cap for log senders (AT-010, part B)+-- ============================================================================+-- Spec §6.1 and §6.3: a Vercel drain sends every request log and the server+-- filters them, so it needs its own cap on what it receives (Growth 20M raw+-- records a month, Enterprise 200M). A Cloudflare Logpush job is filtered to+-- AI bots at the edge, but a job set up without the filter would send every+-- line, so the same cap guards it.+--+-- ai_bot_usage gains the month's raw records and when that cap was reached;+-- ai_bot_ingested_files keeps a retried file from being read twice.+-- ai_bot_count_raw adds a batch's records under the usage row's lock and+-- says whether the batch may be read: yes while the month was under the cap+-- when it arrived, so a month overshoots by at most the batch that crosses+-- it (a drain sends at most 1 MB). After that, batches are only counted.++alter table public.ai_bot_usage+ add column raw_records bigint not null default 0,+ add column raw_capped_at timestamptz;++-- A log file a sender sends again (it timed out waiting, say) is read once:+-- each connection's files by the SHA-256 of their body, claimed before the+-- read and given back when it fails, kept 7 days (the daily prune). No+-- policies: only the service role reads or writes.+create table public.ai_bot_ingested_files (+ connection_id uuid not null references public.ai_bot_connections(id) on delete cascade,+ body_hash text not null,+ received_at timestamptz not null default now(),+ primary key (connection_id, body_hash)+);++create index ai_bot_ingested_files_time_idx on public.ai_bot_ingested_files (received_at);++alter table public.ai_bot_ingested_files enable row level security;++create or replace function public.ai_bot_count_raw(+ p_connection_id uuid,+ p_count bigint,+ p_cap bigint+)+returns jsonb+language plpgsql+volatile+security definer+set search_path to 'public'+as $$+declare+ v_org uuid;+ v_month date := date_trunc('month', now() at time zone 'utc')::date;+ v_raw bigint;+ v_allowed boolean;+begin+ select b.organization_id into v_org+ from public.ai_bot_connections c+ join public.brands b on b.id = c.brand_id+ where c.id = p_connection_id and c.revoked_at is null;+ if v_org is null then+ return jsonb_build_object('error', 'connection');+ end if;++ insert into public.ai_bot_usage (organization_id, month)+ values (v_org, v_month)+ on conflict (organization_id, month) do nothing;++ select u.raw_records into v_raw+ from public.ai_bot_usage u+ where u.organization_id = v_org and u.month = v_month+ for update;++ v_allowed := p_cap < 0 or v_raw < p_cap;++ update public.ai_bot_usage u+ set raw_records = u.raw_records + greatest(p_count, 0),+ raw_capped_at = case+ when p_cap >= 0 and u.raw_capped_at is null and u.raw_records + greatest(p_count, 0) >= p_cap+ then now()+ else u.raw_capped_at+ end+ where u.organization_id = v_org and u.month = v_month;++ return jsonb_build_object(+ 'allowed', v_allowed,+ 'raw', v_raw + greatest(p_count, 0),+ 'cap', p_cap+ );+end;+$$;++revoke all on function public.ai_bot_count_raw(uuid, bigint, bigint) from public, anon, authenticated;+grant execute on function public.ai_bot_count_raw(uuid, bigint, bigint) to service_role;++comment on function public.ai_bot_count_raw(uuid, bigint, bigint) is+ 'AI bots (AT-010, spec §6.3): adds a log sender''s batch to the organization''s raw records this month and says whether it may be read (the month was under p_cap when it arrived; <0 = no cap). Locks the usage row. Service role only.';++notify pgrst, 'reload schema';+diff --git a/web/src/config/plans.ts b/web/src/config/plans.tsindex 6a08622c5..101a33851 100644--- a/web/src/config/plans.ts+++ b/web/src/config/plans.ts@@ -78,6 +78,12 @@ export interface PlanLimits {* adjustable per customer via organizations.plan_overrides.*/maxBotEventsPerMonth: number;+ /**+ * Raw log records a month a Vercel drain or Cloudflare Logpush job may send+ * (AT-010, spec §6.3), before the server filters them. Growth 20M,+ * Enterprise 200M; 0 = locked, -1 = no cap.+ */+ maxBotRawRecordsPerMonth: number;maxDailyOnDemand: number;onDemandCooldownMinutes: number;features: readonly Feature[];@@ -120,6 +126,7 @@ export const PLANS: Record<PlanId, Plan> = {maxOpportunitiesPerRequest: 5,maxSiteAudits: -1,maxBotEventsPerMonth: -1,+ maxBotRawRecordsPerMonth: -1,maxDailyOnDemand: -1,onDemandCooldownMinutes: 0,features: [@@ -161,6 +168,7 @@ export const PLANS: Record<PlanId, Plan> = {maxOpportunitiesPerRequest: 3,maxSiteAudits: 100,maxBotEventsPerMonth: 0,+ maxBotRawRecordsPerMonth: 0,maxDailyOnDemand: 3,onDemandCooldownMinutes: 15,allowedScrapers: ['chatgpt-web', 'copilot-web'],@@ -206,6 +214,7 @@ export const PLANS: Record<PlanId, Plan> = {maxOpportunitiesPerRequest: 5,maxSiteAudits: 500,maxBotEventsPerMonth: 2_000_000,+ maxBotRawRecordsPerMonth: 20_000_000,maxDailyOnDemand: 10,onDemandCooldownMinutes: 5,// API-model tracking (Claude) is not part of Growth — scraper engines@@ -252,6 +261,7 @@ export const PLANS: Record<PlanId, Plan> = {maxOpportunitiesPerRequest: 5,maxSiteAudits: -1,maxBotEventsPerMonth: 20_000_000,+ maxBotRawRecordsPerMonth: 200_000_000,maxDailyOnDemand: -1,onDemandCooldownMinutes: 0,// API-model tracking (Claude) is a per-customer opt-in on Enterprise: