Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.
TypeScript aeoai-agentai-searchanswer-engine-optimizationbrand-monitoringchatgptclaudecopilotdockergeminigeogoogle-ai-overviewsgrokmcpnextjsopen-sourceperplexityseosupabasetypescript
fix(traffic): read a bot log file in one transaction, and prune bot rows in batches (#1223)
Cihana2067b4e489f
diff --git a/server/src/lib/bot-ingest.js b/server/src/lib/bot-ingest.jsindex 7a7c47b1a..0decb0263 100644--- a/server/src/lib/bot-ingest.js+++ b/server/src/lib/bot-ingest.js@@ -6,8 +6,10 @@* A batch is verified request by request (user agent on the list, IP in the* operator's published ranges, ai-bots.js), then written in one step that* holds the organization's monthly cap (00183 ai_bot_ingest): rows while- * there is room, bot × page × day counters always. No IP is kept. Crossing- * 80% of the cap raises the internal alert once a month (§6.3 guardrails).+ * there is room, bot × page × day counters always. A log sender's file is+ * claimed, counted against the raw cap and written in that same transaction+ * (00189 ai_bot_ingest_file). No IP is kept. Crossing 80% of the cap raises+ * the internal alert once a month (§6.3 guardrails).*/import { createHash, randomBytes } from 'crypto';@@ -19,9 +21,6 @@ import { logger } from './logger.js';/** The most requests one Worker batch may carry (a Worker sends about 100). */export const MAX_BATCH = 500;-/** Verified events are written this many at a time. */-export const WRITE_CHUNK = 500;-/** Requests older than this are dropped: a sender catching up after a week isn't the site's reading now. */export const MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000;@@ -102,15 +101,9 @@ function readRequest(raw, now) {};}-/**- * Verify and store one batch for a connection. Returns what was received,- * verified and stored, and why the rest wasn't counted. A Worker batch is- * kept to MAX_BATCH; a log file (`limit: Infinity`) is read whole and its- * verified events written WRITE_CHUNK at a time.- */-export async function ingestBotRequests(connection, requests, deps = {}) {+/** The bots among a sender's requests, as events to write, and why the rest were skipped. */+async function verifyRequests(list, deps) {const now = deps.now ?? Date.now();- const list = Array.isArray(requests) ? requests.slice(0, deps.limit ?? MAX_BATCH) : [];const skipped = { invalid: 0, unknown_agent: 0, unverified_ip: 0, no_ranges: 0 };const events = [];for (const raw of list) {@@ -133,6 +126,31 @@ export async function ingestBotRequests(connection, requests, deps = {}) {at: request.at,});}+ return { events, skipped };+}++/** The internal alert, once a month, when a write takes the month past 80% of the cap. */+function alertAt80(connection, data, cap) {+ if (!data?.crossed80) return;+ logger.warn(+ {+ organizationId: connection.organizationId,+ brandId: connection.brandId,+ used: data.used,+ cap,+ },+ '[ai-bots] 80% of the monthly bot event cap reached',+ );+}++/**+ * Verify and store one Worker batch (at most MAX_BATCH requests) for a+ * connection. Returns what was received, verified and stored, and why the+ * rest wasn't counted.+ */+export async function ingestBotRequests(connection, requests, deps = {}) {+ const list = Array.isArray(requests) ? requests.slice(0, MAX_BATCH) : [];+ const { events, skipped } = await verifyRequests(list, deps);let stored = 0;if (events.length > 0) {@@ -140,106 +158,136 @@ export async function ingestBotRequests(connection, requests, deps = {}) {// A plan without the report (Starter) stores nothing, not even counters.if (cap === 0)return { received: list.length, verified: events.length, stored, skipped, locked: true };- for (let i = 0; i < events.length; i += WRITE_CHUNK) {- const { data, error } = await supabaseAdmin.rpc('ai_bot_ingest', {- p_connection_id: connection.id,- p_cap: cap,- p_events: events.slice(i, i + WRITE_CHUNK),- });- if (error) throw new Error(error.message);- if (data?.error) throw new Error(`ai_bot_ingest: ${data.error}`);- stored += data?.stored ?? 0;- if (data?.crossed80) {- logger.warn(- {- organizationId: connection.organizationId,- brandId: connection.brandId,- used: data.used,- cap,- },- '[ai-bots] 80% of the monthly bot event cap reached',- );- }- }+ const { data, error } = await supabaseAdmin.rpc('ai_bot_ingest', {+ p_connection_id: connection.id,+ p_cap: cap,+ p_events: events,+ });+ if (error) throw new Error(error.message);+ if (data?.error) throw new Error(`ai_bot_ingest: ${data.error}`);+ stored = data?.stored ?? 0;+ alertAt80(connection, data, cap);}return { received: list.length, verified: events.length, stored, skipped };}/**- * Claim a log file for reading (a Logpush job's or a Vercel drain's), by- * the SHA-256 of its body per connection: a sender that times out waiting- * sends the same file again, and its chunks already written would count- * twice. Returns the hash, or null when the file was read already. A read- * that fails gives its claim back (releaseLogFile) so the retry is read.+ * A log file's key (a Logpush job's or a Vercel drain's): the SHA-256 of its+ * body. A sender that times out waiting sends the same file again, and the+ * key keeps it from being read twice.*/-export async function claimLogFile(connection, body) {- const hash = createHash('sha256')+export function logFileHash(body) {+ return createHash('sha256').update(Buffer.isBuffer(body) ? body : Buffer.alloc(0)).digest('hex');+}++/**+ * Whether the connection's file was read already, so a resent one is+ * answered before it is opened. Only a shortcut: ai_bot_ingest_file claims+ * the file again in its own transaction.+ */+export async function logFileRead(connection, hash) {const { data, error } = await supabaseAdmin.from('ai_bot_ingested_files')- .upsert(- { connection_id: connection.id, body_hash: hash },- { onConflict: 'connection_id,body_hash', ignoreDuplicates: true },- )- .select('body_hash');+ .select('body_hash')+ .eq('connection_id', connection.id)+ .eq('body_hash', hash)+ .maybeSingle();if (error) throw new Error(error.message);- return (data ?? []).length > 0 ? hash : null;+ return data !== null;}-/** Give a claim back after a read that failed. */-export async function releaseLogFile(connection, hash) {- const { error } = await supabaseAdmin- .from('ai_bot_ingested_files')- .delete()- .eq('connection_id', connection.id)- .eq('body_hash', hash);- if (error)- logger.warn({ err: error, connectionId: connection.id }, '[ai-bots] claim release failed');+/**+ * Whether the organization's month has reached the raw cap already (00184+ * ai_bot_usage.raw_records), so a file is only counted, not verified. The+ * count itself, compared with today's cap: a plan raised mid-month reads+ * again. Only a shortcut: ai_bot_ingest_file decides under the usage lock.+ */+async function rawCapReached(connection, rawCap, now) {+ if (rawCap < 0) return false;+ const month = `${new Date(now).toISOString().slice(0, 7)}-01`;+ const { data, error } = await supabaseAdmin+ .from('ai_bot_usage')+ .select('raw_records')+ .eq('organization_id', connection.organizationId)+ .eq('month', month)+ .maybeSingle();+ if (error) throw new Error(error.message);+ return (data?.raw_records ?? 0) >= rawCap;}/**- * Count a log sender's records against the month's raw cap (a Vercel drain- * or a Logpush job, §6.3). `allowed` says whether this batch may be read; a- * plan without the report reads none.+ * Read a log sender's file (a Logpush job's or a Vercel drain's, §6.3):+ * verify its requests, then claim the file, count its records against the+ * month's raw cap and write its events in one transaction+ * (ai_bot_ingest_file). When that fails, nothing was claimed or counted,+ * so the sender's retry is read in full. Past the raw cap, the records are+ * counted and nothing is verified or written; a plan without the report+ * reads nothing.*/-export async function countRawRecords(connection, records) {- const cap = await getBotRawCap(connection.organizationId);- if (cap === 0) return { allowed: false, locked: true };- const { data, error } = await supabaseAdmin.rpc('ai_bot_count_raw', {+export async function ingestLogFile(connection, hash, parsed, deps = {}) {+ const rawCap = await getBotRawCap(connection.organizationId);+ const cap = await getBotEventCap(connection.organizationId);+ if (rawCap === 0 || cap === 0) {+ return { received: parsed.records, verified: 0, stored: 0, locked: true };+ }+ const capped = await rawCapReached(connection, rawCap, deps.now ?? Date.now());+ const list = !capped && Array.isArray(parsed.requests) ? parsed.requests : [];+ const { events, skipped } = await verifyRequests(list, deps);++ const { data, error } = await supabaseAdmin.rpc('ai_bot_ingest_file', {p_connection_id: connection.id,- p_count: records,+ p_body_hash: hash,+ p_records: parsed.records,+ p_raw_cap: rawCap,p_cap: cap,+ p_events: events,});if (error) throw new Error(error.message);- if (data?.error) throw new Error(`ai_bot_count_raw: ${data.error}`);- return { allowed: data?.allowed === true, raw: data?.raw ?? null, cap };+ if (data?.duplicate) return { received: 0, verified: 0, stored: 0, duplicate: true };+ if (data?.raw_capped) {+ return { received: parsed.records, verified: 0, stored: 0, rawCapped: true };+ }+ alertAt80(connection, data, cap);+ return { received: list.length, verified: events.length, stored: data?.stored ?? 0, skipped };}/** Verified rows are kept 90 days (§6.2); the bot × page × day counters 400. */export const EVENT_RETENTION_DAYS = 90;export const DAILY_RETENTION_DAYS = 400;-/** Log file hashes kept for spotting a sender's retry (claimLogFile). */+/** Log file hashes kept for spotting a sender's retry (ai_bot_ingest_file). */export const FILE_HASH_RETENTION_DAYS = 7;-/** The daily prune of rows past their retention (the daily run). */+/** Rows the prune deletes per table per call, each call its own statement. */+export const PRUNE_BATCH = 5000;+/** Calls per daily run at most (a million rows a table); the rest waits a day. */+export const MAX_PRUNE_CALLS = 200;++/**+ * The daily prune of rows past their retention (the daily run), PRUNE_BATCH+ * rows a table per call (00189 ai_bot_prune), so no one DELETE runs into the+ * statement timeout. Stops when a call deletes fewer than that from every+ * table.+ */export async function pruneBotEvents(now = Date.now()) {const before = (days) => new Date(now - days * 24 * 60 * 60 * 1000).toISOString();- const { error } = await supabaseAdmin- .from('ai_bot_events')- .delete()- .lt('occurred_at', before(EVENT_RETENTION_DAYS));- if (error) logger.warn({ err: error }, '[ai-bots] event prune failed');- const { error: dailyError } = await supabaseAdmin- .from('ai_bot_daily')- .delete()- .lt('day', before(DAILY_RETENTION_DAYS).slice(0, 10));- if (dailyError) logger.warn({ err: dailyError }, '[ai-bots] daily prune failed');- // A sender retries within minutes; a week of file hashes is plenty.- const { error: filesError } = await supabaseAdmin- .from('ai_bot_ingested_files')- .delete()- .lt('received_at', before(FILE_HASH_RETENTION_DAYS));- if (filesError) logger.warn({ err: filesError }, '[ai-bots] file hash prune failed');+ const bounds = {+ p_events_before: before(EVENT_RETENTION_DAYS),+ p_daily_before: before(DAILY_RETENTION_DAYS).slice(0, 10),+ // A sender retries within minutes; a week of file hashes is plenty.+ p_files_before: before(FILE_HASH_RETENTION_DAYS),+ p_limit: PRUNE_BATCH,+ };+ for (let call = 0; call < MAX_PRUNE_CALLS; call++) {+ const { data, error } = await supabaseAdmin.rpc('ai_bot_prune', bounds);+ if (error) {+ logger.warn({ err: error }, '[ai-bots] prune failed');+ return;+ }+ const counts = [data?.events, data?.daily, data?.files].map((n) => n ?? 0);+ if (counts.every((n) => n < PRUNE_BATCH)) return;+ }+ logger.warn({ calls: MAX_PRUNE_CALLS }, '[ai-bots] prune stopped early; the rest goes tomorrow');}diff --git a/server/src/lib/bot-ingest.test.js b/server/src/lib/bot-ingest.test.jsindex dc9fcca1e..7f612c991 100644--- a/server/src/lib/bot-ingest.test.js+++ b/server/src/lib/bot-ingest.test.js@@ -8,18 +8,23 @@ import { beforeEach, describe, expect, it, vi } from 'vitest';const rpc = vi.fn();let connectionRow = null;+// What a single-row read of each table finds.+let rows = {};const selects = [];vi.mock('../config/supabase.js', () => ({default: {rpc: (...args) => rpc(...args),- from: () => {+ from: (table) => {const chain = {select: () => chain,eq: (col, val) => {selects.push([col, val]);return chain;},- maybeSingle: async () => ({ data: connectionRow, error: null }),+ maybeSingle: async () => ({+ data: table === 'ai_bot_connections' ? connectionRow : (rows[table] ?? null),+ error: null,+ }),};return chain;},@@ -37,12 +42,16 @@ vi.mock('./logger.js', () => ({ logger: { warn: (...a) => warn(...a) } }));const { clearRangeCache } = await import('./ai-bots.js');const {MAX_BATCH,- WRITE_CHUNK,+ MAX_PRUNE_CALLS,+ PRUNE_BATCH,connectionForSecret,- countRawRecords,hashSecret,ingestBotRequests,+ ingestLogFile,+ logFileHash,+ logFileRead,newConnectionSecret,+ pruneBotEvents,publicIngestEndpoint,} = await import('./bot-ingest.js');@@ -72,7 +81,9 @@ beforeEach(() => {getBotEventCap.mockReset().mockResolvedValue(2_000_000);getBotRawCap.mockReset().mockResolvedValue(20_000_000);warn.mockReset();+ fetchImpl.mockClear();selects.length = 0;+ rows = {};});describe('ingestBotRequests', () => {@@ -140,20 +151,6 @@ describe('ingestBotRequests', () => {expect(many.received).toBe(MAX_BATCH);});- it('reads a whole log file and writes its verified events in chunks', async () => {- rpc.mockResolvedValue({ data: { stored: 1, used: 1, crossed80: false }, error: null });- const result = await ingestBotRequests(- connection,- Array.from({ length: WRITE_CHUNK + 20 }, () => gpt()),- { fetchImpl, now: NOW, limit: Infinity },- );- expect(result.verified).toBe(WRITE_CHUNK + 20);- expect(rpc).toHaveBeenCalledTimes(2);- expect(rpc.mock.calls[0][1].p_events).toHaveLength(WRITE_CHUNK);- expect(rpc.mock.calls[1][1].p_events).toHaveLength(20);- expect(result.stored).toBe(2);- });-it('writes nothing on a plan without the report', async () => {getBotEventCap.mockResolvedValue(0);const result = await ingestBotRequests(connection, [gpt()], { fetchImpl, now: NOW });@@ -200,32 +197,126 @@ describe('connection secrets', () => {});});-describe('countRawRecords', () => {- it("counts a log sender's records against the plan's raw cap", async () => {- rpc.mockResolvedValue({ data: { allowed: true, raw: 1200, cap: 20_000_000 }, error: null });- expect(await countRawRecords(connection, 1200)).toEqual({- allowed: true,- raw: 1200,- cap: 20_000_000,+describe('ingestLogFile', () => {+ const drain = { ...connection, kind: 'vercel_drain' };+ const file = (requests, records = requests.length) => ({ records, requests });++ it('claims, counts and writes a whole file in one call', async () => {+ rpc.mockResolvedValue({+ data: { received: 520, stored: 520, used: 520, crossed80: false, raw: 600 },+ error: null,});- expect(rpc).toHaveBeenCalledWith('ai_bot_count_raw', {+ const requests = Array.from({ length: 520 }, () => gpt());+ const result = await ingestLogFile(drain, 'h1', file(requests, 600), { fetchImpl, now: NOW });+ expect(result).toMatchObject({ received: 520, verified: 520, stored: 520 });+ expect(rpc).toHaveBeenCalledTimes(1);+ const [name, args] = rpc.mock.calls[0];+ expect(name).toBe('ai_bot_ingest_file');+ expect(args).toMatchObject({p_connection_id: 'c1',- p_count: 1200,- p_cap: 20_000_000,+ p_body_hash: 'h1',+ p_records: 600,+ p_raw_cap: 20_000_000,+ p_cap: 2_000_000,});+ expect(args.p_events).toHaveLength(520);+ });- rpc.mockResolvedValue({- data: { allowed: false, raw: 20_000_100, cap: 20_000_000 },- error: null,+ it('answers a file read meanwhile as a duplicate', async () => {+ rpc.mockResolvedValue({ data: { duplicate: true }, error: null });+ expect(await ingestLogFile(drain, 'h1', file([gpt()]), { fetchImpl, now: NOW })).toEqual({+ received: 0,+ verified: 0,+ stored: 0,+ duplicate: true,});- expect((await countRawRecords(connection, 100)).allowed).toBe(false);+ });++ it('only counts a file past the raw cap, without verifying it', async () => {+ rows.ai_bot_usage = { raw_records: 20_000_000 };+ rpc.mockResolvedValue({ data: { raw_capped: true, raw: 20_000_010 }, error: null });+ const result = await ingestLogFile(drain, 'h1', file([gpt()], 10), { fetchImpl, now: NOW });+ expect(result).toEqual({ received: 10, verified: 0, stored: 0, rawCapped: true });+ expect(rpc.mock.calls[0][1].p_events).toEqual([]);+ expect(fetchImpl).not.toHaveBeenCalled();+ expect(selects).toContainEqual(['month', '2026-10-01']);+ });++ it('reads again once a raised plan lifts the cap above the month so far', async () => {+ rows.ai_bot_usage = { raw_records: 20_000_000 };+ getBotRawCap.mockResolvedValue(200_000_000);+ rpc.mockResolvedValue({ data: { stored: 1, used: 1, raw: 20_000_001 }, error: null });+ await ingestLogFile(drain, 'h1', file([gpt()]), { fetchImpl, now: NOW });+ expect(rpc.mock.calls[0][1].p_events).toHaveLength(1);});it('reads nothing on a plan without the report', async () => {getBotRawCap.mockResolvedValue(0);- expect(await countRawRecords(connection, 10)).toEqual({ allowed: false, locked: true });+ expect(await ingestLogFile(drain, 'h1', file([gpt()]), { fetchImpl, now: NOW })).toEqual({+ received: 1,+ verified: 0,+ stored: 0,+ locked: true,+ });expect(rpc).not.toHaveBeenCalled();});++ it('raises the internal alert, and throws when the write fails', async () => {+ rpc.mockResolvedValue({ data: { stored: 1, used: 1_600_000, crossed80: true }, error: null });+ await ingestLogFile(drain, 'h1', file([gpt()]), { fetchImpl, now: NOW });+ expect(warn).toHaveBeenCalledWith(+ expect.objectContaining({ used: 1_600_000, cap: 2_000_000 }),+ '[ai-bots] 80% of the monthly bot event cap reached',+ );++ rpc.mockResolvedValue({ data: null, error: { message: 'boom' } });+ await expect(+ ingestLogFile(drain, 'h1', file([gpt()]), { fetchImpl, now: NOW }),+ ).rejects.toThrow('boom');+ });+});++describe('log file keys', () => {+ it("keys a file by its body's hash, and finds one read already", async () => {+ expect(logFileHash(Buffer.from('a'))).toBe(logFileHash(Buffer.from('a')));+ expect(logFileHash(Buffer.from('a'))).not.toBe(logFileHash(Buffer.from('b')));+ expect(await logFileRead(connection, 'h1')).toBe(false);+ rows.ai_bot_ingested_files = { body_hash: 'h1' };+ expect(await logFileRead(connection, 'h1')).toBe(true);+ expect(selects).toContainEqual(['body_hash', 'h1']);+ });+});++describe('pruneBotEvents', () => {+ it('deletes in batches until a call deletes fewer than a batch from every table', async () => {+ const full = { events: PRUNE_BATCH, daily: 10, files: 0 };+ rpc+ .mockResolvedValueOnce({ data: full, error: null })+ .mockResolvedValueOnce({ data: full, error: null })+ .mockResolvedValueOnce({ data: { events: 3, daily: 0, files: 0 }, error: null });+ await pruneBotEvents(NOW);+ expect(rpc).toHaveBeenCalledTimes(3);+ expect(rpc).toHaveBeenCalledWith('ai_bot_prune', {+ p_events_before: '2026-07-21T10:00:00.000Z',+ p_daily_before: '2025-09-14',+ p_files_before: '2026-10-12T10:00:00.000Z',+ p_limit: PRUNE_BATCH,+ });+ });++ it('stops after its call budget, and on an error', async () => {+ rpc.mockResolvedValue({ data: { events: 0, daily: PRUNE_BATCH, files: 0 }, error: null });+ await pruneBotEvents(NOW);+ expect(rpc).toHaveBeenCalledTimes(MAX_PRUNE_CALLS);+ expect(warn).toHaveBeenCalledWith(+ { calls: MAX_PRUNE_CALLS },+ '[ai-bots] prune stopped early; the rest goes tomorrow',+ );++ rpc.mockReset().mockResolvedValue({ data: null, error: { message: 'timeout' } });+ await pruneBotEvents(NOW);+ expect(rpc).toHaveBeenCalledTimes(1);+ });});describe('publicIngestEndpoint', () => {diff --git a/server/src/routes/bot-ingest.js b/server/src/routes/bot-ingest.jsindex 80cd98021..ff43e6660 100644--- a/server/src/routes/bot-ingest.js+++ b/server/src/routes/bot-ingest.js@@ -5,20 +5,20 @@* before its body is read, so a caller without one can't make the server* take in 20 MB. The body is read by the connection's kind* (bot-log-formats.js): a Worker's JSON batch, a Logpush file, or a Vercel- * drain's log entries. Log senders' records count against the month's raw- * cap before they are read, and a file they send again (after timing out,- * say) is read once.+ * drain's log entries. A log sender's file is claimed, counted against the+ * month's raw cap and written in one transaction, so a file it sends again+ * (after timing out, say) is read once, and a read that fails leaves nothing+ * for the retry to count twice.*/import express, { Router } from 'express';import { PlanLimitError } from '../lib/plan-guard.js';import {- MAX_BATCH,- claimLogFile,connectionForSecret,- countRawRecords,ingestBotRequests,- releaseLogFile,+ ingestLogFile,+ logFileHash,+ logFileRead,} from '../lib/bot-ingest.js';import { LogTooLargeError, readSenderBody } from '../lib/bot-log-formats.js';import { logger } from '../lib/logger.js';@@ -50,16 +50,12 @@ async function sender(req, res, next) {router.post('/bots/ingest', botIngestLimiter, sender, body, async (req, res) => {const { connection } = res.locals;- // A log file's claim, given back unless the file was read.- let claim = null;- let read = false;try {const worker = connection.kind === 'cloudflare_worker';- if (!worker) {- claim = await claimLogFile(connection, req.body);- // Answered with 200 so the sender stops sending what was read.- if (!claim) return res.json({ received: 0, verified: 0, stored: 0, duplicate: true });- }+ const hash = worker ? null : logFileHash(req.body);+ // Answered with 200 so the sender stops sending what was read.+ const duplicate = { received: 0, verified: 0, stored: 0, duplicate: true };+ if (!worker && (await logFileRead(connection, hash))) return res.json(duplicate);let parsed;try {@@ -72,24 +68,11 @@ router.post('/bots/ingest', botIngestLimiter, sender, body, async (req, res) =>return res.status(400).json({ error: 'unreadable_body' });}- if (!worker) {- const raw = await countRawRecords(connection, parsed.records);- if (!raw.allowed) {- // Answered with 200 so the sender doesn't retry what won't be read.- read = true;- return res.json({- received: parsed.records,- verified: 0,- stored: 0,- ...(raw.locked ? { locked: true } : { rawCapped: true }),- });- }- }-- const result = await ingestBotRequests(connection, parsed.requests, {- limit: worker ? MAX_BATCH : Infinity,- });- read = true;+ // A capped or locked file is answered with 200 too, so the sender doesn't+ // retry what won't be read.+ const result = worker+ ? await ingestBotRequests(connection, parsed.requests)+ : await ingestLogFile(connection, hash, parsed);return res.json({ ...result, records: parsed.records });} catch (error) {if (error instanceof PlanLimitError) {@@ -98,8 +81,6 @@ router.post('/bots/ingest', botIngestLimiter, sender, body, async (req, res) =>// Mounted before the request-id middleware: there is no req.log here.logger.error({ err: error }, 'bot ingest error');return res.status(500).json({ error: 'ingest_failed' });- } finally {- if (claim && !read) await releaseLogFile(connection, claim);}});diff --git a/server/src/routes/bot-ingest.test.js b/server/src/routes/bot-ingest.test.jsindex 541a5a576..3a5058fcf 100644--- a/server/src/routes/bot-ingest.test.js+++ b/server/src/routes/bot-ingest.test.js@@ -5,21 +5,19 @@ import express from 'express';/*** POST /bots/ingest (AT-010): a connection's secret, then its body read by* kind — a Worker's batch, a Vercel drain's entries or a Logpush file — with- * log senders' records counted against the raw cap first.+ * a log sender's file written in one step (claim, raw count, events).*/const connectionForSecret = vi.fn();const ingestBotRequests = vi.fn();-const countRawRecords = vi.fn();-const claimLogFile = vi.fn();-const releaseLogFile = vi.fn();+const ingestLogFile = vi.fn();+const logFileRead = vi.fn();vi.mock('../lib/bot-ingest.js', () => ({- MAX_BATCH: 500,connectionForSecret: (...a) => connectionForSecret(...a),ingestBotRequests: (...a) => ingestBotRequests(...a),- countRawRecords: (...a) => countRawRecords(...a),- claimLogFile: (...a) => claimLogFile(...a),- releaseLogFile: (...a) => releaseLogFile(...a),+ ingestLogFile: (...a) => ingestLogFile(...a),+ logFileHash: () => 'hash-1',+ logFileRead: (...a) => logFileRead(...a),}));vi.mock('../lib/logger.js', () => ({ logger: { warn: vi.fn(), error: vi.fn() } }));vi.mock('../lib/plan-guard.js', () => ({@@ -45,9 +43,8 @@ beforeEach(async () => {ingestBotRequests.mockReset().mockResolvedValue({ received: 0, verified: 0, stored: 0, skipped: {} });- countRawRecords.mockReset().mockResolvedValue({ allowed: true });- claimLogFile.mockReset().mockResolvedValue('hash-1');- releaseLogFile.mockReset().mockResolvedValue(undefined);+ ingestLogFile.mockReset().mockResolvedValue({ received: 0, verified: 0, stored: 0, skipped: {} });+ logFileRead.mockReset().mockResolvedValue(false);const app = express();app.use(router);await new Promise((resolve) => {@@ -78,15 +75,14 @@ describe('POST /bots/ingest', () => {expect(res.status).toBe(200);expect(await res.json()).toMatchObject({ stored: 1, records: 1 });expect(connectionForSecret).toHaveBeenCalledWith('abot_x');- expect(ingestBotRequests).toHaveBeenCalledWith(- connection,- [{ userAgent: 'GPTBot', ip: '1.2.3.4', url: '/' }],- { limit: 500 },- );- expect(countRawRecords).not.toHaveBeenCalled();+ expect(ingestBotRequests).toHaveBeenCalledWith(connection, [+ { userAgent: 'GPTBot', ip: '1.2.3.4', url: '/' },+ ]);+ expect(ingestLogFile).not.toHaveBeenCalled();+ expect(logFileRead).not.toHaveBeenCalled();});- it("reads a Vercel drain's entries after counting them against the raw cap", async () => {+ it("reads a Vercel drain's entries as one file, keyed by its body", async () => {const connection = { id: 'c2', organizationId: 'o1', kind: 'vercel_drain' };connectionForSecret.mockResolvedValue(connection);const entries = [@@ -105,18 +101,18 @@ describe('POST /bots/ingest', () => {const res = await post('abot_x', entries);expect(res.status).toBe(200);expect((await res.json()).records).toBe(2);- expect(countRawRecords).toHaveBeenCalledWith(connection, 2);- expect(ingestBotRequests.mock.calls[0][1]).toHaveLength(1);- expect(ingestBotRequests.mock.calls[0][2]).toEqual({ limit: Infinity });+ const [conn, hash, parsed] = ingestLogFile.mock.calls[0];+ expect(conn).toBe(connection);+ expect(hash).toBe('hash-1');+ expect(parsed.records).toBe(2);+ expect(parsed.requests).toHaveLength(1);+ expect(ingestBotRequests).not.toHaveBeenCalled();- countRawRecords.mockResolvedValue({ allowed: false });- ingestBotRequests.mockClear();+ // A capped or locked file is answered with 200, so the sender doesn't retry it.+ ingestLogFile.mockResolvedValue({ received: 2, verified: 0, stored: 0, rawCapped: true });const capped = await post('abot_x', entries);+ expect(capped.status).toBe(200);expect(await capped.json()).toMatchObject({ received: 2, stored: 0, rawCapped: true });- expect(ingestBotRequests).not.toHaveBeenCalled();-- countRawRecords.mockResolvedValue({ allowed: false, locked: true });- expect(await (await post('abot_x', entries)).json()).toMatchObject({ locked: true });});it("opens a Logpush job's gzipped file, and refuses an unreadable one", async () => {@@ -131,7 +127,7 @@ describe('POST /bots/ingest', () => {body: gzipSync('{"content":"tests"}'),});expect(ok.status).toBe(200);- expect(countRawRecords).toHaveBeenCalledWith(expect.objectContaining({ id: 'c3' }), 1);+ expect(ingestLogFile.mock.calls[0][2].records).toBe(1);const bad = await fetch(url, {method: 'POST',@@ -141,26 +137,22 @@ describe('POST /bots/ingest', () => {expect(bad.status).toBe(400);});- it('reads a log file once: a resent one is answered without reading it', async () => {+ it('reads a log file once: a resent one is answered without opening it', async () => {const connection = { id: 'c2', organizationId: 'o1', kind: 'vercel_drain' };connectionForSecret.mockResolvedValue(connection);const entries = [{ source: 'build', message: 'done' }];- expect((await post('abot_x', entries)).status).toBe(200);- expect(claimLogFile).toHaveBeenCalledWith(connection, expect.any(Buffer));- expect(releaseLogFile).not.toHaveBeenCalled();-- claimLogFile.mockResolvedValue(null);- countRawRecords.mockClear();+ logFileRead.mockResolvedValue(true);const again = await post('abot_x', entries);expect(again.status).toBe(200);expect(await again.json()).toMatchObject({ duplicate: true, stored: 0 });- expect(countRawRecords).not.toHaveBeenCalled();+ expect(logFileRead).toHaveBeenCalledWith(connection, 'hash-1');+ expect(ingestLogFile).not.toHaveBeenCalled();});- it('gives the claim back when the read fails, so the retry is read', async () => {+ it('answers 500 when the write fails, leaving nothing for the retry to count twice', async () => {const connection = { id: 'c2', organizationId: 'o1', kind: 'vercel_drain' };connectionForSecret.mockResolvedValue(connection);- ingestBotRequests.mockRejectedValue(new Error('database down'));+ ingestLogFile.mockRejectedValue(new Error('database down'));const entries = [{proxy: {@@ -174,7 +166,8 @@ describe('POST /bots/ingest', () => {},];expect((await post('abot_x', entries)).status).toBe(500);- expect(releaseLogFile).toHaveBeenCalledWith(connection, 'hash-1');+ // The claim, count and events were one transaction: nothing to give back.+ expect(ingestLogFile).toHaveBeenCalledTimes(1);});it('answers 413 for a file past the line limit, without reading it', async () => {@@ -189,8 +182,7 @@ describe('POST /bots/ingest', () => {body: gzipSync(Array.from({ length: 20_001 }, () => '{}').join('\n')),});expect(res.status).toBe(413);- expect(countRawRecords).not.toHaveBeenCalled();- expect(releaseLogFile).toHaveBeenCalled();+ expect(ingestLogFile).not.toHaveBeenCalled();});it('refuses an unknown secret, and an organization without a subscription', async () => {@@ -215,7 +207,7 @@ describe('POST /bots/ingest', () => {body: Buffer.alloc(21 * 1024 * 1024),});expect(res.status).toBe(401);- expect(claimLogFile).not.toHaveBeenCalled();+ expect(logFileRead).not.toHaveBeenCalled();});it("gives a matched secret its connection's bucket for the next requests", async () => {diff --git a/supabase/migrations/00189_ai_bot_ingest_file.sql b/supabase/migrations/00189_ai_bot_ingest_file.sqlnew file mode 100644index 000000000..8103b16f9--- /dev/null+++ b/supabase/migrations/00189_ai_bot_ingest_file.sql@@ -0,0 +1,133 @@+-- ============================================================================+-- 00189: a bot log file read in one transaction, and bot rows pruned in batches+-- ============================================================================+-- Follow-ups from #1179's review (AT-010).+--+-- A Logpush file or a Vercel drain's delivery was read in steps, each its own+-- transaction: its claim (ai_bot_ingested_files), its raw records+-- (ai_bot_count_raw), then its verified events 500 at a time (ai_bot_ingest).+-- When a later step failed, the claim was given back, and the sender's retry+-- counted the raw records, and the chunks already written, a second time.+--+-- ai_bot_ingest_file takes all three steps in one transaction: a failure+-- leaves nothing behind, so the retry is read as new. A delivery of the same+-- file that arrives meanwhile waits on the claim's key, then reads as a+-- duplicate. A file holds at most 20,000 lines (bot-log-formats.js), so its+-- events fit one call.+--+-- ai_bot_prune deletes at most p_limit rows past their retention from each+-- bot table per call. The daily run calls it until a call deletes fewer, so+-- no single DELETE runs into the statement timeout at Enterprise volume.++create or replace function public.ai_bot_ingest_file(+ p_connection_id uuid,+ p_body_hash text,+ p_records bigint,+ p_raw_cap bigint,+ p_cap bigint,+ p_events jsonb+)+returns jsonb+language plpgsql+volatile+security definer+set search_path to 'public'+as $$+declare+ v_claimed integer;+ v_raw jsonb;+ v_ingest jsonb;+begin+ insert into public.ai_bot_ingested_files (connection_id, body_hash)+ values (p_connection_id, p_body_hash)+ on conflict (connection_id, body_hash) do nothing;+ get diagnostics v_claimed = row_count;+ if v_claimed = 0 then+ return jsonb_build_object('duplicate', true);+ end if;++ -- Raised rather than returned, so the claim rolls back with the rest.+ v_raw := public.ai_bot_count_raw(p_connection_id, p_records, p_raw_cap);+ if v_raw ? 'error' then+ raise exception 'ai_bot_count_raw: %', v_raw ->> 'error';+ end if;+ -- Past the month's raw cap the file is only counted.+ if not (v_raw ->> 'allowed')::boolean then+ return jsonb_build_object('raw_capped', true, 'raw', v_raw -> 'raw');+ end if;+ if coalesce(jsonb_array_length(p_events), 0) = 0 then+ return jsonb_build_object('received', 0, 'stored', 0, 'raw', v_raw -> 'raw');+ end if;++ v_ingest := public.ai_bot_ingest(p_connection_id, p_cap, p_events);+ if v_ingest ? 'error' then+ raise exception 'ai_bot_ingest: %', v_ingest ->> 'error';+ end if;+ return v_ingest || jsonb_build_object('raw', v_raw -> 'raw');+end;+$$;++revoke all on function public.ai_bot_ingest_file(uuid, text, bigint, bigint, bigint, jsonb)+ from public, anon, authenticated;+grant execute on function public.ai_bot_ingest_file(uuid, text, bigint, bigint, bigint, jsonb)+ to service_role;++comment on function public.ai_bot_ingest_file(uuid, text, bigint, bigint, bigint, jsonb) is+ 'AI bots (AT-010): one log file of a connection in one transaction: claims it by its body hash (a duplicate returns {duplicate}), adds its records to the month''s raw count (past p_raw_cap returns {raw_capped}), then writes its verified events as ai_bot_ingest does. A failure rolls the claim back too. Service role only.';++create or replace function public.ai_bot_prune(+ p_events_before timestamptz,+ p_daily_before date,+ p_files_before timestamptz,+ p_limit integer+)+returns jsonb+language plpgsql+volatile+security definer+set search_path to 'public'+as $$+declare+ v_events integer;+ v_daily integer;+ v_files integer;+begin+ delete from public.ai_bot_events e+ where e.id in (+ select o.id from public.ai_bot_events o+ where o.occurred_at < p_events_before+ limit p_limit+ );+ get diagnostics v_events = row_count;++ delete from public.ai_bot_daily d+ using (+ select o.brand_id, o.day, o.bot, o.page from public.ai_bot_daily o+ where o.day < p_daily_before+ limit p_limit+ ) old+ where d.brand_id = old.brand_id and d.day = old.day and d.bot = old.bot and d.page = old.page;+ get diagnostics v_daily = row_count;++ delete from public.ai_bot_ingested_files f+ using (+ select o.connection_id, o.body_hash from public.ai_bot_ingested_files o+ where o.received_at < p_files_before+ limit p_limit+ ) old+ where f.connection_id = old.connection_id and f.body_hash = old.body_hash;+ get diagnostics v_files = row_count;++ return jsonb_build_object('events', v_events, 'daily', v_daily, 'files', v_files);+end;+$$;++revoke all on function public.ai_bot_prune(timestamptz, date, timestamptz, integer)+ from public, anon, authenticated;+grant execute on function public.ai_bot_prune(timestamptz, date, timestamptz, integer)+ to service_role;++comment on function public.ai_bot_prune(timestamptz, date, timestamptz, integer) is+ 'AI bots (AT-010): deletes at most p_limit rows from each of ai_bot_events (occurred_at), ai_bot_daily (day) and ai_bot_ingested_files (received_at) older than the given bounds, and returns how many it deleted from each. The daily run calls it until a call deletes fewer than p_limit. Service role only.';++notify pgrst, 'reload schema';diff --git a/supabase/schema.sql b/supabase/schema.sqlindex 0b034329b..6f4a42787 100644--- a/supabase/schema.sql+++ b/supabase/schema.sql@@ -26204,3 +26204,140 @@ create policy "ai_traffic_digests: member select" on public.ai_traffic_digestscomment on table public.ai_traffic_digests is'AI Traffic weekly digests sent (AT-013): one per brand and week, claimed before sending so a week is never mailed twice.';+-- ─────────────────────────────────────────────────────────────────────────+-- migrations/00189_ai_bot_ingest_file.sql+-- ─────────────────────────────────────────────────────────────────────────+-- ============================================================================+-- 00189: a bot log file read in one transaction, and bot rows pruned in batches+-- ============================================================================+-- Follow-ups from #1179's review (AT-010).+--+-- A Logpush file or a Vercel drain's delivery was read in steps, each its own+-- transaction: its claim (ai_bot_ingested_files), its raw records+-- (ai_bot_count_raw), then its verified events 500 at a time (ai_bot_ingest).+-- When a later step failed, the claim was given back, and the sender's retry+-- counted the raw records, and the chunks already written, a second time.+--+-- ai_bot_ingest_file takes all three steps in one transaction: a failure+-- leaves nothing behind, so the retry is read as new. A delivery of the same+-- file that arrives meanwhile waits on the claim's key, then reads as a+-- duplicate. A file holds at most 20,000 lines (bot-log-formats.js), so its+-- events fit one call.+--+-- ai_bot_prune deletes at most p_limit rows past their retention from each+-- bot table per call. The daily run calls it until a call deletes fewer, so+-- no single DELETE runs into the statement timeout at Enterprise volume.++create or replace function public.ai_bot_ingest_file(+ p_connection_id uuid,+ p_body_hash text,+ p_records bigint,+ p_raw_cap bigint,+ p_cap bigint,+ p_events jsonb+)+returns jsonb+language plpgsql+volatile+security definer+set search_path to 'public'+as $$+declare+ v_claimed integer;+ v_raw jsonb;+ v_ingest jsonb;+begin+ insert into public.ai_bot_ingested_files (connection_id, body_hash)+ values (p_connection_id, p_body_hash)+ on conflict (connection_id, body_hash) do nothing;+ get diagnostics v_claimed = row_count;+ if v_claimed = 0 then+ return jsonb_build_object('duplicate', true);+ end if;++ -- Raised rather than returned, so the claim rolls back with the rest.+ v_raw := public.ai_bot_count_raw(p_connection_id, p_records, p_raw_cap);+ if v_raw ? 'error' then+ raise exception 'ai_bot_count_raw: %', v_raw ->> 'error';+ end if;+ -- Past the month's raw cap the file is only counted.+ if not (v_raw ->> 'allowed')::boolean then+ return jsonb_build_object('raw_capped', true, 'raw', v_raw -> 'raw');+ end if;+ if coalesce(jsonb_array_length(p_events), 0) = 0 then+ return jsonb_build_object('received', 0, 'stored', 0, 'raw', v_raw -> 'raw');+ end if;++ v_ingest := public.ai_bot_ingest(p_connection_id, p_cap, p_events);+ if v_ingest ? 'error' then+ raise exception 'ai_bot_ingest: %', v_ingest ->> 'error';+ end if;+ return v_ingest || jsonb_build_object('raw', v_raw -> 'raw');+end;+$$;++revoke all on function public.ai_bot_ingest_file(uuid, text, bigint, bigint, bigint, jsonb)+ from public, anon, authenticated;+grant execute on function public.ai_bot_ingest_file(uuid, text, bigint, bigint, bigint, jsonb)+ to service_role;++comment on function public.ai_bot_ingest_file(uuid, text, bigint, bigint, bigint, jsonb) is+ 'AI bots (AT-010): one log file of a connection in one transaction: claims it by its body hash (a duplicate returns {duplicate}), adds its records to the month''s raw count (past p_raw_cap returns {raw_capped}), then writes its verified events as ai_bot_ingest does. A failure rolls the claim back too. Service role only.';++create or replace function public.ai_bot_prune(+ p_events_before timestamptz,+ p_daily_before date,+ p_files_before timestamptz,+ p_limit integer+)+returns jsonb+language plpgsql+volatile+security definer+set search_path to 'public'+as $$+declare+ v_events integer;+ v_daily integer;+ v_files integer;+begin+ delete from public.ai_bot_events e+ where e.id in (+ select o.id from public.ai_bot_events o+ where o.occurred_at < p_events_before+ limit p_limit+ );+ get diagnostics v_events = row_count;++ delete from public.ai_bot_daily d+ using (+ select o.brand_id, o.day, o.bot, o.page from public.ai_bot_daily o+ where o.day < p_daily_before+ limit p_limit+ ) old+ where d.brand_id = old.brand_id and d.day = old.day and d.bot = old.bot and d.page = old.page;+ get diagnostics v_daily = row_count;++ delete from public.ai_bot_ingested_files f+ using (+ select o.connection_id, o.body_hash from public.ai_bot_ingested_files o+ where o.received_at < p_files_before+ limit p_limit+ ) old+ where f.connection_id = old.connection_id and f.body_hash = old.body_hash;+ get diagnostics v_files = row_count;++ return jsonb_build_object('events', v_events, 'daily', v_daily, 'files', v_files);+end;+$$;++revoke all on function public.ai_bot_prune(timestamptz, date, timestamptz, integer)+ from public, anon, authenticated;+grant execute on function public.ai_bot_prune(timestamptz, date, timestamptz, integer)+ to service_role;++comment on function public.ai_bot_prune(timestamptz, date, timestamptz, integer) is+ 'AI bots (AT-010): deletes at most p_limit rows from each of ai_bot_events (occurred_at), ai_bot_daily (day) and ai_bot_ingested_files (received_at) older than the given bounds, and returns how many it deleted from each. The daily run calls it until a call deletes fewer than p_limit. Service role only.';++notify pgrst, 'reload schema';+