Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.
TypeScript aeoai-agentai-searchanswer-engine-optimizationbrand-monitoringchatgptclaudecopilotdockergeminigeogoogle-ai-overviewsgrokmcpnextjsopen-sourceperplexityseosupabasetypescript
feat(traffic): show log senders' records against the raw cap, and why the bot report stops (#1224)
Cihan02463afb5028
diff --git a/server/src/routes/bots.js b/server/src/routes/bots.jsindex c0c7f5e8a..09dd35cb2 100644--- a/server/src/routes/bots.js+++ b/server/src/routes/bots.js@@ -4,7 +4,9 @@* own secret, shown once. Admins and managers make and revoke them, on plans* with the bot report (Starter has it locked, spec §7).*- * GET /api/bots/connections/:brandId — the brand's connections and this month's use+ * GET /api/bots/connections/:brandId — the brand's connections and this month's use:+ * rows against the event cap, log records+ * against the raw cap* POST /api/bots/connections/:brandId — { kind } → a connection, its secret, the endpoint and* what to copy: the Worker, a Logpush job or a Vercel drain* DELETE /api/bots/connections/:brandId/:id — revoke one@@ -14,7 +16,7 @@ import { Router } from 'express';import { z } from 'zod';import supabaseAdmin from '../config/supabase.js';import { assertBrandAccess } from '../lib/access.js';-import { PlanLimitError, getBotEventCap } from '../lib/plan-guard.js';+import { PlanLimitError, getBotEventCap, getBotRawCap } from '../lib/plan-guard.js';import {CONNECTION_KINDS,hashSecret,@@ -56,7 +58,7 @@ router.get('/connections/:brandId', async (req, res) => {try {const { brand } = await assertBrandAccess(req.params.brandId, req.user.id);const month = new Date().toISOString().slice(0, 7) + '-01';- const [{ data: connections, error }, { data: usage }, cap] = await Promise.all([+ const [{ data: connections, error }, { data: usage }, cap, rawCap] = await Promise.all([supabaseAdmin.from('ai_bot_connections').select('id, kind, created_at, last_event_at')@@ -65,11 +67,12 @@ router.get('/connections/:brandId', async (req, res) => {.order('created_at'),supabaseAdmin.from('ai_bot_usage')- .select('stored_events, counted_events, capped_at')+ .select('stored_events, counted_events, capped_at, raw_records, raw_capped_at').eq('organization_id', brand.organization_id).eq('month', month).maybeSingle(),getBotEventCap(brand.organization_id),+ getBotRawCap(brand.organization_id),]);if (error) throw new Error(error.message);return res.json({@@ -80,6 +83,11 @@ router.get('/connections/:brandId', async (req, res) => {counted: usage?.counted_events ?? 0,cappedAt: usage?.capped_at ?? null,cap,+ // A Logpush job's or a Vercel drain's records (00184): past rawCap the+ // files are counted but not read, so the report stops until the month ends.+ raw: usage?.raw_records ?? 0,+ rawCappedAt: usage?.raw_capped_at ?? null,+ rawCap,},});} catch (error) {diff --git a/server/src/routes/bots.test.js b/server/src/routes/bots.test.jsindex 29ccc12ba..388a1ec95 100644--- a/server/src/routes/bots.test.js+++ b/server/src/routes/bots.test.js@@ -31,7 +31,16 @@ function table(name) {};}if (name === 'ai_bot_usage')- return { data: { stored_events: 12, counted_events: 15, capped_at: null }, error: null };+ return {+ data: {+ stored_events: 12,+ counted_events: 15,+ capped_at: null,+ raw_records: 20_000_400,+ raw_capped_at: '2026-10-18T09:00:00Z',+ },+ error: null,+ };return { data: null, error: null };};const chain = new Proxy(@@ -77,6 +86,7 @@ vi.mock('../lib/plan-guard.js', async () => {PlanLimitError,getOrgIdForUser: async () => 'o1',getBotEventCap: async () => cap,+ getBotRawCap: async () => 20_000_000,};});@@ -176,6 +186,12 @@ describe('AI bot connections', () => {{ id: 'c1', kind: 'cloudflare_worker', created_at: 'then', last_event_at: null },]);expect(body.usage).toMatchObject({ stored: 12, counted: 15, cappedAt: null, cap: 2_000_000 });+ // Log records against the raw cap, which stops a log sender's report mid-month.+ expect(body.usage).toMatchObject({+ raw: 20_000_400,+ rawCappedAt: '2026-10-18T09:00:00Z',+ rawCap: 20_000_000,+ });expect(body.usage.month).toMatch(/^\d{4}-\d{2}-01$/);});diff --git a/web/messages/en.json b/web/messages/en.jsonindex 957aa95fb..32d6c6a36 100644--- a/web/messages/en.json+++ b/web/messages/en.json@@ -3550,6 +3550,8 @@"usage": "This month: {stored} of {cap} detailed rows.","usageNoCap": "This month: {stored} detailed {count, plural, one {row} other {rows}}.","paused": "Detailed rows paused until {day}, totals still counted.",+ "rawUsage": "Log records this month: {raw} of {cap}.",+ "logsPaused": "Log files paused until {day}: this month's log records are used up, so requests from Logpush or the Vercel drain aren't read until then.","secretOnce": "Copy these settings now: the secret in them isn't shown again. Lost it? Revoke this connection and create a new one.","copy": "Copy","copied": "Copied",diff --git a/web/src/app/[locale]/(dashboard)/dashboard/traffic/_bots-connect.test.tsx b/web/src/app/[locale]/(dashboard)/dashboard/traffic/_bots-connect.test.tsxindex b0197737c..f292bfa06 100644--- a/web/src/app/[locale]/(dashboard)/dashboard/traffic/_bots-connect.test.tsx+++ b/web/src/app/[locale]/(dashboard)/dashboard/traffic/_bots-connect.test.tsx@@ -32,7 +32,16 @@ const { BotsSource } = await import('./_bots-connect');const info = (over: Partial<BotConnections> = {}): BotConnections => ({connections: [],- usage: { month: '2026-10-01', stored: 0, counted: 0, cappedAt: null, cap: 2_000_000 },+ usage: {+ month: '2026-10-01',+ stored: 0,+ counted: 0,+ cappedAt: null,+ cap: 2_000_000,+ raw: 0,+ rawCappedAt: null,+ rawCap: 20_000_000,+ },...over,});const hoursAgo = (h: number) => new Date(Date.now() - h * 3_600_000).toISOString();@@ -177,6 +186,9 @@ describe('BotsSource', () => {counted: 2_400_000,cappedAt: '2026-10-25T00:00:00Z',cap: 2_000_000,+ raw: 0,+ rawCappedAt: null,+ rawCap: 20_000_000,},}),);@@ -185,6 +197,8 @@ describe('BotsSource', () => {await act(async () => button('Manage')!.click());expect(page()).toContain('This month: 2,000,000 of 2,000,000 detailed rows.');expect(page()).toContain('Detailed rows paused until Nov 1, totals still counted.');+ // A Worker's batches don't count against the log records cap.+ expect(page()).not.toContain('Log records this month');await act(async () => button('Revoke')!.click());await act(async () => button('Revoke: stop reading it')!.click());@@ -192,6 +206,32 @@ describe('BotsSource', () => {expect(toast.success).toHaveBeenCalled();});+ it("shows a log sender's records against the raw cap, and when its files are read again", async () => {+ getBotConnections.mockResolvedValue(+ info({+ connections: [+ { id: 'c1', kind: 'vercel_drain', createdAt: 't', lastEventAt: hoursAgo(30) },+ ],+ usage: {+ month: '2026-10-01',+ stored: 1_000,+ counted: 1_000,+ cappedAt: null,+ cap: 2_000_000,+ raw: 20_000_400,+ rawCappedAt: '2026-10-18T09:00:00Z',+ rawCap: 20_000_000,+ },+ }),+ );+ await render();+ await act(async () => button('Manage')!.click());+ expect(page()).toContain('Log records this month: 20,000,400 of 20,000,000.');+ expect(page()).toContain(+ "Log files paused until Nov 1: this month's log records are used up, so requests from Logpush or the Vercel drain aren't read until then.",+ );+ });+it('waits for the first request, and keeps members from connecting', async () => {getBotConnections.mockResolvedValue(info({diff --git a/web/src/app/[locale]/(dashboard)/dashboard/traffic/_bots-connect.tsx b/web/src/app/[locale]/(dashboard)/dashboard/traffic/_bots-connect.tsxindex 3ddaa7df2..a277d111d 100644--- a/web/src/app/[locale]/(dashboard)/dashboard/traffic/_bots-connect.tsx+++ b/web/src/app/[locale]/(dashboard)/dashboard/traffic/_bots-connect.tsx@@ -17,7 +17,13 @@ import {import { cn } from '@/lib/utils';import { formatNumber } from '@/lib/format';import { formatRelative } from '@/lib/format-relative';-import { botsCardState, lastBotEvent, rowsPausedUntil } from '@/lib/ai-bots';+import {+ botsCardState,+ hasLogSender,+ lastBotEvent,+ logsPausedUntil,+ rowsPausedUntil,+} from '@/lib/ai-bots';import {createBotConnection,getBotConnections,@@ -188,6 +194,13 @@ export function BotsDialog({const { usage, connections } = info;const pausedUntil = rowsPausedUntil(usage);+ const logsPaused = logsPausedUntil(info);+ const shortDay = (day: string) =>+ new Date(`${day}T00:00:00Z`).toLocaleDateString('en-US', {+ month: 'short',+ day: 'numeric',+ timeZone: 'UTC',+ });return (<Dialog open={open} onOpenChange={close}>@@ -265,9 +278,18 @@ export function BotsDialog({stored: formatNumber(usage.stored),cap: formatNumber(usage.cap),})}- {pausedUntil &&- ` ${t('paused', { day: new Date(`${pausedUntil}T00:00:00Z`).toLocaleDateString('en-US', { month: 'short', day: 'numeric', timeZone: 'UTC' }) })}`}+ {pausedUntil && ` ${t('paused', { day: shortDay(pausedUntil) })}`}</p>+ {/* Log senders' records count against their own cap (§6.3). */}+ {hasLogSender(info) && usage.rawCap > 0 && (+ <p className="text-xs text-muted-foreground">+ {t('rawUsage', {+ raw: formatNumber(usage.raw),+ cap: formatNumber(usage.rawCap),+ })}+ {logsPaused && ` ${t('logsPaused', { day: shortDay(logsPaused) })}`}+ </p>+ )}</section>)}diff --git a/web/src/app/[locale]/(dashboard)/dashboard/traffic/_bots-report.test.tsx b/web/src/app/[locale]/(dashboard)/dashboard/traffic/_bots-report.test.tsxindex e8168a81a..12bd51480 100644--- a/web/src/app/[locale]/(dashboard)/dashboard/traffic/_bots-report.test.tsx+++ b/web/src/app/[locale]/(dashboard)/dashboard/traffic/_bots-report.test.tsx@@ -38,7 +38,16 @@ const hoursAgo = (h: number) => new Date(Date.now() - h * 3_600_000).toISOStringconst info = (over: Partial<BotConnections> = {}): BotConnections => ({connections: [{ id: 'c1', kind: 'cloudflare_worker', createdAt: 't', lastEventAt: hoursAgo(1) }],- usage: { month: '2026-10-01', stored: 10, counted: 10, cappedAt: null, cap: 2_000_000 },+ usage: {+ month: '2026-10-01',+ stored: 10,+ counted: 10,+ cappedAt: null,+ cap: 2_000_000,+ raw: 0,+ rawCappedAt: null,+ rawCap: 20_000_000,+ },...over,});const report = (over: Partial<TrafficBots> = {}): TrafficBots => ({@@ -178,6 +187,29 @@ describe('BotsReportSection', () => {expect(text()).not.toContain('The search-bot alert starts on');});+ it("says why the report stops mid-month once a log sender's raw cap is reached", async () => {+ getBotConnections.mockResolvedValue(+ info({+ connections: [+ { id: 'c1', kind: 'cloudflare_logpush', createdAt: 't', lastEventAt: hoursAgo(30) },+ ],+ usage: {+ month: '2026-10-01',+ stored: 10,+ counted: 10,+ cappedAt: null,+ cap: 2_000_000,+ raw: 20_000_400,+ rawCappedAt: '2026-10-18T09:00:00Z',+ rawCap: 20_000_000,+ },+ }),+ );+ await render();+ expect(text()).toContain('Log files paused until');+ expect(text()).toContain("this month's log records are used up");+ });+it('says "last 90 days" for All time, which is what it covers', async () => {await act(async () => {root.render(diff --git a/web/src/app/[locale]/(dashboard)/dashboard/traffic/_bots-report.tsx b/web/src/app/[locale]/(dashboard)/dashboard/traffic/_bots-report.tsxindex 2879f7837..58a0aafc1 100644--- a/web/src/app/[locale]/(dashboard)/dashboard/traffic/_bots-report.tsx+++ b/web/src/app/[locale]/(dashboard)/dashboard/traffic/_bots-report.tsx@@ -21,7 +21,7 @@ import { cn } from '@/lib/utils';import { formatNumber } from '@/lib/format';import { formatRelative } from '@/lib/format-relative';import { botName } from '@/lib/ai-bot-names';-import { botReportState, rowsPausedUntil, sampleBotTypes } from '@/lib/ai-bots';+import { botReportState, logsPausedUntil, rowsPausedUntil, sampleBotTypes } from '@/lib/ai-bots';import { getBotConnections, type BotConnections } from '@/lib/actions/ai-bots';import {getTrafficBots,@@ -257,6 +257,8 @@ export function BotsReportSection({const dialogInfo = info ?? loaded?.info ?? null;const state = info ? botReportState(info, report) : null;const pausedUntil = info ? rowsPausedUntil(info.usage) : null;+ // Past the raw cap log files aren't read: the report stops mid-month.+ const logsPaused = info ? logsPausedUntil(info) : null;const sample = (<div className="relative min-h-[220px]">@@ -357,6 +359,11 @@ export function BotsReportSection({})}</p>)}+ {state === 'data' && logsPaused && (+ <p className="px-6 pb-3 text-xs text-muted-foreground">+ {tBots('logsPaused', { day: formatTrafficBucketDate(logsPaused) })}+ </p>+ )}</CardContent></Card>diff --git a/web/src/lib/actions/ai-bots.test.ts b/web/src/lib/actions/ai-bots.test.tsindex 9955cfc11..1310f95d2 100644--- a/web/src/lib/actions/ai-bots.test.ts+++ b/web/src/lib/actions/ai-bots.test.ts@@ -28,12 +28,30 @@ describe('AI bot connection actions', () => {fetchMock.mockResolvedValue(json({connections: [{ id: 'c1', kind: 'vercel_drain', created_at: 't0', last_event_at: null }],- usage: { month: '2026-10-01', stored: '12', counted: 15, cappedAt: null, cap: 2000000 },+ usage: {+ month: '2026-10-01',+ stored: '12',+ counted: 15,+ cappedAt: null,+ cap: 2000000,+ raw: 20_000_400,+ rawCappedAt: '2026-10-18T09:00:00Z',+ rawCap: 20_000_000,+ },}),);expect(await getBotConnections('b1')).toEqual({connections: [{ id: 'c1', kind: 'vercel_drain', createdAt: 't0', lastEventAt: null }],- usage: { month: '2026-10-01', stored: 12, counted: 15, cappedAt: null, cap: 2000000 },+ usage: {+ month: '2026-10-01',+ stored: 12,+ counted: 15,+ cappedAt: null,+ cap: 2000000,+ raw: 20_000_400,+ rawCappedAt: '2026-10-18T09:00:00Z',+ rawCap: 20_000_000,+ },});expect(fetchMock).toHaveBeenCalledWith('http://api.test/api/bots/connections/b1', {headers: { Authorization: 'Bearer tok' },diff --git a/web/src/lib/actions/ai-bots.ts b/web/src/lib/actions/ai-bots.tsindex dab8f4807..382e48f91 100644--- a/web/src/lib/actions/ai-bots.ts+++ b/web/src/lib/actions/ai-bots.ts@@ -34,6 +34,12 @@ export interface BotUsage {cappedAt: string | null;/** Rows a month: -1 no cap, 0 the report is locked (Starter). */cap: number;+ /** Log records (Logpush, Vercel drain) received this month, read or not. */+ raw: number;+ /** When this month's raw cap was reached: log files aren't read after it. */+ rawCappedAt: string | null;+ /** Log records a month: -1 no cap, 0 the report is locked. */+ rawCap: number;}export interface BotConnections {@@ -105,6 +111,9 @@ export async function getBotConnections(brandId: string): Promise<BotConnectionscounted: n(usage.counted),cappedAt: typeof usage.cappedAt === 'string' ? usage.cappedAt : null,cap: Number(usage.cap ?? 0),+ raw: n(usage.raw),+ rawCappedAt: typeof usage.rawCappedAt === 'string' ? usage.rawCappedAt : null,+ rawCap: Number(usage.rawCap ?? 0),},};}diff --git a/web/src/lib/ai-bots.test.ts b/web/src/lib/ai-bots.test.tsindex bad0c6fcd..8859efd80 100644--- a/web/src/lib/ai-bots.test.ts+++ b/web/src/lib/ai-bots.test.ts@@ -3,6 +3,7 @@ import {botReportState,botsCardState,lastBotEvent,+ logsPausedUntil,rowsPausedUntil,sampleBotTypes,} from './ai-bots';@@ -16,6 +17,9 @@ const usage = (over: Partial<BotUsage> = {}): BotUsage => ({counted: 0,cappedAt: null,cap: 2_000_000,+ raw: 0,+ rawCappedAt: null,+ rawCap: 20_000_000,...over,});const conn = (lastEventAt: string | null, id = 'c1') => ({@@ -56,6 +60,31 @@ describe('lastBotEvent and rowsPausedUntil', () => {expect(rowsPausedUntil(usage())).toBeNull();expect(rowsPausedUntil(usage({ cap: -1, cappedAt: '2026-10-20T00:00:00Z' }))).toBeNull();});++ it('pauses log files past the raw cap, only for a brand with a log sender', () => {+ const drain = { ...conn(null), kind: 'vercel_drain' as const };+ const capped = usage({ raw: 20_000_400, rawCappedAt: '2026-10-18T09:00:00Z' });+ expect(logsPausedUntil({ connections: [drain], usage: capped })).toBe('2026-11-01');+ // A Worker's batches don't count against it.+ expect(logsPausedUntil({ connections: [conn(null)], usage: capped })).toBeNull();+ expect(logsPausedUntil({ connections: [drain], usage: usage() })).toBeNull();+ expect(logsPausedUntil({ connections: [drain], usage: { ...capped, rawCap: -1 } })).toBeNull();+ });++ // gkhngyk on #1224: rawCappedAt is set once a month and never cleared.+ it("reads the pause against today's plan, not when the cap was first hit", () => {+ const drain = { ...conn(null), kind: 'vercel_drain' as const };+ // Growth hit 20M, then upgraded to Enterprise: its files are read again.+ const raised = usage({+ raw: 20_000_400,+ rawCappedAt: '2026-10-18T09:00:00Z',+ rawCap: 200_000_000,+ });+ expect(logsPausedUntil({ connections: [drain], usage: raised })).toBeNull();+ // Enterprise at 30M moved down to Growth: its files stop, with no rawCappedAt yet.+ const lowered = usage({ raw: 30_000_000, rawCappedAt: null, rawCap: 20_000_000 });+ expect(logsPausedUntil({ connections: [drain], usage: lowered })).toBe('2026-11-01');+ });});describe('botReportState and sampleBotTypes (AT-011)', () => {diff --git a/web/src/lib/ai-bots.ts b/web/src/lib/ai-bots.tsindex 6a22f77c4..e3b38f772 100644--- a/web/src/lib/ai-bots.ts+++ b/web/src/lib/ai-bots.ts@@ -28,15 +28,42 @@ export function botsCardState({ connections, usage }: BotConnections): BotsCardSreturn lastBotEvent(connections) ? 'on' : 'waiting';}+/** The first day of the month after `month` ("2026-10-01" → "2026-11-01"); null if malformed. */+function nextMonth(month: string): string | null {+ if (!/^\d{4}-\d{2}-01$/.test(month)) return null;+ const [y, m] = month.split('-').map(Number);+ return new Date(Date.UTC(y, m, 1)).toISOString().slice(0, 10);+}+/*** The day detailed rows start again when this month's cap was reached* ("Detailed rows paused until Nov 1, totals still counted", spec §6.3);* null while under the cap.*/export function rowsPausedUntil(usage: BotUsage): string | null {- if (!usage.cappedAt || usage.cap <= 0 || !/^\d{4}-\d{2}-01$/.test(usage.month)) return null;- const [y, m] = usage.month.split('-').map(Number);- return new Date(Date.UTC(y, m, 1)).toISOString().slice(0, 10);+ if (!usage.cappedAt || usage.cap <= 0) return null;+ return nextMonth(usage.month);+}++/** Connections that send log files, counted against the raw cap (a Worker's batches aren't). */+export function hasLogSender({ connections }: BotConnections): boolean {+ return connections.some((c) => c.kind !== 'cloudflare_worker');+}++/**+ * The day log files are read again when this month's raw cap was reached+ * (§6.3). Past it a Logpush job's or a Vercel drain's files are only+ * counted: no rows and no totals, so the report stops until then. Null+ * while under the cap, or without a log sender.+ *+ * Read against today's plan, as the server decides (raw < cap), not from+ * rawCappedAt: 00184 sets that once a month and never clears it, so a raised+ * plan reads files again under it, and a lowered one stops without it.+ */+export function logsPausedUntil(info: BotConnections): string | null {+ const { usage } = info;+ if (usage.rawCap <= 0 || usage.raw < usage.rawCap || !hasLogSender(info)) return null;+ return nextMonth(usage.month);}/**