Skip to content

Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.

fix(traffic): check a bot sender's secret before reading its body, and rate-limit unknown secrets by IP (#1222)

Cihana661f62c5b6f
diff --git a/server/src/middleware/rate-limiter.js b/server/src/middleware/rate-limiter.js
index 90f1d85d8..ab0e6b143 100644
--- a/server/src/middleware/rate-limiter.js
+++ b/server/src/middleware/rate-limiter.js
@@ -1,6 +1,5 @@
-import { createHash } from 'node:crypto';
import rateLimit from 'express-rate-limit';
-import { verifiedUserId } from './verified-tokens.js';
+import { verifiedBotConnection, verifiedUserId } from './verified-tokens.js';
/**
* Anonymous bucket key. IPv4 (including v4-mapped IPv6) keys on the exact
@@ -79,14 +78,17 @@ export const trafficLimiter = rateLimit({
});
/**
- * AI-bot ingest key (AT-010): the connection, by its secret's hash, since a
- * Cloudflare Worker's requests leave from addresses Cloudflare shares across
- * its customers. Without a secret, the caller's IP.
+ * AI-bot ingest key (AT-010): the connection, since a Cloudflare Worker's
+ * requests leave from addresses Cloudflare shares across its customers. Only
+ * a secret the route has already matched to a live connection
+ * (verified-tokens.js) earns that bucket: an unknown one, or none, counts
+ * against the caller's IP, so a flood of made-up secrets can't take a fresh
+ * bucket, and a lookup, per request.
*/
export function botIngestKey(req) {
- const secret = (req.headers.authorization || '').replace(/^Bearer\s+/i, '');
- if (!secret) return anonymousIpKey(req.ip);
- return 'bot:' + createHash('sha256').update(secret).digest('hex');
+ const connectionId = verifiedBotConnection(req.headers.authorization);
+ if (connectionId) return 'bot:' + connectionId;
+ return anonymousIpKey(req.ip);
}
// AI-bot ingest: 600 requests a minute per connection. A Worker isolate
diff --git a/server/src/middleware/rate-limiter.test.js b/server/src/middleware/rate-limiter.test.js
index eac7f84e5..66764225e 100644
--- a/server/src/middleware/rate-limiter.test.js
+++ b/server/src/middleware/rate-limiter.test.js
@@ -1,6 +1,10 @@
import { describe, it, expect, beforeEach } from 'vitest';
import { apiLimiterKey, botIngestKey } from './rate-limiter.js';
-import { markTokenVerified, clearVerifiedTokens } from './verified-tokens.js';
+import {
+ markBotSenderVerified,
+ markTokenVerified,
+ clearVerifiedTokens,
+} from './verified-tokens.js';
function reqWith({ authorization, ip = '203.0.113.7' } = {}) {
return { headers: authorization ? { authorization } : {}, ip };
@@ -62,17 +66,34 @@ describe('rate-limiter – apiLimiterKey', () => {
});
describe('rate-limiter – botIngestKey', () => {
- it("buckets AI-bot ingest by the connection's secret, whatever the address", () => {
+ beforeEach(() => clearVerifiedTokens());
+
+ it('buckets a matched secret by its connection, whatever the address', () => {
+ markBotSenderVerified('Bearer abot_one', 'c1');
+ markBotSenderVerified('Bearer abot_two', 'c2');
const a = botIngestKey(reqWith({ authorization: 'Bearer abot_one', ip: '1.1.1.1' }));
const b = botIngestKey(reqWith({ authorization: 'Bearer abot_one', ip: '2.2.2.2' }));
const other = botIngestKey(reqWith({ authorization: 'Bearer abot_two', ip: '1.1.1.1' }));
- expect(a).toBe(b);
- expect(a).not.toBe(other);
- // The secret itself isn't kept as the key.
- expect(a).toMatch(/^bot:[0-9a-f]{64}$/);
+ expect(a).toBe('bot:c1');
+ expect(b).toBe('bot:c1');
+ expect(other).toBe('bot:c2');
+ });
+
+ it('keeps unknown secrets in the caller IP bucket, so made-up ones share it', () => {
+ const one = botIngestKey(reqWith({ authorization: 'Bearer abot_made_up_1', ip: '1.1.1.1' }));
+ const two = botIngestKey(reqWith({ authorization: 'Bearer abot_made_up_2', ip: '1.1.1.1' }));
+ expect(one).toBe('ip:1.1.1.1');
+ expect(two).toBe('ip:1.1.1.1');
});
it('falls back to the IP without a secret', () => {
expect(botIngestKey(reqWith({ ip: '1.1.1.1' }))).toBe('ip:1.1.1.1');
});
+
+ it("never gives a bot secret a dashboard user's bucket", () => {
+ markBotSenderVerified('Bearer abot_one', 'c1');
+ expect(apiLimiterKey(reqWith({ authorization: 'Bearer abot_one', ip: '1.1.1.1' }))).toBe(
+ 'ip:1.1.1.1',
+ );
+ });
});
diff --git a/server/src/middleware/verified-tokens.js b/server/src/middleware/verified-tokens.js
index c2af17637..2969bb281 100644
--- a/server/src/middleware/verified-tokens.js
+++ b/server/src/middleware/verified-tokens.js
@@ -13,36 +13,66 @@ import { createHash } from 'node:crypto';
*/
const TTL_MS = 30 * 60 * 1000;
const MAX_ENTRIES = 5000;
-const cache = new Map();
function hashAuthHeader(authorizationHeader) {
return createHash('sha256').update(authorizationHeader).digest('hex').slice(0, 32);
}
+function verifiedCache() {
+ const cache = new Map();
+ return {
+ mark(authorizationHeader, id) {
+ if (!authorizationHeader || !id) return;
+ if (cache.size >= MAX_ENTRIES) {
+ cache.delete(cache.keys().next().value);
+ }
+ cache.set(hashAuthHeader(authorizationHeader), { id, expiresAt: Date.now() + TTL_MS });
+ },
+ get(authorizationHeader) {
+ if (!authorizationHeader) return null;
+ const key = hashAuthHeader(authorizationHeader);
+ const hit = cache.get(key);
+ if (!hit) return null;
+ if (hit.expiresAt < Date.now()) {
+ cache.delete(key);
+ return null;
+ }
+ return hit.id;
+ },
+ clear() {
+ cache.clear();
+ },
+ };
+}
+
+const users = verifiedCache();
+
export function markTokenVerified(authorizationHeader, userId) {
- if (!authorizationHeader || !userId) return;
- if (cache.size >= MAX_ENTRIES) {
- cache.delete(cache.keys().next().value);
- }
- cache.set(hashAuthHeader(authorizationHeader), {
- userId,
- expiresAt: Date.now() + TTL_MS,
- });
+ users.mark(authorizationHeader, userId);
}
export function verifiedUserId(authorizationHeader) {
- if (!authorizationHeader) return null;
- const key = hashAuthHeader(authorizationHeader);
- const hit = cache.get(key);
- if (!hit) return null;
- if (hit.expiresAt < Date.now()) {
- cache.delete(key);
- return null;
- }
- return hit.userId;
+ return users.get(authorizationHeader);
+}
+
+/**
+ * AI-bot senders (AT-010) whose secret named a live connection, mapped to
+ * the connection's id. Kept apart from users, so a bot secret never earns
+ * a dashboard bucket. Each accepted request marks it again, so a sender
+ * that keeps sending keeps its bucket.
+ */
+const botSenders = verifiedCache();
+
+export function markBotSenderVerified(authorizationHeader, connectionId) {
+ botSenders.mark(authorizationHeader, connectionId);
+}
+
+export function verifiedBotConnection(authorizationHeader) {
+ return botSenders.get(authorizationHeader);
}
// Test hook — the limiter's behavior depends on cache state.
export function clearVerifiedTokens() {
- cache.clear();
+ users.clear();
+ botSenders.clear();
}
diff --git a/server/src/routes/bot-ingest.js b/server/src/routes/bot-ingest.js
index ecf268f29..80cd98021 100644
--- a/server/src/routes/bot-ingest.js
+++ b/server/src/routes/bot-ingest.js
@@ -1,11 +1,13 @@
/**
* POST /bots/ingest — AI-bot requests from a brand's own edge (AT-010, spec
* §6.1). Public, outside /api: the sender is the customer's Cloudflare or
- * Vercel, authenticated by its connection secret (Authorization: Bearer).
- * The body is read by the connection's kind (bot-log-formats.js): a Worker's
- * JSON batch, a Logpush file, or a Vercel drain's log entries. Log senders'
- * records count against the month's raw cap before they are read, and a
- * file they send again (after timing out, say) is read once.
+ * Vercel, authenticated by its connection secret (Authorization: Bearer)
+ * before its body is read, so a caller without one can't make the server
+ * take in 20 MB. The body is read by the connection's kind
+ * (bot-log-formats.js): a Worker's JSON batch, a Logpush file, or a Vercel
+ * drain's log entries. Log senders' records count against the month's raw
+ * cap before they are read, and a file they send again (after timing out,
+ * say) is read once.
*/
import express, { Router } from 'express';
@@ -21,23 +23,37 @@ import {
import { LogTooLargeError, readSenderBody } from '../lib/bot-log-formats.js';
import { logger } from '../lib/logger.js';
import { botIngestLimiter } from '../middleware/rate-limiter.js';
+import { markBotSenderVerified } from '../middleware/verified-tokens.js';
const router = Router();
// Raw: each kind has its own body, and a Logpush file comes gzipped. A body
// sent with Content-Encoding: gzip is inflated here, within the limit.
+// Mounted before the app's JSON parser (server.js), so this reads it.
const body = express.raw({ type: () => true, limit: '20mb' });
-router.post('/bots/ingest', botIngestLimiter, body, async (req, res) => {
- let connection = null;
- // A log file's claim, given back unless the file was read.
- let claim = null;
- let read = false;
+/** The live connection the secret names, before the body is read; 401 otherwise. */
+async function sender(req, res, next) {
try {
const secret = (req.headers.authorization || '').replace(/^Bearer\s+/i, '');
- connection = await connectionForSecret(secret);
+ const connection = await connectionForSecret(secret);
if (!connection) return res.status(401).json({ error: 'unknown_connection' });
+ // Its next requests count against the connection's bucket, not the IP's.
+ markBotSenderVerified(req.headers.authorization, connection.id);
+ res.locals.connection = connection;
+ return next();
+ } catch (error) {
+ logger.error({ err: error }, 'bot ingest error');
+ return res.status(500).json({ error: 'ingest_failed' });
+ }
+}
+router.post('/bots/ingest', botIngestLimiter, sender, body, async (req, res) => {
+ const { connection } = res.locals;
+ // A log file's claim, given back unless the file was read.
+ let claim = null;
+ let read = false;
+ try {
const worker = connection.kind === 'cloudflare_worker';
if (!worker) {
claim = await claimLogFile(connection, req.body);
diff --git a/server/src/routes/bot-ingest.test.js b/server/src/routes/bot-ingest.test.js
index f179fdfcd..541a5a576 100644
--- a/server/src/routes/bot-ingest.test.js
+++ b/server/src/routes/bot-ingest.test.js
@@ -33,11 +33,14 @@ vi.mock('../lib/plan-guard.js', () => ({
const { default: router } = await import('./bot-ingest.js');
const { PlanLimitError } = await import('../lib/plan-guard.js');
+const { botIngestKey } = await import('../middleware/rate-limiter.js');
+const { clearVerifiedTokens } = await import('../middleware/verified-tokens.js');
let server;
let url;
beforeEach(async () => {
+ clearVerifiedTokens();
connectionForSecret.mockReset();
ingestBotRequests
.mockReset()
@@ -194,9 +197,38 @@ describe('POST /bots/ingest', () => {
connectionForSecret.mockResolvedValue(null);
expect((await post('abot_bad', { requests: [] })).status).toBe(401);
expect(ingestBotRequests).not.toHaveBeenCalled();
+ // A made-up secret earns no bucket of its own.
+ expect(botIngestKey({ headers: { authorization: 'Bearer abot_bad' }, ip: '1.1.1.1' })).toBe(
+ 'ip:1.1.1.1',
+ );
connectionForSecret.mockResolvedValue({ id: 'c1', kind: 'cloudflare_worker' });
ingestBotRequests.mockRejectedValue(new PlanLimitError('choose a plan', 402));
expect((await post('abot_x', { requests: [] })).status).toBe(402);
});
+
+ it("checks the secret before reading the body: an unknown one gets 401, not the body's 413", async () => {
+ connectionForSecret.mockResolvedValue(null);
+ const res = await fetch(url, {
+ method: 'POST',
+ headers: { Authorization: 'Bearer abot_bad' },
+ body: Buffer.alloc(21 * 1024 * 1024),
+ });
+ expect(res.status).toBe(401);
+ expect(claimLogFile).not.toHaveBeenCalled();
+ });
+
+ it("gives a matched secret its connection's bucket for the next requests", async () => {
+ connectionForSecret.mockResolvedValue({ id: 'c1', kind: 'cloudflare_worker' });
+ const req = { headers: { authorization: 'Bearer abot_x' }, ip: '1.1.1.1' };
+ expect(botIngestKey(req)).toBe('ip:1.1.1.1');
+ expect((await post('abot_x', { requests: [] })).status).toBe(200);
+ expect(botIngestKey(req)).toBe('bot:c1');
+ });
+
+ it('answers 500 when the secret lookup fails, without reading the body', async () => {
+ connectionForSecret.mockRejectedValue(new Error('database down'));
+ expect((await post('abot_x', { requests: [] })).status).toBe(500);
+ expect(ingestBotRequests).not.toHaveBeenCalled();
+ });
});
diff --git a/server/src/server.js b/server/src/server.js
index 242884bac..7db0943cd 100644
--- a/server/src/server.js
+++ b/server/src/server.js
@@ -58,17 +58,16 @@ const server = http.createServer(app);
// payload exactly as sent, and express.json()'s parse/re-serialize would break it.
// Mounted first so the global JSON parser skips this path (body already consumed).
app.use('/cloro/callback', express.raw({ type: 'application/json', limit: '50mb' }));
-// AI-bot senders (AT-010) post a Worker's JSON, a gzipped Logpush file or a
-// Vercel drain's logs: read raw, before the JSON and form parsers take them.
-app.use('/bots/ingest', express.raw({ type: () => true, limit: '20mb' }));
+// AI-bot requests from brands' Cloudflare / Vercel (AT-010): authenticated by
+// each connection's secret, server to server. Mounted before the JSON and form
+// parsers: the route checks the secret first, then reads the Worker's JSON,
+// the gzipped Logpush file or the Vercel drain's logs raw itself.
+app.use('/', botIngestRoutes);
app.use(bodyParser.json({ limit: '50mb' }));
app.use(bodyParser.urlencoded({ limit: '50mb', extended: true }));
// --- Public traffic tracking (before helmet/cors — needs its own CORS for any origin) ---
app.use('/', trafficRoutes);
-// AI-bot requests from brands' Cloudflare / Vercel (AT-010): authenticated by
-// each connection's secret, server to server.
-app.use('/', botIngestRoutes);
// --- Request logger (before everything to catch all requests) ---
app.use(requestIdMiddleware);