Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

fix: bot detection for server side events

Carl-Gerhard Lindesvärdb94d2569766b
diff --git a/apps/api/scripts/get-bots.test.ts b/apps/api/scripts/get-bots.test.ts
new file mode 100644
index 000000000..64def0b2f
--- /dev/null
+++ b/apps/api/scripts/get-bots.test.ts
@@ -0,0 +1,61 @@
+/**
+ * Tests for the bot-list generator's allowlist surgery.
+ *
+ * `stripAllowlistedTokens` removes legitimate runtime/client identifiers from
+ * the upstream device-detector "Generic Bot" patterns so that regenerating the
+ * list (`pnpm gen:bots`) can never re-introduce false positives for backend
+ * traffic. These run without a network call (main() is guarded on direct run).
+ */
+
+import { describe, expect, it } from 'vitest';
+import { ALLOWLISTED_BOT_UA_TOKENS, stripAllowlistedTokens } from './get-bots';
+
+describe('stripAllowlistedTokens', () => {
+ it('removes allowlisted identifiers from an anchored exact-match group', () => {
+ const input = '^(?:chrome|node|Node\\.js|url|Zeus|ZmEu)$';
+ expect(stripAllowlistedTokens(input)).toBe('^(?:chrome|url|Zeus|ZmEu)$');
+ });
+
+ it('preserves the genuinely-suspicious branches', () => {
+ const result = stripAllowlistedTokens(
+ '^(?:chrome|node|Node\\.js|Zeus|ZmEu)$',
+ );
+ expect(result).toContain('Zeus');
+ expect(result).toContain('ZmEu');
+ expect(result).toContain('chrome');
+ });
+
+ it('strips the tokens when the group sits inside a larger alternation', () => {
+ const input = '^xenu|^(?:chrome|node|Node\\.js|url)$|OnlyScans';
+ const result = stripAllowlistedTokens(input);
+ expect(result).toBe('^xenu|^(?:chrome|url)$|OnlyScans');
+ expect(result).not.toContain('|node|');
+ expect(result).not.toContain('Node\\.js');
+ });
+
+ it('leaves groups without allowlisted tokens untouched', () => {
+ const input = '^(?:chrome|firefox|Zeus)$';
+ expect(stripAllowlistedTokens(input)).toBe(input);
+ });
+
+ it('drops an emptied group together with its trailing pipe (no `||`)', () => {
+ const input = '^(?:node|Node\\.js)$|OnlyScans';
+ const result = stripAllowlistedTokens(input);
+ expect(result).toBe('OnlyScans');
+ expect(result).not.toContain('||');
+ });
+
+ it('does not touch plain (non-anchored) patterns', () => {
+ const input = 'AhrefsBot|node-fetch|crawler';
+ expect(stripAllowlistedTokens(input)).toBe(input);
+ });
+
+ it('every allowlisted token is escaped exactly as it appears in a branch', () => {
+ // Guards against e.g. listing `Node.js` (unescaped) which would never match
+ // the upstream `Node\.js` branch and silently fail to strip anything.
+ for (const token of ALLOWLISTED_BOT_UA_TOKENS) {
+ const group = `^(?:${token}|RealBot)$`;
+ expect(stripAllowlistedTokens(group)).toBe('^(?:RealBot)$');
+ }
+ });
+});
diff --git a/apps/api/scripts/get-bots.ts b/apps/api/scripts/get-bots.ts
index 2dfe32e23..5d8281339 100644
--- a/apps/api/scripts/get-bots.ts
+++ b/apps/api/scripts/get-bots.ts
@@ -10,9 +10,44 @@ import yaml from 'js-yaml';
// Regex special characters that indicate we need actual regex
const regexSpecialChars = /[|^$.*+?(){}\[\]\\]/;
+// User agents that must NEVER be treated as bots, even though the upstream
+// device-detector list flags them. Its "Generic Bot" pattern lumps bare
+// runtime/client identifiers (e.g. `node`, `Node.js`) into an anchored
+// alternation because browsers never send them — but legitimate server-side
+// traffic does. We only want to flag traffic whose user agent *explicitly*
+// identifies itself as a bot (handled by the separate `bot|crawler|spider...`
+// heuristic), so these plain identifiers are stripped out of the generated
+// patterns here. This runs on every regen, so upstream can never re-introduce
+// the false positives. Values are matched against single alternation branches,
+// so regex metacharacters must be escaped exactly as they appear upstream.
+export const ALLOWLISTED_BOT_UA_TOKENS = new Set(['node', 'Node\\.js']);
+
+// Removes allowlisted identifiers from anchored exact-match groups such as
+// `^(?:chrome|node|Node\.js|url)$`, leaving the genuinely suspicious branches
+// intact. If a group ends up empty it is dropped together with a trailing `|`
+// so the surrounding alternation stays valid.
+export function stripAllowlistedTokens(regex: string): string {
+ return regex.replace(
+ /\^\(\?:([^)]+)\)\$\|?/g,
+ (match: string, inner: string) => {
+ const kept = inner
+ .split('|')
+ .filter((alt) => !ALLOWLISTED_BOT_UA_TOKENS.has(alt));
+
+ if (kept.length === 0) {
+ return '';
+ }
+
+ const trailingPipe = match.endsWith('|') ? '|' : '';
+ return `^(?:${kept.join('|')})$${trailingPipe}`;
+ },
+ );
+}
+
function transformBots(bots: any[]): any[] {
return bots.map((bot) => {
- const { regex, ...rest } = bot;
+ const { regex: rawRegex, ...rest } = bot;
+ const regex = stripAllowlistedTokens(rawRegex);
const hasRegexChars = regexSpecialChars.test(regex);
if (hasRegexChars) {
@@ -60,4 +95,8 @@ async function main() {
}
}
-main();
+// Only fetch + regenerate when run directly (e.g. `pnpm gen:bots`), so the
+// transformation helpers can be imported and unit-tested without a network call.
+if (process.argv[1] === __filename) {
+ main();
+}
diff --git a/apps/api/src/app.ts b/apps/api/src/app.ts
index ddd7adafc..92f251351 100644
--- a/apps/api/src/app.ts
+++ b/apps/api/src/app.ts
@@ -64,6 +64,7 @@ import { logger } from './utils/logger';
declare module 'fastify' {
interface FastifyRequest {
client: IServiceClientWithProject | null;
+ clientSecretAuth?: boolean;
clientIp: string;
clientIpHeader: string;
timestamp?: number;
diff --git a/apps/api/src/bots/bots.ts b/apps/api/src/bots/bots.ts
index 1eb50b28e..18a0e1e5f 100644
--- a/apps/api/src/bots/bots.ts
+++ b/apps/api/src/bots/bots.ts
@@ -7305,7 +7305,7 @@ const bots = [
},
{
regex:
- 'nuhk|grub-client|Download Demon|SearchExpress|Microsoft URL Control|borg|altavista|dataminr\\.com|teoma|oegp|http%20client|htdig|mogimogi|larbin|scrubby|searchsight|semanticdiscovery|snappy|zeal(?!ot)|dataparksearch|findlinks|BrowserMob|URL2PNG|ZooShot|GomezA|Google SketchUp|Read%20Later|7Siters|centuryb\\.o\\.t9|InterNaetBoten|EasyBib AutoCite|Bidtellect|tomnomnom/meg|cortex|Re-re Studio|adreview|AHC/|NameOfAgent|Request-Promise|ALittle Client|Hello,? world|wp_is_mobile|0xAbyssalDoesntExist|Anarchy99|^revolt|nvd0rz|xfa1|Hakai|gbrmss|fuck-your-hp|IDBTE4M CODE87|Antoine|Insomania|Hells-Net|b3astmode|Linux Gnu \\(cow\\)|Test Certificate Info|iplabel|Magellan|TheSafex?Internetx?Search|Searcherx?web|kirkland-signature|LinkChain|survey-security-dot-txt|infrawatch|Time/|r00ts3c-owned-you|nvdorz|Root Slut|NiggaBalls|BotPoke|GlobalWebSearch|xx032_bo9vs83_2a|sslshed|geckotrail|Wordup|Keydrop|\\(compatible\\)|John Recon|SPARK COMMIT|masjesu|Komaru_The_Cat|Jesus Christ of Nazareth is LORD|Kowai|Hakai|LoliSec|LMAO|^xenu|^(?:chrome|desktop|firefox|Abcd|Dark|KvshClient|node|Node\\.js|Report Runner|url|Zeus|ZmEu)$|OnlyScans|TheInternetSearchx|Laravel Reaver|bang2013|libredtail|Mozilliqa|Tiberius|honeygain|AW-WB-Filter|SaferSoftwashLeadGen|YourUserAgentHere|phpneuralnetwork',
+ 'nuhk|grub-client|Download Demon|SearchExpress|Microsoft URL Control|borg|altavista|dataminr\\.com|teoma|oegp|http%20client|htdig|mogimogi|larbin|scrubby|searchsight|semanticdiscovery|snappy|zeal(?!ot)|dataparksearch|findlinks|BrowserMob|URL2PNG|ZooShot|GomezA|Google SketchUp|Read%20Later|7Siters|centuryb\\.o\\.t9|InterNaetBoten|EasyBib AutoCite|Bidtellect|tomnomnom/meg|cortex|Re-re Studio|adreview|AHC/|NameOfAgent|Request-Promise|ALittle Client|Hello,? world|wp_is_mobile|0xAbyssalDoesntExist|Anarchy99|^revolt|nvd0rz|xfa1|Hakai|gbrmss|fuck-your-hp|IDBTE4M CODE87|Antoine|Insomania|Hells-Net|b3astmode|Linux Gnu \\(cow\\)|Test Certificate Info|iplabel|Magellan|TheSafex?Internetx?Search|Searcherx?web|kirkland-signature|LinkChain|survey-security-dot-txt|infrawatch|Time/|r00ts3c-owned-you|nvdorz|Root Slut|NiggaBalls|BotPoke|GlobalWebSearch|xx032_bo9vs83_2a|sslshed|geckotrail|Wordup|Keydrop|\\(compatible\\)|John Recon|SPARK COMMIT|masjesu|Komaru_The_Cat|Jesus Christ of Nazareth is LORD|Kowai|Hakai|LoliSec|LMAO|^xenu|^(?:chrome|desktop|firefox|Abcd|Dark|KvshClient|Report Runner|url|Zeus|ZmEu)$|OnlyScans|TheInternetSearchx|Laravel Reaver|bang2013|libredtail|Mozilliqa|Tiberius|honeygain|AW-WB-Filter|SaferSoftwashLeadGen|YourUserAgentHere|phpneuralnetwork',
name: 'Generic Bot',
},
{
diff --git a/apps/api/src/bots/index.test.ts b/apps/api/src/bots/index.test.ts
new file mode 100644
index 000000000..624655b2c
--- /dev/null
+++ b/apps/api/src/bots/index.test.ts
@@ -0,0 +1,60 @@
+/**
+ * Tests for bot detection against the generated `bots.ts` list.
+ *
+ * These exercise the pure `detectBot` matcher (no Redis), guarding two things:
+ * 1. Legitimate backend/runtime traffic (e.g. `node`, `Node.js`) is NOT a bot.
+ * The upstream device-detector list flags these via its "Generic Bot"
+ * pattern; we strip them at generation time (scripts/get-bots.ts), so this
+ * test fails loudly if a regen ever re-introduces the false positive.
+ * 2. Genuine bots are still detected — we only removed the allowlisted
+ * identifiers, not the rest of the bot patterns.
+ */
+
+import { describe, expect, it } from 'vitest';
+import { detectBot } from './index';
+
+describe('detectBot', () => {
+ describe('does not flag legitimate runtime/client user agents', () => {
+ // The exact identifiers the upstream "Generic Bot" pattern caught.
+ const allowlisted = ['node', 'Node.js'];
+ // Common server-side HTTP clients that must always pass through.
+ const backendClients = [
+ 'undici',
+ 'axios/1.6.0',
+ 'node-fetch/2.6.7',
+ 'got (https://github.com/sindresorhus/got)',
+ 'PostmanRuntime/7.0',
+ ];
+
+ for (const ua of [...allowlisted, ...backendClients]) {
+ it(`treats "${ua}" as a real user`, () => {
+ expect(detectBot(ua)).toBeNull();
+ });
+ }
+ });
+
+ describe('still flags real bots', () => {
+ const realBots = [
+ 'Googlebot/2.1 (+http://www.google.com/bot.html)',
+ 'Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)',
+ // Heuristic patterns (case-sensitive, as upstream defines them).
+ 'some-crawler/1.0',
+ 'a-scraper-bot',
+ ];
+
+ for (const ua of realBots) {
+ it(`detects "${ua}"`, () => {
+ expect(detectBot(ua)).not.toBeNull();
+ });
+ }
+
+ // Genuinely-suspicious bare tokens that live in the SAME "Generic Bot"
+ // alternation we edited — proves we surgically removed only the allowlisted
+ // identifiers and left the rest of that pattern working.
+ for (const ua of ['ZmEu', 'Zeus']) {
+ it(`still detects bare scanner token "${ua}" as Generic Bot`, () => {
+ expect(detectBot(ua)?.name).toBe('Generic Bot');
+ });
+ }
+ });
+});
diff --git a/apps/api/src/bots/index.ts b/apps/api/src/bots/index.ts
index 391aa2900..4cf09c6c9 100644
--- a/apps/api/src/bots/index.ts
+++ b/apps/api/src/bots/index.ts
@@ -15,30 +15,39 @@ const compiledBots = bots.map((bot) => {
const regexBots = compiledBots.filter((bot) => 'compiledRegex' in bot);
const includesBots = compiledBots.filter((bot) => 'includes' in bot);
-export const isBot = cacheable(
- 'is-bot',
- (ua: string) => {
- // Check simple string patterns first (fast)
- for (const bot of includesBots) {
- if (ua.includes(bot.includes)) {
- return {
- name: bot.name,
- type: 'category' in bot ? bot.category : 'Unknown',
- };
- }
+export interface BotMatch {
+ name: string;
+ type: string;
+}
+
+// Pure, synchronous bot detection against the generated `bots.ts` list.
+// Exported (and unwrapped from the cache) so it can be tested without Redis.
+// The list itself decides what counts as a bot; legitimate runtime/client
+// identifiers (e.g. `node`, `Node.js`) are stripped from the patterns at
+// generation time (see scripts/get-bots.ts), so backend traffic is only
+// flagged when its user agent explicitly identifies itself as a bot.
+export function detectBot(ua: string): BotMatch | null {
+ // Check simple string patterns first (fast)
+ for (const bot of includesBots) {
+ if (ua.includes(bot.includes)) {
+ return {
+ name: bot.name,
+ type: 'category' in bot ? bot.category : 'Unknown',
+ };
}
+ }
- // Check regex patterns (slower)
- for (const bot of regexBots) {
- if (bot.compiledRegex.test(ua)) {
- return {
- name: bot.name,
- type: 'category' in bot ? bot.category : 'Unknown',
- };
- }
+ // Check regex patterns (slower)
+ for (const bot of regexBots) {
+ if (bot.compiledRegex.test(ua)) {
+ return {
+ name: bot.name,
+ type: 'category' in bot ? bot.category : 'Unknown',
+ };
}
+ }
+
+ return null;
+}
- return null;
- },
- 60 * 5
-);
+export const isBot = cacheable('is-bot', detectBot, 60 * 5);
diff --git a/apps/api/src/hooks/is-bot.hook.test.ts b/apps/api/src/hooks/is-bot.hook.test.ts
new file mode 100644
index 000000000..4f1bba10f
--- /dev/null
+++ b/apps/api/src/hooks/is-bot.hook.test.ts
@@ -0,0 +1,87 @@
+/**
+ * Tests for isBotHook — the preHandler that filters bot traffic on ingestion.
+ *
+ * The key behaviour guarded here: requests authenticated with a client secret
+ * (server-side SDKs) are never treated as bots, regardless of user agent. Bot
+ * detection only applies to public/frontend (origin-authenticated) traffic.
+ */
+
+import type { FastifyReply, FastifyRequest } from 'fastify';
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+const isBot = vi.fn();
+const createBotEvent = vi.fn();
+
+vi.mock('@/bots', () => ({ isBot: (ua: string) => isBot(ua) }));
+vi.mock('@openpanel/db', () => ({
+ createBotEvent: (...args: unknown[]) => createBotEvent(...args),
+}));
+
+const { isBotHook } = await import('./is-bot.hook');
+
+function makeReply() {
+ const status = vi.fn();
+ const send = vi.fn();
+ const reply = { status, send };
+ status.mockReturnValue(reply);
+ send.mockReturnValue(reply);
+ return { reply: reply as unknown as FastifyReply, status, send };
+}
+
+function makeReq(overrides: Partial<FastifyRequest> = {}) {
+ return {
+ headers: { 'user-agent': 'Googlebot/2.1' },
+ client: { projectId: 'proj-1' },
+ body: { type: 'track', payload: { properties: { path: '/home' } } },
+ ...overrides,
+ } as unknown as FastifyRequest;
+}
+
+beforeEach(() => {
+ isBot.mockReset();
+ createBotEvent.mockReset();
+});
+
+describe('isBotHook', () => {
+ it('skips bot detection entirely for client-secret (server-side) traffic', async () => {
+ const req = makeReq({ clientSecretAuth: true });
+ const { reply, status } = makeReply();
+
+ await isBotHook(req as never, reply);
+
+ expect(isBot).not.toHaveBeenCalled();
+ expect(createBotEvent).not.toHaveBeenCalled();
+ expect(status).not.toHaveBeenCalled();
+ });
+
+ it('records a bot event and responds 202 for public bot traffic', async () => {
+ isBot.mockResolvedValue({ name: 'Googlebot', type: 'Search bot' });
+ const req = makeReq();
+ const { reply, status } = makeReply();
+
+ await isBotHook(req as never, reply);
+
+ expect(isBot).toHaveBeenCalledWith('Googlebot/2.1');
+ expect(createBotEvent).toHaveBeenCalledWith(
+ expect.objectContaining({
+ name: 'Googlebot',
+ type: 'Search bot',
+ projectId: 'proj-1',
+ path: '/home',
+ }),
+ );
+ expect(status).toHaveBeenCalledWith(202);
+ });
+
+ it('passes legitimate public traffic through untouched', async () => {
+ isBot.mockResolvedValue(null);
+ const req = makeReq({ headers: { 'user-agent': 'node' } as never });
+ const { reply, status } = makeReply();
+
+ const result = await isBotHook(req as never, reply);
+
+ expect(createBotEvent).not.toHaveBeenCalled();
+ expect(status).not.toHaveBeenCalled();
+ expect(result).toBeUndefined();
+ });
+});
diff --git a/apps/api/src/hooks/is-bot.hook.ts b/apps/api/src/hooks/is-bot.hook.ts
index 09346fbde..a787d71cb 100644
--- a/apps/api/src/hooks/is-bot.hook.ts
+++ b/apps/api/src/hooks/is-bot.hook.ts
@@ -12,6 +12,15 @@ export async function isBotHook(
}>,
reply: FastifyReply
) {
+ // Requests authenticated with a client secret come from server-side SDKs
+ // (node, php, go, rust, java, python, …). That auth is a far stronger signal
+ // of legitimate first-party traffic than the user agent, so never treat them
+ // as bots — bot detection is for public/frontend (origin-authenticated)
+ // traffic.
+ if (req.clientSecretAuth) {
+ return;
+ }
+
const bot = req.headers['user-agent']
? await isBot(req.headers['user-agent'])
: null;
diff --git a/apps/api/src/utils/auth.ts b/apps/api/src/utils/auth.ts
index 0b0dd2f14..70cb59862 100644
--- a/apps/api/src/utils/auth.ts
+++ b/apps/api/src/utils/auth.ts
@@ -59,6 +59,10 @@ export async function validateSdkRequest(
clientSecretNew || clientSecretOld || clientSecretFromBody;
const origin = headers.origin;
+ if (clientSecret) {
+ req.clientSecretAuth = true;
+ }
+
const createError = (message: string) =>
new SdkAuthError(message, {
clientId,