OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
fix: bot detection for server side events
Carl-Gerhard Lindesvärdb94d2569766b
diff --git a/apps/api/scripts/get-bots.test.ts b/apps/api/scripts/get-bots.test.tsnew file mode 100644index 000000000..64def0b2f--- /dev/null+++ b/apps/api/scripts/get-bots.test.ts@@ -0,0 +1,61 @@+/**+ * Tests for the bot-list generator's allowlist surgery.+ *+ * `stripAllowlistedTokens` removes legitimate runtime/client identifiers from+ * the upstream device-detector "Generic Bot" patterns so that regenerating the+ * list (`pnpm gen:bots`) can never re-introduce false positives for backend+ * traffic. These run without a network call (main() is guarded on direct run).+ */++import { describe, expect, it } from 'vitest';+import { ALLOWLISTED_BOT_UA_TOKENS, stripAllowlistedTokens } from './get-bots';++describe('stripAllowlistedTokens', () => {+ it('removes allowlisted identifiers from an anchored exact-match group', () => {+ const input = '^(?:chrome|node|Node\\.js|url|Zeus|ZmEu)$';+ expect(stripAllowlistedTokens(input)).toBe('^(?:chrome|url|Zeus|ZmEu)$');+ });++ it('preserves the genuinely-suspicious branches', () => {+ const result = stripAllowlistedTokens(+ '^(?:chrome|node|Node\\.js|Zeus|ZmEu)$',+ );+ expect(result).toContain('Zeus');+ expect(result).toContain('ZmEu');+ expect(result).toContain('chrome');+ });++ it('strips the tokens when the group sits inside a larger alternation', () => {+ const input = '^xenu|^(?:chrome|node|Node\\.js|url)$|OnlyScans';+ const result = stripAllowlistedTokens(input);+ expect(result).toBe('^xenu|^(?:chrome|url)$|OnlyScans');+ expect(result).not.toContain('|node|');+ expect(result).not.toContain('Node\\.js');+ });++ it('leaves groups without allowlisted tokens untouched', () => {+ const input = '^(?:chrome|firefox|Zeus)$';+ expect(stripAllowlistedTokens(input)).toBe(input);+ });++ it('drops an emptied group together with its trailing pipe (no `||`)', () => {+ const input = '^(?:node|Node\\.js)$|OnlyScans';+ const result = stripAllowlistedTokens(input);+ expect(result).toBe('OnlyScans');+ expect(result).not.toContain('||');+ });++ it('does not touch plain (non-anchored) patterns', () => {+ const input = 'AhrefsBot|node-fetch|crawler';+ expect(stripAllowlistedTokens(input)).toBe(input);+ });++ it('every allowlisted token is escaped exactly as it appears in a branch', () => {+ // Guards against e.g. listing `Node.js` (unescaped) which would never match+ // the upstream `Node\.js` branch and silently fail to strip anything.+ for (const token of ALLOWLISTED_BOT_UA_TOKENS) {+ const group = `^(?:${token}|RealBot)$`;+ expect(stripAllowlistedTokens(group)).toBe('^(?:RealBot)$');+ }+ });+});diff --git a/apps/api/scripts/get-bots.ts b/apps/api/scripts/get-bots.tsindex 2dfe32e23..5d8281339 100644--- a/apps/api/scripts/get-bots.ts+++ b/apps/api/scripts/get-bots.ts@@ -10,9 +10,44 @@ import yaml from 'js-yaml';// Regex special characters that indicate we need actual regexconst regexSpecialChars = /[|^$.*+?(){}\[\]\\]/;+// User agents that must NEVER be treated as bots, even though the upstream+// device-detector list flags them. Its "Generic Bot" pattern lumps bare+// runtime/client identifiers (e.g. `node`, `Node.js`) into an anchored+// alternation because browsers never send them — but legitimate server-side+// traffic does. We only want to flag traffic whose user agent *explicitly*+// identifies itself as a bot (handled by the separate `bot|crawler|spider...`+// heuristic), so these plain identifiers are stripped out of the generated+// patterns here. This runs on every regen, so upstream can never re-introduce+// the false positives. Values are matched against single alternation branches,+// so regex metacharacters must be escaped exactly as they appear upstream.+export const ALLOWLISTED_BOT_UA_TOKENS = new Set(['node', 'Node\\.js']);++// Removes allowlisted identifiers from anchored exact-match groups such as+// `^(?:chrome|node|Node\.js|url)$`, leaving the genuinely suspicious branches+// intact. If a group ends up empty it is dropped together with a trailing `|`+// so the surrounding alternation stays valid.+export function stripAllowlistedTokens(regex: string): string {+ return regex.replace(+ /\^\(\?:([^)]+)\)\$\|?/g,+ (match: string, inner: string) => {+ const kept = inner+ .split('|')+ .filter((alt) => !ALLOWLISTED_BOT_UA_TOKENS.has(alt));++ if (kept.length === 0) {+ return '';+ }++ const trailingPipe = match.endsWith('|') ? '|' : '';+ return `^(?:${kept.join('|')})$${trailingPipe}`;+ },+ );+}+function transformBots(bots: any[]): any[] {return bots.map((bot) => {- const { regex, ...rest } = bot;+ const { regex: rawRegex, ...rest } = bot;+ const regex = stripAllowlistedTokens(rawRegex);const hasRegexChars = regexSpecialChars.test(regex);if (hasRegexChars) {@@ -60,4 +95,8 @@ async function main() {}}-main();+// Only fetch + regenerate when run directly (e.g. `pnpm gen:bots`), so the+// transformation helpers can be imported and unit-tested without a network call.+if (process.argv[1] === __filename) {+ main();+}diff --git a/apps/api/src/app.ts b/apps/api/src/app.tsindex ddd7adafc..92f251351 100644--- a/apps/api/src/app.ts+++ b/apps/api/src/app.ts@@ -64,6 +64,7 @@ import { logger } from './utils/logger';declare module 'fastify' {interface FastifyRequest {client: IServiceClientWithProject | null;+ clientSecretAuth?: boolean;clientIp: string;clientIpHeader: string;timestamp?: number;diff --git a/apps/api/src/bots/bots.ts b/apps/api/src/bots/bots.tsindex 1eb50b28e..18a0e1e5f 100644--- a/apps/api/src/bots/bots.ts+++ b/apps/api/src/bots/bots.ts@@ -7305,7 +7305,7 @@ const bots = [},{regex:- 'nuhk|grub-client|Download Demon|SearchExpress|Microsoft URL Control|borg|altavista|dataminr\\.com|teoma|oegp|http%20client|htdig|mogimogi|larbin|scrubby|searchsight|semanticdiscovery|snappy|zeal(?!ot)|dataparksearch|findlinks|BrowserMob|URL2PNG|ZooShot|GomezA|Google SketchUp|Read%20Later|7Siters|centuryb\\.o\\.t9|InterNaetBoten|EasyBib AutoCite|Bidtellect|tomnomnom/meg|cortex|Re-re Studio|adreview|AHC/|NameOfAgent|Request-Promise|ALittle Client|Hello,? world|wp_is_mobile|0xAbyssalDoesntExist|Anarchy99|^revolt|nvd0rz|xfa1|Hakai|gbrmss|fuck-your-hp|IDBTE4M CODE87|Antoine|Insomania|Hells-Net|b3astmode|Linux Gnu \\(cow\\)|Test Certificate Info|iplabel|Magellan|TheSafex?Internetx?Search|Searcherx?web|kirkland-signature|LinkChain|survey-security-dot-txt|infrawatch|Time/|r00ts3c-owned-you|nvdorz|Root Slut|NiggaBalls|BotPoke|GlobalWebSearch|xx032_bo9vs83_2a|sslshed|geckotrail|Wordup|Keydrop|\\(compatible\\)|John Recon|SPARK COMMIT|masjesu|Komaru_The_Cat|Jesus Christ of Nazareth is LORD|Kowai|Hakai|LoliSec|LMAO|^xenu|^(?:chrome|desktop|firefox|Abcd|Dark|KvshClient|node|Node\\.js|Report Runner|url|Zeus|ZmEu)$|OnlyScans|TheInternetSearchx|Laravel Reaver|bang2013|libredtail|Mozilliqa|Tiberius|honeygain|AW-WB-Filter|SaferSoftwashLeadGen|YourUserAgentHere|phpneuralnetwork',+ 'nuhk|grub-client|Download Demon|SearchExpress|Microsoft URL Control|borg|altavista|dataminr\\.com|teoma|oegp|http%20client|htdig|mogimogi|larbin|scrubby|searchsight|semanticdiscovery|snappy|zeal(?!ot)|dataparksearch|findlinks|BrowserMob|URL2PNG|ZooShot|GomezA|Google SketchUp|Read%20Later|7Siters|centuryb\\.o\\.t9|InterNaetBoten|EasyBib AutoCite|Bidtellect|tomnomnom/meg|cortex|Re-re Studio|adreview|AHC/|NameOfAgent|Request-Promise|ALittle Client|Hello,? world|wp_is_mobile|0xAbyssalDoesntExist|Anarchy99|^revolt|nvd0rz|xfa1|Hakai|gbrmss|fuck-your-hp|IDBTE4M CODE87|Antoine|Insomania|Hells-Net|b3astmode|Linux Gnu \\(cow\\)|Test Certificate Info|iplabel|Magellan|TheSafex?Internetx?Search|Searcherx?web|kirkland-signature|LinkChain|survey-security-dot-txt|infrawatch|Time/|r00ts3c-owned-you|nvdorz|Root Slut|NiggaBalls|BotPoke|GlobalWebSearch|xx032_bo9vs83_2a|sslshed|geckotrail|Wordup|Keydrop|\\(compatible\\)|John Recon|SPARK COMMIT|masjesu|Komaru_The_Cat|Jesus Christ of Nazareth is LORD|Kowai|Hakai|LoliSec|LMAO|^xenu|^(?:chrome|desktop|firefox|Abcd|Dark|KvshClient|Report Runner|url|Zeus|ZmEu)$|OnlyScans|TheInternetSearchx|Laravel Reaver|bang2013|libredtail|Mozilliqa|Tiberius|honeygain|AW-WB-Filter|SaferSoftwashLeadGen|YourUserAgentHere|phpneuralnetwork',name: 'Generic Bot',},{diff --git a/apps/api/src/bots/index.test.ts b/apps/api/src/bots/index.test.tsnew file mode 100644index 000000000..624655b2c--- /dev/null+++ b/apps/api/src/bots/index.test.ts@@ -0,0 +1,60 @@+/**+ * Tests for bot detection against the generated `bots.ts` list.+ *+ * These exercise the pure `detectBot` matcher (no Redis), guarding two things:+ * 1. Legitimate backend/runtime traffic (e.g. `node`, `Node.js`) is NOT a bot.+ * The upstream device-detector list flags these via its "Generic Bot"+ * pattern; we strip them at generation time (scripts/get-bots.ts), so this+ * test fails loudly if a regen ever re-introduces the false positive.+ * 2. Genuine bots are still detected — we only removed the allowlisted+ * identifiers, not the rest of the bot patterns.+ */++import { describe, expect, it } from 'vitest';+import { detectBot } from './index';++describe('detectBot', () => {+ describe('does not flag legitimate runtime/client user agents', () => {+ // The exact identifiers the upstream "Generic Bot" pattern caught.+ const allowlisted = ['node', 'Node.js'];+ // Common server-side HTTP clients that must always pass through.+ const backendClients = [+ 'undici',+ 'axios/1.6.0',+ 'node-fetch/2.6.7',+ 'got (https://github.com/sindresorhus/got)',+ 'PostmanRuntime/7.0',+ ];++ for (const ua of [...allowlisted, ...backendClients]) {+ it(`treats "${ua}" as a real user`, () => {+ expect(detectBot(ua)).toBeNull();+ });+ }+ });++ describe('still flags real bots', () => {+ const realBots = [+ 'Googlebot/2.1 (+http://www.google.com/bot.html)',+ 'Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)',+ // Heuristic patterns (case-sensitive, as upstream defines them).+ 'some-crawler/1.0',+ 'a-scraper-bot',+ ];++ for (const ua of realBots) {+ it(`detects "${ua}"`, () => {+ expect(detectBot(ua)).not.toBeNull();+ });+ }++ // Genuinely-suspicious bare tokens that live in the SAME "Generic Bot"+ // alternation we edited — proves we surgically removed only the allowlisted+ // identifiers and left the rest of that pattern working.+ for (const ua of ['ZmEu', 'Zeus']) {+ it(`still detects bare scanner token "${ua}" as Generic Bot`, () => {+ expect(detectBot(ua)?.name).toBe('Generic Bot');+ });+ }+ });+});diff --git a/apps/api/src/bots/index.ts b/apps/api/src/bots/index.tsindex 391aa2900..4cf09c6c9 100644--- a/apps/api/src/bots/index.ts+++ b/apps/api/src/bots/index.ts@@ -15,30 +15,39 @@ const compiledBots = bots.map((bot) => {const regexBots = compiledBots.filter((bot) => 'compiledRegex' in bot);const includesBots = compiledBots.filter((bot) => 'includes' in bot);-export const isBot = cacheable(- 'is-bot',- (ua: string) => {- // Check simple string patterns first (fast)- for (const bot of includesBots) {- if (ua.includes(bot.includes)) {- return {- name: bot.name,- type: 'category' in bot ? bot.category : 'Unknown',- };- }+export interface BotMatch {+ name: string;+ type: string;+}++// Pure, synchronous bot detection against the generated `bots.ts` list.+// Exported (and unwrapped from the cache) so it can be tested without Redis.+// The list itself decides what counts as a bot; legitimate runtime/client+// identifiers (e.g. `node`, `Node.js`) are stripped from the patterns at+// generation time (see scripts/get-bots.ts), so backend traffic is only+// flagged when its user agent explicitly identifies itself as a bot.+export function detectBot(ua: string): BotMatch | null {+ // Check simple string patterns first (fast)+ for (const bot of includesBots) {+ if (ua.includes(bot.includes)) {+ return {+ name: bot.name,+ type: 'category' in bot ? bot.category : 'Unknown',+ };}+ }- // Check regex patterns (slower)- for (const bot of regexBots) {- if (bot.compiledRegex.test(ua)) {- return {- name: bot.name,- type: 'category' in bot ? bot.category : 'Unknown',- };- }+ // Check regex patterns (slower)+ for (const bot of regexBots) {+ if (bot.compiledRegex.test(ua)) {+ return {+ name: bot.name,+ type: 'category' in bot ? bot.category : 'Unknown',+ };}+ }++ return null;+}- return null;- },- 60 * 5-);+export const isBot = cacheable('is-bot', detectBot, 60 * 5);diff --git a/apps/api/src/hooks/is-bot.hook.test.ts b/apps/api/src/hooks/is-bot.hook.test.tsnew file mode 100644index 000000000..4f1bba10f--- /dev/null+++ b/apps/api/src/hooks/is-bot.hook.test.ts@@ -0,0 +1,87 @@+/**+ * Tests for isBotHook — the preHandler that filters bot traffic on ingestion.+ *+ * The key behaviour guarded here: requests authenticated with a client secret+ * (server-side SDKs) are never treated as bots, regardless of user agent. Bot+ * detection only applies to public/frontend (origin-authenticated) traffic.+ */++import type { FastifyReply, FastifyRequest } from 'fastify';+import { beforeEach, describe, expect, it, vi } from 'vitest';++const isBot = vi.fn();+const createBotEvent = vi.fn();++vi.mock('@/bots', () => ({ isBot: (ua: string) => isBot(ua) }));+vi.mock('@openpanel/db', () => ({+ createBotEvent: (...args: unknown[]) => createBotEvent(...args),+}));++const { isBotHook } = await import('./is-bot.hook');++function makeReply() {+ const status = vi.fn();+ const send = vi.fn();+ const reply = { status, send };+ status.mockReturnValue(reply);+ send.mockReturnValue(reply);+ return { reply: reply as unknown as FastifyReply, status, send };+}++function makeReq(overrides: Partial<FastifyRequest> = {}) {+ return {+ headers: { 'user-agent': 'Googlebot/2.1' },+ client: { projectId: 'proj-1' },+ body: { type: 'track', payload: { properties: { path: '/home' } } },+ ...overrides,+ } as unknown as FastifyRequest;+}++beforeEach(() => {+ isBot.mockReset();+ createBotEvent.mockReset();+});++describe('isBotHook', () => {+ it('skips bot detection entirely for client-secret (server-side) traffic', async () => {+ const req = makeReq({ clientSecretAuth: true });+ const { reply, status } = makeReply();++ await isBotHook(req as never, reply);++ expect(isBot).not.toHaveBeenCalled();+ expect(createBotEvent).not.toHaveBeenCalled();+ expect(status).not.toHaveBeenCalled();+ });++ it('records a bot event and responds 202 for public bot traffic', async () => {+ isBot.mockResolvedValue({ name: 'Googlebot', type: 'Search bot' });+ const req = makeReq();+ const { reply, status } = makeReply();++ await isBotHook(req as never, reply);++ expect(isBot).toHaveBeenCalledWith('Googlebot/2.1');+ expect(createBotEvent).toHaveBeenCalledWith(+ expect.objectContaining({+ name: 'Googlebot',+ type: 'Search bot',+ projectId: 'proj-1',+ path: '/home',+ }),+ );+ expect(status).toHaveBeenCalledWith(202);+ });++ it('passes legitimate public traffic through untouched', async () => {+ isBot.mockResolvedValue(null);+ const req = makeReq({ headers: { 'user-agent': 'node' } as never });+ const { reply, status } = makeReply();++ const result = await isBotHook(req as never, reply);++ expect(createBotEvent).not.toHaveBeenCalled();+ expect(status).not.toHaveBeenCalled();+ expect(result).toBeUndefined();+ });+});diff --git a/apps/api/src/hooks/is-bot.hook.ts b/apps/api/src/hooks/is-bot.hook.tsindex 09346fbde..a787d71cb 100644--- a/apps/api/src/hooks/is-bot.hook.ts+++ b/apps/api/src/hooks/is-bot.hook.ts@@ -12,6 +12,15 @@ export async function isBotHook(}>,reply: FastifyReply) {+ // Requests authenticated with a client secret come from server-side SDKs+ // (node, php, go, rust, java, python, …). That auth is a far stronger signal+ // of legitimate first-party traffic than the user agent, so never treat them+ // as bots — bot detection is for public/frontend (origin-authenticated)+ // traffic.+ if (req.clientSecretAuth) {+ return;+ }+const bot = req.headers['user-agent']? await isBot(req.headers['user-agent']): null;diff --git a/apps/api/src/utils/auth.ts b/apps/api/src/utils/auth.tsindex 0b0dd2f14..70cb59862 100644--- a/apps/api/src/utils/auth.ts+++ b/apps/api/src/utils/auth.ts@@ -59,6 +59,10 @@ export async function validateSdkRequest(clientSecretNew || clientSecretOld || clientSecretFromBody;const origin = headers.origin;+ if (clientSecret) {+ req.clientSecretAuth = true;+ }+const createError = (message: string) =>new SdkAuthError(message, {clientId,