OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
import fs from 'node:fs';
import path, { dirname } from 'node:path';
import { fileURLToPath } from 'node:url';
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
import yaml from 'js-yaml';
const regexSpecialChars = /[|^$.*+?(){}[\]\\]/;
// User agents that must NEVER be treated as bots even though the upstream
// device-detector list flags them: its "Generic Bot" pattern lumps bare runtime
// identifiers (`node`, `Node.js`) into an anchored alternation, but legitimate
// server-side traffic sends them. Only user agents that explicitly identify as
// a bot (the separate `bot|crawler|spider...` heuristic) are flagged. Values are
// matched against single alternation branches, so metacharacters must be
// escaped exactly as upstream spells them.
export const ALLOWLISTED_BOT_UA_TOKENS = new Set(['node', 'Node\\.js']);
// Removes allowlisted identifiers from anchored groups such as
// `^(?:chrome|node|Node\.js|url)$`. An emptied group is dropped with its
// trailing `|` so the alternation stays valid.
export function stripAllowlistedTokens(regex: string): string {
return regex.replace(
/\^\(\?:([^)]+)\)\$\|?/g,
(match: string, inner: string) => {
const kept = inner
.split('|')
.filter((alt) => !ALLOWLISTED_BOT_UA_TOKENS.has(alt));
if (kept.length === 0) {
return '';
}
const trailingPipe = match.endsWith('|') ? '|' : '';
return `^(?:${kept.join('|')})$${trailingPipe}`;
}
);
}
function transformBots(bots: any[]): any[] {
return bots.map((bot) => {
const { regex: rawRegex, ...rest } = bot;
const regex = stripAllowlistedTokens(rawRegex);
const hasRegexChars = regexSpecialChars.test(regex);
if (hasRegexChars) {
return { regex, ...rest };
}
return { includes: regex, ...rest };
});
}
async function main() {
try {
const data = await fetch(
'https://raw.githubusercontent.com/matomo-org/device-detector/master/regexes/bots.yml'
).then((res) => res.text());
const parsedData = yaml.load(data) as any[];
const transformedBots = transformBots(parsedData);
fs.writeFileSync(
path.resolve(__dirname, '../src/modules/ingest/src/bots/bots.ts'),
[
'// This file is generated by the script get-bots.ts',
'',
'// The data is fetch from device-detector https://raw.githubusercontent.com/matomo-org/device-detector/master/regexes/bots.yml',
'',
`const bots = ${JSON.stringify(transformedBots, null, 2)} as const;`,
'export default bots;',
'',
].join('\n'),
'utf-8'
);
console.log(
`✅ Generated bots.ts with ${transformedBots.length} bot entries`
);
const regexCount = transformedBots.filter((b) => 'regex' in b).length;
const includesCount = transformedBots.filter((b) => 'includes' in b).length;
console.log(` - ${includesCount} simple string matches (includes)`);
console.log(` - ${regexCount} regex patterns`);
} catch (e) {
console.log(e);
}
}
// Only fetch when run directly, so the helpers can be unit-tested without a network call.
if (process.argv[1] === __filename) {
main();
}