πΈ Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
feat(bots): detect for every site, and catch the paced distributed crawler (#1123)
Goldflag7dcce6fa3e4f
diff --git a/server/src/db/clickhouse/schema/core.ts b/server/src/db/clickhouse/schema/core.tsindex 89a79f9d2..3d4cac0be 100644--- a/server/src/db/clickhouse/schema/core.ts+++ b/server/src/db/clickhouse/schema/core.ts@@ -29,26 +29,30 @@ const BOT_EVENTS_COLUMNS_TO_ENSURE: ColumnDefinition[] = [// Null = the client sent no score and the server inferred nothing.{ name: "client_bot_score", definition: "client_bot_score Nullable(UInt8)" },{ name: "client_signal_mask", definition: "client_signal_mask UInt16 DEFAULT 0" },+ // Which anomaly rules fired and what they summed to. "Rate anomaly" is a+ // dozen rules with different meanings, so without these an audit row cannot+ // say why the request was convicted.+ { name: "anomaly_reasons", definition: "anomaly_reasons String DEFAULT ''" },+ { name: "anomaly_score", definition: "anomaly_score UInt8 DEFAULT 0" },];-async function ensureBotEventsColumns() {- const existingColumns = await getTableColumns("bot_events");+// Runs against both audit tables: they carry the same columns on purpose, so a+// column added to one has to reach the other or the shared queries stop working.+async function ensureBotEventsColumns(table: "bot_events" | "bot_observations") {+ const existingColumns = await getTableColumns(table);const missingColumns = BOT_EVENTS_COLUMNS_TO_ENSURE.filter(column => !existingColumns.has(column.name));if (missingColumns.length === 0) {- logger.debug("Bot events table columns are up to date");+ logger.debug({ table }, "Bot events table columns are up to date");return;}- logger.info(- { missingColumns: missingColumns.map(column => column.name) },- "Adding missing bot events table columns"- );+ logger.info({ table, missingColumns: missingColumns.map(column => column.name) }, "Adding missing bot table columns");await execClickhouseInitStep(- "add missing bot events columns",+ `add missing ${table} columns`,`- ALTER TABLE bot_events+ ALTER TABLE ${table}${missingColumns.map(column => `ADD COLUMN IF NOT EXISTS ${column.definition}`).join(",\n ")}`,{ lockAcquireTimeoutSeconds: 15 }@@ -169,7 +173,9 @@ export async function initializeCoreTables() {matched_ua_pattern String DEFAULT '',bot_category LowCardinality(String) DEFAULT '',client_bot_score Nullable(UInt8),- client_signal_mask UInt16 DEFAULT 0+ client_signal_mask UInt16 DEFAULT 0,+ anomaly_reasons String DEFAULT '',+ anomaly_score UInt8 DEFAULT 0)ENGINE = MergeTree()PARTITION BY toYYYYMM(timestamp)@@ -178,7 +184,59 @@ export async function initializeCoreTables() {`);- await ensureBotEventsColumns();+ await ensureBotEventsColumns("bot_events");++ // Forensic mirror of bot_events for sites with bot blocking turned OFF: the+ // event is still tracked normally, and this row is the only trace that+ // detection fired. Columns match bot_events exactly so the same analysis+ // queries run against either table; only the TTL is shorter.+ await execClickhouseInitStep(+ "create bot observations table",+ `+ CREATE TABLE IF NOT EXISTS bot_observations (+ site_id UInt16,+ timestamp DateTime,+ session_id String,+ user_id String,+ hostname String,+ pathname String,+ querystring String,+ referrer String,+ browser LowCardinality(String),+ browser_version LowCardinality(String),+ operating_system LowCardinality(String),+ operating_system_version LowCardinality(String),+ country LowCardinality(FixedString(2)),+ region LowCardinality(String),+ city String,+ lat Float64,+ lon Float64,+ screen_width UInt16,+ screen_height UInt16,+ device_type LowCardinality(String),+ type LowCardinality(String) DEFAULT 'pageview',+ asn Nullable(UInt32),+ asn_org String DEFAULT '',+ detected_ua_pattern Bool DEFAULT false,+ detected_header_heuristics Bool DEFAULT false,+ detected_client_signals Bool DEFAULT false,+ detected_bot_asn Bool DEFAULT false,+ detected_rate_anomaly Bool DEFAULT false,+ matched_ua_pattern String DEFAULT '',+ bot_category LowCardinality(String) DEFAULT '',+ client_bot_score Nullable(UInt8),+ client_signal_mask UInt16 DEFAULT 0,+ anomaly_reasons String DEFAULT '',+ anomaly_score UInt8 DEFAULT 0+ )+ ENGINE = MergeTree()+ PARTITION BY toYYYYMM(timestamp)+ ORDER BY (site_id, timestamp)+ TTL timestamp + INTERVAL 30 DAY+ `+ );++ await ensureBotEventsColumns("bot_observations");await execClickhouseInitStep("create session replay events table",diff --git a/server/src/db/redis/redis.ts b/server/src/db/redis/redis.tsindex 739be566b..1d16044f0 100644--- a/server/src/db/redis/redis.ts+++ b/server/src/db/redis/redis.ts@@ -232,10 +232,21 @@ export async function apiRateLimitConsume(input: ApiRateLimitInput): Promise<Api// top/total is the modal share, which separates an organic population (one// dominant browser version) from a fleet rotating a fixed list of user agents.//+// Two further kinds serve aggregate rules that ask about a whole cohort rather+// than one visitor, where per-member bookkeeping would be far too expensive to+// keep. Both live inside a fixed time bucket encoded in the key by the caller,+// exactly like a distribution, so the key simply expires. "counter" is a plain+// monotonic INCR β the cheapest possible answer to "how many events did this+// cohort produce this bucket?" β and ignores the member entirely. "cardinality"+// is a HyperLogLog: an approximate distinct count for dimensions such as distinct+// paths or distinct actors, where the true cardinality can reach tens of+// thousands and a sorted set holding every member would be far too large.+//// numberOfKeys is omitted so the key count can be passed per call. KEYS holds the// counter keys; ARGV is [nowMs, then (kind, member, windowMs, maxSize) per key].-// Every key yields three results β [count, 0, 0] for rolling counters and-// [total, top, distinct] for distributions β so the caller can index by position.+// Every key yields three results β [count, 0, 0] for rolling counters, plain+// counters and cardinality estimates, and [total, top, distinct] for+// distributions β so the caller can index by position.redis.defineCommand("anomalyObserve", {lua: `local now = tonumber(ARGV[1])@@ -266,6 +277,21 @@ redis.defineCommand("anomalyObserve", {results[out + 1] = totalresults[out + 2] = topresults[out + 3] = #values+ elseif kind == 'c' then+ -- Plain monotonic counter inside the caller-encoded bucket; member unused.+ results[out + 1] = redis.call('INCR', key)+ redis.call('PEXPIRE', key, windowMs)+ results[out + 2] = 0+ results[out + 3] = 0+ elseif kind == 'p' then+ -- HyperLogLog: an approximate distinct count for dimensions whose true+ -- cardinality can reach tens of thousands (distinct paths, distinct+ -- actors), where a sorted set holding every member would be far too large.+ redis.call('PFADD', key, member)+ redis.call('PEXPIRE', key, windowMs)+ results[out + 1] = redis.call('PFCOUNT', key)+ results[out + 2] = 0+ results[out + 3] = 0elseredis.call('ZADD', key, now, member)redis.call('ZREMRANGEBYSCORE', key, '-inf', '(' .. (now - windowMs))@@ -286,15 +312,25 @@ export interface AnomalyCounterSpec {/** Fully-namespaced Redis key for this counter. */key: string;/**- * Rolling sorted-set counter (default) or fixed-bucket value distribution. A- * distribution key must already encode its time bucket.+ * How this counter stores its observations:+ * - `rolling` (default) β sorted set over a sliding `windowMs`; reports exact+ * cardinality.+ * - `distribution` β hash of value -> count; reports total, top and distinct.+ * - `counter` β plain monotonic INCR; reports the incremented value. `member`+ * and `maxSize` are unused.+ * - `cardinality` β HyperLogLog; reports an approximate distinct count.+ * `maxSize` is unused.+ *+ * `rolling` manages its own window, so its key may be stable. The other three+ * kinds are fixed-bucket: the key must already encode its time bucket, since+ * they only expire rather than trim.*/- kind?: "rolling" | "distribution";- /** Unique per-event token (rate counters) or observed value (distinct/distribution counters). */+ kind?: "rolling" | "distribution" | "counter" | "cardinality";+ /** Unique per-event token (rate counters) or observed value (distinct/distribution/cardinality counters). */member: string;- /** Rolling window for `rolling`; key TTL for `distribution`. */+ /** Rolling window for `rolling`; key TTL for the fixed-bucket kinds. */windowMs: number;- /** Hard cap on stored members/fields; 0 disables trimming. */+ /** Hard cap on stored members/fields; 0 disables trimming. Unused by `counter` and `cardinality`. */maxSize: number;}@@ -309,17 +345,27 @@ interface AnomalyRedis extends Redis {anomalyObserve(...args: (string | number)[]): Promise<number[]>;}+// Wire codes the Lua script dispatches on. Written as an exhaustive map rather+// than a ternary so a new kind cannot silently fall through to a rolling counter.+const ANOMALY_KIND_CODES: Record<NonNullable<AnomalyCounterSpec["kind"]>, string> = {+ rolling: "z",+ distribution: "h",+ counter: "c",+ cardinality: "p",+};+/*** Observe one event against a batch of counters and return each counter's* current reading, in the same order as `specs`. Atomic across all workers; a- * single round-trip regardless of how many counters are passed. Rolling counters- * report cardinality in `total`; distributions fill all three fields.+ * single round-trip regardless of how many counters are passed. Distributions+ * fill all three fields; every other kind reports its single number in `total`+ * and leaves `top`/`distinct` at zero.*/export async function anomalyObserve(nowMs: number, specs: AnomalyCounterSpec[]): Promise<AnomalyDistribution[]> {const keys = specs.map(spec => spec.key);const args: (string | number)[] = [specs.length, ...keys, nowMs];for (const spec of specs) {- args.push(spec.kind === "distribution" ? "h" : "z", spec.member, spec.windowMs, spec.maxSize);+ args.push(ANOMALY_KIND_CODES[spec.kind ?? "rolling"], spec.member, spec.windowMs, spec.maxSize);}const flat = await (redis as AnomalyRedis).anomalyObserve(...args);return specs.map((_, index) => ({diff --git a/server/src/services/tracker/botBlocking/README.md b/server/src/services/tracker/botBlocking/README.mdindex 028ca9030..90ae692fb 100644--- a/server/src/services/tracker/botBlocking/README.md+++ b/server/src/services/tracker/botBlocking/README.md@@ -1,8 +1,24 @@# Bot Blocking-This directory owns tracker-side bot detection for public `/track` ingestion. `trackEvent.ts` validates the payload, resolves the request IP, and calls `checkBotBlocking()`. If any bot method matches, the event is stored in ClickHouse `bot_events` with per-layer bot metadata.+This directory owns tracker-side bot detection for public `/track` ingestion. `trackEvent.ts` validates the payload, resolves the request IP, and calls `checkBotBlocking()`.-Detected bot requests are not inserted into the normal `events` table, so dashboard, report, replay-list, and usage queries do not need a bot filter.+## Detection and enforcement are separate++Detection runs for **every** site. Only trusted server-side ingestion skips it, because it is authenticated first-party traffic reporting its own IP and user agent, which several layers would convict outright.++Site-level `blockBots` decides what happens to a detection, not whether one is looked for. `checkBotBlocking()` returns `enforced` alongside the detection, and ingestion routes on it:++| detection | `blockBots` | event lands in | audit row |+| --- | --- | --- | --- |+| yes | on | β (kept out of `events`) | `bot_events` |+| yes | off | `events`, as normal | `bot_observations` |+| no | either | `events`, as normal | β |++Enforced detections are still not inserted into `events`, so dashboard, report, replay-list, and usage queries need no bot filter.++A site that has turned blocking off previously returned before any layer ran, which left it with neither protection nor any record of what it was receiving. `bot_observations` is that record β a forensic sample of the traffic each layer would have acted on, collected without changing a single number the site owner sees today.++It is a sample and not a ledger, so do not treat a count from it as "what blocking would have removed". The queue drops a batch on ClickHouse failure rather than retrying (the same as `bot_events` β an audit row must never cost an ingest), and it is written after the event itself, so a crash between the two loses the row and keeps the event. Compare shapes and ratios in it; do not reconcile its count against `events`.## Entry Point@@ -29,7 +45,9 @@ Current methods:- `bot_asn`: detects curated bot/scanner/AI provider ASNs as a standalone layer. Generic ipverse `hosting` ASNs are supporting evidence only and are recorded when another layer also matched.- `rate_anomaly`: detects request bursts and crawl-shaped behavior using in-memory sliding-window counters.-The client-side `_bs` value is a cached, weighted score computed once per page lifecycle. Strong signals such as automation APIs, impossible dimensions, or default automation viewport sizes can reach the blocking threshold alone; weaker signals such as SwiftShader, missing Chrome globals, and empty plugin lists only add supporting weight. The client also sends `_bsm`, a compact bitmask used for aggregate component counters. The server supplements that mask from validated screen dimensions so older scripts can still move `800x600`, `1024x768`, and impossible dimensions into the `client_signals` layer.+The client-side `_bs` value is a cached, weighted score computed once per page lifecycle. Strong signals such as automation APIs, impossible dimensions, or default automation viewport sizes can reach the blocking threshold alone; weaker signals such as SwiftShader, missing Chrome globals, and empty plugin lists only add supporting weight. The client also sends `_bsm`, a compact bitmask used for aggregate component counters. The server supplements that mask from validated screen dimensions so older scripts can still move `800x600`, `1024x768`, `1280x1200`, square screens, and impossible dimensions into the `client_signals` layer.++One signal is server-only: `missingScreenDimensions`. A browser always has `window.screen`, so only the server can observe that a hit arrived carrying no dimensions at all. It is weak (weight 1, never strong) and skipped for mobile sites, where a native SDK has no screen to report. It exists so the geometry rules β which are skipped outright when nothing is reported β leave a trace of having been skipped rather than the hit passing silently.## Logging@@ -48,17 +66,20 @@ Each detection object contains compact method-specific details such as matched U`botDetectionStats.ts` also logs process-lifetime totals for tracker requests that reach the bot-blocking entry point every 5 seconds:- `totalRequests`-- `totalBotRequests`+- `totalBotRequests` β what was **detected**+- `totalEnforcedBotRequests` β the subset that was actually diverted out of `events`. The gap between the two is what sites with blocking disabled are absorbing.- `botRequestPercentage`- `botDetectionTotals` by method- `clientBotScoreHistogram` with buckets for missing, `0`, `1`, `2`, and `3+`-- `clientBotSignalTotals` for `_bsm` components: missing mask, automation API, zero outer dimensions, missing Chrome global, SwiftShader, empty plugins, default `800x600` viewport, default `1024x768` viewport, impossible dimensions, outer dimension anomalies, plugin/API absence, and unknown mask bits+- `clientBotSignalTotals` for `_bsm` components: missing mask, automation API, zero outer dimensions, missing Chrome global, SwiftShader, empty plugins, default `800x600` viewport, default `1024x768` viewport, default `1280x1200` viewport, square screen, missing screen dimensions, impossible dimensions, outer dimension anomalies, plugin/API absence, and unknown mask bits++`missingMask` and the histogram's `missing` bucket follow what the **client** reported, so they measure tracker adoption. The per-signal totals count every bit we ended up with, inferred server-side or not β which is what makes a newly defined bit appearing in production the proof that a deploy actually took.A request can increment multiple method totals if multiple methods detected it, so method totals can sum higher than `totalBotRequests`.## Storage-`botEventQueue.ts` enriches detected bot requests with the same browser, device, and geolocation basics as normal events, then inserts a compact audit row into `bot_events`.+`botEventQueue.ts` enriches detected bot requests with the same browser, device, and geolocation basics as normal events, then inserts a compact audit row. It exports two queues over one implementation: `botEventQueue` writes to `bot_events` (enforced), and `botObservationQueue` writes to `bot_observations` (detected but not enforced). The schemas are identical on purpose, so the same forensic query runs against either.The table keeps only the columns needed to inspect bot traffic:@@ -67,8 +88,12 @@ The table keeps only the columns needed to inspect bot traffic:- ASN fields: `asn`, `asn_org`- one boolean column for each detection layer- UA classification fields: `matched_ua_pattern`, `bot_category`+- client evidence: `client_bot_score`, `client_signal_mask`+- anomaly evidence: `anomaly_reasons` (the rules that fired, comma-separated) and `anomaly_score`. The boolean layer columns say *that* the anomaly layer convicted; these say which of its dozen rules did, which is the only form in which the row can be argued with.++Both tables are healed by `ensureBotEventsColumns` at init, so a column added here reaches tables created by earlier versions.-`bot_events` has a 3-month TTL. The main `events` table has no bot-specific columns or bot-specific TTL.+`bot_events` has a 3-month TTL; `bot_observations` has a 30-day TTL, since it exists to answer a current question rather than to hold long-term history. The main `events` table has no bot-specific columns or bot-specific TTL β any such columns present on a production `events` table are drift, and nothing writes them.## ASN Data@@ -84,19 +109,30 @@ npm run update:datacenter-asns## Rate Anomaly Layer-`anomalyScorer.ts` uses per-process memory only. It tracks short rolling windows for:+`anomalyScorer.ts` keeps its counters in Redis, so every worker and replica shares one view; an in-process fallback takes over automatically if Redis is unavailable, and `DISABLE_REDIS_ANOMALY=true` pins scoring to it. All of a request's counters ride one Lua round-trip.-- events per `siteId + IP + UA hash`-- events per `siteId + IP`+It tracks:++- events per `siteId + IP + UA hash`, over 10s and 60s+- interaction-event bursts per visitor tuple, on their own beyond-human threshold- distinct paths per visitor tuple-- distinct UAs per IP-- distinct hostnames per IP+- events, distinct UAs, and distinct hostnames per IP- site-wide volume for a UA hash- missing client bot score volume+- browser-version spread within a cohort (`siteId + screen + language + browser family`)+- events per actor per day+- enumeration shape per cohort, in 15-minute buckets++Reasons come back in two lists, and the distinction is structural rather than a matter of how the scores happen to add up:++- **convicting** β keyed on a single actor, or on a distribution no organic population produces. These can open a conviction.+- **supporting** β keyed on dimensions real visitors share (an IP, a popular browser), plus long-window volume, where one address may be an entire office behind a SASE gateway. These are only ever added to a score convicting evidence has already opened.-The layer is score-based. Strong rules, such as more than 30 events in 10 seconds for one visitor tuple or more than 25 distinct paths in 60 seconds, can block alone. Weak rules, such as missing client score, only add context unless paired with stronger behavior.+Three rules deserve their own note:-This catches obvious floods and fast crawlers, but it is local to a Node process. If the signal is useful in production, move the same keys and thresholds to Redis so counters are shared across workers and containers.+- **Cohort version uniformity** convicts, and is the only rule that can catch a paced distributed crawler β one that stays under every per-identity threshold by design. Its key must carry the browser family: without it, Chrome, Safari and Firefox version numbers pool into one distribution, which inflates the distinct-version count and depresses the modal share, i.e. manufactures exactly the two conditions the rule fires on.+- **Events per actor per day** is supporting-only and scores 1. Its actor is the exact request IP, deliberately *not* the /24 identity bucket the per-visitor rules use β bucketing merges a datacenter range or a SASE egress pool into one actor, and a day is long enough for that to reach any threshold on legitimate traffic. It is additionally dropped outright for an address showing more than three user agents in five minutes, which is what a shared gateway looks like.+- **Enumeration** (`enumerationObserver.ts`) is in shadow mode: it measures path novelty, events per actor, and direct share per cohort, requires two consecutive qualifying buckets, and **never scores**. It logs. Promoting it should follow from reading those logs, not from a threshold nudge. Its direct share is measured on the raw referrer, before self-referrers are cleared for storage β a stored empty referrer means "direct *or* internal navigation", which most ordinary browsing satisfies.## Trust Boundariesdiff --git a/server/src/services/tracker/botBlocking/anomalyScorer.test.ts b/server/src/services/tracker/botBlocking/anomalyScorer.test.tsindex 13f487c82..9621badd7 100644--- a/server/src/services/tracker/botBlocking/anomalyScorer.test.ts+++ b/server/src/services/tracker/botBlocking/anomalyScorer.test.ts@@ -10,6 +10,7 @@ vi.mock("../../../db/redis/redis.js", () => ({}));import {+ BucketedCounter,observeTrackingAnomaly,resetAnomalyScorerForTests,setRedisAnomalyEnabledForTests,@@ -133,6 +134,54 @@ describe("observeTrackingAnomaly (in-process fallback)", () => {});});+describe("BucketedCounter key bound", () => {+ // `cleanup` only frees an entry once its window has passed, and the actor+ // counter's window is a day β so without a key bound a Redis outage would+ // leave it holding an entry for every address seen all day.+ const WINDOW = 60_000;++ it("stops admitting new keys at the cap and reports them as zero", () => {+ const counter = new BucketedCounter();++ expect(counter.observe("a", 0, WINDOW, 2)).toBe(1);+ expect(counter.observe("b", 0, WINDOW, 2)).toBe(1);+ // At capacity: "c" is never held, so it counts as nothing rather than+ // displacing someone. Under-reporting is the safe direction β this evidence+ // can only ever raise a score, so a suppressed count cannot accuse anyone.+ expect(counter.observe("c", 0, WINDOW, 2)).toBe(0);+ expect(counter.observe("c", 0, WINDOW, 2)).toBe(0);+ });++ it("keeps counting the keys it already holds", () => {+ const counter = new BucketedCounter();++ counter.observe("a", 0, WINDOW, 1);+ counter.observe("b", 0, WINDOW, 1);++ expect(counter.observe("a", 0, WINDOW, 1)).toBe(2);+ });++ it("rolls an existing key into a new window even at capacity", () => {+ // A rollover replaces an entry rather than adding one, so refusing it would+ // freeze the counter at its first window's membership forever.+ const counter = new BucketedCounter();+ counter.observe("a", 0, WINDOW, 1);++ expect(counter.observe("a", WINDOW, WINDOW, 1)).toBe(1);+ });++ it("readmits keys once cleanup has freed the window", () => {+ const counter = new BucketedCounter();+ counter.observe("a", 0, WINDOW, 1);++ expect(counter.observe("b", 0, WINDOW, 1)).toBe(0);++ counter.cleanup(WINDOW * 3, WINDOW);++ expect(counter.observe("b", WINDOW * 3, WINDOW, 1)).toBe(1);+ });+});+describe("cohort version uniformity (in-process fallback)", () => {beforeEach(() => {resetAnomalyScorerForTests();@@ -205,6 +254,218 @@ describe("cohort version uniformity (in-process fallback)", () => {expect(result.counters.cohortEvents60s).toBe(0);expect(result.counters.cohortDistinctVersions60s).toBe(0);});++ it("skips the cohort counter when the language is unset", async () => {+ // An unset language would otherwise merge every locale on the Site into one+ // cohort, which is the same defect as merging every browser.+ const result = await observeTrackingAnomaly({ ...cohortInput, language: undefined });++ expect(result.counters.cohortEvents60s).toBe(0);+ });++ /**+ * The regression fixture for the cohort key. Four browser families share one+ * screen and language, each peaked on its own current version the way real+ * populations are. Pooled into a single distribution β which is what the key+ * did before it carried the family β that reads as sixteen versions at a ~21%+ * modal share: every condition of the rule, met by nothing but ordinary+ * traffic on a busy Site.+ *+ * Each family is given enough volume to clear the rule's floor on its own, so+ * this passes because the distributions are separated, not because any of them+ * is too small to count.+ */+ it("does not convict a mixed-browser population sharing one screen and language", async () => {+ const families = [+ (version: number) => desktopChrome(version),+ (version: number) => `Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:${version}.0) Gecko/20100101 Firefox/${version}.0`,+ (version: number) =>+ `Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/${version}.0 Safari/605.1.15`,+ (version: number) =>+ `Mozilla/5.0 (Linux; Android 13) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/${version}.0 Chrome/115.0.0.0 Mobile Safari/537.36`,+ ];+ const dominantVersions = [150, 130, 18, 23];++ let result;+ let index = 0;+ for (const [familyIndex, buildUserAgent] of families.entries()) {+ const dominant = dominantVersions[familyIndex];+ for (let event = 0; event < 320; event++) {+ // 85% on the family's current version, the rest on a stale tail.+ const version = event % 100 < 85 ? dominant : dominant - 1 - (event % 3);+ result = await observeTrackingAnomaly({+ ...cohortInput,+ ipAddress: `198.51.100.${index % 254}`,+ userAgent: buildUserAgent(version),+ pathname: `/article/${index}`,+ nowMs: baseInput.nowMs + index,+ });+ index++;+ }+ }++ expect(result?.reasons.map(reason => reason.rule)).not.toContain("cohort_version_uniformity_60s");+ expect(result?.isAnomalous).toBe(false);+ // The last family's cohort saw only its own traffic, not all 1,280 events.+ expect(result?.counters.cohortEvents60s).toBe(320);+ });++ /**+ * Which family a real user agent resolves to, pinned through the cohort+ * counter: two agents sharing a family accumulate into one counter, and two+ * that don't each keep their own. Chromium derivatives split on where their+ * token sits relative to `Chrome/`, and that split is deliberate β see+ * `getBrowserIdentity`.+ */+ describe("browser family resolution against real user agents", () => {+ const chrome120 =+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36";+ const edge120 = `${chrome120} Edg/120.0.2210.144`;+ const opera120 = `${chrome120} OPR/106.0.4998.70`;+ const samsung23 =+ "Mozilla/5.0 (Linux; Android 13; SAMSUNG SM-S918B) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/23.0 Chrome/115.0.0.0 Mobile Safari/537.36";+ const chromeIos120 =+ "Mozilla/5.0 (iPhone; CPU iPhone OS 17_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/120.0.6099.101 Mobile/15E148 Safari/604.1";+ const firefoxIos121 =+ "Mozilla/5.0 (iPhone; CPU iPhone OS 17_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/121.0 Mobile/15E148 Safari/605.1.15";+ const safari17 =+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15";++ /**+ * Ten events per agent, in order; returns each agent's final cohort event+ * count. A count of ten means that agent had a cohort to itself; anything+ * higher means it landed in one an earlier agent had already filled.+ */+ async function cohortSizesPerAgent(userAgents: string[]) {+ const sizes: number[] = [];+ let index = 0;+ for (const userAgent of userAgents) {+ let result;+ for (let event = 0; event < 10; event++) {+ result = await observeTrackingAnomaly({+ ...cohortInput,+ ipAddress: `198.51.100.${index % 254}`,+ userAgent,+ pathname: `/article/${index}`,+ nowMs: baseInput.nowMs + index,+ });+ index++;+ }+ sizes.push(result!.counters.cohortEvents60s);+ }+ return sizes;+ }++ it("pools Chromium derivatives that report their Chromium version", async () => {+ // Edge and Opera put their own token after `Chrome/`, so the leftmost+ // match is the Chromium version they actually ship. Chrome 120, Edge 120+ // and Opera 106-on-Chromium-120 do belong on one distribution: they track+ // the same release train and peak together, so 10 / 20 / 30 is the count+ // of one shared cohort filling up.+ expect(await cohortSizesPerAgent([chrome120, edge120, opera120])).toEqual([10, 20, 30]);+ });++ it("keeps one version distribution for a pooled derivative", async () => {+ // The pooling has to hold in the distribution too, not just the key: three+ // agents, one Chromium version between them.+ await cohortSizesPerAgent([chrome120, edge120, opera120]);+ const result = await observeTrackingAnomaly({ ...cohortInput, userAgent: edge120 });++ expect(result.counters.cohortDistinctVersions60s).toBe(1);+ });++ it("separates families whose version numbers move independently", async () => {+ // Samsung Internet and the iOS browsers put their token first, so each+ // keeps its own distribution β correct, because their version numbers are+ // on their own release schedules and would never peak together. Every+ // agent seeing exactly its own ten is that separation.+ const sizes = await cohortSizesPerAgent([chrome120, samsung23, chromeIos120, firefoxIos121, safari17]);++ expect(sizes).toEqual([10, 10, 10, 10, 10]);+ });+ });++ it("still convicts a fleet that rotates versions within one browser family", async () => {+ // The fix must not cost the rule its target: a fleet rotating a fixed user+ // agent list stays inside one family, so separating families changes nothing+ // about what it looks like.+ const versions = Array.from({ length: 480 }, (_, index) => 103 + (index % 16));++ const result = await driveCohort(versions);++ expect(result.reasons.map(reason => reason.rule)).toContain("cohort_version_uniformity_60s");+ });+});++describe("enumeration observer (shadow mode)", () => {+ beforeEach(() => {+ resetAnomalyScorerForTests();+ setRedisAnomalyEnabledForTests(false);+ });++ const enumerationInput = {+ ...baseInput,+ screenWidth: 1920,+ screenHeight: 1080,+ language: "en-US",+ referrer: "",+ };++ /**+ * One 15-minute bucket of enumerating traffic: a fresh identity per hit, a+ * path nobody has requested before, no referrer. Every per-identity rule sees+ * a single quiet visitor, which is the whole reason this shape needs its own+ * measurement.+ */+ async function driveEnumerationBucket(bucketStartMs: number) {+ let result;+ for (let index = 0; index < 320; index++) {+ result = await observeTrackingAnomaly({+ ...enumerationInput,+ // A distinct address per hit β the crawler mints a fresh identity each+ // time, which is exactly why no per-identity rule can see it.+ ipAddress: `198.51.${Math.floor(index / 254)}.${index % 254}`,+ pathname: `/status/online/gender/f/hair/white/page/${bucketStartMs}-${index}`,+ nowMs: bucketStartMs + index,+ });+ }+ return result;+ }++ it("observes an enumerating cohort without scoring it", async () => {+ const result = await driveEnumerationBucket(1_000_000);++ expect(result?.enumeration).toMatchObject({ qualifies: true, sustained: false });+ // The observation exists and the verdict does not move. Shadow mode is the+ // entire point: this rule is collecting evidence, not deciding anything.+ expect(result?.score).toBe(0);+ expect(result?.isAnomalous).toBe(false);+ expect(result?.convictingReasons).toEqual([]);+ });++ it("marks the cohort once the shape holds across two consecutive buckets", async () => {+ await driveEnumerationBucket(1_000_000);+ const result = await driveEnumerationBucket(1_000_000 + 15 * 60 * 1000);++ expect(result?.enumeration).toMatchObject({ qualifies: true, sustained: true });+ // Still not a verdict.+ expect(result?.isAnomalous).toBe(false);+ });++ it("does not carry a streak across a gap in the buckets", async () => {+ await driveEnumerationBucket(1_000_000);+ const result = await driveEnumerationBucket(1_000_000 + 3 * 15 * 60 * 1000);++ expect(result?.enumeration).toMatchObject({ qualifies: true, sustained: false });+ });++ it("does not observe when the hit arrived with a referrer", async () => {+ // Only a direct hit increments the direct counter, so only a direct hit can+ // read a meaningful share of them.+ const result = await observeTrackingAnomaly({ ...enumerationInput, referrer: "https://news.example.com/" });++ expect(result.enumeration).toBeUndefined();+ });});describe("observeTrackingAnomaly (Redis-backed)", () => {@@ -215,15 +476,16 @@ describe("observeTrackingAnomaly (Redis-backed)", () => {});it("sends one spec per enabled counter and maps results back by counter name", async () => {- // 8 counters, all enabled (path + host present, no client score).- mocks.anomalyObserve.mockResolvedValue(rollingReadings(31, 5, 2, 9, 3, 1, 4, 7));+ // 9 counters (path + host present, no client score). The cohort and+ // enumeration counters stay out: this input has no screen dimensions.+ mocks.anomalyObserve.mockResolvedValue(rollingReadings(31, 5, 2, 9, 3, 1, 4, 7, 12));const result = await observeTrackingAnomaly({ ...baseInput, hasClientBotScore: false });expect(mocks.anomalyObserve).toHaveBeenCalledTimes(1);const [nowMs, specs] = mocks.anomalyObserve.mock.calls[0];expect(nowMs).toBe(baseInput.nowMs);- expect(specs).toHaveLength(8);+ expect(specs).toHaveLength(9);expect(specs.map((spec: { key: string }) => spec.key)).toEqual([expect.stringContaining("bot:a:te10:"),expect.stringContaining("bot:a:te60:"),@@ -233,6 +495,7 @@ describe("observeTrackingAnomaly (Redis-backed)", () => {expect.stringContaining("bot:a:idh:"),expect.stringContaining("bot:a:sue:"),expect.stringContaining("bot:a:mcs:"),+ expect.stringContaining("bot:a:av:"),]);expect(result.counters.tupleEvents10s).toBe(31);@@ -241,9 +504,48 @@ describe("observeTrackingAnomaly (Redis-backed)", () => {expect(result.reasons.map(reason => reason.rule)).toContain("tuple_events_10s");});+ // Spec order for baseInput (client score present, so no `mcs`):+ // te10, te60, tdp, ie60, idua, idh, sue, av.+ it("never convicts on long-window volume alone", async () => {+ // 1,500 events in a day from one actor β five times the measured p99.99 β+ // and nothing else. A scraper looks like this; so does a single office+ // behind one SASE egress address, which is why this can only corroborate.+ mocks.anomalyObserve.mockResolvedValue(rollingReadings(1, 2, 1, 10, 1, 1, 5, 1500));++ const result = await observeTrackingAnomaly(baseInput);++ expect(result.isAnomalous).toBe(false);+ expect(result.score).toBe(0);+ expect(result.supportingReasons.map(reason => reason.rule)).toContain("actor_events_1d");+ expect(result.convictingReasons).toEqual([]);+ });++ it("drops long-window volume entirely for an actor showing many user agents", async () => {+ // The shared-egress signature: one address, many browsers. Corporate SASE+ // and VPN gateways sit here permanently, and their daily counts are large+ // and legitimate β the rule must not even record against them.+ mocks.anomalyObserve.mockResolvedValue(rollingReadings(1, 2, 1, 10, 12, 1, 5, 1500));++ const result = await observeTrackingAnomaly(baseInput);++ expect(result.reasons.map(reason => reason.rule)).not.toContain("actor_events_1d");+ expect(result.reasons.map(reason => reason.rule)).toContain("ip_distinct_user_agents_5m");+ });++ it("adds long-window volume to a score that convicting evidence already opened", async () => {+ // tuple_events_10s (4, convicting) + actor_events_1d (1, supporting).+ mocks.anomalyObserve.mockResolvedValue(rollingReadings(31, 2, 1, 10, 1, 1, 5, 1500));++ const result = await observeTrackingAnomaly(baseInput);++ expect(result.convictingReasons.map(reason => reason.rule)).toEqual(["tuple_events_10s"]);+ expect(result.score).toBe(5);+ expect(result.isAnomalous).toBe(true);+ });+it("omits conditional counters that don't apply and reports them as zero", async () => {// No pathname, no hostname, client score present β 3 counters dropped.- mocks.anomalyObserve.mockResolvedValue(rollingReadings(1, 1, 1, 1, 1));+ mocks.anomalyObserve.mockResolvedValue(rollingReadings(1, 1, 1, 1, 1, 1));const result = await observeTrackingAnomaly({...baseInput,@@ -253,7 +555,7 @@ describe("observeTrackingAnomaly (Redis-backed)", () => {});const [, specs] = mocks.anomalyObserve.mock.calls[0];- expect(specs).toHaveLength(5);+ expect(specs).toHaveLength(6);expect(result.counters.tupleDistinctPaths60s).toBe(0);expect(result.counters.ipDistinctHosts60s).toBe(0);expect(result.counters.missingClientScore60s).toBe(0);diff --git a/server/src/services/tracker/botBlocking/anomalyScorer.ts b/server/src/services/tracker/botBlocking/anomalyScorer.tsindex aca673e48..c3fa806d0 100644--- a/server/src/services/tracker/botBlocking/anomalyScorer.ts+++ b/server/src/services/tracker/botBlocking/anomalyScorer.ts@@ -1,5 +1,11 @@import { anomalyObserve, type AnomalyCounterSpec } from "../../../db/redis/redis.js";import { createServiceLogger } from "../../../lib/logger/logger.js";+import {+ ENUMERATION_BUCKET_MS,+ observeEnumeration,+ resetEnumerationObserverForTests,+ type EnumerationObservation,+} from "./enumerationObserver.js";const SECOND = 1000;const MINUTE = 60 * SECOND;@@ -9,6 +15,12 @@ const CLEANUP_INTERVAL_MS = 60 * SECOND;const MAX_COUNTER_BUCKET_SIZE = 512;const MAX_DISTINCT_BUCKET_SIZE = 512;const MAX_DISTRIBUTION_FIELDS = 128;+// Only bounds the in-process fallback's distinct sets; Redis uses HyperLogLog,+// which has no equivalent limit.+const MAX_LOCAL_DISTINCT_VALUES = 4096;+// Bounds how many keys the in-process fallback's plain counters hold at once.+// Redis needs no equivalent, since it expires each key on its own window.+const MAX_LOCAL_COUNTER_KEYS = 100_000;/*** Cohort-uniformity rule. A cohort is one exact device fingerprint on a site β@@ -33,6 +45,32 @@ const COHORT_MIN_EVENTS_60S = 300;const COHORT_MIN_DISTINCT_VERSIONS = 8;const COHORT_MAX_MODAL_SHARE = 0.25;+/**+ * Long-window per-actor volume. Every other rate rule here works in 10- and+ * 60-second windows, which a crawler paced at one hit every few seconds slips+ * under indefinitely: 4,000 pageviews a day from a single datacenter address+ * never trips any of them. Measured over production, events per (site, actor,+ * day) has p99.99 = 303 and only 69 actors out of 1.03M exceed 1,000.+ *+ * It is supporting-only, and permanently so. The actor is the exact request IP β+ * note this is NOT the identity actor, which buckets datacenter egress to a /24+ * and would merge far more aggressively. Even at exact-IP granularity an office+ * or a carrier NAT is one address shared by hundreds of people, and a four-figure+ * daily count from one is unremarkable. A rule that convicts on this number+ * convicts an office.+ *+ * The guard below drops it for actors showing many user agents, which is what+ * shared egress usually looks like. It is a weak test, deliberately: a managed+ * fleet where everyone runs the same browser build defeats it, which is another+ * reason the score is 1 and the rule can only ever corroborate. Scoring it+ * higher would let it combine with `missing_client_score_60s` and one crowd rule+ * to reach the threshold on evidence that is entirely shared-dimension.+ */+const ACTOR_EVENTS_1D_THRESHOLD = 1000;+const ACTOR_EVENTS_1D_SCORE = 1;+const PROXY_MERGE_MAX_USER_AGENTS = 3;+const DAY = 24 * 60 * MINUTE;+const logger = createServiceLogger("anomaly-scorer");// Counters live in Redis so every worker/replica shares one view; an in-process@@ -63,6 +101,11 @@ export interface AnomalyCounters {cohortEvents60s: number;cohortTopVersionEvents60s: number;cohortDistinctVersions60s: number;+ actorEvents1d: number;+ enumerationEvents15m: number;+ enumerationDistinctPaths15m: number;+ enumerationDistinctActors15m: number;+ enumerationDirectEvents15m: number;}/**@@ -87,6 +130,13 @@ export interface AnomalyInput {hostname?: string;pathname?: string;eventType?: string;+ /**+ * The referrer as it arrived, NOT the stored value. Self-referrers are cleared+ * before storage, so a stored empty referrer means "direct or internal+ * navigation" β two very different populations. The enumeration observer needs+ * the raw distinction.+ */+ referrer?: string;hasClientBotScore: boolean;/** Cohort dimensions; the cohort rule is skipped unless all three resolve. */screenWidth?: number;@@ -98,8 +148,28 @@ export interface AnomalyInput {export interface AnomalyResult {isAnomalous: boolean;score: number;+ /**+ * Rules describing a single actor. Only these can open a conviction, and the+ * split is the point: keeping the two kinds of evidence in one list left the+ * invariant resting on how the scores happened to add up, one threshold edit+ * away from letting shared-dimension evidence convict on its own.+ */+ convictingReasons: AnomalyReason[];+ /**+ * Rules keyed on dimensions real visitors share β an IP, a popular browser on+ * a busy site β plus long-window volume, where the actor may be an entire+ * office behind one egress. These raise a score that convicting evidence has+ * already opened, and can never open one themselves.+ */+ supportingReasons: AnomalyReason[];+ /** Both lists, in that order, for the audit record written to ClickHouse. */reasons: AnomalyReason[];counters: AnomalyCounters;+ /**+ * Shadow-mode enumeration reading. Recorded, never scored β nothing in this+ * module reads it back into `score`.+ */+ enumeration?: EnumerationObservation;}class RollingCounter {@@ -196,6 +266,89 @@ class BucketedDistributionCounter {}}+/**+ * Plain count inside a fixed time bucket, mirroring the Redis `counter` kind.+ * Tumbling like the distribution counter above: the bucket index is part of the+ * caller's key, so a new bucket simply starts a new entry.+ *+ * Bounded by key count, which is the dimension that actually runs away here.+ * `cleanup` only frees an entry once its window has passed, and `actorEvents1d`+ * is keyed per actor address on a one-day window β so during a Redis outage an+ * unbounded map would hold an entry for every address seen all day.+ *+ * Past the cap a new key counts as zero rather than evicting an existing one.+ * Both halves of that are deliberate: eviction would let a heavy actor clear its+ * own count by cycling keys, and zero keeps the fallback failing towards saying+ * nothing, the same direction as the distinct-value cap below.+ */+export class BucketedCounter {+ private buckets = new Map<string, { startMs: number; count: number }>();++ observe(key: string, nowMs: number, windowMs: number, maxKeys: number): number {+ const bucketStartMs = Math.floor(nowMs / windowMs) * windowMs;+ let bucket = this.buckets.get(key);+ if (!bucket || bucket.startMs !== bucketStartMs) {+ // A key already present is being rolled over, not added, so it is let+ // through at capacity β only a genuinely new key can grow the map.+ if (!bucket && this.buckets.size >= maxKeys) {+ return 0;+ }+ bucket = { startMs: bucketStartMs, count: 0 };+ this.buckets.set(key, bucket);+ }+ bucket.count++;+ return bucket.count;+ }++ cleanup(nowMs: number, windowMs: number) {+ for (const [key, bucket] of this.buckets) {+ if (bucket.startMs + windowMs < nowMs) {+ this.buckets.delete(key);+ }+ }+ }++ clear() {+ this.buckets.clear();+ }+}++/**+ * Distinct-value count inside a fixed time bucket, mirroring the Redis+ * `cardinality` kind. Where Redis uses a HyperLogLog, this holds the values, so+ * it is capped β above the cap it undercounts. That direction is deliberate: a+ * suppressed distinct count lowers path novelty, so the fallback can only fail+ * towards saying nothing, never towards a false accusation.+ */+class BucketedDistinctCounter {+ private buckets = new Map<string, { startMs: number; values: Set<string> }>();++ observe(key: string, value: string, nowMs: number, windowMs: number, maxValues: number): number {+ const bucketStartMs = Math.floor(nowMs / windowMs) * windowMs;+ let bucket = this.buckets.get(key);+ if (!bucket || bucket.startMs !== bucketStartMs) {+ bucket = { startMs: bucketStartMs, values: new Set<string>() };+ this.buckets.set(key, bucket);+ }+ if (bucket.values.size < maxValues) {+ bucket.values.add(value);+ }+ return bucket.values.size;+ }++ cleanup(nowMs: number, windowMs: number) {+ for (const [key, bucket] of this.buckets) {+ if (bucket.startMs + windowMs < nowMs) {+ this.buckets.delete(key);+ }+ }+ }++ clear() {+ this.buckets.clear();+ }+}+interface AnomalyDistributionReading {total: number;top: number;@@ -215,19 +368,51 @@ const ipDistinctHosts60s = new RollingDistinctCounter();const cohortBrowserVersions60s = new BucketedDistributionCounter();+const actorEvents1d = new BucketedCounter();+const enumerationEvents15m = new BucketedCounter();+const enumerationDirectEvents15m = new BucketedCounter();+const enumerationDistinctPaths15m = new BucketedDistinctCounter();+const enumerationDistinctActors15m = new BucketedDistinctCounter();+let lastCleanupMs = 0;+const BRANDED_BROWSER_PATTERN = /(Edg|EdgA|EdgiOS|OPR|SamsungBrowser|Firefox|FxiOS|CriOS|Chrome)\/(\d{1,4})/;+const GENERIC_VERSION_PATTERN = /Version\/(\d{1,4})/;+/**- * Major version of the reported browser engine, used as the cohort rule's- * distribution dimension. A deliberately loose parse: the rule only cares that- * rotating user agents map to different values and stable ones map to the same- * value, so any token that moves with the browser release works.+ * The browser family and major version behind a user agent, used as the cohort+ * rule's grouping key and distribution dimension respectively. A deliberately+ * loose parse: the rule only cares that rotating user agents map to different+ * versions and stable ones map to the same version, so any token that moves with+ * the browser release works.+ *+ * Both come out of the same match, which is the point β a version number is only+ * comparable within its own family. Chrome 120 and Safari 17 are not two points+ * on one distribution, and pooling them is what made a mixed-browser cohort look+ * flat (see the cohort key below).+ *+ * Chromium derivatives that put their own token AFTER `Chrome/` β Edge (`Edg`)+ * and Opera (`OPR`) β resolve to `chrome`, because the leftmost match wins.+ * That is the behaviour we want and not an accident of the regex: their+ * `Chrome/` token carries the Chromium version they actually ship, so Edge 120+ * and Chrome 120 genuinely belong on one distribution and both peak together.+ * Derivatives whose token comes first β Samsung Internet, Chrome and Firefox on+ * iOS β get their own family, which is also right, since their version numbers+ * move independently. `anomalyScorer.test.ts` pins all of these against real+ * user agents.*/-function getBrowserMajorVersion(userAgent: string): string {- const match =- /(?:Edg|EdgA|EdgiOS|OPR|SamsungBrowser|Firefox|FxiOS|CriOS|Chrome)\/(\d{1,4})/.exec(userAgent) ??- /Version\/(\d{1,4})/.exec(userAgent);- return match ? match[1] : "";+function getBrowserIdentity(userAgent: string): { family: string; majorVersion: string } {+ const branded = BRANDED_BROWSER_PATTERN.exec(userAgent);+ if (branded) {+ return { family: branded[1].toLowerCase(), majorVersion: branded[2] };+ }++ const generic = GENERIC_VERSION_PATTERN.exec(userAgent);+ if (generic) {+ return { family: "safari", majorVersion: generic[1] };+ }++ return { family: "", majorVersion: "" };}// Unique-per-event token so each observation is a distinct sorted-set member in@@ -308,6 +493,11 @@ function maybeCleanup(nowMs: number) {ipDistinctUserAgents5m.cleanup(nowMs, 5 * MINUTE);ipDistinctHosts60s.cleanup(nowMs, MINUTE);cohortBrowserVersions60s.cleanup(nowMs, MINUTE);+ actorEvents1d.cleanup(nowMs, DAY);+ enumerationEvents15m.cleanup(nowMs, ENUMERATION_BUCKET_MS);+ enumerationDirectEvents15m.cleanup(nowMs, ENUMERATION_BUCKET_MS);+ enumerationDistinctPaths15m.cleanup(nowMs, ENUMERATION_BUCKET_MS);+ enumerationDistinctActors15m.cleanup(nowMs, ENUMERATION_BUCKET_MS);}// A single counter described once for both backends: its Redis key/member and an@@ -316,7 +506,7 @@ function maybeCleanup(nowMs: number) {interface CounterPlan {name: keyof AnomalyCounters;enabled: boolean;- kind?: "rolling" | "distribution";+ kind?: AnomalyCounterSpec["kind"];redisKey: string;member: string;windowMs: number;@@ -331,7 +521,28 @@ interface CounterPlan {observeLocalDistribution?: (nowMs: number) => AnomalyDistributionReading;}-function buildCounterPlan(input: AnomalyInput, nowMs: number): CounterPlan[] {+/**+ * The cohort bucket the enumeration observer reads, carried out of the plan so+ * the observer does not have to re-derive the cohort key. Null when the event+ * has no cohort or no path, which is when there is nothing to measure.+ */+interface EnumerationPlan {+ cohortKey: string;+ bucket: number;+ /**+ * Whether this hit arrived with no referrer at all, judged on the raw value+ * before self-referrers are cleared for storage. Only a direct hit increments+ * the direct counter, so only a direct hit can read a meaningful share.+ */+ isDirect: boolean;+}++interface AnomalyPlan {+ counters: CounterPlan[];+ enumeration: EnumerationPlan | null;+}++function buildCounterPlan(input: AnomalyInput, nowMs: number): AnomalyPlan {const siteId = input.siteId;const ipAddress = normalizeDimension(input.ipAddress);const userAgentHash = hashValue(input.userAgent || "");@@ -347,17 +558,37 @@ function buildCounterPlan(input: AnomalyInput, nowMs: number): CounterPlan[] {// The cohort's time bucket lives in the key, so the Redis hash expires instead// of needing its own pruning. Skipped unless every dimension resolves β// a partial fingerprint would merge unrelated visitors into one cohort.- const browserMajorVersion = getBrowserMajorVersion(input.userAgent || "");+ //+ // The browser family is part of the key, not just the measured dimension.+ // Without it, one cohort pooled Chrome, Safari, Firefox and Edge version+ // numbers into a single distribution, which inflates the distinct-version+ // count and depresses the modal share β the exact two conditions this rule+ // convicts on. On a large site with a common screen and language that is a+ // false-positive path into a convicting rule, so the key must separate them.+ // Language is required for the same reason: an unset language would merge+ // every locale on the site into one cohort.+ const language = normalizeDimension(input.language);+ const { family: browserFamily, majorVersion: browserMajorVersion } = getBrowserIdentity(input.userAgent || "");const hasCohort =+ browserFamily !== "" &&browserMajorVersion !== "" &&+ language !== "" &&typeof input.screenWidth === "number" &&typeof input.screenHeight === "number" &&input.screenWidth > 0 &&input.screenHeight > 0;- const cohortKey = `${siteId}:${input.screenWidth}x${input.screenHeight}:${normalizeDimension(input.language)}`;+ const cohortKey = `${siteId}:${input.screenWidth}x${input.screenHeight}:${language}:${browserFamily}`;const cohortBucket = Math.floor(nowMs / MINUTE);- return [+ // Enumeration shape is measured over the same cohort, on a much longer bucket:+ // a crawler paced at one hit per identity produces nothing legible in a+ // minute. It needs a path, since path novelty is the whole measurement.+ const hasEnumerationCohort = hasCohort && pathname !== "";+ const enumerationBucket = Math.floor(nowMs / ENUMERATION_BUCKET_MS);+ const isDirect = normalizeDimension(input.referrer) === "";+ const dayBucket = Math.floor(nowMs / DAY);++ const counters: CounterPlan[] = [{name: "tupleEvents10s",enabled: !isInteraction,@@ -453,7 +684,67 @@ function buildCounterPlan(input: AnomalyInput, nowMs: number): CounterPlan[] {observeLocalDistribution: now =>cohortBrowserVersions60s.observe(cohortKey, browserMajorVersion, now, MINUTE, MAX_DISTRIBUTION_FIELDS),},+ {+ name: "actorEvents1d",+ enabled: true,+ kind: "counter",+ redisKey: `bot:a:av:${ipKey}:${dayBucket}`,+ member: "",+ windowMs: DAY,+ maxSize: 0,+ observeLocal: now => actorEvents1d.observe(ipKey, now, DAY, MAX_LOCAL_COUNTER_KEYS),+ },+ {+ name: "enumerationEvents15m",+ enabled: hasEnumerationCohort,+ kind: "counter",+ redisKey: `bot:e:ev:${cohortKey}:${enumerationBucket}`,+ member: "",+ windowMs: ENUMERATION_BUCKET_MS,+ maxSize: 0,+ observeLocal: now => enumerationEvents15m.observe(cohortKey, now, ENUMERATION_BUCKET_MS, MAX_LOCAL_COUNTER_KEYS),+ },+ {+ // A sorted set cannot hold this: a crawler's cohort reaches tens of+ // thousands of distinct paths in a bucket, which is what HyperLogLog is+ // for. The count is approximate; every threshold reading it has margin.+ name: "enumerationDistinctPaths15m",+ enabled: hasEnumerationCohort,+ kind: "cardinality",+ redisKey: `bot:e:pa:${cohortKey}:${enumerationBucket}`,+ member: pathname,+ windowMs: ENUMERATION_BUCKET_MS,+ maxSize: 0,+ observeLocal: now =>+ enumerationDistinctPaths15m.observe(cohortKey, pathname, now, ENUMERATION_BUCKET_MS, MAX_LOCAL_DISTINCT_VALUES),+ },+ {+ name: "enumerationDistinctActors15m",+ enabled: hasEnumerationCohort,+ kind: "cardinality",+ redisKey: `bot:e:ac:${cohortKey}:${enumerationBucket}`,+ member: ipAddress,+ windowMs: ENUMERATION_BUCKET_MS,+ maxSize: 0,+ observeLocal: now =>+ enumerationDistinctActors15m.observe(cohortKey, ipAddress, now, ENUMERATION_BUCKET_MS, MAX_LOCAL_DISTINCT_VALUES),+ },+ {+ name: "enumerationDirectEvents15m",+ enabled: hasEnumerationCohort && isDirect,+ kind: "counter",+ redisKey: `bot:e:di:${cohortKey}:${enumerationBucket}`,+ member: "",+ windowMs: ENUMERATION_BUCKET_MS,+ maxSize: 0,+ observeLocal: now => enumerationDirectEvents15m.observe(cohortKey, now, ENUMERATION_BUCKET_MS, MAX_LOCAL_COUNTER_KEYS),+ },];++ return {+ counters,+ enumeration: hasEnumerationCohort ? { cohortKey, bucket: enumerationBucket, isDirect } : null,+ };}function emptyCounters(): AnomalyCounters {@@ -470,6 +761,11 @@ function emptyCounters(): AnomalyCounters {cohortEvents60s: 0,cohortTopVersionEvents60s: 0,cohortDistinctVersions60s: 0,+ actorEvents1d: 0,+ enumerationEvents15m: 0,+ enumerationDistinctPaths15m: 0,+ enumerationDistinctActors15m: 0,+ enumerationDirectEvents15m: 0,};}@@ -523,15 +819,15 @@ function observeViaLocal(plan: CounterPlan[], nowMs: number): AnomalyCounters {}function computeAnomalyResult(counters: AnomalyCounters): AnomalyResult {- // Individual rules are keyed on (site, ip, ua) β they describe a single actor+ // Convicting rules are keyed on (site, ip, ua) β they describe a single actor// and may convict on their own. The interaction-burst threshold is set beyond// human clicking speed (10/s sustained); ordinary widget bursts stay below it.- const individualReasons: AnomalyReason[] = [];- addReason(individualReasons, "tuple_events_10s", 4, counters.tupleEvents10s, 30, 10);- addReason(individualReasons, "tuple_events_60s", 4, counters.tupleEvents60s, 120, 60);- addReason(individualReasons, "tuple_interaction_events_10s", 4, counters.tupleInteractionEvents10s, 100, 10);- addReason(individualReasons, "tuple_distinct_paths_60s", 4, counters.tupleDistinctPaths60s, 25, 60);- addReason(individualReasons, "missing_client_score_60s", 1, counters.missingClientScore60s, 20, 60);+ const convictingReasons: AnomalyReason[] = [];+ addReason(convictingReasons, "tuple_events_10s", 4, counters.tupleEvents10s, 30, 10);+ addReason(convictingReasons, "tuple_events_60s", 4, counters.tupleEvents60s, 120, 60);+ addReason(convictingReasons, "tuple_interaction_events_10s", 4, counters.tupleInteractionEvents10s, 100, 10);+ addReason(convictingReasons, "tuple_distinct_paths_60s", 4, counters.tupleDistinctPaths60s, 25, 60);+ addReason(convictingReasons, "missing_client_score_60s", 1, counters.missingClientScore60s, 20, 60);// Cohort uniformity. Not a crowd rule: a crowd rule blames a visitor for a// dimension real people share (an IP, a popular browser), whereas this fires@@ -544,7 +840,7 @@ function computeAnomalyResult(counters: AnomalyCounters): AnomalyResult {counters.cohortDistinctVersions60s >= COHORT_MIN_DISTINCT_VERSIONS &&counters.cohortTopVersionEvents60s < counters.cohortEvents60s * COHORT_MAX_MODAL_SHARE) {- individualReasons.push({+ convictingReasons.push({rule: "cohort_version_uniformity_60s",score: 4,value: Math.round((counters.cohortTopVersionEvents60s / counters.cohortEvents60s) * 100),@@ -553,25 +849,46 @@ function computeAnomalyResult(counters: AnomalyCounters): AnomalyResult {});}- // Crowd rules are keyed on shared dimensions (ip, site+ua) that many real+ // Supporting rules are keyed on shared dimensions (ip, site+ua) that many real// visitors legitimately share β one busy CGNAT IP or a popular browser on a// busy site exceeds them with zero per-visitor evidence. Like generic hosting- // ASNs in the layer above, they corroborate but never convict: their scores- // count only when at least one individual rule also fired.- const crowdReasons: AnomalyReason[] = [];- addReason(crowdReasons, "ip_events_60s", 3, counters.ipEvents60s, 200, 60);- addReason(crowdReasons, "ip_distinct_user_agents_5m", 3, counters.ipDistinctUserAgents5m, 10, 300);- addReason(crowdReasons, "ip_distinct_hosts_60s", 2, counters.ipDistinctHosts60s, 6, 60);- addReason(crowdReasons, "site_user_agent_events_60s", 1, counters.siteUserAgentEvents60s, 300, 60);-- const individualScore = individualReasons.reduce((total, reason) => total + reason.score, 0);- const crowdScore = crowdReasons.reduce((total, reason) => total + reason.score, 0);- const score = individualScore + (individualReasons.length > 0 ? crowdScore : 0);+ // ASNs in the layer above, they corroborate but never convict.+ const supportingReasons: AnomalyReason[] = [];+ addReason(supportingReasons, "ip_events_60s", 3, counters.ipEvents60s, 200, 60);+ addReason(supportingReasons, "ip_distinct_user_agents_5m", 3, counters.ipDistinctUserAgents5m, 10, 300);+ addReason(supportingReasons, "ip_distinct_hosts_60s", 2, counters.ipDistinctHosts60s, 6, 60);+ addReason(supportingReasons, "site_user_agent_events_60s", 1, counters.siteUserAgentEvents60s, 300, 60);++ // Long-window volume, skipped entirely for an actor showing many user agents.+ // That is the signature of shared egress β a corporate SASE or VPN gateway,+ // where one address is hundreds of real people and a four-figure daily count+ // is unremarkable. A single scraper presents one user agent, so the guard+ // costs nothing against the traffic this rule exists for.+ if (counters.ipDistinctUserAgents5m <= PROXY_MERGE_MAX_USER_AGENTS) {+ addReason(+ supportingReasons,+ "actor_events_1d",+ ACTOR_EVENTS_1D_SCORE,+ counters.actorEvents1d,+ ACTOR_EVENTS_1D_THRESHOLD,+ 86400+ );+ }++ // The invariant, stated once: supporting evidence is only ever added to a+ // score that convicting evidence has already opened. With the two lists+ // separate this cannot be broken by retuning a threshold β only by editing+ // this line, which is the point.+ const convictingScore = convictingReasons.reduce((total, reason) => total + reason.score, 0);+ const supportingScore = supportingReasons.reduce((total, reason) => total + reason.score, 0);+ const score = convictingReasons.length === 0 ? 0 : convictingScore + supportingScore;return {isAnomalous: score >= ANOMALY_SCORE_THRESHOLD,score,- reasons: [...individualReasons, ...crowdReasons],+ convictingReasons,+ supportingReasons,+ reasons: [...convictingReasons, ...supportingReasons],counters,};}@@ -581,21 +898,49 @@ export async function observeTrackingAnomaly(input: AnomalyInput): Promise<Anomaconst plan = buildCounterPlan(input, nowMs);let counters: AnomalyCounters;+ let usedLocalCounters = !redisAnomalyEnabled;if (redisAnomalyEnabled) {try {- counters = await observeViaRedis(plan, nowMs);+ counters = await observeViaRedis(plan.counters, nowMs);} catch (error) {+ usedLocalCounters = true;// A Redis blip must never break ingestion. Fall back to the in-process// counters β accuracy degrades under clustering, but detection keeps working// and recovers automatically once Redis is back.logger.error({ err: error }, "Redis anomaly counters failed; using in-process fallback");- counters = observeViaLocal(plan, nowMs);+ counters = observeViaLocal(plan.counters, nowMs);}} else {- counters = observeViaLocal(plan, nowMs);+ counters = observeViaLocal(plan.counters, nowMs);+ }++ const result = computeAnomalyResult(counters);++ // Only a direct hit can read a meaningful direct share, since only a direct+ // hit incremented that counter. The observation is attached to the result and+ // deliberately goes no further β it is evidence being gathered, not a verdict.+ if (plan.enumeration?.isDirect) {+ const enumeration = await observeEnumeration(+ plan.enumeration.cohortKey,+ plan.enumeration.bucket,+ {+ events: counters.enumerationEvents15m,+ distinctPaths: counters.enumerationDistinctPaths15m,+ distinctActors: counters.enumerationDistinctActors15m,+ directEvents: counters.enumerationDirectEvents15m,+ },+ // Follow the counters' own backend: reaching for Redis to mark a streak+ // while the counters are in-process would pay a connection timeout per+ // qualifying bucket during exactly the outage the fallback exists for.+ usedLocalCounters+ );++ if (enumeration) {+ result.enumeration = enumeration;+ }}- return computeAnomalyResult(counters);+ return result;}export function resetAnomalyScorerForTests() {@@ -609,6 +954,12 @@ export function resetAnomalyScorerForTests() {ipDistinctUserAgents5m.clear();ipDistinctHosts60s.clear();cohortBrowserVersions60s.clear();+ actorEvents1d.clear();+ enumerationEvents15m.clear();+ enumerationDirectEvents15m.clear();+ enumerationDistinctPaths15m.clear();+ enumerationDistinctActors15m.clear();+ resetEnumerationObserverForTests();lastCleanupMs = 0;eventSeq = 0;}diff --git a/server/src/services/tracker/botBlocking/botDetectionStats.test.ts b/server/src/services/tracker/botBlocking/botDetectionStats.test.tsindex 564c18011..105f576dc 100644--- a/server/src/services/tracker/botBlocking/botDetectionStats.test.ts+++ b/server/src/services/tracker/botBlocking/botDetectionStats.test.ts@@ -38,7 +38,7 @@ describe("botDetectionStats aggregation", () => {it("flushes only the delta since the previous flush", async () => {recordBotBlockingRequest(undefined, undefined); // totalRequests=1, cs:missing=1, sig:missingMask=1- recordBotDetections(["ua_pattern"]); // totalBotRequests=1, m:ua_pattern=1+ recordBotDetections(["ua_pattern"], true); // totalBotRequests=1, m:ua_pattern=1await flushBotDetectionStats();@@ -81,7 +81,7 @@ describe("botDetectionStats aggregation", () => {});it("does not throw when Redis is unavailable", async () => {- recordBotDetections(["rate_anomaly"]);+ recordBotDetections(["rate_anomaly"], true);mocks.exec.mockRejectedValue(new Error("redis down"));await expect(flushBotDetectionStats()).resolves.toBeUndefined();diff --git a/server/src/services/tracker/botBlocking/botDetectionStats.ts b/server/src/services/tracker/botBlocking/botDetectionStats.tsindex 09119f5d1..356f5f1fe 100644--- a/server/src/services/tracker/botBlocking/botDetectionStats.ts+++ b/server/src/services/tracker/botBlocking/botDetectionStats.ts@@ -63,6 +63,7 @@ const SIGNAL_KEYS = Object.keys(clientBotSignalTotals) as ClientBotSignalTotal[]let totalRequests = 0;let totalBotRequests = 0;+let totalEnforcedBotRequests = 0;function getBotRequestPercentage(requests = totalRequests, botRequests = totalBotRequests) {if (requests === 0) {@@ -71,10 +72,29 @@ function getBotRequestPercentage(requests = totalRequests, botRequests = totalBoreturn Number(((botRequests / requests) * 100).toFixed(2));}-export function recordBotBlockingRequest(clientBotScore: number | undefined, clientBotSignalMask: number | undefined) {+/**+ * `hasClientScore`/`hasClientMask` are whether the CLIENT reported them, which+ * is a different question from whether we have them: the server infers signals+ * of its own, so a request from a tracker too old to report anything can still+ * arrive here with a score and a mask. These two counters measure tracker+ * adoption, so they follow the client.+ *+ * The histogram still partitions every request, but its buckets mean two+ * different things by design: `missing` is "the client reported nothing", while+ * the numbered buckets hold the score we ended up with, inferred contributions+ * included. The per-signal totals follow the same rule β they count every bit we+ * ended up with, which is what makes a new bit appearing in production the proof+ * that a deploy actually took.+ */+export function recordBotBlockingRequest(+ clientBotScore: number | undefined,+ clientBotSignalMask: number | undefined,+ hasClientMask = clientBotSignalMask !== undefined,+ hasClientScore = clientBotScore !== undefined+) {totalRequests++;- if (typeof clientBotScore !== "number" || !Number.isFinite(clientBotScore)) {+ if (!hasClientScore || typeof clientBotScore !== "number" || !Number.isFinite(clientBotScore)) {clientBotScoreHistogram.missing++;} else if (clientBotScore === 0) {clientBotScoreHistogram.score0++;@@ -86,9 +106,11 @@ export function recordBotBlockingRequest(clientBotScore: number | undefined, cliclientBotScoreHistogram.score3Plus++;}- if (typeof clientBotSignalMask !== "number" || !Number.isFinite(clientBotSignalMask)) {+ if (!hasClientMask) {clientBotSignalTotals.missingMask++;- } else {+ }++ if (typeof clientBotSignalMask === "number" && Number.isFinite(clientBotSignalMask)) {for (const name of CLIENT_BOT_SIGNAL_NAMES) {if ((clientBotSignalMask & CLIENT_BOT_SIGNAL_MASKS[name]) !== 0) {clientBotSignalTotals[name]++;@@ -101,8 +123,19 @@ export function recordBotBlockingRequest(clientBotScore: number | undefined, cli}}-export function recordBotDetections(methods: readonly BotDetectionMethod[]) {+/**+ * `enforced` is whether the detection was acted on. Detection now runs for every+ * site, including those with bot blocking turned off, so the detection totals+ * and the blocked total are no longer the same number: `totalBotRequests`+ * counts what was detected, `totalEnforcedBotRequests` the subset that was+ * actually diverted out of `events`. The gap between them is what sites with+ * blocking disabled are absorbing.+ */+export function recordBotDetections(methods: readonly BotDetectionMethod[], enforced: boolean) {totalBotRequests++;+ if (enforced) {+ totalEnforcedBotRequests++;+ }for (const method of methods) {totals[method]++;}@@ -112,6 +145,7 @@ export function getBotDetectionStats() {return {totalRequests,totalBotRequests,+ totalEnforcedBotRequests,botRequestPercentage: getBotRequestPercentage(),totals: { ...totals },clientBotScoreHistogram: { ...clientBotScoreHistogram },@@ -122,6 +156,7 @@ export function getBotDetectionStats() {export function resetBotDetectionStatsForTests() {totalRequests = 0;totalBotRequests = 0;+ totalEnforcedBotRequests = 0;for (const method of BOT_DETECTION_METHODS) {totals[method] = 0;}@@ -139,6 +174,7 @@ function flattenLocalCounters(): Record<string, number> {const flat: Record<string, number> = {totalRequests,totalBotRequests,+ totalEnforcedBotRequests,};for (const method of BOT_DETECTION_METHODS) {flat[`m:${method}`] = totals[method];@@ -177,6 +213,7 @@ function structureAggregate(hash: Record<string, string>) {return {totalRequests: requests,totalBotRequests: botRequests,+ totalEnforcedBotRequests: num("totalEnforcedBotRequests"),botRequestPercentage: getBotRequestPercentage(requests, botRequests),botDetectionTotals: aggregateTotals,clientBotScoreHistogram: histogram,diff --git a/server/src/services/tracker/botBlocking/botEventQueue.ts b/server/src/services/tracker/botBlocking/botEventQueue.tsindex d04134a0f..62a937a14 100644--- a/server/src/services/tracker/botBlocking/botEventQueue.ts+++ b/server/src/services/tracker/botBlocking/botEventQueue.ts@@ -19,9 +19,12 @@ class BotEventQueue {private batchSize = BOT_EVENT_BATCH_SIZE;private interval = BOT_EVENT_FLUSH_INTERVAL_MS;private processing = false;- private logger = createServiceLogger("bot-event-queue");+ private table: string;+ private logger: ReturnType<typeof createServiceLogger>;- constructor() {+ constructor(table: string, loggerName: string) {+ this.table = table;+ this.logger = createServiceLogger(loggerName);setInterval(() => this.processQueue(), this.interval);}@@ -77,12 +80,14 @@ class BotEventQueue {bot_category: event.botCategory || "",client_bot_score: event.clientBotScore ?? null,client_signal_mask: event.clientSignalMask ?? 0,+ anomaly_reasons: event.anomalyReasons || "",+ anomaly_score: event.anomalyScore ?? 0,};});try {await clickhouse.insert({- table: "bot_events",+ table: this.table,values: processedBotEvents,format: "JSONEachRow",});@@ -94,4 +99,10 @@ class BotEventQueue {}}-export const botEventQueue = new BotEventQueue();+export const botEventQueue = new BotEventQueue("bot_events", "bot-event-queue");++// Detections that were NOT enforced: on a site with bot blocking disabled the+// event is tracked as normal, so this row is the only evidence that detection+// fired. The bot_observations schema deliberately mirrors bot_events+// column-for-column so the same analysis queries work against either table.+export const botObservationQueue = new BotEventQueue("bot_observations", "bot-observation-queue");diff --git a/server/src/services/tracker/botBlocking/enumerationObserver.test.ts b/server/src/services/tracker/botBlocking/enumerationObserver.test.tsnew file mode 100644index 000000000..37fb3569f--- /dev/null+++ b/server/src/services/tracker/botBlocking/enumerationObserver.test.ts@@ -0,0 +1,132 @@+import { beforeEach, describe, expect, it, vi } from "vitest";++const mocks = vi.hoisted(() => ({+ exists: vi.fn(),+}));++// `multi()` returns a chainable builder; only the second result (the EXISTS on+// the previous bucket) is read back.+vi.mock("../../../db/redis/redis.js", () => ({+ redis: {+ multi: () => ({+ set: () => ({+ exists: () => ({+ exec: async () => [+ [null, "OK"],+ [null, mocks.exists()],+ ],+ }),+ }),+ }),+ },+ anomalyObserve: vi.fn(),+}));++import {+ evaluateEnumerationReading,+ observeEnumeration,+ resetEnumerationObserverForTests,+} from "./enumerationObserver.js";++/** The measured shape of the crawler this rule was built from. */+const crawlerReading = {+ events: 400,+ distinctPaths: 384,+ distinctActors: 342,+ directEvents: 400,+};++describe("evaluateEnumerationReading", () => {+ it("recognises an enumerating cohort", () => {+ const result = evaluateEnumerationReading(crawlerReading);++ expect(result.qualifies).toBe(true);+ expect(result.pathNovelty).toBeGreaterThan(0.9);+ expect(result.eventsPerActor).toBeLessThan(1.25);+ });++ it("leaves ordinary reading traffic alone", () => {+ // A busy content site: visitors read several pages each, and they revisit+ // the same popular ones.+ const result = evaluateEnumerationReading({+ events: 4000,+ distinctPaths: 90,+ distinctActors: 850,+ directEvents: 3960,+ });++ expect(result.qualifies).toBe(false);+ });++ it("leaves a small cohort alone even when its shape matches", () => {+ // Below the volume floor these ratios are noise: twelve visitors who each+ // opened one unlinked page produce exactly the crawler's numbers.+ const result = evaluateEnumerationReading({+ events: 12,+ distinctPaths: 12,+ distinctActors: 12,+ directEvents: 12,+ });++ expect(result.qualifies).toBe(false);+ expect(result.pathNovelty).toBe(1);+ });++ it("leaves a campaign landing burst alone, because it arrives referred", () => {+ // A marketing blast lands on thousands of unique UTM-tagged URLs, one hit+ // per visitor β the crawler's shape in every respect but one.+ const result = evaluateEnumerationReading({+ events: 900,+ distinctPaths: 880,+ distinctActors: 870,+ directEvents: 40,+ });++ expect(result.qualifies).toBe(false);+ expect(result.pathNovelty).toBeGreaterThan(0.9);+ });++ it("reports zeros rather than dividing by zero on an empty bucket", () => {+ const result = evaluateEnumerationReading({+ events: 0,+ distinctPaths: 0,+ distinctActors: 0,+ directEvents: 0,+ });++ expect(result).toMatchObject({ qualifies: false, pathNovelty: 0, eventsPerActor: 0, directShare: 0 });+ });+});++describe("observeEnumeration", () => {+ beforeEach(() => {+ resetEnumerationObserverForTests();+ mocks.exists.mockReset().mockReturnValue(0);+ });++ it("does not mark a cohort on its first qualifying bucket", async () => {+ const result = await observeEnumeration("site:1920x1080:en-us:chrome", 42, crawlerReading);++ expect(result).toMatchObject({ qualifies: true, sustained: false });+ });++ it("marks a cohort once the previous bucket qualified too", async () => {+ mocks.exists.mockReturnValue(1);++ const result = await observeEnumeration("site:1920x1080:en-us:chrome", 43, crawlerReading);++ expect(result).toMatchObject({ qualifies: true, sustained: true });+ });++ it("does not consult the streak for a cohort that did not qualify", async () => {+ const result = await observeEnumeration("site:1920x1080:en-us:chrome", 42, {+ events: 4000,+ distinctPaths: 90,+ distinctActors: 850,+ directEvents: 3960,+ });++ expect(result).toMatchObject({ qualifies: false, sustained: false });+ expect(mocks.exists).not.toHaveBeenCalled();+ });+});diff --git a/server/src/services/tracker/botBlocking/enumerationObserver.ts b/server/src/services/tracker/botBlocking/enumerationObserver.tsnew file mode 100644index 000000000..f1d6087bd--- /dev/null+++ b/server/src/services/tracker/botBlocking/enumerationObserver.ts@@ -0,0 +1,223 @@+import { redis } from "../../../db/redis/redis.js";+import { createServiceLogger } from "../../../lib/logger/logger.js";++/**+ * The enumeration observer β shadow mode.+ *+ * A distributed crawler that walks a site's URL space leaves no per-visitor+ * evidence: it paces itself under every rate threshold, sends one well-formed+ * event per identity, and runs a real browser. Nothing in the per-request layers+ * can see it. What it cannot hide is the *shape* of what a cohort requests: it+ * enumerates permutations, so nearly every path it asks for is one nobody has+ * asked for before, and it burns a fresh identity per hit.+ *+ * That shape is measured here, per (cohort, 15-minute bucket):+ *+ * path novelty distinct paths / events β enumeration, not reading+ * events per actor events / distinct actors β an identity per request+ * direct share events with no referrer β arrived at, not navigated to+ *+ * Two properties keep this honest. First, it is an aggregate statement β it+ * describes a cohort, not the request in front of us β so it is recorded and+ * never convicts. `observeEnumeration` returns an observation; no caller turns+ * it into a score. Promoting it later is a deliberate act, informed by what+ * shadow mode collects, not a threshold nudge.+ *+ * Second, a cohort must qualify in two consecutive buckets before it is marked,+ * which filters a one-off migration crawl or sitemap ping.+ *+ * The direct share is measured on the RAW referrer, before self-referrers are+ * cleared for storage. A stored empty referrer means "direct or internal+ * navigation", which is a different and far more common thing β measuring the+ * stored value would put most ordinary browsing above the threshold.+ *+ * Two known weaknesses, recorded because they are the reason this is not a rule+ * yet and must be resolved before it becomes one:+ *+ * 1. The streak is evaluated against the CURRENT bucket's running counters,+ * not a closed one. A cohort that qualifies transiently early in a bucket+ * stays marked even if the full bucket would not have qualified, and 300+ * events either side of a bucket boundary reads as "sustained" on two+ * seconds of traffic. Promotion needs closed-bucket evaluation.+ * 2. The direct share does not separate a crawler from a genuinely direct+ * campaign. Email, in-app and messenger traffic arrives with no referrer+ * on personalized one-hit-per-recipient URLs β the crawler's shape in every+ * measured respect. Only UTM-tagged campaigns are distinguishable today,+ * and only because they tend to be referred, which is not something to rely+ * on.+ */++const logger = createServiceLogger("enumeration-observer");++export const ENUMERATION_BUCKET_MS = 15 * 60 * 1000;++/**+ * Gates, set well outside anything organic traffic produces. The crawler that+ * motivated them ran at 0.96 novelty and 1.17 events per actor sustained over+ * hours; a busy content site sits near 0.02 novelty with several events per+ * visitor. The volume floor keeps small cohorts β where these ratios are noise β+ * out of it entirely.+ */+const MIN_EVENTS = 300;+const MIN_PATH_NOVELTY = 0.9;+const MAX_EVENTS_PER_ACTOR = 1.25;+const MIN_DIRECT_SHARE = 0.98;++/** Raw counter readings for one cohort's current bucket. */+export interface EnumerationReading {+ events: number;+ distinctPaths: number;+ distinctActors: number;+ directEvents: number;+}++export interface EnumerationObservation {+ /** This bucket meets every gate. */+ qualifies: boolean;+ /** This bucket and the one before it both qualified. */+ sustained: boolean;+ pathNovelty: number;+ eventsPerActor: number;+ directShare: number;+ events: number;+}++// Mirrors the Redis streak markers for when Redis is unavailable, one entry per+// (cohort, bucket) exactly as Redis holds one key per (cohort, bucket). Storing+// only the latest bucket per cohort would not do: every later hit in the same+// bucket would then read its own marker back and see no streak. Bounded because+// a cohort only lands here after clearing the volume and shape gates, which+// almost nothing does.+const localQualifiedBuckets = new Set<string>();+const MAX_LOCAL_STREAK_ENTRIES = 1024;++/**+ * The streak verdict for each (cohort, bucket) this process has already+ * resolved. Without it the cost of observing scales with the crawler: a cohort+ * qualifies at its 300th event, and every event after that in the same bucket+ * would repeat the round-trip and re-log the same finding β thousands of both,+ * on exactly the traffic being measured. The marker only needs writing once, so+ * resolve once per bucket and serve the answer from here afterwards.+ */+const resolvedBuckets = new Map<string, boolean>();+const MAX_RESOLVED_BUCKETS = 1024;++function ratio(numerator: number, denominator: number): number {+ return denominator > 0 ? numerator / denominator : 0;+}++export function evaluateEnumerationReading(reading: EnumerationReading) {+ const pathNovelty = ratio(reading.distinctPaths, reading.events);+ const eventsPerActor = ratio(reading.events, reading.distinctActors);+ const directShare = ratio(reading.directEvents, reading.events);++ const qualifies =+ reading.events >= MIN_EVENTS &&+ pathNovelty >= MIN_PATH_NOVELTY &&+ // An actor count above the event count is not meaningful; HyperLogLog is+ // approximate, so this can happen by a hair on small buckets.+ eventsPerActor > 0 &&+ eventsPerActor <= MAX_EVENTS_PER_ACTOR &&+ directShare >= MIN_DIRECT_SHARE;++ return { qualifies, pathNovelty, eventsPerActor, directShare, events: reading.events };+}++function markLocally(cohortKey: string, bucket: number): boolean {+ const previousQualified = localQualifiedBuckets.has(`${cohortKey}:${bucket - 1}`);+ if (localQualifiedBuckets.size >= MAX_LOCAL_STREAK_ENTRIES) {+ localQualifiedBuckets.clear();+ }+ localQualifiedBuckets.add(`${cohortKey}:${bucket}`);+ return previousQualified;+}++/**+ * Record that this bucket qualified and report whether the previous one did too.+ * Resolved at most once per (cohort, bucket) per process β see `resolvedBuckets`+ * β so the extra round-trip is charged per qualifying bucket, not per event in+ * one.+ *+ * `useLocalState` follows whichever backend the counters themselves used. Going+ * to Redis for the marker while the counters are in-process would mean paying a+ * connection timeout on every qualifying bucket during an outage, which is the+ * moment the hot path can least afford it.+ */+async function resolveStreak(cohortKey: string, bucket: number, useLocalState: boolean): Promise<boolean> {+ const bucketKey = `${cohortKey}:${bucket}`;+ const resolved = resolvedBuckets.get(bucketKey);+ if (resolved !== undefined) {+ return resolved;+ }++ let sustained: boolean;+ if (useLocalState) {+ sustained = markLocally(cohortKey, bucket);+ } else {+ try {+ const results = await redis+ .multi()+ .set(`bot:enum:q:${bucketKey}`, "1", "PX", ENUMERATION_BUCKET_MS * 3)+ .exists(`bot:enum:q:${cohortKey}:${bucket - 1}`)+ .exec();++ // `exec` returns [error, value] pairs in command order; the second is ours.+ sustained = results?.[1]?.[1] === 1;+ } catch (error) {+ logger.error({ err: error }, "Enumeration streak check failed; using in-process fallback");+ // The marker for the previous bucket may only exist in Redis, so a streak+ // spanning an outage is lost. Shadow mode tolerates that; it under-reports+ // rather than inventing a finding.+ sustained = markLocally(cohortKey, bucket);+ }+ }++ if (resolvedBuckets.size >= MAX_RESOLVED_BUCKETS) {+ resolvedBuckets.clear();+ }+ resolvedBuckets.set(bucketKey, sustained);+ return sustained;+}++/**+ * Evaluate one cohort bucket. Returns null when the cohort's counters were not+ * collected (no cohort dimensions, or no path to measure novelty against).+ */+export async function observeEnumeration(+ cohortKey: string,+ bucket: number,+ reading: EnumerationReading,+ useLocalState = false+): Promise<EnumerationObservation | null> {+ const evaluated = evaluateEnumerationReading(reading);+ if (!evaluated.qualifies) {+ return { ...evaluated, sustained: false };+ }++ const alreadyResolved = resolvedBuckets.has(`${cohortKey}:${bucket}`);+ const sustained = await resolveStreak(cohortKey, bucket, useLocalState);++ if (sustained && !alreadyResolved) {+ // The only output of shadow mode. It is deliberately a log line and not a+ // score: promoting this rule should follow from reading these, not from it+ // having quietly been convicting all along.+ logger.warn(+ {+ cohortKey,+ bucket,+ events: evaluated.events,+ pathNovelty: Number(evaluated.pathNovelty.toFixed(3)),+ eventsPerActor: Number(evaluated.eventsPerActor.toFixed(3)),+ directShare: Number(evaluated.directShare.toFixed(3)),+ },+ "Enumeration cohort sustained across two buckets (shadow mode, not enforced)"+ );+ }++ return { ...evaluated, sustained };+}++export function resetEnumerationObserverForTests() {+ localQualifiedBuckets.clear();+ resolvedBuckets.clear();+}diff --git a/server/src/services/tracker/botBlocking/index.test.ts b/server/src/services/tracker/botBlocking/index.test.tsindex f1eaf18ec..77cee7756 100644--- a/server/src/services/tracker/botBlocking/index.test.ts+++ b/server/src/services/tracker/botBlocking/index.test.ts@@ -37,14 +37,39 @@ describe("checkBotBlocking", () => {vi.mocked(lookupAsn).mockReturnValue(null);});- it("does nothing when bot blocking is disabled", async () => {+ it("still detects when bot blocking is disabled, but does not enforce", async () => {const result = await checkBotBlocking({headers: {},blockBots: false,payload: basePayload,});- expect(result).toBeNull();+ // The detection is made either way; `enforced` is what changes. A Site that+ // has opted out of blocking used to be skipped before any layer ran, which+ // left it with no record at all of what it was receiving.+ expect(result).toMatchObject({ isBot: true, enforced: false });+ });++ it("marks the same detection as enforced when bot blocking is enabled", async () => {+ const result = await checkBotBlocking({+ headers: {},+ blockBots: true,+ payload: basePayload,+ });++ expect(result).toMatchObject({ isBot: true, enforced: true });+ });++ it("returns null for a clean request whether or not blocking is enabled", async () => {+ for (const blockBots of [true, false]) {+ const result = await checkBotBlocking({+ headers: browserHeaders,+ blockBots,+ payload: { ...basePayload, screenWidth: 1512, screenHeight: 982, clientBotScore: 0, clientBotSignalMask: 0 },+ });++ expect(result).toBeNull();+ }});it("skips verified trusted server-side ingestion requests", async () => {@@ -71,33 +96,62 @@ describe("checkBotBlocking", () => {});});- it("counts requests before bot-blocking bypasses for percentage stats", async () => {- await checkBotBlocking({- headers: {},- blockBots: false,- payload: basePayload,- });+ it("counts requests before the trusted-ingestion bypass for percentage stats", async () => {await checkBotBlocking({headers: {},blockBots: true,trustedServerSideIngestion: true,- payload: { ...basePayload, clientBotScore: 0, clientBotSignalMask: 0 },+ payload: { ...basePayload, screenWidth: 1512, screenHeight: 982, clientBotScore: 0, clientBotSignalMask: 0 },});expect(getBotDetectionStats()).toMatchObject({- totalRequests: 2,+ totalRequests: 1,totalBotRequests: 0,+ totalEnforcedBotRequests: 0,botRequestPercentage: 0,- clientBotScoreHistogram: {- missing: 1,- score0: 1,- },- clientBotSignalTotals: {- missingMask: 1,- },+ clientBotScoreHistogram: { score0: 1 },+ });+ });++ it("separates what was detected from what was enforced", async () => {+ await checkBotBlocking({ headers: {}, blockBots: true, payload: basePayload });+ await checkBotBlocking({ headers: {}, blockBots: false, payload: basePayload });++ // Detection now runs for every Site, so these two numbers are no longer the+ // same one. The gap is what Sites with blocking disabled are absorbing.+ expect(getBotDetectionStats()).toMatchObject({+ totalRequests: 2,+ totalBotRequests: 2,+ totalEnforcedBotRequests: 1,});});+ it("records missing screen dimensions without ever convicting on them", async () => {+ const result = await checkBotBlocking({+ headers: browserHeaders,+ blockBots: true,+ payload: { ...basePayload, clientBotScore: 0, clientBotSignalMask: 0 },+ });++ // Weight 1 and never strong, so it cannot reach the threshold alone. The+ // point is that a hit reporting no display leaves a trace instead of the+ // geometry rules silently having nothing to say about it.+ expect(result).toBeNull();+ expect(getBotDetectionStats().clientBotSignalTotals.missingScreenDimensions).toBe(1);+ });++ it("does not raise missing screen dimensions for mobile sites", async () => {+ // A native SDK has no screen to report, and is not a bot for it.+ await checkBotBlocking({+ headers: {},+ blockBots: true,+ isMobileSite: true,+ payload: { ...basePayload, clientBotScore: 0, clientBotSignalMask: 0 },+ });++ expect(getBotDetectionStats().clientBotSignalTotals.missingScreenDimensions).toBe(0);+ });+it("returns bot event properties for detected bots", async () => {const result = await checkBotBlocking({headers: {},@@ -174,27 +228,30 @@ describe("checkBotBlocking", () => {it("records client bot score and signal aggregates for inspected requests", async () => {const headers = browserHeaders;+ // Plausible dimensions, so the aggregates reflect what the client reported+ // rather than picking up server-inferred geometry signals as well.+ const webPayload = { ...basePayload, screenWidth: 1512, screenHeight: 982 };await checkBotBlocking({headers,blockBots: true,- payload: basePayload,+ payload: webPayload,});await checkBotBlocking({headers,blockBots: true,- payload: { ...basePayload, clientBotScore: 0, clientBotSignalMask: 0 },+ payload: { ...webPayload, clientBotScore: 0, clientBotSignalMask: 0 },});await checkBotBlocking({headers,blockBots: true,- payload: { ...basePayload, clientBotScore: 1, clientBotSignalMask: clientBotSignalMasks.swiftShader },+ payload: { ...webPayload, clientBotScore: 1, clientBotSignalMask: clientBotSignalMasks.swiftShader },});await checkBotBlocking({headers,blockBots: true,payload: {- ...basePayload,+ ...webPayload,clientBotScore: 2,clientBotSignalMask: clientBotSignalMasks.zeroOuterDimensions,},@@ -202,7 +259,7 @@ describe("checkBotBlocking", () => {await checkBotBlocking({headers,blockBots: true,- payload: { ...basePayload, clientBotScore: 3, clientBotSignalMask: clientBotSignalMasks.automationApi },+ payload: { ...webPayload, clientBotScore: 3, clientBotSignalMask: clientBotSignalMasks.automationApi },});expect(getBotDetectionStats()).toMatchObject({@@ -377,6 +434,10 @@ describe("checkBotBlocking", () => {blockBots: true,payload: {...basePayload,+ // Real dimensions, so this stays a test about weak client signals and+ // does not also pick up the missing-dimensions signal.+ screenWidth: 1512,+ screenHeight: 982,clientBotScore: 3,clientBotSignalMask: clientBotSignalMasks.zeroOuterDimensions | clientBotSignalMasks.swiftShader,},@@ -524,6 +585,23 @@ describe("checkBotBlocking", () => {});expect(result?.detections.map(detection => detection.layer)).toEqual(["rate_anomaly"]);expect(result?.detections[0].anomalyReasons?.map(reason => reason.rule)).toContain("tuple_events_10s");+ // The audit row has to carry which rules fired, not just that the layer+ // did: "rate anomaly" spans a dozen rules, and a row that only records the+ // layer cannot be argued with after the fact.+ expect(result?.eventProperties.anomalyReasons).toContain("tuple_events_10s");+ expect(result?.eventProperties.anomalyScore).toBeGreaterThan(0);+ });++ it("leaves the anomaly audit fields empty when another layer convicted", async () => {+ const userAgent = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";++ const result = await checkBotBlocking({+ headers: { ...browserHeaders, "user-agent": userAgent },+ blockBots: true,+ payload: { ...basePayload, userAgent },+ });++ expect(result?.eventProperties).toMatchObject({ anomalyReasons: "", anomalyScore: 0 });});it("convicts a browser release too old to have run the tracker", async () => {diff --git a/server/src/services/tracker/botBlocking/index.ts b/server/src/services/tracker/botBlocking/index.tsindex 29ac235dc..f16004a85 100644--- a/server/src/services/tracker/botBlocking/index.ts+++ b/server/src/services/tracker/botBlocking/index.ts@@ -36,6 +36,8 @@ interface BotBlockingPayload {hostname?: string;pathname?: string;eventType?: string;+ /** As it arrived, before self-referrers are cleared for storage. */+ referrer?: string;ipAddress: string;}@@ -99,10 +101,27 @@ export interface BotEventProperties {botCategory: string;clientBotScore?: number;clientSignalMask?: number;+ /**+ * Which anomaly rules fired, comma-separated, and what they summed to. The+ * other layers name themselves β a UA pattern or an ASN is its own+ * explanation β but "rate anomaly" covers a dozen rules with very different+ * meanings, and without these an audit row cannot answer why the request was+ * convicted. That question is the entire reason `bot_observations` exists.+ */+ anomalyReasons: string;+ anomalyScore: number;}export interface BotDetectionResult {isBot: true;+ /**+ * Whether this detection is acted on. Detection and enforcement are separate+ * concerns: every site is evaluated, but only a site with bot blocking on has+ * its bot traffic diverted out of `events`. A detection with `enforced: false`+ * is recorded to `bot_observations` and the event is tracked as normal, so a+ * site that has opted out still produces evidence of what it is receiving.+ */+ enforced: boolean;message: string;detections: BotBlockingDetection[];eventProperties: BotEventProperties;@@ -115,6 +134,7 @@ function buildBotEventProperties(): BotEventProperties {const detectionLayers = new Set(detections.map(detection => detection.layer));const uaDetection = detections.find(detection => detection.layer === "ua_pattern");+ const anomalyDetection = detections.find(detection => detection.layer === "rate_anomaly");return {isBot: true,@@ -129,10 +149,16 @@ function buildBotEventProperties(botCategory: uaDetection?.botCategory ?? "",clientBotScore: clientSignalResult.scoreForStats,clientSignalMask: clientSignalResult.maskForStats,+ anomalyReasons: (anomalyDetection?.anomalyReasons ?? []).map(reason => reason.rule).join(","),+ anomalyScore: anomalyDetection?.score ?? 0,};}-function getClientSignalResult(payload: BotBlockingPayload, userAgent: string) {+function getClientSignalResult(+ payload: BotBlockingPayload,+ userAgent: string,+ hasReportableScreen: boolean+) {const hasClientScore = typeof payload.clientBotScore === "number" && Number.isFinite(payload.clientBotScore);const hasClientMask = typeof payload.clientBotSignalMask === "number" && Number.isFinite(payload.clientBotSignalMask);const rawMask = hasClientMask ? payload.clientBotSignalMask! : 0;@@ -152,12 +178,19 @@ function getClientSignalResult(payload: BotBlockingPayload, userAgent: string) {// Re-derived server-side rather than trusted from the client mask, so the// rules apply to every hit regardless of which tracker version sent it. An- // event that reports no dimensions at all says nothing about its display.+ // event that reports no dimensions at all says nothing about its display, so+ // the geometry rules are skipped β but the skip itself is recorded, weakly, so+ // that "no dimensions" is visible in the mask instead of passing silently. It+ // is only raised where a screen was expected: a native SDK and a trusted+ // server-side integration both legitimately have none, and counting them would+ // make the signal useless for reading adoption or confirming a deploy.const { screenWidth, screenHeight } = payload;if (screenWidth !== undefined || screenHeight !== undefined) {for (const signal of getScreenDimensionSignals(screenWidth ?? NaN, screenHeight ?? NaN, userAgent)) {addInferredSignal(signal);}+ } else if (hasReportableScreen) {+ addInferredSignal("missingScreenDimensions");}const score = Math.min((hasClientScore ? payload.clientBotScore! : 0) + inferredScore, MAX_CLIENT_BOT_SCORE);@@ -174,6 +207,8 @@ function getClientSignalResult(payload: BotBlockingPayload, userAgent: string) {signalNames: getClientBotSignalNames(mask),scoreForStats: hasClientScore || inferredScore > 0 ? score : undefined,maskForStats: hasClientMask || mask !== 0 ? mask : undefined,+ hasClientMask,+ hasClientScore,};}@@ -186,10 +221,24 @@ export async function checkBotBlocking({lookupAsn: asnLookup = lookupAsn,}: BotBlockingInput): Promise<BotDetectionResult | null> {const userAgent = payload.userAgent || (headers["user-agent"] as string) || "";- const clientSignalResult = getClientSignalResult(payload, userAgent);- recordBotBlockingRequest(clientSignalResult.scoreForStats, clientSignalResult.maskForStats);-- if (!blockBots || trustedServerSideIngestion) {+ const clientSignalResult = getClientSignalResult(payload, userAgent, !isMobileSite && !trustedServerSideIngestion);+ recordBotBlockingRequest(+ clientSignalResult.scoreForStats,+ clientSignalResult.maskForStats,+ clientSignalResult.hasClientMask,+ clientSignalResult.hasClientScore+ );++ // Trusted server-side ingestion is authenticated first-party traffic that+ // reports its own IP and user agent: none of the layers below are meaningful+ // against it, and several would convict it outright. It is the only traffic+ // that skips detection.+ //+ // `blockBots` deliberately does NOT skip detection. It decides what happens to+ // a detection, not whether one is looked for β a site that has turned blocking+ // off used to get no evaluation at all, which left it with neither protection+ // nor any record of what it was receiving.+ if (trustedServerSideIngestion) {return null;}@@ -295,6 +344,7 @@ export async function checkBotBlocking({hostname: payload.hostname,pathname: payload.pathname,eventType: payload.eventType,+ referrer: payload.referrer,hasClientBotScore: typeof payload.clientBotScore === "number",screenWidth: payload.screenWidth,screenHeight: payload.screenHeight,@@ -335,10 +385,14 @@ export async function checkBotBlocking({);}- recordBotDetections(detections.map(detection => detection.layer));+ recordBotDetections(+ detections.map(detection => detection.layer),+ blockBots+ );return {isBot: true,+ enforced: blockBots,message: blockMessage ?? "Bot detected",detections,eventProperties: buildBotEventProperties(detections, asnInfo, clientSignalResult),diff --git a/server/src/services/tracker/ingestEvent.test.ts b/server/src/services/tracker/ingestEvent.test.tsindex c51fbacbb..7b82d4123 100644--- a/server/src/services/tracker/ingestEvent.test.ts+++ b/server/src/services/tracker/ingestEvent.test.ts@@ -4,6 +4,7 @@ import type { TrackingRequest } from "./trackingRequest.js";const mocks = vi.hoisted(() => ({addBotEvent: vi.fn(),+ addBotObservation: vi.fn(),addPageview: vi.fn(),checkBotBlocking: vi.fn(),decideSiteExclusion: vi.fn(),@@ -31,6 +32,7 @@ vi.mock("./botBlocking/index.js", () => ({vi.mock("./botBlocking/botEventQueue.js", () => ({botEventQueue: { add: mocks.addBotEvent },+ botObservationQueue: { add: mocks.addBotObservation },}));vi.mock("../sites/siteExclusionDecision.js", () => ({@@ -185,8 +187,9 @@ describe("ingestEvent", () => {expect(mocks.addPageview).not.toHaveBeenCalled();});- it("routes detected bots to the bot queue instead of a session", async () => {+ it("routes enforced bot detections to the bot queue instead of a session", async () => {mocks.checkBotBlocking.mockResolvedValue({+ enforced: true,eventProperties: { bot_layer: "ua_pattern" },});@@ -201,6 +204,36 @@ describe("ingestEvent", () => {);expect(mocks.updateSession).not.toHaveBeenCalled();expect(mocks.addPageview).not.toHaveBeenCalled();+ expect(mocks.addBotObservation).not.toHaveBeenCalled();+ });++ it("tracks an unenforced bot detection normally and records the observation", async () => {+ // A Site with bot blocking turned off. The visitor's event must reach+ // `events` exactly as it would have before detection ran for these Sites at+ // all β the detection is recorded alongside it, and changes nothing.+ mocks.checkBotBlocking.mockResolvedValue({+ enforced: false,+ eventProperties: { bot_layer: "ua_pattern" },+ });++ const outcome = await ingestEvent(trackingRequest());++ expect(outcome).toEqual({ status: "tracked", sessionId: "session-alice" });+ expect(mocks.addPageview).toHaveBeenCalledTimes(1);+ expect(mocks.addBotEvent).not.toHaveBeenCalled();+ expect(mocks.addBotObservation).toHaveBeenCalledWith(+ expect.objectContaining({+ bot_layer: "ua_pattern",+ // The real session, not a synthetic bot one: the event is in `events`.+ sessionId: "session-alice",+ })+ );+ });++ it("records no observation when nothing was detected", async () => {+ await ingestEvent(trackingRequest());++ expect(mocks.addBotObservation).not.toHaveBeenCalled();});it("hands bot detection the request's own headers and ASN resolver", async () => {diff --git a/server/src/services/tracker/ingestEvent.ts b/server/src/services/tracker/ingestEvent.tsindex 026c9579c..da35c8524 100644--- a/server/src/services/tracker/ingestEvent.ts+++ b/server/src/services/tracker/ingestEvent.ts@@ -2,7 +2,7 @@ import { createServiceLogger } from "../../lib/logger/logger.js";import { sessionsService } from "../sessions/sessionsService.js";import { decideSiteExclusion, type SiteExclusionDecision } from "../sites/siteExclusionDecision.js";import { usageService } from "../usageService.js";-import { botEventQueue } from "./botBlocking/botEventQueue.js";+import { botEventQueue, botObservationQueue } from "./botBlocking/botEventQueue.js";import { checkBotBlocking } from "./botBlocking/index.js";import { pageviewQueue } from "./pageviewQueue.js";import type { TrackingRequest } from "./trackingRequest.js";@@ -36,9 +36,12 @@ export type IngestOutcome =* still moved every anomaly counter for the Site.* 2. Over-limit β before any Redis or ClickHouse work, since the event is* going to be dropped regardless.- * 3. Bot detection β needs a real event; charges a Redis round-trip.+ * 3. Bot detection β needs a real event; charges a Redis round-trip. It runs+ * for every Site, whether or not the Site blocks bots; `blockBots` decides+ * only where a detection is sent, at stage 5.* 4. Identity and session β the only stage that hashes a fingerprint.- * 5. Enqueue.+ * 5. Enqueue β to `bot_events` if the detection is enforced, otherwise to+ * `events` plus, when something was detected, `bot_observations`.*/export async function ingestEvent(trackingRequest: TrackingRequest): Promise<IngestOutcome> {const { payload, site } = trackingRequest;@@ -80,13 +83,14 @@ export async function ingestEvent(trackingRequest: TrackingRequest): Promise<Inghostname: payload.hostname,pathname: payload.pathname,eventType: payload.type,+ referrer: payload.referrer,ipAddress: trackingRequest.ipAddress,},});const eventPayload = await createBasePayload(trackingRequest);- if (botDetectionResult) {+ if (botDetectionResult?.enforced) {await botEventQueue.add({...eventPayload,...botDetectionResult.eventProperties,@@ -107,5 +111,18 @@ export async function ingestEvent(trackingRequest: TrackingRequest): Promise<IngsessionId,});+ // The Site has bot blocking turned off, so this hit is tracked like any other+ // β but it was detected, and the detection is worth keeping. Mirroring it to+ // bot_observations is what lets the Site's owner see what turning blocking on+ // would have removed, which is otherwise unanswerable: before this, a Site+ // that opted out was never evaluated at all.+ if (botDetectionResult) {+ await botObservationQueue.add({+ ...eventPayload,+ ...botDetectionResult.eventProperties,+ sessionId,+ });+ }+return { status: "tracked", sessionId };}diff --git a/shared/src/botSignalContract.ts b/shared/src/botSignalContract.tsindex 96c9e1da6..8d0c0ffbf 100644--- a/shared/src/botSignalContract.ts+++ b/shared/src/botSignalContract.ts@@ -28,6 +28,7 @@ export const CLIENT_BOT_SIGNAL_MASKS = {pluginApiAbsence: 1 << 9,defaultViewport1280x1200: 1 << 10,squareScreen: 1 << 11,+ missingScreenDimensions: 1 << 12,} as const;export type ClientBotSignalName = keyof typeof CLIENT_BOT_SIGNAL_MASKS;@@ -52,6 +53,7 @@ export const CLIENT_BOT_SIGNAL_WEIGHTS: Record<ClientBotSignalName, number> = {pluginApiAbsence: 0,defaultViewport1280x1200: 3,squareScreen: 3,+ missingScreenDimensions: 1,};/**@@ -68,9 +70,11 @@ export const ALL_CLIENT_BOT_SIGNAL_BITS = CLIENT_BOT_SIGNAL_NAMES.reduce(/*** Signals automation-specific enough to convict on their own. The remaining* (weak) signals β empty plugins, SwiftShader, zero outer dimensions, missing- * window.chrome β all occur on real devices (Android Chrome ships empty- * plugins; low-end GPUs fall back to SwiftShader; prerendered pages report zero- * outer dimensions), so they corroborate other layers but never convict.+ * window.chrome, absent screen dimensions β all occur on real devices (Android+ * Chrome ships empty plugins; low-end GPUs fall back to SwiftShader;+ * prerendered pages report zero outer dimensions; a server-side or native+ * integration has no screen to report), so they corroborate other layers but+ * never convict.*/export const STRONG_CLIENT_BOT_SIGNAL_BITS =CLIENT_BOT_SIGNAL_MASKS.automationApi |@@ -111,6 +115,15 @@ export const IMPLAUSIBLE_DESKTOP_VIEWPORTS: readonly {{ width: 1280, height: 1200, signal: "defaultViewport1280x1200" },];+/**+ * `missingScreenDimensions` is the one signal the tracker never sets: a browser+ * always has `window.screen`, so only the server can observe that a hit arrived+ * carrying no dimensions at all. It is deliberately weak (weight 1, never+ * strong) and web-only β a native SDK or a server-side integration has no+ * screen, and neither is a bot. It exists so that the geometry rules, which are+ * skipped outright when nothing is reported, leave a trace of having been+ * skipped rather than silently passing the hit.+ */export function isPlausibleScreenDimensions(width: number, height: number): boolean {return (Number.isFinite(width) &&