OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
chore(core): trim comments in charts, reports, cohorts and ingest
Carl-Gerhard Lindesvärd7eed740fa514
diff --git a/packages/core/src/buffers/base-buffer.ts b/packages/core/src/buffers/base-buffer.tsindex c4d521604..6d5614723 100644--- a/packages/core/src/buffers/base-buffer.ts+++ b/packages/core/src/buffers/base-buffer.ts@@ -7,11 +7,7 @@ import type { CoreConfig } from '../config';import type { Logger } from '../logger';import type { ServiceDeps } from '../services';-/**- * What a buffer needs from the boot scope. Buffers are built once by- * `createBuffers(deps)` and hung on `AppDeps` — never module singletons, so a- * test constructs its own and closes nothing shared.- */+/** What a buffer needs from the boot scope; a test constructs its own. */export interface BufferDeps {/*** Each buffer logs under its own name. Injected rather than built here so@@ -30,10 +26,8 @@ export interface BufferDeps {*/isCronPaused(): Promise<boolean>;/**- * The parsed environment. Every buffer's batch/chunk/TTL knob used to be a- * module-scope `process.env` read; they arrive here instead, which is also- * what lets a test build a buffer with a different sizing without touching- * the process environment.+ * The parsed environment: every buffer's batch/chunk/TTL knob, so a test can+ * size a buffer without touching the process environment.*/config: CoreConfig;}@@ -316,10 +310,8 @@ export class BaseBuffer {}/**- * Ground-truth size of the buffer (LLEN of the Redis list). O(1). The- * previous implementation kept a shadow counter to "avoid" LLEN — but- * LLEN and GET are both O(1) single-roundtrip ops in Redis, so the- * counter was free complexity that silently drifted. Always use this.+ * Ground-truth size of the buffer (LLEN of the Redis list), O(1). No shadow+ * counter: it silently drifts.*/async getBufferSize(): Promise<number> {return await getRedisCache().llen(this.getRedisListKey());diff --git a/packages/core/src/buffers/bot-buffer.ts b/packages/core/src/buffers/bot-buffer.tsindex d9aefad0b..d985423a6 100644--- a/packages/core/src/buffers/bot-buffer.ts+++ b/packages/core/src/buffers/bot-buffer.ts@@ -58,9 +58,7 @@ export class BotBuffer extends BaseBuffer {return;}- // Raw passthrough: each Redis entry is already a valid JSONEachRow- // line. Streaming raw strings skips JSON.parse + the client's- // re-stringify on the hot path.+ // Raw passthrough: each Redis entry is already a JSONEachRow line.const ch = this.resolveCh();const chStart = performance.now();await ch.insert({diff --git a/packages/core/src/buffers/buffer.metrics.ts b/packages/core/src/buffers/buffer.metrics.tsindex 988a6a373..12dcf60d0 100644--- a/packages/core/src/buffers/buffer.metrics.ts+++ b/packages/core/src/buffers/buffer.metrics.ts@@ -1,7 +1,5 @@-// The twelve `buffer_*` series (names and labels reproduced exactly).-// Registered per boot rather than per import, because the buffers themselves-// are per boot — and only where a role consumes queues, since the LLEN-// gauges cost one Redis round trip per buffer per scrape.+// Registered per boot only where a role consumes queues: the LLEN gauges cost+// one Redis round trip per buffer per scrape.import client from 'prom-client';import { registry } from '../metrics';@@ -82,8 +80,6 @@ export function registerBufferMetrics(}));- // ---- Flush metrics (populated via flushObserver hooks) -----const flushDuration = new client.Histogram({name: 'buffer_flush_duration_ms',help: 'Wall time of a tryFlush call, including lock acquisition',@@ -142,8 +138,6 @@ export function registerBufferMetrics(});register.registerMetric(flushLlenAtStart);- // ---- Add-path metrics -----const addDurationMs = new client.Histogram({name: 'buffer_add_duration_ms',help: 'Duration of a single add() call (per-event ingest path)',diff --git a/packages/core/src/buffers/clickhouse-failover.test.ts b/packages/core/src/buffers/clickhouse-failover.test.tsindex fb520f188..c9233f0b0 100644--- a/packages/core/src/buffers/clickhouse-failover.test.ts+++ b/packages/core/src/buffers/clickhouse-failover.test.ts@@ -9,18 +9,12 @@ function loadClickHouse() {}/**- * The deliberate ECONNREFUSED to a dead ClickHouse node must be caught and- * retried (sin-binning) on the client the buffers are handed as- * `BufferDeps.ch`, exactly as packages/db's- * `sql.round-robin.clickhouse.test.ts` proves for `chQuery`/`chQueryWithMeta`- * directly. That file is NOT touched here; this extends the same proof onto- * `ch.insert` reached via the core buffer seam.+ * An ECONNREFUSED from a dead ClickHouse node must be retried (sin-binned) on+ * the client the buffers are handed as `BufferDeps.ch`, including `ch.insert`.*- * Setup mirrors that file exactly: a two-node `CLICKHOUSE_URL` whose first node- * (port 1, privileged and unbindable without root) is reliably refused. The- * picker starts at index 0, so every call here fails over to node 1 — if the- * seam let the rejection escape instead of retrying, this file would throw or- * hang rather than pass.+ * `CLICKHOUSE_URL` is a two-node list whose first node (port 1, unbindable+ * without root) is reliably refused. The picker starts at index 0, so every+ * call fails over to node 1; a rejection that escaped would throw or hang.*/const CLICKHOUSE_TEST_DATABASE = 'openpanel_test';diff --git a/packages/core/src/buffers/create-buffers.ts b/packages/core/src/buffers/create-buffers.tsindex 09bd1b375..02b460bbe 100644--- a/packages/core/src/buffers/create-buffers.ts+++ b/packages/core/src/buffers/create-buffers.ts@@ -1,5 +1,3 @@-// Built ONCE at boot and hung on `AppDeps.buffers` — never module singletons.-import type { BufferDeps } from './base-buffer';import { BotBuffer } from './bot-buffer';import { EventBuffer } from './event-buffer';diff --git a/packages/core/src/buffers/event-buffer.test.ts b/packages/core/src/buffers/event-buffer.test.tsindex 193afc0b9..c82761947 100644--- a/packages/core/src/buffers/event-buffer.test.ts+++ b/packages/core/src/buffers/event-buffer.test.ts@@ -20,11 +20,9 @@ const chQuery = mock(async (_query: string | SqlFragment): Promise<unknown[]> => []);-// The client comes in as `BufferDeps.ch` and reads go through core's own-// `chQuery` (ch-query.ts) — so the insert path needs no module mock at all, and-// the read path mocks one core module instead of-// `@openpanel/db/src/clickhouse/client` (whose import builds a real client and-// a pino transport worker thread per test file).+// Inserts go through `BufferDeps.ch` (no module mock needed); reads go through+// core's `chQuery`, mocked here instead of `@openpanel/db`'s client, whose+// import builds a real client and a pino transport thread per test file.mock.module('../ch-query', () => ({...realChQuery,chQuery: (_scope: unknown, query: string | SqlFragment) => chQuery(query),@@ -216,15 +214,9 @@ describe('EventBuffer', () => {};expect(callArgs.format).toBe('JSONEachRow');expect(callArgs.table).toBe('events');- // After the raw-passthrough optimisation, `values` is an- // object-mode Readable stream that yields pre-serialized- // JSONEachRow lines as strings. The client (configured with- // `json.stringify: (v) => typeof v === 'string' ? v : JSON.stringify(v)`)- // then passes the strings through unchanged — no JSON.parse on- // our side, no JSON.stringify on the client's side. Object mode- // is mandatory: @clickhouse/client rejects byte streams for- // JSON* formats with "expected Readable Stream with enabled- // object mode".+ // `values` is an object-mode Readable of pre-serialized JSONEachRow lines.+ // Object mode is mandatory: @clickhouse/client rejects byte streams for+ // JSON* formats.const stream = callArgs.values as Readable;expect(stream.readableObjectMode).toBe(true);const lines = await streamToLines(stream);@@ -653,8 +645,7 @@ describe('EventBuffer', () => {chInsert.mockRejectedValueOnce(new Error('ClickHouse unavailable'));- // Errors propagate to tryFlush (which resyncs the counter). The safety- // property — queue preserved on CH failure — still holds.+ // Errors propagate to tryFlush; the queue is preserved on CH failure.await expect(eventBuffer.processBuffer()).rejects.toThrow('ClickHouse unavailable');@@ -702,11 +693,8 @@ describe('extractProjectId', () => {});it('falls back to JSON.parse when properties has project_id BEFORE top-level', () => {- // This is the bug case the regex couldn't handle. If a future- // refactor swaps the field order in the event constructor, the- // fast path would attribute counts to the wrong project. The- // fallback fixes that: as soon as we see two occurrences we- // resolve via real JSON.parse and pick the top-level key.+ // If the field order in the event constructor ever changes, the fast path+ // would misattribute counts; two occurrences must resolve via JSON.parse.const line = JSON.stringify({properties: { project_id: 'user-supplied' },project_id: 'real-project',diff --git a/packages/core/src/buffers/event-buffer.ts b/packages/core/src/buffers/event-buffer.tsindex 2aae5b27e..6d32cf352 100644--- a/packages/core/src/buffers/event-buffer.ts+++ b/packages/core/src/buffers/event-buffer.ts@@ -6,23 +6,13 @@ import { BaseBuffer, type BufferDeps } from './base-buffer';const PROJECT_ID_NEEDLE = '"project_id":"';/**- * Extract the top-level `project_id` from a single JSONEachRow event- * line without doing a full JSON.parse.+ * Extract the top-level `project_id` from a JSONEachRow event line without a+ * full JSON.parse. Returns `null` if absent or malformed.*- * Fast path: scan with `indexOf`. If the needle appears exactly once,- * we know it's the top-level field (any `project_id` nested in a JSON- * string value would be escaped as `\"project_id\":\"...`, which the- * indexOf scan can't match because of the `\` in front).- *- * Slow path: if the needle appears two or more times, the line has a- * legitimate nested `project_id` key (e.g. inside `properties` if a- * user happens to set one with that name). The regex/indexOf can't- * tell which is the top-level one — at that point we fall back to a- * real JSON.parse for correctness. This is rare in practice but- * removes the silent-attribution failure mode entirely.- *- * Returns `null` if no top-level project_id is present or the row is- * malformed.+ * Fast path: `indexOf` on the needle. A nested `project_id` inside a string+ * value is escaped (`\"project_id\"`), so it cannot match; exactly one hit is+ * the top-level field. Two or more hits mean a real nested key (a user+ * property of that name), so fall back to JSON.parse rather than misattribute.*/export function extractProjectId(line: string): string | null {const first = line.indexOf(PROJECT_ID_NEEDLE);@@ -45,9 +35,8 @@ export function extractProjectId(line: string): string | null {const valueStart = first + PROJECT_ID_NEEDLE.length;const valueEnd = line.indexOf('"', valueStart);- // valueEnd === valueStart means the value is empty (`"project_id":""`)- // — treat as missing so we don't pollute pub/sub counts with an empty- // key. Matches the old regex's `[^"]+` (one-or-more) behavior.+ // An empty value is treated as missing so it does not pollute pub/sub+ // counts with an empty key.if (valueEnd <= valueStart) {return null;}@@ -138,8 +127,6 @@ export class EventBuffer extends BaseBuffer {}add(event: IClickhouseEvent) {- // Event-buffer's add() is synchronous (in-memory push). Measured anyway- // for consistency with the other buffers' add-latency tracking.const start = performance.now();this.lastQueuedSeq += 1;this.pendingEvents.push(event);@@ -426,11 +413,9 @@ export class EventBuffer extends BaseBuffer {return;}- // The queued lines are already valid JSONEachRow (one stringified event- // per Redis entry), and the client's custom `json.stringify`- // (CLICKHOUSE_OPTIONS) passes strings through unchanged — so skipping- // JSON.parse here sends the bytes straight from Redis to the CH HTTP- // body, with no parse, no re-stringify and no intermediate allocations.+ // The queued lines are already JSONEachRow, and the client's custom+ // `json.stringify` (CLICKHOUSE_OPTIONS) passes strings through unchanged,+ // so the bytes go from Redis to the CH HTTP body without a parse.const countByProject = new Map<string, number>();const yieldEvery = this.getYieldInterval(queueEvents.length, {min: 1000,@@ -453,8 +438,6 @@ export class EventBuffer extends BaseBuffer {(chunk) =>ch.insert({table: 'events',- // Stream the raw JSONEachRow lines straight through — already- // serialized in Redis, no client-side parse/stringify needed.values: this.jsonEachRowStream(chunk),format: 'JSONEachRow',clickhouse_settings: this.getClickhouseSettings(),diff --git a/packages/core/src/buffers/group-buffer.ts b/packages/core/src/buffers/group-buffer.tsindex 80e9a68c1..024195141 100644--- a/packages/core/src/buffers/group-buffer.ts+++ b/packages/core/src/buffers/group-buffer.ts@@ -172,9 +172,7 @@ export class GroupBuffer extends BaseBuffer {return;}- // Raw passthrough: each Redis entry is already a valid JSONEachRow- // line. Streaming raw strings to CH skips JSON.parse + the client's- // re-stringify on the hot path.+ // Raw passthrough: each Redis entry is already a JSONEachRow line.const ch = this.resolveCh();const chStart = performance.now();await this.parallelLimit(this.chunks(items, this.chunkSize), (chunk) =>diff --git a/packages/core/src/buffers/profile-backfill-buffer.ts b/packages/core/src/buffers/profile-backfill-buffer.tsindex d4a073580..5da6eae7a 100644--- a/packages/core/src/buffers/profile-backfill-buffer.ts+++ b/packages/core/src/buffers/profile-backfill-buffer.ts@@ -72,8 +72,7 @@ export class ProfileBackfillBuffer extends BaseBuffer {const ch = this.resolveCh();// `replicatedTarget`, not `sql.id`: the clustered form carries an- // `ON CLUSTER` clause and is not a bare identifier, so `sql.id` refused it- // and every backfill flush threw before reaching ClickHouse.+ // `ON CLUSTER` clause and is not a bare identifier.const target = replicatedTarget(this.deps.config.clickhouseClustered,TABLE_NAMES.eventsdiff --git a/packages/core/src/buffers/profile-buffer.test.ts b/packages/core/src/buffers/profile-buffer.test.tsindex d76e1138e..b4910488d 100644--- a/packages/core/src/buffers/profile-buffer.test.ts+++ b/packages/core/src/buffers/profile-buffer.test.ts@@ -187,8 +187,7 @@ describe('ProfileBuffer', () => {chInsert.mockRejectedValueOnce(new Error('ClickHouse unavailable'));- // Errors propagate to tryFlush (which resyncs the counter). The safety- // property — queue preserved on CH failure — still holds.+ // Errors propagate to tryFlush; the queue is preserved on CH failure.await expect(profileBuffer.processBuffer()).rejects.toThrow('ClickHouse unavailable');diff --git a/packages/core/src/buffers/profile-buffer.ts b/packages/core/src/buffers/profile-buffer.tsindex 26a2fc46c..28efec9cb 100644--- a/packages/core/src/buffers/profile-buffer.ts+++ b/packages/core/src/buffers/profile-buffer.ts@@ -213,11 +213,8 @@ export class ProfileBuffer extends BaseBuffer {', ');try {- // Table alias `p` is required: without it, WHERE's `last_seen_at`- // resolves to the SELECT-list aggregate alias `max(last_seen_at) AS- // last_seen_at`, which is an aggregate function and illegal in WHERE- // (CH ILLEGAL_AGGREGATION). Qualifying with `p.` bypasses the alias- // lookup and binds to the raw column.+ // Qualify with `p.`: a bare `last_seen_at` resolves to the aggregate+ // alias and is illegal in WHERE (ILLEGAL_AGGREGATION).const rows = await this.chQuery<IClickhouseProfile>(sql`SELECT ${PROFILE_LATEST_AGGREGATE_COLUMNS}FROM ${sql.id(TABLE_NAMES.profiles)} AS p@@ -263,16 +260,10 @@ export class ProfileBuffer extends BaseBuffer {return;}- // Parse + merge within batch in a single pass. Yields back to the- // event loop periodically so concurrent add() calls and BullMQ- // heartbeats can make progress while the merge churns through- // profiles. The merge step is the hottest CPU stage of any buffer —- // deepMergeObjects recursively combines nested `properties` objects- // and was previously responsible for ~500ms event-loop blocks.- //- // Heavy loop (parse + deep merge): narrow band so we yield often- // even with smaller batches but don't yield every-few-items on- // very small ones.+ // Yields to the event loop periodically so concurrent add() calls and+ // BullMQ heartbeats progress: deepMergeObjects on nested `properties` is+ // the hottest CPU stage of any buffer. Narrow band: parse + deep merge+ // is a heavy loop.const mergeYieldEvery = this.getYieldInterval(rawProfiles.length, {min: 100,max: 1000,@@ -317,10 +308,8 @@ export class ProfileBuffer extends BaseBuffer {}}- // Fetch cache misses from ClickHouse in bounded chunks. Timed separately- // from the ch.insert phase because it can dominate flush time for- // profile-buffer specifically (FINAL replaced with argMax + GROUP BY in- // batchFetchFromClickhouse, but worth measuring directly).+ // Fetch cache misses from ClickHouse in bounded chunks, timed apart from+ // ch.insert because the fetch can dominate flush time.let chFetchMs: number | undefined;if (cacheMisses.length > 0) {const chFetchStart = performance.now();@@ -332,9 +321,8 @@ export class ProfileBuffer extends BaseBuffer {}}- // Final merge: in-batch profile + existing (from cache or ClickHouse).- // Second heavy stage — same deepMergeObjects call per unique profile.- // Same band as the in-batch merge above.+ // Final merge: in-batch profile + existing (cache or ClickHouse), same+ // band as the in-batch merge.const finalYieldEvery = this.getYieldInterval(uniqueProfiles.length, {min: 100,max: 1000,diff --git a/packages/core/src/buffers/replay-buffer.ts b/packages/core/src/buffers/replay-buffer.tsindex fe184d17e..dd73aef62 100644--- a/packages/core/src/buffers/replay-buffer.ts+++ b/packages/core/src/buffers/replay-buffer.ts@@ -72,11 +72,8 @@ export class ReplayBuffer extends BaseBuffer {return;}- // Raw passthrough: each Redis entry is already a valid JSONEachRow- // line (we JSON.stringify a single chunk before rpush). Streaming the- // raw strings to CH skips JSON.parse × N on the worker AND the- // client's internal JSON.stringify × N — significant because each- // rrweb chunk's `payload` is 10–100KB.+ // Raw passthrough: each Redis entry is already a JSONEachRow line, and an+ // rrweb chunk's `payload` is 10–100KB, so skipping parse/stringify matters.const ch = this.resolveCh();const chStart = performance.now();await this.parallelLimit(this.chunks(items, this.chunkSize), (chunk) =>diff --git a/packages/core/src/buffers/session-buffer.test.ts b/packages/core/src/buffers/session-buffer.test.tsindex 216299ef8..5555d1fc0 100644--- a/packages/core/src/buffers/session-buffer.test.ts+++ b/packages/core/src/buffers/session-buffer.test.ts@@ -356,9 +356,7 @@ describe('SessionBuffer', () => {chInsert.mockRejectedValueOnce(new Error('ClickHouse unavailable'));- // Errors now propagate to tryFlush (which handles them by resyncing the- // counter). processBuffer no longer swallows — we still verify the- // safety property: the queue is preserved.+ // Errors propagate to tryFlush; the queue is preserved.await expect(sessionBuffer.processBuffer()).rejects.toThrow('ClickHouse unavailable');diff --git a/packages/core/src/buffers/session-buffer.ts b/packages/core/src/buffers/session-buffer.tsindex 20e14ada5..d69e77201 100644--- a/packages/core/src/buffers/session-buffer.ts+++ b/packages/core/src/buffers/session-buffer.ts@@ -6,7 +6,7 @@ import type { IClickhouseSession } from '../modules/session/session.service';import { TABLE_NAMES } from '../shared/ch-tables';import { BaseBuffer, type BufferDeps } from './base-buffer';-// 30min of idle in event-time → session ends. Matches industry default.+// 30min of idle in event-time → session ends.const DEFAULT_SESSION_TIMEOUT_MS = 1000 * 60 * 30;/**@@ -227,9 +227,9 @@ export class SessionBuffer extends BaseBuffer {* call is a no-op — the new session still needs its blob and sorted-set* entry.*- * Blobs no longer have a Redis TTL — this is the only path that removes- * them. The daily vacuum cron is a backstop for the rare case where this- * fails (worker crash, network partition).+ * Blobs have no Redis TTL, so this is the only path that removes them. The+ * daily vacuum cron is a backstop for when it fails (worker crash, network+ * partition).*/async cleanup({projectId,@@ -552,12 +552,10 @@ export class SessionBuffer extends BaseBuffer {parsed.push(s);}- // A single high-activity session produces many (sign=-1, sign=+1)- // pairs within one flush window — every extend appends a pair. The- // VersionedCollapsingMergeTree collapses them at merge time, so the- // final state is correct either way, but inserting all the- // intermediate rows costs network bytes, gzip CPU, and CH ingest +- // merge work. Squash per-version to insert only the boundary rows.+ // A busy session appends a (-1, +1) pair per extend within one flush.+ // VersionedCollapsingMergeTree would collapse them at merge time anyway,+ // but inserting them all costs network, gzip CPU and merge work. Squash+ // per version to insert only the boundary rows.//// Set SESSION_BUFFER_SQUASH=false to disable as a safety hatch.const sessions = this.squashEnableddiff --git a/packages/core/src/jobs/compat.ts b/packages/core/src/jobs/compat.tsindex 3f20adec6..ee85adaad 100644--- a/packages/core/src/jobs/compat.ts+++ b/packages/core/src/jobs/compat.ts@@ -1,7 +1,5 @@-// There is no dual-write window, no drain and no queue rename: a legacy job-// already in the queue is read back through these hooks. A wrong one is-// silent job loss, which is why the tests next to this file were written-// before it.+// A legacy job already in the queue is read back through these hooks. A wrong+// one is silent job loss.import type { CompatHook } from './define';import { isEnvelope, type JobMeta } from './envelope';diff --git a/packages/core/src/jobs/jobs.metrics.test.ts b/packages/core/src/jobs/jobs.metrics.test.tsindex c708dfcb6..a0cda0e61 100644--- a/packages/core/src/jobs/jobs.metrics.test.ts+++ b/packages/core/src/jobs/jobs.metrics.test.ts@@ -31,8 +31,7 @@ test('five gauges per queue, named from the Redis key so dashboards keep workingfor (const suffix of COUNT_SUFFIXES) {expect(names).toContain(`sessions_${suffix}_count`);expect(names).toContain(`cron_${suffix}_count`);- // Acceptance note: the registry key stays `cohortCompute`, so the series- // name does too.+ // The series name follows the registry key, `cohortCompute`.expect(names).toContain(`cohortCompute_${suffix}_count`);}});diff --git a/packages/core/src/jobs/jobs.metrics.ts b/packages/core/src/jobs/jobs.metrics.tsindex 34ee16e9e..3ff1e65f9 100644--- a/packages/core/src/jobs/jobs.metrics.ts+++ b/packages/core/src/jobs/jobs.metrics.ts@@ -1,17 +1,7 @@-// The queue-side collectors, on the one core registry.-//-// Three deliberate differences from the metric set this replaces, written up-// for dashboard owners in `packages/core/docs/OPS_GRAFANA_MIGRATION.md`:-//-// 1. Registers the five `<queue>_*_count` gauges for all seven BullMQ-// queues, not just two — ten new series appear; none of the existing ten-// changes. 2. They register only where the role consumes (main.ts), because-// each gauge is one Redis round trip per scrape and ten api replicas-// exposing them would multiply that for no new information. 3. A gauge whose-// Redis read throws is skipped rather than failing the whole scrape, so one-// Redis blip does not turn into a 500 on /metrics and a gap in every panel;-// core's buffer gauges already swallow the same way-// (buffers/buffer.metrics.ts).+// The `<queue>_*_count` gauges register only where the role consumes: each is+// one Redis round trip per scrape, and api replicas would multiply that for no+// new information. A gauge whose Redis read throws is skipped rather than+// failing the whole scrape.import type { Queue as BullQueue } from 'bullmq';import client from 'prom-client';diff --git a/packages/core/src/jobs/schedulers.ts b/packages/core/src/jobs/schedulers.tsindex 4e998e96f..59e0b6133 100644--- a/packages/core/src/jobs/schedulers.ts+++ b/packages/core/src/jobs/schedulers.ts@@ -4,9 +4,7 @@// needed.//// The scheduler list is derived from the `cron` queue's own jobs, so a job-// and its schedule cannot drift apart. `schedulersFromRegistry` takes the-// registry's `cron` queue as data; the registry itself calls it and exports-// the result as `CRON_SCHEDULES`, so this file imports nothing above it.+// and its schedule cannot drift apart.import type { Logger } from '../logger';import type { AnyJob, QueueDefinition, RepeatSchedule } from './define';@@ -55,8 +53,7 @@ const CONFLICT_JOB_STATES = [const CONFLICT_ERROR_SUBSTRING = 'job ID already exists';/**- * Derives the scheduler list from the `cron` queue's own jobs — the join is by- * construction now, not by a second registry kept in sync by a test. A job's+ * Derives the scheduler list from the `cron` queue's own jobs. A job's* `cron` is `RepeatSchedule` (always-on), `null` (on-demand — e.g. `ping`,* scheduled conditionally elsewhere), or, on the `cron` queue, never simply* absent: `defineQueue`'s overload for that queue name makes omitting it a type@@ -171,7 +168,6 @@ async function upsertWithConflictRetry('job scheduler conflict detected, attempting cleanup');await removeConflictingJobs(queue, scheduler.id, logger);- // Also try removing the scheduler itself to start fresh.await queue.removeJobScheduler(scheduler.id).catch(() => undefined);try {diff --git a/packages/core/src/modules/chart/chart.rpc.ts b/packages/core/src/modules/chart/chart.rpc.tsindex 8907da097..e7bbff83f 100644--- a/packages/core/src/modules/chart/chart.rpc.ts+++ b/packages/core/src/modules/chart/chart.rpc.ts@@ -1,11 +1,10 @@-// The six member-only procedures use `protectedProcedure`, so-// `enforceAccess` reads the top-level `projectId` before the input is parsed.+// The six member-only procedures use `protectedProcedure`, so `enforceAccess`+// reads the top-level `projectId` before the input is parsed.//// `chartProcedure` (funnel, conversion, sankey, chart, aggregate, cohort) is the-// share-aware builder: `publicProcedure` plus one middleware that admits an-// anonymous caller holding a valid share — `shareId` + `id` resolve the saved-// report, and the request renders that report (the caller may only move the-// date window). The resolved report rides on `ctx.report`, so a handler+// share-aware builder: an anonymous caller holding a valid share resolves the+// saved report, and the request renders that report (the caller may only move+// the date window). The resolved report rides on `ctx.report`, so a handler// cannot forget to resolve it.import { z } from 'zod';diff --git a/packages/core/src/modules/chart/chart.service.test.ts b/packages/core/src/modules/chart/chart.service.test.tsindex 28a97da4c..cf097f904 100644--- a/packages/core/src/modules/chart/chart.service.test.ts+++ b/packages/core/src/modules/chart/chart.service.test.ts@@ -27,7 +27,7 @@ const DAY_MS = 24 * 60 * 60 * 1000;const PROJECT_CARD_DAYS = 90;// Bypass Redis caching so a stale entry from another run cannot gate an-// assertion. Captured before mock.module runs — see mcp's tools.test.ts.+// assertion. Captured before mock.module runs.const actualRedis = await import('@openpanel/redis');mock.module('@openpanel/redis', () => ({...actualRedis,@@ -264,8 +264,8 @@ describe('picker queries', () => {it('lists event names with counts, `*` first', async () => {const events = await service.listChartEvents(TEST_PROJECT_ID);- // Counts are `sum(event_count)` over distinct_event_names_mv, so they are- // real event totals. `count(name)` would count insert parts (ISSUES.md M1).+ // Counts are `sum(event_count)` over distinct_event_names_mv, i.e. real event+ // totals; `count(name)` would count insert parts.expect(events[0]?.name).toBe('*');expect(events[0]?.count).toBe(events.slice(1).reduce((total, event) => total + event.count, 0)@@ -284,10 +284,8 @@ describe('picker queries', () => {]);});- // The picker used to show `count(name)` over distinct_event_names_mv, which- // counts one row per name PER INSERT PART, not per event (ISSUES.md M1). On- // the large seed that showed 52 for an event with 974,387 occurrences. The- // only assertion that catches it is one against the events table itself.+ // `count(name)` over distinct_event_names_mv counts one row per name PER INSERT+ // PART, not per event, so only an assertion against the events table catches it.it('counts real events, not materialized-view rows', async () => {const events = await service.listChartEvents(TEST_PROJECT_ID);const actual = await ch.query({diff --git a/packages/core/src/modules/chart/chart.service.ts b/packages/core/src/modules/chart/chart.service.tsindex 99189c813..e140be44f 100644--- a/packages/core/src/modules/chart/chart.service.ts+++ b/packages/core/src/modules/chart/chart.service.ts@@ -1,9 +1,5 @@-// Every ClickHouse statement this module runs is a `sql` fragment from-// src/sql.ts. Funnel, conversion, sankey and retention are dispatched to the-// sibling `*.service.ts` files. The one thing still value-imported from-// `@openpanel/db` under `./src/` is the `sql` tag, which stays in-// `packages/db` by name: a compile-time template tag, no client and no-// request scope.+// Every ClickHouse statement here is a `sql` fragment from src/sql.ts. Funnel,+// conversion, sankey and retention dispatch to the sibling `*.service.ts` files.import { getChartPrevStartEndDate } from '@openpanel/shared';import { flatten, map, pipe, prop, sort, uniq } from 'ramda';@@ -110,17 +106,15 @@ export {type ChartBucketProfilesInput,ChartCohortIdError,} from './src/sql';-// Types only: `packages/core/src/index.ts` re-exports these two. The four-// statement functions beside them had no consumer through this file — the-// engine and the sibling services import ./src/statement directly.+// Types only: `packages/core/src/index.ts` re-exports these two.export type {AggregateChartSqlInput,ChartSqlInput,} from './src/statement';/**- * Cap on distinct event property keys returned to the picker. Projects in the- * 8k range exist, so the previous 10k was reachable in normal use.+ * Cap on distinct event property keys returned to the picker. Projects with+ * ~8k keys exist.*/const EVENT_PROPERTY_KEY_LIMIT = 50_000;@@ -144,12 +138,10 @@ const BUCKET_PROFILES_BATCH_SIZE = 200;const BUCKET_PROFILES_FETCH_CONCURRENCY = 4;/**- * Cap on the ids one chart data point hands the drill-down modal. The bucket- * query used to be unbounded, so a busy project's point returned a 31,447-id- * tail that the modal renders- * through a virtualizer and nobody scrolls to, after paying for every one of- * them in `profiles FINAL` batches. Same number as FUNNEL_PROFILES_LIMIT: both- * feed the same modal, so the two drill-downs should not disagree on depth.+ * Cap on the ids one chart data point hands the drill-down modal. A busy point+ * can hold tens of thousands of ids nobody scrolls to, each paid for in+ * `profiles FINAL` batches. Same number as FUNNEL_PROFILES_LIMIT: both feed the+ * same modal.*/const BUCKET_PROFILES_LIMIT = 1000;@@ -202,8 +194,6 @@ async function getProfilesInBatches(return perBatch.flat();}-// --- shared report input ------------------------------------------------------export type ShareableReportInput = IReportInput & {shareId?: string;id?: string;@@ -229,8 +219,6 @@ export function resolveReportInput(};}-// --- projectCard --------------------------------------------------------------export interface ProjectCardChartRow {value: number;date: Date;@@ -299,8 +287,6 @@ export async function getProjectCard(};}-// --- events / properties / values --------------------------------------------export interface ChartEventOption {name: string;count: number;@@ -448,8 +434,6 @@ export async function getChartPropertyValues(};}-// --- funnel / conversion / sankey / retention ---------------------------------async function currentAndPreviousPeriod(deps: ServiceDeps,chartInput: IReportInput@@ -629,8 +613,6 @@ export async function getRetentionChart(});}-// --- drill-down profiles ------------------------------------------------------export interface ChartBucketProfilesRequest {projectId: string;/** ISO string of the data point's bucket. */@@ -736,30 +718,22 @@ export async function getFunnelStepProfiles(return getProfilesInBatches(deps, ids, projectId, FUNNEL_PROFILES_BATCH_SIZE);}-// --- service -------------------------------------------------------------------// The funnel, conversion, sankey and retention services live in this module's-// own sibling files rather than in modules of their own, so they FOLD INTO-// `chart` rather than becoming four more `Services` members: chart.service.ts-// is already the dispatcher every caller goes through (`getFunnelChart` /-// `getConversionChart` / `getSankeyChart` / `getRetentionChart`), and a-// `Services` key per file would name four things that are not modules.+// The funnel, conversion, sankey and retention services fold into `chart`+// rather than becoming `Services` members: chart.service.ts is the dispatcher+// every caller goes through.export function createChartService(deps: ServiceDeps,services: () => Services) {- // The four chart sub-modules each expose their own `create*Service(deps)`.- // `services.ts` binds each under its own key; `chart` composes them as well,- // so the callers that reach a funnel/retention method through the chart- // facade keep working. Both bind the same stateless closures.+ // Each sub-module exposes its own `create*Service(deps)`; chart composes them+ // too so callers reaching them through the chart facade keep working.const funnel = createFunnelService(deps, services);const conversion = createConversionService(deps, services);const sankey = createSankeyService(deps, services);const retention = createRetentionService(deps, services);return {- // chartexecute: (input: IReportInput): Promise<FinalChart> =>executeChart(deps, input),executeAggregate: (input: IReportInput): Promise<FinalChart> =>@@ -781,7 +755,6 @@ export function createChartService(funnelStepProfiles: (input: FunnelStepProfilesRequest): Promise<IServiceProfile[]> => getFunnelStepProfiles(deps, input),- // funnelgetFunnelGroup: funnel.getFunnelGroup,/** The funnel row -> serie grouping (funnel.service.ts's `toSeries`). */toFunnelSeries: funnel.toSeries,@@ -792,19 +765,16 @@ export function createChartService(getFunnelCore: funnel.getFunnelCore,buildFunnelBase: funnel.buildFunnelBase,getFunnelProfileIds: funnel.getFunnelProfileIds,- // conversiongetConversionChart: (chartInput: IReportInput): ReturnType<typeof getConversionChart> =>getConversionChart(deps, chartInput),getConversion: conversion.getConversion,- // sankeygetRawWhereClause: sankey.getRawWhereClause,getSankeyChart: (input: IReportInput): ReturnType<typeof getSankeyChart> =>getSankeyChart(deps, input),getSankey: sankey.getSankey,getUserFlowCore: sankey.getUserFlowCore,- // retentionprocessCohortData: retention.processCohortData,getRetentionChart: (report: NonNullable<IServiceReport> | null,diff --git a/packages/core/src/modules/chart/funnel.service.ts b/packages/core/src/modules/chart/funnel.service.tsindex 159ad3ef0..18790f038 100644--- a/packages/core/src/modules/chart/funnel.service.ts+++ b/packages/core/src/modules/chart/funnel.service.ts@@ -433,10 +433,8 @@ export async function getFunnelProfileIds(}/**- * `ctx.services.chart`'s funnel half, as its own module factory.- * `createChartService` composes this rather than re-binding these functions- * itself, so the chart module keeps one registry key while this file exposes- * its own factory.+ * `ctx.services.chart`'s funnel half, as its own module factory, composed by+ * `createChartService`.*/export function createFunnelService(deps: ServiceDeps,diff --git a/packages/core/src/modules/chart/retention.service.test.ts b/packages/core/src/modules/chart/retention.service.test.tsindex fce7194f4..c4855d896 100644--- a/packages/core/src/modules/chart/retention.service.test.ts+++ b/packages/core/src/modules/chart/retention.service.test.ts@@ -92,9 +92,8 @@ describe('getRetentionCohort', () => {});it('counts retention across the year boundary (week interval)', async () => {- // The old toWeek() implementation returned week-of-year (resets each Jan),- // so a 2024-W52 -> 2025-W01 return produced a negative diff and was dropped.- // This locks the year-aware behavior.+ // Week-of-year resets each Jan, so a 2024-W52 -> 2025-W01 return must still+ // count (year-aware diff).const rows = await getRetentionCohort(deps, {projectId: PROJECT_ID,firstEvent: ['app_open'],@@ -140,9 +139,9 @@ describe('getRetentionCohort', () => {// fired app_open on D2 (RU1, RU2, RU4) already belongs to an earlier cohort.expect(rows.map((row) => row.cohort_interval)).toEqual([day.d0, day.d1]);- // Characterize the bug we fixed: the legacy "every-occurrence" cohorting- // (no GROUP BY profile_id dedup) places recurring users in every day they- // were active, producing a spurious D2 cohort and inflated sizes.+ // Without the GROUP BY profile_id dedup, every-occurrence cohorting places+ // recurring users in every day they were active, producing a spurious D2+ // cohort and inflated sizes.const legacy = await chQuery<{ cohort_interval: string; size: number }>(`SELECT toDate(created_at) AS cohort_interval, COUNT(DISTINCT profile_id) AS sizeFROM ${TABLE_NAMES.cohort_events_mv}@@ -335,7 +334,7 @@ describe('processCohortData weighted average (pure)', () => {expect(avg.values[0]).toBe(80);expect(avg.percentages[0]).toBe(1);// Period 2 pools C0+C1 only (C2 is immature). C1's genuine zero is included,- // so it's 25/180 = 0.14 — NOT 25/100 = 0.25 (the old "exclude zeros" bug).+ // so it's 25/180 = 0.14, not 25/100 = 0.25.expect(avg.percentages[2]).toBe(0.14);// Period 3 pools only C0 (C1, C2 immature): 10/100 = 0.10.expect(avg.percentages[3]).toBe(0.1);diff --git a/packages/core/src/modules/chart/retention.service.ts b/packages/core/src/modules/chart/retention.service.tsindex b687ab605..5b6dcf097 100644--- a/packages/core/src/modules/chart/retention.service.ts+++ b/packages/core/src/modules/chart/retention.service.ts@@ -1,6 +1,5 @@-// The cohort matrix here is the single source of truth for retention cohorts:-// it powers the dashboard retention chart (via the tRPC `cohort` procedure) as-// well as the MCP / agent / REST retention endpoints.+// The cohort matrix is shared by the dashboard retention chart and the MCP /+// agent / REST retention endpoints.import { DateTime, round } from '@openpanel/shared';import { range } from 'ramda';@@ -194,17 +193,13 @@ export async function getEngagementCore(};}-// ----------------------------------------------------------------------------// Cohort retention matrix-//-// Definition: every user is assigned to exactly ONE cohort, the interval of-// their FIRST `firstEvent` within the window (first-touch). For each cohort we+// Cohort retention matrix: every user is assigned to exactly ONE cohort, the+// interval of their FIRST `firstEvent` within the window. For each cohort we// count how many of those users performed `secondEvent` 0..N intervals later.-// - criteria 'on' -> active exactly k intervals after (=)-// - criteria 'on_or_after' -> active at least k intervals after (>=, cumulative)-// When `firstEvent` / `secondEvent` is empty the name filter is dropped and the-// query measures retention on ANY activity (active-user retention).-// ---------------------------------------------------------------------------+// - criteria 'on' -> active exactly k intervals after (=)+// - criteria 'on_or_after' -> active at least k intervals after (>=, cumulative)+// An empty `firstEvent` / `secondEvent` drops the name filter, measuring+// active-user retention.const LUXON_UNIT: Record<IRetentionInterval, 'days' | 'weeks' | 'months'> = {minute: 'days',diff --git a/packages/core/src/modules/chart/sankey.service.ts b/packages/core/src/modules/chart/sankey.service.tsindex 603ba3dc3..c184a974f 100644--- a/packages/core/src/modules/chart/sankey.service.ts+++ b/packages/core/src/modules/chart/sankey.service.ts@@ -35,8 +35,6 @@ const MIN_LINK_PERCENT = 0.25;const PERCENT = 100;const FIRST_STEP = 1;-// `packages/core/src/index.ts` names this file as the schema's source; the-// schema itself lives in chart.constants.ts.export { type IGetSankeyInput, zGetSankeyInput } from './chart.constants';export interface SankeyNode {@@ -134,12 +132,10 @@ function toSankeyEvent(* so. An unparseable date is left alone — the statement itself is where a* malformed range is rejected.*- * Applied by `getSankeyChart`, the `chart.sankey` entry point, and not by- * `getSankey` itself: it bounds that procedure specifically, because it is- * the range picker that offers `12m`/`lastYear`. The REST- * `/insights/:projectId/user_flow` route, the MCP `get_user_flow` tool and the- * assistant tool take caller-supplied dates, so they are not covered by the- * range picker's cap.+ * Applied by `getSankeyChart` (the `chart.sankey` entry point) and not by+ * `getSankey`: the REST `/insights/:projectId/user_flow` route, the MCP+ * `get_user_flow` tool and the assistant tool take caller-supplied dates and+ * are not covered by the range picker's cap.*/export function assertSankeyWindowIsAnswerable(startDate: string,@@ -388,7 +384,7 @@ function mergeFinalNodes(return b.value - a.value;});- // Links can now point at the same merged target; sum them.+ // Links can point at the same merged target; sum them.const linkMap = new Map<string, number>();for (const link of remappedLinks) {const key = `${link.source}->${link.target}`;diff --git a/packages/core/src/modules/chart/src/compiled.ts b/packages/core/src/modules/chart/src/compiled.tsindex 03999a158..480234e35 100644--- a/packages/core/src/modules/chart/src/compiled.ts+++ b/packages/core/src/modules/chart/src/compiled.ts@@ -1,13 +1,8 @@-// The ONE place the chart module splices pre-compiled SQL text.-//-// Both filter compilers (./filter-where.ts, ./table-filter-where.ts) and the-// field resolver (./field-resolution.ts) return `SqlFragment`s, so nothing-// crosses `compiledText` from them. What is left are the places the chart-// builders splice a name they generated themselves — CTE names, `b_<n>` /-// `label_<n>` aliases, a `toStartOf*` keyword, the backtick-quoted-// `profile.properties.<key>` CTE alias — none of which is a value and none of-// which `sql.id` can express. It stays a named export in one file, not a-// `sql.raw()`, so the census is a grep.+// The ONE place the chart module splices pre-compiled SQL text: names the chart+// builders generate themselves (CTE names, `b_<n>` / `label_<n>` aliases, a+// `toStartOf*` keyword, the backtick-quoted `profile.properties.<key>` CTE+// alias), none of which is a value and none of which `sql.id` can express. It+// stays a named export in one file, not a `sql.raw()`, so the census is a grep.import {isSqlFragment,@@ -55,10 +50,9 @@ function narrowableKey(slot: SqlSlot, keys: string[]): string | null {}/**- * Splice nested fragments into one flat (literals, params) interleaving. The- * rendered statement is unchanged — this only makes "the literal immediately- * before this param" a question with an answer, which is what the rewrite- * below needs now that `sql.id` contributes a nested fragment of its own.+ * Splice nested fragments into one flat (literals, params) interleaving, so+ * "the literal immediately before this param" is answerable, which the rewrite+ * below needs because `sql.id` contributes a nested fragment of its own.*/function flatten(fragment: SqlFragment,@@ -84,16 +78,11 @@ function flatten(}/**- * The same rewrite over a fragment. It runs on the literal parts, and on the- * one shape the field resolver now emits instead of text — the literal+ * The same rewrite over a fragment: on the literal parts, and on the literal* `profile.properties[` followed by the key as a bound `String` param and a- * closing `]`. Collapsing those three pieces is what keeps a narrowed- * `profile.properties.<key>` ref pointing at the CTE's scalar column once the- * key stopped being part of the SQL text.- *- * A value can no longer be rewritten even in principle: only a param the- * resolver placed in that exact position is eligible, and it is replaced by an- * identifier rather than edited.+ * closing `]`. Collapsing those pieces keeps a narrowed ref pointing at the+ * CTE's scalar column. Only a param in that exact position is eligible, and it+ * is replaced by an identifier rather than edited.*/export function fragmentWithProfileRefs(fragment: SqlFragment,diff --git a/packages/core/src/modules/chart/src/conversion.sql.ts b/packages/core/src/modules/chart/src/conversion.sql.tsindex dced41210..01e14fba8 100644--- a/packages/core/src/modules/chart/src/conversion.sql.ts+++ b/packages/core/src/modules/chart/src/conversion.sql.ts@@ -1,10 +1,5 @@// The conversion statement as a pure `sql` fragment.//-// The field resolver and both filter compilers return fragments; their output —-// breakdown expressions, per-step filter clauses and the cohort joins — is-// interpolated directly. `compiledText` is left with the `b_<index>` aliases-// this file generates itself.-//// Cluster note: `events`, `profiles` and `groups` are Distributed on Cloud. The// profile / group / cohort joins are LEFT ANY JOIN and run under the client's// `distributed_product_mode: 'allow'`; no `IN (subquery)` is involved.diff --git a/packages/core/src/modules/chart/src/dates.ts b/packages/core/src/modules/chart/src/dates.tsindex d4e9e7eed..4c3802aed 100644--- a/packages/core/src/modules/chart/src/dates.ts+++ b/packages/core/src/modules/chart/src/dates.ts@@ -1,7 +1,6 @@-// @openpanel/db's clickhouse/client.ts owns these, but importing it constructs-// a client and a pino logger at import time (see insight.service.ts's header),-// and the fragment builders here must stay pure and synchronous. Same choice-// cohort.service.ts made for its table map.+// Importing @openpanel/db's clickhouse/client constructs a client and a pino+// logger at import time, and the fragment builders here must stay pure and+// synchronous.const CLICKHOUSE_DATE_SUFFIX = /(\.\d{3})?Z+$/;diff --git a/packages/core/src/modules/chart/src/engine/format.ts b/packages/core/src/modules/chart/src/engine/format.tsindex 50d8409e9..697dd0025 100644--- a/packages/core/src/modules/chart/src/engine/format.ts+++ b/packages/core/src/modules/chart/src/engine/format.ts@@ -88,7 +88,6 @@ export function format(return {id: slug(cs.id),names: displayName,- // TODO: Do we need this now?event: {id: definition?.id,name: eventName,diff --git a/packages/core/src/modules/chart/src/field-resolution.test.ts b/packages/core/src/modules/chart/src/field-resolution.test.tsindex de44652a1..3e13dc431 100644--- a/packages/core/src/modules/chart/src/field-resolution.test.ts+++ b/packages/core/src/modules/chart/src/field-resolution.test.ts@@ -1,16 +1,9 @@/**- * Unit tests for the event-field resolution helpers.- *- * Background — these helpers exist because the chart, funnel, and conversion- * services used to inline whatever field name the dashboard sent (saved report,- * autocomplete picker, raw API call) directly into SQL. That worked for the- * common case but produced UNKNOWN_IDENTIFIER errors when:- *- * Older clients sent camelCase names (`referrerName`) that don't match the- * snake_case ClickHouse schema; - users picked utm_* in the property filter UI- * without realising those live in the `properties` map on the events table; - a- * saved report referenced a custom property (`temple_name`) as if it were a- * top-level column.+ * Unit tests for the event-field resolution helpers. Field names from the+ * dashboard (saved report, autocomplete picker, raw API call) must not reach SQL+ * verbatim: older clients send camelCase names (`referrerName`), users pick+ * utm_* without realising they live in the `properties` map, and saved reports+ * reference custom properties (`temple_name`) as if they were top-level columns.*/import { describe, expect, it } from 'bun:test';import { isKnownEventField, normalizeEventField } from './field-resolution';diff --git a/packages/core/src/modules/chart/src/field-resolution.ts b/packages/core/src/modules/chart/src/field-resolution.tsindex 787772595..ccc2322fa 100644--- a/packages/core/src/modules/chart/src/field-resolution.ts+++ b/packages/core/src/modules/chart/src/field-resolution.ts@@ -2,17 +2,10 @@// Chart field resolution: how a report's field name (`referrerName`,// `utm_source`, `properties.x.*`, `profile.email`, `group.name`,// `cohort:<id>`, `has_profile`) becomes a ClickHouse expression, plus the-// profile-CTE narrowing helpers.-//-// Each resolver returns a `SqlFragment` whose values — cohort ids, cohort-// labels, project ids and the `properties[...]` map keys — bind as-// `{pN:Type}` params, and whose identifiers go through `sql.id`.-//-// The one piece of text this file still builds is the backtick-quoted CTE alias-// `` `profile.properties.<key>` `` (see `profilePropertiesCteSelect`), which is-// an identifier `sql.id` cannot express — three dot-separated parts — and which-// `collectProfilePropertyKeys` already guards. It goes through `compiled.ts`,-// the module's one text seam.+// profile-CTE narrowing helpers. Values bind as params and identifiers go+// through `sql.id`; the one exception is the backtick-quoted CTE alias+// `` `profile.properties.<key>` `` (see `profilePropertiesCteSelect`), which+// `sql.id` cannot express and which goes through `compiled.ts`.import { type SqlFragment, sql } from '@openpanel/db/src/clickhouse/sql';import type { IChartBreakdown } from '../../report/report.constants';@@ -31,11 +24,8 @@ export const CHART_TABLE = {cohortEventsMv: 'cohort_events_mv',} as const;-// Top-level columns on the events table. Derived from the migration in-// @openpanel/db's code-migrations/3-init-ch.ts (+ revenue added in-// 6-add-revenue-column.ts).-// Used to distinguish real columns from property keys and reject unknown-// identifiers before they reach ClickHouse.+// Top-level columns on the events table, used to tell real columns from property+// keys and to reject unknown identifiers before they reach ClickHouse.export const EVENT_TOP_LEVEL_COLUMNS = new Set<string>(['id','name',@@ -257,8 +247,7 @@ export function cohortBreakdownLabelExpr(/*** The membership subselect both filter compilers use for `inCohort` /- * `notInCohort`. Uses a plain `IN (subquery)`, not `GLOBAL IN` — preserved- * deliberately as an accepted trade-off, not an oversight.+ * `notInCohort`. A plain `IN (subquery)`, not `GLOBAL IN`: an accepted trade-off.*/export function buildCohortMembersSubselect(cohortIds: string[],@@ -395,8 +384,7 @@ function matchPropertyMapPrefix(property: string): string | undefined {* scalar, decided by testing the rendered TEXT for a `%`. That's reachable* two ways: the wildcard branch emits `transformPropertyKey`'s pattern, and a* NON-wildcard key containing a literal `%` also matches — a known defect- * (`properties.a%b` is treated as an array and fails at ClickHouse), kept for- * behavioral parity.+ * (`properties.a%b` is treated as an array and fails at ClickHouse).*/export function isWildcardPropertyKey(rawProperty: string): boolean {const property = normalizeEventField(rawProperty);@@ -423,12 +411,9 @@ export function getSelectPropertyKey(*/eventsAlias?: string): SqlFragment {- // Map camelCase aliases (`referrerName` → `referrer_name`) and bare UTM- // names (`utm_source` → `properties.__query.utm_source`) into their- // canonical form before doing any pattern matching. The fallback at the- // bottom of this function returns `property` verbatim, so without this- // normalization an alias would leak into the generated SQL and fail with- // UNKNOWN_IDENTIFIER.+ // Normalize aliases (`referrerName`, bare `utm_source`) first: the fallback at+ // the bottom returns `property` verbatim, so an alias would leak into the SQL+ // and fail with UNKNOWN_IDENTIFIER.const property = normalizeEventField(rawProperty);const extractedCohortId = cohortId || extractCohortId(property);@@ -440,7 +425,7 @@ export function getSelectPropertyKey(return sql`if(profile_id != device_id, 'true', 'false')`;}- // Handle group properties — requires ARRAY JOIN + _g JOIN to be present in query+ // Group properties need the ARRAY JOIN + _g JOIN to be present in the query.if (property.startsWith('group.') && projectId) {return getGroupPropertySql(property);}@@ -468,7 +453,6 @@ export function getSelectPropertyKey(return sql`${map}[${sql.string(property.replace(new RegExp(`^${match}.`), ''))}]`;}-// --- profile-property CTE narrowing (perf) ---------------------------------// profile.properties.<key> refs render as Map lookups `profile.properties['<key>']`.// Pulling the whole `properties` Map into the profile CTE makes the LEFT ANY// JOIN hash carry the full Map per profile — roughly a kilobyte each on realdiff --git a/packages/core/src/modules/chart/src/field-values.sql.ts b/packages/core/src/modules/chart/src/field-values.sql.tsindex 2a46eca5b..263e8ea7f 100644--- a/packages/core/src/modules/chart/src/field-values.sql.ts+++ b/packages/core/src/modules/chart/src/field-values.sql.ts@@ -28,7 +28,7 @@ export const SESSION_LEVEL_VALUE_COLUMNS: ReadonlySet<string> = new Set([/*** A value picker only needs recently-seen values, so the `events` fallback is- * clamped to this window instead of the old 6-month full scan.+ * clamped to this window.*/export const EVENT_FIELD_VALUES_LOOKBACK_DAYS = 30;diff --git a/packages/core/src/modules/chart/src/filter-cast.ts b/packages/core/src/modules/chart/src/filter-cast.tsindex 4575dab19..b234f7385 100644--- a/packages/core/src/modules/chart/src/filter-cast.ts+++ b/packages/core/src/modules/chart/src/filter-cast.ts@@ -1,9 +1,6 @@/** biome-ignore-all lint/style/useDefaultSwitchClause: switch cases are exhaustive by design */// The typed-cast half of `buildFilterWhere` (./table-filter-where.ts) and of// `./filter-where.ts` (the event-property compiler).-//-// Both filter compilers call the same pair: `castFragment` and-// `buildTypedClauseFragment` bind every value as a `{name:Type}` param.import {type SqlFragment,diff --git a/packages/core/src/modules/chart/src/filter-where.ts b/packages/core/src/modules/chart/src/filter-where.tsindex 9b32df668..7cd0cd630 100644--- a/packages/core/src/modules/chart/src/filter-where.ts+++ b/packages/core/src/modules/chart/src/filter-where.ts@@ -157,7 +157,6 @@ export function getEventFiltersWhereClause(return;}- // Handle group. prefixed filters (requires ARRAY JOIN + _g JOIN in query)if (name.startsWith('group.') && projectId) {const whereFrom = getGroupPropertySql(name);if (hasTypedCast(filter.type) && isTypedOperator(operator)) {diff --git a/packages/core/src/modules/chart/src/funnel.sql.test.ts b/packages/core/src/modules/chart/src/funnel.sql.test.tsindex 0ac5fa3a1..9d7d8060d 100644--- a/packages/core/src/modules/chart/src/funnel.sql.test.ts+++ b/packages/core/src/modules/chart/src/funnel.sql.test.ts@@ -1,18 +1,13 @@/**- * SQL-shape tests for the funnel statements.+ * SQL-shape tests for the funnel statements: render, then `EXPLAIN <sql>` with the+ * bound params against the isolated `openpanel_test` ClickHouse (pinned by+ * test/preload.ts). EXPLAIN resolves columns without executing, so a breakdown+ * expression referencing a join alias that was never added fails here as+ * UNKNOWN_IDENTIFIER.*- * Strategy, same as sql.test.ts: render the statement, then run `EXPLAIN <sql>`- * with its bound params against the isolated `openpanel_test` ClickHouse- * (pinned by test/preload.ts). EXPLAIN parses and resolves columns without- * executing, so a breakdown expression referencing a join alias that was never- * added fails here as UNKNOWN_IDENTIFIER — which is exactly the class of bug- * that made funnel "View Users" return "No users found" for profile-property- * and cohort breakdowns.- *- * Text assertions look at the rendered `query` with placeholder NUMBERS- * normalised away: a fragment reused in two clauses renders `{p3:String}` in- * one and `{p11:String}` in the other, and the property under test is that the- * same condition appears twice, not what its slots were numbered.+ * Text assertions normalise placeholder NUMBERS away: a fragment reused in two+ * clauses renders `{p3:String}` in one and `{p11:String}` in the other, and the+ * property under test is that the same condition appears twice.** The statements' only Postgres read (the breakdown's cohort names) is stubbed,* so this file is independent of whichever partial `prisma-client` mock a@@ -236,7 +231,7 @@ describe('funnel.sql / buildFunnelBase — cohort breakdowns', () => {const alias = `cohort_${COHORT_ID.replace(/-/g, '_')}`;expect(sql).toContain(`${alias}.profile_id`);expect(sql).toContain(`AS ${alias}`);- // The cohort label is a bound value now.+ // The cohort label is a bound value.expect(text).toContain('Power users');await explain(statement);});diff --git a/packages/core/src/modules/chart/src/funnel.sql.ts b/packages/core/src/modules/chart/src/funnel.sql.tsindex 4dbaad0ee..dc22460ea 100644--- a/packages/core/src/modules/chart/src/funnel.sql.ts+++ b/packages/core/src/modules/chart/src/funnel.sql.ts@@ -2,13 +2,8 @@//// The chart and the profile list are built from ONE base: a breakdown// expression only works if the join it references was added, and the joins-// depend on the breakdowns, so building the selects in one place and the joins-// in another is exactly the bug that made funnel "View Users" return "No users-// found" for profile-property and cohort breakdowns.-//-// The field resolver and filter compiler render text (see compiled.ts); their-// output — filter clauses, breakdown expressions, the profile CTE columns and-// the cohort joins — is spliced, everything else is bound.+// depend on the breakdowns, so building them apart made funnel "View Users"+// return no users for profile-property and cohort breakdowns.//// Cluster note: `events`, `profiles`, `groups` and `cohort_members` are// Distributed on Cloud. Every join runs under the client's@@ -218,10 +213,8 @@ function profileCteColumns(const fields = new Set(profileJoinFields(profileFilters, profileBreakdowns));- // Both the breakdown- and the filter-derived names are vetted above; the- // allowlist here is the backstop that makes `sql.id` throw rather than- // inline if either loop ever lets something else through (main #512,- // GHSA-pc3q-gw7f-p2x2).+ // The allowlist is the backstop that makes `sql.id` throw rather than inline+ // if either loop above lets something else through (GHSA-pc3q-gw7f-p2x2).const columns: SqlFragment[] = Array.from(fields).map((field) =>sql.id(field, FUNNEL_JOIN_COLUMNS));@@ -405,10 +398,6 @@ export function funnelProfilesQuery(return sql`${base.ctes}SELECT DISTINCT profile_id FROM funnel WHERE ${sql.join(where, ' AND ')} LIMIT ${sql.uint64(input.limit)}`;}-/**- * No in-repo caller. Reachable only through `packages/core/src/index.ts`,- * which still publishes the wrapper.- */export function funnelSessionsQuery(input: {projectId: string;startDate: string;diff --git a/packages/core/src/modules/chart/src/profile-columns.test.ts b/packages/core/src/modules/chart/src/profile-columns.test.tsindex 303e22aaa..65a47c020 100644--- a/packages/core/src/modules/chart/src/profile-columns.test.ts+++ b/packages/core/src/modules/chart/src/profile-columns.test.ts@@ -1,8 +1,6 @@-// The JOIN column list used to be built by splitting raw `profile.*` filter-// names, so a crafted name could close the subquery (GHSA-pc3q-gw7f-p2x2). The-// rewrite binds every identifier through `sql.id`, so injection is impossible;-// what these assert is that the list is also an allowlist, so a filter cannot-// name a column it has no business reading.+// Every identifier binds through `sql.id`, so injection is impossible+// (GHSA-pc3q-gw7f-p2x2); these assert the list is also an allowlist, so a filter+// cannot name a column it has no business reading.import { describe, expect, it } from 'bun:test';import { JOINABLE_PROFILE_COLUMNS } from '../chart.constants';import { profileJoinFields } from './funnel.sql';diff --git a/packages/core/src/modules/chart/src/retention.sql.test.ts b/packages/core/src/modules/chart/src/retention.sql.test.tsindex 320e6a579..51e52ed20 100644--- a/packages/core/src/modules/chart/src/retention.sql.test.ts+++ b/packages/core/src/modules/chart/src/retention.sql.test.ts@@ -25,8 +25,7 @@ beforeAll(async () => {describe('rollingActiveUsersQuery', () => {// `date + n` smears every active day forward, so without an upper bound the- // series runs up to `days - 1` days past today. On the seeded acme-web at- // days=7 that drew two points into the future (ISSUES.md H8c).+ // series runs up to `days - 1` days past today.it('never projects past today', () => {for (const days of [1, 7, 30]) {const { query } = RET.rollingActiveUsersQuery(diff --git a/packages/core/src/modules/chart/src/retention.sql.ts b/packages/core/src/modules/chart/src/retention.sql.tsindex c14c8610c..06b1bfb4d 100644--- a/packages/core/src/modules/chart/src/retention.sql.ts+++ b/packages/core/src/modules/chart/src/retention.sql.ts@@ -1,8 +1,5 @@// Every ClickHouse query the retention service runs, as pure `sql` fragments.//-// The filter compiler renders text (see compiled.ts); its output is the only-// thing spliced into the cohort statement.-//// Cluster note: `events`, `dau_mv` and `cohort_events_mv` are Distributed on// Cloud. None of these statements contains an `IN (subquery)` — the cohort// matrix is a CTE self-join — so no `IN` / `GLOBAL IN` decision arises.@@ -50,7 +47,6 @@ export interface RetentionSeriesQueryInput {endDate?: string;}-/** Same spelling as `overview/src/pages.sql.ts` and `profile/src/sql.ts`. */function createdAtWithin({startDate,endDate,diff --git a/packages/core/src/modules/chart/src/run-query.ts b/packages/core/src/modules/chart/src/run-query.tsindex c4b6408a3..479489526 100644--- a/packages/core/src/modules/chart/src/run-query.ts+++ b/packages/core/src/modules/chart/src/run-query.ts@@ -1,16 +1,7 @@-// The chart module's one ClickHouse entry point.-//-// The client is `deps.ch` — the same round-robin/retry proxy `main.ts` builds-// and hands to every scope — instead of a lazy-// `import('@openpanel/db/src/clickhouse/client')`. Reaching it through the-// scope is what puts the request's `requestId` on the query's log line; the-// lazy loader had no request scope to carry.-//-// `deps.ch.query` is `chQuery`'s own transport (`withRetry(client =>-// client.query(...))`), and the Int-meta coercion below is the same one-// `chQuery` applies — ClickHouse's JSON format returns every Int*/UInt* column-// as a string, so a caller's `count: number` only holds if it is parsed back-// out here. Same shape as the overview module's run-query.ts.+// The chart module's one ClickHouse entry point. The client is `deps.ch`, which+// puts the request's `requestId` on the query's log line. ClickHouse's JSON+// format returns every Int*/UInt* column as a string, so a caller's+// `count: number` only holds if it is parsed back out here, as `chQuery` does.import {type SqlFragment,diff --git a/packages/core/src/modules/chart/src/sankey.sql.ts b/packages/core/src/modules/chart/src/sankey.sql.tsindex adf40efb6..c4f85e48b 100644--- a/packages/core/src/modules/chart/src/sankey.sql.ts+++ b/packages/core/src/modules/chart/src/sankey.sql.ts@@ -1,9 +1,5 @@// Every ClickHouse statement the sankey (user-flow) service runs, as pure `sql`// fragments.-//-// The filter compiler renders text (see compiled.ts); only its output is-// spliced.-//// Cluster note: `events` is Distributed on Cloud, and the `session_id IN// (SELECT session_id FROM start_event_sessions)` HAVING clauses are plain `IN// (subquery)` over a CTE. Whether these should be `GLOBAL IN` on a cluster is@@ -23,8 +19,7 @@ export const TOP_ENTRY_EVENTS = 3;* lambda indexes its own captured `groupArray` is Theta(n^2) in the longest* session's event count — ClickHouse re-materialises the array once per* element, which is fatal for a tenant with one very long session.- * `arrayCompact` is the built-in for exactly this and is linear; the result- * is byte-identical.+ * `arrayCompact` is the built-in for exactly this and is linear.*/const DEDUPE_CONSECUTIVE = compiledText('arrayCompact(groupArray(event_name)) as events_deduped'diff --git a/packages/core/src/modules/chart/src/sql.test.ts b/packages/core/src/modules/chart/src/sql.test.tsindex e47f58514..ec1255599 100644--- a/packages/core/src/modules/chart/src/sql.test.ts+++ b/packages/core/src/modules/chart/src/sql.test.ts@@ -1,23 +1,16 @@/**- * SQL-shape tests for the chart statements.- *- * Strategy: render the statement, then run `EXPLAIN <sql>` with its bound- * params against the isolated `openpanel_test` ClickHouse (pinned by- * test/preload.ts). EXPLAIN parses, resolves columns and builds the plan- * without executing, so UNKNOWN_IDENTIFIER / AMBIGUOUS_IDENTIFIER / bad JOIN ON+ * SQL-shape tests for the chart statements: render, then `EXPLAIN <sql>` with the+ * bound params against the isolated `openpanel_test` ClickHouse (pinned by+ * test/preload.ts). EXPLAIN resolves columns and builds the plan without+ * executing, so UNKNOWN_IDENTIFIER / AMBIGUOUS_IDENTIFIER / bad JOIN ON* expressions surface without seeded data. WITH FILL TO < FROM is a runtime* check, so it is covered by a plain string assertion instead.*- * Text assertions look at the rendered `query`; chart-level values are- * `{pN:Type}` placeholders there, filter/breakdown expressions are inlined- * verbatim (see field-resolution.ts's header).- ** The statements' only Postgres read (the project's cohorts, for the- * all-cohorts breakdown) is stubbed to "no cohorts": that is the precondition- * the all-cohorts cases assert on, and it keeps this file independent of- * whichever partial `prisma-client` mock a module-root service test left behind- * (bun runs every file in one shared registry without `--isolate`). The real- * Postgres cohort path runs in../chart.service.test.ts.+ * all-cohorts breakdown) is stubbed to "no cohorts", which is the precondition+ * the all-cohorts cases assert on, and keeps this file independent of+ * whichever partial `prisma-client` mock a module-root service test left behind.+ * The real Postgres cohort path runs in ../chart.service.test.ts.*/import { afterAll, beforeAll, describe, expect, it, mock } from 'bun:test';@@ -29,8 +22,8 @@ import type {import type { AggregateChartSqlInput, ChartSqlInput } from './statement';const cohortFindMany = mock(async () => []);-// Plain-object snapshot, not the live binding — restoring from the namespace-// import in afterAll would just re-apply the mock (see cohort.service.test.ts).+// Plain-object snapshot, not the live binding: restoring from the namespace+// import in afterAll would just re-apply the mock.const actualPrismaClient = await import('@openpanel/db/src/prisma-client');const realPrismaClient = { ...actualPrismaClient };mock.module('@openpanel/db/src/prisma-client', () => ({@@ -320,9 +313,8 @@ describe('getChartSql', () => {);});- // main a2cbf2e8: the subquery was aliased `subQuery`, so the `e` every- // breakdown SELECT is built with was out of scope and the chart failed- // with UNKNOWN_IDENTIFIER on `e.properties`.+ // The subquery must not be aliased `subQuery`: the `e` every breakdown SELECT+ // is built with would be out of scope (UNKNOWN_IDENTIFIER on `e.properties`).it('one_event_per_user + a property breakdown resolves the events alias', async () => {const rendered = await getChartSql({event: event({ segment: 'one_event_per_user' }),@@ -335,8 +327,7 @@ describe('getChartSql', () => {});// Saved reports / older clients can send field names that don't match the- // events schema; the chart service used to inline them verbatim, crashing- // parse.+ // events schema; they must not be inlined verbatim.it('normalizes camelCase filter alias (referrerName → referrer_name)', async () => {const rendered = await getChartSql({event: event({@@ -365,9 +356,9 @@ describe('getChartSql', () => {});it('drops unknown breakdown rather than emitting invalid identifier', async () => {- // `temple_name` is a custom property (lives in properties map) but got- // saved as a top-level breakdown. The old code path emitted- // `SELECT temple_name as _uc_label_1 FROM events`, failing parse.+ // `temple_name` is a custom property (lives in properties map) saved as a+ // top-level breakdown; it must not be emitted as+ // `SELECT temple_name as _uc_label_1 FROM events`.const rendered = await getChartSql({event: event(),breakdowns: [breakdown('temple_name')],diff --git a/packages/core/src/modules/chart/src/sql.ts b/packages/core/src/modules/chart/src/sql.tsindex 9d75b47c5..b9f872266 100644--- a/packages/core/src/modules/chart/src/sql.ts+++ b/packages/core/src/modules/chart/src/sql.ts@@ -116,8 +116,6 @@ function assertCohortId(cohortId: string): string {return cohortId;}-// --- breakdown resolution ----------------------------------------------------/*** Drop breakdowns whose field name doesn't resolve to a known events column,* properties path, profile path, group path, or cohort — saved reports with@@ -153,8 +151,6 @@ export function withoutEmptyAllCohortsBreakdown();}-// --- clause rendering --------------------------------------------------------function clauses(parts: Parts): SqlFragment[] {return Object.values(parts);}@@ -202,8 +198,6 @@ function orderByClause(orderBy: Parts): SqlFragment {return sql`ORDER BY ${sql.join(keys, ', ')}`;}-// --- shared chart body -------------------------------------------------------function profileCteSelectField(field: string,profileKeys: string[],@@ -423,11 +417,9 @@ function countExpression(return sql`count(*) as count`;}-// Aliased `e`, not `subQuery`: breakdown SELECT expressions are always built-// with the events alias, so `e.properties` was out of scope in the outer-// query and any chart combining this segment with a property breakdown failed-// with UNKNOWN_IDENTIFIER. The subquery already does `SELECT * FROM events e`,-// so every `e.`-qualified reference outside it resolves unchanged.+// Aliased `e`, not `subQuery`: breakdown SELECT expressions are built with the+// events alias, so `e.properties` must be in scope in the outer query. The+// subquery does `SELECT * FROM events e`, so `e.`-qualified references resolve.function oneEventPerUserFrom(body: ChartBody): SqlFragment {return sql`(SELECT DISTINCT ON (profile_id) * from ${sql.id(CHART_TABLE.events)} e ${joinsClause(body.joins)} WHERE ${sql.join(clauses(body.where), ' AND ')}@@ -448,8 +440,6 @@ function dateRangeWhere(}}-// --- series chart ------------------------------------------------------------function intervalBucket(interval: IInterval,timezone: string@@ -538,8 +528,6 @@ export function chartSeriesQuery(input: ChartSeriesQueryInput): SqlFragment {return sql`${withClause(body.ctes)}SELECT * EXCEPT (_uc_state), ${totalCount} FROM (${selectClause(body.select)} FROM ${sql.id(CHART_TABLE.events)} e ${joinsClause(body.joins)} ${whereClause(body.where)} ${groupByClause(groupBy)}) ${orderByClause(orderBy)} ${fill}`;}-// --- aggregate chart ---------------------------------------------------------/** One row per breakdown combination over the whole range. */export function aggregateChartQuery(input: AggregateChartQueryInput@@ -573,13 +561,10 @@ export function aggregateChartQuery(return sql`${head} FROM ${sql.id(CHART_TABLE.events)} e ${joinsClause(body.joins)} ${whereClause(body.where)} ${groupByClause(groupBy)} ${orderByClause(orderBy)} ${limitClause}`;}-// --- router queries ----------------------------------------------------------/*** projectCard: three months of daily unique visitors + revenue, filled.* The timezone travels as `clickhouse_settings.session_timezone` (the caller- * passes it) rather than an inline `SETTINGS` clause — same setting, same- * query scope, and a bindable position.+ * passes it) rather than an inline `SETTINGS` clause.*/export function projectCardChartQuery(projectId: string): SqlFragment {return sql`SELECT@@ -647,8 +632,6 @@ export function groupPropertyValuesQuery(return sql`SELECT distinct ${expression} as values FROM ${sql.id(CHART_TABLE.groups)} FINAL WHERE project_id = ${sql.string(projectId)} AND deleted = 0 AND ${expression} != '' AND ${expression} IS NOT NULL ORDER BY created_at DESC LIMIT ${sql.uint64(PROPERTY_VALUES_LIMIT)}`;}-// --- getProfiles -------------------------------------------------------------export interface ChartBucketProfilesInput {projectId: string;/** Bucket start as a `YYYY-MM-DD HH:mm:ss` string. */diff --git a/packages/core/src/modules/chart/src/table-filter-where.test.ts b/packages/core/src/modules/chart/src/table-filter-where.test.tsindex 432c904bb..84369aa57 100644--- a/packages/core/src/modules/chart/src/table-filter-where.test.ts+++ b/packages/core/src/modules/chart/src/table-filter-where.test.ts@@ -1,16 +1,10 @@/**- * Unit tests for `buildFilterWhere`.- *- * No ClickHouse needed. The compiler returns bound `SqlFragment`s, so each- * clause is rendered through `toStatement` and its params substituted back- * in: what the assertions say is unchanged, a value that used to appear as- * an escaped literal now appears in `query_params`. The- * interesting part is the `profile.*` branch: the field name arrives from the- * caller (saved report, URL state, raw API call) and used to be concatenated- * into the SQL text as-is, so a name that was not a column produced whatever- * SQL the caller wrote. It now resolves against the known profiles columns and- * the filter is dropped when it does not, which is what the `group.*` and- * `session.*` branches already did.+ * Unit tests for `buildFilterWhere`. No ClickHouse needed: the compiler returns+ * bound `SqlFragment`s, so each clause is rendered through `toStatement` and its+ * params substituted back in. The interesting part is the `profile.*` branch:+ * the field name arrives from the caller (saved report, URL state, raw API+ * call), so it resolves against the known profiles columns and the filter is+ * dropped when it does not, as the `group.*` and `session.*` branches do.*/import { describe, expect, it } from 'bun:test';diff --git a/packages/core/src/modules/chart/src/table-filter-where.ts b/packages/core/src/modules/chart/src/table-filter-where.tsindex 45c8b022b..2d37bc6b4 100644--- a/packages/core/src/modules/chart/src/table-filter-where.ts+++ b/packages/core/src/modules/chart/src/table-filter-where.ts@@ -104,7 +104,7 @@ function integerLiteralDigits(value: number): bigint | null {* have had. Declaring `Float64` for every number would bind a *different constant** — visible wherever the comparand's type takes part in type resolution, and* in the `NO_COMMON_TYPE` message when a numeric filter lands on a String- * property (the proof's two error cases).+ * property.** `filter-cast.ts`'s twin needs none of this: there every value is wrapped in* `toFloat64OrNull(toString(…))` or a sibling cast, which erases the declared@@ -122,7 +122,7 @@ function numericLiteralType(value: number): string {}/**- * Bind one filter value the way `sqlstring.escape` rendered it: a string is a+ * Bind one filter value: a string is a* trimmed quoted literal, a number a numeric literal of the type ClickHouse* would have inferred for it, a boolean `true`/`false` and `null` the SQL* keyword NULL — which only `Nullable` reproduces, since a `String` paramdiff --git a/packages/core/src/modules/cohort/cohort.jobs.test.ts b/packages/core/src/modules/cohort/cohort.jobs.test.tsindex 7f53d2840..54a0fae8e 100644--- a/packages/core/src/modules/cohort/cohort.jobs.test.ts+++ b/packages/core/src/modules/cohort/cohort.jobs.test.ts@@ -1,5 +1,4 @@-// Job/cron-fragment wiring — no ClickHouse/Postgres touched. The service-// methods themselves are exercised in cohort.service.test.ts.+// Job/cron-fragment wiring; no ClickHouse/Postgres touched.import { expect, test } from 'bun:test';import { testCoreConfig } from '../../../test/config-fixture';@@ -131,9 +130,7 @@ test('cohortRefresh fans out one enqueueCompute per non-static cohort', async ()expect(calls).toEqual(['c1', 'c2']);});-// Byte-identity with the id/cadence schedulers.test.ts's golden snapshot-// pins. The cadence now lives on the job itself, so this reads it straight-// off the registry.+// Reads the cadence straight off the registry.test('the cohort cron fragment keeps its scheduler id and cadence', () => {expect(queues.cron.jobs.cohortRefresh.cron).toEqual({pattern: '*/30 * * * *',diff --git a/packages/core/src/modules/cohort/cohort.jobs.ts b/packages/core/src/modules/cohort/cohort.jobs.tsindex 4782cfad7..4d63e6e5a 100644--- a/packages/core/src/modules/cohort/cohort.jobs.ts+++ b/packages/core/src/modules/cohort/cohort.jobs.ts@@ -1,7 +1,5 @@-// `cohortCompute` is this module's own queue — registry key `cohortCompute`-// EXACTLY (no env rename, so COHORTCOMPUTE_CONCURRENCY keeps working).-// `cohortRefresh` is a cron fragment, spread into the ONE `cron` queue by-// jobs.registry.ts.+// `cohortCompute` is this module's own queue; `cohortRefresh` is a cron fragment+// spread into the shared `cron` queue by jobs.registry.ts.import { z } from 'zod';import { defineJob } from '../../jobs/define';diff --git a/packages/core/src/modules/cohort/cohort.rpc.test.ts b/packages/core/src/modules/cohort/cohort.rpc.test.tsindex 2123f7201..7b134c962 100644--- a/packages/core/src/modules/cohort/cohort.rpc.test.ts+++ b/packages/core/src/modules/cohort/cohort.rpc.test.ts@@ -1,8 +1,4 @@-// Only the "is anyone logged in" boundary is exercised here — no database.-// The access-check + business logic ride on @openpanel/db (lazy-loaded, see-// cohort.service.ts's header) and ctx.services.cohort; wiring this router-// end-to-end against a real Postgres is P6's (protectedProcedure) job, not-// this one's — see cohort.rpc.ts's header.+// Only the "is anyone logged in" boundary is exercised here, with no database.import { expect, mock, test } from 'bun:test';import { stubHttpCtx } from '../../../test/rpc-fixtures';@@ -19,7 +15,6 @@ const COOKIE_OPTIONS: CookieOptions = {path: '/',};-// EMPTY_SESSION's shape (packages/db/src/services/auth-session.service.ts) —// `TrpcContext.session` is never literally `null`, only its `userId` is.const EMPTY_SESSION = { session: null, user: null, userId: null };diff --git a/packages/core/src/modules/cohort/cohort.rpc.ts b/packages/core/src/modules/cohort/cohort.rpc.tsindex 3a40b272c..b54f468dc 100644--- a/packages/core/src/modules/cohort/cohort.rpc.ts+++ b/packages/core/src/modules/cohort/cohort.rpc.ts@@ -1,14 +1,6 @@-// `protectedProcedure` runs `enforceUserIsAuthed` + `enforceAccess` BEFORE the-// input parser. The explicit checks in the handlers below stay: `enforceAccess`-// only sees a TOP-LEVEL `projectId` / `organizationId`, so anything resolved-// from another id needs its own.-//-// The per-project access ladder itself is bound once, in auth.service.ts; every-// procedure here reaches it through `ctx.services.auth`.-//-// CRUD (list/get/create/update/delete) reads/writes Prisma's `cohort` table-// directly — cohort.service.ts owns only the compute-heavy and-// ClickHouse-touching operations.+// `protectedProcedure` runs `enforceAccess` BEFORE the input parser, and it only+// sees a TOP-LEVEL `projectId` / `organizationId`, so handlers that resolve a+// project from another id keep their own explicit check.import { z } from 'zod';import { createTRPCRouter, protectedProcedure } from '../../rpc/base';@@ -37,9 +29,8 @@ const EXPORT_PROFILES_MAX_LIMIT = 10_000;const EXPORT_PROFILES_DEFAULT_LIMIT = 10_000;/**- * `mostEvents` and `popularRoutes` read the project's whole history before this- * window existed. The default is the one `profile.powerUsers` got for the same- * cause; the cohort page passes no range, so this is what it shows.+ * Bounds the event scan behind `mostEvents` and `popularRoutes`. The cohort page+ * passes no range, so this is what it shows.*/const COHORT_ACTIVITY_DEFAULT_RANGE = '3m';diff --git a/packages/core/src/modules/cohort/cohort.service.test.ts b/packages/core/src/modules/cohort/cohort.service.test.tsindex c450e676b..ebffdc22a 100644--- a/packages/core/src/modules/cohort/cohort.service.test.ts+++ b/packages/core/src/modules/cohort/cohort.service.test.ts@@ -1,11 +1,5 @@-// Every function under test takes `ServiceDeps`, so `deps.db` and `deps.ch`-// ARE the fakes below. The one module still mocked is core's own-// `ch-query`, which is what the subject calls.-//-// The pure SQL-shape builders (buildEventCriteriaQuery,-// buildPropertyBasedCohortQuery, deriveCohortQuerySettings) have their own-// file, src/cohort-sql.test.ts, and need none of this — they take no ClickHouse-// connection.+// Every function under test takes `ServiceDeps`, so `deps.db` and `deps.ch` are+// the fakes below. The one module still mocked is core's own `ch-query`.import { afterAll, beforeAll, expect, mock, test } from 'bun:test';import { testCoreConfig } from '../../../test/config-fixture';@@ -56,12 +50,9 @@ const chQuery = mock(async () => [] as unknown[]);const chInsert = mock(async (_args: { table: string }) => undefined);const chCommand = mock(async (_args: { query: string }) => undefined);-// The subject reaches ClickHouse through core's own `chQuery(deps, ...)`-// (ch-query.ts) — mocking that one module keeps this file off-// @openpanel/db's live-bound client singleton entirely. `mock.module` still-// has no per-file scope under bare `bun test` (AGENTS.md), so snapshot the-// real module into a plain object FIRST and restore it in afterAll —-// restoring via the live import binding would just re-apply the mock.+// `mock.module` has no per-file scope under bare `bun test`, so snapshot the+// real module into a plain object first and restore it in afterAll; restoring+// via the live import binding would re-apply the mock.const realChQuery = { ...(await import('../../ch-query')) };mock.module('../../ch-query', () => ({...realChQuery,@@ -215,8 +206,7 @@ test('createCohortService() delegates updateMembership and listRefreshableCohortseedCohort({ id: 'c2', isStatic: true });const service = subject.createCohortService(deps, testServices());- // The container's members are closures over `deps`, not the bare functions,- // so identity is no longer the observable — delegation is.+ // The container's members are closures over `deps`; delegation is observable.expect((await service.listRefreshableCohortIds()).sort()).toEqual(['c1']);chCommand.mockClear();diff --git a/packages/core/src/modules/cohort/cohort.service.ts b/packages/core/src/modules/cohort/cohort.service.tsindex ff1b13903..7e42655e4 100644--- a/packages/core/src/modules/cohort/cohort.service.ts+++ b/packages/core/src/modules/cohort/cohort.service.ts@@ -1,10 +1,8 @@// Cohort definitions, membership computation and the compute enqueue.//-// Table names are a local literal map (`TABLE`, below) rather than-// @openpanel/db's `TABLE_NAMES`: that lives in the same module as `ch` /-// `chQuery` (clickhouse/client.ts), which constructs a real pino logger at-// import time, and this module's SQL builders are pure sync functions the-// shape tests call with no ClickHouse connection at all.+// Table names are a local literal map (`TABLE`) rather than @openpanel/db's+// `TABLE_NAMES`: importing that builds a pino logger, and the SQL builders here+// are pure functions the shape tests call without a ClickHouse connection.import type { ClickHouseSettings } from '@clickhouse/client';import { type SqlFragment, sql } from '@openpanel/db/src/clickhouse/sql';@@ -25,8 +23,7 @@ import type {Timeframe,} from './cohort.constants';-// Physical ClickHouse table names this module reads/writes. Literal, not-// imported from @openpanel/db's TABLE_NAMES — see the header comment.+// Physical ClickHouse table names this module reads/writes.const TABLE = {profiles: 'profiles',events: 'events',@@ -36,15 +33,10 @@ const TABLE = {eventPropertyProfileSummaryMv: 'event_property_profile_summary_mv',} as const;-// Max members materialized into cohort_members per compute. Cohorts larger-// than this are silently truncated to an arbitrary subset, so deployments-// with bigger cohorts need to raise it — env-tunable to avoid an image-// rebuild for what is really a sizing knob.-//-// Strictly a positive safe integer, which is what the config loader's-// `optionalPositiveInt` guarantees: '5000junk', '-1' and '0' all arrive as-// undefined, and 0 is falsy at the `limit ? LIMIT ... : ''` call sites, which-// would silently remove the cap entirely.+// Max members materialized into cohort_members per compute. Larger cohorts are+// silently truncated to an arbitrary subset, so raise it for big deployments.+// Must be a positive safe integer: 0 is falsy at the `limit ? LIMIT ... : ''`+// call sites and would silently remove the cap.const DEFAULT_COHORT_MATERIALIZE_LIMIT = 10_000;export function cohortMaterializeLimit(config: CoreConfig): number {@@ -53,24 +45,17 @@ export function cohortMaterializeLimit(config: CoreConfig): number {);}-// Property cohorts aggregate every profile row for the project, so they are the-// one cohort query that can outgrow the server's memory headroom. Two opt-in-// knobs bound them; with NEITHER set, no per-query settings are applied and the-// server's own defaults govern — upstream behavior is unchanged.+// Property cohorts aggregate every profile row for the project, so they can+// outgrow the server's memory. Two opt-in knobs bound them; with neither set the+// server defaults govern.//// COHORT_QUERY_MEMORY_LIMIT_BYTES hard cap for these queries// COHORT_QUERY_SPILL_BYTES GROUP BY spills to disk past this//-// A GROUP BY only starts spilling once it crosses the threshold, so the spill-// threshold must sit BELOW the memory limit — inverted, the query is killed-// before it ever writes to disk (ClickHouse Cloud ships exactly that inversion-// by default, which is how these queries OOM'd instead of spilling). When only-// the limit is set — or the pair is inverted — the threshold derives as-// limit/3. Spilling early costs little: the volume spilled is set by the data,-// not the threshold.-//-// A standalone function of its two parsed inputs (not a module-level read), so-// a test can exercise every branch by calling it directly.+// A GROUP BY only spills once it crosses the threshold, so the threshold must+// sit BELOW the memory limit: inverted, the query is killed before it writes to+// disk (ClickHouse Cloud ships that inversion by default). When only the limit+// is set, or the pair is inverted, the threshold derives as limit/3.export function deriveCohortQuerySettings({memoryLimitBytes,spillBytes: spillBytesParsed,@@ -233,10 +218,6 @@ export function buildEventCriteriaQuery(const timeConstraint = buildTimeConstraint(timeframe, sql.id('event_date'));const project = sql.string(projectId);const eventName = sql.string(name);- // No filter name can start with both 'properties.' and- // 'profile.properties.' (they diverge at the 4th character), so the- // 'profile.properties.' exclusion this used to carry was always true here- // and is dropped rather than kept as dead weight.const hasEventPropertyFilters = filters.some((f) =>f.name.startsWith('properties.'));@@ -327,8 +308,7 @@ export function buildEventCriteriaQuery(// The bare profiles columns a cohort filter may reach. Every name is// qualified by the time it gets here, and anything outside the list is a-// filter naming a column that is not the caller's to read (main #512,-// GHSA-gvwr-5684-wjqc).+// filter naming a column that is not the caller's to read (GHSA-gvwr-5684-wjqc).const COHORT_PROFILE_COLUMNS = PROFILE_SELECT_COLUMNS.map((column) => `profiles.${column}`);@@ -357,16 +337,12 @@ function buildProfileCohortHavingClause(): SqlFragment | null {const { properties, operator } = definition.criteria;- // Every argMax below must order the candidate rows IDENTICALLY, or- // equal-version rows with conflicting fields could each win a different- // column — matching an AND cohort against a synthetic combination no- // stored row contains. One shared key — the version column, tie-broken by- // a hash of every referenced column — makes all aggregates pick their- // value from the same winning row, deterministically. The hash (rather- // than the raw value tuple) keeps the per-group comparison state at a- // fixed 8 bytes, cheaper than comparing the raw tuple. A wrong tie-break- // would need a version tie AND a 64-bit collision between different rows —- // and even then every aggregate in the query still elects the same row.+ // Every argMax below must order candidate rows IDENTICALLY, or equal-version+ // rows with conflicting fields could each win a different column and match an+ // AND cohort against a combination no stored row contains. One shared key (the+ // version column, tie-broken by a hash of every referenced column) makes all+ // aggregates pick the same winning row. The hash keeps comparison state at a+ // fixed 8 bytes.const referencedColumns = Array.from(new Set(properties.map((f) => normalizeProfileColumn(f.name)))).map(profileColumnAccess);@@ -449,12 +425,8 @@ export async function computeEventBasedCohort(const combinedQuery = buildEventBasedCohortQuery(projectId, definition);// The LIMIT has to wrap the combination, not trail it: appended to an- // INTERSECT / UNION chain, ClickHouse applies it to the last SELECT alone.- // That was survivable while every operand was a narrow event-derived set;- // a "never did X" operand is most of the project's profiles, so limiting it- // before the INTERSECT would cut the cohort down to an arbitrary slice —- // and at the preview's limit of 10, almost always to nothing. The count- // query below already wraps for the same reason.+ // INTERSECT / UNION chain, ClickHouse applies it to the last SELECT alone, and+ // a "never did X" operand is most of the project's profiles.const finalQuery = limit? sql`SELECT profile_id FROM (${combinedQuery}) LIMIT ${sql.uint64(limit)}`: combinedQuery;@@ -496,16 +468,10 @@ function getProfileFiltersWhereClause(let columnAccess = profileColumnAccess(normalizeProfileColumn(name));if (latestPerProfileKey) {- // Resolve the profile's newest row inside a GROUP BY instead of- // reading through FINAL. The key is shared by every wrapped column- // (see buildProfileCohortHavingClause), so all aggregates read the- // SAME winning row: last_seen_at is the table's version column but is- // not unique, and per-column tie-breaking would let equal-version- // rows with conflicting fields produce a synthetic combination no- // stored row contains. FINAL breaks the same ties by part order,- // which is not derivable from the data and can shift under a- // background merge — the shared value-tuple tie-break is- // deterministic instead.+ // Resolve the newest row inside a GROUP BY instead of FINAL. The key is+ // shared by every wrapped column (see buildProfileCohortHavingClause), so all+ // aggregates read the SAME winning row; FINAL breaks version ties by part+ // order, which can shift under a background merge.columnAccess = sql`argMax(${columnAccess}, ${latestPerProfileKey})`;}@@ -953,8 +919,7 @@ export async function listCohortMemberProfiles(}/**- * The window `mostEvents` / `popularRoutes` read. Both had no date filter, so- * they scanned the project's whole event history however small the cohort.+ * The window `mostEvents` / `popularRoutes` read, bounding the event scan.* `YYYY-MM-DD HH:mm:ss`, as `getChartStartEndDate` returns it.*/export interface CohortActivityWindow {@@ -964,7 +929,6 @@ export interface CohortActivityWindow {const DEFAULT_COHORT_ACTIVITY_LIMIT = 10;-/** Spelled as `overview/src/pages.sql.ts` and `profile/src/sql.ts` spell it. */function createdAtWithin({startDate,endDate,@@ -989,9 +953,6 @@ export function cohortMemberEventsQuery(window: CohortActivityWindow,limit: number): SqlFragment {- // This plain `IN (subquery)` on `cohort_members` is left as-is — changing- // how the subquery binds its values would not change anything about- // distribution semantics.return sql`SELECT name, count() AS countFROM ${sql.id(TABLE.events)}diff --git a/packages/core/src/modules/cohort/src/cohort-activity-sql.test.ts b/packages/core/src/modules/cohort/src/cohort-activity-sql.test.tsindex c02bef5cb..304a405ef 100644--- a/packages/core/src/modules/cohort/src/cohort-activity-sql.test.ts+++ b/packages/core/src/modules/cohort/src/cohort-activity-sql.test.ts@@ -1,7 +1,5 @@// SQL-shape tests for the cohort detail page's `mostEvents` / `popularRoutes`-// statements. Both are bounded to a window: without it they read the-// project's whole event history however small the cohort was. Pure builders —-// no ClickHouse.+// statements, which are bounded to a window.import { describe, expect, test } from 'bun:test';import {diff --git a/packages/core/src/modules/cohort/src/cohort-event-sql.test.ts b/packages/core/src/modules/cohort/src/cohort-event-sql.test.tsindex 8c5dfa607..556782000 100644--- a/packages/core/src/modules/cohort/src/cohort-event-sql.test.ts+++ b/packages/core/src/modules/cohort/src/cohort-event-sql.test.ts@@ -1,5 +1,4 @@-// SQL-shape tests for event-based cohort criteria. The "never did this event"-// assertions are unchanged in what they claim.+// SQL-shape tests for event-based cohort criteria.import { describe, expect, test } from 'bun:test';import type {diff --git a/packages/core/src/modules/cohort/src/cohort-sql.test.ts b/packages/core/src/modules/cohort/src/cohort-sql.test.tsindex 4a68b9fc7..5330df1e3 100644--- a/packages/core/src/modules/cohort/src/cohort-sql.test.ts+++ b/packages/core/src/modules/cohort/src/cohort-sql.test.ts@@ -1,25 +1,11 @@-// SQL-shape tests for property-based cohort queries. Minus its itCH-// ClickHouse-reachability EXPLAIN check: `mock.module` substitutions apply-// process-wide, not per file, so a sibling file's-// `@openpanel/db/src/clickhouse/client` mock (cohort.service.test.ts) can-// silently replace this file's "real" import too, making a reachability check-// meaningless. No module in core's suite ports that pattern for the same-// reason; the generated SQL was verified against local ClickHouse by hand-// instead.+// SQL-shape tests for property-based cohort queries. There is deliberately no+// ClickHouse EXPLAIN check: `mock.module` substitutions apply process-wide, so a+// sibling file's client mock can silently replace this file's "real" import.//-// String assertions need no mocking at all: cohort.service.ts's db/ch access is-// lazy, and buildPropertyBasedCohortQuery / deriveCohortQuerySettings touch-// neither — see cohort.service.ts's header.-//-// The builders return `SqlFragment`s, so every assertion below runs against-// the rendered statement plus its bound params rather than a finished-// string. The property-key quoting test below asserts a binding, not-// escaped text, because the key never appears in the SQL text at all.-//-// bun:test shares one module registry per file even under `--isolate`, so-// re-importing under different env vars would not re-evaluate a-// module-level const. `deriveCohortQuerySettings` exists so the different-// settings combinations below are exercised as direct calls instead.+// The builders return `SqlFragment`s, so assertions run against the rendered+// statement plus bound params. `deriveCohortQuerySettings` is called directly+// because bun:test shares one module registry per file, so env-var variants+// cannot be re-imported.import { expect, test } from 'bun:test';import {@@ -266,6 +252,3 @@ test('deriveCohortQuerySettings never derives a zero spill threshold (0 would DI).toBeGreaterThanOrEqual(1);}});--// A malformed COHORT_QUERY_* value is rejected by the config loader now, not-// here — `apps/api/src/config/env.test.ts` owns that case.diff --git a/packages/core/src/modules/ingest/ingest.constants.ts b/packages/core/src/modules/ingest/ingest.constants.tsindex f852c34ce..e27c48731 100644--- a/packages/core/src/modules/ingest/ingest.constants.ts+++ b/packages/core/src/modules/ingest/ingest.constants.ts@@ -1,7 +1,3 @@-// The ingestion wire contract. All three enforce the same thing (what a-// caller may name an event and what a track body may contain), so they are-// one file here.-import { z } from 'zod';/** Event names the ingestion pipeline mints itself; a caller may not send them. */@@ -130,16 +126,10 @@ export function isBlockedEventName(name: string): boolean {return false;}-// ----- Hand-written types (source of truth) ------//-// These interfaces are duplicated in code that ships in our SDK type-// declarations. We hand-write them (instead of using `z.infer<…>`) so:-// 1. Generated `.d.ts` files for the SDKs are readable plain TypeScript-// with no `import type { ITrackPayload }` line pointing at a workspace-// package (none of them are published).-// 2. The interfaces stay clean — no zod internals leaking through.-// Each schema below has `satisfies z.ZodType<…>` attached so a drift-// between the interface and the schema fails to compile.+// Hand-written (not `z.infer`) because these interfaces ship in the SDK type+// declarations, which must read as plain TypeScript with no import of an+// unpublished workspace package. Each schema `satisfies` its interface so+// drift fails to compile.export type IProfileId = string | number;@@ -210,8 +200,6 @@ export type ITrackHandlerPayload =| { type: 'group'; payload: IGroupPayload }| { type: 'assign_group'; payload: IAssignGroupPayload };-// ----- Schemas (each `satisfies` its hand-written interface) ------export const zProfileId = z.union([z.string().min(1),z.number(),diff --git a/packages/core/src/modules/ingest/ingest.routes.ts b/packages/core/src/modules/ingest/ingest.routes.tsindex d01f17ad5..af2b1eacd 100644--- a/packages/core/src/modules/ingest/ingest.routes.ts+++ b/packages/core/src/modules/ingest/ingest.routes.ts@@ -1,20 +1,14 @@// THE HOOK ORDER IS THE CONTRACT: duplicate -> clientAuth -> isBot ->-// subscription. It holds by lifecycle phase rather than by registration order-// — verified against Elysia 1.4.30, where a route declaring all three runs-// `derive -> transform -> macro resolve -> beforeHandle`. So the duplicate-// check is a `transform`, `clientAuth` is the macro, and the bot check is a-// `beforeHandle`, which is what lets it read `client.secretVerified` after-// the macro resolves the principal.+// subscription. It holds by lifecycle phase rather than registration order+// (verified against Elysia 1.4.30: `derive -> transform -> macro resolve ->+// beforeHandle`). So the duplicate check is a `transform`, `clientAuth` is the+// macro, and the bot check is a `beforeHandle`, which lets it read+// `client.secretVerified` after the macro resolves the principal.//-// Both routes carry the duplicate check because the guard wrapping them below-// is scoped to both, so `GET /track/device-id` goes through it too.-//-// `POST /event` — a legacy compat route kept because production still posts-// to it — sits beside `/track` with the same hook chain, plus a per-client-// usage counter recorded after authentication (so a label can only ever be a-// client id that exists) and before the hooks that can short-circuit (so a-// client whose events are dropped as bot/wind-down traffic still counts as-// one that would break if `/event` disappeared).+// `POST /event`, a legacy route production still posts to, has the same hook+// chain plus a per-client usage counter recorded after authentication (so a+// label is always a real client id) and before the hooks that can+// short-circuit (so a client whose events are dropped still counts).import type { AppDeps, HttpCtx } from '../../context';import { defineRoutes } from '../../http/define';@@ -32,9 +26,6 @@ import {} from './ingest.service';import { validateIngestRequest } from './src/client-auth';import { recordLegacyEventRequest } from './src/ingest.metrics';-// The Kafka producer is a sibling now, not an `AppDeps` field: kafka.ts moved-// into this module, so there is nothing left for main.ts to inject. It still-// constructs no client at import time.import { produceIncomingEvent } from './src/kafka';const TAGS = ['Track'];@@ -165,8 +156,6 @@ function windDownGuard(deps: AppDeps) {}export const ingestRoutes = defineRoutes((app, deps: AppDeps) => {- // One closure over `deps` for all three routes: this is route-definition- // work, and three identical closures read as if it were per-request.const blockWhenWoundDown = windDownGuard(deps);return app.guard(diff --git a/packages/core/src/modules/ingest/ingest.service.test.ts b/packages/core/src/modules/ingest/ingest.service.test.tsindex 108ecf177..9ed251dd5 100644--- a/packages/core/src/modules/ingest/ingest.service.test.ts+++ b/packages/core/src/modules/ingest/ingest.service.test.ts@@ -94,7 +94,7 @@ const track = (properties?: Record<string, unknown>): ITrackHandlerPayload =>// Back-dating is documented and deliberate, so the guard here is a FLOOR, not// a window: historical imports must keep working, and only values that cannot-// be a real event time fall back to the server clock (ISSUES.md H6).+// be a real event time fall back to the server clock.describe('getTimestamp', () => {const ARRIVED_AT = Date.UTC(2026, 8, 27, 12, 0, 0);const withTimestamp = (value: unknown) =>@@ -112,8 +112,8 @@ describe('getTimestamp', () => {});it('falls back to the server clock for a value that lands in 1970', () => {- // `new Date(12345)` is 1970-01-01T00:00:12.345Z. One such row reached- // ClickHouse during the QA run and every WITH FILL series then began there.+ // `new Date(12345)` is 1970-01-01T00:00:12.345Z. One such row once reached+ // ClickHouse and every WITH FILL series then began there.expect(withTimestamp(12_345).timestamp).toBe(ARRIVED_AT);expect(withTimestamp(0).timestamp).toBe(ARRIVED_AT);});diff --git a/packages/core/src/modules/ingest/ingest.service.ts b/packages/core/src/modules/ingest/ingest.service.tsindex 78cd5d548..f5d99cc38 100644--- a/packages/core/src/modules/ingest/ingest.service.ts+++ b/packages/core/src/modules/ingest/ingest.service.ts@@ -1,10 +1,5 @@-// The ingestion pipeline for POST /track and POST /event.-//// Nothing here throws for a caller error: each transport owns its own status// codes and bodies, so the pipeline returns a `TrackOutcome` instead.-//-// The Kafka producer is injected as an argument, not read from a module-// singleton, so a test can assert on what was produced without a broker.import { generateId } from '@openpanel/shared';import { parseUserAgent } from '@openpanel/shared/server';@@ -80,8 +75,6 @@ export type IngestBuffers = Pick<Buffers, 'session' | 'replay' | 'group'>;export interface IngestTransport {buffers: IngestBuffers;produceIncomingEvent: IncomingEventProducer;- /** The profile writes below reach ClickHouse and the profile buffer- * through the scope, not a module singleton. */deps: ServiceDeps;}@@ -548,7 +541,7 @@ export async function ingestTrack(** It gates on `windDownStep`, not on `subscriptionState`. Every expired trial* is already in `trial_expired`, so gating on the state would block thousands- * of orgs the moment this ships, skipping the four warning emails entirely.+ * of orgs at once, skipping the four warning emails entirely.** It answers 202, not 402 or 403 (the caller does; this returns the verdict).* The SDKs treat only 401 and 2xx as terminal, so a "correct" status codediff --git a/packages/core/src/modules/ingest/legacy-event.test.ts b/packages/core/src/modules/ingest/legacy-event.test.tsindex 7a0ed0179..ff1358e28 100644--- a/packages/core/src/modules/ingest/legacy-event.test.ts+++ b/packages/core/src/modules/ingest/legacy-event.test.ts@@ -2,9 +2,6 @@* Tests for the legacy `POST /event` compat route: the route is KEPT,* produces to Kafka exactly like /track, and is measured per client so a* future removal decision has data.- *- * The `mixan-*` header fallback moved with the validator itself — see- * `src/client-auth.test.ts`.*/import {diff --git a/packages/core/src/modules/ingest/src/client-auth.test.ts b/packages/core/src/modules/ingest/src/client-auth.test.tsindex dc6fe5adb..1f72a3920 100644--- a/packages/core/src/modules/ingest/src/client-auth.test.ts+++ b/packages/core/src/modules/ingest/src/client-auth.test.ts@@ -5,9 +5,9 @@* usage metric confirms `/event` callers have moved off it.** `secretVerified` is the side channel the bot check reads. The `secret- * verification` block guards main #481: `secretVerified` and revenue- * ingestion follow whether the secret verified against the stored hash, not- * whether a secret string was on the request.+ * verification` block guards that it and revenue ingestion follow whether the+ * secret verified against the stored hash, not whether a secret string was on+ * the request.*/import {diff --git a/packages/core/src/modules/ingest/src/client-auth.ts b/packages/core/src/modules/ingest/src/client-auth.tsindex cefb88ff2..a6d31af3a 100644--- a/packages/core/src/modules/ingest/src/client-auth.ts+++ b/packages/core/src/modules/ingest/src/client-auth.ts@@ -1,7 +1,3 @@-// The ingest tier of caller authentication. Core's `clientAuth` macro reaches-// it through http/client-auth.ts, which a route hands this function to as-// `clientAuth: { ingest: validateIngestRequest }`.-//// None of the behaviour below is decoration: the `ignoreCorsAndSecret`// short-circuit, the unanchored wildcard origin regex, CORS-before-secret// ordering, the ip and profile_id project filters, the `__revenue` gate, thediff --git a/packages/core/src/modules/ingest/src/consumer.test.ts b/packages/core/src/modules/ingest/src/consumer.test.tsindex a94f420ac..aab8be1fa 100644--- a/packages/core/src/modules/ingest/src/consumer.test.ts+++ b/packages/core/src/modules/ingest/src/consumer.test.ts@@ -371,15 +371,9 @@ describe('unparseable messages', () => {});/**- * Deleted the offset watermark, whose only outputs were the `reprocessed`- * counter and a warn line — live duplicates occurred but none of those lines- * fired, because `resetWatermarks` fired on GROUP_JOIN and a failed- * durability flush never advanced it.- *- * These are the regression tests for the half of that claim that matters: the- * functional statement in the resolve loop was always `resolveOffset(m.offset)`- * and never read the watermark, so a redelivery must resolve and commit exactly- * what it did before.+ * A redelivery must resolve and commit exactly what the resolve loop marks:+ * no watermark suppresses or alters it, so duplicates are never silently+ * skipped.*/describe('a redelivery after the watermark was deleted', () => {test('resolves every offset again, in ascending order', async () => {@@ -404,9 +398,9 @@ describe('a redelivery after the watermark was deleted', () => {await handler.eachBatch(makeBatch(messages).payload);await handler.eachBatch(makeBatch(messages).payload);- // At-least-once, unchanged: the consumer does not suppress a redelivery,- // and nothing in this file now decides whether one is a duplicate — the- // marker on the event id does (incoming-event-handler.ts).+ // At-least-once: the consumer does not suppress a redelivery. Whether one+ // is a duplicate is decided by the marker on the event id+ // (incoming-event-handler.ts).expect(handleEvent).toHaveBeenCalledTimes(2);});diff --git a/packages/core/src/modules/ingest/src/consumer.ts b/packages/core/src/modules/ingest/src/consumer.tsindex 2988c74b7..295dfe1cc 100644--- a/packages/core/src/modules/ingest/src/consumer.ts+++ b/packages/core/src/modules/ingest/src/consumer.ts@@ -2,10 +2,6 @@// offset walk, at-least-once, bounded in-consumer retry, and a dead-letter on// exhaustion. The dead-letter destination is a capped Redis list, and the// event is dropped whether or not that write lands.-//-// The kafkajs client, the topic/consumer-group names, the DLQ producer and the-// retry bounds are all injected — this file never spells any of them, which is-// what keeps them testable without a broker.import type {Consumer,@@ -57,11 +53,9 @@ export type DeadLetterReason = 'parse_error' | 'handler_error';* What the consumer hands to the dead-letter sink. The value stays the* producer's original bytes, so the record is a faithful copy of what arrived.*- * Two implementations satisfy this, and the seam exists so the choice is one- * line in `apps/api`'s wiring: `@openpanel/redis`'s `createDeadLetterRecorder`- * (the capped list) and `./kafka.ts`'s `produceDeadLetterEvent` (the DLQ- * topic, kept as an explicit "for now" choice; a DLQ message produced from- * these bytes is replayable onto the events topic unchanged).+ * Implemented by `@openpanel/redis`'s `createDeadLetterRecorder` (the capped+ * list) and `./kafka.ts`'s `produceDeadLetterEvent` (the DLQ topic; a message+ * produced from these bytes is replayable onto the events topic unchanged).*/export interface DeadLetterMessage {key: Buffer | null;@@ -112,11 +106,10 @@ export interface EventsBatchHandlerDeps {* not at `consumer.stop`.** The gate MUST reject when the events THIS batch buffered are not in- * Redis: a graceful restart previously lost events whose offsets were- * committed even though the events existed only in the event buffer's- * in-process array. The window is what makes "this batch's events are- * durable" answerable at all, now that a failed write drops the events- * instead of keeping them for the broker to redeliver.+ * Redis: events that exist only in the event buffer's in-process array+ * would otherwise be lost with their committed offsets. The window is what+ * makes "this batch's events are durable" answerable, since a failed write+ * drops events instead of keeping them for the broker to redeliver.*/openDurabilityWindow: () => () => Promise<void>;logger: ConsumerLogger;@@ -169,10 +162,9 @@ export function createEventsBatchHandler(** There is deliberately no retry, no backoff and no unresolved offset here.* The dead-letter destination is Redis, and Redis being unavailable is- * exactly when handlers fail — so a failed write that held the offset back- * would rebuild the redelivery loop this replaced: once something else owns- * the outcome, a safety net underneath it is the bug.- *+ * exactly when handlers fail, so a failed write that held the offset back+ * would recreate the redelivery loop: once something else owns the outcome,+ * a safety net underneath it is the bug.* The two counters are what carry the volume, since a capped list makes* 50,000 drops look like 12: `deadLettered` = recorded and dropped,* `deadLetterFailed` = dropped WITHOUT being recorded. Neither means "will be@@ -476,9 +468,8 @@ export async function startKafkaEventsConsumer(consumer.on(consumer.events.HEARTBEAT, deps.batch.onActivity);- // ---- Lifecycle / rebalance ("re-election") visibility ----- // Without this logging, a rebalance storm (a common source of- // at-least-once duplicates) would be invisible.+ // Lifecycle / rebalance logging: a rebalance storm (a common source of+ // at-least-once duplicates) would otherwise be invisible.consumer.on(consumer.events.GROUP_JOIN, ({ payload }) => {// A new assignment means partitions may have moved between members, so a// consecutive-failure count carried over from the old one would pick thediff --git a/packages/core/src/modules/ingest/src/device-id.test.ts b/packages/core/src/modules/ingest/src/device-id.test.tsindex e5b3c1cdc..eca518c82 100644--- a/packages/core/src/modules/ingest/src/device-id.test.ts+++ b/packages/core/src/modules/ingest/src/device-id.test.ts@@ -18,7 +18,7 @@ import {const NOW = new Date('2026-06-08T12:00:00.000Z').getTime();const MINUTE = 60 * 1000;const SALTS = { current: 'salt-current', previous: 'salt-previous' };-/** SESSION_TIMEOUT_MS's default: the caller resolves it from config now. */+/** SESSION_TIMEOUT_MS's default; the caller resolves it from config. */const SESSION_TIMEOUT_MS = 30 * 60 * 1000;/** The caller's request logger. Inputs here are an IP and a salt, so a@@ -136,12 +136,11 @@ describe('getDeviceId — session resolution', () => {});/**- * Made the cache client fail fast, so this read now rejects in microseconds- * instead of after ioredis's 20-attempt reconnect cycle. The degradation- * itself must be untouched: the api and the worker only agree on a session- * because they derive the SAME id from project/device/time bucket, so a- * faster failure that changed the id would split every session an outage- * touched.+ * The cache client fails fast, so this read rejects in microseconds instead+ * of after ioredis's 20-attempt reconnect cycle. The degradation must not+ * change the id: the api and the worker only agree on a session because they+ * derive the SAME id from project/device/time bucket, so a different id on+ * failure would split every session an outage touched.*/it('mints the SAME deterministic id whether the session read misses or rejects', async () => {const { buffer: emptyStore } = stubBuffer(null);diff --git a/packages/core/src/modules/ingest/src/device-id.ts b/packages/core/src/modules/ingest/src/device-id.tsindex 26672459c..795ca9325 100644--- a/packages/core/src/modules/ingest/src/device-id.ts+++ b/packages/core/src/modules/ingest/src/device-id.ts@@ -100,7 +100,7 @@ export async function getDeviceId({* Returns true when an existing session is recent enough that the incoming* event should EXTEND it rather than start a new session.*- * Critical: blobs no longer have a Redis TTL (so the reaper can always find+ * Critical: blobs have no Redis TTL (so the reaper can always find* them), which means an existing session blob may linger past its idle* window. We must NOT blindly reuse `existing.id` — if we did, the worker's* boundary detection would open a "new" session with the same id as thediff --git a/packages/core/src/modules/ingest/src/incoming-event-handler.test.ts b/packages/core/src/modules/ingest/src/incoming-event-handler.test.tsindex 4aa8e4573..666f75bb1 100644--- a/packages/core/src/modules/ingest/src/incoming-event-handler.test.ts+++ b/packages/core/src/modules/ingest/src/incoming-event-handler.test.ts@@ -467,9 +467,8 @@ describe('incomingEvent', () => {});/**- * The marker replaces the offset watermark, which cannot see every- * duplicate. MARK MEANS COUNT AND LOG: every test below asserts the event- * was still INSERTED.+ * MARK MEANS COUNT AND LOG: the marker does not suppress, so every test below+ * asserts the event was still INSERTED.*/describe('duplicate marker', () => {const TTL_MS = 120_000;diff --git a/packages/core/src/modules/ingest/src/incoming-event-handler.ts b/packages/core/src/modules/ingest/src/incoming-event-handler.tsindex 7ad4066ba..bc0cce2cc 100644--- a/packages/core/src/modules/ingest/src/incoming-event-handler.ts+++ b/packages/core/src/modules/ingest/src/incoming-event-handler.ts@@ -1,14 +1,10 @@-// The Kafka consumer's per-message handler — the step that turns one topic-// payload into session lifecycle decisions and ClickHouse rows.+// The Kafka consumer's per-message handler: turns one topic payload into+// session lifecycle decisions and ClickHouse rows.//-// Everything that touches Redis, Postgres, ClickHouse or a queue is INJECTED-// (`IncomingEventDeps`), the way session-end.ts does it, so the tests drive the-// real code with recording doubles instead of `mock.module`.-// `createIncomingEventDeps` binds it to ONE MESSAGE'S ctx: the buffers, the-// Postgres client and the session-end enqueue all ride the scope carrying that-// envelope's requestId, so the id the producer stamped reaches the row the-// consumer writes. The two lookups a work scope cannot supply arrive as-// `IncomingEventBindings` from the composition root.+// Everything that touches Redis, Postgres, ClickHouse or a queue is injected+// (`IncomingEventDeps`) so tests drive the real code with recording doubles.+// `createIncomingEventDeps` binds it to one message's ctx, so the requestId the+// producer stamped reaches the row the consumer writes.import { getTime, isSameDomain, parsePath } from '@openpanel/shared';import { anyPass, isEmpty, isNil, mergeDeepRight, omit, reject } from 'ramda';@@ -245,11 +241,9 @@ const parseRevenue = (revenue: unknown): number | undefined => {* its answer. A false positive on a suppressing check would be silent data* loss, and whether the events table ever gets dedupe is not decided here.*- * It replaces the offset watermark, which could not see a duplicate across a- * rebalance, an eviction, a durability redelivery or a restart. It catches- * redeliveries of the SAME id: crash, eviction, durability redelivery. It- * does NOT catch a lost ACK, where the SDK re-sends and the producer mints a- * fresh id per request; SDK-supplied ids are what closes that, later.+ * Catches redeliveries of the SAME id (crash, eviction, durability redelivery,+ * rebalance, restart). It does NOT catch a lost ACK, where the SDK re-sends and+ * the producer mints a fresh id per request.** FAILS OPEN, AND NEVER REJECTS. Redis being away is exactly when redeliveries* happen, so an observability counter must not be the thing that delays ordiff --git a/packages/core/src/modules/ingest/src/ingest.metrics.ts b/packages/core/src/modules/ingest/src/ingest.metrics.tsindex 4518fec72..4922ecb44 100644--- a/packages/core/src/modules/ingest/src/ingest.metrics.ts+++ b/packages/core/src/modules/ingest/src/ingest.metrics.ts@@ -6,10 +6,9 @@ import type { ConsumerMetrics, DeadLetterReason } from './consumer';// Events whose producer-minted id had already been seen. This is a MARKER, not// a dedupe: the event is inserted either way and this counter is the whole-// point of the check. It replaces `kafka_events_reprocessed_total`, which was-// keyed on a Kafka offset in one process's heap and therefore blind to every-// duplicate that mattered (measured: 161 real duplicates, 0 reprocessed-// increments).+// point of the check. The older offset-keyed counter lived in one process's+// heap and was blind to every duplicate that mattered (161 real duplicates,+// 0 reprocessed increments).//// It UNDERCOUNTS by design: a duplicate whose marker could not be written// (Redis away) is not counted, because the check fails open rather than@@ -33,8 +32,7 @@ export const kafkaHandlerFailuresTotal = new client.Counter({// Messages recorded in the dead-letter list after their attempts were exhausted// (or after they failed to parse), then DROPPED. One increment per message. The// list is capped at the last N, so this counter — not the list — is the only-// thing that tells 50,000 drops apart from 12. The series name is unchanged:-// the destination moved, the meaning did not.+// thing that tells 50,000 drops apart from 12.export const kafkaDeadLetteredTotal = new client.Counter({name: 'kafka_events_dead_lettered_total',help: 'Kafka event messages recorded in the dead-letter list and dropped',diff --git a/packages/core/src/modules/ingest/src/kafka.ts b/packages/core/src/modules/ingest/src/kafka.tsindex 1d2510dcf..0cb0c598c 100644--- a/packages/core/src/modules/ingest/src/kafka.ts+++ b/packages/core/src/modules/ingest/src/kafka.ts@@ -102,9 +102,8 @@ const getProducer = async (config: CoreConfig): Promise<Producer> => {// OUT_OF_ORDER_SEQUENCE_NUMBER and stick the producer per-partition.// Raising it gained only a couple percent of throughput and was// rejected; batching is what amortises the round-trip.- // KAFKA_PRODUCER_MAX_IN_FLIGHT still raises it for a measurement run —- // read docs/KAFKA_PRODUCER_OPTIONS.md first: it is coupled to the retry- // policy.+ // KAFKA_PRODUCER_MAX_IN_FLIGHT still raises it for a measurement run, but it+ // is coupled to the retry policy.maxInFlightRequests: tuning.maxInFlight,allowAutoTopicCreation: true,retry: {@@ -277,7 +276,7 @@ export const createKafkaEventsConsumer = (return consumer;};-// ── Consumer-group lag (backpressure visibility) ────────────────────────────+// Consumer-group lag (backpressure visibility)// A fast producer can hide a lagging consumer entirely — throughput numbers// alone don't show it. `sampleConsumerGroupLag` reads the same end-offset-// minus-committed-offset a broker-side tool (e.g. `rpk group describe`) woulddiff --git a/packages/core/src/modules/ingest/src/producer-batcher.ts b/packages/core/src/modules/ingest/src/producer-batcher.tsindex f2ac6aabd..8f0f409cb 100644--- a/packages/core/src/modules/ingest/src/producer-batcher.ts+++ b/packages/core/src/modules/ingest/src/producer-batcher.ts@@ -2,8 +2,6 @@// ONE `send` carrying MANY messages, instead of one awaited `send` per// /track request.//-// It is OFF unless the config loader says otherwise — see `producer-tuning.ts`.-//// Two invariants the tests hold this file to://// 1. ORDER. Messages leave in the order they were enqueued: within a batchdiff --git a/packages/core/src/modules/ingest/src/producer-errors.ts b/packages/core/src/modules/ingest/src/producer-errors.tsindex 1d21254ca..6e25dfa3c 100644--- a/packages/core/src/modules/ingest/src/producer-errors.ts+++ b/packages/core/src/modules/ingest/src/producer-errors.ts@@ -1,7 +1,5 @@// The producer's error classification, split out of `kafka.ts` so the batching-// path can be tested against the same code the single-message path uses: a-// batch that fails must reset the producer exactly like a lone message did, and-// that is only provable if the recovery wrapper is reachable without a broker.+// path can be tested against the same recovery wrapper without a broker.// Kafka error codes that mean the producer's PID/sequence state is// permanently out of sync with the broker for some partition — only adiff --git a/packages/core/src/modules/ingest/src/producer-tuning.test.ts b/packages/core/src/modules/ingest/src/producer-tuning.test.tsindex ee0095eb6..151393244 100644--- a/packages/core/src/modules/ingest/src/producer-tuning.test.ts+++ b/packages/core/src/modules/ingest/src/producer-tuning.test.ts@@ -1,8 +1,7 @@/*** Batching is ON at the shipped defaults (size 25, linger 5 ms) and- * `maxInFlightRequests` stays 1. There are no fallbacks left to test — the- * three knobs are required `KafkaConfig` fields, so a config that omits one- * does not compile.+ * `maxInFlightRequests` stays 1. The three knobs are required `KafkaConfig`+ * fields, so there are no fallbacks to test.*/import { describe, expect, test } from 'bun:test';diff --git a/packages/core/src/modules/ingest/src/producer-tuning.ts b/packages/core/src/modules/ingest/src/producer-tuning.tsindex 6be48d146..510b4eaa7 100644--- a/packages/core/src/modules/ingest/src/producer-tuning.ts+++ b/packages/core/src/modules/ingest/src/producer-tuning.ts@@ -1,9 +1,7 @@// The Kafka producer's throughput knobs, resolved in one place.//-// The three fields are required members of `KafkaConfig`-// (`packages/core/src/config.ts`): the config loader always sets them, so there-// is no fallback here and no second default anywhere. This file only decides-// whether the configured size batches at all.+// The config loader always sets the three fields, so there is no fallback+// here. This file only decides whether the configured size batches at all.import type { KafkaConfig } from '../../../config';diff --git a/packages/core/src/modules/ingest/wind-down.test.ts b/packages/core/src/modules/ingest/wind-down.test.tsindex 3f3cee81d..13ed2ec37 100644--- a/packages/core/src/modules/ingest/wind-down.test.ts+++ b/packages/core/src/modules/ingest/wind-down.test.ts@@ -2,11 +2,11 @@* Tests for `isIngestionWoundDown` — the gate that stops ingestion for* organizations blocked by the wind-down sequence.*- * Same three behaviours matter: it gates on the wind-down STEP rather than the+ * Three behaviours matter: it gates on the wind-down STEP rather than the* subscription state (so an expired trial keeps ingesting until it has actually* been warned), the caller answers 202 rather than a 4xx (the SDKs retry* everything except 401 and 2xx), and it fails open. `SELF_HOSTED` is a- * `selfHosted` argument now — core reads no environment.+ * `selfHosted` argument: core reads no environment.*/import {@@ -125,7 +125,7 @@ describe('isIngestionWoundDown', () => {it('accepts ingestion from an expired trial that has not been warned yet', async () => {// The reason the gate reads windDownStep and not subscriptionState: every// lapsed trial is already trial_expired, so gating on state would block- // thousands of orgs the moment this ships.+ // thousands of orgs at once.getOrganizationByProjectIdCached.mockResolvedValue({id: 'org-1',subscriptionState: 'trial_expired',diff --git a/packages/core/src/modules/overview/overview.rpc.ts b/packages/core/src/modules/overview/overview.rpc.tsindex 90cb34744..3e066e75f 100644--- a/packages/core/src/modules/overview/overview.rpc.ts+++ b/packages/core/src/modules/overview/overview.rpc.ts@@ -1,9 +1,6 @@// `overviewProcedure` is `publicProcedure` plus one middleware that resolves// share-or-membership off the raw `projectId`/`shareId`, BEFORE the input// parser.-//-// There is no per-range cache middleware yet; `createCacheMiddleware` in-// rpc/base.ts is the seam that will carry it.import { getChartPrevStartEndDate } from '@openpanel/shared';import { format } from 'date-fns';diff --git a/packages/core/src/modules/overview/pages.service.ts b/packages/core/src/modules/overview/pages.service.tsindex b13923d3a..0d6c7aecd 100644--- a/packages/core/src/modules/overview/pages.service.ts+++ b/packages/core/src/modules/overview/pages.service.ts@@ -1,15 +1,6 @@-// The ClickHouse queries moved to src/pages.sql.ts.-//-// `PagesService` is no longer a class and there is no `pagesService` module-// singleton. It is registered as its OWN service (`services.pages`) rather than-// folded into `overview`, because both expose a `getTopPages` and they are-// different queries with different inputs — one over `events` scoped by search,-// one over `sessions` scoped by filters. Folding them would have had to rename-// one, which is a call-site contract this wave does not change. The-// caller-supplied `constructor(client)` slot is gone: the mcp-// `get_page_performance` tool constructed a fresh instance per call only to-// dodge a module-singleton mocking hazard, and there is no singleton left to-// dodge.+// `services.pages` is its own service rather than folded into `overview`: both+// expose a `getTopPages`, over different tables (`events` scoped by search,+// `sessions` scoped by filters) with different inputs.import type { ServiceDeps, Services } from '../../services';import { getSettingsForProject } from '../organization/organization.service';diff --git a/packages/core/src/modules/overview/src/overview-raw-where.sql.test.ts b/packages/core/src/modules/overview/src/overview-raw-where.sql.test.tsindex 3056a288e..b1240219d 100644--- a/packages/core/src/modules/overview/src/overview-raw-where.sql.test.ts+++ b/packages/core/src/modules/overview/src/overview-raw-where.sql.test.ts@@ -1,19 +1,13 @@/*** SQL-syntax tests for the overview module's `getRawWhereClause` (the UTM- * remapping). Same pattern as the chart module's SQL-syntax tests.- * `getRawWhereClause` sits on the `sql` tag, so these assertions read the- * rendered statement AND its bound params instead of one escaped string; the- * rest of the overview module's queries already run through- * overview.sql.ts, covered by overview.sql.test.ts. It is a plain module- * function, not a class method — it never needed a client, being pure- * fragment building.+ * remapping). It sits on the `sql` tag, so these assertions read the rendered+ * statement AND its bound params instead of one escaped string.*- * Strategy: build the SQL string, then run `EXPLAIN <sql>` against the local- * ClickHouse instance. EXPLAIN parses the query, resolves columns, and builds- * the query plan without executing it — so we catch UNKNOWN_IDENTIFIER,- * AMBIGUOUS_IDENTIFIER and bad JOIN ON expressions without needing seeded data.- * WITH FILL TO < FROM is a runtime check, so it's covered by a plain string- * assertion instead.+ * Strategy: build the SQL, then run `EXPLAIN <sql>` against the local ClickHouse.+ * EXPLAIN resolves columns and builds the plan without executing, so+ * UNKNOWN_IDENTIFIER, AMBIGUOUS_IDENTIFIER and bad JOIN ON expressions surface+ * without seeded data. WITH FILL TO < FROM is a runtime check, so it's covered+ * by a plain string assertion instead.** Requires a locally reachable CH at http://localhost:23123/openpanel. All* `itCH` tests auto-skip if CH is unreachable.diff --git a/packages/core/src/modules/overview/src/overview.sql.test.ts b/packages/core/src/modules/overview/src/overview.sql.test.tsindex c4a8e6cb1..d3b2b9414 100644--- a/packages/core/src/modules/overview/src/overview.sql.test.ts+++ b/packages/core/src/modules/overview/src/overview.sql.test.ts@@ -294,7 +294,7 @@ describe('overview.sql — sql.id() identifier whitelists (R3)', () => {// The MCP tools send a bare `YYYY-MM-DD`; the dashboard sends the day's edges// already. Both have to end up bounding the same last day, or every MCP range-// silently loses it (ISSUES.md H8a).+// silently loses it.describe('overview.sql — date-only range boundaries', () => {function boundsOf(fragment: SqlFragment): string[] {const { query_params } = fragment.toStatement();diff --git a/packages/core/src/modules/overview/src/overview.sql.ts b/packages/core/src/modules/overview/src/overview.sql.tsindex 1e342ad83..87c5a01c7 100644--- a/packages/core/src/modules/overview/src/overview.sql.ts+++ b/packages/core/src/modules/overview/src/overview.sql.ts@@ -1,14 +1,9 @@// Every ClickHouse query OverviewService runs, as pure `sql` fragments.//-// Both filter compilers and the field resolver return fragments and are-// interpolated directly. `compiledText` is left with one job here: the-// `INTERVAL <n> <unit>` step keyword in a WITH FILL clause, which is SQL-// syntax, not a value.-//-// `toStartOf`/`toInterval`/`datetime` deliberately never pass a timezone-// into `toStartOfWeek`/`toStartOfMonth` — ClickHouse resolves those from the-// query's `session_timezone` setting instead, which every statement here-// still sets (see src/run-query.ts).+// `toStartOf`/`toInterval`/`datetime` deliberately never pass a timezone into+// `toStartOfWeek`/`toStartOfMonth`: ClickHouse resolves those from the query's+// `session_timezone` setting, which every statement here sets (see+// src/run-query.ts).import { type SqlFragment, sql } from '@openpanel/db/src/clickhouse/sql';import {@@ -118,8 +113,6 @@ function rawWhere(where: SqlFragment | null): SqlFragment {return where ? sql`AND ${where}` : sql.empty;}-// --- revenue ------------------------------------------------------------------export interface RevenueQueryInput {projectId: string;startDate: string;@@ -144,8 +137,6 @@ export function revenueQuery(input: RevenueQueryInput): SqlFragment {`;}-// --- metrics: sessions-only path -----------------------------------------------export interface SessionMetricsQueryInput {projectId: string;startDate: string;@@ -179,8 +170,6 @@ export function sessionMetricsQuery(`;}-// --- metrics: page-filtered path ------------------------------------------------export interface PageFilterMetricsQueryInput {projectId: string;startDate: string;@@ -329,8 +318,6 @@ export function metricsWithPageFilterQuery(`;}-// --- top pages ------------------------------------------------------------------export interface TopPagesQueryInput {projectId: string;startDate: string;@@ -359,8 +346,6 @@ export function topPagesQuery(input: TopPagesQueryInput): SqlFragment {`;}-// --- top entry/exit ---------------------------------------------------------------export interface DistinctSessionsQueryInput {projectId: string;startDate: string;@@ -437,8 +422,6 @@ export function topEntryExitQuery(input: TopEntryExitQueryInput): SqlFragment {`;}-// --- top generic (breakdown by a single dimension) -------------------------------/** `column`/`prefixColumn` are drawn from a closed zod enum upstream — bound* as identifiers here rather than trusted as pre-validated text. */const TOP_GENERIC_COLUMNS = [@@ -564,8 +547,6 @@ export function topGenericSeriesTimeSeriesQuery(`;}-// --- user journey (sankey of screen_view paths) -----------------------------------export interface OrderedEventsQueryInput {projectId: string;startDate: string;@@ -589,9 +570,8 @@ function orderedEventsQuery(input: OrderedEventsQueryInput): SqlFragment {/*** `arrayCompact` drops consecutive repeats in linear time. An `arrayFilter`- * spelling is Theta(n^2) in the longest session's pageview count and could not- * answer for a tenant with one very long session — the same defect, and the- * same fix, as sankey.sql.ts's DEDUPE_CONSECUTIVE.+ * answer for a tenant with one very long session (see sankey.sql.ts's+ * DEDUPE_CONSECUTIVE).*/function pathsDedupedCte(input: OrderedEventsQueryInput,@@ -675,8 +655,6 @@ export function transitionsQuery(input: TransitionsQueryInput): SqlFragment {`;}-// --- top events -------------------------------------------------------------------export interface TopEventsQueryInput {projectId: string;startDate: string;@@ -703,8 +681,6 @@ export function topEventsQuery(input: TopEventsQueryInput): SqlFragment {`;}-// --- top link-out -------------------------------------------------------------------export interface TopLinkOutQueryInput {projectId: string;startDate: string;@@ -728,8 +704,6 @@ export function topLinkOutQuery(input: TopLinkOutQueryInput): SqlFragment {`;}-// --- map data -------------------------------------------------------------------export interface MapDataQueryInput {projectId: string;startDate: string;@@ -755,8 +729,6 @@ export function mapDataQuery(input: MapDataQueryInput): SqlFragment {`;}-// --- live data (dashboard's 30-minute rolling window) ----------------------------export interface LiveTotalSessionsQueryInput {projectId: string;}diff --git a/packages/core/src/modules/overview/src/run-query.ts b/packages/core/src/modules/overview/src/run-query.tsindex b41eed429..82dd9c28a 100644--- a/packages/core/src/modules/overview/src/run-query.ts+++ b/packages/core/src/modules/overview/src/run-query.ts@@ -1,11 +1,4 @@-// The overview module's one ClickHouse entry point.-//-// The client is `deps.ch` — the same round-robin/retry proxy `main.ts` builds-// and hands to every scope — instead of a lazy-// `import('@openpanel/db/src/clickhouse/client')`. `OverviewService` /-// `PagesService` were classes carrying a caller-supplied client precisely-// because there was no scope to reach one through; there is now, so both are-// factories over `ServiceDeps` and this file just forwards.+// The overview module's one ClickHouse entry point; the client is `deps.ch`.//// `timezone` is mandatory here (every overview query sends one, several as a// literal 'UTC') where the shared helper leaves it optional, which is why thisdiff --git a/packages/core/src/modules/report/report.constants.test.ts b/packages/core/src/modules/report/report.constants.test.tsindex ddd9e5904..0b58b4cce 100644--- a/packages/core/src/modules/report/report.constants.test.ts+++ b/packages/core/src/modules/report/report.constants.test.ts@@ -1,7 +1,6 @@/**- * Locks down `getDefaultIntervalByDates`'s rewrite from date-fns- * (differenceInDays/isSameDay) to plain Date math (see file header) against- * the exact boundaries the original branches used.+ * Locks down `getDefaultIntervalByDates`'s plain Date math against the exact+ * boundaries of the date-fns branches it replaces.*/import { describe, expect, it } from 'bun:test';import { getDefaultIntervalByDates } from './report.constants';diff --git a/packages/core/src/modules/report/report.constants.ts b/packages/core/src/modules/report/report.constants.tsindex 6e8a95b79..6676d0858 100644--- a/packages/core/src/modules/report/report.constants.ts+++ b/packages/core/src/modules/report/report.constants.ts@@ -1,10 +1,8 @@// `getDefaultIntervalByDates` uses plain Date math instead of date-fns. Every// process that constructs these Dates runs under TZ=UTC, so Date's local-// getters (getFullYear/getMonth/getDate) already read UTC — the helpers below-// reproduce date-fns's local-calendar semantics under that invariant. The-// only inputs this function ever receives are date-only strings (midnight+// getters already read UTC, and the inputs are date-only strings (midnight// boundaries), where date-fns's exact-duration-vs-calendar-day nuance cannot-// diverge from a plain ms division anyway.+// diverge from a plain ms division.import { z } from 'zod';@@ -328,7 +326,7 @@ function isSameCalendarDay(a: Date, b: Date): boolean {);}-/** date-fns's differenceInDays without the dependency — see file header. */+/** date-fns's differenceInDays without the dependency. */function differenceInDays(dateLeft: Date, dateRight: Date): number {return Math.trunc((dateLeft.getTime() - dateRight.getTime()) / MS_PER_DAY);}@@ -636,10 +634,7 @@ export const chartColors = [*/const CHART_FORMULA_PATTERN = /^[A-Za-z0-9_ .,+\-*/()%^]*$/;-// Helper, not vocabulary: every consumer copies these 4 lines locally instead-// of importing them (apps/start/src/utils/object-to-zod-enums.ts is the-// frontend's copy). Kept private here, for this file's own use building the-// enums below.+// Private helper for the enums below; consumers keep their own copy.function objectToZodEnums<K extends string>(obj: Record<K, unknown>): [K, ...K[]] {@@ -939,10 +934,7 @@ export const zReport = zReportInput.extend({// Alias for backward compatibilityexport const zChartInput = zReportInput;-// ---------------------------------------------------------------------------// `UnionOmit` has no owning module; it is kept here, with the report types it-// is only ever applied to (apps/start's reportSlice).-// ---------------------------------------------------------------------------+// `UnionOmit` has no owning module; it lives with the report types it is applied to.export type UnionOmit<T, K extends keyof any> = T extends any? Omit<T, K>diff --git a/packages/core/src/modules/report/report.rpc.ts b/packages/core/src/modules/report/report.rpc.tsindex 1a0ccb19c..dbfb2dfd4 100644--- a/packages/core/src/modules/report/report.rpc.ts+++ b/packages/core/src/modules/report/report.rpc.ts@@ -1,12 +1,7 @@-// `protectedProcedure` runs `enforceUserIsAuthed` + `enforceAccess` BEFORE the-// input parser. The explicit checks in the handlers below stay: `enforceAccess`-// only sees a TOP-LEVEL `projectId` / `organizationId`, so anything resolved-// from another id needs its own.-//-// Most of this router's inputs carry a reportId or a dashboardId rather than a-// projectId, so `enforceAccess` is blind to them and the in-handler-// `requireProjectAccess` through `ctx.services.auth` is the only check that-// fires.+// `protectedProcedure` runs `enforceAccess` BEFORE the input parser, and it only+// sees a TOP-LEVEL `projectId` / `organizationId`. Most inputs here carry a+// reportId or dashboardId instead, so the in-handler `requireProjectAccess`+// through `ctx.services.auth` is the only check that fires.import { z } from 'zod';import { createTRPCRouter, protectedProcedure } from '../../rpc/base';diff --git a/packages/core/src/modules/report/report.service.test.ts b/packages/core/src/modules/report/report.service.test.tsindex 0ed23b0f5..4456ae735 100644--- a/packages/core/src/modules/report/report.service.test.ts+++ b/packages/core/src/modules/report/report.service.test.ts@@ -1,8 +1,6 @@/**- * Unit tests for mergeGlobalFilters — the helper that combines report-level- * global filters with each event series' own filters (AND semantics). Pure- * function, no ClickHouse/Postgres needed. See ./src/series.ts for the- * helper itself.+ * Unit tests for mergeGlobalFilters, which combines report-level global filters+ * with each event series' own filters (AND semantics). Pure function.*/import { describe, expect, it } from 'bun:test';import type { IChartEventFilter, IChartEventItem } from './report.constants';diff --git a/packages/core/src/modules/report/report.service.ts b/packages/core/src/modules/report/report.service.tsindex d736a2d40..efc573749 100644--- a/packages/core/src/modules/report/report.service.ts+++ b/packages/core/src/modules/report/report.service.ts@@ -1,7 +1,5 @@// Every function takes `ServiceDeps` and reaches Postgres as `deps.db`; the// Prisma row types below are `import type`, erased at runtime.-// `onlyReportEvents`/`mergeGlobalFilters` live in ./src/series.ts — see that-// file for why.import type {Report as DbReport,@@ -49,7 +47,6 @@ export function transformReportEventItem(index: number): IChartEventItem {if (item.type === 'formula') {- // Transform formulareturn {type: 'formula',id: item.id ?? alphabetIds[index]!,@@ -59,7 +56,6 @@ export function transformReportEventItem(};}- // Transform event with type fieldreturn {type: 'event',segment: item.segment ?? 'event',@@ -231,10 +227,8 @@ export async function getReportDataCore(return { ...meta, data: result };}-// ------------------------------------------------------------------------// Mutations. Each takes the resource already fetched by report.rpc.ts for the-// access check, so this does exactly one fewer Prisma round trip than-// re-fetching.+// Mutations take the resource already fetched by report.rpc.ts for the access+// check, saving a second Prisma round trip.type IReportInputForWrite = Omit<IReport, 'projectId'>;@@ -451,8 +445,6 @@ export async function resetReportLayouts(});}-// --- service -------------------------------------------------------------export function createReportService(deps: ServiceDeps,_services: () => Services