Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

chore(core): trim comments in charts, reports, cohorts and ingest

Carl-Gerhard Lindesvärd7eed740fa514
diff --git a/packages/core/src/buffers/base-buffer.ts b/packages/core/src/buffers/base-buffer.ts
index c4d521604..6d5614723 100644
--- a/packages/core/src/buffers/base-buffer.ts
+++ b/packages/core/src/buffers/base-buffer.ts
@@ -7,11 +7,7 @@ import type { CoreConfig } from '../config';
import type { Logger } from '../logger';
import type { ServiceDeps } from '../services';
-/**
- * What a buffer needs from the boot scope. Buffers are built once by
- * `createBuffers(deps)` and hung on `AppDeps` — never module singletons, so a
- * test constructs its own and closes nothing shared.
- */
+/** What a buffer needs from the boot scope; a test constructs its own. */
export interface BufferDeps {
/**
* Each buffer logs under its own name. Injected rather than built here so
@@ -30,10 +26,8 @@ export interface BufferDeps {
*/
isCronPaused(): Promise<boolean>;
/**
- * The parsed environment. Every buffer's batch/chunk/TTL knob used to be a
- * module-scope `process.env` read; they arrive here instead, which is also
- * what lets a test build a buffer with a different sizing without touching
- * the process environment.
+ * The parsed environment: every buffer's batch/chunk/TTL knob, so a test can
+ * size a buffer without touching the process environment.
*/
config: CoreConfig;
}
@@ -316,10 +310,8 @@ export class BaseBuffer {
}
/**
- * Ground-truth size of the buffer (LLEN of the Redis list). O(1). The
- * previous implementation kept a shadow counter to "avoid" LLEN — but
- * LLEN and GET are both O(1) single-roundtrip ops in Redis, so the
- * counter was free complexity that silently drifted. Always use this.
+ * Ground-truth size of the buffer (LLEN of the Redis list), O(1). No shadow
+ * counter: it silently drifts.
*/
async getBufferSize(): Promise<number> {
return await getRedisCache().llen(this.getRedisListKey());
diff --git a/packages/core/src/buffers/bot-buffer.ts b/packages/core/src/buffers/bot-buffer.ts
index d9aefad0b..d985423a6 100644
--- a/packages/core/src/buffers/bot-buffer.ts
+++ b/packages/core/src/buffers/bot-buffer.ts
@@ -58,9 +58,7 @@ export class BotBuffer extends BaseBuffer {
return;
}
- // Raw passthrough: each Redis entry is already a valid JSONEachRow
- // line. Streaming raw strings skips JSON.parse + the client's
- // re-stringify on the hot path.
+ // Raw passthrough: each Redis entry is already a JSONEachRow line.
const ch = this.resolveCh();
const chStart = performance.now();
await ch.insert({
diff --git a/packages/core/src/buffers/buffer.metrics.ts b/packages/core/src/buffers/buffer.metrics.ts
index 988a6a373..12dcf60d0 100644
--- a/packages/core/src/buffers/buffer.metrics.ts
+++ b/packages/core/src/buffers/buffer.metrics.ts
@@ -1,7 +1,5 @@
-// The twelve `buffer_*` series (names and labels reproduced exactly).
-// Registered per boot rather than per import, because the buffers themselves
-// are per boot — and only where a role consumes queues, since the LLEN
-// gauges cost one Redis round trip per buffer per scrape.
+// Registered per boot only where a role consumes queues: the LLEN gauges cost
+// one Redis round trip per buffer per scrape.
import client from 'prom-client';
import { registry } from '../metrics';
@@ -82,8 +80,6 @@ export function registerBufferMetrics(
})
);
- // ---- Flush metrics (populated via flushObserver hooks) ----
-
const flushDuration = new client.Histogram({
name: 'buffer_flush_duration_ms',
help: 'Wall time of a tryFlush call, including lock acquisition',
@@ -142,8 +138,6 @@ export function registerBufferMetrics(
});
register.registerMetric(flushLlenAtStart);
- // ---- Add-path metrics ----
-
const addDurationMs = new client.Histogram({
name: 'buffer_add_duration_ms',
help: 'Duration of a single add() call (per-event ingest path)',
diff --git a/packages/core/src/buffers/clickhouse-failover.test.ts b/packages/core/src/buffers/clickhouse-failover.test.ts
index fb520f188..c9233f0b0 100644
--- a/packages/core/src/buffers/clickhouse-failover.test.ts
+++ b/packages/core/src/buffers/clickhouse-failover.test.ts
@@ -9,18 +9,12 @@ function loadClickHouse() {
}
/**
- * The deliberate ECONNREFUSED to a dead ClickHouse node must be caught and
- * retried (sin-binning) on the client the buffers are handed as
- * `BufferDeps.ch`, exactly as packages/db's
- * `sql.round-robin.clickhouse.test.ts` proves for `chQuery`/`chQueryWithMeta`
- * directly. That file is NOT touched here; this extends the same proof onto
- * `ch.insert` reached via the core buffer seam.
+ * An ECONNREFUSED from a dead ClickHouse node must be retried (sin-binned) on
+ * the client the buffers are handed as `BufferDeps.ch`, including `ch.insert`.
*
- * Setup mirrors that file exactly: a two-node `CLICKHOUSE_URL` whose first node
- * (port 1, privileged and unbindable without root) is reliably refused. The
- * picker starts at index 0, so every call here fails over to node 1 — if the
- * seam let the rejection escape instead of retrying, this file would throw or
- * hang rather than pass.
+ * `CLICKHOUSE_URL` is a two-node list whose first node (port 1, unbindable
+ * without root) is reliably refused. The picker starts at index 0, so every
+ * call fails over to node 1; a rejection that escaped would throw or hang.
*/
const CLICKHOUSE_TEST_DATABASE = 'openpanel_test';
diff --git a/packages/core/src/buffers/create-buffers.ts b/packages/core/src/buffers/create-buffers.ts
index 09bd1b375..02b460bbe 100644
--- a/packages/core/src/buffers/create-buffers.ts
+++ b/packages/core/src/buffers/create-buffers.ts
@@ -1,5 +1,3 @@
-// Built ONCE at boot and hung on `AppDeps.buffers` — never module singletons.
-
import type { BufferDeps } from './base-buffer';
import { BotBuffer } from './bot-buffer';
import { EventBuffer } from './event-buffer';
diff --git a/packages/core/src/buffers/event-buffer.test.ts b/packages/core/src/buffers/event-buffer.test.ts
index 193afc0b9..c82761947 100644
--- a/packages/core/src/buffers/event-buffer.test.ts
+++ b/packages/core/src/buffers/event-buffer.test.ts
@@ -20,11 +20,9 @@ const chQuery = mock(
async (_query: string | SqlFragment): Promise<unknown[]> => []
);
-// The client comes in as `BufferDeps.ch` and reads go through core's own
-// `chQuery` (ch-query.ts) — so the insert path needs no module mock at all, and
-// the read path mocks one core module instead of
-// `@openpanel/db/src/clickhouse/client` (whose import builds a real client and
-// a pino transport worker thread per test file).
+// Inserts go through `BufferDeps.ch` (no module mock needed); reads go through
+// core's `chQuery`, mocked here instead of `@openpanel/db`'s client, whose
+// import builds a real client and a pino transport thread per test file.
mock.module('../ch-query', () => ({
...realChQuery,
chQuery: (_scope: unknown, query: string | SqlFragment) => chQuery(query),
@@ -216,15 +214,9 @@ describe('EventBuffer', () => {
};
expect(callArgs.format).toBe('JSONEachRow');
expect(callArgs.table).toBe('events');
- // After the raw-passthrough optimisation, `values` is an
- // object-mode Readable stream that yields pre-serialized
- // JSONEachRow lines as strings. The client (configured with
- // `json.stringify: (v) => typeof v === 'string' ? v : JSON.stringify(v)`)
- // then passes the strings through unchanged — no JSON.parse on
- // our side, no JSON.stringify on the client's side. Object mode
- // is mandatory: @clickhouse/client rejects byte streams for
- // JSON* formats with "expected Readable Stream with enabled
- // object mode".
+ // `values` is an object-mode Readable of pre-serialized JSONEachRow lines.
+ // Object mode is mandatory: @clickhouse/client rejects byte streams for
+ // JSON* formats.
const stream = callArgs.values as Readable;
expect(stream.readableObjectMode).toBe(true);
const lines = await streamToLines(stream);
@@ -653,8 +645,7 @@ describe('EventBuffer', () => {
chInsert.mockRejectedValueOnce(new Error('ClickHouse unavailable'));
- // Errors propagate to tryFlush (which resyncs the counter). The safety
- // property — queue preserved on CH failure — still holds.
+ // Errors propagate to tryFlush; the queue is preserved on CH failure.
await expect(eventBuffer.processBuffer()).rejects.toThrow(
'ClickHouse unavailable'
);
@@ -702,11 +693,8 @@ describe('extractProjectId', () => {
});
it('falls back to JSON.parse when properties has project_id BEFORE top-level', () => {
- // This is the bug case the regex couldn't handle. If a future
- // refactor swaps the field order in the event constructor, the
- // fast path would attribute counts to the wrong project. The
- // fallback fixes that: as soon as we see two occurrences we
- // resolve via real JSON.parse and pick the top-level key.
+ // If the field order in the event constructor ever changes, the fast path
+ // would misattribute counts; two occurrences must resolve via JSON.parse.
const line = JSON.stringify({
properties: { project_id: 'user-supplied' },
project_id: 'real-project',
diff --git a/packages/core/src/buffers/event-buffer.ts b/packages/core/src/buffers/event-buffer.ts
index 2aae5b27e..6d32cf352 100644
--- a/packages/core/src/buffers/event-buffer.ts
+++ b/packages/core/src/buffers/event-buffer.ts
@@ -6,23 +6,13 @@ import { BaseBuffer, type BufferDeps } from './base-buffer';
const PROJECT_ID_NEEDLE = '"project_id":"';
/**
- * Extract the top-level `project_id` from a single JSONEachRow event
- * line without doing a full JSON.parse.
+ * Extract the top-level `project_id` from a JSONEachRow event line without a
+ * full JSON.parse. Returns `null` if absent or malformed.
*
- * Fast path: scan with `indexOf`. If the needle appears exactly once,
- * we know it's the top-level field (any `project_id` nested in a JSON
- * string value would be escaped as `\"project_id\":\"...`, which the
- * indexOf scan can't match because of the `\` in front).
- *
- * Slow path: if the needle appears two or more times, the line has a
- * legitimate nested `project_id` key (e.g. inside `properties` if a
- * user happens to set one with that name). The regex/indexOf can't
- * tell which is the top-level one — at that point we fall back to a
- * real JSON.parse for correctness. This is rare in practice but
- * removes the silent-attribution failure mode entirely.
- *
- * Returns `null` if no top-level project_id is present or the row is
- * malformed.
+ * Fast path: `indexOf` on the needle. A nested `project_id` inside a string
+ * value is escaped (`\"project_id\"`), so it cannot match; exactly one hit is
+ * the top-level field. Two or more hits mean a real nested key (a user
+ * property of that name), so fall back to JSON.parse rather than misattribute.
*/
export function extractProjectId(line: string): string | null {
const first = line.indexOf(PROJECT_ID_NEEDLE);
@@ -45,9 +35,8 @@ export function extractProjectId(line: string): string | null {
const valueStart = first + PROJECT_ID_NEEDLE.length;
const valueEnd = line.indexOf('"', valueStart);
- // valueEnd === valueStart means the value is empty (`"project_id":""`)
- // — treat as missing so we don't pollute pub/sub counts with an empty
- // key. Matches the old regex's `[^"]+` (one-or-more) behavior.
+ // An empty value is treated as missing so it does not pollute pub/sub
+ // counts with an empty key.
if (valueEnd <= valueStart) {
return null;
}
@@ -138,8 +127,6 @@ export class EventBuffer extends BaseBuffer {
}
add(event: IClickhouseEvent) {
- // Event-buffer's add() is synchronous (in-memory push). Measured anyway
- // for consistency with the other buffers' add-latency tracking.
const start = performance.now();
this.lastQueuedSeq += 1;
this.pendingEvents.push(event);
@@ -426,11 +413,9 @@ export class EventBuffer extends BaseBuffer {
return;
}
- // The queued lines are already valid JSONEachRow (one stringified event
- // per Redis entry), and the client's custom `json.stringify`
- // (CLICKHOUSE_OPTIONS) passes strings through unchanged — so skipping
- // JSON.parse here sends the bytes straight from Redis to the CH HTTP
- // body, with no parse, no re-stringify and no intermediate allocations.
+ // The queued lines are already JSONEachRow, and the client's custom
+ // `json.stringify` (CLICKHOUSE_OPTIONS) passes strings through unchanged,
+ // so the bytes go from Redis to the CH HTTP body without a parse.
const countByProject = new Map<string, number>();
const yieldEvery = this.getYieldInterval(queueEvents.length, {
min: 1000,
@@ -453,8 +438,6 @@ export class EventBuffer extends BaseBuffer {
(chunk) =>
ch.insert({
table: 'events',
- // Stream the raw JSONEachRow lines straight through — already
- // serialized in Redis, no client-side parse/stringify needed.
values: this.jsonEachRowStream(chunk),
format: 'JSONEachRow',
clickhouse_settings: this.getClickhouseSettings(),
diff --git a/packages/core/src/buffers/group-buffer.ts b/packages/core/src/buffers/group-buffer.ts
index 80e9a68c1..024195141 100644
--- a/packages/core/src/buffers/group-buffer.ts
+++ b/packages/core/src/buffers/group-buffer.ts
@@ -172,9 +172,7 @@ export class GroupBuffer extends BaseBuffer {
return;
}
- // Raw passthrough: each Redis entry is already a valid JSONEachRow
- // line. Streaming raw strings to CH skips JSON.parse + the client's
- // re-stringify on the hot path.
+ // Raw passthrough: each Redis entry is already a JSONEachRow line.
const ch = this.resolveCh();
const chStart = performance.now();
await this.parallelLimit(this.chunks(items, this.chunkSize), (chunk) =>
diff --git a/packages/core/src/buffers/profile-backfill-buffer.ts b/packages/core/src/buffers/profile-backfill-buffer.ts
index d4a073580..5da6eae7a 100644
--- a/packages/core/src/buffers/profile-backfill-buffer.ts
+++ b/packages/core/src/buffers/profile-backfill-buffer.ts
@@ -72,8 +72,7 @@ export class ProfileBackfillBuffer extends BaseBuffer {
const ch = this.resolveCh();
// `replicatedTarget`, not `sql.id`: the clustered form carries an
- // `ON CLUSTER` clause and is not a bare identifier, so `sql.id` refused it
- // and every backfill flush threw before reaching ClickHouse.
+ // `ON CLUSTER` clause and is not a bare identifier.
const target = replicatedTarget(
this.deps.config.clickhouseClustered,
TABLE_NAMES.events
diff --git a/packages/core/src/buffers/profile-buffer.test.ts b/packages/core/src/buffers/profile-buffer.test.ts
index d76e1138e..b4910488d 100644
--- a/packages/core/src/buffers/profile-buffer.test.ts
+++ b/packages/core/src/buffers/profile-buffer.test.ts
@@ -187,8 +187,7 @@ describe('ProfileBuffer', () => {
chInsert.mockRejectedValueOnce(new Error('ClickHouse unavailable'));
- // Errors propagate to tryFlush (which resyncs the counter). The safety
- // property — queue preserved on CH failure — still holds.
+ // Errors propagate to tryFlush; the queue is preserved on CH failure.
await expect(profileBuffer.processBuffer()).rejects.toThrow(
'ClickHouse unavailable'
);
diff --git a/packages/core/src/buffers/profile-buffer.ts b/packages/core/src/buffers/profile-buffer.ts
index 26a2fc46c..28efec9cb 100644
--- a/packages/core/src/buffers/profile-buffer.ts
+++ b/packages/core/src/buffers/profile-buffer.ts
@@ -213,11 +213,8 @@ export class ProfileBuffer extends BaseBuffer {
', '
);
try {
- // Table alias `p` is required: without it, WHERE's `last_seen_at`
- // resolves to the SELECT-list aggregate alias `max(last_seen_at) AS
- // last_seen_at`, which is an aggregate function and illegal in WHERE
- // (CH ILLEGAL_AGGREGATION). Qualifying with `p.` bypasses the alias
- // lookup and binds to the raw column.
+ // Qualify with `p.`: a bare `last_seen_at` resolves to the aggregate
+ // alias and is illegal in WHERE (ILLEGAL_AGGREGATION).
const rows = await this.chQuery<IClickhouseProfile>(
sql`SELECT ${PROFILE_LATEST_AGGREGATE_COLUMNS}
FROM ${sql.id(TABLE_NAMES.profiles)} AS p
@@ -263,16 +260,10 @@ export class ProfileBuffer extends BaseBuffer {
return;
}
- // Parse + merge within batch in a single pass. Yields back to the
- // event loop periodically so concurrent add() calls and BullMQ
- // heartbeats can make progress while the merge churns through
- // profiles. The merge step is the hottest CPU stage of any buffer —
- // deepMergeObjects recursively combines nested `properties` objects
- // and was previously responsible for ~500ms event-loop blocks.
- //
- // Heavy loop (parse + deep merge): narrow band so we yield often
- // even with smaller batches but don't yield every-few-items on
- // very small ones.
+ // Yields to the event loop periodically so concurrent add() calls and
+ // BullMQ heartbeats progress: deepMergeObjects on nested `properties` is
+ // the hottest CPU stage of any buffer. Narrow band: parse + deep merge
+ // is a heavy loop.
const mergeYieldEvery = this.getYieldInterval(rawProfiles.length, {
min: 100,
max: 1000,
@@ -317,10 +308,8 @@ export class ProfileBuffer extends BaseBuffer {
}
}
- // Fetch cache misses from ClickHouse in bounded chunks. Timed separately
- // from the ch.insert phase because it can dominate flush time for
- // profile-buffer specifically (FINAL replaced with argMax + GROUP BY in
- // batchFetchFromClickhouse, but worth measuring directly).
+ // Fetch cache misses from ClickHouse in bounded chunks, timed apart from
+ // ch.insert because the fetch can dominate flush time.
let chFetchMs: number | undefined;
if (cacheMisses.length > 0) {
const chFetchStart = performance.now();
@@ -332,9 +321,8 @@ export class ProfileBuffer extends BaseBuffer {
}
}
- // Final merge: in-batch profile + existing (from cache or ClickHouse).
- // Second heavy stage — same deepMergeObjects call per unique profile.
- // Same band as the in-batch merge above.
+ // Final merge: in-batch profile + existing (cache or ClickHouse), same
+ // band as the in-batch merge.
const finalYieldEvery = this.getYieldInterval(uniqueProfiles.length, {
min: 100,
max: 1000,
diff --git a/packages/core/src/buffers/replay-buffer.ts b/packages/core/src/buffers/replay-buffer.ts
index fe184d17e..dd73aef62 100644
--- a/packages/core/src/buffers/replay-buffer.ts
+++ b/packages/core/src/buffers/replay-buffer.ts
@@ -72,11 +72,8 @@ export class ReplayBuffer extends BaseBuffer {
return;
}
- // Raw passthrough: each Redis entry is already a valid JSONEachRow
- // line (we JSON.stringify a single chunk before rpush). Streaming the
- // raw strings to CH skips JSON.parse × N on the worker AND the
- // client's internal JSON.stringify × N — significant because each
- // rrweb chunk's `payload` is 10–100KB.
+ // Raw passthrough: each Redis entry is already a JSONEachRow line, and an
+ // rrweb chunk's `payload` is 10–100KB, so skipping parse/stringify matters.
const ch = this.resolveCh();
const chStart = performance.now();
await this.parallelLimit(this.chunks(items, this.chunkSize), (chunk) =>
diff --git a/packages/core/src/buffers/session-buffer.test.ts b/packages/core/src/buffers/session-buffer.test.ts
index 216299ef8..5555d1fc0 100644
--- a/packages/core/src/buffers/session-buffer.test.ts
+++ b/packages/core/src/buffers/session-buffer.test.ts
@@ -356,9 +356,7 @@ describe('SessionBuffer', () => {
chInsert.mockRejectedValueOnce(new Error('ClickHouse unavailable'));
- // Errors now propagate to tryFlush (which handles them by resyncing the
- // counter). processBuffer no longer swallows — we still verify the
- // safety property: the queue is preserved.
+ // Errors propagate to tryFlush; the queue is preserved.
await expect(sessionBuffer.processBuffer()).rejects.toThrow(
'ClickHouse unavailable'
);
diff --git a/packages/core/src/buffers/session-buffer.ts b/packages/core/src/buffers/session-buffer.ts
index 20e14ada5..d69e77201 100644
--- a/packages/core/src/buffers/session-buffer.ts
+++ b/packages/core/src/buffers/session-buffer.ts
@@ -6,7 +6,7 @@ import type { IClickhouseSession } from '../modules/session/session.service';
import { TABLE_NAMES } from '../shared/ch-tables';
import { BaseBuffer, type BufferDeps } from './base-buffer';
-// 30min of idle in event-time → session ends. Matches industry default.
+// 30min of idle in event-time → session ends.
const DEFAULT_SESSION_TIMEOUT_MS = 1000 * 60 * 30;
/**
@@ -227,9 +227,9 @@ export class SessionBuffer extends BaseBuffer {
* call is a no-op — the new session still needs its blob and sorted-set
* entry.
*
- * Blobs no longer have a Redis TTL — this is the only path that removes
- * them. The daily vacuum cron is a backstop for the rare case where this
- * fails (worker crash, network partition).
+ * Blobs have no Redis TTL, so this is the only path that removes them. The
+ * daily vacuum cron is a backstop for when it fails (worker crash, network
+ * partition).
*/
async cleanup({
projectId,
@@ -552,12 +552,10 @@ export class SessionBuffer extends BaseBuffer {
parsed.push(s);
}
- // A single high-activity session produces many (sign=-1, sign=+1)
- // pairs within one flush window — every extend appends a pair. The
- // VersionedCollapsingMergeTree collapses them at merge time, so the
- // final state is correct either way, but inserting all the
- // intermediate rows costs network bytes, gzip CPU, and CH ingest +
- // merge work. Squash per-version to insert only the boundary rows.
+ // A busy session appends a (-1, +1) pair per extend within one flush.
+ // VersionedCollapsingMergeTree would collapse them at merge time anyway,
+ // but inserting them all costs network, gzip CPU and merge work. Squash
+ // per version to insert only the boundary rows.
//
// Set SESSION_BUFFER_SQUASH=false to disable as a safety hatch.
const sessions = this.squashEnabled
diff --git a/packages/core/src/jobs/compat.ts b/packages/core/src/jobs/compat.ts
index 3f20adec6..ee85adaad 100644
--- a/packages/core/src/jobs/compat.ts
+++ b/packages/core/src/jobs/compat.ts
@@ -1,7 +1,5 @@
-// There is no dual-write window, no drain and no queue rename: a legacy job
-// already in the queue is read back through these hooks. A wrong one is
-// silent job loss, which is why the tests next to this file were written
-// before it.
+// A legacy job already in the queue is read back through these hooks. A wrong
+// one is silent job loss.
import type { CompatHook } from './define';
import { isEnvelope, type JobMeta } from './envelope';
diff --git a/packages/core/src/jobs/jobs.metrics.test.ts b/packages/core/src/jobs/jobs.metrics.test.ts
index c708dfcb6..a0cda0e61 100644
--- a/packages/core/src/jobs/jobs.metrics.test.ts
+++ b/packages/core/src/jobs/jobs.metrics.test.ts
@@ -31,8 +31,7 @@ test('five gauges per queue, named from the Redis key so dashboards keep working
for (const suffix of COUNT_SUFFIXES) {
expect(names).toContain(`sessions_${suffix}_count`);
expect(names).toContain(`cron_${suffix}_count`);
- // Acceptance note: the registry key stays `cohortCompute`, so the series
- // name does too.
+ // The series name follows the registry key, `cohortCompute`.
expect(names).toContain(`cohortCompute_${suffix}_count`);
}
});
diff --git a/packages/core/src/jobs/jobs.metrics.ts b/packages/core/src/jobs/jobs.metrics.ts
index 34ee16e9e..3ff1e65f9 100644
--- a/packages/core/src/jobs/jobs.metrics.ts
+++ b/packages/core/src/jobs/jobs.metrics.ts
@@ -1,17 +1,7 @@
-// The queue-side collectors, on the one core registry.
-//
-// Three deliberate differences from the metric set this replaces, written up
-// for dashboard owners in `packages/core/docs/OPS_GRAFANA_MIGRATION.md`:
-//
-// 1. Registers the five `<queue>_*_count` gauges for all seven BullMQ
-// queues, not just two — ten new series appear; none of the existing ten
-// changes. 2. They register only where the role consumes (main.ts), because
-// each gauge is one Redis round trip per scrape and ten api replicas
-// exposing them would multiply that for no new information. 3. A gauge whose
-// Redis read throws is skipped rather than failing the whole scrape, so one
-// Redis blip does not turn into a 500 on /metrics and a gap in every panel;
-// core's buffer gauges already swallow the same way
-// (buffers/buffer.metrics.ts).
+// The `<queue>_*_count` gauges register only where the role consumes: each is
+// one Redis round trip per scrape, and api replicas would multiply that for no
+// new information. A gauge whose Redis read throws is skipped rather than
+// failing the whole scrape.
import type { Queue as BullQueue } from 'bullmq';
import client from 'prom-client';
diff --git a/packages/core/src/jobs/schedulers.ts b/packages/core/src/jobs/schedulers.ts
index 4e998e96f..59e0b6133 100644
--- a/packages/core/src/jobs/schedulers.ts
+++ b/packages/core/src/jobs/schedulers.ts
@@ -4,9 +4,7 @@
// needed.
//
// The scheduler list is derived from the `cron` queue's own jobs, so a job
-// and its schedule cannot drift apart. `schedulersFromRegistry` takes the
-// registry's `cron` queue as data; the registry itself calls it and exports
-// the result as `CRON_SCHEDULES`, so this file imports nothing above it.
+// and its schedule cannot drift apart.
import type { Logger } from '../logger';
import type { AnyJob, QueueDefinition, RepeatSchedule } from './define';
@@ -55,8 +53,7 @@ const CONFLICT_JOB_STATES = [
const CONFLICT_ERROR_SUBSTRING = 'job ID already exists';
/**
- * Derives the scheduler list from the `cron` queue's own jobs — the join is by
- * construction now, not by a second registry kept in sync by a test. A job's
+ * Derives the scheduler list from the `cron` queue's own jobs. A job's
* `cron` is `RepeatSchedule` (always-on), `null` (on-demand — e.g. `ping`,
* scheduled conditionally elsewhere), or, on the `cron` queue, never simply
* absent: `defineQueue`'s overload for that queue name makes omitting it a type
@@ -171,7 +168,6 @@ async function upsertWithConflictRetry(
'job scheduler conflict detected, attempting cleanup'
);
await removeConflictingJobs(queue, scheduler.id, logger);
- // Also try removing the scheduler itself to start fresh.
await queue.removeJobScheduler(scheduler.id).catch(() => undefined);
try {
diff --git a/packages/core/src/modules/chart/chart.rpc.ts b/packages/core/src/modules/chart/chart.rpc.ts
index 8907da097..e7bbff83f 100644
--- a/packages/core/src/modules/chart/chart.rpc.ts
+++ b/packages/core/src/modules/chart/chart.rpc.ts
@@ -1,11 +1,10 @@
-// The six member-only procedures use `protectedProcedure`, so
-// `enforceAccess` reads the top-level `projectId` before the input is parsed.
+// The six member-only procedures use `protectedProcedure`, so `enforceAccess`
+// reads the top-level `projectId` before the input is parsed.
//
// `chartProcedure` (funnel, conversion, sankey, chart, aggregate, cohort) is the
-// share-aware builder: `publicProcedure` plus one middleware that admits an
-// anonymous caller holding a valid share — `shareId` + `id` resolve the saved
-// report, and the request renders that report (the caller may only move the
-// date window). The resolved report rides on `ctx.report`, so a handler
+// share-aware builder: an anonymous caller holding a valid share resolves the
+// saved report, and the request renders that report (the caller may only move
+// the date window). The resolved report rides on `ctx.report`, so a handler
// cannot forget to resolve it.
import { z } from 'zod';
diff --git a/packages/core/src/modules/chart/chart.service.test.ts b/packages/core/src/modules/chart/chart.service.test.ts
index 28a97da4c..cf097f904 100644
--- a/packages/core/src/modules/chart/chart.service.test.ts
+++ b/packages/core/src/modules/chart/chart.service.test.ts
@@ -27,7 +27,7 @@ const DAY_MS = 24 * 60 * 60 * 1000;
const PROJECT_CARD_DAYS = 90;
// Bypass Redis caching so a stale entry from another run cannot gate an
-// assertion. Captured before mock.module runs — see mcp's tools.test.ts.
+// assertion. Captured before mock.module runs.
const actualRedis = await import('@openpanel/redis');
mock.module('@openpanel/redis', () => ({
...actualRedis,
@@ -264,8 +264,8 @@ describe('picker queries', () => {
it('lists event names with counts, `*` first', async () => {
const events = await service.listChartEvents(TEST_PROJECT_ID);
- // Counts are `sum(event_count)` over distinct_event_names_mv, so they are
- // real event totals. `count(name)` would count insert parts (ISSUES.md M1).
+ // Counts are `sum(event_count)` over distinct_event_names_mv, i.e. real event
+ // totals; `count(name)` would count insert parts.
expect(events[0]?.name).toBe('*');
expect(events[0]?.count).toBe(
events.slice(1).reduce((total, event) => total + event.count, 0)
@@ -284,10 +284,8 @@ describe('picker queries', () => {
]);
});
- // The picker used to show `count(name)` over distinct_event_names_mv, which
- // counts one row per name PER INSERT PART, not per event (ISSUES.md M1). On
- // the large seed that showed 52 for an event with 974,387 occurrences. The
- // only assertion that catches it is one against the events table itself.
+ // `count(name)` over distinct_event_names_mv counts one row per name PER INSERT
+ // PART, not per event, so only an assertion against the events table catches it.
it('counts real events, not materialized-view rows', async () => {
const events = await service.listChartEvents(TEST_PROJECT_ID);
const actual = await ch.query({
diff --git a/packages/core/src/modules/chart/chart.service.ts b/packages/core/src/modules/chart/chart.service.ts
index 99189c813..e140be44f 100644
--- a/packages/core/src/modules/chart/chart.service.ts
+++ b/packages/core/src/modules/chart/chart.service.ts
@@ -1,9 +1,5 @@
-// Every ClickHouse statement this module runs is a `sql` fragment from
-// src/sql.ts. Funnel, conversion, sankey and retention are dispatched to the
-// sibling `*.service.ts` files. The one thing still value-imported from
-// `@openpanel/db` under `./src/` is the `sql` tag, which stays in
-// `packages/db` by name: a compile-time template tag, no client and no
-// request scope.
+// Every ClickHouse statement here is a `sql` fragment from src/sql.ts. Funnel,
+// conversion, sankey and retention dispatch to the sibling `*.service.ts` files.
import { getChartPrevStartEndDate } from '@openpanel/shared';
import { flatten, map, pipe, prop, sort, uniq } from 'ramda';
@@ -110,17 +106,15 @@ export {
type ChartBucketProfilesInput,
ChartCohortIdError,
} from './src/sql';
-// Types only: `packages/core/src/index.ts` re-exports these two. The four
-// statement functions beside them had no consumer through this file — the
-// engine and the sibling services import ./src/statement directly.
+// Types only: `packages/core/src/index.ts` re-exports these two.
export type {
AggregateChartSqlInput,
ChartSqlInput,
} from './src/statement';
/**
- * Cap on distinct event property keys returned to the picker. Projects in the
- * 8k range exist, so the previous 10k was reachable in normal use.
+ * Cap on distinct event property keys returned to the picker. Projects with
+ * ~8k keys exist.
*/
const EVENT_PROPERTY_KEY_LIMIT = 50_000;
@@ -144,12 +138,10 @@ const BUCKET_PROFILES_BATCH_SIZE = 200;
const BUCKET_PROFILES_FETCH_CONCURRENCY = 4;
/**
- * Cap on the ids one chart data point hands the drill-down modal. The bucket
- * query used to be unbounded, so a busy project's point returned a 31,447-id
- * tail that the modal renders
- * through a virtualizer and nobody scrolls to, after paying for every one of
- * them in `profiles FINAL` batches. Same number as FUNNEL_PROFILES_LIMIT: both
- * feed the same modal, so the two drill-downs should not disagree on depth.
+ * Cap on the ids one chart data point hands the drill-down modal. A busy point
+ * can hold tens of thousands of ids nobody scrolls to, each paid for in
+ * `profiles FINAL` batches. Same number as FUNNEL_PROFILES_LIMIT: both feed the
+ * same modal.
*/
const BUCKET_PROFILES_LIMIT = 1000;
@@ -202,8 +194,6 @@ async function getProfilesInBatches(
return perBatch.flat();
}
-// --- shared report input -----------------------------------------------------
-
export type ShareableReportInput = IReportInput & {
shareId?: string;
id?: string;
@@ -229,8 +219,6 @@ export function resolveReportInput(
};
}
-// --- projectCard -------------------------------------------------------------
-
export interface ProjectCardChartRow {
value: number;
date: Date;
@@ -299,8 +287,6 @@ export async function getProjectCard(
};
}
-// --- events / properties / values -------------------------------------------
-
export interface ChartEventOption {
name: string;
count: number;
@@ -448,8 +434,6 @@ export async function getChartPropertyValues(
};
}
-// --- funnel / conversion / sankey / retention --------------------------------
-
async function currentAndPreviousPeriod(
deps: ServiceDeps,
chartInput: IReportInput
@@ -629,8 +613,6 @@ export async function getRetentionChart(
});
}
-// --- drill-down profiles -----------------------------------------------------
-
export interface ChartBucketProfilesRequest {
projectId: string;
/** ISO string of the data point's bucket. */
@@ -736,30 +718,22 @@ export async function getFunnelStepProfiles(
return getProfilesInBatches(deps, ids, projectId, FUNNEL_PROFILES_BATCH_SIZE);
}
-// --- service -----------------------------------------------------------------
-
-// The funnel, conversion, sankey and retention services live in this module's
-// own sibling files rather than in modules of their own, so they FOLD INTO
-// `chart` rather than becoming four more `Services` members: chart.service.ts
-// is already the dispatcher every caller goes through (`getFunnelChart` /
-// `getConversionChart` / `getSankeyChart` / `getRetentionChart`), and a
-// `Services` key per file would name four things that are not modules.
+// The funnel, conversion, sankey and retention services fold into `chart`
+// rather than becoming `Services` members: chart.service.ts is the dispatcher
+// every caller goes through.
export function createChartService(
deps: ServiceDeps,
services: () => Services
) {
- // The four chart sub-modules each expose their own `create*Service(deps)`.
- // `services.ts` binds each under its own key; `chart` composes them as well,
- // so the callers that reach a funnel/retention method through the chart
- // facade keep working. Both bind the same stateless closures.
+ // Each sub-module exposes its own `create*Service(deps)`; chart composes them
+ // too so callers reaching them through the chart facade keep working.
const funnel = createFunnelService(deps, services);
const conversion = createConversionService(deps, services);
const sankey = createSankeyService(deps, services);
const retention = createRetentionService(deps, services);
return {
- // chart
execute: (input: IReportInput): Promise<FinalChart> =>
executeChart(deps, input),
executeAggregate: (input: IReportInput): Promise<FinalChart> =>
@@ -781,7 +755,6 @@ export function createChartService(
funnelStepProfiles: (
input: FunnelStepProfilesRequest
): Promise<IServiceProfile[]> => getFunnelStepProfiles(deps, input),
- // funnel
getFunnelGroup: funnel.getFunnelGroup,
/** The funnel row -> serie grouping (funnel.service.ts's `toSeries`). */
toFunnelSeries: funnel.toSeries,
@@ -792,19 +765,16 @@ export function createChartService(
getFunnelCore: funnel.getFunnelCore,
buildFunnelBase: funnel.buildFunnelBase,
getFunnelProfileIds: funnel.getFunnelProfileIds,
- // conversion
getConversionChart: (
chartInput: IReportInput
): ReturnType<typeof getConversionChart> =>
getConversionChart(deps, chartInput),
getConversion: conversion.getConversion,
- // sankey
getRawWhereClause: sankey.getRawWhereClause,
getSankeyChart: (input: IReportInput): ReturnType<typeof getSankeyChart> =>
getSankeyChart(deps, input),
getSankey: sankey.getSankey,
getUserFlowCore: sankey.getUserFlowCore,
- // retention
processCohortData: retention.processCohortData,
getRetentionChart: (
report: NonNullable<IServiceReport> | null,
diff --git a/packages/core/src/modules/chart/funnel.service.ts b/packages/core/src/modules/chart/funnel.service.ts
index 159ad3ef0..18790f038 100644
--- a/packages/core/src/modules/chart/funnel.service.ts
+++ b/packages/core/src/modules/chart/funnel.service.ts
@@ -433,10 +433,8 @@ export async function getFunnelProfileIds(
}
/**
- * `ctx.services.chart`'s funnel half, as its own module factory.
- * `createChartService` composes this rather than re-binding these functions
- * itself, so the chart module keeps one registry key while this file exposes
- * its own factory.
+ * `ctx.services.chart`'s funnel half, as its own module factory, composed by
+ * `createChartService`.
*/
export function createFunnelService(
deps: ServiceDeps,
diff --git a/packages/core/src/modules/chart/retention.service.test.ts b/packages/core/src/modules/chart/retention.service.test.ts
index fce7194f4..c4855d896 100644
--- a/packages/core/src/modules/chart/retention.service.test.ts
+++ b/packages/core/src/modules/chart/retention.service.test.ts
@@ -92,9 +92,8 @@ describe('getRetentionCohort', () => {
});
it('counts retention across the year boundary (week interval)', async () => {
- // The old toWeek() implementation returned week-of-year (resets each Jan),
- // so a 2024-W52 -> 2025-W01 return produced a negative diff and was dropped.
- // This locks the year-aware behavior.
+ // Week-of-year resets each Jan, so a 2024-W52 -> 2025-W01 return must still
+ // count (year-aware diff).
const rows = await getRetentionCohort(deps, {
projectId: PROJECT_ID,
firstEvent: ['app_open'],
@@ -140,9 +139,9 @@ describe('getRetentionCohort', () => {
// fired app_open on D2 (RU1, RU2, RU4) already belongs to an earlier cohort.
expect(rows.map((row) => row.cohort_interval)).toEqual([day.d0, day.d1]);
- // Characterize the bug we fixed: the legacy "every-occurrence" cohorting
- // (no GROUP BY profile_id dedup) places recurring users in every day they
- // were active, producing a spurious D2 cohort and inflated sizes.
+ // Without the GROUP BY profile_id dedup, every-occurrence cohorting places
+ // recurring users in every day they were active, producing a spurious D2
+ // cohort and inflated sizes.
const legacy = await chQuery<{ cohort_interval: string; size: number }>(`
SELECT toDate(created_at) AS cohort_interval, COUNT(DISTINCT profile_id) AS size
FROM ${TABLE_NAMES.cohort_events_mv}
@@ -335,7 +334,7 @@ describe('processCohortData weighted average (pure)', () => {
expect(avg.values[0]).toBe(80);
expect(avg.percentages[0]).toBe(1);
// Period 2 pools C0+C1 only (C2 is immature). C1's genuine zero is included,
- // so it's 25/180 = 0.14 — NOT 25/100 = 0.25 (the old "exclude zeros" bug).
+ // so it's 25/180 = 0.14, not 25/100 = 0.25.
expect(avg.percentages[2]).toBe(0.14);
// Period 3 pools only C0 (C1, C2 immature): 10/100 = 0.10.
expect(avg.percentages[3]).toBe(0.1);
diff --git a/packages/core/src/modules/chart/retention.service.ts b/packages/core/src/modules/chart/retention.service.ts
index b687ab605..5b6dcf097 100644
--- a/packages/core/src/modules/chart/retention.service.ts
+++ b/packages/core/src/modules/chart/retention.service.ts
@@ -1,6 +1,5 @@
-// The cohort matrix here is the single source of truth for retention cohorts:
-// it powers the dashboard retention chart (via the tRPC `cohort` procedure) as
-// well as the MCP / agent / REST retention endpoints.
+// The cohort matrix is shared by the dashboard retention chart and the MCP /
+// agent / REST retention endpoints.
import { DateTime, round } from '@openpanel/shared';
import { range } from 'ramda';
@@ -194,17 +193,13 @@ export async function getEngagementCore(
};
}
-// ---------------------------------------------------------------------------
-// Cohort retention matrix
-//
-// Definition: every user is assigned to exactly ONE cohort, the interval of
-// their FIRST `firstEvent` within the window (first-touch). For each cohort we
+// Cohort retention matrix: every user is assigned to exactly ONE cohort, the
+// interval of their FIRST `firstEvent` within the window. For each cohort we
// count how many of those users performed `secondEvent` 0..N intervals later.
-// - criteria 'on' -> active exactly k intervals after (=)
-// - criteria 'on_or_after' -> active at least k intervals after (>=, cumulative)
-// When `firstEvent` / `secondEvent` is empty the name filter is dropped and the
-// query measures retention on ANY activity (active-user retention).
-// ---------------------------------------------------------------------------
+// - criteria 'on' -> active exactly k intervals after (=)
+// - criteria 'on_or_after' -> active at least k intervals after (>=, cumulative)
+// An empty `firstEvent` / `secondEvent` drops the name filter, measuring
+// active-user retention.
const LUXON_UNIT: Record<IRetentionInterval, 'days' | 'weeks' | 'months'> = {
minute: 'days',
diff --git a/packages/core/src/modules/chart/sankey.service.ts b/packages/core/src/modules/chart/sankey.service.ts
index 603ba3dc3..c184a974f 100644
--- a/packages/core/src/modules/chart/sankey.service.ts
+++ b/packages/core/src/modules/chart/sankey.service.ts
@@ -35,8 +35,6 @@ const MIN_LINK_PERCENT = 0.25;
const PERCENT = 100;
const FIRST_STEP = 1;
-// `packages/core/src/index.ts` names this file as the schema's source; the
-// schema itself lives in chart.constants.ts.
export { type IGetSankeyInput, zGetSankeyInput } from './chart.constants';
export interface SankeyNode {
@@ -134,12 +132,10 @@ function toSankeyEvent(
* so. An unparseable date is left alone — the statement itself is where a
* malformed range is rejected.
*
- * Applied by `getSankeyChart`, the `chart.sankey` entry point, and not by
- * `getSankey` itself: it bounds that procedure specifically, because it is
- * the range picker that offers `12m`/`lastYear`. The REST
- * `/insights/:projectId/user_flow` route, the MCP `get_user_flow` tool and the
- * assistant tool take caller-supplied dates, so they are not covered by the
- * range picker's cap.
+ * Applied by `getSankeyChart` (the `chart.sankey` entry point) and not by
+ * `getSankey`: the REST `/insights/:projectId/user_flow` route, the MCP
+ * `get_user_flow` tool and the assistant tool take caller-supplied dates and
+ * are not covered by the range picker's cap.
*/
export function assertSankeyWindowIsAnswerable(
startDate: string,
@@ -388,7 +384,7 @@ function mergeFinalNodes(
return b.value - a.value;
});
- // Links can now point at the same merged target; sum them.
+ // Links can point at the same merged target; sum them.
const linkMap = new Map<string, number>();
for (const link of remappedLinks) {
const key = `${link.source}->${link.target}`;
diff --git a/packages/core/src/modules/chart/src/compiled.ts b/packages/core/src/modules/chart/src/compiled.ts
index 03999a158..480234e35 100644
--- a/packages/core/src/modules/chart/src/compiled.ts
+++ b/packages/core/src/modules/chart/src/compiled.ts
@@ -1,13 +1,8 @@
-// The ONE place the chart module splices pre-compiled SQL text.
-//
-// Both filter compilers (./filter-where.ts, ./table-filter-where.ts) and the
-// field resolver (./field-resolution.ts) return `SqlFragment`s, so nothing
-// crosses `compiledText` from them. What is left are the places the chart
-// builders splice a name they generated themselves — CTE names, `b_<n>` /
-// `label_<n>` aliases, a `toStartOf*` keyword, the backtick-quoted
-// `profile.properties.<key>` CTE alias — none of which is a value and none of
-// which `sql.id` can express. It stays a named export in one file, not a
-// `sql.raw()`, so the census is a grep.
+// The ONE place the chart module splices pre-compiled SQL text: names the chart
+// builders generate themselves (CTE names, `b_<n>` / `label_<n>` aliases, a
+// `toStartOf*` keyword, the backtick-quoted `profile.properties.<key>` CTE
+// alias), none of which is a value and none of which `sql.id` can express. It
+// stays a named export in one file, not a `sql.raw()`, so the census is a grep.
import {
isSqlFragment,
@@ -55,10 +50,9 @@ function narrowableKey(slot: SqlSlot, keys: string[]): string | null {
}
/**
- * Splice nested fragments into one flat (literals, params) interleaving. The
- * rendered statement is unchanged — this only makes "the literal immediately
- * before this param" a question with an answer, which is what the rewrite
- * below needs now that `sql.id` contributes a nested fragment of its own.
+ * Splice nested fragments into one flat (literals, params) interleaving, so
+ * "the literal immediately before this param" is answerable, which the rewrite
+ * below needs because `sql.id` contributes a nested fragment of its own.
*/
function flatten(
fragment: SqlFragment,
@@ -84,16 +78,11 @@ function flatten(
}
/**
- * The same rewrite over a fragment. It runs on the literal parts, and on the
- * one shape the field resolver now emits instead of text — the literal
+ * The same rewrite over a fragment: on the literal parts, and on the literal
* `profile.properties[` followed by the key as a bound `String` param and a
- * closing `]`. Collapsing those three pieces is what keeps a narrowed
- * `profile.properties.<key>` ref pointing at the CTE's scalar column once the
- * key stopped being part of the SQL text.
- *
- * A value can no longer be rewritten even in principle: only a param the
- * resolver placed in that exact position is eligible, and it is replaced by an
- * identifier rather than edited.
+ * closing `]`. Collapsing those pieces keeps a narrowed ref pointing at the
+ * CTE's scalar column. Only a param in that exact position is eligible, and it
+ * is replaced by an identifier rather than edited.
*/
export function fragmentWithProfileRefs(
fragment: SqlFragment,
diff --git a/packages/core/src/modules/chart/src/conversion.sql.ts b/packages/core/src/modules/chart/src/conversion.sql.ts
index dced41210..01e14fba8 100644
--- a/packages/core/src/modules/chart/src/conversion.sql.ts
+++ b/packages/core/src/modules/chart/src/conversion.sql.ts
@@ -1,10 +1,5 @@
// The conversion statement as a pure `sql` fragment.
//
-// The field resolver and both filter compilers return fragments; their output —
-// breakdown expressions, per-step filter clauses and the cohort joins — is
-// interpolated directly. `compiledText` is left with the `b_<index>` aliases
-// this file generates itself.
-//
// Cluster note: `events`, `profiles` and `groups` are Distributed on Cloud. The
// profile / group / cohort joins are LEFT ANY JOIN and run under the client's
// `distributed_product_mode: 'allow'`; no `IN (subquery)` is involved.
diff --git a/packages/core/src/modules/chart/src/dates.ts b/packages/core/src/modules/chart/src/dates.ts
index d4e9e7eed..4c3802aed 100644
--- a/packages/core/src/modules/chart/src/dates.ts
+++ b/packages/core/src/modules/chart/src/dates.ts
@@ -1,7 +1,6 @@
-// @openpanel/db's clickhouse/client.ts owns these, but importing it constructs
-// a client and a pino logger at import time (see insight.service.ts's header),
-// and the fragment builders here must stay pure and synchronous. Same choice
-// cohort.service.ts made for its table map.
+// Importing @openpanel/db's clickhouse/client constructs a client and a pino
+// logger at import time, and the fragment builders here must stay pure and
+// synchronous.
const CLICKHOUSE_DATE_SUFFIX = /(\.\d{3})?Z+$/;
diff --git a/packages/core/src/modules/chart/src/engine/format.ts b/packages/core/src/modules/chart/src/engine/format.ts
index 50d8409e9..697dd0025 100644
--- a/packages/core/src/modules/chart/src/engine/format.ts
+++ b/packages/core/src/modules/chart/src/engine/format.ts
@@ -88,7 +88,6 @@ export function format(
return {
id: slug(cs.id),
names: displayName,
- // TODO: Do we need this now?
event: {
id: definition?.id,
name: eventName,
diff --git a/packages/core/src/modules/chart/src/field-resolution.test.ts b/packages/core/src/modules/chart/src/field-resolution.test.ts
index de44652a1..3e13dc431 100644
--- a/packages/core/src/modules/chart/src/field-resolution.test.ts
+++ b/packages/core/src/modules/chart/src/field-resolution.test.ts
@@ -1,16 +1,9 @@
/**
- * Unit tests for the event-field resolution helpers.
- *
- * Background — these helpers exist because the chart, funnel, and conversion
- * services used to inline whatever field name the dashboard sent (saved report,
- * autocomplete picker, raw API call) directly into SQL. That worked for the
- * common case but produced UNKNOWN_IDENTIFIER errors when:
- *
- * Older clients sent camelCase names (`referrerName`) that don't match the
- * snake_case ClickHouse schema; - users picked utm_* in the property filter UI
- * without realising those live in the `properties` map on the events table; - a
- * saved report referenced a custom property (`temple_name`) as if it were a
- * top-level column.
+ * Unit tests for the event-field resolution helpers. Field names from the
+ * dashboard (saved report, autocomplete picker, raw API call) must not reach SQL
+ * verbatim: older clients send camelCase names (`referrerName`), users pick
+ * utm_* without realising they live in the `properties` map, and saved reports
+ * reference custom properties (`temple_name`) as if they were top-level columns.
*/
import { describe, expect, it } from 'bun:test';
import { isKnownEventField, normalizeEventField } from './field-resolution';
diff --git a/packages/core/src/modules/chart/src/field-resolution.ts b/packages/core/src/modules/chart/src/field-resolution.ts
index 787772595..ccc2322fa 100644
--- a/packages/core/src/modules/chart/src/field-resolution.ts
+++ b/packages/core/src/modules/chart/src/field-resolution.ts
@@ -2,17 +2,10 @@
// Chart field resolution: how a report's field name (`referrerName`,
// `utm_source`, `properties.x.*`, `profile.email`, `group.name`,
// `cohort:<id>`, `has_profile`) becomes a ClickHouse expression, plus the
-// profile-CTE narrowing helpers.
-//
-// Each resolver returns a `SqlFragment` whose values — cohort ids, cohort
-// labels, project ids and the `properties[...]` map keys — bind as
-// `{pN:Type}` params, and whose identifiers go through `sql.id`.
-//
-// The one piece of text this file still builds is the backtick-quoted CTE alias
-// `` `profile.properties.<key>` `` (see `profilePropertiesCteSelect`), which is
-// an identifier `sql.id` cannot express — three dot-separated parts — and which
-// `collectProfilePropertyKeys` already guards. It goes through `compiled.ts`,
-// the module's one text seam.
+// profile-CTE narrowing helpers. Values bind as params and identifiers go
+// through `sql.id`; the one exception is the backtick-quoted CTE alias
+// `` `profile.properties.<key>` `` (see `profilePropertiesCteSelect`), which
+// `sql.id` cannot express and which goes through `compiled.ts`.
import { type SqlFragment, sql } from '@openpanel/db/src/clickhouse/sql';
import type { IChartBreakdown } from '../../report/report.constants';
@@ -31,11 +24,8 @@ export const CHART_TABLE = {
cohortEventsMv: 'cohort_events_mv',
} as const;
-// Top-level columns on the events table. Derived from the migration in
-// @openpanel/db's code-migrations/3-init-ch.ts (+ revenue added in
-// 6-add-revenue-column.ts).
-// Used to distinguish real columns from property keys and reject unknown
-// identifiers before they reach ClickHouse.
+// Top-level columns on the events table, used to tell real columns from property
+// keys and to reject unknown identifiers before they reach ClickHouse.
export const EVENT_TOP_LEVEL_COLUMNS = new Set<string>([
'id',
'name',
@@ -257,8 +247,7 @@ export function cohortBreakdownLabelExpr(
/**
* The membership subselect both filter compilers use for `inCohort` /
- * `notInCohort`. Uses a plain `IN (subquery)`, not `GLOBAL IN` — preserved
- * deliberately as an accepted trade-off, not an oversight.
+ * `notInCohort`. A plain `IN (subquery)`, not `GLOBAL IN`: an accepted trade-off.
*/
export function buildCohortMembersSubselect(
cohortIds: string[],
@@ -395,8 +384,7 @@ function matchPropertyMapPrefix(property: string): string | undefined {
* scalar, decided by testing the rendered TEXT for a `%`. That's reachable
* two ways: the wildcard branch emits `transformPropertyKey`'s pattern, and a
* NON-wildcard key containing a literal `%` also matches — a known defect
- * (`properties.a%b` is treated as an array and fails at ClickHouse), kept for
- * behavioral parity.
+ * (`properties.a%b` is treated as an array and fails at ClickHouse).
*/
export function isWildcardPropertyKey(rawProperty: string): boolean {
const property = normalizeEventField(rawProperty);
@@ -423,12 +411,9 @@ export function getSelectPropertyKey(
*/
eventsAlias?: string
): SqlFragment {
- // Map camelCase aliases (`referrerName` → `referrer_name`) and bare UTM
- // names (`utm_source` → `properties.__query.utm_source`) into their
- // canonical form before doing any pattern matching. The fallback at the
- // bottom of this function returns `property` verbatim, so without this
- // normalization an alias would leak into the generated SQL and fail with
- // UNKNOWN_IDENTIFIER.
+ // Normalize aliases (`referrerName`, bare `utm_source`) first: the fallback at
+ // the bottom returns `property` verbatim, so an alias would leak into the SQL
+ // and fail with UNKNOWN_IDENTIFIER.
const property = normalizeEventField(rawProperty);
const extractedCohortId = cohortId || extractCohortId(property);
@@ -440,7 +425,7 @@ export function getSelectPropertyKey(
return sql`if(profile_id != device_id, 'true', 'false')`;
}
- // Handle group properties — requires ARRAY JOIN + _g JOIN to be present in query
+ // Group properties need the ARRAY JOIN + _g JOIN to be present in the query.
if (property.startsWith('group.') && projectId) {
return getGroupPropertySql(property);
}
@@ -468,7 +453,6 @@ export function getSelectPropertyKey(
return sql`${map}[${sql.string(property.replace(new RegExp(`^${match}.`), ''))}]`;
}
-// --- profile-property CTE narrowing (perf) ---------------------------------
// profile.properties.<key> refs render as Map lookups `profile.properties['<key>']`.
// Pulling the whole `properties` Map into the profile CTE makes the LEFT ANY
// JOIN hash carry the full Map per profile — roughly a kilobyte each on real
diff --git a/packages/core/src/modules/chart/src/field-values.sql.ts b/packages/core/src/modules/chart/src/field-values.sql.ts
index 2a46eca5b..263e8ea7f 100644
--- a/packages/core/src/modules/chart/src/field-values.sql.ts
+++ b/packages/core/src/modules/chart/src/field-values.sql.ts
@@ -28,7 +28,7 @@ export const SESSION_LEVEL_VALUE_COLUMNS: ReadonlySet<string> = new Set([
/**
* A value picker only needs recently-seen values, so the `events` fallback is
- * clamped to this window instead of the old 6-month full scan.
+ * clamped to this window.
*/
export const EVENT_FIELD_VALUES_LOOKBACK_DAYS = 30;
diff --git a/packages/core/src/modules/chart/src/filter-cast.ts b/packages/core/src/modules/chart/src/filter-cast.ts
index 4575dab19..b234f7385 100644
--- a/packages/core/src/modules/chart/src/filter-cast.ts
+++ b/packages/core/src/modules/chart/src/filter-cast.ts
@@ -1,9 +1,6 @@
/** biome-ignore-all lint/style/useDefaultSwitchClause: switch cases are exhaustive by design */
// The typed-cast half of `buildFilterWhere` (./table-filter-where.ts) and of
// `./filter-where.ts` (the event-property compiler).
-//
-// Both filter compilers call the same pair: `castFragment` and
-// `buildTypedClauseFragment` bind every value as a `{name:Type}` param.
import {
type SqlFragment,
diff --git a/packages/core/src/modules/chart/src/filter-where.ts b/packages/core/src/modules/chart/src/filter-where.ts
index 9b32df668..7cd0cd630 100644
--- a/packages/core/src/modules/chart/src/filter-where.ts
+++ b/packages/core/src/modules/chart/src/filter-where.ts
@@ -157,7 +157,6 @@ export function getEventFiltersWhereClause(
return;
}
- // Handle group. prefixed filters (requires ARRAY JOIN + _g JOIN in query)
if (name.startsWith('group.') && projectId) {
const whereFrom = getGroupPropertySql(name);
if (hasTypedCast(filter.type) && isTypedOperator(operator)) {
diff --git a/packages/core/src/modules/chart/src/funnel.sql.test.ts b/packages/core/src/modules/chart/src/funnel.sql.test.ts
index 0ac5fa3a1..9d7d8060d 100644
--- a/packages/core/src/modules/chart/src/funnel.sql.test.ts
+++ b/packages/core/src/modules/chart/src/funnel.sql.test.ts
@@ -1,18 +1,13 @@
/**
- * SQL-shape tests for the funnel statements.
+ * SQL-shape tests for the funnel statements: render, then `EXPLAIN <sql>` with the
+ * bound params against the isolated `openpanel_test` ClickHouse (pinned by
+ * test/preload.ts). EXPLAIN resolves columns without executing, so a breakdown
+ * expression referencing a join alias that was never added fails here as
+ * UNKNOWN_IDENTIFIER.
*
- * Strategy, same as sql.test.ts: render the statement, then run `EXPLAIN <sql>`
- * with its bound params against the isolated `openpanel_test` ClickHouse
- * (pinned by test/preload.ts). EXPLAIN parses and resolves columns without
- * executing, so a breakdown expression referencing a join alias that was never
- * added fails here as UNKNOWN_IDENTIFIER — which is exactly the class of bug
- * that made funnel "View Users" return "No users found" for profile-property
- * and cohort breakdowns.
- *
- * Text assertions look at the rendered `query` with placeholder NUMBERS
- * normalised away: a fragment reused in two clauses renders `{p3:String}` in
- * one and `{p11:String}` in the other, and the property under test is that the
- * same condition appears twice, not what its slots were numbered.
+ * Text assertions normalise placeholder NUMBERS away: a fragment reused in two
+ * clauses renders `{p3:String}` in one and `{p11:String}` in the other, and the
+ * property under test is that the same condition appears twice.
*
* The statements' only Postgres read (the breakdown's cohort names) is stubbed,
* so this file is independent of whichever partial `prisma-client` mock a
@@ -236,7 +231,7 @@ describe('funnel.sql / buildFunnelBase — cohort breakdowns', () => {
const alias = `cohort_${COHORT_ID.replace(/-/g, '_')}`;
expect(sql).toContain(`${alias}.profile_id`);
expect(sql).toContain(`AS ${alias}`);
- // The cohort label is a bound value now.
+ // The cohort label is a bound value.
expect(text).toContain('Power users');
await explain(statement);
});
diff --git a/packages/core/src/modules/chart/src/funnel.sql.ts b/packages/core/src/modules/chart/src/funnel.sql.ts
index 4dbaad0ee..dc22460ea 100644
--- a/packages/core/src/modules/chart/src/funnel.sql.ts
+++ b/packages/core/src/modules/chart/src/funnel.sql.ts
@@ -2,13 +2,8 @@
//
// The chart and the profile list are built from ONE base: a breakdown
// expression only works if the join it references was added, and the joins
-// depend on the breakdowns, so building the selects in one place and the joins
-// in another is exactly the bug that made funnel "View Users" return "No users
-// found" for profile-property and cohort breakdowns.
-//
-// The field resolver and filter compiler render text (see compiled.ts); their
-// output — filter clauses, breakdown expressions, the profile CTE columns and
-// the cohort joins — is spliced, everything else is bound.
+// depend on the breakdowns, so building them apart made funnel "View Users"
+// return no users for profile-property and cohort breakdowns.
//
// Cluster note: `events`, `profiles`, `groups` and `cohort_members` are
// Distributed on Cloud. Every join runs under the client's
@@ -218,10 +213,8 @@ function profileCteColumns(
const fields = new Set(profileJoinFields(profileFilters, profileBreakdowns));
- // Both the breakdown- and the filter-derived names are vetted above; the
- // allowlist here is the backstop that makes `sql.id` throw rather than
- // inline if either loop ever lets something else through (main #512,
- // GHSA-pc3q-gw7f-p2x2).
+ // The allowlist is the backstop that makes `sql.id` throw rather than inline
+ // if either loop above lets something else through (GHSA-pc3q-gw7f-p2x2).
const columns: SqlFragment[] = Array.from(fields).map((field) =>
sql.id(field, FUNNEL_JOIN_COLUMNS)
);
@@ -405,10 +398,6 @@ export function funnelProfilesQuery(
return sql`${base.ctes}SELECT DISTINCT profile_id FROM funnel WHERE ${sql.join(where, ' AND ')} LIMIT ${sql.uint64(input.limit)}`;
}
-/**
- * No in-repo caller. Reachable only through `packages/core/src/index.ts`,
- * which still publishes the wrapper.
- */
export function funnelSessionsQuery(input: {
projectId: string;
startDate: string;
diff --git a/packages/core/src/modules/chart/src/profile-columns.test.ts b/packages/core/src/modules/chart/src/profile-columns.test.ts
index 303e22aaa..65a47c020 100644
--- a/packages/core/src/modules/chart/src/profile-columns.test.ts
+++ b/packages/core/src/modules/chart/src/profile-columns.test.ts
@@ -1,8 +1,6 @@
-// The JOIN column list used to be built by splitting raw `profile.*` filter
-// names, so a crafted name could close the subquery (GHSA-pc3q-gw7f-p2x2). The
-// rewrite binds every identifier through `sql.id`, so injection is impossible;
-// what these assert is that the list is also an allowlist, so a filter cannot
-// name a column it has no business reading.
+// Every identifier binds through `sql.id`, so injection is impossible
+// (GHSA-pc3q-gw7f-p2x2); these assert the list is also an allowlist, so a filter
+// cannot name a column it has no business reading.
import { describe, expect, it } from 'bun:test';
import { JOINABLE_PROFILE_COLUMNS } from '../chart.constants';
import { profileJoinFields } from './funnel.sql';
diff --git a/packages/core/src/modules/chart/src/retention.sql.test.ts b/packages/core/src/modules/chart/src/retention.sql.test.ts
index 320e6a579..51e52ed20 100644
--- a/packages/core/src/modules/chart/src/retention.sql.test.ts
+++ b/packages/core/src/modules/chart/src/retention.sql.test.ts
@@ -25,8 +25,7 @@ beforeAll(async () => {
describe('rollingActiveUsersQuery', () => {
// `date + n` smears every active day forward, so without an upper bound the
- // series runs up to `days - 1` days past today. On the seeded acme-web at
- // days=7 that drew two points into the future (ISSUES.md H8c).
+ // series runs up to `days - 1` days past today.
it('never projects past today', () => {
for (const days of [1, 7, 30]) {
const { query } = RET.rollingActiveUsersQuery(
diff --git a/packages/core/src/modules/chart/src/retention.sql.ts b/packages/core/src/modules/chart/src/retention.sql.ts
index c14c8610c..06b1bfb4d 100644
--- a/packages/core/src/modules/chart/src/retention.sql.ts
+++ b/packages/core/src/modules/chart/src/retention.sql.ts
@@ -1,8 +1,5 @@
// Every ClickHouse query the retention service runs, as pure `sql` fragments.
//
-// The filter compiler renders text (see compiled.ts); its output is the only
-// thing spliced into the cohort statement.
-//
// Cluster note: `events`, `dau_mv` and `cohort_events_mv` are Distributed on
// Cloud. None of these statements contains an `IN (subquery)` — the cohort
// matrix is a CTE self-join — so no `IN` / `GLOBAL IN` decision arises.
@@ -50,7 +47,6 @@ export interface RetentionSeriesQueryInput {
endDate?: string;
}
-/** Same spelling as `overview/src/pages.sql.ts` and `profile/src/sql.ts`. */
function createdAtWithin({
startDate,
endDate,
diff --git a/packages/core/src/modules/chart/src/run-query.ts b/packages/core/src/modules/chart/src/run-query.ts
index c4b6408a3..479489526 100644
--- a/packages/core/src/modules/chart/src/run-query.ts
+++ b/packages/core/src/modules/chart/src/run-query.ts
@@ -1,16 +1,7 @@
-// The chart module's one ClickHouse entry point.
-//
-// The client is `deps.ch` — the same round-robin/retry proxy `main.ts` builds
-// and hands to every scope — instead of a lazy
-// `import('@openpanel/db/src/clickhouse/client')`. Reaching it through the
-// scope is what puts the request's `requestId` on the query's log line; the
-// lazy loader had no request scope to carry.
-//
-// `deps.ch.query` is `chQuery`'s own transport (`withRetry(client =>
-// client.query(...))`), and the Int-meta coercion below is the same one
-// `chQuery` applies — ClickHouse's JSON format returns every Int*/UInt* column
-// as a string, so a caller's `count: number` only holds if it is parsed back
-// out here. Same shape as the overview module's run-query.ts.
+// The chart module's one ClickHouse entry point. The client is `deps.ch`, which
+// puts the request's `requestId` on the query's log line. ClickHouse's JSON
+// format returns every Int*/UInt* column as a string, so a caller's
+// `count: number` only holds if it is parsed back out here, as `chQuery` does.
import {
type SqlFragment,
diff --git a/packages/core/src/modules/chart/src/sankey.sql.ts b/packages/core/src/modules/chart/src/sankey.sql.ts
index adf40efb6..c4f85e48b 100644
--- a/packages/core/src/modules/chart/src/sankey.sql.ts
+++ b/packages/core/src/modules/chart/src/sankey.sql.ts
@@ -1,9 +1,5 @@
// Every ClickHouse statement the sankey (user-flow) service runs, as pure `sql`
// fragments.
-//
-// The filter compiler renders text (see compiled.ts); only its output is
-// spliced.
-//
// Cluster note: `events` is Distributed on Cloud, and the `session_id IN
// (SELECT session_id FROM start_event_sessions)` HAVING clauses are plain `IN
// (subquery)` over a CTE. Whether these should be `GLOBAL IN` on a cluster is
@@ -23,8 +19,7 @@ export const TOP_ENTRY_EVENTS = 3;
* lambda indexes its own captured `groupArray` is Theta(n^2) in the longest
* session's event count — ClickHouse re-materialises the array once per
* element, which is fatal for a tenant with one very long session.
- * `arrayCompact` is the built-in for exactly this and is linear; the result
- * is byte-identical.
+ * `arrayCompact` is the built-in for exactly this and is linear.
*/
const DEDUPE_CONSECUTIVE = compiledText(
'arrayCompact(groupArray(event_name)) as events_deduped'
diff --git a/packages/core/src/modules/chart/src/sql.test.ts b/packages/core/src/modules/chart/src/sql.test.ts
index e47f58514..ec1255599 100644
--- a/packages/core/src/modules/chart/src/sql.test.ts
+++ b/packages/core/src/modules/chart/src/sql.test.ts
@@ -1,23 +1,16 @@
/**
- * SQL-shape tests for the chart statements.
- *
- * Strategy: render the statement, then run `EXPLAIN <sql>` with its bound
- * params against the isolated `openpanel_test` ClickHouse (pinned by
- * test/preload.ts). EXPLAIN parses, resolves columns and builds the plan
- * without executing, so UNKNOWN_IDENTIFIER / AMBIGUOUS_IDENTIFIER / bad JOIN ON
+ * SQL-shape tests for the chart statements: render, then `EXPLAIN <sql>` with the
+ * bound params against the isolated `openpanel_test` ClickHouse (pinned by
+ * test/preload.ts). EXPLAIN resolves columns and builds the plan without
+ * executing, so UNKNOWN_IDENTIFIER / AMBIGUOUS_IDENTIFIER / bad JOIN ON
* expressions surface without seeded data. WITH FILL TO < FROM is a runtime
* check, so it is covered by a plain string assertion instead.
*
- * Text assertions look at the rendered `query`; chart-level values are
- * `{pN:Type}` placeholders there, filter/breakdown expressions are inlined
- * verbatim (see field-resolution.ts's header).
- *
* The statements' only Postgres read (the project's cohorts, for the
- * all-cohorts breakdown) is stubbed to "no cohorts": that is the precondition
- * the all-cohorts cases assert on, and it keeps this file independent of
- * whichever partial `prisma-client` mock a module-root service test left behind
- * (bun runs every file in one shared registry without `--isolate`). The real
- * Postgres cohort path runs in../chart.service.test.ts.
+ * all-cohorts breakdown) is stubbed to "no cohorts", which is the precondition
+ * the all-cohorts cases assert on, and keeps this file independent of
+ * whichever partial `prisma-client` mock a module-root service test left behind.
+ * The real Postgres cohort path runs in ../chart.service.test.ts.
*/
import { afterAll, beforeAll, describe, expect, it, mock } from 'bun:test';
@@ -29,8 +22,8 @@ import type {
import type { AggregateChartSqlInput, ChartSqlInput } from './statement';
const cohortFindMany = mock(async () => []);
-// Plain-object snapshot, not the live binding — restoring from the namespace
-// import in afterAll would just re-apply the mock (see cohort.service.test.ts).
+// Plain-object snapshot, not the live binding: restoring from the namespace
+// import in afterAll would just re-apply the mock.
const actualPrismaClient = await import('@openpanel/db/src/prisma-client');
const realPrismaClient = { ...actualPrismaClient };
mock.module('@openpanel/db/src/prisma-client', () => ({
@@ -320,9 +313,8 @@ describe('getChartSql', () => {
);
});
- // main a2cbf2e8: the subquery was aliased `subQuery`, so the `e` every
- // breakdown SELECT is built with was out of scope and the chart failed
- // with UNKNOWN_IDENTIFIER on `e.properties`.
+ // The subquery must not be aliased `subQuery`: the `e` every breakdown SELECT
+ // is built with would be out of scope (UNKNOWN_IDENTIFIER on `e.properties`).
it('one_event_per_user + a property breakdown resolves the events alias', async () => {
const rendered = await getChartSql({
event: event({ segment: 'one_event_per_user' }),
@@ -335,8 +327,7 @@ describe('getChartSql', () => {
});
// Saved reports / older clients can send field names that don't match the
- // events schema; the chart service used to inline them verbatim, crashing
- // parse.
+ // events schema; they must not be inlined verbatim.
it('normalizes camelCase filter alias (referrerName → referrer_name)', async () => {
const rendered = await getChartSql({
event: event({
@@ -365,9 +356,9 @@ describe('getChartSql', () => {
});
it('drops unknown breakdown rather than emitting invalid identifier', async () => {
- // `temple_name` is a custom property (lives in properties map) but got
- // saved as a top-level breakdown. The old code path emitted
- // `SELECT temple_name as _uc_label_1 FROM events`, failing parse.
+ // `temple_name` is a custom property (lives in properties map) saved as a
+ // top-level breakdown; it must not be emitted as
+ // `SELECT temple_name as _uc_label_1 FROM events`.
const rendered = await getChartSql({
event: event(),
breakdowns: [breakdown('temple_name')],
diff --git a/packages/core/src/modules/chart/src/sql.ts b/packages/core/src/modules/chart/src/sql.ts
index 9d75b47c5..b9f872266 100644
--- a/packages/core/src/modules/chart/src/sql.ts
+++ b/packages/core/src/modules/chart/src/sql.ts
@@ -116,8 +116,6 @@ function assertCohortId(cohortId: string): string {
return cohortId;
}
-// --- breakdown resolution ---------------------------------------------------
-
/**
* Drop breakdowns whose field name doesn't resolve to a known events column,
* properties path, profile path, group path, or cohort — saved reports with
@@ -153,8 +151,6 @@ export function withoutEmptyAllCohortsBreakdown(
);
}
-// --- clause rendering -------------------------------------------------------
-
function clauses(parts: Parts): SqlFragment[] {
return Object.values(parts);
}
@@ -202,8 +198,6 @@ function orderByClause(orderBy: Parts): SqlFragment {
return sql`ORDER BY ${sql.join(keys, ', ')}`;
}
-// --- shared chart body ------------------------------------------------------
-
function profileCteSelectField(
field: string,
profileKeys: string[],
@@ -423,11 +417,9 @@ function countExpression(
return sql`count(*) as count`;
}
-// Aliased `e`, not `subQuery`: breakdown SELECT expressions are always built
-// with the events alias, so `e.properties` was out of scope in the outer
-// query and any chart combining this segment with a property breakdown failed
-// with UNKNOWN_IDENTIFIER. The subquery already does `SELECT * FROM events e`,
-// so every `e.`-qualified reference outside it resolves unchanged.
+// Aliased `e`, not `subQuery`: breakdown SELECT expressions are built with the
+// events alias, so `e.properties` must be in scope in the outer query. The
+// subquery does `SELECT * FROM events e`, so `e.`-qualified references resolve.
function oneEventPerUserFrom(body: ChartBody): SqlFragment {
return sql`(
SELECT DISTINCT ON (profile_id) * from ${sql.id(CHART_TABLE.events)} e ${joinsClause(body.joins)} WHERE ${sql.join(clauses(body.where), ' AND ')}
@@ -448,8 +440,6 @@ function dateRangeWhere(
}
}
-// --- series chart -----------------------------------------------------------
-
function intervalBucket(
interval: IInterval,
timezone: string
@@ -538,8 +528,6 @@ export function chartSeriesQuery(input: ChartSeriesQueryInput): SqlFragment {
return sql`${withClause(body.ctes)}SELECT * EXCEPT (_uc_state), ${totalCount} FROM (${selectClause(body.select)} FROM ${sql.id(CHART_TABLE.events)} e ${joinsClause(body.joins)} ${whereClause(body.where)} ${groupByClause(groupBy)}) ${orderByClause(orderBy)} ${fill}`;
}
-// --- aggregate chart --------------------------------------------------------
-
/** One row per breakdown combination over the whole range. */
export function aggregateChartQuery(
input: AggregateChartQueryInput
@@ -573,13 +561,10 @@ export function aggregateChartQuery(
return sql`${head} FROM ${sql.id(CHART_TABLE.events)} e ${joinsClause(body.joins)} ${whereClause(body.where)} ${groupByClause(groupBy)} ${orderByClause(orderBy)} ${limitClause}`;
}
-// --- router queries ---------------------------------------------------------
-
/**
* projectCard: three months of daily unique visitors + revenue, filled.
* The timezone travels as `clickhouse_settings.session_timezone` (the caller
- * passes it) rather than an inline `SETTINGS` clause — same setting, same
- * query scope, and a bindable position.
+ * passes it) rather than an inline `SETTINGS` clause.
*/
export function projectCardChartQuery(projectId: string): SqlFragment {
return sql`SELECT
@@ -647,8 +632,6 @@ export function groupPropertyValuesQuery(
return sql`SELECT distinct ${expression} as values FROM ${sql.id(CHART_TABLE.groups)} FINAL WHERE project_id = ${sql.string(projectId)} AND deleted = 0 AND ${expression} != '' AND ${expression} IS NOT NULL ORDER BY created_at DESC LIMIT ${sql.uint64(PROPERTY_VALUES_LIMIT)}`;
}
-// --- getProfiles ------------------------------------------------------------
-
export interface ChartBucketProfilesInput {
projectId: string;
/** Bucket start as a `YYYY-MM-DD HH:mm:ss` string. */
diff --git a/packages/core/src/modules/chart/src/table-filter-where.test.ts b/packages/core/src/modules/chart/src/table-filter-where.test.ts
index 432c904bb..84369aa57 100644
--- a/packages/core/src/modules/chart/src/table-filter-where.test.ts
+++ b/packages/core/src/modules/chart/src/table-filter-where.test.ts
@@ -1,16 +1,10 @@
/**
- * Unit tests for `buildFilterWhere`.
- *
- * No ClickHouse needed. The compiler returns bound `SqlFragment`s, so each
- * clause is rendered through `toStatement` and its params substituted back
- * in: what the assertions say is unchanged, a value that used to appear as
- * an escaped literal now appears in `query_params`. The
- * interesting part is the `profile.*` branch: the field name arrives from the
- * caller (saved report, URL state, raw API call) and used to be concatenated
- * into the SQL text as-is, so a name that was not a column produced whatever
- * SQL the caller wrote. It now resolves against the known profiles columns and
- * the filter is dropped when it does not, which is what the `group.*` and
- * `session.*` branches already did.
+ * Unit tests for `buildFilterWhere`. No ClickHouse needed: the compiler returns
+ * bound `SqlFragment`s, so each clause is rendered through `toStatement` and its
+ * params substituted back in. The interesting part is the `profile.*` branch:
+ * the field name arrives from the caller (saved report, URL state, raw API
+ * call), so it resolves against the known profiles columns and the filter is
+ * dropped when it does not, as the `group.*` and `session.*` branches do.
*/
import { describe, expect, it } from 'bun:test';
diff --git a/packages/core/src/modules/chart/src/table-filter-where.ts b/packages/core/src/modules/chart/src/table-filter-where.ts
index 45c8b022b..2d37bc6b4 100644
--- a/packages/core/src/modules/chart/src/table-filter-where.ts
+++ b/packages/core/src/modules/chart/src/table-filter-where.ts
@@ -104,7 +104,7 @@ function integerLiteralDigits(value: number): bigint | null {
* have had. Declaring `Float64` for every number would bind a *different constant*
* — visible wherever the comparand's type takes part in type resolution, and
* in the `NO_COMMON_TYPE` message when a numeric filter lands on a String
- * property (the proof's two error cases).
+ * property.
*
* `filter-cast.ts`'s twin needs none of this: there every value is wrapped in
* `toFloat64OrNull(toString(…))` or a sibling cast, which erases the declared
@@ -122,7 +122,7 @@ function numericLiteralType(value: number): string {
}
/**
- * Bind one filter value the way `sqlstring.escape` rendered it: a string is a
+ * Bind one filter value: a string is a
* trimmed quoted literal, a number a numeric literal of the type ClickHouse
* would have inferred for it, a boolean `true`/`false` and `null` the SQL
* keyword NULL — which only `Nullable` reproduces, since a `String` param
diff --git a/packages/core/src/modules/cohort/cohort.jobs.test.ts b/packages/core/src/modules/cohort/cohort.jobs.test.ts
index 7f53d2840..54a0fae8e 100644
--- a/packages/core/src/modules/cohort/cohort.jobs.test.ts
+++ b/packages/core/src/modules/cohort/cohort.jobs.test.ts
@@ -1,5 +1,4 @@
-// Job/cron-fragment wiring — no ClickHouse/Postgres touched. The service
-// methods themselves are exercised in cohort.service.test.ts.
+// Job/cron-fragment wiring; no ClickHouse/Postgres touched.
import { expect, test } from 'bun:test';
import { testCoreConfig } from '../../../test/config-fixture';
@@ -131,9 +130,7 @@ test('cohortRefresh fans out one enqueueCompute per non-static cohort', async ()
expect(calls).toEqual(['c1', 'c2']);
});
-// Byte-identity with the id/cadence schedulers.test.ts's golden snapshot
-// pins. The cadence now lives on the job itself, so this reads it straight
-// off the registry.
+// Reads the cadence straight off the registry.
test('the cohort cron fragment keeps its scheduler id and cadence', () => {
expect(queues.cron.jobs.cohortRefresh.cron).toEqual({
pattern: '*/30 * * * *',
diff --git a/packages/core/src/modules/cohort/cohort.jobs.ts b/packages/core/src/modules/cohort/cohort.jobs.ts
index 4782cfad7..4d63e6e5a 100644
--- a/packages/core/src/modules/cohort/cohort.jobs.ts
+++ b/packages/core/src/modules/cohort/cohort.jobs.ts
@@ -1,7 +1,5 @@
-// `cohortCompute` is this module's own queue — registry key `cohortCompute`
-// EXACTLY (no env rename, so COHORTCOMPUTE_CONCURRENCY keeps working).
-// `cohortRefresh` is a cron fragment, spread into the ONE `cron` queue by
-// jobs.registry.ts.
+// `cohortCompute` is this module's own queue; `cohortRefresh` is a cron fragment
+// spread into the shared `cron` queue by jobs.registry.ts.
import { z } from 'zod';
import { defineJob } from '../../jobs/define';
diff --git a/packages/core/src/modules/cohort/cohort.rpc.test.ts b/packages/core/src/modules/cohort/cohort.rpc.test.ts
index 2123f7201..7b134c962 100644
--- a/packages/core/src/modules/cohort/cohort.rpc.test.ts
+++ b/packages/core/src/modules/cohort/cohort.rpc.test.ts
@@ -1,8 +1,4 @@
-// Only the "is anyone logged in" boundary is exercised here — no database.
-// The access-check + business logic ride on @openpanel/db (lazy-loaded, see
-// cohort.service.ts's header) and ctx.services.cohort; wiring this router
-// end-to-end against a real Postgres is P6's (protectedProcedure) job, not
-// this one's — see cohort.rpc.ts's header.
+// Only the "is anyone logged in" boundary is exercised here, with no database.
import { expect, mock, test } from 'bun:test';
import { stubHttpCtx } from '../../../test/rpc-fixtures';
@@ -19,7 +15,6 @@ const COOKIE_OPTIONS: CookieOptions = {
path: '/',
};
-// EMPTY_SESSION's shape (packages/db/src/services/auth-session.service.ts) —
// `TrpcContext.session` is never literally `null`, only its `userId` is.
const EMPTY_SESSION = { session: null, user: null, userId: null };
diff --git a/packages/core/src/modules/cohort/cohort.rpc.ts b/packages/core/src/modules/cohort/cohort.rpc.ts
index 3a40b272c..b54f468dc 100644
--- a/packages/core/src/modules/cohort/cohort.rpc.ts
+++ b/packages/core/src/modules/cohort/cohort.rpc.ts
@@ -1,14 +1,6 @@
-// `protectedProcedure` runs `enforceUserIsAuthed` + `enforceAccess` BEFORE the
-// input parser. The explicit checks in the handlers below stay: `enforceAccess`
-// only sees a TOP-LEVEL `projectId` / `organizationId`, so anything resolved
-// from another id needs its own.
-//
-// The per-project access ladder itself is bound once, in auth.service.ts; every
-// procedure here reaches it through `ctx.services.auth`.
-//
-// CRUD (list/get/create/update/delete) reads/writes Prisma's `cohort` table
-// directly — cohort.service.ts owns only the compute-heavy and
-// ClickHouse-touching operations.
+// `protectedProcedure` runs `enforceAccess` BEFORE the input parser, and it only
+// sees a TOP-LEVEL `projectId` / `organizationId`, so handlers that resolve a
+// project from another id keep their own explicit check.
import { z } from 'zod';
import { createTRPCRouter, protectedProcedure } from '../../rpc/base';
@@ -37,9 +29,8 @@ const EXPORT_PROFILES_MAX_LIMIT = 10_000;
const EXPORT_PROFILES_DEFAULT_LIMIT = 10_000;
/**
- * `mostEvents` and `popularRoutes` read the project's whole history before this
- * window existed. The default is the one `profile.powerUsers` got for the same
- * cause; the cohort page passes no range, so this is what it shows.
+ * Bounds the event scan behind `mostEvents` and `popularRoutes`. The cohort page
+ * passes no range, so this is what it shows.
*/
const COHORT_ACTIVITY_DEFAULT_RANGE = '3m';
diff --git a/packages/core/src/modules/cohort/cohort.service.test.ts b/packages/core/src/modules/cohort/cohort.service.test.ts
index c450e676b..ebffdc22a 100644
--- a/packages/core/src/modules/cohort/cohort.service.test.ts
+++ b/packages/core/src/modules/cohort/cohort.service.test.ts
@@ -1,11 +1,5 @@
-// Every function under test takes `ServiceDeps`, so `deps.db` and `deps.ch`
-// ARE the fakes below. The one module still mocked is core's own
-// `ch-query`, which is what the subject calls.
-//
-// The pure SQL-shape builders (buildEventCriteriaQuery,
-// buildPropertyBasedCohortQuery, deriveCohortQuerySettings) have their own
-// file, src/cohort-sql.test.ts, and need none of this — they take no ClickHouse
-// connection.
+// Every function under test takes `ServiceDeps`, so `deps.db` and `deps.ch` are
+// the fakes below. The one module still mocked is core's own `ch-query`.
import { afterAll, beforeAll, expect, mock, test } from 'bun:test';
import { testCoreConfig } from '../../../test/config-fixture';
@@ -56,12 +50,9 @@ const chQuery = mock(async () => [] as unknown[]);
const chInsert = mock(async (_args: { table: string }) => undefined);
const chCommand = mock(async (_args: { query: string }) => undefined);
-// The subject reaches ClickHouse through core's own `chQuery(deps, ...)`
-// (ch-query.ts) — mocking that one module keeps this file off
-// @openpanel/db's live-bound client singleton entirely. `mock.module` still
-// has no per-file scope under bare `bun test` (AGENTS.md), so snapshot the
-// real module into a plain object FIRST and restore it in afterAll —
-// restoring via the live import binding would just re-apply the mock.
+// `mock.module` has no per-file scope under bare `bun test`, so snapshot the
+// real module into a plain object first and restore it in afterAll; restoring
+// via the live import binding would re-apply the mock.
const realChQuery = { ...(await import('../../ch-query')) };
mock.module('../../ch-query', () => ({
...realChQuery,
@@ -215,8 +206,7 @@ test('createCohortService() delegates updateMembership and listRefreshableCohort
seedCohort({ id: 'c2', isStatic: true });
const service = subject.createCohortService(deps, testServices());
- // The container's members are closures over `deps`, not the bare functions,
- // so identity is no longer the observable — delegation is.
+ // The container's members are closures over `deps`; delegation is observable.
expect((await service.listRefreshableCohortIds()).sort()).toEqual(['c1']);
chCommand.mockClear();
diff --git a/packages/core/src/modules/cohort/cohort.service.ts b/packages/core/src/modules/cohort/cohort.service.ts
index ff1b13903..7e42655e4 100644
--- a/packages/core/src/modules/cohort/cohort.service.ts
+++ b/packages/core/src/modules/cohort/cohort.service.ts
@@ -1,10 +1,8 @@
// Cohort definitions, membership computation and the compute enqueue.
//
-// Table names are a local literal map (`TABLE`, below) rather than
-// @openpanel/db's `TABLE_NAMES`: that lives in the same module as `ch` /
-// `chQuery` (clickhouse/client.ts), which constructs a real pino logger at
-// import time, and this module's SQL builders are pure sync functions the
-// shape tests call with no ClickHouse connection at all.
+// Table names are a local literal map (`TABLE`) rather than @openpanel/db's
+// `TABLE_NAMES`: importing that builds a pino logger, and the SQL builders here
+// are pure functions the shape tests call without a ClickHouse connection.
import type { ClickHouseSettings } from '@clickhouse/client';
import { type SqlFragment, sql } from '@openpanel/db/src/clickhouse/sql';
@@ -25,8 +23,7 @@ import type {
Timeframe,
} from './cohort.constants';
-// Physical ClickHouse table names this module reads/writes. Literal, not
-// imported from @openpanel/db's TABLE_NAMES — see the header comment.
+// Physical ClickHouse table names this module reads/writes.
const TABLE = {
profiles: 'profiles',
events: 'events',
@@ -36,15 +33,10 @@ const TABLE = {
eventPropertyProfileSummaryMv: 'event_property_profile_summary_mv',
} as const;
-// Max members materialized into cohort_members per compute. Cohorts larger
-// than this are silently truncated to an arbitrary subset, so deployments
-// with bigger cohorts need to raise it — env-tunable to avoid an image
-// rebuild for what is really a sizing knob.
-//
-// Strictly a positive safe integer, which is what the config loader's
-// `optionalPositiveInt` guarantees: '5000junk', '-1' and '0' all arrive as
-// undefined, and 0 is falsy at the `limit ? LIMIT ... : ''` call sites, which
-// would silently remove the cap entirely.
+// Max members materialized into cohort_members per compute. Larger cohorts are
+// silently truncated to an arbitrary subset, so raise it for big deployments.
+// Must be a positive safe integer: 0 is falsy at the `limit ? LIMIT ... : ''`
+// call sites and would silently remove the cap.
const DEFAULT_COHORT_MATERIALIZE_LIMIT = 10_000;
export function cohortMaterializeLimit(config: CoreConfig): number {
@@ -53,24 +45,17 @@ export function cohortMaterializeLimit(config: CoreConfig): number {
);
}
-// Property cohorts aggregate every profile row for the project, so they are the
-// one cohort query that can outgrow the server's memory headroom. Two opt-in
-// knobs bound them; with NEITHER set, no per-query settings are applied and the
-// server's own defaults govern — upstream behavior is unchanged.
+// Property cohorts aggregate every profile row for the project, so they can
+// outgrow the server's memory. Two opt-in knobs bound them; with neither set the
+// server defaults govern.
//
// COHORT_QUERY_MEMORY_LIMIT_BYTES hard cap for these queries
// COHORT_QUERY_SPILL_BYTES GROUP BY spills to disk past this
//
-// A GROUP BY only starts spilling once it crosses the threshold, so the spill
-// threshold must sit BELOW the memory limit — inverted, the query is killed
-// before it ever writes to disk (ClickHouse Cloud ships exactly that inversion
-// by default, which is how these queries OOM'd instead of spilling). When only
-// the limit is set — or the pair is inverted — the threshold derives as
-// limit/3. Spilling early costs little: the volume spilled is set by the data,
-// not the threshold.
-//
-// A standalone function of its two parsed inputs (not a module-level read), so
-// a test can exercise every branch by calling it directly.
+// A GROUP BY only spills once it crosses the threshold, so the threshold must
+// sit BELOW the memory limit: inverted, the query is killed before it writes to
+// disk (ClickHouse Cloud ships that inversion by default). When only the limit
+// is set, or the pair is inverted, the threshold derives as limit/3.
export function deriveCohortQuerySettings({
memoryLimitBytes,
spillBytes: spillBytesParsed,
@@ -233,10 +218,6 @@ export function buildEventCriteriaQuery(
const timeConstraint = buildTimeConstraint(timeframe, sql.id('event_date'));
const project = sql.string(projectId);
const eventName = sql.string(name);
- // No filter name can start with both 'properties.' and
- // 'profile.properties.' (they diverge at the 4th character), so the
- // 'profile.properties.' exclusion this used to carry was always true here
- // and is dropped rather than kept as dead weight.
const hasEventPropertyFilters = filters.some((f) =>
f.name.startsWith('properties.')
);
@@ -327,8 +308,7 @@ export function buildEventCriteriaQuery(
// The bare profiles columns a cohort filter may reach. Every name is
// qualified by the time it gets here, and anything outside the list is a
-// filter naming a column that is not the caller's to read (main #512,
-// GHSA-gvwr-5684-wjqc).
+// filter naming a column that is not the caller's to read (GHSA-gvwr-5684-wjqc).
const COHORT_PROFILE_COLUMNS = PROFILE_SELECT_COLUMNS.map(
(column) => `profiles.${column}`
);
@@ -357,16 +337,12 @@ function buildProfileCohortHavingClause(
): SqlFragment | null {
const { properties, operator } = definition.criteria;
- // Every argMax below must order the candidate rows IDENTICALLY, or
- // equal-version rows with conflicting fields could each win a different
- // column — matching an AND cohort against a synthetic combination no
- // stored row contains. One shared key — the version column, tie-broken by
- // a hash of every referenced column — makes all aggregates pick their
- // value from the same winning row, deterministically. The hash (rather
- // than the raw value tuple) keeps the per-group comparison state at a
- // fixed 8 bytes, cheaper than comparing the raw tuple. A wrong tie-break
- // would need a version tie AND a 64-bit collision between different rows —
- // and even then every aggregate in the query still elects the same row.
+ // Every argMax below must order candidate rows IDENTICALLY, or equal-version
+ // rows with conflicting fields could each win a different column and match an
+ // AND cohort against a combination no stored row contains. One shared key (the
+ // version column, tie-broken by a hash of every referenced column) makes all
+ // aggregates pick the same winning row. The hash keeps comparison state at a
+ // fixed 8 bytes.
const referencedColumns = Array.from(
new Set(properties.map((f) => normalizeProfileColumn(f.name)))
).map(profileColumnAccess);
@@ -449,12 +425,8 @@ export async function computeEventBasedCohort(
const combinedQuery = buildEventBasedCohortQuery(projectId, definition);
// The LIMIT has to wrap the combination, not trail it: appended to an
- // INTERSECT / UNION chain, ClickHouse applies it to the last SELECT alone.
- // That was survivable while every operand was a narrow event-derived set;
- // a "never did X" operand is most of the project's profiles, so limiting it
- // before the INTERSECT would cut the cohort down to an arbitrary slice —
- // and at the preview's limit of 10, almost always to nothing. The count
- // query below already wraps for the same reason.
+ // INTERSECT / UNION chain, ClickHouse applies it to the last SELECT alone, and
+ // a "never did X" operand is most of the project's profiles.
const finalQuery = limit
? sql`SELECT profile_id FROM (${combinedQuery}) LIMIT ${sql.uint64(limit)}`
: combinedQuery;
@@ -496,16 +468,10 @@ function getProfileFiltersWhereClause(
let columnAccess = profileColumnAccess(normalizeProfileColumn(name));
if (latestPerProfileKey) {
- // Resolve the profile's newest row inside a GROUP BY instead of
- // reading through FINAL. The key is shared by every wrapped column
- // (see buildProfileCohortHavingClause), so all aggregates read the
- // SAME winning row: last_seen_at is the table's version column but is
- // not unique, and per-column tie-breaking would let equal-version
- // rows with conflicting fields produce a synthetic combination no
- // stored row contains. FINAL breaks the same ties by part order,
- // which is not derivable from the data and can shift under a
- // background merge — the shared value-tuple tie-break is
- // deterministic instead.
+ // Resolve the newest row inside a GROUP BY instead of FINAL. The key is
+ // shared by every wrapped column (see buildProfileCohortHavingClause), so all
+ // aggregates read the SAME winning row; FINAL breaks version ties by part
+ // order, which can shift under a background merge.
columnAccess = sql`argMax(${columnAccess}, ${latestPerProfileKey})`;
}
@@ -953,8 +919,7 @@ export async function listCohortMemberProfiles(
}
/**
- * The window `mostEvents` / `popularRoutes` read. Both had no date filter, so
- * they scanned the project's whole event history however small the cohort.
+ * The window `mostEvents` / `popularRoutes` read, bounding the event scan.
* `YYYY-MM-DD HH:mm:ss`, as `getChartStartEndDate` returns it.
*/
export interface CohortActivityWindow {
@@ -964,7 +929,6 @@ export interface CohortActivityWindow {
const DEFAULT_COHORT_ACTIVITY_LIMIT = 10;
-/** Spelled as `overview/src/pages.sql.ts` and `profile/src/sql.ts` spell it. */
function createdAtWithin({
startDate,
endDate,
@@ -989,9 +953,6 @@ export function cohortMemberEventsQuery(
window: CohortActivityWindow,
limit: number
): SqlFragment {
- // This plain `IN (subquery)` on `cohort_members` is left as-is — changing
- // how the subquery binds its values would not change anything about
- // distribution semantics.
return sql`
SELECT name, count() AS count
FROM ${sql.id(TABLE.events)}
diff --git a/packages/core/src/modules/cohort/src/cohort-activity-sql.test.ts b/packages/core/src/modules/cohort/src/cohort-activity-sql.test.ts
index c02bef5cb..304a405ef 100644
--- a/packages/core/src/modules/cohort/src/cohort-activity-sql.test.ts
+++ b/packages/core/src/modules/cohort/src/cohort-activity-sql.test.ts
@@ -1,7 +1,5 @@
// SQL-shape tests for the cohort detail page's `mostEvents` / `popularRoutes`
-// statements. Both are bounded to a window: without it they read the
-// project's whole event history however small the cohort was. Pure builders —
-// no ClickHouse.
+// statements, which are bounded to a window.
import { describe, expect, test } from 'bun:test';
import {
diff --git a/packages/core/src/modules/cohort/src/cohort-event-sql.test.ts b/packages/core/src/modules/cohort/src/cohort-event-sql.test.ts
index 8c5dfa607..556782000 100644
--- a/packages/core/src/modules/cohort/src/cohort-event-sql.test.ts
+++ b/packages/core/src/modules/cohort/src/cohort-event-sql.test.ts
@@ -1,5 +1,4 @@
-// SQL-shape tests for event-based cohort criteria. The "never did this event"
-// assertions are unchanged in what they claim.
+// SQL-shape tests for event-based cohort criteria.
import { describe, expect, test } from 'bun:test';
import type {
diff --git a/packages/core/src/modules/cohort/src/cohort-sql.test.ts b/packages/core/src/modules/cohort/src/cohort-sql.test.ts
index 4a68b9fc7..5330df1e3 100644
--- a/packages/core/src/modules/cohort/src/cohort-sql.test.ts
+++ b/packages/core/src/modules/cohort/src/cohort-sql.test.ts
@@ -1,25 +1,11 @@
-// SQL-shape tests for property-based cohort queries. Minus its itCH
-// ClickHouse-reachability EXPLAIN check: `mock.module` substitutions apply
-// process-wide, not per file, so a sibling file's
-// `@openpanel/db/src/clickhouse/client` mock (cohort.service.test.ts) can
-// silently replace this file's "real" import too, making a reachability check
-// meaningless. No module in core's suite ports that pattern for the same
-// reason; the generated SQL was verified against local ClickHouse by hand
-// instead.
+// SQL-shape tests for property-based cohort queries. There is deliberately no
+// ClickHouse EXPLAIN check: `mock.module` substitutions apply process-wide, so a
+// sibling file's client mock can silently replace this file's "real" import.
//
-// String assertions need no mocking at all: cohort.service.ts's db/ch access is
-// lazy, and buildPropertyBasedCohortQuery / deriveCohortQuerySettings touch
-// neither — see cohort.service.ts's header.
-//
-// The builders return `SqlFragment`s, so every assertion below runs against
-// the rendered statement plus its bound params rather than a finished
-// string. The property-key quoting test below asserts a binding, not
-// escaped text, because the key never appears in the SQL text at all.
-//
-// bun:test shares one module registry per file even under `--isolate`, so
-// re-importing under different env vars would not re-evaluate a
-// module-level const. `deriveCohortQuerySettings` exists so the different
-// settings combinations below are exercised as direct calls instead.
+// The builders return `SqlFragment`s, so assertions run against the rendered
+// statement plus bound params. `deriveCohortQuerySettings` is called directly
+// because bun:test shares one module registry per file, so env-var variants
+// cannot be re-imported.
import { expect, test } from 'bun:test';
import {
@@ -266,6 +252,3 @@ test('deriveCohortQuerySettings never derives a zero spill threshold (0 would DI
).toBeGreaterThanOrEqual(1);
}
});
-
-// A malformed COHORT_QUERY_* value is rejected by the config loader now, not
-// here — `apps/api/src/config/env.test.ts` owns that case.
diff --git a/packages/core/src/modules/ingest/ingest.constants.ts b/packages/core/src/modules/ingest/ingest.constants.ts
index f852c34ce..e27c48731 100644
--- a/packages/core/src/modules/ingest/ingest.constants.ts
+++ b/packages/core/src/modules/ingest/ingest.constants.ts
@@ -1,7 +1,3 @@
-// The ingestion wire contract. All three enforce the same thing (what a
-// caller may name an event and what a track body may contain), so they are
-// one file here.
-
import { z } from 'zod';
/** Event names the ingestion pipeline mints itself; a caller may not send them. */
@@ -130,16 +126,10 @@ export function isBlockedEventName(name: string): boolean {
return false;
}
-// ----- Hand-written types (source of truth) -----
-//
-// These interfaces are duplicated in code that ships in our SDK type
-// declarations. We hand-write them (instead of using `z.infer<…>`) so:
-// 1. Generated `.d.ts` files for the SDKs are readable plain TypeScript
-// with no `import type { ITrackPayload }` line pointing at a workspace
-// package (none of them are published).
-// 2. The interfaces stay clean — no zod internals leaking through.
-// Each schema below has `satisfies z.ZodType<…>` attached so a drift
-// between the interface and the schema fails to compile.
+// Hand-written (not `z.infer`) because these interfaces ship in the SDK type
+// declarations, which must read as plain TypeScript with no import of an
+// unpublished workspace package. Each schema `satisfies` its interface so
+// drift fails to compile.
export type IProfileId = string | number;
@@ -210,8 +200,6 @@ export type ITrackHandlerPayload =
| { type: 'group'; payload: IGroupPayload }
| { type: 'assign_group'; payload: IAssignGroupPayload };
-// ----- Schemas (each `satisfies` its hand-written interface) -----
-
export const zProfileId = z.union([
z.string().min(1),
z.number(),
diff --git a/packages/core/src/modules/ingest/ingest.routes.ts b/packages/core/src/modules/ingest/ingest.routes.ts
index d01f17ad5..af2b1eacd 100644
--- a/packages/core/src/modules/ingest/ingest.routes.ts
+++ b/packages/core/src/modules/ingest/ingest.routes.ts
@@ -1,20 +1,14 @@
// THE HOOK ORDER IS THE CONTRACT: duplicate -> clientAuth -> isBot ->
-// subscription. It holds by lifecycle phase rather than by registration order
-// — verified against Elysia 1.4.30, where a route declaring all three runs
-// `derive -> transform -> macro resolve -> beforeHandle`. So the duplicate
-// check is a `transform`, `clientAuth` is the macro, and the bot check is a
-// `beforeHandle`, which is what lets it read `client.secretVerified` after
-// the macro resolves the principal.
+// subscription. It holds by lifecycle phase rather than registration order
+// (verified against Elysia 1.4.30: `derive -> transform -> macro resolve ->
+// beforeHandle`). So the duplicate check is a `transform`, `clientAuth` is the
+// macro, and the bot check is a `beforeHandle`, which lets it read
+// `client.secretVerified` after the macro resolves the principal.
//
-// Both routes carry the duplicate check because the guard wrapping them below
-// is scoped to both, so `GET /track/device-id` goes through it too.
-//
-// `POST /event` — a legacy compat route kept because production still posts
-// to it — sits beside `/track` with the same hook chain, plus a per-client
-// usage counter recorded after authentication (so a label can only ever be a
-// client id that exists) and before the hooks that can short-circuit (so a
-// client whose events are dropped as bot/wind-down traffic still counts as
-// one that would break if `/event` disappeared).
+// `POST /event`, a legacy route production still posts to, has the same hook
+// chain plus a per-client usage counter recorded after authentication (so a
+// label is always a real client id) and before the hooks that can
+// short-circuit (so a client whose events are dropped still counts).
import type { AppDeps, HttpCtx } from '../../context';
import { defineRoutes } from '../../http/define';
@@ -32,9 +26,6 @@ import {
} from './ingest.service';
import { validateIngestRequest } from './src/client-auth';
import { recordLegacyEventRequest } from './src/ingest.metrics';
-// The Kafka producer is a sibling now, not an `AppDeps` field: kafka.ts moved
-// into this module, so there is nothing left for main.ts to inject. It still
-// constructs no client at import time.
import { produceIncomingEvent } from './src/kafka';
const TAGS = ['Track'];
@@ -165,8 +156,6 @@ function windDownGuard(deps: AppDeps) {
}
export const ingestRoutes = defineRoutes((app, deps: AppDeps) => {
- // One closure over `deps` for all three routes: this is route-definition
- // work, and three identical closures read as if it were per-request.
const blockWhenWoundDown = windDownGuard(deps);
return app.guard(
diff --git a/packages/core/src/modules/ingest/ingest.service.test.ts b/packages/core/src/modules/ingest/ingest.service.test.ts
index 108ecf177..9ed251dd5 100644
--- a/packages/core/src/modules/ingest/ingest.service.test.ts
+++ b/packages/core/src/modules/ingest/ingest.service.test.ts
@@ -94,7 +94,7 @@ const track = (properties?: Record<string, unknown>): ITrackHandlerPayload =>
// Back-dating is documented and deliberate, so the guard here is a FLOOR, not
// a window: historical imports must keep working, and only values that cannot
-// be a real event time fall back to the server clock (ISSUES.md H6).
+// be a real event time fall back to the server clock.
describe('getTimestamp', () => {
const ARRIVED_AT = Date.UTC(2026, 8, 27, 12, 0, 0);
const withTimestamp = (value: unknown) =>
@@ -112,8 +112,8 @@ describe('getTimestamp', () => {
});
it('falls back to the server clock for a value that lands in 1970', () => {
- // `new Date(12345)` is 1970-01-01T00:00:12.345Z. One such row reached
- // ClickHouse during the QA run and every WITH FILL series then began there.
+ // `new Date(12345)` is 1970-01-01T00:00:12.345Z. One such row once reached
+ // ClickHouse and every WITH FILL series then began there.
expect(withTimestamp(12_345).timestamp).toBe(ARRIVED_AT);
expect(withTimestamp(0).timestamp).toBe(ARRIVED_AT);
});
diff --git a/packages/core/src/modules/ingest/ingest.service.ts b/packages/core/src/modules/ingest/ingest.service.ts
index 78cd5d548..f5d99cc38 100644
--- a/packages/core/src/modules/ingest/ingest.service.ts
+++ b/packages/core/src/modules/ingest/ingest.service.ts
@@ -1,10 +1,5 @@
-// The ingestion pipeline for POST /track and POST /event.
-//
// Nothing here throws for a caller error: each transport owns its own status
// codes and bodies, so the pipeline returns a `TrackOutcome` instead.
-//
-// The Kafka producer is injected as an argument, not read from a module
-// singleton, so a test can assert on what was produced without a broker.
import { generateId } from '@openpanel/shared';
import { parseUserAgent } from '@openpanel/shared/server';
@@ -80,8 +75,6 @@ export type IngestBuffers = Pick<Buffers, 'session' | 'replay' | 'group'>;
export interface IngestTransport {
buffers: IngestBuffers;
produceIncomingEvent: IncomingEventProducer;
- /** The profile writes below reach ClickHouse and the profile buffer
- * through the scope, not a module singleton. */
deps: ServiceDeps;
}
@@ -548,7 +541,7 @@ export async function ingestTrack(
*
* It gates on `windDownStep`, not on `subscriptionState`. Every expired trial
* is already in `trial_expired`, so gating on the state would block thousands
- * of orgs the moment this ships, skipping the four warning emails entirely.
+ * of orgs at once, skipping the four warning emails entirely.
*
* It answers 202, not 402 or 403 (the caller does; this returns the verdict).
* The SDKs treat only 401 and 2xx as terminal, so a "correct" status code
diff --git a/packages/core/src/modules/ingest/legacy-event.test.ts b/packages/core/src/modules/ingest/legacy-event.test.ts
index 7a0ed0179..ff1358e28 100644
--- a/packages/core/src/modules/ingest/legacy-event.test.ts
+++ b/packages/core/src/modules/ingest/legacy-event.test.ts
@@ -2,9 +2,6 @@
* Tests for the legacy `POST /event` compat route: the route is KEPT,
* produces to Kafka exactly like /track, and is measured per client so a
* future removal decision has data.
- *
- * The `mixan-*` header fallback moved with the validator itself — see
- * `src/client-auth.test.ts`.
*/
import {
diff --git a/packages/core/src/modules/ingest/src/client-auth.test.ts b/packages/core/src/modules/ingest/src/client-auth.test.ts
index dc6fe5adb..1f72a3920 100644
--- a/packages/core/src/modules/ingest/src/client-auth.test.ts
+++ b/packages/core/src/modules/ingest/src/client-auth.test.ts
@@ -5,9 +5,9 @@
* usage metric confirms `/event` callers have moved off it.
*
* `secretVerified` is the side channel the bot check reads. The `secret
- * verification` block guards main #481: `secretVerified` and revenue
- * ingestion follow whether the secret verified against the stored hash, not
- * whether a secret string was on the request.
+ * verification` block guards that it and revenue ingestion follow whether the
+ * secret verified against the stored hash, not whether a secret string was on
+ * the request.
*/
import {
diff --git a/packages/core/src/modules/ingest/src/client-auth.ts b/packages/core/src/modules/ingest/src/client-auth.ts
index cefb88ff2..a6d31af3a 100644
--- a/packages/core/src/modules/ingest/src/client-auth.ts
+++ b/packages/core/src/modules/ingest/src/client-auth.ts
@@ -1,7 +1,3 @@
-// The ingest tier of caller authentication. Core's `clientAuth` macro reaches
-// it through http/client-auth.ts, which a route hands this function to as
-// `clientAuth: { ingest: validateIngestRequest }`.
-//
// None of the behaviour below is decoration: the `ignoreCorsAndSecret`
// short-circuit, the unanchored wildcard origin regex, CORS-before-secret
// ordering, the ip and profile_id project filters, the `__revenue` gate, the
diff --git a/packages/core/src/modules/ingest/src/consumer.test.ts b/packages/core/src/modules/ingest/src/consumer.test.ts
index a94f420ac..aab8be1fa 100644
--- a/packages/core/src/modules/ingest/src/consumer.test.ts
+++ b/packages/core/src/modules/ingest/src/consumer.test.ts
@@ -371,15 +371,9 @@ describe('unparseable messages', () => {
});
/**
- * Deleted the offset watermark, whose only outputs were the `reprocessed`
- * counter and a warn line — live duplicates occurred but none of those lines
- * fired, because `resetWatermarks` fired on GROUP_JOIN and a failed
- * durability flush never advanced it.
- *
- * These are the regression tests for the half of that claim that matters: the
- * functional statement in the resolve loop was always `resolveOffset(m.offset)`
- * and never read the watermark, so a redelivery must resolve and commit exactly
- * what it did before.
+ * A redelivery must resolve and commit exactly what the resolve loop marks:
+ * no watermark suppresses or alters it, so duplicates are never silently
+ * skipped.
*/
describe('a redelivery after the watermark was deleted', () => {
test('resolves every offset again, in ascending order', async () => {
@@ -404,9 +398,9 @@ describe('a redelivery after the watermark was deleted', () => {
await handler.eachBatch(makeBatch(messages).payload);
await handler.eachBatch(makeBatch(messages).payload);
- // At-least-once, unchanged: the consumer does not suppress a redelivery,
- // and nothing in this file now decides whether one is a duplicate — the
- // marker on the event id does (incoming-event-handler.ts).
+ // At-least-once: the consumer does not suppress a redelivery. Whether one
+ // is a duplicate is decided by the marker on the event id
+ // (incoming-event-handler.ts).
expect(handleEvent).toHaveBeenCalledTimes(2);
});
diff --git a/packages/core/src/modules/ingest/src/consumer.ts b/packages/core/src/modules/ingest/src/consumer.ts
index 2988c74b7..295dfe1cc 100644
--- a/packages/core/src/modules/ingest/src/consumer.ts
+++ b/packages/core/src/modules/ingest/src/consumer.ts
@@ -2,10 +2,6 @@
// offset walk, at-least-once, bounded in-consumer retry, and a dead-letter on
// exhaustion. The dead-letter destination is a capped Redis list, and the
// event is dropped whether or not that write lands.
-//
-// The kafkajs client, the topic/consumer-group names, the DLQ producer and the
-// retry bounds are all injected — this file never spells any of them, which is
-// what keeps them testable without a broker.
import type {
Consumer,
@@ -57,11 +53,9 @@ export type DeadLetterReason = 'parse_error' | 'handler_error';
* What the consumer hands to the dead-letter sink. The value stays the
* producer's original bytes, so the record is a faithful copy of what arrived.
*
- * Two implementations satisfy this, and the seam exists so the choice is one
- * line in `apps/api`'s wiring: `@openpanel/redis`'s `createDeadLetterRecorder`
- * (the capped list) and `./kafka.ts`'s `produceDeadLetterEvent` (the DLQ
- * topic, kept as an explicit "for now" choice; a DLQ message produced from
- * these bytes is replayable onto the events topic unchanged).
+ * Implemented by `@openpanel/redis`'s `createDeadLetterRecorder` (the capped
+ * list) and `./kafka.ts`'s `produceDeadLetterEvent` (the DLQ topic; a message
+ * produced from these bytes is replayable onto the events topic unchanged).
*/
export interface DeadLetterMessage {
key: Buffer | null;
@@ -112,11 +106,10 @@ export interface EventsBatchHandlerDeps {
* not at `consumer.stop`.
*
* The gate MUST reject when the events THIS batch buffered are not in
- * Redis: a graceful restart previously lost events whose offsets were
- * committed even though the events existed only in the event buffer's
- * in-process array. The window is what makes "this batch's events are
- * durable" answerable at all, now that a failed write drops the events
- * instead of keeping them for the broker to redeliver.
+ * Redis: events that exist only in the event buffer's in-process array
+ * would otherwise be lost with their committed offsets. The window is what
+ * makes "this batch's events are durable" answerable, since a failed write
+ * drops events instead of keeping them for the broker to redeliver.
*/
openDurabilityWindow: () => () => Promise<void>;
logger: ConsumerLogger;
@@ -169,10 +162,9 @@ export function createEventsBatchHandler(
*
* There is deliberately no retry, no backoff and no unresolved offset here.
* The dead-letter destination is Redis, and Redis being unavailable is
- * exactly when handlers fail — so a failed write that held the offset back
- * would rebuild the redelivery loop this replaced: once something else owns
- * the outcome, a safety net underneath it is the bug.
- *
+ * exactly when handlers fail, so a failed write that held the offset back
+ * would recreate the redelivery loop: once something else owns the outcome,
+ * a safety net underneath it is the bug.
* The two counters are what carry the volume, since a capped list makes
* 50,000 drops look like 12: `deadLettered` = recorded and dropped,
* `deadLetterFailed` = dropped WITHOUT being recorded. Neither means "will be
@@ -476,9 +468,8 @@ export async function startKafkaEventsConsumer(
consumer.on(consumer.events.HEARTBEAT, deps.batch.onActivity);
- // ---- Lifecycle / rebalance ("re-election") visibility ----
- // Without this logging, a rebalance storm (a common source of
- // at-least-once duplicates) would be invisible.
+ // Lifecycle / rebalance logging: a rebalance storm (a common source of
+ // at-least-once duplicates) would otherwise be invisible.
consumer.on(consumer.events.GROUP_JOIN, ({ payload }) => {
// A new assignment means partitions may have moved between members, so a
// consecutive-failure count carried over from the old one would pick the
diff --git a/packages/core/src/modules/ingest/src/device-id.test.ts b/packages/core/src/modules/ingest/src/device-id.test.ts
index e5b3c1cdc..eca518c82 100644
--- a/packages/core/src/modules/ingest/src/device-id.test.ts
+++ b/packages/core/src/modules/ingest/src/device-id.test.ts
@@ -18,7 +18,7 @@ import {
const NOW = new Date('2026-06-08T12:00:00.000Z').getTime();
const MINUTE = 60 * 1000;
const SALTS = { current: 'salt-current', previous: 'salt-previous' };
-/** SESSION_TIMEOUT_MS's default: the caller resolves it from config now. */
+/** SESSION_TIMEOUT_MS's default; the caller resolves it from config. */
const SESSION_TIMEOUT_MS = 30 * 60 * 1000;
/** The caller's request logger. Inputs here are an IP and a salt, so a
@@ -136,12 +136,11 @@ describe('getDeviceId — session resolution', () => {
});
/**
- * Made the cache client fail fast, so this read now rejects in microseconds
- * instead of after ioredis's 20-attempt reconnect cycle. The degradation
- * itself must be untouched: the api and the worker only agree on a session
- * because they derive the SAME id from project/device/time bucket, so a
- * faster failure that changed the id would split every session an outage
- * touched.
+ * The cache client fails fast, so this read rejects in microseconds instead
+ * of after ioredis's 20-attempt reconnect cycle. The degradation must not
+ * change the id: the api and the worker only agree on a session because they
+ * derive the SAME id from project/device/time bucket, so a different id on
+ * failure would split every session an outage touched.
*/
it('mints the SAME deterministic id whether the session read misses or rejects', async () => {
const { buffer: emptyStore } = stubBuffer(null);
diff --git a/packages/core/src/modules/ingest/src/device-id.ts b/packages/core/src/modules/ingest/src/device-id.ts
index 26672459c..795ca9325 100644
--- a/packages/core/src/modules/ingest/src/device-id.ts
+++ b/packages/core/src/modules/ingest/src/device-id.ts
@@ -100,7 +100,7 @@ export async function getDeviceId({
* Returns true when an existing session is recent enough that the incoming
* event should EXTEND it rather than start a new session.
*
- * Critical: blobs no longer have a Redis TTL (so the reaper can always find
+ * Critical: blobs have no Redis TTL (so the reaper can always find
* them), which means an existing session blob may linger past its idle
* window. We must NOT blindly reuse `existing.id` — if we did, the worker's
* boundary detection would open a "new" session with the same id as the
diff --git a/packages/core/src/modules/ingest/src/incoming-event-handler.test.ts b/packages/core/src/modules/ingest/src/incoming-event-handler.test.ts
index 4aa8e4573..666f75bb1 100644
--- a/packages/core/src/modules/ingest/src/incoming-event-handler.test.ts
+++ b/packages/core/src/modules/ingest/src/incoming-event-handler.test.ts
@@ -467,9 +467,8 @@ describe('incomingEvent', () => {
});
/**
- * The marker replaces the offset watermark, which cannot see every
- * duplicate. MARK MEANS COUNT AND LOG: every test below asserts the event
- * was still INSERTED.
+ * MARK MEANS COUNT AND LOG: the marker does not suppress, so every test below
+ * asserts the event was still INSERTED.
*/
describe('duplicate marker', () => {
const TTL_MS = 120_000;
diff --git a/packages/core/src/modules/ingest/src/incoming-event-handler.ts b/packages/core/src/modules/ingest/src/incoming-event-handler.ts
index 7ad4066ba..bc0cce2cc 100644
--- a/packages/core/src/modules/ingest/src/incoming-event-handler.ts
+++ b/packages/core/src/modules/ingest/src/incoming-event-handler.ts
@@ -1,14 +1,10 @@
-// The Kafka consumer's per-message handler — the step that turns one topic
-// payload into session lifecycle decisions and ClickHouse rows.
+// The Kafka consumer's per-message handler: turns one topic payload into
+// session lifecycle decisions and ClickHouse rows.
//
-// Everything that touches Redis, Postgres, ClickHouse or a queue is INJECTED
-// (`IncomingEventDeps`), the way session-end.ts does it, so the tests drive the
-// real code with recording doubles instead of `mock.module`.
-// `createIncomingEventDeps` binds it to ONE MESSAGE'S ctx: the buffers, the
-// Postgres client and the session-end enqueue all ride the scope carrying that
-// envelope's requestId, so the id the producer stamped reaches the row the
-// consumer writes. The two lookups a work scope cannot supply arrive as
-// `IncomingEventBindings` from the composition root.
+// Everything that touches Redis, Postgres, ClickHouse or a queue is injected
+// (`IncomingEventDeps`) so tests drive the real code with recording doubles.
+// `createIncomingEventDeps` binds it to one message's ctx, so the requestId the
+// producer stamped reaches the row the consumer writes.
import { getTime, isSameDomain, parsePath } from '@openpanel/shared';
import { anyPass, isEmpty, isNil, mergeDeepRight, omit, reject } from 'ramda';
@@ -245,11 +241,9 @@ const parseRevenue = (revenue: unknown): number | undefined => {
* its answer. A false positive on a suppressing check would be silent data
* loss, and whether the events table ever gets dedupe is not decided here.
*
- * It replaces the offset watermark, which could not see a duplicate across a
- * rebalance, an eviction, a durability redelivery or a restart. It catches
- * redeliveries of the SAME id: crash, eviction, durability redelivery. It
- * does NOT catch a lost ACK, where the SDK re-sends and the producer mints a
- * fresh id per request; SDK-supplied ids are what closes that, later.
+ * Catches redeliveries of the SAME id (crash, eviction, durability redelivery,
+ * rebalance, restart). It does NOT catch a lost ACK, where the SDK re-sends and
+ * the producer mints a fresh id per request.
*
* FAILS OPEN, AND NEVER REJECTS. Redis being away is exactly when redeliveries
* happen, so an observability counter must not be the thing that delays or
diff --git a/packages/core/src/modules/ingest/src/ingest.metrics.ts b/packages/core/src/modules/ingest/src/ingest.metrics.ts
index 4518fec72..4922ecb44 100644
--- a/packages/core/src/modules/ingest/src/ingest.metrics.ts
+++ b/packages/core/src/modules/ingest/src/ingest.metrics.ts
@@ -6,10 +6,9 @@ import type { ConsumerMetrics, DeadLetterReason } from './consumer';
// Events whose producer-minted id had already been seen. This is a MARKER, not
// a dedupe: the event is inserted either way and this counter is the whole
-// point of the check. It replaces `kafka_events_reprocessed_total`, which was
-// keyed on a Kafka offset in one process's heap and therefore blind to every
-// duplicate that mattered (measured: 161 real duplicates, 0 reprocessed
-// increments).
+// point of the check. The older offset-keyed counter lived in one process's
+// heap and was blind to every duplicate that mattered (161 real duplicates,
+// 0 reprocessed increments).
//
// It UNDERCOUNTS by design: a duplicate whose marker could not be written
// (Redis away) is not counted, because the check fails open rather than
@@ -33,8 +32,7 @@ export const kafkaHandlerFailuresTotal = new client.Counter({
// Messages recorded in the dead-letter list after their attempts were exhausted
// (or after they failed to parse), then DROPPED. One increment per message. The
// list is capped at the last N, so this counter — not the list — is the only
-// thing that tells 50,000 drops apart from 12. The series name is unchanged:
-// the destination moved, the meaning did not.
+// thing that tells 50,000 drops apart from 12.
export const kafkaDeadLetteredTotal = new client.Counter({
name: 'kafka_events_dead_lettered_total',
help: 'Kafka event messages recorded in the dead-letter list and dropped',
diff --git a/packages/core/src/modules/ingest/src/kafka.ts b/packages/core/src/modules/ingest/src/kafka.ts
index 1d2510dcf..0cb0c598c 100644
--- a/packages/core/src/modules/ingest/src/kafka.ts
+++ b/packages/core/src/modules/ingest/src/kafka.ts
@@ -102,9 +102,8 @@ const getProducer = async (config: CoreConfig): Promise<Producer> => {
// OUT_OF_ORDER_SEQUENCE_NUMBER and stick the producer per-partition.
// Raising it gained only a couple percent of throughput and was
// rejected; batching is what amortises the round-trip.
- // KAFKA_PRODUCER_MAX_IN_FLIGHT still raises it for a measurement run —
- // read docs/KAFKA_PRODUCER_OPTIONS.md first: it is coupled to the retry
- // policy.
+ // KAFKA_PRODUCER_MAX_IN_FLIGHT still raises it for a measurement run, but it
+ // is coupled to the retry policy.
maxInFlightRequests: tuning.maxInFlight,
allowAutoTopicCreation: true,
retry: {
@@ -277,7 +276,7 @@ export const createKafkaEventsConsumer = (
return consumer;
};
-// ── Consumer-group lag (backpressure visibility) ────────────────────────────
+// Consumer-group lag (backpressure visibility)
// A fast producer can hide a lagging consumer entirely — throughput numbers
// alone don't show it. `sampleConsumerGroupLag` reads the same end-offset-
// minus-committed-offset a broker-side tool (e.g. `rpk group describe`) would
diff --git a/packages/core/src/modules/ingest/src/producer-batcher.ts b/packages/core/src/modules/ingest/src/producer-batcher.ts
index f2ac6aabd..8f0f409cb 100644
--- a/packages/core/src/modules/ingest/src/producer-batcher.ts
+++ b/packages/core/src/modules/ingest/src/producer-batcher.ts
@@ -2,8 +2,6 @@
// ONE `send` carrying MANY messages, instead of one awaited `send` per
// /track request.
//
-// It is OFF unless the config loader says otherwise — see `producer-tuning.ts`.
-//
// Two invariants the tests hold this file to:
//
// 1. ORDER. Messages leave in the order they were enqueued: within a batch
diff --git a/packages/core/src/modules/ingest/src/producer-errors.ts b/packages/core/src/modules/ingest/src/producer-errors.ts
index 1d21254ca..6e25dfa3c 100644
--- a/packages/core/src/modules/ingest/src/producer-errors.ts
+++ b/packages/core/src/modules/ingest/src/producer-errors.ts
@@ -1,7 +1,5 @@
// The producer's error classification, split out of `kafka.ts` so the batching
-// path can be tested against the same code the single-message path uses: a
-// batch that fails must reset the producer exactly like a lone message did, and
-// that is only provable if the recovery wrapper is reachable without a broker.
+// path can be tested against the same recovery wrapper without a broker.
// Kafka error codes that mean the producer's PID/sequence state is
// permanently out of sync with the broker for some partition — only a
diff --git a/packages/core/src/modules/ingest/src/producer-tuning.test.ts b/packages/core/src/modules/ingest/src/producer-tuning.test.ts
index ee0095eb6..151393244 100644
--- a/packages/core/src/modules/ingest/src/producer-tuning.test.ts
+++ b/packages/core/src/modules/ingest/src/producer-tuning.test.ts
@@ -1,8 +1,7 @@
/**
* Batching is ON at the shipped defaults (size 25, linger 5 ms) and
- * `maxInFlightRequests` stays 1. There are no fallbacks left to test — the
- * three knobs are required `KafkaConfig` fields, so a config that omits one
- * does not compile.
+ * `maxInFlightRequests` stays 1. The three knobs are required `KafkaConfig`
+ * fields, so there are no fallbacks to test.
*/
import { describe, expect, test } from 'bun:test';
diff --git a/packages/core/src/modules/ingest/src/producer-tuning.ts b/packages/core/src/modules/ingest/src/producer-tuning.ts
index 6be48d146..510b4eaa7 100644
--- a/packages/core/src/modules/ingest/src/producer-tuning.ts
+++ b/packages/core/src/modules/ingest/src/producer-tuning.ts
@@ -1,9 +1,7 @@
// The Kafka producer's throughput knobs, resolved in one place.
//
-// The three fields are required members of `KafkaConfig`
-// (`packages/core/src/config.ts`): the config loader always sets them, so there
-// is no fallback here and no second default anywhere. This file only decides
-// whether the configured size batches at all.
+// The config loader always sets the three fields, so there is no fallback
+// here. This file only decides whether the configured size batches at all.
import type { KafkaConfig } from '../../../config';
diff --git a/packages/core/src/modules/ingest/wind-down.test.ts b/packages/core/src/modules/ingest/wind-down.test.ts
index 3f3cee81d..13ed2ec37 100644
--- a/packages/core/src/modules/ingest/wind-down.test.ts
+++ b/packages/core/src/modules/ingest/wind-down.test.ts
@@ -2,11 +2,11 @@
* Tests for `isIngestionWoundDown` — the gate that stops ingestion for
* organizations blocked by the wind-down sequence.
*
- * Same three behaviours matter: it gates on the wind-down STEP rather than the
+ * Three behaviours matter: it gates on the wind-down STEP rather than the
* subscription state (so an expired trial keeps ingesting until it has actually
* been warned), the caller answers 202 rather than a 4xx (the SDKs retry
* everything except 401 and 2xx), and it fails open. `SELF_HOSTED` is a
- * `selfHosted` argument now — core reads no environment.
+ * `selfHosted` argument: core reads no environment.
*/
import {
@@ -125,7 +125,7 @@ describe('isIngestionWoundDown', () => {
it('accepts ingestion from an expired trial that has not been warned yet', async () => {
// The reason the gate reads windDownStep and not subscriptionState: every
// lapsed trial is already trial_expired, so gating on state would block
- // thousands of orgs the moment this ships.
+ // thousands of orgs at once.
getOrganizationByProjectIdCached.mockResolvedValue({
id: 'org-1',
subscriptionState: 'trial_expired',
diff --git a/packages/core/src/modules/overview/overview.rpc.ts b/packages/core/src/modules/overview/overview.rpc.ts
index 90cb34744..3e066e75f 100644
--- a/packages/core/src/modules/overview/overview.rpc.ts
+++ b/packages/core/src/modules/overview/overview.rpc.ts
@@ -1,9 +1,6 @@
// `overviewProcedure` is `publicProcedure` plus one middleware that resolves
// share-or-membership off the raw `projectId`/`shareId`, BEFORE the input
// parser.
-//
-// There is no per-range cache middleware yet; `createCacheMiddleware` in
-// rpc/base.ts is the seam that will carry it.
import { getChartPrevStartEndDate } from '@openpanel/shared';
import { format } from 'date-fns';
diff --git a/packages/core/src/modules/overview/pages.service.ts b/packages/core/src/modules/overview/pages.service.ts
index b13923d3a..0d6c7aecd 100644
--- a/packages/core/src/modules/overview/pages.service.ts
+++ b/packages/core/src/modules/overview/pages.service.ts
@@ -1,15 +1,6 @@
-// The ClickHouse queries moved to src/pages.sql.ts.
-//
-// `PagesService` is no longer a class and there is no `pagesService` module
-// singleton. It is registered as its OWN service (`services.pages`) rather than
-// folded into `overview`, because both expose a `getTopPages` and they are
-// different queries with different inputs — one over `events` scoped by search,
-// one over `sessions` scoped by filters. Folding them would have had to rename
-// one, which is a call-site contract this wave does not change. The
-// caller-supplied `constructor(client)` slot is gone: the mcp
-// `get_page_performance` tool constructed a fresh instance per call only to
-// dodge a module-singleton mocking hazard, and there is no singleton left to
-// dodge.
+// `services.pages` is its own service rather than folded into `overview`: both
+// expose a `getTopPages`, over different tables (`events` scoped by search,
+// `sessions` scoped by filters) with different inputs.
import type { ServiceDeps, Services } from '../../services';
import { getSettingsForProject } from '../organization/organization.service';
diff --git a/packages/core/src/modules/overview/src/overview-raw-where.sql.test.ts b/packages/core/src/modules/overview/src/overview-raw-where.sql.test.ts
index 3056a288e..b1240219d 100644
--- a/packages/core/src/modules/overview/src/overview-raw-where.sql.test.ts
+++ b/packages/core/src/modules/overview/src/overview-raw-where.sql.test.ts
@@ -1,19 +1,13 @@
/**
* SQL-syntax tests for the overview module's `getRawWhereClause` (the UTM
- * remapping). Same pattern as the chart module's SQL-syntax tests.
- * `getRawWhereClause` sits on the `sql` tag, so these assertions read the
- * rendered statement AND its bound params instead of one escaped string; the
- * rest of the overview module's queries already run through
- * overview.sql.ts, covered by overview.sql.test.ts. It is a plain module
- * function, not a class method — it never needed a client, being pure
- * fragment building.
+ * remapping). It sits on the `sql` tag, so these assertions read the rendered
+ * statement AND its bound params instead of one escaped string.
*
- * Strategy: build the SQL string, then run `EXPLAIN <sql>` against the local
- * ClickHouse instance. EXPLAIN parses the query, resolves columns, and builds
- * the query plan without executing it — so we catch UNKNOWN_IDENTIFIER,
- * AMBIGUOUS_IDENTIFIER and bad JOIN ON expressions without needing seeded data.
- * WITH FILL TO < FROM is a runtime check, so it's covered by a plain string
- * assertion instead.
+ * Strategy: build the SQL, then run `EXPLAIN <sql>` against the local ClickHouse.
+ * EXPLAIN resolves columns and builds the plan without executing, so
+ * UNKNOWN_IDENTIFIER, AMBIGUOUS_IDENTIFIER and bad JOIN ON expressions surface
+ * without seeded data. WITH FILL TO < FROM is a runtime check, so it's covered
+ * by a plain string assertion instead.
*
* Requires a locally reachable CH at http://localhost:23123/openpanel. All
* `itCH` tests auto-skip if CH is unreachable.
diff --git a/packages/core/src/modules/overview/src/overview.sql.test.ts b/packages/core/src/modules/overview/src/overview.sql.test.ts
index c4a8e6cb1..d3b2b9414 100644
--- a/packages/core/src/modules/overview/src/overview.sql.test.ts
+++ b/packages/core/src/modules/overview/src/overview.sql.test.ts
@@ -294,7 +294,7 @@ describe('overview.sql — sql.id() identifier whitelists (R3)', () => {
// The MCP tools send a bare `YYYY-MM-DD`; the dashboard sends the day's edges
// already. Both have to end up bounding the same last day, or every MCP range
-// silently loses it (ISSUES.md H8a).
+// silently loses it.
describe('overview.sql — date-only range boundaries', () => {
function boundsOf(fragment: SqlFragment): string[] {
const { query_params } = fragment.toStatement();
diff --git a/packages/core/src/modules/overview/src/overview.sql.ts b/packages/core/src/modules/overview/src/overview.sql.ts
index 1e342ad83..87c5a01c7 100644
--- a/packages/core/src/modules/overview/src/overview.sql.ts
+++ b/packages/core/src/modules/overview/src/overview.sql.ts
@@ -1,14 +1,9 @@
// Every ClickHouse query OverviewService runs, as pure `sql` fragments.
//
-// Both filter compilers and the field resolver return fragments and are
-// interpolated directly. `compiledText` is left with one job here: the
-// `INTERVAL <n> <unit>` step keyword in a WITH FILL clause, which is SQL
-// syntax, not a value.
-//
-// `toStartOf`/`toInterval`/`datetime` deliberately never pass a timezone
-// into `toStartOfWeek`/`toStartOfMonth` — ClickHouse resolves those from the
-// query's `session_timezone` setting instead, which every statement here
-// still sets (see src/run-query.ts).
+// `toStartOf`/`toInterval`/`datetime` deliberately never pass a timezone into
+// `toStartOfWeek`/`toStartOfMonth`: ClickHouse resolves those from the query's
+// `session_timezone` setting, which every statement here sets (see
+// src/run-query.ts).
import { type SqlFragment, sql } from '@openpanel/db/src/clickhouse/sql';
import {
@@ -118,8 +113,6 @@ function rawWhere(where: SqlFragment | null): SqlFragment {
return where ? sql`AND ${where}` : sql.empty;
}
-// --- revenue -----------------------------------------------------------------
-
export interface RevenueQueryInput {
projectId: string;
startDate: string;
@@ -144,8 +137,6 @@ export function revenueQuery(input: RevenueQueryInput): SqlFragment {
`;
}
-// --- metrics: sessions-only path ----------------------------------------------
-
export interface SessionMetricsQueryInput {
projectId: string;
startDate: string;
@@ -179,8 +170,6 @@ export function sessionMetricsQuery(
`;
}
-// --- metrics: page-filtered path -----------------------------------------------
-
export interface PageFilterMetricsQueryInput {
projectId: string;
startDate: string;
@@ -329,8 +318,6 @@ export function metricsWithPageFilterQuery(
`;
}
-// --- top pages -----------------------------------------------------------------
-
export interface TopPagesQueryInput {
projectId: string;
startDate: string;
@@ -359,8 +346,6 @@ export function topPagesQuery(input: TopPagesQueryInput): SqlFragment {
`;
}
-// --- top entry/exit --------------------------------------------------------------
-
export interface DistinctSessionsQueryInput {
projectId: string;
startDate: string;
@@ -437,8 +422,6 @@ export function topEntryExitQuery(input: TopEntryExitQueryInput): SqlFragment {
`;
}
-// --- top generic (breakdown by a single dimension) ------------------------------
-
/** `column`/`prefixColumn` are drawn from a closed zod enum upstream — bound
* as identifiers here rather than trusted as pre-validated text. */
const TOP_GENERIC_COLUMNS = [
@@ -564,8 +547,6 @@ export function topGenericSeriesTimeSeriesQuery(
`;
}
-// --- user journey (sankey of screen_view paths) ----------------------------------
-
export interface OrderedEventsQueryInput {
projectId: string;
startDate: string;
@@ -589,9 +570,8 @@ function orderedEventsQuery(input: OrderedEventsQueryInput): SqlFragment {
/**
* `arrayCompact` drops consecutive repeats in linear time. An `arrayFilter`
- * spelling is Theta(n^2) in the longest session's pageview count and could not
- * answer for a tenant with one very long session — the same defect, and the
- * same fix, as sankey.sql.ts's DEDUPE_CONSECUTIVE.
+ * answer for a tenant with one very long session (see sankey.sql.ts's
+ * DEDUPE_CONSECUTIVE).
*/
function pathsDedupedCte(
input: OrderedEventsQueryInput,
@@ -675,8 +655,6 @@ export function transitionsQuery(input: TransitionsQueryInput): SqlFragment {
`;
}
-// --- top events ------------------------------------------------------------------
-
export interface TopEventsQueryInput {
projectId: string;
startDate: string;
@@ -703,8 +681,6 @@ export function topEventsQuery(input: TopEventsQueryInput): SqlFragment {
`;
}
-// --- top link-out ------------------------------------------------------------------
-
export interface TopLinkOutQueryInput {
projectId: string;
startDate: string;
@@ -728,8 +704,6 @@ export function topLinkOutQuery(input: TopLinkOutQueryInput): SqlFragment {
`;
}
-// --- map data ------------------------------------------------------------------
-
export interface MapDataQueryInput {
projectId: string;
startDate: string;
@@ -755,8 +729,6 @@ export function mapDataQuery(input: MapDataQueryInput): SqlFragment {
`;
}
-// --- live data (dashboard's 30-minute rolling window) ---------------------------
-
export interface LiveTotalSessionsQueryInput {
projectId: string;
}
diff --git a/packages/core/src/modules/overview/src/run-query.ts b/packages/core/src/modules/overview/src/run-query.ts
index b41eed429..82dd9c28a 100644
--- a/packages/core/src/modules/overview/src/run-query.ts
+++ b/packages/core/src/modules/overview/src/run-query.ts
@@ -1,11 +1,4 @@
-// The overview module's one ClickHouse entry point.
-//
-// The client is `deps.ch` — the same round-robin/retry proxy `main.ts` builds
-// and hands to every scope — instead of a lazy
-// `import('@openpanel/db/src/clickhouse/client')`. `OverviewService` /
-// `PagesService` were classes carrying a caller-supplied client precisely
-// because there was no scope to reach one through; there is now, so both are
-// factories over `ServiceDeps` and this file just forwards.
+// The overview module's one ClickHouse entry point; the client is `deps.ch`.
//
// `timezone` is mandatory here (every overview query sends one, several as a
// literal 'UTC') where the shared helper leaves it optional, which is why this
diff --git a/packages/core/src/modules/report/report.constants.test.ts b/packages/core/src/modules/report/report.constants.test.ts
index ddd9e5904..0b58b4cce 100644
--- a/packages/core/src/modules/report/report.constants.test.ts
+++ b/packages/core/src/modules/report/report.constants.test.ts
@@ -1,7 +1,6 @@
/**
- * Locks down `getDefaultIntervalByDates`'s rewrite from date-fns
- * (differenceInDays/isSameDay) to plain Date math (see file header) against
- * the exact boundaries the original branches used.
+ * Locks down `getDefaultIntervalByDates`'s plain Date math against the exact
+ * boundaries of the date-fns branches it replaces.
*/
import { describe, expect, it } from 'bun:test';
import { getDefaultIntervalByDates } from './report.constants';
diff --git a/packages/core/src/modules/report/report.constants.ts b/packages/core/src/modules/report/report.constants.ts
index 6e8a95b79..6676d0858 100644
--- a/packages/core/src/modules/report/report.constants.ts
+++ b/packages/core/src/modules/report/report.constants.ts
@@ -1,10 +1,8 @@
// `getDefaultIntervalByDates` uses plain Date math instead of date-fns. Every
// process that constructs these Dates runs under TZ=UTC, so Date's local
-// getters (getFullYear/getMonth/getDate) already read UTC — the helpers below
-// reproduce date-fns's local-calendar semantics under that invariant. The
-// only inputs this function ever receives are date-only strings (midnight
+// getters already read UTC, and the inputs are date-only strings (midnight
// boundaries), where date-fns's exact-duration-vs-calendar-day nuance cannot
-// diverge from a plain ms division anyway.
+// diverge from a plain ms division.
import { z } from 'zod';
@@ -328,7 +326,7 @@ function isSameCalendarDay(a: Date, b: Date): boolean {
);
}
-/** date-fns's differenceInDays without the dependency — see file header. */
+/** date-fns's differenceInDays without the dependency. */
function differenceInDays(dateLeft: Date, dateRight: Date): number {
return Math.trunc((dateLeft.getTime() - dateRight.getTime()) / MS_PER_DAY);
}
@@ -636,10 +634,7 @@ export const chartColors = [
*/
const CHART_FORMULA_PATTERN = /^[A-Za-z0-9_ .,+\-*/()%^]*$/;
-// Helper, not vocabulary: every consumer copies these 4 lines locally instead
-// of importing them (apps/start/src/utils/object-to-zod-enums.ts is the
-// frontend's copy). Kept private here, for this file's own use building the
-// enums below.
+// Private helper for the enums below; consumers keep their own copy.
function objectToZodEnums<K extends string>(
obj: Record<K, unknown>
): [K, ...K[]] {
@@ -939,10 +934,7 @@ export const zReport = zReportInput.extend({
// Alias for backward compatibility
export const zChartInput = zReportInput;
-// --------------------------------------------------------------------------
-// `UnionOmit` has no owning module; it is kept here, with the report types it
-// is only ever applied to (apps/start's reportSlice).
-// ---------------------------------------------------------------------------
+// `UnionOmit` has no owning module; it lives with the report types it is applied to.
export type UnionOmit<T, K extends keyof any> = T extends any
? Omit<T, K>
diff --git a/packages/core/src/modules/report/report.rpc.ts b/packages/core/src/modules/report/report.rpc.ts
index 1a0ccb19c..dbfb2dfd4 100644
--- a/packages/core/src/modules/report/report.rpc.ts
+++ b/packages/core/src/modules/report/report.rpc.ts
@@ -1,12 +1,7 @@
-// `protectedProcedure` runs `enforceUserIsAuthed` + `enforceAccess` BEFORE the
-// input parser. The explicit checks in the handlers below stay: `enforceAccess`
-// only sees a TOP-LEVEL `projectId` / `organizationId`, so anything resolved
-// from another id needs its own.
-//
-// Most of this router's inputs carry a reportId or a dashboardId rather than a
-// projectId, so `enforceAccess` is blind to them and the in-handler
-// `requireProjectAccess` through `ctx.services.auth` is the only check that
-// fires.
+// `protectedProcedure` runs `enforceAccess` BEFORE the input parser, and it only
+// sees a TOP-LEVEL `projectId` / `organizationId`. Most inputs here carry a
+// reportId or dashboardId instead, so the in-handler `requireProjectAccess`
+// through `ctx.services.auth` is the only check that fires.
import { z } from 'zod';
import { createTRPCRouter, protectedProcedure } from '../../rpc/base';
diff --git a/packages/core/src/modules/report/report.service.test.ts b/packages/core/src/modules/report/report.service.test.ts
index 0ed23b0f5..4456ae735 100644
--- a/packages/core/src/modules/report/report.service.test.ts
+++ b/packages/core/src/modules/report/report.service.test.ts
@@ -1,8 +1,6 @@
/**
- * Unit tests for mergeGlobalFilters — the helper that combines report-level
- * global filters with each event series' own filters (AND semantics). Pure
- * function, no ClickHouse/Postgres needed. See ./src/series.ts for the
- * helper itself.
+ * Unit tests for mergeGlobalFilters, which combines report-level global filters
+ * with each event series' own filters (AND semantics). Pure function.
*/
import { describe, expect, it } from 'bun:test';
import type { IChartEventFilter, IChartEventItem } from './report.constants';
diff --git a/packages/core/src/modules/report/report.service.ts b/packages/core/src/modules/report/report.service.ts
index d736a2d40..efc573749 100644
--- a/packages/core/src/modules/report/report.service.ts
+++ b/packages/core/src/modules/report/report.service.ts
@@ -1,7 +1,5 @@
// Every function takes `ServiceDeps` and reaches Postgres as `deps.db`; the
// Prisma row types below are `import type`, erased at runtime.
-// `onlyReportEvents`/`mergeGlobalFilters` live in ./src/series.ts — see that
-// file for why.
import type {
Report as DbReport,
@@ -49,7 +47,6 @@ export function transformReportEventItem(
index: number
): IChartEventItem {
if (item.type === 'formula') {
- // Transform formula
return {
type: 'formula',
id: item.id ?? alphabetIds[index]!,
@@ -59,7 +56,6 @@ export function transformReportEventItem(
};
}
- // Transform event with type field
return {
type: 'event',
segment: item.segment ?? 'event',
@@ -231,10 +227,8 @@ export async function getReportDataCore(
return { ...meta, data: result };
}
-// -----------------------------------------------------------------------
-// Mutations. Each takes the resource already fetched by report.rpc.ts for the
-// access check, so this does exactly one fewer Prisma round trip than
-// re-fetching.
+// Mutations take the resource already fetched by report.rpc.ts for the access
+// check, saving a second Prisma round trip.
type IReportInputForWrite = Omit<IReport, 'projectId'>;
@@ -451,8 +445,6 @@ export async function resetReportLayouts(
});
}
-// --- service ------------------------------------------------------------
-
export function createReportService(
deps: ServiceDeps,
_services: () => Services