Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

ralph(M15-103): Per-module fixes: packages/core/src/modules/chart - R1, R8, R13 and the findings no rule covers

ralph5ca50df44732
diff --git a/packages/core/src/modules/chart/chart.constants.ts b/packages/core/src/modules/chart/chart.constants.ts
new file mode 100644
index 000000000..4a8cdc5a4
--- /dev/null
+++ b/packages/core/src/modules/chart/chart.constants.ts
@@ -0,0 +1,62 @@
+// The chart module's isomorphic surface (ADR-022 R8): limits, zod schemas and
+// the column allow-lists shared by more than one file here. Imports zod and
+// sibling `*.constants.ts` only — anything needing `sql`, `deps` or a
+// ClickHouse type belongs in a service or a `src/` file.
+
+import { z } from 'zod';
+import { zChartEvent } from '../report/report.constants';
+
+/** User-flow depth: how many events one sankey path may span. */
+export const DEFAULT_SANKEY_STEPS = 5;
+const MIN_SANKEY_STEPS = 2;
+const MAX_SANKEY_STEPS = 10;
+
+export const zGetSankeyInput = z.object({
+ projectId: z.string(),
+ startDate: z.string(),
+ endDate: z.string(),
+ steps: z
+ .number()
+ .min(MIN_SANKEY_STEPS)
+ .max(MAX_SANKEY_STEPS)
+ .default(DEFAULT_SANKEY_STEPS),
+ mode: z.enum(['between', 'after', 'before']),
+ startEvent: zChartEvent,
+ endEvent: zChartEvent.optional(),
+ exclude: z.array(z.string()).default([]),
+ include: z.array(z.string()).optional(),
+});
+
+export type IGetSankeyInput = z.infer<typeof zGetSankeyInput> & {
+ timezone: string;
+};
+
+/**
+ * Top-level `profiles` columns a `profile.<field>` reference may resolve to,
+ * in V1's order. Both halves of the chart path read this one list: the SELECT
+ * side (`getProfilePropertySelect`, src/field-resolution.ts) and the filter
+ * side (`profileColumnSql`, src/table-filter-where.ts). Anything outside it is
+ * not a column name and must never reach the SQL text.
+ */
+export const PROFILE_SELECT_COLUMNS = [
+ 'id',
+ 'first_name',
+ 'last_name',
+ 'email',
+ 'avatar',
+ 'created_at',
+ 'last_seen_at',
+];
+
+/**
+ * Profile columns the funnel's and conversion's `profiles` join may expose
+ * beyond the `properties` Map. Narrower than PROFILE_SELECT_COLUMNS: those two
+ * statements join per event row, so `id` and `avatar` are never selected.
+ */
+export const JOINABLE_PROFILE_COLUMNS = [
+ 'email',
+ 'first_name',
+ 'last_name',
+ 'created_at',
+ 'last_seen_at',
+];
diff --git a/packages/core/src/modules/chart/chart.service.test.ts b/packages/core/src/modules/chart/chart.service.test.ts
index 1a846b937..781e26294 100644
--- a/packages/core/src/modules/chart/chart.service.test.ts
+++ b/packages/core/src/modules/chart/chart.service.test.ts
@@ -1,7 +1,7 @@
/**
* Integration tests for the chart module against the isolated `openpanel_test`
* databases (pinned by test/preload.ts) — the positive-row counterpart of
- * src/chart.sql.test.ts's EXPLAIN checks, and the one place the cohort and
+ * src/sql.test.ts's EXPLAIN checks, and the one place the cohort and
* all-cohorts paths run against seeded `cohort_members` + a Postgres cohort.
*
* Fixture (test/fixtures.ts, seeded per suite under its own project id so it
diff --git a/packages/core/src/modules/chart/chart.service.ts b/packages/core/src/modules/chart/chart.service.ts
index 8e24572ad..1e1cf7768 100644
--- a/packages/core/src/modules/chart/chart.service.ts
+++ b/packages/core/src/modules/chart/chart.service.ts
@@ -1,9 +1,9 @@
// Ported from packages/db/src/services/chart.service.ts, packages/db/src/
// engine/* and the handler bodies of packages/trpc/src/routers/chart.ts
// (M7-003). Every ClickHouse statement this module runs is a `sql` fragment
-// from src/chart.sql.ts, proven byte-equivalent to V1 in
-// src/chart.sql.proof.md; V1's chart.service and engine are re-export shims
-// onto this module and its router delegates here (DELEGATE PATTERN).
+// from src/sql.ts, proven byte-equivalent to V1 in src/sql.proof.md; V1's
+// chart.service and engine are re-export shims onto this module and its
+// router delegates here (DELEGATE PATTERN).
//
// The funnel, conversion, sankey and retention statements this module
// dispatches to live in the sibling `*.service.ts` files of this module
@@ -51,17 +51,6 @@ import {
getRetentionCohort,
} from './retention.service';
import { createSankeyService, getSankey } from './sankey.service';
-import {
- chartBucketProfilesQuery,
- eventFieldValuesQuery,
- eventNamesWithCountQuery,
- eventPropertyKeysQuery,
- eventPropertyValuesQuery,
- groupPropertyValuesQuery,
- profilePropertyValuesQuery,
- projectCardChartQuery,
- projectCardMetricsQuery,
-} from './src/chart.sql';
import { formatClickhouseDate } from './src/dates';
import { executeAggregateChart, executeChart } from './src/engine/execute';
import {
@@ -72,19 +61,18 @@ import {
normalizeEventField,
} from './src/field-resolution';
import { runQuery } from './src/run-query';
+import {
+ chartBucketProfilesQuery,
+ eventFieldValuesQuery,
+ eventNamesWithCountQuery,
+ eventPropertyKeysQuery,
+ eventPropertyValuesQuery,
+ groupPropertyValuesQuery,
+ profilePropertyValuesQuery,
+ projectCardChartQuery,
+ projectCardMetricsQuery,
+} from './src/sql';
-export {
- type ChartBucketProfilesInput,
- ChartCohortIdError,
-} from './src/chart.sql';
-export {
- type AggregateChartSqlInput,
- type ChartSqlInput,
- fetchCohortsMetadata,
- fetchProjectCohorts,
- getAggregateChartSql,
- getChartSql,
-} from './src/chart-statement';
export { rewriteProfilePropertyRefs } from './src/compiled';
export { executeAggregateChart, executeChart } from './src/engine/execute';
export {
@@ -126,6 +114,17 @@ export {
type FilterTableScope,
getEventFiltersWhereClause,
} from './src/filter-where';
+export {
+ type ChartBucketProfilesInput,
+ ChartCohortIdError,
+} from './src/sql';
+// Types only: `packages/core/src/index.ts` re-exports these two. The four
+// statement functions beside them had no consumer through this file — the
+// engine and the sibling services import ./src/statement directly (R13).
+export type {
+ AggregateChartSqlInput,
+ ChartSqlInput,
+} from './src/statement';
/**
* Cap on distinct event property keys returned to the picker. Projects in the
@@ -770,7 +769,6 @@ export function createChartService(
getFunnelGroup: funnel.getFunnelGroup,
/** The funnel row -> serie grouping (funnel.service.ts's `toSeries`). */
toFunnelSeries: funnel.toSeries,
- buildSessionsCte: funnel.buildSessionsCte,
getFunnelChart: (
chartInput: IReportInput
): ReturnType<typeof getFunnelChart> => getFunnelChart(deps, chartInput),
diff --git a/packages/core/src/modules/chart/conversion.service.ts b/packages/core/src/modules/chart/conversion.service.ts
index 7b39900ee..867b00cf6 100644
--- a/packages/core/src/modules/chart/conversion.service.ts
+++ b/packages/core/src/modules/chart/conversion.service.ts
@@ -9,7 +9,7 @@ import type { ServiceDeps, Services } from '../../services';
import type { IReportInput } from '../report/report.constants';
import { NOT_SET_VALUE } from '../report/report.constants';
import { mergeGlobalFilters, onlyReportEvents } from '../report/src/series';
-import { fetchCohortsMetadata } from './src/chart-statement';
+import { JOINABLE_PROFILE_COLUMNS } from './chart.constants';
import { compiledText } from './src/compiled';
import { type ConversionGroup, conversionQuery } from './src/conversion.sql';
import {
@@ -24,20 +24,12 @@ import {
joinFilterClauses,
} from './src/filter-where';
import { runQuery } from './src/run-query';
+import { fetchCohortsMetadata } from './src/statement';
/** Default funnel window, in hours, when the report does not set one. */
const DEFAULT_FUNNEL_WINDOW_HOURS = 24;
const SECONDS_PER_HOUR = 3600;
-/** Profile columns the join may expose, beyond the `properties` Map. */
-const JOINABLE_PROFILE_COLUMNS = [
- 'email',
- 'first_name',
- 'last_name',
- 'created_at',
- 'last_seen_at',
-];
-
const CONVERSION_STEP_COUNT = 2;
interface ConversionRow {
diff --git a/packages/core/src/modules/chart/funnel.service.ts b/packages/core/src/modules/chart/funnel.service.ts
index aa6055a73..0b533d1d3 100644
--- a/packages/core/src/modules/chart/funnel.service.ts
+++ b/packages/core/src/modules/chart/funnel.service.ts
@@ -13,7 +13,6 @@ import type {
IReportInput,
} from '../report/report.constants';
import { mergeGlobalFilters, onlyReportEvents } from '../report/src/series';
-import { fetchCohortsMetadata } from './src/chart-statement';
import { collectBreakdownCohortIds } from './src/field-resolution';
import {
EMPTY_BREAKDOWN_LABEL,
@@ -27,6 +26,7 @@ import {
knownFunnelBreakdowns,
} from './src/funnel.sql';
import { runQuery } from './src/run-query';
+import { fetchCohortsMetadata } from './src/statement';
export {
EMPTY_BREAKDOWN_LABEL,
@@ -446,7 +446,6 @@ export function createFunnelService(
return {
getFunnelGroup,
toSeries,
- buildSessionsCte,
getFunnel: (
input: Parameters<typeof getFunnel>[1]
): ReturnType<typeof getFunnel> => getFunnel(deps, input),
diff --git a/packages/core/src/modules/chart/sankey.service.ts b/packages/core/src/modules/chart/sankey.service.ts
index 9f46aae6c..98c7c9bf8 100644
--- a/packages/core/src/modules/chart/sankey.service.ts
+++ b/packages/core/src/modules/chart/sankey.service.ts
@@ -4,14 +4,14 @@
// re-export shim onto this module (DELEGATE PATTERN).
import type { SqlFragment } from '@openpanel/db/src/clickhouse/sql';
-import { z } from 'zod';
import type { ServiceDeps, Services } from '../../services';
import { getSettingsForProject } from '../organization/organization.service';
import {
chartColors,
+ type IChartEvent,
type IChartEventFilter,
- zChartEvent,
} from '../report/report.constants';
+import { DEFAULT_SANKEY_STEPS, type IGetSankeyInput } from './chart.constants';
import {
getEventFiltersWhereClause,
joinFilterClauses,
@@ -26,10 +26,6 @@ import {
sankeyTransitionsQuery,
} from './src/sankey.sql';
-const DEFAULT_SANKEY_STEPS = 5;
-const MIN_SANKEY_STEPS = 2;
-const MAX_SANKEY_STEPS = 10;
-
/** Destinations kept per node when the flow branches. */
const TOP_DESTINATIONS_PER_NODE = 3;
/** Links carrying less than this share of the entry sessions are dropped. */
@@ -37,25 +33,9 @@ const MIN_LINK_PERCENT = 0.25;
const PERCENT = 100;
const FIRST_STEP = 1;
-export const zGetSankeyInput = z.object({
- projectId: z.string(),
- startDate: z.string(),
- endDate: z.string(),
- steps: z
- .number()
- .min(MIN_SANKEY_STEPS)
- .max(MAX_SANKEY_STEPS)
- .default(DEFAULT_SANKEY_STEPS),
- mode: z.enum(['between', 'after', 'before']),
- startEvent: zChartEvent,
- endEvent: zChartEvent.optional(),
- exclude: z.array(z.string()).default([]),
- include: z.array(z.string()).optional(),
-});
-
-export type IGetSankeyInput = z.infer<typeof zGetSankeyInput> & {
- timezone: string;
-};
+// `packages/core/src/index.ts` names this file as the schema's source; the
+// schema itself lives in chart.constants.ts (R8).
+export { type IGetSankeyInput, zGetSankeyInput } from './chart.constants';
export interface SankeyNode {
id: string;
@@ -131,7 +111,7 @@ export function getRawWhereClause(
}
function toSankeyEvent(
- event: z.infer<typeof zChartEvent> | undefined
+ event: IChartEvent | undefined
): SankeyEvent | undefined {
return event
? {
diff --git a/packages/core/src/modules/chart/src/conversion.sql.ts b/packages/core/src/modules/chart/src/conversion.sql.ts
index 3fb1d035a..be75fedf2 100644
--- a/packages/core/src/modules/chart/src/conversion.sql.ts
+++ b/packages/core/src/modules/chart/src/conversion.sql.ts
@@ -18,7 +18,7 @@
import { type SqlFragment, sql } from '@openpanel/db/src/clickhouse/sql';
import type { IInterval } from '../../report/report.constants';
import { compiledText } from './compiled';
-import { CHART_TABLE } from './field-resolution';
+import { buildGroupsQuery, CHART_TABLE } from './field-resolution';
/** windowFunnel counts a conversion when both steps fired. */
const CONVERTED_STEP_COUNT = 2;
@@ -88,7 +88,7 @@ function profileJoin(fields: string[], projectId: string): SqlFragment {
function groupJoin(needed: boolean, projectId: string): SqlFragment {
return needed
- ? sql`ARRAY JOIN groups AS _group_id LEFT ANY JOIN (SELECT id, name, type, properties FROM ${sql.id(CHART_TABLE.groups)} FINAL WHERE project_id = ${sql.string(projectId)}) AS _g ON _g.id = _group_id`
+ ? sql`ARRAY JOIN groups AS _group_id LEFT ANY JOIN (${buildGroupsQuery(projectId)}) AS _g ON _g.id = _group_id`
: sql.empty;
}
diff --git a/packages/core/src/modules/chart/src/engine/compute.ts b/packages/core/src/modules/chart/src/engine/compute.ts
index 800bfab24..634a76fa9 100644
--- a/packages/core/src/modules/chart/src/engine/compute.ts
+++ b/packages/core/src/modules/chart/src/engine/compute.ts
@@ -4,10 +4,7 @@ import { alphabetIds } from '../../../report/report.constants';
import { evaluateFormula } from './formula';
import type { ConcreteSeries } from './types';
-/**
- * Compute formula series from fetched event series
- * Formulas reference event series using alphabet IDs (A, B, C, etc.)
- */
+/** Formulas reference the event series by alphabet ID (A, B, C, ...). */
export function compute(
fetchedSeries: ConcreteSeries[],
definitions: Array<{
@@ -18,7 +15,7 @@ export function compute(
): ConcreteSeries[] {
const results: ConcreteSeries[] = [...fetchedSeries];
- // Process formulas in order (they can reference previous formulas)
+ // In definition order: a formula may reference an earlier formula.
definitions.forEach((definition, formulaIndex) => {
if (definition.type !== 'formula') {
return;
@@ -29,20 +26,21 @@ export function compute(
return;
}
- // Group ALL series (events + previously computed formulas) by breakdown signature
- // Series with the same breakdown values should be computed together
+ // Series sharing a breakdown signature are computed together. `results`
+ // opens as a copy of `fetchedSeries`, so only the formulas earlier
+ // iterations appended still need adding.
const seriesByBreakdown = new Map<string, ConcreteSeries[]>();
- // Include both fetched event series AND previously computed formulas
const allSeries = [
...fetchedSeries,
- ...results.filter((s) => s.definitionIndex < formulaIndex),
+ ...results.filter(
+ (serie) =>
+ serie.definitionIndex < formulaIndex && !fetchedSeries.includes(serie)
+ ),
];
allSeries.forEach((serie) => {
- // Create breakdown signature: skip first name part (event/formula name) and use breakdown values
- // If name.length === 1, it means no breakdowns (just event name)
- // If name.length > 1, name[0] is event name, name[1+] are breakdown values
+ // name[0] is the event/formula name; name[1+] are the breakdown values.
const breakdownSignature =
serie.name.length > 1 ? serie.name.slice(1).join(':::') : '';
@@ -52,15 +50,12 @@ export function compute(
seriesByBreakdown.get(breakdownSignature)!.push(serie);
});
- // Compute formula for each breakdown group
for (const [breakdownSignature, breakdownSeries] of seriesByBreakdown) {
- // Map series by their definition index for formula evaluation
const seriesByIndex = new Map<number, ConcreteSeries>();
breakdownSeries.forEach((serie) => {
seriesByIndex.set(serie.definitionIndex, serie);
});
- // Get all unique dates across all series in this breakdown group
const allDates = new Set<string>();
breakdownSeries.forEach((serie) => {
serie.data.forEach((item) => {
@@ -121,7 +116,6 @@ export function compute(
const formulaData = sortedDates.map((date) => {
const scope: Record<string, number> = {};
- // Build scope using alphabet IDs (A, B, C, etc.)
definitions.slice(0, formulaIndex).forEach((_depDef, depIndex) => {
const readableId = alphabetIds[depIndex];
if (!readableId) {
@@ -152,7 +146,6 @@ export function compute(
}
});
- // Evaluate formula
const result = evaluateFormula(formula.formula, scope);
return {
@@ -162,11 +155,8 @@ export function compute(
};
});
- // Create concrete series for this formula
const templateSerie = breakdownSeries[0]!;
- // Extract breakdown values from template series name
- // name[0] is event/formula name, name[1+] are breakdown values
const breakdownValues =
templateSerie.name.length > 1 ? templateSerie.name.slice(1) : [];
diff --git a/packages/core/src/modules/chart/src/engine/execute.ts b/packages/core/src/modules/chart/src/engine/execute.ts
index f2a0110ef..e66e4053c 100644
--- a/packages/core/src/modules/chart/src/engine/execute.ts
+++ b/packages/core/src/modules/chart/src/engine/execute.ts
@@ -1,9 +1,6 @@
import { getChartPrevStartEndDate } from '@openpanel/shared';
import type { ServiceDeps } from '../../../../services';
-import {
- getOrganizationSubscriptionChartEndDate,
- getSettingsForProject,
-} from '../../../organization/organization.service';
+import { getOrganizationSubscriptionChartEndDate } from '../../../organization/organization.service';
import type {
FinalChart,
IReportInput,
@@ -38,7 +35,7 @@ export async function executeChart(
input: IReportInput
): Promise<FinalChart> {
const normalized = await normalizeWithinSubscription(deps, input);
- const executionPlan = await plan(deps, normalized);
+ const executionPlan = plan(normalized);
const computedSeries = compute(
await fetch(deps, executionPlan),
executionPlan.definitions
@@ -46,7 +43,7 @@ export async function executeChart(
let previousSeries: ConcreteSeries[] | null = null;
if (input.previous) {
- const previousPlan = await plan(deps, {
+ const previousPlan = plan({
...normalized,
...getChartPrevStartEndDate({
startDate: normalized.startDate,
@@ -74,7 +71,7 @@ export async function executeAggregateChart(
input: IReportInput
): Promise<FinalChart> {
const normalized = await normalizeWithinSubscription(deps, input);
- const { timezone } = await getSettingsForProject(deps, normalized.projectId);
+ const { timezone } = normalized;
const currentPeriod = {
startDate: normalized.startDate,
endDate: normalized.endDate,
diff --git a/packages/core/src/modules/chart/src/engine/fetch.ts b/packages/core/src/modules/chart/src/engine/fetch.ts
index 35fe0bb89..b48950e84 100644
--- a/packages/core/src/modules/chart/src/engine/fetch.ts
+++ b/packages/core/src/modules/chart/src/engine/fetch.ts
@@ -8,8 +8,8 @@ import type {
IGetChartDataInput,
} from '../../../report/report.constants';
import { alphabetIds } from '../../../report/report.constants';
-import { getAggregateChartSql, getChartSql } from '../chart-statement';
import { runQuery } from '../run-query';
+import { getAggregateChartSql, getChartSql } from '../statement';
import type { NormalizedInput } from './normalize';
import type { ConcreteSeries, Plan, SeriesDefinition } from './types';
diff --git a/packages/core/src/modules/chart/src/engine/formula.ts b/packages/core/src/modules/chart/src/engine/formula.ts
index 4e72e9227..bba41d6b0 100644
--- a/packages/core/src/modules/chart/src/engine/formula.ts
+++ b/packages/core/src/modules/chart/src/engine/formula.ts
@@ -156,7 +156,9 @@ function getCompiledFormula(formula: string): mathjs.EvalFunction {
/**
* Returns true when the formula is plain arithmetic over series references.
- * Used to reject hostile formulas at the API boundary before they are stored.
+ * Nothing validates a formula before it is stored today — the guard that
+ * actually holds is `compileFormula` inside `evaluateFormula`, which rejects
+ * per evaluation and renders the series as 0.
*/
export function isValidFormula(formula: string): boolean {
try {
diff --git a/packages/core/src/modules/chart/src/engine/normalize.ts b/packages/core/src/modules/chart/src/engine/normalize.ts
index babc51e5b..3c3273c0c 100644
--- a/packages/core/src/modules/chart/src/engine/normalize.ts
+++ b/packages/core/src/modules/chart/src/engine/normalize.ts
@@ -12,6 +12,8 @@ import type { SeriesDefinition } from './types';
export type NormalizedInput = IReportInputWithDates & {
series: SeriesDefinition[];
+ /** The project's timezone, resolved once here for the whole pipeline. */
+ timezone: string;
};
type LegacySeriesItem = Partial<IChartEvent> & { type?: string };
@@ -64,5 +66,6 @@ export async function normalize(
),
startDate,
endDate,
+ timezone,
};
}
diff --git a/packages/core/src/modules/chart/src/engine/plan.ts b/packages/core/src/modules/chart/src/engine/plan.ts
index cc1f0418a..a26d9dad3 100644
--- a/packages/core/src/modules/chart/src/engine/plan.ts
+++ b/packages/core/src/modules/chart/src/engine/plan.ts
@@ -1,6 +1,4 @@
import { slug } from '@openpanel/shared';
-import type { ServiceDeps } from '../../../../services';
-import { getSettingsForProject } from '../../../organization/organization.service';
import type { IChartEventItem } from '../../../report/report.constants';
import { alphabetIds } from '../../../report/report.constants';
import type { NormalizedInput } from './normalize';
@@ -10,12 +8,7 @@ import type { ConcreteSeries, Plan } from './types';
* One placeholder per event definition; breakdown expansion happens in fetch.
* Formulas are derived from event series, so they only exist after compute.
*/
-export async function plan(
- deps: ServiceDeps,
- normalized: NormalizedInput
-): Promise<Plan> {
- const { timezone } = await getSettingsForProject(deps, normalized.projectId);
-
+export function plan(normalized: NormalizedInput): Plan {
const concreteSeries: ConcreteSeries[] = [];
normalized.series.forEach((definition, index) => {
if (definition.type !== 'event') {
@@ -40,6 +33,6 @@ export async function plan(
concreteSeries,
definitions: normalized.series,
input: normalized,
- timezone,
+ timezone: normalized.timezone,
};
}
diff --git a/packages/core/src/modules/chart/src/field-resolution.sql.proof.md b/packages/core/src/modules/chart/src/field-resolution.sql.proof.md
index a1a7b11c8..0a825b8dd 100644
--- a/packages/core/src/modules/chart/src/field-resolution.sql.proof.md
+++ b/packages/core/src/modules/chart/src/field-resolution.sql.proof.md
@@ -22,7 +22,7 @@ comparison**.
**`cohort_members` is empty in the prod copy** (`SELECT count() FROM
cohort_members` = 0), so the five cohort cases are 0-row statement
equivalences; positive-row cohort coverage lives in `openpanel_test` (same
- precedent as `chart.sql.proof.md` and `filter-where.sql.proof.md`). The other
+ precedent as `sql.proof.md` and `filter-where.sql.proof.md`). The other
zero-count cases are listed under *Cases that return no rows, and why*.
- **Machine**: single-node ClickHouse 26.1.3.52 on a 4-vCPU box — timings are
directional only; production is 2 shards × 2 replicas
@@ -52,13 +52,13 @@ double quotes, or SQL keywords):
- `packages/core/src/modules/chart/src/field-resolution.ts:203,505` — the
`` `cohort-<id>` `` CTE name and the backtick-quoted CTE alias
`` `profile.properties.<key>` ``.
-- `packages/core/src/modules/chart/src/chart.sql.ts:175` — the CTE name in
+- `packages/core/src/modules/chart/src/sql.ts:175` — the CTE name in
`WITH <name> AS (...)`.
-- `packages/core/src/modules/chart/src/chart.sql.ts:274` — the double-quoted
+- `packages/core/src/modules/chart/src/sql.ts:274` — the double-quoted
`"profile.<field>"` CTE alias.
-- `packages/core/src/modules/chart/src/chart.sql.ts:340` — the
+- `packages/core/src/modules/chart/src/sql.ts:340` — the
`` `cohort-<id>` `` CTE name in a JOIN.
-- `packages/core/src/modules/chart/src/chart.sql.ts:434,435,469,580` — the
+- `packages/core/src/modules/chart/src/sql.ts:434,435,469,580` — the
`label_<n>` aliases / group-by keys and the `MATH_FUNCTION_BY_SEGMENT`
aggregate keyword.
- `packages/core/src/modules/chart/src/conversion.sql.ts:117` and
@@ -162,7 +162,7 @@ substrings are byte-identical to V1's; only the subject they run against
changed. Three test files did this with a local helper that renders
`toStatement()` and substitutes every `{pN:Type}` back to the literal V1
emitted (single-quoted, backslash-escaped, arrays as `(a, b)` — i.e.
-`sqlstring.escape`'s output). `chart.sql.test.ts` and `funnel.sql.test.ts`
+`sqlstring.escape`'s output). `sql.test.ts` and `funnel.sql.test.ts`
expose it as a third field `.text` on their existing `render()` result,
alongside the untouched `.sql` / `.params`; `table-filter-where.test.ts` folds
it into the existing `build()` helper, so its call sites are unchanged, and
@@ -173,11 +173,11 @@ and the params directly.
| test | before | after |
|---|---|---|
-| `chart.sql.test.ts` › `getChartSql` › ``qualifies properties[...] with `e.` when group join is present (fixes AMBIGUOUS_IDENTIFIER)`` | `expect(rendered.sql).not.toMatch(/(?<![._\w])properties\[/)` and `.toContain("e.properties['__query.utm_source']")` | same two assertions on `rendered.text` |
-| `chart.sql.test.ts` › `getChartSql` › `property metric (property_sum) with group join is unambiguous` | `expect(rendered.sql).toContain("e.properties['revenue_amount']")` | same on `rendered.text` |
-| `chart.sql.test.ts` › `getChartSql` › `routes bare utm_source filter through properties map` | `expect(rendered.sql).toContain("properties['__query.utm_source']")`, `.not.toMatch(/(?<![._\w])utm_source\s*=/)` | same two on `rendered.text` |
-| `chart.sql.test.ts` › `getAggregateChartSql` › `properties + group breakdown is unambiguous` | `expect(rendered.sql).toContain("e.properties['__query.utm_source']")` | same on `rendered.text` |
-| `chart.sql.test.ts` › `profile-property narrowing` › all five non-EXPLAIN tests (`projects only the referenced keys as scalar columns in the profile CTE`, `falls back to the full Map for wildcard refs`, `never narrows identifier-unsafe keys (backtick falls back to the Map)`, `collects the math-metric property too`, `creates the profile join for a metric-only profile property`) | `const { sql } = await getChartSql(...)`, assertions on `sql` | `const { text } = ...`, the SAME assertions on `text`. Every one of these tests names a `properties['<key>']` access, so the whole destructure moved rather than half of it; the two assertions in the set that carry no bound value (`properties as "profile.properties"`, `LEFT ANY JOIN profile ON profile.id = profile_id`) would still hold on `.sql` |
+| `sql.test.ts` › `getChartSql` › ``qualifies properties[...] with `e.` when group join is present (fixes AMBIGUOUS_IDENTIFIER)`` | `expect(rendered.sql).not.toMatch(/(?<![._\w])properties\[/)` and `.toContain("e.properties['__query.utm_source']")` | same two assertions on `rendered.text` |
+| `sql.test.ts` › `getChartSql` › `property metric (property_sum) with group join is unambiguous` | `expect(rendered.sql).toContain("e.properties['revenue_amount']")` | same on `rendered.text` |
+| `sql.test.ts` › `getChartSql` › `routes bare utm_source filter through properties map` | `expect(rendered.sql).toContain("properties['__query.utm_source']")`, `.not.toMatch(/(?<![._\w])utm_source\s*=/)` | same two on `rendered.text` |
+| `sql.test.ts` › `getAggregateChartSql` › `properties + group breakdown is unambiguous` | `expect(rendered.sql).toContain("e.properties['__query.utm_source']")` | same on `rendered.text` |
+| `sql.test.ts` › `profile-property narrowing` › all five non-EXPLAIN tests (`projects only the referenced keys as scalar columns in the profile CTE`, `falls back to the full Map for wildcard refs`, `never narrows identifier-unsafe keys (backtick falls back to the Map)`, `collects the math-metric property too`, `creates the profile join for a metric-only profile property`) | `const { sql } = await getChartSql(...)`, assertions on `sql` | `const { text } = ...`, the SAME assertions on `text`. Every one of these tests names a `properties['<key>']` access, so the whole destructure moved rather than half of it; the two assertions in the set that carry no bound value (`properties as "profile.properties"`, `LEFT ANY JOIN profile ON profile.id = profile_id`) would still hold on `.sql` |
| `funnel.sql.test.ts` › `profile breakdowns` › `adds the profiles join for a profile.properties breakdown` | ``expect(sql).toContain("properties['plan'] as `profile.properties.plan`")``; the two join assertions on `sql` | the first assertion on `text`; the two join assertions stay on `sql` |
| `funnel.sql.test.ts` › `cohort breakdowns` › `adds the cohort join for a cohort breakdown` | `expect(sql).toContain('Power users')`; the two alias assertions on `sql` | `expect(text).toContain('Power users')` — the cohort label is a bound value now; the two alias assertions stay on `sql` |
| `funnel.sql.test.ts` › `breakdown attribution` › `attributes event-property breakdowns at the entry step` | `expect(sql).toContain("argMinIf(properties['experiment'], created_at,")`; the GROUP BY assertions on `sql` | that one assertion on `chartText(...)`; the GROUP BY assertions stay on `sql` |
diff --git a/packages/core/src/modules/chart/src/field-resolution.ts b/packages/core/src/modules/chart/src/field-resolution.ts
index ca5f51cc3..a715dd5c6 100644
--- a/packages/core/src/modules/chart/src/field-resolution.ts
+++ b/packages/core/src/modules/chart/src/field-resolution.ts
@@ -19,6 +19,7 @@
import { type SqlFragment, sql } from '@openpanel/db/src/clickhouse/sql';
import type { IChartBreakdown } from '../../report/report.constants';
+import { PROFILE_SELECT_COLUMNS } from '../chart.constants';
import { compiledText } from './compiled';
export const CHART_TABLE = {
@@ -107,6 +108,11 @@ const ALL_COHORTS_UNKNOWN_LABEL = 'Unknown';
const IN_COHORT_LABEL = 'In Cohort';
const NOT_IN_COHORT_LABEL = 'Not In Cohort';
+/**
+ * The scalar `profiles` columns the profile CTE may project. Distinct from
+ * chart.constants.ts's PROFILE_SELECT_COLUMNS, which is the direct-SELECT
+ * allow-list: the CTE never carries `avatar` and does carry `properties`.
+ */
const PROFILE_CTE_SCALAR_FIELDS = [
'email',
'first_name',
@@ -122,19 +128,6 @@ export const PROFILE_CTE_FIELDS = [
...PROFILE_CTE_SCALAR_FIELDS,
];
-// Top-level profiles columns `profile.<field>` may select, in V1's order. Same
-// set as PROFILE_COLUMNS in ./table-filter-where.ts, which resolves the filter
-// side; keep the two in sync.
-const PROFILE_SELECT_COLUMNS = [
- 'id',
- 'first_name',
- 'last_name',
- 'email',
- 'avatar',
- 'created_at',
- 'last_seen_at',
-];
-
// Normalize an incoming field name into its canonical form. Returns a string
// suitable for `getSelectPropertyKey` / `getEventFiltersWhereClause` — i.e.
// either a top-level column name (`referrer_name`), a `properties.foo` /
@@ -196,7 +189,7 @@ export interface CohortMetadata {
/**
* The CTE name a single-cohort breakdown declares. Backtick-quoted because the
* id contains dashes, which `sql.id` rejects, so it goes through the module's
- * one text seam. Callers must validate the id first — chart.sql.ts builds the
+ * one text seam. Callers must validate the id first — sql.ts builds the
* same name behind `assertCohortId`.
*/
export function getCohortCteName(cohortId: string): SqlFragment {
@@ -248,6 +241,34 @@ export function buildAllCohortsMembershipQuery(projectId: string): SqlFragment {
`;
}
+/**
+ * The `if(notEmpty(...))` label a single-cohort breakdown selects. Callers
+ * must have validated the id — sql.ts guards with `assertCohortId`.
+ */
+export function cohortBreakdownLabelExpr(
+ cohortId: string,
+ cohortName?: string
+): SqlFragment {
+ const cohortAlias = getCohortAlias(cohortId);
+ const inLabel = sql.string(cohortName ?? IN_COHORT_LABEL);
+ const notInLabel = sql.string(
+ cohortName ? `Not ${cohortName}` : NOT_IN_COHORT_LABEL
+ );
+ return sql`if(notEmpty(${sql.id(`${cohortAlias}.profile_id`)}), ${inLabel}, ${notInLabel})`;
+}
+
+/**
+ * The membership subselect both filter compilers use for `inCohort` /
+ * `notInCohort`. V1 wrote a plain `IN (subquery)` here; ADR-013 conversions
+ * never change `IN` / `GLOBAL IN` in either direction (docs/ENVIRONMENT.md).
+ */
+export function buildCohortMembersSubselect(
+ cohortIds: string[],
+ projectId: string
+): SqlFragment {
+ return sql`(SELECT profile_id FROM ${sql.id(CHART_TABLE.cohortMembers)} FINAL WHERE cohort_id IN ${sql.array('String', cohortIds)} AND project_id = ${sql.string(projectId)})`;
+}
+
export function buildAllCohortsLabelExpr(
cohorts: CohortMetadata[],
alias = ALL_COHORTS_DEFAULT_ALIAS
@@ -306,6 +327,11 @@ export function transformPropertyKey(property: string) {
return `${match}['${property.replace(new RegExp(`^${match}.`), '')}']`;
}
+/** The `groups` lookup the chart, funnel and conversion statements all join. */
+export function buildGroupsQuery(projectId: string): SqlFragment {
+ return sql`SELECT id, name, type, properties FROM ${sql.id(CHART_TABLE.groups)} FINAL WHERE project_id = ${sql.string(projectId)}`;
+}
+
// Returns a SQL expression for a group property via the _g JOIN alias
// property format: "group.name", "group.type", "group.properties.plan"
export function getGroupPropertySql(property: string): SqlFragment {
@@ -345,8 +371,6 @@ export function getGroupPropertySelect(property: string): SqlFragment {
// Returns the SELECT expression when querying the profiles table directly (no join alias).
// Use for fetching distinct values for profile.* properties.
-// Lists the same profiles columns as PROFILE_COLUMNS in filter-where.service.ts,
-// which resolves profile.* on the filter side; keep the two in sync.
export function getProfilePropertySelect(property: string): SqlFragment {
const withoutPrefix = property.replace(/^profile\./, '');
if (PROFILE_SELECT_COLUMNS.includes(withoutPrefix)) {
@@ -411,12 +435,7 @@ export function getSelectPropertyKey(
const extractedCohortId = cohortId || extractCohortId(property);
if (extractedCohortId && projectId) {
- const cohortAlias = getCohortAlias(extractedCohortId);
- const inLabel = sql.string(cohortName ?? IN_COHORT_LABEL);
- const notInLabel = sql.string(
- cohortName ? `Not ${cohortName}` : NOT_IN_COHORT_LABEL
- );
- return sql`if(notEmpty(${sql.id(`${cohortAlias}.profile_id`)}), ${inLabel}, ${notInLabel})`;
+ return cohortBreakdownLabelExpr(extractedCohortId, cohortName);
}
if (property === 'has_profile') {
diff --git a/packages/core/src/modules/chart/src/filter-where.sql.proof.md b/packages/core/src/modules/chart/src/filter-where.sql.proof.md
index 03ba7ede6..2eeeddf92 100644
--- a/packages/core/src/modules/chart/src/filter-where.sql.proof.md
+++ b/packages/core/src/modules/chart/src/filter-where.sql.proof.md
@@ -22,7 +22,7 @@ statement here returns a single aggregate row, so no case needed set comparison.
prove nothing. **`cohort_members` is empty in the prod copy** (`SELECT count()
FROM cohort_members` = 0), so the two `inCohort` cases are 0-row statement
equivalences; positive-row cohort coverage lives in `openpanel_test` (same
- precedent as `chart.sql.proof.md`).
+ precedent as `sql.proof.md`).
- **Machine**: single-node ClickHouse 26.1.3.52 on a 4-vCPU box — timings are
directional only; production is 2 shards × 2 replicas
(`docs/ENVIRONMENT.md`). Two statements contain `IN (subquery)` on a
@@ -405,7 +405,7 @@ rejected. All three are pre-existing V1 behaviour and are reproduced, not fixed.
## `rewriteProfilePropertyRefs` — where the rewrite happens now
-V1 ran the rewrite over finished TEXT: `chart.sql.ts` and `funnel.sql.ts` called
+V1 ran the rewrite over finished TEXT: `sql.ts` and `funnel.sql.ts` called
`compiledTextWithProfileRefs(clause, keys)` on the compiler's string. With the
compiler returning fragments there is no string to rewrite, so the rewrite moved
to `compiled.ts`'s `fragmentWithProfileRefs(fragment, keys)`, which maps
@@ -417,9 +417,9 @@ That is the same rewrite, and now it is the same rewrite *by construction*.
`compiled.ts` recorded why the text version was safe: an escaped literal cannot
contain `['`, so the rewrite never matched a value. A bound param is not in the
text at all, so it cannot match even in principle — the accident became an
-invariant. Callers: `chart.sql.ts:358` and `funnel.sql.ts:149`.
+invariant. Callers: `sql.ts:358` and `funnel.sql.ts:149`.
`compiledTextWithProfileRefs` survives for the ONE thing still rendering text,
-the field resolver (`chart.sql.ts:332`, `funnel.sql.ts:180`).
+the field resolver (`sql.ts:332`, `funnel.sql.ts:180`).
Two proof cases exercise it end to end against the narrowed profile CTE — the
wrapper selects ``properties['country'] as `profile.properties.country` `` and
@@ -549,7 +549,7 @@ SELECT toStartOfDay(created_at) AS date, sum(sign) AS total FROM sessions FINAL
## `compiledText` callers left
Every `compiledText(` / `compiledTextWithProfileRefs(` call that wrapped a
-`getEventFiltersWhereClause` result is gone — `chart.sql.ts` (two sites),
+`getEventFiltersWhereClause` result is gone — `sql.ts` (two sites),
`funnel.sql.ts`, `retention.sql.ts`, `conversion.sql.ts` (via `stepCondition`),
`sankey.sql.ts` (via `sessionEventCte`) and `overview.sql.ts` (via `rawWhere`)
all interpolate the fragment directly now. What remains wraps **only**
@@ -560,18 +560,18 @@ field-resolution output or a static SQL literal — M12-003's job — plus
|---|---|
| `packages/core/src/modules/chart/src/filter-where.ts:144` | `getGroupPropertySql(name)` — field resolver |
| `packages/core/src/modules/chart/src/filter-where.ts:243` | `getSelectPropertyKey(...)` — field resolver |
-| `packages/core/src/modules/chart/src/chart.sql.ts:178` | a CTE name (static) |
-| `packages/core/src/modules/chart/src/chart.sql.ts:282` | a select-field list built from field-resolution output |
-| `packages/core/src/modules/chart/src/chart.sql.ts:332` | `spliced` — field resolver + profile-ref rewrite |
-| `packages/core/src/modules/chart/src/chart.sql.ts:348` | a cohort CTE name (static) |
-| `packages/core/src/modules/chart/src/chart.sql.ts:442` | a `label_N` alias (static) |
-| `packages/core/src/modules/chart/src/chart.sql.ts:443` | a `label_N` alias (static) |
-| `packages/core/src/modules/chart/src/chart.sql.ts:476` | a math aggregate name (closed set) |
-| `packages/core/src/modules/chart/src/chart.sql.ts:587` | a `label_N` alias (static) |
-| `packages/core/src/modules/chart/src/chart.sql.ts:691` | field-resolution select expression |
-| `packages/core/src/modules/chart/src/chart.sql.ts:700` | field-resolution select expression |
-| `packages/core/src/modules/chart/src/chart.sql.ts:713` | field-resolution select expression |
-| `packages/core/src/modules/chart/src/chart.sql.ts:797` | `getSelectPropertyKey(...)` — field resolver |
+| `packages/core/src/modules/chart/src/sql.ts:178` | a CTE name (static) |
+| `packages/core/src/modules/chart/src/sql.ts:282` | a select-field list built from field-resolution output |
+| `packages/core/src/modules/chart/src/sql.ts:332` | `spliced` — field resolver + profile-ref rewrite |
+| `packages/core/src/modules/chart/src/sql.ts:348` | a cohort CTE name (static) |
+| `packages/core/src/modules/chart/src/sql.ts:442` | a `label_N` alias (static) |
+| `packages/core/src/modules/chart/src/sql.ts:443` | a `label_N` alias (static) |
+| `packages/core/src/modules/chart/src/sql.ts:476` | a math aggregate name (closed set) |
+| `packages/core/src/modules/chart/src/sql.ts:587` | a `label_N` alias (static) |
+| `packages/core/src/modules/chart/src/sql.ts:691` | field-resolution select expression |
+| `packages/core/src/modules/chart/src/sql.ts:700` | field-resolution select expression |
+| `packages/core/src/modules/chart/src/sql.ts:713` | field-resolution select expression |
+| `packages/core/src/modules/chart/src/sql.ts:797` | `getSelectPropertyKey(...)` — field resolver |
| `packages/core/src/modules/chart/src/funnel.sql.ts:79` | `STRICT_INCREASE_MODE` (static) |
| `packages/core/src/modules/chart/src/funnel.sql.ts:180` | `getSelectPropertyKey(...)` — field resolver |
| `packages/core/src/modules/chart/src/funnel.sql.ts:189` | a `b_N` alias (static) |
@@ -617,7 +617,7 @@ byte-for-byte; only their `filterClauses` INPUT fixtures moved from
because `CompiledFilterClauses` is now `Record<string, SqlFragment>` while
`buildFilterWhere` (M12-003) still returns text. `overview.sql.test.ts` moved
its `rawFilterWhere: ''` fixtures to `rawFilterWhere: null` for the same reason.
-`chart.sql.test.ts` and `funnel.sql.test.ts` are untouched — they assert on
+`sql.test.ts` and `funnel.sql.test.ts` are untouched — they assert on
column expressions, which still come from the field resolver.
## Reproducing a case
diff --git a/packages/core/src/modules/chart/src/filter-where.ts b/packages/core/src/modules/chart/src/filter-where.ts
index 414704cae..7d18d3bf4 100644
--- a/packages/core/src/modules/chart/src/filter-where.ts
+++ b/packages/core/src/modules/chart/src/filter-where.ts
@@ -14,7 +14,7 @@ import { stripLeadingAndTrailingSlashes } from '@openpanel/shared';
import type { IChartEventFilter } from '../../report/report.constants';
import { getCohortIds } from '../../report/report.constants';
import {
- CHART_TABLE,
+ buildCohortMembersSubselect,
EVENT_FIELD_ALIASES,
EVENT_TOP_LEVEL_COLUMNS,
getGroupPropertySql,
@@ -111,9 +111,7 @@ export function getEventFiltersWhereClause(
const profileIdExpr = eventsAlias
? sql.id(`${eventsAlias}.profile_id`)
: sql.id('profile_id');
- // V1 wrote a plain `IN (subquery)` here; ADR-013 conversions never
- // change IN/GLOBAL IN in either direction (docs/ENVIRONMENT.md).
- const members = sql`(SELECT profile_id FROM ${sql.id(CHART_TABLE.cohortMembers)} FINAL WHERE cohort_id IN ${sql.array('String', cohortIds)} AND project_id = ${sql.string(projectId)})`;
+ const members = buildCohortMembersSubselect(cohortIds, projectId);
where[id] =
operator === 'notInCohort'
? sql`${profileIdExpr} NOT IN ${members}`
diff --git a/packages/core/src/modules/chart/src/funnel.sql.test.ts b/packages/core/src/modules/chart/src/funnel.sql.test.ts
index e5887f75b..16085f1bf 100644
--- a/packages/core/src/modules/chart/src/funnel.sql.test.ts
+++ b/packages/core/src/modules/chart/src/funnel.sql.test.ts
@@ -2,7 +2,7 @@
* SQL-shape tests for the funnel statements. Ported from
* packages/db/src/services/funnel-sql.test.ts (M7-004).
*
- * Strategy, same as chart.sql.test.ts: render the statement, then run
+ * Strategy, same as sql.test.ts: render the statement, then run
* `EXPLAIN <sql>` with its bound params against the isolated `openpanel_test`
* ClickHouse (pinned by test/preload.ts). EXPLAIN parses and resolves columns
* without executing, so a breakdown expression referencing a join alias that
diff --git a/packages/core/src/modules/chart/src/funnel.sql.ts b/packages/core/src/modules/chart/src/funnel.sql.ts
index b664f7210..695a91808 100644
--- a/packages/core/src/modules/chart/src/funnel.sql.ts
+++ b/packages/core/src/modules/chart/src/funnel.sql.ts
@@ -28,8 +28,10 @@ import type {
IChartBreakdown,
IChartEvent,
} from '../../report/report.constants';
+import { JOINABLE_PROFILE_COLUMNS } from '../chart.constants';
import { compiledText, fragmentWithProfileRefs } from './compiled';
import {
+ buildGroupsQuery,
buildInlineCohortJoin,
CHART_TABLE,
type CohortMetadata,
@@ -53,15 +55,6 @@ export const EMPTY_BREAKDOWN_LABEL = 'Not set';
/** The funnel CTE's events alias — `getSelectPropertyKey` needs the same one. */
const EVENTS_ALIAS = 'events';
-/** Profile columns the funnel's profiles CTE may expose beyond `properties`. */
-const JOINABLE_PROFILE_COLUMNS = [
- 'email',
- 'first_name',
- 'last_name',
- 'created_at',
- 'last_seen_at',
-];
-
/**
* windowFunnel's 'strict_increase' mode requires every step's timestamp to be
* strictly greater than the previous step's, so same-timestamp sequences
@@ -349,7 +342,7 @@ export function funnelBase(input: FunnelBaseInput): FunnelBase {
)} AND ${stepPreFilter} GROUP BY ${sql.join(groupByKeys)}`;
const groupsCte = needsGroupArrayJoin
- ? sql`_g AS (SELECT id, name, type, properties FROM ${sql.id(CHART_TABLE.groups)} FINAL WHERE project_id = ${sql.string(projectId)}), `
+ ? sql`_g AS (${buildGroupsQuery(projectId)}), `
: sql.empty;
// windowFunnel is computed per primary key, so the `funnel` CTE only has to
@@ -416,7 +409,10 @@ export function funnelProfilesQuery(
return sql`${base.ctes}SELECT DISTINCT profile_id FROM funnel WHERE ${sql.join(where, ' AND ')} LIMIT ${sql.uint64(input.limit)}`;
}
-/** Exported by V1 as `buildSessionsCte`; no in-repo caller, kept as published API. */
+/**
+ * Exported by V1 as `buildSessionsCte`; no in-repo caller. Reachable only
+ * through `packages/core/src/index.ts`, which still publishes the wrapper.
+ */
export function funnelSessionsQuery(input: {
projectId: string;
startDate: string;
diff --git a/packages/core/src/modules/chart/src/chart.sql.proof.md b/packages/core/src/modules/chart/src/sql.proof.md
similarity index 99%
rename from packages/core/src/modules/chart/src/chart.sql.proof.md
rename to packages/core/src/modules/chart/src/sql.proof.md
index 851e5a407..77eb0860c 100644
--- a/packages/core/src/modules/chart/src/chart.sql.proof.md
+++ b/packages/core/src/modules/chart/src/sql.proof.md
@@ -1,8 +1,8 @@
-# chart.sql.ts — V1 → V2 result-set proof (M7-003)
+# sql.ts — V1 → V2 result-set proof (M7-003)
-Every query builder in `chart.sql.ts` was executed twice against the same data — once as V1 (the `git HEAD` `packages/db/src/services/chart.service.ts` `getChartSql` / `getAggregateChartSql`, and the `packages/trpc/src/routers/chart.ts` clix / `createSqlBuilder` texts, `f1573b1a`) and once as V2 (the fragment returned by the core builder, bound through `query_params`) — through one `@clickhouse/client` with `format: 'JSON'` and the same `session_timezone`. The two responses were compared on `data` (every row, every column, as JSON) and `meta` (column names and types). Chart statements were compared as row **sets**: V1's `ORDER BY date ASC WITH FILL` leaves rows sharing a `date` in arbitrary order, and three V1-vs-V1 runs of the 191-row wildcard-breakdown case produced three different orders; the engine groups rows by label, so order is not part of the contract. Statements without an `ORDER BY` (values lists, getProfiles) were compared as sets for the same reason.
+Every query builder in `sql.ts` was executed twice against the same data — once as V1 (the `git HEAD` `packages/db/src/services/chart.service.ts` `getChartSql` / `getAggregateChartSql`, and the `packages/trpc/src/routers/chart.ts` clix / `createSqlBuilder` texts, `f1573b1a`) and once as V2 (the fragment returned by the core builder, bound through `query_params`) — through one `@clickhouse/client` with `format: 'JSON'` and the same `session_timezone`. The two responses were compared on `data` (every row, every column, as JSON) and `meta` (column names and types). Chart statements were compared as row **sets**: V1's `ORDER BY date ASC WITH FILL` leaves rows sharing a `date` in arbitrary order, and three V1-vs-V1 runs of the 191-row wildcard-breakdown case produced three different orders; the engine groups rows by label, so order is not part of the contract. Statements without an `ORDER BY` (values lists, getProfiles) were compared as sets for the same reason.
-- **Date**: 2026-09-04. **Data**: local prod-copy `openpanel` (319,499,000 events, static). Projects: `verdict` / `secure-privacy` (UTC), `bayse` (America/New_York), `website-8103` (Asia/Kolkata), `dream-mate` (Australia/Sydney), `authc-project-a1` (the only project with a Postgres cohort; `cohort_members` is empty in the prod-copy, so cohort cases prove statement equivalence on 0 rows — positive-row coverage is `chart.sql.test.ts` on `openpanel_test`).
+- **Date**: 2026-09-04. **Data**: local prod-copy `openpanel` (319,499,000 events, static). Projects: `verdict` / `secure-privacy` (UTC), `bayse` (America/New_York), `website-8103` (Asia/Kolkata), `dream-mate` (Australia/Sydney), `authc-project-a1` (the only project with a Postgres cohort; `cohort_members` is empty in the prod-copy, so cohort cases prove statement equivalence on 0 rows — positive-row coverage is `sql.test.ts` on `openpanel_test`).
- **Machine**: single-node ClickHouse 26.1.3.52 on a 4-vCPU box — timings are directional only; production is 2 shards × 2 replicas (`docs/ENVIRONMENT.md`). The chart statements contain no `IN (subquery)`; cohort membership is a CTE joined on `profile_id`, unchanged from V1. `rows_read` is often lower for V2 (e.g. 245717 → 98299 on the multi-series case): ClickHouse's index analysis treats a bound `{p:String}` differently from an inline literal for the `created_at` range, with identical result sets — a perf note, not a semantic change.
- **Verdict**: 73 cases (28 `chartSeriesQuery`, 15 `aggregateChartQuery`, 30 router queries), all IDENTICAL. 5 of the 73 are **IDENTICAL ERROR**: V1 defects reproduced byte-for-byte on both sides, listed below, not fixed here (out of scope — the task is byte-equivalence, and each is recorded as debt in the task report).
diff --git a/packages/core/src/modules/chart/src/chart.sql.test.ts b/packages/core/src/modules/chart/src/sql.test.ts
similarity index 98%
rename from packages/core/src/modules/chart/src/chart.sql.test.ts
rename to packages/core/src/modules/chart/src/sql.test.ts
index 22ea2af44..3552cedcf 100644
--- a/packages/core/src/modules/chart/src/chart.sql.test.ts
+++ b/packages/core/src/modules/chart/src/sql.test.ts
@@ -28,7 +28,7 @@ import type {
IChartBreakdown,
IChartEvent,
} from '../../report/report.constants';
-import type { AggregateChartSqlInput, ChartSqlInput } from './chart-statement';
+import type { AggregateChartSqlInput, ChartSqlInput } from './statement';
const cohortFindMany = mock(async () => []);
// Plain-object snapshot, not the live binding — restoring from the namespace
@@ -46,8 +46,8 @@ afterAll(async () => {
let ch: typeof import('@openpanel/db/src/clickhouse/client').ch;
let deps: import('../../../services').ServiceDeps;
-let buildChartSql: typeof import('./chart-statement').getChartSql;
-let buildAggregateChartSql: typeof import('./chart-statement').getAggregateChartSql;
+let buildChartSql: typeof import('./statement').getChartSql;
+let buildAggregateChartSql: typeof import('./statement').getAggregateChartSql;
// IGetChartDataInput has display-only fields (metric, chartType, previous,
// etc.) that the SQL builders ignore; loosen the input type here.
@@ -134,7 +134,7 @@ beforeAll(async () => {
({
getChartSql: buildChartSql,
getAggregateChartSql: buildAggregateChartSql,
- } = await import('./chart-statement'));
+ } = await import('./statement'));
// EXPLAIN needs the openpanel_test schema to resolve columns against.
const { bootstrapTestDatabases } = await import(
'../../../../../../test/bootstrap-databases'
diff --git a/packages/core/src/modules/chart/src/chart.sql.ts b/packages/core/src/modules/chart/src/sql.ts
similarity index 92%
rename from packages/core/src/modules/chart/src/chart.sql.ts
rename to packages/core/src/modules/chart/src/sql.ts
index f0ffe638f..9b9ddb521 100644
--- a/packages/core/src/modules/chart/src/chart.sql.ts
+++ b/packages/core/src/modules/chart/src/sql.ts
@@ -5,7 +5,7 @@
// project id, event name, dates, timezone, cohort ids and names, limits —
// bound as a `{pN:Type}` parameter instead of an escaped literal. Each
// builder's result set was diffed against V1's on the local prod-copy; the
-// statements, params, row counts and timings are in chart.sql.proof.md.
+// statements, params, row counts and timings are in sql.proof.md.
//
// The field resolver and filter compiler still render text (see compiled.ts);
// their output is spliced, everything else is bound. Dates bind as V1's own
@@ -32,8 +32,13 @@ import type {
import { compiledText, fragmentWithProfileRefs } from './compiled';
import { formatClickhouseDate } from './dates';
import {
+ buildAllCohortsLabelExpr,
+ buildAllCohortsMembershipQuery,
+ buildCohortMembershipQuery,
+ buildGroupsQuery,
CHART_TABLE,
type CohortMetadata,
+ cohortBreakdownLabelExpr,
collectBreakdownCohortIds,
collectProfileCteFields,
collectProfilePropertyKeys,
@@ -50,7 +55,6 @@ import { getEventFiltersWhereClause } from './filter-where';
const EVENTS_ALIAS = 'e';
const ALL_COHORTS_ALIAS = '_all_cohorts';
-const ALL_COHORTS_UNKNOWN_LABEL = 'Unknown';
// Cohort ids are Postgres uuids; they also name a CTE and a join alias, so
// anything else is refused rather than inlined (V1 inlined it unchecked).
const COHORT_ID_PATTERN = /^[A-Za-z0-9_-]+$/;
@@ -211,56 +215,6 @@ function orderByClause(orderBy: Parts): SqlFragment {
// --- shared chart body ------------------------------------------------------
-function cohortMembershipCte(cohortId: string, projectId: string): SqlFragment {
- return sql`
- SELECT profile_id
- FROM ${sql.id(CHART_TABLE.cohortMembers)} FINAL
- WHERE cohort_id = ${sql.string(cohortId)}
- AND project_id = ${sql.string(projectId)}
- `;
-}
-
-function allCohortsMembershipCte(projectId: string): SqlFragment {
- return sql`
- SELECT profile_id, cohort_id
- FROM ${sql.id(CHART_TABLE.cohortMembers)} FINAL
- WHERE project_id = ${sql.string(projectId)}
- `;
-}
-
-function allCohortsLabelExpr(cohorts: CohortMetadata[]): SqlFragment {
- if (cohorts.length === 0) {
- return sql`${sql.string(ALL_COHORTS_UNKNOWN_LABEL)}`;
- }
- const ids = sql.array(
- 'String',
- cohorts.map((cohort) => cohort.id)
- );
- const names = sql.array(
- 'String',
- cohorts.map((cohort) => cohort.name)
- );
- return sql`transform(${sql.id(`${ALL_COHORTS_ALIAS}.cohort_id`)}, ${ids}, ${names}, ${sql.string(ALL_COHORTS_UNKNOWN_LABEL)})`;
-}
-
-function cohortBreakdownLabel(
- cohortId: string,
- cohortName: string | undefined
-): SqlFragment {
- const alias = sql.id(
- `${getCohortAlias(assertCohortId(cohortId))}.profile_id`
- );
- const inLabel = sql.string(cohortName ?? 'In Cohort');
- const notInLabel = sql.string(
- cohortName ? `Not ${cohortName}` : 'Not In Cohort'
- );
- return sql`if(notEmpty(${alias}), ${inLabel}, ${notInLabel})`;
-}
-
-function groupsCte(projectId: string): SqlFragment {
- return sql`SELECT id, name, type, properties FROM ${sql.id(CHART_TABLE.groups)} FINAL WHERE project_id = ${sql.string(projectId)}`;
-}
-
function profileCteSelectField(
field: string,
profileKeys: string[],
@@ -327,7 +281,7 @@ function chartBody({
const joins: Parts = {};
if (hasAllCohortsBreakdown) {
- ctes[ALL_COHORTS_ALIAS] = allCohortsMembershipCte(projectId);
+ ctes[ALL_COHORTS_ALIAS] = buildAllCohortsMembershipQuery(projectId);
joins[ALL_COHORTS_ALIAS] =
sql`INNER JOIN _all_cohorts ON _all_cohorts.profile_id = e.profile_id`;
}
@@ -335,7 +289,7 @@ function chartBody({
for (const cohortId of cohortIds) {
const cteName = `\`cohort-${cohortId}\``;
const alias = getCohortAlias(cohortId);
- ctes[cteName] = cohortMembershipCte(cohortId, projectId);
+ ctes[cteName] = buildCohortMembershipQuery(cohortId, projectId);
joins[`cohort_${cohortId}`] =
sql`LEFT ANY JOIN ${compiledText(cteName)} AS ${sql.id(alias)} ON ${sql.id(`${alias}.profile_id`)} = e.profile_id`;
}
@@ -365,7 +319,7 @@ function chartBody({
anyRefOn('group.', profileRefs) || event.segment === 'group';
if (needsGroupArrayJoin) {
- ctes._g = groupsCte(projectId);
+ ctes._g = buildGroupsQuery(projectId);
joins.groups = sql`ARRAY JOIN groups AS _group_id`;
joins.groups_table = sql`LEFT ANY JOIN _g ON _g.id = _group_id`;
}
@@ -401,11 +355,14 @@ function breakdownLabel(
Pick<ChartBody, 'profileKeys'>
): SqlFragment {
if (isAllCohortsBreakdown(breakdown.name)) {
- return allCohortsLabelExpr(allCohorts);
+ return buildAllCohortsLabelExpr(allCohorts, ALL_COHORTS_ALIAS);
}
const cohortId = extractCohortId(breakdown.name);
if (cohortId) {
- return cohortBreakdownLabel(cohortId, cohortMetadata.get(cohortId)?.name);
+ return cohortBreakdownLabelExpr(
+ assertCohortId(cohortId),
+ cohortMetadata.get(cohortId)?.name
+ );
}
return fragmentWithProfileRefs(
getSelectPropertyKey(
@@ -781,7 +738,7 @@ export function chartBucketProfilesQuery(
].some((name) => name.startsWith('group.'));
if (needsGroupJoin) {
joins.groups = sql`ARRAY JOIN groups AS _group_id`;
- joins.groups_cte = sql`LEFT ANY JOIN (${groupsCte(projectId)}) AS _g ON _g.id = _group_id`;
+ joins.groups_cte = sql`LEFT ANY JOIN (${buildGroupsQuery(projectId)}) AS _g ON _g.id = _group_id`;
}
for (const [key, value] of Object.entries(breakdowns)) {
diff --git a/packages/core/src/modules/chart/src/chart-statement.ts b/packages/core/src/modules/chart/src/statement.ts
similarity index 92%
rename from packages/core/src/modules/chart/src/chart-statement.ts
rename to packages/core/src/modules/chart/src/statement.ts
index eb78d5c81..2518bef25 100644
--- a/packages/core/src/modules/chart/src/chart-statement.ts
+++ b/packages/core/src/modules/chart/src/statement.ts
@@ -1,11 +1,13 @@
// V1's `getChartSql` / `getAggregateChartSql`: the Postgres cohort lookups a
-// chart needs, then the pure fragment builders in chart.sql.ts. Lives beside
-// them (not in chart.service.ts) so the engine can import it without pulling
-// the service — which imports the engine — into a cycle.
+// chart needs, then the pure fragment builders in sql.ts. Lives beside them
+// (not in chart.service.ts) so the engine can import it without pulling the
+// service — which imports the engine — into a cycle.
import type { SqlFragment } from '@openpanel/db/src/clickhouse/sql';
import type { ServiceDeps } from '../../../services';
import type { IGetChartDataInput } from '../../report/report.constants';
+import type { CohortMetadata } from './field-resolution';
+import { collectBreakdownCohortIds } from './field-resolution';
import {
aggregateChartQuery,
chartSeriesQuery,
@@ -13,9 +15,7 @@ import {
type ResolvedChartBreakdowns,
requestsAllCohortsBreakdown,
withoutEmptyAllCohortsBreakdown,
-} from './chart.sql';
-import type { CohortMetadata } from './field-resolution';
-import { collectBreakdownCohortIds } from './field-resolution';
+} from './sql';
export type ChartSqlInput = IGetChartDataInput & { timezone: string };
export type AggregateChartSqlInput = Omit<
diff --git a/packages/core/src/modules/chart/src/table-filter-where.ts b/packages/core/src/modules/chart/src/table-filter-where.ts
index a2df18cc9..6f2d9ff73 100644
--- a/packages/core/src/modules/chart/src/table-filter-where.ts
+++ b/packages/core/src/modules/chart/src/table-filter-where.ts
@@ -19,8 +19,9 @@ import {
type IChartEventFilterValue,
type IChartFilterValueType,
} from '../../report/report.constants';
+import { PROFILE_SELECT_COLUMNS } from '../chart.constants';
import { formatClickhouseDate } from './dates';
-import { CHART_TABLE } from './field-resolution';
+import { buildCohortMembersSubselect, CHART_TABLE } from './field-resolution';
import {
buildTypedClauseFragment,
hasTypedCast,
@@ -268,22 +269,6 @@ function compileScalarClause(
return null;
}
-/**
- * Profiles columns a `profile.<field>` filter may resolve to. Anything else is
- * not a column name and must not reach the SQL text. Kept in sync with
- * `getProfilePropertySelect` in field-resolution.ts, which lists the same set
- * for the SELECT side.
- */
-const PROFILE_COLUMNS = [
- 'id',
- 'first_name',
- 'last_name',
- 'email',
- 'avatar',
- 'created_at',
- 'last_seen_at',
-];
-
/** Group fields that are columns rather than a `properties` lookup. */
const GROUP_COLUMNS = ['name', 'type', 'id'];
@@ -299,8 +284,8 @@ function profileColumnSql(name: string): SqlFragment | null {
const key = withoutPrefix.replace(/^properties\./, '');
return sql`properties[${sql.string(key)}]`;
}
- if (PROFILE_COLUMNS.includes(withoutPrefix)) {
- return sql.id(withoutPrefix, PROFILE_COLUMNS);
+ if (PROFILE_SELECT_COLUMNS.includes(withoutPrefix)) {
+ return sql.id(withoutPrefix, PROFILE_SELECT_COLUMNS);
}
return null;
}
@@ -351,7 +336,7 @@ function buildCohortClause(
return null;
}
const profileId = sql.id(ctx.profileIdExpr);
- const members = sql`(SELECT profile_id FROM ${sql.id(CHART_TABLE.cohortMembers)} FINAL WHERE cohort_id IN ${sql.array('String', cohortIds)} AND project_id = ${sql.string(projectId)})`;
+ const members = buildCohortMembersSubselect(cohortIds, projectId);
return filter.operator === 'notInCohort'
? sql`${profileId} NOT IN ${members}`
: sql`${profileId} IN ${members}`;