πͺ’ Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
chore(query-ast): add golden-SQL recording/diff harness (#16364)
Niklas Semmler6cf7c2ac18fe
diff --git a/packages/shared/AGENTS.md b/packages/shared/AGENTS.mdindex fb10de4923..814a48eec9 100644--- a/packages/shared/AGENTS.md+++ b/packages/shared/AGENTS.md@@ -24,6 +24,11 @@- Queue payload schemas: `src/server/queues.ts`- Queue helpers: `src/server/redis/*`- Dashboard/monitor query feature (data model + server-only builder/executor): `src/features/query/*`+- Query-builder AST (server half, WIP): `src/server/query-ast/*` β golden-SQL+ recording/diff harness that captures the current SQL at the+ `src/server/repositories/clickhouse.ts` exec seam and normalizes it via+ `clickhouse format` for snapshot comparison. Every migrated call site is+ proven against its baseline here.- Postgres schema: `prisma/schema.prisma`- For unstable public eval APIs, the public `evaluatorId` is currently theexact `EvalTemplate.id`. Latest-version family grouping is derived fromdiff --git a/packages/shared/src/server/query-ast/README.md b/packages/shared/src/server/query-ast/README.mdnew file mode 100644index 0000000000..a094261dd9--- /dev/null+++ b/packages/shared/src/server/query-ast/README.md@@ -0,0 +1,27 @@+# query-ast (server)++Server-only half of the query-builder AST module. Compiler,+validation passes, physical registry, execution context, exec integration, and+golden-check test utils live here, behind the `@langfuse/shared/src/server`+export boundary. The client-safe half (node types, predicate AST, builder+sugar, semantic field metadata, `FilterState` embedding) will land under+`packages/shared/src/query-ast/`.++## What's here today++Only the golden-SQL harness β the thing every later migration is tested+against:++- `goldenHarness.ts` β captures SQL at the `repositories/clickhouse.ts` exec+ seam in test mode, then normalizes it (`clickhouse format` + positional param+ names) for snapshot comparison. No ClickHouse server needed.+- `environments.golden.test.ts` β baselines the current SQL shapes of+ `environments.getEnvironmentsForProject` (both write-mode branches Γ the+ timestamp bound). This is the tier-0 exit target for the spike: once+ `environments.ts` compiles from the AST, it must reproduce these snapshots.++Regenerate baselines with `-u` after an intentional SQL change:++```+pnpm --filter @langfuse/shared run test src/server/query-ast -- -u+```diff --git a/packages/shared/src/server/query-ast/__snapshots__/environments.golden.test.ts.snap b/packages/shared/src/server/query-ast/__snapshots__/environments.golden.test.ts.snapnew file mode 100644index 0000000000..59b29e1a43--- /dev/null+++ b/packages/shared/src/server/query-ast/__snapshots__/environments.golden.test.ts.snap@@ -0,0 +1,137 @@+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html++exports[`golden: environments.getEnvironmentsForProject > writeMode=events_only fromTimestamp=set 1`] = `+[+ {+ "fn": "queryClickhouse",+ "params": {+ "param1": "golden-project",+ "param2": 2026-01-01T00:00:00.000Z,+ },+ "route": "environments.getEnvironmentsForProject",+ "sql": "SELECT DISTINCT environment+FROM events_core+WHERE (project_id = {param1:String}) AND (start_time >= {param2:DateTime64(3)})",+ },+ {+ "fn": "queryClickhouse",+ "params": {+ "param1": "golden-project",+ "param2": [+ "NUMERIC",+ "BOOLEAN",+ "CATEGORICAL",+ "TEXT",+ ],+ "param3": 2026-01-01T00:00:00.000Z,+ },+ "route": "environments.getEnvironmentsForProject",+ "sql": "SELECT DISTINCT environment+FROM scores+WHERE (project_id = {param1:String}) AND (data_type IN ({param2:Array(String)})) AND (timestamp >= {param3:DateTime64(3)})",+ },+]+`;++exports[`golden: environments.getEnvironmentsForProject > writeMode=events_only fromTimestamp=unset 1`] = `+[+ {+ "fn": "queryClickhouse",+ "params": {+ "param1": "golden-project",+ },+ "route": "environments.getEnvironmentsForProject",+ "sql": "SELECT DISTINCT environment+FROM events_core+WHERE project_id = {param1:String}",+ },+ {+ "fn": "queryClickhouse",+ "params": {+ "param1": "golden-project",+ "param2": [+ "NUMERIC",+ "BOOLEAN",+ "CATEGORICAL",+ "TEXT",+ ],+ },+ "route": "environments.getEnvironmentsForProject",+ "sql": "SELECT DISTINCT environment+FROM scores+WHERE (project_id = {param1:String}) AND (data_type IN ({param2:Array(String)}))",+ },+]+`;++exports[`golden: environments.getEnvironmentsForProject > writeMode=legacy fromTimestamp=set 1`] = `+[+ {+ "fn": "queryClickhouse",+ "params": {+ "param1": "golden-project",+ "param2": 2026-01-01T00:00:00.000Z,+ },+ "route": "environments.getEnvironmentsForProject",+ "sql": "SELECT DISTINCT environment+FROM traces+WHERE (project_id = {param1:String}) AND (timestamp >= {param2:DateTime64(3)})+UNION ALL+SELECT DISTINCT environment+FROM observations+WHERE (project_id = {param1:String}) AND (start_time >= {param2:DateTime64(3)})",+ },+ {+ "fn": "queryClickhouse",+ "params": {+ "param1": "golden-project",+ "param2": [+ "NUMERIC",+ "BOOLEAN",+ "CATEGORICAL",+ "TEXT",+ ],+ "param3": 2026-01-01T00:00:00.000Z,+ },+ "route": "environments.getEnvironmentsForProject",+ "sql": "SELECT DISTINCT environment+FROM scores+WHERE (project_id = {param1:String}) AND (data_type IN ({param2:Array(String)})) AND (timestamp >= {param3:DateTime64(3)})",+ },+]+`;++exports[`golden: environments.getEnvironmentsForProject > writeMode=legacy fromTimestamp=unset 1`] = `+[+ {+ "fn": "queryClickhouse",+ "params": {+ "param1": "golden-project",+ },+ "route": "environments.getEnvironmentsForProject",+ "sql": "SELECT DISTINCT environment+FROM traces+WHERE project_id = {param1:String}+UNION ALL+SELECT DISTINCT environment+FROM observations+WHERE project_id = {param1:String}",+ },+ {+ "fn": "queryClickhouse",+ "params": {+ "param1": "golden-project",+ "param2": [+ "NUMERIC",+ "BOOLEAN",+ "CATEGORICAL",+ "TEXT",+ ],+ },+ "route": "environments.getEnvironmentsForProject",+ "sql": "SELECT DISTINCT environment+FROM scores+WHERE (project_id = {param1:String}) AND (data_type IN ({param2:Array(String)}))",+ },+]+`;diff --git a/packages/shared/src/server/query-ast/environments.golden.test.ts b/packages/shared/src/server/query-ast/environments.golden.test.tsnew file mode 100644index 0000000000..c8d9996f09--- /dev/null+++ b/packages/shared/src/server/query-ast/environments.golden.test.ts@@ -0,0 +1,76 @@+import { afterAll, beforeEach, describe, expect, it, vi } from "vitest";++import { env } from "../../env";+import {+ capturedQueries,+ clickhouseFormatAvailable,+ normalizeCapturedQueries,+ resetCaptures,+} from "./goldenHarness";++// Record the exec seam instead of hitting ClickHouse. The factory is hoisted+// above imports, so it pulls the harness in via dynamic import; the captured+// store is a module singleton shared with the assertions below.+vi.mock("../repositories/clickhouse", async (importOriginal) => {+ const actual =+ await importOriginal<typeof import("../repositories/clickhouse")>();+ const { buildClickhouseMock } = await import("./goldenHarness.js");+ return buildClickhouseMock(actual);+});++import { getEnvironmentsForProject } from "../repositories/environments";++const FIXED_PROJECT_ID = "golden-project";+// Fixed so the captured params are deterministic across runs.+const FIXED_FROM_TIMESTAMP = new Date("2026-01-01T00:00:00.000Z");++// clickhouse format is the normalizer; without it a comparison would be+// unnormalized and could pass on cosmetic-only differences. Skip loudly.+const describeWithClickhouse = clickhouseFormatAvailable()+ ? describe+ : describe.skip;++if (!clickhouseFormatAvailable()) {+ console.warn(+ "[golden-harness] `clickhouse format` unavailable β skipping golden SQL tests. Install clickhouse-local to run them.",+ );+}++describeWithClickhouse("golden: environments.getEnvironmentsForProject", () => {+ const originalWriteMode = env.LANGFUSE_MIGRATION_V4_WRITE_MODE;++ beforeEach(() => resetCaptures());+ afterAll(() => {+ env.LANGFUSE_MIGRATION_V4_WRITE_MODE = originalWriteMode;+ });++ // Variants enumerated by the inputs that branch the emitted SQL: the write+ // mode (legacy traces+observations UNION vs. events_core) and whether a+ // fromTimestamp bound is present. `dual` and `events_only` take the same code+ // branch, so `events_only` covers both.+ const variants = [+ { writeMode: "legacy" as const, fromTimestamp: undefined },+ { writeMode: "legacy" as const, fromTimestamp: FIXED_FROM_TIMESTAMP },+ { writeMode: "events_only" as const, fromTimestamp: undefined },+ { writeMode: "events_only" as const, fromTimestamp: FIXED_FROM_TIMESTAMP },+ ];++ for (const variant of variants) {+ const name = `writeMode=${variant.writeMode} fromTimestamp=${+ variant.fromTimestamp ? "set" : "unset"+ }`;++ it(name, async () => {+ env.LANGFUSE_MIGRATION_V4_WRITE_MODE = variant.writeMode;++ await getEnvironmentsForProject({+ projectId: FIXED_PROJECT_ID,+ fromTimestamp: variant.fromTimestamp,+ });++ // One repository call emits its tracing read then its scores read; the+ // order is part of the baseline.+ expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();+ });+ }+});diff --git a/packages/shared/src/server/query-ast/goldenHarness.test.ts b/packages/shared/src/server/query-ast/goldenHarness.test.tsnew file mode 100644index 0000000000..276652919c--- /dev/null+++ b/packages/shared/src/server/query-ast/goldenHarness.test.ts@@ -0,0 +1,34 @@+import { describe, expect, it } from "vitest";++import { normalizeParams } from "./goldenHarness";++// Pure logic β no `clickhouse` binary required, so this always runs.+describe("normalizeParams", () => {+ it("rewrites placeholders to positional tokens in first-occurrence order", () => {+ const { sql, params } = normalizeParams(+ "SELECT * FROM t WHERE project_id = {projectId:String} AND ts >= {fromTimestamp:DateTime64(3)}",+ { projectId: "p", fromTimestamp: "2026-01-01" },+ );+ expect(sql).toBe(+ "SELECT * FROM t WHERE project_id = {param1:String} AND ts >= {param2:DateTime64(3)}",+ );+ expect(params).toEqual({ param1: "p", param2: "2026-01-01" });+ });++ it("collapses random-suffixed names that repeat to one stable token", () => {+ const { sql, params } = normalizeParams(+ "SELECT {p_a1b2:String}, {p_a1b2:String}",+ { p_a1b2: "x" },+ );+ expect(sql).toBe("SELECT {param1:String}, {param1:String}");+ expect(params).toEqual({ param1: "x" });+ });++ it("drops params not referenced in the SQL", () => {+ const { params } = normalizeParams("SELECT {used:String}", {+ used: "y",+ unused: "z",+ });+ expect(params).toEqual({ param1: "y" });+ });+});diff --git a/packages/shared/src/server/query-ast/goldenHarness.ts b/packages/shared/src/server/query-ast/goldenHarness.tsnew file mode 100644index 0000000000..980f91ef0f--- /dev/null+++ b/packages/shared/src/server/query-ast/goldenHarness.ts@@ -0,0 +1,233 @@+import { spawnSync } from "child_process";+import { Readable } from "stream";++import type * as ClickhouseModule from "../repositories/clickhouse";++/**+ * Golden-SQL recording/diff harness for the query-builder AST refactor.+ *+ * The query-builder AST refactor migrates repository call sites one family at a+ * time. Every migration must prove that the AST-compiled SQL is equivalent to+ * the SQL emitted today. This harness captures that "today" baseline once and+ * lets each later migration replay through it.+ *+ * Two independent pieces, both server-only and test-scoped:+ *+ * 1. Capture β {@link buildClickhouseMock} replaces the six exec functions in+ * `repositories/clickhouse.ts` with recorders that push `(fn, query,+ * params, tags)` into {@link capturedQueries} and return benign empty+ * results. A test drives a real repository function with fixed inputs; the+ * SQL it would have sent is captured without touching a ClickHouse server.+ *+ * 2. Golden check β {@link normalizeCapturedQueries} normalizes each captured+ * query so two structurally-equal statements compare equal regardless of+ * formatting or param naming: it pipes the SQL through `clickhouse format`+ * (via `clickhouse-local`'s multi-binary β no server state) and rewrites+ * param placeholders to positional tokens (today's are random-suffixed at+ * several sites). The normalized value is what a Vitest snapshot stores.+ *+ * `system.query_log` is a corpus-discovery source only, never a harness source.+ */++/**+ * A single exec-seam call captured in test mode.+ *+ * We capture only the SQL text, its params, and the tags β the parts the AST+ * compiler is responsible for producing. Every other `ClickhouseQueryOpts`+ * field (`clickhouseSettings`, `preferredClickhouseService`,+ * `useMultipartParamsAuto`, β¦) is a client-side execution/routing knob that+ * doesn't change the query text ClickHouse parses, so it's intentionally left+ * out of the baseline.+ *+ * KNOWN GAP: `queryClickhouseExecRaw` is the one exception β its real seam sends+ * `${query}\nFORMAT ${format}`, so `format` *is* part of the wire-level query+ * text. We deliberately ignore it for now: no exec-raw call site is golden-+ * tested yet (this file only drives `queryClickhouse`). Before the first+ * exec-raw site is migrated onto this harness, thread `format` into+ * CapturedQuery/GoldenQuery and fold `FORMAT <format>` into the normalized SQL,+ * or two exec-raw calls differing only in `format` (e.g. JSONEachRow vs+ * Parquet) would snapshot identically.+ */+export type CapturedQuery = {+ /** Which of the six exec functions the repository called. */+ fn: string;+ query: string;+ params?: Record<string, unknown>;+ /** The `tags` bag, whose `route` identifies the call site (see queryTags.ts). */+ tags?: unknown;+};++/** Module-singleton capture store. Same instance for the mock and the test. */+export const capturedQueries: CapturedQuery[] = [];++export function resetCaptures(): void {+ capturedQueries.length = 0;+}++function capture(+ fn: string,+ opts: { query: string; params?: Record<string, unknown>; tags?: unknown },+): void {+ capturedQueries.push({+ fn,+ query: opts.query,+ params: opts.params,+ tags: opts.tags,+ });+}++/**+ * Build the mock module object for `vi.mock("../repositories/clickhouse")`.+ * Spreads the real module so non-exec exports (`upsertClickhouse`, error+ * classes, re-exports) keep working, then overrides the six read/exec seams+ * with recorders. Each recorder honors its original signature β arrays,+ * async generators, and the exec-raw stream shape β so callers that iterate or+ * read a stream don't blow up while being recorded.+ */+export function buildClickhouseMock(+ actual: typeof ClickhouseModule,+): typeof ClickhouseModule {+ return {+ ...actual,+ queryClickhouse: async (opts) => {+ capture("queryClickhouse", opts);+ return [];+ },+ queryClickhouseStream: async function* (opts) {+ capture("queryClickhouseStream", opts);+ yield* [];+ },+ queryClickhouseStreamRawText: async function* (opts) {+ capture("queryClickhouseStreamRawText", opts);+ yield* [];+ },+ queryClickhouseWithProgress: async function* (opts) {+ capture("queryClickhouseWithProgress", opts);+ yield* [];+ },+ queryClickhouseExecRaw: async (opts) => {+ // NOTE: `opts.format` is dropped here on purpose β see CapturedQuery's+ // "KNOWN GAP". Revisit before any exec-raw call site is golden-tested.+ capture("queryClickhouseExecRaw", opts);+ return {+ queryId: "golden-harness",+ stream: Readable.from([]),+ responseHeaders: {},+ };+ },+ commandClickhouse: async (opts) => {+ capture("commandClickhouse", opts);+ },+ } as typeof ClickhouseModule;+}++/** The normalized form of one captured query β the unit a golden snapshot holds. */+export type GoldenQuery = {+ fn: string;+ /** `route` tag if the call site set one, else `"unknown"`. */+ route: string;+ /** Formatted (via `clickhouse format`) and param-normalized SQL. */+ sql: string;+ /** Params bound by the query, keyed by their positional token; only those+ * actually referenced in the SQL are kept (unreferenced binds are no-ops). */+ params: Record<string, unknown>;+};++// The `clickhouse` multi-binary's `format` subcommand. Not `clickhouse-local`,+// which forces local mode and won't accept `format` as a subcommand.+const clickhouseBin = "clickhouse";++let formatAvailability: boolean | undefined;++/**+ * Whether `clickhouse format` can run here. The golden check needs it; gate the+ * suite on this so a machine (or CI job) without the binary skips loudly rather+ * than silently passing an unnormalized comparison.+ */+export function clickhouseFormatAvailable(): boolean {+ if (formatAvailability !== undefined) return formatAvailability;+ try {+ const res = spawnSync(clickhouseBin, ["format"], {+ input: "SELECT 1",+ encoding: "utf8",+ });+ formatAvailability = res.status === 0;+ } catch {+ formatAvailability = false;+ }+ return formatAvailability;+}++/**+ * Canonicalize SQL through `clickhouse format` (whitespace, parenthesization,+ * keyword casing). Uses the `clickhouse` multi-binary's `format` subcommand,+ * which `clickhouse-local` provides β no server, no state. Throws on a parse+ * error so malformed SQL fails the test instead of comparing raw.+ */+export function formatSql(sql: string): string {+ const res = spawnSync(clickhouseBin, ["format"], {+ input: sql,+ encoding: "utf8",+ maxBuffer: 64 * 1024 * 1024,+ });+ if (res.error) throw res.error;+ if (res.status !== 0) {+ throw new Error(+ `clickhouse format failed (status ${res.status}): ${res.stderr?.trim()}\n--- SQL ---\n${sql}`,+ );+ }+ return res.stdout.trimEnd();+}++const PARAM_PLACEHOLDER = /\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*:/g;++/**+ * Rewrite ClickHouse `{name:Type}` placeholders to positional tokens+ * (`param1`, `param2`, β¦) in first-occurrence order, remapping the params map+ * to match. Defeats random-suffixed param names so equivalent queries compare+ * equal. Params never referenced in the SQL are dropped β they bind nothing.+ */+export function normalizeParams(+ sql: string,+ params?: Record<string, unknown>,+): { sql: string; params: Record<string, unknown> } {+ const mapping = new Map<string, string>();+ const normalizedSql = sql.replace(+ PARAM_PLACEHOLDER,+ (_match, name: string) => {+ if (!mapping.has(name)) mapping.set(name, `param${mapping.size + 1}`);+ return `{${mapping.get(name)}:`;+ },+ );++ const normalizedParams: Record<string, unknown> = {};+ if (params) {+ for (const [key, token] of mapping) {+ if (key in params) normalizedParams[token] = params[key];+ }+ }+ return { sql: normalizedSql, params: normalizedParams };+}++function routeOf(tags: unknown): string {+ if (tags && typeof tags === "object" && "route" in tags) {+ const route = (tags as { route?: unknown }).route;+ if (typeof route === "string" && route.trim()) return route.trim();+ }+ return "unknown";+}++/**+ * Normalize an ordered list of captured queries into golden entries. The order+ * is significant β one repository call may emit several queries (e.g.+ * environments emits its tracing read and its scores read), and their sequence+ * is part of the baseline.+ */+export function normalizeCapturedQueries(+ captured: readonly CapturedQuery[],+): GoldenQuery[] {+ return captured.map((c) => {+ const { sql, params } = normalizeParams(formatSql(c.query), c.params);+ return { fn: c.fn, route: routeOf(c.tags), sql, params };+ });+}diff --git a/packages/shared/src/server/repositories/environments.ts b/packages/shared/src/server/repositories/environments.tsindex 4f8648a840..42d80140b9 100644--- a/packages/shared/src/server/repositories/environments.ts+++ b/packages/shared/src/server/repositories/environments.ts@@ -35,7 +35,7 @@ export const getEnvironmentsForProject = async ()`,params: { projectId, fromTimestamp },- tags: { projectId },+ tags: { projectId, route: "environments.getEnvironmentsForProject" },preferredClickhouseService: "ReadOnly",}): queryClickhouse<{ environment: string }>({@@ -46,7 +46,7 @@ export const getEnvironmentsForProject = async (${fromTimestamp ? "AND start_time >= {fromTimestamp: DateTime64(3)}" : ""}`,params: { projectId, fromTimestamp },- tags: { projectId },+ tags: { projectId, route: "environments.getEnvironmentsForProject" },preferredClickhouseService: "EventsReadOnly",});@@ -59,7 +59,7 @@ export const getEnvironmentsForProject = async (${fromTimestamp ? "AND timestamp >= {fromTimestamp: DateTime64(3)}" : ""}`,params: { projectId, fromTimestamp, dataTypes: LISTABLE_SCORE_TYPES },- tags: { projectId },+ tags: { projectId, route: "environments.getEnvironmentsForProject" },preferredClickhouseService: "ReadOnly",});