🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(native): restore writer parity coverage and preserve map keys (#17940)
Valery Meleshkin0451dbeca19c
diff --git a/packages/native/src/native_js.rs b/packages/native/src/native_js.rsindex f9f41011ca..7ef453ae96 100644--- a/packages/native/src/native_js.rs+++ b/packages/native/src/native_js.rs@@ -309,16 +309,19 @@ impl<T: FromJsValue> FromJsValue for BTreeMap<String, T> {.map_err(|error| error.to_string())?;let mut result = BTreeMap::new();for index in 0..length {- let key: String = keys.get_element(index).map_err(|error| error.to_string())?;+ let key: Unknown<'_> = keys.get_element(index).map_err(|error| error.to_string())?;+ let name: String = unsafe { key.cast() }.map_err(|error| error.to_string())?;// JSONEachRow includes only own enumerable string keys.if !object- .has_own_property(&key)+ .has_own_property(&name).map_err(|error| error.to_string())?{continue;}- let value = get_js_property(&object, &key)?;- result.insert(key, T::from_js(value)?);+ let value: Unknown<'_> = object+ .get_property_unchecked(key)+ .map_err(|error| format!("cannot read property {name}: {error}"))?;+ result.insert(name, T::from_js(value)?);}Ok(result)}diff --git a/worker/src/__tests__/helpers/nativeCodecFixtures.ts b/worker/src/__tests__/helpers/nativeCodecFixtures.tsnew file mode 100644index 0000000000..64286fbd8a--- /dev/null+++ b/worker/src/__tests__/helpers/nativeCodecFixtures.ts@@ -0,0 +1,223 @@+import { readFileSync, readdirSync } from "node:fs";+import { resolve } from "node:path";+import type { EventRecordInsertType } from "@langfuse/shared/src/server";+import type { EventInput } from "../../services/IngestionService/index.js";++// Read the existing trace captures and exercise production createEventRecord/writeEventRecord.+// Model lookup responses are replayed from captures; no fixture JSON is copied into Rust.+type JsonObject = Record<string, unknown>;+const repoRoot = resolve(__dirname, "../../../..");++const parseObject = (value: unknown): JsonObject => {+ if (value && typeof value === "object" && !Array.isArray(value)) {+ return value as JsonObject;+ }+ if (typeof value === "string") {+ try {+ const parsed: unknown = JSON.parse(value);+ return parseObject(parsed);+ } catch {+ return {};+ }+ }+ return {};+};++const numberMap = (value: unknown): Record<string, number> =>+ Object.fromEntries(+ Object.entries(parseObject(value)).flatMap(([key, item]) => {+ const number = typeof item === "number" ? item : Number(item);+ return Number.isFinite(number) ? [[key, number]] : [];+ }),+ );++const stringArray = (value: unknown): string[] =>+ Array.isArray(value)+ ? value.filter((item): item is string => typeof item === "string")+ : [];++const stringMap = (value: unknown): Record<string, string> =>+ Object.fromEntries(+ Object.entries(parseObject(value)).map(([key, item]) => [+ key,+ typeof item === "string" ? item : JSON.stringify(item),+ ]),+ );++const optionalString = (value: unknown): string | undefined =>+ typeof value === "string" ? value : undefined;++const mergeObjects = (...values: unknown[]): JsonObject =>+ Object.assign({}, ...values.map(parseObject));++const fixtureDirectories = [+ resolve(repoRoot, "packages/shared/scripts/seeder/utils/framework-traces"),+ resolve(repoRoot, "worker/src/__tests__/chatml/framework-traces"),+];++const fixturePaths = fixtureDirectories+ .flatMap((directory) =>+ readdirSync(directory)+ .filter((name) =>+ directory.endsWith("chatml/framework-traces")+ ? name.endsWith(".trace.json")+ : name.endsWith(".json"),+ )+ .map((name) => resolve(directory, name)),+ )+ .sort();++const eventInputFromCapture = (+ observation: JsonObject,+ trace: JsonObject,+): EventInput => {+ const isRoot = observation.parentObservationId == null;+ const usageDetails = numberMap(observation.usageDetails);+ const costDetails = numberMap(observation.costDetails);+ const providedUsageDetails = numberMap(observation.providedUsageDetails);+ const providedCostDetails = numberMap(observation.providedCostDetails);++ return {+ projectId: String(observation.projectId ?? trace.projectId ?? "fixture"),+ traceId: String(observation.traceId ?? trace.id),+ spanId: String(observation.id),+ parentSpanId:+ typeof observation.parentObservationId === "string"+ ? observation.parentObservationId+ : undefined,+ startTimeISO: String(observation.startTime),+ endTimeISO: String(observation.endTime ?? observation.startTime),+ completionStartTime: optionalString(observation.completionStartTime),+ name: typeof observation.name === "string" ? observation.name : "",+ type: typeof observation.type === "string" ? observation.type : "SPAN",+ environment: String(+ observation.environment ?? trace.environment ?? "default",+ ),+ version:+ typeof observation.version === "string" ? observation.version : undefined,+ release: typeof trace.release === "string" ? trace.release : undefined,+ traceName: typeof trace.name === "string" ? trace.name : undefined,+ tags: Array.isArray(trace.tags)+ ? trace.tags.filter((tag): tag is string => typeof tag === "string")+ : [],+ public: trace.public === true,+ bookmarked: isRoot && trace.bookmarked === true,+ userId: typeof trace.userId === "string" ? trace.userId : undefined,+ sessionId:+ typeof trace.sessionId === "string" ? trace.sessionId : undefined,+ level:+ typeof observation.level === "string" ? observation.level : undefined,+ statusMessage:+ typeof observation.statusMessage === "string"+ ? observation.statusMessage+ : undefined,+ modelName:+ typeof observation.model === "string" ? observation.model : undefined,+ modelId: optionalString(observation.internalModelId),+ modelParameters:+ observation.modelParameters as EventInput["modelParameters"],+ providedUsageDetails:+ Object.keys(providedUsageDetails).length > 0+ ? providedUsageDetails+ : usageDetails,+ usageDetails,+ providedCostDetails:+ Object.keys(providedCostDetails).length > 0+ ? providedCostDetails+ : costDetails,+ costDetails,+ usagePricingTierId: optionalString(observation.usagePricingTierId),+ usagePricingTierName: optionalString(observation.usagePricingTierName),+ promptId: optionalString(observation.promptId),+ promptName: optionalString(observation.promptName),+ promptVersion:+ observation.promptVersion == null+ ? undefined+ : String(observation.promptVersion),+ toolDefinitions: stringMap(observation.toolDefinitions),+ toolCalls: stringArray(observation.toolCalls),+ toolCallNames: stringArray(observation.toolCallNames),+ input: observation.input as string | undefined,+ output: observation.output as string | undefined,+ metadata: mergeObjects(trace.metadata, observation.metadata),+ source: "API",+ ingestionApiKey: "",+ ingestionSdkName: "",+ ingestionSdkVersion: "",+ blobStorageFilePath: "fixtures",+ eventBytes: 0,+ };+};++export const capturedPreparedRows = async () => {+ const { IngestionService } =+ await import("../../services/IngestionService/index.js");++ const normalizedRows: EventRecordInsertType[] = [];+ const capturedGenerationUsage = new Map<+ string,+ Pick<+ EventRecordInsertType,+ | "model_id"+ | "usage_pricing_tier_id"+ | "usage_pricing_tier_name"+ | "usage_details"+ | "cost_details"+ >+ >();+ const writer = {+ addToQueue: (_table: unknown, record: EventRecordInsertType) => {+ normalizedRows.push(record);+ },+ };+ const service = new IngestionService(+ {} as never,+ {} as never,+ writer as never,+ {} as never,+ );++ // Avoid external model and prompt lookups while retaining the production method's field mapping.+ // Captured generation enrichment is replayed where the source trace already contains it.+ (service as any).getGenerationUsage = async ({+ observationRecord,+ }: {+ observationRecord: EventRecordInsertType;+ }) => ({+ usage_details: capturedGenerationUsage.get(observationRecord.id)+ ?.usage_details,+ cost_details: capturedGenerationUsage.get(observationRecord.id)+ ?.cost_details,+ internal_model_id: capturedGenerationUsage.get(observationRecord.id)+ ?.model_id,+ usage_pricing_tier_id: capturedGenerationUsage.get(observationRecord.id)+ ?.usage_pricing_tier_id,+ usage_pricing_tier_name: capturedGenerationUsage.get(observationRecord.id)+ ?.usage_pricing_tier_name,+ });++ for (const fixturePath of fixturePaths) {+ const fixture = JSON.parse(readFileSync(fixturePath, "utf8")) as {+ trace?: JsonObject;+ observations?: JsonObject[];+ };+ const trace = fixture.trace ?? {};+ for (const observation of fixture.observations ?? []) {+ const eventInput = eventInputFromCapture(observation, trace);+ capturedGenerationUsage.set(eventInput.spanId, {+ usage_details: eventInput.usageDetails ?? {},+ cost_details: eventInput.costDetails ?? {},+ model_id: eventInput.modelId,+ usage_pricing_tier_id: eventInput.usagePricingTierId,+ usage_pricing_tier_name: eventInput.usagePricingTierName,+ });+ const eventRecord = await service.createEventRecord(+ eventInput,+ `fixtures/${fixturePath.split("/").pop()}`,+ );+ await service.writeEventRecord(eventRecord);+ }+ }++ return normalizedRows;+};diff --git a/worker/src/__tests__/nativeCodec.integration.test.ts b/worker/src/__tests__/nativeCodec.integration.test.tsnew file mode 100644index 0000000000..98a6f3f396--- /dev/null+++ b/worker/src/__tests__/nativeCodec.integration.test.ts@@ -0,0 +1,377 @@+import { randomInt, randomUUID } from "node:crypto";+import { afterAll, beforeAll, describe, expect, it } from "vitest";+import fc, { type Arbitrary } from "fast-check";+import { PreparedEvent } from "@langfuse/native";+import {+ clickhouseClient,+ type ClickhouseClientType,+ type EventRecordInsertType,+} from "@langfuse/shared/src/server";+import { ClickhouseWriter, TableName } from "../services/ClickhouseWriter";+import { prepareNativeEvent } from "../services/IngestionService/prepareNativeEvent";+import { capturedPreparedRows } from "./helpers/nativeCodecFixtures";++type Row = Record<string, unknown>;+type Column = { name: string; type: string; default_kind: string };++// The explicit CI/package command requires ClickHouse. Both branches use the production+// writer; Native rows cross the real NAPI boundary as JS objects, without a JSON test driver.+describe.runIf(+ process.env.LANGFUSE_NATIVE_FAIL_IF_CLICKHOUSE_UNAVAILABLE === "1",+)("TypeScript and Rust Native writer parity", () => {+ let columns: Column[];+ const client = clickhouseClient();++ beforeAll(async () => {+ const result = await client.query({+ query:+ "SELECT name, type, default_kind FROM system.columns WHERE database=currentDatabase() AND table='events_full' ORDER BY position",+ format: "JSONEachRow",+ });+ columns = await result.json<Column>();+ expect(columns.length).toBeGreaterThan(0);+ const normalizeType = (type: string) =>+ type+ .replaceAll(/LowCardinality\(([^()]*)\)/g, "$1")+ .replaceAll(/\s/g, "");+ // Names and types come from the same Rust declaration as JS reads and column writes.+ // Check the full insertable set, including columns that captured traffic omits.+ const liveColumns = columns.filter(+ (column) => !["MATERIALIZED", "ALIAS"].includes(column.default_kind),+ );+ const nativeColumns = PreparedEvent.columns();+ expect(+ nativeColumns+ .map((column) => [column.name, normalizeType(column.columnType)])+ .sort(),+ ).toEqual(+ liveColumns+ .map((column) => [column.name, normalizeType(column.type)])+ .sort(),+ );+ for (const column of nativeColumns.filter((column) => column.usesDefault)) {+ expect(+ liveColumns.find((live) => live.name === column.name)?.default_kind,+ `${column.name} must retain its DEFAULT expression`,+ ).toBe("DEFAULT");+ }+ });++ afterAll(async () => {+ await client.close();+ });++ const write = async (+ rows: Row[],+ table: string,+ native: boolean,+ maxRows: number,+ ) => {+ let inserts = 0;+ const adapter = {+ insert: async (params: Parameters<ClickhouseClientType["insert"]>[0]) => {+ expect(native).toBe(false);+ expect(params.table).toBe(TableName.EventsFull);+ inserts++;+ return client.insert({ ...params, table });+ },+ exec: async (params: Parameters<ClickhouseClientType["exec"]>[0]) => {+ expect(native).toBe(true);+ expect(params.query).toBe("INSERT INTO events_full FORMAT Native");+ inserts++;+ return client.exec({+ ...params,+ query: `INSERT INTO ${table} FORMAT Native`,+ });+ },+ } as ClickhouseClientType;+ let writer:+ | ClickhouseWriter+ | ReturnType<typeof ClickhouseWriter.getNativeInstance>;+ let writerShutdown = false;+ try {+ if (native) {+ writer = ClickhouseWriter.getNativeInstance(adapter);+ writer.batchSize = maxRows;+ for (const row of rows) {+ writer.addToQueue(+ TableName.EventsFull,+ prepareNativeEvent(row as EventRecordInsertType),+ );+ }+ } else {+ writer = ClickhouseWriter.getInstance(adapter);+ writer.batchSize = maxRows;+ for (const row of rows) {+ writer.addToQueue(+ TableName.EventsFull,+ structuredClone(row) as EventRecordInsertType,+ );+ }+ }+ await ClickhouseWriter.shutdownAll();+ writerShutdown = true;+ expect(writer.queue[TableName.EventsFull]).toHaveLength(0);+ expect(inserts).toBeGreaterThan(0);+ } finally {+ if (!writerShutdown) await ClickhouseWriter.shutdownAll();+ }+ };++ const compare = async (rows: Row[], maxRows: number) => {+ const suffix = randomUUID().replaceAll("-", "");+ const jsonTable = `native_codec_json_${suffix}`;+ const nativeTable = `native_codec_native_${suffix}`;+ try {+ for (const table of [jsonTable, nativeTable]) {+ await client.command({+ query: `CREATE TABLE ${table} AS events_full ENGINE=Memory`,+ });+ }+ await write(rows, jsonTable, false, maxRows);+ await write(rows, nativeTable, true, maxRows);++ // Compare inside ClickHouse to preserve Decimal/UInt64 precision, duplicate rows,+ // defaults, materialized and alias columns. Only map entry order is immaterial.+ const maps = columns+ .filter((c) => c.type.startsWith("Map("))+ .map((c) => c.name);+ const projection = `* EXCEPT (${maps.join(", ")}), ${maps.map((name) => `mapSort((k, v) -> k, ${name}) AS ${name}`).join(", ")}`;+ const select = (table: string) => `SELECT ${projection} FROM ${table}`;+ const response = await client.query({+ query: `SELECT+ (SELECT count() FROM ${jsonTable}) AS json_count,+ (SELECT count() FROM ${nativeTable}) AS native_count,+ (SELECT count() FROM (${select(jsonTable)} EXCEPT ALL ${select(nativeTable)})) AS json_only,+ (SELECT count() FROM (${select(nativeTable)} EXCEPT ALL ${select(jsonTable)})) AS native_only,+ (SELECT toString(sum(event_bytes)) FROM ${nativeTable}) AS native_bytes+ SETTINGS asterisk_include_materialized_columns=1, asterisk_include_alias_columns=1`,+ format: "JSONEachRow",+ });+ const [result] = await response.json<Record<string, string | number>>();+ expect(Number(result.json_count)).toBe(rows.length);+ expect(Number(result.native_count)).toBe(rows.length);+ if (Number(result.json_only) !== 0 || Number(result.native_only) !== 0) {+ const differences = await Promise.all(+ [jsonTable, nativeTable].map(async (table) => {+ const result = await client.query({+ query: `${select(table)} ORDER BY trace_id, span_id LIMIT 5`,+ format: "JSONEachRow",+ });+ return result.json();+ }),+ );+ expect(differences[1]).toEqual(differences[0]);+ }+ expect(Number(result.json_only)).toBe(0);+ expect(Number(result.native_only)).toBe(0);+ expect(BigInt(result.native_bytes)).toBe(+ rows.reduce((sum, row) => sum + BigInt(row.event_bytes as number), 0n),+ );+ } finally {+ for (const table of [jsonTable, nativeTable]) {+ await client.command({ query: `DROP TABLE IF EXISTS ${table}` });+ }+ }+ };++ it("compares captured traffic through production TS preparation and both encodings", async () => {+ const rows = await capturedPreparedRows();+ expect(rows.length).toBeGreaterThan(0);+ const enriched = rows.find(+ (row) => row.span_id === "0d8f5427-c91a-44c8-86c5-e12d806edd57",+ );+ expect(enriched?.provided_cost_details).toEqual({+ input: 0,+ output: 0,+ total: 0,+ });+ expect(enriched?.cost_details).toEqual({+ input: 0.00147,+ output: 0.0024,+ total: 0.00387,+ });+ await compare(rows, 16);+ }, 120_000);++ // Vitest retries must exercise the same generated cases after a failure.+ const seed = Number(process.env.FAST_CHECK_SEED ?? randomInt(2 ** 31));++ it("persists generated rows identically through the production JS adapter", async () => {+ process.stderr.write(`Native writer parity: seed=${seed}\n`);+ await fc.assert(+ fc.asyncProperty(+ fc.array(preparedRowArbitrary(columns), { minLength: 1, maxLength: 8 }),+ fc.integer({ min: 1, max: 8 }),+ async (rows, maxRows) => compare(rows, maxRows),+ ),+ { seed, numRuns: 32 },+ );+ }, 120_000);++ it("preserves prepared JS values, defaults and accounting through NAPI", async () => {+ const base: Row = {+ project_id: "native-boundary",+ trace_id: "trace",+ span_id: "span",+ start_time: "2026-09-24 00:00:00.123456",+ created_at: "2026-09-24 00:00:00.123456",+ updated_at: "2026-09-24 00:00:00.123456",+ event_ts: "2026-09-24 00:00:00.123456",+ input: '{"emoji":"🔥","escaped":"\\n"}',+ output: "café",+ model_parameters: {+ temperature: 1,+ nested: ["🔥", -0, 1e-7, 2 ** 63, null],+ omitted: undefined,+ },+ event_bytes: 123,+ metadata_names: [+ "evaluator_id",+ "job_configuration_id",+ "evaluator_test",+ ],+ metadata_values: ["evaluator", "job", "true"],+ environment: undefined,+ prompt_version: null,+ end_time: undefined,+ usage_pricing_tier_id: null,+ usage_details: {+ large: Number.MAX_SAFE_INTEGER,+ // JSONEachRow stores the decimal JS spelling of this double, which differs+ // from the integer obtained by a direct f64-to-u64 cast.+ aboveSignedRange: 2 ** 63,+ zero: -0,+ "\u0000": 0,+ },+ cost_details: {+ precise: 510407.65505697404,+ tiny: 1e-13,+ negative: -1.9999999999999,+ overflow: 1e6,+ "\u0000": 0,+ },+ tool_definitions: Object.fromEntries([+ ["__proto__", "tool"],+ ["é", "🔥"],+ ["\u0000", "nul-key-tool"],+ ]),+ tags: ["", "🔥", "line\nbreak"],+ };+ await compare(+ [+ base,+ {+ ...base,+ environment: "",+ evaluator_id: "",+ evaluation_rule_id: "",+ evaluator_execution_is_test: false,+ prompt_version: 65535,+ model_parameters: null,+ event_bytes: 0,+ },+ { ...base },+ ],+ 2,+ );+ }, 30_000);+});++// The live schema supplies the domain independently of Rust's declaration. These bounded+// values exercise every storage column while keeping server-backed failures cheap to shrink.+function preparedRowArbitrary(columns: Column[]): Arbitrary<Row> {+ const fields = Object.fromEntries(+ columns+ .filter(+ (column) => !["MATERIALIZED", "ALIAS"].includes(column.default_kind),+ )+ .map((column) => {+ const value = columnValueArbitrary(column.type);+ return [+ column.name,+ // Preparation supplies timestamps explicitly. Falling back to now64() would compare+ // different insertion times, outside the prepared-row contract.+ column.default_kind === "DEFAULT" &&+ !["created_at", "updated_at"].includes(column.name)+ ? fc.option(value, { nil: undefined })+ : value,+ ];+ }),+ );+ return fc+ .tuple(fc.record(fields), fc.boolean())+ .map(([row, evaluatorKeys]) => {+ // Prepared metadata has parallel arrays. Mix arbitrary names with evaluator keys so both+ // ordinary metadata preservation and metadata-derived DEFAULT expressions are exercised.+ for (const prefix of [+ "metadata",+ "experiment_metadata",+ "experiment_item_metadata",+ ]) {+ const values = row[`${prefix}_values`] as string[];+ const names = evaluatorKeys+ ? [+ "evaluator_id",+ "evaluation_rule_id",+ "job_configuration_id",+ "evaluator_test",+ ]+ : (row[`${prefix}_names`] as string[]);+ row[`${prefix}_names`] = names.slice(0, values.length);+ row[`${prefix}_values`] = values.slice(0, names.length);+ }+ return row;+ });+}++function columnValueArbitrary(type: string): Arbitrary<unknown> {+ const normalized = type+ .replaceAll(/LowCardinality\(([^()]*)\)/g, "$1")+ .replaceAll(/\s/g, "");+ const text = fc.oneof(+ fc.fullUnicodeString({ maxLength: 24 }),+ fc.constantFrom("", "true"),+ );+ const nullable = normalized.match(/^Nullable\((.*)\)$/);+ if (nullable)+ return fc.option(columnValueArbitrary(nullable[1]), { nil: null });+ const map = normalized.match(/^Map\(String,(.*)\)$/);+ if (map)+ return fc+ .array(fc.tuple(text, columnValueArbitrary(map[1])), { maxLength: 4 })+ .map(Object.fromEntries);+ switch (normalized) {+ case "String":+ return text;+ case "Bool":+ return fc.boolean();+ case "UInt8":+ return fc.integer({ min: 0, max: 1 });+ case "UInt16":+ return fc.integer({ min: 0, max: 65535 });+ case "UInt64":+ return fc.integer({ min: 0, max: Number.MAX_SAFE_INTEGER });+ case "Decimal(18,12)":+ return fc.double({+ min: -999999,+ max: 999999,+ noNaN: true,+ noDefaultInfinity: true,+ });+ case "Array(String)":+ return fc.array(text, { maxLength: 4 });+ case "DateTime64(6)":+ return fc+ .tuple(+ fc.integer({ min: 0, max: 2000000000 }),+ fc.integer({ min: 0, max: 999999 }),+ )+ .map(+ ([seconds, micros]) =>+ `${new Date(seconds * 1000).toISOString().slice(0, 19).replace("T", " ")}.${String(micros).padStart(6, "0")}`,+ );+ default:+ throw new Error(`Unsupported live events_full type: ${type}`);+ }+}