🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
chore(worker): cover OTEL replay with provider fixtures and properties (#17742)
Valery Meleshkin2ba080996b35
diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex ef64f0eb1e..7495b6538f 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -1254,6 +1254,9 @@ importers:eslint:specifier: 'catalog:'version: 10.8.0([email protected])([email protected])+ fast-check:+ specifier: 3.23.2+ version: 3.23.2msw:specifier: ^2.6.5version: 2.6.5(@types/[email protected])(@typescript/[email protected])diff --git a/worker/package.json b/worker/package.jsonindex de2f426293..c791ab95b7 100644--- a/worker/package.json+++ b/worker/package.json@@ -93,6 +93,7 @@"@types/node": "^24.3.0","@vitest/coverage-v8": "^4.1.11","eslint": "catalog:",+ "fast-check": "3.23.2","msw": "^2.6.5","tsx": "^4.20.5","typescript": "catalog:",diff --git a/worker/src/__tests__/helpers/arbitraries/json.ts b/worker/src/__tests__/helpers/arbitraries/json.tsnew file mode 100644index 0000000000..1833ad4d55--- /dev/null+++ b/worker/src/__tests__/helpers/arbitraries/json.ts@@ -0,0 +1,48 @@+import fc, { type Arbitrary } from "fast-check";++export type Json =+ | null+ | boolean+ | number+ | string+ | Json[]+ | { [key: string]: Json };++// Unicode and object-like key names are valid JSON data, including empty keys.+export const jsonKeyArbitrary = fc.oneof(+ { withCrossShrink: true },+ fc.string({ maxLength: 12, unit: "grapheme" }),+ fc.constantFrom("", "a.b", "__proto__", "constructor", "prototype", "🌍\n"),+);++// Bounded recursive JSON with finite numbers and shrinking from containers to+// leaves. Depth and width bound test cost without restricting JSON value kinds.+export function recursiveJsonArbitrary(+ depth: number,+ maxWidth: number,+): Arbitrary<Json> {+ const leaf = fc.oneof(+ { withCrossShrink: true },+ fc.constant(null),+ fc.boolean(),+ fc.double({ noNaN: true, noDefaultInfinity: true }),+ fc.string({ maxLength: 24, unit: "grapheme" }),+ ) as Arbitrary<Json>;+ if (depth === 0) return leaf;+ const child = recursiveJsonArbitrary(depth - 1, maxWidth);++ return fc.oneof(+ { withCrossShrink: true },+ { weight: 4, arbitrary: leaf },+ {+ weight: 2,+ arbitrary: fc.array(child, {+ maxLength: maxWidth,+ }),+ },+ {+ weight: 2,+ arbitrary: fc.dictionary(jsonKeyArbitrary, child, { maxKeys: maxWidth }),+ },+ );+}diff --git a/worker/src/__tests__/otelReplay.corpus.test.ts b/worker/src/__tests__/otelReplay.corpus.test.tsnew file mode 100644index 0000000000..f427900f05--- /dev/null+++ b/worker/src/__tests__/otelReplay.corpus.test.ts@@ -0,0 +1,426 @@+import "./helpers/otelReplaySetup";+import { describe, expect, it } from "vitest";+import type { ResourceSpan } from "@langfuse/shared/src/server";+import { vercelAiSdkMixedToolMessagesFixture } from "../../../packages/shared/src/utils/normalized-io/conventions/providers/ai-sdk/fixtures";+import { langgraphProductionShapeFixture } from "../../../packages/shared/src/utils/normalized-io/conventions/providers/langchain/fixtures";+import { pydanticAiProductionShapeFixture } from "../../../packages/shared/src/utils/normalized-io/conventions/providers/pydantic-ai/fixtures";+import { microsoftAgentProductionShapeFixture } from "../../../packages/shared/src/utils/normalized-io/conventions/providers/otel-genai/fixtures";+import { runOtelReplay } from "./helpers/otelReplayHarness";++type ScopeSpan = NonNullable<ResourceSpan["scopeSpans"]>[number];+type Span = NonNullable<ScopeSpan["spans"]>[number];++type ReplayFixture = {+ name: string;+ otel: {+ scopeSpan: ScopeSpan;+ resourceAttributes: Record<string, unknown>;+ };+ spanIO: {+ input: unknown;+ output: unknown;+ };+};++type ProviderCase = {+ name: string;+ fixture: ReplayFixture;+ toolDefinitionsAttribute?: string;+ addToolDefinitionsToInput?: boolean;+ expectedObservationType?: string;+ expectedModelName: string;+ expectedUsageDetails: Record<string, number>;+ expectedFinalUsageDetails: Record<string, number>;+ expectedToolCalls?: Array<{+ name: string;+ id: string;+ arguments: string;+ type: string;+ index: number;+ }>;+ expectedMetadata?: Record<string, string>;+ expectedModelParameters?: Record<string, string | number>;+ stringRepairs?: Record<string, string>;+ numericRepairs: Record<string, string>;+};++const providerCases: ProviderCase[] = [+ {+ name: "AI SDK",+ fixture: vercelAiSdkMixedToolMessagesFixture,+ toolDefinitionsAttribute: "gen_ai.tool.definitions",+ // The captured spanIO input omits tools; the replay stores the OTLP tool+ // definitions with the messages in its input column.+ addToolDefinitionsToInput: true,+ expectedModelName: "gpt-5-2025-08-07",+ // This fixture has no usage attributes. The Vercel `ai` scope branch fills+ // missing input/output counts with zero; the generic `gen_ai` branch does not.+ expectedUsageDetails: { input: 0, output: 0 },+ expectedFinalUsageDetails: { input: 0, output: 0, total: 0 },+ expectedToolCalls: [+ {+ name: "extract",+ id: "toolu_01XXtujJ3DBaYEZGzn96xpGt",+ arguments: JSON.stringify({+ score: "0",+ reasoning:+ "The last user message is a how-to question asking for guidance on using Langfuse evaluation features. It does not express feedback about a feature.",+ }),+ type: "tool_use",+ index: 0,+ },+ ],+ numericRepairs: {},+ },+ {+ name: "LangGraph",+ fixture: langgraphProductionShapeFixture,+ expectedModelName: "synthetic-value-044",+ expectedUsageDetails: {+ synthetic_field_010: 13252,+ synthetic_field_011: 0,+ input: 1,+ output: 406,+ synthetic_field_012: 0,+ synthetic_field_013: 13252,+ },+ expectedFinalUsageDetails: {+ synthetic_field_010: 13252,+ synthetic_field_011: 0,+ input: 1,+ output: 406,+ synthetic_field_012: 0,+ synthetic_field_013: 13252,+ total: 26911,+ },+ expectedMetadata: {+ langgraph_step: "1",+ ls_max_tokens: "256",+ },+ numericRepairs: {+ "langfuse.observation.metadata.langgraph_step": "1",+ "langfuse.observation.metadata.ls_max_tokens": "256",+ },+ },+ {+ name: "Pydantic AI",+ fixture: pydanticAiProductionShapeFixture,+ toolDefinitionsAttribute: "gen_ai.tool.definitions",+ expectedModelName: "synthetic-value-403",+ expectedUsageDetails: {+ input: 31,+ output: 96,+ accepted_prediction_tokens: 0,+ audio_tokens: 0,+ reasoning_tokens: 7,+ rejected_prediction_tokens: 0,+ },+ expectedFinalUsageDetails: {+ input: 31,+ output: 96,+ accepted_prediction_tokens: 0,+ audio_tokens: 0,+ reasoning_tokens: 7,+ rejected_prediction_tokens: 0,+ total: 134,+ },+ // Known limitation: this provider puts tool calls in `parts`, which the+ // current extractor does not split into tool-call columns. The full output+ // is persisted while those columns remain empty.+ expectedToolCalls: [],+ expectedModelParameters: { max_tokens: 1024 },+ numericRepairs: {+ "gen_ai.request.max_tokens": "1024",+ "gen_ai.usage.input_tokens": "31",+ "gen_ai.usage.output_tokens": "96",+ "gen_ai.usage.details.accepted_prediction_tokens": "0",+ "gen_ai.usage.details.audio_tokens": "0",+ "gen_ai.usage.details.reasoning_tokens": "7",+ "gen_ai.usage.details.rejected_prediction_tokens": "0",+ },+ },+ {+ name: "OTel GenAI",+ fixture: microsoftAgentProductionShapeFixture,+ toolDefinitionsAttribute: "gen_ai.tool.definitions",+ // The anonymizer replaced this Langfuse enum with an invalid placeholder.+ // For this replay contract, the GenAI model and message payload are treated+ // as a generation; this does not claim the original enum value was recovered.+ stringRepairs: { "langfuse.observation.type": "generation" },+ expectedObservationType: "GENERATION",+ // Current conversion suppresses model and usage for any invoke_agent span,+ // including this fixture with gen_ai.request.model. Keep the observed blank+ // model and absent usage explicit until that behavior is changed separately.+ expectedModelName: "",+ expectedUsageDetails: {},+ expectedFinalUsageDetails: {},+ expectedToolCalls: [],+ expectedModelParameters: { "choice.count": 1 },+ numericRepairs: {+ "gen_ai.request.choice.count": "1",+ },+ },+];++function cloneId(id: Span["traceId"]): Span["traceId"] {+ if (Buffer.isBuffer(id)) return Buffer.from(id);+ if (id.data) return { data: Buffer.from(id.data) };+ return id;+}++function idToHex(id: Span["traceId"]): string {+ return Buffer.from(+ Buffer.isBuffer(id) ? id : (id.data ?? Buffer.alloc(0)),+ ).toString("hex");+}++function fixtureStringAttribute(+ fixture: ReplayFixture,+ key: string,+): string | undefined {+ const attribute = fixture.otel.scopeSpan.spans?.[0]?.attributes?.find(+ (candidate) => candidate.key === key,+ );+ const value = attribute?.value as { stringValue?: unknown } | undefined;+ return typeof value?.stringValue === "string" ? value.stringValue : undefined;+}++function fixtureJsonAttribute(fixture: ReplayFixture, key: string): unknown {+ const value = fixtureStringAttribute(fixture, key);+ if (value === undefined) throw new Error(`Fixture is missing ${key}`);+ return JSON.parse(value);+}++function parseFixtureValue(value: unknown): unknown {+ if (typeof value !== "string") return value;++ try {+ return JSON.parse(value);+ } catch {+ return value;+ }+}++function persistedJsonColumn(+ row: Record<string, unknown>,+ key: string,+): unknown {+ const value = row[key];+ if (typeof value !== "string") return value;++ try {+ return JSON.parse(value);+ } catch {+ return value;+ }+}++function expectedToolDefinitions(+ definitions: unknown[],+): Record<string, string> {+ return Object.fromEntries(+ definitions.map((definition) => {+ const value = definition as {+ function?: Record<string, unknown>;+ name?: string;+ description?: string;+ parameters?: unknown;+ inputSchema?: unknown;+ parameters_json_schema?: unknown;+ };+ const tool = value.function ?? value;+ const name = tool.name as string;+ const parameters =+ tool.parameters ?? tool.parameters_json_schema ?? tool.inputSchema;++ return [+ name,+ JSON.stringify({+ description: (tool.description as string | undefined) ?? "",+ parameters: parameters ? JSON.stringify(parameters) : "",+ }),+ ];+ }),+ );+}++function persistedToolCalls(row: Record<string, unknown>): unknown[] {+ const calls = persistedJsonColumn(row, "tool_calls");+ if (!Array.isArray(calls)) {+ throw new Error("ClickHouse replay row is missing tool_calls");+ }+ return calls.map((call) =>+ typeof call === "string" ? JSON.parse(call) : call,+ );+}++function assertMetadataValues(+ row: Record<string, unknown>,+ expected: Record<string, string>,+): void {+ const names = row.metadata_names as string[];+ const values = row.metadata_values as string[];++ for (const [name, value] of Object.entries(expected)) {+ const index = names.indexOf(name);+ expect(index, `metadata field ${name}`).toBeGreaterThanOrEqual(0);+ expect(values[index]).toBe(value);+ }+}++function buildResourceSpans(+ fixture: ReplayFixture,+ startTimeUnixNano: string,+ endTimeUnixNano: string,+ stringRepairs: Record<string, string>,+ numericRepairs: Record<string, string>,+): ResourceSpan[] {+ const sourceScopeSpan = fixture.otel.scopeSpan;++ return [+ {+ resource: {+ attributes: Object.entries(fixture.otel.resourceAttributes).map(+ ([key, value]) => ({+ key,+ value: { stringValue: String(value) },+ }),+ ),+ },+ scopeSpans: [+ {+ ...sourceScopeSpan,+ scope: sourceScopeSpan.scope+ ? {+ ...sourceScopeSpan.scope,+ attributes: sourceScopeSpan.scope.attributes?.map((attribute) =>+ structuredClone(attribute),+ ),+ }+ : undefined,+ spans: (sourceScopeSpan.spans ?? []).map((sourceSpan) => ({+ ...sourceSpan,+ traceId: cloneId(sourceSpan.traceId),+ spanId: cloneId(sourceSpan.spanId),+ parentSpanId: sourceSpan.parentSpanId+ ? cloneId(sourceSpan.parentSpanId)+ : undefined,+ startTimeUnixNano,+ endTimeUnixNano,+ attributes: sourceSpan.attributes?.map((attribute) => {+ const repairedValue = numericRepairs[attribute.key];+ if (repairedValue !== undefined) {+ return { ...attribute, value: { intValue: repairedValue } };+ }++ const repairedString = stringRepairs[attribute.key];+ return repairedString === undefined+ ? structuredClone(attribute)+ : { ...attribute, value: { stringValue: repairedString } };+ }),+ events: sourceSpan.events+ ? structuredClone(sourceSpan.events)+ : sourceSpan.events,+ status: sourceSpan.status+ ? { ...sourceSpan.status }+ : sourceSpan.status,+ })),+ },+ ],+ },+ ];+}++// The writer owns retries; a Vitest retry could overlap an in-flight ClickHouse replay.+describe("OTEL replay provider corpus", { retry: 0, timeout: 120_000 }, () => {+ it.each(providerCases)(+ "$name persists provider payloads and identity through ClickHouse",+ async (providerCase) => {+ const {+ expectedMetadata,+ expectedModelParameters,+ expectedToolCalls,+ expectedModelName,+ expectedObservationType,+ expectedFinalUsageDetails,+ expectedUsageDetails,+ fixture,+ name,+ numericRepairs,+ stringRepairs,+ toolDefinitionsAttribute,+ addToolDefinitionsToInput,+ } = providerCase;+ const resourceSpans = buildResourceSpans(+ fixture,+ "1714488530686000000",+ "1714488530687000000",+ stringRepairs ?? {},+ numericRepairs,+ );+ const sourceSpan = resourceSpans[0].scopeSpans?.[0].spans?.[0];+ expect(sourceSpan).toBeDefined();++ const { storedRows } = await runOtelReplay({+ resourceSpans,+ projectId: `otel-replay-${name.toLowerCase().replaceAll(" ", "-")}`,+ });++ expect(storedRows).toHaveLength(1);+ const row = storedRows[0];++ expect(row.trace_id).toBe(idToHex(sourceSpan!.traceId));+ expect(row.span_id).toBe(idToHex(sourceSpan!.spanId));+ if (expectedObservationType) {+ expect(row.type).toBe(expectedObservationType);+ }+ expect(row.provided_model_name).toBe(expectedModelName);++ const spanIOInput = parseFixtureValue(fixture.spanIO.input);+ const expectedOutput = parseFixtureValue(fixture.spanIO.output);+ const storedInput = persistedJsonColumn(row, "input");+ expect(persistedJsonColumn(row, "output")).toEqual(expectedOutput);++ if (toolDefinitionsAttribute) {+ const toolDefinitions = fixtureJsonAttribute(+ fixture,+ toolDefinitionsAttribute,+ ) as unknown[];+ const expectedInput = addToolDefinitionsToInput+ ? {+ ...(spanIOInput as Record<string, unknown>),+ tools: toolDefinitions,+ }+ : spanIOInput;+ expect(storedInput).toEqual(expectedInput);+ expect(persistedJsonColumn(row, "tool_definitions")).toEqual(+ expectedToolDefinitions(toolDefinitions),+ );+ } else {+ expect(storedInput).toEqual(spanIOInput);+ }++ expect(persistedJsonColumn(row, "provided_usage_details")).toEqual(+ expectedUsageDetails,+ );+ expect(persistedJsonColumn(row, "usage_details")).toEqual(+ expectedFinalUsageDetails,+ );++ if (expectedToolCalls) {+ expect(row.tool_call_names).toEqual(+ expectedToolCalls.map((call) => call.name),+ );+ expect(persistedToolCalls(row)).toEqual(+ expectedToolCalls.map(({ name: _name, ...call }) => call),+ );+ }+ if (expectedMetadata) assertMetadataValues(row, expectedMetadata);+ if (expectedModelParameters) {+ expect(persistedJsonColumn(row, "model_parameters")).toMatchObject(+ expectedModelParameters,+ );+ }+ },+ );+});diff --git a/worker/src/__tests__/otelReplay.properties.test.ts b/worker/src/__tests__/otelReplay.properties.test.tsnew file mode 100644index 0000000000..c319ea7908--- /dev/null+++ b/worker/src/__tests__/otelReplay.properties.test.ts@@ -0,0 +1,703 @@+import "./helpers/otelReplaySetup";++import { randomInt } from "node:crypto";+import fc, { type Arbitrary } from "fast-check";+import { describe, expect, it } from "vitest";+import type { ResourceSpan } from "@langfuse/shared/src/server";+import { runOtelReplay } from "./helpers/otelReplayHarness";+import {+ type Json,+ jsonKeyArbitrary,+ recursiveJsonArbitrary,+} from "./helpers/arbitraries/json";++const PROJECT_PREFIX = "otel-replay-property";+const FILE_KEY = "otel-replay/test.json";+const RESOURCE_ENVIRONMENT = "resource-env";+const PROVIDER_MODEL = "provider-model";+const SCOPE_NAME = "langfuse-sdk-adversarial";+const SERVICE_NAME = "otel-property-service";+const RESOURCE_VERSION = "resource-version";+const RESOURCE_RELEASE = "resource-release";++type JsonObject = Record<string, Json>;++type RootPayload =+ | { kind: "json"; value: Json }+ | { kind: "raw"; value: string };++type TypedScalar =+ | { kind: "string"; value: string }+ | { kind: "int"; value: number }+ | { kind: "double"; value: number | "NaN" | "Infinity" | "-Infinity" }+ | { kind: "bool"; value: boolean };++type MetadataAttribute = {+ key: string;+ value: TypedScalar | { kind: "array"; values: TypedScalar[] };+};++type SpanSeed = {+ input: RootPayload;+ output: RootPayload;+ observationMetadata: JsonObject;+ traceMetadata: JsonObject;+ observationAttributes: MetadataAttribute[];+ traceAttributes: MetadataAttribute[];+ parentSelector: number;+ orderKey: number;+ timestampPresence: "both" | "startOnly" | "endOnly";+ promptVersion: number | null;+ version: string | null;+ release: string | null;+ environment: "span" | "lower" | "empty" | "resource";+ modelPrecedence: "canonical" | "response" | "request" | "none";+};++type ReplayCase = {+ traceId: string;+ spanIdBase: bigint;+ baseTimeMillis: number;+ spanSeeds: SpanSeed[];+};++// Null means the attribute is absent; empty strings are sent verbatim.+function stringAttributes(values: Record<string, string | null>) {+ return Object.entries(values).flatMap(([key, value]) =>+ value === null ? [] : [{ key, value: { stringValue: value } }],+ );+}++function typedAttribute(attribute: MetadataAttribute) {+ const encodeScalar = (value: TypedScalar) => {+ switch (value.kind) {+ case "string":+ return { stringValue: value.value };+ case "int":+ return { intValue: String(value.value) };+ case "double":+ return { doubleValue: value.value };+ case "bool":+ return { boolValue: value.value };+ }+ };++ return {+ key: attribute.key,+ value:+ attribute.value.kind === "array"+ ? { arrayValue: { values: attribute.value.values.map(encodeScalar) } }+ : encodeScalar(attribute.value),+ };+}++function typedAttributeValue(attribute: MetadataAttribute): unknown {+ if (attribute.value.kind === "array") {+ return attribute.value.values.map((value) => value.value);+ }+ return attribute.value.value;+}++function unixNanos(millis: number, remainder: number): string {+ return (BigInt(millis) * 1_000_000n + BigInt(remainder)).toString();+}++function paddedHex(value: bigint, digits: number): string {+ return value.toString(16).padStart(digits, "0");+}++function rootPayloadText(value: RootPayload): string {+ return value.kind === "json" ? JSON.stringify(value.value) : value.value;+}++function metadataFromSource(+ raw: JsonObject,+ attributes: MetadataAttribute[],+): Record<string, unknown> {+ // Prefixed __proto__ entries are currently dropped during extraction; an+ // own __proto__ key in the raw JSON metadata remains part of the stored data.+ const dottedValues = Object.fromEntries(+ attributes+ .filter((attribute) => attribute.key !== "__proto__")+ .map((attribute) => [attribute.key, typedAttributeValue(attribute)]),+ );+ // OTEL's raw JSON metadata is merged first; prefixed attributes then replace+ // an identical top-level key while leaving literal-dot/nested collisions.+ // JSON transport canonicalizes numbers such as -0 before metadata is flattened.+ return { ...JSON.parse(JSON.stringify(raw)), ...dottedValues };+}++function expectedMetadata(seed: SpanSeed): Record<string, unknown> {+ const base = {+ resourceAttributes: {+ "service.name": SERVICE_NAME,+ "service.version": RESOURCE_VERSION,+ "langfuse.environment": RESOURCE_ENVIRONMENT,+ "langfuse.release": RESOURCE_RELEASE,+ },+ scope: { name: SCOPE_NAME, version: "1.0.0", attributes: {} },+ };++ return {+ ...base,+ ...metadataFromSource(seed.observationMetadata, seed.observationAttributes),+ ...metadataFromSource(seed.traceMetadata, seed.traceAttributes),+ };+}++// Test-side contract oracle for the persisted flattened metadata. Arrays stay+// leaves; returning pairs preserves duplicate flattened paths.+function expectedMetadataPairs(+ metadata: Record<string, unknown>,+ prefix = "",+): Array<[string, string]> {+ return Object.entries(metadata).flatMap(([key, value]) => {+ const path = prefix ? `${prefix}.${key}` : key;+ if (value !== null && typeof value === "object" && !Array.isArray(value)) {+ return expectedMetadataPairs(value as Record<string, unknown>, path);+ }+ const persistedValue =+ value === null || value === undefined+ ? ""+ : typeof value === "string"+ ? value+ : JSON.stringify(value);+ return [[path, persistedValue]];+ });+}++function buildReplayCase(params: ReplayCase) {+ const { traceId, spanSeeds, baseTimeMillis } = params;+ const projectId = `${PROJECT_PREFIX}-${traceId}`;+ const spanIds = spanSeeds.map((_, ordinal) =>+ paddedHex(params.spanIdBase - BigInt(ordinal), 16),+ );++ const cases = spanSeeds.map((seed, ordinal) => {+ const spanId = spanIds[ordinal];+ const parentSpanId =+ ordinal === 0 ? "" : spanIds[seed.parentSelector % ordinal];+ const marker = `${traceId}-${ordinal}`;+ const start = baseTimeMillis + ordinal * 1_000;+ const end = start + 250;+ const modelName = {+ canonical: `explicit-model-${marker}`,+ response: `response-model-${marker}`,+ request: `request-model-${marker}`,+ none: "",+ }[seed.modelPrecedence];+ const metadata = expectedMetadataPairs(expectedMetadata(seed));+ const expected = {+ project_id: projectId,+ trace_id: traceId,+ span_id: spanId,+ parent_span_id: parentSpanId,+ name: `property-span-${marker}`,+ type: "GENERATION",+ input: rootPayloadText(seed.input),+ output: rootPayloadText(seed.output),+ start_time: seed.timestampPresence === "endOnly" ? end : start,+ end_time: seed.timestampPresence === "startOnly" ? start : end,+ prompt_version: seed.promptVersion,+ release: seed.release ?? RESOURCE_RELEASE,+ version: seed.version ?? RESOURCE_VERSION,+ environment:+ seed.environment === "span"+ ? `span-env-${traceId.slice(-8)}-${ordinal}`+ : RESOURCE_ENVIRONMENT,+ provided_model_name: modelName,+ trace_name: `property-trace-${traceId}`,+ service_name: SERVICE_NAME,+ service_version: RESOURCE_VERSION,+ scope_name: SCOPE_NAME,+ scope_version: "1.0.0",+ source: "otel",+ ingestion_sdk_name: "otel-replay",+ ingestion_sdk_version: "test",+ blob_storage_file_path: FILE_KEY,+ metadata_names: metadata.map(([name]) => name),+ metadata_values: metadata.map(([, value]) => value),+ };+ const attributes = stringAttributes({+ "langfuse.observation.type": "generation",+ "langfuse.observation.input": expected.input,+ "langfuse.observation.output": expected.output,+ "gen_ai.input.messages": JSON.stringify([+ { role: "user", content: `provider-input-${marker}` },+ ]),+ "gen_ai.output.messages": JSON.stringify([+ { role: "assistant", content: `provider-output-${marker}` },+ ]),+ "gen_ai.request.model":+ seed.modelPrecedence === "request"+ ? modelName+ : seed.modelPrecedence === "none"+ ? ""+ : PROVIDER_MODEL,+ "langfuse.observation.prompt.version":+ seed.promptVersion?.toString() ?? null,+ "gen_ai.response.model":+ seed.modelPrecedence === "request" || seed.modelPrecedence === "none"+ ? ""+ : `response-model-${marker}`,+ "langfuse.trace.name": expected.trace_name,+ "langfuse.observation.metadata": JSON.stringify(seed.observationMetadata),+ "langfuse.trace.metadata": JSON.stringify(seed.traceMetadata),+ });+ const metadataAttributes = [+ ...seed.observationAttributes.map((attribute) =>+ typedAttribute({+ ...attribute,+ key: `langfuse.observation.metadata.${attribute.key}`,+ }),+ ),+ ...seed.traceAttributes.map((attribute) =>+ typedAttribute({+ ...attribute,+ key: `langfuse.trace.metadata.${attribute.key}`,+ }),+ ),+ ];+ const overrides = stringAttributes({+ "langfuse.observation.model.name":+ seed.modelPrecedence === "canonical" ? modelName : "",+ "langfuse.version": seed.version,+ "langfuse.release": seed.release,+ "langfuse.environment":+ seed.environment === "span"+ ? expected.environment+ : seed.environment === "empty"+ ? ""+ : null,+ "deployment.environment.name":+ seed.environment === "lower" ? "span-fallback-env" : null,+ });+ return {+ expected,+ orderKey: seed.orderKey,+ span: {+ traceId: Buffer.from(traceId, "hex"),+ spanId: Buffer.from(spanId, "hex"),+ ...(ordinal === 0+ ? {}+ : { parentSpanId: Buffer.from(parentSpanId, "hex") }),+ name: expected.name,+ kind: 1,+ ...(seed.timestampPresence === "endOnly"+ ? {}+ : {+ startTimeUnixNano: unixNanos(+ start,+ seed.timestampPresence === "both" ? 0 : 999_999,+ ),+ }),+ ...(seed.timestampPresence === "startOnly"+ ? {}+ : {+ endTimeUnixNano: unixNanos(+ end,+ seed.timestampPresence === "both" ? 1 : 999_999,+ ),+ }),+ attributes: [...attributes, ...metadataAttributes, ...overrides],+ status: { code: 0 },+ },+ };+ });++ return {+ projectId,+ expectedRows: cases.map(({ expected }) => expected),+ resourceSpans: [+ {+ resource: {+ attributes: stringAttributes({+ "service.name": SERVICE_NAME,+ "service.version": RESOURCE_VERSION,+ "langfuse.environment": RESOURCE_ENVIRONMENT,+ "langfuse.release": RESOURCE_RELEASE,+ }),+ },+ scopeSpans: [+ {+ scope: { name: SCOPE_NAME, version: "1.0.0" },+ // Children are shuffled and the root remains last in the batch.+ spans: [+ ...cases+ .slice(1)+ .sort((left, right) => left.orderKey - right.orderKey)+ .map(({ span }) => span),+ cases[0].span,+ ],+ },+ ],+ },+ ] satisfies ResourceSpan[],+ };+}++function persistedMillis(value: unknown): number {+ if (typeof value !== "string") {+ throw new Error(+ `Expected a ClickHouse datetime string, received ${String(value)}`,+ );+ }+ const iso = value.includes("T") ? value : `${value.replace(" ", "T")}Z`;+ const millis = Date.parse(iso);+ if (!Number.isFinite(millis)) {+ throw new Error(`Invalid persisted ClickHouse datetime: ${value}`);+ }+ return millis;+}++// These top-level keys trigger tool-definition migration in the downstream+// observation normalizer. That behavior has its own tests; this property owns+// generic metadata JSON persistence and excludes those two recognized forms.+const metadataObjectKeyArbitrary = jsonKeyArbitrary.filter(+ (key) => key !== "tools" && key !== "attributes",+);+const metadataAttributeKeyArbitrary = jsonKeyArbitrary.filter(+ (key) =>+ // Empty suffixes are ignored; tool-normalizer reserved forms are out of scope.+ key !== "" && key !== "tools" && key !== "attributes",+);++const specialJsonValues: Json[] = [+ null,+ "",+ { emptyObject: {}, emptyArray: [], nullLeaf: null },+ JSON.parse(+ '{"__proto__":{"preserved":true},"constructor":{"prototype":false},"a.b":"literal dotted key","nested":{"a":{"b":"nested dotted path"}}}',+ ) as Json,+ {+ line: 'line\nreturn\rtab\t nul\u0000 quote" slash/ backslash\\',+ unicode: "🌍🧪\u2028\u2029",+ deep: { one: { two: { three: { four: [null, false, "終"] } } } },+ },+ [+ { role: "human", content: [{ type: "text", text: "question" }] },+ { role: "ai", content: [{ type: "text", text: "answer" }] },+ ],+];++// OTLP distinguishes scalar types even when JSON would encode the same value;+// typed arrays exercise the metadata conversion before flattening.+function scalarAttributeArbitrary(): Arbitrary<TypedScalar> {+ return fc.oneof(+ { withCrossShrink: true },+ fc.record({+ kind: fc.constant("string" as const),+ value: fc.string({ maxLength: 20, unit: "grapheme" }),+ }),+ fc.record({+ kind: fc.constant("int" as const),+ value: fc.integer({ min: -1_000_000, max: 1_000_000 }),+ }),+ fc.record({+ kind: fc.constant("double" as const),+ value: fc.oneof(+ { withCrossShrink: true },+ fc.double({ noNaN: true, noDefaultInfinity: true }),+ fc.constantFrom(+ "NaN" as const,+ "Infinity" as const,+ "-Infinity" as const,+ ),+ ),+ }),+ fc.record({ kind: fc.constant("bool" as const), value: fc.boolean() }),+ );+}++function metadataAttributeArbitrary(+ keyArbitrary: Arbitrary<string>,+): Arbitrary<MetadataAttribute> {+ const scalar = scalarAttributeArbitrary();+ return fc.record({+ key: keyArbitrary,+ value: fc.oneof(+ { withCrossShrink: true },+ scalar,+ fc.record({+ kind: fc.constant("array" as const),+ values: fc.array(scalar, { maxLength: 4 }),+ }),+ ),+ });+}++// The Langfuse IO attributes accept both JSON text and raw strings. Whitespace+// and invalid JSON must survive, independently for input and output.+function rootPayloadArbitrary(+ jsonValue: Arbitrary<Json>,+): Arbitrary<RootPayload> {+ return fc.oneof(+ { withCrossShrink: true },+ fc.record({+ kind: fc.constant("raw" as const),+ value: fc.oneof(+ { withCrossShrink: true },+ fc.string({ maxLength: 24, unit: "grapheme" }),+ fc.constantFrom(+ "",+ "plain text",+ "{invalid",+ " \r\n\t",+ " null ",+ " {} ",+ ),+ ),+ }),+ fc.record({ kind: fc.constant("json" as const), value: jsonValue }),+ );+}++// Both campaigns vary every axis independently. The biased campaign adds+// provider-shaped state and metadata collisions that broad JSON seldom produces.+function replayCaseArbitrary(+ depth: number,+ maxWidth: number,+ biased: boolean,+): Arbitrary<ReplayCase> {+ const broadJson = recursiveJsonArbitrary(depth, maxWidth);+ const jsonValue = biased+ ? fc.oneof(+ { withCrossShrink: true },+ { weight: 2, arbitrary: broadJson },+ { weight: 5, arbitrary: fc.constantFrom(...specialJsonValues) },+ )+ : broadJson;+ const metadataValue = recursiveJsonArbitrary(+ Math.max(1, depth - 1),+ maxWidth,+ );+ const broadMetadataRoot = fc.dictionary(+ metadataObjectKeyArbitrary,+ metadataValue,+ {+ maxKeys: maxWidth,+ },+ );+ const metadataRoot = biased+ ? fc.oneof(+ { withCrossShrink: true },+ { weight: 2, arbitrary: broadMetadataRoot },+ {+ weight: 5,+ arbitrary: fc+ .record({+ nestedStatus: metadataValue,+ literalStatus: metadataValue,+ messages: fc.array(metadataValue, { maxLength: maxWidth }),+ deep: recursiveJsonArbitrary(4, maxWidth),+ wide: fc.array(metadataValue, { maxLength: maxWidth }),+ })+ .map(({ nestedStatus, literalStatus, messages, deep, wide }) => ({+ route: { status: nestedStatus },+ "route.status": literalStatus,+ langgraph: { messages, state: { values: wide } },+ resourceAttributes: { custom: { deep } },+ scope: { name: "generated-scope", attributes: { values: wide } },+ })),+ },+ )+ : broadMetadataRoot;+ const metadataAttrKey = biased+ ? fc.oneof(+ { withCrossShrink: true },+ { weight: 2, arbitrary: metadataAttributeKeyArbitrary },+ {+ weight: 5,+ arbitrary: fc.constantFrom(+ "a.b",+ "route.status",+ "__proto__",+ "tags",+ "model",+ "scope.name",+ ),+ },+ )+ : metadataAttributeKeyArbitrary;+ const metadataAttrs = fc.array(metadataAttributeArbitrary(metadataAttrKey), {+ maxLength: 3,+ });++ return fc.record({+ traceId: fc.bigInt({ min: 1n, max: (1n << 128n) - 1n }).map(+ (value) => paddedHex(value, 32),+ (value) => {+ if (typeof value !== "string" || !/^[0-9a-f]{32}$/.test(value)) {+ throw new Error("Expected a 32-digit hexadecimal trace ID");+ }+ return BigInt(`0x${value}`);+ },+ ),+ spanIdBase: fc.bigInt({ min: 6n, max: (1n << 64n) - 1n }),+ baseTimeMillis: fc.integer({+ min: 1_700_000_000_000,+ max: 1_800_000_000_000,+ }),+ // A filtered zero-minimum array can shrink away trailing spans even when+ // its first span is the failure; fast-check's minLength: 1 retains that tail.+ spanSeeds: fc+ .array(+ fc.record({+ input: rootPayloadArbitrary(jsonValue),+ output: rootPayloadArbitrary(jsonValue),+ observationMetadata: metadataRoot,+ traceMetadata: metadataRoot,+ observationAttributes: metadataAttrs,+ traceAttributes: metadataAttrs,+ parentSelector: fc.nat(65_535),+ orderKey: fc.integer(),+ timestampPresence: fc.constantFrom(+ "both" as const,+ "startOnly" as const,+ "endOnly" as const,+ ),+ promptVersion: fc.constantFrom(null, 0, 1, 65_535),+ version: fc.option(fc.string({ maxLength: 20 }), { nil: null }),+ release: fc.option(fc.string({ maxLength: 20 }), { nil: null }),+ environment: fc.constantFrom(+ "resource" as const,+ "span" as const,+ "lower" as const,+ "empty" as const,+ ),+ modelPrecedence: fc.constantFrom(+ "none" as const,+ "canonical" as const,+ "response" as const,+ "request" as const,+ ),+ }),+ { maxLength: 6 },+ )+ .filter((seeds) => seeds.length > 0),+ });+}++// Small defaults keep directed witnesses within the same shrinkable input domain.+const minimalSpan: SpanSeed = {+ input: { kind: "raw", value: "" },+ output: { kind: "raw", value: "" },+ observationMetadata: {},+ traceMetadata: {},+ observationAttributes: [],+ traceAttributes: [],+ parentSelector: 0,+ orderKey: 0,+ timestampPresence: "both",+ promptVersion: null,+ version: null,+ release: null,+ environment: "resource",+ modelPrecedence: "none",+};++const directedCases: ReplayCase[] = [+ {+ traceId: paddedHex(1n, 32),+ spanIdBase: 6n,+ baseTimeMillis: 1_700_000_000_000,+ spanSeeds: [+ {+ ...minimalSpan,+ observationMetadata: {+ route: { status: "nested" },+ "route.status": "literal",+ },+ traceAttributes: [+ { key: "route.status", value: { kind: "string", value: "trace" } },+ ],+ },+ ],+ },+ {+ traceId: paddedHex(2n, 32),+ spanIdBase: 7n,+ baseTimeMillis: 1_700_000_000_000,+ spanSeeds: [+ {+ ...minimalSpan,+ input: { kind: "raw", value: "{invalid" },+ observationMetadata: JSON.parse('{"__proto__":"raw-value"}'),+ observationAttributes: [+ { key: "a", value: { kind: "array", values: [] } },+ ],+ traceAttributes: [+ { key: "__proto__", value: { kind: "string", value: "kept" } },+ ],+ },+ {+ ...minimalSpan,+ input: { kind: "json", value: [null, false, {}] },+ output: { kind: "raw", value: " null " },+ },+ ],+ },+];++async function assertReplayPersists(params: ReplayCase): Promise<void> {+ const replay = buildReplayCase(params);+ const { storedRows } = await runOtelReplay({+ resourceSpans: replay.resourceSpans,+ projectId: replay.projectId,+ fileKey: FILE_KEY,+ });++ expect(storedRows).toHaveLength(replay.expectedRows.length);+ const rowsBySpanId = new Map(+ storedRows.map((row) => [String(row.span_id), row]),+ );+ expect([...rowsBySpanId.keys()].sort()).toEqual(+ replay.expectedRows.map((row) => row.span_id).sort(),+ );++ for (const expected of replay.expectedRows) {+ const row = rowsBySpanId.get(expected.span_id)!;+ // Compare every contracted field exactly, including both metadata arrays:+ // objectContaining permits unrelated stored columns, not extra array elements.+ expect({+ ...row,+ start_time: persistedMillis(row.start_time),+ end_time: persistedMillis(row.end_time),+ }).toEqual(expect.objectContaining(expected));+ }+}++// The replay writer retries internally; Vitest retries could overlap an+// unfinished replay and race the shared writer singleton.+describe(+ "OTEL replay adversarial properties",+ { retry: 0, timeout: 120_000 },+ () => {+ it.each([+ ["broad recursive JSON", replayCaseArbitrary(4, 4, false), []],+ [+ "edge-biased recursive JSON",+ replayCaseArbitrary(6, 7, true),+ directedCases.map((example) => [example]),+ ],+ ])(+ "persists %s input/output and recursive metadata with identity and precedence intact",+ async (name, arbitrary, examples) => {+ const seed = randomInt(2 ** 31);+ // Log before running so a Vitest timeout still leaves the input sequence reproducible.+ process.stderr.write(`OTEL replay ${name}: seed=${seed}\n`);+ await fc.assert(+ fc.asyncProperty(+ arbitrary as Arbitrary<ReplayCase>,+ assertReplayPersists,+ ),+ {+ seed,+ numRuns: 128,+ ...(examples.length > 0 ? { examples } : {}),+ },+ );+ },+ );+ },+);