🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix: harden OTel attribute path reconstruction (#16681)
Valery Meleshkin02f1d8ecf67b
diff --git a/packages/shared/src/server/otel/OtelIngestionProcessor.metadataDropped.test.ts b/packages/shared/src/server/otel/OtelIngestionProcessor.metadataDropped.test.tsindex 699f26ea41..0ff5ac3b92 100644--- a/packages/shared/src/server/otel/OtelIngestionProcessor.metadataDropped.test.ts+++ b/packages/shared/src/server/otel/OtelIngestionProcessor.metadataDropped.test.ts@@ -661,25 +661,44 @@ describe("OTel reconstructed array drop telemetry", () => {([stat]) => stat === ARRAY_ATTRIBUTE_DROPPED_METRIC,);- it("reports an out-of-range array attribute", async () => {+ it("reports validation drops with their own reasons", async () => {const processor = createProcessor();const batch = buildBatch([{key: "llm.input_messages.10001.content",value: { stringValue: "dropped" },},+ {+ key: `llm.input_messages.${Array.from(+ { length: 65 },+ () => "nested",+ ).join(".")}`,+ value: { stringValue: "dropped" },+ },]);await processor.processToIngestionEvents(batch);- expect(arrayDropCalls()).toContainEqual([- ARRAY_ATTRIBUTE_DROPPED_METRIC,- 1,- {- reason: "reconstruction_budget_exceeded",- prefix: "llm.input_messages",- },- ]);+ expect(arrayDropCalls()).toEqual(+ expect.arrayContaining([+ [+ ARRAY_ATTRIBUTE_DROPPED_METRIC,+ 1,+ {+ reason: "reconstruction_array_index_exceeded",+ prefix: "llm.input_messages",+ },+ ],+ [+ ARRAY_ATTRIBUTE_DROPPED_METRIC,+ 1,+ {+ reason: "reconstruction_path_depth_exceeded",+ prefix: "llm.input_messages",+ },+ ],+ ]),+ );});it("caps warnings without logging rejected attribute keys or values", async () => {@@ -693,7 +712,7 @@ describe("OTel reconstructed array drop telemetry", () => {(_, index) =>buildBatch([{- key: `llm.input_messages.${10_001 + index}.secret-content`,+ key: "llm.input_messages.5000.messages.5000.secret-content",value: { stringValue: `customer-secret-${index}` },},])[0],@@ -723,4 +742,46 @@ describe("OTel reconstructed array drop telemetry", () => {warnSpy.mockRestore();}});++ it("does not let validation drops consume the array-budget warning cap", async () => {+ const warnSpy = vi+ .spyOn(serverBarrel.logger, "warn")+ .mockImplementation(() => serverBarrel.logger);++ try {+ const deepPath = Array.from({ length: 65 }, () => "nested").join(".");+ const processor = createProcessor();+ const validationBatches = Array.from(+ { length: 12 },+ () =>+ buildBatch([+ {+ key: `llm.input_messages.${deepPath}`,+ value: { stringValue: "dropped" },+ },+ ])[0],+ );++ await processor.processToIngestionEvents(validationBatches);+ warnSpy.mockClear();++ await processor.processToIngestionEvents(+ buildBatch([+ {+ key: "llm.input_messages.5000.messages.5000.content",+ value: { stringValue: "dropped" },+ },+ ]),+ );++ expect(warnSpy).toHaveBeenCalledWith("OTEL array attribute dropped", {+ projectId: PROJECT_ID,+ prefix: "llm.input_messages",+ reason: "reconstruction_budget_exceeded",+ droppedAttributeCount: 1,+ });+ } finally {+ warnSpy.mockRestore();+ }+ });});diff --git a/packages/shared/src/server/otel/OtelIngestionProcessor.ts b/packages/shared/src/server/otel/OtelIngestionProcessor.tsindex f5e1e6ea0c..6067d2af6f 100644--- a/packages/shared/src/server/otel/OtelIngestionProcessor.ts+++ b/packages/shared/src/server/otel/OtelIngestionProcessor.ts@@ -99,6 +99,23 @@ interface MetadataDropContext {dropScope: object;}+const DANGEROUS_OTEL_PATH_SEGMENTS = new Set([+ "__proto__",+ "constructor",+ "prototype",+]);+const CANONICAL_ARRAY_INDEX = /^(0|[1-9]\d*)$/;+type ReconstructedAttributeDropReason =+ | "reconstruction_array_index_exceeded"+ | "reconstruction_budget_exceeded"+ | "reconstruction_path_depth_exceeded";++const getOtelArrayIndex = (segment: string): number | undefined => {+ if (!CANONICAL_ARRAY_INDEX.test(segment)) return undefined;+ const index = Number(segment);+ return Number.isSafeInteger(index) ? index : undefined;+};+/*** Tracks the total number of array slots that dotted attribute paths would* materialize. A path is reserved transactionally: when it would exceed the@@ -121,12 +138,13 @@ class ArraySlotBudget {for (let index = 0; index < pathParts.length; index++) {const segment = pathParts[index];- if (!/^\d+$/.test(segment)) continue;+ const arrayIndex = getOtelArrayIndex(segment);+ if (arrayIndex === undefined) continue;const arrayPath = pathParts.slice(0, index).join(".");const previousLength =pendingLengths.get(arrayPath) ?? this.lengthsByPath.get(arrayPath) ?? 0;- const requiredLength = Number(segment) + 1;+ const requiredLength = arrayIndex + 1;if (requiredLength > previousLength) {additionalSlots += requiredLength - previousLength;pendingLengths.set(arrayPath, requiredLength);@@ -244,9 +262,10 @@ export class OtelIngestionProcessor {private static readonly OTEL_ARRAY_ATTRIBUTE_DROPPED_METRIC ="langfuse.ingestion.otel.array_attribute_dropped";- // Flattened OTel message attributes name array slots directly, so cap the- // total logical array length before reconstructing the value.+ // Flattened OTel message attributes can create deeply nested objects and+ // sparse arrays, so bound both forms of structural expansion.private static readonly MAX_OTEL_RECONSTRUCTED_ARRAY_SLOTS = 10_001;+ private static readonly MAX_OTEL_RECONSTRUCTED_PATH_DEPTH = 64;private static readonly METADATA_DROP_WARN_CAP = 10;private static readonly ARRAY_ATTRIBUTE_DROP_WARN_CAP = 10;@@ -254,7 +273,10 @@ export class OtelIngestionProcessor {private seenTraces: Set<string> = new Set();private reportedMetadataDrops = new WeakMap<object, Set<string>>();private metadataDropWarnCount = 0;- private arrayAttributeDropWarnCount = 0;+ private arrayAttributeDropWarnCounts = new Map<+ ReconstructedAttributeDropReason,+ number+ >();private isInitialized = false;private traceEventCounts = {shallow: 0,@@ -1530,9 +1552,9 @@ export class OtelIngestionProcessor {** TraceLoop and OpenInference encode structured messages in attribute names;* for example, `gen_ai.prompt.0.content` becomes `[{ content: value }]`.- * Numeric path segments therefore become JavaScript array indices. The total- * logical slots across all nested arrays are bounded, since dimensions can- * otherwise multiply into a huge sparse structure during serialization.+ * Paths retain the legacy insertion-order conflict policy. Generated objects+ * have no prototype, arrays only accept bounded numeric indices, and the+ * writer never descends into an object supplied as an attribute value.*/private convertKeyPathToNestedObject(input: Record<string, unknown>,@@ -1542,103 +1564,184 @@ export class OtelIngestionProcessor {return input[prefix];}- const keys: string[] = [];- const overBudgetKeys: string[] = [];+ type ReconstructedContainer = Record<string, unknown> | unknown[];+ const droppedAttributeCounts = new Map<+ ReconstructedAttributeDropReason,+ number+ >();+ const recordDrop = (reason: ReconstructedAttributeDropReason): void => {+ droppedAttributeCounts.set(+ reason,+ (droppedAttributeCounts.get(reason) ?? 0) + 1,+ );+ };+ const parsePath = (+ key: string,+ recordLimitDrop: boolean,+ ): string[] | undefined => {+ const segments = key.split(".");+ if (+ segments.length >+ OtelIngestionProcessor.MAX_OTEL_RECONSTRUCTED_PATH_DEPTH+ ) {+ if (recordLimitDrop) recordDrop("reconstruction_path_depth_exceeded");+ return undefined;+ }++ for (const segment of segments) {+ if (segment.length === 0 || DANGEROUS_OTEL_PATH_SEGMENTS.has(segment)) {+ return undefined;+ }++ if (CANONICAL_ARRAY_INDEX.test(segment)) {+ const index = getOtelArrayIndex(segment);+ if (+ index === undefined ||+ index >= OtelIngestionProcessor.MAX_OTEL_RECONSTRUCTED_ARRAY_SLOTS+ ) {+ if (recordLimitDrop) {+ recordDrop("reconstruction_array_index_exceeded");+ }+ return undefined;+ }+ }+ }+ return segments;+ };++ const prefixWithDot = `${prefix}.`;+ const createdContainers = new WeakSet<object>();const arraySlotBudget = new ArraySlotBudget(OtelIngestionProcessor.MAX_OTEL_RECONSTRUCTED_ARRAY_SLOTS,);- for (const inputKey of Object.keys(input)) {- const key = inputKey.replace(`${prefix}.`, "");- if (!arraySlotBudget.tryReserve(key.split("."))) {- overBudgetKeys.push(key);- continue;+ const rootArraySlotBudget = new ArraySlotBudget(+ OtelIngestionProcessor.MAX_OTEL_RECONSTRUCTED_ARRAY_SLOTS,+ );++ const inputKeys = Object.keys(input);+ const useArray = inputKeys.some((inputKey) => {+ if (!inputKey.startsWith(prefixWithDot)) return false;+ const path = parsePath(inputKey.slice(prefixWithDot.length), false);+ if (path === undefined || !rootArraySlotBudget.tryReserve(path)) {+ return false;}+ return path.length > 1 && getOtelArrayIndex(path[0]) !== undefined;+ });+ const result: ReconstructedContainer = useArray ? [] : Object.create(null);+ createdContainers.add(result);++ const isWritableKey = (+ container: ReconstructedContainer,+ key: string,+ ): boolean => {+ if (!Array.isArray(container)) return true;+ const index = getOtelArrayIndex(key);+ return (+ index !== undefined &&+ index < OtelIngestionProcessor.MAX_OTEL_RECONSTRUCTED_ARRAY_SLOTS+ );+ };- keys.push(key);- }- this.recordArrayAttributesDropped(prefix, overBudgetKeys);- const useArray = keys.some((key) => key.match(/^\d+\./));+ const defineOwn = (+ container: ReconstructedContainer,+ key: string,+ value: unknown,+ ): void => {+ Object.defineProperty(container, key, {+ configurable: true,+ enumerable: true,+ value,+ writable: true,+ });+ };- // Blocklist to prevent prototype pollution via crafted OTel attribute keys- const DANGEROUS_KEYS = new Set(["__proto__", "constructor", "prototype"]);+ const applyPath = (+ root: ReconstructedContainer,+ path: readonly string[],+ value: unknown,+ commit: boolean,+ ): boolean => {+ let current = root;+ for (let index = 0; index < path.length - 1; index++) {+ const key = path[index];+ if (!isWritableKey(current, key)) return false;- // Helper function to set a value at a nested path- const setNestedValue = (obj: any, path: string[], value: unknown): void => {- let current = obj;- for (let i = 0; i < path.length - 1; i++) {- const key = path[i];- if (DANGEROUS_KEYS.has(key)) return;if (!Object.hasOwn(current, key)) {- // Check if next key is a number to decide if we need an array or object- current[key] = /^\d+$/.test(path[i + 1]) ? [] : Object.create(null);+ if (!commit) return true;+ const child: ReconstructedContainer =+ getOtelArrayIndex(path[index + 1]) !== undefined+ ? []+ : Object.create(null);+ createdContainers.add(child);+ defineOwn(current, key, child);+ current = child;+ continue;}- const next = current[key];- if (typeof next !== "object" || next === null) {- // Preserve an earlier leaf value when a later attribute conflicts.- return;++ const next = Reflect.get(current, key);+ if (+ typeof next !== "object" ||+ next === null ||+ !createdContainers.has(next)+ ) {+ return false;}- current = next;+ current = next as ReconstructedContainer;}const finalKey = path[path.length - 1];- if (!DANGEROUS_KEYS.has(finalKey)) {- current[finalKey] = value;+ if (!isWritableKey(current, finalKey)) return false;+ if (Object.hasOwn(current, finalKey)) {+ const existing = Reflect.get(current, finalKey);+ if (+ typeof existing === "object" &&+ existing !== null &&+ createdContainers.has(existing)+ ) {+ return false;+ }}+ if (commit) defineOwn(current, finalKey, value);+ return true;};- if (useArray) {- const result: any[] = [];- for (const key of keys) {- const pathParts = key.split(".");- const indexSegment = pathParts[0];- const index = Number(indexSegment);- if (!result[index]) {- result[index] = Object.create(null);- }- if (pathParts.length === 2) {- // Simple case: 0.content -> result[0].content- result[index][pathParts[1]] = input[`${prefix}.${key}`];- } else {- // Nested case: 0.message.content -> result[0].message.content- setNestedValue(- result[index],- pathParts.slice(1),- input[`${prefix}.${key}`],- );- }+ for (const inputKey of inputKeys) {+ if (!inputKey.startsWith(prefixWithDot)) continue;+ const path = parsePath(inputKey.slice(prefixWithDot.length), true);+ if (!path) continue;+ const value = input[inputKey];+ if (!applyPath(result, path, value, false)) continue;+ if (!arraySlotBudget.tryReserve(path)) {+ recordDrop("reconstruction_budget_exceeded");+ continue;}- return result;+ applyPath(result, path, value, true);}-- const result: Record<string, unknown> = Object.create(null);- for (const key of keys) {- const pathParts = key.split(".");- if (pathParts.length === 1) {- result[key] = input[`${prefix}.${key}`];- } else {- setNestedValue(result, pathParts, input[`${prefix}.${key}`]);- }+ for (const [reason, droppedAttributeCount] of droppedAttributeCounts) {+ this.recordArrayAttributesDropped(prefix, reason, droppedAttributeCount);}return result;}- private recordArrayAttributesDropped(prefix: string, keys: string[]): void {- if (keys.length === 0) return;+ private recordArrayAttributesDropped(+ prefix: string,+ reason: ReconstructedAttributeDropReason,+ droppedAttributeCount: number,+ ): void {+ if (droppedAttributeCount === 0) return;recordIncrement(OtelIngestionProcessor.OTEL_ARRAY_ATTRIBUTE_DROPPED_METRIC,- keys.length,- { reason: "reconstruction_budget_exceeded", prefix },+ droppedAttributeCount,+ { reason, prefix },);- if (- this.arrayAttributeDropWarnCount <- OtelIngestionProcessor.ARRAY_ATTRIBUTE_DROP_WARN_CAP- ) {- this.arrayAttributeDropWarnCount += 1;+ const warningCount = this.arrayAttributeDropWarnCounts.get(reason) ?? 0;+ if (warningCount < OtelIngestionProcessor.ARRAY_ATTRIBUTE_DROP_WARN_CAP) {+ this.arrayAttributeDropWarnCounts.set(reason, warningCount + 1);logger.warn("OTEL array attribute dropped", {projectId: this.projectId,prefix,- reason: "reconstruction_budget_exceeded",- droppedAttributeCount: keys.length,+ reason,+ droppedAttributeCount,});}}diff --git a/web/src/__tests__/server/api/otel/otelMapping.servertest.ts b/web/src/__tests__/server/api/otel/otelMapping.servertest.tsindex 2e46d6bdc8..83c1dad292 100644--- a/web/src/__tests__/server/api/otel/otelMapping.servertest.ts+++ b/web/src/__tests__/server/api/otel/otelMapping.servertest.ts@@ -9693,6 +9693,61 @@ describe("OTel Resource Span Mapping", () => {}};+ it.each([+ [+ "flat object",+ [+ ["role", "user"],+ ["content", "hello"],+ ],+ { role: "user", content: "hello" },+ ],+ [+ "root array with shared prefixes",+ [+ ["1.content", "world"],+ ["0.role", "user"],+ ["0.content", "hello"],+ ["1.role", "assistant"],+ ],+ [+ { role: "user", content: "hello" },+ { role: "assistant", content: "world" },+ ],+ ],+ [+ "nested objects and scalar arrays",+ [+ ["request.model", "test-model"],+ ["request.parts.1", "second"],+ ["request.parts.0", "first"],+ ["request.options.temperature", "0.5"],+ ],+ {+ request: {+ model: "test-model",+ parts: ["first", "second"],+ options: { temperature: "0.5" },+ },+ },+ ],+ ] as const)(+ "should preserve legacy reconstruction for %s",+ (_name, attributes, expected) => {+ const events = createTestOtelProcessor({ publicKey }).processToEvent([+ createResourceSpan(+ attributes.map(([path, stringValue]) => ({+ key: `gen_ai.prompt.${path}`,+ value: { stringValue },+ })),+ ),+ ]);++ expect(events).toHaveLength(1);+ expect(events[0].input).toEqual(expected);+ },+ );+it("should not traverse inherited properties in nested gen_ai.prompt attributes", async () => {const { events, observedCall, originalCall } =await convertAndObserveHasOwnPropertyCall([@@ -9734,10 +9789,10 @@ describe("OTel Resource Span Mapping", () => {const input = events.find((event) => event.type === "span-create")?.body.input as { a?: unknown[] } | undefined;expect(input?.a?.[0]).toBe("seed");- expect(Object.hasOwn(input?.a ?? [], "hasOwnProperty")).toBe(true);- expect(Reflect.get(input?.a ?? [], "hasOwnProperty")).toMatchObject({- call: "pwned",- });+ expect(Object.hasOwn(input?.a ?? [], "hasOwnProperty")).toBe(false);+ expect(Reflect.get(input?.a ?? [], "hasOwnProperty")).toBe(+ Object.prototype.hasOwnProperty,+ );});it("should preserve a scalar leaf and ignore a conflicting nested attribute", () => {@@ -9765,6 +9820,152 @@ describe("OTel Resource Span Mapping", () => {});});+ it("should preserve a reconstructed container and ignore a conflicting leaf", () => {+ const events = createTestOtelProcessor({ publicKey }).processToEvent([+ createResourceSpan([+ {+ key: "gen_ai.prompt.0.content",+ value: { stringValue: "nested" },+ },+ {+ key: "gen_ai.prompt.0",+ value: { stringValue: "leaf" },+ },+ ]),+ ]);++ expect(events).toHaveLength(1);+ expect(events[0].input).toEqual([{ content: "nested" }]);+ });++ it.each([+ {+ name: "array first",+ attributes: [+ {+ key: "gen_ai.prompt.a.0",+ value: { stringValue: "first" },+ },+ {+ key: "gen_ai.prompt.a.length",+ value: { stringValue: "pwned" },+ },+ ],+ expected: ["first"],+ },+ {+ name: "object first",+ attributes: [+ {+ key: "gen_ai.prompt.a.length",+ value: { stringValue: "pwned" },+ },+ {+ key: "gen_ai.prompt.a.0",+ value: { stringValue: "first" },+ },+ ],+ expected: { 0: "first", length: "pwned" },+ },+ ])(+ "should safely preserve the first reconstructed container kind ($name)",+ ({ attributes, expected }) => {+ const events = createTestOtelProcessor({ publicKey }).processToEvent([+ createResourceSpan([+ ...attributes,+ {+ key: "gen_ai.prompt.safe",+ value: { stringValue: "still-here" },+ },+ ]),+ ]);++ expect(events).toHaveLength(1);+ expect(events[0].input).toMatchObject({+ a: expected,+ safe: "still-here",+ });+ },+ );++ it("should not interpret noncanonical numeric syntax as an array index", () => {+ const events = createTestOtelProcessor({ publicKey }).processToEvent([+ createResourceSpan([+ {+ key: "gen_ai.prompt.0.content",+ value: { stringValue: "safe" },+ },+ {+ key: "gen_ai.prompt.1e6.content",+ value: { stringValue: "pwned" },+ },+ ]),+ ]);++ expect(events).toHaveLength(1);+ const input = events[0].input as Array<{ content: string }>;+ expect(input).toHaveLength(1);+ expect(input[0]).toEqual({ content: "safe" });+ });++ it("should drop paths that exceed the reconstruction depth limit", () => {+ const deepPath = Array.from({ length: 65 }, () => "nested").join(".");+ const events = createTestOtelProcessor({ publicKey }).processToEvent([+ createResourceSpan([+ {+ key: `gen_ai.prompt.${deepPath}`,+ value: { stringValue: "pwned" },+ },+ {+ key: "gen_ai.prompt.safe",+ value: { stringValue: "still-here" },+ },+ ]),+ ]);++ expect(events).toHaveLength(1);+ expect(events[0].input).toEqual({ safe: "still-here" });+ });++ it("should not let a rejected path consume the array-slot budget", () => {+ const events = createTestOtelProcessor({ publicKey }).processToEvent([+ createResourceSpan([+ {+ key: "gen_ai.prompt.0.message.__proto__.9999.content",+ value: { stringValue: "pwned" },+ },+ {+ key: "gen_ai.prompt.1.content",+ value: { stringValue: "still-here" },+ },+ ]),+ ]);++ expect(events).toHaveLength(1);+ const input = events[0].input as Array<{ content?: string }>;+ expect(input).toHaveLength(2);+ expect(input[0]).toBeUndefined();+ expect(input[1]).toEqual({ content: "still-here" });+ });++ it("should not let an over-budget array path change the root container type", () => {+ const events = createTestOtelProcessor({ publicKey }).processToEvent([+ createResourceSpan([+ {+ key: "gen_ai.prompt.5000.notes.5000.content",+ value: { stringValue: "dropped" },+ },+ {+ key: "gen_ai.prompt.name",+ value: { stringValue: "still-here" },+ },+ ]),+ ]);++ expect(events).toHaveLength(1);+ expect(events[0].input).toEqual({ name: "still-here" });+ });+it("should not pollute Object.prototype via __proto__ in gen_ai.prompt attributes", async () => {const traceId = "abcdef1234567890abcdef1234567890";const spanId = "abcdef1234567890";diff --git a/worker/src/__tests__/chatml/integration.test.ts b/worker/src/__tests__/chatml/integration.test.tsindex 3cace624b4..28df9cb23e 100644--- a/worker/src/__tests__/chatml/integration.test.ts+++ b/worker/src/__tests__/chatml/integration.test.ts@@ -1,4 +1,5 @@import { describe, it, expect, vi } from "vitest";+import { createHash } from "crypto";import { readFileSync, existsSync, writeFileSync, readdirSync } from "fs";import path from "path";@@ -12,6 +13,7 @@ import {} from "@langfuse/shared/src/utils/chatml";import { deepParseJson } from "@langfuse/shared";+import { OtelIngestionProcessor } from "@langfuse/shared/src/server";describe("ChatML Integration", () => {it("should handle OpenAI multimodal format", () => {@@ -522,6 +524,177 @@ const tracesDir = path.resolve(__dirname, "framework-traces");const traceFiles = readdirSync(tracesDir).filter((f) =>f.endsWith(".trace.json"),);++const otelReconstructionPrefixes = [+ "gen_ai.prompt",+ "gen_ai.completion",+ "llm.input_messages",+ "llm.output_messages",+] as const;++const expectedOtelReconstructionDigests = {+ "agno-2025-06-11.trace.json:b38a82eaa62b551e:llm.input_messages":+ "79dae2697161990380f3cc2409030d1e958b30706babd1a73613b10f5b807348",+ "agno-2025-06-11.trace.json:ca136de468e156c9:llm.input_messages":+ "b754777938eee8274ad8891da07446072ab8fe7898d99fd9430abb26aa278ec5",+ "beeai-2025-08-01.trace.json:52df0f7f9dad6b23:llm.input_messages":+ "9e09b39eafc8bf335608817d86dca4c7d6debbe8ff464bf68407388fc2be70c9",+ "beeai-2025-08-01.trace.json:f50f7bbf13112df9:llm.output_messages":+ "d874643405382404cadb4e70a9046f50357bdfb8bf5a4e44cc4d50a0216c101c",+ "beeai-2025-08-01.trace.json:4c5c5a7936ca24bc:llm.input_messages":+ "a736bdc132475ed501b72ecc14087aaa73bd01d54ed0b86f660f7059e93b6f4b",+ "beeai-2025-08-01.trace.json:1f956cbea34725bd:llm.input_messages":+ "f50c6ed16b46a5fd0c38aa5d1649759e6bd26bf3b279c058340ea4f2d1b5f17c",+ "crewai-2025-07-11.trace.json:231c43964b7e7e63:llm.input_messages":+ "090af8a41c384a74784270cecd1743fbe64b61af1344a7ad327eca06d5027bda",+ "crewai-2025-07-11.trace.json:231c43964b7e7e63:llm.output_messages":+ "5d2c3c0a6f1aafd174f5015efbda809a4bcddaf2cfd19c1b26ead101542f6e9d",+ "google-adk-2025-08-28.trace.json:bded677884f031ce:llm.input_messages":+ "077595470a234fdd1638cd54626bf6d9de33413b065f885d744d5384854c948d",+ "google-adk-2025-08-28.trace.json:bded677884f031ce:llm.output_messages":+ "f7c165eb9d7053e004498ff13be315d4c8dd1be1994854a14a3d1f2e2908b934",+ "google-adk-2025-08-28.trace.json:f6e4c65297e40c96:llm.input_messages":+ "a2c27fe417936852bb3ec99f8a5e333b56caf3658738bf133e8d1ec329f95e34",+ "google-adk-2025-08-28.trace.json:f6e4c65297e40c96:llm.output_messages":+ "3ec3d61cd82e7d8193ad0b9c4a4663fd19613afbe3014ec44f33399275400f71",+ "google-gemini-2025-08-01.trace.json:b7a63ca7e1d083bc:llm.input_messages":+ "02fb5023a8ab3234b69b0af60ef2fffe78e0443d452323ea733b10ba8d2e6757",+ "google-gemini-2025-08-01.trace.json:b7a63ca7e1d083bc:llm.output_messages":+ "9d51690e135533dbb44127193472d729d5afd48b2d69a16b1e79ccb68860470b",+ "koog-2025-08-26.trace.json:e44e72cf2d221781:gen_ai.prompt":+ "12541a8d701c9962ebcc381fbf54539d443a093a8c8cf3e22acd63f0ea146910",+ "koog-2025-08-26.trace.json:e44e72cf2d221781:gen_ai.completion":+ "41b2ea825990164dcecd9fe607a45ff5601fbd7d3146ce79b32dc69e9b2f1c6d",+ "koog-2025-08-26.trace.json:042f9350c3729147:gen_ai.prompt":+ "05dc418f9c1829e3b0c5d8d91086313dac3e6d7cc6d37655fe8d76ae082fab41",+ "koog-2025-08-26.trace.json:042f9350c3729147:gen_ai.completion":+ "d913abf5a058217e9436318ed64e54570a477063514aa499ef0941024ffdc6a5",+ "openai-agents-2025-09-30.trace.json:90d94774e8e3724d:llm.input_messages":+ "e7e0d957608131d91ff7486fff064b9455817f005b7b29d309a3bbc231742e1c",+ "openai-agents-2025-09-30.trace.json:90d94774e8e3724d:llm.output_messages":+ "0d1254e7d61fdf2126da1820a5be2c7146287fe69c8b10b686dc1c2d5b9f333d",+ "openai-agents-2025-09-30.trace.json:98870087af69bf06:llm.input_messages":+ "feb45e1214f1e634d58a5888f58438d2ebe3ebddad1e67a75d67e3539df77971",+ "openai-agents-2025-09-30.trace.json:98870087af69bf06:llm.output_messages":+ "f819f9144a675fd46f4e445ac29084941bee1f3404f99f291d5a8cd799b2c06c",+ "vertex-ai-2025-08-01.trace.json:125abcbf5c41f4df:llm.input_messages":+ "1ec974ca95d73babe518739e05965f4dafe17572a47cec6fe66d2653303d0492",+ "vertex-ai-2025-08-01.trace.json:125abcbf5c41f4df:llm.output_messages":+ "acf3c7eeb02edd9d558bd3e9138cf022926c40d9a87c4e917a1ec98e9a467cc4",+};++type StructuralValue =+ | { type: "primitive"; value: unknown }+ | { type: "array"; length: number; entries: [string, StructuralValue][] }+ | { type: "object"; entries: [string, StructuralValue][] };++const encodeStructure = (value: unknown): StructuralValue => {+ if (Array.isArray(value)) {+ return {+ type: "array",+ length: value.length,+ entries: Object.keys(value).map((key) => [+ key,+ encodeStructure((value as unknown as Record<string, unknown>)[key]),+ ]),+ };+ }++ if (typeof value === "object" && value !== null) {+ const record = value as Record<string, unknown>;+ return {+ type: "object",+ entries: Object.keys(record).map((key) => [+ key,+ encodeStructure(record[key]),+ ]),+ };+ }++ return { type: "primitive", value };+};++const getStructuralDigest = (value: unknown): string =>+ createHash("sha256")+ .update(JSON.stringify(encodeStructure(value)))+ .digest("hex");++describe("OTel reconstruction compatibility against real observations", () => {+ it("matches the output recorded from the pre-hardening implementation", () => {+ const processor = new OtelIngestionProcessor({+ projectId: "test-project",+ publicKey: "pk-test",+ sdkName: "test",+ sdkVersion: "test",+ });+ const reconstruct = (+ processor as unknown as {+ convertKeyPathToNestedObject: (+ input: Record<string, unknown>,+ prefix: string,+ ) => unknown;+ }+ ).convertKeyPathToNestedObject.bind(processor);+ const actualDigests: Record<string, string> = {};+ const relevantFiles = new Set<string>();+ let relevantAttributeCount = 0;++ for (const traceFile of [...traceFiles].sort()) {+ const traceContent = readFileSync(+ path.join(tracesDir, traceFile),+ "utf-8",+ );+ const observations = JSON.parse(traceContent).observations as Array<{+ id: string;+ metadata?: string | { attributes?: Record<string, unknown> };+ }>;++ for (const observation of observations) {+ if (!observation.metadata) continue;++ const metadata =+ typeof observation.metadata === "string"+ ? (JSON.parse(observation.metadata) as {+ attributes?: Record<string, unknown>;+ })+ : observation.metadata;+ const attributes = metadata.attributes;+ if (!attributes) continue;++ for (const prefix of otelReconstructionPrefixes) {+ const attributeKeys = Object.keys(attributes).filter((key) =>+ key.startsWith(prefix),+ );+ if (attributeKeys.length === 0) continue;++ relevantFiles.add(traceFile);+ relevantAttributeCount += attributeKeys.length;+ const prefixedAttributes = Object.fromEntries(+ attributeKeys.map((key) => [key, attributes[key]]),+ );+ const caseId = `${traceFile}:${observation.id}:${prefix}`;+ actualDigests[caseId] = getStructuralDigest(+ reconstruct(prefixedAttributes, prefix),+ );+ }+ }+ }++ expect([...relevantFiles]).toEqual([+ "agno-2025-06-11.trace.json",+ "beeai-2025-08-01.trace.json",+ "crewai-2025-07-11.trace.json",+ "google-adk-2025-08-28.trace.json",+ "google-gemini-2025-08-01.trace.json",+ "koog-2025-08-26.trace.json",+ "openai-agents-2025-09-30.trace.json",+ "vertex-ai-2025-08-01.trace.json",+ ]);+ expect(relevantAttributeCount).toBe(232);+ expect(actualDigests).toEqual(expectedOtelReconstructionDigests);+ });+});+// use this to update the expected mapping result when changing/fixing the mapping logicconst updateExpectedFilesOnFailure = false;