Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix: harden OTel attribute path reconstruction (#16681)

Valery Meleshkin02f1d8ecf67b
diff --git a/packages/shared/src/server/otel/OtelIngestionProcessor.metadataDropped.test.ts b/packages/shared/src/server/otel/OtelIngestionProcessor.metadataDropped.test.ts
index 699f26ea41..0ff5ac3b92 100644
--- a/packages/shared/src/server/otel/OtelIngestionProcessor.metadataDropped.test.ts
+++ b/packages/shared/src/server/otel/OtelIngestionProcessor.metadataDropped.test.ts
@@ -661,25 +661,44 @@ describe("OTel reconstructed array drop telemetry", () => {
([stat]) => stat === ARRAY_ATTRIBUTE_DROPPED_METRIC,
);
- it("reports an out-of-range array attribute", async () => {
+ it("reports validation drops with their own reasons", async () => {
const processor = createProcessor();
const batch = buildBatch([
{
key: "llm.input_messages.10001.content",
value: { stringValue: "dropped" },
},
+ {
+ key: `llm.input_messages.${Array.from(
+ { length: 65 },
+ () => "nested",
+ ).join(".")}`,
+ value: { stringValue: "dropped" },
+ },
]);
await processor.processToIngestionEvents(batch);
- expect(arrayDropCalls()).toContainEqual([
- ARRAY_ATTRIBUTE_DROPPED_METRIC,
- 1,
- {
- reason: "reconstruction_budget_exceeded",
- prefix: "llm.input_messages",
- },
- ]);
+ expect(arrayDropCalls()).toEqual(
+ expect.arrayContaining([
+ [
+ ARRAY_ATTRIBUTE_DROPPED_METRIC,
+ 1,
+ {
+ reason: "reconstruction_array_index_exceeded",
+ prefix: "llm.input_messages",
+ },
+ ],
+ [
+ ARRAY_ATTRIBUTE_DROPPED_METRIC,
+ 1,
+ {
+ reason: "reconstruction_path_depth_exceeded",
+ prefix: "llm.input_messages",
+ },
+ ],
+ ]),
+ );
});
it("caps warnings without logging rejected attribute keys or values", async () => {
@@ -693,7 +712,7 @@ describe("OTel reconstructed array drop telemetry", () => {
(_, index) =>
buildBatch([
{
- key: `llm.input_messages.${10_001 + index}.secret-content`,
+ key: "llm.input_messages.5000.messages.5000.secret-content",
value: { stringValue: `customer-secret-${index}` },
},
])[0],
@@ -723,4 +742,46 @@ describe("OTel reconstructed array drop telemetry", () => {
warnSpy.mockRestore();
}
});
+
+ it("does not let validation drops consume the array-budget warning cap", async () => {
+ const warnSpy = vi
+ .spyOn(serverBarrel.logger, "warn")
+ .mockImplementation(() => serverBarrel.logger);
+
+ try {
+ const deepPath = Array.from({ length: 65 }, () => "nested").join(".");
+ const processor = createProcessor();
+ const validationBatches = Array.from(
+ { length: 12 },
+ () =>
+ buildBatch([
+ {
+ key: `llm.input_messages.${deepPath}`,
+ value: { stringValue: "dropped" },
+ },
+ ])[0],
+ );
+
+ await processor.processToIngestionEvents(validationBatches);
+ warnSpy.mockClear();
+
+ await processor.processToIngestionEvents(
+ buildBatch([
+ {
+ key: "llm.input_messages.5000.messages.5000.content",
+ value: { stringValue: "dropped" },
+ },
+ ]),
+ );
+
+ expect(warnSpy).toHaveBeenCalledWith("OTEL array attribute dropped", {
+ projectId: PROJECT_ID,
+ prefix: "llm.input_messages",
+ reason: "reconstruction_budget_exceeded",
+ droppedAttributeCount: 1,
+ });
+ } finally {
+ warnSpy.mockRestore();
+ }
+ });
});
diff --git a/packages/shared/src/server/otel/OtelIngestionProcessor.ts b/packages/shared/src/server/otel/OtelIngestionProcessor.ts
index f5e1e6ea0c..6067d2af6f 100644
--- a/packages/shared/src/server/otel/OtelIngestionProcessor.ts
+++ b/packages/shared/src/server/otel/OtelIngestionProcessor.ts
@@ -99,6 +99,23 @@ interface MetadataDropContext {
dropScope: object;
}
+const DANGEROUS_OTEL_PATH_SEGMENTS = new Set([
+ "__proto__",
+ "constructor",
+ "prototype",
+]);
+const CANONICAL_ARRAY_INDEX = /^(0|[1-9]\d*)$/;
+type ReconstructedAttributeDropReason =
+ | "reconstruction_array_index_exceeded"
+ | "reconstruction_budget_exceeded"
+ | "reconstruction_path_depth_exceeded";
+
+const getOtelArrayIndex = (segment: string): number | undefined => {
+ if (!CANONICAL_ARRAY_INDEX.test(segment)) return undefined;
+ const index = Number(segment);
+ return Number.isSafeInteger(index) ? index : undefined;
+};
+
/**
* Tracks the total number of array slots that dotted attribute paths would
* materialize. A path is reserved transactionally: when it would exceed the
@@ -121,12 +138,13 @@ class ArraySlotBudget {
for (let index = 0; index < pathParts.length; index++) {
const segment = pathParts[index];
- if (!/^\d+$/.test(segment)) continue;
+ const arrayIndex = getOtelArrayIndex(segment);
+ if (arrayIndex === undefined) continue;
const arrayPath = pathParts.slice(0, index).join(".");
const previousLength =
pendingLengths.get(arrayPath) ?? this.lengthsByPath.get(arrayPath) ?? 0;
- const requiredLength = Number(segment) + 1;
+ const requiredLength = arrayIndex + 1;
if (requiredLength > previousLength) {
additionalSlots += requiredLength - previousLength;
pendingLengths.set(arrayPath, requiredLength);
@@ -244,9 +262,10 @@ export class OtelIngestionProcessor {
private static readonly OTEL_ARRAY_ATTRIBUTE_DROPPED_METRIC =
"langfuse.ingestion.otel.array_attribute_dropped";
- // Flattened OTel message attributes name array slots directly, so cap the
- // total logical array length before reconstructing the value.
+ // Flattened OTel message attributes can create deeply nested objects and
+ // sparse arrays, so bound both forms of structural expansion.
private static readonly MAX_OTEL_RECONSTRUCTED_ARRAY_SLOTS = 10_001;
+ private static readonly MAX_OTEL_RECONSTRUCTED_PATH_DEPTH = 64;
private static readonly METADATA_DROP_WARN_CAP = 10;
private static readonly ARRAY_ATTRIBUTE_DROP_WARN_CAP = 10;
@@ -254,7 +273,10 @@ export class OtelIngestionProcessor {
private seenTraces: Set<string> = new Set();
private reportedMetadataDrops = new WeakMap<object, Set<string>>();
private metadataDropWarnCount = 0;
- private arrayAttributeDropWarnCount = 0;
+ private arrayAttributeDropWarnCounts = new Map<
+ ReconstructedAttributeDropReason,
+ number
+ >();
private isInitialized = false;
private traceEventCounts = {
shallow: 0,
@@ -1530,9 +1552,9 @@ export class OtelIngestionProcessor {
*
* TraceLoop and OpenInference encode structured messages in attribute names;
* for example, `gen_ai.prompt.0.content` becomes `[{ content: value }]`.
- * Numeric path segments therefore become JavaScript array indices. The total
- * logical slots across all nested arrays are bounded, since dimensions can
- * otherwise multiply into a huge sparse structure during serialization.
+ * Paths retain the legacy insertion-order conflict policy. Generated objects
+ * have no prototype, arrays only accept bounded numeric indices, and the
+ * writer never descends into an object supplied as an attribute value.
*/
private convertKeyPathToNestedObject(
input: Record<string, unknown>,
@@ -1542,103 +1564,184 @@ export class OtelIngestionProcessor {
return input[prefix];
}
- const keys: string[] = [];
- const overBudgetKeys: string[] = [];
+ type ReconstructedContainer = Record<string, unknown> | unknown[];
+ const droppedAttributeCounts = new Map<
+ ReconstructedAttributeDropReason,
+ number
+ >();
+ const recordDrop = (reason: ReconstructedAttributeDropReason): void => {
+ droppedAttributeCounts.set(
+ reason,
+ (droppedAttributeCounts.get(reason) ?? 0) + 1,
+ );
+ };
+ const parsePath = (
+ key: string,
+ recordLimitDrop: boolean,
+ ): string[] | undefined => {
+ const segments = key.split(".");
+ if (
+ segments.length >
+ OtelIngestionProcessor.MAX_OTEL_RECONSTRUCTED_PATH_DEPTH
+ ) {
+ if (recordLimitDrop) recordDrop("reconstruction_path_depth_exceeded");
+ return undefined;
+ }
+
+ for (const segment of segments) {
+ if (segment.length === 0 || DANGEROUS_OTEL_PATH_SEGMENTS.has(segment)) {
+ return undefined;
+ }
+
+ if (CANONICAL_ARRAY_INDEX.test(segment)) {
+ const index = getOtelArrayIndex(segment);
+ if (
+ index === undefined ||
+ index >= OtelIngestionProcessor.MAX_OTEL_RECONSTRUCTED_ARRAY_SLOTS
+ ) {
+ if (recordLimitDrop) {
+ recordDrop("reconstruction_array_index_exceeded");
+ }
+ return undefined;
+ }
+ }
+ }
+ return segments;
+ };
+
+ const prefixWithDot = `${prefix}.`;
+ const createdContainers = new WeakSet<object>();
const arraySlotBudget = new ArraySlotBudget(
OtelIngestionProcessor.MAX_OTEL_RECONSTRUCTED_ARRAY_SLOTS,
);
- for (const inputKey of Object.keys(input)) {
- const key = inputKey.replace(`${prefix}.`, "");
- if (!arraySlotBudget.tryReserve(key.split("."))) {
- overBudgetKeys.push(key);
- continue;
+ const rootArraySlotBudget = new ArraySlotBudget(
+ OtelIngestionProcessor.MAX_OTEL_RECONSTRUCTED_ARRAY_SLOTS,
+ );
+
+ const inputKeys = Object.keys(input);
+ const useArray = inputKeys.some((inputKey) => {
+ if (!inputKey.startsWith(prefixWithDot)) return false;
+ const path = parsePath(inputKey.slice(prefixWithDot.length), false);
+ if (path === undefined || !rootArraySlotBudget.tryReserve(path)) {
+ return false;
}
+ return path.length > 1 && getOtelArrayIndex(path[0]) !== undefined;
+ });
+ const result: ReconstructedContainer = useArray ? [] : Object.create(null);
+ createdContainers.add(result);
+
+ const isWritableKey = (
+ container: ReconstructedContainer,
+ key: string,
+ ): boolean => {
+ if (!Array.isArray(container)) return true;
+ const index = getOtelArrayIndex(key);
+ return (
+ index !== undefined &&
+ index < OtelIngestionProcessor.MAX_OTEL_RECONSTRUCTED_ARRAY_SLOTS
+ );
+ };
- keys.push(key);
- }
- this.recordArrayAttributesDropped(prefix, overBudgetKeys);
- const useArray = keys.some((key) => key.match(/^\d+\./));
+ const defineOwn = (
+ container: ReconstructedContainer,
+ key: string,
+ value: unknown,
+ ): void => {
+ Object.defineProperty(container, key, {
+ configurable: true,
+ enumerable: true,
+ value,
+ writable: true,
+ });
+ };
- // Blocklist to prevent prototype pollution via crafted OTel attribute keys
- const DANGEROUS_KEYS = new Set(["__proto__", "constructor", "prototype"]);
+ const applyPath = (
+ root: ReconstructedContainer,
+ path: readonly string[],
+ value: unknown,
+ commit: boolean,
+ ): boolean => {
+ let current = root;
+ for (let index = 0; index < path.length - 1; index++) {
+ const key = path[index];
+ if (!isWritableKey(current, key)) return false;
- // Helper function to set a value at a nested path
- const setNestedValue = (obj: any, path: string[], value: unknown): void => {
- let current = obj;
- for (let i = 0; i < path.length - 1; i++) {
- const key = path[i];
- if (DANGEROUS_KEYS.has(key)) return;
if (!Object.hasOwn(current, key)) {
- // Check if next key is a number to decide if we need an array or object
- current[key] = /^\d+$/.test(path[i + 1]) ? [] : Object.create(null);
+ if (!commit) return true;
+ const child: ReconstructedContainer =
+ getOtelArrayIndex(path[index + 1]) !== undefined
+ ? []
+ : Object.create(null);
+ createdContainers.add(child);
+ defineOwn(current, key, child);
+ current = child;
+ continue;
}
- const next = current[key];
- if (typeof next !== "object" || next === null) {
- // Preserve an earlier leaf value when a later attribute conflicts.
- return;
+
+ const next = Reflect.get(current, key);
+ if (
+ typeof next !== "object" ||
+ next === null ||
+ !createdContainers.has(next)
+ ) {
+ return false;
}
- current = next;
+ current = next as ReconstructedContainer;
}
const finalKey = path[path.length - 1];
- if (!DANGEROUS_KEYS.has(finalKey)) {
- current[finalKey] = value;
+ if (!isWritableKey(current, finalKey)) return false;
+ if (Object.hasOwn(current, finalKey)) {
+ const existing = Reflect.get(current, finalKey);
+ if (
+ typeof existing === "object" &&
+ existing !== null &&
+ createdContainers.has(existing)
+ ) {
+ return false;
+ }
}
+ if (commit) defineOwn(current, finalKey, value);
+ return true;
};
- if (useArray) {
- const result: any[] = [];
- for (const key of keys) {
- const pathParts = key.split(".");
- const indexSegment = pathParts[0];
- const index = Number(indexSegment);
- if (!result[index]) {
- result[index] = Object.create(null);
- }
- if (pathParts.length === 2) {
- // Simple case: 0.content -> result[0].content
- result[index][pathParts[1]] = input[`${prefix}.${key}`];
- } else {
- // Nested case: 0.message.content -> result[0].message.content
- setNestedValue(
- result[index],
- pathParts.slice(1),
- input[`${prefix}.${key}`],
- );
- }
+ for (const inputKey of inputKeys) {
+ if (!inputKey.startsWith(prefixWithDot)) continue;
+ const path = parsePath(inputKey.slice(prefixWithDot.length), true);
+ if (!path) continue;
+ const value = input[inputKey];
+ if (!applyPath(result, path, value, false)) continue;
+ if (!arraySlotBudget.tryReserve(path)) {
+ recordDrop("reconstruction_budget_exceeded");
+ continue;
}
- return result;
+ applyPath(result, path, value, true);
}
-
- const result: Record<string, unknown> = Object.create(null);
- for (const key of keys) {
- const pathParts = key.split(".");
- if (pathParts.length === 1) {
- result[key] = input[`${prefix}.${key}`];
- } else {
- setNestedValue(result, pathParts, input[`${prefix}.${key}`]);
- }
+ for (const [reason, droppedAttributeCount] of droppedAttributeCounts) {
+ this.recordArrayAttributesDropped(prefix, reason, droppedAttributeCount);
}
return result;
}
- private recordArrayAttributesDropped(prefix: string, keys: string[]): void {
- if (keys.length === 0) return;
+ private recordArrayAttributesDropped(
+ prefix: string,
+ reason: ReconstructedAttributeDropReason,
+ droppedAttributeCount: number,
+ ): void {
+ if (droppedAttributeCount === 0) return;
recordIncrement(
OtelIngestionProcessor.OTEL_ARRAY_ATTRIBUTE_DROPPED_METRIC,
- keys.length,
- { reason: "reconstruction_budget_exceeded", prefix },
+ droppedAttributeCount,
+ { reason, prefix },
);
- if (
- this.arrayAttributeDropWarnCount <
- OtelIngestionProcessor.ARRAY_ATTRIBUTE_DROP_WARN_CAP
- ) {
- this.arrayAttributeDropWarnCount += 1;
+ const warningCount = this.arrayAttributeDropWarnCounts.get(reason) ?? 0;
+ if (warningCount < OtelIngestionProcessor.ARRAY_ATTRIBUTE_DROP_WARN_CAP) {
+ this.arrayAttributeDropWarnCounts.set(reason, warningCount + 1);
logger.warn("OTEL array attribute dropped", {
projectId: this.projectId,
prefix,
- reason: "reconstruction_budget_exceeded",
- droppedAttributeCount: keys.length,
+ reason,
+ droppedAttributeCount,
});
}
}
diff --git a/web/src/__tests__/server/api/otel/otelMapping.servertest.ts b/web/src/__tests__/server/api/otel/otelMapping.servertest.ts
index 2e46d6bdc8..83c1dad292 100644
--- a/web/src/__tests__/server/api/otel/otelMapping.servertest.ts
+++ b/web/src/__tests__/server/api/otel/otelMapping.servertest.ts
@@ -9693,6 +9693,61 @@ describe("OTel Resource Span Mapping", () => {
}
};
+ it.each([
+ [
+ "flat object",
+ [
+ ["role", "user"],
+ ["content", "hello"],
+ ],
+ { role: "user", content: "hello" },
+ ],
+ [
+ "root array with shared prefixes",
+ [
+ ["1.content", "world"],
+ ["0.role", "user"],
+ ["0.content", "hello"],
+ ["1.role", "assistant"],
+ ],
+ [
+ { role: "user", content: "hello" },
+ { role: "assistant", content: "world" },
+ ],
+ ],
+ [
+ "nested objects and scalar arrays",
+ [
+ ["request.model", "test-model"],
+ ["request.parts.1", "second"],
+ ["request.parts.0", "first"],
+ ["request.options.temperature", "0.5"],
+ ],
+ {
+ request: {
+ model: "test-model",
+ parts: ["first", "second"],
+ options: { temperature: "0.5" },
+ },
+ },
+ ],
+ ] as const)(
+ "should preserve legacy reconstruction for %s",
+ (_name, attributes, expected) => {
+ const events = createTestOtelProcessor({ publicKey }).processToEvent([
+ createResourceSpan(
+ attributes.map(([path, stringValue]) => ({
+ key: `gen_ai.prompt.${path}`,
+ value: { stringValue },
+ })),
+ ),
+ ]);
+
+ expect(events).toHaveLength(1);
+ expect(events[0].input).toEqual(expected);
+ },
+ );
+
it("should not traverse inherited properties in nested gen_ai.prompt attributes", async () => {
const { events, observedCall, originalCall } =
await convertAndObserveHasOwnPropertyCall([
@@ -9734,10 +9789,10 @@ describe("OTel Resource Span Mapping", () => {
const input = events.find((event) => event.type === "span-create")?.body
.input as { a?: unknown[] } | undefined;
expect(input?.a?.[0]).toBe("seed");
- expect(Object.hasOwn(input?.a ?? [], "hasOwnProperty")).toBe(true);
- expect(Reflect.get(input?.a ?? [], "hasOwnProperty")).toMatchObject({
- call: "pwned",
- });
+ expect(Object.hasOwn(input?.a ?? [], "hasOwnProperty")).toBe(false);
+ expect(Reflect.get(input?.a ?? [], "hasOwnProperty")).toBe(
+ Object.prototype.hasOwnProperty,
+ );
});
it("should preserve a scalar leaf and ignore a conflicting nested attribute", () => {
@@ -9765,6 +9820,152 @@ describe("OTel Resource Span Mapping", () => {
});
});
+ it("should preserve a reconstructed container and ignore a conflicting leaf", () => {
+ const events = createTestOtelProcessor({ publicKey }).processToEvent([
+ createResourceSpan([
+ {
+ key: "gen_ai.prompt.0.content",
+ value: { stringValue: "nested" },
+ },
+ {
+ key: "gen_ai.prompt.0",
+ value: { stringValue: "leaf" },
+ },
+ ]),
+ ]);
+
+ expect(events).toHaveLength(1);
+ expect(events[0].input).toEqual([{ content: "nested" }]);
+ });
+
+ it.each([
+ {
+ name: "array first",
+ attributes: [
+ {
+ key: "gen_ai.prompt.a.0",
+ value: { stringValue: "first" },
+ },
+ {
+ key: "gen_ai.prompt.a.length",
+ value: { stringValue: "pwned" },
+ },
+ ],
+ expected: ["first"],
+ },
+ {
+ name: "object first",
+ attributes: [
+ {
+ key: "gen_ai.prompt.a.length",
+ value: { stringValue: "pwned" },
+ },
+ {
+ key: "gen_ai.prompt.a.0",
+ value: { stringValue: "first" },
+ },
+ ],
+ expected: { 0: "first", length: "pwned" },
+ },
+ ])(
+ "should safely preserve the first reconstructed container kind ($name)",
+ ({ attributes, expected }) => {
+ const events = createTestOtelProcessor({ publicKey }).processToEvent([
+ createResourceSpan([
+ ...attributes,
+ {
+ key: "gen_ai.prompt.safe",
+ value: { stringValue: "still-here" },
+ },
+ ]),
+ ]);
+
+ expect(events).toHaveLength(1);
+ expect(events[0].input).toMatchObject({
+ a: expected,
+ safe: "still-here",
+ });
+ },
+ );
+
+ it("should not interpret noncanonical numeric syntax as an array index", () => {
+ const events = createTestOtelProcessor({ publicKey }).processToEvent([
+ createResourceSpan([
+ {
+ key: "gen_ai.prompt.0.content",
+ value: { stringValue: "safe" },
+ },
+ {
+ key: "gen_ai.prompt.1e6.content",
+ value: { stringValue: "pwned" },
+ },
+ ]),
+ ]);
+
+ expect(events).toHaveLength(1);
+ const input = events[0].input as Array<{ content: string }>;
+ expect(input).toHaveLength(1);
+ expect(input[0]).toEqual({ content: "safe" });
+ });
+
+ it("should drop paths that exceed the reconstruction depth limit", () => {
+ const deepPath = Array.from({ length: 65 }, () => "nested").join(".");
+ const events = createTestOtelProcessor({ publicKey }).processToEvent([
+ createResourceSpan([
+ {
+ key: `gen_ai.prompt.${deepPath}`,
+ value: { stringValue: "pwned" },
+ },
+ {
+ key: "gen_ai.prompt.safe",
+ value: { stringValue: "still-here" },
+ },
+ ]),
+ ]);
+
+ expect(events).toHaveLength(1);
+ expect(events[0].input).toEqual({ safe: "still-here" });
+ });
+
+ it("should not let a rejected path consume the array-slot budget", () => {
+ const events = createTestOtelProcessor({ publicKey }).processToEvent([
+ createResourceSpan([
+ {
+ key: "gen_ai.prompt.0.message.__proto__.9999.content",
+ value: { stringValue: "pwned" },
+ },
+ {
+ key: "gen_ai.prompt.1.content",
+ value: { stringValue: "still-here" },
+ },
+ ]),
+ ]);
+
+ expect(events).toHaveLength(1);
+ const input = events[0].input as Array<{ content?: string }>;
+ expect(input).toHaveLength(2);
+ expect(input[0]).toBeUndefined();
+ expect(input[1]).toEqual({ content: "still-here" });
+ });
+
+ it("should not let an over-budget array path change the root container type", () => {
+ const events = createTestOtelProcessor({ publicKey }).processToEvent([
+ createResourceSpan([
+ {
+ key: "gen_ai.prompt.5000.notes.5000.content",
+ value: { stringValue: "dropped" },
+ },
+ {
+ key: "gen_ai.prompt.name",
+ value: { stringValue: "still-here" },
+ },
+ ]),
+ ]);
+
+ expect(events).toHaveLength(1);
+ expect(events[0].input).toEqual({ name: "still-here" });
+ });
+
it("should not pollute Object.prototype via __proto__ in gen_ai.prompt attributes", async () => {
const traceId = "abcdef1234567890abcdef1234567890";
const spanId = "abcdef1234567890";
diff --git a/worker/src/__tests__/chatml/integration.test.ts b/worker/src/__tests__/chatml/integration.test.ts
index 3cace624b4..28df9cb23e 100644
--- a/worker/src/__tests__/chatml/integration.test.ts
+++ b/worker/src/__tests__/chatml/integration.test.ts
@@ -1,4 +1,5 @@
import { describe, it, expect, vi } from "vitest";
+import { createHash } from "crypto";
import { readFileSync, existsSync, writeFileSync, readdirSync } from "fs";
import path from "path";
@@ -12,6 +13,7 @@ import {
} from "@langfuse/shared/src/utils/chatml";
import { deepParseJson } from "@langfuse/shared";
+import { OtelIngestionProcessor } from "@langfuse/shared/src/server";
describe("ChatML Integration", () => {
it("should handle OpenAI multimodal format", () => {
@@ -522,6 +524,177 @@ const tracesDir = path.resolve(__dirname, "framework-traces");
const traceFiles = readdirSync(tracesDir).filter((f) =>
f.endsWith(".trace.json"),
);
+
+const otelReconstructionPrefixes = [
+ "gen_ai.prompt",
+ "gen_ai.completion",
+ "llm.input_messages",
+ "llm.output_messages",
+] as const;
+
+const expectedOtelReconstructionDigests = {
+ "agno-2025-06-11.trace.json:b38a82eaa62b551e:llm.input_messages":
+ "79dae2697161990380f3cc2409030d1e958b30706babd1a73613b10f5b807348",
+ "agno-2025-06-11.trace.json:ca136de468e156c9:llm.input_messages":
+ "b754777938eee8274ad8891da07446072ab8fe7898d99fd9430abb26aa278ec5",
+ "beeai-2025-08-01.trace.json:52df0f7f9dad6b23:llm.input_messages":
+ "9e09b39eafc8bf335608817d86dca4c7d6debbe8ff464bf68407388fc2be70c9",
+ "beeai-2025-08-01.trace.json:f50f7bbf13112df9:llm.output_messages":
+ "d874643405382404cadb4e70a9046f50357bdfb8bf5a4e44cc4d50a0216c101c",
+ "beeai-2025-08-01.trace.json:4c5c5a7936ca24bc:llm.input_messages":
+ "a736bdc132475ed501b72ecc14087aaa73bd01d54ed0b86f660f7059e93b6f4b",
+ "beeai-2025-08-01.trace.json:1f956cbea34725bd:llm.input_messages":
+ "f50c6ed16b46a5fd0c38aa5d1649759e6bd26bf3b279c058340ea4f2d1b5f17c",
+ "crewai-2025-07-11.trace.json:231c43964b7e7e63:llm.input_messages":
+ "090af8a41c384a74784270cecd1743fbe64b61af1344a7ad327eca06d5027bda",
+ "crewai-2025-07-11.trace.json:231c43964b7e7e63:llm.output_messages":
+ "5d2c3c0a6f1aafd174f5015efbda809a4bcddaf2cfd19c1b26ead101542f6e9d",
+ "google-adk-2025-08-28.trace.json:bded677884f031ce:llm.input_messages":
+ "077595470a234fdd1638cd54626bf6d9de33413b065f885d744d5384854c948d",
+ "google-adk-2025-08-28.trace.json:bded677884f031ce:llm.output_messages":
+ "f7c165eb9d7053e004498ff13be315d4c8dd1be1994854a14a3d1f2e2908b934",
+ "google-adk-2025-08-28.trace.json:f6e4c65297e40c96:llm.input_messages":
+ "a2c27fe417936852bb3ec99f8a5e333b56caf3658738bf133e8d1ec329f95e34",
+ "google-adk-2025-08-28.trace.json:f6e4c65297e40c96:llm.output_messages":
+ "3ec3d61cd82e7d8193ad0b9c4a4663fd19613afbe3014ec44f33399275400f71",
+ "google-gemini-2025-08-01.trace.json:b7a63ca7e1d083bc:llm.input_messages":
+ "02fb5023a8ab3234b69b0af60ef2fffe78e0443d452323ea733b10ba8d2e6757",
+ "google-gemini-2025-08-01.trace.json:b7a63ca7e1d083bc:llm.output_messages":
+ "9d51690e135533dbb44127193472d729d5afd48b2d69a16b1e79ccb68860470b",
+ "koog-2025-08-26.trace.json:e44e72cf2d221781:gen_ai.prompt":
+ "12541a8d701c9962ebcc381fbf54539d443a093a8c8cf3e22acd63f0ea146910",
+ "koog-2025-08-26.trace.json:e44e72cf2d221781:gen_ai.completion":
+ "41b2ea825990164dcecd9fe607a45ff5601fbd7d3146ce79b32dc69e9b2f1c6d",
+ "koog-2025-08-26.trace.json:042f9350c3729147:gen_ai.prompt":
+ "05dc418f9c1829e3b0c5d8d91086313dac3e6d7cc6d37655fe8d76ae082fab41",
+ "koog-2025-08-26.trace.json:042f9350c3729147:gen_ai.completion":
+ "d913abf5a058217e9436318ed64e54570a477063514aa499ef0941024ffdc6a5",
+ "openai-agents-2025-09-30.trace.json:90d94774e8e3724d:llm.input_messages":
+ "e7e0d957608131d91ff7486fff064b9455817f005b7b29d309a3bbc231742e1c",
+ "openai-agents-2025-09-30.trace.json:90d94774e8e3724d:llm.output_messages":
+ "0d1254e7d61fdf2126da1820a5be2c7146287fe69c8b10b686dc1c2d5b9f333d",
+ "openai-agents-2025-09-30.trace.json:98870087af69bf06:llm.input_messages":
+ "feb45e1214f1e634d58a5888f58438d2ebe3ebddad1e67a75d67e3539df77971",
+ "openai-agents-2025-09-30.trace.json:98870087af69bf06:llm.output_messages":
+ "f819f9144a675fd46f4e445ac29084941bee1f3404f99f291d5a8cd799b2c06c",
+ "vertex-ai-2025-08-01.trace.json:125abcbf5c41f4df:llm.input_messages":
+ "1ec974ca95d73babe518739e05965f4dafe17572a47cec6fe66d2653303d0492",
+ "vertex-ai-2025-08-01.trace.json:125abcbf5c41f4df:llm.output_messages":
+ "acf3c7eeb02edd9d558bd3e9138cf022926c40d9a87c4e917a1ec98e9a467cc4",
+};
+
+type StructuralValue =
+ | { type: "primitive"; value: unknown }
+ | { type: "array"; length: number; entries: [string, StructuralValue][] }
+ | { type: "object"; entries: [string, StructuralValue][] };
+
+const encodeStructure = (value: unknown): StructuralValue => {
+ if (Array.isArray(value)) {
+ return {
+ type: "array",
+ length: value.length,
+ entries: Object.keys(value).map((key) => [
+ key,
+ encodeStructure((value as unknown as Record<string, unknown>)[key]),
+ ]),
+ };
+ }
+
+ if (typeof value === "object" && value !== null) {
+ const record = value as Record<string, unknown>;
+ return {
+ type: "object",
+ entries: Object.keys(record).map((key) => [
+ key,
+ encodeStructure(record[key]),
+ ]),
+ };
+ }
+
+ return { type: "primitive", value };
+};
+
+const getStructuralDigest = (value: unknown): string =>
+ createHash("sha256")
+ .update(JSON.stringify(encodeStructure(value)))
+ .digest("hex");
+
+describe("OTel reconstruction compatibility against real observations", () => {
+ it("matches the output recorded from the pre-hardening implementation", () => {
+ const processor = new OtelIngestionProcessor({
+ projectId: "test-project",
+ publicKey: "pk-test",
+ sdkName: "test",
+ sdkVersion: "test",
+ });
+ const reconstruct = (
+ processor as unknown as {
+ convertKeyPathToNestedObject: (
+ input: Record<string, unknown>,
+ prefix: string,
+ ) => unknown;
+ }
+ ).convertKeyPathToNestedObject.bind(processor);
+ const actualDigests: Record<string, string> = {};
+ const relevantFiles = new Set<string>();
+ let relevantAttributeCount = 0;
+
+ for (const traceFile of [...traceFiles].sort()) {
+ const traceContent = readFileSync(
+ path.join(tracesDir, traceFile),
+ "utf-8",
+ );
+ const observations = JSON.parse(traceContent).observations as Array<{
+ id: string;
+ metadata?: string | { attributes?: Record<string, unknown> };
+ }>;
+
+ for (const observation of observations) {
+ if (!observation.metadata) continue;
+
+ const metadata =
+ typeof observation.metadata === "string"
+ ? (JSON.parse(observation.metadata) as {
+ attributes?: Record<string, unknown>;
+ })
+ : observation.metadata;
+ const attributes = metadata.attributes;
+ if (!attributes) continue;
+
+ for (const prefix of otelReconstructionPrefixes) {
+ const attributeKeys = Object.keys(attributes).filter((key) =>
+ key.startsWith(prefix),
+ );
+ if (attributeKeys.length === 0) continue;
+
+ relevantFiles.add(traceFile);
+ relevantAttributeCount += attributeKeys.length;
+ const prefixedAttributes = Object.fromEntries(
+ attributeKeys.map((key) => [key, attributes[key]]),
+ );
+ const caseId = `${traceFile}:${observation.id}:${prefix}`;
+ actualDigests[caseId] = getStructuralDigest(
+ reconstruct(prefixedAttributes, prefix),
+ );
+ }
+ }
+ }
+
+ expect([...relevantFiles]).toEqual([
+ "agno-2025-06-11.trace.json",
+ "beeai-2025-08-01.trace.json",
+ "crewai-2025-07-11.trace.json",
+ "google-adk-2025-08-28.trace.json",
+ "google-gemini-2025-08-01.trace.json",
+ "koog-2025-08-26.trace.json",
+ "openai-agents-2025-09-30.trace.json",
+ "vertex-ai-2025-08-01.trace.json",
+ ]);
+ expect(relevantAttributeCount).toBe(232);
+ expect(actualDigests).toEqual(expectedOtelReconstructionDigests);
+ });
+});
+
// use this to update the expected mapping result when changing/fixing the mapping logic
const updateExpectedFilesOnFailure = false;