Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

66.1 KB
Raw Blame
/* eslint-disable @repo/no-exotic-operators */
/**
 * Test suite to verify that OTEL events coming through the ingestion pipeline
 * correctly conform to the ObservationForEval schema.
 *
 * This is critical because:
 * 1. OTEL events are the primary ingestion path for observation-level evals
 * 2. If the schema validation fails, observation evals become ineffective
 * 3. We need to ensure all typical fields are correctly mapped
 *
 * Flow tested:
 * ResourceSpan -> processToEvent() -> createEventRecord() -> convertEventRecordToObservationForEval()
 */
import { describe, it, expect, beforeEach, vi } from "vitest";
import { OtelIngestionProcessor } from "@langfuse/shared/src/server";
import {
  convertEventRecordToObservationForEval,
  observationForEvalSchema,
  type ObservationForEval,
} from "@langfuse/shared";
import { prisma } from "@langfuse/shared/src/db";
import { IngestionService } from "../../services/IngestionService";
import * as clickhouseWriterExports from "../../services/ClickhouseWriter";

const createTestOtelProcessor = (projectId = "test-project") =>
  new OtelIngestionProcessor({
    projectId,
    publicKey: "",
    sdkName: "",
    sdkVersion: "",
  });

// Mock ClickhouseWriter to avoid actual database writes.
// vi.hoisted ensures this is declared before vi.mock's hoisted factory runs.
// Without it, the variable would be undefined when the factory executes.
const { mockAddToClickhouseWriter } = vi.hoisted(() => ({
  mockAddToClickhouseWriter: vi.fn(),
}));
vi.mock("../../services/ClickhouseWriter", async (importOriginal) => {
  const original = (await importOriginal()) as object;
  return {
    ...original,
    ClickhouseWriter: {
      getInstance: () => ({
        addToQueue: mockAddToClickhouseWriter,
      }),
    },
  };
});

// Mock ClickhouseClient to return empty results (no existing records)
const mockClickhouseClient = {
  query: async () => ({
    json: async () => [],
    query_id: "test-query-id",
    response_headers: { "x-clickhouse-summary": "[]" },
  }),
};

// Create IngestionService instance with mocked dependencies
// Note: redis is null as we don't need it for createEventRecord
// prisma is used for prompt/model lookups (will return null if not found)
const ingestionService = new IngestionService(
  null as any,
  prisma,
  clickhouseWriterExports.ClickhouseWriter.getInstance() as any,
  mockClickhouseClient as any,
);

/**
 * Helper to create protobuf-style timestamp from nanoseconds
 */
function createNanoTimestamp(nanoTime: bigint): {
  low: number;
  high: number;
  unsigned: boolean;
} {
  const low = Number(nanoTime & BigInt(0xffffffff));
  const high = Number(nanoTime >> BigInt(32));
  return { low, high, unsigned: true };
}

/**
 * Helper to create Buffer-style ID from hex string
 */
function createBufferId(hexString: string): { type: "Buffer"; data: number[] } {
  const buffer = Buffer.from(hexString, "hex");
  return { type: "Buffer", data: Array.from(buffer) };
}

/**
 * Process a ResourceSpan through the full pipeline and return the ObservationForEval
 */
async function processOtelSpanToObservationForEval(
  resourceSpan: any,
  projectId = "test-project",
): Promise<ObservationForEval[]> {
  const processor = createTestOtelProcessor(projectId);

  // Process to EventInput array using the real OtelIngestionProcessor
  const eventInputs = processor.processToEvent([resourceSpan]);

  // Convert each EventInput to EventRecordInsertType using the real IngestionService
  // Then convert to ObservationForEval
  const results: ObservationForEval[] = [];
  for (const eventInput of eventInputs) {
    const eventRecord = await ingestionService.createEventRecord(
      eventInput,
      "test/otel/test.json",
    );
    const observation = convertEventRecordToObservationForEval(eventRecord);
    results.push(observation);
  }

  return results;
}

describe("OTEL to ObservationForEval Schema Validation", () => {
  beforeEach(() => {
    vi.clearAllMocks();
  });

  describe("Langfuse SDK spans", () => {
    it("should convert a Langfuse SDK generation span to valid ObservationForEval", async () => {
      const langfuseOtelSpan = {
        resource: {
          attributes: [
            { key: "telemetry.sdk.language", value: { stringValue: "python" } },
            {
              key: "telemetry.sdk.name",
              value: { stringValue: "opentelemetry" },
            },
            { key: "telemetry.sdk.version", value: { stringValue: "1.32.0" } },
            {
              key: "langfuse.environment",
              value: { stringValue: "production" },
            },
            { key: "langfuse.release", value: { stringValue: "0.0.1" } },
            { key: "service.name", value: { stringValue: "test-service" } },
          ],
        },
        scopeSpans: [
          {
            scope: {
              name: "langfuse-sdk",
              version: "3.0.0",
              attributes: [
                { key: "public_key", value: { stringValue: "pk-lf-test" } },
              ],
            },
            spans: [
              {
                traceId: createBufferId("2cce18f7e8cd065a0b4e634eef728391"),
                spanId: createBufferId("57f0255417974100"),
                parentSpanId: createBufferId("dfe387fea7ef3b02"),
                name: "test-generation",
                kind: 1,
                startTimeUnixNano: createNanoTimestamp(
                  BigInt(1714488530686000000),
                ),
                endTimeUnixNano: createNanoTimestamp(
                  BigInt(1714488530687000000),
                ),
                attributes: [
                  {
                    key: "langfuse.observation.type",
                    value: { stringValue: "generation" },
                  },
                  {
                    key: "langfuse.observation.level",
                    value: { stringValue: "WARNING" },
                  },
                  {
                    key: "langfuse.observation.status_message",
                    value: { stringValue: "test status" },
                  },
                  {
                    key: "langfuse.observation.input",
                    value: {
                      stringValue: '[{"role": "user", "content": "hello"}]',
                    },
                  },
                  {
                    key: "langfuse.observation.output",
                    value: {
                      stringValue:
                        '{"role": "assistant", "content": "hi there"}',
                    },
                  },
                  {
                    key: "langfuse.observation.model.name",
                    value: { stringValue: "gpt-4" },
                  },
                  {
                    key: "langfuse.observation.prompt.name",
                    value: { stringValue: "my-prompt" },
                  },
                  {
                    key: "langfuse.observation.prompt.version",
                    value: { intValue: { low: 1, high: 0, unsigned: false } },
                  },
                  {
                    key: "langfuse.observation.usage_details",
                    value: {
                      stringValue: '{"input": 100, "output": 50}',
                    },
                  },
                  {
                    key: "langfuse.observation.cost_details",
                    value: {
                      stringValue: '{"input": 0.001, "output": 0.002}',
                    },
                  },
                  // Model parameters are extracted from gen_ai.request.* attributes
                  {
                    key: "gen_ai.request.temperature",
                    value: { doubleValue: 0.7 },
                  },
                  {
                    key: "langfuse.observation.metadata.custom_key",
                    value: { stringValue: '"custom_value"' },
                  },
                  {
                    key: "langfuse.trace.name",
                    value: { stringValue: "test-trace" },
                  },
                  { key: "user.id", value: { stringValue: "user-123" } },
                  { key: "session.id", value: { stringValue: "session-456" } },
                  {
                    key: "langfuse.version",
                    value: { stringValue: "v1.0.0" },
                  },
                  {
                    key: "langfuse.trace.tags",
                    value: {
                      arrayValue: {
                        values: [
                          { stringValue: "tag1" },
                          { stringValue: "tag2" },
                        ],
                      },
                    },
                  },
                ],
                status: {},
              },
            ],
          },
        ],
      };

      const observations = await processOtelSpanToObservationForEval(
        langfuseOtelSpan,
        "test-project",
      );

      expect(observations).toHaveLength(1);
      const obs = observations[0];

      // Validate schema conformance first
      const result = observationForEvalSchema.safeParse(obs);
      expect(result.success).toBe(true);

      // Validate required identifiers
      expect(obs.span_id).toBe("57f0255417974100");
      expect(obs.trace_id).toBe("2cce18f7e8cd065a0b4e634eef728391");
      expect(obs.project_id).toBe("test-project");
      expect(obs.parent_span_id).toBe("dfe387fea7ef3b02");

      // Validate core properties
      expect(obs.type).toBe("GENERATION");
      expect(obs.name).toBe("test-generation");
      expect(obs.environment).toBe("production");
      expect(obs.level).toBe("WARNING");
      expect(obs.status_message).toBe("test status");
      expect(obs.version).toBe("v1.0.0");

      // Validate trace-level properties
      expect(obs.trace_name).toBe("test-trace");
      expect(obs.user_id).toBe("user-123");
      expect(obs.session_id).toBe("session-456");
      expect(obs.tags).toEqual(["tag1", "tag2"]);
      expect(obs.release).toBe("0.0.1");

      // Validate model properties
      expect(obs.provided_model_name).toBe("gpt-4");
      // model_parameters comes as object from createEventRecord (schema accepts both string and object)
      expect(obs.model_parameters).toEqual({ temperature: 0.7 });

      // Validate prompt properties
      expect(obs.prompt_name).toBe("my-prompt");
      // prompt_version comes as number from createEventRecord (schema accepts both string and number)
      expect(obs.prompt_version).toBe(1);

      // Validate usage/cost
      expect(obs.provided_usage_details).toEqual({ input: 100, output: 50 });
      expect(obs.provided_cost_details).toEqual({
        input: 0.001,
        output: 0.002,
      });

      // Validate I/O
      expect(obs.input).toBe('[{"role": "user", "content": "hello"}]');
      expect(obs.output).toBe('{"role": "assistant", "content": "hi there"}');

      // Validate metadata is an object
      expect(typeof obs.metadata).toBe("object");
    });

    it("should convert a Langfuse SDK span with minimal attributes", async () => {
      const minimalOtelSpan = {
        resource: { attributes: [] },
        scopeSpans: [
          {
            scope: { name: "langfuse-sdk", version: "3.0.0" },
            spans: [
              {
                traceId: createBufferId("1234567890abcdef1234567890abcdef"),
                spanId: createBufferId("abcdef1234567890"),
                name: "minimal-span",
                kind: 1,
                startTimeUnixNano: createNanoTimestamp(
                  BigInt(1714488530686000000),
                ),
                endTimeUnixNano: createNanoTimestamp(
                  BigInt(1714488530687000000),
                ),
                attributes: [
                  {
                    key: "langfuse.observation.type",
                    value: { stringValue: "span" },
                  },
                ],
                status: {},
              },
            ],
          },
        ],
      };

      const observations = await processOtelSpanToObservationForEval(
        minimalOtelSpan,
        "test-project",
      );

      expect(observations).toHaveLength(1);
      const obs = observations[0];

      // Validate schema conformance
      const result = observationForEvalSchema.safeParse(obs);
      expect(result.success).toBe(true);

      // Required fields should be present
      expect(obs.span_id).toBe("abcdef1234567890");
      expect(obs.trace_id).toBe("1234567890abcdef1234567890abcdef");
      expect(obs.project_id).toBe("test-project");
      expect(obs.type).toBe("SPAN");
      expect(obs.name).toBe("minimal-span");
      expect(obs.level).toBe("DEFAULT");
      expect(obs.environment).toBe("default");

      // Nullable fields should be null or undefined (nullish)
      expect(obs.parent_span_id).toBeFalsy();
      expect(obs.user_id).toBeFalsy();
      expect(obs.session_id).toBeFalsy();
      expect(obs.provided_model_name).toBeFalsy();
      expect(obs.input).toBeFalsy();
      expect(obs.output).toBeFalsy();

      // Array/object fields should have defaults
      expect(obs.tags).toEqual([]);
      expect(obs.tool_calls).toEqual([]);
      expect(obs.tool_definitions).toEqual({});
    });
  });

  describe("Vendor SDK spans", () => {
    it("should convert an OpenLit span to valid ObservationForEval", async () => {
      const openLitSpan = {
        resource: {
          attributes: [
            { key: "telemetry.sdk.language", value: { stringValue: "python" } },
            { key: "telemetry.sdk.name", value: { stringValue: "openlit" } },
            { key: "telemetry.sdk.version", value: { stringValue: "1.27.0" } },
            { key: "service.name", value: { stringValue: "default" } },
            {
              key: "deployment.environment",
              value: { stringValue: "production" },
            },
          ],
        },
        scopeSpans: [
          {
            scope: { name: "openlit.otel.tracing" },
            spans: [
              {
                traceId: createBufferId("ea6737084c1c2984a54a3e3962d3595f"),
                spanId: createBufferId("b904bffb20be6d7e"),
                name: "openai.chat.completions",
                kind: 3,
                startTimeUnixNano: createNanoTimestamp(
                  BigInt(1738241187653000000),
                ),
                endTimeUnixNano: createNanoTimestamp(
                  BigInt(1738241188827000000),
                ),
                attributes: [
                  { key: "gen_ai.system", value: { stringValue: "openai" } },
                  {
                    key: "gen_ai.operation.name",
                    value: { stringValue: "chat" },
                  },
                  {
                    key: "gen_ai.request.model",
                    value: { stringValue: "gpt-3.5-turbo" },
                  },
                  {
                    key: "gen_ai.usage.input_tokens",
                    value: { intValue: { low: 14, high: 0, unsigned: false } },
                  },
                  {
                    key: "gen_ai.usage.output_tokens",
                    value: { intValue: { low: 96, high: 0, unsigned: false } },
                  },
                  {
                    key: "gen_ai.usage.total_tokens",
                    value: { intValue: { low: 110, high: 0, unsigned: false } },
                  },
                  {
                    key: "gen_ai.usage.cost",
                    value: { doubleValue: 0.000151 },
                  },
                ],
                events: [
                  {
                    name: "gen_ai.content.prompt",
                    attributes: [
                      {
                        key: "gen_ai.prompt",
                        value: {
                          stringValue: "user: What is LLM Observability?",
                        },
                      },
                    ],
                  },
                  {
                    name: "gen_ai.content.completion",
                    attributes: [
                      {
                        key: "gen_ai.completion",
                        value: { stringValue: "LLM Observability is..." },
                      },
                    ],
                  },
                ],
                status: { code: 1 },
              },
            ],
          },
        ],
      };

      const observations =
        await processOtelSpanToObservationForEval(openLitSpan);

      expect(observations).toHaveLength(1);
      const obs = observations[0];

      // Validate schema conformance
      const result = observationForEvalSchema.safeParse(obs);
      expect(result.success).toBe(true);

      // Validate key fields
      expect(obs.type).toBe("GENERATION");
      expect(obs.environment).toBe("production");
      expect(obs.provided_model_name).toBe("gpt-3.5-turbo");

      // Usage should be extracted
      expect(obs.provided_usage_details).toMatchObject({
        input: 14,
        output: 96,
        total: 110,
      });
    });

    it("should convert a TraceLoop span to valid ObservationForEval", async () => {
      const traceLoopSpan = {
        resource: {
          attributes: [
            { key: "service.name", value: { stringValue: "traceloop-app" } },
          ],
        },
        scopeSpans: [
          {
            scope: {
              name: "opentelemetry.instrumentation.openai.v1",
              version: "0.33.9",
            },
            spans: [
              {
                traceId: createBufferId("e4ef45025c9b40924bff175e2b125b5b"),
                spanId: createBufferId("aabf16e416ae4952"),
                name: "openai.chat",
                kind: 3,
                startTimeUnixNano: createNanoTimestamp(
                  BigInt(1738241287865000000),
                ),
                endTimeUnixNano: createNanoTimestamp(
                  BigInt(1738241289310000000),
                ),
                attributes: [
                  { key: "llm.request.type", value: { stringValue: "chat" } },
                  { key: "gen_ai.system", value: { stringValue: "OpenAI" } },
                  {
                    key: "gen_ai.request.model",
                    value: { stringValue: "gpt-3.5-turbo" },
                  },
                  {
                    key: "gen_ai.response.model",
                    value: { stringValue: "gpt-3.5-turbo-0125" },
                  },
                  {
                    key: "llm.usage.total_tokens",
                    value: { intValue: { low: 187, high: 0, unsigned: false } },
                  },
                  {
                    key: "gen_ai.usage.completion_tokens",
                    value: { intValue: { low: 173, high: 0, unsigned: false } },
                  },
                  {
                    key: "gen_ai.usage.prompt_tokens",
                    value: { intValue: { low: 14, high: 0, unsigned: false } },
                  },
                  {
                    key: "gen_ai.prompt.0.role",
                    value: { stringValue: "user" },
                  },
                  {
                    key: "gen_ai.prompt.0.content",
                    value: { stringValue: "What is LLM Observability?" },
                  },
                  {
                    key: "gen_ai.completion.0.role",
                    value: { stringValue: "assistant" },
                  },
                  {
                    key: "gen_ai.completion.0.content",
                    value: {
                      stringValue: "LLM Observability is a strategy...",
                    },
                  },
                ],
                status: {},
              },
            ],
          },
        ],
      };

      const observations =
        await processOtelSpanToObservationForEval(traceLoopSpan);

      expect(observations).toHaveLength(1);
      const obs = observations[0];

      // Validate schema conformance
      const result = observationForEvalSchema.safeParse(obs);
      expect(result.success).toBe(true);

      // Validate model extraction (should get response.model)
      expect(obs.provided_model_name).toBe("gpt-3.5-turbo-0125");

      // Validate usage extraction (TraceLoop uses different keys)
      expect(obs.provided_usage_details).toMatchObject({
        input: 14,
        output: 173,
      });
    });

    it("should convert a Vercel AI SDK span to valid ObservationForEval", async () => {
      const vercelAISpan = {
        resource: {
          attributes: [
            { key: "telemetry.sdk.language", value: { stringValue: "nodejs" } },
            { key: "service.name", value: { stringValue: "vercel-app" } },
          ],
        },
        scopeSpans: [
          {
            scope: { name: "ai" },
            spans: [
              {
                traceId: createBufferId("f1e2d3c4b5a69788f1e2d3c4b5a69788"),
                spanId: createBufferId("1122334455667788"),
                name: "ai.generateText",
                kind: 1,
                startTimeUnixNano: createNanoTimestamp(
                  BigInt(1738241387865000000),
                ),
                endTimeUnixNano: createNanoTimestamp(
                  BigInt(1738241389310000000),
                ),
                attributes: [
                  { key: "ai.model.id", value: { stringValue: "gpt-4-turbo" } },
                  {
                    key: "ai.operationId",
                    value: { stringValue: "generateText" },
                  },
                  {
                    key: "ai.telemetry.functionId",
                    value: { stringValue: "chat" },
                  },
                  {
                    key: "ai.prompt.messages",
                    value: {
                      stringValue: '[{"role":"user","content":"Say hello"}]',
                    },
                  },
                  {
                    key: "ai.response.text",
                    value: { stringValue: "Hello! How can I help you today?" },
                  },
                  {
                    key: "gen_ai.usage.input_tokens",
                    value: { intValue: { low: 10, high: 0, unsigned: false } },
                  },
                  {
                    key: "gen_ai.usage.output_tokens",
                    value: { intValue: { low: 8, high: 0, unsigned: false } },
                  },
                  {
                    key: "ai.telemetry.metadata.userId",
                    value: { stringValue: "vercel-user-123" },
                  },
                  {
                    key: "ai.telemetry.metadata.sessionId",
                    value: { stringValue: "vercel-session-456" },
                  },
                ],
                status: {},
              },
            ],
          },
        ],
      };

      const observations =
        await processOtelSpanToObservationForEval(vercelAISpan);

      expect(observations).toHaveLength(1);
      const obs = observations[0];

      // Validate schema conformance
      const result = observationForEvalSchema.safeParse(obs);
      expect(result.success).toBe(true);

      // Validate model extraction
      expect(obs.provided_model_name).toBe("gpt-4-turbo");

      // Validate user/session from Vercel AI SDK metadata
      expect(obs.user_id).toBe("vercel-user-123");
      expect(obs.session_id).toBe("vercel-session-456");

      // Validate I/O extraction
      expect(obs.input).toBe('[{"role":"user","content":"Say hello"}]');
      expect(obs.output).toBe("Hello! How can I help you today?");
    });

    it("should extract AI SDK tool calls from stringified OTel toolCalls attributes", async () => {
      const vercelAIToolCallSpan = {
        resource: { attributes: [] },
        scopeSpans: [
          {
            scope: { name: "ai" },
            spans: [
              {
                traceId: createBufferId("a1e2d3c4b5a69788a1e2d3c4b5a69788"),
                spanId: createBufferId("1122334455667799"),
                name: "ai.generateText.doGenerate",
                kind: 1,
                startTimeUnixNano: createNanoTimestamp(
                  BigInt(1738241387865000000),
                ),
                endTimeUnixNano: createNanoTimestamp(
                  BigInt(1738241389310000000),
                ),
                attributes: [
                  {
                    key: "ai.model.id",
                    value: { stringValue: "claude-opus-4-6" },
                  },
                  {
                    key: "ai.operationId",
                    value: { stringValue: "ai.generateText.doGenerate" },
                  },
                  {
                    key: "ai.prompt.messages",
                    value: {
                      stringValue: JSON.stringify([
                        {
                          role: "user",
                          content: "Check the weather in Berlin",
                        },
                      ]),
                    },
                  },
                  {
                    key: "ai.response.text",
                    value: { stringValue: "I'll check the weather." },
                  },
                  {
                    key: "ai.response.toolCalls",
                    value: {
                      stringValue: JSON.stringify([
                        {
                          toolCallId: "tooluse_weather",
                          toolName: "getWeather",
                          input: {
                            location: "Berlin",
                            unit: "celsius",
                          },
                        },
                      ]),
                    },
                  },
                ],
                status: {},
              },
            ],
          },
        ],
      };

      const observations =
        await processOtelSpanToObservationForEval(vercelAIToolCallSpan);

      expect(observations).toHaveLength(1);
      const obs = observations[0];

      const result = observationForEvalSchema.safeParse(obs);
      expect(result.success).toBe(true);

      expect(obs.tool_call_names).toEqual(["getWeather"]);
      expect(obs.tool_calls).toHaveLength(1);
      expect(JSON.parse(obs.tool_calls[0])).toMatchObject({
        id: "tooluse_weather",
        arguments: JSON.stringify({
          location: "Berlin",
          unit: "celsius",
        }),
      });
    });

    it("should create ClickHouse-valid event records for event-based OTel spans with available tools", async () => {
      const tool = {
        type: "function",
        name: "calculator",
        description: "Do math",
        parameters: {
          type: "object",
          properties: {
            expression: { type: "string" },
          },
          required: ["expression"],
        },
      };
      const resourceSpan = {
        resource: {
          attributes: [
            { key: "service.name", value: { stringValue: "agent-service" } },
          ],
        },
        scopeSpans: [
          {
            scope: { name: "agent_framework", version: "1.0.0" },
            spans: [
              {
                traceId: createBufferId("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"),
                spanId: createBufferId("bbbbbbbbbbbbbbbb"),
                name: "agent-event-span",
                kind: 1,
                startTimeUnixNano: createNanoTimestamp(
                  BigInt(1714488530686000000),
                ),
                endTimeUnixNano: createNanoTimestamp(
                  BigInt(1714488530687000000),
                ),
                attributes: [
                  {
                    key: "gen_ai.tool.definitions",
                    value: { stringValue: JSON.stringify([tool]) },
                  },
                ],
                events: [
                  {
                    name: "gen_ai.user.message",
                    timeUnixNano: createNanoTimestamp(
                      BigInt(1714488530686000000),
                    ),
                    attributes: [
                      {
                        key: "content",
                        value: { stringValue: "What is 2 + 2?" },
                      },
                    ],
                  },
                  {
                    name: "gen_ai.choice",
                    timeUnixNano: createNanoTimestamp(
                      BigInt(1714488530687000000),
                    ),
                    attributes: [
                      {
                        key: "tool_calls",
                        value: {
                          stringValue: JSON.stringify([
                            {
                              id: "call_1",
                              name: "calculator",
                              arguments: JSON.stringify({
                                expression: "2 + 2",
                              }),
                            },
                          ]),
                        },
                      },
                    ],
                  },
                ],
                status: {},
              },
            ],
          },
        ],
      };

      const processor = createTestOtelProcessor();
      const eventInputs = processor.processToEvent([resourceSpan]);

      expect(eventInputs).toHaveLength(1);
      expect(typeof eventInputs[0].input).toBe("object");
      expect(
        eventInputs[0].metadata.attributes?.["gen_ai.tool.definitions"],
      ).toBeUndefined();
      expect(eventInputs[0].toolDefinitions).toHaveProperty("calculator");
      expect(eventInputs[0].toolCallNames).toEqual(["calculator"]);

      const eventRecord = await ingestionService.createEventRecord(
        eventInputs[0],
        "test/otel/tools.json",
      );

      expect(typeof eventRecord.input).toBe("string");
      expect(JSON.parse(eventRecord.input)).toEqual({
        messages: [{ role: "user", content: "What is 2 + 2?" }],
        tools: [tool],
      });
      expect(eventRecord.metadata_names).not.toContain(
        "attributes.gen_ai.tool.definitions",
      );
      expect(eventRecord.tool_definitions).toHaveProperty("calculator");
      expect(JSON.parse(eventRecord.tool_definitions.calculator)).toEqual({
        description: "Do math",
        parameters: JSON.stringify(tool.parameters),
      });
      expect(eventRecord.tool_call_names).toEqual(["calculator"]);
      expect(eventRecord.tool_calls).toHaveLength(1);
      expect(JSON.parse(eventRecord.tool_calls[0])).toEqual({
        id: "call_1",
        arguments: JSON.stringify({ expression: "2 + 2" }),
        type: "",
        index: 0,
      });

      const observation = convertEventRecordToObservationForEval(eventRecord);
      const result = observationForEvalSchema.safeParse(observation);
      expect(result.success).toBe(true);
      expect(observation.tool_definitions).toEqual(
        eventRecord.tool_definitions,
      );
    });
  });

  describe("Schema field coverage", () => {
    it("should correctly map all ObservationForEval fields", async () => {
      // Create a span with all possible fields populated
      const fullOtelSpan = {
        resource: {
          attributes: [
            { key: "telemetry.sdk.language", value: { stringValue: "python" } },
            { key: "langfuse.environment", value: { stringValue: "staging" } },
            { key: "langfuse.release", value: { stringValue: "v2.0.0" } },
            {
              key: "service.name",
              value: { stringValue: "full-test-service" },
            },
          ],
        },
        scopeSpans: [
          {
            scope: { name: "langfuse-sdk", version: "3.5.0" },
            spans: [
              {
                traceId: createBufferId("abcdef1234567890abcdef1234567890"),
                spanId: createBufferId("1234567890abcdef"),
                parentSpanId: createBufferId("fedcba0987654321"),
                name: "full-test-observation",
                kind: 1,
                startTimeUnixNano: createNanoTimestamp(
                  BigInt(1738241487865000000),
                ),
                endTimeUnixNano: createNanoTimestamp(
                  BigInt(1738241489310000000),
                ),
                attributes: [
                  {
                    key: "langfuse.observation.type",
                    value: { stringValue: "generation" },
                  },
                  {
                    key: "langfuse.observation.level",
                    value: { stringValue: "ERROR" },
                  },
                  {
                    key: "langfuse.observation.status_message",
                    value: { stringValue: "Rate limit exceeded" },
                  },
                  {
                    key: "langfuse.observation.input",
                    value: { stringValue: '{"query": "test input"}' },
                  },
                  {
                    key: "langfuse.observation.output",
                    value: { stringValue: '{"result": "test output"}' },
                  },
                  {
                    key: "langfuse.observation.model.name",
                    value: { stringValue: "claude-3-opus" },
                  },
                  // Model parameters are extracted from gen_ai.request.* attributes
                  {
                    key: "gen_ai.request.temperature",
                    value: { doubleValue: 0.5 },
                  },
                  {
                    key: "gen_ai.request.max_tokens",
                    value: {
                      intValue: { low: 1000, high: 0, unsigned: false },
                    },
                  },
                  {
                    key: "langfuse.observation.prompt.name",
                    value: { stringValue: "analysis-prompt" },
                  },
                  {
                    key: "langfuse.observation.prompt.version",
                    value: { intValue: { low: 3, high: 0, unsigned: false } },
                  },
                  {
                    key: "langfuse.observation.usage_details",
                    value: {
                      stringValue:
                        '{"input": 500, "output": 200, "total": 700}',
                    },
                  },
                  {
                    key: "langfuse.observation.cost_details",
                    value: {
                      stringValue: '{"input": 0.015, "output": 0.030}',
                    },
                  },
                  {
                    key: "langfuse.observation.metadata.request_id",
                    value: { stringValue: '"req-12345"' },
                  },
                  {
                    key: "langfuse.observation.metadata.source",
                    value: { stringValue: '"api"' },
                  },
                  {
                    key: "langfuse.trace.name",
                    value: { stringValue: "full-test-trace" },
                  },
                  { key: "user.id", value: { stringValue: "user-full-test" } },
                  {
                    key: "session.id",
                    value: { stringValue: "session-full-test" },
                  },
                  {
                    key: "langfuse.version",
                    value: { stringValue: "v1.5.0" },
                  },
                  {
                    key: "langfuse.trace.tags",
                    value: {
                      arrayValue: {
                        values: [
                          { stringValue: "production" },
                          { stringValue: "high-priority" },
                          { stringValue: "monitored" },
                        ],
                      },
                    },
                  },
                ],
                status: { code: 2, message: "Error" },
              },
            ],
          },
        ],
      };

      const observations =
        await processOtelSpanToObservationForEval(fullOtelSpan);

      expect(observations).toHaveLength(1);
      const obs = observations[0];

      // Validate schema conformance
      const result = observationForEvalSchema.safeParse(obs);
      expect(result.success).toBe(true);

      // Validate all fields from observationForEvalSchema are present
      const schemaKeys = Object.keys(observationForEvalSchema.shape);
      for (const key of schemaKeys) {
        expect(obs).toHaveProperty(key);
      }

      // Validate specific field values
      expect(obs.span_id).toBe("1234567890abcdef");
      expect(obs.trace_id).toBe("abcdef1234567890abcdef1234567890");
      expect(obs.parent_span_id).toBe("fedcba0987654321");
      expect(obs.type).toBe("GENERATION");
      expect(obs.name).toBe("full-test-observation");
      expect(obs.environment).toBe("staging");
      expect(obs.version).toBe("v1.5.0");
      expect(obs.release).toBe("v2.0.0");
      expect(obs.level).toBe("ERROR");
      expect(obs.status_message).toBe("Rate limit exceeded");
      expect(obs.trace_name).toBe("full-test-trace");
      expect(obs.user_id).toBe("user-full-test");
      expect(obs.session_id).toBe("session-full-test");
      expect(obs.tags).toEqual(["production", "high-priority", "monitored"]);
      expect(obs.provided_model_name).toBe("claude-3-opus");
      expect(obs.model_parameters).toEqual({
        temperature: 0.5,
        max_tokens: 1000,
      });
      expect(obs.prompt_name).toBe("analysis-prompt");
      expect(obs.prompt_version).toBe(3);
      expect(obs.input).toBe('{"query": "test input"}');
      expect(obs.output).toBe('{"result": "test output"}');
    });

    it("should handle experiment fields correctly", async () => {
      const experimentSpan = {
        resource: { attributes: [] },
        scopeSpans: [
          {
            scope: { name: "langfuse-sdk", version: "3.0.0" },
            spans: [
              {
                traceId: createBufferId("1234567890123456789012345678abcd"),
                spanId: createBufferId("exp1234567890123"),
                name: "experiment-run",
                kind: 1,
                startTimeUnixNano: createNanoTimestamp(
                  BigInt(1738241587865000000),
                ),
                endTimeUnixNano: createNanoTimestamp(
                  BigInt(1738241589310000000),
                ),
                attributes: [
                  {
                    key: "langfuse.observation.type",
                    value: { stringValue: "generation" },
                  },
                  {
                    key: "langfuse.experiment.id",
                    value: { stringValue: "exp-001" },
                  },
                  {
                    key: "langfuse.experiment.name",
                    value: { stringValue: "Prompt Optimization v2" },
                  },
                  {
                    key: "langfuse.experiment.description",
                    value: { stringValue: "Testing new prompt templates" },
                  },
                  {
                    key: "langfuse.experiment.dataset.id",
                    value: { stringValue: "dataset-test-001" },
                  },
                  {
                    key: "langfuse.experiment.item.id",
                    value: { stringValue: "item-123" },
                  },
                  {
                    key: "langfuse.experiment.item.expected_output",
                    value: { stringValue: '{"expected": "result"}' },
                  },
                  {
                    key: "langfuse.observation.input",
                    value: { stringValue: '{"test": "input"}' },
                  },
                  {
                    key: "langfuse.observation.output",
                    value: { stringValue: '{"test": "output"}' },
                  },
                ],
                status: {},
              },
            ],
          },
        ],
      };

      const observations =
        await processOtelSpanToObservationForEval(experimentSpan);

      expect(observations).toHaveLength(1);
      const obs = observations[0];

      // Validate schema conformance
      const result = observationForEvalSchema.safeParse(obs);
      expect(result.success).toBe(true);

      // Validate experiment fields
      expect(obs.experiment_id).toBe("exp-001");
      expect(obs.experiment_name).toBe("Prompt Optimization v2");
      expect(obs.experiment_description).toBe("Testing new prompt templates");
      expect(obs.experiment_dataset_id).toBe("dataset-test-001");
      expect(obs.experiment_item_id).toBe("item-123");
      expect(obs.experiment_item_expected_output).toBe(
        '{"expected": "result"}',
      );
    });
  });

  describe("Edge cases", () => {
    it("should handle spans with empty strings correctly", async () => {
      const emptyStringSpan = {
        resource: { attributes: [] },
        scopeSpans: [
          {
            scope: { name: "langfuse-sdk", version: "3.0.0" },
            spans: [
              {
                traceId: createBufferId("12345678901234567890123456789012"),
                spanId: createBufferId("empty123456789ab"),
                name: "",
                kind: 1,
                startTimeUnixNano: createNanoTimestamp(
                  BigInt(1738241687865000000),
                ),
                endTimeUnixNano: createNanoTimestamp(
                  BigInt(1738241689310000000),
                ),
                attributes: [
                  {
                    key: "langfuse.observation.type",
                    value: { stringValue: "span" },
                  },
                  {
                    key: "langfuse.observation.input",
                    value: { stringValue: "" },
                  },
                  {
                    key: "langfuse.observation.output",
                    value: { stringValue: "" },
                  },
                ],
                status: {},
              },
            ],
          },
        ],
      };

      const observations =
        await processOtelSpanToObservationForEval(emptyStringSpan);

      expect(observations).toHaveLength(1);
      const obs = observations[0];

      // Validate schema conformance
      const result = observationForEvalSchema.safeParse(obs);
      expect(result.success).toBe(true);

      // Empty name should use span name (which is empty)
      expect(obs.name).toBe("");
    });

    it("should handle spans with special characters in values", async () => {
      const specialCharsSpan = {
        resource: { attributes: [] },
        scopeSpans: [
          {
            scope: { name: "langfuse-sdk", version: "3.0.0" },
            spans: [
              {
                traceId: createBufferId("12345678901234561234567890123456"),
                spanId: createBufferId("spec123456789abc"),
                name: "span-with-special-chars",
                kind: 1,
                startTimeUnixNano: createNanoTimestamp(
                  BigInt(1738241787865000000),
                ),
                endTimeUnixNano: createNanoTimestamp(
                  BigInt(1738241789310000000),
                ),
                attributes: [
                  {
                    key: "langfuse.observation.type",
                    value: { stringValue: "span" },
                  },
                  {
                    key: "langfuse.observation.input",
                    value: {
                      stringValue:
                        '{"message": "Hello! <script>alert(1)</script>"}',
                    },
                  },
                  {
                    key: "langfuse.observation.output",
                    value: {
                      stringValue: 'Line1\\nLine2\\tTabbed\\"Quoted\\"',
                    },
                  },
                  {
                    key: "langfuse.trace.tags",
                    value: {
                      arrayValue: {
                        values: [
                          { stringValue: "tag-with-dash" },
                          { stringValue: "tag_with_underscore" },
                          { stringValue: "tag.with" },
                        ],
                      },
                    },
                  },
                ],
                status: {},
              },
            ],
          },
        ],
      };

      const observations =
        await processOtelSpanToObservationForEval(specialCharsSpan);

      expect(observations).toHaveLength(1);
      const obs = observations[0];

      // Validate schema conformance
      const result = observationForEvalSchema.safeParse(obs);
      expect(result.success).toBe(true);

      // Validate special characters are preserved
      expect(obs.input).toContain("<script>");
      expect(obs.tags).toEqual([
        "tag-with-dash",
        "tag_with_underscore",
        "tag.with",
      ]);
    });

    it("should handle spans with very large input/output", async () => {
      const largeContent = "x".repeat(100000); // 100KB of data
      const largeSpan = {
        resource: { attributes: [] },
        scopeSpans: [
          {
            scope: { name: "langfuse-sdk", version: "3.0.0" },
            spans: [
              {
                traceId: createBufferId("12345678901234567890123456781234"),
                spanId: createBufferId("large12345678901"),
                name: "large-io-span",
                kind: 1,
                startTimeUnixNano: createNanoTimestamp(
                  BigInt(1738241887865000000),
                ),
                endTimeUnixNano: createNanoTimestamp(
                  BigInt(1738241889310000000),
                ),
                attributes: [
                  {
                    key: "langfuse.observation.type",
                    value: { stringValue: "generation" },
                  },
                  {
                    key: "langfuse.observation.input",
                    value: { stringValue: largeContent },
                  },
                  {
                    key: "langfuse.observation.output",
                    value: { stringValue: largeContent },
                  },
                ],
                status: {},
              },
            ],
          },
        ],
      };

      const observations = await processOtelSpanToObservationForEval(largeSpan);

      expect(observations).toHaveLength(1);
      const obs = observations[0];

      // Validate schema conformance
      const result = observationForEvalSchema.safeParse(obs);
      expect(result.success).toBe(true);

      // Validate large content is preserved
      expect(obs.input?.length).toBe(100000);
      expect(obs.output?.length).toBe(100000);
    });

    it("should handle different observation types", async () => {
      const observationTypes = [
        "span",
        "generation",
        "event",
        "agent",
        "tool",
        "chain",
        "retriever",
        "evaluator",
        "guardrail",
        "embedding",
      ];

      for (const obsType of observationTypes) {
        // Create valid hex string for traceId (32 chars = 16 bytes)
        const typeHex = obsType.padEnd(16, "0").slice(0, 16);
        const traceIdHex = `${typeHex}${typeHex}`;
        // Create valid hex string for spanId (16 chars = 8 bytes)
        const spanIdHex = `${obsType.slice(0, 8).padEnd(8, "0")}12345678`;

        const typeSpan = {
          resource: { attributes: [] },
          scopeSpans: [
            {
              scope: { name: "langfuse-sdk", version: "3.0.0" },
              spans: [
                {
                  traceId: createBufferId(traceIdHex),
                  spanId: createBufferId(spanIdHex),
                  name: `${obsType}-test`,
                  kind: 1,
                  startTimeUnixNano: createNanoTimestamp(
                    BigInt(1738241987865000000),
                  ),
                  endTimeUnixNano: createNanoTimestamp(
                    BigInt(1738241989310000000),
                  ),
                  attributes: [
                    {
                      key: "langfuse.observation.type",
                      value: { stringValue: obsType },
                    },
                  ],
                  status: {},
                },
              ],
            },
          ],
        };

        const observations =
          await processOtelSpanToObservationForEval(typeSpan);

        expect(observations).toHaveLength(1);
        expect(observations[0].type).toBe(obsType.toUpperCase());

        // Validate schema conformance for each type
        const result = observationForEvalSchema.safeParse(observations[0]);
        expect(result.success).toBe(true);
      }
    });

    it("should handle metadata with nested objects", async () => {
      const nestedMetadataSpan = {
        resource: { attributes: [] },
        scopeSpans: [
          {
            scope: { name: "langfuse-sdk", version: "3.0.0" },
            spans: [
              {
                traceId: createBufferId("12345678901234567890123456789abc"),
                spanId: createBufferId("nest123456789012"),
                name: "nested-metadata-span",
                kind: 1,
                startTimeUnixNano: createNanoTimestamp(
                  BigInt(1738242087865000000),
                ),
                endTimeUnixNano: createNanoTimestamp(
                  BigInt(1738242089310000000),
                ),
                attributes: [
                  {
                    key: "langfuse.observation.type",
                    value: { stringValue: "span" },
                  },
                  {
                    key: "langfuse.observation.metadata",
                    value: {
                      stringValue: JSON.stringify({
                        nested: {
                          deeply: {
                            value: "found",
                          },
                        },
                        array: [1, 2, 3],
                        mixed: {
                          string: "text",
                          number: 42,
                          boolean: true,
                        },
                      }),
                    },
                  },
                ],
                status: {},
              },
            ],
          },
        ],
      };

      const observations =
        await processOtelSpanToObservationForEval(nestedMetadataSpan);

      expect(observations).toHaveLength(1);
      const obs = observations[0];

      // Validate schema conformance
      const result = observationForEvalSchema.safeParse(obs);
      expect(result.success).toBe(true);

      // Validate metadata is present (as flattened or stringified)
      expect(obs.metadata).toBeDefined();
      expect(typeof obs.metadata).toBe("object");
    });
  });

  describe("Multiple spans in single batch", () => {
    it("should process multiple spans and all conform to schema", async () => {
      const multiSpanBatch = {
        resource: {
          attributes: [
            { key: "langfuse.environment", value: { stringValue: "test" } },
          ],
        },
        scopeSpans: [
          {
            scope: { name: "langfuse-sdk", version: "3.0.0" },
            spans: [
              {
                traceId: createBufferId("12345678901234567890123456789def"),
                spanId: createBufferId("aaaa1234567890ab"),
                name: "span-1",
                kind: 1,
                startTimeUnixNano: createNanoTimestamp(
                  BigInt(1738242187865000000),
                ),
                endTimeUnixNano: createNanoTimestamp(
                  BigInt(1738242189310000000),
                ),
                attributes: [
                  {
                    key: "langfuse.observation.type",
                    value: { stringValue: "span" },
                  },
                ],
                status: {},
              },
              {
                traceId: createBufferId("12345678901234567890123456789def"),
                spanId: createBufferId("bbbb2345678901bc"),
                parentSpanId: createBufferId("aaaa1234567890ab"),
                name: "generation-1",
                kind: 1,
                startTimeUnixNano: createNanoTimestamp(
                  BigInt(1738242189400000000),
                ),
                endTimeUnixNano: createNanoTimestamp(
                  BigInt(1738242189500000000),
                ),
                attributes: [
                  {
                    key: "langfuse.observation.type",
                    value: { stringValue: "generation" },
                  },
                  {
                    key: "langfuse.observation.model.name",
                    value: { stringValue: "gpt-4" },
                  },
                ],
                status: {},
              },
              {
                traceId: createBufferId("12345678901234567890123456789def"),
                spanId: createBufferId("cccc3456789012cd"),
                parentSpanId: createBufferId("aaaa1234567890ab"),
                name: "tool-call",
                kind: 1,
                startTimeUnixNano: createNanoTimestamp(
                  BigInt(1738242189600000000),
                ),
                endTimeUnixNano: createNanoTimestamp(
                  BigInt(1738242189700000000),
                ),
                attributes: [
                  {
                    key: "langfuse.observation.type",
                    value: { stringValue: "tool" },
                  },
                ],
                status: {},
              },
            ],
          },
        ],
      };

      const observations =
        await processOtelSpanToObservationForEval(multiSpanBatch);

      expect(observations).toHaveLength(3);

      // All observations should have the same trace_id
      expect(observations[0].trace_id).toBe("12345678901234567890123456789def");
      expect(observations[1].trace_id).toBe("12345678901234567890123456789def");
      expect(observations[2].trace_id).toBe("12345678901234567890123456789def");

      // Parent-child relationships should be preserved
      // Note: nullish fields may be null or undefined depending on the processing path
      expect(observations[0].parent_span_id).toBeFalsy();
      expect(observations[1].parent_span_id).toBe("aaaa1234567890ab");
      expect(observations[2].parent_span_id).toBe("aaaa1234567890ab");

      // All should conform to schema
      for (const obs of observations) {
        const result = observationForEvalSchema.safeParse(obs);
        expect(result.success).toBe(true);
      }
    });
  });

  describe("Usage and cost details", () => {
    it("should correctly extract and transform usage details from OTEL spans", async () => {
      const usageSpan = {
        resource: { attributes: [] },
        scopeSpans: [
          {
            scope: { name: "langfuse-sdk", version: "3.0.0" },
            spans: [
              {
                traceId: createBufferId("1234567890123456789012345678cdef"),
                spanId: createBufferId("usage12345678901"),
                name: "usage-test",
                kind: 1,
                startTimeUnixNano: createNanoTimestamp(
                  BigInt(1738242287865000000),
                ),
                endTimeUnixNano: createNanoTimestamp(
                  BigInt(1738242289310000000),
                ),
                attributes: [
                  {
                    key: "langfuse.observation.type",
                    value: { stringValue: "generation" },
                  },
                  {
                    key: "langfuse.observation.model.name",
                    value: { stringValue: "gpt-4" },
                  },
                  {
                    key: "langfuse.observation.usage_details",
                    value: {
                      stringValue: JSON.stringify({
                        input: 100,
                        output: 200,
                        total: 300,
                        cached_input: 50,
                      }),
                    },
                  },
                  {
                    key: "langfuse.observation.cost_details",
                    value: {
                      stringValue: JSON.stringify({
                        input: 0.01,
                        output: 0.02,
                        total: 0.03,
                      }),
                    },
                  },
                ],
                status: {},
              },
            ],
          },
        ],
      };

      const observations = await processOtelSpanToObservationForEval(usageSpan);

      expect(observations).toHaveLength(1);
      const obs = observations[0];

      // Validate schema conformance
      const result = observationForEvalSchema.safeParse(obs);
      expect(result.success).toBe(true);

      // Validate usage details
      expect(obs.provided_usage_details).toEqual({
        input: 100,
        output: 200,
        total: 300,
        cached_input: 50,
      });

      // Validate cost details
      expect(obs.provided_cost_details).toEqual({
        input: 0.01,
        output: 0.02,
        total: 0.03,
      });
    });
  });

  describe("Tool calls extraction", () => {
    it("should correctly extract tool definitions and calls", async () => {
      const toolCallSpan = {
        resource: { attributes: [] },
        scopeSpans: [
          {
            scope: { name: "langfuse-sdk", version: "3.0.0" },
            spans: [
              {
                traceId: createBufferId("123456789012345678901234567890ab"),
                spanId: createBufferId("tool1234567890ab"),
                name: "tool-call-test",
                kind: 1,
                startTimeUnixNano: createNanoTimestamp(
                  BigInt(1738242387865000000),
                ),
                endTimeUnixNano: createNanoTimestamp(
                  BigInt(1738242389310000000),
                ),
                attributes: [
                  {
                    key: "langfuse.observation.type",
                    value: { stringValue: "generation" },
                  },
                  {
                    key: "langfuse.observation.model.name",
                    value: { stringValue: "gpt-4" },
                  },
                  {
                    key: "langfuse.observation.input",
                    value: {
                      stringValue: JSON.stringify({
                        messages: [
                          { role: "user", content: "What is the weather?" },
                        ],
                        tools: [
                          {
                            type: "function",
                            function: {
                              name: "get_weather",
                              description: "Get current weather",
                              parameters: {
                                type: "object",
                                properties: {
                                  location: { type: "string" },
                                },
                              },
                            },
                          },
                        ],
                      }),
                    },
                  },
                  {
                    key: "langfuse.observation.output",
                    value: {
                      stringValue: JSON.stringify({
                        choices: [
                          {
                            message: {
                              role: "assistant",
                              tool_calls: [
                                {
                                  id: "call_123",
                                  type: "function",
                                  function: {
                                    name: "get_weather",
                                    arguments: '{"location": "San Francisco"}',
                                  },
                                },
                              ],
                            },
                          },
                        ],
                      }),
                    },
                  },
                ],
                status: {},
              },
            ],
          },
        ],
      };

      const observations =
        await processOtelSpanToObservationForEval(toolCallSpan);

      expect(observations).toHaveLength(1);
      const obs = observations[0];

      // Validate schema conformance
      const result = observationForEvalSchema.safeParse(obs);
      expect(result.success).toBe(true);

      // Tool definitions and calls are extracted by IngestionService
      // They should be present in the observation
      expect(obs.tool_definitions).toBeDefined();
      expect(obs.tool_calls).toBeDefined();
      expect(obs.tool_call_names).toBeDefined();

      // tool_call_count is derived from tool_call_names
      expect(obs.tool_call_count).toBe(obs.tool_call_names.length);
    });

    it("should set tool_call_count to 0 when no tool calls are present", async () => {
      const noToolsSpan = {
        resource: { attributes: [] },
        scopeSpans: [
          {
            scope: { name: "langfuse-sdk", version: "3.0.0" },
            spans: [
              {
                traceId: createBufferId("123456789012345678901234567890ab"),
                spanId: createBufferId("00b001567890abcd"),
                name: "no-tools-test",
                kind: 1,
                startTimeUnixNano: createNanoTimestamp(
                  BigInt(1738242387865000000),
                ),
                endTimeUnixNano: createNanoTimestamp(
                  BigInt(1738242389310000000),
                ),
                attributes: [
                  {
                    key: "langfuse.observation.type",
                    value: { stringValue: "generation" },
                  },
                  {
                    key: "langfuse.observation.input",
                    value: { stringValue: '"Hello"' },
                  },
                  {
                    key: "langfuse.observation.output",
                    value: { stringValue: '"Hi there"' },
                  },
                ],
                status: {},
              },
            ],
          },
        ],
      };

      const observations =
        await processOtelSpanToObservationForEval(noToolsSpan);

      expect(observations).toHaveLength(1);
      expect(observations[0].tool_call_count).toBe(0);
    });
  });

  describe("Malformed usage_details and cost_details handling", () => {
    function buildSpanWithRawAttributes(
      extraAttrs: Array<{ key: string; value: Record<string, unknown> }>,
      scopeName = "langfuse-sdk",
    ) {
      return {
        resource: { attributes: [] },
        scopeSpans: [
          {
            scope: { name: scopeName, version: "3.0.0" },
            spans: [
              {
                traceId: createBufferId("1234567890abcdef1234567890malfor"),
                spanId: createBufferId("malformed12345678"),
                name: "malformed-test",
                kind: 1,
                startTimeUnixNano: createNanoTimestamp(
                  BigInt(1738242287865000000),
                ),
                endTimeUnixNano: createNanoTimestamp(
                  BigInt(1738242289310000000),
                ),
                attributes: [
                  {
                    key: "langfuse.observation.type",
                    value: { stringValue: "generation" },
                  },
                  {
                    key: "langfuse.observation.model.name",
                    value: { stringValue: "gpt-4" },
                  },
                  ...extraAttrs,
                ],
                status: {},
              },
            ],
          },
        ],
      };
    }

    it.each([
      {
        name: "invalid JSON",
        usage: { stringValue: "not json{{{" },
        cost: { stringValue: "{broken" },
        expectedUsage: {},
        expectedCost: {},
      },
      {
        name: "JSON array",
        usage: { stringValue: "[100, 200]" },
        cost: { stringValue: "[0.01]" },
        expectedUsage: {},
        expectedCost: {},
      },
      {
        name: "JSON primitive",
        usage: { stringValue: "42" },
        cost: { stringValue: '"a string"' },
        expectedUsage: {},
        expectedCost: {},
      },
      {
        name: "non-string attribute type",
        usage: { intValue: 12345 },
        cost: { doubleValue: 0.99 },
        expectedUsage: {},
        expectedCost: {},
      },
      {
        name: "attributes omitted entirely",
        usage: null,
        cost: null,
        expectedUsage: {},
        expectedCost: {},
      },
      {
        name: "valid usage but malformed cost",
        usage: { stringValue: '{"input":100,"output":200}' },
        cost: { stringValue: "not json" },
        expectedUsage: { input: 100, output: 200 },
        expectedCost: {},
      },
      {
        name: "non-numeric cost entries",
        usage: null,
        cost: {
          stringValue:
            '{"total":0.01,"input":"0.004","output":-0.002,"currency":"USD"}',
        },
        expectedUsage: {},
        expectedCost: { total: 0.01, input: 0.004, output: -0.002 },
      },
    ])(
      "should handle $name without data corruption",
      async ({ usage, cost, expectedUsage, expectedCost }) => {
        const attrs: Array<{
          key: string;
          value: Record<string, unknown>;
        }> = [];
        if (usage)
          attrs.push({
            key: "langfuse.observation.usage_details",
            value: usage,
          });
        if (cost)
          attrs.push({
            key: "langfuse.observation.cost_details",
            value: cost,
          });

        const span = buildSpanWithRawAttributes(attrs);
        const observations = await processOtelSpanToObservationForEval(span);

        expect(observations).toHaveLength(1);
        const obs = observations[0];
        expect(observationForEvalSchema.safeParse(obs).success).toBe(true);
        expect(obs.provided_usage_details).toEqual(expectedUsage);
        expect(obs.provided_cost_details).toEqual(expectedCost);
      },
    );

    it("should fall through to gen_ai.usage.* when usage_details is invalid JSON", async () => {
      const span = buildSpanWithRawAttributes(
        [
          {
            key: "langfuse.observation.usage_details",
            value: { stringValue: "{{bad}}" },
          },
          {
            key: "gen_ai.usage.input_tokens",
            value: { intValue: { low: 50, high: 0, unsigned: false } },
          },
          {
            key: "gen_ai.usage.output_tokens",
            value: { intValue: { low: 75, high: 0, unsigned: false } },
          },
        ],
        "ai", // scope name triggers gen_ai.usage.* fallback path
      );

      const observations = await processOtelSpanToObservationForEval(span);
      expect(observations).toHaveLength(1);
      expect(observations[0].provided_usage_details).toMatchObject({
        input: 50,
        output: 75,
      });
    });
  });
});