Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

28.4 KB
Raw Blame
import { randomUUID } from "crypto";
import { makeAPICall } from "@/src/__tests__/test-utils";
import waitForExpect from "wait-for-expect";
import {
  clickhouseClient,
  getBlobStorageByProjectAndEntityId,
  getObservationById,
  getScoreById,
  getTraceById,
  createOrgProjectAndApiKey,
  ingestionEvent,
} from "@langfuse/shared/src/server";
import { v4 } from "uuid";

describe("ingestion timestamp compatibility", () => {
  it.each(["2024-02-29T06:15Z", "2024-02-29T06:15:00.123+02:30"])(
    "accepts %s in both the envelope and body",
    (timestamp) => {
      const event = {
        id: "event-id",
        type: "trace-create",
        timestamp,
        body: { id: "trace-id", timestamp },
      };
      expect(ingestionEvent.parse(event)).toMatchObject(event);
    },
  );
});

describe("ingestion modelParameters compatibility", () => {
  const parseModelParameters = (modelParameters: Record<string, unknown>) =>
    ingestionEvent.parse({
      id: "event-id",
      type: "generation-create",
      timestamp: new Date().toISOString(),
      body: { id: "generation-id", modelParameters },
    }).body;

  it("keeps supported values unchanged", () => {
    const modelParameters = {
      temperature: 0.2,
      stream: false,
      stop: ["\n"],
      reasoning: { effort: "high" },
      seed: null,
    };

    expect(parseModelParameters(modelParameters)).toMatchObject({
      modelParameters,
    });
  });

  it("serializes nested values instead of rejecting the generation", () => {
    const responseFormat = {
      type: "json_schema",
      json_schema: { name: "answer", strict: true, schema: { type: "object" } },
    };

    expect(
      parseModelParameters({
        temperature: 0,
        response_format: responseFormat,
        thinking: { type: "enabled", budget_tokens: 1024 },
        stop_token_ids: [1, 2],
      }),
    ).toMatchObject({
      modelParameters: {
        temperature: 0,
        response_format: JSON.stringify(responseFormat),
        thinking: '{"type":"enabled","budget_tokens":1024}',
        stop_token_ids: "[1,2]",
      },
    });
  });
});

let projectId: string;
let auth: string;
let publicKey: string;

const postIngestion = (body: unknown, customHeaders?: Record<string, string>) =>
  makeAPICall("POST", "/api/public/ingestion", body, auth, customHeaders);

type IngestionAttributionRow = {
  ingestion_api_key: string;
  ingestion_sdk_name: string;
  ingestion_sdk_version: string;
};

const getIngestionAttribution = async (table: "scores", id: string) => {
  const result = await clickhouseClient().query({
    query: `
      SELECT
        ingestion_api_key,
        ingestion_sdk_name,
        ingestion_sdk_version
      FROM ${table}
      WHERE project_id = {projectId: String}
        AND id = {id: String}
      ORDER BY event_ts DESC
      LIMIT 1
    `,
    query_params: {
      projectId,
      id,
    },
    format: "JSONEachRow",
  });

  const rows = await result.json<IngestionAttributionRow>();
  return rows[0];
};

describe("/api/public/ingestion API Endpoint", () => {
  beforeEach(async () => {
    const fixture = await createOrgProjectAndApiKey();
    projectId = fixture.projectId;
    auth = fixture.auth;
    publicKey = fixture.publicKey;
  });
  it.each([
    [
      "plain",
      {
        id: randomUUID(),
        type: "trace-create",
        timestamp: new Date().toISOString(),
        body: {
          id: randomUUID(),
          timestamp: new Date().toISOString(),
        },
      },
    ],
    [
      "metadata+io",
      {
        id: randomUUID(),
        type: "trace-create",
        timestamp: new Date().toISOString(),
        body: {
          id: randomUUID(),
          timestamp: new Date().toISOString(),
          metadata: { hello: "world" },
          input: "input",
          output: "output",
        },
      },
    ],
    [
      "complex-io",
      {
        id: randomUUID(),
        type: "trace-create",
        timestamp: new Date().toISOString(),
        body: {
          id: randomUUID(),
          timestamp: new Date().toISOString(),
          metadata: { hello: "world" },
          input: ["hello", { world: "world" }, [1, 2, 3]],
        },
      },
    ],
    [
      "non-default-environment",
      {
        id: randomUUID(),
        type: "trace-create",
        timestamp: new Date().toISOString(),
        body: {
          id: randomUUID(),
          timestamp: new Date().toISOString(),
          metadata: { hello: "world" },
          input: ["hello", { world: "world" }, [1, 2, 3]],
          environment: "production",
        },
      },
    ],
    [
      "emojis in i/o",
      {
        id: randomUUID(),
        type: "trace-create",
        timestamp: new Date().toISOString(),
        body: {
          id: randomUUID(),
          timestamp: new Date().toISOString(),
          metadata: { hello: "world" },
          input: "🪢🚀🌖",
          output: "👋🌍",
        },
      },
    ],
  ])(
    "should create traces via the ingestion API (%s)",
    async (_name: string, entity: any) => {
      const response = await postIngestion({
        batch: [entity],
      });

      expect(response.status).toBe(207);

      await waitForExpect(async () => {
        const trace = await getTraceById({
          traceId: entity.body.id,
          projectId,
        });
        expect(trace).toBeDefined();
        expect(trace!.id).toBe(entity.body.id);
        expect(trace!.projectId).toBe(projectId);
        expect(trace!.metadata).toEqual(entity.body?.metadata ?? {});
        expect(trace!.input).toEqual(entity.body?.input ?? null);
        expect(trace!.output).toEqual(entity.body?.output ?? null);
        expect(trace!.environment).toEqual(
          entity.body?.environment ?? "default",
        );
      });
    },
  );

  // Disabled within test sequence as we're using a clickhouse version which doesn't support this
  // it("should replace bad escape sequences on clickhouse", async () => {
  //   const entity = {
  //     id: randomUUID(),
  //     type: "trace-create",
  //     timestamp: new Date().toISOString(),
  //     body: {
  //       id: randomUUID(),
  //       timestamp: new Date().toISOString(),
  //       metadata: { hello: "world" },
  //       input: "test\\ud8000test",
  //       environment: "production",
  //     },
  //   };
  //   const response = await postIngestion({
  //     batch: [entity],
  //   });
  //
  //   expect(response.status).toBe(207);
  //   await waitForExpect(async () => {
  //     const trace = await getTraceById({ traceId: entity.body.id, projectId });
  //     expect(trace).toBeDefined();
  //     expect(trace!.id).toBe(entity.body.id);
  //     expect(trace!.projectId).toBe(projectId);
  //     expect(trace!.input).toContain("test");
  //   });
  // });

  it.each([
    [
      "generation",
      "GENERATION",
      {
        id: randomUUID(),
        type: "generation-create",
        timestamp: new Date().toISOString(),
        body: {
          id: randomUUID(),
          traceId: randomUUID(),
          parentObservationId: randomUUID(),
          startTime: new Date().toISOString(),
          model: "gpt-4",
          input: { text: "input" },
          output: { text: "output" },
        },
      },
    ],
    [
      "span",
      "SPAN",
      {
        id: randomUUID(),
        type: "span-create",
        timestamp: new Date().toISOString(),
        body: {
          id: randomUUID(),
          traceId: randomUUID(),
          startTime: new Date().toISOString(),
        },
      },
    ],
    [
      "span-complex-io",
      "SPAN",
      {
        id: randomUUID(),
        type: "span-create",
        timestamp: new Date().toISOString(),
        body: {
          id: randomUUID(),
          traceId: randomUUID(),
          startTime: new Date().toISOString(),
          input: ["hello", { world: "world" }, [1, 2, 3]],
          output: ["hello", { world: [2, 3, "test"] }, [1, 2, 3]],
        },
      },
    ],
    [
      "span-non-default-environment",
      "SPAN",
      {
        id: randomUUID(),
        type: "span-create",
        timestamp: new Date().toISOString(),
        body: {
          id: randomUUID(),
          traceId: randomUUID(),
          startTime: new Date().toISOString(),
          input: ["hello", { world: "world" }, [1, 2, 3]],
          output: ["hello", { world: [2, 3, "test"] }, [1, 2, 3]],
          environment: "production",
        },
      },
    ],
    [
      "agent",
      "AGENT",
      {
        id: randomUUID(),
        type: "agent-create",
        timestamp: new Date().toISOString(),
        body: {
          id: randomUUID(),
          traceId: randomUUID(),
          startTime: new Date().toISOString(),
          endTime: new Date(Date.now() + 1000).toISOString(),
          name: "AI Agent",
          input: "Process user request",
          output: "Request processed successfully",
          model: "claude-3-haiku",
          modelParameters: { temperature: 0.7, max_tokens: 1000 },
          usage: {
            input: 150,
            output: 75,
            total: 225,
            unit: "TOKENS",
            inputCost: 0.0015,
            outputCost: 0.003,
            totalCost: 0.0045,
          },
          usageDetails: { input: 150, output: 75, total: 225 },
          costDetails: { input: 0.0015, output: 0.003, total: 0.0045 },
        },
      },
    ],
    [
      "tool",
      "TOOL",
      {
        id: randomUUID(),
        type: "tool-create",
        timestamp: new Date().toISOString(),
        body: {
          id: randomUUID(),
          traceId: randomUUID(),
          startTime: new Date().toISOString(),
          endTime: new Date(Date.now() + 2000).toISOString(),
          name: "Web Search Tool",
          input: "Search for current weather",
          output: "Weather data retrieved",
          model: "gpt-4o-mini",
          usage: {
            input: 50,
            output: 25,
            total: 75,
            unit: "TOKENS",
            inputCost: 0.0001,
            outputCost: 0.0002,
            totalCost: 0.0003,
          },
        },
      },
    ],
    [
      "chain",
      "CHAIN",
      {
        id: randomUUID(),
        type: "chain-create",
        timestamp: new Date().toISOString(),
        body: {
          id: randomUUID(),
          traceId: randomUUID(),
          startTime: new Date().toISOString(),
          endTime: new Date(Date.now() + 3000).toISOString(),
          name: "Processing Chain",
          input: "Multi-step task",
          output: "All steps completed",
          model: "gpt-4",
          usageDetails: { input: 800, output: 400, total: 1200 },
          costDetails: { input: 0.024, output: 0.048, total: 0.072 },
        },
      },
    ],
    [
      "retriever",
      "RETRIEVER",
      {
        id: randomUUID(),
        type: "retriever-create",
        timestamp: new Date().toISOString(),
        body: {
          id: randomUUID(),
          traceId: randomUUID(),
          startTime: new Date().toISOString(),
          endTime: new Date(Date.now() + 1500).toISOString(),
          name: "Document Retriever",
          input: "Query document database",
          output: "Retrieved 5 relevant documents",
        },
      },
    ],
    [
      "evaluator",
      "EVALUATOR",
      {
        id: randomUUID(),
        type: "evaluator-create",
        timestamp: new Date().toISOString(),
        body: {
          id: randomUUID(),
          traceId: randomUUID(),
          startTime: new Date().toISOString(),
          endTime: new Date(Date.now() + 800).toISOString(),
          name: "Quality Evaluator",
          input: "Evaluate response quality",
          output: "Quality score: 0.85",
        },
      },
    ],
    [
      "embedding",
      "EMBEDDING",
      {
        id: randomUUID(),
        type: "embedding-create",
        timestamp: new Date().toISOString(),
        body: {
          id: randomUUID(),
          traceId: randomUUID(),
          startTime: new Date().toISOString(),
          endTime: new Date(Date.now() + 500).toISOString(),
          name: "Text Embedding",
          input: "Text to embed",
          output: "Embedding vector generated",
          model: "text-embedding-ada-002",
          usage: {
            input: 20,
            output: 0,
            total: 20,
            unit: "TOKENS",
            totalCost: 0.00004,
          },
        },
      },
    ],
    [
      "guardrail",
      "GUARDRAIL",
      {
        id: randomUUID(),
        type: "guardrail-create",
        timestamp: new Date().toISOString(),
        body: {
          id: randomUUID(),
          traceId: randomUUID(),
          startTime: new Date().toISOString(),
        },
      },
    ],
  ])(
    "should create observations via the ingestion API (%s)",
    async (_name: string, type: string, entity: any) => {
      const response = await postIngestion({
        batch: [entity],
      });

      expect(response.status).toBe(207);

      await waitForExpect(async () => {
        const observation = await getObservationById({
          id: entity.body.id,
          projectId,
          fetchWithInputOutput: true,
        });
        expect(observation).toBeDefined();
        expect(observation!.id).toBe(entity.body.id);
        expect(observation!.projectId).toBe(projectId);
        expect(observation!.metadata).toEqual(entity.body?.metadata ?? {});
        expect(observation!.input).toEqual(entity.body?.input ?? null);
        expect(observation!.output).toEqual(entity.body?.output ?? null);
        expect(observation!.type).toBe(type);
        expect(observation!.environment).toEqual(
          entity.body?.environment ?? "default",
        );
      }, 15_000);
    },
    20_000,
  );

  it.each([
    [
      "plain",
      {
        id: randomUUID(),
        type: "score-create",
        timestamp: new Date().toISOString(),
        body: {
          id: randomUUID(),
          name: "score-name",
          traceId: randomUUID(),
          value: 100.5,
          observationId: randomUUID(),
        },
      },
    ],
    [
      "non-default-environment",
      {
        id: randomUUID(),
        type: "score-create",
        timestamp: new Date().toISOString(),
        body: {
          id: randomUUID(),
          name: "score-name",
          traceId: randomUUID(),
          value: 100.5,
          observationId: randomUUID(),
          environment: "production",
        },
      },
    ],
  ])(
    "should create scores via the ingestion API (%s)",
    async (_name: string, entity: any) => {
      const response = await postIngestion({
        batch: [entity],
      });

      expect(response.status).toBe(207);

      await waitForExpect(async () => {
        const score = await getScoreById({
          projectId,
          scoreId: entity.body.id,
        });
        expect(score).toBeDefined();
        expect(score!.id).toBe(entity.body.id);
        expect(score!.projectId).toBe(projectId);
        expect(score!.value).toEqual(100.5);
        expect(score!.environment).toEqual(
          entity.body?.environment ?? "default",
        );
      });
    },
  );

  it("should persist ingestion attribution for scores", async () => {
    const timestamp = new Date().toISOString();
    const traceId = randomUUID();
    const observationId = randomUUID();
    const scoreId = randomUUID();

    const response = await postIngestion(
      {
        batch: [
          {
            id: randomUUID(),
            type: "generation-create",
            timestamp,
            body: {
              id: observationId,
              traceId,
              startTime: timestamp,
              model: "gpt-4",
              environment: "default",
            },
          },
          {
            id: randomUUID(),
            type: "score-create",
            timestamp,
            body: {
              id: scoreId,
              name: "score-name",
              traceId,
              observationId,
              value: 100.5,
              environment: "default",
            },
          },
        ],
      },
      {
        "x-langfuse-sdk-name": "python",
        "x-langfuse-sdk-version": "3.4.0",
      },
    );

    expect(response.status).toBe(207);

    await waitForExpect(async () => {
      const observation = await getObservationById({
        id: observationId,
        projectId,
      });
      expect(observation).toBeDefined();

      const score = await getScoreById({
        projectId,
        scoreId,
      });
      expect(score).toBeDefined();

      expect(await getIngestionAttribution("scores", scoreId)).toEqual({
        ingestion_api_key: publicKey,
        ingestion_sdk_name: "python",
        ingestion_sdk_version: "3.4.0",
      });
    }, 15_000);
  }, 20_000);

  it.each([
    "&",
    "$",
    "@",
    "=",
    ";",
    "/",
    "+",
    " ",
    ",",
    "?",
    // "\\",
    "{",
    "}",
    "^",
    "%",
    "`",
    "]",
    '"',
    ">",
    "[",
    "~",
    "<",
    "#",
    "|",
  ])(
    "should test special S3 characters in IDs (%s)",
    async (char: string) => {
      const traceId = randomUUID();

      const response = await postIngestion({
        batch: [
          {
            id: randomUUID(),
            type: "trace-create",
            timestamp: new Date().toISOString(),
            body: {
              id: `${traceId}-${char}-test`,
              timestamp: new Date().toISOString(),
            },
          },
        ],
      });

      expect(response.status).toBe(207);

      await waitForExpect(async () => {
        const trace = await getTraceById({
          traceId: `${traceId}-${char}-test`,
          projectId,
        });
        expect(trace).toBeDefined();
        expect(trace!.id).toBe(`${traceId}-${char}-test`);
        expect(trace!.projectId).toBe(projectId);
        expect(trace!.environment).toEqual("default");
      });
    },
    10000,
  );

  it("should fail for \\r in id", async () => {
    const traceId = v4();

    const response = await postIngestion({
      batch: [
        {
          id: `${v4()}-\r-test`,
          type: "trace-create",
          timestamp: new Date().toISOString(),
          body: {
            id: traceId,
            userId: "user-1",
            metadata: { key: "value" },
            release: "1.0.0",
            version: "2.0.0",
          },
        },
      ],
    });

    expect(response.status).toBe(207);
    expect("errors" in response.body).toBe(true);

    expect(response.body.errors.length).toBe(1);
    expect(response.body.errors[0].message).toBe("Invalid request data");
    expect(response.body.errors[0].error).toContain(
      "ID cannot contain carriage return characters",
    );
  });

  it("should fail for long trace name", async () => {
    const traceId = v4();

    const baseString =
      "Lorem ipsum dolor sit amet, consectetur adipiscing elit. ";
    const repeatCount = Math.ceil(1500 / baseString.length);
    const name = baseString.repeat(repeatCount);

    const response = await postIngestion({
      batch: [
        {
          id: v4(),
          type: "trace-create",
          timestamp: new Date().toISOString(),
          body: {
            id: traceId,
            name,
            userId: "user-1",
            metadata: { key: "value" },
            release: "1.0.0",
            version: "2.0.0",
          },
        },
      ],
    });

    expect(response.status).toBe(207);
    expect("errors" in response.body).toBe(true);

    expect(response.body.errors.length).toBe(1);
    expect(response.body.errors[0].message).toBe("Invalid request data");
  });

  it("should silently drop invalid float values in usageDetails", async () => {
    const response = await postIngestion({
      batch: [
        {
          id: v4(),
          type: "generation-create",
          timestamp: new Date().toISOString(),
          body: {
            id: randomUUID(),
            traceId: randomUUID(),
            parentObservationId: randomUUID(),
            startTime: new Date().toISOString(),
            model: "gpt-4",
            input: { text: "input" },
            output: { text: "output" },
            usageDetails: {
              key: 0.1, // invalid float - should be silently dropped
            },
          },
        },
      ],
    });

    // Event should succeed, invalid usageDetails values are silently dropped
    expect(response.status).toBe(207);
    expect(response.body.successes.length).toBe(1);
    expect(response.body.errors.length).toBe(0);
  });

  it.each([
    "langfuse-test",
    ".invalidcharacter!",
    "incrediblylongstringwithmorethan40characters",
  ])(
    "should fallback to default for invalid environments (%s)",
    async (environment: string) => {
      const entity = {
        id: randomUUID(),
        type: "score-create",
        timestamp: new Date().toISOString(),
        body: {
          id: randomUUID(),
          name: "score-name",
          traceId: randomUUID(),
          value: 100.5,
          observationId: randomUUID(),
          environment,
        },
      };

      const response = await postIngestion({
        batch: [entity],
      });

      expect(response.status).toBe(207);
      expect(response.body.successes.length).toBe(1);
      expect(response.body.errors.length).toBe(0);
    },
  );

  // Disabled until eventLog becomes the default behaviour.
  it("should create a log entry for the S3 file", async () => {
    const traceId = v4();
    const eventId = v4();

    const response = await postIngestion({
      batch: [
        {
          id: eventId,
          type: "trace-create",
          timestamp: new Date().toISOString(),
          body: {
            id: traceId,
            name: "Foo Bar",
            userId: "user-1",
            metadata: { key: "value" },
            release: "1.0.0",
            version: "2.0.0",
          },
        },
      ],
    });
    expect(response.status).toBe(207);

    await waitForExpect(async () => {
      const logs = await getBlobStorageByProjectAndEntityId(
        projectId,
        "trace",
        traceId,
      );
      expect(logs.length).toBeGreaterThan(0);
      expect(logs[0].bucket_path).toBe(
        `events/${projectId}/trace/${traceId}/${eventId}.json`,
      );
    });
  });

  it.each([
    ["string", { testId: "this is a string metadata" }],
    ["big-number", { testId: "1983516295378495150" }],
    ["small-number", { testId: 5 }],
    ["float-number", { testId: 5.5 }],
  ])(
    "#6123: should treat %s metadata for traces as such",
    async (_type, metadataValue) => {
      const traceId = randomUUID();

      const entity = {
        id: randomUUID(),
        type: "trace-create",
        timestamp: new Date().toISOString(),
        body: {
          id: traceId,
          timestamp: new Date().toISOString(),
          metadata: metadataValue,
        },
      };

      const response = await postIngestion({
        batch: [entity],
      });

      expect(response.status).toBe(207);

      await waitForExpect(async () => {
        const trace = await getTraceById({ traceId, projectId });
        expect(trace).toBeDefined();
        expect(trace!.id).toBe(traceId);
        expect(JSON.stringify(trace!.metadata)).toBe(
          JSON.stringify(metadataValue),
        );
      });
    },
    10000,
  );

  it.each([
    ["string", { testId: "this is a string metadata" }],
    ["big-number", { testId: "1983516295378495150" }],
    ["small-number", { testId: 5 }],
    ["float-number", { testId: 5.5 }],
  ])(
    "#6123: should treat %s metadata for observations as such",
    async (_type, metadataValue) => {
      const observationId = randomUUID();
      const traceId = randomUUID();

      const entity = {
        id: randomUUID(),
        type: "span-create",
        timestamp: new Date().toISOString(),
        body: {
          id: observationId,
          traceId: traceId,
          startTime: new Date().toISOString(),
          metadata: metadataValue,
        },
      };

      const response = await postIngestion({
        batch: [entity],
      });

      expect(response.status).toBe(207);

      await waitForExpect(async () => {
        const observation = await getObservationById({
          id: observationId,
          projectId,
          fetchWithInputOutput: true,
        });
        expect(observation).toBeDefined();
        expect(observation!.id).toBe(observationId);
        expect(JSON.stringify(observation!.metadata)).toBe(
          JSON.stringify(metadataValue),
        );
      });
    },
  );

  it.each([
    ["string", { testId: "this is a string metadata" }],
    ["big-number", { testId: "1983516295378495150" }],
    ["small-number", { testId: 5 }],
    ["float-number", { testId: 5.5 }],
  ])(
    "#6123: should treat %s metadata for scores as such",
    async (_type, metadataValue) => {
      const scoreId = randomUUID();
      const traceId = randomUUID();

      const entity = {
        id: randomUUID(),
        type: "score-create",
        timestamp: new Date().toISOString(),
        body: {
          id: scoreId,
          name: "score-name",
          traceId: traceId,
          value: 100.5,
          metadata: metadataValue,
        },
      };

      const response = await postIngestion({
        batch: [entity],
      });

      expect(response.status).toBe(207);

      await waitForExpect(async () => {
        const score = await getScoreById({ projectId, scoreId });
        expect(score).toBeDefined();
        expect(score!.id).toBe(scoreId);
        expect(JSON.stringify(score!.metadata)).toBe(
          JSON.stringify(metadataValue),
        );
      });
    },
  );

  it("should merge metadata correctly across multiple trace updates", async () => {
    const traceId = randomUUID();

    // First update with initial metadata: {"step": 1, "status": "started"}
    const traceUpdate1 = {
      id: randomUUID(),
      type: "trace-create",
      timestamp: new Date().toISOString(),
      body: {
        id: traceId,
        name: "operation",
        timestamp: new Date().toISOString(),
        metadata: { step: 1, status: "started" },
      },
    };

    // Second update with additional metadata: {"step": 2, "error": ""}
    // This should merge with the first update
    const traceUpdate2 = {
      id: randomUUID(),
      type: "trace-create",
      timestamp: new Date(Date.now() + 1000).toISOString(), // Later timestamp
      body: {
        id: traceId,
        name: "operation",
        timestamp: new Date(Date.now() + 1000).toISOString(),
        metadata: { step: 2, error: "" },
      },
    };

    const response1 = await postIngestion({
      batch: [traceUpdate1],
    });
    expect(response1.status).toBe(207);

    await waitForExpect(async () => {
      const trace = await getTraceById({ traceId, projectId });
      expect(trace?.metadata).toEqual({
        step: 1,
        status: "started",
      });
    }, 15_000);

    const response2 = await postIngestion({
      batch: [traceUpdate2],
    });
    expect(response2.status).toBe(207);

    await waitForExpect(async () => {
      const trace = await getTraceById({ traceId, projectId });
      expect(trace).toBeDefined();
      expect(trace!.id).toBe(traceId);
      expect(trace!.projectId).toBe(projectId);

      // Expected final metadata: {"step": 2, "status": "started", "error": ""}
      // This verifies that:
      // - "step" is updated to the latest value (2)
      // - "status" is preserved from the first update ("started")
      // - "error" is added from the second update ("")
      expect(trace!.metadata).toEqual({
        step: 2,
        status: "started",
        error: "",
      });
    });
  }, 20000);

  it("#4900: should clear score comment on update with `null`", async () => {
    const scoreId = randomUUID();
    const score1 = {
      id: randomUUID(),
      type: "score-create",
      timestamp: new Date().toISOString(),
      body: {
        id: scoreId,
        name: "score-name",
        traceId: randomUUID(),
        value: 100.5,
        observationId: randomUUID(),
        comment: "Foo Bar",
      },
    };

    const score2 = {
      id: randomUUID(),
      type: "score-create",
      timestamp: new Date(Date.now() + 1000).toISOString(),
      body: {
        id: scoreId,
        name: "score-name",
        traceId: randomUUID(),
        value: 100.5,
        observationId: randomUUID(),
        comment: null, // Explicitly set to null to clear the comment
      },
    };

    const response = await postIngestion({
      batch: [score1, score2],
    });

    expect(response.status).toBe(207);

    await waitForExpect(async () => {
      const score = await getScoreById({ projectId, scoreId });
      expect(score).toBeDefined();
      expect(score!.id).toBe(scoreId);
      expect(score!.projectId).toBe(projectId);
      expect(score!.value).toEqual(100.5);
      expect(score!.comment).toBe(null);
    });
  }, 10000);
});