Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

9.2 KB
Raw Blame
import { v4 } from "uuid";
import {
  createObservationsCh,
  createTracesCh,
  createObservation,
  createTrace,
  getTracesTable,
  getTracesTableCount,
  type TracesTableUiReturnType,
  type ObservationRecordInsertType,
  type TraceRecordInsertType,
} from "@langfuse/shared/src/server";
import { type FilterState, type TracingSearchType } from "@langfuse/shared";

describe("Traces table API test", () => {
  it("should get a correct trace without observation", async () => {
    const project_id = v4();
    const trace_id = v4();

    const trace = createTrace({ id: trace_id, project_id });
    await createTracesCh([trace]);

    const tableRows = await getTracesTable({
      projectId: project_id,
      filter: [],
      searchQuery: undefined,
      orderBy: undefined,
      limit: 1,
      page: 0,
    });

    expect(tableRows).toHaveLength(1);
    expect(tableRows[0].id).toEqual(trace_id);
    expect(tableRows[0].projectId).toEqual(project_id);
    expect(tableRows[0].tags.sort()).toEqual(trace.tags.sort());
    expect(tableRows[0].name).toEqual(trace.name);
    expect(tableRows[0].bookmarked).toEqual(trace.bookmarked);
    expect(tableRows[0].release).toEqual(trace.release);
    expect(tableRows[0].version).toEqual(trace.version);
    expect(tableRows[0].userId).toEqual(trace.user_id);
    expect(tableRows[0].sessionId).toEqual(trace.session_id);
    expect(tableRows[0].public).toEqual(trace.public);
  });

  it("#5274: should get a traces in expected default order", async () => {
    const project_id = v4();
    const trace_id = v4();

    // Trace1 happened after Trace2, but Trace2 got updated.
    const trace1 = createTrace({
      id: `${trace_id}-1`,
      project_id,
      timestamp: new Date().getTime(),
      event_ts: new Date().getTime(),
    });
    const trace2 = createTrace({
      id: `${trace_id}-2`,
      project_id,
      timestamp: new Date().getTime() - 5000,
      event_ts: new Date().getTime() + 5000,
    });
    await createTracesCh([trace1, trace2]);

    const tableRows = await getTracesTable({
      projectId: project_id,
      filter: [],
      searchQuery: undefined,
      orderBy: { column: "timestamp", order: "DESC" },
      limit: 2,
      page: 0,
    });

    expect(tableRows).toHaveLength(2);
    expect(tableRows[0].id).toEqual(`${trace_id}-1`);
    expect(tableRows[1].id).toEqual(`${trace_id}-2`);
  });

  it("should get a correct trace with observations", async () => {
    const project_id = v4();
    const trace_id = v4();

    const trace = createTrace({ id: trace_id, project_id });
    await createTracesCh([trace]);

    const obs1 = createObservation({ trace_id, project_id });
    const obs2 = createObservation({ trace_id, project_id });
    await createObservationsCh([obs1, obs2]);

    const tableRows = await getTracesTable({
      projectId: project_id,
      filter: [],
      searchQuery: undefined,
      orderBy: undefined,
      limit: 1,
      page: 0,
    });

    expect(tableRows).toHaveLength(1);
    expect(tableRows[0].id).toEqual(trace_id);
    expect(tableRows[0].projectId).toEqual(project_id);
    expect(tableRows[0].tags.sort()).toEqual(trace.tags.sort());
    expect(tableRows[0].name).toEqual(trace.name);
    expect(tableRows[0].bookmarked).toEqual(trace.bookmarked);
    expect(tableRows[0].release).toEqual(trace.release);
    expect(tableRows[0].version).toEqual(trace.version);
    expect(tableRows[0].userId).toEqual(trace.user_id);
    expect(tableRows[0].sessionId).toEqual(trace.session_id);
    expect(tableRows[0].public).toEqual(trace.public);
  });

  type TestCase = {
    traceInput: Partial<TraceRecordInsertType>;
    observationInput: Partial<ObservationRecordInsertType>[];
    filterstate: FilterState;
    expected: Partial<TracesTableUiReturnType>[];
  };

  [
    {
      traceInput: {},
      observationInput: [
        { cost_details: { total: 100 } },
        { cost_details: { total: 200 } },
      ],
      filterstate: [
        {
          column: "totalCost",
          operator: ">" as const,
          value: 100000,
          type: "number" as const,
        },
      ],
      expected: [],
    },
    {
      traceInput: {},
      observationInput: [
        { usage_details: { total: 100 } },
        { usage_details: { total: 200 } },
      ],
      filterstate: [
        {
          column: "totalTokens",
          operator: ">" as const,
          value: 3456789,
          type: "number" as const,
        },
      ],
      expected: [],
    },
    {
      traceInput: {},
      observationInput: [],
      filterstate: [
        {
          column: "id",
          operator: "=" as const,
          value: "some-id",
          type: "string" as const,
        },
      ],
      expected: [],
    },
    {
      traceInput: {},
      observationInput: [],
      filterstate: [
        {
          column: "Latency (s)",
          operator: ">" as const,
          value: 5_000_000, // Verify that we can pass large values
          type: "number" as const,
        },
      ],
      expected: [],
    },
  ].forEach(async (testConfig: TestCase) => {
    it(`should get a correct trace with filters ${JSON.stringify(testConfig)}`, async () => {
      const project_id = v4();
      const trace_id = v4();

      const trace = createTrace({
        id: trace_id,
        project_id,
        ...testConfig.traceInput,
      });
      await createTracesCh([trace]);

      expect(testConfig.observationInput.length).not.toBeGreaterThan(2);

      const obs1 = createObservation({
        trace_id,
        project_id,
        ...(testConfig.observationInput.length > 0
          ? testConfig.observationInput[0]
          : {}),
      });
      const obs2 = createObservation({
        trace_id,
        project_id,
        ...(testConfig.observationInput.length > 1
          ? testConfig.observationInput[1]
          : {}),
      });
      await createObservationsCh([obs1, obs2]);

      const tableRows = await getTracesTable({
        projectId: project_id,
        filter: testConfig.filterstate,
        searchQuery: undefined,
        orderBy: undefined,
        limit: 1,
        page: 0,
      });

      expect(tableRows).toHaveLength(testConfig.expected.length);
      testConfig.expected.forEach((expectedTrace, index) => {
        if (expectedTrace.id !== undefined) {
          expect(tableRows[index].id).toEqual(expectedTrace.id);
        }
        if (expectedTrace.projectId !== undefined) {
          expect(tableRows[index].projectId).toEqual(expectedTrace.projectId);
        }
        if (expectedTrace.tags !== undefined) {
          expect(tableRows[index].tags).toEqual(expectedTrace.tags);
        }
        if (expectedTrace.name !== undefined) {
          expect(tableRows[index].name).toEqual(expectedTrace.name);
        }
        if (expectedTrace.bookmarked !== undefined) {
          expect(tableRows[index].bookmarked).toEqual(expectedTrace.bookmarked);
        }
        if (expectedTrace.release !== undefined) {
          expect(tableRows[index].release).toEqual(expectedTrace.release);
        }
        if (expectedTrace.version !== undefined) {
          expect(tableRows[index].version).toEqual(expectedTrace.version);
        }
        if (expectedTrace.userId !== undefined) {
          expect(tableRows[index].userId).toEqual(expectedTrace.userId);
        }
        if (expectedTrace.sessionId !== undefined) {
          expect(tableRows[index].sessionId).toEqual(expectedTrace.sessionId);
        }
        if (expectedTrace.public !== undefined) {
          expect(tableRows[index].public).toEqual(expectedTrace.public);
        }
      });
    });
  });

  it.each(["ASC", "DESC"] as const)(
    "pages past the first page with a metadata filter and content search (%s)",
    async (order) => {
      const project_id = v4();
      const documentId = `doc-${v4()}`;
      const needle = `needle-${v4()}`;
      const traces = Array.from({ length: 12 }, (_, i) =>
        createTrace({
          id: `trace-${String(i).padStart(2, "0")}-${v4()}`,
          project_id,
          timestamp: Date.now() - i * 1_000,
          metadata: { documentId },
          input: `${needle} payload ${i}`,
        }),
      );
      await createTracesCh(traces);

      const filter: FilterState = [
        {
          column: "metadata",
          type: "stringObject",
          key: "documentId",
          operator: "=",
          value: documentId,
        },
      ];
      const searchType: TracingSearchType[] = ["content"];
      const query = {
        projectId: project_id,
        filter,
        searchQuery: needle,
        searchType,
        orderBy: { column: "timestamp" as const, order },
        limit: 5,
      };

      const [page0, page1, totalCount] = await Promise.all([
        getTracesTable({ ...query, page: 0 }),
        getTracesTable({ ...query, page: 1 }),
        // Count matches the UI: orderBy is always null so the aggregate
        // is not wrapped in a timestamp ORDER BY.
        getTracesTableCount({
          projectId: project_id,
          filter,
          searchQuery: needle,
          searchType: ["content"],
        }),
      ]);

      expect(totalCount).toBe(12);
      expect(page0).toHaveLength(5);
      expect(page1).toHaveLength(5);
      const page0Ids = new Set(page0.map((row) => row.id));
      expect(page1.every((row) => !page0Ids.has(row.id))).toBe(true);
    },
  );
});