Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

14.9 KB
Raw Blame
import { beforeEach, expect, it, vi } from "vitest";
import { assembleTranscript } from "@langfuse/shared/src/server/transcript/transcript";
import { loadTraceTranscript } from "@/src/features/events/server/loadTraceTranscript";

const mocks = vi.hoisted(() => ({
  assemble: vi.fn(),
  observations: vi.fn(),
  root: vi.fn(),
}));
vi.mock("@langfuse/shared/src/server", async () => {
  const { normalizeSpanIO } =
    await import("@langfuse/shared/src/utils/normalized-io");
  return {
    MAX_OBSERVATIONS_PER_TRACE: 10000,
    orderObservations: (observations: unknown[]) => observations,
    assembleTranscript: mocks.assemble,
    getObservationsForTraceFromEventsTable: mocks.observations,
    getObservationByIdFromEventsTable: mocks.root,
    normalizeIO: ({ io }: { io: Parameters<typeof normalizeSpanIO>[0] }) =>
      normalizeSpanIO(io),
  };
});
const trace = {
  projectId: "project",
  traceId: "trace",
  timestamp: new Date(0),
};
beforeEach(() => {
  vi.clearAllMocks();
  mocks.assemble.mockReset();
  mocks.observations.mockReset();
  mocks.root.mockReset();
});

it.each(
  ["ERROR", "WARNING"].flatMap((level) =>
    [null, "", []].map((output) => ({ level, output })),
  ),
)(
  "retains tool status with empty output without metadata recovery: %j",
  async ({ level, output }) => {
    const generation = {
      id: "generation",
      traceId: "trace",
      type: "GENERATION" as const,
      name: "generation",
      parentObservationId: null,
      nestingLevel: 0,
      startTime: new Date(0),
      endTime: new Date(1),
      input: [{ role: "user", content: "Run a tool" }],
      output: {
        role: "assistant",
        tool_calls: [
          {
            id: "call-1",
            type: "function",
            function: { name: "search", arguments: "{}" },
          },
        ],
      },
      metadata: {},
    };
    const tool = {
      ...generation,
      id: "tool",
      type: "TOOL" as const,
      name: "search",
      parentObservationId: generation.id,
      startTime: new Date(2),
      input: {},
      output,
      level,
      statusMessage: "Search unavailable",
      metadata: { callID: "conflicting" },
    };
    const observations = [generation, tool];
    const snapshot = structuredClone(observations);
    mocks.observations.mockResolvedValue({ observations, totalCount: 2 });
    mocks.assemble.mockImplementation(assembleTranscript);
    const result = await loadTraceTranscript(trace);
    expect(
      result.transcript?.threads[0]?.currentTurn.messages.find(
        (message) => message.observationId === "tool",
      ),
    ).toMatchObject({
      role: "tool",
      level,
      statusMessage: tool.statusMessage,
      parts: [
        {
          type: "tool-result",
          toolCallId: "call-1",
          output: { type: "json", value: null },
          isError: level === "ERROR",
        },
      ],
    });
    expect(observations).toEqual(snapshot);
  },
);

it("matches parallel same-name TOOL results by payload ID, not metadata", async () => {
  const generation = {
    id: "generation",
    traceId: "trace",
    type: "GENERATION" as const,
    name: "generation",
    parentObservationId: null,
    nestingLevel: 0,
    startTime: new Date(0),
    endTime: new Date(1),
    input: [{ role: "user", content: "Run tools" }],
    output: {
      role: "assistant",
      tool_calls: ["call-a", "call-b"].map((id) => ({
        id,
        type: "function",
        function: { name: "search", arguments: "{}" },
      })),
    },
    metadata: {},
  };
  const tools = ["call-b", "call-a"].map((id, index) => ({
    ...generation,
    id: `tool-${index}`,
    type: "TOOL" as const,
    name: "search",
    parentObservationId: generation.id,
    startTime: new Date(index + 2),
    input: {},
    output: { role: "tool", tool_call_id: id, content: `result-${id}` },
    metadata: { callID: "conflicting", toolCallId: "conflicting" },
  }));
  const observations = [generation, ...tools];
  const snapshot = structuredClone(observations);
  mocks.observations.mockResolvedValue({ observations, totalCount: 3 });
  mocks.assemble.mockImplementation(assembleTranscript);

  const result = await loadTraceTranscript(trace);
  const messages = result.transcript!.threads[0]!.currentTurn.messages;
  expect(messages.filter((message) => message.role === "tool")).toHaveLength(2);
  for (const tool of tools) {
    expect(
      messages.find((message) => message.observationId === tool.id),
    ).toMatchObject({
      parts: [
        {
          type: "tool-result",
          toolCallId: tool.output.tool_call_id,
          output: { type: "text", value: tool.output.content },
        },
      ],
    });
  }
  expect(observations).toEqual(snapshot);
});

it.each(["callID", "toolCallId"])(
  "does not synthesize generation calls from metadata.%s",
  async (metadataKey) => {
    const generation = {
      id: "generation",
      traceId: "trace",
      type: "GENERATION" as const,
      name: "generation",
      parentObservationId: null,
      nestingLevel: 0,
      startTime: new Date(0),
      endTime: new Date(1),
      input: "Question",
      output: { answer: 42 },
      metadata: {},
    };
    const observations = [
      generation,
      {
        ...generation,
        id: "tool",
        type: "TOOL" as const,
        name: "search",
        parentObservationId: generation.id,
        output: { result: 42 },
        metadata: { [metadataKey]: "call-1" },
      },
    ];
    mocks.observations.mockResolvedValue({ observations, totalCount: 2 });
    mocks.assemble.mockImplementation(assembleTranscript);

    const result = await loadTraceTranscript(trace);
    expect(result.transcript).toEqual(assembleTranscript([generation]));
  },
);

it.each([
  { structureCount: 10000, contentCount: 1000, cutoff: false },
  { structureCount: 10001, contentCount: 1000, cutoff: true },
  { structureCount: 10000, contentCount: 1001, cutoff: true },
])(
  "merges capped I/O with structure and reports cutoff: %j",
  async ({ structureCount, contentCount, cutoff }) => {
    const span = { id: "span", type: "SPAN" };
    const generation = { id: "generation", type: "GENERATION" };
    const tool = { id: "tool", type: "TOOL" };
    const generationWithContent = {
      ...generation,
      input: "question",
      output: "answer",
    };
    mocks.observations
      .mockResolvedValueOnce({
        observations: [span, generation, tool],
        totalCount: structureCount,
      })
      .mockResolvedValueOnce({
        observations: [generationWithContent],
        totalCount: contentCount,
      });

    const result = await loadTraceTranscript(trace);

    expect(mocks.assemble).toHaveBeenCalledWith([span, generationWithContent]);
    expect(result.cutoff).toBe(cutoff);
    expect(mocks.observations).toHaveBeenNthCalledWith(1, trace);
    expect(mocks.observations).toHaveBeenNthCalledWith(2, {
      ...trace,
      selectIOAndMetadata: true,
      types: ["GENERATION", "TOOL"],
      limit: 1000,
    });
  },
);

it.each([
  { parentObservationId: null, isRootObservation: true },
  { parentObservationId: "", isRootObservation: undefined },
  { parentObservationId: "external-parent", isRootObservation: true },
])(
  "falls back to normalized root input/output with root provenance: %j",
  async ({ parentObservationId, isRootObservation }) => {
    const root = {
      id: "root",
      traceId: "trace",
      parentObservationId,
      isRootObservation,
      type: "SPAN",
      name: "Root",
      startTime: new Date(0),
      endTime: new Date(1000),
      input: "Question",
      output: "Answer",
      metadata: {},
    };
    mocks.assemble.mockReturnValue(null);
    mocks.observations.mockResolvedValue({
      observations: [root],
      totalCount: 1,
    });
    mocks.root.mockResolvedValue(root);

    const result = await loadTraceTranscript({
      ...trace,
      fallbackToRootIO: true,
    });

    expect(mocks.root).toHaveBeenCalledWith({
      projectId: "project",
      traceId: "trace",
      id: "root",
      startTime: new Date(0),
      fetchWithInputOutput: true,
    });
    expect(result.cutoff).toBe(false);
    expect(result.transcript?.threads).toHaveLength(1);
    expect(result.transcript?.threads[0]?.currentTurn).toMatchObject({
      nestingLevel: 0,
      observations: [{ id: "root", traceId: "trace" }],
      messages: [
        {
          role: "user",
          source: "input",
          parts: [{ type: "text", text: "Question" }],
          observationId: "root",
          traceId: "trace",
          startTime: root.startTime,
          endTime: root.endTime,
        },
        {
          role: "assistant",
          source: "output",
          parts: [{ type: "text", text: "Answer" }],
          observationId: "root",
          traceId: "trace",
          startTime: root.startTime,
          endTime: root.endTime,
        },
      ],
    });
  },
);

it("falls back to seeded JSON chat messages with an empty-string root parent", async () => {
  const root = {
    id: "t-seed-media-image-only",
    traceId: "seed-media-image-only",
    parentObservationId: "",
    startTime: new Date(0),
    endTime: null,
    input: JSON.stringify([
      {
        role: "user",
        content: [
          { type: "text", text: "Please analyze the seeded image attachment." },
          {
            type: "image_url",
            image_url: {
              url: "@@@langfuseMedia:type=image/png|id=seed-image|source=base64_data_uri@@@",
            },
          },
        ],
      },
    ]),
    output: JSON.stringify([{ role: "assistant", content: "Image analysis" }]),
    metadata: "{}",
  };
  mocks.assemble.mockReturnValue(null);
  mocks.observations.mockResolvedValue({ observations: [root], totalCount: 1 });
  mocks.root.mockResolvedValue(root);

  const result = await loadTraceTranscript({
    ...trace,
    traceId: root.traceId,
    fallbackToRootIO: true,
  });
  expect(result.transcript?.threads[0]?.currentTurn.messages).toMatchObject([
    {
      observationId: root.id,
      role: "user",
      parts: expect.arrayContaining([
        { type: "text", text: "Please analyze the seeded image attachment." },
      ]),
    },
    {
      observationId: root.id,
      role: "assistant",
      parts: [{ type: "text", text: "Image analysis" }],
    },
  ]);
});

it("returns the shared transcript unchanged without fetching root I/O", async () => {
  const original = { threads: [] };
  mocks.assemble.mockReturnValue(original);
  mocks.observations.mockResolvedValue({ observations: [], totalCount: 0 });
  const result = await loadTraceTranscript({
    ...trace,
    fallbackToRootIO: true,
  });
  expect(result.transcript).toBe(original);
  expect(mocks.root).not.toHaveBeenCalled();
});

it("leaves an empty transcript unchanged unless fallback is enabled", async () => {
  mocks.assemble.mockReturnValue(null);
  mocks.observations.mockResolvedValue({
    observations: [{ id: "root", parentObservationId: null }],
    totalCount: 1,
  });
  expect((await loadTraceTranscript(trace)).transcript).toBeNull();
  expect(mocks.root).not.toHaveBeenCalled();
});

it("leaves the transcript empty when no root was found", async () => {
  mocks.assemble.mockReturnValue(null);
  mocks.observations.mockResolvedValue({ observations: [], totalCount: 0 });
  expect(
    (await loadTraceTranscript({ ...trace, fallbackToRootIO: true }))
      .transcript,
  ).toBeNull();
  expect(mocks.root).not.toHaveBeenCalled();
});

it("leaves the transcript empty when the root has no input/output", async () => {
  mocks.assemble.mockReturnValue(null);
  mocks.observations.mockResolvedValue({
    observations: [
      { id: "root", parentObservationId: null, startTime: new Date(0) },
    ],
    totalCount: 1,
  });
  mocks.root.mockResolvedValue({ input: null, output: null, metadata: {} });
  expect(
    (await loadTraceTranscript({ ...trace, fallbackToRootIO: true }))
      .transcript,
  ).toBeNull();
});

it("loads and normalizes matching span I/O without assembly or ancestors", async () => {
  const message = {
    role: "assistant" as const,
    source: "output" as const,
    parts: [{ type: "text" as const, text: "Response" }],
    observationId: "match",
    traceId: "trace",
    startTime: new Date(0),
    endTime: new Date(1),
  };
  mocks.observations.mockResolvedValue({
    observations: [
      {
        id: "match",
        type: "SPAN",
        startTime: new Date(0),
        endTime: new Date(1),
        input: null,
        output: "Response",
        metadata: {},
      },
    ],
    totalCount: 1,
  });
  const filter = [
    {
      column: "name",
      type: "string" as const,
      operator: "=" as const,
      value: "child",
    },
  ];
  const result = await loadTraceTranscript({ ...trace, filter });
  expect(mocks.observations).toHaveBeenCalledExactlyOnceWith({
    ...trace,
    filter,
    selectIOAndMetadata: true,
    limit: 1000,
  });
  expect(mocks.assemble).not.toHaveBeenCalled();
  expect(result.transcript?.threads).toEqual([
    {
      conversationHistory: [],
      currentTurn: {
        nestingLevel: 0,
        observations: [{ id: "match", traceId: "trace" }],
        messages: [message],
      },
    },
  ]);
});

it("does not restore unmatched root I/O when no observations match", async () => {
  mocks.assemble.mockReturnValue(null);
  mocks.observations.mockResolvedValue({ observations: [], totalCount: 0 });
  const result = await loadTraceTranscript({
    ...trace,
    fallbackToRootIO: true,
    filter: [
      { column: "level", type: "string", operator: "=", value: "ERROR" },
    ],
  });
  expect(result.transcript).toBeNull();
  expect(mocks.root).not.toHaveBeenCalled();
});

it("bypasses assembly for trace-only filters and keeps each observation's full I/O in chronological order", async () => {
  const observations = [2, 1].map((index) => ({
    id: `span-${index}`,
    type: "SPAN",
    startTime: new Date(index),
    endTime: new Date(index + 1),
    input: [{ role: "user", content: "Repeated history" }],
    output: `Answer ${index}`,
    metadata: {},
  }));
  mocks.observations.mockResolvedValue({ observations, totalCount: 1001 });
  const result = await loadTraceTranscript({
    ...trace,
    fallbackToRootIO: true,
    filter: [
      { column: "traceName", type: "string", operator: "=", value: "chat" },
    ],
  });
  expect(mocks.observations).toHaveBeenCalledExactlyOnceWith({
    ...trace,
    fallbackToRootIO: true,
    filter: [],
    selectIOAndMetadata: true,
    limit: 1000,
  });
  expect(mocks.assemble).not.toHaveBeenCalled();
  expect(mocks.root).not.toHaveBeenCalled();
  expect(result.cutoff).toBe(true);
  expect(
    result.transcript?.threads.map((thread) => ({
      history: thread.conversationHistory,
      id: thread.currentTurn.observations[0]?.id,
      messages: thread.currentTurn.messages.map((message) => message.parts),
    })),
  ).toEqual(
    [1, 2].map((index) => ({
      history: [],
      id: `span-${index}`,
      messages: [
        [{ type: "text", text: "Repeated history" }],
        [{ type: "text", text: `Answer ${index}` }],
      ],
    })),
  );
});