Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

75.7 KB
Raw Blame
import {
  afterAll,
  afterEach,
  beforeAll,
  beforeEach,
  describe,
  expect,
  it,
  vi,
} from "vitest";
import { type Job } from "bullmq";
import { context, SpanStatusCode, trace } from "@opentelemetry/api";
import { NodeSDK, tracing } from "@opentelemetry/sdk-node";
import {
  getCurrentSpan,
  convertObservation,
  createObservation,
  getTraceBatchEventStream,
  logger,
  QueueJobs,
  TraceBatchQueue,
  type QueueName,
  recordDistribution,
  recordGauge,
  recordIncrement,
  type TraceBatchInputStorage,
  type TQueueJobTypes,
} from "@langfuse/shared/src/server";
import { env } from "../../env";
import { tokenCountAsync } from "../../features/tokenisation/async-usage";
import { tokenCount } from "../../features/tokenisation/usage";
import { TopicsProviderUnavailable } from "../../features/topics/provider-error";
import { summarizeAssembledTrace } from "../../features/topics/summarizeAssembledTrace";
import { enqueueTopicsAutomation } from "../../features/topics/topicsAutomation";
import {
  getEnabledTopicsModels,
  listTopicFacets,
  listTopicRules,
  isValidTopicSummary,
  pauseTopicsModels,
  writeTopicSummaries,
  type TopicsModels,
} from "@langfuse/shared/topics/server";
import type {
  TopicFacet,
  TopicRule,
  TopicSummary,
} from "@langfuse/shared/topics";
import { recordTraceBatchTranscript } from "../../features/traceBatching/traceBatchTranscript";
import * as traceBatchTranscript from "../../features/traceBatching/traceBatchTranscript";
import {
  recordTraceBatchActiveReads,
  traceBatchQueueProcessor,
} from "../traceBatchQueue";

const inputStorage = vi.hoisted(() => ({
  put: vi.fn<TraceBatchInputStorage["put"]>(),
  get: vi.fn<TraceBatchInputStorage["get"]>(),
}));

vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({
  ...(await importOriginal<typeof import("@langfuse/shared/src/server")>()),
  getTraceBatchEventStream: vi.fn(),
  getCurrentSpan: vi.fn(),
  recordDistribution: vi.fn(),
  recordGauge: vi.fn(),
  recordIncrement: vi.fn(),
  createTraceBatchInputStorage: () => inputStorage,
}));

vi.mock("../../features/topics/summarizeAssembledTrace", () => ({
  summarizeAssembledTrace: vi.fn(async () => ({
    outcome: "unchanged",
    rows: [],
  })),
}));

vi.mock("../../features/topics/topicsAutomation", () => ({
  enqueueTopicsAutomation: vi.fn(async () => {}),
}));

vi.mock("@langfuse/shared/topics/server", () => ({
  isTopicsProjectEnabled: vi.fn(() => true),
  getEnabledTopicsModels: vi.fn(
    async (projectIds: string[]) =>
      new Map(projectIds.map((projectId) => [projectId, { projectId }])),
  ),
  listTopicFacets: vi.fn(async () => []),
  listTopicRules: vi.fn(async () => []),
  pauseTopicsModels: vi.fn(),
  writeTopicSummaries: vi.fn(),
  isValidTopicSummary: vi.fn(() => true),
}));

// Exercise real tokenization without starting the compiled worker-thread pool.
vi.mock("../../features/tokenisation/async-usage", () => ({
  tokenCountAsync: vi.fn(async (params) => tokenCount(params)),
}));

const originalReadEnabled = env.LANGFUSE_TRACE_BATCH_READ_ENABLED;
const originalTranscriptMetricsEnabled =
  env.LANGFUSE_TRACE_BATCH_TRANSCRIPT_METRICS_ENABLED;
const originalTranscriptStorageEnabled =
  env.LANGFUSE_TRACE_BATCH_TRANSCRIPT_STORAGE_ENABLED;
const originalCloudRegion = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;
const originalExperimentId = env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID;
const processingSpan = trace.wrapSpanContext({
  traceId: "0123456789abcdef0123456789abcdef",
  spanId: "0123456789abcdef",
  traceFlags: 1,
});
const exporter = new tracing.InMemorySpanExporter();
const sdk = new NodeSDK({
  spanProcessors: [new tracing.SimpleSpanProcessor(exporter)],
  autoDetectResources: false,
  instrumentations: [],
});
beforeAll(() => sdk.start());
afterAll(async () => {
  await sdk.shutdown();
  trace.disable();
  context.disable();
});
beforeEach(() => {
  exporter.reset();
  env.LANGFUSE_TRACE_BATCH_READ_ENABLED = "true";
  env.LANGFUSE_TRACE_BATCH_TRANSCRIPT_STORAGE_ENABLED = "false";
  inputStorage.put.mockResolvedValue({
    outcome: "stored",
    jsonLength: 100,
  });
  env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "DEV";
  vi.mocked(getCurrentSpan).mockReturnValue(processingSpan);
  vi.spyOn(processingSpan, "setAttributes");
});
afterEach(() => {
  env.LANGFUSE_TRACE_BATCH_READ_ENABLED = originalReadEnabled;
  env.LANGFUSE_TRACE_BATCH_TRANSCRIPT_METRICS_ENABLED =
    originalTranscriptMetricsEnabled;
  env.LANGFUSE_TRACE_BATCH_TRANSCRIPT_STORAGE_ENABLED =
    originalTranscriptStorageEnabled;
  env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = originalCloudRegion;
  env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID = originalExperimentId;
  vi.restoreAllMocks();
  vi.clearAllMocks();
});

describe("trace batch queue", () => {
  it("measures rendered Topics text for an admitted trace", async () => {
    const observations = [
      convertObservation(
        createObservation({
          type: "GENERATION",
          name: "explain",
          trace_id: "topics-sample",
          input: JSON.stringify([
            { role: "user", content: "Explain the result" },
          ]),
          output: JSON.stringify({ role: "assistant", content: "The answer" }),
        }),
      ),
    ];

    await recordTraceBatchTranscript(observations);

    expect(tokenCountAsync).toHaveBeenCalledTimes(4);
    const baselineJson = vi.mocked(tokenCountAsync).mock.calls[1][0]
      .text as string;
    expect(
      JSON.parse(baselineJson).threads[0].currentTurn.observations,
    ).toEqual([expect.objectContaining({ traceId: "topics-sample" })]);
    const genericText = vi.mocked(tokenCountAsync).mock.calls[2][0].text;
    expect(genericText).toBe(
      "[user] Explain the result\n[assistant] The answer",
    );
    const renderedText = vi.mocked(tokenCountAsync).mock.calls[3][0].text;
    expect(renderedText).toBe(
      [
        "<run_facts>",
        "threads: 1 · rendered user entries: 1 · rendered tool calls: 0 · error signals: 0 · omitted lines: 0",
        "</run_facts>",
        "<this_run>",
        "[generation] explain",
        "[user · request] Explain the result",
        "[assistant · final output] The answer",
        "</this_run>",
        "<end_of_run>",
        "last action: assistant text",
        "final output: assistant",
        "</end_of_run>",
      ].join("\n"),
    );
    expect(recordDistribution).not.toHaveBeenCalled();
    expect(exporter.getFinishedSpans()[0].attributes).toMatchObject({
      "langfuse.trace_batch.transcript_json_characters": baselineJson.length,
      "langfuse.trace_batch.transcript_json_tokens": tokenCount({
        model: vi.mocked(tokenCountAsync).mock.calls[1][0].model,
        text: baselineJson,
      }),
      "langfuse.trace_batch.generic_transcript_characters": genericText.length,
      "langfuse.trace_batch.generic_transcript_tokens": tokenCount({
        model: vi.mocked(tokenCountAsync).mock.calls[2][0].model,
        text: genericText,
      }),
      "langfuse.trace_batch.topics_transcript_characters": renderedText.length,
      "langfuse.trace_batch.topics_transcript_blocks_cut": 0,
      "langfuse.trace_batch.topics_transcript_history_share": 0,
      "langfuse.trace_batch.topics_transcript_tokens": tokenCount({
        model: vi.mocked(tokenCountAsync).mock.calls[3][0].model,
        text: renderedText,
      }),
      "langfuse.trace_batch.topics_transcript_block_user_raw_characters":
        "Explain the result".length,
      "langfuse.trace_batch.topics_transcript_block_observations_raw_characters":
        "explain".length,
      "langfuse.trace_batch.transcript_observation_count": 1,
      "langfuse.trace_batch.transcript_history_message_count": 0,
      "langfuse.trace_batch.transcript_current_turn_message_count": 2,
      "langfuse.trace_batch.transcript_current_turn_part_count": 2,
    });
  });

  it("emits transcript distributions only when the switch is on", async () => {
    env.LANGFUSE_TRACE_BATCH_TRANSCRIPT_METRICS_ENABLED = "true";
    await recordTraceBatchTranscript([
      convertObservation(
        createObservation({
          type: "GENERATION",
          name: "explain",
          trace_id: "topics-metrics",
          input: JSON.stringify([
            { role: "user", content: "Explain the result" },
          ]),
          output: JSON.stringify({ role: "assistant", content: "The answer" }),
        }),
      ),
    ]);

    const renderedText = vi.mocked(tokenCountAsync).mock.calls[3][0].text;
    expect(recordDistribution).toHaveBeenCalledWith(
      "langfuse.trace_batch.topics_transcript_characters",
      String(renderedText).length,
    );
    expect(recordDistribution).toHaveBeenCalledWith(
      "langfuse.trace_batch.topics_transcript_tokens",
      expect.any(Number),
      { tokenizer: "o200k_base" },
    );
    expect(recordDistribution).toHaveBeenCalledWith(
      "langfuse.trace_batch.transcript_json_characters",
      expect.any(Number),
    );
  });

  it("keeps the batch successful when Topics token estimation fails", async () => {
    vi.mocked(tokenCountAsync)
      .mockResolvedValueOnce(10)
      .mockResolvedValueOnce(20)
      .mockResolvedValueOnce(30)
      .mockRejectedValueOnce(new Error("tokenizer unavailable"));

    await expect(
      recordTraceBatchTranscript([
        convertObservation(
          createObservation({
            type: "GENERATION",
            trace_id: "topics-failure",
            input: JSON.stringify([{ role: "user", content: "Question" }]),
            output: JSON.stringify({ role: "assistant", content: "Answer" }),
          }),
        ),
      ]),
    ).resolves.toBeUndefined();

    expect(recordIncrement).toHaveBeenCalledWith(
      "langfuse.trace_batch.topics_transcript_failed",
      1,
    );
    expect(recordDistribution).not.toHaveBeenCalled();
    const span = exporter.getFinishedSpans()[0];
    expect(span.attributes).toMatchObject({
      "langfuse.trace_batch.topics_transcript": "failed",
      "langfuse.trace_batch.topics_transcript_characters": expect.any(Number),
      "langfuse.trace_batch.transcript_message_tokens": 10,
      "langfuse.trace_batch.transcript_json_tokens": 20,
      "langfuse.trace_batch.generic_transcript_tokens": 30,
    });
    expect(span.attributes).not.toHaveProperty(
      "langfuse.trace_batch.topics_transcript_tokens",
    );
  });

  it("measures history, current turn, threads and deduplicated tool responses", async () => {
    const history = [
      { role: "user", content: "earlier question" },
      { role: "assistant", content: "earlier answer" },
    ];
    const question = { role: "user", content: "current question" };
    const call = {
      role: "assistant",
      content: null,
      tool_calls: [
        {
          id: "call",
          type: "function",
          function: { name: "lookup", arguments: "{}" },
        },
      ],
    };
    const observation = (
      id: string,
      input: unknown,
      output: unknown,
      type: "GENERATION" | "TOOL" = "GENERATION",
    ) =>
      convertObservation(
        createObservation({
          id,
          trace_id: "trace",
          project_id: "project",
          type,
          name: type === "TOOL" ? "lookup" : "generation",
          start_time: `2026-01-01T12:00:0${id}.000Z`,
          input: JSON.stringify(input),
          output: JSON.stringify(output),
        }),
      );
    await recordTraceBatchTranscript([
      observation("1", [...history, question], call),
      observation("2", {}, "tool payload", "TOOL"),
      observation(
        "3",
        [
          ...history,
          question,
          call,
          { role: "tool", tool_call_id: "call", content: "tool payload" },
        ],
        "current answer",
      ),
      observation(
        "4",
        [{ role: "user", content: "independent question" }],
        "independent answer",
      ),
    ]);
    const estimates = vi
      .mocked(tokenCountAsync)
      .mock.calls.map(([params]) => params);
    expect(estimates).toHaveLength(5);
    const [current, historyOnly] = estimates
      .slice(0, 2)
      .map(({ text }) => JSON.stringify(text));
    expect(current.split("tool payload")).toHaveLength(2);
    expect(current).not.toContain("earlier question");
    expect(current).toContain("current question");
    expect(current).toContain("independent question");
    expect(historyOnly).toContain("earlier question");
    expect(historyOnly).not.toContain("current question");
    expect(recordDistribution).not.toHaveBeenCalled();
    const span = exporter.getFinishedSpans()[0];
    expect(span.attributes).toMatchObject({
      "langfuse.trace_batch.transcript_thread_count": 2,
      "langfuse.trace_batch.transcript_observation_count": 4,
      "langfuse.trace_batch.transcript_history_message_count": 2,
      "langfuse.trace_batch.transcript_history_part_count": 2,
      "langfuse.trace_batch.transcript_current_turn_tool_call_count": 1,
      "langfuse.trace_batch.transcript_current_turn_tool_result_count": 1,
      "langfuse.trace_batch.transcript_current_turn_tokens": tokenCount(
        estimates[0],
      ),
      "langfuse.trace_batch.transcript_history_tokens": tokenCount(
        estimates[1],
      ),
      "langfuse.trace_batch.transcript_message_tokens":
        tokenCount(estimates[0])! + tokenCount(estimates[1])!,
    });
    const toolCharacters = JSON.stringify({
      type: "tool-result",
      toolCallId: "call",
      toolName: "lookup",
      output: { type: "text", value: "tool payload" },
    }).length;
    expect(
      span.attributes[
        "langfuse.trace_batch.transcript_tool_response_characters"
      ],
    ).toBe(toolCharacters);
    expect(
      span.attributes["langfuse.trace_batch.transcript_content_characters"],
    ).toBeGreaterThan(toolCharacters);
    let phaseTotal = 0;
    for (const phase of ["normalization", "matching"]) {
      const duration = span.attributes[
        `langfuse.trace_batch.transcript_assembly_${phase}_duration_ms`
      ] as number;
      expect(duration).toBeGreaterThanOrEqual(0);
      phaseTotal += duration;
    }
    expect(phaseTotal).toBeLessThanOrEqual(
      span.attributes[
        "langfuse.trace_batch.transcript_assembly_duration_ms"
      ] as number,
    );
  });

  it("counts unmatched tool results without counting adjacent user text", async () => {
    await recordTraceBatchTranscript([
      convertObservation(
        createObservation({
          type: "GENERATION",
          trace_id: "trace",
          input: JSON.stringify([
            {
              role: "user",
              content: [
                { type: "text", text: "Please explain this result" },
                {
                  type: "tool_result",
                  tool_use_id: "unmatched",
                  content: "tool payload",
                },
              ],
            },
          ]),
          output: "Answer",
        }),
      ),
    ]);
    expect(tokenCountAsync).toHaveBeenCalledTimes(4);
    const toolCharacters = JSON.stringify({
      type: "tool-result",
      toolCallId: "unmatched",
      output: { type: "text", value: "tool payload" },
    }).length;
    const totalCharacters =
      toolCharacters +
      JSON.stringify({ type: "text", text: "Please explain this result" })
        .length +
      JSON.stringify({ type: "text", text: "Answer" }).length;
    expect(recordDistribution).not.toHaveBeenCalled();
    expect(exporter.getFinishedSpans()[0].attributes).toMatchObject({
      "langfuse.trace_batch.transcript_tool_response_characters":
        toolCharacters,
      "langfuse.trace_batch.transcript_content_characters": totalCharacters,
      "langfuse.trace_batch.transcript_history_tokens": 0,
    });
  });

  it.each(["unavailable", "failed"])(
    "retains current-turn tokens and character counts but omits the sum when history is %s",
    async (outcome) => {
      vi.mocked(tokenCountAsync).mockResolvedValueOnce(100);
      if (outcome === "unavailable")
        vi.mocked(tokenCountAsync).mockResolvedValueOnce(undefined);
      else
        vi.mocked(tokenCountAsync).mockRejectedValueOnce(
          new Error("tokenizer failed"),
        );
      await recordTraceBatchTranscript([
        convertObservation(
          createObservation({
            type: "GENERATION",
            trace_id: "trace",
            input: JSON.stringify([
              { role: "assistant", content: "history" },
              { role: "user", content: "question" },
            ]),
            output: "answer",
          }),
        ),
      ]);
      const span = exporter.getFinishedSpans()[0];
      expect(
        span.attributes["langfuse.trace_batch.transcript_current_turn_tokens"],
      ).toBe(100);
      expect(span.attributes).not.toHaveProperty(
        "langfuse.trace_batch.transcript_history_tokens",
      );
      expect(span.attributes).not.toHaveProperty(
        "langfuse.trace_batch.transcript_message_tokens",
      );
      expect(
        span.attributes["langfuse.trace_batch.transcript_content_characters"],
      ).toBeGreaterThan(0);
      expect(span.attributes["langfuse.trace_batch.token_estimation"]).toBe(
        outcome,
      );
      expect(recordIncrement).toHaveBeenCalledWith(
        `langfuse.trace_batch.token_estimation_${outcome}`,
        1,
      );
    },
  );

  it.each([false, true])(
    "keeps streaming while one tokenization is pending and drains it (stream fails: %s)",
    async (streamFails) => {
      let resolveFirst!: (tokens: number) => void;
      let rejectSecond!: (error: Error) => void;
      let cleaningUp = false;
      vi.mocked(tokenCountAsync)
        .mockImplementationOnce(
          () => new Promise<number>((resolve) => (resolveFirst = resolve)),
        )
        .mockImplementationOnce(() => Promise.resolve(0))
        .mockImplementationOnce(() => Promise.resolve(0))
        .mockImplementationOnce(() => Promise.resolve(0))
        .mockImplementationOnce(() =>
          cleaningUp
            ? Promise.resolve(0)
            : new Promise<number>((_, reject) => (rejectSecond = reject)),
        );
      let suppliedThirdTrace = false;
      let reachedEnd = false;
      let settled = false;
      const failure = new Error("stream failed");
      vi.mocked(getTraceBatchEventStream).mockImplementation(
        async function* () {
          for (const [index, traceId] of ["a", "b", "b", "c"].entries()) {
            if (traceId === "c") suppliedThirdTrace = true;
            yield {
              project_id: "project",
              trace_id: traceId,
              environment: "default",
              span_id: `span-${index}`,
              parent_span_id: null,
              start_time: "2026-09-11 00:00:00.000000",
              event_ts: "2026-09-11 00:00:00.000000",
              type: traceId === "c" ? "SPAN" : "GENERATION",
              name: "generation",
              input: "hello",
              output: "world",
              metadata: {},
              level: "DEFAULT",
              status_message: null,
              tool_definitions: {},
              tool_calls: [],
              tool_call_names: [],
            };
          }
          reachedEnd = true;
          if (streamFails) throw failure;
        },
      );
      const job = {
        data: {
          id: "bounded-tokenization",
          name: QueueJobs.TraceBatch,
          timestamp: new Date(),
          payload: {
            traces: ["a", "b", "c"].map((traceId) => ({
              projectId: "project",
              traceId,
              minStart: 0,
              maxStart: 1,
              revision: "r",
            })),
          },
        },
      } as Job<TQueueJobTypes[QueueName.TraceBatch]>;
      const processing = context
        .with(trace.setSpan(context.active(), processingSpan), () =>
          traceBatchQueueProcessor(job, undefined),
        )
        .then(
          (result) => ({ result, error: undefined }),
          (error: unknown) => ({ result: undefined, error }),
        );
      void processing.then(() => (settled = true));
      try {
        // The stream drains while a is being tokenized, so the socket never
        // idles; b and c wait behind a for the shared tokenizer pool.
        await vi.waitFor(() => expect(reachedEnd).toBe(true));
        expect(suppliedThirdTrace).toBe(true);
        expect(tokenCountAsync).toHaveBeenCalledTimes(1);
        expect(exporter.getFinishedSpans()).toHaveLength(0);
        resolveFirst(100);
        await vi.waitFor(() =>
          expect(tokenCountAsync).toHaveBeenCalledTimes(5),
        );
        expect(settled).toBe(false);
        // An unavailable token estimate must not retry a whole batch or mask
        // the original stream failure; its promise is drained before returning.
        rejectSecond(new Error("token count timed out"));
        const outcome = await processing;
        expect(outcome.error).toBe(streamFails ? failure : undefined);
        const spans = exporter.getFinishedSpans();
        expect(spans).toHaveLength(streamFails ? 2 : 3);
        expect(spans[0].attributes).toMatchObject({
          "langfuse.trace.id": "a",
          "langfuse.trace_batch.transcript_message_tokens": 100,
        });
        expect(spans[1].attributes).toMatchObject({
          "langfuse.trace.id": "b",
          "langfuse.trace_batch.token_estimation": "failed",
        });
        expect(spans[1].status.code).toBe(SpanStatusCode.UNSET);
        expect(spans[1].attributes).not.toHaveProperty(
          "langfuse.trace_batch.transcript_message_tokens",
        );
        expect(
          spans[1].attributes["langfuse.trace_batch.transcript_characters"],
        ).toBeGreaterThan(0);
        for (const span of spans) {
          expect(span.parentSpanContext?.spanId).toBe(
            processingSpan.spanContext().spanId,
          );
        }
        expect(recordIncrement).toHaveBeenCalledWith(
          "langfuse.trace_batch.token_estimation_failed",
          1,
        );
        expect(
          spans.filter(
            (span) =>
              "langfuse.trace_batch.transcript_assembly_duration_ms" in
              span.attributes,
          ),
        ).toHaveLength(streamFails ? 2 : 3);
      } finally {
        cleaningUp = true;
        resolveFirst?.(100);
        rejectSecond?.(new Error("test cleanup"));
        await processing;
      }
    },
  );
  it("summarizes traces concurrently and resumes streaming when any summary brings payloads below the cap", async () => {
    const originalCap = env.LANGFUSE_TRACE_BATCH_MAX_PENDING_SUMMARY_BYTES;
    env.LANGFUSE_TRACE_BATCH_MAX_PENDING_SUMMARY_BYTES = 1_000;
    const payload = "x".repeat(600);
    let reachedEnd = false;
    let resolveFirst!: () => void;
    let resolveSecond!: () => void;
    const first = new Promise<void>((resolve) => (resolveFirst = resolve));
    const second = new Promise<void>((resolve) => (resolveSecond = resolve));
    vi.mocked(summarizeAssembledTrace)
      .mockImplementationOnce(async () => {
        await first;
        return { outcome: "summarized", rows: [] };
      })
      .mockImplementationOnce(async () => {
        await second;
        return { outcome: "summarized", rows: [] };
      });
    vi.mocked(getTraceBatchEventStream).mockImplementation(async function* () {
      for (const traceId of ["a", "b", "c"]) {
        yield {
          project_id: "project",
          trace_id: traceId,
          environment: "default",
          span_id: traceId,
          parent_span_id: null,
          start_time: "2026-09-11 00:00:00.000000",
          event_ts: "2026-09-11 00:00:00.000000",
          type: "GENERATION",
          name: "generation",
          input: JSON.stringify([{ role: "user", content: "hello" }]),
          output: "",
          metadata: traceId === "c" ? {} : { payload },
          level: "DEFAULT",
          status_message: null,
          tool_definitions: {},
          tool_calls: [],
          tool_call_names: [],
        };
      }
      reachedEnd = true;
    });
    const job = {
      data: {
        id: "summary-backpressure",
        name: QueueJobs.TraceBatch,
        timestamp: new Date(),
        payload: {
          traces: ["a", "b", "c"].map((traceId) => ({
            projectId: "project",
            traceId,
            minStart: 0,
            maxStart: 1,
            revision: "r",
          })),
        },
      },
    } as Job<TQueueJobTypes[QueueName.TraceBatch]>;
    let settled = false;
    const processing = Promise.resolve(
      traceBatchQueueProcessor(job, undefined),
    );
    void processing.then(() => (settled = true));
    try {
      await vi.waitFor(() =>
        expect(recordIncrement).toHaveBeenCalledWith(
          "langfuse.trace_batch.summary_backpressure",
          1,
        ),
      );
      expect(reachedEnd).toBe(false);
      await vi.waitFor(() =>
        expect(summarizeAssembledTrace).toHaveBeenCalledTimes(2),
      );
      // The later trace finishing first releases its payload.
      resolveSecond();
      await vi.waitFor(() => expect(reachedEnd).toBe(true));
      await vi.waitFor(() =>
        expect(summarizeAssembledTrace).toHaveBeenCalledTimes(3),
      );
      expect(settled).toBe(false);
      resolveFirst();
      await expect(processing).resolves.toEqual(
        expect.objectContaining({ traceCount: 3 }),
      );
    } finally {
      resolveFirst();
      resolveSecond();
      await processing;
      env.LANGFUSE_TRACE_BATCH_MAX_PENDING_SUMMARY_BYTES = originalCap;
      vi.mocked(summarizeAssembledTrace).mockResolvedValue({
        outcome: "unchanged",
        rows: [],
      });
    }
  });
  it("finishes the batch past a failed trace, writes rows once, retries transient errors and drops only failing rows", async () => {
    const traces = ["a", "b", "c"];
    vi.mocked(getTraceBatchEventStream).mockImplementation(async function* () {
      for (const projectId of traces)
        yield {
          project_id: projectId,
          trace_id: "trace",
          environment: "default",
          span_id: "span",
          parent_span_id: null,
          start_time: "2026-09-11 00:00:00.000000",
          event_ts: "2026-09-11 00:00:00.000000",
          type: "GENERATION",
          name: "generation",
          input: JSON.stringify([{ role: "user", content: projectId }]),
          output: JSON.stringify({ role: "assistant", content: "answer" }),
          metadata: {},
          level: "DEFAULT",
          status_message: null,
          tool_definitions: {},
          tool_calls: [],
          tool_call_names: [],
        };
    });
    vi.mocked(summarizeAssembledTrace).mockImplementation(
      async ({ projectId }) => ({
        outcome: "summarized",
        rows: [{ projectId, state: "complete" } as TopicSummary],
        // A failed trace still returns its valid facets.
        ...(projectId === "b"
          ? {
              failure: new TopicsProviderUnavailable(
                "bad output",
                "invalid_output",
              ),
            }
          : {}),
      }),
    );
    const add = vi.fn();
    vi.spyOn(TraceBatchQueue, "getInstance").mockReturnValue({
      add,
    } as unknown as ReturnType<typeof TraceBatchQueue.getInstance>);
    const entry = (projectId: string) => ({
      projectId,
      traceId: "trace",
      minStart: 0,
      maxStart: 1,
      revision: "r",
    });
    const job = {
      data: {
        id: "outcomes",
        name: QueueJobs.TraceBatch,
        timestamp: new Date(),
        payload: { traces: traces.map(entry) },
      },
    } as Job<TQueueJobTypes[QueueName.TraceBatch]>;

    await expect(traceBatchQueueProcessor(job, undefined)).resolves.toEqual(
      expect.objectContaining({ traceCount: 3 }),
    );
    expect(
      vi.mocked(summarizeAssembledTrace).mock.calls.map(([t]) => t.projectId),
    ).toEqual(traces);
    expect(recordIncrement).toHaveBeenCalledWith(
      "langfuse.topics.trace_outcomes",
      2,
      { outcome: "summarized" },
    );
    expect(recordIncrement).toHaveBeenCalledWith(
      "langfuse.topics.trace_outcomes",
      1,
      { outcome: "failed", reason: "invalid_output" },
    );
    expect(writeTopicSummaries).toHaveBeenCalledOnce();
    expect(
      vi
        .mocked(writeTopicSummaries)
        .mock.calls[0][0].map((row) => row.projectId),
    ).toEqual(traces);
    expect(add).not.toHaveBeenCalled();

    // c's invalid row is dropped before the insert; a transient error retries the rest.
    vi.mocked(recordIncrement).mockClear();
    vi.mocked(isValidTopicSummary).mockImplementation(
      (row) => row.projectId !== "c",
    );
    vi.mocked(writeTopicSummaries).mockRejectedValueOnce(
      new Error("Timeout error."),
    );
    await expect(traceBatchQueueProcessor(job, undefined)).resolves.toEqual(
      expect.objectContaining({ traceCount: 3 }),
    );
    const outcomeCalls = vi
      .mocked(recordIncrement)
      .mock.calls.filter(([name]) => name === "langfuse.topics.trace_outcomes");
    expect(outcomeCalls).toEqual(
      expect.arrayContaining([
        ["langfuse.topics.trace_outcomes", 1, { outcome: "summarized" }],
        [
          "langfuse.topics.trace_outcomes",
          1,
          { outcome: "failed", reason: "invalid_output" },
        ],
        [
          "langfuse.topics.trace_outcomes",
          1,
          { outcome: "failed", reason: "storage" },
        ],
      ]),
    );
    expect(outcomeCalls).toHaveLength(3);
    expect(writeTopicSummaries).toHaveBeenCalledTimes(3);
    expect(
      vi
        .mocked(writeTopicSummaries)
        .mock.lastCall![0].map((row) => row.projectId),
    ).toEqual(["a", "b"]);
    // Only stored rows are queued for assignment.
    expect(
      vi
        .mocked(enqueueTopicsAutomation)
        .mock.lastCall![1].map((row) => row.projectId),
    ).toEqual(["a", "b"]);

    // Like ClickhouseWriter, an outage drops the insert once instead of splitting it.
    vi.mocked(recordIncrement).mockClear();
    vi.mocked(isValidTopicSummary).mockImplementation(() => true);
    vi.mocked(writeTopicSummaries)
      .mockReset()
      .mockRejectedValue(new Error("connect ECONNREFUSED"));
    await traceBatchQueueProcessor(job, undefined);
    expect(writeTopicSummaries).toHaveBeenCalledOnce();
    expect(recordIncrement).toHaveBeenCalledWith(
      "langfuse.topics.trace_outcomes",
      2,
      { outcome: "failed", reason: "storage" },
    );
    expect(vi.mocked(enqueueTopicsAutomation).mock.lastCall![1]).toEqual([]);
    vi.mocked(writeTopicSummaries).mockReset();
    vi.mocked(summarizeAssembledTrace).mockImplementation(async () => ({
      outcome: "unchanged",
      rows: [],
    }));
  });

  it("stores and queues completed traces' rows before rethrowing a failed stream, so a retry skips them", async () => {
    const failure = new Error("query failed");
    const event = (projectId: string) => ({
      project_id: projectId,
      trace_id: "trace",
      environment: "default",
      span_id: "span",
      parent_span_id: null,
      start_time: "2026-09-11 00:00:00.000000",
      event_ts: "2026-09-11 00:00:00.000000",
      type: "GENERATION",
      name: "generation",
      input: JSON.stringify([{ role: "user", content: projectId }]),
      output: JSON.stringify({ role: "assistant", content: "answer" }),
      metadata: {},
      level: "DEFAULT",
      status_message: null,
      tool_definitions: {},
      tool_calls: [],
      tool_call_names: [],
    });
    vi.mocked(getTraceBatchEventStream).mockImplementation(async function* () {
      yield event("a");
      // b's boundary completes a; b itself is cut off by the failure.
      yield event("b");
      throw failure;
    });
    vi.mocked(summarizeAssembledTrace).mockImplementation(
      async ({ projectId }) => ({
        outcome: "summarized",
        rows: [{ projectId, state: "complete" } as TopicSummary],
      }),
    );
    const job = {
      data: {
        id: "stream-failure",
        name: QueueJobs.TraceBatch,
        timestamp: new Date(),
        payload: {
          traces: ["a", "b"].map((projectId) => ({
            projectId,
            traceId: "trace",
            minStart: 0,
            maxStart: 1,
            revision: "r",
          })),
        },
      },
    } as Job<TQueueJobTypes[QueueName.TraceBatch]>;

    await expect(traceBatchQueueProcessor(job, undefined)).rejects.toBe(
      failure,
    );
    expect(writeTopicSummaries).toHaveBeenCalledOnce();
    expect(vi.mocked(writeTopicSummaries).mock.calls[0][0]).toEqual([
      { projectId: "a", state: "complete" },
    ]);
    // A retry skips stored rows, so they are queued before the failure rethrows.
    expect(enqueueTopicsAutomation).toHaveBeenCalledWith("stream-failure", [
      { projectId: "a", state: "complete" },
    ]);
    vi.mocked(summarizeAssembledTrace).mockImplementation(async () => ({
      outcome: "unchanged",
      rows: [],
    }));
  });

  it("keeps the batch successful when Topics assignment cannot be queued", async () => {
    vi.mocked(getTraceBatchEventStream).mockImplementation(async function* () {
      yield {
        project_id: "a",
        trace_id: "trace",
        environment: "default",
        span_id: "span",
        parent_span_id: null,
        start_time: "2026-09-11 00:00:00.000000",
        event_ts: "2026-09-11 00:00:00.000000",
        type: "GENERATION",
        name: "generation",
        input: JSON.stringify([{ role: "user", content: "question" }]),
        output: JSON.stringify({ role: "assistant", content: "answer" }),
        metadata: {},
        level: "DEFAULT",
        status_message: null,
        tool_definitions: {},
        tool_calls: [],
        tool_call_names: [],
      };
    });
    vi.mocked(summarizeAssembledTrace).mockImplementation(async () => ({
      outcome: "summarized",
      rows: [{ projectId: "a", state: "complete" } as TopicSummary],
    }));
    vi.mocked(enqueueTopicsAutomation).mockRejectedValueOnce(
      new Error("Redis unavailable"),
    );
    const job = {
      data: {
        id: "enqueue-failure",
        name: QueueJobs.TraceBatch,
        timestamp: new Date(),
        payload: {
          traces: [
            {
              projectId: "a",
              traceId: "trace",
              minStart: 0,
              maxStart: 1,
              revision: "r",
            },
          ],
        },
      },
    } as Job<TQueueJobTypes[QueueName.TraceBatch]>;

    await expect(traceBatchQueueProcessor(job, undefined)).resolves.toEqual(
      expect.objectContaining({ traceCount: 1 }),
    );
    expect(writeTopicSummaries).toHaveBeenCalledOnce();
    expect(recordIncrement).toHaveBeenCalledWith(
      "langfuse.topics.trace_outcomes",
      1,
      { outcome: "summarized" },
    );
    vi.mocked(summarizeAssembledTrace).mockImplementation(async () => ({
      outcome: "unchanged",
      rows: [],
    }));
  });

  it.each(["disabled", "stored", "too_large", "upload_failed"] as const)(
    "preserves Topics processing when optional input storage is %s",
    async (outcome) => {
      env.LANGFUSE_TRACE_BATCH_TRANSCRIPT_STORAGE_ENABLED =
        outcome === "disabled" ? "false" : "true";
      const info = vi.spyOn(logger, "info");
      if (outcome === "upload_failed")
        inputStorage.put.mockRejectedValueOnce(
          new Error("Failed to upload file to S3"),
        );
      if (outcome === "too_large")
        inputStorage.put.mockResolvedValueOnce({
          outcome,
          jsonLength: 1_000_001,
        });
      vi.mocked(getTraceBatchEventStream).mockImplementation(
        async function* () {
          yield {
            project_id: "project",
            trace_id: "trace",
            environment: "default",
            span_id: "span",
            parent_span_id: null,
            start_time: "2026-09-11 00:00:00.000000",
            event_ts: "2026-09-11 00:00:00.000000",
            type: "GENERATION",
            name: "generation",
            input: JSON.stringify([{ role: "user", content: "private input" }]),
            output: JSON.stringify({
              role: "assistant",
              content: "private output",
            }),
            metadata: {},
            level: "DEFAULT",
            status_message: null,
            tool_definitions: {},
            tool_calls: [],
            tool_call_names: [],
          };
        },
      );
      const job = {
        data: {
          id: "storage",
          name: QueueJobs.TraceBatch,
          timestamp: new Date(),
          payload: {
            traces: [
              {
                projectId: "project",
                traceId: "trace",
                minStart: 0,
                maxStart: 1,
                revision: "queued-revision",
              },
            ],
          },
        },
      } as Job<TQueueJobTypes[QueueName.TraceBatch]>;

      await expect(
        traceBatchQueueProcessor(job, undefined),
      ).resolves.toBeDefined();
      expect(summarizeAssembledTrace).toHaveBeenCalledTimes(1);
      expect(
        vi.mocked(summarizeAssembledTrace).mock.calls[0][0].transcript?.threads,
      ).toHaveLength(1);
      expect(inputStorage.put).toHaveBeenCalledTimes(
        outcome === "disabled" ? 0 : 1,
      );
      if (outcome !== "disabled") {
        expect(inputStorage.put).toHaveBeenCalledWith(
          {
            projectId: "project",
            traceId: "trace",
            revision: "queued-revision",
          },
          {
            transcript: vi.mocked(summarizeAssembledTrace).mock.calls[0][0]
              .transcript,
            executionIssues: [],
            traceContext: {
              sessionId: null,
              traceTimestamp: "2026-09-11T00:00:00.000Z",
              environment: "default",
              traceName: "generation",
            },
          },
        );
        const storageSpan = exporter
          .getFinishedSpans()
          .find((span) => span.name === "trace-batch-store-inputs");
        expect(
          storageSpan?.attributes["langfuse.trace_batch.input_storage.outcome"],
        ).toBe(outcome);
        if (outcome === "upload_failed")
          expect(storageSpan?.status.code).toBe(SpanStatusCode.ERROR);
      }
      if (outcome === "too_large") {
        expect(info).toHaveBeenCalledWith("Trace batch input storage skipped", {
          projectId: "project",
          traceId: "trace",
          revision: "queued-revision",
          reason: "input_too_large",
          jsonLength: 1_000_001,
          limit: 1_000_000,
        });
        expect(JSON.stringify(info.mock.calls)).not.toContain("private input");
        expect(JSON.stringify(info.mock.calls)).not.toContain("private output");
      }
    },
  );

  it("stores execution issues next to the transcript", async () => {
    env.LANGFUSE_TRACE_BATCH_TRANSCRIPT_STORAGE_ENABLED = "true";
    vi.mocked(getTraceBatchEventStream).mockImplementation(async function* () {
      yield {
        project_id: "project",
        trace_id: "trace",
        environment: "default",
        span_id: "span",
        parent_span_id: null,
        start_time: "2026-09-11 00:00:00.000000",
        event_ts: "2026-09-11 00:00:00.000000",
        type: "GENERATION",
        name: "generation",
        input: JSON.stringify([{ role: "user", content: "Hi" }]),
        output: JSON.stringify({ role: "assistant", content: "Hello" }),
        metadata: {},
        level: "WARNING",
        status_message: "Rate limited, retried",
        tool_definitions: {},
        tool_calls: [],
        tool_call_names: [],
      };
    });
    const job = {
      data: {
        id: "issues",
        name: QueueJobs.TraceBatch,
        timestamp: new Date(),
        payload: {
          traces: [
            {
              projectId: "project",
              traceId: "trace",
              minStart: 0,
              maxStart: 1,
              revision: "queued-revision",
            },
          ],
        },
      },
    } as Job<TQueueJobTypes[QueueName.TraceBatch]>;

    await traceBatchQueueProcessor(job, undefined);

    expect(inputStorage.put).toHaveBeenCalledExactlyOnceWith(
      expect.anything(),
      expect.objectContaining({
        executionIssues: [
          expect.objectContaining({
            observationId: "span",
            level: "WARNING",
            statusMessage: "Rate limited, retried",
          }),
        ],
      }),
    );
  });

  it.each([false, true])(
    "runs Topics alongside capture and holds the slot until both settle (Topics rejects: %s)",
    async (summaryRejects) => {
      env.LANGFUSE_TRACE_BATCH_TRANSCRIPT_STORAGE_ENABLED = "true";
      const originalConcurrency = env.LANGFUSE_TOPICS_TRACE_CONCURRENCY;
      env.LANGFUSE_TOPICS_TRACE_CONCURRENCY = 1;
      let finishUpload!: () => void;
      inputStorage.put.mockImplementationOnce(
        () =>
          new Promise((resolve) => {
            finishUpload = () =>
              resolve({ outcome: "stored", jsonLength: 100 });
          }),
      );
      vi.mocked(summarizeAssembledTrace).mockImplementationOnce(async () => {
        if (summaryRejects) throw new Error("summary failed");
        return { outcome: "unchanged", rows: [] };
      });
      vi.mocked(getTraceBatchEventStream).mockImplementation(
        async function* () {
          for (const traceId of ["a", "b"])
            yield {
              project_id: "project",
              trace_id: traceId,
              environment: "default",
              span_id: "span",
              parent_span_id: null,
              start_time: "2026-09-11 00:00:00.000000",
              event_ts: "2026-09-11 00:00:00.000000",
              type: "GENERATION",
              name: "generation",
              input: JSON.stringify([{ role: "user", content: "hello" }]),
              output: JSON.stringify({ role: "assistant", content: "answer" }),
              metadata: {},
              level: "DEFAULT",
              status_message: null,
              tool_definitions: {},
              tool_calls: [],
              tool_call_names: [],
            };
        },
      );
      const job = {
        data: {
          id: "concurrent-storage",
          name: QueueJobs.TraceBatch,
          timestamp: new Date(),
          payload: {
            traces: ["a", "b"].map((traceId) => ({
              projectId: "project",
              traceId,
              minStart: 0,
              maxStart: 1,
              revision: "revision",
            })),
          },
        },
      } as Job<TQueueJobTypes[QueueName.TraceBatch]>;
      const processing = traceBatchQueueProcessor(job, undefined);
      try {
        await vi.waitFor(() =>
          expect(summarizeAssembledTrace).toHaveBeenCalledTimes(1),
        );
        await new Promise<void>((resolve) => setImmediate(resolve));
        expect(inputStorage.put).toHaveBeenCalledTimes(1);
        expect(summarizeAssembledTrace).toHaveBeenCalledTimes(1);
        finishUpload();
        await processing;
        expect(inputStorage.put).toHaveBeenCalledTimes(2);
        expect(summarizeAssembledTrace).toHaveBeenCalledTimes(2);
      } finally {
        finishUpload?.();
        await processing;
        env.LANGFUSE_TOPICS_TRACE_CONCURRENCY = originalConcurrency;
      }
    },
  );

  it("summarizes only the saved rule's facets and skips traces outside its sampling", async () => {
    env.LANGFUSE_TRACE_BATCH_TRANSCRIPT_STORAGE_ENABLED = "true";
    vi.mocked(getTraceBatchEventStream).mockImplementation(async function* () {
      for (const projectId of ["picked", "sampled-out"])
        yield {
          project_id: projectId,
          trace_id: "trace",
          environment: "default",
          span_id: "span",
          parent_span_id: null,
          start_time: "2026-09-11 00:00:00.000000",
          event_ts: "2026-09-11 00:00:00.000000",
          type: "GENERATION",
          name: "generation",
          input: JSON.stringify([{ role: "user", content: projectId }]),
          output: JSON.stringify({ role: "assistant", content: "answer" }),
          metadata: {},
          level: "DEFAULT",
          status_message: null,
          tool_definitions: {},
          tool_calls: [],
          tool_call_names: [],
        };
    });
    const facet = (id: string) => ({ id }) as TopicFacet;
    vi.mocked(listTopicFacets).mockResolvedValue([
      facet("intent"),
      facet("issues"),
    ]);
    vi.mocked(listTopicRules).mockImplementation(async (projectId) => [
      {
        facetIds: ["issues"],
        sampling: projectId === "picked" ? 1 : 0,
      } as TopicRule,
    ]);
    const job = {
      data: {
        id: "rule",
        name: QueueJobs.TraceBatch,
        timestamp: new Date(),
        payload: {
          traces: ["picked", "sampled-out"].map((projectId) => ({
            projectId,
            traceId: "trace",
            minStart: 0,
            maxStart: 1,
            revision: `revision-${projectId}`,
          })),
        },
      },
    } as Job<TQueueJobTypes[QueueName.TraceBatch]>;

    await traceBatchQueueProcessor(job, undefined);

    expect(
      vi
        .mocked(summarizeAssembledTrace)
        .mock.calls.map(([input]) => [
          input.projectId,
          input.facets.map(({ id }) => id),
        ]),
    ).toEqual([["picked", ["issues"]]]);
    expect(inputStorage.put).toHaveBeenCalledTimes(1);
    expect(inputStorage.put).toHaveBeenCalledWith(
      { projectId: "picked", traceId: "trace", revision: "revision-picked" },
      expect.objectContaining({
        transcript: expect.objectContaining({ threads: expect.any(Array) }),
      }),
    );
    vi.mocked(listTopicFacets).mockResolvedValue([]);
    vi.mocked(listTopicRules).mockResolvedValue([]);
  });

  it("passes the session of the newest event that has one, breaking timestamp ties by span ID", async () => {
    env.LANGFUSE_TRACE_BATCH_TRANSCRIPT_STORAGE_ENABLED = "true";
    vi.mocked(getTraceBatchEventStream).mockImplementation(async function* () {
      for (const [span, eventTs, sessionId] of [
        ["a", "2026-09-11 00:00:01.000000", "session-old"],
        ["b", "2026-09-11 00:00:02.000000", "session-new"],
        ["d", "2026-09-11 00:00:02.000000", "session-tie"],
        ["c", "2026-09-11 00:00:03.000000", ""],
      ])
        yield {
          project_id: "project",
          trace_id: "trace",
          session_id: sessionId,
          environment: `env-${span}`,
          span_id: span,
          parent_span_id: null,
          start_time: `2026-09-11 00:00:0${span === "d" ? "0" : "1"}.000000`,
          event_ts: eventTs,
          type: "GENERATION",
          name: `generation-${span}`,
          input: JSON.stringify([{ role: "user", content: span }]),
          output: JSON.stringify({ role: "assistant", content: "answer" }),
          metadata: {},
          level: "DEFAULT",
          status_message: null,
          tool_definitions: {},
          tool_calls: [],
          tool_call_names: [],
        };
    });
    vi.mocked(listTopicFacets).mockResolvedValue([
      { id: "intent" } as TopicFacet,
    ]);
    vi.mocked(listTopicRules).mockResolvedValue([
      { facetIds: ["intent"], sampling: 1 } as TopicRule,
    ]);
    const job = {
      data: {
        id: "session",
        name: QueueJobs.TraceBatch,
        timestamp: new Date(),
        payload: {
          traces: [
            {
              projectId: "project",
              traceId: "trace",
              minStart: 0,
              maxStart: 1,
              revision: "r",
            },
          ],
        },
      },
    } as Job<TQueueJobTypes[QueueName.TraceBatch]>;

    await traceBatchQueueProcessor(job, undefined);

    expect(
      vi.mocked(summarizeAssembledTrace).mock.calls.at(-1)?.[0],
    ).toMatchObject({
      traceId: "trace",
      sessionId: "session-tie",
      traceTimestamp: "2026-09-11T00:00:00.000Z",
      environment: "env-c",
      traceName: "generation-a",
    });
    expect(inputStorage.put).toHaveBeenCalledWith(
      { projectId: "project", traceId: "trace", revision: "r" },
      expect.objectContaining({
        traceContext: {
          sessionId: "session-tie",
          traceTimestamp: "2026-09-11T00:00:00.000Z",
          environment: "env-c",
          traceName: "generation-a",
        },
      }),
    );
    vi.mocked(listTopicFacets).mockResolvedValue([]);
    vi.mocked(listTopicRules).mockResolvedValue([]);
  });

  it("skips projects without enabled Topics models and pauses a project whose connection is rejected", async () => {
    env.LANGFUSE_TRACE_BATCH_TRANSCRIPT_STORAGE_ENABLED = "true";
    const rows = [
      ["off", "trace-1"],
      ["on", "trace-1"],
      ["on", "trace-2"],
      ["on", "trace-3"],
    ] as const;
    vi.mocked(getTraceBatchEventStream).mockImplementation(async function* () {
      for (const [projectId, traceId] of rows)
        yield {
          project_id: projectId,
          trace_id: traceId,
          environment: "default",
          span_id: "span",
          parent_span_id: null,
          start_time: "2026-09-11 00:00:00.000000",
          event_ts: "2026-09-11 00:00:00.000000",
          type: "GENERATION",
          name: "generation",
          input: JSON.stringify([{ role: "user", content: traceId }]),
          output: JSON.stringify({ role: "assistant", content: "answer" }),
          metadata: {},
          level: "DEFAULT",
          status_message: null,
          tool_definitions: {},
          tool_calls: [],
          tool_call_names: [],
        };
    });
    const models = { projectId: "on" } as TopicsModels;
    vi.mocked(getEnabledTopicsModels).mockResolvedValueOnce(
      new Map([["on", models]]),
    );
    vi.mocked(summarizeAssembledTrace).mockImplementation(
      async ({ traceId }) => {
        if (traceId === "trace-2")
          throw new TopicsProviderUnavailable(
            "Connection rejected (HTTP 401).",
            "authentication",
            "summary",
          );
        return { outcome: "summarized", rows: [] };
      },
    );
    const job = {
      data: {
        id: "kill-switch",
        name: QueueJobs.TraceBatch,
        timestamp: new Date(),
        payload: {
          traces: rows.map(([projectId, traceId]) => ({
            projectId,
            traceId,
            minStart: 0,
            maxStart: 1,
            revision: "r",
          })),
        },
      },
    } as Job<TQueueJobTypes[QueueName.TraceBatch]>;

    await traceBatchQueueProcessor(job, undefined);

    // One Postgres read for the whole batch, deduplicated by project.
    expect(getEnabledTopicsModels).toHaveBeenCalledExactlyOnceWith([
      "off",
      "on",
    ]);
    expect(listTopicFacets).toHaveBeenCalledExactlyOnceWith("on");
    expect(inputStorage.put.mock.calls.map(([reference]) => reference)).toEqual(
      [
        { projectId: "on", traceId: "trace-1", revision: "r" },
        { projectId: "on", traceId: "trace-2", revision: "r" },
      ],
    );
    expect(
      vi
        .mocked(summarizeAssembledTrace)
        .mock.calls.map(([input]) => [
          input.projectId,
          input.traceId,
          input.models,
        ]),
    ).toEqual([
      ["on", "trace-1", models],
      ["on", "trace-2", models],
    ]);
    expect(pauseTopicsModels).toHaveBeenCalledExactlyOnceWith("on", {
      blockReason: "LLM_CONNECTION_AUTH_INVALID",
      blockMessage: expect.stringContaining("Connection rejected (HTTP 401)."),
    });
    expect(recordIncrement).toHaveBeenCalledWith(
      "langfuse.topics.trace_outcomes",
      2,
      { outcome: "disabled" },
    );
    vi.mocked(summarizeAssembledTrace).mockImplementation(async () => ({
      outcome: "unchanged",
      rows: [],
    }));
  });

  it("uses the latest environment to exclude internal traces from Topics and storage", async () => {
    env.LANGFUSE_TRACE_BATCH_TRANSCRIPT_STORAGE_ENABLED = "true";
    const userMessage = { role: "user", content: "first question" };
    const answer = { role: "assistant", content: "first answer" };
    const row = {
      project_id: "a",
      trace_id: "shared-trace",
      environment: "langfuse-topics",
      span_id: "first",
      parent_span_id: null,
      start_time: "2026-09-11 00:00:00.000000",
      event_ts: "2026-09-11 00:00:00.000002",
      type: "GENERATION",
      name: "generation",
      input: JSON.stringify([userMessage]),
      output: JSON.stringify(answer),
      metadata: {},
      level: "DEFAULT",
      status_message: null,
      tool_definitions: {},
      tool_calls: [],
      tool_call_names: [],
    };
    vi.mocked(getTraceBatchEventStream).mockImplementation(async function* () {
      // SQL groups traces but does not order observations within a trace.
      yield {
        ...row,
        span_id: "second",
        environment: "production",
        event_ts: "2026-09-11 00:00:00.000001",
        start_time: "2026-09-11 00:01:00.000000",
        input: JSON.stringify([
          userMessage,
          answer,
          { role: "user", content: "second question" },
        ]),
        output: JSON.stringify({ role: "assistant", content: "second answer" }),
      };
      // The newer internal event has an earlier start time and smaller span ID.
      yield row;
      expect(tokenCountAsync).not.toHaveBeenCalled();
      yield {
        ...row,
        project_id: "b",
        environment: "langfuse-topics",
        type: "SPAN",
      };
      yield {
        ...row,
        project_id: "b",
        environment: "production",
        type: "SPAN",
        span_id: "second",
      };
    });
    const job = {
      data: {
        id: "transcripts",
        name: QueueJobs.TraceBatch,
        timestamp: new Date(),
        payload: {
          traces: ["a", "b"].map((projectId) => ({
            projectId,
            traceId: "shared-trace",
            minStart: 0,
            maxStart: 1,
            revision: "r",
          })),
        },
      },
    } as Job<TQueueJobTypes[QueueName.TraceBatch]>;
    await traceBatchQueueProcessor(job, undefined);
    expect(
      vi.mocked(summarizeAssembledTrace).mock.calls.map(([input]) => ({
        projectId: input.projectId,
        environment: input.environment,
      })),
    ).toEqual([{ projectId: "b", environment: "production" }]);
    expect(inputStorage.put.mock.calls.map(([reference]) => reference)).toEqual(
      [{ projectId: "b", traceId: "shared-trace", revision: "r" }],
    );
    const estimates = vi.mocked(tokenCountAsync).mock.calls;
    expect(estimates).toHaveLength(5);
    const serializedTranscript = JSON.stringify(estimates[0][0].text);
    for (const content of [
      "first question",
      "first answer",
      "second question",
      "second answer",
    ]) {
      expect(serializedTranscript.split(content)).toHaveLength(2);
    }
    const spans = exporter
      .getFinishedSpans()
      .filter((span) => span.name === "trace-batch-transcript");
    expect(spans).toHaveLength(2);
    for (const [index, projectId] of ["a", "b"].entries()) {
      const span = spans[index];
      expect(span.name).toBe("trace-batch-transcript");
      expect(span.attributes).toMatchObject({
        "langfuse.project.id": projectId,
        "langfuse.trace.id": "shared-trace",
        "langfuse.trace_batch.observation_count": 2,
        "langfuse.trace_batch.has_transcript": index === 0,
        "langfuse.trace_batch.transcript_message_tokens":
          index === 0 ? tokenCount(estimates[0][0]) : 0,
        "langfuse.trace_batch.transcript_current_turn_tokens":
          index === 0 ? expect.any(Number) : 0,
        "langfuse.trace_batch.transcript_history_tokens":
          index === 0 ? expect.any(Number) : 0,
        "langfuse.trace_batch.transcript_content_characters":
          index === 0 ? expect.any(Number) : 0,
        "langfuse.trace_batch.transcript_tool_response_characters":
          index === 0 ? expect.any(Number) : 0,
        "langfuse.trace_batch.transcript_thread_count": index === 0 ? 1 : 0,
        "langfuse.trace_batch.transcript_characters":
          index === 0 ? expect.any(Number) : 0,
        "langfuse.trace_batch.transcript_assembly_duration_ms":
          expect.any(Number),
      });
      const url = new URL(String(span.attributes["langfuse.trace.url"]));
      expect(url.pathname).toBe(`/project/${projectId}/traces`);
      expect(url.searchParams.get("peek")).toBe("shared-trace");
      expect(JSON.stringify(span.attributes)).not.toContain("first question");
    }
  });
  it("discards self-hosted jobs before parsing even when reads are enabled", async () => {
    env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID = "discarded-read";
    env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = undefined;
    const job = {
      opts: {},
      data: { incompatiblePayload: true },
    } as unknown as Job<TQueueJobTypes[QueueName.TraceBatch]>;
    await expect(traceBatchQueueProcessor(job, undefined)).resolves.toEqual({
      discarded: "not_cloud",
    });
    expect(getTraceBatchEventStream).not.toHaveBeenCalled();
    expect(recordIncrement).toHaveBeenCalledExactlyOnceWith(
      "langfuse.trace_batch.discarded_jobs",
      1,
      { reason: "not_cloud" },
    );
    expect(recordDistribution).not.toHaveBeenCalled();
    expect(recordGauge).not.toHaveBeenCalled();
    expect(job.opts.removeOnComplete).toBe(true);
    expect(processingSpan.setAttributes).toHaveBeenCalledWith(
      expect.objectContaining({ "langfuse.trace_batch.outcome": "discard" }),
    );
    expect(processingSpan.setAttributes).not.toHaveBeenCalledWith(
      expect.objectContaining({
        "langfuse.trace_batch.observation_count": expect.any(Number),
      }),
    );
  });

  it("discards an expired batch before querying, including on retry", async () => {
    const job = {
      opts: {},
      data: {
        id: "expired",
        name: QueueJobs.TraceBatch,
        timestamp: new Date(Date.now() - 3 * 60 * 60_000).toISOString(),
        payload: {
          traces: [
            {
              projectId: "project",
              traceId: "trace",
              minStart: 0,
              maxStart: 1,
              revision: "r",
            },
          ],
        },
      },
    } as unknown as Job<TQueueJobTypes[QueueName.TraceBatch]>;
    for (let attempt = 0; attempt < 2; attempt++) {
      await expect(traceBatchQueueProcessor(job, undefined)).resolves.toEqual({
        discarded: "expired",
      });
    }
    expect(getTraceBatchEventStream).not.toHaveBeenCalled();
    expect(vi.mocked(recordIncrement).mock.calls).toEqual([
      ["langfuse.trace_batch.discarded_jobs", 1, { reason: "expired" }],
      ["langfuse.trace_batch.discarded_jobs", 1, { reason: "expired" }],
    ]);
    expect(recordDistribution).not.toHaveBeenCalled();
    expect(recordGauge).not.toHaveBeenCalled();
    expect(job.opts.removeOnComplete).toBe(true);
  });

  it("forwards overrides and rejects a partially consumed stream without reporting success", async () => {
    const originalEnv = { ...env };
    Object.assign(env, {
      LANGFUSE_TRACE_BATCH_MAX_THREADS: 1,
      LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE: 512,
      LANGFUSE_TRACE_BATCH_REQUEST_TIMEOUT_MS: 45_000,
      LANGFUSE_TRACE_BATCH_EXPERIMENT_ID: "arm-b",
      BUILD_ID: "test-build",
    });
    const log = vi.spyOn(logger, "info").mockImplementation(() => logger);
    const failure = new Error("query failed");
    vi.mocked(getTraceBatchEventStream).mockImplementation(async function* () {
      expect(processingSpan.setAttributes).toHaveBeenCalledWith(
        expect.objectContaining({
          "langfuse.trace_batch.batch_trace_count": 2,
          "langfuse.trace_batch.batch_project_count": 2,
          "langfuse.trace_batch.event_time_span_ms": 11_000,
          "langfuse.trace_batch.max_trace_span_ms": 3_000,
        }),
      );
      // Completion must annotate the processing span captured before child work.
      vi.mocked(getCurrentSpan).mockReturnValue(undefined);
      yield {
        project_id: "project",
        trace_id: "trace",
        environment: "default",
        span_id: "span",
        parent_span_id: null,
        start_time: "2026-09-11 00:00:00.000000",
        event_ts: "2026-09-11 00:00:00.000000",
        type: "GENERATION",
        name: "generation",
        input: "input",
        output: "output",
        metadata: { é: "界🙂" },
        level: "DEFAULT",
        status_message: null,
        tool_definitions: {},
        tool_calls: [],
        tool_call_names: [],
      };
      throw failure;
    });
    const payload = {
      traces: [
        {
          projectId: "project",
          traceId: "trace",
          minStart: 1_000,
          maxStart: 2_000,
          revision: "r",
        },
        {
          projectId: "other",
          traceId: "trace",
          minStart: 9_000,
          maxStart: 12_000,
          revision: "r",
        },
      ],
    };
    const job = {
      id: "batch-job",
      attemptsMade: 1,
      data: {
        id: "batch",
        name: QueueJobs.TraceBatch,
        timestamp: new Date(),
        payload,
      },
    } as Job<TQueueJobTypes[QueueName.TraceBatch]>;
    try {
      await expect(traceBatchQueueProcessor(job, undefined)).rejects.toBe(
        failure,
      );
      expect(tokenCountAsync).not.toHaveBeenCalled();
      expect(getTraceBatchEventStream).toHaveBeenCalledWith(payload, {
        maxThreads: 1,
        maxBlockSize: 512,
        requestTimeoutMs: 45_000,
        experimentId: "arm-b",
        queryId: expect.any(String),
      });
      expect(processingSpan.setAttributes).toHaveBeenCalledWith(
        expect.objectContaining({
          "langfuse.trace_batch.experiment_id": "arm-b",
          "langfuse.trace_batch.job_id": "batch-job",
          "langfuse.trace_batch.attempt": 2,
        }),
      );
      expect(processingSpan.setAttributes).toHaveBeenCalledWith(
        expect.objectContaining({
          "langfuse.trace_batch.query_id": vi.mocked(getTraceBatchEventStream)
            .mock.lastCall![1]!.queryId,
        }),
      );
      expect(processingSpan.setAttributes).toHaveBeenLastCalledWith({
        "langfuse.trace_batch.outcome": "failure",
        "langfuse.trace_batch.duration_ms": expect.any(Number),
        "langfuse.trace_batch.observation_count": 1,
        "langfuse.trace_batch.found_trace_count": 1,
        "langfuse.trace_batch.found_project_count": 1,
        "langfuse.trace_batch.input_bytes": 5,
        "langfuse.trace_batch.output_bytes": 6,
        "langfuse.trace_batch.metadata_bytes": 9,
        "langfuse.trace_batch.partial": true,
      });
      expect(log).toHaveBeenCalledWith(
        "Trace batch experiment read",
        expect.objectContaining({
          experimentId: "arm-b",
          buildId: "test-build",
          maxThreads: 1,
          maxBlockSize: 512,
          batchTraceCount: 2,
          jobId: "batch-job",
          attempt: 2,
          queryId: vi.mocked(getTraceBatchEventStream).mock.lastCall![1]!
            .queryId,
        }),
      );
      expect(log).toHaveBeenCalledWith(
        "Trace batch experiment read completed",
        expect.objectContaining({
          jobId: "batch-job",
          attempt: 2,
          queryId: vi.mocked(getTraceBatchEventStream).mock.lastCall![1]!
            .queryId,
          outcome: "failure",
          batchTraceCount: 2,
          batchProjectCount: 2,
          eventTimeSpanMs: 11_000,
          maxTraceSpanMs: 3_000,
          observationCount: 1,
          foundTraceCount: 1,
          foundProjectCount: 1,
          inputBytes: 5,
          outputBytes: 6,
          metadataBytes: 9,
          partial: true,
          durationMs: expect.any(Number),
        }),
      );
      expect(vi.mocked(recordDistribution).mock.calls).toEqual([
        ["langfuse.trace_batch.failed_read_observation_count", 1],
        ["langfuse.trace_batch.failed_read_input_bytes", 5],
      ]);
      expect(recordIncrement).not.toHaveBeenCalled();
    } finally {
      env.LANGFUSE_TRACE_BATCH_MAX_THREADS =
        originalEnv.LANGFUSE_TRACE_BATCH_MAX_THREADS;
      env.LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE =
        originalEnv.LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE;
      env.LANGFUSE_TRACE_BATCH_REQUEST_TIMEOUT_MS =
        originalEnv.LANGFUSE_TRACE_BATCH_REQUEST_TIMEOUT_MS;
      env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID =
        originalEnv.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID;
      env.BUILD_ID = originalEnv.BUILD_ID;
    }
  });
  it.each([false, true])(
    "records partial-read metrics once when transcript processing fails (stream also fails: %s)",
    async (streamFails) => {
      const summaryError = new Error("transcript assembly failed");
      const streamError = new Error("stream failed");
      vi.spyOn(
        traceBatchTranscript,
        "recordTraceBatchTranscript",
      ).mockRejectedValue(summaryError);
      vi.mocked(getTraceBatchEventStream).mockImplementation(
        async function* () {
          for (const traceId of ["a", "b"]) {
            yield {
              project_id: "project",
              trace_id: traceId,
              environment: "default",
              span_id: traceId,
              parent_span_id: null,
              start_time: "2026-09-11 00:00:00.000000",
              event_ts: "2026-09-11 00:00:00.000000",
              type: "GENERATION",
              name: "generation",
              input: "hello",
              output: "",
              metadata: {},
              level: "DEFAULT",
              status_message: null,
              tool_definitions: {},
              tool_calls: [],
              tool_call_names: [],
            };
          }
          if (streamFails) throw streamError;
        },
      );
      const job = {
        data: {
          id: "failed-transcript",
          name: QueueJobs.TraceBatch,
          timestamp: new Date(),
          payload: {
            traces: ["a", "b"].map((traceId) => ({
              projectId: "project",
              traceId,
              minStart: 0,
              maxStart: 1,
              revision: "r",
            })),
          },
        },
      } as Job<TQueueJobTypes[QueueName.TraceBatch]>;
      await expect(traceBatchQueueProcessor(job, undefined)).rejects.toBe(
        streamFails ? streamError : summaryError,
      );
      expect(
        vi
          .mocked(recordDistribution)
          .mock.calls.filter(([name]) =>
            name.startsWith("langfuse.trace_batch.failed_read_"),
          ),
      ).toEqual([
        ["langfuse.trace_batch.failed_read_observation_count", 2],
        ["langfuse.trace_batch.failed_read_input_bytes", 10],
      ]);
      expect(recordDistribution).not.toHaveBeenCalledWith(
        "langfuse.trace_batch.observation_count",
        expect.any(Number),
      );
      expect(summarizeAssembledTrace).not.toHaveBeenCalled();
      expect(recordGauge).toHaveBeenLastCalledWith(
        "langfuse.trace_batch.active_reads",
        0,
      );
    },
  );
  it("retains the active read count when overlapping reads succeed or fail", async () => {
    env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID = "overlapping";
    const log = vi.spyOn(logger, "info").mockImplementation(() => logger);
    const first = Promise.withResolvers<void>();
    const second = Promise.withResolvers<void>();
    const failure = new Error("stream failed");
    vi.mocked(getTraceBatchEventStream)
      .mockImplementationOnce(async function* () {
        await first.promise;
      })
      .mockImplementationOnce(async function* () {
        await second.promise;
        throw failure;
      });
    const job = {
      data: {
        id: "batch",
        name: QueueJobs.TraceBatch,
        timestamp: new Date(),
        payload: {
          traces: [
            {
              projectId: "project",
              traceId: "trace",
              minStart: 0,
              maxStart: 1,
              revision: "r",
            },
          ],
        },
      },
    } as Job<TQueueJobTypes[QueueName.TraceBatch]>;

    const successfulRead = traceBatchQueueProcessor(job, undefined);
    const failedRead = traceBatchQueueProcessor(job, undefined);
    expect(recordGauge).toHaveBeenLastCalledWith(
      "langfuse.trace_batch.active_reads",
      2,
    );
    // Periodic samples keep unchanged, long-running reads visible.
    recordTraceBatchActiveReads();
    first.resolve();
    await successfulRead;
    expect(recordGauge).toHaveBeenLastCalledWith(
      "langfuse.trace_batch.active_reads",
      1,
    );
    second.resolve();
    await expect(failedRead).rejects.toBe(failure);
    for (const outcome of ["success", "failure"]) {
      expect(log).toHaveBeenCalledWith(
        "Trace batch experiment read completed",
        expect.objectContaining({
          outcome,
          batchTraceCount: 1,
          batchProjectCount: 1,
          eventTimeSpanMs: 1,
          maxTraceSpanMs: 1,
          observationCount: 0,
          foundTraceCount: 0,
          foundProjectCount: 0,
          inputBytes: 0,
          outputBytes: 0,
          metadataBytes: 0,
          partial: outcome === "failure",
        }),
      );
    }
    expect(vi.mocked(recordGauge).mock.calls).toEqual([
      ["langfuse.trace_batch.active_reads", 1],
      ["langfuse.trace_batch.active_reads", 2],
      ["langfuse.trace_batch.active_reads", 2],
      ["langfuse.trace_batch.active_reads", 1],
      ["langfuse.trace_batch.active_reads", 0],
    ]);
    expect(recordIncrement).not.toHaveBeenCalled();
  });
  it("counts project/trace pairs with producers disabled and safely repeats reads without retaining payloads", async () => {
    env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID = "successful-read";
    const log = vi.spyOn(logger, "info").mockImplementation(() => logger);
    const ingestionEnabled = env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED;
    const dispatcherEnabled = env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED;
    env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = "false";
    env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED = "false";
    try {
      const payload = {
        traces: [
          ["project", "trace-a"],
          ["project", "trace-b"],
          ["other", "trace-a"],
          ["project", "trace-missing"],
          ["absent", "trace-missing"],
        ].map(([projectId, traceId], index) => ({
          projectId,
          traceId,
          minStart: 1_000 + index * 10_000,
          maxStart: 2_000 + index * 11_000,
          revision: "revision",
        })),
      };
      const job = {
        data: {
          id: "batch",
          name: QueueJobs.TraceBatch,
          timestamp: new Date(),
          payload,
        },
      } as Job<TQueueJobTypes[QueueName.TraceBatch]>;
      let yieldedRows = 0;
      vi.mocked(getTraceBatchEventStream).mockImplementation(
        async function* () {
          for (const [projectId, traceId] of [
            ["project", "trace-a"],
            ["project", "trace-a"],
            ["project", "trace-b"],
            ["other", "trace-a"],
          ]) {
            yieldedRows++;
            yield {
              project_id: projectId,
              trace_id: traceId,
              environment: "default",
              span_id: `span-${yieldedRows}`,
              parent_span_id: null,
              start_time: "2026-09-11 00:00:00.000000",
              event_ts: "2026-09-11 00:00:00.000000",
              type: "GENERATION",
              name: "generation",
              input: "hello",
              output: "世界",
              metadata: { é: "界🙂" },
              level: "DEFAULT",
              status_message: null,
              tool_definitions: {},
              tool_calls: [],
              tool_call_names: [],
            };
          }
        },
      );

      for (let attempt = 0; attempt < 2; attempt++) {
        await expect(traceBatchQueueProcessor(job, undefined)).resolves.toEqual(
          {
            observationCount: 4,
            traceCount: 3,
            projectCount: 2,
            inputBytes: 20,
            outputBytes: 24,
            metadataBytes: 36,
            ioMetadataBytes: 80,
          },
        );
      }
      expect(getTraceBatchEventStream).toHaveBeenCalledTimes(2);
      expect(getTraceBatchEventStream).toHaveBeenCalledWith(
        payload,
        expect.anything(),
      );
      expect(yieldedRows).toBe(8);
      expect(processingSpan.setAttributes).toHaveBeenLastCalledWith({
        "langfuse.trace_batch.outcome": "success",
        "langfuse.trace_batch.duration_ms": expect.any(Number),
        "langfuse.trace_batch.observation_count": 4,
        "langfuse.trace_batch.found_trace_count": 3,
        "langfuse.trace_batch.found_project_count": 2,
        "langfuse.trace_batch.input_bytes": 20,
        "langfuse.trace_batch.output_bytes": 24,
        "langfuse.trace_batch.metadata_bytes": 36,
        "langfuse.trace_batch.partial": false,
      });
      expect(log).toHaveBeenCalledWith(
        "Trace batch experiment read completed",
        expect.objectContaining({
          outcome: "success",
          batchTraceCount: 5,
          batchProjectCount: 3,
          eventTimeSpanMs: 45_000,
          maxTraceSpanMs: 5_000,
          observationCount: 4,
          foundTraceCount: 3,
          foundProjectCount: 2,
          inputBytes: 20,
          outputBytes: 24,
          metadataBytes: 36,
          partial: false,
          durationMs: expect.any(Number),
        }),
      );
      for (const [name, bytes] of [
        ["input_bytes", 20],
        ["output_bytes", 24],
        ["metadata_bytes", 36],
      ] as const) {
        expect(
          vi
            .mocked(recordDistribution)
            .mock.calls.filter(
              ([metric]) => metric === `langfuse.trace_batch.${name}`,
            )
            .map(([, value]) => value),
        ).toEqual([bytes, bytes]);
      }
      expect(recordDistribution).not.toHaveBeenCalledWith(
        "langfuse.trace_batch.io_metadata_bytes",
        expect.anything(),
      );
      expect(recordDistribution).not.toHaveBeenCalledWith(
        "langfuse.trace_batch.found_project_count",
        expect.anything(),
      );
      expect(recordDistribution).toHaveBeenCalledWith(
        "langfuse.trace_batch.missing_trace_count",
        2,
      );
    } finally {
      env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = ingestionEnabled;
      env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED = dispatcherEnabled;
    }
  });

  it.each([true, false])(
    "normalizes persisted single-project jobs and counts returned projects (has rows: %s)",
    async (hasRows) => {
      env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID = hasRows ? undefined : "no-span";
      if (!hasRows) vi.mocked(getCurrentSpan).mockReturnValue(undefined);
      const trace = {
        traceId: "trace",
        minStart: 1_000,
        maxStart: 2_000,
        revision: "revision",
      };
      const job = {
        data: {
          id: "legacy-batch",
          name: QueueJobs.TraceBatch,
          timestamp: new Date().toISOString(),
          payload: { projectId: "project", traces: [trace] },
        },
      } as unknown as Job<TQueueJobTypes[QueueName.TraceBatch]>;
      vi.mocked(getTraceBatchEventStream).mockImplementation(
        async function* () {
          if (!hasRows) return;
          yield {
            project_id: "project",
            trace_id: "trace",
            environment: "default",
            span_id: "span",
            parent_span_id: null,
            start_time: "2026-09-11 00:00:00.000000",
            event_ts: "2026-09-11 00:00:00.000000",
            type: "GENERATION",
            name: "generation",
            input: "hello",
            output: "world",
            metadata: {},
            level: "DEFAULT",
            status_message: null,
            tool_definitions: {},
            tool_calls: [],
            tool_call_names: [],
          };
        },
      );

      await expect(traceBatchQueueProcessor(job, undefined)).resolves.toEqual({
        observationCount: Number(hasRows),
        traceCount: Number(hasRows),
        projectCount: Number(hasRows),
        inputBytes: hasRows ? 5 : 0,
        outputBytes: hasRows ? 5 : 0,
        metadataBytes: 0,
        ioMetadataBytes: hasRows ? 10 : 0,
      });
      // Only Topics batch totals; no experiment attributes without an experiment.
      expect(vi.mocked(processingSpan.setAttributes).mock.calls).toEqual(
        hasRows ? [[{ "langfuse.topics.traces.unchanged": 1 }]] : [],
      );
      expect(recordDistribution).toHaveBeenCalledWith(
        "langfuse.trace_batch.input_bytes",
        hasRows ? 5 : 0,
      );
      expect(recordDistribution).toHaveBeenCalledWith(
        "langfuse.trace_batch.output_bytes",
        hasRows ? 5 : 0,
      );
      expect(recordDistribution).toHaveBeenCalledWith(
        "langfuse.trace_batch.metadata_bytes",
        0,
      );
      expect(recordDistribution).not.toHaveBeenCalledWith(
        "langfuse.trace_batch.found_project_count",
        expect.anything(),
      );
      expect(getTraceBatchEventStream).toHaveBeenCalledWith(
        { traces: [{ ...trace, projectId: "project" }] },
        expect.anything(),
      );
    },
  );
});