Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(trace-batching): expose read insights on processing spans (#17654)

Lysander Kiesel17561cdc21ae
diff --git a/worker/src/features/traceBatching/README.md b/worker/src/features/traceBatching/README.md
index 8c2f4cf8d5..69b5c4cdbf 100644
--- a/worker/src/features/traceBatching/README.md
+++ b/worker/src/features/traceBatching/README.md
@@ -171,6 +171,19 @@ not unique ingested events. Discarded/unparsed jobs omit stream counters.
Use these existing logs to compare batch shape and volume across outcomes by
query ID; no payload contents, project-ID lists or per-batch metric tags are added.
+With an experiment ID, the BullMQ processing span also carries attributes under
+`langfuse.trace_batch.*`: `experiment_id`, `job_id`, `attempt` (one-based),
+`query_id`, `batch_trace_count`, `batch_project_count`, `event_time_span_ms`,
+and `max_trace_span_ms`. Shape and query ID are attached before reading, so they
+remain available if the stream fails. Completion adds `outcome`, `duration_ms`,
+`observation_count`, `found_trace_count`, `found_project_count`, `input_bytes`,
+`output_bytes`, `metadata_bytes`, and `partial`, with the same semantics as the
+logs. Discards and validation failures omit query/stream fields. These reuse
+existing aggregates without additional queries or serialization. Payloads and
+project/trace-ID lists are not attached. Dispatcher size estimates remain aggregate
+metrics, not per-job span attributes; consumed bytes are not estimated total size.
+Span availability follows the existing tracing sampling and retention settings.
+
For the initial scaling curve, use deterministic trace sampling at 10%, 20%,
50%, then 100% across the same project population. Annotate fixed UTC windows;
pause tracking and drain old readiness/queued work between settings. Warm up at
diff --git a/worker/src/queues/__tests__/traceBatchQueue.test.ts b/worker/src/queues/__tests__/traceBatchQueue.test.ts
index 0f42729d51..7fd8df8b60 100644
--- a/worker/src/queues/__tests__/traceBatchQueue.test.ts
+++ b/worker/src/queues/__tests__/traceBatchQueue.test.ts
@@ -1,6 +1,8 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { type Job } from "bullmq";
+import { trace } from "@opentelemetry/api";
import {
+ getCurrentSpan,
getTraceBatchEventStream,
logger,
QueueJobs,
@@ -19,6 +21,7 @@ import {
vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({
...(await importOriginal<typeof import("@langfuse/shared/src/server")>()),
getTraceBatchEventStream: vi.fn(),
+ getCurrentSpan: vi.fn(),
recordDistribution: vi.fn(),
recordGauge: vi.fn(),
recordIncrement: vi.fn(),
@@ -27,9 +30,16 @@ vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({
const originalReadEnabled = env.LANGFUSE_TRACE_BATCH_READ_ENABLED;
const originalCloudRegion = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;
const originalExperimentId = env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID;
+const processingSpan = trace.wrapSpanContext({
+ traceId: "0123456789abcdef0123456789abcdef",
+ spanId: "0123456789abcdef",
+ traceFlags: 1,
+});
beforeEach(() => {
env.LANGFUSE_TRACE_BATCH_READ_ENABLED = "true";
env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "DEV";
+ vi.mocked(getCurrentSpan).mockReturnValue(processingSpan);
+ vi.spyOn(processingSpan, "setAttributes");
});
afterEach(() => {
env.LANGFUSE_TRACE_BATCH_READ_ENABLED = originalReadEnabled;
@@ -41,6 +51,7 @@ afterEach(() => {
describe("trace batch queue", () => {
it("discards self-hosted jobs before parsing even when reads are enabled", async () => {
+ env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID = "discarded-read";
env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = undefined;
const job = {
opts: {},
@@ -62,6 +73,14 @@ describe("trace batch queue", () => {
);
expect(recordGauge).not.toHaveBeenCalled();
expect(job.opts.removeOnComplete).toBe(true);
+ expect(processingSpan.setAttributes).toHaveBeenCalledWith(
+ expect.objectContaining({ "langfuse.trace_batch.outcome": "discard" }),
+ );
+ expect(processingSpan.setAttributes).not.toHaveBeenCalledWith(
+ expect.objectContaining({
+ "langfuse.trace_batch.observation_count": expect.any(Number),
+ }),
+ );
});
it("discards an expired batch before querying, including on retry", async () => {
@@ -117,6 +136,16 @@ describe("trace batch queue", () => {
const log = vi.spyOn(logger, "info").mockImplementation(() => logger);
const failure = new Error("query failed");
vi.mocked(getTraceBatchEventStream).mockImplementation(async function* () {
+ expect(processingSpan.setAttributes).toHaveBeenCalledWith(
+ expect.objectContaining({
+ "langfuse.trace_batch.batch_trace_count": 2,
+ "langfuse.trace_batch.batch_project_count": 2,
+ "langfuse.trace_batch.event_time_span_ms": 11_000,
+ "langfuse.trace_batch.max_trace_span_ms": 3_000,
+ }),
+ );
+ // Completion must annotate the processing span captured before child work.
+ vi.mocked(getCurrentSpan).mockReturnValue(undefined);
yield {
project_id: "project",
trace_id: "trace",
@@ -173,6 +202,30 @@ describe("trace batch queue", () => {
experimentId: "arm-b",
queryId: expect.any(String),
});
+ expect(processingSpan.setAttributes).toHaveBeenCalledWith(
+ expect.objectContaining({
+ "langfuse.trace_batch.experiment_id": "arm-b",
+ "langfuse.trace_batch.job_id": "batch-job",
+ "langfuse.trace_batch.attempt": 2,
+ }),
+ );
+ expect(processingSpan.setAttributes).toHaveBeenCalledWith(
+ expect.objectContaining({
+ "langfuse.trace_batch.query_id": vi.mocked(getTraceBatchEventStream)
+ .mock.lastCall![1]!.queryId,
+ }),
+ );
+ expect(processingSpan.setAttributes).toHaveBeenLastCalledWith({
+ "langfuse.trace_batch.outcome": "failure",
+ "langfuse.trace_batch.duration_ms": expect.any(Number),
+ "langfuse.trace_batch.observation_count": 1,
+ "langfuse.trace_batch.found_trace_count": 1,
+ "langfuse.trace_batch.found_project_count": 1,
+ "langfuse.trace_batch.input_bytes": 5,
+ "langfuse.trace_batch.output_bytes": 6,
+ "langfuse.trace_batch.metadata_bytes": 9,
+ "langfuse.trace_batch.partial": true,
+ });
expect(log).toHaveBeenCalledWith(
"Trace batch experiment read",
expect.objectContaining({
@@ -396,6 +449,17 @@ describe("trace batch queue", () => {
expect.anything(),
);
expect(yieldedRows).toBe(8);
+ expect(processingSpan.setAttributes).toHaveBeenLastCalledWith({
+ "langfuse.trace_batch.outcome": "success",
+ "langfuse.trace_batch.duration_ms": expect.any(Number),
+ "langfuse.trace_batch.observation_count": 4,
+ "langfuse.trace_batch.found_trace_count": 3,
+ "langfuse.trace_batch.found_project_count": 2,
+ "langfuse.trace_batch.input_bytes": 20,
+ "langfuse.trace_batch.output_bytes": 24,
+ "langfuse.trace_batch.metadata_bytes": 36,
+ "langfuse.trace_batch.partial": false,
+ });
expect(log).toHaveBeenCalledWith(
"Trace batch experiment read completed",
expect.objectContaining({
@@ -450,6 +514,8 @@ describe("trace batch queue", () => {
it.each([true, false])(
"normalizes persisted single-project jobs and counts returned projects (has rows: %s)",
async (hasRows) => {
+ env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID = hasRows ? undefined : "no-span";
+ if (!hasRows) vi.mocked(getCurrentSpan).mockReturnValue(undefined);
const trace = {
traceId: "trace",
minStart: 1_000,
@@ -495,6 +561,7 @@ describe("trace batch queue", () => {
metadataBytes: 0,
ioMetadataBytes: hasRows ? 10 : 0,
});
+ expect(processingSpan.setAttributes).not.toHaveBeenCalled();
expect(recordDistribution).toHaveBeenCalledWith(
"langfuse.trace_batch.input_bytes",
hasRows ? 5 : 0,
diff --git a/worker/src/queues/traceBatchQueue.ts b/worker/src/queues/traceBatchQueue.ts
index aae8cac673..deb544c28d 100644
--- a/worker/src/queues/traceBatchQueue.ts
+++ b/worker/src/queues/traceBatchQueue.ts
@@ -2,6 +2,7 @@
import { type Processor } from "bullmq";
import { randomUUID } from "node:crypto";
import {
+ getCurrentSpan,
getTraceBatchEventStream,
logger,
recordDistribution,
@@ -24,6 +25,10 @@ export const traceBatchQueueProcessor: Processor<
TQueueJobTypes[QueueName.TraceBatch]
> = async (job) => {
const startedAt = performance.now();
+ // Keep attributes on the processing span rather than a nested read span.
+ const span = env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID
+ ? getCurrentSpan()
+ : undefined;
let outcome: "success" | "failure" | "discard" = "failure";
let queryId: string | undefined;
let batchShape:
@@ -41,6 +46,12 @@ export const traceBatchQueueProcessor: Processor<
let outputBytes = 0;
let metadataBytes = 0;
try {
+ span?.setAttributes({
+ "langfuse.trace_batch.experiment_id":
+ env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID,
+ "langfuse.trace_batch.job_id": job.id,
+ "langfuse.trace_batch.attempt": job.attemptsMade + 1,
+ });
const disabledReason = !env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION
? "not_cloud"
: env.LANGFUSE_TRACE_BATCH_READ_ENABLED !== "true"
@@ -92,6 +103,14 @@ export const traceBatchQueueProcessor: Processor<
eventTimeSpanMs: batch.traces.length ? maxStart - minStart : 0,
maxTraceSpanMs,
};
+ span?.setAttributes({
+ "langfuse.trace_batch.query_id": queryId,
+ "langfuse.trace_batch.batch_trace_count": batchShape.batchTraceCount,
+ "langfuse.trace_batch.batch_project_count":
+ batchShape.batchProjectCount,
+ "langfuse.trace_batch.event_time_span_ms": batchShape.eventTimeSpanMs,
+ "langfuse.trace_batch.max_trace_span_ms": batchShape.maxTraceSpanMs,
+ });
logger.info("Trace batch experiment read", {
...queryOptions,
jobId: job.id,
@@ -177,6 +196,21 @@ export const traceBatchQueueProcessor: Processor<
};
} finally {
const durationMs = performance.now() - startedAt;
+ span?.setAttributes({
+ "langfuse.trace_batch.outcome": outcome,
+ "langfuse.trace_batch.duration_ms": durationMs,
+ ...(queryId
+ ? {
+ "langfuse.trace_batch.observation_count": observationCount,
+ "langfuse.trace_batch.found_trace_count": foundTraces.size,
+ "langfuse.trace_batch.found_project_count": foundProjects.size,
+ "langfuse.trace_batch.input_bytes": inputBytes,
+ "langfuse.trace_batch.output_bytes": outputBytes,
+ "langfuse.trace_batch.metadata_bytes": metadataBytes,
+ "langfuse.trace_batch.partial": outcome !== "success",
+ }
+ : {}),
+ });
if (env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID) {
logger.info("Trace batch experiment read completed", {
experimentId: env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID,