🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(trace-batching): expose read insights on processing spans (#17654)
Lysander Kiesel17561cdc21ae
diff --git a/worker/src/features/traceBatching/README.md b/worker/src/features/traceBatching/README.mdindex 8c2f4cf8d5..69b5c4cdbf 100644--- a/worker/src/features/traceBatching/README.md+++ b/worker/src/features/traceBatching/README.md@@ -171,6 +171,19 @@ not unique ingested events. Discarded/unparsed jobs omit stream counters.Use these existing logs to compare batch shape and volume across outcomes byquery ID; no payload contents, project-ID lists or per-batch metric tags are added.+With an experiment ID, the BullMQ processing span also carries attributes under+`langfuse.trace_batch.*`: `experiment_id`, `job_id`, `attempt` (one-based),+`query_id`, `batch_trace_count`, `batch_project_count`, `event_time_span_ms`,+and `max_trace_span_ms`. Shape and query ID are attached before reading, so they+remain available if the stream fails. Completion adds `outcome`, `duration_ms`,+`observation_count`, `found_trace_count`, `found_project_count`, `input_bytes`,+`output_bytes`, `metadata_bytes`, and `partial`, with the same semantics as the+logs. Discards and validation failures omit query/stream fields. These reuse+existing aggregates without additional queries or serialization. Payloads and+project/trace-ID lists are not attached. Dispatcher size estimates remain aggregate+metrics, not per-job span attributes; consumed bytes are not estimated total size.+Span availability follows the existing tracing sampling and retention settings.+For the initial scaling curve, use deterministic trace sampling at 10%, 20%,50%, then 100% across the same project population. Annotate fixed UTC windows;pause tracking and drain old readiness/queued work between settings. Warm up atdiff --git a/worker/src/queues/__tests__/traceBatchQueue.test.ts b/worker/src/queues/__tests__/traceBatchQueue.test.tsindex 0f42729d51..7fd8df8b60 100644--- a/worker/src/queues/__tests__/traceBatchQueue.test.ts+++ b/worker/src/queues/__tests__/traceBatchQueue.test.ts@@ -1,6 +1,8 @@import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";import { type Job } from "bullmq";+import { trace } from "@opentelemetry/api";import {+ getCurrentSpan,getTraceBatchEventStream,logger,QueueJobs,@@ -19,6 +21,7 @@ import {vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({...(await importOriginal<typeof import("@langfuse/shared/src/server")>()),getTraceBatchEventStream: vi.fn(),+ getCurrentSpan: vi.fn(),recordDistribution: vi.fn(),recordGauge: vi.fn(),recordIncrement: vi.fn(),@@ -27,9 +30,16 @@ vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({const originalReadEnabled = env.LANGFUSE_TRACE_BATCH_READ_ENABLED;const originalCloudRegion = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;const originalExperimentId = env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID;+const processingSpan = trace.wrapSpanContext({+ traceId: "0123456789abcdef0123456789abcdef",+ spanId: "0123456789abcdef",+ traceFlags: 1,+});beforeEach(() => {env.LANGFUSE_TRACE_BATCH_READ_ENABLED = "true";env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "DEV";+ vi.mocked(getCurrentSpan).mockReturnValue(processingSpan);+ vi.spyOn(processingSpan, "setAttributes");});afterEach(() => {env.LANGFUSE_TRACE_BATCH_READ_ENABLED = originalReadEnabled;@@ -41,6 +51,7 @@ afterEach(() => {describe("trace batch queue", () => {it("discards self-hosted jobs before parsing even when reads are enabled", async () => {+ env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID = "discarded-read";env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = undefined;const job = {opts: {},@@ -62,6 +73,14 @@ describe("trace batch queue", () => {);expect(recordGauge).not.toHaveBeenCalled();expect(job.opts.removeOnComplete).toBe(true);+ expect(processingSpan.setAttributes).toHaveBeenCalledWith(+ expect.objectContaining({ "langfuse.trace_batch.outcome": "discard" }),+ );+ expect(processingSpan.setAttributes).not.toHaveBeenCalledWith(+ expect.objectContaining({+ "langfuse.trace_batch.observation_count": expect.any(Number),+ }),+ );});it("discards an expired batch before querying, including on retry", async () => {@@ -117,6 +136,16 @@ describe("trace batch queue", () => {const log = vi.spyOn(logger, "info").mockImplementation(() => logger);const failure = new Error("query failed");vi.mocked(getTraceBatchEventStream).mockImplementation(async function* () {+ expect(processingSpan.setAttributes).toHaveBeenCalledWith(+ expect.objectContaining({+ "langfuse.trace_batch.batch_trace_count": 2,+ "langfuse.trace_batch.batch_project_count": 2,+ "langfuse.trace_batch.event_time_span_ms": 11_000,+ "langfuse.trace_batch.max_trace_span_ms": 3_000,+ }),+ );+ // Completion must annotate the processing span captured before child work.+ vi.mocked(getCurrentSpan).mockReturnValue(undefined);yield {project_id: "project",trace_id: "trace",@@ -173,6 +202,30 @@ describe("trace batch queue", () => {experimentId: "arm-b",queryId: expect.any(String),});+ expect(processingSpan.setAttributes).toHaveBeenCalledWith(+ expect.objectContaining({+ "langfuse.trace_batch.experiment_id": "arm-b",+ "langfuse.trace_batch.job_id": "batch-job",+ "langfuse.trace_batch.attempt": 2,+ }),+ );+ expect(processingSpan.setAttributes).toHaveBeenCalledWith(+ expect.objectContaining({+ "langfuse.trace_batch.query_id": vi.mocked(getTraceBatchEventStream)+ .mock.lastCall![1]!.queryId,+ }),+ );+ expect(processingSpan.setAttributes).toHaveBeenLastCalledWith({+ "langfuse.trace_batch.outcome": "failure",+ "langfuse.trace_batch.duration_ms": expect.any(Number),+ "langfuse.trace_batch.observation_count": 1,+ "langfuse.trace_batch.found_trace_count": 1,+ "langfuse.trace_batch.found_project_count": 1,+ "langfuse.trace_batch.input_bytes": 5,+ "langfuse.trace_batch.output_bytes": 6,+ "langfuse.trace_batch.metadata_bytes": 9,+ "langfuse.trace_batch.partial": true,+ });expect(log).toHaveBeenCalledWith("Trace batch experiment read",expect.objectContaining({@@ -396,6 +449,17 @@ describe("trace batch queue", () => {expect.anything(),);expect(yieldedRows).toBe(8);+ expect(processingSpan.setAttributes).toHaveBeenLastCalledWith({+ "langfuse.trace_batch.outcome": "success",+ "langfuse.trace_batch.duration_ms": expect.any(Number),+ "langfuse.trace_batch.observation_count": 4,+ "langfuse.trace_batch.found_trace_count": 3,+ "langfuse.trace_batch.found_project_count": 2,+ "langfuse.trace_batch.input_bytes": 20,+ "langfuse.trace_batch.output_bytes": 24,+ "langfuse.trace_batch.metadata_bytes": 36,+ "langfuse.trace_batch.partial": false,+ });expect(log).toHaveBeenCalledWith("Trace batch experiment read completed",expect.objectContaining({@@ -450,6 +514,8 @@ describe("trace batch queue", () => {it.each([true, false])("normalizes persisted single-project jobs and counts returned projects (has rows: %s)",async (hasRows) => {+ env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID = hasRows ? undefined : "no-span";+ if (!hasRows) vi.mocked(getCurrentSpan).mockReturnValue(undefined);const trace = {traceId: "trace",minStart: 1_000,@@ -495,6 +561,7 @@ describe("trace batch queue", () => {metadataBytes: 0,ioMetadataBytes: hasRows ? 10 : 0,});+ expect(processingSpan.setAttributes).not.toHaveBeenCalled();expect(recordDistribution).toHaveBeenCalledWith("langfuse.trace_batch.input_bytes",hasRows ? 5 : 0,diff --git a/worker/src/queues/traceBatchQueue.ts b/worker/src/queues/traceBatchQueue.tsindex aae8cac673..deb544c28d 100644--- a/worker/src/queues/traceBatchQueue.ts+++ b/worker/src/queues/traceBatchQueue.ts@@ -2,6 +2,7 @@import { type Processor } from "bullmq";import { randomUUID } from "node:crypto";import {+ getCurrentSpan,getTraceBatchEventStream,logger,recordDistribution,@@ -24,6 +25,10 @@ export const traceBatchQueueProcessor: Processor<TQueueJobTypes[QueueName.TraceBatch]> = async (job) => {const startedAt = performance.now();+ // Keep attributes on the processing span rather than a nested read span.+ const span = env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID+ ? getCurrentSpan()+ : undefined;let outcome: "success" | "failure" | "discard" = "failure";let queryId: string | undefined;let batchShape:@@ -41,6 +46,12 @@ export const traceBatchQueueProcessor: Processor<let outputBytes = 0;let metadataBytes = 0;try {+ span?.setAttributes({+ "langfuse.trace_batch.experiment_id":+ env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID,+ "langfuse.trace_batch.job_id": job.id,+ "langfuse.trace_batch.attempt": job.attemptsMade + 1,+ });const disabledReason = !env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION? "not_cloud": env.LANGFUSE_TRACE_BATCH_READ_ENABLED !== "true"@@ -92,6 +103,14 @@ export const traceBatchQueueProcessor: Processor<eventTimeSpanMs: batch.traces.length ? maxStart - minStart : 0,maxTraceSpanMs,};+ span?.setAttributes({+ "langfuse.trace_batch.query_id": queryId,+ "langfuse.trace_batch.batch_trace_count": batchShape.batchTraceCount,+ "langfuse.trace_batch.batch_project_count":+ batchShape.batchProjectCount,+ "langfuse.trace_batch.event_time_span_ms": batchShape.eventTimeSpanMs,+ "langfuse.trace_batch.max_trace_span_ms": batchShape.maxTraceSpanMs,+ });logger.info("Trace batch experiment read", {...queryOptions,jobId: job.id,@@ -177,6 +196,21 @@ export const traceBatchQueueProcessor: Processor<};} finally {const durationMs = performance.now() - startedAt;+ span?.setAttributes({+ "langfuse.trace_batch.outcome": outcome,+ "langfuse.trace_batch.duration_ms": durationMs,+ ...(queryId+ ? {+ "langfuse.trace_batch.observation_count": observationCount,+ "langfuse.trace_batch.found_trace_count": foundTraces.size,+ "langfuse.trace_batch.found_project_count": foundProjects.size,+ "langfuse.trace_batch.input_bytes": inputBytes,+ "langfuse.trace_batch.output_bytes": outputBytes,+ "langfuse.trace_batch.metadata_bytes": metadataBytes,+ "langfuse.trace_batch.partial": outcome !== "success",+ }+ : {}),+ });if (env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID) {logger.info("Trace batch experiment read completed", {experimentId: env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID,