🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
chore(worker): drop duplicate trace-batch metrics (#18241)
Lysander Kiesel5fdf5a7088dc
diff --git a/worker/src/__tests__/traceBatching.test.ts b/worker/src/__tests__/traceBatching.test.tsindex ef9e37e4dd..4fe9f45858 100644--- a/worker/src/__tests__/traceBatching.test.ts+++ b/worker/src/__tests__/traceBatching.test.ts@@ -537,13 +537,16 @@ describe("trace micro-batch scheduling with Redis", () => {);expect(Number(sample?.[1])).toBeGreaterThan(0);}- for (const type of ["waiting", "delayed"]) {- expect(recordGauge).toHaveBeenCalledWith(- "langfuse.trace_batch.queue_depth",- type === "waiting" ? 2 : 1,- { type, unit: "records" },- );- }+ expect(recordGauge).toHaveBeenCalledWith(+ "langfuse.trace_batch.queue_depth",+ 1,+ { type: "delayed", unit: "records" },+ );+ expect(recordGauge).not.toHaveBeenCalledWith(+ "langfuse.trace_batch.queue_depth",+ expect.anything(),+ expect.objectContaining({ type: "waiting" }),+ );expect(recordGauge).toHaveBeenCalledWith("langfuse.trace_batch.queue_waiting_head_age_ms",5_000,@@ -563,7 +566,7 @@ describe("trace micro-batch scheduling with Redis", () => {"langfuse.trace_batch.active_reads",0,);- expect(vi.mocked(recordGauge).mock.calls).toHaveLength(10);+ expect(vi.mocked(recordGauge).mock.calls).toHaveLength(7);expect(vi.mocked(recordGauge).mock.calls.every(([, value]) => value === 0),).toBe(true);@@ -854,10 +857,6 @@ describe("trace micro-batch scheduling with Redis", () => {traceCount: 2,projectCount: 1,});- expect(recordDistribution).toHaveBeenCalledWith(- "langfuse.trace_batch.found_project_count",- 1,- );// All three observations must carry their untruncated I/O and metadata.expect(result.ioMetadataBytes).toBeGreaterThanOrEqual(3 * (Buffer.byteLength(input) + Buffer.byteLength(output) + 8_000),@@ -866,9 +865,9 @@ describe("trace micro-batch scheduling with Redis", () => {"langfuse.trace_batch.size",2,);- expect(recordDistribution).toHaveBeenCalledWith(+ expect(recordDistribution).not.toHaveBeenCalledWith("langfuse.trace_batch.io_metadata_bytes",- result.ioMetadataBytes,+ expect.anything(),);expect(await client().zcard(dueKey)).toBe(0);expect(await client().hlen(stateKey)).toBe(0);@@ -927,9 +926,13 @@ describe("trace micro-batch scheduling with Redis", () => {1_000 - expectedCount,{ decision: "excluded" },);+ expect(recordIncrement).not.toHaveBeenCalledWith(+ "langfuse.trace_batch.event_updates",+ expect.anything(),+ { stage: "sampled" },+ );for (const [stage, count] of [["eligible", 1_000],- ["sampled", expectedCount],...(expectedCount > 0 ? [["recorded", expectedCount] as const] : []),] as const) {expect(recordIncrement).toHaveBeenCalledWith(@@ -968,9 +971,9 @@ describe("trace micro-batch scheduling with Redis", () => {1_001,{ stage: "eligible" },);- expect(recordIncrement).toHaveBeenCalledWith(+ expect(recordIncrement).not.toHaveBeenCalledWith("langfuse.trace_batch.event_updates",- 1_001,+ expect.anything(),{ stage: "sampled" },);expect(recordIncrement).toHaveBeenCalledWith(@@ -1696,6 +1699,7 @@ describe("trace micro-batch scheduling with Redis", () => {expect(recordIncrement).toHaveBeenCalledWith("langfuse.trace_batch.expired_traces",1_000,+ { source: "ingestion" },);await trackTraceBatchActivity("project", [event("refreshed", 3_000_000)]);expect((await client().hkeys(stateKey)).sort()).toEqual([@@ -1706,6 +1710,7 @@ describe("trace micro-batch scheduling with Redis", () => {expect(recordIncrement).toHaveBeenCalledWith("langfuse.trace_batch.expired_traces",1,+ { source: "ingestion" },);expect(await queue.getWaitingCount()).toBe(0);});@@ -1727,6 +1732,11 @@ describe("trace micro-batch scheduling with Redis", () => {env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = "false";const dispatcher = runner();await dispatcher.processBatch();+ expect(recordIncrement).toHaveBeenCalledWith(+ "langfuse.trace_batch.expired_traces",+ expect.any(Number),+ { source: "dispatcher" },+ );expect(vi.mocked(recordIncrement)diff --git a/worker/src/env.ts b/worker/src/env.tsindex 482d20dd59..288f17a32a 100644--- a/worker/src/env.ts+++ b/worker/src/env.ts@@ -121,6 +121,9 @@ const EnvSchema = z.object({.enum(["true", "false"]).default("false"),LANGFUSE_TRACE_BATCH_READ_ENABLED: z.enum(["true", "false"]).default("false"),+ LANGFUSE_TRACE_BATCH_TRANSCRIPT_METRICS_ENABLED: z+ .enum(["true", "false"])+ .default("false"),LANGFUSE_TRACE_BATCH_CONCURRENCY: z.coerce.number().int()diff --git a/worker/src/features/otel-ingestion/processOtelEvents.ts b/worker/src/features/otel-ingestion/processOtelEvents.tsindex df3f49eca4..fb652b12f0 100644--- a/worker/src/features/otel-ingestion/processOtelEvents.ts+++ b/worker/src/features/otel-ingestion/processOtelEvents.ts@@ -2,7 +2,6 @@ import { convertEventRecordToObservationForEval } from "@langfuse/shared";import {logger,OtelIngestionProcessor,- recordDistribution,traceException,} from "@langfuse/shared/src/server";import { env } from "../../env";@@ -174,10 +173,6 @@ export async function processOtelEvents({);if (traceBatchEvents.length > 0) {- recordDistribution(- "langfuse.trace_batch.ingestion_trace_count",- new Set(traceBatchEvents.map((event) => event.traceId)).size,- );// The writer has accepted these records; its flush completes separately.await trackTraceBatchActivity(projectId, traceBatchEvents);}diff --git a/worker/src/features/traceBatching/README.md b/worker/src/features/traceBatching/README.mdindex 4753e34379..2bc8324c31 100644--- a/worker/src/features/traceBatching/README.md+++ b/worker/src/features/traceBatching/README.md@@ -2,28 +2,30 @@This read-only experiment measures full-event reads and transcript assembly foridle traces. It requires-`NEXT_PUBLIC_LANGFUSE_CLOUD_REGION` and explicit opt-in. All four enablement-flags below default to `false`; an ordinary release needs no infrastructure-changes. These internal controls are intentionally absent from env templates.+`NEXT_PUBLIC_LANGFUSE_CLOUD_REGION` and explicit opt-in. The enablement flags+and the transcript-metrics switch below default to `false`; an ordinary release+needs no infrastructure changes. These internal controls are intentionally+absent from env templates.## Controls-| Variable | Default | Role |-| --------------------------------------------- | --------- | ---------------------------------------------------------------------------------------------------------- |-| `LANGFUSE_TRACE_BATCH_INGESTION_ENABLED` | `false` | Track accepted direct-v4 event writes in Redis. |-| `LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED` | `false` | Schedule idle traces into BullMQ jobs. |-| `QUEUE_CONSUMER_TRACE_BATCH_QUEUE_IS_ENABLED` | `false` | Register the batch worker. |-| `LANGFUSE_TRACE_BATCH_READ_ENABLED` | `false` | Allow the worker to query ClickHouse; otherwise discard jobs. |-| `LANGFUSE_TRACE_BATCH_SAMPLING_RATE` | `1` | Fraction admitted by ingestion, from 0 to 1. |-| `LANGFUSE_TRACE_BATCH_STRATEGY` | `project` | Choose project-order packing or opt-in locality grouping. |-| `LANGFUSE_TRACE_BATCH_MAX_SIZE` | `60` | Maximum traces per job, up to 10,000. |-| `LANGFUSE_TRACE_BATCH_MAX_THREADS` | `2` | ClickHouse threads per query (positive integer). |-| `LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE` | unset | Optional positive `max_block_size` hint; unset inherits the server profile. |-| `LANGFUSE_TRACE_BATCH_EXPERIMENT_ID` | unset | Optional 1–64 character attempt/query label: letters, digits, `.`, `_`, `-`; first character alphanumeric. |-| `LANGFUSE_TRACE_BATCH_CONCURRENCY` | `2` | Concurrent batch jobs per worker process. |-| `LANGFUSE_TRACE_BATCH_IDLE_MS` | `600000` | Idle time before a trace is ready (10 minutes). |-| `LANGFUSE_TRACE_BATCH_PENDING_TTL_MS` | `7200000` | Pending retention and inactivity expiry (2 hours). |-| `LANGFUSE_TRACE_BATCH_DISPATCH_INTERVAL_MS` | `30000` | Dispatcher cadence (30 seconds). |+| Variable | Default | Role |+| ------------------------------------------------- | --------- | ---------------------------------------------------------------------------------------------------------- |+| `LANGFUSE_TRACE_BATCH_INGESTION_ENABLED` | `false` | Track accepted direct-v4 event writes in Redis. |+| `LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED` | `false` | Schedule idle traces into BullMQ jobs. |+| `QUEUE_CONSUMER_TRACE_BATCH_QUEUE_IS_ENABLED` | `false` | Register the batch worker. |+| `LANGFUSE_TRACE_BATCH_READ_ENABLED` | `false` | Allow the worker to query ClickHouse; otherwise discard jobs. |+| `LANGFUSE_TRACE_BATCH_TRANSCRIPT_METRICS_ENABLED` | `false` | Emit transcript size distributions. Span attributes and failure counters stay on either way. |+| `LANGFUSE_TRACE_BATCH_SAMPLING_RATE` | `1` | Fraction admitted by ingestion, from 0 to 1. |+| `LANGFUSE_TRACE_BATCH_STRATEGY` | `project` | Choose project-order packing or opt-in locality grouping. |+| `LANGFUSE_TRACE_BATCH_MAX_SIZE` | `60` | Maximum traces per job, up to 10,000. |+| `LANGFUSE_TRACE_BATCH_MAX_THREADS` | `2` | ClickHouse threads per query (positive integer). |+| `LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE` | unset | Optional positive `max_block_size` hint; unset inherits the server profile. |+| `LANGFUSE_TRACE_BATCH_EXPERIMENT_ID` | unset | Optional 1–64 character attempt/query label: letters, digits, `.`, `_`, `-`; first character alphanumeric. |+| `LANGFUSE_TRACE_BATCH_CONCURRENCY` | `2` | Concurrent batch jobs per worker process. |+| `LANGFUSE_TRACE_BATCH_IDLE_MS` | `600000` | Idle time before a trace is ready (10 minutes). |+| `LANGFUSE_TRACE_BATCH_PENDING_TTL_MS` | `7200000` | Pending retention and inactivity expiry (2 hours). |+| `LANGFUSE_TRACE_BATCH_DISPATCH_INTERVAL_MS` | `30000` | Dispatcher cadence (30 seconds). |Producer, dispatcher and consumer can run independently. Sampling reuses theevaluator's deterministic trace-ID sampler; every observation of the same trace@@ -77,17 +79,20 @@ every member; `estimates_unavailable` counts unknown traces and affected batchesEstimates stay out of queue payloads, job IDs and batch selection. They do not capwork or change oversized-trace handling. Dispatch retries can emit another sample.-Serialized event bytes differ from the reader's `io_metadata_bytes` metric: they-include the serialized event fields and JSON encoding, and do not represent RAM,-network transfer or ClickHouse scan bytes. After rollout, collect a stable hour+Serialized event bytes differ from the reader's `input_bytes`, `output_bytes`+and `metadata_bytes` metrics: they include the serialized event fields and JSON+encoding, and do not represent RAM, network transfer or ClickHouse scan bytes.+The job return still sums those three as `ioMetadataBytes`; that sum is not a+custom metric. After rollout, collect a stable hourwith unchanged sampling and batching settings; compare trace/batch distributions,unknown-estimate counts, ingestion latency and Redis command rate/CPU/memory withthe baseline before choosing any size-aware batching policy.The worker streams full `events_full` payloads and records batch observation,-trace, project and logical payload-byte metrics. The `input_bytes`, `output_bytes`+trace and logical payload-byte metrics. The `input_bytes`, `output_bytes`and `metadata_bytes` distributions under `langfuse.trace_batch` each report one-total per completed batch read; `io_metadata_bytes` retains their combined total.+total per completed batch read. Distinct project count stays on the span and+completion log.Sizes count UTF-8 bytes, including metadata keys and values, excluding JSONtransport framing/escaping and compression. They do not measure ClickHouse bytesscanned or per-trace ingestion size; retries can record another batch sample.@@ -117,33 +122,37 @@ events can arrive. Event versions follow the current ClickHouse merge state;the shared ordering function keeps one observation per ID. Retries can repeatper-trace samples.-These distributions use the `langfuse.trace_batch` prefix:--| Metric | Sample |-| --- | --- |-| `transcript_assembly_duration_ms` | One trace's ordering and assembly time, excluding I/O conversion, stream waits and tokenization; `has_transcript:true\|false`. |-| `transcript_json_characters` | UTF-16 length of `JSON.stringify(assembleTranscript(...))`, including thread/message wrappers and provenance; zero for a null transcript. Same admitted traces as the Topics length metrics. |-| `transcript_json_tokens` | o200k estimate of that complete JSON string, tagged `tokenizer:o200k_base`; zero for a null transcript. Attempted on the same admitted traces as `topics_transcript_tokens`; unavailable estimates are omitted. |-| `generic_transcript_characters` | UTF-16 length of the previous generic plain-text layout, rendered with the same Topics block caps and inclusions. Tool definitions or errors may still render when assembly returns null. |-| `generic_transcript_tokens` | o200k estimate of that generic text, tagged `tokenizer:o200k_base`. |-| `transcript_message_tokens` | Sum of current-turn and history message-projection token estimates; zero when empty, omitted if either estimate is unavailable. Distinct from full-transcript JSON. |-| `transcript_current_turn_tokens` | Token estimate of current-turn messages across all threads, using role and parts only; zero when empty. |-| `transcript_history_tokens` | Token estimate of history messages across all threads, using the same role/parts representation; zero when empty. |-| `transcript_content_characters` | Sum of JSON-serialized message-part lengths across history and current turn, in UTF-16 code units; excludes message wrappers and provenance. |-| `transcript_tool_response_characters` | The subset of content characters belonging to tool-role messages or unmatched tool-result parts; zero when absent. |-| `transcript_thread_count` | Number of assembled conversation threads, or zero for a null transcript. |-| `transcript_observation_count` | Ordered, deduplicated observations passed to assembly and the Topics renderer. The `observation_count` span attribute is the raw row count before deduplication. |-| `transcript_history_message_count`, `transcript_history_part_count` | Replayed messages and their normalized parts across all assembled threads. |-| `transcript_current_turn_message_count`, `transcript_current_turn_part_count` | This-trace messages and their normalized parts across all assembled threads. |-| `transcript_current_turn_tool_call_count`, `transcript_current_turn_tool_result_count` | Tool-call parts and tool-result parts in this trace; a tool-role part also counts as a result. |-| `transcript_assembly_phase_duration_ms` | Two non-overlapping samples per trace tagged `phase:normalization` or `phase:matching`. Normalization includes initial message-key construction and partitioning; matching covers remaining assembly work, including tool matching, deduplication, any rebuilt keys and finalization. Observation ordering is outside these phases but remains in total assembly time. |-| `topics_transcript_characters` | UTF-16 length of the complete rendered Topics text, including labels, section headings and line breaks; admitted traces only. |-| `topics_transcript_tokens` | o200k estimate of that same text, tagged `tokenizer:o200k_base`; admitted traces only. |-| `transcript_comparison_render_duration_ms` | Time to serialize assembled JSON and render generic and Topics text, excluding tokenization. |-| `transcript_comparison_tokenization_duration_ms` | Wall time for the three sequential JSON, generic and Topics token estimates, excluding the existing current-turn/history estimates. |-| `topics_transcript_block_characters` | Present block content length tagged `block:user\|assistant\|system\|reasoning\|tool_calls\|tool_results\|tool_definitions\|errors\|run_io\|observations` and `stage:raw\|clipped`; absent blocks emit no sample. |-| `topics_transcript_blocks_cut` | Number of content blocks cut by their per-block character caps. |-| `topics_transcript_history_characters`, `topics_transcript_current_turn_characters`, `topics_transcript_history_share` | Rendered content-line characters from replayed input and this run, plus replayed input divided by their sum (0 when both are empty). This-run content includes fallback trace-level I/O, observation markers, and inline errors. |+These values are always attributes on the `trace-batch-transcript` span. With+`LANGFUSE_TRACE_BATCH_TRANSCRIPT_METRICS_ENABLED=true` they are also+`langfuse.trace_batch.*` distributions. That switch defaults to `false`.+Failure and unavailable estimates still increment the counters named in the+paragraphs below, with or without the switch.++| Metric | Sample |+| ---------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |+| `transcript_assembly_duration_ms` | One trace's ordering and assembly time, excluding I/O conversion, stream waits and tokenization; `has_transcript:true\|false`. |+| `transcript_json_characters` | UTF-16 length of `JSON.stringify(assembleTranscript(...))`, including thread/message wrappers and provenance; zero for a null transcript. Same admitted traces as the Topics length metrics. |+| `transcript_json_tokens` | o200k estimate of that complete JSON string, tagged `tokenizer:o200k_base`; zero for a null transcript. Attempted on the same admitted traces as `topics_transcript_tokens`; unavailable estimates are omitted. |+| `generic_transcript_characters` | UTF-16 length of the previous generic plain-text layout, rendered with the same Topics block caps and inclusions. Tool definitions or errors may still render when assembly returns null. |+| `generic_transcript_tokens` | o200k estimate of that generic text, tagged `tokenizer:o200k_base`. |+| `transcript_message_tokens` | Sum of current-turn and history message-projection token estimates; zero when empty, omitted if either estimate is unavailable. Distinct from full-transcript JSON. |+| `transcript_current_turn_tokens` | Token estimate of current-turn messages across all threads, using role and parts only; zero when empty. |+| `transcript_history_tokens` | Token estimate of history messages across all threads, using the same role/parts representation; zero when empty. |+| `transcript_content_characters` | Sum of JSON-serialized message-part lengths across history and current turn, in UTF-16 code units; excludes message wrappers and provenance. |+| `transcript_tool_response_characters` | The subset of content characters belonging to tool-role messages or unmatched tool-result parts; zero when absent. |+| `transcript_thread_count` | Number of assembled conversation threads, or zero for a null transcript. |+| `transcript_observation_count` | Ordered, deduplicated observations passed to assembly and the Topics renderer. The `observation_count` span attribute is the raw row count before deduplication. |+| `transcript_history_message_count`, `transcript_history_part_count` | Replayed messages and their normalized parts across all assembled threads. |+| `transcript_current_turn_message_count`, `transcript_current_turn_part_count` | This-trace messages and their normalized parts across all assembled threads. |+| `transcript_current_turn_tool_call_count`, `transcript_current_turn_tool_result_count` | Tool-call parts and tool-result parts in this trace; a tool-role part also counts as a result. |+| `transcript_assembly_phase_duration_ms` | Two non-overlapping samples per trace tagged `phase:normalization` or `phase:matching`. Normalization includes initial message-key construction and partitioning; matching covers remaining assembly work, including tool matching, deduplication, any rebuilt keys and finalization. Observation ordering is outside these phases but remains in total assembly time. |+| `topics_transcript_characters` | UTF-16 length of the complete rendered Topics text, including labels, section headings and line breaks; admitted traces only. |+| `topics_transcript_tokens` | o200k estimate of that same text, tagged `tokenizer:o200k_base`; admitted traces only. |+| `transcript_comparison_render_duration_ms` | Time to serialize assembled JSON and render generic and Topics text, excluding tokenization. |+| `transcript_comparison_tokenization_duration_ms` | Wall time for the three sequential JSON, generic and Topics token estimates, excluding the existing current-turn/history estimates. |+| `topics_transcript_block_characters` | Present block content length tagged `block:user\|assistant\|system\|reasoning\|tool_calls\|tool_results\|tool_definitions\|errors\|run_io\|observations` and `stage:raw\|clipped`; absent blocks emit no sample. |+| `topics_transcript_blocks_cut` | Number of content blocks cut by their per-block character caps. |+| `topics_transcript_history_characters`, `topics_transcript_current_turn_characters`, `topics_transcript_history_share` | Rendered content-line characters from replayed input and this run, plus replayed input divided by their sum (0 when both are empty). This-run content includes fallback trace-level I/O, observation markers, and inline errors. |Topics measurement uses the existing assembled transcript and the inclusiveTopics preset in `packages/shared/src/server/transcript/topics-renderer-config.ts`. It@@ -182,20 +191,19 @@ and worker; every admitted trace receives Topics measurements.Transcript token counts use the existing local worker-thread pool and bundledtiktoken WASM, without a network or model API call. The `gpt-4o` configurationselects `o200k_base`, also used by GPT-5 mini and nano-([OpenAI mapping](https://github.com/openai/tiktoken/blob/main/tiktoken/model.py)); metrics are tagged-`tokenizer:o200k_base`. These are serialized-payload estimates, not provider+([OpenAI mapping](https://github.com/openai/tiktoken/blob/main/tiktoken/model.py)).+The span records `tokenizer:o200k_base`. These are serialized-payload estimates, not providerbilling counts or a model's full request framing.Current turn and history use identical `{ messages: [{ role, parts }] }` JSONframing, without observation provenance. Only nonempty partitions are tokenized,at most twice per trace. Their sum is `transcript_message_tokens`; it is notdirectly comparable to the full assembled-transcript JSON or rendered text. The-`transcript_json_*`, `generic_transcript_*`, and `topics_transcript_*` length-distributions are attempted on the same admitted trace cohort and can be-compared at p50/p90/p99 after checking their sample counts. The generic+`transcript_json_*`, `generic_transcript_*`, and `topics_transcript_*` lengths+are span attributes on the same admitted trace cohort. The genericrenderer preserves the previous plain-text layout under the same Topicsper-block preset; it has no trace-root I/O or run-state sections. Emptytranscripts have zero JSON length and tokens. Thread counts are numeric-samples, never metric tags.+span attributes.To diagnose large Topics texts, compare `topics_transcript_tokens` with`transcript_observation_count`, the history/current-turn message and part counts,@@ -237,8 +245,11 @@ trace's observations. Before submitting another transcript it awaits the previoupromise. Projections are tokenized sequentially, keeping at most one pendingtokenizer request per batch. Success and streamfailure both drain accepted promises; a stream failure never assembles its-partial final trace. Assembly itself remains synchronous. `read_duration_ms`-includes all this processing.+partial final trace. Assembly itself remains synchronous. Wall-clock duration+of that work, including tokenization, is the span attribute `duration_ms` and+the experiment completion log. The generic queue series+`langfuse.queue.trace_batch.time_distribution` with `type:processing` records+the same duration when the processor returns.Memory includes the current trace, the previous transcript being tokenized andstream buffers, multiplied by active batch jobs. Each trace also retains its@@ -255,11 +266,10 @@ CPU and queue depth before raising batch concurrency.ClickHouse must sort the filtered result, so compare query memory and latency against the unorderedbaseline before increasing load.-Enable percentile aggregations for these distribution metrics in Datadog-Metrics Summary, then select p50, p75, p90, p95 and p99 in Metrics Explorer.-Datadog computes these across workers; do not average worker percentiles.-See [Datadog distributions](https://docs.datadoghq.com/metrics/distributions/).-Metric delivery and percentile configuration must be verified after deployment.+Transcript sizes are always span attributes. The same measurements are also+dogstatsd distributions when `LANGFUSE_TRACE_BATCH_TRANSCRIPT_METRICS_ENABLED`+is `true`. That switch defaults to `false`. Use the trace explorer query below+either way.### Inspect individual transcripts in Datadog@@ -281,7 +291,7 @@ It also records the remaining Topics metrics above. Block character attributes u`topics_transcript_block_<block>_<stage>_characters`. Topics character metricsare absent if rendering failed; the token metric can also be absent if itsestimate failed.-No transcript content is attached, and IDs are not distribution metric tags.+No transcript content is attached. Project and trace IDs stay on the span.`transcript_characters` is the legacy span-only name for the same value as`transcript_json_characters`. The full JSON includes syntax and provenance, souse `transcript_content_characters` as the tool-response denominator. JSON@@ -311,8 +321,8 @@ and retries can produce different or repeated samples for the same trace.These spans follow existing APM ingestion sampling and retention. Configure a[custom retention filter](https://docs.datadoghq.com/tracing/trace_pipeline/trace_retention/)at 100% for the outlier query to keep matching ingested spans searchable. This-cannot recover spans dropped before ingestion. Distribution metrics remain-independent and cannot identify a historical sample's trace retroactively.+cannot recover spans dropped before ingestion. Transcript sizes live on these+spans and follow APM sampling; they are not separate custom metrics.## Capacity measurements@@ -320,19 +330,17 @@ The following metrics use the `langfuse.trace_batch` prefix. All additions areinside the existing cloud experiment paths; normal ingestion with trackingdisabled performs no new aggregation or Redis calls.-| Metric | Kind / tags | Meaning |-| --- | --- | --- |-| `event_updates`, `serialized_event_bytes` | Counters; `stage:eligible\|sampled\|recorded` | Accepted, valid-start-time updates before sampling, after sampling, and in Redis-acknowledged chunks. Repeated updates count again. |-| `read_attempts` | Counter; `outcome:success\|failure\|discard` | One outcome per processor invocation, including retries, validation failures and disabled/expired discards. |-| `read_duration_ms` | Distribution; same outcome tags | Wall-clock processor duration, including failed and discarded attempts. |-| `active_reads` | Per-process gauge | Streams currently being consumed; failures decrement the count too. |-| `failed_read_observation_count`, `failed_read_input_bytes`, `failed_read_output_bytes`, `failed_read_metadata_bytes`, `failed_read_io_metadata_bytes` | Distributions | Partial logical rows/bytes consumed before a stream failure. Separate from successful throughput. |-| `queue_depth` | Gauge; `type:waiting\|active\|delayed\|failed` | Global BullMQ snapshots; waiting includes paused work. |-| `queue_waiting_head_age_ms` | Gauge | Maximum creation age of the next FIFO jobs in waiting/paused lists. Zero when empty. |-| `redis_key_bytes` | Gauge; `key:due\|state` | Estimated memory of each readiness key, including Redis overhead. |-| `redis_key_entries` | Gauge; `key:due\|state` | `ZCARD` / `HLEN` at the memory snapshot. |--The eligible/sampled counters reuse the already-computed serialized size and+| Metric | Kind / tags | Meaning |+| ---------------------------------------------------------- | ------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |+| `event_updates`, `serialized_event_bytes` | Counters; `stage:eligible\|recorded` | Accepted, valid-start-time updates before sampling, and in Redis-acknowledged chunks. Repeated updates count again. Selected volume is `sampling_decisions`. |+| `active_reads` | Per-process gauge | Streams currently being consumed; failures decrement the count too. |+| `failed_read_observation_count`, `failed_read_input_bytes` | Distributions | Partial logical rows and input bytes consumed before a stream failure. Separate from successful throughput. Output and metadata bytes stay on the completion log. |+| `queue_depth` | Gauge; `type:delayed` | Delayed BullMQ jobs. Waiting, active, and failed depth stay on `langfuse.queue.trace_batch.depth`. Request rate, wait time, and processing time use the generic `langfuse.queue.trace_batch` series, the same as every other queue. Processing time is recorded when the processor returns. Discard reasons stay on `discarded_jobs`. |+| `queue_waiting_head_age_ms` | Gauge | Maximum creation age of the next FIFO jobs in waiting/paused lists. Zero when empty. |+| `redis_key_bytes` | Gauge; `key:due\|state` | Estimated memory of each readiness key, including Redis overhead. |+| `redis_key_entries` | Gauge; `key:due\|state` | `ZCARD` / `HLEN` at the memory snapshot. |++The eligible and recorded counters reuse the already-computed serialized size andper-trace aggregates. Recorded volume is emitted after each acknowledged Luachunk: earlier chunks remain counted if a later one fails. A timeout can meanRedis applied an update without acknowledgement, so the stage difference includes@@ -466,8 +474,9 @@ cost is O(n × k × cap), where k is the fewest feasible jobs. Measure durationbefore increasing scale, especially with distant event times that force many jobs.Compare `dispatched_batches` (tags `strategy`, `fill`),-`event_time_envelope_ms`, `observed_start_span_ms`, `candidate_buffer_size` and-`selector_duration_ms` under `langfuse.trace_batch`. The envelope metric includes+`event_time_envelope_ms`, `candidate_buffer_size` and+`selector_duration_ms` under `langfuse.trace_batch`. Per-trace event span stays+on the completion log as `maxTraceSpanMs`. The envelope metric includesthe reader's two-minute margin on each side; the selector's envelope limit doesnot. Payload and Redis state are unchanged.@@ -485,6 +494,8 @@ worker/container rather than a fleet average:- `runtime.node.mem.heap_used`: JavaScript heap in use.- `langfuse.trace_batch.snapshot_size`, `pending_traces`, `ready_traces` and`oldest_due_age_ms`: snapshot and backlog growth.+- `langfuse.trace_batch.expired_traces`: tagged `source:ingestion` or+ `source:dispatcher`. Sum within one source; the two writers are different events.- `langfuse.periodic_runner.duration_ms` and `completed`, filtered by`runner:trace_batch_dispatcher`: run duration and outcome.diff --git a/worker/src/features/traceBatching/TraceBatchMetricsRunner.ts b/worker/src/features/traceBatching/TraceBatchMetricsRunner.tsindex fd79cbc1ad..afa70fa532 100644--- a/worker/src/features/traceBatching/TraceBatchMetricsRunner.ts+++ b/worker/src/features/traceBatching/TraceBatchMetricsRunner.ts@@ -75,25 +75,12 @@ export class TraceBatchMetricsRunner extends PeriodicRunner {private async collectQueueDepth(): Promise<void> {const queue = TraceBatchQueue.getInstance();if (!queue) throw new Error("Trace batch queue is unavailable");- const counts = await queue.getJobCounts(- "waiting",- "paused",- "active",- "delayed",- "failed",- );- const depths = {- waiting: (counts.waiting ?? 0) + (counts.paused ?? 0),- active: counts.active ?? 0,- delayed: counts.delayed ?? 0,- failed: counts.failed ?? 0,- };- for (const [type, depth] of Object.entries(depths)) {- recordGauge(`${METRIC_PREFIX}.queue_depth`, depth, {- type,- unit: "records",- });- }+ // Waiting, active, and failed depth are already on the generic queue gauge.+ const counts = await queue.getJobCounts("delayed");+ recordGauge(`${METRIC_PREFIX}.queue_depth`, counts.delayed ?? 0, {+ type: "delayed",+ unit: "records",+ });}private async collectWaitingHeadAge(): Promise<void> {diff --git a/worker/src/features/traceBatching/traceBatchTranscript.ts b/worker/src/features/traceBatching/traceBatchTranscript.tsindex babde8d7c4..d0c0c7f6d9 100644--- a/worker/src/features/traceBatching/traceBatchTranscript.ts+++ b/worker/src/features/traceBatching/traceBatchTranscript.ts@@ -42,14 +42,25 @@ const TOKEN_METRICS = ["transcript_history_tokens",] as const;+function recordTranscriptDistribution(+ metric: string,+ value: number,+ tags?: { [tag: string]: string | number },+): void {+ if (env.LANGFUSE_TRACE_BATCH_TRANSCRIPT_METRICS_ENABLED !== "true") return;+ if (tags) {+ recordDistribution(`langfuse.trace_batch.${metric}`, value, tags);+ return;+ }+ recordDistribution(`langfuse.trace_batch.${metric}`, value);+}+function recordTokens(span: Span,metric: (typeof TOKEN_METRICS)[number],tokens: number,) {- recordDistribution(`langfuse.trace_batch.${metric}`, tokens, {- tokenizer: "o200k_base",- });+ recordTranscriptDistribution(metric, tokens, { tokenizer: "o200k_base" });span.setAttribute(`langfuse.trace_batch.${metric}`, tokens);}@@ -97,7 +108,7 @@ function recordContentAndStructureCounts(["transcript_content_characters", total],["transcript_tool_response_characters", toolResponses],] as const) {- recordDistribution(`langfuse.trace_batch.${metric}`, characters);+ recordTranscriptDistribution(metric, characters);span.setAttribute(`langfuse.trace_batch.${metric}`, characters);}for (const [metric, count] of [@@ -109,7 +120,7 @@ function recordContentAndStructureCounts(["transcript_current_turn_tool_call_count", currentTurnToolCalls],["transcript_current_turn_tool_result_count", currentTurnToolResults],] as const) {- recordDistribution(`langfuse.trace_batch.${metric}`, count);+ recordTranscriptDistribution(metric, count);span.setAttribute(`langfuse.trace_batch.${metric}`, count);}}@@ -169,7 +180,7 @@ function recordTopicsRendering(topicsTranscriptConfig,);const metric = (name: string, value: number) => {- recordDistribution(`langfuse.trace_batch.${name}`, value);+ recordTranscriptDistribution(name, value);span.setAttribute(`langfuse.trace_batch.${name}`, value);};metric("topics_transcript_characters", text.length);@@ -189,8 +200,8 @@ function recordTopicsRendering(if (stats.blockCharacters[block].raw === 0) continue;for (const stage of ["raw", "clipped"] as const) {const characters = stats.blockCharacters[block][stage];- recordDistribution(- "langfuse.trace_batch.topics_transcript_block_characters",+ recordTranscriptDistribution(+ "topics_transcript_block_characters",characters,{ block, stage },);@@ -220,7 +231,7 @@ async function recordTopicsTokens(text: string, span: Span) {span.setAttribute("langfuse.trace_batch.topics_transcript", "unavailable");return;}- recordDistribution("langfuse.trace_batch.topics_transcript_tokens", tokens, {+ recordTranscriptDistribution("topics_transcript_tokens", tokens, {tokenizer: "o200k_base",});span.setAttribute("langfuse.trace_batch.topics_transcript_tokens", tokens);@@ -237,7 +248,7 @@ async function recordTranscriptJsonTokens(text: string | null, span: Span) {span.setAttribute("langfuse.trace_batch.transcript_json", "unavailable");return;}- recordDistribution("langfuse.trace_batch.transcript_json_tokens", tokens, {+ recordTranscriptDistribution("transcript_json_tokens", tokens, {tokenizer: "o200k_base",});span.setAttribute("langfuse.trace_batch.transcript_json_tokens", tokens);@@ -255,7 +266,7 @@ async function recordGenericTranscriptTokens(text: string, span: Span) {span.setAttribute("langfuse.trace_batch.generic_transcript", "unavailable");return;}- recordDistribution("langfuse.trace_batch.generic_transcript_tokens", tokens, {+ recordTranscriptDistribution("generic_transcript_tokens", tokens, {tokenizer: "o200k_base",});span.setAttribute("langfuse.trace_batch.generic_transcript_tokens", tokens);@@ -268,10 +279,7 @@ function recordGenericRendering(): string | undefined {try {const text = renderGenericTranscript(transcript, observations);- recordDistribution(- "langfuse.trace_batch.generic_transcript_characters",- text.length,- );+ recordTranscriptDistribution("generic_transcript_characters", text.length);span.setAttribute("langfuse.trace_batch.generic_transcript_characters",text.length,@@ -316,8 +324,8 @@ export function recordTraceBatchTranscript(phaseTimings = timings;});const assemblyDurationMs = performance.now() - startedAt;- recordDistribution(- "langfuse.trace_batch.transcript_assembly_duration_ms",+ recordTranscriptDistribution(+ "transcript_assembly_duration_ms",assemblyDurationMs,{ has_transcript: String(transcript !== null) },);@@ -330,8 +338,8 @@ export function recordTraceBatchTranscript(["normalization", phaseTimings.normalizationMs],["matching", phaseTimings.matchingMs],] as const) {- recordDistribution(- "langfuse.trace_batch.transcript_assembly_phase_duration_ms",+ recordTranscriptDistribution(+ "transcript_assembly_phase_duration_ms",duration,{ phase },);@@ -341,10 +349,7 @@ export function recordTraceBatchTranscript();}const threadCount = transcript?.threads.length ?? 0;- recordDistribution(- "langfuse.trace_batch.transcript_thread_count",- threadCount,- );+ recordTranscriptDistribution("transcript_thread_count", threadCount);span.setAttribute("langfuse.trace_batch.transcript_thread_count",threadCount,@@ -358,8 +363,8 @@ export function recordTraceBatchTranscript(const transcriptJson =transcript === null ? null : JSON.stringify(transcript);const transcriptJsonCharacters = transcriptJson?.length ?? 0;- recordDistribution(- "langfuse.trace_batch.transcript_json_characters",+ recordTranscriptDistribution(+ "transcript_json_characters",transcriptJsonCharacters,);span.setAttribute(@@ -385,8 +390,8 @@ export function recordTraceBatchTranscript();const comparisonRenderDurationMs =performance.now() - comparisonRenderStart;- recordDistribution(- "langfuse.trace_batch.transcript_comparison_render_duration_ms",+ recordTranscriptDistribution(+ "transcript_comparison_render_duration_ms",comparisonRenderDurationMs,);span.setAttribute(@@ -437,8 +442,8 @@ export function recordTraceBatchTranscript(}const comparisonTokenizationDurationMs =performance.now() - comparisonTokenizationStart;- recordDistribution(- "langfuse.trace_batch.transcript_comparison_tokenization_duration_ms",+ recordTranscriptDistribution(+ "transcript_comparison_tokenization_duration_ms",comparisonTokenizationDurationMs,);span.setAttribute(diff --git a/worker/src/features/traceBatching/traceBatching.ts b/worker/src/features/traceBatching/traceBatching.tsindex a048f72d41..73f1f81e6b 100644--- a/worker/src/features/traceBatching/traceBatching.ts+++ b/worker/src/features/traceBatching/traceBatching.ts@@ -549,7 +549,7 @@ const ACKNOWLEDGE_SCRIPT = ``;function recordTrackingVolume(- stage: "eligible" | "sampled" | "recorded",+ stage: "eligible" | "recorded",entries: Iterable<readonly [string,@@ -620,7 +620,6 @@ export async function trackTraceBatchActivity(samplingRate,})),);- recordGauge("langfuse.trace_batch.sampling_rate", samplingRate);// Counts distinct trace IDs per ingestion batch, not globally unique traces.recordIncrement("langfuse.trace_batch.sampling_decisions", entries.length, {decision: "selected",@@ -630,7 +629,6 @@ export async function trackTraceBatchActivity(bounds.size - entries.length,{ decision: "excluded" },);- recordTrackingVolume("sampled", entries);if (entries.length === 0) return;if (!redis) throw new Error("Trace batching requires Redis");// Bounded scripts keep ingestion from monopolizing the global Redis slot.@@ -657,7 +655,9 @@ export async function trackTraceBatchActivity();// Count only acknowledged chunks; a timeout can leave Redis's outcome unknown.recordTrackingVolume("recorded", chunk);- recordIncrement("langfuse.trace_batch.expired_traces", expired);+ recordIncrement("langfuse.trace_batch.expired_traces", expired, {+ source: "ingestion",+ });recordIncrement("langfuse.trace_batch.tracked_traces", chunk.length);}} catch (error) {@@ -751,7 +751,9 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {const [clock, , pending, ready, oldest, expired] = snapshot.map(Number);now = clock;cutoff = String(snapshot[1]);- recordIncrement("langfuse.trace_batch.expired_traces", expired);+ recordIncrement("langfuse.trace_batch.expired_traces", expired, {+ source: "dispatcher",+ });recordGauge("langfuse.trace_batch.pending_traces", pending);recordGauge("langfuse.trace_batch.ready_traces", ready);recordGauge(@@ -832,9 +834,6 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {2 * QUERY_BUFFER_MS,{ strategy },);- recordIncrement("langfuse.trace_batch.dispatched_traces", batch.length, {- batch_kind: batch.length === 1 ? "singleton" : "multi",- });let estimatedEventUpdateCount = 0;let estimatedSerializedEventBytes = 0;let unavailableEstimates = 0;@@ -855,15 +854,6 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {} else {unavailableEstimates++;}- recordDistribution(- "langfuse.trace_batch.observed_start_span_ms",- entry.trace.maxStart - entry.trace.minStart,- { strategy },- );- recordDistribution(- "langfuse.trace_batch.due_lag_ms",- Math.max(0, Date.now() - entry.due),- );}if (unavailableEstimates === 0) {recordDistribution(diff --git a/worker/src/queues/__tests__/otelDirectEventWrite.test.ts b/worker/src/queues/__tests__/otelDirectEventWrite.test.tsindex 6a1e388ec4..d35f700f92 100644--- a/worker/src/queues/__tests__/otelDirectEventWrite.test.ts+++ b/worker/src/queues/__tests__/otelDirectEventWrite.test.ts@@ -5,7 +5,6 @@ import {OtelIngestionProcessor,QueueJobs,type QueueName,- recordDistribution,type TQueueJobTypes,} from "@langfuse/shared/src/server";import { env } from "../../env";@@ -29,7 +28,6 @@ import {vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({...(await importOriginal<typeof import("@langfuse/shared/src/server")>()),getS3EventStorageClient: vi.fn(),- recordDistribution: vi.fn(),}));vi.mock("../../features/evaluation/observationEval",@@ -133,10 +131,6 @@ describe("direct-v4 trace batch tracking", () => {serializedEventBytes: index === 0 ? 101 : 202,})),);- expect(recordDistribution).toHaveBeenCalledWith(- "langfuse.trace_batch.ingestion_trace_count",- 1,- );vi.mocked(trackTraceBatchActivity).mockClear();env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = undefined;diff --git a/worker/src/queues/__tests__/traceBatchQueue.test.ts b/worker/src/queues/__tests__/traceBatchQueue.test.tsindex ae96d882a1..f1e7e1d5cc 100644--- a/worker/src/queues/__tests__/traceBatchQueue.test.ts+++ b/worker/src/queues/__tests__/traceBatchQueue.test.ts@@ -48,6 +48,8 @@ vi.mock("../../features/tokenisation/async-usage", () => ({}));const originalReadEnabled = env.LANGFUSE_TRACE_BATCH_READ_ENABLED;+const originalTranscriptMetricsEnabled =+ env.LANGFUSE_TRACE_BATCH_TRANSCRIPT_METRICS_ENABLED;const originalCloudRegion = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;const originalExperimentId = env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID;const processingSpan = trace.wrapSpanContext({@@ -76,6 +78,8 @@ beforeEach(() => {});afterEach(() => {env.LANGFUSE_TRACE_BATCH_READ_ENABLED = originalReadEnabled;+ env.LANGFUSE_TRACE_BATCH_TRANSCRIPT_METRICS_ENABLED =+ originalTranscriptMetricsEnabled;env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = originalCloudRegion;env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID = originalExperimentId;vi.restoreAllMocks();@@ -106,34 +110,10 @@ describe("trace batch queue", () => {expect(JSON.parse(baselineJson).threads[0].currentTurn.observations,).toEqual([expect.objectContaining({ traceId: "topics-sample" })]);- expect(recordDistribution).toHaveBeenCalledWith(- "langfuse.trace_batch.transcript_json_characters",- baselineJson.length,- );- expect(recordDistribution).toHaveBeenCalledWith(- "langfuse.trace_batch.transcript_json_tokens",- tokenCount({- model: vi.mocked(tokenCountAsync).mock.calls[1][0].model,- text: baselineJson,- }),- { tokenizer: "o200k_base" },- );const genericText = vi.mocked(tokenCountAsync).mock.calls[2][0].text;expect(genericText).toBe("[user] Explain the result\n[assistant] The answer",);- expect(recordDistribution).toHaveBeenCalledWith(- "langfuse.trace_batch.generic_transcript_characters",- genericText.length,- );- expect(recordDistribution).toHaveBeenCalledWith(- "langfuse.trace_batch.generic_transcript_tokens",- tokenCount({- model: vi.mocked(tokenCountAsync).mock.calls[2][0].model,- text: genericText,- }),- { tokenizer: "o200k_base" },- );const renderedText = vi.mocked(tokenCountAsync).mock.calls[3][0].text;expect(renderedText).toBe([@@ -151,29 +131,29 @@ describe("trace batch queue", () => {"</end_of_run>",].join("\n"),);- expect(recordDistribution).toHaveBeenCalledWith(- "langfuse.trace_batch.topics_transcript_tokens",- tokenCount({- model: vi.mocked(tokenCountAsync).mock.calls[3][0].model,- text: renderedText,- }),- { tokenizer: "o200k_base" },- );- expect(recordDistribution).toHaveBeenCalledWith(- "langfuse.trace_batch.topics_transcript_block_characters",- "Explain the result".length,- { block: "user", stage: "raw" },- );- expect(recordDistribution).toHaveBeenCalledWith(- "langfuse.trace_batch.topics_transcript_block_characters",- "explain".length,- { block: "observations", stage: "raw" },- );+ expect(recordDistribution).not.toHaveBeenCalled();expect(exporter.getFinishedSpans()[0].attributes).toMatchObject({+ "langfuse.trace_batch.transcript_json_characters": baselineJson.length,+ "langfuse.trace_batch.transcript_json_tokens": tokenCount({+ model: vi.mocked(tokenCountAsync).mock.calls[1][0].model,+ text: baselineJson,+ }),+ "langfuse.trace_batch.generic_transcript_characters": genericText.length,+ "langfuse.trace_batch.generic_transcript_tokens": tokenCount({+ model: vi.mocked(tokenCountAsync).mock.calls[2][0].model,+ text: genericText,+ }),"langfuse.trace_batch.topics_transcript_characters": renderedText.length,"langfuse.trace_batch.topics_transcript_blocks_cut": 0,"langfuse.trace_batch.topics_transcript_history_share": 0,- "langfuse.trace_batch.topics_transcript_tokens": expect.any(Number),+ "langfuse.trace_batch.topics_transcript_tokens": tokenCount({+ model: vi.mocked(tokenCountAsync).mock.calls[3][0].model,+ text: renderedText,+ }),+ "langfuse.trace_batch.topics_transcript_block_user_raw_characters":+ "Explain the result".length,+ "langfuse.trace_batch.topics_transcript_block_observations_raw_characters":+ "explain".length,"langfuse.trace_batch.transcript_observation_count": 1,"langfuse.trace_batch.transcript_history_message_count": 0,"langfuse.trace_batch.transcript_current_turn_message_count": 2,@@ -181,6 +161,38 @@ describe("trace batch queue", () => {});});+ it("emits transcript distributions only when the switch is on", async () => {+ env.LANGFUSE_TRACE_BATCH_TRANSCRIPT_METRICS_ENABLED = "true";+ await recordTraceBatchTranscript([+ convertObservation(+ createObservation({+ type: "GENERATION",+ name: "explain",+ trace_id: "topics-metrics",+ input: JSON.stringify([+ { role: "user", content: "Explain the result" },+ ]),+ output: JSON.stringify({ role: "assistant", content: "The answer" }),+ }),+ ),+ ]);++ const renderedText = vi.mocked(tokenCountAsync).mock.calls[3][0].text;+ expect(recordDistribution).toHaveBeenCalledWith(+ "langfuse.trace_batch.topics_transcript_characters",+ String(renderedText).length,+ );+ expect(recordDistribution).toHaveBeenCalledWith(+ "langfuse.trace_batch.topics_transcript_tokens",+ expect.any(Number),+ { tokenizer: "o200k_base" },+ );+ expect(recordDistribution).toHaveBeenCalledWith(+ "langfuse.trace_batch.transcript_json_characters",+ expect.any(Number),+ );+ });+it("keeps the batch successful when Topics token estimation fails", async () => {vi.mocked(tokenCountAsync).mockResolvedValueOnce(10)@@ -205,21 +217,18 @@ describe("trace batch queue", () => {"langfuse.trace_batch.topics_transcript_failed",1,);- expect(recordDistribution).toHaveBeenCalledWith(- "langfuse.trace_batch.topics_transcript_characters",- expect.any(Number),- );- expect(recordDistribution).not.toHaveBeenCalledWith(- "langfuse.trace_batch.topics_transcript_tokens",- expect.any(Number),- expect.anything(),- );- expect(exporter.getFinishedSpans()[0].attributes).toMatchObject({+ expect(recordDistribution).not.toHaveBeenCalled();+ const span = exporter.getFinishedSpans()[0];+ expect(span.attributes).toMatchObject({"langfuse.trace_batch.topics_transcript": "failed",+ "langfuse.trace_batch.topics_transcript_characters": expect.any(Number),"langfuse.trace_batch.transcript_message_tokens": 10,"langfuse.trace_batch.transcript_json_tokens": 20,"langfuse.trace_batch.generic_transcript_tokens": 30,});+ expect(span.attributes).not.toHaveProperty(+ "langfuse.trace_batch.topics_transcript_tokens",+ );});it("measures history, current turn, threads and deduplicated tool responses", async () => {@@ -289,54 +298,33 @@ describe("trace batch queue", () => {expect(current).toContain("independent question");expect(historyOnly).toContain("earlier question");expect(historyOnly).not.toContain("current question");- expect(recordDistribution).toHaveBeenCalledWith(- "langfuse.trace_batch.transcript_thread_count",- 2,- );- expect(recordDistribution).toHaveBeenCalledWith(- "langfuse.trace_batch.transcript_observation_count",- 4,- );- expect(recordDistribution).toHaveBeenCalledWith(- "langfuse.trace_batch.transcript_history_message_count",- 2,- );- expect(recordDistribution).toHaveBeenCalledWith(- "langfuse.trace_batch.transcript_history_part_count",- 2,- );- expect(recordDistribution).toHaveBeenCalledWith(- "langfuse.trace_batch.transcript_current_turn_tool_call_count",- 1,- );- expect(recordDistribution).toHaveBeenCalledWith(- "langfuse.trace_batch.transcript_current_turn_tool_result_count",- 1,- );- for (const [index, suffix] of [- "current_turn_tokens",- "history_tokens",- ].entries()) {- expect(recordDistribution).toHaveBeenCalledWith(- `langfuse.trace_batch.transcript_${suffix}`,- tokenCount(estimates[index]),- { tokenizer: "o200k_base" },- );- }- expect(recordDistribution).toHaveBeenCalledWith(- "langfuse.trace_batch.transcript_message_tokens",- tokenCount(estimates[0])! + tokenCount(estimates[1])!,- { tokenizer: "o200k_base" },- );+ expect(recordDistribution).not.toHaveBeenCalled();const span = exporter.getFinishedSpans()[0];+ expect(span.attributes).toMatchObject({+ "langfuse.trace_batch.transcript_thread_count": 2,+ "langfuse.trace_batch.transcript_observation_count": 4,+ "langfuse.trace_batch.transcript_history_message_count": 2,+ "langfuse.trace_batch.transcript_history_part_count": 2,+ "langfuse.trace_batch.transcript_current_turn_tool_call_count": 1,+ "langfuse.trace_batch.transcript_current_turn_tool_result_count": 1,+ "langfuse.trace_batch.transcript_current_turn_tokens": tokenCount(+ estimates[0],+ ),+ "langfuse.trace_batch.transcript_history_tokens": tokenCount(+ estimates[1],+ ),+ "langfuse.trace_batch.transcript_message_tokens":+ tokenCount(estimates[0])! + tokenCount(estimates[1])!,+ });const toolCharacters = JSON.stringify({type: "text",text: "tool payload",}).length;- expect(recordDistribution).toHaveBeenCalledWith(- "langfuse.trace_batch.transcript_tool_response_characters",- toolCharacters,- );+ expect(+ span.attributes[+ "langfuse.trace_batch.transcript_tool_response_characters"+ ],+ ).toBe(toolCharacters);expect(span.attributes["langfuse.trace_batch.transcript_content_characters"],).toBeGreaterThan(toolCharacters);@@ -347,11 +335,6 @@ describe("trace batch queue", () => {] as number;expect(duration).toBeGreaterThanOrEqual(0);phaseTotal += duration;- expect(recordDistribution).toHaveBeenCalledWith(- "langfuse.trace_batch.transcript_assembly_phase_duration_ms",- duration,- { phase },- );}expect(phaseTotal).toBeLessThanOrEqual(span.attributes[@@ -394,19 +377,13 @@ describe("trace batch queue", () => {JSON.stringify({ type: "text", text: "Please explain this result" }).length +JSON.stringify({ type: "text", text: "Answer" }).length;- expect(recordDistribution).toHaveBeenCalledWith(- "langfuse.trace_batch.transcript_tool_response_characters",- toolCharacters,- );- expect(recordDistribution).toHaveBeenCalledWith(- "langfuse.trace_batch.transcript_content_characters",- totalCharacters,- );- expect(recordDistribution).toHaveBeenCalledWith(- "langfuse.trace_batch.transcript_history_tokens",- 0,- { tokenizer: "o200k_base" },- );+ expect(recordDistribution).not.toHaveBeenCalled();+ expect(exporter.getFinishedSpans()[0].attributes).toMatchObject({+ "langfuse.trace_batch.transcript_tool_response_characters":+ toolCharacters,+ "langfuse.trace_batch.transcript_content_characters": totalCharacters,+ "langfuse.trace_batch.transcript_history_tokens": 0,+ });});it.each(["unavailable", "failed"])(@@ -570,12 +547,11 @@ describe("trace batch queue", () => {1,);expect(- vi- .mocked(recordDistribution)- .mock.calls.filter(- ([name]) =>- name === "langfuse.trace_batch.transcript_assembly_duration_ms",- ),+ spans.filter(+ (span) =>+ "langfuse.trace_batch.transcript_assembly_duration_ms" in+ span.attributes,+ ),).toHaveLength(streamFails ? 2 : 3);} finally {cleaningUp = true;@@ -652,40 +628,6 @@ describe("trace batch queue", () => {]) {expect(serializedTranscript.split(content)).toHaveLength(2);}- expect(recordDistribution).toHaveBeenCalledWith(- "langfuse.trace_batch.transcript_message_tokens",- tokenCount(estimates[0][0]),- { tokenizer: "o200k_base" },- );- expect(recordDistribution).toHaveBeenCalledWith(- "langfuse.trace_batch.transcript_message_tokens",- 0,- { tokenizer: "o200k_base" },- );- for (const suffix of ["current_turn", "history"]) {- expect(recordDistribution).toHaveBeenCalledWith(- `langfuse.trace_batch.transcript_${suffix}_tokens`,- 0,- { tokenizer: "o200k_base" },- );- }- for (const suffix of ["content", "tool_response"]) {- expect(recordDistribution).toHaveBeenCalledWith(- `langfuse.trace_batch.transcript_${suffix}_characters`,- 0,- );- }- expect(recordDistribution).toHaveBeenCalledWith(- "langfuse.trace_batch.transcript_thread_count",- 0,- );- for (const hasTranscript of ["true", "false"]) {- expect(recordDistribution).toHaveBeenCalledWith(- "langfuse.trace_batch.transcript_assembly_duration_ms",- expect.any(Number),- { has_transcript: hasTranscript },- );- }const spans = exporter.getFinishedSpans();expect(spans).toHaveLength(2);for (const [index, projectId] of ["a", "b"].entries()) {@@ -698,6 +640,15 @@ describe("trace batch queue", () => {"langfuse.trace_batch.has_transcript": index === 0,"langfuse.trace_batch.transcript_message_tokens":index === 0 ? tokenCount(estimates[0][0]) : 0,+ "langfuse.trace_batch.transcript_current_turn_tokens":+ index === 0 ? expect.any(Number) : 0,+ "langfuse.trace_batch.transcript_history_tokens":+ index === 0 ? expect.any(Number) : 0,+ "langfuse.trace_batch.transcript_content_characters":+ index === 0 ? expect.any(Number) : 0,+ "langfuse.trace_batch.transcript_tool_response_characters":+ index === 0 ? expect.any(Number) : 0,+ "langfuse.trace_batch.transcript_thread_count": index === 0 ? 1 : 0,"langfuse.trace_batch.transcript_characters":index === 0 ? expect.any(Number) : 0,"langfuse.trace_batch.transcript_assembly_duration_ms":@@ -720,16 +671,12 @@ describe("trace batch queue", () => {discarded: "not_cloud",});expect(getTraceBatchEventStream).not.toHaveBeenCalled();- expect(recordIncrement).toHaveBeenCalledWith(- "langfuse.trace_batch.read_attempts",+ expect(recordIncrement).toHaveBeenCalledExactlyOnceWith(+ "langfuse.trace_batch.discarded_jobs",1,- { outcome: "discard" },- );- expect(recordDistribution).toHaveBeenCalledExactlyOnceWith(- "langfuse.trace_batch.read_duration_ms",- expect.any(Number),- { outcome: "discard" },+ { reason: "not_cloud" },);+ expect(recordDistribution).not.toHaveBeenCalled();expect(recordGauge).not.toHaveBeenCalled();expect(job.opts.removeOnComplete).toBe(true);expect(processingSpan.setAttributes).toHaveBeenCalledWith(@@ -768,18 +715,11 @@ describe("trace batch queue", () => {});}expect(getTraceBatchEventStream).not.toHaveBeenCalled();- expect(vi.mocked(recordDistribution).mock.calls).toEqual([- [- "langfuse.trace_batch.read_duration_ms",- expect.any(Number),- { outcome: "discard" },- ],- [- "langfuse.trace_batch.read_duration_ms",- expect.any(Number),- { outcome: "discard" },- ],+ expect(vi.mocked(recordIncrement).mock.calls).toEqual([+ ["langfuse.trace_batch.discarded_jobs", 1, { reason: "expired" }],+ ["langfuse.trace_batch.discarded_jobs", 1, { reason: "expired" }],]);+ expect(recordDistribution).not.toHaveBeenCalled();expect(recordGauge).not.toHaveBeenCalled();expect(job.opts.removeOnComplete).toBe(true);});@@ -925,20 +865,8 @@ describe("trace batch queue", () => {expect(vi.mocked(recordDistribution).mock.calls).toEqual([["langfuse.trace_batch.failed_read_observation_count", 1],["langfuse.trace_batch.failed_read_input_bytes", 5],- ["langfuse.trace_batch.failed_read_output_bytes", 6],- ["langfuse.trace_batch.failed_read_metadata_bytes", 9],- ["langfuse.trace_batch.failed_read_io_metadata_bytes", 20],- [- "langfuse.trace_batch.read_duration_ms",- expect.any(Number),- { outcome: "failure" },- ],]);- expect(recordIncrement).toHaveBeenCalledWith(- "langfuse.trace_batch.read_attempts",- 1,- { outcome: "failure" },- );+ expect(recordIncrement).not.toHaveBeenCalled();} finally {env.LANGFUSE_TRACE_BATCH_MAX_THREADS =originalEnv.LANGFUSE_TRACE_BATCH_MAX_THREADS;@@ -1024,10 +952,7 @@ describe("trace batch queue", () => {["langfuse.trace_batch.active_reads", 1],["langfuse.trace_batch.active_reads", 0],]);- expect(vi.mocked(recordIncrement).mock.calls).toEqual([- ["langfuse.trace_batch.read_attempts", 1, { outcome: "success" }],- ["langfuse.trace_batch.read_attempts", 1, { outcome: "failure" }],- ]);+ expect(recordIncrement).not.toHaveBeenCalled();});it("counts project/trace pairs with producers disabled and safely repeats reads without retaining payloads", async () => {env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID = "successful-read";@@ -1142,7 +1067,6 @@ describe("trace batch queue", () => {["input_bytes", 20],["output_bytes", 24],["metadata_bytes", 36],- ["io_metadata_bytes", 80],] as const) {expect(vi@@ -1153,14 +1077,14 @@ describe("trace batch queue", () => {.map(([, value]) => value),).toEqual([bytes, bytes]);}- expect(- vi- .mocked(recordDistribution)- .mock.calls.filter(- ([name]) => name === "langfuse.trace_batch.found_project_count",- )- .map(([, value]) => value),- ).toEqual([2, 2]);+ expect(recordDistribution).not.toHaveBeenCalledWith(+ "langfuse.trace_batch.io_metadata_bytes",+ expect.anything(),+ );+ expect(recordDistribution).not.toHaveBeenCalledWith(+ "langfuse.trace_batch.found_project_count",+ expect.anything(),+ );expect(recordDistribution).toHaveBeenCalledWith("langfuse.trace_batch.missing_trace_count",2,@@ -1234,9 +1158,9 @@ describe("trace batch queue", () => {"langfuse.trace_batch.metadata_bytes",0,);- expect(recordDistribution).toHaveBeenCalledWith(+ expect(recordDistribution).not.toHaveBeenCalledWith("langfuse.trace_batch.found_project_count",- Number(hasRows),+ expect.anything(),);expect(getTraceBatchEventStream).toHaveBeenCalledWith({ traces: [{ ...trace, projectId: "project" }] },diff --git a/worker/src/queues/traceBatchQueue.ts b/worker/src/queues/traceBatchQueue.tsindex afbc03e92a..cf83bebea3 100644--- a/worker/src/queues/traceBatchQueue.ts+++ b/worker/src/queues/traceBatchQueue.ts@@ -187,9 +187,6 @@ export const traceBatchQueueProcessor: Processor<for (const [name, value] of [["observation_count", observationCount],["input_bytes", inputBytes],- ["output_bytes", outputBytes],- ["metadata_bytes", metadataBytes],- ["io_metadata_bytes", inputBytes + outputBytes + metadataBytes],] as const) {recordDistribution(`langfuse.trace_batch.failed_read_${name}`, value);}@@ -216,14 +213,6 @@ export const traceBatchQueueProcessor: Processor<recordDistribution("langfuse.trace_batch.input_bytes", inputBytes);recordDistribution("langfuse.trace_batch.output_bytes", outputBytes);recordDistribution("langfuse.trace_batch.metadata_bytes", metadataBytes);- recordDistribution(- "langfuse.trace_batch.io_metadata_bytes",- ioMetadataBytes,- );- recordDistribution(- "langfuse.trace_batch.found_project_count",- foundProjects.size,- );recordDistribution("langfuse.trace_batch.missing_trace_count",batch.traces.length - foundTraces.size,@@ -279,9 +268,5 @@ export const traceBatchQueueProcessor: Processor<: {}),});}- recordIncrement("langfuse.trace_batch.read_attempts", 1, { outcome });- recordDistribution("langfuse.trace_batch.read_duration_ms", durationMs, {- outcome,- });}};