Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

chore(worker): drop duplicate trace-batch metrics (#18241)

Lysander Kiesel5fdf5a7088dc
diff --git a/worker/src/__tests__/traceBatching.test.ts b/worker/src/__tests__/traceBatching.test.ts
index ef9e37e4dd..4fe9f45858 100644
--- a/worker/src/__tests__/traceBatching.test.ts
+++ b/worker/src/__tests__/traceBatching.test.ts
@@ -537,13 +537,16 @@ describe("trace micro-batch scheduling with Redis", () => {
);
expect(Number(sample?.[1])).toBeGreaterThan(0);
}
- for (const type of ["waiting", "delayed"]) {
- expect(recordGauge).toHaveBeenCalledWith(
- "langfuse.trace_batch.queue_depth",
- type === "waiting" ? 2 : 1,
- { type, unit: "records" },
- );
- }
+ expect(recordGauge).toHaveBeenCalledWith(
+ "langfuse.trace_batch.queue_depth",
+ 1,
+ { type: "delayed", unit: "records" },
+ );
+ expect(recordGauge).not.toHaveBeenCalledWith(
+ "langfuse.trace_batch.queue_depth",
+ expect.anything(),
+ expect.objectContaining({ type: "waiting" }),
+ );
expect(recordGauge).toHaveBeenCalledWith(
"langfuse.trace_batch.queue_waiting_head_age_ms",
5_000,
@@ -563,7 +566,7 @@ describe("trace micro-batch scheduling with Redis", () => {
"langfuse.trace_batch.active_reads",
0,
);
- expect(vi.mocked(recordGauge).mock.calls).toHaveLength(10);
+ expect(vi.mocked(recordGauge).mock.calls).toHaveLength(7);
expect(
vi.mocked(recordGauge).mock.calls.every(([, value]) => value === 0),
).toBe(true);
@@ -854,10 +857,6 @@ describe("trace micro-batch scheduling with Redis", () => {
traceCount: 2,
projectCount: 1,
});
- expect(recordDistribution).toHaveBeenCalledWith(
- "langfuse.trace_batch.found_project_count",
- 1,
- );
// All three observations must carry their untruncated I/O and metadata.
expect(result.ioMetadataBytes).toBeGreaterThanOrEqual(
3 * (Buffer.byteLength(input) + Buffer.byteLength(output) + 8_000),
@@ -866,9 +865,9 @@ describe("trace micro-batch scheduling with Redis", () => {
"langfuse.trace_batch.size",
2,
);
- expect(recordDistribution).toHaveBeenCalledWith(
+ expect(recordDistribution).not.toHaveBeenCalledWith(
"langfuse.trace_batch.io_metadata_bytes",
- result.ioMetadataBytes,
+ expect.anything(),
);
expect(await client().zcard(dueKey)).toBe(0);
expect(await client().hlen(stateKey)).toBe(0);
@@ -927,9 +926,13 @@ describe("trace micro-batch scheduling with Redis", () => {
1_000 - expectedCount,
{ decision: "excluded" },
);
+ expect(recordIncrement).not.toHaveBeenCalledWith(
+ "langfuse.trace_batch.event_updates",
+ expect.anything(),
+ { stage: "sampled" },
+ );
for (const [stage, count] of [
["eligible", 1_000],
- ["sampled", expectedCount],
...(expectedCount > 0 ? [["recorded", expectedCount] as const] : []),
] as const) {
expect(recordIncrement).toHaveBeenCalledWith(
@@ -968,9 +971,9 @@ describe("trace micro-batch scheduling with Redis", () => {
1_001,
{ stage: "eligible" },
);
- expect(recordIncrement).toHaveBeenCalledWith(
+ expect(recordIncrement).not.toHaveBeenCalledWith(
"langfuse.trace_batch.event_updates",
- 1_001,
+ expect.anything(),
{ stage: "sampled" },
);
expect(recordIncrement).toHaveBeenCalledWith(
@@ -1696,6 +1699,7 @@ describe("trace micro-batch scheduling with Redis", () => {
expect(recordIncrement).toHaveBeenCalledWith(
"langfuse.trace_batch.expired_traces",
1_000,
+ { source: "ingestion" },
);
await trackTraceBatchActivity("project", [event("refreshed", 3_000_000)]);
expect((await client().hkeys(stateKey)).sort()).toEqual([
@@ -1706,6 +1710,7 @@ describe("trace micro-batch scheduling with Redis", () => {
expect(recordIncrement).toHaveBeenCalledWith(
"langfuse.trace_batch.expired_traces",
1,
+ { source: "ingestion" },
);
expect(await queue.getWaitingCount()).toBe(0);
});
@@ -1727,6 +1732,11 @@ describe("trace micro-batch scheduling with Redis", () => {
env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = "false";
const dispatcher = runner();
await dispatcher.processBatch();
+ expect(recordIncrement).toHaveBeenCalledWith(
+ "langfuse.trace_batch.expired_traces",
+ expect.any(Number),
+ { source: "dispatcher" },
+ );
expect(
vi
.mocked(recordIncrement)
diff --git a/worker/src/env.ts b/worker/src/env.ts
index 482d20dd59..288f17a32a 100644
--- a/worker/src/env.ts
+++ b/worker/src/env.ts
@@ -121,6 +121,9 @@ const EnvSchema = z.object({
.enum(["true", "false"])
.default("false"),
LANGFUSE_TRACE_BATCH_READ_ENABLED: z.enum(["true", "false"]).default("false"),
+ LANGFUSE_TRACE_BATCH_TRANSCRIPT_METRICS_ENABLED: z
+ .enum(["true", "false"])
+ .default("false"),
LANGFUSE_TRACE_BATCH_CONCURRENCY: z.coerce
.number()
.int()
diff --git a/worker/src/features/otel-ingestion/processOtelEvents.ts b/worker/src/features/otel-ingestion/processOtelEvents.ts
index df3f49eca4..fb652b12f0 100644
--- a/worker/src/features/otel-ingestion/processOtelEvents.ts
+++ b/worker/src/features/otel-ingestion/processOtelEvents.ts
@@ -2,7 +2,6 @@ import { convertEventRecordToObservationForEval } from "@langfuse/shared";
import {
logger,
OtelIngestionProcessor,
- recordDistribution,
traceException,
} from "@langfuse/shared/src/server";
import { env } from "../../env";
@@ -174,10 +173,6 @@ export async function processOtelEvents({
);
if (traceBatchEvents.length > 0) {
- recordDistribution(
- "langfuse.trace_batch.ingestion_trace_count",
- new Set(traceBatchEvents.map((event) => event.traceId)).size,
- );
// The writer has accepted these records; its flush completes separately.
await trackTraceBatchActivity(projectId, traceBatchEvents);
}
diff --git a/worker/src/features/traceBatching/README.md b/worker/src/features/traceBatching/README.md
index 4753e34379..2bc8324c31 100644
--- a/worker/src/features/traceBatching/README.md
+++ b/worker/src/features/traceBatching/README.md
@@ -2,28 +2,30 @@
This read-only experiment measures full-event reads and transcript assembly for
idle traces. It requires
-`NEXT_PUBLIC_LANGFUSE_CLOUD_REGION` and explicit opt-in. All four enablement
-flags below default to `false`; an ordinary release needs no infrastructure
-changes. These internal controls are intentionally absent from env templates.
+`NEXT_PUBLIC_LANGFUSE_CLOUD_REGION` and explicit opt-in. The enablement flags
+and the transcript-metrics switch below default to `false`; an ordinary release
+needs no infrastructure changes. These internal controls are intentionally
+absent from env templates.
## Controls
-| Variable | Default | Role |
-| --------------------------------------------- | --------- | ---------------------------------------------------------------------------------------------------------- |
-| `LANGFUSE_TRACE_BATCH_INGESTION_ENABLED` | `false` | Track accepted direct-v4 event writes in Redis. |
-| `LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED` | `false` | Schedule idle traces into BullMQ jobs. |
-| `QUEUE_CONSUMER_TRACE_BATCH_QUEUE_IS_ENABLED` | `false` | Register the batch worker. |
-| `LANGFUSE_TRACE_BATCH_READ_ENABLED` | `false` | Allow the worker to query ClickHouse; otherwise discard jobs. |
-| `LANGFUSE_TRACE_BATCH_SAMPLING_RATE` | `1` | Fraction admitted by ingestion, from 0 to 1. |
-| `LANGFUSE_TRACE_BATCH_STRATEGY` | `project` | Choose project-order packing or opt-in locality grouping. |
-| `LANGFUSE_TRACE_BATCH_MAX_SIZE` | `60` | Maximum traces per job, up to 10,000. |
-| `LANGFUSE_TRACE_BATCH_MAX_THREADS` | `2` | ClickHouse threads per query (positive integer). |
-| `LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE` | unset | Optional positive `max_block_size` hint; unset inherits the server profile. |
-| `LANGFUSE_TRACE_BATCH_EXPERIMENT_ID` | unset | Optional 1–64 character attempt/query label: letters, digits, `.`, `_`, `-`; first character alphanumeric. |
-| `LANGFUSE_TRACE_BATCH_CONCURRENCY` | `2` | Concurrent batch jobs per worker process. |
-| `LANGFUSE_TRACE_BATCH_IDLE_MS` | `600000` | Idle time before a trace is ready (10 minutes). |
-| `LANGFUSE_TRACE_BATCH_PENDING_TTL_MS` | `7200000` | Pending retention and inactivity expiry (2 hours). |
-| `LANGFUSE_TRACE_BATCH_DISPATCH_INTERVAL_MS` | `30000` | Dispatcher cadence (30 seconds). |
+| Variable | Default | Role |
+| ------------------------------------------------- | --------- | ---------------------------------------------------------------------------------------------------------- |
+| `LANGFUSE_TRACE_BATCH_INGESTION_ENABLED` | `false` | Track accepted direct-v4 event writes in Redis. |
+| `LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED` | `false` | Schedule idle traces into BullMQ jobs. |
+| `QUEUE_CONSUMER_TRACE_BATCH_QUEUE_IS_ENABLED` | `false` | Register the batch worker. |
+| `LANGFUSE_TRACE_BATCH_READ_ENABLED` | `false` | Allow the worker to query ClickHouse; otherwise discard jobs. |
+| `LANGFUSE_TRACE_BATCH_TRANSCRIPT_METRICS_ENABLED` | `false` | Emit transcript size distributions. Span attributes and failure counters stay on either way. |
+| `LANGFUSE_TRACE_BATCH_SAMPLING_RATE` | `1` | Fraction admitted by ingestion, from 0 to 1. |
+| `LANGFUSE_TRACE_BATCH_STRATEGY` | `project` | Choose project-order packing or opt-in locality grouping. |
+| `LANGFUSE_TRACE_BATCH_MAX_SIZE` | `60` | Maximum traces per job, up to 10,000. |
+| `LANGFUSE_TRACE_BATCH_MAX_THREADS` | `2` | ClickHouse threads per query (positive integer). |
+| `LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE` | unset | Optional positive `max_block_size` hint; unset inherits the server profile. |
+| `LANGFUSE_TRACE_BATCH_EXPERIMENT_ID` | unset | Optional 1–64 character attempt/query label: letters, digits, `.`, `_`, `-`; first character alphanumeric. |
+| `LANGFUSE_TRACE_BATCH_CONCURRENCY` | `2` | Concurrent batch jobs per worker process. |
+| `LANGFUSE_TRACE_BATCH_IDLE_MS` | `600000` | Idle time before a trace is ready (10 minutes). |
+| `LANGFUSE_TRACE_BATCH_PENDING_TTL_MS` | `7200000` | Pending retention and inactivity expiry (2 hours). |
+| `LANGFUSE_TRACE_BATCH_DISPATCH_INTERVAL_MS` | `30000` | Dispatcher cadence (30 seconds). |
Producer, dispatcher and consumer can run independently. Sampling reuses the
evaluator's deterministic trace-ID sampler; every observation of the same trace
@@ -77,17 +79,20 @@ every member; `estimates_unavailable` counts unknown traces and affected batches
Estimates stay out of queue payloads, job IDs and batch selection. They do not cap
work or change oversized-trace handling. Dispatch retries can emit another sample.
-Serialized event bytes differ from the reader's `io_metadata_bytes` metric: they
-include the serialized event fields and JSON encoding, and do not represent RAM,
-network transfer or ClickHouse scan bytes. After rollout, collect a stable hour
+Serialized event bytes differ from the reader's `input_bytes`, `output_bytes`
+and `metadata_bytes` metrics: they include the serialized event fields and JSON
+encoding, and do not represent RAM, network transfer or ClickHouse scan bytes.
+The job return still sums those three as `ioMetadataBytes`; that sum is not a
+custom metric. After rollout, collect a stable hour
with unchanged sampling and batching settings; compare trace/batch distributions,
unknown-estimate counts, ingestion latency and Redis command rate/CPU/memory with
the baseline before choosing any size-aware batching policy.
The worker streams full `events_full` payloads and records batch observation,
-trace, project and logical payload-byte metrics. The `input_bytes`, `output_bytes`
+trace and logical payload-byte metrics. The `input_bytes`, `output_bytes`
and `metadata_bytes` distributions under `langfuse.trace_batch` each report one
-total per completed batch read; `io_metadata_bytes` retains their combined total.
+total per completed batch read. Distinct project count stays on the span and
+completion log.
Sizes count UTF-8 bytes, including metadata keys and values, excluding JSON
transport framing/escaping and compression. They do not measure ClickHouse bytes
scanned or per-trace ingestion size; retries can record another batch sample.
@@ -117,33 +122,37 @@ events can arrive. Event versions follow the current ClickHouse merge state;
the shared ordering function keeps one observation per ID. Retries can repeat
per-trace samples.
-These distributions use the `langfuse.trace_batch` prefix:
-
-| Metric | Sample |
-| --- | --- |
-| `transcript_assembly_duration_ms` | One trace's ordering and assembly time, excluding I/O conversion, stream waits and tokenization; `has_transcript:true\|false`. |
-| `transcript_json_characters` | UTF-16 length of `JSON.stringify(assembleTranscript(...))`, including thread/message wrappers and provenance; zero for a null transcript. Same admitted traces as the Topics length metrics. |
-| `transcript_json_tokens` | o200k estimate of that complete JSON string, tagged `tokenizer:o200k_base`; zero for a null transcript. Attempted on the same admitted traces as `topics_transcript_tokens`; unavailable estimates are omitted. |
-| `generic_transcript_characters` | UTF-16 length of the previous generic plain-text layout, rendered with the same Topics block caps and inclusions. Tool definitions or errors may still render when assembly returns null. |
-| `generic_transcript_tokens` | o200k estimate of that generic text, tagged `tokenizer:o200k_base`. |
-| `transcript_message_tokens` | Sum of current-turn and history message-projection token estimates; zero when empty, omitted if either estimate is unavailable. Distinct from full-transcript JSON. |
-| `transcript_current_turn_tokens` | Token estimate of current-turn messages across all threads, using role and parts only; zero when empty. |
-| `transcript_history_tokens` | Token estimate of history messages across all threads, using the same role/parts representation; zero when empty. |
-| `transcript_content_characters` | Sum of JSON-serialized message-part lengths across history and current turn, in UTF-16 code units; excludes message wrappers and provenance. |
-| `transcript_tool_response_characters` | The subset of content characters belonging to tool-role messages or unmatched tool-result parts; zero when absent. |
-| `transcript_thread_count` | Number of assembled conversation threads, or zero for a null transcript. |
-| `transcript_observation_count` | Ordered, deduplicated observations passed to assembly and the Topics renderer. The `observation_count` span attribute is the raw row count before deduplication. |
-| `transcript_history_message_count`, `transcript_history_part_count` | Replayed messages and their normalized parts across all assembled threads. |
-| `transcript_current_turn_message_count`, `transcript_current_turn_part_count` | This-trace messages and their normalized parts across all assembled threads. |
-| `transcript_current_turn_tool_call_count`, `transcript_current_turn_tool_result_count` | Tool-call parts and tool-result parts in this trace; a tool-role part also counts as a result. |
-| `transcript_assembly_phase_duration_ms` | Two non-overlapping samples per trace tagged `phase:normalization` or `phase:matching`. Normalization includes initial message-key construction and partitioning; matching covers remaining assembly work, including tool matching, deduplication, any rebuilt keys and finalization. Observation ordering is outside these phases but remains in total assembly time. |
-| `topics_transcript_characters` | UTF-16 length of the complete rendered Topics text, including labels, section headings and line breaks; admitted traces only. |
-| `topics_transcript_tokens` | o200k estimate of that same text, tagged `tokenizer:o200k_base`; admitted traces only. |
-| `transcript_comparison_render_duration_ms` | Time to serialize assembled JSON and render generic and Topics text, excluding tokenization. |
-| `transcript_comparison_tokenization_duration_ms` | Wall time for the three sequential JSON, generic and Topics token estimates, excluding the existing current-turn/history estimates. |
-| `topics_transcript_block_characters` | Present block content length tagged `block:user\|assistant\|system\|reasoning\|tool_calls\|tool_results\|tool_definitions\|errors\|run_io\|observations` and `stage:raw\|clipped`; absent blocks emit no sample. |
-| `topics_transcript_blocks_cut` | Number of content blocks cut by their per-block character caps. |
-| `topics_transcript_history_characters`, `topics_transcript_current_turn_characters`, `topics_transcript_history_share` | Rendered content-line characters from replayed input and this run, plus replayed input divided by their sum (0 when both are empty). This-run content includes fallback trace-level I/O, observation markers, and inline errors. |
+These values are always attributes on the `trace-batch-transcript` span. With
+`LANGFUSE_TRACE_BATCH_TRANSCRIPT_METRICS_ENABLED=true` they are also
+`langfuse.trace_batch.*` distributions. That switch defaults to `false`.
+Failure and unavailable estimates still increment the counters named in the
+paragraphs below, with or without the switch.
+
+| Metric | Sample |
+| ---------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `transcript_assembly_duration_ms` | One trace's ordering and assembly time, excluding I/O conversion, stream waits and tokenization; `has_transcript:true\|false`. |
+| `transcript_json_characters` | UTF-16 length of `JSON.stringify(assembleTranscript(...))`, including thread/message wrappers and provenance; zero for a null transcript. Same admitted traces as the Topics length metrics. |
+| `transcript_json_tokens` | o200k estimate of that complete JSON string, tagged `tokenizer:o200k_base`; zero for a null transcript. Attempted on the same admitted traces as `topics_transcript_tokens`; unavailable estimates are omitted. |
+| `generic_transcript_characters` | UTF-16 length of the previous generic plain-text layout, rendered with the same Topics block caps and inclusions. Tool definitions or errors may still render when assembly returns null. |
+| `generic_transcript_tokens` | o200k estimate of that generic text, tagged `tokenizer:o200k_base`. |
+| `transcript_message_tokens` | Sum of current-turn and history message-projection token estimates; zero when empty, omitted if either estimate is unavailable. Distinct from full-transcript JSON. |
+| `transcript_current_turn_tokens` | Token estimate of current-turn messages across all threads, using role and parts only; zero when empty. |
+| `transcript_history_tokens` | Token estimate of history messages across all threads, using the same role/parts representation; zero when empty. |
+| `transcript_content_characters` | Sum of JSON-serialized message-part lengths across history and current turn, in UTF-16 code units; excludes message wrappers and provenance. |
+| `transcript_tool_response_characters` | The subset of content characters belonging to tool-role messages or unmatched tool-result parts; zero when absent. |
+| `transcript_thread_count` | Number of assembled conversation threads, or zero for a null transcript. |
+| `transcript_observation_count` | Ordered, deduplicated observations passed to assembly and the Topics renderer. The `observation_count` span attribute is the raw row count before deduplication. |
+| `transcript_history_message_count`, `transcript_history_part_count` | Replayed messages and their normalized parts across all assembled threads. |
+| `transcript_current_turn_message_count`, `transcript_current_turn_part_count` | This-trace messages and their normalized parts across all assembled threads. |
+| `transcript_current_turn_tool_call_count`, `transcript_current_turn_tool_result_count` | Tool-call parts and tool-result parts in this trace; a tool-role part also counts as a result. |
+| `transcript_assembly_phase_duration_ms` | Two non-overlapping samples per trace tagged `phase:normalization` or `phase:matching`. Normalization includes initial message-key construction and partitioning; matching covers remaining assembly work, including tool matching, deduplication, any rebuilt keys and finalization. Observation ordering is outside these phases but remains in total assembly time. |
+| `topics_transcript_characters` | UTF-16 length of the complete rendered Topics text, including labels, section headings and line breaks; admitted traces only. |
+| `topics_transcript_tokens` | o200k estimate of that same text, tagged `tokenizer:o200k_base`; admitted traces only. |
+| `transcript_comparison_render_duration_ms` | Time to serialize assembled JSON and render generic and Topics text, excluding tokenization. |
+| `transcript_comparison_tokenization_duration_ms` | Wall time for the three sequential JSON, generic and Topics token estimates, excluding the existing current-turn/history estimates. |
+| `topics_transcript_block_characters` | Present block content length tagged `block:user\|assistant\|system\|reasoning\|tool_calls\|tool_results\|tool_definitions\|errors\|run_io\|observations` and `stage:raw\|clipped`; absent blocks emit no sample. |
+| `topics_transcript_blocks_cut` | Number of content blocks cut by their per-block character caps. |
+| `topics_transcript_history_characters`, `topics_transcript_current_turn_characters`, `topics_transcript_history_share` | Rendered content-line characters from replayed input and this run, plus replayed input divided by their sum (0 when both are empty). This-run content includes fallback trace-level I/O, observation markers, and inline errors. |
Topics measurement uses the existing assembled transcript and the inclusive
Topics preset in `packages/shared/src/server/transcript/topics-renderer-config.ts`. It
@@ -182,20 +191,19 @@ and worker; every admitted trace receives Topics measurements.
Transcript token counts use the existing local worker-thread pool and bundled
tiktoken WASM, without a network or model API call. The `gpt-4o` configuration
selects `o200k_base`, also used by GPT-5 mini and nano
-([OpenAI mapping](https://github.com/openai/tiktoken/blob/main/tiktoken/model.py)); metrics are tagged
-`tokenizer:o200k_base`. These are serialized-payload estimates, not provider
+([OpenAI mapping](https://github.com/openai/tiktoken/blob/main/tiktoken/model.py)).
+The span records `tokenizer:o200k_base`. These are serialized-payload estimates, not provider
billing counts or a model's full request framing.
Current turn and history use identical `{ messages: [{ role, parts }] }` JSON
framing, without observation provenance. Only nonempty partitions are tokenized,
at most twice per trace. Their sum is `transcript_message_tokens`; it is not
directly comparable to the full assembled-transcript JSON or rendered text. The
-`transcript_json_*`, `generic_transcript_*`, and `topics_transcript_*` length
-distributions are attempted on the same admitted trace cohort and can be
-compared at p50/p90/p99 after checking their sample counts. The generic
+`transcript_json_*`, `generic_transcript_*`, and `topics_transcript_*` lengths
+are span attributes on the same admitted trace cohort. The generic
renderer preserves the previous plain-text layout under the same Topics
per-block preset; it has no trace-root I/O or run-state sections. Empty
transcripts have zero JSON length and tokens. Thread counts are numeric
-samples, never metric tags.
+span attributes.
To diagnose large Topics texts, compare `topics_transcript_tokens` with
`transcript_observation_count`, the history/current-turn message and part counts,
@@ -237,8 +245,11 @@ trace's observations. Before submitting another transcript it awaits the previou
promise. Projections are tokenized sequentially, keeping at most one pending
tokenizer request per batch. Success and stream
failure both drain accepted promises; a stream failure never assembles its
-partial final trace. Assembly itself remains synchronous. `read_duration_ms`
-includes all this processing.
+partial final trace. Assembly itself remains synchronous. Wall-clock duration
+of that work, including tokenization, is the span attribute `duration_ms` and
+the experiment completion log. The generic queue series
+`langfuse.queue.trace_batch.time_distribution` with `type:processing` records
+the same duration when the processor returns.
Memory includes the current trace, the previous transcript being tokenized and
stream buffers, multiplied by active batch jobs. Each trace also retains its
@@ -255,11 +266,10 @@ CPU and queue depth before raising batch concurrency.
ClickHouse must sort the filtered result, so compare query memory and latency against the unordered
baseline before increasing load.
-Enable percentile aggregations for these distribution metrics in Datadog
-Metrics Summary, then select p50, p75, p90, p95 and p99 in Metrics Explorer.
-Datadog computes these across workers; do not average worker percentiles.
-See [Datadog distributions](https://docs.datadoghq.com/metrics/distributions/).
-Metric delivery and percentile configuration must be verified after deployment.
+Transcript sizes are always span attributes. The same measurements are also
+dogstatsd distributions when `LANGFUSE_TRACE_BATCH_TRANSCRIPT_METRICS_ENABLED`
+is `true`. That switch defaults to `false`. Use the trace explorer query below
+either way.
### Inspect individual transcripts in Datadog
@@ -281,7 +291,7 @@ It also records the remaining Topics metrics above. Block character attributes u
`topics_transcript_block_<block>_<stage>_characters`. Topics character metrics
are absent if rendering failed; the token metric can also be absent if its
estimate failed.
-No transcript content is attached, and IDs are not distribution metric tags.
+No transcript content is attached. Project and trace IDs stay on the span.
`transcript_characters` is the legacy span-only name for the same value as
`transcript_json_characters`. The full JSON includes syntax and provenance, so
use `transcript_content_characters` as the tool-response denominator. JSON
@@ -311,8 +321,8 @@ and retries can produce different or repeated samples for the same trace.
These spans follow existing APM ingestion sampling and retention. Configure a
[custom retention filter](https://docs.datadoghq.com/tracing/trace_pipeline/trace_retention/)
at 100% for the outlier query to keep matching ingested spans searchable. This
-cannot recover spans dropped before ingestion. Distribution metrics remain
-independent and cannot identify a historical sample's trace retroactively.
+cannot recover spans dropped before ingestion. Transcript sizes live on these
+spans and follow APM sampling; they are not separate custom metrics.
## Capacity measurements
@@ -320,19 +330,17 @@ The following metrics use the `langfuse.trace_batch` prefix. All additions are
inside the existing cloud experiment paths; normal ingestion with tracking
disabled performs no new aggregation or Redis calls.
-| Metric | Kind / tags | Meaning |
-| --- | --- | --- |
-| `event_updates`, `serialized_event_bytes` | Counters; `stage:eligible\|sampled\|recorded` | Accepted, valid-start-time updates before sampling, after sampling, and in Redis-acknowledged chunks. Repeated updates count again. |
-| `read_attempts` | Counter; `outcome:success\|failure\|discard` | One outcome per processor invocation, including retries, validation failures and disabled/expired discards. |
-| `read_duration_ms` | Distribution; same outcome tags | Wall-clock processor duration, including failed and discarded attempts. |
-| `active_reads` | Per-process gauge | Streams currently being consumed; failures decrement the count too. |
-| `failed_read_observation_count`, `failed_read_input_bytes`, `failed_read_output_bytes`, `failed_read_metadata_bytes`, `failed_read_io_metadata_bytes` | Distributions | Partial logical rows/bytes consumed before a stream failure. Separate from successful throughput. |
-| `queue_depth` | Gauge; `type:waiting\|active\|delayed\|failed` | Global BullMQ snapshots; waiting includes paused work. |
-| `queue_waiting_head_age_ms` | Gauge | Maximum creation age of the next FIFO jobs in waiting/paused lists. Zero when empty. |
-| `redis_key_bytes` | Gauge; `key:due\|state` | Estimated memory of each readiness key, including Redis overhead. |
-| `redis_key_entries` | Gauge; `key:due\|state` | `ZCARD` / `HLEN` at the memory snapshot. |
-
-The eligible/sampled counters reuse the already-computed serialized size and
+| Metric | Kind / tags | Meaning |
+| ---------------------------------------------------------- | ------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `event_updates`, `serialized_event_bytes` | Counters; `stage:eligible\|recorded` | Accepted, valid-start-time updates before sampling, and in Redis-acknowledged chunks. Repeated updates count again. Selected volume is `sampling_decisions`. |
+| `active_reads` | Per-process gauge | Streams currently being consumed; failures decrement the count too. |
+| `failed_read_observation_count`, `failed_read_input_bytes` | Distributions | Partial logical rows and input bytes consumed before a stream failure. Separate from successful throughput. Output and metadata bytes stay on the completion log. |
+| `queue_depth` | Gauge; `type:delayed` | Delayed BullMQ jobs. Waiting, active, and failed depth stay on `langfuse.queue.trace_batch.depth`. Request rate, wait time, and processing time use the generic `langfuse.queue.trace_batch` series, the same as every other queue. Processing time is recorded when the processor returns. Discard reasons stay on `discarded_jobs`. |
+| `queue_waiting_head_age_ms` | Gauge | Maximum creation age of the next FIFO jobs in waiting/paused lists. Zero when empty. |
+| `redis_key_bytes` | Gauge; `key:due\|state` | Estimated memory of each readiness key, including Redis overhead. |
+| `redis_key_entries` | Gauge; `key:due\|state` | `ZCARD` / `HLEN` at the memory snapshot. |
+
+The eligible and recorded counters reuse the already-computed serialized size and
per-trace aggregates. Recorded volume is emitted after each acknowledged Lua
chunk: earlier chunks remain counted if a later one fails. A timeout can mean
Redis applied an update without acknowledgement, so the stage difference includes
@@ -466,8 +474,9 @@ cost is O(n × k × cap), where k is the fewest feasible jobs. Measure duration
before increasing scale, especially with distant event times that force many jobs.
Compare `dispatched_batches` (tags `strategy`, `fill`),
-`event_time_envelope_ms`, `observed_start_span_ms`, `candidate_buffer_size` and
-`selector_duration_ms` under `langfuse.trace_batch`. The envelope metric includes
+`event_time_envelope_ms`, `candidate_buffer_size` and
+`selector_duration_ms` under `langfuse.trace_batch`. Per-trace event span stays
+on the completion log as `maxTraceSpanMs`. The envelope metric includes
the reader's two-minute margin on each side; the selector's envelope limit does
not. Payload and Redis state are unchanged.
@@ -485,6 +494,8 @@ worker/container rather than a fleet average:
- `runtime.node.mem.heap_used`: JavaScript heap in use.
- `langfuse.trace_batch.snapshot_size`, `pending_traces`, `ready_traces` and
`oldest_due_age_ms`: snapshot and backlog growth.
+- `langfuse.trace_batch.expired_traces`: tagged `source:ingestion` or
+ `source:dispatcher`. Sum within one source; the two writers are different events.
- `langfuse.periodic_runner.duration_ms` and `completed`, filtered by
`runner:trace_batch_dispatcher`: run duration and outcome.
diff --git a/worker/src/features/traceBatching/TraceBatchMetricsRunner.ts b/worker/src/features/traceBatching/TraceBatchMetricsRunner.ts
index fd79cbc1ad..afa70fa532 100644
--- a/worker/src/features/traceBatching/TraceBatchMetricsRunner.ts
+++ b/worker/src/features/traceBatching/TraceBatchMetricsRunner.ts
@@ -75,25 +75,12 @@ export class TraceBatchMetricsRunner extends PeriodicRunner {
private async collectQueueDepth(): Promise<void> {
const queue = TraceBatchQueue.getInstance();
if (!queue) throw new Error("Trace batch queue is unavailable");
- const counts = await queue.getJobCounts(
- "waiting",
- "paused",
- "active",
- "delayed",
- "failed",
- );
- const depths = {
- waiting: (counts.waiting ?? 0) + (counts.paused ?? 0),
- active: counts.active ?? 0,
- delayed: counts.delayed ?? 0,
- failed: counts.failed ?? 0,
- };
- for (const [type, depth] of Object.entries(depths)) {
- recordGauge(`${METRIC_PREFIX}.queue_depth`, depth, {
- type,
- unit: "records",
- });
- }
+ // Waiting, active, and failed depth are already on the generic queue gauge.
+ const counts = await queue.getJobCounts("delayed");
+ recordGauge(`${METRIC_PREFIX}.queue_depth`, counts.delayed ?? 0, {
+ type: "delayed",
+ unit: "records",
+ });
}
private async collectWaitingHeadAge(): Promise<void> {
diff --git a/worker/src/features/traceBatching/traceBatchTranscript.ts b/worker/src/features/traceBatching/traceBatchTranscript.ts
index babde8d7c4..d0c0c7f6d9 100644
--- a/worker/src/features/traceBatching/traceBatchTranscript.ts
+++ b/worker/src/features/traceBatching/traceBatchTranscript.ts
@@ -42,14 +42,25 @@ const TOKEN_METRICS = [
"transcript_history_tokens",
] as const;
+function recordTranscriptDistribution(
+ metric: string,
+ value: number,
+ tags?: { [tag: string]: string | number },
+): void {
+ if (env.LANGFUSE_TRACE_BATCH_TRANSCRIPT_METRICS_ENABLED !== "true") return;
+ if (tags) {
+ recordDistribution(`langfuse.trace_batch.${metric}`, value, tags);
+ return;
+ }
+ recordDistribution(`langfuse.trace_batch.${metric}`, value);
+}
+
function recordTokens(
span: Span,
metric: (typeof TOKEN_METRICS)[number],
tokens: number,
) {
- recordDistribution(`langfuse.trace_batch.${metric}`, tokens, {
- tokenizer: "o200k_base",
- });
+ recordTranscriptDistribution(metric, tokens, { tokenizer: "o200k_base" });
span.setAttribute(`langfuse.trace_batch.${metric}`, tokens);
}
@@ -97,7 +108,7 @@ function recordContentAndStructureCounts(
["transcript_content_characters", total],
["transcript_tool_response_characters", toolResponses],
] as const) {
- recordDistribution(`langfuse.trace_batch.${metric}`, characters);
+ recordTranscriptDistribution(metric, characters);
span.setAttribute(`langfuse.trace_batch.${metric}`, characters);
}
for (const [metric, count] of [
@@ -109,7 +120,7 @@ function recordContentAndStructureCounts(
["transcript_current_turn_tool_call_count", currentTurnToolCalls],
["transcript_current_turn_tool_result_count", currentTurnToolResults],
] as const) {
- recordDistribution(`langfuse.trace_batch.${metric}`, count);
+ recordTranscriptDistribution(metric, count);
span.setAttribute(`langfuse.trace_batch.${metric}`, count);
}
}
@@ -169,7 +180,7 @@ function recordTopicsRendering(
topicsTranscriptConfig,
);
const metric = (name: string, value: number) => {
- recordDistribution(`langfuse.trace_batch.${name}`, value);
+ recordTranscriptDistribution(name, value);
span.setAttribute(`langfuse.trace_batch.${name}`, value);
};
metric("topics_transcript_characters", text.length);
@@ -189,8 +200,8 @@ function recordTopicsRendering(
if (stats.blockCharacters[block].raw === 0) continue;
for (const stage of ["raw", "clipped"] as const) {
const characters = stats.blockCharacters[block][stage];
- recordDistribution(
- "langfuse.trace_batch.topics_transcript_block_characters",
+ recordTranscriptDistribution(
+ "topics_transcript_block_characters",
characters,
{ block, stage },
);
@@ -220,7 +231,7 @@ async function recordTopicsTokens(text: string, span: Span) {
span.setAttribute("langfuse.trace_batch.topics_transcript", "unavailable");
return;
}
- recordDistribution("langfuse.trace_batch.topics_transcript_tokens", tokens, {
+ recordTranscriptDistribution("topics_transcript_tokens", tokens, {
tokenizer: "o200k_base",
});
span.setAttribute("langfuse.trace_batch.topics_transcript_tokens", tokens);
@@ -237,7 +248,7 @@ async function recordTranscriptJsonTokens(text: string | null, span: Span) {
span.setAttribute("langfuse.trace_batch.transcript_json", "unavailable");
return;
}
- recordDistribution("langfuse.trace_batch.transcript_json_tokens", tokens, {
+ recordTranscriptDistribution("transcript_json_tokens", tokens, {
tokenizer: "o200k_base",
});
span.setAttribute("langfuse.trace_batch.transcript_json_tokens", tokens);
@@ -255,7 +266,7 @@ async function recordGenericTranscriptTokens(text: string, span: Span) {
span.setAttribute("langfuse.trace_batch.generic_transcript", "unavailable");
return;
}
- recordDistribution("langfuse.trace_batch.generic_transcript_tokens", tokens, {
+ recordTranscriptDistribution("generic_transcript_tokens", tokens, {
tokenizer: "o200k_base",
});
span.setAttribute("langfuse.trace_batch.generic_transcript_tokens", tokens);
@@ -268,10 +279,7 @@ function recordGenericRendering(
): string | undefined {
try {
const text = renderGenericTranscript(transcript, observations);
- recordDistribution(
- "langfuse.trace_batch.generic_transcript_characters",
- text.length,
- );
+ recordTranscriptDistribution("generic_transcript_characters", text.length);
span.setAttribute(
"langfuse.trace_batch.generic_transcript_characters",
text.length,
@@ -316,8 +324,8 @@ export function recordTraceBatchTranscript(
phaseTimings = timings;
});
const assemblyDurationMs = performance.now() - startedAt;
- recordDistribution(
- "langfuse.trace_batch.transcript_assembly_duration_ms",
+ recordTranscriptDistribution(
+ "transcript_assembly_duration_ms",
assemblyDurationMs,
{ has_transcript: String(transcript !== null) },
);
@@ -330,8 +338,8 @@ export function recordTraceBatchTranscript(
["normalization", phaseTimings.normalizationMs],
["matching", phaseTimings.matchingMs],
] as const) {
- recordDistribution(
- "langfuse.trace_batch.transcript_assembly_phase_duration_ms",
+ recordTranscriptDistribution(
+ "transcript_assembly_phase_duration_ms",
duration,
{ phase },
);
@@ -341,10 +349,7 @@ export function recordTraceBatchTranscript(
);
}
const threadCount = transcript?.threads.length ?? 0;
- recordDistribution(
- "langfuse.trace_batch.transcript_thread_count",
- threadCount,
- );
+ recordTranscriptDistribution("transcript_thread_count", threadCount);
span.setAttribute(
"langfuse.trace_batch.transcript_thread_count",
threadCount,
@@ -358,8 +363,8 @@ export function recordTraceBatchTranscript(
const transcriptJson =
transcript === null ? null : JSON.stringify(transcript);
const transcriptJsonCharacters = transcriptJson?.length ?? 0;
- recordDistribution(
- "langfuse.trace_batch.transcript_json_characters",
+ recordTranscriptDistribution(
+ "transcript_json_characters",
transcriptJsonCharacters,
);
span.setAttribute(
@@ -385,8 +390,8 @@ export function recordTraceBatchTranscript(
);
const comparisonRenderDurationMs =
performance.now() - comparisonRenderStart;
- recordDistribution(
- "langfuse.trace_batch.transcript_comparison_render_duration_ms",
+ recordTranscriptDistribution(
+ "transcript_comparison_render_duration_ms",
comparisonRenderDurationMs,
);
span.setAttribute(
@@ -437,8 +442,8 @@ export function recordTraceBatchTranscript(
}
const comparisonTokenizationDurationMs =
performance.now() - comparisonTokenizationStart;
- recordDistribution(
- "langfuse.trace_batch.transcript_comparison_tokenization_duration_ms",
+ recordTranscriptDistribution(
+ "transcript_comparison_tokenization_duration_ms",
comparisonTokenizationDurationMs,
);
span.setAttribute(
diff --git a/worker/src/features/traceBatching/traceBatching.ts b/worker/src/features/traceBatching/traceBatching.ts
index a048f72d41..73f1f81e6b 100644
--- a/worker/src/features/traceBatching/traceBatching.ts
+++ b/worker/src/features/traceBatching/traceBatching.ts
@@ -549,7 +549,7 @@ const ACKNOWLEDGE_SCRIPT = `
`;
function recordTrackingVolume(
- stage: "eligible" | "sampled" | "recorded",
+ stage: "eligible" | "recorded",
entries: Iterable<
readonly [
string,
@@ -620,7 +620,6 @@ export async function trackTraceBatchActivity(
samplingRate,
})),
);
- recordGauge("langfuse.trace_batch.sampling_rate", samplingRate);
// Counts distinct trace IDs per ingestion batch, not globally unique traces.
recordIncrement("langfuse.trace_batch.sampling_decisions", entries.length, {
decision: "selected",
@@ -630,7 +629,6 @@ export async function trackTraceBatchActivity(
bounds.size - entries.length,
{ decision: "excluded" },
);
- recordTrackingVolume("sampled", entries);
if (entries.length === 0) return;
if (!redis) throw new Error("Trace batching requires Redis");
// Bounded scripts keep ingestion from monopolizing the global Redis slot.
@@ -657,7 +655,9 @@ export async function trackTraceBatchActivity(
);
// Count only acknowledged chunks; a timeout can leave Redis's outcome unknown.
recordTrackingVolume("recorded", chunk);
- recordIncrement("langfuse.trace_batch.expired_traces", expired);
+ recordIncrement("langfuse.trace_batch.expired_traces", expired, {
+ source: "ingestion",
+ });
recordIncrement("langfuse.trace_batch.tracked_traces", chunk.length);
}
} catch (error) {
@@ -751,7 +751,9 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {
const [clock, , pending, ready, oldest, expired] = snapshot.map(Number);
now = clock;
cutoff = String(snapshot[1]);
- recordIncrement("langfuse.trace_batch.expired_traces", expired);
+ recordIncrement("langfuse.trace_batch.expired_traces", expired, {
+ source: "dispatcher",
+ });
recordGauge("langfuse.trace_batch.pending_traces", pending);
recordGauge("langfuse.trace_batch.ready_traces", ready);
recordGauge(
@@ -832,9 +834,6 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {
2 * QUERY_BUFFER_MS,
{ strategy },
);
- recordIncrement("langfuse.trace_batch.dispatched_traces", batch.length, {
- batch_kind: batch.length === 1 ? "singleton" : "multi",
- });
let estimatedEventUpdateCount = 0;
let estimatedSerializedEventBytes = 0;
let unavailableEstimates = 0;
@@ -855,15 +854,6 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {
} else {
unavailableEstimates++;
}
- recordDistribution(
- "langfuse.trace_batch.observed_start_span_ms",
- entry.trace.maxStart - entry.trace.minStart,
- { strategy },
- );
- recordDistribution(
- "langfuse.trace_batch.due_lag_ms",
- Math.max(0, Date.now() - entry.due),
- );
}
if (unavailableEstimates === 0) {
recordDistribution(
diff --git a/worker/src/queues/__tests__/otelDirectEventWrite.test.ts b/worker/src/queues/__tests__/otelDirectEventWrite.test.ts
index 6a1e388ec4..d35f700f92 100644
--- a/worker/src/queues/__tests__/otelDirectEventWrite.test.ts
+++ b/worker/src/queues/__tests__/otelDirectEventWrite.test.ts
@@ -5,7 +5,6 @@ import {
OtelIngestionProcessor,
QueueJobs,
type QueueName,
- recordDistribution,
type TQueueJobTypes,
} from "@langfuse/shared/src/server";
import { env } from "../../env";
@@ -29,7 +28,6 @@ import {
vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({
...(await importOriginal<typeof import("@langfuse/shared/src/server")>()),
getS3EventStorageClient: vi.fn(),
- recordDistribution: vi.fn(),
}));
vi.mock(
"../../features/evaluation/observationEval",
@@ -133,10 +131,6 @@ describe("direct-v4 trace batch tracking", () => {
serializedEventBytes: index === 0 ? 101 : 202,
})),
);
- expect(recordDistribution).toHaveBeenCalledWith(
- "langfuse.trace_batch.ingestion_trace_count",
- 1,
- );
vi.mocked(trackTraceBatchActivity).mockClear();
env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = undefined;
diff --git a/worker/src/queues/__tests__/traceBatchQueue.test.ts b/worker/src/queues/__tests__/traceBatchQueue.test.ts
index ae96d882a1..f1e7e1d5cc 100644
--- a/worker/src/queues/__tests__/traceBatchQueue.test.ts
+++ b/worker/src/queues/__tests__/traceBatchQueue.test.ts
@@ -48,6 +48,8 @@ vi.mock("../../features/tokenisation/async-usage", () => ({
}));
const originalReadEnabled = env.LANGFUSE_TRACE_BATCH_READ_ENABLED;
+const originalTranscriptMetricsEnabled =
+ env.LANGFUSE_TRACE_BATCH_TRANSCRIPT_METRICS_ENABLED;
const originalCloudRegion = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;
const originalExperimentId = env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID;
const processingSpan = trace.wrapSpanContext({
@@ -76,6 +78,8 @@ beforeEach(() => {
});
afterEach(() => {
env.LANGFUSE_TRACE_BATCH_READ_ENABLED = originalReadEnabled;
+ env.LANGFUSE_TRACE_BATCH_TRANSCRIPT_METRICS_ENABLED =
+ originalTranscriptMetricsEnabled;
env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = originalCloudRegion;
env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID = originalExperimentId;
vi.restoreAllMocks();
@@ -106,34 +110,10 @@ describe("trace batch queue", () => {
expect(
JSON.parse(baselineJson).threads[0].currentTurn.observations,
).toEqual([expect.objectContaining({ traceId: "topics-sample" })]);
- expect(recordDistribution).toHaveBeenCalledWith(
- "langfuse.trace_batch.transcript_json_characters",
- baselineJson.length,
- );
- expect(recordDistribution).toHaveBeenCalledWith(
- "langfuse.trace_batch.transcript_json_tokens",
- tokenCount({
- model: vi.mocked(tokenCountAsync).mock.calls[1][0].model,
- text: baselineJson,
- }),
- { tokenizer: "o200k_base" },
- );
const genericText = vi.mocked(tokenCountAsync).mock.calls[2][0].text;
expect(genericText).toBe(
"[user] Explain the result\n[assistant] The answer",
);
- expect(recordDistribution).toHaveBeenCalledWith(
- "langfuse.trace_batch.generic_transcript_characters",
- genericText.length,
- );
- expect(recordDistribution).toHaveBeenCalledWith(
- "langfuse.trace_batch.generic_transcript_tokens",
- tokenCount({
- model: vi.mocked(tokenCountAsync).mock.calls[2][0].model,
- text: genericText,
- }),
- { tokenizer: "o200k_base" },
- );
const renderedText = vi.mocked(tokenCountAsync).mock.calls[3][0].text;
expect(renderedText).toBe(
[
@@ -151,29 +131,29 @@ describe("trace batch queue", () => {
"</end_of_run>",
].join("\n"),
);
- expect(recordDistribution).toHaveBeenCalledWith(
- "langfuse.trace_batch.topics_transcript_tokens",
- tokenCount({
- model: vi.mocked(tokenCountAsync).mock.calls[3][0].model,
- text: renderedText,
- }),
- { tokenizer: "o200k_base" },
- );
- expect(recordDistribution).toHaveBeenCalledWith(
- "langfuse.trace_batch.topics_transcript_block_characters",
- "Explain the result".length,
- { block: "user", stage: "raw" },
- );
- expect(recordDistribution).toHaveBeenCalledWith(
- "langfuse.trace_batch.topics_transcript_block_characters",
- "explain".length,
- { block: "observations", stage: "raw" },
- );
+ expect(recordDistribution).not.toHaveBeenCalled();
expect(exporter.getFinishedSpans()[0].attributes).toMatchObject({
+ "langfuse.trace_batch.transcript_json_characters": baselineJson.length,
+ "langfuse.trace_batch.transcript_json_tokens": tokenCount({
+ model: vi.mocked(tokenCountAsync).mock.calls[1][0].model,
+ text: baselineJson,
+ }),
+ "langfuse.trace_batch.generic_transcript_characters": genericText.length,
+ "langfuse.trace_batch.generic_transcript_tokens": tokenCount({
+ model: vi.mocked(tokenCountAsync).mock.calls[2][0].model,
+ text: genericText,
+ }),
"langfuse.trace_batch.topics_transcript_characters": renderedText.length,
"langfuse.trace_batch.topics_transcript_blocks_cut": 0,
"langfuse.trace_batch.topics_transcript_history_share": 0,
- "langfuse.trace_batch.topics_transcript_tokens": expect.any(Number),
+ "langfuse.trace_batch.topics_transcript_tokens": tokenCount({
+ model: vi.mocked(tokenCountAsync).mock.calls[3][0].model,
+ text: renderedText,
+ }),
+ "langfuse.trace_batch.topics_transcript_block_user_raw_characters":
+ "Explain the result".length,
+ "langfuse.trace_batch.topics_transcript_block_observations_raw_characters":
+ "explain".length,
"langfuse.trace_batch.transcript_observation_count": 1,
"langfuse.trace_batch.transcript_history_message_count": 0,
"langfuse.trace_batch.transcript_current_turn_message_count": 2,
@@ -181,6 +161,38 @@ describe("trace batch queue", () => {
});
});
+ it("emits transcript distributions only when the switch is on", async () => {
+ env.LANGFUSE_TRACE_BATCH_TRANSCRIPT_METRICS_ENABLED = "true";
+ await recordTraceBatchTranscript([
+ convertObservation(
+ createObservation({
+ type: "GENERATION",
+ name: "explain",
+ trace_id: "topics-metrics",
+ input: JSON.stringify([
+ { role: "user", content: "Explain the result" },
+ ]),
+ output: JSON.stringify({ role: "assistant", content: "The answer" }),
+ }),
+ ),
+ ]);
+
+ const renderedText = vi.mocked(tokenCountAsync).mock.calls[3][0].text;
+ expect(recordDistribution).toHaveBeenCalledWith(
+ "langfuse.trace_batch.topics_transcript_characters",
+ String(renderedText).length,
+ );
+ expect(recordDistribution).toHaveBeenCalledWith(
+ "langfuse.trace_batch.topics_transcript_tokens",
+ expect.any(Number),
+ { tokenizer: "o200k_base" },
+ );
+ expect(recordDistribution).toHaveBeenCalledWith(
+ "langfuse.trace_batch.transcript_json_characters",
+ expect.any(Number),
+ );
+ });
+
it("keeps the batch successful when Topics token estimation fails", async () => {
vi.mocked(tokenCountAsync)
.mockResolvedValueOnce(10)
@@ -205,21 +217,18 @@ describe("trace batch queue", () => {
"langfuse.trace_batch.topics_transcript_failed",
1,
);
- expect(recordDistribution).toHaveBeenCalledWith(
- "langfuse.trace_batch.topics_transcript_characters",
- expect.any(Number),
- );
- expect(recordDistribution).not.toHaveBeenCalledWith(
- "langfuse.trace_batch.topics_transcript_tokens",
- expect.any(Number),
- expect.anything(),
- );
- expect(exporter.getFinishedSpans()[0].attributes).toMatchObject({
+ expect(recordDistribution).not.toHaveBeenCalled();
+ const span = exporter.getFinishedSpans()[0];
+ expect(span.attributes).toMatchObject({
"langfuse.trace_batch.topics_transcript": "failed",
+ "langfuse.trace_batch.topics_transcript_characters": expect.any(Number),
"langfuse.trace_batch.transcript_message_tokens": 10,
"langfuse.trace_batch.transcript_json_tokens": 20,
"langfuse.trace_batch.generic_transcript_tokens": 30,
});
+ expect(span.attributes).not.toHaveProperty(
+ "langfuse.trace_batch.topics_transcript_tokens",
+ );
});
it("measures history, current turn, threads and deduplicated tool responses", async () => {
@@ -289,54 +298,33 @@ describe("trace batch queue", () => {
expect(current).toContain("independent question");
expect(historyOnly).toContain("earlier question");
expect(historyOnly).not.toContain("current question");
- expect(recordDistribution).toHaveBeenCalledWith(
- "langfuse.trace_batch.transcript_thread_count",
- 2,
- );
- expect(recordDistribution).toHaveBeenCalledWith(
- "langfuse.trace_batch.transcript_observation_count",
- 4,
- );
- expect(recordDistribution).toHaveBeenCalledWith(
- "langfuse.trace_batch.transcript_history_message_count",
- 2,
- );
- expect(recordDistribution).toHaveBeenCalledWith(
- "langfuse.trace_batch.transcript_history_part_count",
- 2,
- );
- expect(recordDistribution).toHaveBeenCalledWith(
- "langfuse.trace_batch.transcript_current_turn_tool_call_count",
- 1,
- );
- expect(recordDistribution).toHaveBeenCalledWith(
- "langfuse.trace_batch.transcript_current_turn_tool_result_count",
- 1,
- );
- for (const [index, suffix] of [
- "current_turn_tokens",
- "history_tokens",
- ].entries()) {
- expect(recordDistribution).toHaveBeenCalledWith(
- `langfuse.trace_batch.transcript_${suffix}`,
- tokenCount(estimates[index]),
- { tokenizer: "o200k_base" },
- );
- }
- expect(recordDistribution).toHaveBeenCalledWith(
- "langfuse.trace_batch.transcript_message_tokens",
- tokenCount(estimates[0])! + tokenCount(estimates[1])!,
- { tokenizer: "o200k_base" },
- );
+ expect(recordDistribution).not.toHaveBeenCalled();
const span = exporter.getFinishedSpans()[0];
+ expect(span.attributes).toMatchObject({
+ "langfuse.trace_batch.transcript_thread_count": 2,
+ "langfuse.trace_batch.transcript_observation_count": 4,
+ "langfuse.trace_batch.transcript_history_message_count": 2,
+ "langfuse.trace_batch.transcript_history_part_count": 2,
+ "langfuse.trace_batch.transcript_current_turn_tool_call_count": 1,
+ "langfuse.trace_batch.transcript_current_turn_tool_result_count": 1,
+ "langfuse.trace_batch.transcript_current_turn_tokens": tokenCount(
+ estimates[0],
+ ),
+ "langfuse.trace_batch.transcript_history_tokens": tokenCount(
+ estimates[1],
+ ),
+ "langfuse.trace_batch.transcript_message_tokens":
+ tokenCount(estimates[0])! + tokenCount(estimates[1])!,
+ });
const toolCharacters = JSON.stringify({
type: "text",
text: "tool payload",
}).length;
- expect(recordDistribution).toHaveBeenCalledWith(
- "langfuse.trace_batch.transcript_tool_response_characters",
- toolCharacters,
- );
+ expect(
+ span.attributes[
+ "langfuse.trace_batch.transcript_tool_response_characters"
+ ],
+ ).toBe(toolCharacters);
expect(
span.attributes["langfuse.trace_batch.transcript_content_characters"],
).toBeGreaterThan(toolCharacters);
@@ -347,11 +335,6 @@ describe("trace batch queue", () => {
] as number;
expect(duration).toBeGreaterThanOrEqual(0);
phaseTotal += duration;
- expect(recordDistribution).toHaveBeenCalledWith(
- "langfuse.trace_batch.transcript_assembly_phase_duration_ms",
- duration,
- { phase },
- );
}
expect(phaseTotal).toBeLessThanOrEqual(
span.attributes[
@@ -394,19 +377,13 @@ describe("trace batch queue", () => {
JSON.stringify({ type: "text", text: "Please explain this result" })
.length +
JSON.stringify({ type: "text", text: "Answer" }).length;
- expect(recordDistribution).toHaveBeenCalledWith(
- "langfuse.trace_batch.transcript_tool_response_characters",
- toolCharacters,
- );
- expect(recordDistribution).toHaveBeenCalledWith(
- "langfuse.trace_batch.transcript_content_characters",
- totalCharacters,
- );
- expect(recordDistribution).toHaveBeenCalledWith(
- "langfuse.trace_batch.transcript_history_tokens",
- 0,
- { tokenizer: "o200k_base" },
- );
+ expect(recordDistribution).not.toHaveBeenCalled();
+ expect(exporter.getFinishedSpans()[0].attributes).toMatchObject({
+ "langfuse.trace_batch.transcript_tool_response_characters":
+ toolCharacters,
+ "langfuse.trace_batch.transcript_content_characters": totalCharacters,
+ "langfuse.trace_batch.transcript_history_tokens": 0,
+ });
});
it.each(["unavailable", "failed"])(
@@ -570,12 +547,11 @@ describe("trace batch queue", () => {
1,
);
expect(
- vi
- .mocked(recordDistribution)
- .mock.calls.filter(
- ([name]) =>
- name === "langfuse.trace_batch.transcript_assembly_duration_ms",
- ),
+ spans.filter(
+ (span) =>
+ "langfuse.trace_batch.transcript_assembly_duration_ms" in
+ span.attributes,
+ ),
).toHaveLength(streamFails ? 2 : 3);
} finally {
cleaningUp = true;
@@ -652,40 +628,6 @@ describe("trace batch queue", () => {
]) {
expect(serializedTranscript.split(content)).toHaveLength(2);
}
- expect(recordDistribution).toHaveBeenCalledWith(
- "langfuse.trace_batch.transcript_message_tokens",
- tokenCount(estimates[0][0]),
- { tokenizer: "o200k_base" },
- );
- expect(recordDistribution).toHaveBeenCalledWith(
- "langfuse.trace_batch.transcript_message_tokens",
- 0,
- { tokenizer: "o200k_base" },
- );
- for (const suffix of ["current_turn", "history"]) {
- expect(recordDistribution).toHaveBeenCalledWith(
- `langfuse.trace_batch.transcript_${suffix}_tokens`,
- 0,
- { tokenizer: "o200k_base" },
- );
- }
- for (const suffix of ["content", "tool_response"]) {
- expect(recordDistribution).toHaveBeenCalledWith(
- `langfuse.trace_batch.transcript_${suffix}_characters`,
- 0,
- );
- }
- expect(recordDistribution).toHaveBeenCalledWith(
- "langfuse.trace_batch.transcript_thread_count",
- 0,
- );
- for (const hasTranscript of ["true", "false"]) {
- expect(recordDistribution).toHaveBeenCalledWith(
- "langfuse.trace_batch.transcript_assembly_duration_ms",
- expect.any(Number),
- { has_transcript: hasTranscript },
- );
- }
const spans = exporter.getFinishedSpans();
expect(spans).toHaveLength(2);
for (const [index, projectId] of ["a", "b"].entries()) {
@@ -698,6 +640,15 @@ describe("trace batch queue", () => {
"langfuse.trace_batch.has_transcript": index === 0,
"langfuse.trace_batch.transcript_message_tokens":
index === 0 ? tokenCount(estimates[0][0]) : 0,
+ "langfuse.trace_batch.transcript_current_turn_tokens":
+ index === 0 ? expect.any(Number) : 0,
+ "langfuse.trace_batch.transcript_history_tokens":
+ index === 0 ? expect.any(Number) : 0,
+ "langfuse.trace_batch.transcript_content_characters":
+ index === 0 ? expect.any(Number) : 0,
+ "langfuse.trace_batch.transcript_tool_response_characters":
+ index === 0 ? expect.any(Number) : 0,
+ "langfuse.trace_batch.transcript_thread_count": index === 0 ? 1 : 0,
"langfuse.trace_batch.transcript_characters":
index === 0 ? expect.any(Number) : 0,
"langfuse.trace_batch.transcript_assembly_duration_ms":
@@ -720,16 +671,12 @@ describe("trace batch queue", () => {
discarded: "not_cloud",
});
expect(getTraceBatchEventStream).not.toHaveBeenCalled();
- expect(recordIncrement).toHaveBeenCalledWith(
- "langfuse.trace_batch.read_attempts",
+ expect(recordIncrement).toHaveBeenCalledExactlyOnceWith(
+ "langfuse.trace_batch.discarded_jobs",
1,
- { outcome: "discard" },
- );
- expect(recordDistribution).toHaveBeenCalledExactlyOnceWith(
- "langfuse.trace_batch.read_duration_ms",
- expect.any(Number),
- { outcome: "discard" },
+ { reason: "not_cloud" },
);
+ expect(recordDistribution).not.toHaveBeenCalled();
expect(recordGauge).not.toHaveBeenCalled();
expect(job.opts.removeOnComplete).toBe(true);
expect(processingSpan.setAttributes).toHaveBeenCalledWith(
@@ -768,18 +715,11 @@ describe("trace batch queue", () => {
});
}
expect(getTraceBatchEventStream).not.toHaveBeenCalled();
- expect(vi.mocked(recordDistribution).mock.calls).toEqual([
- [
- "langfuse.trace_batch.read_duration_ms",
- expect.any(Number),
- { outcome: "discard" },
- ],
- [
- "langfuse.trace_batch.read_duration_ms",
- expect.any(Number),
- { outcome: "discard" },
- ],
+ expect(vi.mocked(recordIncrement).mock.calls).toEqual([
+ ["langfuse.trace_batch.discarded_jobs", 1, { reason: "expired" }],
+ ["langfuse.trace_batch.discarded_jobs", 1, { reason: "expired" }],
]);
+ expect(recordDistribution).not.toHaveBeenCalled();
expect(recordGauge).not.toHaveBeenCalled();
expect(job.opts.removeOnComplete).toBe(true);
});
@@ -925,20 +865,8 @@ describe("trace batch queue", () => {
expect(vi.mocked(recordDistribution).mock.calls).toEqual([
["langfuse.trace_batch.failed_read_observation_count", 1],
["langfuse.trace_batch.failed_read_input_bytes", 5],
- ["langfuse.trace_batch.failed_read_output_bytes", 6],
- ["langfuse.trace_batch.failed_read_metadata_bytes", 9],
- ["langfuse.trace_batch.failed_read_io_metadata_bytes", 20],
- [
- "langfuse.trace_batch.read_duration_ms",
- expect.any(Number),
- { outcome: "failure" },
- ],
]);
- expect(recordIncrement).toHaveBeenCalledWith(
- "langfuse.trace_batch.read_attempts",
- 1,
- { outcome: "failure" },
- );
+ expect(recordIncrement).not.toHaveBeenCalled();
} finally {
env.LANGFUSE_TRACE_BATCH_MAX_THREADS =
originalEnv.LANGFUSE_TRACE_BATCH_MAX_THREADS;
@@ -1024,10 +952,7 @@ describe("trace batch queue", () => {
["langfuse.trace_batch.active_reads", 1],
["langfuse.trace_batch.active_reads", 0],
]);
- expect(vi.mocked(recordIncrement).mock.calls).toEqual([
- ["langfuse.trace_batch.read_attempts", 1, { outcome: "success" }],
- ["langfuse.trace_batch.read_attempts", 1, { outcome: "failure" }],
- ]);
+ expect(recordIncrement).not.toHaveBeenCalled();
});
it("counts project/trace pairs with producers disabled and safely repeats reads without retaining payloads", async () => {
env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID = "successful-read";
@@ -1142,7 +1067,6 @@ describe("trace batch queue", () => {
["input_bytes", 20],
["output_bytes", 24],
["metadata_bytes", 36],
- ["io_metadata_bytes", 80],
] as const) {
expect(
vi
@@ -1153,14 +1077,14 @@ describe("trace batch queue", () => {
.map(([, value]) => value),
).toEqual([bytes, bytes]);
}
- expect(
- vi
- .mocked(recordDistribution)
- .mock.calls.filter(
- ([name]) => name === "langfuse.trace_batch.found_project_count",
- )
- .map(([, value]) => value),
- ).toEqual([2, 2]);
+ expect(recordDistribution).not.toHaveBeenCalledWith(
+ "langfuse.trace_batch.io_metadata_bytes",
+ expect.anything(),
+ );
+ expect(recordDistribution).not.toHaveBeenCalledWith(
+ "langfuse.trace_batch.found_project_count",
+ expect.anything(),
+ );
expect(recordDistribution).toHaveBeenCalledWith(
"langfuse.trace_batch.missing_trace_count",
2,
@@ -1234,9 +1158,9 @@ describe("trace batch queue", () => {
"langfuse.trace_batch.metadata_bytes",
0,
);
- expect(recordDistribution).toHaveBeenCalledWith(
+ expect(recordDistribution).not.toHaveBeenCalledWith(
"langfuse.trace_batch.found_project_count",
- Number(hasRows),
+ expect.anything(),
);
expect(getTraceBatchEventStream).toHaveBeenCalledWith(
{ traces: [{ ...trace, projectId: "project" }] },
diff --git a/worker/src/queues/traceBatchQueue.ts b/worker/src/queues/traceBatchQueue.ts
index afbc03e92a..cf83bebea3 100644
--- a/worker/src/queues/traceBatchQueue.ts
+++ b/worker/src/queues/traceBatchQueue.ts
@@ -187,9 +187,6 @@ export const traceBatchQueueProcessor: Processor<
for (const [name, value] of [
["observation_count", observationCount],
["input_bytes", inputBytes],
- ["output_bytes", outputBytes],
- ["metadata_bytes", metadataBytes],
- ["io_metadata_bytes", inputBytes + outputBytes + metadataBytes],
] as const) {
recordDistribution(`langfuse.trace_batch.failed_read_${name}`, value);
}
@@ -216,14 +213,6 @@ export const traceBatchQueueProcessor: Processor<
recordDistribution("langfuse.trace_batch.input_bytes", inputBytes);
recordDistribution("langfuse.trace_batch.output_bytes", outputBytes);
recordDistribution("langfuse.trace_batch.metadata_bytes", metadataBytes);
- recordDistribution(
- "langfuse.trace_batch.io_metadata_bytes",
- ioMetadataBytes,
- );
- recordDistribution(
- "langfuse.trace_batch.found_project_count",
- foundProjects.size,
- );
recordDistribution(
"langfuse.trace_batch.missing_trace_count",
batch.traces.length - foundTraces.size,
@@ -279,9 +268,5 @@ export const traceBatchQueueProcessor: Processor<
: {}),
});
}
- recordIncrement("langfuse.trace_batch.read_attempts", 1, { outcome });
- recordDistribution("langfuse.trace_batch.read_duration_ms", durationMs, {
- outcome,
- });
}
};