🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { type Processor } from "bullmq";
import pLimit from "p-limit";
import { backOff } from "exponential-backoff";
import { randomUUID } from "node:crypto";
import { type Observation } from "@langfuse/shared";
import {
convertObservation,
createTraceBatchInputStorage,
getCurrentSpan,
getTraceBatchEventStream,
isInternalEvalEnvironment,
logger,
recordDistribution,
recordGauge,
recordIncrement,
TraceBatchEventSchema,
type QueueName,
type TQueueJobTypes,
type TranscriptContext,
type TraceBatchInputReference,
} from "@langfuse/shared/src/server";
import {
getEnabledTopicsModels,
isTopicsProjectEnabled,
isValidTopicSummary,
listTopicFacets,
listTopicRules,
writeTopicSummaries,
type TopicsModels,
} from "@langfuse/shared/topics/server";
import type { TopicFacet, TopicSummary } from "@langfuse/shared/topics";
import { env } from "../env";
import {
getDeterministicSamplingValue,
shouldSampleEvaluation,
} from "../features/evaluation/deterministicSampling";
import {
isRejectedConnection,
pauseForRejectedConnection,
TopicsProviderUnavailable,
} from "../features/topics/provider-error";
import { TopicMetrics } from "../features/topics/metrics";
import { isRetryableClickhouseInsertError } from "../services/ClickhouseWriter";
import { summarizeAssembledTrace } from "../features/topics/summarizeAssembledTrace";
import { enqueueTopicsAutomation } from "../features/topics/topicsAutomation";
import { recordTraceBatchTranscript } from "../features/traceBatching/traceBatchTranscript";
import { storeInputs } from "../features/traceBatching/storeInputs";
const inputStorage = createTraceBatchInputStorage();
type TraceBatchObservation = Observation & {
eventTimestamp: string;
sessionId: string | null;
};
// Rows include a failed trace's valid facets; the batch writes them once.
type TraceOutcome = (
| { outcome: "disabled" | "unchanged" | "summarized" }
| { outcome: "failed"; reason: string }
) & { rows: TopicSummary[] };
/** Identity of one trace within a batch; a batch spans projects. */
const traceKey = (projectId: string, traceId: string) =>
JSON.stringify([projectId, traceId]);
/**
* Topics settings of one batch, loaded with one query while the read starts.
* A project paused mid-batch is skipped for the rest of the batch.
*/
class TopicsBatchProjects {
private readonly models: Promise<Map<string, TopicsModels>>;
private readonly settings = new Map<
string,
Promise<{ facets: TopicFacet[]; sampling: number }>
>();
private readonly paused = new Set<string>();
constructor(projectIds: Iterable<string>) {
const allowed = [...new Set(projectIds)].filter(isTopicsProjectEnabled);
this.models = getEnabledTopicsModels(allowed);
// A failed load fails each trace that reads it, not the whole batch.
this.models.catch(() => {});
}
async get(projectId: string) {
if (this.paused.has(projectId)) return null;
const models = (await this.models).get(projectId);
if (!models) return null;
let settings = this.settings.get(projectId);
if (!settings) {
settings = loadProjectRule(projectId);
this.settings.set(projectId, settings);
// A failed load fails this trace only; the next trace loads again.
settings.catch(() => this.settings.delete(projectId));
}
return { models, ...(await settings) };
}
async pause(projectId: string, error: TopicsProviderUnavailable) {
if (this.paused.has(projectId)) return;
// The project is skipped for this batch even if recording the pause fails.
this.paused.add(projectId);
await pauseForRejectedConnection(projectId, error);
}
}
/** The saved rule's facets and sampling; without a rule, every facet runs. */
async function loadProjectRule(projectId: string) {
const [facets, rules] = await Promise.all([
listTopicFacets(projectId),
listTopicRules(projectId),
]);
const rule = rules[0];
if (!rule) return { facets, sampling: 1 };
const selected = new Set(rule.facetIds);
return {
facets: facets.filter((facet) => selected.has(facet.id)),
sampling: rule.sampling,
};
}
/** Orders events like manual processing: event_ts, then span ID, then environment. */
function newestEvent(observations: TraceBatchObservation[]) {
return observations.reduce<TraceBatchObservation | undefined>(
(current, row) => {
if (!current) return row;
if (row.eventTimestamp !== current.eventTimestamp)
return row.eventTimestamp > current.eventTimestamp ? row : current;
if (row.id !== current.id) return row.id > current.id ? row : current;
return row.environment > current.environment ? row : current;
},
undefined,
);
}
/** Session of the newest event that has one; events of a trace can carry different or no sessions. */
function newestSessionId(observations: TraceBatchObservation[]) {
return (
newestEvent(observations.filter((row) => row.sessionId))?.sessionId ?? null
);
}
async function summarizeTraceBatch(
observations: TraceBatchObservation[],
context: TranscriptContext | null,
topics: TopicsBatchProjects,
jobEnqueuedAt: Date,
revisions: ReadonlyMap<string, string>,
): Promise<TraceOutcome> {
const first = observations[0];
const traceId = first?.traceId;
if (
!first ||
!traceId ||
observations.some(
(row) => row.projectId !== first.projectId || row.traceId !== traceId,
)
)
throw new Error(
"Trace batch observations crossed a project or trace boundary.",
);
// Match manual processing and discovery, preserving event_ts microseconds.
const latest = newestEvent(observations)!;
if (isInternalEvalEnvironment(latest.environment))
return { outcome: "disabled", rows: [] };
const revision = revisions.get(traceKey(first.projectId, traceId));
const inputReference: TraceBatchInputReference | undefined =
revision === undefined
? undefined
: { projectId: first.projectId, traceId, revision };
const traceTimestamp = new Date(
Math.min(...observations.map((row) => row.startTime.getTime())),
).toISOString();
const sessionId = newestSessionId(observations);
const traceContext = {
sessionId,
traceTimestamp,
environment: latest.environment,
traceName: first.name,
};
let rows: TopicSummary[] = [];
try {
const project = await topics.get(first.projectId);
if (
!project ||
!shouldSampleEvaluation({
samplingValue: getDeterministicSamplingValue(traceId),
samplingRate: project.sampling,
})
)
return { outcome: "disabled", rows };
// Capture reports its own failures; drain both before releasing the Topics slot.
const [summary] = await Promise.allSettled([
summarizeAssembledTrace({
projectId: first.projectId,
traceId,
...traceContext,
jobEnqueuedAt,
traceName: traceContext.traceName ?? "",
transcript: context?.transcript ?? null,
models: project.models,
facets: project.facets,
}),
inputReference
? storeInputs(
inputReference,
{
transcript: context?.transcript ?? null,
executionIssues: context?.executionIssues ?? [],
traceContext,
},
inputStorage,
)
: Promise.resolve(),
]);
if (summary.status === "rejected") throw summary.reason;
const result = summary.value;
rows = result.rows;
if (result.failure === undefined) return { outcome: result.outcome, rows };
throw result.failure;
} catch (error) {
// One trace's failure must not fail or re-read the shared batch.
const reason =
error instanceof TopicsProviderUnavailable ? error.reason : "other";
if (isRejectedConnection(error)) await topics.pause(first.projectId, error);
logger.warn("Topics summary failed for trace", {
projectId: first.projectId,
traceId,
reason,
error: error instanceof Error ? error.message : String(error),
});
return { outcome: "failed", reason, rows };
}
}
/**
* Writes the batch's rows in one insert and returns the rows it dropped. Invalid rows are
* dropped up front so they cannot fail the others; like ClickhouseWriter, transient errors
* are retried and any other error drops the insert once.
*/
async function writeBatchSummaries(
rows: TopicSummary[],
): Promise<TopicSummary[]> {
const valid = rows.filter(isValidTopicSummary);
const invalid = rows.filter((row) => !isValidTopicSummary(row));
if (invalid.length > 0)
logger.warn("Topics summary rows failed validation", {
rowCount: invalid.length,
});
if (valid.length === 0) return invalid;
try {
await new TopicMetrics().measure(
"storage",
() =>
backOff(() => writeTopicSummaries(valid), {
numOfAttempts: env.LANGFUSE_INGESTION_CLICKHOUSE_MAX_ATTEMPTS,
retry: isRetryableClickhouseInsertError,
}),
"storage",
);
} catch (error) {
logger.warn("Topics summary write failed for trace batch", {
rowCount: valid.length,
error: error instanceof Error ? error.message : String(error),
});
return rows;
}
for (const row of valid)
recordIncrement("langfuse.topics.facet_summaries", 1, { state: row.state });
return invalid;
}
/** Failed traces are counted, not retried. */
function recordTopicsOutcomes(outcomes: TraceOutcome[]): void {
const counts = new Map<string, number>();
for (const result of outcomes) {
const key = JSON.stringify([
result.outcome,
result.outcome === "failed" ? result.reason : undefined,
]);
counts.set(key, (counts.get(key) ?? 0) + 1);
}
const perOutcome: Record<string, number> = {};
for (const [key, count] of counts) {
const [outcome, reason] = JSON.parse(key) as [string, string | null];
recordIncrement("langfuse.topics.trace_outcomes", count, {
outcome,
...(reason ? { reason } : {}),
});
if (outcome === "disabled") continue;
const attribute = `langfuse.topics.traces.${outcome}`;
perOutcome[attribute] = (perOutcome[attribute] ?? 0) + count;
}
// Batch totals of Topics work on the job span; each trace has its own child span.
if (Object.keys(perOutcome).length > 0)
getCurrentSpan()?.setAttributes(perOutcome);
}
const JOB_MAX_AGE_MS = 2 * 60 * 60_000;
let activeReads = 0;
export function recordTraceBatchActiveReads(): void {
recordGauge("langfuse.trace_batch.active_reads", activeReads);
}
export const traceBatchQueueProcessor: Processor<
TQueueJobTypes[QueueName.TraceBatch]
> = async (job) => {
const startedAt = performance.now();
// Keep attributes on the processing span rather than a nested read span.
const span = env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID
? getCurrentSpan()
: undefined;
let outcome: "success" | "failure" | "discard" = "failure";
let queryId: string | undefined;
let batchShape:
| {
batchTraceCount: number;
batchProjectCount: number;
eventTimeSpanMs: number;
maxTraceSpanMs: number;
}
| undefined;
const foundTraces = new Set<string>();
const foundProjects = new Set<string>();
let observationCount = 0;
let inputBytes = 0;
let outputBytes = 0;
let metadataBytes = 0;
try {
span?.setAttributes({
"langfuse.trace_batch.experiment_id":
env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID,
"langfuse.trace_batch.job_id": job.id,
"langfuse.trace_batch.attempt": job.attemptsMade + 1,
});
const disabledReason = (() => {
if (!env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION) {
return "not_cloud";
}
if (env.LANGFUSE_TRACE_BATCH_READ_ENABLED !== "true") {
return "reads_disabled";
}
return undefined;
})();
if (disabledReason) {
outcome = "discard";
// BullMQ reads these options after the processor returns, then removes atomically.
job.opts.removeOnComplete = true;
recordIncrement("langfuse.trace_batch.discarded_jobs", 1, {
reason: disabledReason,
});
return { discarded: disabledReason };
}
const event = TraceBatchEventSchema.parse(job.data);
if (Date.now() - event.timestamp.getTime() >= JOB_MAX_AGE_MS) {
outcome = "discard";
job.opts.removeOnComplete = true;
recordIncrement("langfuse.trace_batch.discarded_jobs", 1, {
reason: "expired",
});
return { discarded: "expired" };
}
const batch = event.payload;
const revisions = new Map(
batch.traces.map((trace) => [
traceKey(trace.projectId, trace.traceId),
trace.revision,
]),
);
const topicsOutcomes: TraceOutcome[] = [];
let droppedRows = new Set<TopicSummary>();
const topics = new TopicsBatchProjects(
batch.traces.map((trace) => trace.projectId),
);
const queryOptions = {
maxThreads: env.LANGFUSE_TRACE_BATCH_MAX_THREADS,
maxBlockSize: env.LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE,
requestTimeoutMs: env.LANGFUSE_TRACE_BATCH_REQUEST_TIMEOUT_MS,
experimentId: env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID,
queryId: randomUUID(),
};
queryId = queryOptions.queryId;
if (queryOptions.experimentId) {
const projects = new Set<string>();
let minStart = Infinity;
let maxStart = -Infinity;
let maxTraceSpanMs = 0;
for (const trace of batch.traces) {
projects.add(trace.projectId);
minStart = Math.min(minStart, trace.minStart);
maxStart = Math.max(maxStart, trace.maxStart);
maxTraceSpanMs = Math.max(
maxTraceSpanMs,
trace.maxStart - trace.minStart,
);
}
batchShape = {
batchTraceCount: batch.traces.length,
batchProjectCount: projects.size,
eventTimeSpanMs: batch.traces.length ? maxStart - minStart : 0,
maxTraceSpanMs,
};
span?.setAttributes({
"langfuse.trace_batch.query_id": queryId,
"langfuse.trace_batch.batch_trace_count": batchShape.batchTraceCount,
"langfuse.trace_batch.batch_project_count":
batchShape.batchProjectCount,
"langfuse.trace_batch.event_time_span_ms": batchShape.eventTimeSpanMs,
"langfuse.trace_batch.max_trace_span_ms": batchShape.maxTraceSpanMs,
});
logger.info("Trace batch experiment read", {
...queryOptions,
jobId: job.id,
attempt: job.attemptsMade + 1,
// null means no override; the server profile's value is not known here.
maxBlockSize: queryOptions.maxBlockSize ?? null,
buildId: env.BUILD_ID,
batchTraceCount: batch.traces.length,
maxBatchSize: env.LANGFUSE_TRACE_BATCH_MAX_SIZE,
concurrency: env.LANGFUSE_TRACE_BATCH_CONCURRENCY,
samplingRate: env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE,
idleMs: env.LANGFUSE_TRACE_BATCH_IDLE_MS,
dispatchIntervalMs: env.LANGFUSE_TRACE_BATCH_DISPATCH_INTERVAL_MS,
});
}
// ClickHouse sockets time out when idle, so the reader never waits on
// transcript or Topics work unless queued payloads exceed the cap.
// Transcripts are tokenized one trace at a time to keep at most one pending
// request in the tokenizer pool shared with ingestion; Topics model calls
// then run for up to LANGFUSE_TOPICS_TRACE_CONCURRENCY traces at once.
const pendingSummaries = new Set<Promise<void>>();
const topicsLimit = pLimit(env.LANGFUSE_TOPICS_TRACE_CONCURRENCY);
let transcribing: Promise<unknown> = Promise.resolve();
let pendingSummaryBytes = 0;
let summaryFailed = false;
let summaryError: unknown;
const enqueueSummary = (
observations: TraceBatchObservation[],
bytes: number,
) => {
pendingSummaryBytes += bytes;
const enqueuedAt = performance.now();
const transcribed = transcribing.then(async () => {
// Let socket reads run between traces' synchronous assembly.
await new Promise((resolve) => setImmediate(resolve));
recordDistribution(
"langfuse.trace_batch.summary_queue_wait_ms",
performance.now() - enqueuedAt,
);
if (summaryFailed) return null;
let context: TranscriptContext | null = null;
await recordTraceBatchTranscript(observations, async (assembled) => {
context = assembled;
});
return { context };
});
transcribing = transcribed.catch(() => {});
const summary: Promise<void> = transcribed
.then(async (assembled) => {
if (!assembled) return;
await topicsLimit(async () => {
if (!summaryFailed)
topicsOutcomes.push(
await summarizeTraceBatch(
observations,
assembled.context,
topics,
event.timestamp,
revisions,
),
);
});
})
.catch((error: unknown) => {
if (!summaryFailed) summaryError = error;
summaryFailed = true;
})
.finally(() => {
pendingSummaryBytes -= bytes;
pendingSummaries.delete(summary);
});
pendingSummaries.add(summary);
};
activeReads++;
try {
recordTraceBatchActiveReads();
let traceObservations: TraceBatchObservation[] = [];
let traceBytes = 0;
for await (const event of getTraceBatchEventStream(batch, queryOptions)) {
observationCount++;
foundTraces.add(traceKey(event.project_id, event.trace_id));
foundProjects.add(event.project_id);
// Logical UTF-8 payload size, excluding JSON transport and compression.
const eventInputBytes = Buffer.byteLength(event.input);
const eventOutputBytes = Buffer.byteLength(event.output);
let eventBytes = eventInputBytes + eventOutputBytes;
inputBytes += eventInputBytes;
outputBytes += eventOutputBytes;
for (const [key, value] of Object.entries(event.metadata)) {
const bytes = Buffer.byteLength(key) + Buffer.byteLength(value);
metadataBytes += bytes;
eventBytes += bytes;
}
const previous = traceObservations[0];
if (
previous &&
(previous.projectId !== event.project_id ||
previous.traceId !== event.trace_id)
) {
if (summaryFailed) break;
enqueueSummary(traceObservations, traceBytes);
traceObservations = [];
traceBytes = 0;
if (
pendingSummaryBytes >
env.LANGFUSE_TRACE_BATCH_MAX_PENDING_SUMMARY_BYTES
) {
recordIncrement("langfuse.trace_batch.summary_backpressure", 1);
// Resume as soon as completed work brings the queue below the cap.
while (
pendingSummaryBytes >
env.LANGFUSE_TRACE_BATCH_MAX_PENDING_SUMMARY_BYTES
)
await Promise.race(pendingSummaries);
}
if (summaryFailed) break;
}
traceBytes += eventBytes;
traceObservations.push({
...convertObservation({
...event,
id: event.span_id,
parent_observation_id: event.parent_span_id,
// These required converter fields are not used by the transcript.
created_at: event.event_ts,
updated_at: event.event_ts,
is_deleted: 0,
provided_usage_details: {},
provided_cost_details: {},
usage_details: {},
cost_details: {},
}),
eventTimestamp: event.event_ts,
sessionId: event.session_id || null,
});
}
// Reaching EOF completes the last trace; a failed stream must not flush it.
if (traceObservations.length && !summaryFailed)
enqueueSummary(traceObservations, traceBytes);
} finally {
try {
// Drain and store accepted summaries even when the stream fails, so a retry skips them.
await Promise.all(pendingSummaries);
droppedRows = new Set(
await writeBatchSummaries(topicsOutcomes.flatMap(({ rows }) => rows)),
);
} finally {
activeReads--;
recordTraceBatchActiveReads();
}
// Stored summaries skip summarization on retry, so queue them before a failure rethrows.
await enqueueTopicsAutomation(
event.id,
topicsOutcomes
.flatMap(({ rows }) => rows)
.filter((row) => !droppedRows.has(row)),
).catch((error: unknown) =>
logger.warn("Topics assignment could not be queued for trace batch", {
error: error instanceof Error ? error.message : String(error),
}),
);
}
if (summaryFailed) throw summaryError;
recordDistribution(
"langfuse.trace_batch.observation_count",
observationCount,
);
recordDistribution(
"langfuse.trace_batch.found_trace_count",
foundTraces.size,
);
const ioMetadataBytes = inputBytes + outputBytes + metadataBytes;
recordDistribution("langfuse.trace_batch.input_bytes", inputBytes);
recordDistribution("langfuse.trace_batch.output_bytes", outputBytes);
recordDistribution("langfuse.trace_batch.metadata_bytes", metadataBytes);
recordDistribution(
"langfuse.trace_batch.missing_trace_count",
batch.traces.length - foundTraces.size,
);
recordTopicsOutcomes(
topicsOutcomes.map((result) =>
result.outcome === "summarized" &&
result.rows.some((row) => droppedRows.has(row))
? { outcome: "failed", reason: "storage", rows: result.rows }
: result,
),
);
// Retries can repeat this read; observation payloads never enter job results.
outcome = "success";
return {
observationCount,
traceCount: foundTraces.size,
projectCount: foundProjects.size,
inputBytes,
outputBytes,
metadataBytes,
ioMetadataBytes,
};
} finally {
if (queryId && outcome === "failure") {
// Consumed rows from failed attempts are not successful throughput.
for (const [name, value] of [
["observation_count", observationCount],
["input_bytes", inputBytes],
] as const) {
recordDistribution(`langfuse.trace_batch.failed_read_${name}`, value);
}
}
const durationMs = performance.now() - startedAt;
span?.setAttributes({
"langfuse.trace_batch.outcome": outcome,
"langfuse.trace_batch.duration_ms": durationMs,
...(queryId
? {
"langfuse.trace_batch.observation_count": observationCount,
"langfuse.trace_batch.found_trace_count": foundTraces.size,
"langfuse.trace_batch.found_project_count": foundProjects.size,
"langfuse.trace_batch.input_bytes": inputBytes,
"langfuse.trace_batch.output_bytes": outputBytes,
"langfuse.trace_batch.metadata_bytes": metadataBytes,
"langfuse.trace_batch.partial": outcome !== "success",
}
: {}),
});
if (env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID) {
logger.info("Trace batch experiment read completed", {
experimentId: env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID,
jobId: job.id,
attempt: job.attemptsMade + 1,
queryId,
outcome,
durationMs,
...batchShape,
...(queryId
? {
observationCount,
foundTraceCount: foundTraces.size,
foundProjectCount: foundProjects.size,
inputBytes,
outputBytes,
metadataBytes,
partial: outcome !== "success",
}
: {}),
});
}
}
};