🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import pLimit from "p-limit";
import { prisma } from "@langfuse/shared/src/db";
import { BatchActionStatus, observationForEvalSchema } from "@langfuse/shared";
import { logger, traceException } from "@langfuse/shared/src/server";
import { createObservationEvalSchedulerDeps } from "../evaluation/observationEval/createSchedulerDeps";
import { scheduleObservationEvals } from "../evaluation/observationEval/scheduleObservationEvals";
import type { ObservationEvalRule } from "../evaluation/observationEval/types";
const BATCH_SIZE = 500;
const CONCURRENCY_LIMIT = 50;
const MAX_ERROR_LOG_LINES = 20;
export async function processBatchedObservationEval(params: {
projectId: string;
batchActionId: string;
evaluators: ObservationEvalRule[];
evaluatorLabels?: string[];
observationStream: AsyncIterable<Record<string, unknown>>;
}): Promise<void> {
const {
projectId,
batchActionId,
evaluators,
evaluatorLabels,
observationStream,
} = params;
const limit = pLimit(CONCURRENCY_LIMIT);
const schedulerDeps = createObservationEvalSchedulerDeps();
await prisma.batchAction.update({
where: { id: batchActionId, projectId },
data: {
status: BatchActionStatus.Processing,
totalCount: 0,
processedCount: 0,
failedCount: 0,
log: null,
},
});
let totalCount = 0;
let processedCount = 0;
let failedCount = 0;
const errors: string[] = [];
let buffer: Record<string, unknown>[] = [];
const processBatch = async (batch: Record<string, unknown>[]) => {
const results = await Promise.allSettled(
batch.map((record) =>
limit(async () => {
const toolCallNames = Array.isArray(record.tool_call_names)
? record.tool_call_names
: [];
const observation = observationForEvalSchema.parse({
...record,
tool_call_count: toolCallNames.length,
});
await scheduleObservationEvals({
observation,
configs: evaluators,
schedulerDeps,
executionMode: "MANUAL",
executionScopeId: batchActionId,
batchActionId,
});
}),
),
);
for (let i = 0; i < results.length; i++) {
const result = results[i];
if (result.status === "fulfilled") {
processedCount++;
} else {
failedCount++;
traceException(result.reason);
if (errors.length < MAX_ERROR_LOG_LINES) {
const errorMessage =
result.reason instanceof Error
? result.reason.message
: "Unknown error";
errors.push(
`Row ${totalCount - batch.length + i + 1}: ${errorMessage}`,
);
}
}
}
await prisma.batchAction.update({
where: { id: batchActionId, projectId },
data: { totalCount, processedCount, failedCount },
});
};
for await (const record of observationStream) {
buffer.push(record);
totalCount++;
if (buffer.length >= BATCH_SIZE) {
await processBatch(buffer);
buffer = [];
}
}
// Process remaining records
if (buffer.length > 0) {
await processBatch(buffer);
}
const finalStatus = (() => {
if (failedCount === 0) {
return BatchActionStatus.Completed;
}
if (processedCount === 0) {
return BatchActionStatus.Failed;
}
return BatchActionStatus.Partial;
})();
const errorSummary =
errors.length > 0
? `${failedCount} observations failed while scheduling ${evaluators.length} evaluator(s): ${(evaluatorLabels ?? evaluators.map((evaluator) => ("scoreName" in evaluator ? evaluator.scoreName : evaluator.id))).join(", ")}.\n${errors.join("\n")}`
: null;
await prisma.batchAction.update({
where: { id: batchActionId, projectId },
data: {
status: finalStatus,
finishedAt: new Date(),
totalCount,
processedCount,
failedCount,
log: errorSummary,
},
});
logger.info(
`Completed observation-run-batched-evaluation action ${batchActionId}`,
{
totalCount,
processedCount,
failedCount,
finalStatus,
},
);
}