🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
/* eslint-disable @typescript-eslint/no-use-before-define */
import { z } from "zod";
import {
DEFAULT_TRACE_ENVIRONMENT,
ObservationEvalExecutionEventSchema,
extractObservationVariables,
logger,
recordIncrement,
buildEvalExecutionData,
} from "@langfuse/shared/src/server";
import { isEvalTargetEnvironmentAllowed } from "../isEvalTargetEnvironmentAllowed";
import {
getCodeEvalVariableMapping,
observationForEvalSchema,
observationVariableMappingList,
type EvalTemplateWithType,
canRunEvalRule,
} from "@langfuse/shared";
import {
EvalTemplateType,
JobConfigState,
Prisma,
type EvalTemplate,
type JobConfiguration,
type JobExecution,
} from "@prisma/client";
import { prisma, JobExecutionStatus } from "@langfuse/shared/src/db";
import { UnrecoverableError } from "../../../errors/UnrecoverableError";
import {
EvalFailureError,
EvalFailureReason,
UnrecoverableEvalFailureError,
} from "../evalFailureErrors";
import {
completeEvalExecution,
type EvalExecutionResult,
} from "../evalCompletion";
import {
createProductionEvalExecutionDeps,
type EvalExecutionDeps,
} from "../evalExecutionDeps";
import { runLLMAsJudgeEvaluation } from "../evalService";
import { executeCodeBasedEvaluation } from "../codeBased/executeCodeBasedEvaluation";
import { runDecisionModelEvaluation } from "../decisionModel/runDecisionModelEvaluation";
import { getEvalS3StorageClient } from "../s3StorageClient";
import { type ObservationForEval } from "./types";
/**
* Dependencies for processing observation evals.
* Allows S3 operations to be injected for testability.
*/
export interface ObservationEvalProcessorDeps {
downloadObservationFromS3: (path: string) => Promise<string>;
evalExecutionDeps: EvalExecutionDeps;
}
/**
* Creates production dependencies for the observation eval processor.
*/
function createObservationEvalProcessorDeps(): ObservationEvalProcessorDeps {
return {
downloadObservationFromS3: async (path: string) => {
const s3Client = getEvalS3StorageClient();
return s3Client.download(path);
},
evalExecutionDeps: createProductionEvalExecutionDeps(),
};
}
/**
* Processes an observation-level evaluation job.
*
* This function:
* 1. Fetches job execution, config, and the expected template type
* 2. Downloads observation data from S3 (stored during scheduling)
* 3. Extracts variables from the observation
* 4. Executes the evaluator-specific implementation
* 5. Completes the eval execution with shared score persistence
*/
type ObservationEvalExecutionType =
| typeof EvalTemplateType.LLM_AS_JUDGE
| typeof EvalTemplateType.CODE;
const EVALUATOR_TYPES_BY_EXECUTION_TYPE: Record<
ObservationEvalExecutionType,
EvalTemplateType[]
> = {
[EvalTemplateType.LLM_AS_JUDGE]: [
EvalTemplateType.LLM_AS_JUDGE,
EvalTemplateType.DECISION_MODEL,
],
[EvalTemplateType.CODE]: [EvalTemplateType.CODE],
};
export type ObservationEvalProcessorOutcome =
| "completed"
| "cancelled"
| "skipped";
type ProcessObservationEvalParams = {
event: z.infer<typeof ObservationEvalExecutionEventSchema>;
executionType: ObservationEvalExecutionType;
deps?: ObservationEvalProcessorDeps;
};
export async function processObservationEval(
params: ProcessObservationEvalParams,
): Promise<ObservationEvalProcessorOutcome> {
const { event, deps = createObservationEvalProcessorDeps() } = params;
logger.debug(
`Processing observation eval job ${event.jobExecutionId} for project ${event.projectId}`,
);
// Fetch job execution
const job = await prisma.jobExecution.findFirst({
where: {
id: event.jobExecutionId,
projectId: event.projectId,
},
});
if (!job) {
logger.info(
`Job execution ${event.jobExecutionId} not found. It may have been deleted.`,
);
return "skipped";
}
// Observation eval executions may already be terminal if the evaluator was
// blocked after scheduling or the queue job was redelivered after processing.
if (
job.status === "CANCELLED" ||
job.status === "ERROR" ||
job.status === "COMPLETED"
) {
logger.debug(`Job execution ${event.jobExecutionId} is already terminal.`);
return "skipped";
}
const resolved = await resolveObservationEvalExecution({
event,
job,
executionType: params.executionType,
});
if (resolved.type === "cancelled") {
await cancelJobExecution(job, event.projectId, resolved.reason);
return "cancelled";
}
const { config: evalJobConfig, template } = resolved;
if (!canRunEvalRule(evalJobConfig, event.executionMode)) {
logger.debug(
`Job execution ${event.jobExecutionId} is not executable because the evaluator is blocked or inactive.`,
);
await prisma.jobExecution.update({
where: {
id: job.id,
projectId: event.projectId,
},
data: {
status: JobExecutionStatus.CANCELLED,
endTime: new Date(),
},
});
return "cancelled";
}
// Download observation data from S3
let observationData: ObservationForEval;
let downloadedString: string;
try {
downloadedString = await deps.downloadObservationFromS3(
event.observationS3Path,
);
} catch (e) {
// S3 download failures are retryable (network issues, temporary unavailability)
throw new EvalFailureError(
`Failed to download observation from S3 at ${event.observationS3Path}: ${e}`,
EvalFailureReason.ObservationDownload,
);
}
// Parse and validate the downloaded data - these are permanent failures
try {
const parsedJson = JSON.parse(downloadedString);
observationData = observationForEvalSchema.parse(parsedJson);
} catch (e) {
// JSON parse errors are permanent - the data won't change on retry
throw new UnrecoverableEvalFailureError(
`Invalid observation data from S3 at ${event.observationS3Path}: invalid JSON - ${e}`,
EvalFailureReason.InvalidObservationData,
);
}
logger.debug(
`Downloaded observation data for job ${job.id}: span_id=${observationData.span_id}`,
);
// Final fail-closed loop safeguard: never execute an eval whose target
// lives in an internal Langfuse environment, regardless of which scheduling
// path created the job. See isEvalTargetEnvironmentAllowed.
if (!isEvalTargetEnvironmentAllowed(observationData.environment)) {
logger.warn(
"Cancelling eval job targeting an internal Langfuse environment",
{
jobExecutionId: event.jobExecutionId,
projectId: event.projectId,
environment: observationData.environment,
observationId: observationData.span_id,
},
);
recordIncrement(
"langfuse.evaluation-execution.internal_target_blocked",
1,
{
source: "observation-eval",
},
);
await prisma.jobExecution.update({
where: { id: job.id, projectId: event.projectId },
data: { status: JobExecutionStatus.CANCELLED, endTime: new Date() },
});
return "cancelled";
}
// Extract variables from observation
const parsedVariableMapping = observationVariableMappingList.parse(
evalJobConfig.variableMapping,
);
const extractedVariables = extractObservationVariables({
observation: observationData,
variableMapping: parsedVariableMapping,
});
logger.debug(
`Extracted ${extractedVariables.length} variables for job ${job.id}`,
);
const executionParams = {
projectId: event.projectId,
organizationId: resolved.organizationId,
jobExecutionId: event.jobExecutionId,
job,
config: evalJobConfig,
extractedVariables,
hasExperimentContext: Boolean(observationData.experiment_id),
environment: observationData.environment ?? DEFAULT_TRACE_ENVIRONMENT,
...buildEvalExecutionData({
type: "JOB",
jobExecutionId: event.jobExecutionId,
jobConfigurationId: job.jobConfigurationId,
...(resolved.type === "v2"
? {
evaluationRuleId: resolved.evaluationRuleId,
assignmentId: resolved.assignmentId,
evaluatorId: resolved.evaluatorId,
evaluatorVersionId: resolved.evaluatorVersionId,
}
: {}),
targetTraceId: job.jobInputTraceId,
targetObservationId: job.jobInputObservationId,
targetDatasetItemId: job.jobInputDatasetItemId,
batchActionId: event.batchActionId,
}),
deps: deps.evalExecutionDeps,
};
let executionResult: EvalExecutionResult;
switch (template.type) {
case EvalTemplateType.LLM_AS_JUDGE:
executionResult = await runLLMAsJudgeEvaluation({
...executionParams,
template,
// A self-inflicted pause targets the evaluator, so every rule using it
// stops; legacy executions pause their job configuration instead.
...(resolved.type === "v2"
? { evaluatorId: resolved.evaluatorId }
: {}),
});
break;
case EvalTemplateType.CODE:
executionResult = await executeCodeBasedEvaluation({
...executionParams,
template,
...(resolved.type === "v2"
? { evaluatorId: resolved.evaluatorId }
: {}),
});
break;
case EvalTemplateType.DECISION_MODEL:
executionResult = await runDecisionModelEvaluation({
...executionParams,
template,
...(resolved.type === "v2"
? { evaluatorId: resolved.evaluatorId }
: {}),
});
break;
}
await completeEvalExecution({
projectId: executionParams.projectId,
jobExecutionId: executionParams.jobExecutionId,
traceId: executionParams.job.jobInputTraceId,
observationId: executionParams.job.jobInputObservationId,
environment: executionParams.environment,
executionType: params.executionType,
deps: executionParams.deps,
result: executionResult,
});
return "completed";
}
async function cancelJobExecution(
job: JobExecution,
projectId: string,
reason: string,
) {
logger.debug("Cancelling observation evaluation job", {
jobExecutionId: job.id,
projectId,
reason,
});
await prisma.jobExecution.update({
where: { id: job.id, projectId },
data: { status: JobExecutionStatus.CANCELLED, endTime: new Date() },
});
}
/**
* Resolves which evaluator definition a queued job must execute.
*
* Jobs scheduled since evaluator v2 carry their evaluator identity in the
* queue payload, so this is a direct lookup. For jobs queued by an older
* worker, the backfill preserves `job_configurations.id` as the rule id, so we
* resolve through that rule's assignment.
*/
async function resolveObservationEvalExecution(params: {
event: z.infer<typeof ObservationEvalExecutionEventSchema>;
job: JobExecution;
executionType: ObservationEvalExecutionType;
}) {
const { event, job, executionType } = params;
const { projectId, evaluatorId, evaluationRuleId } = event;
const evaluatorTypeFilter = {
in: EVALUATOR_TYPES_BY_EXECUTION_TYPE[executionType],
};
if (!evaluatorId) {
const migratedAssignment =
await prisma.evaluationRuleEvaluatorAssignment.findFirst({
where: {
projectId,
evaluationRuleId: job.jobConfigurationId,
evaluator: { projectId, type: evaluatorTypeFilter },
},
include: {
evaluationRule: true,
evaluator: { include: evaluatorInclude },
},
});
if (migratedAssignment) {
const { evaluationRule: rule, evaluator } = migratedAssignment;
if (
rule.status !== JobConfigState.ACTIVE &&
event.executionMode !== "MANUAL"
) {
return { type: "cancelled" as const, reason: "rule-inactive" };
}
return buildV2Execution({
rule,
assignment: migratedAssignment,
evaluator,
});
}
return { type: "cancelled" as const, reason: "assignment-unavailable" };
}
// Manual batch runs address the evaluator directly, with no rule to re-check:
// the user's selection already authorized the run.
if (!evaluationRuleId) {
const evaluator = await prisma.evaluator.findFirst({
where: { id: evaluatorId, projectId, type: evaluatorTypeFilter },
include: evaluatorInclude,
});
if (!evaluator) {
return { type: "cancelled" as const, reason: "evaluator-unavailable" };
}
// Ruleless batch runs have no assignment row. A mapping on the queue
// payload is the override; omitting it inherits the version mapping.
const assignment =
event.variableMapping !== undefined
? { variableMapping: event.variableMapping }
: null;
return buildV2Execution({ rule: null, assignment, evaluator });
}
// The assignment is what authorizes this execution — it is the (rule,
// evaluator) pairing that was scheduled, and it carries the pairing's
// mapping override. Resolving it directly answers "does this pairing still
// exist" and loads the rule and evaluator in the same round trip; a rule
// that was deleted, an evaluator that was detached or deleted, and an
// evaluator whose type no longer matches this queue all collapse to no row.
const assignment = await prisma.evaluationRuleEvaluatorAssignment.findFirst({
where: {
projectId,
evaluationRuleId,
evaluatorId,
evaluator: { projectId, type: evaluatorTypeFilter },
},
include: {
evaluationRule: true,
evaluator: { include: evaluatorInclude },
},
});
if (!assignment) {
return { type: "cancelled" as const, reason: "assignment-unavailable" };
}
const { evaluationRule: rule, evaluator } = assignment;
if (
rule.status !== JobConfigState.ACTIVE &&
event.executionMode !== "MANUAL"
) {
return { type: "cancelled" as const, reason: "rule-inactive" };
}
return buildV2Execution({ rule, assignment, evaluator });
}
const evaluatorInclude = {
project: { select: { orgId: true } },
// A rule runs its evaluator's current version, so the definition is resolved
// here rather than pinned at dispatch.
versions: { orderBy: { version: "desc" }, take: 1 },
} satisfies Prisma.EvaluatorInclude;
function normalizeEvalTemplate(
template: EvalTemplate & { promptMessages?: unknown; questions?: unknown },
evaluatorType: EvalTemplateType,
): EvalTemplateWithType {
switch (evaluatorType) {
case EvalTemplateType.LLM_AS_JUDGE:
if (
template.type !== evaluatorType ||
template.prompt === null ||
template.outputDefinition === null
) {
throw new UnrecoverableError(
"Evaluator template is incomplete for LLM_AS_JUDGE execution",
);
}
return {
...template,
type: evaluatorType,
prompt: template.prompt,
outputDefinition: template.outputDefinition,
sourceCode: null,
sourceCodeLanguage: null,
};
case EvalTemplateType.CODE:
if (
template.type !== evaluatorType ||
template.sourceCode === null ||
template.sourceCodeLanguage === null
) {
throw new UnrecoverableError(
"Evaluator template is incomplete for CODE execution",
);
}
return {
...template,
type: evaluatorType,
prompt: null,
outputDefinition: null,
sourceCode: template.sourceCode,
sourceCodeLanguage: template.sourceCodeLanguage,
};
case EvalTemplateType.DECISION_MODEL:
if (
template.type !== evaluatorType ||
template.questions === null ||
template.questions === undefined
) {
throw new UnrecoverableError(
"Evaluator template is incomplete for DECISION_MODEL execution",
);
}
return {
...template,
type: evaluatorType,
prompt: null,
outputDefinition: null,
sourceCode: null,
sourceCodeLanguage: null,
questions: template.questions,
};
case EvalTemplateType.FACET:
throw new UnrecoverableError("Facets cannot run as evaluators");
}
}
type ResolvedEvaluator = Prisma.EvaluatorGetPayload<{
include: typeof evaluatorInclude;
}>;
type ResolvedEvaluationRule = Prisma.EvaluationRuleGetPayload<object>;
/**
* Shapes the resolved rows into the `JobConfiguration`/`EvalTemplate` pair the
* shared executors still speak, and returns the v2 identity recorded on the
* execution. Returns a cancellation when the evaluator is unusable, which is
* the one condition that survives the resolution query: it is state on rows
* that do exist.
*/
function buildV2Execution(params: {
rule: ResolvedEvaluationRule | null;
assignment: {
id?: string;
variableMapping: Prisma.JsonValue;
} | null;
evaluator: ResolvedEvaluator;
}) {
const { rule, assignment, evaluator } = params;
if (evaluator.blockedAt) {
return { type: "cancelled" as const, reason: "evaluator-blocked" };
}
const version = evaluator.versions[0];
if (!version) {
return { type: "cancelled" as const, reason: "version-missing" };
}
const organizationId = evaluator.project.orgId;
const variableMapping =
assignment?.variableMapping ??
version.variableMapping ??
(evaluator.type === EvalTemplateType.CODE
? getCodeEvalVariableMapping()
: []);
const config = {
id: rule?.id ?? evaluator.id,
createdAt: rule?.createdAt ?? evaluator.createdAt,
updatedAt: rule?.updatedAt ?? evaluator.updatedAt,
projectId: rule?.projectId ?? evaluator.projectId,
jobType: "EVAL",
status: rule?.status ?? JobConfigState.ACTIVE,
blockedAt: null,
blockReason: null,
blockMessage: null,
evalTemplateId: version.id,
scoreName: evaluator.name,
filter: rule?.filter ?? [],
targetObject: rule?.targetObject ?? "event",
variableMapping,
sampling: rule?.sampling ?? new Prisma.Decimal(1),
delay: rule?.delay ?? 0,
timeScope: rule?.timeScope ?? ["NEW"],
} satisfies JobConfiguration;
const template = {
id: version.id,
createdAt: version.createdAt,
updatedAt: version.createdAt,
projectId: rule?.projectId ?? evaluator.projectId,
name: evaluator.name,
version: version.version,
prompt: version.prompt,
promptMessages: version.promptMessages,
type: evaluator.type,
partner: version.partner,
model: version.model,
provider: version.provider,
modelParams: version.modelParams,
vars: version.vars,
outputDefinition: version.outputDefinition,
sourceCode: version.sourceCode,
sourceCodeLanguage: version.sourceCodeLanguage,
questions: version.questions,
};
return {
type: "v2" as const,
config,
template: normalizeEvalTemplate(template, evaluator.type),
organizationId,
evaluationRuleId: rule?.id,
assignmentId: assignment?.id,
evaluatorId: evaluator.id,
evaluatorVersionId: version.id,
};
}