Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(evals): propagate observation scheduling failures (#18310)

Tobias Wochinger54bb9ea8c09b
diff --git a/worker/src/features/batchAction/processBatchedObservationEval.test.ts b/worker/src/features/batchAction/processBatchedObservationEval.test.ts
index 790d4a5929..b32e9d4cc4 100644
--- a/worker/src/features/batchAction/processBatchedObservationEval.test.ts
+++ b/worker/src/features/batchAction/processBatchedObservationEval.test.ts
@@ -1,10 +1,14 @@
import { beforeEach, describe, expect, it, vi, type Mock } from "vitest";
import {
+ BatchActionStatus,
EvalTargetObject,
EvalTemplateType,
JobConfigState,
} from "@langfuse/shared";
-import { type ObservationEvalRule } from "../evaluation/observationEval";
+import {
+ type ObservationEvalRule,
+ type ObservationEvalSchedulerDeps,
+} from "../evaluation/observationEval";
vi.mock("@langfuse/shared/src/db", () => ({
prisma: {
@@ -20,7 +24,10 @@ vi.mock("../evaluation/observationEval", () => ({
}));
import { prisma } from "@langfuse/shared/src/db";
-import { scheduleObservationEvals } from "../evaluation/observationEval";
+import {
+ createObservationEvalSchedulerDeps,
+ scheduleObservationEvals,
+} from "../evaluation/observationEval";
import { processBatchedObservationEval } from "./processBatchedObservationEval";
describe("processBatchedObservationEval", () => {
@@ -236,4 +243,75 @@ describe("processBatchedObservationEval", () => {
const secondCall = (scheduleObservationEvals as Mock).mock.calls[1][0];
expect(secondCall.observation.tool_call_count).toBe(0);
});
+
+ it("marks an observation as failed when its evaluator job cannot be enqueued", async () => {
+ const projectId = "project-1";
+ const enqueueError = new Error("Simulated queue enqueue failure");
+ const schedulerDeps: ObservationEvalSchedulerDeps = {
+ uploadObservationToS3: vi.fn().mockResolvedValue("observation.json"),
+ upsertJobExecution: vi.fn().mockResolvedValue({ id: "execution-1" }),
+ enqueueEvalJob: vi.fn().mockRejectedValue(enqueueError),
+ };
+ const actualModule =
+ await import("../evaluation/observationEval/scheduleObservationEvals");
+
+ vi.mocked(createObservationEvalSchedulerDeps).mockReturnValueOnce(
+ schedulerDeps,
+ );
+ vi.mocked(scheduleObservationEvals).mockImplementationOnce(
+ actualModule.scheduleObservationEvals,
+ );
+
+ await processBatchedObservationEval({
+ projectId,
+ batchActionId: "batch-action-failed-enqueue",
+ evaluators: [
+ {
+ id: "config-1",
+ projectId,
+ filter: [],
+ sampling: { toNumber: () => 1 } as ObservationEvalRule["sampling"],
+ evalTemplateId: "template-1",
+ evalTemplate: { type: EvalTemplateType.LLM_AS_JUDGE },
+ scoreName: "quality",
+ targetObject: EvalTargetObject.EVENT,
+ variableMapping: [],
+ status: JobConfigState.ACTIVE,
+ blockedAt: null,
+ },
+ ],
+ observationStream: (async function* () {
+ yield {
+ span_id: "obs-failed-enqueue",
+ trace_id: "trace-1",
+ project_id: projectId,
+ parent_span_id: null,
+ type: "GENERATION",
+ name: "test",
+ usage_details: {},
+ cost_details: {},
+ provided_usage_details: {},
+ provided_cost_details: {},
+ tags: [],
+ input: "input",
+ output: "output",
+ metadata: {},
+ };
+ })(),
+ });
+
+ expect(prisma.batchAction.update).toHaveBeenLastCalledWith({
+ where: {
+ id: "batch-action-failed-enqueue",
+ projectId,
+ },
+ data: expect.objectContaining({
+ status: BatchActionStatus.Failed,
+ totalCount: 1,
+ processedCount: 0,
+ failedCount: 1,
+ log: expect.stringContaining(enqueueError.message),
+ }),
+ });
+ });
});
diff --git a/worker/src/features/evaluation/observationEval/__tests__/scheduleObservationEvals.test.ts b/worker/src/features/evaluation/observationEval/__tests__/scheduleObservationEvals.test.ts
index 9c4f652867..03fc3734d8 100644
--- a/worker/src/features/evaluation/observationEval/__tests__/scheduleObservationEvals.test.ts
+++ b/worker/src/features/evaluation/observationEval/__tests__/scheduleObservationEvals.test.ts
@@ -445,6 +445,33 @@ describe("scheduleObservationEvals", () => {
});
});
+ it("should propagate assignment failures after all assignments settle", async () => {
+ const schedulerDeps = createMockSchedulerDeps();
+ schedulerDeps.enqueueEvalJob = vi
+ .fn<ObservationEvalSchedulerDeps["enqueueEvalJob"]>()
+ .mockRejectedValueOnce(new Error("First enqueue failed"))
+ .mockRejectedValueOnce(new Error("Second enqueue failed"));
+
+ const schedulingPromise = scheduleObservationEvals({
+ observation: createMockObservation(),
+ configs: [
+ createMockConfig({ id: "config-1" }),
+ createMockConfig({ id: "config-2" }),
+ ],
+ schedulerDeps,
+ });
+
+ await expect(schedulingPromise).rejects.toMatchObject({
+ name: "AggregateError",
+ message: expect.stringContaining("First enqueue failed"),
+ errors: [
+ expect.objectContaining({ message: "First enqueue failed" }),
+ expect.objectContaining({ message: "Second enqueue failed" }),
+ ],
+ });
+ expect(schedulerDeps.enqueueEvalJob).toHaveBeenCalledTimes(2);
+ });
+
it("should pass code template type to the scheduler deps", async () => {
const schedulerDeps = createMockSchedulerDeps();
const observation = createMockObservation();
diff --git a/worker/src/features/evaluation/observationEval/scheduleObservationEvals.ts b/worker/src/features/evaluation/observationEval/scheduleObservationEvals.ts
index 032381858d..9306682f77 100644
--- a/worker/src/features/evaluation/observationEval/scheduleObservationEvals.ts
+++ b/worker/src/features/evaluation/observationEval/scheduleObservationEvals.ts
@@ -177,30 +177,53 @@ export async function scheduleObservationEvals(
data: observation,
});
- // Process each assignment of every matching rule/config.
- await Promise.all(
- matchingConfigs.flatMap(({ config, assignments }) =>
- assignments.map((assignment) =>
- processMatchingConfig({
- observation,
- matchingConfig: config,
- assignment,
- observationS3Path,
- schedulerDeps,
- executionMode,
- executionScopeId,
- }).catch((error) => {
- logger.error("Failed to process observation eval assignment", {
- configId: config.id,
- assignmentId: assignment.id,
- observationId: observation.span_id,
- projectId: observation.project_id,
- error,
- });
- }),
- ),
+ const assignmentAttempts = matchingConfigs.flatMap(
+ ({ config, assignments }) =>
+ assignments.map((assignment) => ({ config, assignment })),
+ );
+
+ const results = await Promise.allSettled(
+ assignmentAttempts.map(({ config, assignment }) =>
+ processMatchingConfig({
+ observation,
+ matchingConfig: config,
+ assignment,
+ observationS3Path,
+ schedulerDeps,
+ executionMode,
+ executionScopeId,
+ }),
),
);
+
+ const failures = results.flatMap((result, index) => {
+ if (result.status === "fulfilled") return [];
+
+ const { config, assignment } = assignmentAttempts[index];
+ logger.error("Failed to process observation eval assignment", {
+ configId: config.id,
+ assignmentId: assignment.id,
+ observationId: observation.span_id,
+ projectId: observation.project_id,
+ error: result.reason,
+ });
+
+ return [result.reason];
+ });
+
+ if (failures.length > 0) {
+ const firstFailure = failures[0];
+ const firstFailureMessage =
+ firstFailure instanceof Error
+ ? firstFailure.message
+ : "Unknown scheduling error";
+ const remainingFailureCount = failures.length - 1;
+
+ throw new AggregateError(
+ failures,
+ `Failed to schedule ${failures.length} of ${assignmentAttempts.length} observation eval assignment(s): ${firstFailureMessage}${remainingFailureCount > 0 ? ` (${remainingFailureCount} more failure(s))` : ""}`,
+ );
+ }
}
interface ProcessConfigParams {