🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(evals): propagate observation scheduling failures (#18310)
Tobias Wochinger54bb9ea8c09b
diff --git a/worker/src/features/batchAction/processBatchedObservationEval.test.ts b/worker/src/features/batchAction/processBatchedObservationEval.test.tsindex 790d4a5929..b32e9d4cc4 100644--- a/worker/src/features/batchAction/processBatchedObservationEval.test.ts+++ b/worker/src/features/batchAction/processBatchedObservationEval.test.ts@@ -1,10 +1,14 @@import { beforeEach, describe, expect, it, vi, type Mock } from "vitest";import {+ BatchActionStatus,EvalTargetObject,EvalTemplateType,JobConfigState,} from "@langfuse/shared";-import { type ObservationEvalRule } from "../evaluation/observationEval";+import {+ type ObservationEvalRule,+ type ObservationEvalSchedulerDeps,+} from "../evaluation/observationEval";vi.mock("@langfuse/shared/src/db", () => ({prisma: {@@ -20,7 +24,10 @@ vi.mock("../evaluation/observationEval", () => ({}));import { prisma } from "@langfuse/shared/src/db";-import { scheduleObservationEvals } from "../evaluation/observationEval";+import {+ createObservationEvalSchedulerDeps,+ scheduleObservationEvals,+} from "../evaluation/observationEval";import { processBatchedObservationEval } from "./processBatchedObservationEval";describe("processBatchedObservationEval", () => {@@ -236,4 +243,75 @@ describe("processBatchedObservationEval", () => {const secondCall = (scheduleObservationEvals as Mock).mock.calls[1][0];expect(secondCall.observation.tool_call_count).toBe(0);});++ it("marks an observation as failed when its evaluator job cannot be enqueued", async () => {+ const projectId = "project-1";+ const enqueueError = new Error("Simulated queue enqueue failure");+ const schedulerDeps: ObservationEvalSchedulerDeps = {+ uploadObservationToS3: vi.fn().mockResolvedValue("observation.json"),+ upsertJobExecution: vi.fn().mockResolvedValue({ id: "execution-1" }),+ enqueueEvalJob: vi.fn().mockRejectedValue(enqueueError),+ };+ const actualModule =+ await import("../evaluation/observationEval/scheduleObservationEvals");++ vi.mocked(createObservationEvalSchedulerDeps).mockReturnValueOnce(+ schedulerDeps,+ );+ vi.mocked(scheduleObservationEvals).mockImplementationOnce(+ actualModule.scheduleObservationEvals,+ );++ await processBatchedObservationEval({+ projectId,+ batchActionId: "batch-action-failed-enqueue",+ evaluators: [+ {+ id: "config-1",+ projectId,+ filter: [],+ sampling: { toNumber: () => 1 } as ObservationEvalRule["sampling"],+ evalTemplateId: "template-1",+ evalTemplate: { type: EvalTemplateType.LLM_AS_JUDGE },+ scoreName: "quality",+ targetObject: EvalTargetObject.EVENT,+ variableMapping: [],+ status: JobConfigState.ACTIVE,+ blockedAt: null,+ },+ ],+ observationStream: (async function* () {+ yield {+ span_id: "obs-failed-enqueue",+ trace_id: "trace-1",+ project_id: projectId,+ parent_span_id: null,+ type: "GENERATION",+ name: "test",+ usage_details: {},+ cost_details: {},+ provided_usage_details: {},+ provided_cost_details: {},+ tags: [],+ input: "input",+ output: "output",+ metadata: {},+ };+ })(),+ });++ expect(prisma.batchAction.update).toHaveBeenLastCalledWith({+ where: {+ id: "batch-action-failed-enqueue",+ projectId,+ },+ data: expect.objectContaining({+ status: BatchActionStatus.Failed,+ totalCount: 1,+ processedCount: 0,+ failedCount: 1,+ log: expect.stringContaining(enqueueError.message),+ }),+ });+ });});diff --git a/worker/src/features/evaluation/observationEval/__tests__/scheduleObservationEvals.test.ts b/worker/src/features/evaluation/observationEval/__tests__/scheduleObservationEvals.test.tsindex 9c4f652867..03fc3734d8 100644--- a/worker/src/features/evaluation/observationEval/__tests__/scheduleObservationEvals.test.ts+++ b/worker/src/features/evaluation/observationEval/__tests__/scheduleObservationEvals.test.ts@@ -445,6 +445,33 @@ describe("scheduleObservationEvals", () => {});});+ it("should propagate assignment failures after all assignments settle", async () => {+ const schedulerDeps = createMockSchedulerDeps();+ schedulerDeps.enqueueEvalJob = vi+ .fn<ObservationEvalSchedulerDeps["enqueueEvalJob"]>()+ .mockRejectedValueOnce(new Error("First enqueue failed"))+ .mockRejectedValueOnce(new Error("Second enqueue failed"));++ const schedulingPromise = scheduleObservationEvals({+ observation: createMockObservation(),+ configs: [+ createMockConfig({ id: "config-1" }),+ createMockConfig({ id: "config-2" }),+ ],+ schedulerDeps,+ });++ await expect(schedulingPromise).rejects.toMatchObject({+ name: "AggregateError",+ message: expect.stringContaining("First enqueue failed"),+ errors: [+ expect.objectContaining({ message: "First enqueue failed" }),+ expect.objectContaining({ message: "Second enqueue failed" }),+ ],+ });+ expect(schedulerDeps.enqueueEvalJob).toHaveBeenCalledTimes(2);+ });+it("should pass code template type to the scheduler deps", async () => {const schedulerDeps = createMockSchedulerDeps();const observation = createMockObservation();diff --git a/worker/src/features/evaluation/observationEval/scheduleObservationEvals.ts b/worker/src/features/evaluation/observationEval/scheduleObservationEvals.tsindex 032381858d..9306682f77 100644--- a/worker/src/features/evaluation/observationEval/scheduleObservationEvals.ts+++ b/worker/src/features/evaluation/observationEval/scheduleObservationEvals.ts@@ -177,30 +177,53 @@ export async function scheduleObservationEvals(data: observation,});- // Process each assignment of every matching rule/config.- await Promise.all(- matchingConfigs.flatMap(({ config, assignments }) =>- assignments.map((assignment) =>- processMatchingConfig({- observation,- matchingConfig: config,- assignment,- observationS3Path,- schedulerDeps,- executionMode,- executionScopeId,- }).catch((error) => {- logger.error("Failed to process observation eval assignment", {- configId: config.id,- assignmentId: assignment.id,- observationId: observation.span_id,- projectId: observation.project_id,- error,- });- }),- ),+ const assignmentAttempts = matchingConfigs.flatMap(+ ({ config, assignments }) =>+ assignments.map((assignment) => ({ config, assignment })),+ );++ const results = await Promise.allSettled(+ assignmentAttempts.map(({ config, assignment }) =>+ processMatchingConfig({+ observation,+ matchingConfig: config,+ assignment,+ observationS3Path,+ schedulerDeps,+ executionMode,+ executionScopeId,+ }),),);++ const failures = results.flatMap((result, index) => {+ if (result.status === "fulfilled") return [];++ const { config, assignment } = assignmentAttempts[index];+ logger.error("Failed to process observation eval assignment", {+ configId: config.id,+ assignmentId: assignment.id,+ observationId: observation.span_id,+ projectId: observation.project_id,+ error: result.reason,+ });++ return [result.reason];+ });++ if (failures.length > 0) {+ const firstFailure = failures[0];+ const firstFailureMessage =+ firstFailure instanceof Error+ ? firstFailure.message+ : "Unknown scheduling error";+ const remainingFailureCount = failures.length - 1;++ throw new AggregateError(+ failures,+ `Failed to schedule ${failures.length} of ${assignmentAttempts.length} observation eval assignment(s): ${firstFailureMessage}${remainingFailureCount > 0 ? ` (${remainingFailureCount} more failure(s))` : ""}`,+ );+ }}interface ProcessConfigParams {