Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

44.4 KB
Raw Blame
import { beforeEach, describe, expect, it, vi, type Mock } from "vitest";
import { JobExecutionStatus } from "@prisma/client";
import z from "zod";

const observabilityMocks = vi.hoisted(() => ({
  spans: [] as Array<{
    name: string;
    attributes: Record<string, unknown>;
  }>,
  blockEvaluator: vi.fn().mockResolvedValue({ blockedEvaluatorIds: [] }),
  recordIncrement: vi.fn(),
}));

vi.mock("@langfuse/shared/src/server", async (importOriginal) => {
  const actual =
    await importOriginal<typeof import("@langfuse/shared/src/server")>();

  return {
    ...actual,
    blockEvaluator: observabilityMocks.blockEvaluator,
    recordIncrement: observabilityMocks.recordIncrement,
    instrumentAsync: vi.fn(
      async (
        { name }: { name: string },
        callback: (span: {
          setAttribute: (key: string, value: unknown) => unknown;
          setAttributes: (attributes: Record<string, unknown>) => unknown;
        }) => Promise<unknown>,
      ) => {
        const captured = { name, attributes: {} as Record<string, unknown> };
        const span = {
          setAttribute(key: string, value: unknown) {
            captured.attributes[key] = value;
            return span;
          },
          setAttributes(attributes: Record<string, unknown>) {
            Object.assign(captured.attributes, attributes);
            return span;
          },
        };
        observabilityMocks.spans.push(captured);
        return callback(span);
      },
    ),
  };
});

import { executeLLMAsJudgeEvaluation } from "./evalService";
import { createMockEvalExecutionDeps } from "./evalExecutionDeps";
import { UnrecoverableError } from "../../errors/UnrecoverableError";
import {
  LLMValidationError,
  recordIncrement,
  type ExtractedVariable,
} from "@langfuse/shared/src/server";
import {
  EvalTargetObject,
  type PersistedEvalOutputDefinition,
  ScoreDataTypeEnum,
} from "@langfuse/shared";

/**
 * Unit tests for executeLLMAsJudgeEvaluation with mocked dependencies.
 *
 * These tests verify the orchestration logic without hitting real databases,
 * LLMs, or external services. Each test creates mock deps with specific
 * overrides to test different scenarios.
 *
 * Note: Job fetching, config fetching, template fetching, and cancelled job
 * handling are now the responsibility of callers (evaluate() and
 * processObservationEval()), not this function. Tests for those scenarios
 * have been removed.
 */
describe("executeLLMAsJudgeEvaluation", () => {
  const projectId = "test-project-123";
  const jobExecutionId = "test-job-execution-456";
  const getScorePersistenceMetricCalls = () =>
    vi
      .mocked(recordIncrement)
      .mock.calls.filter(
        ([metric]) => metric === "langfuse.evaluation.score_persistence",
      );

  beforeEach(() => {
    observabilityMocks.spans.length = 0;
    observabilityMocks.blockEvaluator.mockClear();
    observabilityMocks.recordIncrement.mockClear();
  });

  // ============================================================================
  // Test Fixtures
  // ============================================================================

  const mockJobExecution = {
    id: jobExecutionId,
    createdAt: new Date(),
    updatedAt: new Date(),
    projectId,
    status: JobExecutionStatus.PENDING,
    startTime: new Date(),
    endTime: null,
    jobConfigurationId: "config-789",
    jobInputTraceId: "trace-abc",
    jobInputTraceTimestamp: new Date(),
    jobInputObservationId: null,
    jobInputDatasetItemId: null,
    jobTemplateId: "template-xyz",
    error: null,
    jobOutputScoreId: null,
    executionTraceId: null,
  };

  const mockJobConfiguration = {
    id: "config-789",
    createdAt: new Date(),
    updatedAt: new Date(),
    projectId,
    jobType: "EVAL" as const,
    evalTemplateId: "template-xyz",
    scoreName: "accuracy",
    targetObject: EvalTargetObject.TRACE,
    filter: [],
    variableMapping: [],
    sampling: "1.0",
    delay: 0,
    status: "ACTIVE" as const,
    timeScope: ["NEW" as const],
  };

  const mockEvalTemplate: {
    id: string;
    createdAt: Date;
    updatedAt: Date;
    name: string;
    version: number;
    prompt: string;
    model: string;
    provider: string;
    modelParams: Record<string, unknown>;
    outputDefinition: PersistedEvalOutputDefinition;
    vars: string[];
    projectId: string;
  } = {
    id: "template-xyz",
    createdAt: new Date(),
    updatedAt: new Date(),
    name: "Accuracy Evaluator",
    version: 1,
    prompt: "Evaluate accuracy of {{output}}",
    model: "gpt-4",
    provider: "openai",
    modelParams: {},
    outputDefinition: {
      score: "A number between 0 and 1",
      reasoning: "Explain your reasoning",
    },
    vars: ["output"],
    projectId,
  };

  const extractedVariables: ExtractedVariable[] = [
    {
      var: "output",
      value: "The model response text",
      environment: "production",
    },
  ];

  // ============================================================================
  // Mock Helpers
  // ============================================================================

  /** Default valid model configuration for OpenAI GPT-4 */
  const defaultModelConfig = {
    provider: "openai",
    model: "gpt-4",
    apiKey: { adapter: "openai", secretKey: "test-key" },
    adapter: "openai",
    modelParams: {},
  };

  /** Creates a mock for fetchModelConfig that returns a valid config */
  const mockValidFetchModelConfig = () =>
    vi.fn().mockResolvedValue({
      valid: true,
      config: defaultModelConfig,
    });

  /** Creates a mock for callLLM with a successful response */
  const mockSuccessfulLLMCall = (
    score: number | boolean | string | string[],
    reasoning: string,
  ) => vi.fn().mockResolvedValue({ score, reasoning });

  /** Creates standard deps with all mocks for a successful execution flow */
  const createSuccessfulDeps = (
    overrides: {
      callLLM?: Mock;
      uploadScore?: Mock;
      enqueueScoreIngestion?: Mock;
      updateJobExecution?: Mock;
    } = {},
  ) =>
    createMockEvalExecutionDeps({
      fetchModelConfig: mockValidFetchModelConfig(),
      callLLM: overrides.callLLM ?? mockSuccessfulLLMCall(0.8, "Good"),
      uploadScore: overrides.uploadScore ?? vi.fn(),
      enqueueScoreIngestion: overrides.enqueueScoreIngestion ?? vi.fn(),
      updateJobExecution: overrides.updateJobExecution ?? vi.fn(),
    });

  /** Standard execution params for most tests */
  const createExecutionParams = (
    overrides: {
      job?: typeof mockJobExecution;
      template?: typeof mockEvalTemplate;
      variables?: ExtractedVariable[];
      environment?: string;
      deps?: ReturnType<typeof createMockEvalExecutionDeps>;
    } = {},
  ) => ({
    projectId,
    jobExecutionId,
    job: overrides.job ?? mockJobExecution,
    config: mockJobConfiguration,
    template: overrides.template ?? mockEvalTemplate,
    extractedVariables: overrides.variables ?? extractedVariables,
    environment: overrides.environment ?? "production",
    deps: overrides.deps ?? createSuccessfulDeps(),
  });

  describe("successful execution", () => {
    it("should complete evaluation successfully", async () => {
      const callLLM = mockSuccessfulLLMCall(0.85, "High accuracy observed");
      const uploadScore = vi.fn();
      const enqueueScoreIngestion = vi.fn();
      const updateJobExecution = vi.fn();

      const deps = createSuccessfulDeps({
        callLLM,
        uploadScore,
        enqueueScoreIngestion,
        updateJobExecution,
      });

      await executeLLMAsJudgeEvaluation(createExecutionParams({ deps }));

      expect(callLLM).toHaveBeenCalledTimes(1);
      expect(uploadScore).toHaveBeenCalledTimes(1);
      expect(uploadScore).toHaveBeenCalledWith(
        expect.objectContaining({
          projectId,
          event: expect.objectContaining({
            body: expect.objectContaining({
              name: "accuracy",
              value: 0.85,
              comment: "High accuracy observed",
            }),
          }),
        }),
      );
      expect(enqueueScoreIngestion).toHaveBeenCalledTimes(1);
      expect(getScorePersistenceMetricCalls()).toEqual([
        [
          "langfuse.evaluation.score_persistence",
          1,
          {
            evaluator_type: "llm_as_judge",
            outcome: "success",
            score_count_bucket: "1",
          },
        ],
      ]);
      expect(updateJobExecution).toHaveBeenCalledWith(
        expect.objectContaining({
          id: jobExecutionId,
          projectId,
          data: expect.objectContaining({
            status: JobExecutionStatus.COMPLETED,
            endTime: expect.any(Date),
            jobOutputScoreId: expect.any(String),
            executionTraceId: expect.any(String),
          }),
        }),
      );

      const executeSpan = observabilityMocks.spans.find(
        ({ name }) => name === "eval.execute-llm-as-judge",
      );
      const callSpan = observabilityMocks.spans.find(
        ({ name }) => name === "eval.call-llm",
      );

      expect(executeSpan?.attributes).toMatchObject({
        "langfuse.project.id": projectId,
        "eval.job_execution.id": jobExecutionId,
        "eval.execution.trace_id": expect.any(String),
        "eval.execution.stage": "completed",
        "eval.execution.outcome": "success",
        "eval.model.provider": "openai",
        "eval.model.name": "gpt-4",
        "eval.model.adapter": "openai",
      });
      expect(callSpan?.attributes).toMatchObject({
        "langfuse.project.id": projectId,
        "eval.job_execution.id": jobExecutionId,
        "eval.execution.trace_id":
          executeSpan?.attributes["eval.execution.trace_id"],
        "eval.llm.outcome": "success",
      });
    });

    it("should include observation ID in score when present", async () => {
      const uploadScore = vi.fn();
      const deps = createSuccessfulDeps({
        callLLM: mockSuccessfulLLMCall(0.9, "Excellent"),
        uploadScore,
      });

      await executeLLMAsJudgeEvaluation(
        createExecutionParams({
          job: { ...mockJobExecution, jobInputObservationId: "obs-123" },
          deps,
        }),
      );

      expect(uploadScore).toHaveBeenCalledWith(
        expect.objectContaining({
          event: expect.objectContaining({
            body: expect.objectContaining({
              observationId: "obs-123",
            }),
          }),
        }),
      );
    });

    it("should use provided environment", async () => {
      const uploadScore = vi.fn();
      const deps = createSuccessfulDeps({ uploadScore });

      await executeLLMAsJudgeEvaluation(
        createExecutionParams({
          variables: [{ var: "output", value: "test" }],
          environment: "staging",
          deps,
        }),
      );

      expect(uploadScore).toHaveBeenCalledWith(
        expect.objectContaining({
          event: expect.objectContaining({
            body: expect.objectContaining({
              environment: "staging",
            }),
          }),
        }),
      );
    });

    it("should use 'default' environment when explicitly passed", async () => {
      const uploadScore = vi.fn();
      const deps = createSuccessfulDeps({
        callLLM: mockSuccessfulLLMCall(0.5, "Average"),
        uploadScore,
      });

      await executeLLMAsJudgeEvaluation(
        createExecutionParams({
          variables: [{ var: "output", value: "test" }],
          environment: "default",
          deps,
        }),
      );

      expect(uploadScore).toHaveBeenCalledWith(
        expect.objectContaining({
          event: expect.objectContaining({
            body: expect.objectContaining({
              environment: "default",
            }),
          }),
        }),
      );
    });
  });

  describe("configuration errors", () => {
    it("should throw UnrecoverableError if model config invalid", async () => {
      const deps = createMockEvalExecutionDeps({
        fetchModelConfig: vi.fn().mockResolvedValue({
          valid: false,
          error: "No API key configured",
        }),
      });

      await expect(
        executeLLMAsJudgeEvaluation(createExecutionParams({ deps })),
      ).rejects.toThrow(UnrecoverableError);
    });

    it("should throw UnrecoverableError if output definition is invalid", async () => {
      const templateWithBadSchema = {
        ...mockEvalTemplate,
        outputDefinition: { invalidKey: "value" },
      };

      await expect(
        executeLLMAsJudgeEvaluation(
          createExecutionParams({
            template: templateWithBadSchema,
            deps: createMockEvalExecutionDeps({}),
          }),
        ),
      ).rejects.toThrow(UnrecoverableError);
    });
  });

  describe("LLM response errors", () => {
    it("should throw UnrecoverableError for invalid LLM response", async () => {
      const deps = createMockEvalExecutionDeps({
        fetchModelConfig: mockValidFetchModelConfig(),
        callLLM: vi.fn().mockResolvedValue({
          score: "high", // Invalid - should be number
          reasoning: "Good response",
        }),
      });

      await expect(
        executeLLMAsJudgeEvaluation(createExecutionParams({ deps })),
      ).rejects.toThrow(UnrecoverableError);
    });

    it("should throw UnrecoverableError for missing LLM response fields", async () => {
      const deps = createMockEvalExecutionDeps({
        fetchModelConfig: mockValidFetchModelConfig(),
        callLLM: vi.fn().mockResolvedValue({
          score: 0.8,
          // Missing reasoning field
        }),
      });

      await expect(
        executeLLMAsJudgeEvaluation(createExecutionParams({ deps })),
      ).rejects.toThrow(UnrecoverableError);
    });
  });

  describe("score persistence errors", () => {
    it("should throw error if score upload fails", async () => {
      const deps = createMockEvalExecutionDeps({
        fetchModelConfig: mockValidFetchModelConfig(),
        callLLM: mockSuccessfulLLMCall(0.8, "Good"),
        uploadScore: vi.fn().mockRejectedValue(new Error("S3 upload failed")),
      });

      await expect(
        executeLLMAsJudgeEvaluation(createExecutionParams({ deps })),
      ).rejects.toThrow("Failed to write score");
      expect(getScorePersistenceMetricCalls()).toEqual([
        [
          "langfuse.evaluation.score_persistence",
          1,
          {
            evaluator_type: "llm_as_judge",
            outcome: "failure",
            score_count_bucket: "1",
          },
        ],
      ]);
    });

    it("should throw error if score ingestion queue fails", async () => {
      const deps = createMockEvalExecutionDeps({
        fetchModelConfig: mockValidFetchModelConfig(),
        callLLM: mockSuccessfulLLMCall(0.8, "Good"),
        uploadScore: vi.fn(),
        enqueueScoreIngestion: vi
          .fn()
          .mockRejectedValue(new Error("Queue unavailable")),
      });

      await expect(
        executeLLMAsJudgeEvaluation(createExecutionParams({ deps })),
      ).rejects.toThrow("Failed to write score");
    });
  });

  describe("prompt compilation", () => {
    it("should handle prompt compilation errors gracefully", async () => {
      const templateWithBadPrompt = {
        ...mockEvalTemplate,
        prompt: "Evaluate {{unclosed_bracket",
      };

      const uploadScore = vi.fn();
      const deps = createSuccessfulDeps({
        callLLM: mockSuccessfulLLMCall(0.5, "Could not parse properly"),
        uploadScore,
      });

      // Should not throw - falls back to raw template
      await executeLLMAsJudgeEvaluation(
        createExecutionParams({ template: templateWithBadPrompt, deps }),
      );

      expect(uploadScore).toHaveBeenCalled();
    });

    it("should compile multiple variables into prompt", async () => {
      const multiVarTemplate = {
        ...mockEvalTemplate,
        prompt: "Compare input: {{input}} with output: {{output}}",
        vars: ["input", "output"],
      };

      const multiVariables: ExtractedVariable[] = [
        { var: "input", value: "What is 2+2?" },
        { var: "output", value: "The answer is 4" },
      ];

      const callLLM = mockSuccessfulLLMCall(1.0, "Perfect match");
      const deps = createSuccessfulDeps({ callLLM });

      await executeLLMAsJudgeEvaluation(
        createExecutionParams({
          template: multiVarTemplate,
          variables: multiVariables,
          deps,
        }),
      );

      expect(callLLM).toHaveBeenCalledWith(
        expect.objectContaining({
          messages: expect.arrayContaining([
            expect.objectContaining({
              content:
                "Compare input: What is 2+2? with output: The answer is 4",
            }),
          ]),
        }),
      );
    });
  });

  describe("LLM call parameters", () => {
    it("should pass correct traceSinkParams to LLM for observability", async () => {
      const callLLM = mockSuccessfulLLMCall(0.8, "Good response");
      const deps = createSuccessfulDeps({ callLLM });

      await executeLLMAsJudgeEvaluation(createExecutionParams({ deps }));

      expect(callLLM).toHaveBeenCalledWith(
        expect.objectContaining({
          traceSinkParams: expect.objectContaining({
            targetProjectId: projectId,
            traceId: expect.any(String),
            traceName: `Execute evaluator: ${mockEvalTemplate.name}`,
            environment: "langfuse-llm-as-a-judge",
            evaluationContext: {
              evaluationRuleId: mockJobExecution.jobConfigurationId,
              evaluatorExecutionIsTest: false,
            },
            metadata: expect.objectContaining({
              job_execution_id: jobExecutionId,
              target_trace_id: mockJobExecution.jobInputTraceId,
            }),
          }),
        }),
      );
    });

    it("should pass structured output schema to LLM", async () => {
      const callLLM = mockSuccessfulLLMCall(0.8, "Good response");
      const deps = createSuccessfulDeps({ callLLM });

      await executeLLMAsJudgeEvaluation(
        createExecutionParams({
          deps,
          template: {
            ...mockEvalTemplate,
            outputDefinition: {
              version: 2,
              dataType: ScoreDataTypeEnum.NUMERIC,
              reasoning: { description: "Explain the score" },
              score: {
                description: "Return a score between 0 and 1",
                minValue: 0,
                maxValue: 1,
              },
            },
          },
        }),
      );

      const structuredOutputSchema = callLLM.mock.calls[0][0]
        .structuredOutputSchema as z.ZodTypeAny;

      expect(
        structuredOutputSchema.safeParse({
          score: 0.8,
          reasoning: "Good response",
        }).success,
      ).toBe(true);
      expect(
        structuredOutputSchema.safeParse({
          score: 1.1,
          reasoning: "Outside the configured range",
        }).success,
      ).toBe(false);
      expect(z.toJSONSchema(structuredOutputSchema)).toMatchObject({
        properties: {
          score: {
            minimum: 0,
            maximum: 1,
          },
        },
      });
    });

    it("should pass categorical structured output schema to LLM", async () => {
      const callLLM = mockSuccessfulLLMCall("correct", "Fully supported");
      const deps = createSuccessfulDeps({ callLLM });

      await executeLLMAsJudgeEvaluation(
        createExecutionParams({
          deps,
          template: {
            ...mockEvalTemplate,
            outputDefinition: {
              version: 2,
              dataType: ScoreDataTypeEnum.CATEGORICAL,
              reasoning: {
                description: "Explain the selected category",
              },
              score: {
                description: "Choose the best matching category",
                categories: ["correct", "partial"],
              },
            },
          },
        }),
      );

      const structuredOutputSchema = callLLM.mock.calls[0][0]
        .structuredOutputSchema as z.ZodTypeAny;

      expect(
        structuredOutputSchema.safeParse({
          score: "correct",
          reasoning: "Fully supported",
        }).success,
      ).toBe(true);
      expect(
        structuredOutputSchema.safeParse({
          score: "incorrect",
          reasoning: "Not part of the allowed categories",
        }).success,
      ).toBe(false);
    });

    it("should pass categorical multi-match structured output schema to LLM", async () => {
      const callLLM = mockSuccessfulLLMCall(
        ["correct", "partial"],
        "Both categories apply.",
      );
      const deps = createSuccessfulDeps({ callLLM });

      await executeLLMAsJudgeEvaluation(
        createExecutionParams({
          deps,
          template: {
            ...mockEvalTemplate,
            outputDefinition: {
              version: 2,
              dataType: ScoreDataTypeEnum.CATEGORICAL,
              reasoning: {
                description: "Explain the selected categories",
              },
              score: {
                description: "Choose all matching categories",
                categories: ["correct", "partial"],
                shouldAllowMultipleMatches: true,
              },
            },
          },
        }),
      );

      const structuredOutputSchema = callLLM.mock.calls[0][0]
        .structuredOutputSchema as z.ZodTypeAny;

      expect(
        structuredOutputSchema.safeParse({
          score: ["correct", "partial"],
          reasoning: "Both categories apply.",
        }).success,
      ).toBe(true);
      expect(
        structuredOutputSchema.safeParse({
          score: "correct",
          reasoning: "Single values should be rejected for multi-match.",
        }).success,
      ).toBe(false);
      expect(
        structuredOutputSchema.safeParse({
          score: ["correct", "correct"],
          reasoning: "Duplicates should be rejected.",
        }).success,
      ).toBe(false);
    });

    it("should pass boolean structured output schema to LLM", async () => {
      const callLLM = mockSuccessfulLLMCall(
        false,
        "The answer does not satisfy the criteria.",
      );
      const deps = createSuccessfulDeps({ callLLM });

      await executeLLMAsJudgeEvaluation(
        createExecutionParams({
          deps,
          template: {
            ...mockEvalTemplate,
            outputDefinition: {
              version: 2,
              dataType: ScoreDataTypeEnum.BOOLEAN,
              reasoning: {
                description: "Explain the verdict",
              },
              score: {
                description:
                  "Return true if the answer satisfies the criteria, otherwise false",
              },
            },
          },
        }),
      );

      const structuredOutputSchema = callLLM.mock.calls[0][0]
        .structuredOutputSchema as z.ZodTypeAny;

      expect(
        structuredOutputSchema.safeParse({
          score: true,
          reasoning: "The answer satisfies the criteria.",
        }).success,
      ).toBe(true);
      expect(
        structuredOutputSchema.safeParse({
          score: "true",
          reasoning: "String values should be rejected.",
        }).success,
      ).toBe(false);
    });

    it("should pass model config to LLM", async () => {
      const customModelConfig = {
        provider: "anthropic",
        model: "claude-3-opus",
        apiKey: { adapter: "anthropic", secretKey: "anthropic-key" },
        modelParams: { temperature: 0.5 },
      };

      const callLLM = mockSuccessfulLLMCall(0.9, "Excellent");
      const deps = createMockEvalExecutionDeps({
        fetchModelConfig: vi.fn().mockResolvedValue({
          valid: true,
          config: customModelConfig,
        }),
        callLLM,
        uploadScore: vi.fn(),
        enqueueScoreIngestion: vi.fn(),
        updateJobExecution: vi.fn(),
      });

      await executeLLMAsJudgeEvaluation(createExecutionParams({ deps }));

      expect(callLLM).toHaveBeenCalledWith(
        expect.objectContaining({
          modelConfig: customModelConfig,
        }),
      );
    });
  });

  describe("execution metadata", () => {
    it("should include dataset item ID in metadata when present", async () => {
      const callLLM = mockSuccessfulLLMCall(
        0.75,
        "Dataset evaluation complete",
      );
      const uploadScore = vi.fn();
      const deps = createSuccessfulDeps({ callLLM, uploadScore });

      await executeLLMAsJudgeEvaluation(
        createExecutionParams({
          job: {
            ...mockJobExecution,
            jobInputDatasetItemId: "dataset-item-123",
          },
          deps,
        }),
      );

      expect(callLLM).toHaveBeenCalledWith(
        expect.objectContaining({
          traceSinkParams: expect.objectContaining({
            metadata: expect.objectContaining({
              target_dataset_item_id: "dataset-item-123",
            }),
          }),
        }),
      );

      expect(uploadScore).toHaveBeenCalledWith(
        expect.objectContaining({
          event: expect.objectContaining({
            body: expect.objectContaining({
              metadata: expect.objectContaining({
                target_dataset_item_id: "dataset-item-123",
              }),
            }),
          }),
        }),
      );
    });

    it("should include observation ID in metadata when present", async () => {
      const callLLM = mockSuccessfulLLMCall(
        0.85,
        "Observation evaluation complete",
      );
      const uploadScore = vi.fn();
      const deps = createSuccessfulDeps({ callLLM, uploadScore });

      await executeLLMAsJudgeEvaluation(
        createExecutionParams({
          job: { ...mockJobExecution, jobInputObservationId: "obs-456" },
          deps,
        }),
      );

      expect(callLLM).toHaveBeenCalledWith(
        expect.objectContaining({
          traceSinkParams: expect.objectContaining({
            metadata: expect.objectContaining({
              target_observation_id: "obs-456",
            }),
          }),
        }),
      );

      expect(uploadScore).toHaveBeenCalledWith(
        expect.objectContaining({
          event: expect.objectContaining({
            body: expect.objectContaining({
              metadata: expect.objectContaining({
                target_observation_id: "obs-456",
              }),
            }),
          }),
        }),
      );
    });

    it("should include all job identifiers in full evaluation context", async () => {
      const fullJob = {
        ...mockJobExecution,
        jobInputObservationId: "obs-full",
        jobInputDatasetItemId: "dataset-full",
      };

      const callLLM = mockSuccessfulLLMCall(0.95, "Complete evaluation");
      const uploadScore = vi.fn();
      const deps = createSuccessfulDeps({ callLLM, uploadScore });

      await executeLLMAsJudgeEvaluation(
        createExecutionParams({ job: fullJob, deps }),
      );

      const expectedTraceMetadata = {
        job_execution_id: jobExecutionId,
        target_trace_id: fullJob.jobInputTraceId,
        target_observation_id: "obs-full",
        target_dataset_item_id: "dataset-full",
      };

      expect(callLLM).toHaveBeenCalledWith(
        expect.objectContaining({
          traceSinkParams: expect.objectContaining({
            evaluationContext: {
              evaluationRuleId: fullJob.jobConfigurationId,
              evaluatorExecutionIsTest: false,
            },
            metadata: expect.objectContaining(expectedTraceMetadata),
          }),
        }),
      );

      expect(uploadScore).toHaveBeenCalledWith(
        expect.objectContaining({
          event: expect.objectContaining({
            body: expect.objectContaining({
              evaluationRuleId: fullJob.jobConfigurationId,
              metadata: expect.objectContaining({
                job_execution_id: jobExecutionId,
                target_trace_id: fullJob.jobInputTraceId,
                target_observation_id: "obs-full",
                target_dataset_item_id: "dataset-full",
              }),
            }),
          }),
        }),
      );
    });
  });

  describe("score event structure", () => {
    it("should build complete score event with all required fields", async () => {
      const uploadScore = vi.fn();
      const deps = createSuccessfulDeps({
        callLLM: mockSuccessfulLLMCall(0.88, "Well structured response"),
        uploadScore,
      });

      await executeLLMAsJudgeEvaluation(createExecutionParams({ deps }));

      expect(uploadScore).toHaveBeenCalledWith({
        projectId,
        scoreId: expect.any(String),
        eventId: expect.any(String),
        event: {
          id: expect.any(String),
          timestamp: expect.any(String),
          type: "score-create",
          body: {
            id: expect.any(String),
            traceId: mockJobExecution.jobInputTraceId,
            observationId: null,
            name: mockJobConfiguration.scoreName,
            value: 0.88,
            comment: "Well structured response",
            configId: undefined,
            source: "EVAL",
            environment: "production",
            executionTraceId: expect.any(String),
            evaluatorId: undefined,
            evaluationRuleId: mockJobExecution.jobConfigurationId,
            metadata: expect.objectContaining({
              job_execution_id: jobExecutionId,
              target_trace_id: mockJobExecution.jobInputTraceId,
            }),
            dataType: "NUMERIC",
          },
        },
      });
    });

    it("should use same scoreId in uploadScore and updateJobExecution", async () => {
      const uploadScore = vi.fn();
      const updateJobExecution = vi.fn();
      const deps = createSuccessfulDeps({
        callLLM: mockSuccessfulLLMCall(0.7, "Consistent IDs"),
        uploadScore,
        updateJobExecution,
      });

      await executeLLMAsJudgeEvaluation(createExecutionParams({ deps }));

      const uploadCall = uploadScore.mock.calls[0][0];
      const scoreId = uploadCall.scoreId;

      expect(updateJobExecution).toHaveBeenCalledWith(
        expect.objectContaining({
          data: expect.objectContaining({
            jobOutputScoreId: scoreId,
          }),
        }),
      );
    });

    it("should use same executionTraceId in score event and job update", async () => {
      const uploadScore = vi.fn();
      const updateJobExecution = vi.fn();
      const deps = createSuccessfulDeps({
        callLLM: mockSuccessfulLLMCall(0.6, "Trace ID consistency"),
        uploadScore,
        updateJobExecution,
      });

      await executeLLMAsJudgeEvaluation(createExecutionParams({ deps }));

      const uploadCall = uploadScore.mock.calls[0][0];
      const executionTraceId = uploadCall.event.body.executionTraceId;

      expect(updateJobExecution).toHaveBeenCalledWith(
        expect.objectContaining({
          data: expect.objectContaining({
            executionTraceId,
          }),
        }),
      );
    });

    it("should persist categorical eval scores", async () => {
      const uploadScore = vi.fn();
      const deps = createSuccessfulDeps({
        callLLM: mockSuccessfulLLMCall(
          "correct",
          "The answer is fully supported by the context.",
        ),
        uploadScore,
      });

      await executeLLMAsJudgeEvaluation(
        createExecutionParams({
          deps,
          template: {
            ...mockEvalTemplate,
            outputDefinition: {
              version: 2,
              dataType: ScoreDataTypeEnum.CATEGORICAL,
              reasoning: {
                description: "Explain the selected category",
              },
              score: {
                description: "Choose the best matching category",
                categories: ["correct", "partial"],
              },
            },
          },
        }),
      );

      expect(uploadScore).toHaveBeenCalledWith(
        expect.objectContaining({
          event: expect.objectContaining({
            body: expect.objectContaining({
              value: "correct",
              comment: "The answer is fully supported by the context.",
              dataType: "CATEGORICAL",
            }),
          }),
        }),
      );
    });

    it("should persist boolean eval scores", async () => {
      const uploadScore = vi.fn();
      const deps = createSuccessfulDeps({
        callLLM: mockSuccessfulLLMCall(
          true,
          "The answer satisfies the criteria.",
        ),
        uploadScore,
      });

      await executeLLMAsJudgeEvaluation(
        createExecutionParams({
          deps,
          template: {
            ...mockEvalTemplate,
            outputDefinition: {
              version: 2,
              dataType: ScoreDataTypeEnum.BOOLEAN,
              reasoning: {
                description: "Explain the verdict",
              },
              score: {
                description:
                  "Return true if the answer satisfies the criteria, otherwise false",
              },
            },
          },
        }),
      );

      expect(uploadScore).toHaveBeenCalledWith(
        expect.objectContaining({
          event: expect.objectContaining({
            body: expect.objectContaining({
              value: 1,
              comment: "The answer satisfies the criteria.",
              dataType: "BOOLEAN",
            }),
          }),
        }),
      );
    });

    it("should fail boolean evals when the model returns a string", async () => {
      const deps = createSuccessfulDeps({
        callLLM: mockSuccessfulLLMCall(
          "true",
          "Strings should not be accepted for boolean evaluators.",
        ),
      });

      await expect(
        executeLLMAsJudgeEvaluation(
          createExecutionParams({
            deps,
            template: {
              ...mockEvalTemplate,
              outputDefinition: {
                version: 2,
                dataType: ScoreDataTypeEnum.BOOLEAN,
                reasoning: {
                  description: "Explain the verdict",
                },
                score: {
                  description:
                    "Return true if the answer satisfies the criteria, otherwise false",
                },
              },
            },
          }),
        ),
      ).rejects.toThrow(UnrecoverableError);
    });

    it("should fail categorical evals when the model returns a category outside the allowed set", async () => {
      const deps = createSuccessfulDeps({
        callLLM: mockSuccessfulLLMCall(
          "incorrect",
          "This category is not configured on the evaluator.",
        ),
      });

      await expect(
        executeLLMAsJudgeEvaluation(
          createExecutionParams({
            deps,
            template: {
              ...mockEvalTemplate,
              outputDefinition: {
                version: 2,
                dataType: ScoreDataTypeEnum.CATEGORICAL,
                reasoning: {
                  description: "Explain the selected category",
                },
                score: {
                  description: "Choose the best matching category",
                  categories: ["correct", "partial"],
                },
              },
            },
          }),
        ),
      ).rejects.toThrow(UnrecoverableError);
    });

    it("should fail categorical evals when the model returns multiple matches for a single-match evaluator", async () => {
      const deps = createSuccessfulDeps({
        callLLM: mockSuccessfulLLMCall(
          ["correct", "partial"],
          "This should fail because only one match is allowed.",
        ),
      });

      await expect(
        executeLLMAsJudgeEvaluation(
          createExecutionParams({
            deps,
            template: {
              ...mockEvalTemplate,
              outputDefinition: {
                version: 2,
                dataType: ScoreDataTypeEnum.CATEGORICAL,
                reasoning: {
                  description: "Explain the selected category",
                },
                score: {
                  description: "Choose the best matching category",
                  categories: ["correct", "partial"],
                },
              },
            },
          }),
        ),
      ).rejects.toThrow(UnrecoverableError);
    });

    it("should persist one score per categorical match", async () => {
      const uploadScore = vi.fn();
      const enqueueScoreIngestion = vi.fn();
      const updateJobExecution = vi.fn();
      const deps = createSuccessfulDeps({
        callLLM: mockSuccessfulLLMCall(
          ["correct", "partial"],
          "Both categories apply to the answer.",
        ),
        uploadScore,
        enqueueScoreIngestion,
        updateJobExecution,
      });

      await executeLLMAsJudgeEvaluation(
        createExecutionParams({
          deps,
          template: {
            ...mockEvalTemplate,
            outputDefinition: {
              version: 2,
              dataType: ScoreDataTypeEnum.CATEGORICAL,
              reasoning: {
                description: "Explain the selected categories",
              },
              score: {
                description: "Choose all matching categories",
                categories: ["correct", "partial"],
                shouldAllowMultipleMatches: true,
              },
            },
          },
        }),
      );

      expect(uploadScore).toHaveBeenCalledTimes(2);
      expect(enqueueScoreIngestion).toHaveBeenCalledTimes(2);
      expect(getScorePersistenceMetricCalls()).toEqual([
        [
          "langfuse.evaluation.score_persistence",
          1,
          {
            evaluator_type: "llm_as_judge",
            outcome: "success",
            score_count_bucket: "2",
          },
        ],
      ]);

      const firstUploadCall = uploadScore.mock.calls[0][0];
      const secondUploadCall = uploadScore.mock.calls[1][0];

      expect(firstUploadCall.event.body.value).toBe("correct");
      expect(secondUploadCall.event.body.value).toBe("partial");
      expect(firstUploadCall.event.body.comment).toBe(
        "Both categories apply to the answer.",
      );
      expect(secondUploadCall.event.body.comment).toBe(
        "Both categories apply to the answer.",
      );
      expect(firstUploadCall.scoreId).not.toBe(secondUploadCall.scoreId);
      expect(firstUploadCall.event.body.executionTraceId).toBe(
        secondUploadCall.event.body.executionTraceId,
      );
      expect(firstUploadCall.event.body.metadata).toEqual(
        secondUploadCall.event.body.metadata,
      );

      expect(updateJobExecution).toHaveBeenCalledWith(
        expect.objectContaining({
          data: expect.objectContaining({
            jobOutputScoreId: firstUploadCall.scoreId,
          }),
        }),
      );
    });

    it("should fail multi-match categorical evals when the model returns a scalar", async () => {
      const deps = createSuccessfulDeps({
        callLLM: mockSuccessfulLLMCall(
          "correct",
          "This should fail because multi-match expects an array.",
        ),
      });

      await expect(
        executeLLMAsJudgeEvaluation(
          createExecutionParams({
            deps,
            template: {
              ...mockEvalTemplate,
              outputDefinition: {
                version: 2,
                dataType: ScoreDataTypeEnum.CATEGORICAL,
                reasoning: {
                  description: "Explain the selected categories",
                },
                score: {
                  description: "Choose all matching categories",
                  categories: ["correct", "partial"],
                  shouldAllowMultipleMatches: true,
                },
              },
            },
          }),
        ),
      ).rejects.toThrow(UnrecoverableError);
    });

    it("should fail multi-match categorical evals when the model returns duplicate matches", async () => {
      const deps = createSuccessfulDeps({
        callLLM: mockSuccessfulLLMCall(
          ["correct", "correct"],
          "This should fail because duplicates are not allowed.",
        ),
      });

      await expect(
        executeLLMAsJudgeEvaluation(
          createExecutionParams({
            deps,
            template: {
              ...mockEvalTemplate,
              outputDefinition: {
                version: 2,
                dataType: ScoreDataTypeEnum.CATEGORICAL,
                reasoning: {
                  description: "Explain the selected categories",
                },
                score: {
                  description: "Choose all matching categories",
                  categories: ["correct", "partial"],
                  shouldAllowMultipleMatches: true,
                },
              },
            },
          }),
        ),
      ).rejects.toThrow(UnrecoverableError);
    });
  });

  describe("LLM errors", () => {
    it("should propagate LLM exception for BullMQ retry", async () => {
      const deps = createMockEvalExecutionDeps({
        fetchModelConfig: mockValidFetchModelConfig(),
        callLLM: vi
          .fn()
          .mockRejectedValue(new Error("LLM service unavailable")),
      });

      await expect(
        executeLLMAsJudgeEvaluation(createExecutionParams({ deps })),
      ).rejects.toThrow("LLM service unavailable");
    });

    it("should propagate rate limit errors from LLM", async () => {
      const rateLimitError = new Error("Rate limit exceeded");
      (rateLimitError as unknown as { isRetryable: boolean }).isRetryable =
        true;

      const deps = createMockEvalExecutionDeps({
        fetchModelConfig: mockValidFetchModelConfig(),
        callLLM: vi.fn().mockRejectedValue(rateLimitError),
      });

      await expect(
        executeLLMAsJudgeEvaluation(createExecutionParams({ deps })),
      ).rejects.toThrow("Rate limit exceeded");
    });

    it("records native error policy and blocks the evaluator", async () => {
      const llmError = new LLMValidationError({
        code: "endpoint-unreachable",
        message: "Cannot reach custom endpoint",
      });
      const deps = createMockEvalExecutionDeps({
        fetchModelConfig: mockValidFetchModelConfig(),
        callLLM: vi.fn().mockRejectedValue(llmError),
      });

      await expect(
        executeLLMAsJudgeEvaluation({
          ...createExecutionParams({ deps }),
          evaluatorId: "evaluator-123",
        }),
      ).rejects.toBe(llmError);

      const executeSpan = observabilityMocks.spans.find(
        ({ name }) => name === "eval.execute-llm-as-judge",
      );
      const callSpan = observabilityMocks.spans.find(
        ({ name }) => name === "eval.call-llm",
      );
      const expectedErrorAttributes = {
        "eval.llm.error.kind": "validation",
        "eval.llm.error.retryable": false,
        "eval.llm.error.status_code": 400,
        "eval.llm.blocked": true,
        "eval.llm.block.reason": "LLM_CONNECTION_ENDPOINT_UNREACHABLE",
        "eval.llm.block.source": "llm_completion_error",
      };

      expect(callSpan?.attributes).toMatchObject({
        ...expectedErrorAttributes,
        "eval.llm.outcome": "blocked",
      });
      expect(executeSpan?.attributes).toMatchObject({
        ...expectedErrorAttributes,
        "eval.llm.block.applied": true,
        "eval.execution.stage": "call_llm",
        "eval.execution.outcome": "blocked",
      });
      expect(callSpan?.attributes).not.toHaveProperty(
        "http.response.status_code",
      );
      expect(observabilityMocks.blockEvaluator).toHaveBeenCalledWith(
        expect.objectContaining({
          evaluatorId: "evaluator-123",
          blockReason: "LLM_CONNECTION_ENDPOINT_UNREACHABLE",
          source: "llm_completion_error",
        }),
      );
    });
  });

  describe("job execution update errors", () => {
    it("should propagate updateJobExecution errors", async () => {
      const deps = createMockEvalExecutionDeps({
        fetchModelConfig: mockValidFetchModelConfig(),
        callLLM: mockSuccessfulLLMCall(0.8, "Good"),
        uploadScore: vi.fn(),
        enqueueScoreIngestion: vi.fn(),
        updateJobExecution: vi
          .fn()
          .mockRejectedValue(new Error("Database connection lost")),
      });

      await expect(
        executeLLMAsJudgeEvaluation(createExecutionParams({ deps })),
      ).rejects.toThrow("Database connection lost");
    });

    it("should have persisted score before job update fails", async () => {
      const uploadScore = vi.fn();
      const enqueueScoreIngestion = vi.fn();

      const deps = createMockEvalExecutionDeps({
        fetchModelConfig: mockValidFetchModelConfig(),
        callLLM: mockSuccessfulLLMCall(0.8, "Good"),
        uploadScore,
        enqueueScoreIngestion,
        updateJobExecution: vi
          .fn()
          .mockRejectedValue(new Error("Database connection lost")),
      });

      try {
        await executeLLMAsJudgeEvaluation(createExecutionParams({ deps }));
      } catch {
        // Expected to throw
      }

      // Score should have been persisted before the job update error
      expect(uploadScore).toHaveBeenCalledTimes(1);
      expect(enqueueScoreIngestion).toHaveBeenCalledTimes(1);
    });
  });
});