🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(evals): measure score persistence outcomes (#18600)
Tobias Wochinger3795b3021fb3
diff --git a/worker/src/features/evaluation/evalCompletion.ts b/worker/src/features/evaluation/evalCompletion.tsindex 910e3a2a16..12aadde966 100644--- a/worker/src/features/evaluation/evalCompletion.ts+++ b/worker/src/features/evaluation/evalCompletion.ts@@ -7,6 +7,10 @@ import {} from "@langfuse/shared/src/server";import { buildEvalScoreWritePayloads } from "./evalScoreEvent";import { type EvalExecutionDeps } from "./evalExecutionDeps";+import {+ recordEvalScorePersistenceOutcome,+ type ObservationEvalExecutionType,+} from "./evalExecutionMetrics";/*** Common result returned by every evaluator executor (LLM-as-judge,@@ -27,6 +31,7 @@ export async function completeEvalExecution({traceId,observationId,environment,+ executionType,deps,}: {projectId: string;@@ -35,6 +40,7 @@ export async function completeEvalExecution({traceId: string | null;observationId: string | null;environment: string;+ executionType: ObservationEvalExecutionType;deps: EvalExecutionDeps;}): Promise<{ scoreCount: number }> {const scoreWritePayloads = buildEvalScoreWritePayloads({@@ -72,6 +78,11 @@ export async function completeEvalExecution({}),);} catch (e) {+ recordEvalScorePersistenceOutcome(+ executionType,+ "failure",+ scoreWritePayloads.length,+ );logger.error(`Failed to persist score: ${e}`, e);traceException(e);throw new Error(@@ -79,6 +90,11 @@ export async function completeEvalExecution({);}+ recordEvalScorePersistenceOutcome(+ executionType,+ "success",+ scoreWritePayloads.length,+ );logger.debug(`Persisted ${scoreWritePayloads.length} score(s) for job ${jobExecutionId}`,);diff --git a/worker/src/features/evaluation/evalExecutionMetrics.test.ts b/worker/src/features/evaluation/evalExecutionMetrics.test.tsindex ba5bd48617..c6df2cf255 100644--- a/worker/src/features/evaluation/evalExecutionMetrics.test.ts+++ b/worker/src/features/evaluation/evalExecutionMetrics.test.ts@@ -8,8 +8,10 @@ import {} from "@langfuse/shared/src/server";import { beforeEach, describe, expect, it, vi } from "vitest";import {+ __test,getCodeEvalTerminalErrorOutcome,getLlmEvalTerminalErrorOutcome,+ recordEvalScorePersistenceOutcome,recordEvalTerminalOutcome,recordEvalTimeToFirstAttempt,} from "./evalExecutionMetrics";@@ -28,6 +30,11 @@ describe("evaluation execution metrics", () => {it("records the bounded metric contract", () => {recordEvalTimeToFirstAttempt(EvalTemplateType.LLM_AS_JUDGE, 1_234);recordEvalTerminalOutcome(EvalTemplateType.CODE, "success");+ recordEvalScorePersistenceOutcome(+ EvalTemplateType.LLM_AS_JUDGE,+ "failure",+ 7,+ );expect(recordDistribution).toHaveBeenCalledWith("langfuse.evaluation.execution.time_to_first_attempt_ms",@@ -42,6 +49,28 @@ describe("evaluation execution metrics", () => {1,{ evaluator_type: "code_as_judge", outcome: "success" },);+ expect(recordIncrement).toHaveBeenCalledWith(+ "langfuse.evaluation.score_persistence",+ 1,+ {+ evaluator_type: "llm_as_judge",+ outcome: "failure",+ score_count_bucket: "6-10",+ },+ );+ });++ it.each([+ { scoreCount: 0, expected: "0" },+ { scoreCount: 1, expected: "1" },+ { scoreCount: 2, expected: "2" },+ { scoreCount: 3, expected: "3-5" },+ { scoreCount: 5, expected: "3-5" },+ { scoreCount: 6, expected: "6-10" },+ { scoreCount: 10, expected: "6-10" },+ { scoreCount: 11, expected: "11+" },+ ])("buckets $scoreCount scores as $expected", ({ scoreCount, expected }) => {+ expect(__test.getEvalScoreCountBucket(scoreCount)).toBe(expected);});it.each([diff --git a/worker/src/features/evaluation/evalExecutionMetrics.ts b/worker/src/features/evaluation/evalExecutionMetrics.tsindex 242e15a9e6..8a7cbcc7a1 100644--- a/worker/src/features/evaluation/evalExecutionMetrics.ts+++ b/worker/src/features/evaluation/evalExecutionMetrics.ts@@ -18,6 +18,8 @@ export type EvalExecutionTerminalOutcome =| "customer_error"| "cancelled";+type EvalScoreCountBucket = "0" | "1" | "2" | "3-5" | "6-10" | "11+";+const LLM_EVAL_CUSTOMER_OUTPUT_ERROR_NAMES: ReadonlySet<string> = new Set(["AI_NoObjectGeneratedError","AI_NoOutputGeneratedError",@@ -58,6 +60,18 @@ export function recordEvalTerminalOutcome(});}+export function recordEvalScorePersistenceOutcome(+ executionType: ObservationEvalExecutionType,+ outcome: "success" | "failure",+ scoreCount: number,+): void {+ recordIncrement("langfuse.evaluation.score_persistence", 1, {+ evaluator_type: getEvaluatorTypeTag(executionType),+ outcome,+ score_count_bucket: getEvalScoreCountBucket(scoreCount),+ });+}+export function getLlmEvalTerminalErrorOutcome(classification: EvaluatorLlmErrorClassification | null,): Extract<@@ -139,3 +153,16 @@ function getEvaluatorTypeTag(? "llm_as_judge": "code_as_judge";}++function getEvalScoreCountBucket(scoreCount: number): EvalScoreCountBucket {+ if (scoreCount === 0) return "0";+ if (scoreCount === 1) return "1";+ if (scoreCount === 2) return "2";+ if (scoreCount <= 5) return "3-5";+ if (scoreCount <= 10) return "6-10";+ return "11+";+}++export const __test = {+ getEvalScoreCountBucket,+};diff --git a/worker/src/features/evaluation/evalService.ts b/worker/src/features/evaluation/evalService.tsindex 7634fbedc5..b4fd49a703 100644--- a/worker/src/features/evaluation/evalService.ts+++ b/worker/src/features/evaluation/evalService.ts@@ -1310,6 +1310,7 @@ export async function executeLLMAsJudgeEvaluation(traceId: params.job.jobInputTraceId,observationId: params.job.jobInputObservationId,environment: params.environment,+ executionType: EvalTemplateType.LLM_AS_JUDGE,deps,result,});diff --git a/worker/src/features/evaluation/executeLLMAsJudgeEvaluation.test.ts b/worker/src/features/evaluation/executeLLMAsJudgeEvaluation.test.tsindex 2ca85d6a37..279cc45faf 100644--- a/worker/src/features/evaluation/executeLLMAsJudgeEvaluation.test.ts+++ b/worker/src/features/evaluation/executeLLMAsJudgeEvaluation.test.ts@@ -8,6 +8,7 @@ const observabilityMocks = vi.hoisted(() => ({attributes: Record<string, unknown>;}>,blockEvaluator: vi.fn().mockResolvedValue({ blockedEvaluatorIds: [] }),+ recordIncrement: vi.fn(),}));vi.mock("@langfuse/shared/src/server", async (importOriginal) => {@@ -17,6 +18,7 @@ vi.mock("@langfuse/shared/src/server", async (importOriginal) => {return {...actual,blockEvaluator: observabilityMocks.blockEvaluator,+ recordIncrement: observabilityMocks.recordIncrement,instrumentAsync: vi.fn(async ({ name }: { name: string },@@ -48,6 +50,7 @@ import { createMockEvalExecutionDeps } from "./evalExecutionDeps";import { UnrecoverableError } from "../../errors/UnrecoverableError";import {LLMValidationError,+ recordIncrement,type ExtractedVariable,} from "@langfuse/shared/src/server";import {@@ -71,10 +74,17 @@ import {describe("executeLLMAsJudgeEvaluation", () => {const projectId = "test-project-123";const jobExecutionId = "test-job-execution-456";+ const getScorePersistenceMetricCalls = () =>+ vi+ .mocked(recordIncrement)+ .mock.calls.filter(+ ([metric]) => metric === "langfuse.evaluation.score_persistence",+ );beforeEach(() => {observabilityMocks.spans.length = 0;observabilityMocks.blockEvaluator.mockClear();+ observabilityMocks.recordIncrement.mockClear();});// ============================================================================@@ -250,6 +260,17 @@ describe("executeLLMAsJudgeEvaluation", () => {}),);expect(enqueueScoreIngestion).toHaveBeenCalledTimes(1);+ expect(getScorePersistenceMetricCalls()).toEqual([+ [+ "langfuse.evaluation.score_persistence",+ 1,+ {+ evaluator_type: "llm_as_judge",+ outcome: "success",+ score_count_bucket: "1",+ },+ ],+ ]);expect(updateJobExecution).toHaveBeenCalledWith(expect.objectContaining({id: jobExecutionId,@@ -436,6 +457,17 @@ describe("executeLLMAsJudgeEvaluation", () => {await expect(executeLLMAsJudgeEvaluation(createExecutionParams({ deps })),).rejects.toThrow("Failed to write score");+ expect(getScorePersistenceMetricCalls()).toEqual([+ [+ "langfuse.evaluation.score_persistence",+ 1,+ {+ evaluator_type: "llm_as_judge",+ outcome: "failure",+ score_count_bucket: "1",+ },+ ],+ ]);});it("should throw error if score ingestion queue fails", async () => {@@ -1187,6 +1219,17 @@ describe("executeLLMAsJudgeEvaluation", () => {expect(uploadScore).toHaveBeenCalledTimes(2);expect(enqueueScoreIngestion).toHaveBeenCalledTimes(2);+ expect(getScorePersistenceMetricCalls()).toEqual([+ [+ "langfuse.evaluation.score_persistence",+ 1,+ {+ evaluator_type: "llm_as_judge",+ outcome: "success",+ score_count_bucket: "2",+ },+ ],+ ]);const firstUploadCall = uploadScore.mock.calls[0][0];const secondUploadCall = uploadScore.mock.calls[1][0];diff --git a/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts b/worker/src/features/evaluation/observationEval/observationEvalProcessor.tsindex a707256450..6c5f3a89ef 100644--- a/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts+++ b/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts@@ -303,6 +303,7 @@ export async function processObservationEval(traceId: executionParams.job.jobInputTraceId,observationId: executionParams.job.jobInputObservationId,environment: executionParams.environment,+ executionType: params.executionType,deps: executionParams.deps,result: executionResult,});