Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(evals): measure score persistence outcomes (#18600)

Tobias Wochinger3795b3021fb3
diff --git a/worker/src/features/evaluation/evalCompletion.ts b/worker/src/features/evaluation/evalCompletion.ts
index 910e3a2a16..12aadde966 100644
--- a/worker/src/features/evaluation/evalCompletion.ts
+++ b/worker/src/features/evaluation/evalCompletion.ts
@@ -7,6 +7,10 @@ import {
} from "@langfuse/shared/src/server";
import { buildEvalScoreWritePayloads } from "./evalScoreEvent";
import { type EvalExecutionDeps } from "./evalExecutionDeps";
+import {
+ recordEvalScorePersistenceOutcome,
+ type ObservationEvalExecutionType,
+} from "./evalExecutionMetrics";
/**
* Common result returned by every evaluator executor (LLM-as-judge,
@@ -27,6 +31,7 @@ export async function completeEvalExecution({
traceId,
observationId,
environment,
+ executionType,
deps,
}: {
projectId: string;
@@ -35,6 +40,7 @@ export async function completeEvalExecution({
traceId: string | null;
observationId: string | null;
environment: string;
+ executionType: ObservationEvalExecutionType;
deps: EvalExecutionDeps;
}): Promise<{ scoreCount: number }> {
const scoreWritePayloads = buildEvalScoreWritePayloads({
@@ -72,6 +78,11 @@ export async function completeEvalExecution({
}),
);
} catch (e) {
+ recordEvalScorePersistenceOutcome(
+ executionType,
+ "failure",
+ scoreWritePayloads.length,
+ );
logger.error(`Failed to persist score: ${e}`, e);
traceException(e);
throw new Error(
@@ -79,6 +90,11 @@ export async function completeEvalExecution({
);
}
+ recordEvalScorePersistenceOutcome(
+ executionType,
+ "success",
+ scoreWritePayloads.length,
+ );
logger.debug(
`Persisted ${scoreWritePayloads.length} score(s) for job ${jobExecutionId}`,
);
diff --git a/worker/src/features/evaluation/evalExecutionMetrics.test.ts b/worker/src/features/evaluation/evalExecutionMetrics.test.ts
index ba5bd48617..c6df2cf255 100644
--- a/worker/src/features/evaluation/evalExecutionMetrics.test.ts
+++ b/worker/src/features/evaluation/evalExecutionMetrics.test.ts
@@ -8,8 +8,10 @@ import {
} from "@langfuse/shared/src/server";
import { beforeEach, describe, expect, it, vi } from "vitest";
import {
+ __test,
getCodeEvalTerminalErrorOutcome,
getLlmEvalTerminalErrorOutcome,
+ recordEvalScorePersistenceOutcome,
recordEvalTerminalOutcome,
recordEvalTimeToFirstAttempt,
} from "./evalExecutionMetrics";
@@ -28,6 +30,11 @@ describe("evaluation execution metrics", () => {
it("records the bounded metric contract", () => {
recordEvalTimeToFirstAttempt(EvalTemplateType.LLM_AS_JUDGE, 1_234);
recordEvalTerminalOutcome(EvalTemplateType.CODE, "success");
+ recordEvalScorePersistenceOutcome(
+ EvalTemplateType.LLM_AS_JUDGE,
+ "failure",
+ 7,
+ );
expect(recordDistribution).toHaveBeenCalledWith(
"langfuse.evaluation.execution.time_to_first_attempt_ms",
@@ -42,6 +49,28 @@ describe("evaluation execution metrics", () => {
1,
{ evaluator_type: "code_as_judge", outcome: "success" },
);
+ expect(recordIncrement).toHaveBeenCalledWith(
+ "langfuse.evaluation.score_persistence",
+ 1,
+ {
+ evaluator_type: "llm_as_judge",
+ outcome: "failure",
+ score_count_bucket: "6-10",
+ },
+ );
+ });
+
+ it.each([
+ { scoreCount: 0, expected: "0" },
+ { scoreCount: 1, expected: "1" },
+ { scoreCount: 2, expected: "2" },
+ { scoreCount: 3, expected: "3-5" },
+ { scoreCount: 5, expected: "3-5" },
+ { scoreCount: 6, expected: "6-10" },
+ { scoreCount: 10, expected: "6-10" },
+ { scoreCount: 11, expected: "11+" },
+ ])("buckets $scoreCount scores as $expected", ({ scoreCount, expected }) => {
+ expect(__test.getEvalScoreCountBucket(scoreCount)).toBe(expected);
});
it.each([
diff --git a/worker/src/features/evaluation/evalExecutionMetrics.ts b/worker/src/features/evaluation/evalExecutionMetrics.ts
index 242e15a9e6..8a7cbcc7a1 100644
--- a/worker/src/features/evaluation/evalExecutionMetrics.ts
+++ b/worker/src/features/evaluation/evalExecutionMetrics.ts
@@ -18,6 +18,8 @@ export type EvalExecutionTerminalOutcome =
| "customer_error"
| "cancelled";
+type EvalScoreCountBucket = "0" | "1" | "2" | "3-5" | "6-10" | "11+";
+
const LLM_EVAL_CUSTOMER_OUTPUT_ERROR_NAMES: ReadonlySet<string> = new Set([
"AI_NoObjectGeneratedError",
"AI_NoOutputGeneratedError",
@@ -58,6 +60,18 @@ export function recordEvalTerminalOutcome(
});
}
+export function recordEvalScorePersistenceOutcome(
+ executionType: ObservationEvalExecutionType,
+ outcome: "success" | "failure",
+ scoreCount: number,
+): void {
+ recordIncrement("langfuse.evaluation.score_persistence", 1, {
+ evaluator_type: getEvaluatorTypeTag(executionType),
+ outcome,
+ score_count_bucket: getEvalScoreCountBucket(scoreCount),
+ });
+}
+
export function getLlmEvalTerminalErrorOutcome(
classification: EvaluatorLlmErrorClassification | null,
): Extract<
@@ -139,3 +153,16 @@ function getEvaluatorTypeTag(
? "llm_as_judge"
: "code_as_judge";
}
+
+function getEvalScoreCountBucket(scoreCount: number): EvalScoreCountBucket {
+ if (scoreCount === 0) return "0";
+ if (scoreCount === 1) return "1";
+ if (scoreCount === 2) return "2";
+ if (scoreCount <= 5) return "3-5";
+ if (scoreCount <= 10) return "6-10";
+ return "11+";
+}
+
+export const __test = {
+ getEvalScoreCountBucket,
+};
diff --git a/worker/src/features/evaluation/evalService.ts b/worker/src/features/evaluation/evalService.ts
index 7634fbedc5..b4fd49a703 100644
--- a/worker/src/features/evaluation/evalService.ts
+++ b/worker/src/features/evaluation/evalService.ts
@@ -1310,6 +1310,7 @@ export async function executeLLMAsJudgeEvaluation(
traceId: params.job.jobInputTraceId,
observationId: params.job.jobInputObservationId,
environment: params.environment,
+ executionType: EvalTemplateType.LLM_AS_JUDGE,
deps,
result,
});
diff --git a/worker/src/features/evaluation/executeLLMAsJudgeEvaluation.test.ts b/worker/src/features/evaluation/executeLLMAsJudgeEvaluation.test.ts
index 2ca85d6a37..279cc45faf 100644
--- a/worker/src/features/evaluation/executeLLMAsJudgeEvaluation.test.ts
+++ b/worker/src/features/evaluation/executeLLMAsJudgeEvaluation.test.ts
@@ -8,6 +8,7 @@ const observabilityMocks = vi.hoisted(() => ({
attributes: Record<string, unknown>;
}>,
blockEvaluator: vi.fn().mockResolvedValue({ blockedEvaluatorIds: [] }),
+ recordIncrement: vi.fn(),
}));
vi.mock("@langfuse/shared/src/server", async (importOriginal) => {
@@ -17,6 +18,7 @@ vi.mock("@langfuse/shared/src/server", async (importOriginal) => {
return {
...actual,
blockEvaluator: observabilityMocks.blockEvaluator,
+ recordIncrement: observabilityMocks.recordIncrement,
instrumentAsync: vi.fn(
async (
{ name }: { name: string },
@@ -48,6 +50,7 @@ import { createMockEvalExecutionDeps } from "./evalExecutionDeps";
import { UnrecoverableError } from "../../errors/UnrecoverableError";
import {
LLMValidationError,
+ recordIncrement,
type ExtractedVariable,
} from "@langfuse/shared/src/server";
import {
@@ -71,10 +74,17 @@ import {
describe("executeLLMAsJudgeEvaluation", () => {
const projectId = "test-project-123";
const jobExecutionId = "test-job-execution-456";
+ const getScorePersistenceMetricCalls = () =>
+ vi
+ .mocked(recordIncrement)
+ .mock.calls.filter(
+ ([metric]) => metric === "langfuse.evaluation.score_persistence",
+ );
beforeEach(() => {
observabilityMocks.spans.length = 0;
observabilityMocks.blockEvaluator.mockClear();
+ observabilityMocks.recordIncrement.mockClear();
});
// ============================================================================
@@ -250,6 +260,17 @@ describe("executeLLMAsJudgeEvaluation", () => {
}),
);
expect(enqueueScoreIngestion).toHaveBeenCalledTimes(1);
+ expect(getScorePersistenceMetricCalls()).toEqual([
+ [
+ "langfuse.evaluation.score_persistence",
+ 1,
+ {
+ evaluator_type: "llm_as_judge",
+ outcome: "success",
+ score_count_bucket: "1",
+ },
+ ],
+ ]);
expect(updateJobExecution).toHaveBeenCalledWith(
expect.objectContaining({
id: jobExecutionId,
@@ -436,6 +457,17 @@ describe("executeLLMAsJudgeEvaluation", () => {
await expect(
executeLLMAsJudgeEvaluation(createExecutionParams({ deps })),
).rejects.toThrow("Failed to write score");
+ expect(getScorePersistenceMetricCalls()).toEqual([
+ [
+ "langfuse.evaluation.score_persistence",
+ 1,
+ {
+ evaluator_type: "llm_as_judge",
+ outcome: "failure",
+ score_count_bucket: "1",
+ },
+ ],
+ ]);
});
it("should throw error if score ingestion queue fails", async () => {
@@ -1187,6 +1219,17 @@ describe("executeLLMAsJudgeEvaluation", () => {
expect(uploadScore).toHaveBeenCalledTimes(2);
expect(enqueueScoreIngestion).toHaveBeenCalledTimes(2);
+ expect(getScorePersistenceMetricCalls()).toEqual([
+ [
+ "langfuse.evaluation.score_persistence",
+ 1,
+ {
+ evaluator_type: "llm_as_judge",
+ outcome: "success",
+ score_count_bucket: "2",
+ },
+ ],
+ ]);
const firstUploadCall = uploadScore.mock.calls[0][0];
const secondUploadCall = uploadScore.mock.calls[1][0];
diff --git a/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts b/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts
index a707256450..6c5f3a89ef 100644
--- a/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts
+++ b/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts
@@ -303,6 +303,7 @@ export async function processObservationEval(
traceId: executionParams.job.jobInputTraceId,
observationId: executionParams.job.jobInputObservationId,
environment: executionParams.environment,
+ executionType: params.executionType,
deps: executionParams.deps,
result: executionResult,
});