🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(evals): make trace eval job dedup atomic (#16562)
Tobias Wochinger35362460f4f0
diff --git a/worker/src/__tests__/evalService.test.ts b/worker/src/__tests__/evalService.test.tsindex 377c39c62d..3236c2c814 100644--- a/worker/src/__tests__/evalService.test.ts+++ b/worker/src/__tests__/evalService.test.ts@@ -43,7 +43,10 @@ vi.mock("@langfuse/shared/src/server", async () => {});// Import the mocked function-import { generateLLMText } from "@langfuse/shared/src/server";+import {+ EvalExecutionQueue,+ generateLLMText,+} from "@langfuse/shared/src/server";import { UnrecoverableError } from "../errors/UnrecoverableError";let OPENAI_API_KEY = process.env.OPENAI_API_KEY;@@ -751,6 +754,116 @@ Respond with JSON: {"score": <number>, "reasoning": "<explanation>"}`;expect(jobs[0].startTime).not.toBeNull();}, 10_000);+ test("creates a single eval job when two producers race for the same trace", async () => {+ const { projectId } = await createOrgProjectAndApiKey();+ const traceId = randomUUID();++ await upsertTrace({+ id: traceId,+ project_id: projectId,+ timestamp: convertDateToClickhouseDateTime(new Date()),+ created_at: convertDateToClickhouseDateTime(new Date()),+ updated_at: convertDateToClickhouseDateTime(new Date()),+ });++ await createMigratedEvalConfig({+ data: {+ id: randomUUID(),+ projectId,+ filter: JSON.parse("[]"),+ jobType: "EVAL",+ delay: 0,+ sampling: new Decimal("1"),+ targetObject: EvalTargetObject.TRACE,+ scoreName: "score",+ variableMapping: JSON.parse("[]"),+ },+ });++ const payload = { projectId, traceId };++ // Two shard workers observing the same trace concurrently. Both pass the+ // dedup existence check before either inserts.+ await Promise.all([+ createEvalJobs({+ sourceEventType: "trace-upsert",+ event: payload,+ jobTimestamp,+ }),+ createEvalJobs({+ sourceEventType: "trace-upsert",+ event: payload,+ jobTimestamp,+ }),+ ]);++ const jobs = await prisma.jobExecution.findMany({ where: { projectId } });++ expect(jobs.length).toBe(1);+ }, 10_000);++ test("does not leave a PENDING eval job behind when enqueueing fails", async () => {+ const { projectId } = await createOrgProjectAndApiKey();+ const traceId = randomUUID();++ await upsertTrace({+ id: traceId,+ project_id: projectId,+ timestamp: convertDateToClickhouseDateTime(new Date()),+ created_at: convertDateToClickhouseDateTime(new Date()),+ updated_at: convertDateToClickhouseDateTime(new Date()),+ });++ await createMigratedEvalConfig({+ data: {+ id: randomUUID(),+ projectId,+ filter: JSON.parse("[]"),+ jobType: "EVAL",+ delay: 0,+ sampling: new Decimal("1"),+ targetObject: EvalTargetObject.TRACE,+ scoreName: "score",+ variableMapping: JSON.parse("[]"),+ },+ });++ const getInstanceSpy = vi+ .spyOn(EvalExecutionQueue, "getInstance")+ .mockReturnValue({+ add: vi.fn().mockRejectedValue(new Error("redis is down")),+ } as never);++ try {+ await expect(+ createEvalJobs({+ sourceEventType: "trace-upsert",+ event: { projectId, traceId },+ jobTimestamp,+ }),+ ).rejects.toThrow("redis is down");+ } finally {+ getInstanceSpy.mockRestore();+ }++ // The row must be gone, otherwise the BullMQ redelivery hits the dedup+ // check and the execution stays PENDING forever.+ await expect(+ prisma.jobExecution.count({ where: { projectId } }),+ ).resolves.toBe(0);++ // The retry recreates and enqueues it.+ await createEvalJobs({+ sourceEventType: "trace-upsert",+ event: { projectId, traceId },+ jobTimestamp,+ });++ await expect(+ prisma.jobExecution.count({ where: { projectId } }),+ ).resolves.toBe(1);+ }, 10_000);+test("creates new 'dataset' eval job", async () => {const { projectId } = await createOrgProjectAndApiKey();const traceId = randomUUID();diff --git a/worker/src/features/evaluation/evalService.ts b/worker/src/features/evaluation/evalService.tsindex 6a770ccae3..aa41bad1b2 100644--- a/worker/src/features/evaluation/evalService.ts+++ b/worker/src/features/evaluation/evalService.ts@@ -266,6 +266,36 @@ function toTraceEvalConfig(rule: TraceRule): TraceEvalConfig | null {};}+/**+ * Stable id for a trace/dataset eval execution, derived from the same key the+ * dedup lookup uses.+ *+ * Two producers (trace-upsert shards, dataset-run-item-upsert, CreateEvalQueue)+ * can observe the same trace concurrently and both pass the read-then-write+ * existence check. A random id lets both inserts succeed, which doubles LLM+ * spend and — because score ids are derived from the job execution id — shows+ * up as duplicate scores. Deriving the id from the dedup key lets the primary+ * key reject the loser instead.+ */+function createDeterministicJobExecutionId(params: {+ projectId: string;+ configId: string;+ traceId: string;+ datasetItemId: string | null;+ observationId: string | null;+}): string {+ return createW3CTraceId(+ JSON.stringify([+ "trace-eval",+ params.projectId,+ params.configId,+ params.traceId,+ params.datasetItemId,+ params.observationId,+ ]),+ );+}+export const createEvalJobs = async ({event,sourceEventType,@@ -728,7 +758,18 @@ export const createEvalJobs = async ({// If we matched a trace for a trace event, we create a job or// if we have both trace and datasetItem.if (traceExists && (!isDatasetConfig || Boolean(datasetItem))) {- const jobExecutionId = randomUUID();+ // Derive the id from the dedup key instead of randomising it, so two+ // producers racing on the same (config, trace, dataset item, observation)+ // compute the same primary key. The insert below then relies on the+ // primary key to reject the loser atomically, which the read-then-write+ // existence check above cannot do on its own.+ const jobExecutionId = createDeterministicJobExecutionId({+ projectId: event.projectId,+ configId: config.id,+ traceId: event.traceId,+ datasetItemId: datasetItem?.id ?? null,+ observationId: observationId ?? null,+ });// deduplication: if a job exists already for a trace event, we do not create a new one.if (existingJob.length > 0) {@@ -755,56 +796,88 @@ export const createEvalJobs = async ({`Creating eval job execution for config ${config.id} and trace ${event.traceId}`,);- await prisma.jobExecution.create({- data: {- id: jobExecutionId,- projectId: event.projectId,- jobConfigurationId: config.id,- jobInputTraceId: event.traceId,- jobInputTraceTimestamp: traceTimestamp,- jobTemplateId: config.evalTemplateId,- status: "PENDING",- startTime: new Date(),- ...(datasetItem- ? {- jobInputDatasetItemId: datasetItem.id,- ...("validFrom" in datasetItem && {- jobInputDatasetItemValidFrom: datasetItem.validFrom,- }),- jobInputObservationId: observationId || null,- }- : {}),- },- });-- // add the job to the next queue so that eval can be executed- const shardingKey = `${event.projectId}-${jobExecutionId}`;- await EvalExecutionQueue.getInstance({ shardingKey })?.add(- QueueName.EvaluationExecution,- {- name: QueueJobs.EvaluationExecution,- id: randomUUID(),- timestamp: new Date(),- payload: {+ // `createMany` with `skipDuplicates` turns the insert into an+ // INSERT ... ON CONFLICT DO NOTHING on the primary key. The racing loser+ // gets `count: 0` and must not enqueue, so one execution stays one+ // execution (and one score, since score ids derive from this id).+ const { count: insertedCount } = await prisma.jobExecution.createMany({+ data: [+ {+ id: jobExecutionId,projectId: event.projectId,- jobExecutionId: jobExecutionId,- delay: config.delay,- ...(config.evaluatorId+ jobConfigurationId: config.id,+ jobInputTraceId: event.traceId,+ jobInputTraceTimestamp: traceTimestamp,+ jobTemplateId: config.evalTemplateId,+ status: "PENDING",+ startTime: new Date(),+ ...(datasetItem? {- evaluatorId: config.evaluatorId,- evaluationRuleId: config.evaluationRuleId,+ jobInputDatasetItemId: datasetItem.id,+ ...("validFrom" in datasetItem && {+ jobInputDatasetItemValidFrom: datasetItem.validFrom,+ }),+ jobInputObservationId: observationId || null,}: {}),},- retryBaggage: {- originalJobTimestamp: new Date(),- attempt: 0,+ ],+ skipDuplicates: true,+ });++ if (insertedCount === 0) {+ logger.debug(+ `Concurrent producer already created eval job ${jobExecutionId} for config ${config.id} and trace ${event.traceId}`,+ );+ continue;+ }++ try {+ // add the job to the next queue so that eval can be executed+ const shardingKey = `${event.projectId}-${jobExecutionId}`;+ await EvalExecutionQueue.getInstance({ shardingKey })?.add(+ QueueName.EvaluationExecution,+ {+ name: QueueJobs.EvaluationExecution,+ id: randomUUID(),+ timestamp: new Date(),+ payload: {+ projectId: event.projectId,+ jobExecutionId: jobExecutionId,+ delay: config.delay,+ ...(config.evaluatorId+ ? {+ evaluatorId: config.evaluatorId,+ evaluationRuleId: config.evaluationRuleId,+ }+ : {}),+ },+ retryBaggage: {+ originalJobTimestamp: new Date(),+ attempt: 0,+ },},- },- {- delay: config.delay, // milliseconds- },- );+ {+ delay: config.delay, // milliseconds+ },+ );+ } catch (e) {+ // The row exists but nothing will ever pick it up. Without this+ // compensating delete the BullMQ redelivery would hit the dedup check+ // above and skip re-enqueueing, stranding the execution at PENDING.+ logger.warn(+ `Failed to enqueue eval execution ${jobExecutionId}, removing the orphaned job execution so the retry can recreate it`,+ e,+ );+ await prisma.jobExecution.deleteMany({+ where: {+ id: jobExecutionId,+ projectId: event.projectId,+ status: JobExecutionStatus.PENDING,+ },+ });+ throw e;+ }} else {// if we do not have a match, and execution exists, we mark the job as cancelled// we do this, because a second trace event might 'deselect' a trace