Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(evals): make trace eval job dedup atomic (#16562)

Tobias Wochinger35362460f4f0
diff --git a/worker/src/__tests__/evalService.test.ts b/worker/src/__tests__/evalService.test.ts
index 377c39c62d..3236c2c814 100644
--- a/worker/src/__tests__/evalService.test.ts
+++ b/worker/src/__tests__/evalService.test.ts
@@ -43,7 +43,10 @@ vi.mock("@langfuse/shared/src/server", async () => {
});
// Import the mocked function
-import { generateLLMText } from "@langfuse/shared/src/server";
+import {
+ EvalExecutionQueue,
+ generateLLMText,
+} from "@langfuse/shared/src/server";
import { UnrecoverableError } from "../errors/UnrecoverableError";
let OPENAI_API_KEY = process.env.OPENAI_API_KEY;
@@ -751,6 +754,116 @@ Respond with JSON: {"score": <number>, "reasoning": "<explanation>"}`;
expect(jobs[0].startTime).not.toBeNull();
}, 10_000);
+ test("creates a single eval job when two producers race for the same trace", async () => {
+ const { projectId } = await createOrgProjectAndApiKey();
+ const traceId = randomUUID();
+
+ await upsertTrace({
+ id: traceId,
+ project_id: projectId,
+ timestamp: convertDateToClickhouseDateTime(new Date()),
+ created_at: convertDateToClickhouseDateTime(new Date()),
+ updated_at: convertDateToClickhouseDateTime(new Date()),
+ });
+
+ await createMigratedEvalConfig({
+ data: {
+ id: randomUUID(),
+ projectId,
+ filter: JSON.parse("[]"),
+ jobType: "EVAL",
+ delay: 0,
+ sampling: new Decimal("1"),
+ targetObject: EvalTargetObject.TRACE,
+ scoreName: "score",
+ variableMapping: JSON.parse("[]"),
+ },
+ });
+
+ const payload = { projectId, traceId };
+
+ // Two shard workers observing the same trace concurrently. Both pass the
+ // dedup existence check before either inserts.
+ await Promise.all([
+ createEvalJobs({
+ sourceEventType: "trace-upsert",
+ event: payload,
+ jobTimestamp,
+ }),
+ createEvalJobs({
+ sourceEventType: "trace-upsert",
+ event: payload,
+ jobTimestamp,
+ }),
+ ]);
+
+ const jobs = await prisma.jobExecution.findMany({ where: { projectId } });
+
+ expect(jobs.length).toBe(1);
+ }, 10_000);
+
+ test("does not leave a PENDING eval job behind when enqueueing fails", async () => {
+ const { projectId } = await createOrgProjectAndApiKey();
+ const traceId = randomUUID();
+
+ await upsertTrace({
+ id: traceId,
+ project_id: projectId,
+ timestamp: convertDateToClickhouseDateTime(new Date()),
+ created_at: convertDateToClickhouseDateTime(new Date()),
+ updated_at: convertDateToClickhouseDateTime(new Date()),
+ });
+
+ await createMigratedEvalConfig({
+ data: {
+ id: randomUUID(),
+ projectId,
+ filter: JSON.parse("[]"),
+ jobType: "EVAL",
+ delay: 0,
+ sampling: new Decimal("1"),
+ targetObject: EvalTargetObject.TRACE,
+ scoreName: "score",
+ variableMapping: JSON.parse("[]"),
+ },
+ });
+
+ const getInstanceSpy = vi
+ .spyOn(EvalExecutionQueue, "getInstance")
+ .mockReturnValue({
+ add: vi.fn().mockRejectedValue(new Error("redis is down")),
+ } as never);
+
+ try {
+ await expect(
+ createEvalJobs({
+ sourceEventType: "trace-upsert",
+ event: { projectId, traceId },
+ jobTimestamp,
+ }),
+ ).rejects.toThrow("redis is down");
+ } finally {
+ getInstanceSpy.mockRestore();
+ }
+
+ // The row must be gone, otherwise the BullMQ redelivery hits the dedup
+ // check and the execution stays PENDING forever.
+ await expect(
+ prisma.jobExecution.count({ where: { projectId } }),
+ ).resolves.toBe(0);
+
+ // The retry recreates and enqueues it.
+ await createEvalJobs({
+ sourceEventType: "trace-upsert",
+ event: { projectId, traceId },
+ jobTimestamp,
+ });
+
+ await expect(
+ prisma.jobExecution.count({ where: { projectId } }),
+ ).resolves.toBe(1);
+ }, 10_000);
+
test("creates new 'dataset' eval job", async () => {
const { projectId } = await createOrgProjectAndApiKey();
const traceId = randomUUID();
diff --git a/worker/src/features/evaluation/evalService.ts b/worker/src/features/evaluation/evalService.ts
index 6a770ccae3..aa41bad1b2 100644
--- a/worker/src/features/evaluation/evalService.ts
+++ b/worker/src/features/evaluation/evalService.ts
@@ -266,6 +266,36 @@ function toTraceEvalConfig(rule: TraceRule): TraceEvalConfig | null {
};
}
+/**
+ * Stable id for a trace/dataset eval execution, derived from the same key the
+ * dedup lookup uses.
+ *
+ * Two producers (trace-upsert shards, dataset-run-item-upsert, CreateEvalQueue)
+ * can observe the same trace concurrently and both pass the read-then-write
+ * existence check. A random id lets both inserts succeed, which doubles LLM
+ * spend and — because score ids are derived from the job execution id — shows
+ * up as duplicate scores. Deriving the id from the dedup key lets the primary
+ * key reject the loser instead.
+ */
+function createDeterministicJobExecutionId(params: {
+ projectId: string;
+ configId: string;
+ traceId: string;
+ datasetItemId: string | null;
+ observationId: string | null;
+}): string {
+ return createW3CTraceId(
+ JSON.stringify([
+ "trace-eval",
+ params.projectId,
+ params.configId,
+ params.traceId,
+ params.datasetItemId,
+ params.observationId,
+ ]),
+ );
+}
+
export const createEvalJobs = async ({
event,
sourceEventType,
@@ -728,7 +758,18 @@ export const createEvalJobs = async ({
// If we matched a trace for a trace event, we create a job or
// if we have both trace and datasetItem.
if (traceExists && (!isDatasetConfig || Boolean(datasetItem))) {
- const jobExecutionId = randomUUID();
+ // Derive the id from the dedup key instead of randomising it, so two
+ // producers racing on the same (config, trace, dataset item, observation)
+ // compute the same primary key. The insert below then relies on the
+ // primary key to reject the loser atomically, which the read-then-write
+ // existence check above cannot do on its own.
+ const jobExecutionId = createDeterministicJobExecutionId({
+ projectId: event.projectId,
+ configId: config.id,
+ traceId: event.traceId,
+ datasetItemId: datasetItem?.id ?? null,
+ observationId: observationId ?? null,
+ });
// deduplication: if a job exists already for a trace event, we do not create a new one.
if (existingJob.length > 0) {
@@ -755,56 +796,88 @@ export const createEvalJobs = async ({
`Creating eval job execution for config ${config.id} and trace ${event.traceId}`,
);
- await prisma.jobExecution.create({
- data: {
- id: jobExecutionId,
- projectId: event.projectId,
- jobConfigurationId: config.id,
- jobInputTraceId: event.traceId,
- jobInputTraceTimestamp: traceTimestamp,
- jobTemplateId: config.evalTemplateId,
- status: "PENDING",
- startTime: new Date(),
- ...(datasetItem
- ? {
- jobInputDatasetItemId: datasetItem.id,
- ...("validFrom" in datasetItem && {
- jobInputDatasetItemValidFrom: datasetItem.validFrom,
- }),
- jobInputObservationId: observationId || null,
- }
- : {}),
- },
- });
-
- // add the job to the next queue so that eval can be executed
- const shardingKey = `${event.projectId}-${jobExecutionId}`;
- await EvalExecutionQueue.getInstance({ shardingKey })?.add(
- QueueName.EvaluationExecution,
- {
- name: QueueJobs.EvaluationExecution,
- id: randomUUID(),
- timestamp: new Date(),
- payload: {
+ // `createMany` with `skipDuplicates` turns the insert into an
+ // INSERT ... ON CONFLICT DO NOTHING on the primary key. The racing loser
+ // gets `count: 0` and must not enqueue, so one execution stays one
+ // execution (and one score, since score ids derive from this id).
+ const { count: insertedCount } = await prisma.jobExecution.createMany({
+ data: [
+ {
+ id: jobExecutionId,
projectId: event.projectId,
- jobExecutionId: jobExecutionId,
- delay: config.delay,
- ...(config.evaluatorId
+ jobConfigurationId: config.id,
+ jobInputTraceId: event.traceId,
+ jobInputTraceTimestamp: traceTimestamp,
+ jobTemplateId: config.evalTemplateId,
+ status: "PENDING",
+ startTime: new Date(),
+ ...(datasetItem
? {
- evaluatorId: config.evaluatorId,
- evaluationRuleId: config.evaluationRuleId,
+ jobInputDatasetItemId: datasetItem.id,
+ ...("validFrom" in datasetItem && {
+ jobInputDatasetItemValidFrom: datasetItem.validFrom,
+ }),
+ jobInputObservationId: observationId || null,
}
: {}),
},
- retryBaggage: {
- originalJobTimestamp: new Date(),
- attempt: 0,
+ ],
+ skipDuplicates: true,
+ });
+
+ if (insertedCount === 0) {
+ logger.debug(
+ `Concurrent producer already created eval job ${jobExecutionId} for config ${config.id} and trace ${event.traceId}`,
+ );
+ continue;
+ }
+
+ try {
+ // add the job to the next queue so that eval can be executed
+ const shardingKey = `${event.projectId}-${jobExecutionId}`;
+ await EvalExecutionQueue.getInstance({ shardingKey })?.add(
+ QueueName.EvaluationExecution,
+ {
+ name: QueueJobs.EvaluationExecution,
+ id: randomUUID(),
+ timestamp: new Date(),
+ payload: {
+ projectId: event.projectId,
+ jobExecutionId: jobExecutionId,
+ delay: config.delay,
+ ...(config.evaluatorId
+ ? {
+ evaluatorId: config.evaluatorId,
+ evaluationRuleId: config.evaluationRuleId,
+ }
+ : {}),
+ },
+ retryBaggage: {
+ originalJobTimestamp: new Date(),
+ attempt: 0,
+ },
},
- },
- {
- delay: config.delay, // milliseconds
- },
- );
+ {
+ delay: config.delay, // milliseconds
+ },
+ );
+ } catch (e) {
+ // The row exists but nothing will ever pick it up. Without this
+ // compensating delete the BullMQ redelivery would hit the dedup check
+ // above and skip re-enqueueing, stranding the execution at PENDING.
+ logger.warn(
+ `Failed to enqueue eval execution ${jobExecutionId}, removing the orphaned job execution so the retry can recreate it`,
+ e,
+ );
+ await prisma.jobExecution.deleteMany({
+ where: {
+ id: jobExecutionId,
+ projectId: event.projectId,
+ status: JobExecutionStatus.PENDING,
+ },
+ });
+ throw e;
+ }
} else {
// if we do not have a match, and execution exists, we mark the job as cancelled
// we do this, because a second trace event might 'deselect' a trace