Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(web): make evaluator create idempotent for client-generated ids (#16956)

cursor[bot]483e0bcf5ee1
diff --git a/web/src/__tests__/server/evaluator-v2-service.servertest.ts b/web/src/__tests__/server/evaluator-v2-service.servertest.ts
index 1280b1e998..b003970824 100644
--- a/web/src/__tests__/server/evaluator-v2-service.servertest.ts
+++ b/web/src/__tests__/server/evaluator-v2-service.servertest.ts
@@ -302,6 +302,77 @@ describe("EvaluatorService", () => {
);
});
+ it("returns the existing evaluator when create is retried with the same id and content", async () => {
+ const audit = vi.fn();
+ const service = new EvaluatorService(prisma, audit);
+ const evaluatorId = crypto.randomUUID();
+ const input = { ...llmInput("Retry create"), evaluatorId };
+ const created = await service.create(input, null);
+
+ const retried = await service.create(input, null);
+
+ expect(retried).toMatchObject({
+ id: created.id,
+ projectId,
+ name: "Retry create",
+ });
+ expect(audit).toHaveBeenCalledTimes(1);
+ expect(mocks.invalidateProjectEvalConfigCaches).toHaveBeenCalledTimes(1);
+ await expect(
+ prisma.evaluator.count({ where: { id: evaluatorId } }),
+ ).resolves.toBe(1);
+ });
+
+ it.each([
+ ["name", (input: CreateEvaluatorInput) => ({ ...input, name: "Changed" })],
+ [
+ "description",
+ (input: CreateEvaluatorInput) => ({
+ ...input,
+ description: "Changed description",
+ }),
+ ],
+ [
+ "definition",
+ (input: CreateEvaluatorInput) => ({
+ ...input,
+ definition: {
+ ...input.definition,
+ promptMessages: [{ role: "user" as const, content: "Changed" }],
+ },
+ }),
+ ],
+ ])("rejects a same-project retry with changed %s", async (_field, change) => {
+ const service = createService();
+ const evaluatorId = crypto.randomUUID();
+ const input = { ...llmInput("Retry create"), evaluatorId };
+ await service.create(input, null);
+
+ await expect(service.create(change(input), null)).rejects.toThrow(
+ "An evaluator with this id already exists",
+ );
+ });
+
+ it("rejects a client id that already exists in another project", async () => {
+ const service = createService();
+ const evaluatorId = crypto.randomUUID();
+ await service.create(
+ {
+ ...llmInput("Other project evaluator"),
+ projectId: otherProjectId,
+ evaluatorId,
+ },
+ null,
+ );
+
+ await expect(
+ service.create(
+ { ...llmInput("This project evaluator"), evaluatorId },
+ null,
+ ),
+ ).rejects.toThrow("An evaluator with this id already exists");
+ });
+
it("writes the canonical mapping when creating a code evaluator", async () => {
const created = await createService().create(
{
diff --git a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx
index d76f622c9d..02a44a2fe6 100644
--- a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx
+++ b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx
@@ -237,6 +237,9 @@ export function EvaluatorSetupPage(
);
const [rawResultOpen, setRawResultOpen] = useState(false);
const hasRequestedName = useRef(false);
+ const saveInFlightRef = useRef(false);
+ const hasCreatedRef = useRef(false);
+ const [saveInFlight, setSaveInFlight] = useState(false);
const [historyOpen, setHistoryOpen] = useState(false);
const [deleteOpen, setDeleteOpen] = useState(false);
const [discardOpen, setDiscardOpen] = useState(false);
@@ -444,6 +447,9 @@ export function EvaluatorSetupPage(
};
const save = async () => {
+ if (saveInFlightRef.current || hasCreatedRef.current) return;
+ saveInFlightRef.current = true;
+ setSaveInFlight(true);
try {
let state = evaluatorSetupStore.getState();
const metadata = await prepareEvaluatorMetadataForSave({
@@ -526,6 +532,7 @@ export function EvaluatorSetupPage(
description,
definition,
});
+ hasCreatedRef.current = true;
capture("evaluators:create", {
...getEvaluatorCreationAnalyticsProperties({
evaluatorType: state.type,
@@ -572,6 +579,7 @@ export function EvaluatorSetupPage(
testRunCostUsd: lastTestRunCostUsd,
});
} catch (error) {
+ hasCreatedRef.current = false;
if (
initialEvaluator &&
error instanceof TRPCClientError &&
@@ -581,6 +589,9 @@ export function EvaluatorSetupPage(
} else {
trpcErrorToast(error);
}
+ } finally {
+ saveInFlightRef.current = false;
+ setSaveInFlight(false);
}
};
@@ -794,6 +805,7 @@ export function EvaluatorSetupPage(
initialSnapshot={initialSnapshot.current}
isEditing={Boolean(initialEvaluator)}
isSaving={
+ saveInFlight ||
create.isPending ||
update.isPending ||
suggestName.isPending ||
diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorService.ts b/web/src/features/evals/v2/server/evaluators/evaluatorService.ts
index 253729e508..ad48e754c2 100644
--- a/web/src/features/evals/v2/server/evaluators/evaluatorService.ts
+++ b/web/src/features/evals/v2/server/evaluators/evaluatorService.ts
@@ -282,8 +282,8 @@ export class EvaluatorService {
async create(input: CreateEvaluatorInput, createdByUserId: string | null) {
const block = await validateEvaluatorForPersistence(input);
- const evaluator = await this.prisma
- .$transaction((prisma) =>
+ try {
+ const evaluator = await this.prisma.$transaction((prisma) =>
repository.createEvaluator({
prisma,
input: {
@@ -295,28 +295,48 @@ export class EvaluatorService {
createdByUserId,
block,
}),
- )
- .catch((error) => {
- // Callers may pre-generate the id so test runs can be attributed
- // before the first save. Ids are globally unique, so a collision with
- // another project must not surface as an unhandled 500.
- if (
- error instanceof Prisma.PrismaClientKnownRequestError &&
- error.code === "P2002"
- ) {
- throw new LangfuseConflictError(
- "An evaluator with this id already exists",
- );
- }
- throw error;
+ );
+ await invalidateProjectEvalConfigCaches(input.projectId);
+ await this.audit({
+ action: "create",
+ projectId: input.projectId,
+ evaluatorId: evaluator.id,
});
- await invalidateProjectEvalConfigCaches(input.projectId);
- await this.audit({
- action: "create",
- projectId: input.projectId,
- evaluatorId: evaluator.id,
- });
- return normalizeEvaluatorPromptMessages(evaluator);
+ return normalizeEvaluatorPromptMessages(evaluator);
+ } catch (error) {
+ // Callers may pre-generate the id so test runs can be attributed
+ // before the first save. An exact retry returns the existing evaluator.
+ // Reusing the id with different content remains a conflict.
+ if (
+ error instanceof Prisma.PrismaClientKnownRequestError &&
+ error.code === "P2002"
+ ) {
+ if (input.evaluatorId) {
+ const existing = await repository.findEvaluator({
+ prisma: this.prisma,
+ projectId: input.projectId,
+ evaluatorId: input.evaluatorId,
+ });
+ const latest = existing?.versions[0];
+ if (
+ existing &&
+ latest &&
+ existing.name === input.name &&
+ existing.description === input.description &&
+ isDeepStrictEqual(
+ toEvaluatorDefinition(existing.type, latest),
+ input.definition,
+ )
+ ) {
+ return normalizeEvaluatorPromptMessages(existing);
+ }
+ }
+ throw new LangfuseConflictError(
+ "An evaluator with this id already exists",
+ );
+ }
+ throw error;
+ }
}
// Temporary fallback for the unstable Evaluators API until the final API