🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(web): make evaluator create idempotent for client-generated ids (#16956)
cursor[bot]483e0bcf5ee1
diff --git a/web/src/__tests__/server/evaluator-v2-service.servertest.ts b/web/src/__tests__/server/evaluator-v2-service.servertest.tsindex 1280b1e998..b003970824 100644--- a/web/src/__tests__/server/evaluator-v2-service.servertest.ts+++ b/web/src/__tests__/server/evaluator-v2-service.servertest.ts@@ -302,6 +302,77 @@ describe("EvaluatorService", () => {);});+ it("returns the existing evaluator when create is retried with the same id and content", async () => {+ const audit = vi.fn();+ const service = new EvaluatorService(prisma, audit);+ const evaluatorId = crypto.randomUUID();+ const input = { ...llmInput("Retry create"), evaluatorId };+ const created = await service.create(input, null);++ const retried = await service.create(input, null);++ expect(retried).toMatchObject({+ id: created.id,+ projectId,+ name: "Retry create",+ });+ expect(audit).toHaveBeenCalledTimes(1);+ expect(mocks.invalidateProjectEvalConfigCaches).toHaveBeenCalledTimes(1);+ await expect(+ prisma.evaluator.count({ where: { id: evaluatorId } }),+ ).resolves.toBe(1);+ });++ it.each([+ ["name", (input: CreateEvaluatorInput) => ({ ...input, name: "Changed" })],+ [+ "description",+ (input: CreateEvaluatorInput) => ({+ ...input,+ description: "Changed description",+ }),+ ],+ [+ "definition",+ (input: CreateEvaluatorInput) => ({+ ...input,+ definition: {+ ...input.definition,+ promptMessages: [{ role: "user" as const, content: "Changed" }],+ },+ }),+ ],+ ])("rejects a same-project retry with changed %s", async (_field, change) => {+ const service = createService();+ const evaluatorId = crypto.randomUUID();+ const input = { ...llmInput("Retry create"), evaluatorId };+ await service.create(input, null);++ await expect(service.create(change(input), null)).rejects.toThrow(+ "An evaluator with this id already exists",+ );+ });++ it("rejects a client id that already exists in another project", async () => {+ const service = createService();+ const evaluatorId = crypto.randomUUID();+ await service.create(+ {+ ...llmInput("Other project evaluator"),+ projectId: otherProjectId,+ evaluatorId,+ },+ null,+ );++ await expect(+ service.create(+ { ...llmInput("This project evaluator"), evaluatorId },+ null,+ ),+ ).rejects.toThrow("An evaluator with this id already exists");+ });+it("writes the canonical mapping when creating a code evaluator", async () => {const created = await createService().create({diff --git a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsxindex d76f622c9d..02a44a2fe6 100644--- a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx+++ b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx@@ -237,6 +237,9 @@ export function EvaluatorSetupPage();const [rawResultOpen, setRawResultOpen] = useState(false);const hasRequestedName = useRef(false);+ const saveInFlightRef = useRef(false);+ const hasCreatedRef = useRef(false);+ const [saveInFlight, setSaveInFlight] = useState(false);const [historyOpen, setHistoryOpen] = useState(false);const [deleteOpen, setDeleteOpen] = useState(false);const [discardOpen, setDiscardOpen] = useState(false);@@ -444,6 +447,9 @@ export function EvaluatorSetupPage(};const save = async () => {+ if (saveInFlightRef.current || hasCreatedRef.current) return;+ saveInFlightRef.current = true;+ setSaveInFlight(true);try {let state = evaluatorSetupStore.getState();const metadata = await prepareEvaluatorMetadataForSave({@@ -526,6 +532,7 @@ export function EvaluatorSetupPage(description,definition,});+ hasCreatedRef.current = true;capture("evaluators:create", {...getEvaluatorCreationAnalyticsProperties({evaluatorType: state.type,@@ -572,6 +579,7 @@ export function EvaluatorSetupPage(testRunCostUsd: lastTestRunCostUsd,});} catch (error) {+ hasCreatedRef.current = false;if (initialEvaluator &&error instanceof TRPCClientError &&@@ -581,6 +589,9 @@ export function EvaluatorSetupPage(} else {trpcErrorToast(error);}+ } finally {+ saveInFlightRef.current = false;+ setSaveInFlight(false);}};@@ -794,6 +805,7 @@ export function EvaluatorSetupPage(initialSnapshot={initialSnapshot.current}isEditing={Boolean(initialEvaluator)}isSaving={+ saveInFlight ||create.isPending ||update.isPending ||suggestName.isPending ||diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorService.ts b/web/src/features/evals/v2/server/evaluators/evaluatorService.tsindex 253729e508..ad48e754c2 100644--- a/web/src/features/evals/v2/server/evaluators/evaluatorService.ts+++ b/web/src/features/evals/v2/server/evaluators/evaluatorService.ts@@ -282,8 +282,8 @@ export class EvaluatorService {async create(input: CreateEvaluatorInput, createdByUserId: string | null) {const block = await validateEvaluatorForPersistence(input);- const evaluator = await this.prisma- .$transaction((prisma) =>+ try {+ const evaluator = await this.prisma.$transaction((prisma) =>repository.createEvaluator({prisma,input: {@@ -295,28 +295,48 @@ export class EvaluatorService {createdByUserId,block,}),- )- .catch((error) => {- // Callers may pre-generate the id so test runs can be attributed- // before the first save. Ids are globally unique, so a collision with- // another project must not surface as an unhandled 500.- if (- error instanceof Prisma.PrismaClientKnownRequestError &&- error.code === "P2002"- ) {- throw new LangfuseConflictError(- "An evaluator with this id already exists",- );- }- throw error;+ );+ await invalidateProjectEvalConfigCaches(input.projectId);+ await this.audit({+ action: "create",+ projectId: input.projectId,+ evaluatorId: evaluator.id,});- await invalidateProjectEvalConfigCaches(input.projectId);- await this.audit({- action: "create",- projectId: input.projectId,- evaluatorId: evaluator.id,- });- return normalizeEvaluatorPromptMessages(evaluator);+ return normalizeEvaluatorPromptMessages(evaluator);+ } catch (error) {+ // Callers may pre-generate the id so test runs can be attributed+ // before the first save. An exact retry returns the existing evaluator.+ // Reusing the id with different content remains a conflict.+ if (+ error instanceof Prisma.PrismaClientKnownRequestError &&+ error.code === "P2002"+ ) {+ if (input.evaluatorId) {+ const existing = await repository.findEvaluator({+ prisma: this.prisma,+ projectId: input.projectId,+ evaluatorId: input.evaluatorId,+ });+ const latest = existing?.versions[0];+ if (+ existing &&+ latest &&+ existing.name === input.name &&+ existing.description === input.description &&+ isDeepStrictEqual(+ toEvaluatorDefinition(existing.type, latest),+ input.definition,+ )+ ) {+ return normalizeEvaluatorPromptMessages(existing);+ }+ }+ throw new LangfuseConflictError(+ "An evaluator with this id already exists",+ );+ }+ throw error;+ }}// Temporary fallback for the unstable Evaluators API until the final API