Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(web): return NOT_FOUND for missing model and LLM-connection deletes (#17672)

Max Deichmann88384a5ba17c
diff --git a/web/src/__tests__/server/model-and-llm-connection-deletes.servertest.ts b/web/src/__tests__/server/model-and-llm-connection-deletes.servertest.ts
new file mode 100644
index 0000000000..c61d7ffd33
--- /dev/null
+++ b/web/src/__tests__/server/model-and-llm-connection-deletes.servertest.ts
@@ -0,0 +1,88 @@
+import type { Session } from "next-auth";
+import { randomUUID } from "crypto";
+import { prisma } from "@langfuse/shared/src/db";
+import { appRouter } from "@/src/server/api/root";
+import { createInnerTRPCContext } from "@/src/server/api/trpc";
+import { createOrgProjectAndApiKey } from "@langfuse/shared/src/server";
+
+const buildSession = (orgId: string, projectId: string): Session => ({
+ expires: "1",
+ user: {
+ id: "user-1",
+ name: "Demo User",
+ canCreateOrganizations: true,
+ organizations: [
+ {
+ id: orgId,
+ role: "OWNER",
+ plan: "cloud:hobby",
+ cloudConfig: undefined,
+ name: "Test Organization",
+ metadata: {},
+ aiFeaturesEnabled: false,
+ aiTelemetryEnabled: false,
+ projects: [
+ {
+ id: projectId,
+ role: "ADMIN",
+ name: "Test Project",
+ deletedAt: null,
+ retentionDays: null,
+ hasTraces: false,
+ metadata: {},
+ createdAt: new Date().toISOString(),
+ },
+ ],
+ },
+ ],
+ featureFlags: {
+ searchBar: false,
+ templateFlag: true,
+ excludeClickhouseRead: false,
+ observationEvals: false,
+ v4BetaToggleVisible: false,
+ experimentsV4Enabled: false,
+ },
+ admin: true,
+ },
+ environment: {} as any,
+});
+
+describe("missing-record deletes map to NOT_FOUND", () => {
+ let projectId: string;
+ let caller: ReturnType<typeof appRouter.createCaller>;
+
+ beforeEach(async () => {
+ const setup = await createOrgProjectAndApiKey();
+ projectId = setup.projectId;
+ const ctx = createInnerTRPCContext({
+ session: buildSession(setup.orgId, setup.projectId),
+ headers: {},
+ });
+ caller = appRouter.createCaller({ ...ctx, prisma });
+ });
+
+ it("returns NOT_FOUND when deleting a model that is already gone", async () => {
+ await expect(
+ caller.models.delete({
+ projectId,
+ modelId: randomUUID(),
+ }),
+ ).rejects.toMatchObject({ code: "NOT_FOUND" });
+ });
+
+ it("returns NOT_FOUND when deleting an LLM API key that is already gone", async () => {
+ await expect(
+ caller.llmApiKey.delete({
+ projectId,
+ id: randomUUID(),
+ }),
+ ).rejects.toMatchObject({ code: "NOT_FOUND" });
+ });
+
+ it("returns NOT_FOUND when deleting a default eval model that is already gone", async () => {
+ await expect(
+ caller.defaultLlmModel.deleteDefaultModel({ projectId }),
+ ).rejects.toMatchObject({ code: "NOT_FOUND" });
+ });
+});
diff --git a/web/src/features/evals/server/defaultEvalModelRouter.ts b/web/src/features/evals/server/defaultEvalModelRouter.ts
index 0fd07faa1c..8683e5f5da 100644
--- a/web/src/features/evals/server/defaultEvalModelRouter.ts
+++ b/web/src/features/evals/server/defaultEvalModelRouter.ts
@@ -1,3 +1,4 @@
+import { isPrismaRecordNotFoundError } from "@/src/features/analytics-integrations/server";
import { throwIfNoProjectAccess } from "@/src/features/rbac";
import {
createTRPCRouter,
@@ -5,7 +6,11 @@ import {
} from "@/src/server/api/trpc";
import { z } from "zod";
-import { EvaluatorBlockReason, ZodModelConfig } from "@langfuse/shared";
+import {
+ EvaluatorBlockReason,
+ LangfuseNotFoundError,
+ ZodModelConfig,
+} from "@langfuse/shared";
import {
blockEvaluatorsUsingDefaultModel,
DefaultEvalModelService,
@@ -68,22 +73,30 @@ export const defaultEvalModelRouter = createTRPCRouter({
scope: "evalDefaultModel:CUD",
});
- const result = await ctx.prisma.$transaction(async (tx) => {
- const blockResult = await blockEvaluatorsUsingDefaultModel({
- tx,
- projectId: input.projectId,
- });
-
- // Delete the default model within the transaction
- await tx.defaultLlmModel.delete({
- // unique constraint on projectId
- where: {
+ let result;
+ try {
+ result = await ctx.prisma.$transaction(async (tx) => {
+ const blockResult = await blockEvaluatorsUsingDefaultModel({
+ tx,
projectId: input.projectId,
- },
- });
+ });
- return blockResult;
- });
+ // Delete the default model within the transaction
+ await tx.defaultLlmModel.delete({
+ // unique constraint on projectId
+ where: {
+ projectId: input.projectId,
+ },
+ });
+
+ return blockResult;
+ });
+ } catch (error) {
+ if (isPrismaRecordNotFoundError(error)) {
+ throw new LangfuseNotFoundError("Default evaluation model not found");
+ }
+ throw error;
+ }
await finalizeEvaluatorBlocks({
projectId: input.projectId,
diff --git a/web/src/features/llm-api-key/server/router.ts b/web/src/features/llm-api-key/server/router.ts
index 5533c36693..d88ed1ace4 100644
--- a/web/src/features/llm-api-key/server/router.ts
+++ b/web/src/features/llm-api-key/server/router.ts
@@ -8,6 +8,7 @@ import {
SafeLlmApiKeySchema,
type BedrockAuthMethod,
} from "@/src/features/llm-api-key/types";
+import { isPrismaRecordNotFoundError } from "@/src/features/analytics-integrations/server";
import { throwIfNoProjectAccess } from "@/src/features/rbac";
import {
createTRPCRouter,
@@ -26,6 +27,7 @@ import {
BEDROCK_USE_DEFAULT_CREDENTIALS,
VERTEXAI_USE_DEFAULT_CREDENTIALS,
EvaluatorBlockReason,
+ LangfuseNotFoundError,
type LLMConnectionConfig,
} from "@langfuse/shared";
@@ -418,50 +420,62 @@ export const llmApiKeyRouter = createTRPCRouter({
},
});
- const result = await ctx.prisma.$transaction(async (tx) => {
- // Check if the llm api key is used for the default evaluation model
- const defaultModel = await tx.defaultLlmModel.findFirst({
- where: {
- projectId: input.projectId,
- },
- select: {
- llmApiKeyId: true,
- },
- });
+ if (!llmApiKey) {
+ throw new LangfuseNotFoundError("LLM API key not found");
+ }
- const providerBlock = llmApiKey?.provider
- ? await blockEvaluatorsUsingProvider({
- tx,
+ let result;
+ try {
+ result = await ctx.prisma.$transaction(async (tx) => {
+ // Check if the llm api key is used for the default evaluation model
+ const defaultModel = await tx.defaultLlmModel.findFirst({
+ where: {
projectId: input.projectId,
- provider: llmApiKey.provider,
- })
- : EMPTY_EVALUATOR_BLOCK;
+ },
+ select: {
+ llmApiKeyId: true,
+ },
+ });
- const defaultModelBlock =
- !!defaultModel && defaultModel.llmApiKeyId === llmApiKey?.id
- ? await blockEvaluatorsUsingDefaultModel({
+ const providerBlock = llmApiKey.provider
+ ? await blockEvaluatorsUsingProvider({
tx,
projectId: input.projectId,
+ provider: llmApiKey.provider,
})
: EMPTY_EVALUATOR_BLOCK;
- await tx.llmApiKeys.delete({
- where: {
- id: input.id,
- projectId: input.projectId,
- },
- });
+ const defaultModelBlock =
+ !!defaultModel && defaultModel.llmApiKeyId === llmApiKey.id
+ ? await blockEvaluatorsUsingDefaultModel({
+ tx,
+ projectId: input.projectId,
+ })
+ : EMPTY_EVALUATOR_BLOCK;
+
+ await tx.llmApiKeys.delete({
+ where: {
+ id: input.id,
+ projectId: input.projectId,
+ },
+ });
- await auditLog({
- session: ctx.session,
- resourceType: "llmApiKey",
- resourceId: input.id,
- before: llmApiKey,
- action: "delete",
- });
+ await auditLog({
+ session: ctx.session,
+ resourceType: "llmApiKey",
+ resourceId: input.id,
+ before: llmApiKey,
+ action: "delete",
+ });
- return { providerBlock, defaultModelBlock };
- });
+ return { providerBlock, defaultModelBlock };
+ });
+ } catch (error) {
+ if (isPrismaRecordNotFoundError(error)) {
+ throw new LangfuseNotFoundError("LLM API key not found");
+ }
+ throw error;
+ }
await finalizeEvaluatorBlocks({
projectId: input.projectId,
diff --git a/web/src/server/api/routers/models.ts b/web/src/server/api/routers/models.ts
index e7da28fc3f..232aab6993 100644
--- a/web/src/server/api/routers/models.ts
+++ b/web/src/server/api/routers/models.ts
@@ -1,6 +1,7 @@
import { v4 as uuidv4 } from "uuid";
import { z } from "zod";
+import { isPrismaRecordNotFoundError } from "@/src/features/analytics-integrations/server";
import { auditLog } from "@/src/features/audit-logs/server";
import { isValidPostgresRegex } from "@/src/features/models/server/isValidPostgresRegex";
import {
@@ -13,7 +14,12 @@ import {
createTRPCRouter,
protectedProjectProcedure,
} from "@/src/server/api/trpc";
-import { ModelUsageUnit, paginationZod, Prisma } from "@langfuse/shared";
+import {
+ LangfuseNotFoundError,
+ ModelUsageUnit,
+ paginationZod,
+ Prisma,
+} from "@langfuse/shared";
import {
clearModelCacheForProject,
queryClickhouse,
@@ -390,12 +396,20 @@ export const modelRouter = createTRPCRouter({
scope: "models:CUD",
});
- const deletedModel = await ctx.prisma.model.delete({
- where: {
- id: input.modelId,
- projectId: input.projectId,
- },
- });
+ let deletedModel;
+ try {
+ deletedModel = await ctx.prisma.model.delete({
+ where: {
+ id: input.modelId,
+ projectId: input.projectId,
+ },
+ });
+ } catch (error) {
+ if (isPrismaRecordNotFoundError(error)) {
+ throw new LangfuseNotFoundError("Model not found");
+ }
+ throw error;
+ }
await auditLog({
session: ctx.session,