🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(web): return NOT_FOUND for missing model and LLM-connection deletes (#17672)
Max Deichmann88384a5ba17c
diff --git a/web/src/__tests__/server/model-and-llm-connection-deletes.servertest.ts b/web/src/__tests__/server/model-and-llm-connection-deletes.servertest.tsnew file mode 100644index 0000000000..c61d7ffd33--- /dev/null+++ b/web/src/__tests__/server/model-and-llm-connection-deletes.servertest.ts@@ -0,0 +1,88 @@+import type { Session } from "next-auth";+import { randomUUID } from "crypto";+import { prisma } from "@langfuse/shared/src/db";+import { appRouter } from "@/src/server/api/root";+import { createInnerTRPCContext } from "@/src/server/api/trpc";+import { createOrgProjectAndApiKey } from "@langfuse/shared/src/server";++const buildSession = (orgId: string, projectId: string): Session => ({+ expires: "1",+ user: {+ id: "user-1",+ name: "Demo User",+ canCreateOrganizations: true,+ organizations: [+ {+ id: orgId,+ role: "OWNER",+ plan: "cloud:hobby",+ cloudConfig: undefined,+ name: "Test Organization",+ metadata: {},+ aiFeaturesEnabled: false,+ aiTelemetryEnabled: false,+ projects: [+ {+ id: projectId,+ role: "ADMIN",+ name: "Test Project",+ deletedAt: null,+ retentionDays: null,+ hasTraces: false,+ metadata: {},+ createdAt: new Date().toISOString(),+ },+ ],+ },+ ],+ featureFlags: {+ searchBar: false,+ templateFlag: true,+ excludeClickhouseRead: false,+ observationEvals: false,+ v4BetaToggleVisible: false,+ experimentsV4Enabled: false,+ },+ admin: true,+ },+ environment: {} as any,+});++describe("missing-record deletes map to NOT_FOUND", () => {+ let projectId: string;+ let caller: ReturnType<typeof appRouter.createCaller>;++ beforeEach(async () => {+ const setup = await createOrgProjectAndApiKey();+ projectId = setup.projectId;+ const ctx = createInnerTRPCContext({+ session: buildSession(setup.orgId, setup.projectId),+ headers: {},+ });+ caller = appRouter.createCaller({ ...ctx, prisma });+ });++ it("returns NOT_FOUND when deleting a model that is already gone", async () => {+ await expect(+ caller.models.delete({+ projectId,+ modelId: randomUUID(),+ }),+ ).rejects.toMatchObject({ code: "NOT_FOUND" });+ });++ it("returns NOT_FOUND when deleting an LLM API key that is already gone", async () => {+ await expect(+ caller.llmApiKey.delete({+ projectId,+ id: randomUUID(),+ }),+ ).rejects.toMatchObject({ code: "NOT_FOUND" });+ });++ it("returns NOT_FOUND when deleting a default eval model that is already gone", async () => {+ await expect(+ caller.defaultLlmModel.deleteDefaultModel({ projectId }),+ ).rejects.toMatchObject({ code: "NOT_FOUND" });+ });+});diff --git a/web/src/features/evals/server/defaultEvalModelRouter.ts b/web/src/features/evals/server/defaultEvalModelRouter.tsindex 0fd07faa1c..8683e5f5da 100644--- a/web/src/features/evals/server/defaultEvalModelRouter.ts+++ b/web/src/features/evals/server/defaultEvalModelRouter.ts@@ -1,3 +1,4 @@+import { isPrismaRecordNotFoundError } from "@/src/features/analytics-integrations/server";import { throwIfNoProjectAccess } from "@/src/features/rbac";import {createTRPCRouter,@@ -5,7 +6,11 @@ import {} from "@/src/server/api/trpc";import { z } from "zod";-import { EvaluatorBlockReason, ZodModelConfig } from "@langfuse/shared";+import {+ EvaluatorBlockReason,+ LangfuseNotFoundError,+ ZodModelConfig,+} from "@langfuse/shared";import {blockEvaluatorsUsingDefaultModel,DefaultEvalModelService,@@ -68,22 +73,30 @@ export const defaultEvalModelRouter = createTRPCRouter({scope: "evalDefaultModel:CUD",});- const result = await ctx.prisma.$transaction(async (tx) => {- const blockResult = await blockEvaluatorsUsingDefaultModel({- tx,- projectId: input.projectId,- });-- // Delete the default model within the transaction- await tx.defaultLlmModel.delete({- // unique constraint on projectId- where: {+ let result;+ try {+ result = await ctx.prisma.$transaction(async (tx) => {+ const blockResult = await blockEvaluatorsUsingDefaultModel({+ tx,projectId: input.projectId,- },- });+ });- return blockResult;- });+ // Delete the default model within the transaction+ await tx.defaultLlmModel.delete({+ // unique constraint on projectId+ where: {+ projectId: input.projectId,+ },+ });++ return blockResult;+ });+ } catch (error) {+ if (isPrismaRecordNotFoundError(error)) {+ throw new LangfuseNotFoundError("Default evaluation model not found");+ }+ throw error;+ }await finalizeEvaluatorBlocks({projectId: input.projectId,diff --git a/web/src/features/llm-api-key/server/router.ts b/web/src/features/llm-api-key/server/router.tsindex 5533c36693..d88ed1ace4 100644--- a/web/src/features/llm-api-key/server/router.ts+++ b/web/src/features/llm-api-key/server/router.ts@@ -8,6 +8,7 @@ import {SafeLlmApiKeySchema,type BedrockAuthMethod,} from "@/src/features/llm-api-key/types";+import { isPrismaRecordNotFoundError } from "@/src/features/analytics-integrations/server";import { throwIfNoProjectAccess } from "@/src/features/rbac";import {createTRPCRouter,@@ -26,6 +27,7 @@ import {BEDROCK_USE_DEFAULT_CREDENTIALS,VERTEXAI_USE_DEFAULT_CREDENTIALS,EvaluatorBlockReason,+ LangfuseNotFoundError,type LLMConnectionConfig,} from "@langfuse/shared";@@ -418,50 +420,62 @@ export const llmApiKeyRouter = createTRPCRouter({},});- const result = await ctx.prisma.$transaction(async (tx) => {- // Check if the llm api key is used for the default evaluation model- const defaultModel = await tx.defaultLlmModel.findFirst({- where: {- projectId: input.projectId,- },- select: {- llmApiKeyId: true,- },- });+ if (!llmApiKey) {+ throw new LangfuseNotFoundError("LLM API key not found");+ }- const providerBlock = llmApiKey?.provider- ? await blockEvaluatorsUsingProvider({- tx,+ let result;+ try {+ result = await ctx.prisma.$transaction(async (tx) => {+ // Check if the llm api key is used for the default evaluation model+ const defaultModel = await tx.defaultLlmModel.findFirst({+ where: {projectId: input.projectId,- provider: llmApiKey.provider,- })- : EMPTY_EVALUATOR_BLOCK;+ },+ select: {+ llmApiKeyId: true,+ },+ });- const defaultModelBlock =- !!defaultModel && defaultModel.llmApiKeyId === llmApiKey?.id- ? await blockEvaluatorsUsingDefaultModel({+ const providerBlock = llmApiKey.provider+ ? await blockEvaluatorsUsingProvider({tx,projectId: input.projectId,+ provider: llmApiKey.provider,}): EMPTY_EVALUATOR_BLOCK;- await tx.llmApiKeys.delete({- where: {- id: input.id,- projectId: input.projectId,- },- });+ const defaultModelBlock =+ !!defaultModel && defaultModel.llmApiKeyId === llmApiKey.id+ ? await blockEvaluatorsUsingDefaultModel({+ tx,+ projectId: input.projectId,+ })+ : EMPTY_EVALUATOR_BLOCK;++ await tx.llmApiKeys.delete({+ where: {+ id: input.id,+ projectId: input.projectId,+ },+ });- await auditLog({- session: ctx.session,- resourceType: "llmApiKey",- resourceId: input.id,- before: llmApiKey,- action: "delete",- });+ await auditLog({+ session: ctx.session,+ resourceType: "llmApiKey",+ resourceId: input.id,+ before: llmApiKey,+ action: "delete",+ });- return { providerBlock, defaultModelBlock };- });+ return { providerBlock, defaultModelBlock };+ });+ } catch (error) {+ if (isPrismaRecordNotFoundError(error)) {+ throw new LangfuseNotFoundError("LLM API key not found");+ }+ throw error;+ }await finalizeEvaluatorBlocks({projectId: input.projectId,diff --git a/web/src/server/api/routers/models.ts b/web/src/server/api/routers/models.tsindex e7da28fc3f..232aab6993 100644--- a/web/src/server/api/routers/models.ts+++ b/web/src/server/api/routers/models.ts@@ -1,6 +1,7 @@import { v4 as uuidv4 } from "uuid";import { z } from "zod";+import { isPrismaRecordNotFoundError } from "@/src/features/analytics-integrations/server";import { auditLog } from "@/src/features/audit-logs/server";import { isValidPostgresRegex } from "@/src/features/models/server/isValidPostgresRegex";import {@@ -13,7 +14,12 @@ import {createTRPCRouter,protectedProjectProcedure,} from "@/src/server/api/trpc";-import { ModelUsageUnit, paginationZod, Prisma } from "@langfuse/shared";+import {+ LangfuseNotFoundError,+ ModelUsageUnit,+ paginationZod,+ Prisma,+} from "@langfuse/shared";import {clearModelCacheForProject,queryClickhouse,@@ -390,12 +396,20 @@ export const modelRouter = createTRPCRouter({scope: "models:CUD",});- const deletedModel = await ctx.prisma.model.delete({- where: {- id: input.modelId,- projectId: input.projectId,- },- });+ let deletedModel;+ try {+ deletedModel = await ctx.prisma.model.delete({+ where: {+ id: input.modelId,+ projectId: input.projectId,+ },+ });+ } catch (error) {+ if (isPrismaRecordNotFoundError(error)) {+ throw new LangfuseNotFoundError("Model not found");+ }+ throw error;+ }await auditLog({session: ctx.session,