🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(in-app-agent): treat missing MCP keys as successful cleanup (#16309)
Lysander Kiesela5d901fc5d9d
diff --git a/packages/shared/src/in-app-agent/server/runLifecycle.test.ts b/packages/shared/src/in-app-agent/server/runLifecycle.test.tsindex c132e74da8..21d844010c 100644--- a/packages/shared/src/in-app-agent/server/runLifecycle.test.ts+++ b/packages/shared/src/in-app-agent/server/runLifecycle.test.ts@@ -5,8 +5,10 @@ import {InAppAgentRunErrorCode,InAppAgentRunStatus,} from "../../features/inAppAgent/types";-import type { PrismaClient } from "../../db";+import { Prisma, type PrismaClient } from "../../db";+import { logger } from "../../server";import {+ cleanupTerminalRunMcpApiKeys,createQueuedRun,reconcileConversationRuns,requestRunCancellation,@@ -338,3 +340,44 @@ describe("in-app agent run lifecycle races", () => {expect(metricMocks.recordRunTerminalOutcome).not.toHaveBeenCalled();});});++describe("cleanupTerminalRunMcpApiKeys", () => {+ const missingKeyError = new Prisma.PrismaClientKnownRequestError(+ "No record was found for a delete.",+ { code: "P2025", clientVersion: "test" },+ );++ it("clears the pointer when the API key row is already gone", async () => {+ const updateMany = vi.fn().mockResolvedValue({ count: 1 });+ const prisma = {+ inAppAgentRun: {+ findMany: vi+ .fn()+ .mockResolvedValue([{ id: "run-1", mcpApiKeyId: "key-1" }]),+ updateMany,+ },+ } as unknown as PrismaClient;+ const errorSpy = vi.spyOn(logger, "error");++ try {+ await expect(+ cleanupTerminalRunMcpApiKeys({+ prisma,+ projectId: "project-1",+ conversationId: "conversation-1",+ deleteApiKey: async () => {+ throw missingKeyError;+ },+ }),+ ).resolves.toBe(1);+ } finally {+ errorSpy.mockRestore();+ }++ expect(updateMany).toHaveBeenCalledWith({+ where: { id: "run-1", projectId: "project-1" },+ data: { mcpApiKeyId: null },+ });+ expect(errorSpy).not.toHaveBeenCalled();+ });+});diff --git a/packages/shared/src/in-app-agent/server/runLifecycle.ts b/packages/shared/src/in-app-agent/server/runLifecycle.tsindex 4eb6896e15..65986dcbd8 100644--- a/packages/shared/src/in-app-agent/server/runLifecycle.ts+++ b/packages/shared/src/in-app-agent/server/runLifecycle.ts@@ -746,6 +746,13 @@ export function classifyStaleRun(return null;}+export function isMissingInAppAgentMcpApiKeyError(error: unknown): boolean {+ return (+ error instanceof Prisma.PrismaClientKnownRequestError &&+ error.code === "P2025"+ );+}+/** Retry terminal-run MCP-key cleanup outside the lifecycle transaction. */export async function cleanupTerminalRunMcpApiKeys(params: {prisma: PrismaClient;@@ -769,7 +776,14 @@ export async function cleanupTerminalRunMcpApiKeys(params: {if (!run.mcpApiKeyId) continue;try {- await params.deleteApiKey(run.mcpApiKeyId);+ try {+ await params.deleteApiKey(run.mcpApiKeyId);+ } catch (error) {+ // Concurrent cleanup or a prior delete already removed the row.+ if (!isMissingInAppAgentMcpApiKeyError(error)) {+ throw error;+ }+ }await clearRunMcpApiKeyPointer({prisma: params.prisma,projectId: params.projectId,diff --git a/worker/src/features/in-app-agent/executeInAppAgentRun.test.ts b/worker/src/features/in-app-agent/executeInAppAgentRun.test.tsindex 8efa10fdcf..41c037cfcc 100644--- a/worker/src/features/in-app-agent/executeInAppAgentRun.test.ts+++ b/worker/src/features/in-app-agent/executeInAppAgentRun.test.ts@@ -1,9 +1,10 @@import { randomUUID } from "crypto";import { beforeEach, describe, expect, it, vi } from "vitest";-import { createOrgProjectAndApiKey } from "@langfuse/shared/src/server";+import { createOrgProjectAndApiKey, logger } from "@langfuse/shared/src/server";import { prisma } from "@langfuse/shared/src/db";import { env as sharedEnv } from "@langfuse/shared/src/env";+import { createAndAddApiKeysToDb } from "@langfuse/shared/src/server/auth/apiKeys";import { ResumeForwardedPropsSchema } from "./runtime/types";import { env } from "../../env";@@ -823,13 +824,24 @@ describe("executeInAppAgentRun", () => {});it("reconciles a stale RUNNING delivery and acknowledges without starting the agent loop", async () => {- const { projectId, run } = await seedBackgroundRun({ status: "RUNNING" });+ const { projectId, run, user } = await seedBackgroundRun({+ status: "RUNNING",+ });const twoMinutesAgo = new Date(Date.now() - 2 * 60_000);+ const key = await createAndAddApiKeysToDb({+ prisma,+ entityId: projectId,+ scope: "PROJECT",+ note: "stale-run mcp key",+ isInAppAgentKey: true,+ createdByUserId: user.id,+ });await prisma.inAppAgentRun.update({where: { id_projectId: { id: run.id, projectId } },data: {claimedAt: twoMinutesAgo,heartbeatAt: twoMinutesAgo,+ mcpApiKeyId: key.id,},});@@ -847,6 +859,59 @@ describe("executeInAppAgentRun", () => {expect(failed.status).toBe("FAILED");expect(failed.errorCode).toBe("worker_lost");expect(failed.finishedAt).not.toBeNull();+ expect(failed.mcpApiKeyId).toBeNull();+ expect(await getInAppAgentApiKeys(projectId)).toHaveLength(0);+ });++ it("clears the MCP pointer on claim-miss even when the key row is already gone", async () => {+ const { projectId, run, user } = await seedBackgroundRun({+ status: "RUNNING",+ });+ const twoMinutesAgo = new Date(Date.now() - 2 * 60_000);+ const key = await createAndAddApiKeysToDb({+ prisma,+ entityId: projectId,+ scope: "PROJECT",+ note: "already-deleted mcp key",+ isInAppAgentKey: true,+ createdByUserId: user.id,+ });+ await prisma.inAppAgentRun.update({+ where: { id_projectId: { id: run.id, projectId } },+ data: {+ claimedAt: twoMinutesAgo,+ heartbeatAt: twoMinutesAgo,+ mcpApiKeyId: key.id,+ },+ });+ await prisma.apiKey.delete({ where: { id: key.id } });++ const errorSpy = vi.spyOn(logger, "error").mockImplementation(() => {});+ scenarioRef.current = async () => {+ throw new Error(+ "agent loop must not start on stale unclaimable delivery",+ );+ };++ try {+ await expect(+ executeInAppAgentRun({ projectId, runId: run.id }),+ ).resolves.toBeUndefined();+ } finally {+ errorSpy.mockRestore();+ }++ const failed = await getRun(projectId, run.id);+ expect(failed.status).toBe("FAILED");+ expect(failed.errorCode).toBe("worker_lost");+ expect(failed.mcpApiKeyId).toBeNull();+ expect(+ errorSpy.mock.calls.filter(([message]) =>+ String(message).includes(+ "Failed to clean up in-app agent MCP key on reconcile",+ ),+ ),+ ).toHaveLength(0);});it("acknowledges delivery against an already-FAILED run without changing it", async () => {@@ -950,6 +1015,35 @@ describe("executeInAppAgentRun", () => {expect(await getInAppAgentApiKeys(projectId)).toHaveLength(1);});+ it("treats a missing MCP key as cleaned up and still nulls the pointer", async () => {+ const { projectId, run } = await seedBackgroundRun();+ const errorSpy = vi.spyOn(logger, "error").mockImplementation(() => {});++ scenarioRef.current = async ({ options }) => {+ const keys = await getInAppAgentApiKeys(projectId);+ expect(keys).toHaveLength(1);+ await prisma.apiKey.delete({ where: { id: keys[0].id } });+ await options.onComplete();+ await options.onFinish();+ };++ try {+ await executeInAppAgentRun({ projectId, runId: run.id });+ } finally {+ errorSpy.mockRestore();+ }++ const finished = await getRun(projectId, run.id);+ expect(finished.status).toBe("SUCCEEDED");+ expect(finished.mcpApiKeyId).toBeNull();+ expect(await getInAppAgentApiKeys(projectId)).toHaveLength(0);+ expect(+ errorSpy.mock.calls.filter(([message]) =>+ String(message).includes("Failed to clean up in-app agent MCP API key"),+ ),+ ).toHaveLength(0);+ });+it("aborts active runs on shutdown as FAILED (worker_shutdown)", async () => {const { projectId, run } = await seedBackgroundRun();diff --git a/worker/src/features/in-app-agent/executeInAppAgentRun.ts b/worker/src/features/in-app-agent/executeInAppAgentRun.tsindex cbf9534624..eb06c30730 100644--- a/worker/src/features/in-app-agent/executeInAppAgentRun.ts+++ b/worker/src/features/in-app-agent/executeInAppAgentRun.ts@@ -36,9 +36,11 @@ import {} from "@langfuse/shared/in-app-agent/server/modelProvider";import {claimQueuedRun,+ cleanupTerminalRunMcpApiKeys,clearRunMcpApiKeyPointer,finishClaimedRun,heartbeatClaimedRun,+ isMissingInAppAgentMcpApiKeyError,reconcileConversationRuns,} from "@langfuse/shared/in-app-agent/server/runLifecycle";import {@@ -77,6 +79,27 @@ export function abortActiveInAppAgentRuns(): void {/** Thrown for claim-time revalidation failures; maps to FAILED (init_failed). */class InAppAgentRunInitError extends Error {}+async function deleteInAppAgentMcpApiKey(params: {+ projectId: string;+ apiKeyId: string;+}): Promise<void> {+ try {+ await deleteApiKeyFromDb({+ prisma,+ id: params.apiKeyId,+ entityId: params.projectId,+ scope: "PROJECT",+ redis,+ });+ } catch (error) {+ // Concurrent cleanup or a prior delete already removed the row. Treat+ // that as success so the mcpApiKeyId pointer can still be cleared.+ if (!isMissingInAppAgentMcpApiKeyError(error)) {+ throw error;+ }+ }+}+export async function executeInAppAgentRun(params: {projectId: string;runId: string;@@ -105,6 +128,13 @@ export async function executeInAppAgentRun(params: {projectId,conversationId: existing.conversationId,});+ await cleanupTerminalRunMcpApiKeys({+ prisma,+ projectId,+ conversationId: existing.conversationId,+ deleteApiKey: (apiKeyId) =>+ deleteInAppAgentMcpApiKey({ projectId, apiKeyId }),+ });}return;@@ -144,16 +174,8 @@ export async function executeInAppAgentRun(params: {if (!mcpApiKey) return Promise.resolve();const keyId = mcpApiKey.id;mcpApiKeyCleanup ??= (async () => {- await deleteApiKeyFromDb({- prisma,- id: keyId,- entityId: projectId,- scope: "PROJECT",- redis,- });- // Pointer is nulled only after the delete is confirmed; if the delete- // failed above, the terminal run keeps the pointer so reconciliation- // retries the cleanup.+ await deleteInAppAgentMcpApiKey({ projectId, apiKeyId: keyId });+ // Pointer is nulled after delete succeeds or the key is already gone.await clearRunMcpApiKeyPointer({ prisma, projectId, runId });})().catch((error: unknown) => {mcpApiKeyCleanup = undefined;