The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
fix(runner): keep agents running when app connections expire (#12670)
Dotta86ebdf842ef9
diff --git a/server/src/__tests__/heartbeat-runtime-mcp-servers.test.ts b/server/src/__tests__/heartbeat-runtime-mcp-servers.test.tsindex d0dc7082dc..c93d328d5f 100644--- a/server/src/__tests__/heartbeat-runtime-mcp-servers.test.ts+++ b/server/src/__tests__/heartbeat-runtime-mcp-servers.test.ts@@ -58,7 +58,7 @@ describeEmbeddedPostgres("heartbeat runtime MCP servers", () => {await tempDb?.cleanup();});- it("provisions one aggregate gateway and filters degraded access without blocking direct adapters", async () => {+ it("provisions one aggregate gateway and omits unavailable access without blocking any runtime", async () => {process.env.PAPERCLIP_API_URL = "https://paperclip.example.test";const [company] = await db.insert(companies).values({name: `Runtime MCP ${randomUUID()}`,@@ -158,22 +158,35 @@ describeEmbeddedPostgres("heartbeat runtime MCP servers", () => {}expect(JSON.stringify(tokens)).not.toContain(first[0]!.token);+ await expect(+ buildPaperclipRuntimeMcpServers({+ db,+ agent: agent!,+ runId: randomUUID(),+ expectedAssignmentDigest: "0".repeat(64),+ }),+ ).resolves.toEqual([]);+ expect(await db.select().from(toolMcpGatewayTokens)).toHaveLength(2);+await db.update(toolConnections).set({ healthStatus: "degraded", healthMessage: "fixture unavailable" }).where(eq(toolConnections.id, installedConnection!.id));- await expect(- buildPaperclipRuntimeMcpServers({ db, agent: agent!, runId: randomUUID() }),- ).resolves.toEqual([]);+ const unavailableReports: Array<Array<{ id: string; name: string }>> = [];await expect(buildPaperclipRuntimeMcpServers({db,agent: agent!,runId: randomUUID(),- failOnUnavailableAssignedConnection: true,+ expectedAssignmentDigest: first[0]!.connectionId.slice("assignment:".length),+ onUnavailableAssignedConnections: (connections) => {+ unavailableReports.push(connections);+ },}),- ).rejects.toThrow(- `assigned native MCP connection is unavailable: ${installedConnection!.id}`,- );+ ).resolves.toEqual([]);+ expect(unavailableReports).toEqual([[+ { id: installedConnection!.id, name: installedConnection!.name },+ ]]);+ expect(await db.select().from(toolMcpGatewayTokens)).toHaveLength(2);await expect(createManagedMcpRunConfig({db,diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.tsindex 503c350d08..20a0e3a0d6 100644--- a/server/src/services/heartbeat.ts+++ b/server/src/services/heartbeat.ts@@ -31,6 +31,7 @@ import {PROVIDER_QUOTA_MONITOR_SERVICE_NAME,envBindingSchema,isEnvironmentDriverSupportedForAdapter,+ isToolConnectionAttentionHealth,type BillingType,type CostStatus,type EnvironmentLeaseStatus,@@ -3989,7 +3990,10 @@ export async function buildPaperclipRuntimeMcpServers(input: {db: Db;agent: Pick<typeof agents.$inferSelect, "id" | "companyId" | "name">;runId: string;- failOnUnavailableAssignedConnection?: boolean;+ expectedAssignmentDigest?: string | null;+ onUnavailableAssignedConnections?: (+ connections: Array<{ id: string; name: string }>,+ ) => void | Promise<void>;}): Promise<AdapterRuntimeMcpServer[]> {const access = toolAccessService(input.db);const effective = await access.getEffectiveProfilesForAgent(@@ -4025,19 +4029,39 @@ export async function buildPaperclipRuntimeMcpServers(input: {permittedConnectionIds.has(connection.id)&& connection.status === "active"&& connection.enabled- && !["degraded", "failed", "error", "missing_secret"].includes(connection.healthStatus)+ && !isToolConnectionAttentionHealth(connection.healthStatus)&& (connection.transport === "mcp_remote" || connection.transport === "local_stdio"));const unhealthyConnections = effective.installedConnections.filter((connection) =>permittedConnectionIds.has(connection.id)&& (connection.transport === "mcp_remote" || connection.transport === "local_stdio")- && (!connection.enabled || connection.status !== "active" || ["degraded", "failed", "error", "missing_secret"].includes(connection.healthStatus)),+ && (!connection.enabled || connection.status !== "active" || isToolConnectionAttentionHealth(connection.healthStatus)),);- if (input.failOnUnavailableAssignedConnection && unhealthyConnections.length) {- throw new Error(- `assigned native MCP connection is unavailable: ${unhealthyConnections.map((connection) => connection.id).join(", ")}`,- );+ if (unhealthyConnections.length && input.onUnavailableAssignedConnections) {+ try {+ await input.onUnavailableAssignedConnections(+ unhealthyConnections+ .map(({ id, name }) => ({ id, name }))+ .sort((a, b) => a.name.localeCompare(b.name)),+ );+ } catch (error) {+ logger.warn(+ {+ companyId: input.agent.companyId,+ agentId: input.agent.id,+ runId: input.runId,+ err: error,+ },+ "failed to report unavailable runtime MCP connections",+ );+ }}+ const assignedConnectionIds = new Set(+ assignedConnections.map((connection) => connection.id),+ );+ const assignedTools = effective.allowedTools.filter((tool) =>+ assignedConnectionIds.has(tool.connectionId)+ );const service = createToolGatewayService(input.db);if (assignedConnections.length === 0) {await service.recordRuntimeMcpDeliveryDiagnostic({@@ -4052,11 +4076,19 @@ export async function buildPaperclipRuntimeMcpServers(input: {version: 1,agentId: input.agent.id,connections: assignedConnections.map((connection) => connection.id).sort(),- tools: effective.allowedTools.map((tool) => tool.id).sort(),+ tools: assignedTools.map((tool) => tool.id).sort(),};const assignmentDigest = createHash("sha256").update(JSON.stringify(assignment)).digest("hex");+ // Native runs may lose access after their immutable context is captured, but+ // they must never gain a new or changed assignment during dispatch.+ if (+ input.expectedAssignmentDigest !== undefined+ && input.expectedAssignmentDigest !== assignmentDigest+ ) {+ return [];+ }const profileKey = `native:${input.agent.id}:${assignmentDigest}`;let [profile] = await input.db.select()@@ -4082,7 +4114,7 @@ export async function buildPaperclipRuntimeMcpServers(input: {applicationId: connection.applicationId,connectionId: connection.id,})),- ...effective.allowedTools+ ...assignedTools.filter((tool) => !fullConnectionIds.has(tool.connectionId)).map((tool) => ({selectorType: "catalog_entry" as const,@@ -20522,20 +20554,29 @@ export function heartbeatService(if (nativeRuntimeResolution.kind === "native") {if (!nativeExecution || !nativeRunnerInstanceId)throw new Error("native_runtime_selection_not_persisted");+ const expectedNativeMcpDigest =+ "runtimeContext" in nativeExecution+ && nativeExecution.runtimeContext.mcp.bindingId+ ? nativeExecution.runtimeContext.mcp.digest+ : null;const nativeMcpServers = await buildPaperclipRuntimeMcpServers({db,agent,runId: run.id,- failOnUnavailableAssignedConnection: true,+ expectedAssignmentDigest: expectedNativeMcpDigest,+ onUnavailableAssignedConnections: async (connections) => {+ const names = connections.map((connection) => connection.name).join(", ");+ await onLog(+ "stderr",+ `[paperclip] App connection${connections.length === 1 ? "" : "s"} unavailable: ${names}. Continuing this run without ${connections.length === 1 ? "it" : "them"}; reconnect from Apps to restore access.\n`,+ );+ },});- if (!("runtimeContext" in nativeExecution) && nativeMcpServers.length) {- throw new Error("historical native runs cannot acquire newly assigned MCP access");- }if ("runtimeContext" in nativeExecution) {if (nativeMcpServers.length > 1) throw new Error("native MCP realization must produce one aggregate gateway");const server = nativeMcpServers[0] ?? null;const digest = server?.connectionId.startsWith("assignment:") ? server.connectionId.slice("assignment:".length) : null;- if (digest !== (nativeExecution.runtimeContext.mcp.bindingId ? nativeExecution.runtimeContext.mcp.digest : null)) {+ if (digest && digest !== expectedNativeMcpDigest) {throw new Error("native MCP assignment digest mismatch");}}diff --git a/server/src/services/native-runtime/runtime-context.test.ts b/server/src/services/native-runtime/runtime-context.test.tsindex 475b249ecc..0f4db322dd 100644--- a/server/src/services/native-runtime/runtime-context.test.ts+++ b/server/src/services/native-runtime/runtime-context.test.ts@@ -83,6 +83,102 @@ afterEach(async () => {});describe("buildNativeRuntimeContext", () => {+ it.each(["disabled", "degraded"] as const)(+ "omits an unavailable native MCP connection when it is %s without aborting runtime context creation",+ async (unavailableState) => {+ serviceMocks.exportFiles.mockResolvedValue({+ entryFile: "AGENTS.md",+ files: { "AGENTS.md": "Continue work without unavailable apps.\n" },+ });+ serviceMocks.getEffectiveProfilesForAgent.mockResolvedValue({+ agentId: "agent-1",+ profiles: [],+ entries: [{ effect: "include", connectionId: "connection-1" }],+ bindings: [],+ allowedTools: [{ id: "tool-1", connectionId: "connection-1" }],+ allowedToolNames: ["issues.read"],+ installedConnections: [{+ id: "connection-1",+ transport: "mcp_remote",+ enabled: unavailableState !== "disabled",+ status: unavailableState === "disabled" ? "disabled" : "active",+ healthStatus: unavailableState === "degraded" ? "degraded" : "healthy",+ }],+ });++ const context = await buildNativeRuntimeContext({+ db: {} as Db,+ agent: {+ id: "agent-1",+ companyId: "company-1",+ name: "Reviewer",+ adapterType: "paperclip_runner",+ adapterConfig: {},+ },+ runId: "run-1",+ runtimeConfig: {},+ runtimeSkillEntries: [],+ });++ expect(context.mcp.bindingId).toBeNull();+ expect(context.mcp.assignmentSetId).toMatch(/^sha256:[a-f0-9]{64}$/);+ },+ );++ it("keeps healthy native MCP connections when another assigned connection is unavailable", async () => {+ serviceMocks.exportFiles.mockResolvedValue({+ entryFile: "AGENTS.md",+ files: { "AGENTS.md": "Continue work with the apps that are available.\n" },+ });+ serviceMocks.getEffectiveProfilesForAgent.mockResolvedValue({+ agentId: "agent-1",+ profiles: [],+ entries: [+ { effect: "include", connectionId: "connection-expired" },+ { effect: "include", connectionId: "connection-healthy" },+ ],+ bindings: [],+ allowedTools: [+ { id: "tool-expired", connectionId: "connection-expired" },+ { id: "tool-healthy", connectionId: "connection-healthy" },+ ],+ allowedToolNames: ["expired.read", "healthy.read"],+ installedConnections: [+ {+ id: "connection-expired",+ transport: "mcp_remote",+ enabled: true,+ status: "active",+ healthStatus: "degraded",+ },+ {+ id: "connection-healthy",+ transport: "mcp_remote",+ enabled: true,+ status: "active",+ healthStatus: "healthy",+ },+ ],+ });++ const context = await buildNativeRuntimeContext({+ db: {} as Db,+ agent: {+ id: "agent-1",+ companyId: "company-1",+ name: "Reviewer",+ adapterType: "paperclip_runner",+ adapterConfig: {},+ },+ runId: "run-1",+ runtimeConfig: {},+ runtimeSkillEntries: [],+ });++ expect(context.mcp.bindingId).toBe("native-mcp:run-1");+ expect(context.mcp.assignmentSetId).toMatch(/^sha256:[a-f0-9]{64}$/);+ });+it("materializes every instruction and selected-skill file as immutable, content-addressed context", async () => {serviceMocks.exportFiles.mockResolvedValue({entryFile: "AGENTS.md",diff --git a/server/src/services/native-runtime/runtime-context.ts b/server/src/services/native-runtime/runtime-context.tsindex ea84917a4e..7bc3abbe63 100644--- a/server/src/services/native-runtime/runtime-context.ts+++ b/server/src/services/native-runtime/runtime-context.ts@@ -3,6 +3,7 @@ import fs from "node:fs/promises";import path from "node:path";import type { Db } from "@paperclipai/db";import type { PaperclipSkillEntry } from "@paperclipai/adapter-utils/server-utils";+import { isToolConnectionAttentionHealth } from "@paperclipai/shared";import {PAPERCLIP_OPERATIONAL_SKILL_KEY,resolvePaperclipDesiredSkillNames,@@ -165,17 +166,23 @@ async function materializeSelectedSkills(runtimeConfig: Record<string, unknown>,export async function resolveNativeRuntimeMcpSnapshot(input: { db: Db; agent: Pick<RuntimeAgent, "id" | "companyId">; runId: string }) {const effective = await toolAccessService(input.db).getEffectiveProfilesForAgent(input.agent.companyId, input.agent.id);const permitted = new Set([...effective.entries.filter((entry) => entry.effect === "include" && entry.connectionId).map((entry) => entry.connectionId!), ...effective.allowedTools.map((tool) => tool.connectionId)]);- const unhealthy = effective.installedConnections.filter((connection) =>+ // App access is optional runtime context. Keep usable assignments pinned, but+ // do not stop unrelated work because an assigned app needs attention.+ const availableConnectionIds = new Set(effective.installedConnections.filter((connection) =>permitted.has(connection.id)+ && connection.status === "active"+ && connection.enabled+ && !isToolConnectionAttentionHealth(connection.healthStatus)&& ["mcp_remote", "local_stdio"].includes(connection.transport)- && (!connection.enabled || connection.status !== "active" || ["degraded", "failed", "error", "missing_secret"].includes(connection.healthStatus)),- );- if (unhealthy.length) throw new Error(`assigned native MCP connection is unavailable: ${unhealthy.map((connection) => connection.id).join(", ")}`);+ ).map((connection) => connection.id));const assignment = {version: 1,agentId: input.agent.id,- connections: effective.installedConnections.filter((connection) => permitted.has(connection.id) && connection.status === "active" && connection.enabled && ["mcp_remote", "local_stdio"].includes(connection.transport)).map((connection) => connection.id).sort(),- tools: effective.allowedTools.map((tool) => tool.id).sort(),+ connections: [...availableConnectionIds].sort(),+ tools: effective.allowedTools+ .filter((tool) => availableConnectionIds.has(tool.connectionId))+ .map((tool) => tool.id)+ .sort(),};const assignmentDigest = sha256(JSON.stringify(assignment));return { assignmentSetId: `sha256:${assignmentDigest}`, digest: assignmentDigest, bindingId: assignment.connections.length ? `native-mcp:${input.runId}` : null };