Skip to content

The open-source app everyone uses to manage agents at work

fix(runner): keep agents running when app connections expire (#12670)

Dotta86ebdf842ef9
diff --git a/server/src/__tests__/heartbeat-runtime-mcp-servers.test.ts b/server/src/__tests__/heartbeat-runtime-mcp-servers.test.ts
index d0dc7082dc..c93d328d5f 100644
--- a/server/src/__tests__/heartbeat-runtime-mcp-servers.test.ts
+++ b/server/src/__tests__/heartbeat-runtime-mcp-servers.test.ts
@@ -58,7 +58,7 @@ describeEmbeddedPostgres("heartbeat runtime MCP servers", () => {
await tempDb?.cleanup();
});
- it("provisions one aggregate gateway and filters degraded access without blocking direct adapters", async () => {
+ it("provisions one aggregate gateway and omits unavailable access without blocking any runtime", async () => {
process.env.PAPERCLIP_API_URL = "https://paperclip.example.test";
const [company] = await db.insert(companies).values({
name: `Runtime MCP ${randomUUID()}`,
@@ -158,22 +158,35 @@ describeEmbeddedPostgres("heartbeat runtime MCP servers", () => {
}
expect(JSON.stringify(tokens)).not.toContain(first[0]!.token);
+ await expect(
+ buildPaperclipRuntimeMcpServers({
+ db,
+ agent: agent!,
+ runId: randomUUID(),
+ expectedAssignmentDigest: "0".repeat(64),
+ }),
+ ).resolves.toEqual([]);
+ expect(await db.select().from(toolMcpGatewayTokens)).toHaveLength(2);
+
await db.update(toolConnections)
.set({ healthStatus: "degraded", healthMessage: "fixture unavailable" })
.where(eq(toolConnections.id, installedConnection!.id));
- await expect(
- buildPaperclipRuntimeMcpServers({ db, agent: agent!, runId: randomUUID() }),
- ).resolves.toEqual([]);
+ const unavailableReports: Array<Array<{ id: string; name: string }>> = [];
await expect(
buildPaperclipRuntimeMcpServers({
db,
agent: agent!,
runId: randomUUID(),
- failOnUnavailableAssignedConnection: true,
+ expectedAssignmentDigest: first[0]!.connectionId.slice("assignment:".length),
+ onUnavailableAssignedConnections: (connections) => {
+ unavailableReports.push(connections);
+ },
}),
- ).rejects.toThrow(
- `assigned native MCP connection is unavailable: ${installedConnection!.id}`,
- );
+ ).resolves.toEqual([]);
+ expect(unavailableReports).toEqual([[
+ { id: installedConnection!.id, name: installedConnection!.name },
+ ]]);
+ expect(await db.select().from(toolMcpGatewayTokens)).toHaveLength(2);
await expect(
createManagedMcpRunConfig({
db,
diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts
index 503c350d08..20a0e3a0d6 100644
--- a/server/src/services/heartbeat.ts
+++ b/server/src/services/heartbeat.ts
@@ -31,6 +31,7 @@ import {
PROVIDER_QUOTA_MONITOR_SERVICE_NAME,
envBindingSchema,
isEnvironmentDriverSupportedForAdapter,
+ isToolConnectionAttentionHealth,
type BillingType,
type CostStatus,
type EnvironmentLeaseStatus,
@@ -3989,7 +3990,10 @@ export async function buildPaperclipRuntimeMcpServers(input: {
db: Db;
agent: Pick<typeof agents.$inferSelect, "id" | "companyId" | "name">;
runId: string;
- failOnUnavailableAssignedConnection?: boolean;
+ expectedAssignmentDigest?: string | null;
+ onUnavailableAssignedConnections?: (
+ connections: Array<{ id: string; name: string }>,
+ ) => void | Promise<void>;
}): Promise<AdapterRuntimeMcpServer[]> {
const access = toolAccessService(input.db);
const effective = await access.getEffectiveProfilesForAgent(
@@ -4025,19 +4029,39 @@ export async function buildPaperclipRuntimeMcpServers(input: {
permittedConnectionIds.has(connection.id)
&& connection.status === "active"
&& connection.enabled
- && !["degraded", "failed", "error", "missing_secret"].includes(connection.healthStatus)
+ && !isToolConnectionAttentionHealth(connection.healthStatus)
&& (connection.transport === "mcp_remote" || connection.transport === "local_stdio")
);
const unhealthyConnections = effective.installedConnections.filter((connection) =>
permittedConnectionIds.has(connection.id)
&& (connection.transport === "mcp_remote" || connection.transport === "local_stdio")
- && (!connection.enabled || connection.status !== "active" || ["degraded", "failed", "error", "missing_secret"].includes(connection.healthStatus)),
+ && (!connection.enabled || connection.status !== "active" || isToolConnectionAttentionHealth(connection.healthStatus)),
);
- if (input.failOnUnavailableAssignedConnection && unhealthyConnections.length) {
- throw new Error(
- `assigned native MCP connection is unavailable: ${unhealthyConnections.map((connection) => connection.id).join(", ")}`,
- );
+ if (unhealthyConnections.length && input.onUnavailableAssignedConnections) {
+ try {
+ await input.onUnavailableAssignedConnections(
+ unhealthyConnections
+ .map(({ id, name }) => ({ id, name }))
+ .sort((a, b) => a.name.localeCompare(b.name)),
+ );
+ } catch (error) {
+ logger.warn(
+ {
+ companyId: input.agent.companyId,
+ agentId: input.agent.id,
+ runId: input.runId,
+ err: error,
+ },
+ "failed to report unavailable runtime MCP connections",
+ );
+ }
}
+ const assignedConnectionIds = new Set(
+ assignedConnections.map((connection) => connection.id),
+ );
+ const assignedTools = effective.allowedTools.filter((tool) =>
+ assignedConnectionIds.has(tool.connectionId)
+ );
const service = createToolGatewayService(input.db);
if (assignedConnections.length === 0) {
await service.recordRuntimeMcpDeliveryDiagnostic({
@@ -4052,11 +4076,19 @@ export async function buildPaperclipRuntimeMcpServers(input: {
version: 1,
agentId: input.agent.id,
connections: assignedConnections.map((connection) => connection.id).sort(),
- tools: effective.allowedTools.map((tool) => tool.id).sort(),
+ tools: assignedTools.map((tool) => tool.id).sort(),
};
const assignmentDigest = createHash("sha256")
.update(JSON.stringify(assignment))
.digest("hex");
+ // Native runs may lose access after their immutable context is captured, but
+ // they must never gain a new or changed assignment during dispatch.
+ if (
+ input.expectedAssignmentDigest !== undefined
+ && input.expectedAssignmentDigest !== assignmentDigest
+ ) {
+ return [];
+ }
const profileKey = `native:${input.agent.id}:${assignmentDigest}`;
let [profile] = await input.db
.select()
@@ -4082,7 +4114,7 @@ export async function buildPaperclipRuntimeMcpServers(input: {
applicationId: connection.applicationId,
connectionId: connection.id,
})),
- ...effective.allowedTools
+ ...assignedTools
.filter((tool) => !fullConnectionIds.has(tool.connectionId))
.map((tool) => ({
selectorType: "catalog_entry" as const,
@@ -20522,20 +20554,29 @@ export function heartbeatService(
if (nativeRuntimeResolution.kind === "native") {
if (!nativeExecution || !nativeRunnerInstanceId)
throw new Error("native_runtime_selection_not_persisted");
+ const expectedNativeMcpDigest =
+ "runtimeContext" in nativeExecution
+ && nativeExecution.runtimeContext.mcp.bindingId
+ ? nativeExecution.runtimeContext.mcp.digest
+ : null;
const nativeMcpServers = await buildPaperclipRuntimeMcpServers({
db,
agent,
runId: run.id,
- failOnUnavailableAssignedConnection: true,
+ expectedAssignmentDigest: expectedNativeMcpDigest,
+ onUnavailableAssignedConnections: async (connections) => {
+ const names = connections.map((connection) => connection.name).join(", ");
+ await onLog(
+ "stderr",
+ `[paperclip] App connection${connections.length === 1 ? "" : "s"} unavailable: ${names}. Continuing this run without ${connections.length === 1 ? "it" : "them"}; reconnect from Apps to restore access.\n`,
+ );
+ },
});
- if (!("runtimeContext" in nativeExecution) && nativeMcpServers.length) {
- throw new Error("historical native runs cannot acquire newly assigned MCP access");
- }
if ("runtimeContext" in nativeExecution) {
if (nativeMcpServers.length > 1) throw new Error("native MCP realization must produce one aggregate gateway");
const server = nativeMcpServers[0] ?? null;
const digest = server?.connectionId.startsWith("assignment:") ? server.connectionId.slice("assignment:".length) : null;
- if (digest !== (nativeExecution.runtimeContext.mcp.bindingId ? nativeExecution.runtimeContext.mcp.digest : null)) {
+ if (digest && digest !== expectedNativeMcpDigest) {
throw new Error("native MCP assignment digest mismatch");
}
}
diff --git a/server/src/services/native-runtime/runtime-context.test.ts b/server/src/services/native-runtime/runtime-context.test.ts
index 475b249ecc..0f4db322dd 100644
--- a/server/src/services/native-runtime/runtime-context.test.ts
+++ b/server/src/services/native-runtime/runtime-context.test.ts
@@ -83,6 +83,102 @@ afterEach(async () => {
});
describe("buildNativeRuntimeContext", () => {
+ it.each(["disabled", "degraded"] as const)(
+ "omits an unavailable native MCP connection when it is %s without aborting runtime context creation",
+ async (unavailableState) => {
+ serviceMocks.exportFiles.mockResolvedValue({
+ entryFile: "AGENTS.md",
+ files: { "AGENTS.md": "Continue work without unavailable apps.\n" },
+ });
+ serviceMocks.getEffectiveProfilesForAgent.mockResolvedValue({
+ agentId: "agent-1",
+ profiles: [],
+ entries: [{ effect: "include", connectionId: "connection-1" }],
+ bindings: [],
+ allowedTools: [{ id: "tool-1", connectionId: "connection-1" }],
+ allowedToolNames: ["issues.read"],
+ installedConnections: [{
+ id: "connection-1",
+ transport: "mcp_remote",
+ enabled: unavailableState !== "disabled",
+ status: unavailableState === "disabled" ? "disabled" : "active",
+ healthStatus: unavailableState === "degraded" ? "degraded" : "healthy",
+ }],
+ });
+
+ const context = await buildNativeRuntimeContext({
+ db: {} as Db,
+ agent: {
+ id: "agent-1",
+ companyId: "company-1",
+ name: "Reviewer",
+ adapterType: "paperclip_runner",
+ adapterConfig: {},
+ },
+ runId: "run-1",
+ runtimeConfig: {},
+ runtimeSkillEntries: [],
+ });
+
+ expect(context.mcp.bindingId).toBeNull();
+ expect(context.mcp.assignmentSetId).toMatch(/^sha256:[a-f0-9]{64}$/);
+ },
+ );
+
+ it("keeps healthy native MCP connections when another assigned connection is unavailable", async () => {
+ serviceMocks.exportFiles.mockResolvedValue({
+ entryFile: "AGENTS.md",
+ files: { "AGENTS.md": "Continue work with the apps that are available.\n" },
+ });
+ serviceMocks.getEffectiveProfilesForAgent.mockResolvedValue({
+ agentId: "agent-1",
+ profiles: [],
+ entries: [
+ { effect: "include", connectionId: "connection-expired" },
+ { effect: "include", connectionId: "connection-healthy" },
+ ],
+ bindings: [],
+ allowedTools: [
+ { id: "tool-expired", connectionId: "connection-expired" },
+ { id: "tool-healthy", connectionId: "connection-healthy" },
+ ],
+ allowedToolNames: ["expired.read", "healthy.read"],
+ installedConnections: [
+ {
+ id: "connection-expired",
+ transport: "mcp_remote",
+ enabled: true,
+ status: "active",
+ healthStatus: "degraded",
+ },
+ {
+ id: "connection-healthy",
+ transport: "mcp_remote",
+ enabled: true,
+ status: "active",
+ healthStatus: "healthy",
+ },
+ ],
+ });
+
+ const context = await buildNativeRuntimeContext({
+ db: {} as Db,
+ agent: {
+ id: "agent-1",
+ companyId: "company-1",
+ name: "Reviewer",
+ adapterType: "paperclip_runner",
+ adapterConfig: {},
+ },
+ runId: "run-1",
+ runtimeConfig: {},
+ runtimeSkillEntries: [],
+ });
+
+ expect(context.mcp.bindingId).toBe("native-mcp:run-1");
+ expect(context.mcp.assignmentSetId).toMatch(/^sha256:[a-f0-9]{64}$/);
+ });
+
it("materializes every instruction and selected-skill file as immutable, content-addressed context", async () => {
serviceMocks.exportFiles.mockResolvedValue({
entryFile: "AGENTS.md",
diff --git a/server/src/services/native-runtime/runtime-context.ts b/server/src/services/native-runtime/runtime-context.ts
index ea84917a4e..7bc3abbe63 100644
--- a/server/src/services/native-runtime/runtime-context.ts
+++ b/server/src/services/native-runtime/runtime-context.ts
@@ -3,6 +3,7 @@ import fs from "node:fs/promises";
import path from "node:path";
import type { Db } from "@paperclipai/db";
import type { PaperclipSkillEntry } from "@paperclipai/adapter-utils/server-utils";
+import { isToolConnectionAttentionHealth } from "@paperclipai/shared";
import {
PAPERCLIP_OPERATIONAL_SKILL_KEY,
resolvePaperclipDesiredSkillNames,
@@ -165,17 +166,23 @@ async function materializeSelectedSkills(runtimeConfig: Record<string, unknown>,
export async function resolveNativeRuntimeMcpSnapshot(input: { db: Db; agent: Pick<RuntimeAgent, "id" | "companyId">; runId: string }) {
const effective = await toolAccessService(input.db).getEffectiveProfilesForAgent(input.agent.companyId, input.agent.id);
const permitted = new Set([...effective.entries.filter((entry) => entry.effect === "include" && entry.connectionId).map((entry) => entry.connectionId!), ...effective.allowedTools.map((tool) => tool.connectionId)]);
- const unhealthy = effective.installedConnections.filter((connection) =>
+ // App access is optional runtime context. Keep usable assignments pinned, but
+ // do not stop unrelated work because an assigned app needs attention.
+ const availableConnectionIds = new Set(effective.installedConnections.filter((connection) =>
permitted.has(connection.id)
+ && connection.status === "active"
+ && connection.enabled
+ && !isToolConnectionAttentionHealth(connection.healthStatus)
&& ["mcp_remote", "local_stdio"].includes(connection.transport)
- && (!connection.enabled || connection.status !== "active" || ["degraded", "failed", "error", "missing_secret"].includes(connection.healthStatus)),
- );
- if (unhealthy.length) throw new Error(`assigned native MCP connection is unavailable: ${unhealthy.map((connection) => connection.id).join(", ")}`);
+ ).map((connection) => connection.id));
const assignment = {
version: 1,
agentId: input.agent.id,
- connections: effective.installedConnections.filter((connection) => permitted.has(connection.id) && connection.status === "active" && connection.enabled && ["mcp_remote", "local_stdio"].includes(connection.transport)).map((connection) => connection.id).sort(),
- tools: effective.allowedTools.map((tool) => tool.id).sort(),
+ connections: [...availableConnectionIds].sort(),
+ tools: effective.allowedTools
+ .filter((tool) => availableConnectionIds.has(tool.connectionId))
+ .map((tool) => tool.id)
+ .sort(),
};
const assignmentDigest = sha256(JSON.stringify(assignment));
return { assignmentSetId: `sha256:${assignmentDigest}`, digest: assignmentDigest, bindingId: assignment.connections.length ? `native-mcp:${input.runId}` : null };