The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
fix: retain diagnostic reasons for native runner failures (#14481)
Devin Foleyad1f7e98ead6
diff --git a/doc/observability.md b/doc/observability.mdindex fb2a8b15ee..491e8e35c8 100644--- a/doc/observability.md+++ b/doc/observability.md@@ -486,6 +486,17 @@ These fields contain build identifiers; they add no tenant or user identity.`errorCode`, and `agentAdapter`. The server redacts the error message andthe error code before it sends the event.+Native runner identity and harness failures retain their existing error prefixes.+Their terminal messages now include a bounded guard reason, such as+`session_scope_mismatch`, `durable_identity_unreadable`, or+`backup_without_reusable_lease`. Provider-pack read failures distinguish a missing+file, invalid JSON, permission denial, invalid path type, and other I/O errors.+These reasons contain no session identifiers, provider output, or filesystem+paths. They help diagnose recurrence; they do not authorize a retry, quarantine,+replacement, or a weaker identity check. Existing chat recovery recognizes the+same failure category with or without a reason suffix; it still requires the+exact cleanup receipt, checkpoint, and absence of provider work.+**Server events the default integrations add**- `OnUncaughtException` — each uncaught exception on the main thread, atdiff --git a/server/src/__tests__/chat-channels.integration.test.ts b/server/src/__tests__/chat-channels.integration.test.tsindex 85d968cbc1..d04154ee97 100644--- a/server/src/__tests__/chat-channels.integration.test.ts+++ b/server/src/__tests__/chat-channels.integration.test.ts@@ -56157,9 +56157,13 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {"leased","wrong_thread","source_edited",- ])(- "retries only the original pre-provider Telegram request after exact cleanup: %s",- async (mode) => {+ "different_error",+ ].flatMap((mode) => [+ "runner_state_identity_mismatch",+ "runner_state_identity_mismatch: prior_owner_active",+ ].map((errorMessage) => ({ mode, errorMessage }))))(+ "retries only the original pre-provider Telegram request after exact cleanup: $mode ($errorMessage)",+ async ({ mode, errorMessage }) => {const context = await committedChatResponseRecoveryFixture("telegram");const providerAccount =mode === "null_account"@@ -56286,7 +56290,9 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {agentId: context.fixture.assignedAgentId,status: "failed",errorCode: "adapter_failed",- error: "runner_state_identity_mismatch",+ error: mode === "different_error"+ ? errorMessage.replace("runner_state_identity_mismatch", "runner_state_identity_mismatch_other")+ : errorMessage,finishedAt: new Date(),wakeupRequestId: action.id,runtimeMode: "native",diff --git a/server/src/services/chat-channels.ts b/server/src/services/chat-channels.tsindex dd3694dc6a..a4c67d20b3 100644--- a/server/src/services/chat-channels.ts+++ b/server/src/services/chat-channels.ts@@ -12006,7 +12006,11 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {run.nativeIssueId === issueId &&run.status === "failed" &&run.errorCode === "adapter_failed" &&- run.error === "runner_state_identity_mismatch" &&+ // A diagnostic reason does not change this failure category. The exact+ // checkpoint, cleanup receipt, and no-provider-work proofs below still+ // decide whether the original request can be retried.+ (run.error === "runner_state_identity_mismatch" ||+ run.error?.startsWith("runner_state_identity_mismatch: ")) &&run.nativePhase === "observed" &&coordinator.phase === "observed" &&coordinator.attempt === 0 &&diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.tsindex d533986d90..b6d7316911 100644--- a/server/src/services/native-runtime/native-session-executor.test.ts+++ b/server/src/services/native-runtime/native-session-executor.test.ts@@ -1,4 +1,5 @@import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import { readFileSync } from "node:fs";import {access,cp,@@ -56,6 +57,11 @@ import { buildNativeHeartbeatPreparationSpans } from "./native-run-trace.js";import { NativeRunnerOwnershipUnverifiedError } from "./native-runner-ownership.js";import type { AdapterRuntimeEvent } from "../../adapters/index.js";+vi.mock("node:fs", async (importOriginal) => {+ const actual = await importOriginal<typeof import("node:fs")>();+ return { ...actual, readFileSync: vi.fn(actual.readFileSync) };+});+const githubAccess = vi.hoisted(() => ({activate: vi.fn((_binding: { runId: string }) => vi.fn()),stop: vi.fn(async () => undefined),@@ -1171,6 +1177,54 @@ describe("remote provider pack manifest", () => {});});+describe("provider pack read diagnostics", () => {+ it.each([+ ["EACCES", "permission_denied"],+ ["EPERM", "permission_denied"],+ ["EIO", "io_error"],+ ])("reports %s without exposing the underlying filesystem message", (code, reason) => {+ const root = join(tmpdir(), "private-provider-pack");+ const manifestPath = join(root, "provider-pack.json");+ const cause = Object.assign(new Error(`${code}: cannot read ${manifestPath}`), {+ code,+ path: manifestPath,+ });+ vi.mocked(readFileSync).mockImplementationOnce(() => { throw cause; });+ let failure: Error | undefined;+ try { readRemoteProviderPackManifest(root); } catch (error) { failure = error as Error; }+ expect(readFileSync).toHaveBeenLastCalledWith(manifestPath, "utf8");+ expect(failure?.message).toBe(`runner_remote_provider_artifact_incompatible: provider-pack.json is unreadable (${reason})`);+ expect(failure?.message).not.toContain(root);+ expect(failure?.message).not.toContain(code);+ expect(failure?.cause).toBe(cause);+ });++ it("classifies a JSON null manifest as incompatible instead of a TypeError", async () => {+ const root = await mkdtemp(join(tmpdir(), "paperclip-null-pack-"));+ try {+ await writeFile(join(root, "provider-pack.json"), "null");+ expect(() => readRemoteProviderPackManifest(root)).toThrow(+ "runner_remote_provider_artifact_incompatible: provider pack pins or source revision do not match",+ );+ } finally { await rm(root, { recursive: true, force: true }); }+ });++ it.each(["missing", "invalid_json", "invalid_path_type"])("reports %s without putting the path in the terminal message", async (reason) => {+ const root = await mkdtemp(join(tmpdir(), "paperclip-private-pack-"));+ try {+ const manifestPath = join(root, "provider-pack.json");+ if (reason === "invalid_json") await writeFile(manifestPath, "{ private-invalid-json");+ if (reason === "invalid_path_type") await mkdir(manifestPath);+ let failure: Error | undefined;+ try { readRemoteProviderPackManifest(root); } catch (error) { failure = error as Error; }+ expect(failure?.message).toBe(`runner_remote_provider_artifact_incompatible: provider-pack.json is unreadable (${reason})`);+ expect(failure?.message).not.toContain(root);+ expect(failure?.message).not.toContain("private-invalid-json");+ expect(failure?.cause).toBeDefined();+ } finally { await rm(root, { recursive: true, force: true }); }+ });+});+describe("native harness persistence profiles", () => {const profile = (provider: Record<string, unknown>, driverKind: string) =>resolveNativeHarnessPersistenceProfile({@@ -1477,7 +1531,7 @@ describe("verified native harness backups", () => {},sourceProviderLeaseId: "sandbox-1",}),- ).toThrow("runner_harness_state_mismatch");+ ).toThrow("runner_harness_state_mismatch: backup_provider_identity_missing");} finally {await rm(root, { recursive: true, force: true });}@@ -9119,7 +9173,7 @@ describe("runnerd provider runtime wiring", () => {execution: currentExecution,runnerInstanceId: "runner-after-running-prior-scope",}),- ).rejects.toThrow("runner_state_identity_mismatch");+ ).rejects.toThrow("runner_state_identity_mismatch: prior_owner_active");await expect(access(scopedRoot)).resolves.toBeUndefined();await expect(access(join(stateBase, "quarantine"))).rejects.toThrow();expect(state.createBackend).not.toHaveBeenCalled();@@ -10691,7 +10745,7 @@ describe("runnerd provider runtime wiring", () => {}// Ambiguous ordinary recovery must still fail closed. Only the runtime's// explicitly admitted replacement may retire these prior-session backups.- await expect(prepareReplacement()).rejects.toThrow("runner_harness_state_mismatch");+ await expect(prepareReplacement()).rejects.toThrow("runner_harness_state_mismatch: backup_without_reusable_lease");await expect(backend.openReplacementSession!({identity: { runId: execution.binding.runId }, workingDirectory: execution.workspace.cwd,} as never, {} as never)).resolves.toBe(replacement);diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.tsindex 7b682232f8..1e75231a0b 100644--- a/server/src/services/native-runtime/native-session-executor.ts+++ b/server/src/services/native-runtime/native-session-executor.ts@@ -1691,7 +1691,7 @@ function migrateLegacyRunnerdStateRoot(input: {// A legacy path does not encode the full session scope. A mismatch may be// valid live state owned by another agent/workspace, so refusing the claim// is safe but moving that ambiguous directory is not.- throw new Error("runner_state_identity_mismatch");+ throw new Error("runner_state_identity_mismatch: legacy_owner_unverified");}if (exactRun &&@@ -1704,7 +1704,7 @@ function migrateLegacyRunnerdStateRoot(input: {)) {quarantineRunnerdStateRoot(input.legacy, "identity_indeterminate");- throw new Error("runner_state_identity_mismatch");+ throw new Error("runner_state_identity_mismatch: legacy_authority_indeterminate");}try {renameSync(input.legacy, input.scoped);@@ -1728,7 +1728,7 @@ function migrateLegacyRunnerdStateRoot(input: {durableIdentityMatchesSession(scopedIdentity, input.execution),);if (!exactScopedRun && !sameVerifiedPriorRun) {- throw new Error("runner_state_identity_mismatch");+ throw new Error("runner_state_identity_mismatch: migration_destination_owner_changed");}}return input.scoped;@@ -4573,7 +4573,7 @@ async function migrateRunnerdStateRootForExecution(input: {await recoverQuiescentRunnerdState({ ...input, scoped });}if (input.restartRecovery?.kind === "reattach_remote_runner" && !existsSync(scoped)) {- throw new Error("runner_state_identity_mismatch");+ throw new Error("runner_state_identity_mismatch: remote_reattach_root_missing");}if (existsSync(scoped)) {if (!isSafeNativeStateDirectory(scoped)) {@@ -4593,14 +4593,14 @@ async function migrateRunnerdStateRootForExecution(input: {input.restartRecovery?.kind !== "reattach_remote_runner") {quarantineRunnerdStateRoot(scoped, "identity_indeterminate");}- throw new Error("runner_state_identity_mismatch");+ throw new Error("runner_state_identity_mismatch: durable_identity_unreadable");}if (!durableIdentityMatchesSession(identity, input.execution)) {if (input.restartRecovery?.kind !== "reattach_existing_runner" &&input.restartRecovery?.kind !== "reattach_remote_runner") {quarantineRunnerdStateRoot(scoped, "identity_mismatch");}- throw new Error("runner_state_identity_mismatch");+ throw new Error("runner_state_identity_mismatch: session_scope_mismatch");}if (input.restartRecovery?.kind === "bootstrap_incomplete") {if (runnerdStateProvesIncompleteBootstrap(scoped)) {@@ -4610,7 +4610,7 @@ async function migrateRunnerdStateRootForExecution(input: {// Database evidence alone cannot distinguish a never-connected runner// from a partially-persisted provider bootstrap. Only the durable PRP// root can authorize a fresh bootstrap; anything else stays fail-closed.- throw new Error("runner_state_identity_mismatch");+ throw new Error("runner_state_identity_mismatch: bootstrap_not_proven_incomplete");}if (input.restartRecovery?.kind === "reattach_remote_runner") {await verifyRemoteRunnerReattachment({@@ -4631,7 +4631,7 @@ async function migrateRunnerdStateRootForExecution(input: {if (input.restartRecovery?.kind !== "reattach_existing_runner") {quarantineRunnerdStateRoot(scoped, "identity_indeterminate");}- throw new Error("runner_state_identity_mismatch");+ throw new Error("runner_state_identity_mismatch: authority_indeterminate");}} else {const verification = await verifyPriorRunnerdStateForSessionScope({@@ -4654,7 +4654,7 @@ async function migrateRunnerdStateRootForExecution(input: {: "identity_indeterminate",);}- throw new Error("runner_state_identity_mismatch");+ throw new Error(`runner_state_identity_mismatch: prior_owner_${verification}`);}}return;@@ -4672,7 +4672,7 @@ async function migrateRunnerdStateRootForExecution(input: {// Unlike the full-scope target above, this legacy name can legitimately// belong to another scope. Leave it in place for its owner and fail the// attempted migration visibly.- throw new Error("runner_state_identity_mismatch");+ throw new Error("runner_state_identity_mismatch: legacy_session_scope_mismatch");}let verifiedPriorRunId: string | undefined;if (!durableIdentityMatchesExecution(identity, input.execution)) {@@ -4695,7 +4695,7 @@ async function migrateRunnerdStateRootForExecution(input: {// untouched because the legacy name may still belong to them.quarantineRunnerdStateRoot(legacy, "identity_indeterminate");}- throw new Error("runner_state_identity_mismatch");+ throw new Error(`runner_state_identity_mismatch: legacy_prior_owner_${verification}`);}verifiedPriorRunId = identity.runId;}@@ -4745,7 +4745,7 @@ async function recoverQuiescentRunnerdState(input: {) {// Do not roll back to an older valid checkpoint when a newer quarantined// root contains unconfirmed work, even if the newer root is unreadable.- throw new Error("runner_state_identity_mismatch");+ throw new Error("runner_state_identity_mismatch: quarantine_candidates_ambiguous");}const verified: Array<{root: string;@@ -4932,7 +4932,7 @@ async function recoverQuiescentRunnerdState(input: {) {// Known provider history is not permission to start a replacement when// recovery cannot prove a unique, settled owner.- throw new Error("runner_state_identity_mismatch");+ throw new Error("runner_state_identity_mismatch: quarantine_owner_unverified");}return;}@@ -4951,14 +4951,14 @@ async function recoverQuiescentRunnerdState(input: {"recovery_state_too_large",).toString("utf8") !== expected) {- throw new Error("runner_state_identity_mismatch");+ throw new Error("runner_state_identity_mismatch: recovery_evidence_changed");}}if (!localProcessDefinitelyGone(candidate.processPid) ||!localProcessDefinitelyGone(candidate.processGroupId, true)) {- throw new Error("runner_state_identity_mismatch");+ throw new Error("runner_state_identity_mismatch: recovery_process_not_gone");}if (candidate.root !== input.scoped) {if (existsSync(input.scoped)) {@@ -5673,12 +5673,12 @@ export function buildNativeHarnessBackupManifest(input: {completedAt?: string;}): NativeHarnessBackupManifest {if (!providerSessionIdentityIsPresent(input.providerSessionIdentity)) {- throw new Error("runner_harness_state_mismatch");+ throw new Error("runner_harness_state_mismatch: backup_provider_identity_missing");}const profile = resolveNativeHarnessPersistenceProfile(input.execution);const directories = profile.directories.map((directory) => {const path = resolve(input.backupRoot, directory.name);- if (!existsSync(path)) throw new Error("runner_harness_state_mismatch");+ if (!existsSync(path)) throw new Error("runner_harness_state_mismatch: backup_directory_missing");return { name: directory.name, ...digestBackupDirectory(path) };});return {@@ -9364,14 +9364,22 @@ export function readRemoteProviderPackManifest(readFileSync(resolve(packRoot, "provider-pack.json"), "utf8"),) as RemoteProviderPackManifest;} catch (error) {+ // The terminal run report keeps the outer message, not the cause chain.+ // Keep a bounded reason there; raw filesystem errors include private paths.+ const code = (error as NodeJS.ErrnoException | null)?.code;+ const reason = error instanceof SyntaxError ? "invalid_json"+ : code === "ENOENT" ? "missing"+ : code === "EACCES" || code === "EPERM" ? "permission_denied"+ : code === "EISDIR" || code === "ENOTDIR" ? "invalid_path_type"+ : "io_error";throw new Error(- "runner_remote_provider_artifact_incompatible: provider-pack.json is unreadable",+ `runner_remote_provider_artifact_incompatible: provider-pack.json is unreadable (${reason})`,{ cause: error },);}const payload = manifest?.payload;if (- manifest.schema !== REMOTE_PROVIDER_PACK_SCHEMA ||+ manifest?.schema !== REMOTE_PROVIDER_PACK_SCHEMA ||!payload ||canonicalJson(payload.pins) !== canonicalJson(REMOTE_PROVIDER_PACK_PINS) ||canonicalJson(payload.acpxProfileDigests) !==@@ -11543,7 +11551,7 @@ async function createRunnerdBackendWithinSessionClaim(async () => {for (const directory of persistenceProfile.directories) {const targetPath = remotePersistencePath(directory);- if (!targetPath) throw new Error("runner_harness_state_mismatch");+ if (!targetPath) throw new Error("runner_harness_state_mismatch: restore_target_unavailable");await stageRemoteRunnerDirectory({target: remoteTarget,runner: remoteCommandRunner,@@ -11571,7 +11579,7 @@ async function createRunnerdBackendWithinSessionClaim(canonicalJson(restored.providerSessionIdentity) !==canonicalJson(backup.manifest.providerSessionIdentity)) {- throw new Error("runner_harness_state_mismatch");+ throw new Error("runner_harness_state_mismatch: restored_provider_identity_changed");}// A deliberately non-reusable environment receives a fresh provider lease// for every turn. Stamp that new lease as soon as the verified host backup@@ -11586,7 +11594,7 @@ async function createRunnerdBackendWithinSessionClaim(for (const directory of persistenceProfile.directories) {if (directory.location !== "filesystem") continue;const targetPath = remotePersistencePath(directory);- if (!targetPath) throw new Error("runner_harness_state_mismatch");+ if (!targetPath) throw new Error("runner_harness_state_mismatch: bootstrap_target_unavailable");const escapedTarget = targetPath.replaceAll("'", "'\\''");const created = await remoteCommandRunner.execute({command: "sh",@@ -11766,7 +11774,7 @@ async function createRunnerdBackendWithinSessionClaim(// A continuation that has a durable backup but no recorded reusable// lease was not provider-confirmed lost. Never silently create a new// provider session from that ambiguous state.- throw new Error("runner_harness_state_mismatch");+ throw new Error("runner_harness_state_mismatch: backup_without_reusable_lease");}}} else if (remoteTarget && remoteCommandRunner) {@@ -11950,7 +11958,7 @@ async function createRunnerdBackendWithinSessionClaim(!verified.runnerState ||!verified.providerSessionIdentity) {- throw new Error("runner_harness_state_mismatch");+ throw new Error("runner_harness_state_mismatch: checkpoint_identity_incomplete");}const providerSessionIdentity = verified.providerSessionIdentity;@@ -11965,7 +11973,7 @@ async function createRunnerdBackendWithinSessionClaim(for (const directory of persistenceProfile.directories) {const sourcePath = remotePersistencePath(directory);if (!sourcePath)- throw new Error("runner_harness_state_mismatch");+ throw new Error("runner_harness_state_mismatch: checkpoint_source_unavailable");const targetPath = resolve(pendingRoot, directory.name);await syncRemoteRunnerDirectoryOut({runner: remoteCommandRunner,@@ -11975,7 +11983,7 @@ async function createRunnerdBackendWithinSessionClaim(excludeEntries: directory.excludeEntries,});if (!existsSync(targetPath)) {- throw new Error("runner_harness_state_mismatch");+ throw new Error("runner_harness_state_mismatch: checkpoint_directory_missing");}}const manifest = buildNativeHarnessBackupManifest({