Skip to content

The open-source app everyone uses to manage agents at work

fix: retain diagnostic reasons for native runner failures (#14481)

Devin Foleyad1f7e98ead6
diff --git a/doc/observability.md b/doc/observability.md
index fb2a8b15ee..491e8e35c8 100644
--- a/doc/observability.md
+++ b/doc/observability.md
@@ -486,6 +486,17 @@ These fields contain build identifiers; they add no tenant or user identity.
`errorCode`, and `agentAdapter`. The server redacts the error message and
the error code before it sends the event.
+Native runner identity and harness failures retain their existing error prefixes.
+Their terminal messages now include a bounded guard reason, such as
+`session_scope_mismatch`, `durable_identity_unreadable`, or
+`backup_without_reusable_lease`. Provider-pack read failures distinguish a missing
+file, invalid JSON, permission denial, invalid path type, and other I/O errors.
+These reasons contain no session identifiers, provider output, or filesystem
+paths. They help diagnose recurrence; they do not authorize a retry, quarantine,
+replacement, or a weaker identity check. Existing chat recovery recognizes the
+same failure category with or without a reason suffix; it still requires the
+exact cleanup receipt, checkpoint, and absence of provider work.
+
**Server events the default integrations add**
- `OnUncaughtException` — each uncaught exception on the main thread, at
diff --git a/server/src/__tests__/chat-channels.integration.test.ts b/server/src/__tests__/chat-channels.integration.test.ts
index 85d968cbc1..d04154ee97 100644
--- a/server/src/__tests__/chat-channels.integration.test.ts
+++ b/server/src/__tests__/chat-channels.integration.test.ts
@@ -56157,9 +56157,13 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
"leased",
"wrong_thread",
"source_edited",
- ])(
- "retries only the original pre-provider Telegram request after exact cleanup: %s",
- async (mode) => {
+ "different_error",
+ ].flatMap((mode) => [
+ "runner_state_identity_mismatch",
+ "runner_state_identity_mismatch: prior_owner_active",
+ ].map((errorMessage) => ({ mode, errorMessage }))))(
+ "retries only the original pre-provider Telegram request after exact cleanup: $mode ($errorMessage)",
+ async ({ mode, errorMessage }) => {
const context = await committedChatResponseRecoveryFixture("telegram");
const providerAccount =
mode === "null_account"
@@ -56286,7 +56290,9 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
agentId: context.fixture.assignedAgentId,
status: "failed",
errorCode: "adapter_failed",
- error: "runner_state_identity_mismatch",
+ error: mode === "different_error"
+ ? errorMessage.replace("runner_state_identity_mismatch", "runner_state_identity_mismatch_other")
+ : errorMessage,
finishedAt: new Date(),
wakeupRequestId: action.id,
runtimeMode: "native",
diff --git a/server/src/services/chat-channels.ts b/server/src/services/chat-channels.ts
index dd3694dc6a..a4c67d20b3 100644
--- a/server/src/services/chat-channels.ts
+++ b/server/src/services/chat-channels.ts
@@ -12006,7 +12006,11 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {
run.nativeIssueId === issueId &&
run.status === "failed" &&
run.errorCode === "adapter_failed" &&
- run.error === "runner_state_identity_mismatch" &&
+ // A diagnostic reason does not change this failure category. The exact
+ // checkpoint, cleanup receipt, and no-provider-work proofs below still
+ // decide whether the original request can be retried.
+ (run.error === "runner_state_identity_mismatch" ||
+ run.error?.startsWith("runner_state_identity_mismatch: ")) &&
run.nativePhase === "observed" &&
coordinator.phase === "observed" &&
coordinator.attempt === 0 &&
diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts
index d533986d90..b6d7316911 100644
--- a/server/src/services/native-runtime/native-session-executor.test.ts
+++ b/server/src/services/native-runtime/native-session-executor.test.ts
@@ -1,4 +1,5 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { readFileSync } from "node:fs";
import {
access,
cp,
@@ -56,6 +57,11 @@ import { buildNativeHeartbeatPreparationSpans } from "./native-run-trace.js";
import { NativeRunnerOwnershipUnverifiedError } from "./native-runner-ownership.js";
import type { AdapterRuntimeEvent } from "../../adapters/index.js";
+vi.mock("node:fs", async (importOriginal) => {
+ const actual = await importOriginal<typeof import("node:fs")>();
+ return { ...actual, readFileSync: vi.fn(actual.readFileSync) };
+});
+
const githubAccess = vi.hoisted(() => ({
activate: vi.fn((_binding: { runId: string }) => vi.fn()),
stop: vi.fn(async () => undefined),
@@ -1171,6 +1177,54 @@ describe("remote provider pack manifest", () => {
});
});
+describe("provider pack read diagnostics", () => {
+ it.each([
+ ["EACCES", "permission_denied"],
+ ["EPERM", "permission_denied"],
+ ["EIO", "io_error"],
+ ])("reports %s without exposing the underlying filesystem message", (code, reason) => {
+ const root = join(tmpdir(), "private-provider-pack");
+ const manifestPath = join(root, "provider-pack.json");
+ const cause = Object.assign(new Error(`${code}: cannot read ${manifestPath}`), {
+ code,
+ path: manifestPath,
+ });
+ vi.mocked(readFileSync).mockImplementationOnce(() => { throw cause; });
+ let failure: Error | undefined;
+ try { readRemoteProviderPackManifest(root); } catch (error) { failure = error as Error; }
+ expect(readFileSync).toHaveBeenLastCalledWith(manifestPath, "utf8");
+ expect(failure?.message).toBe(`runner_remote_provider_artifact_incompatible: provider-pack.json is unreadable (${reason})`);
+ expect(failure?.message).not.toContain(root);
+ expect(failure?.message).not.toContain(code);
+ expect(failure?.cause).toBe(cause);
+ });
+
+ it("classifies a JSON null manifest as incompatible instead of a TypeError", async () => {
+ const root = await mkdtemp(join(tmpdir(), "paperclip-null-pack-"));
+ try {
+ await writeFile(join(root, "provider-pack.json"), "null");
+ expect(() => readRemoteProviderPackManifest(root)).toThrow(
+ "runner_remote_provider_artifact_incompatible: provider pack pins or source revision do not match",
+ );
+ } finally { await rm(root, { recursive: true, force: true }); }
+ });
+
+ it.each(["missing", "invalid_json", "invalid_path_type"])("reports %s without putting the path in the terminal message", async (reason) => {
+ const root = await mkdtemp(join(tmpdir(), "paperclip-private-pack-"));
+ try {
+ const manifestPath = join(root, "provider-pack.json");
+ if (reason === "invalid_json") await writeFile(manifestPath, "{ private-invalid-json");
+ if (reason === "invalid_path_type") await mkdir(manifestPath);
+ let failure: Error | undefined;
+ try { readRemoteProviderPackManifest(root); } catch (error) { failure = error as Error; }
+ expect(failure?.message).toBe(`runner_remote_provider_artifact_incompatible: provider-pack.json is unreadable (${reason})`);
+ expect(failure?.message).not.toContain(root);
+ expect(failure?.message).not.toContain("private-invalid-json");
+ expect(failure?.cause).toBeDefined();
+ } finally { await rm(root, { recursive: true, force: true }); }
+ });
+});
+
describe("native harness persistence profiles", () => {
const profile = (provider: Record<string, unknown>, driverKind: string) =>
resolveNativeHarnessPersistenceProfile({
@@ -1477,7 +1531,7 @@ describe("verified native harness backups", () => {
},
sourceProviderLeaseId: "sandbox-1",
}),
- ).toThrow("runner_harness_state_mismatch");
+ ).toThrow("runner_harness_state_mismatch: backup_provider_identity_missing");
} finally {
await rm(root, { recursive: true, force: true });
}
@@ -9119,7 +9173,7 @@ describe("runnerd provider runtime wiring", () => {
execution: currentExecution,
runnerInstanceId: "runner-after-running-prior-scope",
}),
- ).rejects.toThrow("runner_state_identity_mismatch");
+ ).rejects.toThrow("runner_state_identity_mismatch: prior_owner_active");
await expect(access(scopedRoot)).resolves.toBeUndefined();
await expect(access(join(stateBase, "quarantine"))).rejects.toThrow();
expect(state.createBackend).not.toHaveBeenCalled();
@@ -10691,7 +10745,7 @@ describe("runnerd provider runtime wiring", () => {
}
// Ambiguous ordinary recovery must still fail closed. Only the runtime's
// explicitly admitted replacement may retire these prior-session backups.
- await expect(prepareReplacement()).rejects.toThrow("runner_harness_state_mismatch");
+ await expect(prepareReplacement()).rejects.toThrow("runner_harness_state_mismatch: backup_without_reusable_lease");
await expect(backend.openReplacementSession!({
identity: { runId: execution.binding.runId }, workingDirectory: execution.workspace.cwd,
} as never, {} as never)).resolves.toBe(replacement);
diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts
index 7b682232f8..1e75231a0b 100644
--- a/server/src/services/native-runtime/native-session-executor.ts
+++ b/server/src/services/native-runtime/native-session-executor.ts
@@ -1691,7 +1691,7 @@ function migrateLegacyRunnerdStateRoot(input: {
// A legacy path does not encode the full session scope. A mismatch may be
// valid live state owned by another agent/workspace, so refusing the claim
// is safe but moving that ambiguous directory is not.
- throw new Error("runner_state_identity_mismatch");
+ throw new Error("runner_state_identity_mismatch: legacy_owner_unverified");
}
if (
exactRun &&
@@ -1704,7 +1704,7 @@ function migrateLegacyRunnerdStateRoot(input: {
)
) {
quarantineRunnerdStateRoot(input.legacy, "identity_indeterminate");
- throw new Error("runner_state_identity_mismatch");
+ throw new Error("runner_state_identity_mismatch: legacy_authority_indeterminate");
}
try {
renameSync(input.legacy, input.scoped);
@@ -1728,7 +1728,7 @@ function migrateLegacyRunnerdStateRoot(input: {
durableIdentityMatchesSession(scopedIdentity, input.execution),
);
if (!exactScopedRun && !sameVerifiedPriorRun) {
- throw new Error("runner_state_identity_mismatch");
+ throw new Error("runner_state_identity_mismatch: migration_destination_owner_changed");
}
}
return input.scoped;
@@ -4573,7 +4573,7 @@ async function migrateRunnerdStateRootForExecution(input: {
await recoverQuiescentRunnerdState({ ...input, scoped });
}
if (input.restartRecovery?.kind === "reattach_remote_runner" && !existsSync(scoped)) {
- throw new Error("runner_state_identity_mismatch");
+ throw new Error("runner_state_identity_mismatch: remote_reattach_root_missing");
}
if (existsSync(scoped)) {
if (!isSafeNativeStateDirectory(scoped)) {
@@ -4593,14 +4593,14 @@ async function migrateRunnerdStateRootForExecution(input: {
input.restartRecovery?.kind !== "reattach_remote_runner") {
quarantineRunnerdStateRoot(scoped, "identity_indeterminate");
}
- throw new Error("runner_state_identity_mismatch");
+ throw new Error("runner_state_identity_mismatch: durable_identity_unreadable");
}
if (!durableIdentityMatchesSession(identity, input.execution)) {
if (input.restartRecovery?.kind !== "reattach_existing_runner" &&
input.restartRecovery?.kind !== "reattach_remote_runner") {
quarantineRunnerdStateRoot(scoped, "identity_mismatch");
}
- throw new Error("runner_state_identity_mismatch");
+ throw new Error("runner_state_identity_mismatch: session_scope_mismatch");
}
if (input.restartRecovery?.kind === "bootstrap_incomplete") {
if (runnerdStateProvesIncompleteBootstrap(scoped)) {
@@ -4610,7 +4610,7 @@ async function migrateRunnerdStateRootForExecution(input: {
// Database evidence alone cannot distinguish a never-connected runner
// from a partially-persisted provider bootstrap. Only the durable PRP
// root can authorize a fresh bootstrap; anything else stays fail-closed.
- throw new Error("runner_state_identity_mismatch");
+ throw new Error("runner_state_identity_mismatch: bootstrap_not_proven_incomplete");
}
if (input.restartRecovery?.kind === "reattach_remote_runner") {
await verifyRemoteRunnerReattachment({
@@ -4631,7 +4631,7 @@ async function migrateRunnerdStateRootForExecution(input: {
if (input.restartRecovery?.kind !== "reattach_existing_runner") {
quarantineRunnerdStateRoot(scoped, "identity_indeterminate");
}
- throw new Error("runner_state_identity_mismatch");
+ throw new Error("runner_state_identity_mismatch: authority_indeterminate");
}
} else {
const verification = await verifyPriorRunnerdStateForSessionScope({
@@ -4654,7 +4654,7 @@ async function migrateRunnerdStateRootForExecution(input: {
: "identity_indeterminate",
);
}
- throw new Error("runner_state_identity_mismatch");
+ throw new Error(`runner_state_identity_mismatch: prior_owner_${verification}`);
}
}
return;
@@ -4672,7 +4672,7 @@ async function migrateRunnerdStateRootForExecution(input: {
// Unlike the full-scope target above, this legacy name can legitimately
// belong to another scope. Leave it in place for its owner and fail the
// attempted migration visibly.
- throw new Error("runner_state_identity_mismatch");
+ throw new Error("runner_state_identity_mismatch: legacy_session_scope_mismatch");
}
let verifiedPriorRunId: string | undefined;
if (!durableIdentityMatchesExecution(identity, input.execution)) {
@@ -4695,7 +4695,7 @@ async function migrateRunnerdStateRootForExecution(input: {
// untouched because the legacy name may still belong to them.
quarantineRunnerdStateRoot(legacy, "identity_indeterminate");
}
- throw new Error("runner_state_identity_mismatch");
+ throw new Error(`runner_state_identity_mismatch: legacy_prior_owner_${verification}`);
}
verifiedPriorRunId = identity.runId;
}
@@ -4745,7 +4745,7 @@ async function recoverQuiescentRunnerdState(input: {
) {
// Do not roll back to an older valid checkpoint when a newer quarantined
// root contains unconfirmed work, even if the newer root is unreadable.
- throw new Error("runner_state_identity_mismatch");
+ throw new Error("runner_state_identity_mismatch: quarantine_candidates_ambiguous");
}
const verified: Array<{
root: string;
@@ -4932,7 +4932,7 @@ async function recoverQuiescentRunnerdState(input: {
) {
// Known provider history is not permission to start a replacement when
// recovery cannot prove a unique, settled owner.
- throw new Error("runner_state_identity_mismatch");
+ throw new Error("runner_state_identity_mismatch: quarantine_owner_unverified");
}
return;
}
@@ -4951,14 +4951,14 @@ async function recoverQuiescentRunnerdState(input: {
"recovery_state_too_large",
).toString("utf8") !== expected
) {
- throw new Error("runner_state_identity_mismatch");
+ throw new Error("runner_state_identity_mismatch: recovery_evidence_changed");
}
}
if (
!localProcessDefinitelyGone(candidate.processPid) ||
!localProcessDefinitelyGone(candidate.processGroupId, true)
) {
- throw new Error("runner_state_identity_mismatch");
+ throw new Error("runner_state_identity_mismatch: recovery_process_not_gone");
}
if (candidate.root !== input.scoped) {
if (existsSync(input.scoped)) {
@@ -5673,12 +5673,12 @@ export function buildNativeHarnessBackupManifest(input: {
completedAt?: string;
}): NativeHarnessBackupManifest {
if (!providerSessionIdentityIsPresent(input.providerSessionIdentity)) {
- throw new Error("runner_harness_state_mismatch");
+ throw new Error("runner_harness_state_mismatch: backup_provider_identity_missing");
}
const profile = resolveNativeHarnessPersistenceProfile(input.execution);
const directories = profile.directories.map((directory) => {
const path = resolve(input.backupRoot, directory.name);
- if (!existsSync(path)) throw new Error("runner_harness_state_mismatch");
+ if (!existsSync(path)) throw new Error("runner_harness_state_mismatch: backup_directory_missing");
return { name: directory.name, ...digestBackupDirectory(path) };
});
return {
@@ -9364,14 +9364,22 @@ export function readRemoteProviderPackManifest(
readFileSync(resolve(packRoot, "provider-pack.json"), "utf8"),
) as RemoteProviderPackManifest;
} catch (error) {
+ // The terminal run report keeps the outer message, not the cause chain.
+ // Keep a bounded reason there; raw filesystem errors include private paths.
+ const code = (error as NodeJS.ErrnoException | null)?.code;
+ const reason = error instanceof SyntaxError ? "invalid_json"
+ : code === "ENOENT" ? "missing"
+ : code === "EACCES" || code === "EPERM" ? "permission_denied"
+ : code === "EISDIR" || code === "ENOTDIR" ? "invalid_path_type"
+ : "io_error";
throw new Error(
- "runner_remote_provider_artifact_incompatible: provider-pack.json is unreadable",
+ `runner_remote_provider_artifact_incompatible: provider-pack.json is unreadable (${reason})`,
{ cause: error },
);
}
const payload = manifest?.payload;
if (
- manifest.schema !== REMOTE_PROVIDER_PACK_SCHEMA ||
+ manifest?.schema !== REMOTE_PROVIDER_PACK_SCHEMA ||
!payload ||
canonicalJson(payload.pins) !== canonicalJson(REMOTE_PROVIDER_PACK_PINS) ||
canonicalJson(payload.acpxProfileDigests) !==
@@ -11543,7 +11551,7 @@ async function createRunnerdBackendWithinSessionClaim(
async () => {
for (const directory of persistenceProfile.directories) {
const targetPath = remotePersistencePath(directory);
- if (!targetPath) throw new Error("runner_harness_state_mismatch");
+ if (!targetPath) throw new Error("runner_harness_state_mismatch: restore_target_unavailable");
await stageRemoteRunnerDirectory({
target: remoteTarget,
runner: remoteCommandRunner,
@@ -11571,7 +11579,7 @@ async function createRunnerdBackendWithinSessionClaim(
canonicalJson(restored.providerSessionIdentity) !==
canonicalJson(backup.manifest.providerSessionIdentity)
) {
- throw new Error("runner_harness_state_mismatch");
+ throw new Error("runner_harness_state_mismatch: restored_provider_identity_changed");
}
// A deliberately non-reusable environment receives a fresh provider lease
// for every turn. Stamp that new lease as soon as the verified host backup
@@ -11586,7 +11594,7 @@ async function createRunnerdBackendWithinSessionClaim(
for (const directory of persistenceProfile.directories) {
if (directory.location !== "filesystem") continue;
const targetPath = remotePersistencePath(directory);
- if (!targetPath) throw new Error("runner_harness_state_mismatch");
+ if (!targetPath) throw new Error("runner_harness_state_mismatch: bootstrap_target_unavailable");
const escapedTarget = targetPath.replaceAll("'", "'\\''");
const created = await remoteCommandRunner.execute({
command: "sh",
@@ -11766,7 +11774,7 @@ async function createRunnerdBackendWithinSessionClaim(
// A continuation that has a durable backup but no recorded reusable
// lease was not provider-confirmed lost. Never silently create a new
// provider session from that ambiguous state.
- throw new Error("runner_harness_state_mismatch");
+ throw new Error("runner_harness_state_mismatch: backup_without_reusable_lease");
}
}
} else if (remoteTarget && remoteCommandRunner) {
@@ -11950,7 +11958,7 @@ async function createRunnerdBackendWithinSessionClaim(
!verified.runnerState ||
!verified.providerSessionIdentity
) {
- throw new Error("runner_harness_state_mismatch");
+ throw new Error("runner_harness_state_mismatch: checkpoint_identity_incomplete");
}
const providerSessionIdentity = verified.providerSessionIdentity;
@@ -11965,7 +11973,7 @@ async function createRunnerdBackendWithinSessionClaim(
for (const directory of persistenceProfile.directories) {
const sourcePath = remotePersistencePath(directory);
if (!sourcePath)
- throw new Error("runner_harness_state_mismatch");
+ throw new Error("runner_harness_state_mismatch: checkpoint_source_unavailable");
const targetPath = resolve(pendingRoot, directory.name);
await syncRemoteRunnerDirectoryOut({
runner: remoteCommandRunner,
@@ -11975,7 +11983,7 @@ async function createRunnerdBackendWithinSessionClaim(
excludeEntries: directory.excludeEntries,
});
if (!existsSync(targetPath)) {
- throw new Error("runner_harness_state_mismatch");
+ throw new Error("runner_harness_state_mismatch: checkpoint_directory_missing");
}
}
const manifest = buildNativeHarnessBackupManifest({