The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
fix(native): verify warm ownership before configuration rotation
Dottaafe5b3418dbd
diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.mdindex b4b8437a28..65f279c166 100644--- a/doc/DEVELOPING.md+++ b/doc/DEVELOPING.md@@ -1892,6 +1892,14 @@ Only successful turns retain a live warm provider session. A structured failedor cancelled result must retire that session and collect its managed files beforeheartbeat stops the reusable sandbox. A later retry can resume the same sandboxwithout inheriting the stopped provider transport.++An idle warm owner proves only the exact prior run in the same company, session,+workspace and environment. Mutable configuration compatibility is checked+separately: changed settings retire the owned provider before a replacement can+execute. A copied older owner record, an active or missing owner, and a different+environment cannot use this in-memory proof. Database terminal status alone is+not proof that an unsuspended remote runner can be adopted.+The shared lease-release boundary rechecks the durable run status for recoveryand ordinary teardown. Successful workspace copy-back alone must not keep afailed turn's sandbox running.diff --git a/doc/plans/2026-10-15-native-warm-owner-proof.md b/doc/plans/2026-10-15-native-warm-owner-proof.mdnew file mode 100644index 0000000000..ac74ccb8a3--- /dev/null+++ b/doc/plans/2026-10-15-native-warm-owner-proof.md@@ -0,0 +1,38 @@+# Exact warm-owner proof across native configuration changes++A follow-up can fail with `prior_owner_terminal_state_indeterminate` before+provider execution even when its prior heartbeat succeeded. The old predicate+required the next configuration digest to match before recognizing the idle+owner; this prevented the existing compatibility step from retiring a process+whose configuration changed.++Keep the session cache, durable-root admission and compatibility retirement in+their current coordinator. They share process and instruction-copy ownership;+this fix does not move their lifetime or weaken their gates. Cache each actual+owner's heartbeat run ID and require it to match the durable prior-run identity.+Keep company/session/workspace/environment checks and the terminal database+check. Let the later compatibility step decide reuse versus retirement, and+continue to reject unconfirmed retirement before another provider executes.++Verification is through the production session entry point with scripted native+backends and durable local owner files. The six existing selected controls pass.+The initial three-case reproduction has one pass and two failures at the same+prior-owner guard; after the fix all three pass. Eight focused controls now cover+unchanged reuse through three turns, changed settings, changed runtime context,+failed retirement, stale recorded identity, active owner, missing owner and a+different environment. The runtime-context case uses the current v5 input and+fails on the baseline at the same prior-owner guard before passing the repair. Replacing the exact+run-ID comparison with an any-owner check makes the stale-owner control fail;+the source is restored afterward.++The first complete affected executor and resume suites passed 764 cases before+the additional v5 runtime-context case. Full typecheck initially found an+optional-argument error in the test database wrapper; after that correction,+full typecheck and build passed. The v5 fixture expansion then exposed a legacy+v1-only type on the shared database fixture. Its implementation only uses the+common binding, so it now accepts the full native input union. After integrating the separate idle-expiry admission repair from #16432, the+complete executor and resume suites passed 767 cases with zero selected skips.+Full repository typecheck and build passed on the same frozen code and tests. Scripted-provider results do not establish real Daytona recovery,+managed subscription behavior or hosted latency. Recheck the failed hosted+continuation after normal deployment, preserving its existing recovery evidence.+The broader response-latency target remains open independently of this repair.diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.tsindex fac9f42011..572e126883 100644--- a/server/src/services/native-runtime/native-session-executor.test.ts+++ b/server/src/services/native-runtime/native-session-executor.test.ts@@ -35,6 +35,7 @@ import {type Db,} from "@paperclipai/db";import {+ canonicalNativeRuntimeContextDigest,CURSOR_DISTRIBUTION_PINS,QUALIFIED_ACPX_PROFILES,acpxRuntimeSessionDirectoryName,@@ -5158,7 +5159,7 @@ type LeaseCoordinator = {};function leaseDb(- boundExecution: NativeExecutionInputV1 = execution,+ boundExecution: NativeExecutionInput = execution,coordinatorOverrides: Partial<LeaseCoordinator> = {},runResultJson: Record<string, unknown> = {},updates: Array<{ table: unknown; values: Record<string, unknown> }> = [],@@ -9727,7 +9728,7 @@ describe("runnerd provider runtime wiring", () => {};}- function priorNativeRunDb(priorExecution: NativeExecutionInputV1, status = "succeeded"): Db {+ function priorNativeRunDb(priorExecution: NativeExecutionInput, status = "succeeded"): Db {return {select: () => ({ from: () => ({ where: () => ({ limit: async () => [{status,@@ -9736,7 +9737,7 @@ describe("runnerd provider runtime wiring", () => {} as unknown as Db;}- async function initializeScopedRunnerRoot(current: NativeExecutionInputV1, runnerInstanceId: string) {+ async function initializeScopedRunnerRoot(current: NativeExecutionInput, runnerInstanceId: string) {state.createBackend.mockClear();state.createTransport.mockClear();await createRunnerdBackend({ db: leaseDb(current), execution: current, runnerInstanceId });@@ -9771,6 +9772,107 @@ describe("runnerd provider runtime wiring", () => {await rm(isolatedStateDirectory, { recursive: true, force: true });});+ it.each(["unchanged", "changed", "runtime-context", "retirement-failed", "stale-owner", "active-owner", "missing-owner", "foreign-environment"] as const)("rotates an owned warm remote runner before checking changed configuration: %s", async scenario => {+ const first = { ...scopedNativeExecution(`warm-admission-${scenario}`),+ schema: "paperclip.native-execution-input.v5" as const,+ provider: { kind: "codex" as const, model: null, approvalPolicy: "never" as const },+ executionMode: "default" as const, planningContext: null, runtimeContext: nativeRuntimeContextFixture() };+ first.session.lifecyclePolicy = { mode: "warm", idleTimeoutMs: 60_000 };+ const next = { ...first, binding: { ...first.binding, runId: `${first.binding.runId}-next` },+ session: { ...first.session, lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: ["unchanged", "runtime-context", "stale-owner", "missing-owner"].includes(scenario) ? 60_000 : 90_000 } } };+ if (scenario === "runtime-context") {+ next.runtimeContext = { ...first.runtimeContext,+ mcp: { ...first.runtimeContext.mcp, digest: "1".repeat(64) } };+ next.runtimeContext.aggregateDigest = canonicalNativeRuntimeContextDigest(next.runtimeContext);+ }+ const environmentId = `warm-admission-${scenario}`;+ const target = { kind: "remote" as const, transport: "sandbox" as const, environmentId,+ remoteCwd: "/home/daytona/workspace", runner: { execute: vi.fn() } };+ const identity = { runId: first.binding.runId, normalizedSessionId: first.session.normalizedSessionId,+ runnerInstanceId: `runner-${scenario}`, environmentLeaseId: first.binding.executionWorkspaceId };+ const root = await initializeScopedRunnerRoot(first, identity.runnerInstanceId);+ await writeRunnerAuthority(root, identity, "ready");+ const order: string[] = [];+ let failRetirement = scenario === "retirement-failed";+ const firstSession = { close: vi.fn(async () => {+ order.push("retire");+ if (failRetirement) throw new Error("retirement unconfirmed");+ await writeRunnerAuthority(root, identity, "suspended");+ }) };+ const nextSession = { close: vi.fn(async () => undefined) };+ const result = { result: { summary: "done" }, terminal: { runTerminalState: "succeeded" },+ turnId: "turn", normalizedSessionId: first.session.normalizedSessionId,+ providerSessionId: "provider", driverKind: "test", driverVersion: "1",+ nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null };+ state.execute.mockReset().mockImplementationOnce(async options => {+ await options.onSession?.(firstSession);+ return result;+ }).mockImplementationOnce(async options => {+ order.push("execute-next");+ expect(options.existingSession).toBe(scenario === "unchanged" ? firstSession : undefined);+ await options.onSession?.(scenario === "unchanged" ? firstSession : nextSession);+ return result;+ });+ const older = { ...first, binding: { ...first.binding, runId: `${first.binding.runId}-older` } };+ const nextDb = leaseDb(next);+ const ordinarySelect = nextDb.select.bind(nextDb);+ nextDb.select = ((fields?: Record<string, unknown>) => fields?.status === heartbeatRuns.status+ && fields?.runnerProfileJson === heartbeatRuns.runnerProfileJson+ ? priorNativeRunDb(scenario === "stale-owner" ? older : first, scenario === "active-owner" ? "running" : "succeeded").select(fields as never)+ : ordinarySelect(fields as never)) as Db["select"];+ try {+ await executePaperclipNativeSession({ db: leaseDb(first), execution: first,+ runnerInstanceId: identity.runnerInstanceId, useRunnerd: true, runnerExecutionTarget: target as never });+ if (scenario === "stale-owner") await writeRunnerAuthority(root, { ...identity, runId: older.binding.runId }, "ready");+ if (scenario === "missing-owner") {+ await closeWarmNativeSessionsForEnvironment({ environmentId, reason: "fixture owner loss" });+ await writeRunnerAuthority(root, identity, "ready");+ order.length = 0;+ firstSession.close.mockClear();+ }+ const continuation = executePaperclipNativeSession({ db: nextDb, execution: next,+ runnerInstanceId: `runner-next-${scenario}`, useRunnerd: true,+ runnerExecutionTarget: (scenario === "foreign-environment" ? { ...target, environmentId: `${environmentId}-other` } : target) as never });+ if (["stale-owner", "active-owner", "missing-owner", "foreign-environment"].includes(scenario)) {+ await expect(continuation).rejects.toThrow(scenario === "active-owner"+ ? "prior_owner_active" : "prior_owner_terminal_state_indeterminate");+ expect(state.execute).toHaveBeenCalledOnce();+ expect(firstSession.close).not.toHaveBeenCalled();+ expect(order).toEqual([]);+ } else if (scenario === "retirement-failed") {+ await expect(continuation).rejects.toThrow("retirement unconfirmed");+ expect(state.execute).toHaveBeenCalledOnce();+ expect(order).toEqual(["retire"]);+ } else {+ await expect(continuation).resolves.toMatchObject({ exitCode: 0 });+ expect(order).toEqual(scenario === "unchanged" ? ["execute-next"] : ["retire", "execute-next"]);+ if (scenario === "unchanged") {+ await writeRunnerAuthority(root, { ...identity, runId: next.binding.runId }, "ready");+ const third = { ...next, binding: { ...next.binding, runId: `${next.binding.runId}-third` } };+ const thirdDb = leaseDb(third);+ const thirdSelect = thirdDb.select.bind(thirdDb);+ thirdDb.select = ((fields?: Record<string, unknown>) => fields?.status === heartbeatRuns.status+ && fields?.runnerProfileJson === heartbeatRuns.runnerProfileJson+ ? priorNativeRunDb(next).select(fields as never)+ : thirdSelect(fields as never)) as Db["select"];+ state.execute.mockImplementationOnce(async options => {+ expect(options.existingSession).toBe(firstSession);+ await options.onSession?.(firstSession);+ return result;+ });+ await expect(executePaperclipNativeSession({ db: thirdDb, execution: third,+ runnerInstanceId: `runner-third-${scenario}`, useRunnerd: true,+ runnerExecutionTarget: target as never })).resolves.toMatchObject({ exitCode: 0 });+ expect(firstSession.close).not.toHaveBeenCalled();+ expect(state.execute).toHaveBeenCalledTimes(3);+ }+ }+ } finally {+ failRetirement = false;+ await closeWarmNativeSessionsForEnvironment({ environmentId, reason: "test cleanup" });+ }+ });+it.each([["open", "before-close"],["open", "during-close"],diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.tsindex 04dcb09e8a..f90d6a5506 100644--- a/server/src/services/native-runtime/native-session-executor.ts+++ b/server/src/services/native-runtime/native-session-executor.ts@@ -435,6 +435,7 @@ export type NativeInstructionWorkingCopy = {};type WarmNativeSession = {+ runId: string;instructionWorkingCopy?: NativeInstructionWorkingCopy;// Keep the admitted physical root independent of the per-run collection// capability, which instruction preparation adopts before final admission.@@ -4244,7 +4245,7 @@ async function verifyPriorRunnerdStateForSessionScope(input: {identity: RunnerdDurableIdentity;execution: NativeExecutionInput;allowVerifiedBackup: boolean;- allowRetainedWarmRunner: boolean;+ retainedWarmOwnerRunId?: string;}): Promise<PriorRunnerdStateVerification> {let priorRun: {status: string;@@ -4291,7 +4292,7 @@ async function verifyPriorRunnerdStateForSessionScope(input: {input.identity,);if (- input.allowRetainedWarmRunner &&+ input.retainedWarmOwnerRunId === input.identity.runId &&(lifecycle === "not_suspended" ||// A remote runner keeps runner-state.json in the sandbox rather than// beside the controller's PRP journal. During a genuinely warm handoff@@ -4753,7 +4754,7 @@ async function migrateRunnerdStateRootForExecution(input: {db: Db;execution: NativeExecutionInput;allowVerifiedBackup: boolean;- allowRetainedWarmRunner: boolean;+ retainedWarmOwnerRunId?: string;allowLocalRecovery: boolean;onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise<void>;restartRecovery?: NativeRestartRecoveryClaim;@@ -4770,7 +4771,7 @@ async function migrateRunnerdStateRootForExecution(input: {});if (input.allowLocalRecovery &&- !input.allowRetainedWarmRunner &&+ !input.retainedWarmOwnerRunId &&!input.restartRecovery) {await recoverQuiescentRunnerdState({ ...input, scoped });@@ -4843,7 +4844,7 @@ async function migrateRunnerdStateRootForExecution(input: {identity,execution: input.execution,allowVerifiedBackup: input.allowVerifiedBackup,- allowRetainedWarmRunner: input.allowRetainedWarmRunner,+ retainedWarmOwnerRunId: input.retainedWarmOwnerRunId,});if (verification !== "verified" &&@@ -4885,7 +4886,7 @@ async function migrateRunnerdStateRootForExecution(input: {identity,execution: input.execution,allowVerifiedBackup: input.allowVerifiedBackup,- allowRetainedWarmRunner: input.allowRetainedWarmRunner,+ retainedWarmOwnerRunId: input.retainedWarmOwnerRunId,});if (verification !== "verified" &&@@ -5406,27 +5407,17 @@ function nativeSessionConfigDigest(.digest("hex")}`;}-function hasIdleWarmNativeSessionOwner(input: {+function idleWarmNativeSessionOwnerRunId(input: {execution: NativeExecutionInput;runnerExecutionTarget?: AdapterExecutionTarget | null;-}): boolean {- if (input.execution.session.lifecyclePolicy.mode !== "warm") return false;+}): string | undefined {+ if (input.execution.session.lifecyclePolicy.mode !== "warm") return undefined;const entry = warmNativeSessions.get(nativeSessionScopeKey(input.execution));- if (!entry || (entry.busy && entry.instructionPreparationRunId !== input.execution.binding.runId)) return false;- // A verified idle owner proves the checkpoint belongs to this session even- // when its process must later rotate to a new run-scoped broker capability.- // Local environments also have an id. Compare the same environment binding- // stored at acquisition instead of treating every local target as unbound.+ if (!entry || (entry.busy && entry.instructionPreparationRunId !== input.execution.binding.runId)) return undefined;const environmentId = input.runnerExecutionTarget?.environmentId ?? null;- return (- entry.companyId === input.execution.binding.companyId &&- entry.environmentId === environmentId &&- entry.configDigest ===- nativeSessionConfigDigest(- input.execution,- input.runnerExecutionTarget?.kind ?? "local",- )- );+ return entry.companyId === input.execution.binding.companyId && entry.environmentId === environmentId+ ? entry.runId+ : undefined;}function nativeHarnessEnvironmentFingerprint(@@ -7762,7 +7753,7 @@ async function executePaperclipNativeSessionWithinScope(// suspended. Only the exact idle in-process owner may rotate that// prior-run authority; after a hard restart the map is empty and the// durable-state verifier continues to require a suspended runner.- allowRetainedWarmRunner: hasIdleWarmNativeSessionOwner(input),+ retainedWarmOwnerRunId: idleWarmNativeSessionOwnerRunId(input),allowLocalRecovery: input.runnerExecutionTarget?.kind !== "remote",onLog: input.onLog,restartRecovery: input.restartRecovery,@@ -8864,8 +8855,10 @@ async function executePaperclipNativeSessionWithinScope(throw new Error("native_session_supervisor_busy");}existing.session = session;+ existing.runId = input.execution.binding.runId;} elsewarmNativeSessions.set(warmSessionId, {+ runId: input.execution.binding.runId,instructionWorkingCopy: input.instructionWorkingCopy?.checkpointWarm ? undefined : input.instructionWorkingCopy,instructionRoot: input.instructionWorkingCopy?.root,agentId: input.execution.binding.agentId,@@ -11220,7 +11213,7 @@ export async function createRunnerdBackend(input: {db: input.db,execution: input.execution,allowVerifiedBackup: input.runnerExecutionTarget?.kind === "remote",- allowRetainedWarmRunner: false,+ retainedWarmOwnerRunId: undefined,allowLocalRecovery: input.runnerExecutionTarget?.kind !== "remote",onLog: input.onLog,restartRecovery: input.restartRecovery,