The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
fix(runner): persist warm Daytona workspaces (#12901)
Dotta1dceee9a4e75
diff --git a/.github/workflows/runner-full-stack-e2e.yml b/.github/workflows/runner-full-stack-e2e.ymlindex 0c4fa59387..3179f5c8e1 100644--- a/.github/workflows/runner-full-stack-e2e.yml+++ b/.github/workflows/runner-full-stack-e2e.yml@@ -14,7 +14,7 @@ on:type: booleandefault: truegroup:- description: "Comma-separated groups (AND semantics: legacy,native,local,daytona,core,breadth)"+ description: "Comma-separated groups (AND semantics: legacy,native,local,daytona,warm,core,breadth)"type: stringrequired: falsesuite:diff --git a/packages/adapter-utils/src/command-managed-runtime.ts b/packages/adapter-utils/src/command-managed-runtime.tsindex 36500fdc70..cdbae889cd 100644--- a/packages/adapter-utils/src/command-managed-runtime.ts+++ b/packages/adapter-utils/src/command-managed-runtime.ts@@ -12,11 +12,15 @@ import {type SandboxRemoteExecutionSpec,type SandboxSyncOperation,type SandboxSyncResult,+ type WorkspaceDurableSeedPaths,+ type WorkspaceInboundMode,} from "./sandbox-managed-runtime.js";import { preferredShellForSandbox, shellCommandArgs } from "./sandbox-shell.js";import type { RunProcessResult } from "./server-utils.js";import type { RuntimeProgressSink, RuntimeStatusSink } from "./runtime-progress.js";import type { RuntimeSpanRunner } from "./acpx-engine/startup-timing.js";+import type { GitWorkspaceSnapshot } from "./git-workspace-sync.js";+import type { DirectorySnapshot } from "./workspace-restore-merge.js";/*** Input for a duplex channel open. The caller supplies only the command argument@@ -516,6 +520,10 @@ export async function prepareCommandManagedRuntime(input: {workspaceLocalDir: string;workspaceRemoteDir?: string;syncWorkspace?: boolean;+ workspaceInboundMode?: WorkspaceInboundMode;+ workspaceDurableSeed?: WorkspaceDurableSeedPaths;+ workspaceBaseline?: DirectorySnapshot;+ workspaceGitSnapshot?: GitWorkspaceSnapshot | null;workspaceExclude?: string[];preserveAbsentOnRestore?: string[];assets?: CommandManagedRuntimeAsset[];@@ -577,6 +585,10 @@ export async function prepareCommandManagedRuntime(input: {workspaceLocalDir: input.workspaceLocalDir,workspaceRemoteDir,syncWorkspace: input.syncWorkspace,+ workspaceInboundMode: input.workspaceInboundMode,+ workspaceDurableSeed: input.workspaceDurableSeed,+ workspaceBaseline: input.workspaceBaseline,+ workspaceGitSnapshot: input.workspaceGitSnapshot,workspaceExclude: mergeRuntimeExcludes(input.workspaceExclude),preserveAbsentOnRestore: input.preserveAbsentOnRestore,assets: input.assets,@@ -616,6 +628,10 @@ export async function prepareCommandManagedRuntime(input: {workspaceLocalDir: input.workspaceLocalDir,workspaceRemoteDir,syncWorkspace: input.syncWorkspace,+ workspaceInboundMode: input.workspaceInboundMode,+ workspaceDurableSeed: input.workspaceDurableSeed,+ workspaceBaseline: input.workspaceBaseline,+ workspaceGitSnapshot: input.workspaceGitSnapshot,workspaceExclude: mergeRuntimeExcludes(input.workspaceExclude),preserveAbsentOnRestore: input.preserveAbsentOnRestore,assets: input.assets,diff --git a/packages/adapter-utils/src/execution-target.ts b/packages/adapter-utils/src/execution-target.tsindex db0fed0d72..9ea8730cb2 100644--- a/packages/adapter-utils/src/execution-target.ts+++ b/packages/adapter-utils/src/execution-target.ts@@ -18,10 +18,12 @@ import {import type {AdditionalSourceStagingFailure,SandboxAdditionalSource,+ WorkspaceDurableSeedPaths,+ WorkspaceInboundMode,} from "./sandbox-managed-runtime.js";-export {- resolveReferencedSourceIgnore,-} from "./sandbox-managed-runtime.js";+import type { GitWorkspaceSnapshot } from "./git-workspace-sync.js";+import type { DirectorySnapshot } from "./workspace-restore-merge.js";+export { resolveReferencedSourceIgnore } from "./sandbox-managed-runtime.js";export type {AdditionalSourceStagingFailure,ReferencedSourceIgnoreResolution,@@ -249,6 +251,10 @@ export interface PreparedAdapterExecutionTargetRuntime {* stage referenced projects, or when every requested project staged.*/additionalSourceFailures: AdditionalSourceStagingFailure[];+ workspaceSyncSnapshot: {+ baseline: DirectorySnapshot;+ gitSnapshot: GitWorkspaceSnapshot | null;+ } | null;restoreWorkspace(onProgress?: RuntimeProgressSink): Promise<void>;}@@ -1374,6 +1380,10 @@ export async function prepareAdapterExecutionTargetRuntime(input: {timeoutSec?: number;workspaceRemoteDir?: string;syncWorkspace?: boolean;+ workspaceInboundMode?: WorkspaceInboundMode;+ workspaceDurableSeed?: WorkspaceDurableSeedPaths;+ workspaceBaseline?: DirectorySnapshot;+ workspaceGitSnapshot?: GitWorkspaceSnapshot | null;workspaceExclude?: string[];preserveAbsentOnRestore?: string[];assets?: AdapterManagedRuntimeAsset[];@@ -1403,6 +1413,7 @@ export async function prepareAdapterExecutionTargetRuntime(input: {assetDirs: {},additionalSourceDirs: {},additionalSourceFailures: [],+ workspaceSyncSnapshot: null,restoreWorkspace: async () => {},};}@@ -1428,6 +1439,7 @@ export async function prepareAdapterExecutionTargetRuntime(input: {// The SSH transport does not stage referenced projects (it is out of scope), so it never// reports a per-project staging failure.additionalSourceFailures: [],+ workspaceSyncSnapshot: null,restoreWorkspace: prepared.restoreWorkspace,};}@@ -1448,6 +1460,10 @@ export async function prepareAdapterExecutionTargetRuntime(input: {workspaceLocalDir: input.workspaceLocalDir,workspaceRemoteDir: input.workspaceRemoteDir,syncWorkspace: input.syncWorkspace,+ workspaceInboundMode: input.workspaceInboundMode,+ workspaceDurableSeed: input.workspaceDurableSeed,+ workspaceBaseline: input.workspaceBaseline,+ workspaceGitSnapshot: input.workspaceGitSnapshot,workspaceExclude: input.workspaceExclude,preserveAbsentOnRestore: input.preserveAbsentOnRestore,assets: input.assets,@@ -1465,6 +1481,7 @@ export async function prepareAdapterExecutionTargetRuntime(input: {assetDirs: prepared.assetDirs,additionalSourceDirs: prepared.additionalSourceDirs,additionalSourceFailures: prepared.additionalSourceFailures,+ workspaceSyncSnapshot: prepared.workspaceSyncSnapshot,restoreWorkspace: prepared.restoreWorkspace,};}diff --git a/packages/adapter-utils/src/sandbox-managed-runtime.test.ts b/packages/adapter-utils/src/sandbox-managed-runtime.test.tsindex be36585a1e..387ac215df 100644--- a/packages/adapter-utils/src/sandbox-managed-runtime.test.ts+++ b/packages/adapter-utils/src/sandbox-managed-runtime.test.ts@@ -348,6 +348,129 @@ describe("sandbox managed runtime", () => {}});+ it("adopts a warm remote workspace without inbound overwrite and still merges outbound changes", async () => {+ const rootDir = await mkdtemp(+ path.join(os.tmpdir(), "paperclip-sandbox-adopt-"),+ );+ cleanupDirs.push(rootDir);+ const localWorkspaceDir = path.join(rootDir, "local-workspace");+ const remoteWorkspaceDir = path.join(rootDir, "remote-workspace");+ await mkdir(localWorkspaceDir, { recursive: true });+ await mkdir(remoteWorkspaceDir, { recursive: true });+ await writeFile(+ path.join(localWorkspaceDir, "continuity.txt"),+ "host baseline\n",+ "utf8",+ );+ await writeFile(+ path.join(remoteWorkspaceDir, "continuity.txt"),+ "remote retained\n",+ "utf8",+ );+ const client = makeFilesystemClient();+ const syncIn = vi.spyOn(client, "syncIn");++ const prepared = await prepareSandboxManagedRuntime({+ spec: {+ transport: "sandbox",+ provider: "test",+ sandboxId: "sandbox-warm",+ remoteCwd: remoteWorkspaceDir,+ timeoutMs: 30_000,+ apiKey: null,+ },+ adapterKey: "test-adapter",+ client,+ workspaceLocalDir: localWorkspaceDir,+ workspaceInboundMode: "adopt_remote",+ });++ expect(syncIn).not.toHaveBeenCalled();+ await expect(+ readFile(path.join(remoteWorkspaceDir, "continuity.txt"), "utf8"),+ ).resolves.toBe("remote retained\n");+ expect(prepared.workspaceSyncSnapshot).not.toBeNull();++ await writeFile(+ path.join(remoteWorkspaceDir, "continuity.txt"),+ "remote finalized\n",+ "utf8",+ );+ await prepared.restoreWorkspace();+ await expect(+ readFile(path.join(localWorkspaceDir, "continuity.txt"), "utf8"),+ ).resolves.toBe("remote finalized\n");+ });++ it("reconstructs a replacement workspace from the exact durable pre-turn seed", async () => {+ const rootDir = await mkdtemp(+ path.join(os.tmpdir(), "paperclip-sandbox-durable-seed-"),+ );+ cleanupDirs.push(rootDir);+ const localWorkspaceDir = path.join(rootDir, "local-workspace");+ const firstRemoteDir = path.join(rootDir, "first-remote");+ const replacementRemoteDir = path.join(rootDir, "replacement-remote");+ const durableSeed = {+ workspaceArchivePath: path.join(rootDir, "state", "workspace.tar"),+ gitArchivePath: path.join(rootDir, "state", "git.tar"),+ };+ await mkdir(localWorkspaceDir, { recursive: true });+ await writeFile(+ path.join(localWorkspaceDir, "continuity.txt"),+ "durable pre-turn bytes\n",+ "utf8",+ );++ const first = await prepareSandboxManagedRuntime({+ spec: {+ transport: "sandbox",+ provider: "test",+ sandboxId: "sandbox-first",+ remoteCwd: firstRemoteDir,+ timeoutMs: 30_000,+ apiKey: null,+ },+ adapterKey: "test-adapter",+ client: makeFilesystemClient(),+ workspaceLocalDir: localWorkspaceDir,+ workspaceInboundMode: "host_current",+ workspaceDurableSeed: durableSeed,+ });+ expect(first.workspaceSyncSnapshot).not.toBeNull();+ await expect(stat(durableSeed.workspaceArchivePath)).resolves.toMatchObject(+ {+ mode: expect.any(Number),+ },+ );++ await writeFile(+ path.join(localWorkspaceDir, "continuity.txt"),+ "concurrent host edit must not enter replacement\n",+ "utf8",+ );+ await prepareSandboxManagedRuntime({+ spec: {+ transport: "sandbox",+ provider: "test",+ sandboxId: "sandbox-replacement",+ remoteCwd: replacementRemoteDir,+ timeoutMs: 30_000,+ apiKey: null,+ },+ adapterKey: "test-adapter",+ client: makeFilesystemClient(),+ workspaceLocalDir: localWorkspaceDir,+ workspaceInboundMode: "durable_seed",+ workspaceDurableSeed: durableSeed,+ workspaceBaseline: first.workspaceSyncSnapshot!.baseline,+ workspaceGitSnapshot: first.workspaceSyncSnapshot!.gitSnapshot,+ });++ await expect(+ readFile(path.join(replacementRemoteDir, "continuity.txt"), "utf8"),+ ).resolves.toBe("durable pre-turn bytes\n");+ });+it("preserves excluded local workspace artifacts during restore mirroring", async () => {const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-sandbox-restore-"));cleanupDirs.push(rootDir);diff --git a/packages/adapter-utils/src/sandbox-managed-runtime.ts b/packages/adapter-utils/src/sandbox-managed-runtime.tsindex d3de812500..937c812ebf 100644--- a/packages/adapter-utils/src/sandbox-managed-runtime.ts+++ b/packages/adapter-utils/src/sandbox-managed-runtime.ts@@ -1,6 +1,10 @@import { execFile as execFileCallback } from "node:child_process";-import { randomUUID } from "node:crypto";-import { constants as fsConstants, promises as fs } from "node:fs";+import { createHash, randomUUID } from "node:crypto";+import {+ constants as fsConstants,+ createReadStream,+ promises as fs,+} from "node:fs";import os from "node:os";import path from "node:path";import { promisify } from "node:util";@@ -12,6 +16,7 @@ import {deleteLocalGitRef,fetchGitBundleIntoLocalRef,GIT_ARCHIVE_EXCLUDES,+ type GitWorkspaceSnapshot,integrateImportedGitHead,readGitWorkspaceSnapshot,ReferencedSourceIgnoreScanLimitExceededError,@@ -20,7 +25,11 @@ import {withShallowGitWorkspaceClone,WORKSPACE_GIT_SCAN_SATURATED_CODE,} from "./git-workspace-sync.js";-import { captureDirectorySnapshot, mergeDirectoryWithBaseline } from "./workspace-restore-merge.js";+import {+ captureDirectorySnapshot,+ mergeDirectoryWithBaseline,+ type DirectorySnapshot,+} from "./workspace-restore-merge.js";import {createRuntimeProgressReporter,type RuntimeProgressDirection,@@ -553,9 +562,29 @@ export interface PreparedSandboxManagedRuntime {* staged, or when no additional sources were requested.*/additionalSourceFailures: AdditionalSourceStagingFailure[];+ /** Durable merge inputs used to resume an outbound restore after host restart. */+ workspaceSyncSnapshot: {+ baseline: DirectorySnapshot;+ gitSnapshot: GitWorkspaceSnapshot | null;+ } | null;restoreWorkspace(onProgress?: RuntimeProgressSink): Promise<void>;}+export type WorkspaceInboundMode =+ "host_current" | "durable_seed" | "adopt_remote";++/**+ * Controller-owned archives for replaying the exact pre-turn workspace into a+ * replacement sandbox. Paths are never sent to the provider as credentials or+ * persisted in database metadata.+ */+export interface WorkspaceDurableSeedPaths {+ workspaceArchivePath: string;+ workspaceArchiveSha256?: string;+ gitArchivePath?: string | null;+ gitArchiveSha256?: string | null;+}+/** One additional (referenced) project that failed to stage into the sandbox. */export interface AdditionalSourceStagingFailure {projectId: string;@@ -719,6 +748,57 @@ async function withTempDir<T>(prefix: string, fn: (dir: string) => Promise<T>):}}+async function sha256File(filePath: string): Promise<string> {+ return await new Promise((resolveDigest, rejectDigest) => {+ const digest = createHash("sha256");+ const stream = createReadStream(filePath);+ stream.on("data", (chunk) => digest.update(chunk));+ stream.on("error", rejectDigest);+ stream.on("end", () => resolveDigest(digest.digest("hex")));+ });+}++async function copyDurableSeedArchive(input: {+ sourcePath: string;+ targetPath: string;+ expectedSha256?: string | null;+}): Promise<void> {+ const source = await fs.lstat(input.sourcePath);+ if (source.isSymbolicLink() || !source.isFile()) {+ throw new Error("workspace_durable_seed_invalid");+ }+ if (+ input.expectedSha256 &&+ (await sha256File(input.sourcePath)) !== input.expectedSha256+ ) {+ throw new Error("workspace_durable_seed_digest_mismatch");+ }+ await fs.copyFile(input.sourcePath, input.targetPath);+}++async function persistDurableSeedArchive(input: {+ sourcePath: string;+ targetPath: string;+}): Promise<void> {+ const parent = path.dirname(input.targetPath);+ await fs.mkdir(parent, { recursive: true, mode: 0o700 });+ const parentStat = await fs.lstat(parent);+ if (parentStat.isSymbolicLink() || !parentStat.isDirectory()) {+ throw new Error("workspace_durable_seed_root_invalid");+ }+ const temporary = path.join(+ parent,+ `.${path.basename(input.targetPath)}.${randomUUID()}.tmp`,+ );+ try {+ await fs.copyFile(input.sourcePath, temporary);+ await fs.chmod(temporary, 0o600);+ await fs.rename(temporary, input.targetPath);+ } finally {+ await fs.rm(temporary, { force: true }).catch(() => undefined);+ }+}+async function execTar(args: string[]): Promise<void> {await execFile("tar", args, {env: {@@ -976,6 +1056,12 @@ export async function prepareSandboxManagedRuntime(input: {workspaceLocalDir: string;workspaceRemoteDir?: string;syncWorkspace?: boolean;+ /** Selects authoritative host staging, exact durable-seed replay, or no-overwrite adoption. */+ workspaceInboundMode?: WorkspaceInboundMode;+ workspaceDurableSeed?: WorkspaceDurableSeedPaths;+ /** Durable snapshots supplied when reconstructing an interrupted restore. */+ workspaceBaseline?: DirectorySnapshot;+ workspaceGitSnapshot?: GitWorkspaceSnapshot | null;workspaceExclude?: string[];preserveAbsentOnRestore?: string[];assets?: SandboxManagedRuntimeAsset[];@@ -999,6 +1085,16 @@ export async function prepareSandboxManagedRuntime(input: {const workspaceRemoteDir = input.workspaceRemoteDir ?? input.spec.remoteCwd;const runtimeRootDir = path.posix.join(workspaceRemoteDir, ".paperclip-runtime", input.adapterKey);const syncWorkspace = input.syncWorkspace !== false;+ const workspaceInboundMode = input.workspaceInboundMode ?? "host_current";+ const stageWorkspace =+ syncWorkspace && workspaceInboundMode !== "adopt_remote";+ const prepareWorkspaceSeed =+ syncWorkspace &&+ workspaceInboundMode === "adopt_remote" &&+ input.workspaceDurableSeed !== undefined;+ if (workspaceInboundMode === "durable_seed" && !input.workspaceDurableSeed) {+ throw new Error("workspace_durable_seed_missing");+ }// Reject any unsafe asset key before an archive path or an asset directory is// built from it. This runs before the git snapshot work so a bad key fails fast.@@ -1025,7 +1121,11 @@ export async function prepareSandboxManagedRuntime(input: {// It reads git's own bookkeeping to decide what to include/exclude, so it is// usually fast, but on a large working tree the `--ignored` walk is not free.const gitSnapshot = syncWorkspace- ? await runStepSpan("snapshot.git", () => readGitWorkspaceSnapshot(input.workspaceLocalDir))+ ? input.workspaceGitSnapshot !== undefined+ ? input.workspaceGitSnapshot+ : await runStepSpan("snapshot.git", () =>+ readGitWorkspaceSnapshot(input.workspaceLocalDir),+ ): null;const gitIgnoredExcludes = gitSnapshot?.ignoredPaths;const workspaceArchiveExclude = mergeExcludes(@@ -1047,9 +1147,12 @@ export async function prepareSandboxManagedRuntime(input: {// dominant cost in the pre-`pack` window — it reads the content of every// non-excluded file, serially — so it earns its own span.const baselineSnapshot = syncWorkspace- ? await runStepSpan("snapshot.baseline", () =>- captureDirectorySnapshot(input.workspaceLocalDir, { exclude: restoreExclude }),- )+ ? (input.workspaceBaseline ??+ (await runStepSpan("snapshot.baseline", () =>+ captureDirectorySnapshot(input.workspaceLocalDir, {+ exclude: restoreExclude,+ }),+ ))): null;// Every inbound staging step delegates to the provider through `client.syncIn`:@@ -1172,7 +1275,7 @@ export async function prepareSandboxManagedRuntime(input: {// records its own failure and never rejects.const inboundTaskIsRequired: boolean[] = [];- if (syncWorkspace) {+ if (stageWorkspace || prepareWorkspaceSeed) {inboundTaskIsRequired.push(true);inboundTasks.push(() =>runStepSpan("stage.workspace", async () => {@@ -1203,18 +1306,49 @@ export async function prepareSandboxManagedRuntime(input: {if (gitSnapshot) {await emitRuntimeStatus(input.onRuntimeProgress, "git_sync", "Syncing git history to environment");const gitTarPath = path.join(tempDir, "git-workspace.tar");- const remoteGitTar = path.posix.join(runtimeRootDir, "git-workspace-upload.tar");- await withShallowGitWorkspaceClone({- localDir: input.workspaceLocalDir,- snapshot: gitSnapshot,- }, async (cloneDir) => {- await createTarballFromDirectory({- localDir: cloneDir,- archivePath: gitTarPath,- exclude: [".paperclip-runtime"],+ const remoteGitTar = path.posix.join(+ runtimeRootDir,+ "git-workspace-upload.tar",+ );+ if (workspaceInboundMode === "durable_seed") {+ const durableGitArchive =+ input.workspaceDurableSeed?.gitArchivePath;+ if (!durableGitArchive) {+ throw new Error("workspace_durable_seed_git_missing");+ }+ await copyDurableSeedArchive({+ sourcePath: durableGitArchive,+ targetPath: gitTarPath,+ expectedSha256: input.workspaceDurableSeed?.gitArchiveSha256,});+ } else {+ await withShallowGitWorkspaceClone(+ {+ localDir: input.workspaceLocalDir,+ snapshot: gitSnapshot,+ },+ async (cloneDir) => {+ await createTarballFromDirectory({+ localDir: cloneDir,+ archivePath: gitTarPath,+ exclude: [".paperclip-runtime"],+ });+ },+ );+ if (input.workspaceDurableSeed?.gitArchivePath) {+ await persistDurableSeedArchive({+ sourcePath: gitTarPath,+ targetPath: input.workspaceDurableSeed.gitArchivePath,+ });+ }+ }+ workspaceFiles.push({+ sourcePath: gitTarPath,+ targetPath: remoteGitTar,+ kind: "file",+ access: "rw",+ writablePath: workspaceRemoteDir,});- workspaceFiles.push({ sourcePath: gitTarPath, targetPath: remoteGitTar, kind: "file", access: "rw", writablePath: workspaceRemoteDir });workspacePostUploadCommands.push({command: buildWorkspaceTarExtractCommand({workspaceRemoteDir,@@ -1230,22 +1364,48 @@ export async function prepareSandboxManagedRuntime(input: {// the preserved names first. The extract runs AFTER the git extract.await emitRuntimeStatus(input.onRuntimeProgress, "config_sync", "Syncing workspace to environment");const workspaceTarPath = path.join(tempDir, "workspace.tar");- const workspaceArchiveDir = gitSnapshot ? path.join(tempDir, "workspace-overlay") : input.workspaceLocalDir;- if (gitSnapshot) {- await copySelectedWorkspaceEntries({- sourceDir: input.workspaceLocalDir,- targetDir: workspaceArchiveDir,- relativePaths: gitSnapshot.overlayPaths,- exclude: workspaceArchiveExclude,+ if (workspaceInboundMode === "durable_seed") {+ await copyDurableSeedArchive({+ sourcePath: input.workspaceDurableSeed!.workspaceArchivePath,+ targetPath: workspaceTarPath,+ expectedSha256:+ input.workspaceDurableSeed!.workspaceArchiveSha256,+ });+ } else {+ const workspaceArchiveDir = gitSnapshot+ ? path.join(tempDir, "workspace-overlay")+ : input.workspaceLocalDir;+ if (gitSnapshot) {+ await copySelectedWorkspaceEntries({+ sourceDir: input.workspaceLocalDir,+ targetDir: workspaceArchiveDir,+ relativePaths: gitSnapshot.overlayPaths,+ exclude: workspaceArchiveExclude,+ });+ }+ await createTarballFromDirectory({+ localDir: workspaceArchiveDir,+ archivePath: workspaceTarPath,+ exclude: gitSnapshot ? undefined : workspaceArchiveExclude,});+ if (input.workspaceDurableSeed) {+ await persistDurableSeedArchive({+ sourcePath: workspaceTarPath,+ targetPath: input.workspaceDurableSeed.workspaceArchivePath,+ });+ }}- await createTarballFromDirectory({- localDir: workspaceArchiveDir,- archivePath: workspaceTarPath,- exclude: gitSnapshot ? undefined : workspaceArchiveExclude,+ const remoteWorkspaceTar = path.posix.join(+ runtimeRootDir,+ "workspace-upload.tar",+ );+ workspaceFiles.push({+ sourcePath: workspaceTarPath,+ targetPath: remoteWorkspaceTar,+ kind: "file",+ access: "rw",+ writablePath: workspaceRemoteDir,});- const remoteWorkspaceTar = path.posix.join(runtimeRootDir, "workspace-upload.tar");- workspaceFiles.push({ sourcePath: workspaceTarPath, targetPath: remoteWorkspaceTar, kind: "file", access: "rw", writablePath: workspaceRemoteDir });workspacePostUploadCommands.push({command: buildWorkspaceTarExtractCommand({workspaceRemoteDir,@@ -1265,6 +1425,8 @@ export async function prepareSandboxManagedRuntime(input: {workspaceUploadBytes += (await fs.stat(workspaceTarPath)).size;});+ if (!stageWorkspace) return;+// One confined `syncIn` for the whole merged workspace file set. The confine// guard covers every mapping BEFORE any bytes upload (fail-closed): a source// or target escape in EITHER tar mapping stops the upload of both.@@ -1457,6 +1619,10 @@ export async function prepareSandboxManagedRuntime(input: {assetDirs,additionalSourceDirs,additionalSourceFailures,+ workspaceSyncSnapshot:+ syncWorkspace && baselineSnapshot+ ? { baseline: baselineSnapshot, gitSnapshot }+ : null,restoreWorkspace: async (onProgress?: RuntimeProgressSink) => {const restoreSink = onProgress ?? input.onProgress;diff --git a/packages/adapter-utils/src/workspace-restore-merge.test.ts b/packages/adapter-utils/src/workspace-restore-merge.test.tsindex d3a06c08bc..516c6dcc81 100644--- a/packages/adapter-utils/src/workspace-restore-merge.test.ts+++ b/packages/adapter-utils/src/workspace-restore-merge.test.ts@@ -9,9 +9,12 @@ import { afterEach, describe, expect, it, vi } from "vitest";import { resolvePaperclipInstanceRootForAdapter } from "./server-utils.js";import {captureDirectorySnapshot,+ directorySnapshotSha256,classifyWorkspaceRestoreFailure,describeWorkspaceRestoreFailure,mergeDirectoryWithBaseline,+ parseDirectorySnapshot,+ serializeDirectorySnapshot,withDirectoryMergeLock,WORKSPACE_RESTORE_LOCK_TIMEOUT_CODE,} from "./workspace-restore-merge.js";@@ -27,6 +30,36 @@ describe("workspace restore merge", () => {}});+ it("round-trips a deterministic durable snapshot and rejects traversal", async () => {+ const rootDir = await mkdtemp(+ path.join(os.tmpdir(), "paperclip-snapshot-"),+ );+ cleanupDirs.push(rootDir);+ await mkdir(path.join(rootDir, "nested"), { recursive: true });+ await writeFile(path.join(rootDir, "b.txt"), "bravo\n", "utf8");+ await writeFile(path.join(rootDir, "nested", "a.txt"), "alpha\n", "utf8");++ const snapshot = await captureDirectorySnapshot(rootDir, { exclude: [] });+ const serialized = serializeDirectorySnapshot(snapshot);+ const restored = parseDirectorySnapshot(serialized);++ expect(serialized.entries.map(([relativePath]) => relativePath)).toEqual([+ "b.txt",+ "nested",+ "nested/a.txt",+ ]);+ expect(restored).not.toBeNull();+ expect(directorySnapshotSha256(restored!)).toBe(+ directorySnapshotSha256(snapshot),+ );+ expect(+ parseDirectorySnapshot({+ ...serialized,+ entries: [["../escape", serialized.entries[0]![1]]],+ }),+ ).toBeNull();+ });+it("preserves sibling files when sequential stale-baseline restores create the same nested directory tree", async () => {const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-restore-merge-"));cleanupDirs.push(rootDir);diff --git a/packages/adapter-utils/src/workspace-restore-merge.ts b/packages/adapter-utils/src/workspace-restore-merge.tsindex 8d59cd06db..92992e684c 100644--- a/packages/adapter-utils/src/workspace-restore-merge.ts+++ b/packages/adapter-utils/src/workspace-restore-merge.ts@@ -5,7 +5,7 @@ import path from "node:path";import { shouldExcludePath } from "./exclude-patterns.js";import { resolvePaperclipInstanceRootForAdapter } from "./server-utils.js";-type SnapshotEntry =+export type SnapshotEntry =| { kind: "dir" }| { kind: "file"; mode: number; hash: string }| { kind: "symlink"; target: string };@@ -15,6 +15,87 @@ export interface DirectorySnapshot {entries: Map<string, SnapshotEntry>;}+export interface SerializedDirectorySnapshot {+ version: 1;+ exclude: string[];+ entries: Array<[string, SnapshotEntry]>;+}++function isSafeSnapshotRelativePath(value: string): boolean {+ if (!value || path.posix.isAbsolute(value) || path.win32.isAbsolute(value)) {+ return false;+ }+ return !value.split(/[\\/]/).some((segment) => segment === "..");+}++function parseSnapshotEntry(value: unknown): SnapshotEntry | null {+ if (!value || typeof value !== "object" || Array.isArray(value)) return null;+ const candidate = value as Record<string, unknown>;+ if (candidate.kind === "dir") return { kind: "dir" };+ if (candidate.kind === "symlink" && typeof candidate.target === "string") {+ return { kind: "symlink", target: candidate.target };+ }+ if (+ candidate.kind === "file" &&+ typeof candidate.mode === "number" &&+ Number.isInteger(candidate.mode) &&+ candidate.mode >= 0 &&+ typeof candidate.hash === "string" &&+ /^[0-9a-f]{64}$/.test(candidate.hash)+ ) {+ return { kind: "file", mode: candidate.mode, hash: candidate.hash };+ }+ return null;+}++export function serializeDirectorySnapshot(+ snapshot: DirectorySnapshot,+): SerializedDirectorySnapshot {+ return {+ version: 1,+ exclude: [...snapshot.exclude],+ entries: [...snapshot.entries.entries()].sort(([left], [right]) =>+ left.localeCompare(right),+ ),+ };+}++export function parseDirectorySnapshot(+ value: unknown,+): DirectorySnapshot | null {+ if (!value || typeof value !== "object" || Array.isArray(value)) return null;+ const candidate = value as Record<string, unknown>;+ if (+ candidate.version !== 1 ||+ !Array.isArray(candidate.exclude) ||+ !candidate.exclude.every((entry) => typeof entry === "string") ||+ !Array.isArray(candidate.entries)+ ) {+ return null;+ }+ const entries = new Map<string, SnapshotEntry>();+ for (const rawEntry of candidate.entries) {+ if (!Array.isArray(rawEntry) || rawEntry.length !== 2) return null;+ const [relative, rawSnapshotEntry] = rawEntry;+ if (typeof relative !== "string" || !isSafeSnapshotRelativePath(relative)) {+ return null;+ }+ const entry = parseSnapshotEntry(rawSnapshotEntry);+ if (!entry || entries.has(relative)) return null;+ entries.set(relative, entry);+ }+ return {+ exclude: [...new Set(candidate.exclude as string[])],+ entries,+ };+}++export function directorySnapshotSha256(snapshot: DirectorySnapshot): string {+ return createHash("sha256")+ .update(JSON.stringify(serializeDirectorySnapshot(snapshot)))+ .digest("hex");+}+async function hashFile(filePath: string): Promise<string> {return await new Promise((resolve, reject) => {const hash = createHash("sha256");diff --git a/packages/adapters/codex-local/src/server/execute.test.ts b/packages/adapters/codex-local/src/server/execute.test.tsindex a49073d8b0..375687c89f 100644--- a/packages/adapters/codex-local/src/server/execute.test.ts+++ b/packages/adapters/codex-local/src/server/execute.test.ts@@ -117,11 +117,10 @@ describe("codex execute — outbound auth copy-back restore contribution", () =>);}- async function runTeardown(input: {- sandboxAuth: string;- hostAuth: string;- }): Promise<{ finalHostAuth: string; finalHostMode: number }> {- const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-codex-copyback-e2e-"));+ async function runTeardown(input: { sandboxAuth: string; hostAuth: string }) {+ const rootDir = await mkdtemp(+ path.join(os.tmpdir(), "paperclip-codex-copyback-e2e-"),+ );cleanupDirs.push(rootDir);const workspaceDir = path.join(rootDir, "workspace");// The shared host home is what `resolveSharedCodexHomeDir` returns@@ -137,7 +136,7 @@ describe("codex execute — outbound auth copy-back restore contribution", () =>process.env.CODEX_HOME = sharedHostHome;sandboxAuthFixture.bytes = Buffer.from(input.sandboxAuth, "utf8");- await execute({+ const executionResult = await execute({runId: "run-copyback-e2e",agent: {id: "agent-1",@@ -178,6 +177,7 @@ describe("codex execute — outbound auth copy-back restore contribution", () =>return {finalHostAuth: await readFile(hostAuthPath, "utf8"),finalHostMode: (await lstat(hostAuthPath)).mode & 0o777,+ executionResult,};}@@ -231,4 +231,50 @@ describe("codex execute — outbound auth copy-back restore contribution", () =>expect(result.finalHostMode, entry.name).toBe(0o600);}});++ it("surfaces workspace restore failure after successful provider execution", async () => {+ prepareAdapterExecutionTargetRuntime.mockResolvedValueOnce({+ target: { kind: "remote", transport: "ssh" },+ workspaceRemoteDir: "/remote/workspace",+ runtimeRootDir: REMOTE_RUNTIME_ROOT,+ assetDirs: { home: `${REMOTE_RUNTIME_ROOT}/home` },+ restoreWorkspace: async () => {+ throw new Error("workspace copy-back failed");+ },+ });++ await expect(+ runTeardown({+ sandboxAuth: subscriptionAuth({ accountId: "acct", marker: "sandbox" }),+ hostAuth: subscriptionAuth({ accountId: "acct", marker: "host" }),+ }),+ ).rejects.toThrow("workspace copy-back failed");+ });++ it("preserves a provider failure when workspace restore also fails", async () => {+ runChildProcess.mockResolvedValueOnce({+ exitCode: 1,+ signal: null,+ timedOut: false,+ stdout: "",+ stderr: "provider failed first",+ pid: 321,+ startedAt: new Date().toISOString(),+ });+ prepareAdapterExecutionTargetRuntime.mockResolvedValueOnce({+ target: { kind: "remote", transport: "ssh" },+ workspaceRemoteDir: "/remote/workspace",+ runtimeRootDir: REMOTE_RUNTIME_ROOT,+ assetDirs: { home: `${REMOTE_RUNTIME_ROOT}/home` },+ restoreWorkspace: async () => {+ throw new Error("workspace copy-back failed second");+ },+ });++ const result = await runTeardown({+ sandboxAuth: subscriptionAuth({ accountId: "acct", marker: "sandbox" }),+ hostAuth: subscriptionAuth({ accountId: "acct", marker: "host" }),+ });+ expect(result.executionResult.errorMessage).toBe("provider failed first");+ });});diff --git a/packages/adapters/codex-local/src/server/execute.ts b/packages/adapters/codex-local/src/server/execute.tsindex 430a77bcc2..436cd585b4 100644--- a/packages/adapters/codex-local/src/server/execute.ts+++ b/packages/adapters/codex-local/src/server/execute.ts@@ -1526,6 +1526,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec};};+ let executionError: unknown = null;try {const initial = await runAttempt(sessionId);if (@@ -1539,22 +1540,26 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec`[paperclip] Codex resume session "${sessionId}" is unavailable; retrying with a fresh session.\n`,);const retry = await runAttempt(null);- return toResult(retry, true, true);+ const retryResult = toResult(retry, true, true);+ if (retryResult.errorMessage) {+ executionError = new Error(retryResult.errorMessage);+ }+ return retryResult;}- return toResult(initial, false, false);+ const result = toResult(initial, false, false);+ if (result.errorMessage) {+ executionError = new Error(result.errorMessage);+ }+ return result;+ } catch (error) {+ executionError = error;+ throw error;} finally {if (paperclipBridge) {await paperclipBridge.stop();}if (restoreRemoteWorkspace) {- // This teardown runs in a `finally`, so a throw here replaces the- // already-computed run result (`return toResult(...)`) and turns a- // successful Codex run into a failure. The workspace restore — and the- // host credential copy-back inside it — is a best-effort teardown step.- // Keep it rejection-safe: log a fault loudly and keep the pending- // result. The host copy-back installs the credential on disk before any- // diagnostic log runs, so it is already durable when this block returns.try {await onLog("stdout",@@ -1570,6 +1575,10 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec)}: ${error instanceof Error ? error.message : String(error)}\n`,),).catch(() => undefined);+ // A provider failure remains the primary outcome. When provider work+ // succeeded, however, silently accepting a failed copy-back can lose+ // the only workspace edits before a replacement sandbox starts.+ if (executionError === null) throw error;}}}diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rsindex 402b71a094..92e7323b49 100644--- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs+++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs@@ -1345,6 +1345,13 @@ impl CommandExecutor for AcpxCommandExecutor {}}+ fn rotate_authority(&mut self, config: &DurableRunnerConfig) {+ self.context.run_id = config.run_id.clone();+ self.context.normalized_session_id = config.normalized_session_id.clone();+ self.context.turn_id = config.turn_id.clone();+ self.context.item_id = config.item_id.clone();+ }+fn poll_events(&mut self) -> Result<Vec<PolledEvent>, DurableRunnerError> {self.restore()?;if selfdiff --git a/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs b/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rsindex 41f1685cb0..06b9d0d0fd 100644--- a/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs+++ b/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs@@ -54,7 +54,7 @@ pub(crate) const MAX_SETTLED_PROVIDER_TURN_IDS: usize = 4_096;type QuestionOptionLabels = BTreeMap<String, BTreeMap<String, String>>;type QuestionSetMapping = (String, Value, QuestionOptionLabels);-#[derive(Clone)]+#[derive(Clone, PartialEq)]struct ProviderCompletionContract {revision: String,criterion_ids: Vec<String>,@@ -807,6 +807,46 @@ impl CodexProvider {self.durable_tool_call_replays = true;}+ pub(crate) fn attach_run_in_place(+ &mut self,+ authorized_tools: impl IntoIterator<Item = AuthorizedTool>,+ completion_contract: Option<(&str, &[String])>,+ ) -> Result<bool, LocalRunnerError> {+ let authorized_tools = authorized_tools.into_iter().collect::<Vec<_>>();+ let completion_contract =+ completion_contract.map(|(revision, criterion_ids)| ProviderCompletionContract {+ revision: revision.to_owned(),+ criterion_ids: criterion_ids.to_vec(),+ });+ if authorized_tools != self.authorized_tools+ || completion_contract != self.completion_contract+ {+ return Ok(false);+ }+ if self.process.try_wait()?.is_some()+ || self.quarantined+ || self.active_provider_turn_id.is_some()+ || self.ambiguous_turn_start_pending+ || !self.pending_messages.is_empty()+ || !self.deferred_ambiguous_messages.is_empty()+ || !self.pending_tool_requests.is_empty()+ || !self.pending_runtime_requests.is_empty()+ {+ return Err(LocalRunnerError::invalid(+ "Codex warm run attachment requires an idle live provider with no pending work",+ ));+ }+ // The provider process and its thread remain authoritative. Exact+ // settled-turn identities stay in memory so delayed output from an+ // earlier run cannot be accepted as the next turn. A changed semantic+ // tool or completion contract returns false so the caller can preserve+ // the existing cold-resume behavior for that incompatible boundary.+ self.completed_turn_authority = None;+ self.completion_reconciliation_pending = false;+ self.expected_shutdown = false;+ Ok(true)+ }+pub(crate) fn restore_completed_turn_authority(&mut self,authoritative: bool,diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs b/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rsindex 6d0ae23dfc..f88dc0bbc2 100644--- a/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs+++ b/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs@@ -116,9 +116,121 @@ impl CommandLifecycle {}}+fn next_authority_config(+ command: &Command,+ current: &DurableRunnerConfig,+) -> Result<Option<DurableRunnerConfig>, DurableRunnerError> {+ if command.command_type != "run.attach" {+ return Ok(None);+ }+ let Some(boundary) = command.payload.get("paperclipNextAuthority") else {+ return Ok(None);+ };+ let identity = boundary+ .get("identity")+ .and_then(Value::as_object)+ .ok_or_else(|| DurableRunnerError::invalid("run.attach authority identity is required"))?;+ let read_identity = |key: &str| {+ identity+ .get(key)+ .and_then(Value::as_str)+ .filter(|value| !value.is_empty())+ .map(str::to_owned)+ .ok_or_else(|| {+ DurableRunnerError::invalid(format!(+ "run.attach authority identity field {key} is required"+ ))+ })+ };+ let connection = boundary+ .get("connection")+ .and_then(Value::as_object)+ .ok_or_else(|| {+ DurableRunnerError::invalid("run.attach authority connection is required")+ })?;+ let connect_url = match connection.get("mode").and_then(Value::as_str) {+ Some("connect") => connection+ .get("connectUrl")+ .and_then(Value::as_str)+ .filter(|value| !value.is_empty())+ .map(str::to_owned)+ .ok_or_else(|| DurableRunnerError::invalid("run.attach connect URL is required"))?,+ Some("listen") => {+ let address = connection+ .get("listenAddress")+ .and_then(Value::as_str)+ .ok_or_else(|| {+ DurableRunnerError::invalid("run.attach listen address is required")+ })?;+ let port = connection+ .get("listenPort")+ .and_then(Value::as_u64)+ .ok_or_else(|| DurableRunnerError::invalid("run.attach listen port is required"))?;+ let path = connection+ .get("listenPath")+ .and_then(Value::as_str)+ .ok_or_else(|| DurableRunnerError::invalid("run.attach listen path is required"))?;+ format!("listen://{address}:{port}{path}")+ }+ _ => {+ return Err(DurableRunnerError::invalid(+ "run.attach authority connection mode is invalid",+ ));+ }+ };+ let mut next = current.clone();+ next.connect_url = connect_url;+ next.ca_bundle_path = connection+ .get("caBundlePath")+ .and_then(Value::as_str)+ .filter(|value| !value.is_empty())+ .map(Into::into);+ next.runner_instance_id = read_identity("runnerInstanceId")?;+ next.environment_lease_id = read_identity("environmentLeaseId")?;+ next.run_id = read_identity("runId")?;+ next.normalized_session_id = read_identity("normalizedSessionId")?;+ next.turn_id = read_identity("turnId")?;+ next.item_id = read_identity("itemId")?;+ next.validate()?;+ if next.runner_instance_id != current.runner_instance_id+ || next.environment_lease_id != current.environment_lease_id+ || next.normalized_session_id != current.normalized_session_id+ || next.run_id == current.run_id+ {+ return Err(DurableRunnerError::invalid(+ "run.attach authority changed an immutable session binding",+ ));+ }+ Ok(Some(next))+}++fn apply_authority_rotation(+ state: &mut DurableState,+ store: &DurableStateStore,+ config: &mut DurableRunnerConfig,+ endpoint: &mut RunnerTransportEndpoint,+ next: DurableRunnerConfig,+) -> Result<(), DurableRunnerError> {+ let reconnect_count = state.reconnect_count.saturating_add(1);+ let mut diagnostics = std::mem::take(&mut state.diagnostics);+ *endpoint = RunnerTransportEndpoint::new(&next.connect_url, &next.run_id)?;+ *config = next;+ let mut rotated = DurableState::new(config);+ rotated.reconnect_count = reconnect_count;+ rotated.diagnostics.append(&mut diagnostics);+ rotated.record_diagnostic("runner advanced to a new warm run authority");+ *state = rotated;+ store.save(state)+}+pub trait CommandExecutor {fn execute(&mut self, command: &Command) -> Result<CommandExecution, DurableRunnerError>;+ /// Advances provider-side event correlation after a durable `run.attach`+ /// has moved runnerd to the next run-bound authority. The runner validates+ /// and persists the new authority before invoking this infallible hook.+ fn rotate_authority(&mut self, _config: &DurableRunnerConfig) {}+fn poll_events(&mut self) -> Result<Vec<PolledEvent>, DurableRunnerError> {Ok(Vec::new())}@@ -136,7 +248,7 @@ pub trait CommandExecutor {}pub fn run_durable_runner<E: CommandExecutor>(- config: DurableRunnerConfig,+ mut config: DurableRunnerConfig,bootstrap_ticket: BootstrapTicket,mut executor: E,) -> Result<(), DurableRunnerError> {@@ -162,7 +274,7 @@ pub fn run_durable_runner<E: CommandExecutor>(// Bind listener mode or resolve dial mode before processing commands. Dial// reconnects retain the same validated addresses so DNS cannot redirect a// retry after the trust decision.- let endpoint = RunnerTransportEndpoint::new(&config.connect_url, &config.run_id)?;+ let mut endpoint = RunnerTransportEndpoint::new(&config.connect_url, &config.run_id)?;let started = Instant::now();let mut bootstrap_ticket = Some(bootstrap_ticket);let mut lease: Option<LeaseCredential> = None;@@ -287,8 +399,10 @@ pub fn run_durable_runner<E: CommandExecutor>(let mut sent_source_seq = state.acked_source_seq;let mut lifecycle_after_reply = CommandLifecycle::Continue;+ let mut authority_rotation = None;let mut disconnected = false;for command in welcome.pending_commands {+ let next_authority = next_authority_config(&command, &config)?;let (result, lifecycle) =process_command(&mut state, &store, &config, &mut executor, &command)?;if let Some(durable_lifecycle) = lifecycle.durable_state() {@@ -333,6 +447,16 @@ pub fn run_durable_runner<E: CommandExecutor>(// then release the executor without observing later commands.break;}+ if next_authority.is_some() {+ authority_rotation = next_authority;+ break;+ }+ }+ if let Some(next) = authority_rotation {+ apply_authority_rotation(&mut state, &store, &mut config, &mut endpoint, next)?;+ executor.rotate_authority(&config);+ disconnected_since = Some(Instant::now());+ continue;}if !disconnected {if let Err(error) = send_outbox(&mut transport, &state, &mut sent_source_seq) {@@ -425,6 +549,7 @@ pub fn run_durable_runner<E: CommandExecutor>(.map_err(|error| {DurableRunnerError::invalid(format!("command is malformed: {error}"))})?;+ let next_authority = next_authority_config(&command, &config)?;let (result, lifecycle) =process_command(&mut state, &store, &config, &mut executor, &command)?;if let Some(durable_lifecycle) = lifecycle.durable_state() {@@ -468,6 +593,18 @@ pub fn run_durable_runner<E: CommandExecutor>();}}+ if let Some(next) = next_authority {+ apply_authority_rotation(+ &mut state,+ &store,+ &mut config,+ &mut endpoint,+ next,+ )?;+ executor.rotate_authority(&config);+ disconnected_since = Some(Instant::now());+ break;+ }if let Err(error) = send_outbox(&mut transport, &state, &mut sent_source_seq) {state.record_diagnostic("outbox delivery failed; unacknowledged suffix remains durable",@@ -1018,6 +1155,59 @@ mod tests {}}+ #[test]+ fn warm_run_attachment_rotates_only_the_run_authority() {+ let directory = std::env::temp_dir().join(format!(+ "paperclip-runner-warm-authority-{}",+ std::process::id()+ ));+ let _ = fs::remove_dir_all(&directory);+ let mut current = config(directory.clone());+ current.runner_digest = format!("sha256:{}", "a".repeat(64));+ let mut attach = command("run.attach");+ attach.payload = json!({+ "paperclipNextAuthority": {+ "identity": {+ "runnerInstanceId": current.runner_instance_id,+ "environmentLeaseId": current.environment_lease_id,+ "runId": "run_2",+ "normalizedSessionId": current.normalized_session_id,+ "turnId": "turn_2",+ "itemId": "item_2"+ },+ "connection": {+ "mode": "connect",+ "connectUrl": "ws://127.0.0.1:3001/path"+ }+ }+ });++ let next = next_authority_config(&attach, ¤t)+ .unwrap()+ .expect("attachment should carry a new authority");+ assert_eq!(next.run_id, "run_2");+ assert_eq!(next.connect_url, "ws://127.0.0.1:3001/path");++ let store = DurableStateStore::new(&directory).unwrap();+ let (mut state, _) = store.load_or_create(¤t).unwrap();+ state.outbox.push(crate::durable::state::StoredOutboxEvent {+ source_seq: 1,+ priority: 0,+ event_type: "run.attached".to_owned(),+ byte_size: 1,+ envelope: json!({}),+ });+ let mut endpoint =+ RunnerTransportEndpoint::new(¤t.connect_url, ¤t.run_id).unwrap();+ apply_authority_rotation(&mut state, &store, &mut current, &mut endpoint, next).unwrap();++ assert_eq!(state.run_id, "run_2");+ assert_eq!(state.next_source_seq, 1);+ assert!(state.outbox.is_empty());+ assert_eq!(current.run_id, "run_2");+ fs::remove_dir_all(directory).unwrap();+ }+#[test]fn terminal_lifecycle_is_durable_before_fallible_cleanup() {let directory = std::env::temp_dir().join(format!(diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/managed_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/managed_provider_backend.rsindex 5456fc9824..dcdfb5ed0e 100644--- a/packages/paperclip-runner/runner/crates/runner-core/src/managed_provider_backend.rs+++ b/packages/paperclip-runner/runner/crates/runner-core/src/managed_provider_backend.rs@@ -1670,6 +1670,10 @@ impl CommandExecutor for ManagedProviderCommandExecutor {}}+ fn rotate_authority(&mut self, config: &DurableRunnerConfig) {+ self.config = config.clone();+ }+fn poll_events(&mut self) -> Result<Vec<PolledEvent>, DurableRunnerError> {self.poll_provider()?;Ok(selfdiff --git a/packages/paperclip-runner/runner/crates/runner-core/src/native_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/native_provider_backend.rsindex 959b06c8d1..4d54b48a20 100644--- a/packages/paperclip-runner/runner/crates/runner-core/src/native_provider_backend.rs+++ b/packages/paperclip-runner/runner/crates/runner-core/src/native_provider_backend.rs@@ -35,6 +35,14 @@ impl CommandExecutor for SelectedExecutor {}}+ fn rotate_authority(&mut self, config: &DurableRunnerConfig) {+ match self {+ Self::LocalFacade(executor) => executor.rotate_authority(config),+ Self::Acpx(executor) => executor.rotate_authority(config),+ Self::Managed(executor) => executor.rotate_authority(config),+ }+ }+fn acknowledge_events(&mut self, count: usize) -> Result<(), DurableRunnerError> {match self {Self::LocalFacade(executor) => executor.acknowledge_events(count),@@ -163,6 +171,13 @@ impl CommandExecutor for NativeProviderCommandExecutor {.map_or_else(|| Ok(Vec::new()), CommandExecutor::poll_events)}+ fn rotate_authority(&mut self, config: &DurableRunnerConfig) {+ self.config = config.clone();+ if let Some(executor) = self.selected.as_mut() {+ executor.rotate_authority(config);+ }+ }+fn acknowledge_events(&mut self, count: usize) -> Result<(), DurableRunnerError> {self.select_recovery()?;if let Some(executor) = self.selected.as_mut() {diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rsindex f11fe402df..c335667184 100644--- a/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs+++ b/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs@@ -1675,19 +1675,40 @@ impl CodexCommandExecutor {next_state.active_provider_result_fingerprint = None;next_state.active_provider_result_disposition = None;next_state.last_agent_message = None;- if let Some(provider) = self.provider.as_mut() {+ let provider = self.provider.as_mut().ok_or_else(|| {+ DurableRunnerError::invalid("run.attach requires the restored Codex provider process")+ })?;+ let retained_provider = provider+ .attach_run_in_place(+ next_state.tool_bridge.authorized_tools().cloned(),+ next_state.completion_contract.as_ref().map(|contract| {+ (+ contract.revision.as_str(),+ contract.criterion_ids.as_slice(),+ )+ }),+ )+ .map_err(|error| {+ DurableRunnerError::invalid(format!(+ "failed to retain Codex for warm run attachment: {error}"+ ))+ })?;+ if !retained_provider {provider.shutdown().map_err(|error| {DurableRunnerError::invalid(format!("failed to checkpoint Codex before attaching a new run: {error}"))})?;+ self.provider = None;}- self.provider = None;next_state.pending_events.clear();- // Persist the checkpoint as not-open before open_session resumes it for- // the new authority. Otherwise recovery emits a second session.resumed- // notice into the provider queue in addition to the command event.- next_state.lifecycle = "prepared".to_owned();+ next_state.lifecycle = if retained_provider {+ "session_open".to_owned()+ } else {+ // The next provider command restores the same checkpointed session+ // with the rotated tool/completion authority.+ "prepared".to_owned()+ };self.persist_state(&next_state)?;self.state = Some(next_state);Ok(())@@ -3009,6 +3030,10 @@ impl CommandExecutor for CodexCommandExecutor {}}+ fn rotate_authority(&mut self, config: &DurableRunnerConfig) {+ self.event_identity = Some(ProviderEventIdentity::from_config(config));+ }+fn poll_events(&mut self) -> Result<Vec<PolledEvent>, DurableRunnerError> {self.poll_provider()?;Ok(selfdiff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.tsindex fb76579f69..85ee0ab0e1 100644--- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts+++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts@@ -939,7 +939,7 @@ class AuthorityConnection {/** Authenticated, replay-safe PRP transport authority. Business operations are caller supplied. */export class DurablePrpControlPlane {- readonly #identity: DurableRecoveryIdentity;+ #identity: DurableRecoveryIdentity;readonly #store: DurableCoreStore;#expectedRunnerVersion: string;#expectedRunnerDigest: string;@@ -1041,6 +1041,36 @@ export class DurablePrpControlPlane {).length;}+ /**+ * Atomically advances a settled reusable runner to a new run authority while+ * retaining its existing connection lease secret. The runner performs the+ * matching state transition only after acknowledging `run.attach`.+ */+ rotateRunIdentity(identity: DurableRecoveryIdentity): void {+ if (+ !Object.values(identity).every(+ (value) => typeof value === "string" && stableIdPattern.test(value),+ ) ||+ identity.runnerInstanceId !== this.#identity.runnerInstanceId ||+ identity.environmentLeaseId !== this.#identity.environmentLeaseId ||+ identity.normalizedSessionId !== this.#identity.normalizedSessionId ||+ identity.runId === this.#identity.runId ||+ this.#store.state.commands.some((command) => command.status === "pending")+ ) {+ throw new Error("Durable PRP run identity rotation is invalid.");+ }+ this.disconnectActiveRunner();+ const leases = Object.fromEntries(+ Object.entries(this.#store.state.leases).map(([key, lease]) => [+ key,+ { ...lease, identity: structuredClone(identity) },+ ]),+ );+ Object.assign(this.#store.state, initialCoreState(identity), { leases });+ this.#identity = structuredClone(identity);+ this.#store.save();+ }+issueBootstrapTicket(ttlMs = 5_000): string {if (!Number.isInteger(ttlMs) || ttlMs < 1_000 || ttlMs > 60_000) {throw new Error("Durable PRP bootstrap TTL is invalid.");diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.tsindex dcbfc49159..b8ffc2d367 100644--- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts+++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts@@ -1965,7 +1965,7 @@ it("steers the active provider turn through the durable PRP command path", async}}, 30_000);-it("does not expose cross-run attachment before PRP authority can rotate atomically", async () => {+it("rotates PRP authority in place for a warm cross-run attachment", async () => {const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-warm-attach-"));const bundle = createCapabilityRunnerdCodexTransport({runnerBinary: defaultCapabilityRunnerdBinary(),@@ -1978,22 +1978,36 @@ it("does not expose cross-run attachment before PRP authority can rotate atomicasuccess: true,contentItems: [],}));+ const within = async <T>(label: string, promise: Promise<T>) =>+ await Promise.race([+ promise,+ new Promise<never>((_, reject) =>+ setTimeout(() => reject(new Error(`${label} timeout`)), 5_000),+ ),+ ]);try {- await bundle.transport.request("initialize", {});- await bundle.transport.request("thread/start", {- cwd: tmpdir(),- dynamicTools: [- {- name: "get_task_context",- description: "Read the active task.",- inputSchema: {- type: "object",- properties: {},- additionalProperties: false,+ await within("initialize", bundle.transport.request("initialize", {}));+ await within(+ "thread start",+ bundle.transport.request("thread/start", {+ cwd: tmpdir(),+ dynamicTools: [+ {+ name: "get_task_context",+ description: "Read the active task.",+ inputSchema: {+ type: "object",+ properties: {},+ additionalProperties: false,+ },},+ ],+ completionContract: {+ revision: "sha256:warm-three-turn-contract",+ criterionIds: ["objective"],},- ],- });+ }),+ );const runnerPid = bundle.evidence().runnerPid;const providerPid = bundle.evidence().codexPid;const notifications = bundle.transport@@ -2015,12 +2029,45 @@ it("does not expose cross-run attachment before PRP authority can rotate atomica}throw new Error(`${label} completion timeout`);};- await bundle.transport.request("turn/start", {- input: [{ type: "text", text: "first run" }],- });+ await within(+ "first turn start",+ bundle.transport.request("turn/start", {+ input: [{ type: "text", text: "first run" }],+ }),+ );await waitForCompletion("first run");- expect(bundle.transport.attachRun).toBeUndefined();+ await within(+ "warm attach",+ bundle.transport.attachRun!({+ runId: "run-warm-second",+ turnId: "turn-warm-second",+ itemId: "item-warm-second",+ }),+ );+ await within(+ "second turn start",+ bundle.transport.request("turn/start", {+ input: [{ type: "text", text: "second run" }],+ }),+ );+ await waitForCompletion("second run");++ await within(+ "second warm attach",+ bundle.transport.attachRun!({+ runId: "run-warm-third",+ turnId: "turn-warm-third",+ itemId: "item-warm-third",+ }),+ );+ await within(+ "third turn start",+ bundle.transport.request("turn/start", {+ input: [{ type: "text", text: "third run" }],+ }),+ );+ await waitForCompletion("third run");expect(bundle.evidence()).toMatchObject({runnerPid,@@ -2033,6 +2080,99 @@ it("does not expose cross-run attachment before PRP authority can rotate atomica}}, 30_000);+it("releases both PRP authorities when warm rotation activation fails", async () => {+ const stateDirectory = await mkdtemp(+ join(tmpdir(), "runnerd-warm-attach-activation-failure-"),+ );+ const server = createServer();+ const authorities = new Map<string, DurablePrpControlPlane>();+ const released: string[] = [];+ server.on("upgrade", (request, socket, head) => {+ const route = request.url ?? "";+ const authority = authorities.get(route);+ if (!authority) {+ socket.destroy();+ return;+ }+ authority.handleUpgrade(request, socket, route, head);+ });+ await new Promise<void>((resolveListen) =>+ server.listen(0, "127.0.0.1", resolveListen),+ );+ const address = server.address();+ if (!address || typeof address === "string") {+ throw new Error("Expected warm activation failure test listener");+ }+ let registrationCount = 0;+ const bundle = createCapabilityRunnerdCodexTransport({+ runnerBinary: defaultCapabilityRunnerdBinary(),+ codexCommand: fakeCodex,+ codexArgs: fakeCodexArgs(stateDirectory),+ stateDirectory,+ lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 },+ controlPlaneRegistration: async (authority) => {+ registrationCount += 1;+ const route = `/runner-${registrationCount}`;+ authorities.set(route, authority);+ return {+ connectUrl: `ws://127.0.0.1:${address.port}${route}`,+ ...(registrationCount === 1+ ? {}+ : {+ activate: () => {+ throw new Error("rotation activation failed");+ },+ }),+ release: () => {+ released.push(route);+ if (authorities.get(route) === authority) authorities.delete(route);+ },+ };+ },+ });+ bundle.transport.setServerRequestHandler(async () => ({+ success: true,+ contentItems: [],+ }));+ let runnerPid: number | null = null;+ try {+ await bundle.transport.request("thread/start", {+ cwd: tmpdir(),+ dynamicTools: codexSemanticToolSpecs(),+ });+ runnerPid = bundle.evidence().runnerPid;++ await expect(+ bundle.transport.attachRun!({+ runId: "run-warm-activation-failure",+ turnId: "turn-warm-activation-failure",+ itemId: "item-warm-activation-failure",+ }),+ ).rejects.toThrow("rotation activation failed");+ expect(new Set(released)).toEqual(new Set(["/runner-1", "/runner-2"]));+ expect(authorities.size).toBe(0);+ await expect(bundle.transport.request("thread/read", {})).rejects.toThrow(+ "rotation activation failed",+ );+ } finally {+ await bundle.transport.close().catch(() => undefined);+ if (runnerPid) {+ try {+ process.kill(-runnerPid, "SIGKILL");+ } catch {+ // A successful durable close already stopped the runner process group.+ }+ }+ server.closeAllConnections();+ if (server.listening) {+ await new Promise<void>((resolveClose) =>+ server.close(() => resolveClose()),+ );+ }+ await rm(stateDirectory, { recursive: true, force: true });+ }+}, 30_000);+it.each([{binding: "runner instance",diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.tsindex 5ab150c78f..fe384d7336 100644--- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts+++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts@@ -400,7 +400,7 @@ async function rotateExternalAuthorityEpoch(}function rotatedRunAttachPayload(- state: Record<string, unknown>,+ state: { commands?: unknown },desired: DurableRecoveryIdentity,authorizedTools: Record<string, unknown> | null,completionContract:@@ -418,7 +418,22 @@ function rotatedRunAttachPayload();if (!seed)throw new Error("native_runner_authority_rotation_seed_unavailable");- const payload = structuredClone(record(seed.payload));+ return retargetRunAttachPayload(+ record(seed.payload),+ desired,+ authorizedTools,+ completionContract,+ );+}++function retargetRunAttachPayload(+ seedPayload: Record<string, unknown>,+ desired: DurableRecoveryIdentity,+ authorizedTools: Record<string, unknown> | null,+ completionContract:+ { revision: string; criterionIds: readonly string[] } | undefined,+): Record<string, unknown> {+ const payload = structuredClone(seedPayload);const provider = record(payload.provider);if (provider.kind === "acpx" || provider.provider === "acpx") {provider.runId = desired.runId;@@ -958,7 +973,10 @@ export interface CapabilityRunnerdCodexTransportOptions {itemId: string;};/** Registers the run-bound PRP authority on Paperclip's shared HTTP server. */- controlPlaneRegistration?: (authority: DurablePrpControlPlane) => Promise<{+ controlPlaneRegistration?: (+ authority: DurablePrpControlPlane,+ identity?: DurableRecoveryIdentity,+ ) => Promise<{connectUrl?: string;connection?: RunnerProcessConnection;activate?: () => Promise<void> | void;@@ -1942,6 +1960,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {#expectedProviderTurnId: string | null = null;#durableTurnId = "";#authorizedTools: Record<string, unknown> | null = null;+ #runAttachTemplate: Record<string, unknown> | null = null;#closed = false;#closePromise: Promise<void> | null = null;#failure: Error | null = null;@@ -2216,6 +2235,94 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {this.#handler = handler;}+ async attachRun(input: {+ runId: string;+ turnId: string;+ itemId: string;+ }): Promise<void> {+ const core = this.#core;+ if (!core || !this.#startupComplete) {+ throw new Error("native_runner_prp_run_rotation_unavailable");+ }+ const prior = core.store.state.identity;+ const desired: DurableRecoveryIdentity = {+ ...prior,+ runId: input.runId,+ turnId: input.turnId,+ itemId: input.itemId,+ };+ const registration = this.options.controlPlaneRegistration+ ? await this.options.controlPlaneRegistration(core, desired)+ : null;+ const connection: RunnerProcessConnection =+ registration?.connection ??+ (registration?.connectUrl+ ? { mode: "connect", connectUrl: registration.connectUrl }+ : { mode: "connect", connectUrl: core.connectUrl });+ const commandId = `command_attach_${createHash("sha256")+ .update(`${prior.runId}:${desired.runId}:${desired.turnId}`)+ .digest("hex")+ .slice(0, 32)}`;+ const runAttachTemplate = this.#runAttachTemplate+ ? retargetRunAttachPayload(+ this.#runAttachTemplate,+ desired,+ this.#authorizedTools,+ this.options.resumeCompletionContract,+ )+ : rotatedRunAttachPayload(+ core.store.state,+ desired,+ this.#authorizedTools,+ this.options.resumeCompletionContract,+ );+ this.#runAttachTemplate = structuredClone(runAttachTemplate);+ const payload = {+ ...runAttachTemplate,+ paperclipNextAuthority: { identity: desired, connection },+ };+ core.queueCommand("run.attach", payload, commandId, true);+ await this.#waitCommand("run.attach", commandId);+ const attached = core.store.state.commands.find(+ (command) => command.commandId === commandId,+ );+ if (attached?.status !== "completed") {+ await Promise.resolve(registration?.release()).catch(() => undefined);+ throw new Error("native_runner_prp_run_rotation_failed");+ }++ const previousRelease = this.#controlPlaneRelease;+ core.rotateRunIdentity(desired);+ this.#eventIndex = 0;+ this.#durableTurnId = desired.turnId;+ this.#controlPlaneRelease = registration?.release ?? null;+ let previousReleased = false;+ try {+ await registration?.activate?.();+ if (registration?.failure) {+ void registration.failure.catch((error: unknown) => {+ this.#failTransport(+ error instanceof Error ? error : new Error(String(error)),+ );+ });+ }+ await previousRelease?.();+ previousReleased = true;+ await this.#awaitRegistrationReady(registration?.ready);+ } catch (error) {+ const failure = error instanceof Error ? error : new Error(String(error));+ this.#controlPlaneRelease = null;+ await Promise.allSettled([+ Promise.resolve().then(() => registration?.release()),+ ...(previousReleased+ ? []+ : [Promise.resolve().then(() => previousRelease?.())]),+ ]);+ this.#failTransport(failure);+ throw failure;+ }+ }+async resolveRuntimeRequest(input: {requestId: string;turnId: string;@@ -3298,15 +3405,14 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {});this.#core = core;if (rotatedAuthority) {- core.queueCommand(- "run.attach",- rotatedRunAttachPayload(- controlPlaneState,- desiredIdentity,- this.#authorizedTools,- this.options.resumeCompletionContract,- ),+ const runAttachTemplate = rotatedRunAttachPayload(+ controlPlaneState,+ desiredIdentity,+ this.#authorizedTools,+ this.options.resumeCompletionContract,);+ this.#runAttachTemplate = structuredClone(runAttachTemplate);+ core.queueCommand("run.attach", runAttachTemplate);}const committedEvents = core.store.state.committedEvents;const runAttachment = recoveredRunAttachment(core.store.state);diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.test.tsindex 41da01fb5a..84787dff01 100644--- a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts+++ b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts@@ -1240,6 +1240,8 @@ describe("Daytona sandbox provider plugin", () => {providerLeaseId: "sandbox-reuse",metadata: {resumedLease: true,+ resumedFromState: "stopped",+ sandboxState: "started",workspaceSentinel: {result: "matched",token: "sentinel-token",@@ -4060,6 +4062,35 @@ describe("daytona native file-sync hooks", () => {});});+ it("classifies a deleted sandbox during syncOut with a stable unrecoverable code", async () => {+ const hostDir = await makeHostDir();+ mockGet.mockRejectedValue(+ new MockDaytonaNotFoundError("provider detail must not escape"),+ );++ await expect(+ plugin.definition.onEnvironmentSyncOut?.({+ driverKey: "daytona",+ companyId: "company-1",+ environmentId: "env-1",+ config: { timeoutMs: 300000, reuseLease: true },+ lease: syncLease(),+ operations: [+ {+ operationId: "sync-op-missing-sandbox",+ files: [+ {+ sourcePath: `${REMOTE_DIR}/out/result.txt`,+ targetPath: path.join(hostDir, "result.txt"),+ kind: "file",+ },+ ],+ },+ ],+ }),+ ).rejects.toThrow("daytona_sandbox_not_found");+ });+it("syncOut snapshot guard re-checks the resolved source is a non-symlink regular file immediately before copying (validation→copy TOCTOU)", async () => {const hostDir = await makeHostDir();const sandbox = createMockSandbox();diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.tsindex 52819929b5..afb988d64e 100644--- a/packages/plugins/sandbox-providers/daytona/src/plugin.ts+++ b/packages/plugins/sandbox-providers/daytona/src/plugin.ts@@ -619,6 +619,8 @@ function leaseMetadata(input: {shellCommand: "bash" | "sh";remoteCwd: string;resumedLease: boolean;+ resumedFromState?: string | null;+ sandboxState?: string | null;workspaceSentinel?: WorkspaceSentinelResult;}) {return {@@ -626,7 +628,7 @@ function leaseMetadata(input: {shellCommand: input.shellCommand,sandboxId: input.sandbox.id,sandboxName: input.sandbox.name,- sandboxState: input.sandbox.state ?? null,+ sandboxState: input.sandboxState ?? input.sandbox.state ?? null,image: input.config.image,snapshot: input.config.snapshot,target: input.sandbox.target,@@ -637,6 +639,9 @@ function leaseMetadata(input: {...(input.config.archiveOnRelease ? { archiveOnRelease: true } : {}),remoteCwd: input.remoteCwd,resumedLease: input.resumedLease,+ ...(input.resumedLease+ ? { resumedFromState: input.resumedFromState ?? null }+ : {}),// Record the resources Paperclip attempted to request so future diagnosis// can compare requested allocation against what Daytona provisioned....(input.config.cpu != null ? { cpu: input.config.cpu } : {}),@@ -2187,61 +2192,72 @@ const plugin = definePlugin({return { providerLeaseId: null, metadata: { expired: true } };}- // A stopped sandbox loses its session shell, so the stored session id is- // stale after a real restart. Clear the id only when the sandbox is not- // already running, and clear it before the restart. A stopped sandbox has- // no live session, so the clear drops a dead id and a later command opens- // a fresh session. A running sandbox keeps its live session, so the resume- // leaves the id in place; a concurrent command still finds it and teardown- // deletes one session. An unconditional clear would drop the id of a live- // session and leak its shell until sandbox reaping.- if (sandbox.state !== "started") {- sandboxHandleSessionStore.clear(scope);- // A stopped sandbox loses its pseudo-terminals, so a stored duplex channel- // is dead after a real restart. Close and drop every channel on this lease- // before the restart, so no stale channel id survives the resume.- await closeDaytonaDuplexChannelsForLease(params.providerLeaseId);- }- await ensureSandboxStarted(sandbox, toTimeoutSeconds(config.timeoutMs));- try {- const remoteCwd = await resolveSandboxWorkingDirectory(sandbox);- // C3: a resumed lease must clear the workspace sentinel before it is- // trusted, even when the handle came from the cache. On any non-match we- // evict the cached handle and expire the lease so a stale/foreign sandbox- // is never reused on the subsequent (sentinel-skipping) exec path.- const workspaceSentinel = await verifyWorkspaceSentinel({- sandbox,- remoteCwd,- leaseMetadata: params.leaseMetadata,- timeoutSeconds: toTimeoutSeconds(config.timeoutMs),- });- if (workspaceSentinel.result !== "matched") {- evictSandboxHandle(scope);- return { providerLeaseId: null, metadata: { expired: true, workspaceSentinel } };- }- const shellCommand = await detectSandboxShellCommand(sandbox, toTimeoutSeconds(config.timeoutMs));- sandboxHandleCache.markFresh(scope);- sandboxHandleLeaseAdmissionStates.open(scope);- return {- providerLeaseId: sandbox.id,- metadata: leaseMetadata({- config,- sandbox,- shellCommand,- remoteCwd,- resumedLease: true,- workspaceSentinel,- }),- };- } catch (error) {- evictSandboxHandle(scope);- // A timeout, rate limit, or provider 5xx does not prove this sandbox is- // lost. Preserve the exact resource and let the host retry its recorded- // lease; replacement is permitted only after an explicit not-found or- // an immutable workspace identity mismatch.- throw error;- }- }, { allowClosed: true });+ // A stopped sandbox loses its session shell, so the stored session id is+ // stale after a real restart. Clear the id only when the sandbox is not+ // already running, and clear it before the restart. A stopped sandbox has+ // no live session, so the clear drops a dead id and a later command opens+ // a fresh session. A running sandbox keeps its live session, so the resume+ // leaves the id in place; a concurrent command still finds it and teardown+ // deletes one session. An unconditional clear would drop the id of a live+ // session and leak its shell until sandbox reaping.+ if (sandbox.state !== "started") {+ sandboxHandleSessionStore.clear(scope);+ // A stopped sandbox loses its pseudo-terminals, so a stored duplex channel+ // is dead after a real restart. Close and drop every channel on this lease+ // before the restart, so no stale channel id survives the resume.+ await closeDaytonaDuplexChannelsForLease(params.providerLeaseId);+ }+ const resumedFromState = sandbox.state ?? null;+ await ensureSandboxStarted(sandbox, toTimeoutSeconds(config.timeoutMs));+ try {+ const remoteCwd = await resolveSandboxWorkingDirectory(sandbox);+ // C3: a resumed lease must clear the workspace sentinel before it is+ // trusted, even when the handle came from the cache. On any non-match we+ // evict the cached handle and expire the lease so a stale/foreign sandbox+ // is never reused on the subsequent (sentinel-skipping) exec path.+ const workspaceSentinel = await verifyWorkspaceSentinel({+ sandbox,+ remoteCwd,+ leaseMetadata: params.leaseMetadata,+ timeoutSeconds: toTimeoutSeconds(config.timeoutMs),+ });+ if (workspaceSentinel.result !== "matched") {+ evictSandboxHandle(scope);+ return {+ providerLeaseId: null,+ metadata: { expired: true, workspaceSentinel },+ };+ }+ const shellCommand = await detectSandboxShellCommand(+ sandbox,+ toTimeoutSeconds(config.timeoutMs),+ );+ sandboxHandleCache.markFresh(scope);+ sandboxHandleLeaseAdmissionStates.open(scope);+ return {+ providerLeaseId: sandbox.id,+ metadata: leaseMetadata({+ config,+ sandbox,+ shellCommand,+ remoteCwd,+ resumedLease: true,+ resumedFromState,+ sandboxState: "started",+ workspaceSentinel,+ }),+ };+ } catch (error) {+ evictSandboxHandle(scope);+ // A timeout, rate limit, or provider 5xx does not prove this sandbox is+ // lost. Preserve the exact resource and let the host retry its recorded+ // lease; replacement is permitted only after an explicit not-found or+ // an immutable workspace identity mismatch.+ throw error;+ }+ },+ { allowClosed: true },+ );},async onEnvironmentReleaseLease(@@ -2828,18 +2844,28 @@ const plugin = definePlugin({providerLeaseId: params.lease.providerLeaseId,config,};- return await withSandboxActivityGate(scope, async () => {- const sandbox = await getSandbox(scope, { bypassTeardownGate: true });- await ensureSandboxStarted(sandbox, timeoutSeconds);- const result = await performSyncOut({- sandbox,- operations: params.operations,- remoteDir,- timeoutSeconds,+ try {+ return await withSandboxActivityGate(scope, async () => {+ const sandbox = await getSandbox(scope, { bypassTeardownGate: true });+ await ensureSandboxStarted(sandbox, timeoutSeconds);+ const result = await performSyncOut({+ sandbox,+ operations: params.operations,+ remoteDir,+ timeoutSeconds,+ });+ sandboxHandleCache.markFresh(scope);+ return result;});- sandboxHandleCache.markFresh(scope);- return result;- });+ } catch (error) {+ // A deleted Daytona sandbox is the one provider failure that proves its+ // unexported workspace bytes no longer exist. Convert the SDK class to a+ // stable cross-worker message; every other error remains retryable.+ if (error instanceof DaytonaNotFoundError) {+ throw new Error("daytona_sandbox_not_found");+ }+ throw error;+ }},// Open one live login pseudo-terminal. Resolve the cached sandbox by thediff --git a/server/src/__tests__/environment-runtime.test.ts b/server/src/__tests__/environment-runtime.test.tsindex e69e2fa9a9..d24350465f 100644--- a/server/src/__tests__/environment-runtime.test.ts+++ b/server/src/__tests__/environment-runtime.test.ts@@ -561,6 +561,18 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {mode: "shared_workspace",},});+ const workspaceSyncStamp = {+ schema: "paperclip.native-workspace-stamp/v1",+ workspaceId: seeded.executionWorkspaceId,+ providerLeaseId: "sandbox-exact-resume",+ remoteCwd: "/workspace",+ hostSha256: "a".repeat(64),+ finalizedRunId: seeded.runId,+ };+ await environmentService(db).updateLeaseMetadata(first.lease.id, {+ ...(first.lease.metadata ?? {}),+ nativeWorkspaceSync: workspaceSyncStamp,+ });await runtimeWithPlugin.releaseRunLeases(seeded.runId,"released",@@ -592,9 +604,17 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {expect(first.lease.metadata?.sandboxLeaseAcquisition).toEqual({ outcome: "created" });expect(acquired.lease.providerLeaseId).toBe("sandbox-exact-resume");- expect(acquired.lease.metadata?.sandboxLeaseAcquisition).toEqual({ outcome: "resumed" });- expect(workerManager.call.mock.calls.filter((call) => call[1] === "environmentAcquireLease"))- .toHaveLength(1);+ expect(acquired.lease.metadata?.sandboxLeaseAcquisition).toEqual({+ outcome: "resumed",+ });+ expect(acquired.lease.metadata?.nativeWorkspaceSync).toEqual(+ workspaceSyncStamp,+ );+ expect(+ workerManager.call.mock.calls.filter(+ (call) => call[1] === "environmentAcquireLease",+ ),+ ).toHaveLength(1);});it("destroys a disposable paperclip_runner sandbox after the turn", async () => {diff --git a/server/src/__tests__/native-sandbox-lifecycle.test.ts b/server/src/__tests__/native-sandbox-lifecycle.test.tsindex 4cc88c1a20..13b2553b9f 100644--- a/server/src/__tests__/native-sandbox-lifecycle.test.ts+++ b/server/src/__tests__/native-sandbox-lifecycle.test.ts@@ -1,5 +1,9 @@import { describe, expect, it } from "vitest";-import { resolveNativeSandboxLifecycle } from "../services/heartbeat.js";+import {+ providerResourceDispositionForTerminalRun,+ resolveNativeSandboxLifecycle,+ resolveReusableSandboxLifecycle,+} from "../services/heartbeat.js";const reusableSandbox = {kind: "remote" as const,@@ -21,6 +25,19 @@ describe("paperclip_runner sandbox lifecycle", () => {});});+ it("keeps the same warm reusable sandbox for a legacy adapter", () => {+ expect(+ resolveReusableSandboxLifecycle({+ lifecyclePolicy: { mode: "warm", idleTimeoutMs: 300_000 },+ target: reusableSandbox,+ }),+ ).toEqual({+ runnerProcess: "warm",+ sandboxResource: "keep_running",+ failoverBackup: "verified",+ });+ });+it("stops and reuses a per-turn reusable sandbox", () => {expect(resolveNativeSandboxLifecycle({adapterType: "paperclip_runner",@@ -71,4 +88,22 @@ describe("paperclip_runner sandbox lifecycle", () => {target: { kind: "local" },})).toBeNull();});++ it("keeps a warm sandbox only after a successful turn", () => {+ expect(+ providerResourceDispositionForTerminalRun("keep_running", "succeeded"),+ ).toBe("keep_running");+ expect(+ providerResourceDispositionForTerminalRun("keep_running", "failed"),+ ).toBe("stop_and_retain");+ expect(+ providerResourceDispositionForTerminalRun("keep_running", "cancelled"),+ ).toBe("stop_and_retain");+ expect(+ providerResourceDispositionForTerminalRun("keep_running", "timed_out"),+ ).toBe("stop_and_retain");+ expect(providerResourceDispositionForTerminalRun("destroy", "failed")).toBe(+ "destroy",+ );+ });});diff --git a/server/src/__tests__/native-workspace-sync.test.ts b/server/src/__tests__/native-workspace-sync.test.tsnew file mode 100644index 0000000000..a55ee16512--- /dev/null+++ b/server/src/__tests__/native-workspace-sync.test.ts@@ -0,0 +1,297 @@+import { mkdtemp, readdir, rm } from "node:fs/promises";+import os from "node:os";+import path from "node:path";+import { afterEach, describe, expect, it, vi } from "vitest";+import {+ directorySnapshotSha256,+ serializeDirectorySnapshot,+} from "@paperclipai/adapter-utils/workspace-restore-merge";++import {+ classifyNativeWorkspaceInbound,+ nativeWorkspaceSyncInternals,+ readNativeWorkspaceSyncReference,+ resumeNativeWorkspaceSync,+} from "../services/native-runtime/native-workspace-sync.js";++const digest = "a".repeat(64);++describe("native workspace sync durable metadata", () => {+ const originalPaperclipHome = process.env.PAPERCLIP_HOME;+ const originalPaperclipInstanceId = process.env.PAPERCLIP_INSTANCE_ID;+ const cleanupDirs: string[] = [];++ afterEach(async () => {+ if (originalPaperclipHome === undefined) delete process.env.PAPERCLIP_HOME;+ else process.env.PAPERCLIP_HOME = originalPaperclipHome;+ if (originalPaperclipInstanceId === undefined)+ delete process.env.PAPERCLIP_INSTANCE_ID;+ else process.env.PAPERCLIP_INSTANCE_ID = originalPaperclipInstanceId;+ await Promise.all(+ cleanupDirs+ .splice(0)+ .map((directory) => rm(directory, { recursive: true, force: true })),+ );+ });++ it("classifies fresh, warm, replacement, and same-run recovery inputs", () => {+ expect(+ classifyNativeWorkspaceInbound({+ kind: "new_run",+ acquisition: "created",+ hasPriorStamp: false,+ }),+ ).toBe("host_current");+ expect(+ classifyNativeWorkspaceInbound({+ kind: "new_run",+ acquisition: "resumed",+ hasPriorStamp: true,+ }),+ ).toBe("adopt_remote");+ expect(+ classifyNativeWorkspaceInbound({+ kind: "new_run",+ acquisition: "resumed",+ hasPriorStamp: false,+ }),+ ).toBe("host_current");+ expect(+ classifyNativeWorkspaceInbound({+ kind: "new_run",+ acquisition: "replacement",+ hasPriorStamp: true,+ }),+ ).toBe("host_current");+ expect(+ classifyNativeWorkspaceInbound({+ kind: "existing_run",+ restartRecovery: true,+ sameProviderLease: true,+ }),+ ).toBe("adopt_remote");+ expect(+ classifyNativeWorkspaceInbound({+ kind: "existing_run",+ restartRecovery: true,+ sameProviderLease: false,+ }),+ ).toBe("durable_seed");+ expect(() =>+ classifyNativeWorkspaceInbound({+ kind: "existing_run",+ restartRecovery: false,+ sameProviderLease: true,+ }),+ ).toThrow("native_workspace_sync_unexpected_existing_descriptor");+ });++ it("reads backward-compatible references and the resource disposition", () => {+ const base = {+ schema: "paperclip.native-workspace-sync/v1",+ state: "prepared",+ descriptorSha256: digest,+ baselineSha256: digest,+ finalHostSha256: null,+ workspaceId: "workspace-1",+ leaseId: "lease-1",+ providerLeaseId: "sandbox-1",+ remoteCwd: "/workspace",+ };++ expect(readNativeWorkspaceSyncReference(base)).toEqual({+ ...base,+ resourceDisposition: null,+ });+ expect(+ readNativeWorkspaceSyncReference({+ ...base,+ resourceDisposition: "keep_running",+ }),+ ).toEqual({ ...base, resourceDisposition: "keep_running" });+ expect(+ readNativeWorkspaceSyncReference({+ ...base,+ resourceDisposition: "delete_everything",+ }),+ ).toBeNull();+ });++ it("rejects traversal before constructing a durable state path", () => {+ expect(() =>+ nativeWorkspaceSyncInternals.descriptorPath("../run", digest),+ ).toThrow("native_workspace_sync_invalid_run_id");+ expect(() =>+ nativeWorkspaceSyncInternals.descriptorPath("run-1", "../descriptor"),+ ).toThrow("native_workspace_sync_descriptor_digest_invalid");+ });++ it("writes one immutable descriptor when the same state is replayed", async () => {+ const paperclipHome = await mkdtemp(+ path.join(os.tmpdir(), "paperclip-native-workspace-sync-"),+ );+ cleanupDirs.push(paperclipHome);+ process.env.PAPERCLIP_HOME = paperclipHome;+ process.env.PAPERCLIP_INSTANCE_ID = "descriptor-test";+ const baseline = {+ exclude: [".paperclip-runtime"],+ entries: new Map([+ [+ "continuity.txt",+ { kind: "file" as const, mode: 0o644, hash: digest },+ ],+ ]),+ };+ const descriptor = {+ schema: "paperclip.native-workspace-sync/v1" as const,+ binding: {+ runId: "run-idempotent",+ companyId: "company-1",+ workspaceId: "workspace-1",+ leaseId: "lease-1",+ providerLeaseId: "sandbox-1",+ localCwd: path.join(paperclipHome, "workspace"),+ remoteCwd: "/workspace",+ },+ state: "prepared" as const,+ baselineSha256: directorySnapshotSha256(baseline),+ baseline: serializeDirectorySnapshot(baseline),+ gitSnapshot: null,+ seed: null,+ createdAt: "2026-01-01T00:00:00.000Z",+ finalizedAt: null,+ finalHostSha256: null,+ resourceDisposition: "keep_running" as const,+ };++ const first =+ await nativeWorkspaceSyncInternals.writeDescriptor(descriptor);+ const second =+ await nativeWorkspaceSyncInternals.writeDescriptor(descriptor);++ expect(second).toEqual(first);+ const files = await readdir(+ path.dirname(+ nativeWorkspaceSyncInternals.descriptorPath(+ descriptor.binding.runId,+ first.descriptorSha256,+ ),+ ),+ );+ expect(files.filter((file) => file.endsWith(".json"))).toEqual([+ `descriptor.${first.descriptorSha256}.json`,+ ]);+ await expect(+ nativeWorkspaceSyncInternals.readDescriptor({+ runId: descriptor.binding.runId,+ reference: first,+ }),+ ).resolves.toMatchObject({ descriptor });+ });++ it("repairs finalized remote and lease stamps after an interrupted commit", async () => {+ const paperclipHome = await mkdtemp(+ path.join(os.tmpdir(), "paperclip-native-workspace-sync-repair-"),+ );+ cleanupDirs.push(paperclipHome);+ process.env.PAPERCLIP_HOME = paperclipHome;+ process.env.PAPERCLIP_INSTANCE_ID = "descriptor-repair-test";+ const baseline = {+ exclude: [".paperclip-runtime"],+ entries: new Map([+ [+ "continuity.txt",+ { kind: "file" as const, mode: 0o644, hash: digest },+ ],+ ]),+ };+ const finalHostSha256 = "b".repeat(64);+ const descriptor = {+ schema: "paperclip.native-workspace-sync/v1" as const,+ binding: {+ runId: "run-finalized-repair",+ companyId: "company-1",+ workspaceId: "workspace-1",+ leaseId: "lease-1",+ providerLeaseId: "sandbox-1",+ localCwd: path.join(paperclipHome, "workspace"),+ remoteCwd: "/workspace",+ },+ state: "finalized" as const,+ baselineSha256: directorySnapshotSha256(baseline),+ baseline: serializeDirectorySnapshot(baseline),+ gitSnapshot: null,+ seed: null,+ createdAt: "2026-01-01T00:00:00.000Z",+ finalizedAt: "2026-01-01T00:01:00.000Z",+ finalHostSha256,+ resourceDisposition: "keep_running" as const,+ };+ const reference =+ await nativeWorkspaceSyncInternals.writeDescriptor(descriptor);+ const rows = (values: unknown[]) => {+ const query = {+ from: () => query,+ where: () => query,+ for: () => query,+ limit: () => query,+ then: <TResult1 = unknown, TResult2 = never>(+ onfulfilled?:+ ((value: unknown[]) => TResult1 | PromiseLike<TResult1>) | null,+ onrejected?:+ ((reason: unknown) => TResult2 | PromiseLike<TResult2>) | null,+ ) => Promise.resolve(values).then(onfulfilled, onrejected),+ };+ return query;+ };+ let persistedLeaseMetadata: Record<string, unknown> | null = null;+ const db = {+ select: () =>+ rows([{ runnerProfileJson: { nativeWorkspaceSync: reference } }]),+ transaction: async (callback: (tx: unknown) => Promise<unknown>) =>+ callback({+ select: () => rows([{ metadata: { retained: true } }]),+ update: () => ({+ set: (value: { metadata: Record<string, unknown> }) => ({+ where: async () => {+ persistedLeaseMetadata = value.metadata;+ },+ }),+ }),+ }),+ };+ const execute = vi+ .fn()+ .mockResolvedValueOnce({ timedOut: false, exitCode: 1, stdout: "" })+ .mockResolvedValueOnce({ timedOut: false, exitCode: 0, stdout: "" });++ await expect(+ resumeNativeWorkspaceSync({+ db: db as never,+ runId: descriptor.binding.runId,+ target: {+ kind: "remote",+ transport: "sandbox",+ remoteCwd: descriptor.binding.remoteCwd,+ sandboxLeaseAcquisition: {+ providerLeaseId: descriptor.binding.providerLeaseId,+ },+ runner: { execute },+ } as never,+ }),+ ).resolves.toBe(true);++ expect(execute).toHaveBeenCalledTimes(2);+ expect(persistedLeaseMetadata).toMatchObject({+ retained: true,+ nativeWorkspaceSync: {+ schema: "paperclip.native-workspace-stamp/v1",+ workspaceId: descriptor.binding.workspaceId,+ providerLeaseId: descriptor.binding.providerLeaseId,+ remoteCwd: descriptor.binding.remoteCwd,+ hostSha256: finalHostSha256,+ finalizedRunId: descriptor.binding.runId,+ },+ });+ });+});diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.tsindex 5451d53c0c..f35b16fb40 100644--- a/server/src/services/environment-runtime.ts+++ b/server/src/services/environment-runtime.ts@@ -2060,6 +2060,11 @@ function createSandboxEnvironmentDriver(...(reusableLease?.metadata?.nativeHarnessBackup? { nativeHarnessBackup: reusableLease.metadata.nativeHarnessBackup }: {}),+ ...(providerLease && reusableLease?.metadata?.nativeWorkspaceSync+ ? {+ nativeWorkspaceSync: reusableLease.metadata.nativeWorkspaceSync,+ }+ : {}),...(reusableScope ? { reusableSandboxLease: reusableScope } : {}),};try {@@ -2282,6 +2287,13 @@ function createSandboxEnvironmentDriver(...(reusableLease?.metadata?.nativeHarnessBackup? { nativeHarnessBackup: reusableLease.metadata.nativeHarnessBackup }: {}),+ ...(reusableLease &&+ providerLease.providerLeaseId === reusableLease.providerLeaseId &&+ reusableLease.metadata?.nativeWorkspaceSync+ ? {+ nativeWorkspaceSync: reusableLease.metadata.nativeWorkspaceSync,+ }+ : {}),...(reusableScope ? { reusableSandboxLease: reusableScope } : {}),};try {@@ -3154,6 +3166,7 @@ const INTERNAL_PLUGIN_SANDBOX_CONFIG_KEYS = new Set(["sandboxProviderPlugin","sandboxLeaseAcquisition","nativeHarnessBackup",+ "nativeWorkspaceSync",]);// Drop the host-internal and per-lease runtime keys from a sandbox configdiff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.tsindex 195ad4810b..94a52139ee 100644--- a/server/src/services/heartbeat.ts+++ b/server/src/services/heartbeat.ts@@ -142,6 +142,9 @@ import {materializeLegacyQuestionResponseWakeProjection,materializeNativeInteractionResponses,NativeCancellationPendingRecoveryError,+ prepareNativeWorkspaceSync,+ readNativeWorkspaceSyncReference,+ recordNativeFinalizationFailure,type NativeRestartRecoveryClaim,rebindNativeSessionCheckpoint,reconcileNativeFinalizations,@@ -1746,14 +1749,22 @@ export function leaseReleaseStatusForRunStatus(return status === "failed" || status === "timed_out" ? "failed" : "released";}+export function providerResourceDispositionForTerminalRun(+ desired: ProviderResourceDisposition | undefined,+ status: string | null | undefined,+): ProviderResourceDisposition | undefined {+ return desired === "keep_running" && status !== "succeeded"+ ? "stop_and_retain"+ : desired;+}+export interface NativeSandboxLifecycle {runnerProcess: "per_turn" | "warm";sandboxResource: "keep_running" | "stop_and_reuse" | "destroy_after_turn";failoverBackup: "verified";}-export function resolveNativeSandboxLifecycle(input: {- adapterType: string;+export function resolveReusableSandboxLifecycle(input: {lifecyclePolicy:| { mode: "per_turn"; idleTimeoutMs: null }| { mode: "warm"; idleTimeoutMs: number };@@ -1764,12 +1775,9 @@ export function resolveNativeSandboxLifecycle(input: {effectiveCapabilities?: { reusableLeases: boolean } | null;} | null;}): NativeSandboxLifecycle | null {- if (- input.adapterType !== "paperclip_runner" ||- input.target?.kind !== "remote" ||- input.target.transport !== "sandbox"- )+ if (input.target?.kind !== "remote" || input.target.transport !== "sandbox") {return null;+ }const reusableLease =input.target.reusableLeaseConfigured === true &&input.target.effectiveCapabilities?.reusableLeases === true;@@ -1788,6 +1796,27 @@ export function resolveNativeSandboxLifecycle(input: {};}+export function resolveNativeSandboxLifecycle(input: {+ adapterType: string;+ lifecyclePolicy:+ | { mode: "per_turn"; idleTimeoutMs: null }+ | { mode: "warm"; idleTimeoutMs: number };+ target: {+ kind: "local" | "remote";+ transport?: string;+ reusableLeaseConfigured?: boolean;+ effectiveCapabilities?: { reusableLeases: boolean } | null;+ } | null;+}): NativeSandboxLifecycle | null {+ if (+ input.adapterType !== "paperclip_runner" ||+ input.target?.kind !== "remote" ||+ input.target.transport !== "sandbox"+ )+ return null;+ return resolveReusableSandboxLifecycle(input);+}+export function applyPersistedExecutionWorkspaceConfig(input: {config: Record<string, unknown>;workspaceConfig: ExecutionWorkspaceConfig | null;@@ -8170,6 +8199,18 @@ class NativeSessionResumeScheduledError extends Error {}}+class NativeWorkspaceFinalizeScheduledError extends Error {+ constructor(+ readonly original: unknown,+ readonly terminalFailure: boolean,+ readonly reasonCode:+ "workspace_sync_out_failed" | "workspace_sync_out_unrecoverable",+ ) {+ super("Native workspace finalization recovery has been scheduled.");+ this.name = "NativeWorkspaceFinalizeScheduledError";+ }+}+type WorkspaceReadyCommentWriter = {addComment: (issueId: string,@@ -12945,7 +12986,10 @@ export function heartbeatService(// A result committed before the old controller stopped outranks process// recovery. Finish its durable workspace/status suffix before deciding// whether any provider authority needs to be reopened.- await reconcileNativeFinalizations(db);+ await reconcileNativeFinalizations(db, undefined, {+ environmentRuntime,+ onWorkspaceSettled: settleRecoveredNativeWorkspace,+ });const intent = await readHotRestartIntent().catch((error) => {logger.warn({ err: error },@@ -16854,6 +16898,41 @@ export function heartbeatService(return blocked;}+ async function settleRecoveredNativeWorkspace(input: {+ runId: string;+ companyId: string;+ agentId: string;+ succeeded: boolean;+ }) {+ const settledRun = await getRun(input.runId);+ const workspaceSyncReference = readNativeWorkspaceSyncReference(+ parseObject(settledRun?.runnerProfileJson).nativeWorkspaceSync,+ );+ await releaseEnvironmentLeasesForRun({+ runId: input.runId,+ companyId: input.companyId,+ agentId: input.agentId,+ status: settledRun?.status,+ failureReason: settledRun?.error ?? undefined,+ providerResourceDisposition: input.succeeded+ ? (workspaceSyncReference?.resourceDisposition ?? "stop_and_retain")+ : "stop_and_retain",+ });+ await releaseRuntimeServicesForRun(input.runId).catch(() => undefined);+ await finalizeAgentStatus(+ input.agentId,+ input.succeeded ? "succeeded" : "failed",+ input.succeeded+ ? null+ : (settledRun?.error ?? "native_workspace_sync_out_failed"),+ {+ wasFirstHeartbeat: settledRun+ ? timerClaimWasFirstHeartbeat(settledRun)+ : undefined,+ },+ ).catch(() => undefined);+ }+async function reapOrphanedRuns(opts?: { staleThresholdMs?: number }) {const staleThresholdMs = opts?.staleThresholdMs ?? 0;const now = new Date();@@ -16861,7 +16940,10 @@ export function heartbeatService(// Complete persisted native results before generic orphan recovery. The// reconciler reads the durable workspace barrier and persisted runtime// mode, never the current feature flag.- await reconcileNativeFinalizations(db).catch((error) => {+ await reconcileNativeFinalizations(db, undefined, {+ environmentRuntime,+ onWorkspaceSettled: settleRecoveredNativeWorkspace,+ }).catch((error) => {logger.warn({ err: error },"failed to reconcile persisted native finalizations before orphan reaping",@@ -17876,6 +17958,10 @@ export function heartbeatService(activeRunExecutions.add(run.id);let runScratch: HeartbeatRunScratch | null = null;let nativeSessionResumeScheduled = false;+ let nativeWorkspaceFinalizeScheduled = false;+ let nativeWorkspaceSync: Awaited<+ ReturnType<typeof prepareNativeWorkspaceSync>+ > = null;let providerResourceDispositionForRun:ProviderResourceDisposition | undefined;let nativeLifecycleTelemetryForRun:@@ -19659,6 +19745,9 @@ export function heartbeatService(driver: selectedEnvironment.driver,leaseId: activeEnvironmentLease.lease.id,workspaceRealization,+ sandboxLeaseAcquisition:+ activeEnvironmentLease.lease.metadata?.sandboxLeaseAcquisition ??+ null,...(typeof activeEnvironmentLease.lease.metadata?.remoteCwd === "string"? {remoteCwd: activeEnvironmentLease.lease.metadata.remoteCwd,@@ -20363,6 +20452,9 @@ export function heartbeatService(agentId: heartbeatRuns.agentId,runnerInstanceId: heartbeatRuns.runnerInstanceId,nativeSessionId: heartbeatRuns.nativeSessionId,+ processPid: heartbeatRuns.processPid,+ processGroupId: heartbeatRuns.processGroupId,+ processStartedAt: heartbeatRuns.processStartedAt,runnerProfileJson: heartbeatRuns.runnerProfileJson,}).from(heartbeatRuns)@@ -20718,6 +20810,21 @@ export function heartbeatService(? previousNativeRun.runnerInstanceId: (lockedRun.runnerInstanceId ?? nativeRunnerInstanceId),nativeSessionId: lockedRun.nativeSessionId ?? nativeSessionId,+ processPid:+ lockedRun.processPid ??+ (previousNativeRun?.nativeSessionId === nativeSessionId+ ? previousNativeRun.processPid+ : null),+ processGroupId:+ lockedRun.processGroupId ??+ (previousNativeRun?.nativeSessionId === nativeSessionId+ ? previousNativeRun.processGroupId+ : null),+ processStartedAt:+ lockedRun.processStartedAt ??+ (previousNativeRun?.nativeSessionId === nativeSessionId+ ? previousNativeRun.processStartedAt+ : null),nativeIssueId: lockedRun.nativeIssueId ?? issueRef.id,driverKind:lockedRun.driverKind ??@@ -20745,7 +20852,30 @@ export function heartbeatService(}).onConflictDoNothing();});+ nativeWorkspaceSync = await prepareNativeWorkspaceSync({+ db,+ runId: run.id,+ companyId: agent.companyId,+ workspaceId: nativeExecutionWorkspaceId,+ workspaceLocalDir: executionWorkspace.cwd,+ target: executionTarget,+ lease: activeEnvironmentLease.lease,+ restartRecovery: runOptions.nativeRestartRecovery,+ resourceDisposition: providerResourceDispositionForRun,+ });} else {+ const legacyWarmLifecycle =+ executionTarget?.kind === "remote" &&+ executionTarget.transport === "sandbox" &&+ executionTarget.runnerLifecyclePolicy?.mode === "warm"+ ? resolveReusableSandboxLifecycle({+ lifecyclePolicy: executionTarget.runnerLifecyclePolicy,+ target: executionTarget,+ })+ : null;+ if (legacyWarmLifecycle?.sandboxResource === "keep_running") {+ providerResourceDispositionForRun = "keep_running";+ }await db.update(heartbeatRuns).set({@@ -21315,19 +21445,26 @@ export function heartbeatService(// If recording the barrier itself fails, propagate as a run failure// rather than silently leaving dependents stranded behind a missing// finalize row.+ if (nativeWorkspaceSync) {+ await nativeWorkspaceSync.restoreWorkspace();+ }await recordWorkspaceFinalize("succeeded");if (adapterResult.nativeFinalization) {adapterResult.nativeFinalization.workspaceFinalizeStatus ="succeeded";try {- await finalizeNativeRun({+ const finalized = await finalizeNativeRun({db,runId: run.id,workspaceFinalizeStatus: "succeeded",+ preserveProviderAttempt: Boolean(nativeWorkspaceSync),});await dispatchPendingNativeStatusWakeups({companyId: run.companyId,});+ if (finalized.phase === "committed") {+ await nativeWorkspaceSync?.cleanup();+ }} catch (finalizeErr) {logger.warn({ err: finalizeErr, runId: run.id },@@ -21379,6 +21516,40 @@ export function heartbeatService();}if (nativeRuntimeResolution.kind === "native") {+ const proposedResult = await db+ .select({ resultId: nativeRunFinalizations.resultId })+ .from(nativeRunFinalizations)+ .where(eq(nativeRunFinalizations.runId, run.id))+ .limit(1)+ .then((rows) => rows[0]?.resultId ?? null);+ if (proposedResult && nativeWorkspaceSync) {+ const workspaceFailureMessage =+ adapterErr instanceof Error ? adapterErr.message : "";+ const unrecoverable =+ workspaceFailureMessage ===+ "workspace_sync_out_unrecoverable" ||+ workspaceFailureMessage.includes("daytona_sandbox_not_found");+ const failure = await recordNativeFinalizationFailure({+ db,+ runId: run.id,+ error: new Error(+ unrecoverable+ ? "native_workspace_sync_out_unrecoverable"+ : "native_workspace_sync_out_failed",+ ),+ projectRunStatus: true,+ failureScope: "workspace",+ permanent: unrecoverable,+ });+ nativeWorkspaceFinalizeScheduled = true;+ throw new NativeWorkspaceFinalizeScheduledError(+ adapterErr,+ failure.phase === "terminal_failure",+ unrecoverable+ ? "workspace_sync_out_unrecoverable"+ : "workspace_sync_out_failed",+ );+ }try {await finalizeNativeRun({db,@@ -22064,6 +22235,45 @@ export function heartbeatService(}return;}+ if (err instanceof NativeWorkspaceFinalizeScheduledError) {+ const coordinator = await db+ .select({+ nextAttemptAt: nativeRunFinalizations.nextAttemptAt,+ attempt: nativeRunFinalizations.attempt,+ })+ .from(nativeRunFinalizations)+ .where(eq(nativeRunFinalizations.runId, run.id))+ .limit(1)+ .then((rows) => rows[0] ?? null);+ await appendRunEvent(run, {+ eventType: "lifecycle",+ stream: "system",+ level: err.terminalFailure ? "error" : "warn",+ message: err.terminalFailure+ ? "native result is durable, but the sandbox containing unexported workspace changes is unrecoverable"+ : "native result is durable; workspace copy-back will retry without another provider turn",+ payload: {+ attempt: coordinator?.attempt ?? null,+ nextAttemptAt: coordinator?.nextAttemptAt?.toISOString() ?? null,+ fallbackSuppressed: true,+ retryReasonCode: err.reasonCode,+ },+ }).catch(() => undefined);+ if (err.terminalFailure) {+ // The durable coordinator already failed the run, blocked the+ // issue, and cleared its execution lock. Let ordinary teardown+ // release the now-useless lease and return the agent to service.+ nativeWorkspaceFinalizeScheduled = false;+ providerResourceDispositionForRun = "stop_and_retain";+ await finalizeAgentStatus(+ run.agentId,+ "failed",+ "native_workspace_sync_out_unrecoverable",+ { wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run) },+ ).catch(() => undefined);+ }+ return;+ }const message = redactCurrentUserText(err instanceof Error ? err.message : "Unknown adapter failure",await getCurrentUserRedactionOptions(),@@ -22456,7 +22666,11 @@ export function heartbeatService(// terminal". When the teardown reaches this point with the run still// running or queued, force a terminal status before the lease is// released, so the UI never shows a finished task as "Live".- if (latestRun && !nativeSessionResumeScheduled) {+ if (+ latestRun &&+ !nativeSessionResumeScheduled &&+ !nativeWorkspaceFinalizeScheduled+ ) {latestRun = await terminalizeRunOnLeaseRelease(latestRun).catch((terminalizeErr) => {logger.error(@@ -22467,7 +22681,15 @@ export function heartbeatService(},);}- if (!nativeSessionResumeScheduled) {+ // Warm retention is earned only by a fully successful turn. A failed,+ // cancelled, or timed-out run stops the reusable sandbox so the next+ // acquisition must revalidate and explicitly resume it.+ providerResourceDispositionForRun =+ providerResourceDispositionForTerminalRun(+ providerResourceDispositionForRun,+ latestRun?.status,+ );+ if (!nativeSessionResumeScheduled && !nativeWorkspaceFinalizeScheduled) {await releaseEnvironmentLeasesForRun({runId: run.id,companyId: run.companyId,@@ -22539,7 +22761,11 @@ export function heartbeatService(}}activeRunExecutions.delete(run.id);- if (!nativeSessionResumeScheduled && !shutdownInProgress) {+ if (+ !nativeSessionResumeScheduled &&+ !nativeWorkspaceFinalizeScheduled &&+ !shutdownInProgress+ ) {await startNextQueuedRunForAgent(run.agentId);}}diff --git a/server/src/services/native-runtime/index.ts b/server/src/services/native-runtime/index.tsindex f5edaaacb5..643e18b9a9 100644--- a/server/src/services/native-runtime/index.ts+++ b/server/src/services/native-runtime/index.ts@@ -9,4 +9,5 @@ export * from "./paperclip-control-plane-port.js";export * from "./native-run-finalizer.js";export * from "./native-finalization-reconciler.js";export * from "./native-restart-recovery.js";+export * from "./native-workspace-sync.js";export * from "./status-arbiter.js";diff --git a/server/src/services/native-runtime/native-finalization-reconciler.ts b/server/src/services/native-runtime/native-finalization-reconciler.tsindex 4e7073c42c..294a92b985 100644--- a/server/src/services/native-runtime/native-finalization-reconciler.ts+++ b/server/src/services/native-runtime/native-finalization-reconciler.ts@@ -24,6 +24,11 @@ import { issueRecoveryActionService } from "../issue-recovery-actions.js";import { issueService } from "../issues.js";import { emitAgentTaskRun } from "../agent-task-run-telemetry.js";import { resumeNativeWorkspaceFinalization } from "./native-workspace-finalizer.js";+import {+ cleanupNativeWorkspaceSync,+ readNativeWorkspaceSyncReference,+} from "./native-workspace-sync.js";+import type { EnvironmentRuntimeService } from "../environment-runtime.js";import { classifyNativeEvidence } from "./evidence-classifier.js";import { recordNativeWorkAssessment } from "./work-assessments.js";import {@@ -373,19 +378,33 @@ export async function claimNativeSessionResumptions(input: {}/** Recovery is keyed only by persisted mode/coordinator state, never the live flag. */-export async function reconcileNativeFinalizations(db: Db, runIds?: string[]) {- const rows = await db.select({- runId: heartbeatRuns.id,- companyId: heartbeatRuns.companyId,- agentId: heartbeatRuns.agentId,- issueId: nativeRunFinalizations.issueId,- issueStatus: issues.status,- issueStatusVersion: issues.statusVersion,- issueDecisionId: issues.lastStatusDecisionId,- coordinatorPhase: nativeRunFinalizations.phase,- assessmentId: nativeRunFinalizations.assessmentId,- decisionId: nativeRunFinalizations.decisionId,- })+export async function reconcileNativeFinalizations(+ db: Db,+ runIds?: string[],+ options: {+ environmentRuntime?: EnvironmentRuntimeService;+ onWorkspaceSettled?: (input: {+ runId: string;+ companyId: string;+ agentId: string;+ succeeded: boolean;+ }) => Promise<void>;+ } = {},+) {+ const rows = await db+ .select({+ runId: heartbeatRuns.id,+ companyId: heartbeatRuns.companyId,+ agentId: heartbeatRuns.agentId,+ issueId: nativeRunFinalizations.issueId,+ issueStatus: issues.status,+ issueStatusVersion: issues.statusVersion,+ issueDecisionId: issues.lastStatusDecisionId,+ coordinatorPhase: nativeRunFinalizations.phase,+ assessmentId: nativeRunFinalizations.assessmentId,+ decisionId: nativeRunFinalizations.decisionId,+ runnerProfileJson: heartbeatRuns.runnerProfileJson,+ }).from(heartbeatRuns).innerJoin(nativeRunFinalizations, eq(nativeRunFinalizations.runId, heartbeatRuns.id)).innerJoin(issues, and(@@ -617,14 +636,59 @@ export async function reconcileNativeFinalizations(db: Db, runIds?: string[]) {if (!recoveryDecision.effects.some((effect) => effect.kind === "resume_workspace_operation")) {throw new Error("native_reconciliation_workspace_resume_policy_missing");}- const operation = await resumeNativeWorkspaceFinalization({ db, runId: row.runId });- const workspaceFinalizeStatus = operation.status === "succeeded" ? "succeeded" : "failed";+ const operation = await resumeNativeWorkspaceFinalization({+ db,+ runId: row.runId,+ environmentRuntime: options.environmentRuntime,+ });+ const workspaceFinalizeStatus =+ operation.status === "succeeded" ? "succeeded" : "failed";+ if (workspaceFinalizeStatus === "failed") {+ const unrecoverable = operation.stderrExcerpt?.includes(+ "workspace_sync_out_unrecoverable",+ );+ const failure = await recordNativeFinalizationFailure({+ db,+ runId: row.runId,+ error: new Error(+ unrecoverable+ ? "native_workspace_sync_out_unrecoverable"+ : "native_workspace_sync_out_failed",+ ),+ projectRunStatus: true,+ failureScope: "workspace",+ permanent: unrecoverable,+ });+ results.push({+ ...failure,+ reconciliationAction: "resume_workspace_operation" as const,+ workspaceOperationId: operation.id,+ workspaceFinalizeStatus,+ reconciliationDecision: recoveryDecision,+ });+ if (failure.phase === "terminal_failure") {+ await options+ .onWorkspaceSettled?.({+ runId: row.runId,+ companyId: row.companyId,+ agentId: row.agentId,+ succeeded: false,+ })+ .catch(() => undefined);+ }+ continue;+ }try {const finalized = await finalizeNativeRun({db,runId: row.runId,workspaceFinalizeStatus,projectRunStatus: true,+ preserveProviderAttempt: Boolean(+ readNativeWorkspaceSyncReference(+ record(row.runnerProfileJson).nativeWorkspaceSync,+ ),+ ),});results.push({...finalized,@@ -633,6 +697,20 @@ export async function reconcileNativeFinalizations(db: Db, runIds?: string[]) {workspaceFinalizeStatus,reconciliationDecision: recoveryDecision,});+ if (+ workspaceFinalizeStatus === "succeeded" &&+ finalized.phase === "committed"+ ) {+ await cleanupNativeWorkspaceSync(row.runId).catch(() => undefined);+ await options+ .onWorkspaceSettled?.({+ runId: row.runId,+ companyId: row.companyId,+ agentId: row.agentId,+ succeeded: true,+ })+ .catch(() => undefined);+ }} catch (error) {const failure = await recordNativeFinalizationFailure({db,@@ -659,12 +737,29 @@ export async function reconcileNativeFinalizations(db: Db, runIds?: string[]) {throw new Error("native_reconciliation_replay_policy_invalid");}try {- results.push(await finalizeNativeRun({+ const finalized = await finalizeNativeRun({db,runId: row.runId,workspaceFinalizeStatus: barrier.status as "succeeded" | "failed",projectRunStatus: true,- }));+ preserveProviderAttempt: Boolean(+ readNativeWorkspaceSyncReference(+ record(row.runnerProfileJson).nativeWorkspaceSync,+ ),+ ),+ });+ results.push(finalized);+ if (finalized.phase === "committed") {+ await cleanupNativeWorkspaceSync(row.runId).catch(() => undefined);+ await options+ .onWorkspaceSettled?.({+ runId: row.runId,+ companyId: row.companyId,+ agentId: row.agentId,+ succeeded: true,+ })+ .catch(() => undefined);+ }} catch (error) {results.push(await recordNativeFinalizationFailure({db,diff --git a/server/src/services/native-runtime/native-run-finalizer-telemetry.test.ts b/server/src/services/native-runtime/native-run-finalizer-telemetry.test.tsindex ab0446d59a..c06e11c78b 100644--- a/server/src/services/native-runtime/native-run-finalizer-telemetry.test.ts+++ b/server/src/services/native-runtime/native-run-finalizer-telemetry.test.ts@@ -279,6 +279,93 @@ describeEmbeddedPostgres("native run finalizer / status decision committer — aexpect(run?.status).toBe("running");});+ it("bounds workspace-only retries without consuming the provider attempt", async () => {+ const fixture = await seedNativeRun();+ await db.insert(nativeRunFinalizations).values({+ runId: fixture.runId,+ companyId,+ issueId: fixture.issueId,+ phase: "observed",+ attempt: 1,+ });++ for (+ let expectedAttempt = 1;+ expectedAttempt <= 3;+ expectedAttempt += 1+ ) {+ await recordNativeFinalizationFailure({+ db,+ runId: fixture.runId,+ error: new Error("native_workspace_sync_out_failed"),+ projectRunStatus: true,+ failureScope: "workspace",+ });+ const coordinator = await db+ .select()+ .from(nativeRunFinalizations)+ .where(eq(nativeRunFinalizations.runId, fixture.runId))+ .then((rows) => rows[0]!);+ expect(coordinator.attempt).toBe(1);+ expect(coordinator.failureDetail).toMatchObject({+ workspaceFinalizeAttempt: expectedAttempt,+ });+ expect(coordinator.phase).toBe(+ expectedAttempt === 3 ? "terminal_failure" : "retryable_failure",+ );+ }++ await expect(+ db+ .select({ status: heartbeatRuns.status })+ .from(heartbeatRuns)+ .where(eq(heartbeatRuns.id, fixture.runId))+ .then((rows) => rows[0]?.status),+ ).resolves.toBe("failed");+ });++ it("blocks immediately when the sandbox with unexported changes is gone", async () => {+ const fixture = await seedNativeRun();+ await db.insert(nativeRunFinalizations).values({+ runId: fixture.runId,+ companyId,+ issueId: fixture.issueId,+ phase: "workspace_finalizing",+ attempt: 1,+ resultId: null,+ });++ const failure = await recordNativeFinalizationFailure({+ db,+ runId: fixture.runId,+ error: new Error("native_workspace_sync_out_unrecoverable"),+ projectRunStatus: true,+ failureScope: "workspace",+ permanent: true,+ });++ expect(failure).toMatchObject({+ phase: "terminal_failure",+ failureCode: "native_workspace_sync_out_unrecoverable",+ nextAttemptAt: null,+ attempt: 1,+ });+ await expect(+ db+ .select({ status: heartbeatRuns.status })+ .from(heartbeatRuns)+ .where(eq(heartbeatRuns.id, fixture.runId))+ .then((rows) => rows[0]?.status),+ ).resolves.toBe("failed");+ await expect(+ db+ .select({ status: issues.status })+ .from(issues)+ .where(eq(issues.id, fixture.issueId))+ .then((rows) => rows[0]?.status),+ ).resolves.toBe("blocked");+ });+it("emits exactly one event for a cancel_continuations write (trap 2: :685/:518 overlap)", async () => {const fixture = await seedNativeRun();// Build the minimal real rows commitNativeStatusDecision's foreign keys@@ -329,7 +416,10 @@ describeEmbeddedPostgres("native run finalizer / status decision committer — atoStatus: "cancelled",reasonCode: "cancellation_issue_authorized",unblockDescriptor: null,- effects: [{ kind: "release_checkout" }, { kind: "cancel_continuations" }],+ effects: [+ { kind: "release_checkout" },+ { kind: "cancel_continuations" },+ ],};const callsBefore = mockTelemetryClient.track.mock.calls.length;diff --git a/server/src/services/native-runtime/native-run-finalizer.ts b/server/src/services/native-runtime/native-run-finalizer.tsindex 979013c4d5..6042c748fb 100644--- a/server/src/services/native-runtime/native-run-finalizer.ts+++ b/server/src/services/native-runtime/native-run-finalizer.ts@@ -126,7 +126,11 @@ async function acceptedInteractionFromRun(input: {.then((rows) => rows[0] ?? null);}-async function claimCoordinator(input: { db: Db; runId: string }) {+async function claimCoordinator(input: {+ db: Db;+ runId: string;+ preserveProviderAttempt?: boolean;+}) {const leaseOwner = `native-finalizer:${randomUUID()}`;const now = new Date();const claimed = await input.db.transaction(async (tx) => {@@ -152,23 +156,33 @@ async function claimCoordinator(input: { db: Db; runId: string }) {}if (coordinator.phase === "terminal_failure") throw new Error("native_finalization_terminal_failure");if (- coordinator.leaseOwner- && coordinator.leaseExpiresAt- && coordinator.leaseExpiresAt > now- && coordinator.leaseOwner !== leaseOwner- ) throw new Error("native_finalization_lease_busy");- const [updated] = await tx.update(nativeRunFinalizations).set({- leaseOwner,- leaseExpiresAt: new Date(now.getTime() + 5 * 60_000),- attempt: coordinator.attempt + 1,- phase: coordinator.phase === "retryable_failure"- ? coordinator.assessmentId ? "arbitrating" : "workspace_finalizing"- : coordinator.phase,- failureCode: null,- failureDetail: null,- nextAttemptAt: null,- updatedAt: now,- }).where(eq(nativeRunFinalizations.runId, input.runId)).returning();+ coordinator.leaseOwner &&+ coordinator.leaseExpiresAt &&+ coordinator.leaseExpiresAt > now &&+ coordinator.leaseOwner !== leaseOwner+ )+ throw new Error("native_finalization_lease_busy");+ const [updated] = await tx+ .update(nativeRunFinalizations)+ .set({+ leaseOwner,+ leaseExpiresAt: new Date(now.getTime() + 5 * 60_000),+ attempt: input.preserveProviderAttempt+ ? coordinator.attempt+ : coordinator.attempt + 1,+ phase:+ coordinator.phase === "retryable_failure"+ ? coordinator.assessmentId+ ? "arbitrating"+ : "workspace_finalizing"+ : coordinator.phase,+ failureCode: null,+ failureDetail: null,+ nextAttemptAt: null,+ updatedAt: now,+ })+ .where(eq(nativeRunFinalizations.runId, input.runId))+ .returning();if (!updated) throw new Error("native_finalization_claim_failed");return { coordinator: updated, leaseOwner };});@@ -183,6 +197,8 @@ async function recordRetryableFailure(input: {message: string;nextAction: string;projectRunStatus?: boolean;+ failureScope?: "provider" | "workspace";+ permanent?: boolean;}) {const now = new Date();const nextAttemptAt = new Date(now.getTime() + 30_000);@@ -228,50 +244,87 @@ async function recordRetryableFailure(input: {const supersededByNewerRun = Boolean(latestDecisionRun && latestDecisionRun.createdAt > input.run.createdAt,);- const exhausted = input.coordinator.attempt >= 3;- const phase = supersededByNewerRun || exhausted- ? "terminal_failure" as const- : "retryable_failure" as const;+ const priorFailureDetail = record(input.coordinator.failureDetail);+ const workspaceFinalizeAttempt =+ input.failureScope === "workspace"+ ? (typeof priorFailureDetail.workspaceFinalizeAttempt === "number" &&+ Number.isInteger(priorFailureDetail.workspaceFinalizeAttempt) &&+ priorFailureDetail.workspaceFinalizeAttempt >= 0+ ? priorFailureDetail.workspaceFinalizeAttempt+ : 0) + 1+ : null;+ const exhausted =+ input.permanent === true ||+ (workspaceFinalizeAttempt !== null+ ? workspaceFinalizeAttempt >= 3+ : input.coordinator.attempt >= 3);+ const phase =+ supersededByNewerRun || exhausted+ ? ("terminal_failure" as const)+ : ("retryable_failure" as const);const failureCode = supersededByNewerRun? "native_finalization_superseded"- : exhausted- ? "native_finalization_retry_exhausted"- : input.failureCode;- await tx.update(nativeRunFinalizations).set({- phase,- leaseOwner: null,- leaseExpiresAt: null,- failureCode,- failureDetail: {- message: input.message.slice(0, 2_000),- originalFailureCode: input.failureCode,- recoveryOwner: supersededByNewerRun- ? { kind: "none", reason: "newer_native_decision" }- : exhausted- ? { kind: "board" }- : { kind: "agent", agentId: input.run.agentId },- nextAction: input.nextAction,- },- nextAttemptAt: supersededByNewerRun || exhausted ? null : nextAttemptAt,- updatedAt: now,- }).where(eq(nativeRunFinalizations.runId, input.run.id));- const projectsTerminalStatus = exhausted && !supersededByNewerRun && input.projectRunStatus;- const [updatedRun] = await tx.update(heartbeatRuns).set({- ...(projectsTerminalStatus ? {- status: exhaustedRunStatus,- finishedAt: input.run.finishedAt ?? now,- } : {}),- nativePhase: phase,- nativePhaseUpdatedAt: now,- resultJson: {- ...record(input.run.resultJson),- finalizationPhase: phase,+ : input.permanent+ ? input.failureCode+ : exhausted+ ? input.failureScope === "workspace"+ ? "native_workspace_sync_out_retry_exhausted"+ : "native_finalization_retry_exhausted"+ : input.failureCode;+ await tx+ .update(nativeRunFinalizations)+ .set({+ phase,+ leaseOwner: null,+ leaseExpiresAt: null,failureCode,- originalFailureCode: input.failureCode,- nextAttemptAt: supersededByNewerRun || exhausted ? null : nextAttemptAt.toISOString(),- },- updatedAt: now,- }).where(eq(heartbeatRuns.id, input.run.id)).returning();+ failureDetail: {+ message: input.message.slice(0, 2_000),+ originalFailureCode: input.failureCode,+ ...(workspaceFinalizeAttempt === null+ ? {}+ : { workspaceFinalizeAttempt }),+ recoveryOwner: supersededByNewerRun+ ? { kind: "none", reason: "newer_native_decision" }+ : exhausted+ ? { kind: "board" }+ : { kind: "agent", agentId: input.run.agentId },+ nextAction: input.nextAction,+ },+ nextAttemptAt: supersededByNewerRun || exhausted ? null : nextAttemptAt,+ updatedAt: now,+ })+ .where(eq(nativeRunFinalizations.runId, input.run.id));+ const projectsTerminalStatus =+ exhausted && !supersededByNewerRun && input.projectRunStatus;+ const [updatedRun] = await tx+ .update(heartbeatRuns)+ .set({+ ...(projectsTerminalStatus+ ? {+ status:+ input.failureScope === "workspace"+ ? "failed"+ : exhaustedRunStatus,+ finishedAt: input.run.finishedAt ?? now,+ }+ : {}),+ nativePhase: phase,+ nativePhaseUpdatedAt: now,+ resultJson: {+ ...record(input.run.resultJson),+ finalizationPhase: phase,+ failureCode,+ originalFailureCode: input.failureCode,+ nextAttemptAt:+ supersededByNewerRun || exhausted+ ? null+ : nextAttemptAt.toISOString(),+ },+ updatedAt: now,+ })+ .where(eq(heartbeatRuns.id, input.run.id))+ .returning();if (projectsTerminalStatus) terminalRunToEmit = updatedRun ?? null;if (supersededByNewerRun) {await issueRecoveryActionService(tx as unknown as Db).resolveActiveForIssue({@@ -284,9 +337,16 @@ async function recordRetryableFailure(input: {resolutionNote: "A newer native run already committed the authoritative issue decision; the stale finalizer was retired without changing issue state.",}, tx);} else if (exhausted) {- await issueService(tx as unknown as Db).update(input.coordinator.issueId, {- status: "in_review",- }, tx);+ await issueService(tx as unknown as Db).update(+ input.coordinator.issueId,+ {+ status:+ input.permanent && input.failureScope === "workspace"+ ? "blocked"+ : "in_review",+ },+ tx,+ );}if (!supersededByNewerRun) {await issueRecoveryActionService(tx as unknown as Db).upsertSourceScoped({@@ -301,10 +361,15 @@ async function recordRetryableFailure(input: {evidence: {runId: input.run.id,coordinatorAttempt: input.coordinator.attempt,+ ...(workspaceFinalizeAttempt === null+ ? {}+ : { workspaceFinalizeAttempt }),originalFailureCode: input.failureCode,},nextAction: exhausted- ? `Finalization retry budget exhausted. ${input.nextAction}`+ ? input.permanent+ ? input.nextAction+ : `Finalization retry budget exhausted. ${input.nextAction}`: input.nextAction,wakePolicy: exhausted? null@@ -328,6 +393,8 @@ export async function recordNativeFinalizationFailure(input: {runId: string;error: unknown;projectRunStatus?: boolean;+ failureScope?: "provider" | "workspace";+ permanent?: boolean;}) {const [run, coordinator] = await Promise.all([input.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId))@@ -344,8 +411,15 @@ export async function recordNativeFinalizationFailure(input: {coordinator,failureCode,message,- nextAction: "Repair the persisted native result or contract discriminator, then resume finalization from the coordinator.",+ nextAction:+ input.failureScope === "workspace"+ ? input.permanent+ ? "Restore the exact sandbox containing the unexported workspace changes, or resolve the task manually from durable evidence."+ : "Retry workspace export and merge from the retained sandbox; do not submit another provider turn."+ : "Repair the persisted native result or contract discriminator, then resume finalization from the coordinator.",projectRunStatus: input.projectRunStatus,+ failureScope: input.failureScope,+ permanent: input.permanent,});}@@ -395,12 +469,23 @@ export async function finalizeNativeRun(input: {workspaceFinalizeStatus: "succeeded" | "failed";/** Reconciliation owns terminal run projection; the live heartbeat does it afterward. */projectRunStatus?: boolean;+ /** Workspace-only replay must not consume the provider recovery budget. */+ preserveProviderAttempt?: boolean;failpoint?: NativeStatusCommitFailpoint;}) {- const run = await input.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId))- .limit(1).then((rows) => rows[0] ?? null);- if (!run || run.runtimeMode !== "native") throw new Error("native_finalization_run_missing");- const claim = await claimCoordinator({ db: input.db, runId: input.runId });+ const run = await input.db+ .select()+ .from(heartbeatRuns)+ .where(eq(heartbeatRuns.id, input.runId))+ .limit(1)+ .then((rows) => rows[0] ?? null);+ if (!run || run.runtimeMode !== "native")+ throw new Error("native_finalization_run_missing");+ const claim = await claimCoordinator({+ db: input.db,+ runId: input.runId,+ preserveProviderAttempt: input.preserveProviderAttempt,+ });const coordinator = claim.coordinator;if (!claim.leaseOwner && coordinator.phase === "committed") {if (input.projectRunStatus) await projectCommittedRun({ db: input.db, run, coordinator });diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.tsindex 7a9c857026..e6e70a53d5 100644--- a/server/src/services/native-runtime/native-session-executor.test.ts+++ b/server/src/services/native-runtime/native-session-executor.test.ts@@ -2880,7 +2880,7 @@ describe("native warm session supervision", () => {);});- it("rehydrates a runnerd warm session from its checkpoint under a fresh run authority", async () => {+ it("reattaches a live runnerd warm session under a fresh run authority", async () => {const stateBase = await mkdtemp(join(tmpdir(), "paperclip-runnerd-warm-authority-"),);@@ -2889,9 +2889,7 @@ describe("native warm session supervision", () => {process.env.PAPERCLIP_RUNNER_STATE_DIR = stateBase;process.env.PAPERCLIP_HOME = stateBase;const firstClose = vi.fn(async () => undefined);- const secondClose = vi.fn(async () => undefined);const firstSession = { close: firstClose };- const secondSession = { close: secondClose };const first = {...execution,binding: {@@ -2947,18 +2945,8 @@ describe("native warm session supervision", () => {return result;}).mockImplementationOnce(async (options) => {- expect(options.existingSession).toBeUndefined();- expect(options.persistedSession).toEqual(- expect.objectContaining({- identity: expect.objectContaining({- runId: second.binding.runId,- sessionId: second.session.normalizedSessionId,- }),- providerSessionId: "provider-runnerd-warm",- activeTurnId: null,- }),- );- options.onSession?.(secondSession);+ expect(options.existingSession).toBe(firstSession);+ expect(options.persistedSession).toBeUndefined();return result;});@@ -3014,10 +3002,10 @@ describe("native warm session supervision", () => {runnerInstanceId: "runner-runnerd-warm",useRunnerd: true,});- expect(firstClose).toHaveBeenCalledWith({- reason: "warm native session authority epoch rotated",- });- await vi.waitFor(() => expect(secondClose).toHaveBeenCalled(), {+ expect(firstClose).not.toHaveBeenCalled();+ await vi.waitFor(() => expect(firstClose).toHaveBeenCalledWith({+ reason: "warm native session idle timeout",+ }), {timeout: 500,});} finally {@@ -5392,9 +5380,13 @@ describe("runnerd provider runtime wiring", () => {const firstOptions = state.createBackend.mock.calls[0]![1];const continuationOptions = state.createBackend.mock.calls[1]![1];- await expect(firstOptions.dynamicToolHandler!({})).rejects.toThrow(- "native_tool_authority_epoch_revoked",- );+ // The retained runner backend owns one stable callback. After run.attach,+ // that callback routes through the session-scope authority registry to+ // the new run; stale provider calls are rejected earlier by runnerd's+ // turn identity boundary.+ await expect(firstOptions.dynamicToolHandler!({})).resolves.toEqual({+ runId: continuation.binding.runId,+ });await expect(continuationOptions.dynamicToolHandler!({}),).resolves.toEqual({ runId: continuation.binding.runId });diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.tsindex f7fc0f2a3e..5af22bd551 100644--- a/server/src/services/native-runtime/native-session-executor.ts+++ b/server/src/services/native-runtime/native-session-executor.ts@@ -2623,7 +2623,13 @@ export async function nativeProviderRecoveryEvidence(input: {const durableEvents = await input.db.select({ eventType: heartbeatRunEvents.eventType }).from(heartbeatRunEvents)- .where(eq(heartbeatRunEvents.runId, input.runId));+ .where(+ and(+ eq(heartbeatRunEvents.runId, input.runId),+ inArray(heartbeatRunEvents.eventType, [...PROVIDER_DURABLE_EVENT_TYPES]),+ ),+ )+ .limit(1);const providerEventsExist = durableEvents.some((event) =>PROVIDER_DURABLE_EVENT_TYPES.has(event.eventType),);@@ -4171,23 +4177,7 @@ async function executePaperclipNativeSessionWithinScope(entry.busy = true;if (entry.idleTimer !== null) clearTimeout(entry.idleTimer);entry.idleTimer = null;- if (input.useRunnerd) {- // A retained driver closes over the tool authority from the run that- // created it. Preserve provider continuity through its durable- // checkpoint, but rebuild the runnerd backend so the next run gets a- // fresh, immutable authority epoch. Reusing the live driver would- // either retain stale authority or require rebinding old callbacks.- warmNativeSessions.delete(warmSessionId);- await entry.session.close({- reason: "warm native session authority epoch rotated",- });- persistedWarmSession = loadWarmNativeCheckpoint(- input.execution,- warmConfigDigest,- );- } else {- existingWarmSession = entry.session;- }+ existingWarmSession = entry.session;}} else {persistedWarmSession = loadWarmNativeCheckpoint(@@ -7235,6 +7225,13 @@ async function createRunnerdBackendWithinSessionClaim(input.restartRecovery?.kind === "reattach_existing_runner"? input.restartRecovery.process: null;+ const executeCurrentToolAuthority = (+ call: Parameters<SessionToolAuthorityEpoch["execute"]>[0],+ ) => {+ const current = sessionToolAuthorityEpochs.get(sessionScopeId);+ if (!current) throw new Error("native_session_tool_authority_unavailable");+ return current.execute(call);+ };const backend = createNativeSessionBackend(runnerExecution, {runnerInstanceId: input.runnerInstanceId,environment: effectiveRunnerEnvironment,@@ -7243,8 +7240,8 @@ async function createRunnerdBackendWithinSessionClaim(: "local_filesystem",onSpawn: input.onSpawn,dynamicTools,- dynamicToolHandler: (call) => authorityEpoch.execute(call),- acpxDynamicToolHandler: (call) => authorityEpoch.execute(call),+ dynamicToolHandler: executeCurrentToolAuthority,+ acpxDynamicToolHandler: executeCurrentToolAuthority,opencodeRuntimeDirectory: resolve(resolvePaperclipInstanceRoot(),"runtime",@@ -7419,7 +7416,7 @@ async function createRunnerdBackendWithinSessionClaim(turnId: `turn-${input.execution.binding.runId}`,itemId: `item-${input.execution.binding.runId}`,},- controlPlaneRegistration: (authority) =>+ controlPlaneRegistration: (authority, attachmentIdentity) =>measureNativeRunnerSpan(input.trace,"runner.transport.connect",@@ -7446,7 +7443,9 @@ async function createRunnerdBackendWithinSessionClaim(() =>registerRunnerPrpAuthority({companyId: input.execution.binding.companyId,- runId: input.execution.binding.runId,+ runId:+ attachmentIdentity?.runId ??+ input.execution.binding.runId,authority,}),);@@ -7469,7 +7468,9 @@ async function createRunnerdBackendWithinSessionClaim(() =>resolvePaperclipRunnerTransport({target,- runId: input.execution.binding.runId,+ runId:+ attachmentIdentity?.runId ??+ input.execution.binding.runId,localConnectUrl: "ws://127.0.0.1/unused",runnerPublicUrl: input.runnerPublicUrl,runnerCaBundlePath: input.runnerCaBundlePath,@@ -7505,7 +7506,9 @@ async function createRunnerdBackendWithinSessionClaim(() =>resolvePaperclipRunnerTransport({target,- runId: input.execution.binding.runId,+ runId:+ attachmentIdentity?.runId ??+ input.execution.binding.runId,localConnectUrl: "ws://127.0.0.1/unused",runnerPublicUrl: input.runnerPublicUrl,runnerCaBundlePath: input.runnerCaBundlePath,@@ -7541,7 +7544,9 @@ async function createRunnerdBackendWithinSessionClaim(() =>registerRunnerPrpAuthority({companyId: input.execution.binding.companyId,- runId: input.execution.binding.runId,+ runId:+ attachmentIdentity?.runId ??+ input.execution.binding.runId,authority,}),{ parentName: "runner.transport.connect" },diff --git a/server/src/services/native-runtime/native-workspace-finalizer.ts b/server/src/services/native-runtime/native-workspace-finalizer.tsindex 49e87e7795..edec11741c 100644--- a/server/src/services/native-runtime/native-workspace-finalizer.ts+++ b/server/src/services/native-runtime/native-workspace-finalizer.ts@@ -10,6 +10,13 @@ import {} from "@paperclipai/db";import { workspaceOperationService } from "../workspace-operations.js";import { inspectManagedGitWorktreeBranch } from "../workspace-runtime.js";+import { environmentService } from "../environments.js";+import type { EnvironmentRuntimeService } from "../environment-runtime.js";+import { resolveEnvironmentExecutionTarget } from "../environment-execution-target.js";+import {+ readNativeWorkspaceSyncReference,+ resumeNativeWorkspaceSync,+} from "./native-workspace-sync.js";function record(value: unknown): Record<string, unknown> {return value && typeof value === "object" && !Array.isArray(value)@@ -21,14 +28,27 @@ function readString(value: unknown) {return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;}+function workspaceSyncFailure(+ code: "workspace_sync_out_failed" | "workspace_sync_out_unrecoverable",+) {+ return {+ status: "failed" as const,+ exitCode: 1,+ stderr: `${code}\n`,+ metadata: { workspaceSync: { code } },+ };+}+/*** Resume the real workspace-finalization action for a result-bearing native- * run. This service observes the workspace; it never fabricates a successful- * marker. The caller decides whether a failed observation is retryable.+ * run. Durable sandbox-backed runs export and merge the remote workspace;+ * older runs retain the local workspace validation path. The caller owns the+ * durable retry and failure policy.*/export async function resumeNativeWorkspaceFinalization(input: {db: Db;runId: string;+ environmentRuntime?: EnvironmentRuntimeService;}) {const bound = await input.db.select({companyId: heartbeatRuns.companyId,@@ -57,7 +77,12 @@ export async function resumeNativeWorkspaceFinalization(input: {eq(workspaceOperations.phase, "workspace_finalize"),)).orderBy(desc(workspaceOperations.createdAt)).limit(1).then((rows) => rows[0] ?? null);- const persistedInput = record(record(bound.runnerProfileJson).nativeExecutionInput);+ const persistedInput = record(+ record(bound.runnerProfileJson).nativeExecutionInput,+ );+ const nativeWorkspaceSync = readNativeWorkspaceSyncReference(+ record(bound.runnerProfileJson).nativeWorkspaceSync,+ );const binding = record(persistedInput.binding);const workspaceId = previous?.executionWorkspaceId?? bound.issueWorkspaceId@@ -87,6 +112,88 @@ export async function resumeNativeWorkspaceFinalization(input: {: "workspace_directory",},run: async () => {+ if (nativeWorkspaceSync) {+ if (!input.environmentRuntime) {+ return {+ status: "failed",+ exitCode: 1,+ stderr:+ "Native workspace finalization cannot access the environment runtime.\n",+ };+ }+ const environmentsSvc = environmentService(input.db);+ const lease = await environmentsSvc.getLeaseById(+ nativeWorkspaceSync.leaseId,+ );+ const environment = lease?.environmentId+ ? await environmentsSvc.getById(lease.environmentId)+ : null;+ if (+ !lease ||+ !environment ||+ lease.companyId !== bound.companyId ||+ environment.id !== lease.environmentId+ ) {+ return workspaceSyncFailure("workspace_sync_out_unrecoverable");+ }+ if (+ lease.status === "expired" ||+ lease.status === "failed" ||+ lease.status === "pending_cleanup" ||+ !lease.providerLeaseId ||+ lease.providerLeaseId !== nativeWorkspaceSync.providerLeaseId+ ) {+ return workspaceSyncFailure("workspace_sync_out_unrecoverable");+ }+ const target = await resolveEnvironmentExecutionTarget({+ db: input.db,+ companyId: bound.companyId,+ adapterType: "paperclip_runner",+ environment,+ leaseId: lease.id,+ leaseMetadata: lease.metadata,+ lease,+ environmentRuntime: input.environmentRuntime,+ });+ if (!target) {+ return {+ status: "failed",+ exitCode: 1,+ stderr: "Native workspace finalization target is unavailable.\n",+ };+ }+ try {+ const restored = await resumeNativeWorkspaceSync({+ db: input.db,+ runId: input.runId,+ target,+ });+ if (!restored) {+ return workspaceSyncFailure("workspace_sync_out_unrecoverable");+ }+ return {+ status: "succeeded",+ exitCode: 0,+ system:+ "Native workspace finalization restored the remote workspace.\n",+ metadata: {+ workspaceSync: {+ schema: nativeWorkspaceSync.schema,+ workspaceId: nativeWorkspaceSync.workspaceId,+ leaseId: nativeWorkspaceSync.leaseId,+ },+ },+ };+ } catch (error) {+ const code =+ error instanceof Error &&+ (error.message === "workspace_sync_out_unrecoverable" ||+ error.message.includes("daytona_sandbox_not_found"))+ ? "workspace_sync_out_unrecoverable"+ : "workspace_sync_out_failed";+ return workspaceSyncFailure(code);+ }+ }if (!cwd) {return {status: "failed",diff --git a/server/src/services/native-runtime/native-workspace-sync.ts b/server/src/services/native-runtime/native-workspace-sync.tsnew file mode 100644index 0000000000..e8f9f95884--- /dev/null+++ b/server/src/services/native-runtime/native-workspace-sync.ts@@ -0,0 +1,1004 @@+import fs from "node:fs/promises";+import { createReadStream } from "node:fs";+import path from "node:path";+import { createHash, randomUUID } from "node:crypto";+import { eq } from "drizzle-orm";+import type { Db } from "@paperclipai/db";+import { environmentLeases, heartbeatRuns } from "@paperclipai/db";+import type { EnvironmentLease } from "@paperclipai/shared";+import {+ prepareAdapterExecutionTargetRuntime,+ type AdapterExecutionTarget,+ type PreparedAdapterExecutionTargetRuntime,+} from "@paperclipai/adapter-utils/execution-target";+import type { GitWorkspaceSnapshot } from "@paperclipai/adapter-utils/git-workspace-sync";+import {+ directorySnapshotSha256,+ parseDirectorySnapshot,+ serializeDirectorySnapshot,+ type DirectorySnapshot,+ type SerializedDirectorySnapshot,+} from "@paperclipai/adapter-utils/workspace-restore-merge";+import type {+ WorkspaceDurableSeedPaths,+ WorkspaceInboundMode,+} from "@paperclipai/adapter-utils/sandbox-managed-runtime";+import { resolvePaperclipInstanceRoot } from "../../home-paths.js";+import { parseObject } from "../../adapters/utils.js";+import type { NativeRestartRecoveryClaim } from "./native-restart-recovery.js";++const DESCRIPTOR_SCHEMA = "paperclip.native-workspace-sync/v1";+const STAMP_SCHEMA = "paperclip.native-workspace-stamp/v1";+const STATE_ROOT_NAME = "native-workspace-sync";+const DESCRIPTOR_NAME = "descriptor";+const WORKSPACE_SEED_NAME = "workspace-seed.tar";+const GIT_SEED_NAME = "git-seed.tar";+const REMOTE_STAMP_NAME = "workspace-sync-v1.json";+const MAX_DESCRIPTOR_BYTES = 64 * 1024 * 1024;+const SAFE_SEGMENT_RE = /^[A-Za-z0-9_-]+$/;++type NativeWorkspaceSyncState = "prepared" | "finalized";+export type NativeWorkspaceResourceDisposition =+ "keep_running" | "stop_and_retain" | "destroy";++interface NativeWorkspaceSyncDescriptor {+ schema: typeof DESCRIPTOR_SCHEMA;+ binding: {+ runId: string;+ companyId: string;+ workspaceId: string;+ leaseId: string;+ providerLeaseId: string;+ localCwd: string;+ remoteCwd: string;+ };+ state: NativeWorkspaceSyncState;+ baselineSha256: string;+ baseline: SerializedDirectorySnapshot;+ gitSnapshot: GitWorkspaceSnapshot | null;+ seed: {+ workspaceArchiveSha256: string;+ gitArchiveSha256: string | null;+ } | null;+ createdAt: string;+ finalizedAt: string | null;+ finalHostSha256: string | null;+ resourceDisposition: NativeWorkspaceResourceDisposition | null;+}++export interface NativeWorkspaceSyncReference {+ schema: typeof DESCRIPTOR_SCHEMA;+ state: NativeWorkspaceSyncState;+ descriptorSha256: string;+ baselineSha256: string;+ finalHostSha256: string | null;+ workspaceId: string;+ leaseId: string;+ providerLeaseId: string;+ remoteCwd: string;+ resourceDisposition: NativeWorkspaceResourceDisposition | null;+}++export interface PreparedNativeWorkspaceSync {+ mode: WorkspaceInboundMode;+ reference: NativeWorkspaceSyncReference;+ restoreWorkspace(): Promise<void>;+ cleanup(): Promise<void>;+}++export type NativeWorkspaceInboundEvidence =+ | {+ kind: "existing_run";+ restartRecovery: boolean;+ sameProviderLease: boolean;+ }+ | {+ kind: "new_run";+ acquisition: "created" | "resumed" | "replacement" | null;+ hasPriorStamp: boolean;+ };++export function classifyNativeWorkspaceInbound(+ evidence: NativeWorkspaceInboundEvidence,+): WorkspaceInboundMode {+ if (evidence.kind === "existing_run") {+ if (!evidence.restartRecovery) {+ throw new Error("native_workspace_sync_unexpected_existing_descriptor");+ }+ return evidence.sameProviderLease ? "adopt_remote" : "durable_seed";+ }+ return evidence.acquisition === "resumed" && evidence.hasPriorStamp+ ? "adopt_remote"+ : "host_current";+}++function canonicalJson(value: unknown): string {+ if (Array.isArray(value)) {+ return `[${value.map((entry) => canonicalJson(entry)).join(",")}]`;+ }+ if (value && typeof value === "object") {+ const record = value as Record<string, unknown>;+ return `{${Object.keys(record)+ .sort()+ .map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`)+ .join(",")}}`;+ }+ return JSON.stringify(value);+}++function sha256(value: unknown): string {+ return createHash("sha256").update(canonicalJson(value)).digest("hex");+}++function requireSafeSegment(value: string, label: string): string {+ if (!SAFE_SEGMENT_RE.test(value)) {+ throw new Error(`native_workspace_sync_invalid_${label}`);+ }+ return value;+}++function descriptorDirectory(runId: string): string {+ return path.join(+ resolvePaperclipInstanceRoot(),+ STATE_ROOT_NAME,+ requireSafeSegment(runId, "run_id"),+ );+}++function descriptorPath(runId: string, descriptorSha256: string): string {+ if (!/^[0-9a-f]{64}$/.test(descriptorSha256)) {+ throw new Error("native_workspace_sync_descriptor_digest_invalid");+ }+ return path.join(+ descriptorDirectory(runId),+ `${DESCRIPTOR_NAME}.${descriptorSha256}.json`,+ );+}++function legacyDescriptorPath(runId: string): string {+ return path.join(descriptorDirectory(runId), `${DESCRIPTOR_NAME}.json`);+}++function durableSeedPaths(runId: string): {+ workspaceArchivePath: string;+ gitArchivePath: string;+} {+ const directory = descriptorDirectory(runId);+ return {+ workspaceArchivePath: path.join(directory, WORKSPACE_SEED_NAME),+ gitArchivePath: path.join(directory, GIT_SEED_NAME),+ };+}++async function sha256File(filePath: string): Promise<string> {+ return await new Promise((resolveDigest, rejectDigest) => {+ const digest = createHash("sha256");+ const stream = createReadStream(filePath);+ stream.on("data", (chunk) => digest.update(chunk));+ stream.on("error", rejectDigest);+ stream.on("end", () => resolveDigest(digest.digest("hex")));+ });+}++async function verifiedDurableSeed(input: {+ runId: string;+ seed: NonNullable<NativeWorkspaceSyncDescriptor["seed"]>;+ gitSnapshot: GitWorkspaceSnapshot | null;+}): Promise<WorkspaceDurableSeedPaths> {+ try {+ const paths = durableSeedPaths(input.runId);+ const workspaceStat = await fs.lstat(paths.workspaceArchivePath);+ if (workspaceStat.isSymbolicLink() || !workspaceStat.isFile()) {+ throw new Error("workspace_sync_out_unrecoverable");+ }+ if (+ (await sha256File(paths.workspaceArchivePath)) !==+ input.seed.workspaceArchiveSha256+ ) {+ throw new Error("workspace_sync_out_unrecoverable");+ }+ if (input.gitSnapshot) {+ const gitStat = await fs.lstat(paths.gitArchivePath);+ if (+ gitStat.isSymbolicLink() ||+ !gitStat.isFile() ||+ !input.seed.gitArchiveSha256 ||+ (await sha256File(paths.gitArchivePath)) !== input.seed.gitArchiveSha256+ ) {+ throw new Error("workspace_sync_out_unrecoverable");+ }+ } else if (input.seed.gitArchiveSha256 !== null) {+ throw new Error("workspace_sync_out_unrecoverable");+ }+ return {+ workspaceArchivePath: paths.workspaceArchivePath,+ workspaceArchiveSha256: input.seed.workspaceArchiveSha256,+ gitArchivePath: input.gitSnapshot ? paths.gitArchivePath : null,+ gitArchiveSha256: input.seed.gitArchiveSha256,+ };+ } catch (error) {+ if (+ error instanceof Error &&+ error.message === "workspace_sync_out_unrecoverable"+ ) {+ throw error;+ }+ throw new Error("workspace_sync_out_unrecoverable", { cause: error });+ }+}++async function ensurePrivateDirectory(dir: string): Promise<void> {+ await fs.mkdir(dir, { recursive: true, mode: 0o700 });+ const stat = await fs.lstat(dir);+ if (stat.isSymbolicLink() || !stat.isDirectory()) {+ throw new Error("native_workspace_sync_state_root_invalid");+ }+}++async function writeDescriptor(+ descriptor: NativeWorkspaceSyncDescriptor,+): Promise<NativeWorkspaceSyncReference> {+ const dir = descriptorDirectory(descriptor.binding.runId);+ await ensurePrivateDirectory(dir);+ const body = `${canonicalJson(descriptor)}\n`;+ const descriptorSha256 = createHash("sha256").update(body).digest("hex");+ const finalPath = descriptorPath(descriptor.binding.runId, descriptorSha256);+ const existingBody = await fs.readFile(finalPath, "utf8").catch(() => null);+ if (existingBody !== null && existingBody !== body) {+ throw new Error("native_workspace_sync_descriptor_digest_collision");+ }+ const tempPath = path.join(dir, `${DESCRIPTOR_NAME}.${randomUUID()}.tmp`);+ try {+ if (existingBody === null) {+ await fs.writeFile(tempPath, body, { encoding: "utf8", mode: 0o600 });+ await fs.rename(tempPath, finalPath);+ }+ } finally {+ await fs.rm(tempPath, { force: true }).catch(() => undefined);+ }+ return {+ schema: DESCRIPTOR_SCHEMA,+ state: descriptor.state,+ descriptorSha256,+ baselineSha256: descriptor.baselineSha256,+ finalHostSha256: descriptor.finalHostSha256,+ workspaceId: descriptor.binding.workspaceId,+ leaseId: descriptor.binding.leaseId,+ providerLeaseId: descriptor.binding.providerLeaseId,+ remoteCwd: descriptor.binding.remoteCwd,+ resourceDisposition: descriptor.resourceDisposition,+ };+}++export function readNativeWorkspaceSyncReference(+ value: unknown,+): NativeWorkspaceSyncReference | null {+ const candidate = parseObject(value);+ if (+ candidate.schema !== DESCRIPTOR_SCHEMA ||+ (candidate.state !== "prepared" && candidate.state !== "finalized") ||+ typeof candidate.descriptorSha256 !== "string" ||+ !/^[0-9a-f]{64}$/.test(candidate.descriptorSha256) ||+ typeof candidate.baselineSha256 !== "string" ||+ !/^[0-9a-f]{64}$/.test(candidate.baselineSha256) ||+ (candidate.finalHostSha256 !== null &&+ candidate.finalHostSha256 !== undefined &&+ (typeof candidate.finalHostSha256 !== "string" ||+ !/^[0-9a-f]{64}$/.test(candidate.finalHostSha256))) ||+ typeof candidate.workspaceId !== "string" ||+ typeof candidate.leaseId !== "string" ||+ typeof candidate.providerLeaseId !== "string" ||+ typeof candidate.remoteCwd !== "string" ||+ (candidate.resourceDisposition !== null &&+ candidate.resourceDisposition !== undefined &&+ candidate.resourceDisposition !== "keep_running" &&+ candidate.resourceDisposition !== "stop_and_retain" &&+ candidate.resourceDisposition !== "destroy")+ ) {+ return null;+ }+ return {+ schema: DESCRIPTOR_SCHEMA,+ state: candidate.state,+ descriptorSha256: candidate.descriptorSha256,+ baselineSha256: candidate.baselineSha256,+ finalHostSha256:+ typeof candidate.finalHostSha256 === "string"+ ? candidate.finalHostSha256+ : null,+ workspaceId: candidate.workspaceId,+ leaseId: candidate.leaseId,+ providerLeaseId: candidate.providerLeaseId,+ remoteCwd: candidate.remoteCwd,+ resourceDisposition:+ candidate.resourceDisposition === "keep_running" ||+ candidate.resourceDisposition === "stop_and_retain" ||+ candidate.resourceDisposition === "destroy"+ ? candidate.resourceDisposition+ : null,+ };+}++function parseGitSnapshot(+ value: unknown,+): GitWorkspaceSnapshot | null | undefined {+ if (value === null) return null;+ const candidate = parseObject(value);+ const paths = [+ candidate.overlayPaths,+ candidate.deletedPaths,+ candidate.ignoredPaths,+ ];+ if (+ typeof candidate.headCommit !== "string" ||+ (candidate.branchName !== null &&+ typeof candidate.branchName !== "string") ||+ !paths.every(+ (entries) =>+ Array.isArray(entries) &&+ entries.every(+ (entry) =>+ typeof entry === "string" &&+ !path.posix.isAbsolute(entry) &&+ !path.win32.isAbsolute(entry) &&+ !entry.split(/[\\/]/).some((segment) => segment === ".."),+ ),+ )+ ) {+ return undefined;+ }+ return {+ headCommit: candidate.headCommit,+ branchName: candidate.branchName as string | null,+ overlayPaths: [...(candidate.overlayPaths as string[])],+ deletedPaths: [...(candidate.deletedPaths as string[])],+ ignoredPaths: [...(candidate.ignoredPaths as string[])],+ };+}++async function readDescriptor(input: {+ runId: string;+ reference: NativeWorkspaceSyncReference;+}): Promise<{+ descriptor: NativeWorkspaceSyncDescriptor;+ baseline: DirectorySnapshot;+}> {+ const contentAddressedPath = descriptorPath(+ input.runId,+ input.reference.descriptorSha256,+ );+ // Early v1 writers used descriptor.json. Prefer the immutable,+ // content-addressed name, but keep the digest-verified legacy path readable+ // so an interrupted upgrade can still finish its already-proposed result.+ const filePath = await fs+ .lstat(contentAddressedPath)+ .then(() => contentAddressedPath)+ .catch(async (error: NodeJS.ErrnoException) => {+ if (error.code !== "ENOENT") throw error;+ const legacyPath = legacyDescriptorPath(input.runId);+ await fs.lstat(legacyPath);+ return legacyPath;+ });+ const stat = await fs.lstat(filePath);+ if (+ stat.isSymbolicLink() ||+ !stat.isFile() ||+ stat.size > MAX_DESCRIPTOR_BYTES+ ) {+ throw new Error("native_workspace_sync_descriptor_invalid");+ }+ const body = await fs.readFile(filePath, "utf8");+ if (+ createHash("sha256").update(body).digest("hex") !==+ input.reference.descriptorSha256+ ) {+ throw new Error("native_workspace_sync_descriptor_digest_mismatch");+ }+ const parsed = JSON.parse(body) as unknown;+ const candidate = parseObject(parsed);+ const binding = parseObject(candidate.binding);+ const baseline = parseDirectorySnapshot(candidate.baseline);+ const gitSnapshot = parseGitSnapshot(candidate.gitSnapshot);+ const rawSeed =+ candidate.seed === null || candidate.seed === undefined+ ? null+ : parseObject(candidate.seed);+ const seed =+ rawSeed === null+ ? null+ : typeof rawSeed.workspaceArchiveSha256 === "string" &&+ /^[0-9a-f]{64}$/.test(rawSeed.workspaceArchiveSha256) &&+ (rawSeed.gitArchiveSha256 === null ||+ (typeof rawSeed.gitArchiveSha256 === "string" &&+ /^[0-9a-f]{64}$/.test(rawSeed.gitArchiveSha256)))+ ? {+ workspaceArchiveSha256: rawSeed.workspaceArchiveSha256,+ gitArchiveSha256: rawSeed.gitArchiveSha256 as string | null,+ }+ : undefined;+ const finalHostSha256 =+ typeof candidate.finalHostSha256 === "string" &&+ /^[0-9a-f]{64}$/.test(candidate.finalHostSha256)+ ? candidate.finalHostSha256+ : candidate.finalHostSha256 === null+ ? null+ : undefined;+ const resourceDisposition =+ candidate.resourceDisposition === "keep_running" ||+ candidate.resourceDisposition === "stop_and_retain" ||+ candidate.resourceDisposition === "destroy"+ ? candidate.resourceDisposition+ : candidate.resourceDisposition === null+ ? null+ : undefined;+ if (+ candidate.schema !== DESCRIPTOR_SCHEMA ||+ (candidate.state !== "prepared" && candidate.state !== "finalized") ||+ !baseline ||+ gitSnapshot === undefined ||+ seed === undefined ||+ binding.runId !== input.runId ||+ binding.workspaceId !== input.reference.workspaceId ||+ binding.leaseId !== input.reference.leaseId ||+ binding.providerLeaseId !== input.reference.providerLeaseId ||+ binding.remoteCwd !== input.reference.remoteCwd ||+ typeof binding.companyId !== "string" ||+ typeof binding.localCwd !== "string" ||+ !path.isAbsolute(binding.localCwd) ||+ typeof binding.remoteCwd !== "string" ||+ !path.posix.isAbsolute(binding.remoteCwd) ||+ typeof candidate.baselineSha256 !== "string" ||+ candidate.baselineSha256 !== input.reference.baselineSha256 ||+ directorySnapshotSha256(baseline) !== candidate.baselineSha256 ||+ typeof candidate.createdAt !== "string" ||+ !Number.isFinite(Date.parse(candidate.createdAt)) ||+ candidate.state !== input.reference.state ||+ finalHostSha256 === undefined ||+ finalHostSha256 !== input.reference.finalHostSha256 ||+ resourceDisposition === undefined ||+ resourceDisposition !== input.reference.resourceDisposition ||+ (candidate.state === "prepared" &&+ (candidate.finalizedAt !== null || finalHostSha256 !== null)) ||+ (candidate.state === "finalized" &&+ (typeof candidate.finalizedAt !== "string" ||+ !Number.isFinite(Date.parse(candidate.finalizedAt)) ||+ finalHostSha256 === null))+ ) {+ throw new Error("native_workspace_sync_descriptor_binding_mismatch");+ }+ return {+ descriptor: {+ schema: DESCRIPTOR_SCHEMA,+ binding: {+ runId: binding.runId as string,+ companyId: binding.companyId,+ workspaceId: binding.workspaceId as string,+ leaseId: binding.leaseId as string,+ providerLeaseId: binding.providerLeaseId as string,+ localCwd: binding.localCwd,+ remoteCwd: binding.remoteCwd as string,+ },+ state: candidate.state,+ baselineSha256: candidate.baselineSha256,+ baseline: candidate.baseline as SerializedDirectorySnapshot,+ gitSnapshot,+ seed,+ createdAt: candidate.createdAt,+ finalizedAt:+ typeof candidate.finalizedAt === "string"+ ? candidate.finalizedAt+ : null,+ finalHostSha256,+ resourceDisposition,+ },+ baseline,+ };+}++async function persistRunReference(+ db: Db,+ runId: string,+ reference: NativeWorkspaceSyncReference,+): Promise<void> {+ await db.transaction(async (tx) => {+ const run = await tx+ .select({ runnerProfileJson: heartbeatRuns.runnerProfileJson })+ .from(heartbeatRuns)+ .where(eq(heartbeatRuns.id, runId))+ .for("update")+ .limit(1)+ .then((rows) => rows[0] ?? null);+ if (!run) throw new Error("native_workspace_sync_run_missing");+ await tx+ .update(heartbeatRuns)+ .set({+ runnerProfileJson: {+ ...parseObject(run.runnerProfileJson),+ nativeWorkspaceSync: reference,+ },+ updatedAt: new Date(),+ })+ .where(eq(heartbeatRuns.id, runId));+ });+}++async function persistLeaseStamp(input: {+ db: Db;+ leaseId: string;+ stamp: Record<string, unknown>;+}): Promise<void> {+ await input.db.transaction(async (tx) => {+ const lease = await tx+ .select({ metadata: environmentLeases.metadata })+ .from(environmentLeases)+ .where(eq(environmentLeases.id, input.leaseId))+ .for("update")+ .limit(1)+ .then((rows) => rows[0] ?? null);+ if (!lease) throw new Error("native_workspace_sync_lease_missing");+ await tx+ .update(environmentLeases)+ .set({+ metadata: {+ ...parseObject(lease.metadata),+ nativeWorkspaceSync: input.stamp,+ },+ updatedAt: new Date(),+ })+ .where(eq(environmentLeases.id, input.leaseId));+ });+}++function providerLeaseIdFor(input: {+ target: Extract<AdapterExecutionTarget, { transport: "sandbox" }>;+ lease: EnvironmentLease;+}): string {+ const providerLeaseId =+ input.target.sandboxLeaseAcquisition?.providerLeaseId ??+ input.lease.providerLeaseId;+ if (!providerLeaseId) {+ throw new Error("native_workspace_sync_provider_lease_missing");+ }+ return providerLeaseId;+}++function shellQuote(value: string): string {+ return `'${value.replaceAll("'", "'\\''")}'`;+}++async function writeRemoteStamp(input: {+ target: Extract<AdapterExecutionTarget, { transport: "sandbox" }>;+ stamp: Record<string, unknown>;+}): Promise<void> {+ if (!input.target.runner) {+ throw new Error("native_workspace_sync_runner_missing");+ }+ const runtimeDir = path.posix.join(+ input.target.remoteCwd,+ ".paperclip-runtime",+ "paperclip-runner",+ );+ const stampPath = path.posix.join(runtimeDir, REMOTE_STAMP_NAME);+ const tempPath = `${stampPath}.tmp`;+ const encoded = Buffer.from(+ `${canonicalJson(input.stamp)}\n`,+ "utf8",+ ).toString("base64");+ const result = await input.target.runner.execute({+ command: input.target.shellCommand ?? "sh",+ args: [+ "-c",+ `umask 077; mkdir -p ${shellQuote(runtimeDir)} && printf %s ${shellQuote(encoded)} | base64 -d > ${shellQuote(tempPath)} && mv ${shellQuote(tempPath)} ${shellQuote(stampPath)}`,+ ],+ cwd: "/",+ timeoutMs: 15_000,+ bypassSession: true,+ });+ if (result.timedOut || result.exitCode !== 0) {+ throw new Error("native_workspace_sync_remote_stamp_failed");+ }+}++async function remoteStampMatches(input: {+ target: Extract<AdapterExecutionTarget, { transport: "sandbox" }>;+ expected: Record<string, unknown>;+}): Promise<boolean> {+ if (!input.target.runner) return false;+ const stampPath = path.posix.join(+ input.target.remoteCwd,+ ".paperclip-runtime",+ "paperclip-runner",+ REMOTE_STAMP_NAME,+ );+ const result = await input.target.runner.execute({+ command: input.target.shellCommand ?? "sh",+ args: [+ "-c",+ `test -f ${shellQuote(stampPath)} && cat ${shellQuote(stampPath)}`,+ ],+ cwd: "/",+ timeoutMs: 15_000,+ bypassSession: true,+ });+ if (result.timedOut || result.exitCode !== 0) return false;+ try {+ return (+ canonicalJson(JSON.parse(result.stdout)) === canonicalJson(input.expected)+ );+ } catch {+ return false;+ }+}++function leaseStamp(input: {+ lease: EnvironmentLease;+ workspaceId: string;+ providerLeaseId: string;+ remoteCwd: string;+}): Record<string, unknown> | null {+ const stamp = parseObject(+ parseObject(input.lease.metadata).nativeWorkspaceSync,+ );+ if (+ stamp.schema !== STAMP_SCHEMA ||+ stamp.workspaceId !== input.workspaceId ||+ stamp.providerLeaseId !== input.providerLeaseId ||+ stamp.remoteCwd !== input.remoteCwd ||+ typeof stamp.hostSha256 !== "string" ||+ !/^[0-9a-f]{64}$/.test(stamp.hostSha256)+ ) {+ return null;+ }+ return stamp;+}++function finalizedWorkspaceStamp(input: {+ descriptor: NativeWorkspaceSyncDescriptor;+ hostSha256: string;+}): Record<string, unknown> {+ return {+ schema: STAMP_SCHEMA,+ workspaceId: input.descriptor.binding.workspaceId,+ providerLeaseId: input.descriptor.binding.providerLeaseId,+ remoteCwd: input.descriptor.binding.remoteCwd,+ hostSha256: input.hostSha256,+ finalizedRunId: input.descriptor.binding.runId,+ };+}++async function prepareRuntime(input: {+ runId: string;+ target: Extract<AdapterExecutionTarget, { transport: "sandbox" }>;+ workspaceLocalDir: string;+ mode: WorkspaceInboundMode;+ baseline?: DirectorySnapshot;+ gitSnapshot?: GitWorkspaceSnapshot | null;+ durableSeed?: WorkspaceDurableSeedPaths;+}): Promise<PreparedAdapterExecutionTargetRuntime> {+ return prepareAdapterExecutionTargetRuntime({+ runId: input.runId,+ target: input.target,+ adapterKey: "paperclip-runner",+ workspaceLocalDir: input.workspaceLocalDir,+ workspaceRemoteDir: input.target.remoteCwd,+ workspaceInboundMode: input.mode,+ workspaceDurableSeed: input.durableSeed,+ workspaceBaseline: input.baseline,+ workspaceGitSnapshot: input.gitSnapshot,+ });+}++async function finalizePreparedRuntime(input: {+ db: Db;+ runId: string;+ target: Extract<AdapterExecutionTarget, { transport: "sandbox" }>;+ runtime: PreparedAdapterExecutionTargetRuntime;+ descriptor: NativeWorkspaceSyncDescriptor;+}): Promise<NativeWorkspaceSyncReference> {+ await input.runtime.restoreWorkspace();+ const finalSnapshot =+ await import("@paperclipai/adapter-utils/workspace-restore-merge").then(+ ({ captureDirectorySnapshot }) =>+ captureDirectorySnapshot(input.descriptor.binding.localCwd, {+ exclude: input.runtime.workspaceSyncSnapshot?.baseline.exclude ?? [],+ }),+ );+ const finalHostSha256 = directorySnapshotSha256(finalSnapshot);+ const stamp = finalizedWorkspaceStamp({+ descriptor: input.descriptor,+ hostSha256: finalHostSha256,+ });+ await writeRemoteStamp({ target: input.target, stamp });+ const finalizedDescriptor: NativeWorkspaceSyncDescriptor = {+ ...input.descriptor,+ state: "finalized",+ finalizedAt: new Date().toISOString(),+ finalHostSha256,+ };+ const reference = await writeDescriptor(finalizedDescriptor);+ await persistRunReference(input.db, input.runId, reference);+ await persistLeaseStamp({+ db: input.db,+ leaseId: input.descriptor.binding.leaseId,+ stamp,+ });+ return reference;+}++export async function prepareNativeWorkspaceSync(input: {+ db: Db;+ runId: string;+ companyId: string;+ workspaceId: string;+ workspaceLocalDir: string;+ target: AdapterExecutionTarget | null;+ lease: EnvironmentLease;+ restartRecovery?: NativeRestartRecoveryClaim;+ resourceDisposition?: NativeWorkspaceResourceDisposition;+}): Promise<PreparedNativeWorkspaceSync | null> {+ if (input.target?.kind !== "remote" || input.target.transport !== "sandbox") {+ return null;+ }+ const target = input.target;+ const providerLeaseId = providerLeaseIdFor({ target, lease: input.lease });+ const run = await input.db+ .select({ runnerProfileJson: heartbeatRuns.runnerProfileJson })+ .from(heartbeatRuns)+ .where(eq(heartbeatRuns.id, input.runId))+ .limit(1)+ .then((rows) => rows[0] ?? null);+ if (!run) throw new Error("native_workspace_sync_run_missing");+ const existingReference = readNativeWorkspaceSyncReference(+ parseObject(run.runnerProfileJson).nativeWorkspaceSync,+ );++ let runtime: PreparedAdapterExecutionTargetRuntime;+ let descriptor: NativeWorkspaceSyncDescriptor;+ let mode: WorkspaceInboundMode = "host_current";+ const seedPaths = durableSeedPaths(input.runId);+ await ensurePrivateDirectory(descriptorDirectory(input.runId));++ if (existingReference) {+ const existing = await readDescriptor({+ runId: input.runId,+ reference: existingReference,+ });+ if (+ existing.descriptor.binding.companyId !== input.companyId ||+ existing.descriptor.binding.workspaceId !== input.workspaceId ||+ path.resolve(existing.descriptor.binding.localCwd) !==+ path.resolve(input.workspaceLocalDir) ||+ existing.descriptor.binding.remoteCwd !== target.remoteCwd+ ) {+ throw new Error("native_workspace_sync_descriptor_binding_mismatch");+ }+ const sameProviderLease =+ existing.descriptor.binding.providerLeaseId === providerLeaseId;+ mode = classifyNativeWorkspaceInbound({+ kind: "existing_run",+ restartRecovery: Boolean(input.restartRecovery),+ sameProviderLease,+ });+ const durableSeed =+ mode === "durable_seed"+ ? existing.descriptor.seed+ ? await verifiedDurableSeed({+ runId: input.runId,+ seed: existing.descriptor.seed,+ gitSnapshot: existing.descriptor.gitSnapshot,+ })+ : (() => {+ throw new Error("workspace_sync_out_unrecoverable");+ })()+ : undefined;+ runtime = await prepareRuntime({+ runId: input.runId,+ target,+ workspaceLocalDir: input.workspaceLocalDir,+ mode,+ baseline: existing.baseline,+ gitSnapshot: existing.descriptor.gitSnapshot,+ durableSeed,+ });+ descriptor = {+ ...existing.descriptor,+ binding: {+ ...existing.descriptor.binding,+ leaseId: target.leaseId ?? input.lease.id,+ providerLeaseId,+ },+ };+ } else {+ const acquisition = target.sandboxLeaseAcquisition?.outcome ?? null;+ const priorStamp = leaseStamp({+ lease: input.lease,+ workspaceId: input.workspaceId,+ providerLeaseId,+ remoteCwd: target.remoteCwd,+ });+ mode = classifyNativeWorkspaceInbound({+ kind: "new_run",+ acquisition,+ hasPriorStamp: priorStamp !== null,+ });+ runtime = await prepareRuntime({+ runId: input.runId,+ target,+ workspaceLocalDir: input.workspaceLocalDir,+ mode,+ durableSeed: {+ workspaceArchivePath: seedPaths.workspaceArchivePath,+ gitArchivePath: seedPaths.gitArchivePath,+ },+ });+ const currentSnapshot = runtime.workspaceSyncSnapshot;+ if (!currentSnapshot) {+ throw new Error("native_workspace_sync_snapshot_missing");+ }+ if (acquisition === "resumed" && priorStamp) {+ const currentHostSha256 = directorySnapshotSha256(+ currentSnapshot.baseline,+ );+ const verifiedWarmAdoption =+ priorStamp.hostSha256 === currentHostSha256 &&+ (await remoteStampMatches({ target, expected: priorStamp }));+ if (verifiedWarmAdoption) {+ mode = "adopt_remote";+ } else {+ mode = "host_current";+ runtime = await prepareRuntime({+ runId: input.runId,+ target,+ workspaceLocalDir: input.workspaceLocalDir,+ mode,+ baseline: currentSnapshot.baseline,+ gitSnapshot: currentSnapshot.gitSnapshot,+ durableSeed: {+ workspaceArchivePath: seedPaths.workspaceArchivePath,+ gitArchivePath: seedPaths.gitArchivePath,+ },+ });+ }+ }+ const snapshot = runtime.workspaceSyncSnapshot;+ if (!snapshot) throw new Error("native_workspace_sync_snapshot_missing");+ const now = new Date().toISOString();+ const workspaceArchiveSha256 = await sha256File(+ seedPaths.workspaceArchivePath,+ );+ const gitArchiveSha256 = snapshot.gitSnapshot+ ? await sha256File(seedPaths.gitArchivePath)+ : null;+ descriptor = {+ schema: DESCRIPTOR_SCHEMA,+ binding: {+ runId: input.runId,+ companyId: input.companyId,+ workspaceId: input.workspaceId,+ leaseId: target.leaseId ?? input.lease.id,+ providerLeaseId,+ localCwd: path.resolve(input.workspaceLocalDir),+ remoteCwd: target.remoteCwd,+ },+ state: "prepared",+ baselineSha256: directorySnapshotSha256(snapshot.baseline),+ baseline: serializeDirectorySnapshot(snapshot.baseline),+ gitSnapshot: snapshot.gitSnapshot,+ seed: { workspaceArchiveSha256, gitArchiveSha256 },+ createdAt: now,+ finalizedAt: null,+ finalHostSha256: null,+ resourceDisposition: input.resourceDisposition ?? null,+ };+ }++ let reference = await writeDescriptor(descriptor);+ await persistRunReference(input.db, input.runId, reference);+ let restorePromise: Promise<void> | null = null;+ return {+ mode,+ get reference() {+ return reference;+ },+ restoreWorkspace: async () => {+ if (!restorePromise) {+ restorePromise = finalizePreparedRuntime({+ db: input.db,+ runId: input.runId,+ target,+ runtime,+ descriptor,+ })+ .then((finalizedReference) => {+ reference = finalizedReference;+ })+ .catch((error) => {+ restorePromise = null;+ throw error;+ });+ }+ await restorePromise;+ },+ cleanup: () => cleanupNativeWorkspaceSync(input.runId),+ };+}++export async function resumeNativeWorkspaceSync(input: {+ db: Db;+ runId: string;+ target: AdapterExecutionTarget;+}): Promise<boolean> {+ if (input.target.kind !== "remote" || input.target.transport !== "sandbox") {+ throw new Error("workspace_sync_out_unrecoverable");+ }+ const run = await input.db+ .select({ runnerProfileJson: heartbeatRuns.runnerProfileJson })+ .from(heartbeatRuns)+ .where(eq(heartbeatRuns.id, input.runId))+ .limit(1)+ .then((rows) => rows[0] ?? null);+ const reference = readNativeWorkspaceSyncReference(+ parseObject(run?.runnerProfileJson).nativeWorkspaceSync,+ );+ if (!reference) return false;+ const existing = await readDescriptor({ runId: input.runId, reference });+ const providerLeaseId =+ input.target.sandboxLeaseAcquisition?.providerLeaseId ??+ reference.providerLeaseId;+ if (+ providerLeaseId !== reference.providerLeaseId ||+ input.target.remoteCwd !== reference.remoteCwd+ ) {+ throw new Error("workspace_sync_out_unrecoverable");+ }+ if (+ existing.descriptor.state === "finalized" &&+ existing.descriptor.finalHostSha256+ ) {+ const stamp = finalizedWorkspaceStamp({+ descriptor: existing.descriptor,+ hostSha256: existing.descriptor.finalHostSha256,+ });+ if (+ !(await remoteStampMatches({ target: input.target, expected: stamp }))+ ) {+ await writeRemoteStamp({ target: input.target, stamp });+ }+ await persistLeaseStamp({+ db: input.db,+ leaseId: existing.descriptor.binding.leaseId,+ stamp,+ });+ return true;+ }+ const runtime = await prepareRuntime({+ runId: input.runId,+ target: input.target,+ workspaceLocalDir: existing.descriptor.binding.localCwd,+ mode: "adopt_remote",+ baseline: existing.baseline,+ gitSnapshot: existing.descriptor.gitSnapshot,+ });+ await finalizePreparedRuntime({+ db: input.db,+ runId: input.runId,+ target: input.target,+ runtime,+ descriptor: existing.descriptor,+ });+ return true;+}++export async function cleanupNativeWorkspaceSync(runId: string): Promise<void> {+ await fs.rm(descriptorDirectory(runId), { recursive: true, force: true });+}++export const nativeWorkspaceSyncInternals = {+ descriptorPath,+ legacyDescriptorPath,+ durableSeedPaths,+ writeDescriptor,+ readDescriptor,+ readReference: readNativeWorkspaceSyncReference,+ sha256,+};diff --git a/tests/runner-e2e/FIXTURES.md b/tests/runner-e2e/FIXTURES.mdindex 5d533b2d01..01c229242d 100644--- a/tests/runner-e2e/FIXTURES.md+++ b/tests/runner-e2e/FIXTURES.md@@ -58,9 +58,13 @@ registry therefore discovers that row through the public environments API.This still provides full isolation because every cell starts a new Paperclipinstance and database.-Daytona creates a sandbox environment through the public API. Keep-`reuseLease:false`, `runnerLifecycleMode:"per_turn"`, short provider cleanup-backstops, a Daytona secret reference, and an immutable image digest. Teardown+Daytona creates sandbox environments through the public API. The core fixture+keeps `reuseLease:false` and `runnerLifecycleMode:"per_turn"`. The dedicated+warm-continuity fixture uses `reuseLease:true` and+`runnerLifecycleMode:"warm"`; its distinct `configurationKey` is part of the+suite fingerprint even though both fixtures report `environmentId:"daytona"`.+Keep short provider cleanup backstops, a Daytona secret reference, and an+immutable image digest. Teardownmust delete the environment with reusable-lease destruction and must fail thecell if cleanup cannot be confirmed. Keep CPU, memory, and disk explicit: leasemetadata and the per-test public-list-price runtime estimate depend on that@@ -93,7 +97,10 @@ marker factories. `question_resume_completion` must define the deterministicbrowser answer and prove exactly two successful runs with no pendinginteraction. `plan_approval_completion` must target the exact two-stepcanonical Plan revision, capture its pending UI, approve in the browser, and-prove exactly two successful runs.+prove exactly two successful runs. `warm_three_turn` provides exactly two+browser follow-up messages, preserves one project/execution-workspace scope,+verifies host file contents after every turn, and finishes within three+ten-minute turn deadlines.Every selected case runs in its own isolated Paperclip process, and independentcases may run concurrently. Follow-up turns inside one case retain their shareddiff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.mdindex ed9fed5caa..b9ffb1dd6c 100644--- a/tests/runner-e2e/README.md+++ b/tests/runner-e2e/README.md@@ -68,10 +68,11 @@ pnpm test:e2e:runner -- --group native --environment localpnpm test:e2e:runner -- --profile runner-codex --case message-markerpnpm test:e2e:runner -- --case plan-revise-accept --group localpnpm test:e2e:runner -- --case ask-question --group native+pnpm test:e2e:runner -- --suite daytona-warm-continuitypnpm test:e2e:runner -- --all```-The catalog contains three suites. `core-compatibility` (**Core Runner+The catalog contains four suites. `core-compatibility` (**Core RunnerCompatibility**) is seven major runner profiles × local/Daytona × threeworkflows: 42 cells. Its cases are:@@ -107,7 +108,19 @@ duplicating the final response. The second workflow restarts the isolatedPaperclip server while the interaction is waiting, reloads that state, andthen resumes it. The suite has no Daytona cells.-The complete catalog is 66 cells (45 local and 21 Daytona) and 114 expected+`daytona-warm-continuity` (**Daytona Warm Continuity**) is exactly two paid+cells: legacy Codex and Runner Codex against one reusable warm Daytona+configuration. Each cell creates a real project with a primary local-path+workspace through the API, selects it in the browser task dialog, and performs+three browser-driven turns on one issue. Every turn reads and extends the same+nonce file, verifies host copy-back, records scheduler/run/end-to-end timing,+and asserts `created`, `resumed`, `resumed` lease acquisition on one sandbox.+Runner Codex additionally proves stable native session, provider session,+runner instance, PID, and process-start identity. Each turn is bounded to ten+minutes, the cell to thirty minutes, and cleanup explicitly deletes the+sandbox rather than waiting for Daytona's idle timeout.++The complete catalog is 68 cells (45 local and 23 Daytona) and 120 expectedpaid agent turns. Follow-up steps remain ordered within their cell; all othercells are independent. Narrow selectors are strongly recommended whiledeveloping fixtures.@@ -351,7 +364,7 @@ Set `RUNNER_E2E_AWS_ENABLED=true` to route paid cells to the repository-scopedephemeral AWS RunsOn fleet selected by`runs-on/fleet=paperclip-public-pr-x64/env=public-ci`. Any other value uses theproven GitHub-hosted `ubuntu-latest` target. Set `RUNNER_E2E_MAX_PARALLEL` to an-integer from 1–100 on AWS (default 100); use at least 66 to run the current+integer from 1–100 on AWS (default 100); use at least 68 to run the currentcomplete catalog in one wave. The fallback runner retains its 1–57 limit anddefault of 32. Multi-turn steps are sequential inside their cell whileindependent cells overlap. Artifacts and merged HTML/JUnit/normalized reportsdiff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.tsindex 83ecd13b1f..685c18add8 100644--- a/tests/runner-e2e/catalog.test.ts+++ b/tests/runner-e2e/catalog.test.ts@@ -10,7 +10,10 @@ import {runnerProfiles,runnerSuites,runnerTasks,+ daytonaWarmContinuityTask,+ daytonaWarmEnvironment,isImmutableDaytonaImage,+ suiteDefinitionHash,validateRunnerCatalog,} from "./catalog.js";import {@@ -21,7 +24,7 @@ import {} from "./selectors.js";describe("runner E2E catalog", () => {- it("validates the core, local-integrity, and breadth suites", () => {+ it("validates the core, local-integrity, breadth, and warm suites", () => {expect(runnerProfiles).toHaveLength(7);expect(openRouterBreadthProfiles).toHaveLength(4);expect(runnerEnvironments).toHaveLength(2);@@ -29,10 +32,10 @@ describe("runner E2E catalog", () => {expect(localIntegrityTasks).toHaveLength(2);expect(openRouterBreadthTasks).toHaveLength(3);expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([- 42, 14, 10,+ 42, 14, 10, 2,]);- expect(validateRunnerCatalog()).toHaveLength(66);- expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(66);+ expect(validateRunnerCatalog()).toHaveLength(68);+ expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(68);expect(runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"),).toHaveLength(42);@@ -46,18 +49,81 @@ describe("runner E2E catalog", () => {(entry) => entry.suite.id === "openrouter-model-breadth",),).toHaveLength(10);+ expect(+ runnerMatrix.filter(+ (entry) => entry.suite.id === "daytona-warm-continuity",+ ),+ ).toHaveLength(2);expect(runnerMatrix.reduce((total, execution) => total + execution.task.expectedRunCount,0,),- ).toBe(114);+ ).toBe(120);expect(runnerTasks.find((task) => task.id === "plan-revise-accept")?.attemptTimeoutMs,).toEqual({ local: 8 * 60_000, daytona: 12 * 60_000 });});+ it("defines the warm Daytona continuity fixture as exactly two Codex cells", () => {+ expect(daytonaWarmEnvironment).toMatchObject({+ id: "daytona",+ configurationKey: "warm-reuse-v1",+ groups: ["daytona", "warm"],+ });+ expect(+ daytonaWarmEnvironment.buildEnvironment({+ secretRefs: {+ DAYTONA_API_KEY: {+ type: "secret_ref",+ secretId: "22222222-2222-4222-8222-222222222222",+ version: "latest",+ },+ },+ daytonaImage: `runner@sha256:${"a".repeat(64)}`,+ executionId: "warm",+ }),+ ).toMatchObject({+ config: {+ reuseLease: true,+ runnerLifecycleMode: "warm",+ autoStopInterval: 5,+ autoArchiveInterval: 15,+ autoDeleteInterval: 60,+ },+ });+ expect(daytonaWarmContinuityTask).toMatchObject({+ flow: "warm_three_turn",+ expectedRunCount: 3,+ turnTimeoutMs: 600_000,+ });+ expect(+ daytonaWarmContinuityTask.buildFollowupMessages?.("nonce"),+ ).toHaveLength(2);+ const cells = runnerMatrix.filter(+ (entry) => entry.suite.id === "daytona-warm-continuity",+ );+ expect(cells.map((entry) => entry.profile.id)).toEqual([+ "legacy-codex",+ "runner-codex",+ ]);+ expect(cells.every((entry) => entry.environment.id === "daytona")).toBe(+ true,+ );+ const suite = runnerSuites.find(+ (candidate) => candidate.id === "daytona-warm-continuity",+ )!;+ expect(+ suiteDefinitionHash({+ ...suite,+ environments: [+ { ...daytonaWarmEnvironment, configurationKey: "changed" },+ ],+ }),+ ).not.toBe(suiteDefinitionHash(suite));+ });+it("derives the qualified local native OpenCode profiles from the ranked snapshot", () => {expect(openRouterBreadthExcludedModelIds).toEqual(["xiaomi/mimo-v2.5"]);expect(openRouterBreadthExcludedExecutionIds).toEqual([@@ -181,9 +247,10 @@ describe("runner E2E catalog", () => {question?.buildPrompt("nonce"),...breadthTasks,]) {- expect(prompt).toContain("then emit exactly");+ const terminalTextInstruction = prompt?.match(/then emit (?:exactly|only)/)?.[0];+ expect(terminalTextInstruction).toBeDefined();expect(prompt!.indexOf("paperclip_finish exactly once")).toBeLessThan(- prompt!.indexOf("then emit exactly"),+ prompt!.indexOf(terminalTextInstruction!),);expect(prompt).toContain("Wait for that tool call to succeed");}@@ -336,7 +403,7 @@ describe("runner E2E catalog", () => {"those two tool calls form one indivisible response sequence",);expect(task!.buildPrompt("nonce")).toContain(- "Do not emit assistant text, end the heartbeat, or stop after write_document alone",+ "Do not emit assistant text, end the response or heartbeat, or stop after write_document alone",);expect(task!.buildPrompt("nonce")).toContain("one atomic issue PATCH with status `done` and that exact comment",@@ -433,7 +500,7 @@ describe("runner E2E selectors", () => {"daytona",]);const selected = selectRunnerExecutions(options);- expect(selected).toHaveLength(12);+ expect(selected).toHaveLength(13);expect(selected.every((entry) =>@@ -443,7 +510,7 @@ describe("runner E2E selectors", () => {).toBe(true);});- it("rejects groups outside the advertised four", () => {+ it("rejects unknown groups", () => {const options = parseRunnerSelectors(["--group", "codex"]);expect(() => selectRunnerExecutions(options)).toThrow("Unknown group");});@@ -452,10 +519,10 @@ describe("runner E2E selectors", () => {const jobs = buildMatrixJobs(selectRunnerExecutions(parseRunnerSelectors(["--all"])),);- expect(jobs).toHaveLength(66);- expect(jobs.filter((job) => job.needsDaytona)).toHaveLength(21);+ expect(jobs).toHaveLength(68);+ expect(jobs.filter((job) => job.needsDaytona)).toHaveLength(23);expect(jobs.filter((job) => !job.needsDaytona)).toHaveLength(45);- expect(new Set(jobs.map((job) => job.executionId)).size).toBe(66);+ expect(new Set(jobs.map((job) => job.executionId)).size).toBe(68);expect(jobs.find((job) =>diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.tsindex 6b887aaec9..586879eace 100644--- a/tests/runner-e2e/catalog.ts+++ b/tests/runner-e2e/catalog.ts@@ -27,6 +27,7 @@ const SELECTABLE_GROUPS = ["native","local","daytona",+ "warm","core","breadth",] as const;@@ -374,6 +375,50 @@ export const runnerEnvironments: readonly EnvironmentFixture[] = [},] as const;+export const daytonaWarmEnvironment: EnvironmentFixture = {+ id: "daytona",+ configurationKey: "warm-reuse-v1",+ label: "Daytona warm reusable sandbox",+ groups: ["daytona", "warm"],+ driver: "sandbox",+ provider: "daytona",+ credential: "DAYTONA_API_KEY",+ lifecycle: {+ setup: "create_via_api",+ probe: "run_context_via_api",+ cleanup: "delete_via_api_and_destroy_leases",+ },+ expectedExecutionTarget: { kind: "remote", transport: "sandbox" },+ buildEnvironment(input) {+ if (!isImmutableDaytonaImage(input.daytonaImage)) {+ throw new Error(+ "PAPERCLIP_E2E_DAYTONA_IMAGE must be an immutable image digest",+ );+ }+ return {+ name: `Runner E2E Daytona warm ${input.executionId}`,+ description: "Ephemeral reusable Daytona runner E2E environment",+ driver: "sandbox",+ config: {+ provider: "daytona",+ apiKey: requiredDaytonaSecret(input),+ image: input.daytonaImage,+ cpu: 4,+ memory: 4,+ disk: 10,+ reuseLease: true,+ runnerLifecycleMode: "warm",+ autoStopInterval: 5,+ autoArchiveInterval: 15,+ autoDeleteInterval: 60,+ timeoutMs: 300_000,+ livenessTimeoutMs: 30_000,+ },+ envVars: {},+ };+ },+};+export const runnerTasks: readonly RunnerTaskFixture[] = [{id: "message-marker",@@ -719,6 +764,80 @@ const localEnvironment = runnerEnvironments.find((environment) => environment.id === "local",)!;+function warmTurnMarker(turn: 1 | 2 | 3, nonce: string) {+ return `PAPERCLIP_E2E_WARM_T${turn}_${nonce}`;+}++function warmWorkspaceLine(turn: 1 | 2 | 3, nonce: string) {+ return `T${turn}_${nonce}`;+}++function warmTurnInstructions(turn: 1 | 2 | 3, nonce: string) {+ const file = `daytona-warm-${nonce}.txt`;+ const lines = Array.from({ length: turn }, (_, index) =>+ warmWorkspaceLine((index + 1) as 1 | 2 | 3, nonce),+ );+ const marker = warmTurnMarker(turn, nonce);+ const finalTurn = turn === 3;+ return [+ `This is warm Daytona continuity turn ${turn} of 3. Work only in the current execution workspace.`,+ turn === 1+ ? `Create ${file} with exactly this one line followed by a newline: ${lines[0]}`+ : `Before changing anything, read ${file} and verify its content is exactly ${lines.slice(0, -1).join("\\n")} followed by a newline. Then append exactly ${lines.at(-1)} followed by a newline.`,+ `After the write, verify ${file} contains exactly these lines, once each and in order: ${lines.join(" | ")}.`,+ `In a native runner, call paperclip_finish exactly once with {reportedWorkDisposition:"${finalTurn ? "done" : "needs_review"}",summary:"${marker}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[{commandOrCheck:"read ${file}",status:"passed"}]}. Wait for that tool call to succeed, then emit exactly ${marker} once as the complete user-facing final response.`,+ `In a legacy runner, make exactly one public-API completion write after verification: PATCH /api/issues/$PAPERCLIP_TASK_ID with {"status":"${finalTurn ? "done" : "in_review"}","comment":"${marker}"}. Include Authorization and X-Paperclip-Run-Id. Do not POST a separate comment.`,+ `Do not include ${marker} in any other visible response or write. Do not recreate, truncate, reorder, or duplicate prior lines.`,+ ].join("\n");+}++export const daytonaWarmContinuityTask: RunnerTaskFixture = {+ id: "warm-three-turn",+ label: "Warm three-turn workspace continuity",+ groups: ["warm"],+ workMode: "standard",+ flow: "warm_three_turn",+ expectedRunCount: 3,+ attemptTimeoutMs: { local: 30 * 60_000, daytona: 30 * 60_000 },+ turnTimeoutMs: 10 * 60_000,+ expectedTerminalState: { issue: "done", run: "succeeded" },+ buildTitle: (nonce) => `Runner E2E warm Daytona continuity ${nonce}`,+ buildVisibleMarker: (nonce) => warmTurnMarker(3, nonce),+ buildPrompt: (nonce) => warmTurnInstructions(1, nonce),+ buildFollowupMessages: (nonce) => [+ warmTurnInstructions(2, nonce),+ warmTurnInstructions(3, nonce),+ ],+ buildMatchers(nonce, execution) {+ const markers = ([1, 2, 3] as const).map((turn) =>+ warmTurnMarker(turn, nonce),+ );+ return [+ { kind: "message_exact", expected: markers[2] },+ ...markers.map(+ (expected) =>+ ({ kind: "message_occurrences", expected, count: 1 }) as const,+ ),+ { kind: "message_ordered", expected: markers },+ {+ kind: "file_exact",+ path: `daytona-warm-${nonce}.txt`,+ expected: `${([1, 2, 3] as const)+ .map((turn) => warmWorkspaceLine(turn, nonce))+ .join("\n")}\n`,+ },+ { kind: "issue_status", expected: "done" },+ { kind: "run_status", expected: "succeeded" },+ { kind: "runtime_mode", expected: execution.profile.expectedRuntimeMode },+ { kind: "environment", expected: "daytona" },+ ];+ },+};++const codexContinuityProfiles = runnerProfiles.filter((profile) =>+ ["legacy-codex", "runner-codex"].includes(profile.id),+);+export const runnerSuites: readonly RunnerSuiteFixture[] = [{id: "core-compatibility",@@ -762,6 +881,17 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [excludedExecutionIds: openRouterBreadthExcludedExecutionIds,},},+ {+ id: "daytona-warm-continuity",+ label: "Daytona Warm Continuity",+ description:+ "Three browser-driven turns on one reusable Daytona sandbox for legacy and native Codex.",+ groups: ["daytona", "warm"],+ profiles: codexContinuityProfiles,+ environments: [daytonaWarmEnvironment],+ tasks: [daytonaWarmContinuityTask],+ expectedMatrixSize: 2,+ },] as const;export function suiteDefinitionHash(suite: RunnerSuiteFixture) {@@ -774,7 +904,10 @@ export function suiteDefinitionHash(suite: RunnerSuiteFixture) {model: profile.model,qualification: profile.modelQualification,})),- environments: suite.environments.map((environment) => environment.id),+ environments: suite.environments.map((environment) => ({+ id: environment.id,+ configurationKey: environment.configurationKey ?? "default",+ })),tasks: suite.tasks.map((task) => ({id: task.id,flow: task.flow,@@ -869,6 +1002,7 @@ export function validateRunnerCatalog(): MatrixExecution[] {...runnerTasks,...localIntegrityTasks,...openRouterBreadthTasks,+ daytonaWarmContinuityTask,];for (const [label, values] of [["suite", runnerSuites],@@ -888,6 +1022,7 @@ export function validateRunnerCatalog(): MatrixExecution[] {...runnerSuites,...allProfiles,...runnerEnvironments,+ daytonaWarmEnvironment,...allTasks,]) {const unknownGroups = fixture.groups.filter(@@ -916,7 +1051,7 @@ export function validateRunnerCatalog(): MatrixExecution[] {]),);- for (const environment of runnerEnvironments) {+ for (const environment of [...runnerEnvironments, daytonaWarmEnvironment]) {const payload = environment.buildEnvironment({secretRefs: sampleRefs,daytonaImage:@@ -968,8 +1103,8 @@ export function validateRunnerCatalog(): MatrixExecution[] {);}}- if (matrix.length !== 66)- throw new Error(`Expected 66 runner executions; received ${matrix.length}`);+ if (matrix.length !== 68)+ throw new Error(`Expected 68 runner executions; received ${matrix.length}`);return matrix;}diff --git a/tests/runner-e2e/dashboard.ts b/tests/runner-e2e/dashboard.tsindex 820615ff92..6fe3f8d782 100644--- a/tests/runner-e2e/dashboard.ts+++ b/tests/runner-e2e/dashboard.ts@@ -196,6 +196,18 @@ function renderCase(</tr>`,).join("");+ const turnTimingRows = (entry?.result.turnTimings ?? [])+ .map(+ (timing) => `<tr>+ <td>${timing.turn}</td>+ <td><code>${html(timing.runId)}</code></td>+ <td>${html(timing.leaseAcquisitionOutcome)}</td>+ <td>${html(timing.schedulerLatencyMs === null ? "unavailable" : durationLabel(timing.schedulerLatencyMs))}</td>+ <td>${html(timing.runDurationMs === null ? "unavailable" : durationLabel(timing.runDurationMs))}</td>+ <td>${html(timing.responseLatencyMs === null ? "unavailable" : durationLabel(timing.responseLatencyMs))}</td>+ </tr>`,+ )+ .join("");const gallery = screenshots.length? `<div class="gallery" aria-label="Screenshots for ${html(execution.id)}">${screenshots.map(@@ -268,6 +280,7 @@ function renderCase(: ""}<p class="detail">${html(detail)}</p>+ ${turnTimingRows ? `<div class="matcher-wrap"><table class="matchers"><thead><tr><th>Turn</th><th>Run</th><th>Lease</th><th>Scheduler</th><th>Run duration</th><th>Response</th></tr></thead><tbody>${turnTimingRows}</tbody></table></div>` : ""}${matcherRows ? `<div class="matcher-wrap"><table class="matchers"><thead><tr><th>Result</th><th>Matcher</th><th>Expectation</th><th>Detail</th></tr></thead><tbody>${matcherRows}</tbody></table></div>` : `<p class="detail">No matcher result was recorded.</p>`}${entry ? `<details class="usage"><summary>Usage and billing metadata</summary><pre>${html(JSON.stringify({ billing, rawUsage: entry.result.usage ?? null }, null, 2))}</pre></details>` : ""}${links}diff --git a/tests/runner-e2e/history.test.ts b/tests/runner-e2e/history.test.tsindex c11f97e0db..32c6a2bc53 100644--- a/tests/runner-e2e/history.test.ts+++ b/tests/runner-e2e/history.test.ts@@ -195,8 +195,8 @@ describe("runner E2E campaign history", () => {expect(index).toContain("Runner E2E campaigns");expect(index).toContain("complete-green");expect(index).toContain("complete-red");- expect(index).toContain("66/66 passed");- expect(index).toContain("65/66 passed");+ expect(index).toContain("68/68 passed");+ expect(index).toContain("67/68 passed");expect(index).toContain("Open report →");expect(index).toContain("campaigns/complete-red/public-images/campaign-summary.png",diff --git a/tests/runner-e2e/live-fixtures.test.ts b/tests/runner-e2e/live-fixtures.test.tsindex 1fe34be8e0..a919f07c2b 100644--- a/tests/runner-e2e/live-fixtures.test.ts+++ b/tests/runner-e2e/live-fixtures.test.ts@@ -70,4 +70,80 @@ describe("live runner fixtures", () => {"DELETE /api/environments/environment-1?destroyReusableSandboxLeases=true",);});++ it("creates a primary project workspace for reusable Daytona scope", async () => {+ const calls: string[] = [];+ const api = {+ async post(path: string, data?: Record<string, unknown>) {+ calls.push(`POST ${path}`);+ if (path === "/api/plugins/install") {+ return {+ id: "plugin-daytona",+ pluginKey: "paperclip.daytona-sandbox-provider",+ status: "ready",+ };+ }+ if (path === "/api/companies") {+ return { id: "company-1", name: "Runner E2E" };+ }+ if (path.endsWith("/environments")) {+ return { id: "environment-1", driver: "sandbox" };+ }+ if (path.endsWith("/agents")) {+ return { id: "agent-1", name: "Agent", companyId: "company-1" };+ }+ if (path.endsWith("/projects")) {+ expect(data).toMatchObject({+ executionWorkspacePolicy: {+ enabled: true,+ defaultMode: "shared_workspace",+ environmentId: "environment-1",+ },+ workspace: {+ sourceType: "local_path",+ cwd: "/tmp/workspace",+ isPrimary: true,+ },+ });+ return {+ id: "project-1",+ name: data?.name,+ primaryWorkspace: {+ id: "project-workspace-1",+ cwd: "/tmp/workspace",+ },+ };+ }+ throw new Error(`Unexpected POST ${path}`);+ },+ async postSensitive(_path: string, data?: Record<string, unknown>) {+ return { id: `secret-${String(data?.key).toLowerCase()}` };+ },+ async delete() {},+ } as unknown as RunnerApi;+ const execution = runnerMatrix.find(+ (candidate) =>+ candidate.id ===+ "daytona-warm-continuity.runner-codex.daytona.warm-three-turn",+ )!;++ const fixtures = await setupLiveFixtures({+ api,+ execution,+ executionNonce: "nonce",+ workspacePath: "/tmp/workspace",+ credentials: {+ OPENAI_API_KEY: "openai-test-value",+ DAYTONA_API_KEY: "daytona-test-value",+ },+ daytonaImage:+ "ghcr.io/paperclip/image@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+ });++ expect(fixtures.project?.primaryWorkspace?.id).toBe("project-workspace-1");+ expect(+ calls.indexOf("POST /api/companies/company-1/environments"),+ ).toBeLessThan(calls.indexOf("POST /api/companies/company-1/projects"));+ await fixtures.teardown();+ });});diff --git a/tests/runner-e2e/live-fixtures.ts b/tests/runner-e2e/live-fixtures.tsindex 4dba281fb7..de9c242a42 100644--- a/tests/runner-e2e/live-fixtures.ts+++ b/tests/runner-e2e/live-fixtures.ts@@ -32,12 +32,21 @@ interface AgentRecord {name: string;companyId: string;}+interface ProjectRecord {+ id: string;+ name: string;+ primaryWorkspace?: {+ id: string;+ cwd?: string | null;+ } | null;+}export interface LiveFixtureValues {company: CompanyRecord;secretRefs: SecretReferenceMap;environment: EnvironmentRecord;agent: AgentRecord;+ project?: ProjectRecord;teardown(): Promise<void>;}@@ -210,12 +219,51 @@ export async function setupLiveFixtures(input: {},});+ if (execution.environment.configurationKey === "warm-reuse-v1") {+ registry.register<ProjectRecord>({+ id: "project",+ dependencies: ["company", "environment"],+ async setup(resolved) {+ const company = value<CompanyRecord>(resolved, "company");+ const environment = value<EnvironmentRecord>(resolved, "environment");+ return api.post<ProjectRecord>(+ `/api/companies/${company.id}/projects`,+ {+ name: `Runner E2E warm project ${input.executionNonce}`,+ description:+ "Ephemeral project anchoring a reusable Daytona execution workspace",+ executionWorkspacePolicy: {+ enabled: true,+ defaultMode: "shared_workspace",+ sharedWorkspaceConcurrency: "serialize",+ allowIssueOverride: false,+ environmentId: environment.id,+ workspaceStrategy: { type: "project_primary" },+ },+ workspace: {+ name: "Primary",+ sourceType: "local_path",+ cwd: input.workspacePath,+ isPrimary: true,+ },+ },+ );+ },+ async teardown() {+ // The isolated instance is deleted after provider resources are gone.+ },+ });+ }+const setup = await registry.setupAll();return {company: value<CompanyRecord>(setup.values, "company"),secretRefs: value<SecretReferenceMap>(setup.values, "secrets"),environment: value<EnvironmentRecord>(setup.values, "environment"),agent: value<AgentRecord>(setup.values, "agent"),+ ...(setup.values.has("project")+ ? { project: value<ProjectRecord>(setup.values, "project") }+ : {}),teardown: setup.teardown,};}diff --git a/tests/runner-e2e/matchers.ts b/tests/runner-e2e/matchers.tsindex 8ba474fa15..650fd907f4 100644--- a/tests/runner-e2e/matchers.ts+++ b/tests/runner-e2e/matchers.ts@@ -116,6 +116,7 @@ export async function evaluateMatcher(passed = actual === matcher.expected;} else if (matcher.kind === "file_exists" ||+ matcher.kind === "file_exact" ||matcher.kind === "file_contains") {try {@@ -124,7 +125,9 @@ export async function evaluateMatcher((await readFile(matcher.path, "utf8"));passed =matcher.kind === "file_exists" ||- String(actual).includes(matcher.expected);+ (matcher.kind === "file_exact"+ ? String(actual) === matcher.expected+ : String(actual).includes(matcher.expected));} catch {actual = undefined;passed = false;diff --git a/tests/runner-e2e/report.test.ts b/tests/runner-e2e/report.test.tsindex 2f949b6bc5..4ac6453cf0 100644--- a/tests/runner-e2e/report.test.ts+++ b/tests/runner-e2e/report.test.ts@@ -44,6 +44,19 @@ describe("runner E2E report aggregation", () => {workflowRunUrl: "https://example.test/actions/runs/forged",},runIds: ["run-2"],+ turnTimings: [+ {+ turn: 1,+ submittedAt: "2026-08-26T00:00:00.000Z",+ runStartedAt: "2026-08-26T00:00:00.100Z",+ runFinishedAt: "2026-08-26T00:00:01.000Z",+ schedulerLatencyMs: 100,+ runDurationMs: 900,+ responseLatencyMs: 1_000,+ runId: "run-2",+ leaseAcquisitionOutcome: "created",+ },+ ],usage: {inputTokens: 1_250,outputTokens: 75,@@ -201,6 +214,9 @@ describe("runner E2E report aggregation", () => {);expect(dashboard).toContain("message_contains");expect(dashboard).toContain("Matchers and test context");+ expect(dashboard).toContain("Scheduler");+ expect(dashboard).toContain("Run duration");+ expect(dashboard).toContain("100ms");expect(dashboard).toContain("Campaign billing summary");expect(dashboard).toContain("LLM reported subtotal");expect(dashboard).toContain("Agent execution time");diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.tsindex a1c2682b4e..f97cd952d6 100644--- a/tests/runner-e2e/runner.spec.ts+++ b/tests/runner-e2e/runner.spec.ts@@ -44,6 +44,9 @@ interface IssueRecord {status: string;workMode?: string;assigneeAgentId?: string | null;+ projectId?: string | null;+ projectWorkspaceId?: string | null;+ executionWorkspaceId?: string | null;executionRunId?: string | null;checkoutRunId?: string | null;}@@ -66,6 +69,9 @@ interface RunRecord {continuationAttempt?: number;retryOfRunId?: string | null;runnerInstanceId?: string | null;+ nativeSessionId?: string | null;+ processPid?: number | null;+ processStartedAt?: string | null;contextSnapshot?: Record<string, unknown> | null;runnerProfileJson?: Record<string, unknown> | null;usageJson?: Record<string, unknown> | null;@@ -80,13 +86,17 @@ interface RunRecord {interface EnvironmentLeaseRecord {id: string;+ status?: string;+ leasePolicy?: string;+ providerLeaseId?: string | null;+ executionWorkspaceId?: string | null;+ metadata?: Record<string, unknown> | null;issueId?: string | null;heartbeatRunId?: string | null;provider?: string | null;acquiredAt?: string | null;releasedAt?: string | null;updatedAt?: string | null;- metadata?: Record<string, unknown> | null;}interface InteractionRecord {@@ -290,6 +300,7 @@ async function createTaskThroughUi(input: {title: string;prompt: string;workMode: "standard" | "planning" | "ask";+ projectName?: string;}) {const issuesUrl = `/${encodeURIComponent(input.issuePrefix)}/issues`;const newTask = input.page.getByRole("button", { name: "New Task" }).first();@@ -336,9 +347,29 @@ async function createTaskThroughUi(input: {.getByPlaceholder("Search assignees...").fill(input.agentName);await input.page.getByText(input.agentName, { exact: true }).last().click();+ if (input.projectName) {+ const dialog = input.page.getByRole("dialog");+ await dialog.getByRole("button", { name: "Project", exact: true }).click();+ await dialog.getByPlaceholder("Search projects...").fill(input.projectName);+ await dialog.getByText(input.projectName, { exact: true }).last().click();+ }+ const submittedAtMs = Date.now();await input.page.getByRole("button", { name: "Create Task", exact: true }).click();+ return submittedAtMs;+}++async function submitTaskReply(page: Page, body: string): Promise<number> {+ const composer = page.getByTestId("task-chat-composer-input").last();+ await expect(composer).toBeVisible({ timeout: 30_000 });+ await composer+ .locator('[contenteditable="true"], textarea')+ .first()+ .fill(body);+ const submittedAtMs = Date.now();+ await page.getByTestId("task-chat-composer-send").last().click();+ return submittedAtMs;}function matchingRuns(runs: RunRecord[], issue: IssueRecord) {@@ -524,7 +555,11 @@ for (const execution of executions) {page,request,}, testInfo) => {- test.setTimeout(deadlineMs + 90_000);+ test.setTimeout(+ execution.task.flow === "warm_three_turn"+ ? deadlineMs+ : deadlineMs + 90_000,+ );const startedAtMs = Date.now();const startedAt = new Date(startedAtMs).toISOString();const nonce = `${randomBytes(6).toString("hex")}-${attempt}`;@@ -541,6 +576,8 @@ for (const execution of executions) {let selectedRuns: RunRecord[] = [];let runtimeLeases: EnvironmentLeaseRecord[] = [];let matcherResults: MatcherResult[] = [];+ let turnTimings: NonNullable<RunnerE2EResult["turnTimings"]> | undefined;+ const turnSubmissionTimesMs: number[] = [];const screenshots: NonNullable<RunnerE2EResult["screenshots"]> = [];let primaryError: unknown;let failureClassOverride: FailureClass | undefined;@@ -746,14 +783,17 @@ for (const execution of executions) {throw new Error("Created fixture company did not return an issue prefix",);- await createTaskThroughUi({- page,- issuePrefix,- agentName: fixtures.agent.name,- title,- prompt,- workMode: execution.task.workMode,- });+ turnSubmissionTimesMs.push(+ await createTaskThroughUi({+ page,+ issuePrefix,+ agentName: fixtures.agent.name,+ title,+ prompt,+ workMode: execution.task.workMode,+ projectName: fixtures.project?.name,+ }),+ );const deadlineAt = startedAtMs + deadlineMs;issue = await pollUntil({@@ -781,6 +821,16 @@ for (const execution of executions) {`UI-created issue work mode was ${String(issue.workMode)}; expected ${execution.task.workMode}`,);}+ if (fixtures.project) {+ if (+ issue.projectId !== fixtures.project.id ||+ issue.projectWorkspaceId !== fixtures.project.primaryWorkspace?.id+ ) {+ throw new Error(+ `Warm task did not retain its project/workspace scope: ${JSON.stringify({ projectId: issue.projectId, projectWorkspaceId: issue.projectWorkspaceId })}`,+ );+ }+ }await page.goto(`/${encodeURIComponent(issuePrefix)}/issues/${encodeURIComponent(issue.identifier ?? issue.id)}`,@@ -827,6 +877,7 @@ for (const execution of executions) {let planLifecycleEvidence: Record<string, unknown> | null = null;let questionLifecycleEvidence: Record<string, unknown> | null = null;+ let warmLifecycleEvidence: Record<string, unknown> | null = null;let expectedQuestionResolution: {interactionId: string;optionId: string;@@ -1214,12 +1265,165 @@ for (const execution of executions) {.last().click();planLifecycleEvidence = { interaction, plan };+ } else if (execution.task.flow === "warm_three_turn") {+ const followups = execution.task.buildFollowupMessages?.(nonce);+ if (!followups || !fixtures.project?.primaryWorkspace?.id) {+ throw new Error(+ `Warm fixture ${execution.task.id} is missing its project or follow-up messages`,+ );+ }+ const workspaceFile = path.join(+ workspacePath,+ `daytona-warm-${nonce}.txt`,+ );+ const turnEvidence: Array<Record<string, unknown>> = [];+ for (const completedTurn of [1, 2] as const) {+ const turnDeadlineAt = Math.min(+ deadlineAt,+ turnSubmissionTimesMs[completedTurn - 1]! ++ (execution.task.turnTimeoutMs ?? 10 * 60_000),+ );+ const waitingState = await pollUntil({+ label: `warm Daytona turn ${completedTurn} review state for issue ${issue.id}`,+ deadlineAt: turnDeadlineAt,+ load: loadTaskState,+ accept: ({ currentIssue, taskRuns }) =>+ currentIssue.status === "in_review" &&+ taskRuns.length === completedTurn &&+ taskRuns.every((run) => run.status === "succeeded"),+ reject: ({ taskRuns }) =>+ definitiveRunFailure(taskRuns) ??+ (taskRuns.length > completedTurn+ ? `warm turn ${completedTurn} dispatched duplicate runs`+ : undefined),+ });+ const expectedPrefix = `${Array.from(+ { length: completedTurn },+ (_, index) => `T${index + 1}_${nonce}`,+ ).join("\n")}\n`;+ const hostContent = await readFile(workspaceFile, "utf8");+ if (hostContent !== expectedPrefix) {+ throw new Error(+ `Host workspace was not finalized after warm turn ${completedTurn}: expected ${JSON.stringify(expectedPrefix)}, observed ${JSON.stringify(hostContent)}`,+ );+ }+ if (+ waitingState.currentIssue.projectId !== fixtures.project.id ||+ waitingState.currentIssue.projectWorkspaceId !==+ fixtures.project.primaryWorkspace.id ||+ !waitingState.currentIssue.executionWorkspaceId+ ) {+ throw new Error(+ `Warm turn ${completedTurn} lost its project execution-workspace scope`,+ );+ }+ const completedRunIds = new Set(+ waitingState.taskRuns.map((candidate) => candidate.id),+ );+ const retainedTurnLeases = await pollUntil({+ label: `retained Daytona leases after warm turn ${completedTurn}`,+ deadlineAt: Math.min(turnDeadlineAt, Date.now() + 30_000),+ intervalMs: 500,+ load: () =>+ api.get<EnvironmentLeaseRecord[]>(+ `/api/environments/${fixtures!.environment.id}/leases`,+ ),+ accept: (leases) => {+ const runOrder = new Map(+ waitingState.taskRuns.map((candidate, index) => [+ candidate.id,+ index,+ ]),+ );+ const completed = leases+ .filter(+ (lease) =>+ lease.heartbeatRunId &&+ completedRunIds.has(lease.heartbeatRunId),+ )+ .sort(+ (left, right) =>+ (runOrder.get(left.heartbeatRunId ?? "") ?? 0) -+ (runOrder.get(right.heartbeatRunId ?? "") ?? 0),+ );+ return (+ completed.length === completedTurn &&+ completed.every(+ (lease) =>+ lease.status === "retained" &&+ lease.leasePolicy === "reuse_by_environment" &&+ typeof lease.providerLeaseId === "string" &&+ record(lease.metadata).sandboxState === "started",+ ) &&+ completed+ .slice(1)+ .every(+ (lease) =>+ record(lease.metadata).resumedFromState === "started",+ )+ );+ },+ });+ const turnRunOrder = new Map(+ waitingState.taskRuns.map((candidate, index) => [+ candidate.id,+ index,+ ]),+ );+ const completedLeases = retainedTurnLeases+ .filter(+ (lease) =>+ lease.heartbeatRunId &&+ completedRunIds.has(lease.heartbeatRunId),+ )+ .sort(+ (left, right) =>+ (turnRunOrder.get(left.heartbeatRunId ?? "") ?? 0) -+ (turnRunOrder.get(right.heartbeatRunId ?? "") ?? 0),+ );+ if (+ new Set(completedLeases.map((lease) => lease.providerLeaseId))+ .size !== 1+ ) {+ throw new Error(+ `Warm turn ${completedTurn} replaced its Daytona sandbox`,+ );+ }+ turnEvidence.push({+ turn: completedTurn,+ issue: waitingState.currentIssue,+ run: waitingState.taskRuns.at(-1),+ hostContent,+ leases: completedLeases,+ });+ await page.goto(+ `/${encodeURIComponent(issuePrefix)}/issues/${encodeURIComponent(issue.identifier ?? issue.id)}`,+ { waitUntil: "domcontentloaded" },+ );+ await captureScreenshot(+ `warm-turn-${completedTurn}`,+ `Warm Daytona turn ${completedTurn} awaiting review`,+ `warm-turn-${completedTurn}.png`,+ );+ turnSubmissionTimesMs.push(+ await submitTaskReply(page, followups[completedTurn - 1]),+ );+ }+ warmLifecycleEvidence = { turns: turnEvidence };}const taskMatchers = execution.task.buildMatchers(nonce, execution);+ const terminalDeadlineAt =+ execution.task.flow === "warm_three_turn"+ ? Math.min(+ deadlineAt,+ turnSubmissionTimesMs.at(-1)! ++ (execution.task.turnTimeoutMs ?? 10 * 60_000),+ )+ : deadlineAt;let terminal = await pollUntil({label: `issue ${issue.id} and heartbeat run terminal state`,- deadlineAt,+ deadlineAt: terminalDeadlineAt,load: loadTaskState,accept: ({ currentIssue, taskRuns }) =>currentIssue.status === execution.task.expectedTerminalState.issue &&@@ -1289,6 +1493,45 @@ for (const execution of executions) {),);selectedRuns = sortRunsChronologically(selectedRuns);+ if (execution.task.flow === "warm_three_turn") {+ turnTimings = selectedRuns.map((candidate, index) => {+ const submittedAtMs = turnSubmissionTimesMs[index]!;+ const runStartedAtMs = candidate.startedAt+ ? Date.parse(candidate.startedAt)+ : Number.NaN;+ const runFinishedAtMs = candidate.finishedAt+ ? Date.parse(candidate.finishedAt)+ : Number.NaN;+ const acquisition = record(+ record(candidate.contextSnapshot).paperclipEnvironment,+ ).sandboxLeaseAcquisition;+ const acquisitionOutcome = record(acquisition).outcome;+ return {+ turn: index + 1,+ submittedAt: new Date(submittedAtMs).toISOString(),+ runStartedAt: candidate.startedAt ?? null,+ runFinishedAt: candidate.finishedAt ?? null,+ schedulerLatencyMs: Number.isFinite(runStartedAtMs)+ ? Math.max(0, runStartedAtMs - submittedAtMs)+ : null,+ runDurationMs:+ Number.isFinite(runStartedAtMs) &&+ Number.isFinite(runFinishedAtMs)+ ? Math.max(0, runFinishedAtMs - runStartedAtMs)+ : null,+ responseLatencyMs: Number.isFinite(runFinishedAtMs)+ ? Math.max(0, runFinishedAtMs - submittedAtMs)+ : null,+ runId: candidate.id,+ leaseAcquisitionOutcome:+ acquisitionOutcome === "created" ||+ acquisitionOutcome === "resumed" ||+ acquisitionOutcome === "replacement"+ ? acquisitionOutcome+ : "unknown",+ };+ });+ }const finalRun = selectedRuns.at(-1)!;const run =selectedRuns.find(@@ -1500,10 +1743,31 @@ for (const execution of executions) {interactions: terminal.interactions,},};+ const fileObservations = Object.fromEntries(+ await Promise.all(+ taskMatchers+ .filter(+ (matcher) =>+ matcher.kind === "file_exists" ||+ matcher.kind === "file_exact" ||+ matcher.kind === "file_contains",+ )+ .map(async (matcher) => [+ matcher.path,+ await readFile(+ path.isAbsolute(matcher.path)+ ? matcher.path+ : path.join(workspacePath, matcher.path),+ "utf8",+ ).catch(() => undefined),+ ]),+ ),+ );matcherResults = await Promise.all(taskMatchers.map((matcher) =>evaluateMatcher(matcher, {...matcherObservation,+ files: fileObservations,// Multi-run tasks intentionally retain earlier waiting/revision// replies. Exact completion text belongs to the chronological// final run, while occurrence checks still span every agent@@ -1571,6 +1835,190 @@ for (const execution of executions) {"expected a Daytona sandbox lease on the run context",);}+ if (execution.task.flow === "warm_three_turn") {+ const runEnvironmentContexts = selectedRuns.map((candidate) =>+ record(record(candidate.contextSnapshot).paperclipEnvironment),+ );+ const leaseIds = runEnvironmentContexts.map((entry) => entry.leaseId);+ const acquisitionOutcomes = runEnvironmentContexts.map(+ (entry) => record(entry.sandboxLeaseAcquisition).outcome,+ );+ const projectWorkspaceIds = selectedRuns.map(+ (candidate) =>+ record(record(candidate.contextSnapshot).paperclipWorkspace)+ .workspaceId,+ );+ const executionWorkspaceIds = selectedRuns.map(+ (candidate) => record(candidate.contextSnapshot).executionWorkspaceId,+ );+ if (+ leaseIds.some(+ (leaseId) => typeof leaseId !== "string" || leaseId.length === 0,+ )+ ) {+ invariantFailures.push(+ `expected a persisted Daytona lease row for every warm turn; observed ${JSON.stringify(leaseIds)}`,+ );+ }+ if (+ JSON.stringify(acquisitionOutcomes) !==+ JSON.stringify(["created", "resumed", "resumed"])+ ) {+ invariantFailures.push(+ `expected warm lease outcomes created,resumed,resumed; observed ${JSON.stringify(acquisitionOutcomes)}`,+ );+ }+ if (+ !fixtures.project?.primaryWorkspace?.id ||+ projectWorkspaceIds.some(+ (workspaceId) =>+ workspaceId !== fixtures!.project!.primaryWorkspace!.id,+ ) ||+ new Set(executionWorkspaceIds).size !== 1 ||+ executionWorkspaceIds[0] !== issue.executionWorkspaceId ||+ issue.projectId !== fixtures.project.id ||+ issue.projectWorkspaceId !== fixtures.project.primaryWorkspace.id ||+ !issue.executionWorkspaceId+ ) {+ invariantFailures.push(+ `warm task did not preserve project, project-workspace, and execution-workspace identity: ${JSON.stringify({ projectWorkspaceIds, executionWorkspaceIds, projectId: issue.projectId, projectWorkspaceId: issue.projectWorkspaceId, executionWorkspaceId: issue.executionWorkspaceId })}`,+ );+ }+ if (execution.profile.generation === "native") {+ const stableIdentityFields: Array<{+ label: string;+ values: unknown[];+ }> = [+ {+ label: "native session",+ values: selectedRuns.map(+ (candidate) => candidate.nativeSessionId,+ ),+ },+ {+ label: "runner instance",+ values: selectedRuns.map(+ (candidate) => candidate.runnerInstanceId,+ ),+ },+ {+ label: "provider session",+ values: selectedRuns.map((candidate) => candidate.sessionIdAfter),+ },+ {+ label: "runner pid",+ values: selectedRuns.map((candidate) => candidate.processPid),+ },+ {+ label: "runner process fingerprint",+ values: selectedRuns.map(+ (candidate) => candidate.processStartedAt,+ ),+ },+ ];+ for (const { label, values } of stableIdentityFields) {+ if (+ values.some(+ (value) =>+ value === null ||+ value === undefined ||+ String(value).length === 0,+ ) ||+ new Set(values).size !== 1+ ) {+ invariantFailures.push(+ `expected one stable ${label} across native warm turns; observed ${JSON.stringify(values)}`,+ );+ }+ }+ }+ if (+ turnTimings?.length !== 3 ||+ turnTimings.some(+ (timing) =>+ timing.runStartedAt === null ||+ timing.runFinishedAt === null ||+ timing.schedulerLatencyMs === null ||+ timing.runDurationMs === null ||+ timing.responseLatencyMs === null ||+ timing.responseLatencyMs >+ (execution.task.turnTimeoutMs ?? 10 * 60_000),+ )+ ) {+ invariantFailures.push(+ `warm turn timing data was incomplete or exceeded its structural deadline: ${JSON.stringify(turnTimings)}`,+ );+ }+ const retainedLeases = await pollUntil({+ label: `retained warm Daytona lease for issue ${issue.id}`,+ deadlineAt: Math.min(deadlineAt, Date.now() + 30_000),+ intervalMs: 500,+ load: () =>+ api.get<EnvironmentLeaseRecord[]>(+ `/api/environments/${fixtures!.environment.id}/leases`,+ ),+ accept: (leases) => {+ const warmLeases = leases.filter(+ (lease) =>+ lease.issueId === issue!.id &&+ selectedRuns.some(+ (candidate) => candidate.id === lease.heartbeatRunId,+ ),+ );+ return (+ warmLeases.length === 3 &&+ warmLeases.every(+ (lease) =>+ lease.status === "retained" &&+ lease.leasePolicy === "reuse_by_environment" &&+ typeof lease.providerLeaseId === "string" &&+ record(lease.metadata).sandboxState === "started",+ )+ );+ },+ });+ const selectedRunOrder = new Map(+ selectedRuns.map((candidate, index) => [candidate.id, index]),+ );+ const warmLeases = retainedLeases+ .filter(+ (lease) =>+ lease.issueId === issue!.id &&+ selectedRunOrder.has(lease.heartbeatRunId ?? ""),+ )+ .sort(+ (left, right) =>+ (selectedRunOrder.get(left.heartbeatRunId ?? "") ?? 0) -+ (selectedRunOrder.get(right.heartbeatRunId ?? "") ?? 0),+ );+ const providerLeaseIds = warmLeases.map(+ (lease) => lease.providerLeaseId,+ );+ const resumedFromStates = warmLeases+ .slice(1)+ .map((lease) => record(lease.metadata).resumedFromState);+ if (+ new Set(providerLeaseIds).size !== 1 ||+ typeof providerLeaseIds[0] !== "string" ||+ JSON.stringify(resumedFromStates) !==+ JSON.stringify(["started", "started"])+ ) {+ invariantFailures.push(+ `expected one continuously-started Daytona sandbox; observed ${JSON.stringify({ providerLeaseIds, resumedFromStates })}`,+ );+ }+ warmLifecycleEvidence = {+ ...(warmLifecycleEvidence ?? {}),+ leaseIds,+ acquisitionOutcomes,+ projectWorkspaceIds,+ executionWorkspaceIds,+ providerLeaseIds,+ resumedFromStates,+ retainedLeases: warmLeases,+ turnTimings,+ };+ }if (execution.environment.id === "local") {const runLogContent = String(record(runLog).content ?? "");// The log endpoint returns NDJSON, so quotes inside each `chunk` are@@ -1723,6 +2171,7 @@ for (const execution of executions) {interactions: terminal.interactions,planLifecycleEvidence,questionLifecycleEvidence,+ warmLifecycleEvidence,matcherResults,invariantFailures,runEvents,@@ -1940,6 +2389,7 @@ for (const execution of executions) {issueId: issue?.id,issueIdentifier: issue?.identifier ?? null,runIds: selectedRuns.map((run) => run.id),+ ...(turnTimings ? { turnTimings } : {}),startedAt,finishedAt: new Date(finishedAtMs).toISOString(),durationMs: finishedAtMs - startedAtMs,diff --git a/tests/runner-e2e/support.test.ts b/tests/runner-e2e/support.test.tsindex 79fbe6d3e8..e56d9923db 100644--- a/tests/runner-e2e/support.test.ts+++ b/tests/runner-e2e/support.test.ts@@ -341,6 +341,23 @@ describe("runner E2E matchers", () => {expect(result?.detail).toContain("observed 2");});+ it("matches finalized workspace files byte-for-byte", async () => {+ const [matched, extraLine] = await evaluateMatchers(+ [+ { kind: "file_exact", path: "continuity.txt", expected: "T1\nT2\n" },+ { kind: "file_exact", path: "duplicate.txt", expected: "T1\nT2\n" },+ ],+ {+ files: {+ "continuity.txt": "T1\nT2\n",+ "duplicate.txt": "T1\nT2\nT2\n",+ },+ },+ );+ expect(matched?.passed).toBe(true);+ expect(extraLine?.passed).toBe(false);+ });+it("normalizes ordered fragments and evaluates nested JSON Schema", async () => {const results = await evaluateMatchers([diff --git a/tests/runner-e2e/types.ts b/tests/runner-e2e/types.tsindex d607e60893..fcb5093f7b 100644--- a/tests/runner-e2e/types.ts+++ b/tests/runner-e2e/types.ts@@ -13,7 +13,8 @@ export type RunnerTaskFlow =| "single_turn"| "plan_revision_acceptance"| "question_resume_completion"- | "plan_approval_completion";+ | "plan_approval_completion"+ | "warm_three_turn";export interface SecretReference {type: "secret_ref";@@ -73,6 +74,8 @@ export interface RunnerProfileFixture {export interface EnvironmentFixture {id: RunnerEnvironmentId;+ /** Distinguishes materially different configurations that share a provider ID. */+ configurationKey?: string;label: string;groups: readonly string[];driver: "local" | "sandbox";@@ -103,6 +106,7 @@ export type Matcher =| { kind: "runtime_mode"; expected: RunnerGeneration }| { kind: "environment"; expected: RunnerEnvironmentId }| { kind: "file_exists"; path: string }+ | { kind: "file_exact"; path: string; expected: string }| { kind: "file_contains"; path: string; expected: string }| { kind: "artifact_exists"; name: string; mimeType?: string }| { kind: "json_path"; path: string; expected: unknown }@@ -124,6 +128,8 @@ export interface RunnerTaskFixture {buildPrompt(nonce: string): string;buildVisibleMarker(nonce: string): string;buildRevisionRequest?(nonce: string): string;+ buildFollowupMessages?(nonce: string): readonly [string, string];+ turnTimeoutMs?: number;buildQuestionAnswer?(nonce: string): {optionLabel: string;expectedMarker: string;@@ -260,6 +266,17 @@ export interface RunnerE2EResult {issueId?: string;issueIdentifier?: string | null;runIds?: string[];+ turnTimings?: Array<{+ turn: number;+ submittedAt: string;+ runStartedAt: string | null;+ runFinishedAt: string | null;+ schedulerLatencyMs: number | null;+ runDurationMs: number | null;+ responseLatencyMs: number | null;+ runId: string;+ leaseAcquisitionOutcome: "created" | "resumed" | "replacement" | "unknown";+ }>;startedAt: string;finishedAt: string;durationMs: number;diff --git a/tests/runner-e2e/workflow-security.test.ts b/tests/runner-e2e/workflow-security.test.tsindex 55997ce9bb..b875add0d3 100644--- a/tests/runner-e2e/workflow-security.test.ts+++ b/tests/runner-e2e/workflow-security.test.ts@@ -4,7 +4,7 @@ import { describe, expect, it } from "vitest";const repositoryRoot = path.resolve(import.meta.dirname, "../..");const ordinaryPrTrustedWorkflowRevision =- "a0a78ee60946a5f79f85b2bd0584fc766fae43bb";+ "03609aa6ecc9a047ed53d6b6469d8be554fbc46d";const fullStackTestNeeds =/needs:\s*\[\s*authorize,\s*target_lock,\s*catalog,\s*daytona_image,\s*build_runner_artifacts,\s*build_remote_provider_pack,?\s*\]/u;const buildRunnerNeeds =