Skip to content

The open-source app everyone uses to manage agents at work

fix(runner): persist warm Daytona workspaces (#12901)

Dotta1dceee9a4e75
diff --git a/.github/workflows/runner-full-stack-e2e.yml b/.github/workflows/runner-full-stack-e2e.yml
index 0c4fa59387..3179f5c8e1 100644
--- a/.github/workflows/runner-full-stack-e2e.yml
+++ b/.github/workflows/runner-full-stack-e2e.yml
@@ -14,7 +14,7 @@ on:
type: boolean
default: true
group:
- description: "Comma-separated groups (AND semantics: legacy,native,local,daytona,core,breadth)"
+ description: "Comma-separated groups (AND semantics: legacy,native,local,daytona,warm,core,breadth)"
type: string
required: false
suite:
diff --git a/packages/adapter-utils/src/command-managed-runtime.ts b/packages/adapter-utils/src/command-managed-runtime.ts
index 36500fdc70..cdbae889cd 100644
--- a/packages/adapter-utils/src/command-managed-runtime.ts
+++ b/packages/adapter-utils/src/command-managed-runtime.ts
@@ -12,11 +12,15 @@ import {
type SandboxRemoteExecutionSpec,
type SandboxSyncOperation,
type SandboxSyncResult,
+ type WorkspaceDurableSeedPaths,
+ type WorkspaceInboundMode,
} from "./sandbox-managed-runtime.js";
import { preferredShellForSandbox, shellCommandArgs } from "./sandbox-shell.js";
import type { RunProcessResult } from "./server-utils.js";
import type { RuntimeProgressSink, RuntimeStatusSink } from "./runtime-progress.js";
import type { RuntimeSpanRunner } from "./acpx-engine/startup-timing.js";
+import type { GitWorkspaceSnapshot } from "./git-workspace-sync.js";
+import type { DirectorySnapshot } from "./workspace-restore-merge.js";
/**
* Input for a duplex channel open. The caller supplies only the command argument
@@ -516,6 +520,10 @@ export async function prepareCommandManagedRuntime(input: {
workspaceLocalDir: string;
workspaceRemoteDir?: string;
syncWorkspace?: boolean;
+ workspaceInboundMode?: WorkspaceInboundMode;
+ workspaceDurableSeed?: WorkspaceDurableSeedPaths;
+ workspaceBaseline?: DirectorySnapshot;
+ workspaceGitSnapshot?: GitWorkspaceSnapshot | null;
workspaceExclude?: string[];
preserveAbsentOnRestore?: string[];
assets?: CommandManagedRuntimeAsset[];
@@ -577,6 +585,10 @@ export async function prepareCommandManagedRuntime(input: {
workspaceLocalDir: input.workspaceLocalDir,
workspaceRemoteDir,
syncWorkspace: input.syncWorkspace,
+ workspaceInboundMode: input.workspaceInboundMode,
+ workspaceDurableSeed: input.workspaceDurableSeed,
+ workspaceBaseline: input.workspaceBaseline,
+ workspaceGitSnapshot: input.workspaceGitSnapshot,
workspaceExclude: mergeRuntimeExcludes(input.workspaceExclude),
preserveAbsentOnRestore: input.preserveAbsentOnRestore,
assets: input.assets,
@@ -616,6 +628,10 @@ export async function prepareCommandManagedRuntime(input: {
workspaceLocalDir: input.workspaceLocalDir,
workspaceRemoteDir,
syncWorkspace: input.syncWorkspace,
+ workspaceInboundMode: input.workspaceInboundMode,
+ workspaceDurableSeed: input.workspaceDurableSeed,
+ workspaceBaseline: input.workspaceBaseline,
+ workspaceGitSnapshot: input.workspaceGitSnapshot,
workspaceExclude: mergeRuntimeExcludes(input.workspaceExclude),
preserveAbsentOnRestore: input.preserveAbsentOnRestore,
assets: input.assets,
diff --git a/packages/adapter-utils/src/execution-target.ts b/packages/adapter-utils/src/execution-target.ts
index db0fed0d72..9ea8730cb2 100644
--- a/packages/adapter-utils/src/execution-target.ts
+++ b/packages/adapter-utils/src/execution-target.ts
@@ -18,10 +18,12 @@ import {
import type {
AdditionalSourceStagingFailure,
SandboxAdditionalSource,
+ WorkspaceDurableSeedPaths,
+ WorkspaceInboundMode,
} from "./sandbox-managed-runtime.js";
-export {
- resolveReferencedSourceIgnore,
-} from "./sandbox-managed-runtime.js";
+import type { GitWorkspaceSnapshot } from "./git-workspace-sync.js";
+import type { DirectorySnapshot } from "./workspace-restore-merge.js";
+export { resolveReferencedSourceIgnore } from "./sandbox-managed-runtime.js";
export type {
AdditionalSourceStagingFailure,
ReferencedSourceIgnoreResolution,
@@ -249,6 +251,10 @@ export interface PreparedAdapterExecutionTargetRuntime {
* stage referenced projects, or when every requested project staged.
*/
additionalSourceFailures: AdditionalSourceStagingFailure[];
+ workspaceSyncSnapshot: {
+ baseline: DirectorySnapshot;
+ gitSnapshot: GitWorkspaceSnapshot | null;
+ } | null;
restoreWorkspace(onProgress?: RuntimeProgressSink): Promise<void>;
}
@@ -1374,6 +1380,10 @@ export async function prepareAdapterExecutionTargetRuntime(input: {
timeoutSec?: number;
workspaceRemoteDir?: string;
syncWorkspace?: boolean;
+ workspaceInboundMode?: WorkspaceInboundMode;
+ workspaceDurableSeed?: WorkspaceDurableSeedPaths;
+ workspaceBaseline?: DirectorySnapshot;
+ workspaceGitSnapshot?: GitWorkspaceSnapshot | null;
workspaceExclude?: string[];
preserveAbsentOnRestore?: string[];
assets?: AdapterManagedRuntimeAsset[];
@@ -1403,6 +1413,7 @@ export async function prepareAdapterExecutionTargetRuntime(input: {
assetDirs: {},
additionalSourceDirs: {},
additionalSourceFailures: [],
+ workspaceSyncSnapshot: null,
restoreWorkspace: async () => {},
};
}
@@ -1428,6 +1439,7 @@ export async function prepareAdapterExecutionTargetRuntime(input: {
// The SSH transport does not stage referenced projects (it is out of scope), so it never
// reports a per-project staging failure.
additionalSourceFailures: [],
+ workspaceSyncSnapshot: null,
restoreWorkspace: prepared.restoreWorkspace,
};
}
@@ -1448,6 +1460,10 @@ export async function prepareAdapterExecutionTargetRuntime(input: {
workspaceLocalDir: input.workspaceLocalDir,
workspaceRemoteDir: input.workspaceRemoteDir,
syncWorkspace: input.syncWorkspace,
+ workspaceInboundMode: input.workspaceInboundMode,
+ workspaceDurableSeed: input.workspaceDurableSeed,
+ workspaceBaseline: input.workspaceBaseline,
+ workspaceGitSnapshot: input.workspaceGitSnapshot,
workspaceExclude: input.workspaceExclude,
preserveAbsentOnRestore: input.preserveAbsentOnRestore,
assets: input.assets,
@@ -1465,6 +1481,7 @@ export async function prepareAdapterExecutionTargetRuntime(input: {
assetDirs: prepared.assetDirs,
additionalSourceDirs: prepared.additionalSourceDirs,
additionalSourceFailures: prepared.additionalSourceFailures,
+ workspaceSyncSnapshot: prepared.workspaceSyncSnapshot,
restoreWorkspace: prepared.restoreWorkspace,
};
}
diff --git a/packages/adapter-utils/src/sandbox-managed-runtime.test.ts b/packages/adapter-utils/src/sandbox-managed-runtime.test.ts
index be36585a1e..387ac215df 100644
--- a/packages/adapter-utils/src/sandbox-managed-runtime.test.ts
+++ b/packages/adapter-utils/src/sandbox-managed-runtime.test.ts
@@ -348,6 +348,129 @@ describe("sandbox managed runtime", () => {
}
});
+ it("adopts a warm remote workspace without inbound overwrite and still merges outbound changes", async () => {
+ const rootDir = await mkdtemp(
+ path.join(os.tmpdir(), "paperclip-sandbox-adopt-"),
+ );
+ cleanupDirs.push(rootDir);
+ const localWorkspaceDir = path.join(rootDir, "local-workspace");
+ const remoteWorkspaceDir = path.join(rootDir, "remote-workspace");
+ await mkdir(localWorkspaceDir, { recursive: true });
+ await mkdir(remoteWorkspaceDir, { recursive: true });
+ await writeFile(
+ path.join(localWorkspaceDir, "continuity.txt"),
+ "host baseline\n",
+ "utf8",
+ );
+ await writeFile(
+ path.join(remoteWorkspaceDir, "continuity.txt"),
+ "remote retained\n",
+ "utf8",
+ );
+ const client = makeFilesystemClient();
+ const syncIn = vi.spyOn(client, "syncIn");
+
+ const prepared = await prepareSandboxManagedRuntime({
+ spec: {
+ transport: "sandbox",
+ provider: "test",
+ sandboxId: "sandbox-warm",
+ remoteCwd: remoteWorkspaceDir,
+ timeoutMs: 30_000,
+ apiKey: null,
+ },
+ adapterKey: "test-adapter",
+ client,
+ workspaceLocalDir: localWorkspaceDir,
+ workspaceInboundMode: "adopt_remote",
+ });
+
+ expect(syncIn).not.toHaveBeenCalled();
+ await expect(
+ readFile(path.join(remoteWorkspaceDir, "continuity.txt"), "utf8"),
+ ).resolves.toBe("remote retained\n");
+ expect(prepared.workspaceSyncSnapshot).not.toBeNull();
+
+ await writeFile(
+ path.join(remoteWorkspaceDir, "continuity.txt"),
+ "remote finalized\n",
+ "utf8",
+ );
+ await prepared.restoreWorkspace();
+ await expect(
+ readFile(path.join(localWorkspaceDir, "continuity.txt"), "utf8"),
+ ).resolves.toBe("remote finalized\n");
+ });
+
+ it("reconstructs a replacement workspace from the exact durable pre-turn seed", async () => {
+ const rootDir = await mkdtemp(
+ path.join(os.tmpdir(), "paperclip-sandbox-durable-seed-"),
+ );
+ cleanupDirs.push(rootDir);
+ const localWorkspaceDir = path.join(rootDir, "local-workspace");
+ const firstRemoteDir = path.join(rootDir, "first-remote");
+ const replacementRemoteDir = path.join(rootDir, "replacement-remote");
+ const durableSeed = {
+ workspaceArchivePath: path.join(rootDir, "state", "workspace.tar"),
+ gitArchivePath: path.join(rootDir, "state", "git.tar"),
+ };
+ await mkdir(localWorkspaceDir, { recursive: true });
+ await writeFile(
+ path.join(localWorkspaceDir, "continuity.txt"),
+ "durable pre-turn bytes\n",
+ "utf8",
+ );
+
+ const first = await prepareSandboxManagedRuntime({
+ spec: {
+ transport: "sandbox",
+ provider: "test",
+ sandboxId: "sandbox-first",
+ remoteCwd: firstRemoteDir,
+ timeoutMs: 30_000,
+ apiKey: null,
+ },
+ adapterKey: "test-adapter",
+ client: makeFilesystemClient(),
+ workspaceLocalDir: localWorkspaceDir,
+ workspaceInboundMode: "host_current",
+ workspaceDurableSeed: durableSeed,
+ });
+ expect(first.workspaceSyncSnapshot).not.toBeNull();
+ await expect(stat(durableSeed.workspaceArchivePath)).resolves.toMatchObject(
+ {
+ mode: expect.any(Number),
+ },
+ );
+
+ await writeFile(
+ path.join(localWorkspaceDir, "continuity.txt"),
+ "concurrent host edit must not enter replacement\n",
+ "utf8",
+ );
+ await prepareSandboxManagedRuntime({
+ spec: {
+ transport: "sandbox",
+ provider: "test",
+ sandboxId: "sandbox-replacement",
+ remoteCwd: replacementRemoteDir,
+ timeoutMs: 30_000,
+ apiKey: null,
+ },
+ adapterKey: "test-adapter",
+ client: makeFilesystemClient(),
+ workspaceLocalDir: localWorkspaceDir,
+ workspaceInboundMode: "durable_seed",
+ workspaceDurableSeed: durableSeed,
+ workspaceBaseline: first.workspaceSyncSnapshot!.baseline,
+ workspaceGitSnapshot: first.workspaceSyncSnapshot!.gitSnapshot,
+ });
+
+ await expect(
+ readFile(path.join(replacementRemoteDir, "continuity.txt"), "utf8"),
+ ).resolves.toBe("durable pre-turn bytes\n");
+ });
+
it("preserves excluded local workspace artifacts during restore mirroring", async () => {
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-sandbox-restore-"));
cleanupDirs.push(rootDir);
diff --git a/packages/adapter-utils/src/sandbox-managed-runtime.ts b/packages/adapter-utils/src/sandbox-managed-runtime.ts
index d3de812500..937c812ebf 100644
--- a/packages/adapter-utils/src/sandbox-managed-runtime.ts
+++ b/packages/adapter-utils/src/sandbox-managed-runtime.ts
@@ -1,6 +1,10 @@
import { execFile as execFileCallback } from "node:child_process";
-import { randomUUID } from "node:crypto";
-import { constants as fsConstants, promises as fs } from "node:fs";
+import { createHash, randomUUID } from "node:crypto";
+import {
+ constants as fsConstants,
+ createReadStream,
+ promises as fs,
+} from "node:fs";
import os from "node:os";
import path from "node:path";
import { promisify } from "node:util";
@@ -12,6 +16,7 @@ import {
deleteLocalGitRef,
fetchGitBundleIntoLocalRef,
GIT_ARCHIVE_EXCLUDES,
+ type GitWorkspaceSnapshot,
integrateImportedGitHead,
readGitWorkspaceSnapshot,
ReferencedSourceIgnoreScanLimitExceededError,
@@ -20,7 +25,11 @@ import {
withShallowGitWorkspaceClone,
WORKSPACE_GIT_SCAN_SATURATED_CODE,
} from "./git-workspace-sync.js";
-import { captureDirectorySnapshot, mergeDirectoryWithBaseline } from "./workspace-restore-merge.js";
+import {
+ captureDirectorySnapshot,
+ mergeDirectoryWithBaseline,
+ type DirectorySnapshot,
+} from "./workspace-restore-merge.js";
import {
createRuntimeProgressReporter,
type RuntimeProgressDirection,
@@ -553,9 +562,29 @@ export interface PreparedSandboxManagedRuntime {
* staged, or when no additional sources were requested.
*/
additionalSourceFailures: AdditionalSourceStagingFailure[];
+ /** Durable merge inputs used to resume an outbound restore after host restart. */
+ workspaceSyncSnapshot: {
+ baseline: DirectorySnapshot;
+ gitSnapshot: GitWorkspaceSnapshot | null;
+ } | null;
restoreWorkspace(onProgress?: RuntimeProgressSink): Promise<void>;
}
+export type WorkspaceInboundMode =
+ "host_current" | "durable_seed" | "adopt_remote";
+
+/**
+ * Controller-owned archives for replaying the exact pre-turn workspace into a
+ * replacement sandbox. Paths are never sent to the provider as credentials or
+ * persisted in database metadata.
+ */
+export interface WorkspaceDurableSeedPaths {
+ workspaceArchivePath: string;
+ workspaceArchiveSha256?: string;
+ gitArchivePath?: string | null;
+ gitArchiveSha256?: string | null;
+}
+
/** One additional (referenced) project that failed to stage into the sandbox. */
export interface AdditionalSourceStagingFailure {
projectId: string;
@@ -719,6 +748,57 @@ async function withTempDir<T>(prefix: string, fn: (dir: string) => Promise<T>):
}
}
+async function sha256File(filePath: string): Promise<string> {
+ return await new Promise((resolveDigest, rejectDigest) => {
+ const digest = createHash("sha256");
+ const stream = createReadStream(filePath);
+ stream.on("data", (chunk) => digest.update(chunk));
+ stream.on("error", rejectDigest);
+ stream.on("end", () => resolveDigest(digest.digest("hex")));
+ });
+}
+
+async function copyDurableSeedArchive(input: {
+ sourcePath: string;
+ targetPath: string;
+ expectedSha256?: string | null;
+}): Promise<void> {
+ const source = await fs.lstat(input.sourcePath);
+ if (source.isSymbolicLink() || !source.isFile()) {
+ throw new Error("workspace_durable_seed_invalid");
+ }
+ if (
+ input.expectedSha256 &&
+ (await sha256File(input.sourcePath)) !== input.expectedSha256
+ ) {
+ throw new Error("workspace_durable_seed_digest_mismatch");
+ }
+ await fs.copyFile(input.sourcePath, input.targetPath);
+}
+
+async function persistDurableSeedArchive(input: {
+ sourcePath: string;
+ targetPath: string;
+}): Promise<void> {
+ const parent = path.dirname(input.targetPath);
+ await fs.mkdir(parent, { recursive: true, mode: 0o700 });
+ const parentStat = await fs.lstat(parent);
+ if (parentStat.isSymbolicLink() || !parentStat.isDirectory()) {
+ throw new Error("workspace_durable_seed_root_invalid");
+ }
+ const temporary = path.join(
+ parent,
+ `.${path.basename(input.targetPath)}.${randomUUID()}.tmp`,
+ );
+ try {
+ await fs.copyFile(input.sourcePath, temporary);
+ await fs.chmod(temporary, 0o600);
+ await fs.rename(temporary, input.targetPath);
+ } finally {
+ await fs.rm(temporary, { force: true }).catch(() => undefined);
+ }
+}
+
async function execTar(args: string[]): Promise<void> {
await execFile("tar", args, {
env: {
@@ -976,6 +1056,12 @@ export async function prepareSandboxManagedRuntime(input: {
workspaceLocalDir: string;
workspaceRemoteDir?: string;
syncWorkspace?: boolean;
+ /** Selects authoritative host staging, exact durable-seed replay, or no-overwrite adoption. */
+ workspaceInboundMode?: WorkspaceInboundMode;
+ workspaceDurableSeed?: WorkspaceDurableSeedPaths;
+ /** Durable snapshots supplied when reconstructing an interrupted restore. */
+ workspaceBaseline?: DirectorySnapshot;
+ workspaceGitSnapshot?: GitWorkspaceSnapshot | null;
workspaceExclude?: string[];
preserveAbsentOnRestore?: string[];
assets?: SandboxManagedRuntimeAsset[];
@@ -999,6 +1085,16 @@ export async function prepareSandboxManagedRuntime(input: {
const workspaceRemoteDir = input.workspaceRemoteDir ?? input.spec.remoteCwd;
const runtimeRootDir = path.posix.join(workspaceRemoteDir, ".paperclip-runtime", input.adapterKey);
const syncWorkspace = input.syncWorkspace !== false;
+ const workspaceInboundMode = input.workspaceInboundMode ?? "host_current";
+ const stageWorkspace =
+ syncWorkspace && workspaceInboundMode !== "adopt_remote";
+ const prepareWorkspaceSeed =
+ syncWorkspace &&
+ workspaceInboundMode === "adopt_remote" &&
+ input.workspaceDurableSeed !== undefined;
+ if (workspaceInboundMode === "durable_seed" && !input.workspaceDurableSeed) {
+ throw new Error("workspace_durable_seed_missing");
+ }
// Reject any unsafe asset key before an archive path or an asset directory is
// built from it. This runs before the git snapshot work so a bad key fails fast.
@@ -1025,7 +1121,11 @@ export async function prepareSandboxManagedRuntime(input: {
// It reads git's own bookkeeping to decide what to include/exclude, so it is
// usually fast, but on a large working tree the `--ignored` walk is not free.
const gitSnapshot = syncWorkspace
- ? await runStepSpan("snapshot.git", () => readGitWorkspaceSnapshot(input.workspaceLocalDir))
+ ? input.workspaceGitSnapshot !== undefined
+ ? input.workspaceGitSnapshot
+ : await runStepSpan("snapshot.git", () =>
+ readGitWorkspaceSnapshot(input.workspaceLocalDir),
+ )
: null;
const gitIgnoredExcludes = gitSnapshot?.ignoredPaths;
const workspaceArchiveExclude = mergeExcludes(
@@ -1047,9 +1147,12 @@ export async function prepareSandboxManagedRuntime(input: {
// dominant cost in the pre-`pack` window — it reads the content of every
// non-excluded file, serially — so it earns its own span.
const baselineSnapshot = syncWorkspace
- ? await runStepSpan("snapshot.baseline", () =>
- captureDirectorySnapshot(input.workspaceLocalDir, { exclude: restoreExclude }),
- )
+ ? (input.workspaceBaseline ??
+ (await runStepSpan("snapshot.baseline", () =>
+ captureDirectorySnapshot(input.workspaceLocalDir, {
+ exclude: restoreExclude,
+ }),
+ )))
: null;
// Every inbound staging step delegates to the provider through `client.syncIn`:
@@ -1172,7 +1275,7 @@ export async function prepareSandboxManagedRuntime(input: {
// records its own failure and never rejects.
const inboundTaskIsRequired: boolean[] = [];
- if (syncWorkspace) {
+ if (stageWorkspace || prepareWorkspaceSeed) {
inboundTaskIsRequired.push(true);
inboundTasks.push(() =>
runStepSpan("stage.workspace", async () => {
@@ -1203,18 +1306,49 @@ export async function prepareSandboxManagedRuntime(input: {
if (gitSnapshot) {
await emitRuntimeStatus(input.onRuntimeProgress, "git_sync", "Syncing git history to environment");
const gitTarPath = path.join(tempDir, "git-workspace.tar");
- const remoteGitTar = path.posix.join(runtimeRootDir, "git-workspace-upload.tar");
- await withShallowGitWorkspaceClone({
- localDir: input.workspaceLocalDir,
- snapshot: gitSnapshot,
- }, async (cloneDir) => {
- await createTarballFromDirectory({
- localDir: cloneDir,
- archivePath: gitTarPath,
- exclude: [".paperclip-runtime"],
+ const remoteGitTar = path.posix.join(
+ runtimeRootDir,
+ "git-workspace-upload.tar",
+ );
+ if (workspaceInboundMode === "durable_seed") {
+ const durableGitArchive =
+ input.workspaceDurableSeed?.gitArchivePath;
+ if (!durableGitArchive) {
+ throw new Error("workspace_durable_seed_git_missing");
+ }
+ await copyDurableSeedArchive({
+ sourcePath: durableGitArchive,
+ targetPath: gitTarPath,
+ expectedSha256: input.workspaceDurableSeed?.gitArchiveSha256,
});
+ } else {
+ await withShallowGitWorkspaceClone(
+ {
+ localDir: input.workspaceLocalDir,
+ snapshot: gitSnapshot,
+ },
+ async (cloneDir) => {
+ await createTarballFromDirectory({
+ localDir: cloneDir,
+ archivePath: gitTarPath,
+ exclude: [".paperclip-runtime"],
+ });
+ },
+ );
+ if (input.workspaceDurableSeed?.gitArchivePath) {
+ await persistDurableSeedArchive({
+ sourcePath: gitTarPath,
+ targetPath: input.workspaceDurableSeed.gitArchivePath,
+ });
+ }
+ }
+ workspaceFiles.push({
+ sourcePath: gitTarPath,
+ targetPath: remoteGitTar,
+ kind: "file",
+ access: "rw",
+ writablePath: workspaceRemoteDir,
});
- workspaceFiles.push({ sourcePath: gitTarPath, targetPath: remoteGitTar, kind: "file", access: "rw", writablePath: workspaceRemoteDir });
workspacePostUploadCommands.push({
command: buildWorkspaceTarExtractCommand({
workspaceRemoteDir,
@@ -1230,22 +1364,48 @@ export async function prepareSandboxManagedRuntime(input: {
// the preserved names first. The extract runs AFTER the git extract.
await emitRuntimeStatus(input.onRuntimeProgress, "config_sync", "Syncing workspace to environment");
const workspaceTarPath = path.join(tempDir, "workspace.tar");
- const workspaceArchiveDir = gitSnapshot ? path.join(tempDir, "workspace-overlay") : input.workspaceLocalDir;
- if (gitSnapshot) {
- await copySelectedWorkspaceEntries({
- sourceDir: input.workspaceLocalDir,
- targetDir: workspaceArchiveDir,
- relativePaths: gitSnapshot.overlayPaths,
- exclude: workspaceArchiveExclude,
+ if (workspaceInboundMode === "durable_seed") {
+ await copyDurableSeedArchive({
+ sourcePath: input.workspaceDurableSeed!.workspaceArchivePath,
+ targetPath: workspaceTarPath,
+ expectedSha256:
+ input.workspaceDurableSeed!.workspaceArchiveSha256,
+ });
+ } else {
+ const workspaceArchiveDir = gitSnapshot
+ ? path.join(tempDir, "workspace-overlay")
+ : input.workspaceLocalDir;
+ if (gitSnapshot) {
+ await copySelectedWorkspaceEntries({
+ sourceDir: input.workspaceLocalDir,
+ targetDir: workspaceArchiveDir,
+ relativePaths: gitSnapshot.overlayPaths,
+ exclude: workspaceArchiveExclude,
+ });
+ }
+ await createTarballFromDirectory({
+ localDir: workspaceArchiveDir,
+ archivePath: workspaceTarPath,
+ exclude: gitSnapshot ? undefined : workspaceArchiveExclude,
});
+ if (input.workspaceDurableSeed) {
+ await persistDurableSeedArchive({
+ sourcePath: workspaceTarPath,
+ targetPath: input.workspaceDurableSeed.workspaceArchivePath,
+ });
+ }
}
- await createTarballFromDirectory({
- localDir: workspaceArchiveDir,
- archivePath: workspaceTarPath,
- exclude: gitSnapshot ? undefined : workspaceArchiveExclude,
+ const remoteWorkspaceTar = path.posix.join(
+ runtimeRootDir,
+ "workspace-upload.tar",
+ );
+ workspaceFiles.push({
+ sourcePath: workspaceTarPath,
+ targetPath: remoteWorkspaceTar,
+ kind: "file",
+ access: "rw",
+ writablePath: workspaceRemoteDir,
});
- const remoteWorkspaceTar = path.posix.join(runtimeRootDir, "workspace-upload.tar");
- workspaceFiles.push({ sourcePath: workspaceTarPath, targetPath: remoteWorkspaceTar, kind: "file", access: "rw", writablePath: workspaceRemoteDir });
workspacePostUploadCommands.push({
command: buildWorkspaceTarExtractCommand({
workspaceRemoteDir,
@@ -1265,6 +1425,8 @@ export async function prepareSandboxManagedRuntime(input: {
workspaceUploadBytes += (await fs.stat(workspaceTarPath)).size;
});
+ if (!stageWorkspace) return;
+
// One confined `syncIn` for the whole merged workspace file set. The confine
// guard covers every mapping BEFORE any bytes upload (fail-closed): a source
// or target escape in EITHER tar mapping stops the upload of both.
@@ -1457,6 +1619,10 @@ export async function prepareSandboxManagedRuntime(input: {
assetDirs,
additionalSourceDirs,
additionalSourceFailures,
+ workspaceSyncSnapshot:
+ syncWorkspace && baselineSnapshot
+ ? { baseline: baselineSnapshot, gitSnapshot }
+ : null,
restoreWorkspace: async (onProgress?: RuntimeProgressSink) => {
const restoreSink = onProgress ?? input.onProgress;
diff --git a/packages/adapter-utils/src/workspace-restore-merge.test.ts b/packages/adapter-utils/src/workspace-restore-merge.test.ts
index d3a06c08bc..516c6dcc81 100644
--- a/packages/adapter-utils/src/workspace-restore-merge.test.ts
+++ b/packages/adapter-utils/src/workspace-restore-merge.test.ts
@@ -9,9 +9,12 @@ import { afterEach, describe, expect, it, vi } from "vitest";
import { resolvePaperclipInstanceRootForAdapter } from "./server-utils.js";
import {
captureDirectorySnapshot,
+ directorySnapshotSha256,
classifyWorkspaceRestoreFailure,
describeWorkspaceRestoreFailure,
mergeDirectoryWithBaseline,
+ parseDirectorySnapshot,
+ serializeDirectorySnapshot,
withDirectoryMergeLock,
WORKSPACE_RESTORE_LOCK_TIMEOUT_CODE,
} from "./workspace-restore-merge.js";
@@ -27,6 +30,36 @@ describe("workspace restore merge", () => {
}
});
+ it("round-trips a deterministic durable snapshot and rejects traversal", async () => {
+ const rootDir = await mkdtemp(
+ path.join(os.tmpdir(), "paperclip-snapshot-"),
+ );
+ cleanupDirs.push(rootDir);
+ await mkdir(path.join(rootDir, "nested"), { recursive: true });
+ await writeFile(path.join(rootDir, "b.txt"), "bravo\n", "utf8");
+ await writeFile(path.join(rootDir, "nested", "a.txt"), "alpha\n", "utf8");
+
+ const snapshot = await captureDirectorySnapshot(rootDir, { exclude: [] });
+ const serialized = serializeDirectorySnapshot(snapshot);
+ const restored = parseDirectorySnapshot(serialized);
+
+ expect(serialized.entries.map(([relativePath]) => relativePath)).toEqual([
+ "b.txt",
+ "nested",
+ "nested/a.txt",
+ ]);
+ expect(restored).not.toBeNull();
+ expect(directorySnapshotSha256(restored!)).toBe(
+ directorySnapshotSha256(snapshot),
+ );
+ expect(
+ parseDirectorySnapshot({
+ ...serialized,
+ entries: [["../escape", serialized.entries[0]![1]]],
+ }),
+ ).toBeNull();
+ });
+
it("preserves sibling files when sequential stale-baseline restores create the same nested directory tree", async () => {
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-restore-merge-"));
cleanupDirs.push(rootDir);
diff --git a/packages/adapter-utils/src/workspace-restore-merge.ts b/packages/adapter-utils/src/workspace-restore-merge.ts
index 8d59cd06db..92992e684c 100644
--- a/packages/adapter-utils/src/workspace-restore-merge.ts
+++ b/packages/adapter-utils/src/workspace-restore-merge.ts
@@ -5,7 +5,7 @@ import path from "node:path";
import { shouldExcludePath } from "./exclude-patterns.js";
import { resolvePaperclipInstanceRootForAdapter } from "./server-utils.js";
-type SnapshotEntry =
+export type SnapshotEntry =
| { kind: "dir" }
| { kind: "file"; mode: number; hash: string }
| { kind: "symlink"; target: string };
@@ -15,6 +15,87 @@ export interface DirectorySnapshot {
entries: Map<string, SnapshotEntry>;
}
+export interface SerializedDirectorySnapshot {
+ version: 1;
+ exclude: string[];
+ entries: Array<[string, SnapshotEntry]>;
+}
+
+function isSafeSnapshotRelativePath(value: string): boolean {
+ if (!value || path.posix.isAbsolute(value) || path.win32.isAbsolute(value)) {
+ return false;
+ }
+ return !value.split(/[\\/]/).some((segment) => segment === "..");
+}
+
+function parseSnapshotEntry(value: unknown): SnapshotEntry | null {
+ if (!value || typeof value !== "object" || Array.isArray(value)) return null;
+ const candidate = value as Record<string, unknown>;
+ if (candidate.kind === "dir") return { kind: "dir" };
+ if (candidate.kind === "symlink" && typeof candidate.target === "string") {
+ return { kind: "symlink", target: candidate.target };
+ }
+ if (
+ candidate.kind === "file" &&
+ typeof candidate.mode === "number" &&
+ Number.isInteger(candidate.mode) &&
+ candidate.mode >= 0 &&
+ typeof candidate.hash === "string" &&
+ /^[0-9a-f]{64}$/.test(candidate.hash)
+ ) {
+ return { kind: "file", mode: candidate.mode, hash: candidate.hash };
+ }
+ return null;
+}
+
+export function serializeDirectorySnapshot(
+ snapshot: DirectorySnapshot,
+): SerializedDirectorySnapshot {
+ return {
+ version: 1,
+ exclude: [...snapshot.exclude],
+ entries: [...snapshot.entries.entries()].sort(([left], [right]) =>
+ left.localeCompare(right),
+ ),
+ };
+}
+
+export function parseDirectorySnapshot(
+ value: unknown,
+): DirectorySnapshot | null {
+ if (!value || typeof value !== "object" || Array.isArray(value)) return null;
+ const candidate = value as Record<string, unknown>;
+ if (
+ candidate.version !== 1 ||
+ !Array.isArray(candidate.exclude) ||
+ !candidate.exclude.every((entry) => typeof entry === "string") ||
+ !Array.isArray(candidate.entries)
+ ) {
+ return null;
+ }
+ const entries = new Map<string, SnapshotEntry>();
+ for (const rawEntry of candidate.entries) {
+ if (!Array.isArray(rawEntry) || rawEntry.length !== 2) return null;
+ const [relative, rawSnapshotEntry] = rawEntry;
+ if (typeof relative !== "string" || !isSafeSnapshotRelativePath(relative)) {
+ return null;
+ }
+ const entry = parseSnapshotEntry(rawSnapshotEntry);
+ if (!entry || entries.has(relative)) return null;
+ entries.set(relative, entry);
+ }
+ return {
+ exclude: [...new Set(candidate.exclude as string[])],
+ entries,
+ };
+}
+
+export function directorySnapshotSha256(snapshot: DirectorySnapshot): string {
+ return createHash("sha256")
+ .update(JSON.stringify(serializeDirectorySnapshot(snapshot)))
+ .digest("hex");
+}
+
async function hashFile(filePath: string): Promise<string> {
return await new Promise((resolve, reject) => {
const hash = createHash("sha256");
diff --git a/packages/adapters/codex-local/src/server/execute.test.ts b/packages/adapters/codex-local/src/server/execute.test.ts
index a49073d8b0..375687c89f 100644
--- a/packages/adapters/codex-local/src/server/execute.test.ts
+++ b/packages/adapters/codex-local/src/server/execute.test.ts
@@ -117,11 +117,10 @@ describe("codex execute — outbound auth copy-back restore contribution", () =>
);
}
- async function runTeardown(input: {
- sandboxAuth: string;
- hostAuth: string;
- }): Promise<{ finalHostAuth: string; finalHostMode: number }> {
- const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-codex-copyback-e2e-"));
+ async function runTeardown(input: { sandboxAuth: string; hostAuth: string }) {
+ const rootDir = await mkdtemp(
+ path.join(os.tmpdir(), "paperclip-codex-copyback-e2e-"),
+ );
cleanupDirs.push(rootDir);
const workspaceDir = path.join(rootDir, "workspace");
// The shared host home is what `resolveSharedCodexHomeDir` returns
@@ -137,7 +136,7 @@ describe("codex execute — outbound auth copy-back restore contribution", () =>
process.env.CODEX_HOME = sharedHostHome;
sandboxAuthFixture.bytes = Buffer.from(input.sandboxAuth, "utf8");
- await execute({
+ const executionResult = await execute({
runId: "run-copyback-e2e",
agent: {
id: "agent-1",
@@ -178,6 +177,7 @@ describe("codex execute — outbound auth copy-back restore contribution", () =>
return {
finalHostAuth: await readFile(hostAuthPath, "utf8"),
finalHostMode: (await lstat(hostAuthPath)).mode & 0o777,
+ executionResult,
};
}
@@ -231,4 +231,50 @@ describe("codex execute — outbound auth copy-back restore contribution", () =>
expect(result.finalHostMode, entry.name).toBe(0o600);
}
});
+
+ it("surfaces workspace restore failure after successful provider execution", async () => {
+ prepareAdapterExecutionTargetRuntime.mockResolvedValueOnce({
+ target: { kind: "remote", transport: "ssh" },
+ workspaceRemoteDir: "/remote/workspace",
+ runtimeRootDir: REMOTE_RUNTIME_ROOT,
+ assetDirs: { home: `${REMOTE_RUNTIME_ROOT}/home` },
+ restoreWorkspace: async () => {
+ throw new Error("workspace copy-back failed");
+ },
+ });
+
+ await expect(
+ runTeardown({
+ sandboxAuth: subscriptionAuth({ accountId: "acct", marker: "sandbox" }),
+ hostAuth: subscriptionAuth({ accountId: "acct", marker: "host" }),
+ }),
+ ).rejects.toThrow("workspace copy-back failed");
+ });
+
+ it("preserves a provider failure when workspace restore also fails", async () => {
+ runChildProcess.mockResolvedValueOnce({
+ exitCode: 1,
+ signal: null,
+ timedOut: false,
+ stdout: "",
+ stderr: "provider failed first",
+ pid: 321,
+ startedAt: new Date().toISOString(),
+ });
+ prepareAdapterExecutionTargetRuntime.mockResolvedValueOnce({
+ target: { kind: "remote", transport: "ssh" },
+ workspaceRemoteDir: "/remote/workspace",
+ runtimeRootDir: REMOTE_RUNTIME_ROOT,
+ assetDirs: { home: `${REMOTE_RUNTIME_ROOT}/home` },
+ restoreWorkspace: async () => {
+ throw new Error("workspace copy-back failed second");
+ },
+ });
+
+ const result = await runTeardown({
+ sandboxAuth: subscriptionAuth({ accountId: "acct", marker: "sandbox" }),
+ hostAuth: subscriptionAuth({ accountId: "acct", marker: "host" }),
+ });
+ expect(result.executionResult.errorMessage).toBe("provider failed first");
+ });
});
diff --git a/packages/adapters/codex-local/src/server/execute.ts b/packages/adapters/codex-local/src/server/execute.ts
index 430a77bcc2..436cd585b4 100644
--- a/packages/adapters/codex-local/src/server/execute.ts
+++ b/packages/adapters/codex-local/src/server/execute.ts
@@ -1526,6 +1526,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
};
};
+ let executionError: unknown = null;
try {
const initial = await runAttempt(sessionId);
if (
@@ -1539,22 +1540,26 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
`[paperclip] Codex resume session "${sessionId}" is unavailable; retrying with a fresh session.\n`,
);
const retry = await runAttempt(null);
- return toResult(retry, true, true);
+ const retryResult = toResult(retry, true, true);
+ if (retryResult.errorMessage) {
+ executionError = new Error(retryResult.errorMessage);
+ }
+ return retryResult;
}
- return toResult(initial, false, false);
+ const result = toResult(initial, false, false);
+ if (result.errorMessage) {
+ executionError = new Error(result.errorMessage);
+ }
+ return result;
+ } catch (error) {
+ executionError = error;
+ throw error;
} finally {
if (paperclipBridge) {
await paperclipBridge.stop();
}
if (restoreRemoteWorkspace) {
- // This teardown runs in a `finally`, so a throw here replaces the
- // already-computed run result (`return toResult(...)`) and turns a
- // successful Codex run into a failure. The workspace restore — and the
- // host credential copy-back inside it — is a best-effort teardown step.
- // Keep it rejection-safe: log a fault loudly and keep the pending
- // result. The host copy-back installs the credential on disk before any
- // diagnostic log runs, so it is already durable when this block returns.
try {
await onLog(
"stdout",
@@ -1570,6 +1575,10 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
)}: ${error instanceof Error ? error.message : String(error)}\n`,
),
).catch(() => undefined);
+ // A provider failure remains the primary outcome. When provider work
+ // succeeded, however, silently accepting a failed copy-back can lose
+ // the only workspace edits before a replacement sandbox starts.
+ if (executionError === null) throw error;
}
}
}
diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs
index 402b71a094..92e7323b49 100644
--- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs
+++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs
@@ -1345,6 +1345,13 @@ impl CommandExecutor for AcpxCommandExecutor {
}
}
+ fn rotate_authority(&mut self, config: &DurableRunnerConfig) {
+ self.context.run_id = config.run_id.clone();
+ self.context.normalized_session_id = config.normalized_session_id.clone();
+ self.context.turn_id = config.turn_id.clone();
+ self.context.item_id = config.item_id.clone();
+ }
+
fn poll_events(&mut self) -> Result<Vec<PolledEvent>, DurableRunnerError> {
self.restore()?;
if self
diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs b/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs
index 41f1685cb0..06b9d0d0fd 100644
--- a/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs
+++ b/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs
@@ -54,7 +54,7 @@ pub(crate) const MAX_SETTLED_PROVIDER_TURN_IDS: usize = 4_096;
type QuestionOptionLabels = BTreeMap<String, BTreeMap<String, String>>;
type QuestionSetMapping = (String, Value, QuestionOptionLabels);
-#[derive(Clone)]
+#[derive(Clone, PartialEq)]
struct ProviderCompletionContract {
revision: String,
criterion_ids: Vec<String>,
@@ -807,6 +807,46 @@ impl CodexProvider {
self.durable_tool_call_replays = true;
}
+ pub(crate) fn attach_run_in_place(
+ &mut self,
+ authorized_tools: impl IntoIterator<Item = AuthorizedTool>,
+ completion_contract: Option<(&str, &[String])>,
+ ) -> Result<bool, LocalRunnerError> {
+ let authorized_tools = authorized_tools.into_iter().collect::<Vec<_>>();
+ let completion_contract =
+ completion_contract.map(|(revision, criterion_ids)| ProviderCompletionContract {
+ revision: revision.to_owned(),
+ criterion_ids: criterion_ids.to_vec(),
+ });
+ if authorized_tools != self.authorized_tools
+ || completion_contract != self.completion_contract
+ {
+ return Ok(false);
+ }
+ if self.process.try_wait()?.is_some()
+ || self.quarantined
+ || self.active_provider_turn_id.is_some()
+ || self.ambiguous_turn_start_pending
+ || !self.pending_messages.is_empty()
+ || !self.deferred_ambiguous_messages.is_empty()
+ || !self.pending_tool_requests.is_empty()
+ || !self.pending_runtime_requests.is_empty()
+ {
+ return Err(LocalRunnerError::invalid(
+ "Codex warm run attachment requires an idle live provider with no pending work",
+ ));
+ }
+ // The provider process and its thread remain authoritative. Exact
+ // settled-turn identities stay in memory so delayed output from an
+ // earlier run cannot be accepted as the next turn. A changed semantic
+ // tool or completion contract returns false so the caller can preserve
+ // the existing cold-resume behavior for that incompatible boundary.
+ self.completed_turn_authority = None;
+ self.completion_reconciliation_pending = false;
+ self.expected_shutdown = false;
+ Ok(true)
+ }
+
pub(crate) fn restore_completed_turn_authority(
&mut self,
authoritative: bool,
diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs b/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs
index 6d0ae23dfc..f88dc0bbc2 100644
--- a/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs
+++ b/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs
@@ -116,9 +116,121 @@ impl CommandLifecycle {
}
}
+fn next_authority_config(
+ command: &Command,
+ current: &DurableRunnerConfig,
+) -> Result<Option<DurableRunnerConfig>, DurableRunnerError> {
+ if command.command_type != "run.attach" {
+ return Ok(None);
+ }
+ let Some(boundary) = command.payload.get("paperclipNextAuthority") else {
+ return Ok(None);
+ };
+ let identity = boundary
+ .get("identity")
+ .and_then(Value::as_object)
+ .ok_or_else(|| DurableRunnerError::invalid("run.attach authority identity is required"))?;
+ let read_identity = |key: &str| {
+ identity
+ .get(key)
+ .and_then(Value::as_str)
+ .filter(|value| !value.is_empty())
+ .map(str::to_owned)
+ .ok_or_else(|| {
+ DurableRunnerError::invalid(format!(
+ "run.attach authority identity field {key} is required"
+ ))
+ })
+ };
+ let connection = boundary
+ .get("connection")
+ .and_then(Value::as_object)
+ .ok_or_else(|| {
+ DurableRunnerError::invalid("run.attach authority connection is required")
+ })?;
+ let connect_url = match connection.get("mode").and_then(Value::as_str) {
+ Some("connect") => connection
+ .get("connectUrl")
+ .and_then(Value::as_str)
+ .filter(|value| !value.is_empty())
+ .map(str::to_owned)
+ .ok_or_else(|| DurableRunnerError::invalid("run.attach connect URL is required"))?,
+ Some("listen") => {
+ let address = connection
+ .get("listenAddress")
+ .and_then(Value::as_str)
+ .ok_or_else(|| {
+ DurableRunnerError::invalid("run.attach listen address is required")
+ })?;
+ let port = connection
+ .get("listenPort")
+ .and_then(Value::as_u64)
+ .ok_or_else(|| DurableRunnerError::invalid("run.attach listen port is required"))?;
+ let path = connection
+ .get("listenPath")
+ .and_then(Value::as_str)
+ .ok_or_else(|| DurableRunnerError::invalid("run.attach listen path is required"))?;
+ format!("listen://{address}:{port}{path}")
+ }
+ _ => {
+ return Err(DurableRunnerError::invalid(
+ "run.attach authority connection mode is invalid",
+ ));
+ }
+ };
+ let mut next = current.clone();
+ next.connect_url = connect_url;
+ next.ca_bundle_path = connection
+ .get("caBundlePath")
+ .and_then(Value::as_str)
+ .filter(|value| !value.is_empty())
+ .map(Into::into);
+ next.runner_instance_id = read_identity("runnerInstanceId")?;
+ next.environment_lease_id = read_identity("environmentLeaseId")?;
+ next.run_id = read_identity("runId")?;
+ next.normalized_session_id = read_identity("normalizedSessionId")?;
+ next.turn_id = read_identity("turnId")?;
+ next.item_id = read_identity("itemId")?;
+ next.validate()?;
+ if next.runner_instance_id != current.runner_instance_id
+ || next.environment_lease_id != current.environment_lease_id
+ || next.normalized_session_id != current.normalized_session_id
+ || next.run_id == current.run_id
+ {
+ return Err(DurableRunnerError::invalid(
+ "run.attach authority changed an immutable session binding",
+ ));
+ }
+ Ok(Some(next))
+}
+
+fn apply_authority_rotation(
+ state: &mut DurableState,
+ store: &DurableStateStore,
+ config: &mut DurableRunnerConfig,
+ endpoint: &mut RunnerTransportEndpoint,
+ next: DurableRunnerConfig,
+) -> Result<(), DurableRunnerError> {
+ let reconnect_count = state.reconnect_count.saturating_add(1);
+ let mut diagnostics = std::mem::take(&mut state.diagnostics);
+ *endpoint = RunnerTransportEndpoint::new(&next.connect_url, &next.run_id)?;
+ *config = next;
+ let mut rotated = DurableState::new(config);
+ rotated.reconnect_count = reconnect_count;
+ rotated.diagnostics.append(&mut diagnostics);
+ rotated.record_diagnostic("runner advanced to a new warm run authority");
+ *state = rotated;
+ store.save(state)
+}
+
pub trait CommandExecutor {
fn execute(&mut self, command: &Command) -> Result<CommandExecution, DurableRunnerError>;
+ /// Advances provider-side event correlation after a durable `run.attach`
+ /// has moved runnerd to the next run-bound authority. The runner validates
+ /// and persists the new authority before invoking this infallible hook.
+ fn rotate_authority(&mut self, _config: &DurableRunnerConfig) {}
+
fn poll_events(&mut self) -> Result<Vec<PolledEvent>, DurableRunnerError> {
Ok(Vec::new())
}
@@ -136,7 +248,7 @@ pub trait CommandExecutor {
}
pub fn run_durable_runner<E: CommandExecutor>(
- config: DurableRunnerConfig,
+ mut config: DurableRunnerConfig,
bootstrap_ticket: BootstrapTicket,
mut executor: E,
) -> Result<(), DurableRunnerError> {
@@ -162,7 +274,7 @@ pub fn run_durable_runner<E: CommandExecutor>(
// Bind listener mode or resolve dial mode before processing commands. Dial
// reconnects retain the same validated addresses so DNS cannot redirect a
// retry after the trust decision.
- let endpoint = RunnerTransportEndpoint::new(&config.connect_url, &config.run_id)?;
+ let mut endpoint = RunnerTransportEndpoint::new(&config.connect_url, &config.run_id)?;
let started = Instant::now();
let mut bootstrap_ticket = Some(bootstrap_ticket);
let mut lease: Option<LeaseCredential> = None;
@@ -287,8 +399,10 @@ pub fn run_durable_runner<E: CommandExecutor>(
let mut sent_source_seq = state.acked_source_seq;
let mut lifecycle_after_reply = CommandLifecycle::Continue;
+ let mut authority_rotation = None;
let mut disconnected = false;
for command in welcome.pending_commands {
+ let next_authority = next_authority_config(&command, &config)?;
let (result, lifecycle) =
process_command(&mut state, &store, &config, &mut executor, &command)?;
if let Some(durable_lifecycle) = lifecycle.durable_state() {
@@ -333,6 +447,16 @@ pub fn run_durable_runner<E: CommandExecutor>(
// then release the executor without observing later commands.
break;
}
+ if next_authority.is_some() {
+ authority_rotation = next_authority;
+ break;
+ }
+ }
+ if let Some(next) = authority_rotation {
+ apply_authority_rotation(&mut state, &store, &mut config, &mut endpoint, next)?;
+ executor.rotate_authority(&config);
+ disconnected_since = Some(Instant::now());
+ continue;
}
if !disconnected {
if let Err(error) = send_outbox(&mut transport, &state, &mut sent_source_seq) {
@@ -425,6 +549,7 @@ pub fn run_durable_runner<E: CommandExecutor>(
.map_err(|error| {
DurableRunnerError::invalid(format!("command is malformed: {error}"))
})?;
+ let next_authority = next_authority_config(&command, &config)?;
let (result, lifecycle) =
process_command(&mut state, &store, &config, &mut executor, &command)?;
if let Some(durable_lifecycle) = lifecycle.durable_state() {
@@ -468,6 +593,18 @@ pub fn run_durable_runner<E: CommandExecutor>(
);
}
}
+ if let Some(next) = next_authority {
+ apply_authority_rotation(
+ &mut state,
+ &store,
+ &mut config,
+ &mut endpoint,
+ next,
+ )?;
+ executor.rotate_authority(&config);
+ disconnected_since = Some(Instant::now());
+ break;
+ }
if let Err(error) = send_outbox(&mut transport, &state, &mut sent_source_seq) {
state.record_diagnostic(
"outbox delivery failed; unacknowledged suffix remains durable",
@@ -1018,6 +1155,59 @@ mod tests {
}
}
+ #[test]
+ fn warm_run_attachment_rotates_only_the_run_authority() {
+ let directory = std::env::temp_dir().join(format!(
+ "paperclip-runner-warm-authority-{}",
+ std::process::id()
+ ));
+ let _ = fs::remove_dir_all(&directory);
+ let mut current = config(directory.clone());
+ current.runner_digest = format!("sha256:{}", "a".repeat(64));
+ let mut attach = command("run.attach");
+ attach.payload = json!({
+ "paperclipNextAuthority": {
+ "identity": {
+ "runnerInstanceId": current.runner_instance_id,
+ "environmentLeaseId": current.environment_lease_id,
+ "runId": "run_2",
+ "normalizedSessionId": current.normalized_session_id,
+ "turnId": "turn_2",
+ "itemId": "item_2"
+ },
+ "connection": {
+ "mode": "connect",
+ "connectUrl": "ws://127.0.0.1:3001/path"
+ }
+ }
+ });
+
+ let next = next_authority_config(&attach, &current)
+ .unwrap()
+ .expect("attachment should carry a new authority");
+ assert_eq!(next.run_id, "run_2");
+ assert_eq!(next.connect_url, "ws://127.0.0.1:3001/path");
+
+ let store = DurableStateStore::new(&directory).unwrap();
+ let (mut state, _) = store.load_or_create(&current).unwrap();
+ state.outbox.push(crate::durable::state::StoredOutboxEvent {
+ source_seq: 1,
+ priority: 0,
+ event_type: "run.attached".to_owned(),
+ byte_size: 1,
+ envelope: json!({}),
+ });
+ let mut endpoint =
+ RunnerTransportEndpoint::new(&current.connect_url, &current.run_id).unwrap();
+ apply_authority_rotation(&mut state, &store, &mut current, &mut endpoint, next).unwrap();
+
+ assert_eq!(state.run_id, "run_2");
+ assert_eq!(state.next_source_seq, 1);
+ assert!(state.outbox.is_empty());
+ assert_eq!(current.run_id, "run_2");
+ fs::remove_dir_all(directory).unwrap();
+ }
+
#[test]
fn terminal_lifecycle_is_durable_before_fallible_cleanup() {
let directory = std::env::temp_dir().join(format!(
diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/managed_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/managed_provider_backend.rs
index 5456fc9824..dcdfb5ed0e 100644
--- a/packages/paperclip-runner/runner/crates/runner-core/src/managed_provider_backend.rs
+++ b/packages/paperclip-runner/runner/crates/runner-core/src/managed_provider_backend.rs
@@ -1670,6 +1670,10 @@ impl CommandExecutor for ManagedProviderCommandExecutor {
}
}
+ fn rotate_authority(&mut self, config: &DurableRunnerConfig) {
+ self.config = config.clone();
+ }
+
fn poll_events(&mut self) -> Result<Vec<PolledEvent>, DurableRunnerError> {
self.poll_provider()?;
Ok(self
diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/native_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/native_provider_backend.rs
index 959b06c8d1..4d54b48a20 100644
--- a/packages/paperclip-runner/runner/crates/runner-core/src/native_provider_backend.rs
+++ b/packages/paperclip-runner/runner/crates/runner-core/src/native_provider_backend.rs
@@ -35,6 +35,14 @@ impl CommandExecutor for SelectedExecutor {
}
}
+ fn rotate_authority(&mut self, config: &DurableRunnerConfig) {
+ match self {
+ Self::LocalFacade(executor) => executor.rotate_authority(config),
+ Self::Acpx(executor) => executor.rotate_authority(config),
+ Self::Managed(executor) => executor.rotate_authority(config),
+ }
+ }
+
fn acknowledge_events(&mut self, count: usize) -> Result<(), DurableRunnerError> {
match self {
Self::LocalFacade(executor) => executor.acknowledge_events(count),
@@ -163,6 +171,13 @@ impl CommandExecutor for NativeProviderCommandExecutor {
.map_or_else(|| Ok(Vec::new()), CommandExecutor::poll_events)
}
+ fn rotate_authority(&mut self, config: &DurableRunnerConfig) {
+ self.config = config.clone();
+ if let Some(executor) = self.selected.as_mut() {
+ executor.rotate_authority(config);
+ }
+ }
+
fn acknowledge_events(&mut self, count: usize) -> Result<(), DurableRunnerError> {
self.select_recovery()?;
if let Some(executor) = self.selected.as_mut() {
diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs
index f11fe402df..c335667184 100644
--- a/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs
+++ b/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs
@@ -1675,19 +1675,40 @@ impl CodexCommandExecutor {
next_state.active_provider_result_fingerprint = None;
next_state.active_provider_result_disposition = None;
next_state.last_agent_message = None;
- if let Some(provider) = self.provider.as_mut() {
+ let provider = self.provider.as_mut().ok_or_else(|| {
+ DurableRunnerError::invalid("run.attach requires the restored Codex provider process")
+ })?;
+ let retained_provider = provider
+ .attach_run_in_place(
+ next_state.tool_bridge.authorized_tools().cloned(),
+ next_state.completion_contract.as_ref().map(|contract| {
+ (
+ contract.revision.as_str(),
+ contract.criterion_ids.as_slice(),
+ )
+ }),
+ )
+ .map_err(|error| {
+ DurableRunnerError::invalid(format!(
+ "failed to retain Codex for warm run attachment: {error}"
+ ))
+ })?;
+ if !retained_provider {
provider.shutdown().map_err(|error| {
DurableRunnerError::invalid(format!(
"failed to checkpoint Codex before attaching a new run: {error}"
))
})?;
+ self.provider = None;
}
- self.provider = None;
next_state.pending_events.clear();
- // Persist the checkpoint as not-open before open_session resumes it for
- // the new authority. Otherwise recovery emits a second session.resumed
- // notice into the provider queue in addition to the command event.
- next_state.lifecycle = "prepared".to_owned();
+ next_state.lifecycle = if retained_provider {
+ "session_open".to_owned()
+ } else {
+ // The next provider command restores the same checkpointed session
+ // with the rotated tool/completion authority.
+ "prepared".to_owned()
+ };
self.persist_state(&next_state)?;
self.state = Some(next_state);
Ok(())
@@ -3009,6 +3030,10 @@ impl CommandExecutor for CodexCommandExecutor {
}
}
+ fn rotate_authority(&mut self, config: &DurableRunnerConfig) {
+ self.event_identity = Some(ProviderEventIdentity::from_config(config));
+ }
+
fn poll_events(&mut self) -> Result<Vec<PolledEvent>, DurableRunnerError> {
self.poll_provider()?;
Ok(self
diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts
index fb76579f69..85ee0ab0e1 100644
--- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts
+++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts
@@ -939,7 +939,7 @@ class AuthorityConnection {
/** Authenticated, replay-safe PRP transport authority. Business operations are caller supplied. */
export class DurablePrpControlPlane {
- readonly #identity: DurableRecoveryIdentity;
+ #identity: DurableRecoveryIdentity;
readonly #store: DurableCoreStore;
#expectedRunnerVersion: string;
#expectedRunnerDigest: string;
@@ -1041,6 +1041,36 @@ export class DurablePrpControlPlane {
).length;
}
+ /**
+ * Atomically advances a settled reusable runner to a new run authority while
+ * retaining its existing connection lease secret. The runner performs the
+ * matching state transition only after acknowledging `run.attach`.
+ */
+ rotateRunIdentity(identity: DurableRecoveryIdentity): void {
+ if (
+ !Object.values(identity).every(
+ (value) => typeof value === "string" && stableIdPattern.test(value),
+ ) ||
+ identity.runnerInstanceId !== this.#identity.runnerInstanceId ||
+ identity.environmentLeaseId !== this.#identity.environmentLeaseId ||
+ identity.normalizedSessionId !== this.#identity.normalizedSessionId ||
+ identity.runId === this.#identity.runId ||
+ this.#store.state.commands.some((command) => command.status === "pending")
+ ) {
+ throw new Error("Durable PRP run identity rotation is invalid.");
+ }
+ this.disconnectActiveRunner();
+ const leases = Object.fromEntries(
+ Object.entries(this.#store.state.leases).map(([key, lease]) => [
+ key,
+ { ...lease, identity: structuredClone(identity) },
+ ]),
+ );
+ Object.assign(this.#store.state, initialCoreState(identity), { leases });
+ this.#identity = structuredClone(identity);
+ this.#store.save();
+ }
+
issueBootstrapTicket(ttlMs = 5_000): string {
if (!Number.isInteger(ttlMs) || ttlMs < 1_000 || ttlMs > 60_000) {
throw new Error("Durable PRP bootstrap TTL is invalid.");
diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts
index dcbfc49159..b8ffc2d367 100644
--- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts
+++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts
@@ -1965,7 +1965,7 @@ it("steers the active provider turn through the durable PRP command path", async
}
}, 30_000);
-it("does not expose cross-run attachment before PRP authority can rotate atomically", async () => {
+it("rotates PRP authority in place for a warm cross-run attachment", async () => {
const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-warm-attach-"));
const bundle = createCapabilityRunnerdCodexTransport({
runnerBinary: defaultCapabilityRunnerdBinary(),
@@ -1978,22 +1978,36 @@ it("does not expose cross-run attachment before PRP authority can rotate atomica
success: true,
contentItems: [],
}));
+ const within = async <T>(label: string, promise: Promise<T>) =>
+ await Promise.race([
+ promise,
+ new Promise<never>((_, reject) =>
+ setTimeout(() => reject(new Error(`${label} timeout`)), 5_000),
+ ),
+ ]);
try {
- await bundle.transport.request("initialize", {});
- await bundle.transport.request("thread/start", {
- cwd: tmpdir(),
- dynamicTools: [
- {
- name: "get_task_context",
- description: "Read the active task.",
- inputSchema: {
- type: "object",
- properties: {},
- additionalProperties: false,
+ await within("initialize", bundle.transport.request("initialize", {}));
+ await within(
+ "thread start",
+ bundle.transport.request("thread/start", {
+ cwd: tmpdir(),
+ dynamicTools: [
+ {
+ name: "get_task_context",
+ description: "Read the active task.",
+ inputSchema: {
+ type: "object",
+ properties: {},
+ additionalProperties: false,
+ },
},
+ ],
+ completionContract: {
+ revision: "sha256:warm-three-turn-contract",
+ criterionIds: ["objective"],
},
- ],
- });
+ }),
+ );
const runnerPid = bundle.evidence().runnerPid;
const providerPid = bundle.evidence().codexPid;
const notifications = bundle.transport
@@ -2015,12 +2029,45 @@ it("does not expose cross-run attachment before PRP authority can rotate atomica
}
throw new Error(`${label} completion timeout`);
};
- await bundle.transport.request("turn/start", {
- input: [{ type: "text", text: "first run" }],
- });
+ await within(
+ "first turn start",
+ bundle.transport.request("turn/start", {
+ input: [{ type: "text", text: "first run" }],
+ }),
+ );
await waitForCompletion("first run");
- expect(bundle.transport.attachRun).toBeUndefined();
+ await within(
+ "warm attach",
+ bundle.transport.attachRun!({
+ runId: "run-warm-second",
+ turnId: "turn-warm-second",
+ itemId: "item-warm-second",
+ }),
+ );
+ await within(
+ "second turn start",
+ bundle.transport.request("turn/start", {
+ input: [{ type: "text", text: "second run" }],
+ }),
+ );
+ await waitForCompletion("second run");
+
+ await within(
+ "second warm attach",
+ bundle.transport.attachRun!({
+ runId: "run-warm-third",
+ turnId: "turn-warm-third",
+ itemId: "item-warm-third",
+ }),
+ );
+ await within(
+ "third turn start",
+ bundle.transport.request("turn/start", {
+ input: [{ type: "text", text: "third run" }],
+ }),
+ );
+ await waitForCompletion("third run");
expect(bundle.evidence()).toMatchObject({
runnerPid,
@@ -2033,6 +2080,99 @@ it("does not expose cross-run attachment before PRP authority can rotate atomica
}
}, 30_000);
+it("releases both PRP authorities when warm rotation activation fails", async () => {
+ const stateDirectory = await mkdtemp(
+ join(tmpdir(), "runnerd-warm-attach-activation-failure-"),
+ );
+ const server = createServer();
+ const authorities = new Map<string, DurablePrpControlPlane>();
+ const released: string[] = [];
+ server.on("upgrade", (request, socket, head) => {
+ const route = request.url ?? "";
+ const authority = authorities.get(route);
+ if (!authority) {
+ socket.destroy();
+ return;
+ }
+ authority.handleUpgrade(request, socket, route, head);
+ });
+ await new Promise<void>((resolveListen) =>
+ server.listen(0, "127.0.0.1", resolveListen),
+ );
+ const address = server.address();
+ if (!address || typeof address === "string") {
+ throw new Error("Expected warm activation failure test listener");
+ }
+ let registrationCount = 0;
+ const bundle = createCapabilityRunnerdCodexTransport({
+ runnerBinary: defaultCapabilityRunnerdBinary(),
+ codexCommand: fakeCodex,
+ codexArgs: fakeCodexArgs(stateDirectory),
+ stateDirectory,
+ lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 },
+ controlPlaneRegistration: async (authority) => {
+ registrationCount += 1;
+ const route = `/runner-${registrationCount}`;
+ authorities.set(route, authority);
+ return {
+ connectUrl: `ws://127.0.0.1:${address.port}${route}`,
+ ...(registrationCount === 1
+ ? {}
+ : {
+ activate: () => {
+ throw new Error("rotation activation failed");
+ },
+ }),
+ release: () => {
+ released.push(route);
+ if (authorities.get(route) === authority) authorities.delete(route);
+ },
+ };
+ },
+ });
+ bundle.transport.setServerRequestHandler(async () => ({
+ success: true,
+ contentItems: [],
+ }));
+ let runnerPid: number | null = null;
+ try {
+ await bundle.transport.request("thread/start", {
+ cwd: tmpdir(),
+ dynamicTools: codexSemanticToolSpecs(),
+ });
+ runnerPid = bundle.evidence().runnerPid;
+
+ await expect(
+ bundle.transport.attachRun!({
+ runId: "run-warm-activation-failure",
+ turnId: "turn-warm-activation-failure",
+ itemId: "item-warm-activation-failure",
+ }),
+ ).rejects.toThrow("rotation activation failed");
+ expect(new Set(released)).toEqual(new Set(["/runner-1", "/runner-2"]));
+ expect(authorities.size).toBe(0);
+ await expect(bundle.transport.request("thread/read", {})).rejects.toThrow(
+ "rotation activation failed",
+ );
+ } finally {
+ await bundle.transport.close().catch(() => undefined);
+ if (runnerPid) {
+ try {
+ process.kill(-runnerPid, "SIGKILL");
+ } catch {
+ // A successful durable close already stopped the runner process group.
+ }
+ }
+ server.closeAllConnections();
+ if (server.listening) {
+ await new Promise<void>((resolveClose) =>
+ server.close(() => resolveClose()),
+ );
+ }
+ await rm(stateDirectory, { recursive: true, force: true });
+ }
+}, 30_000);
+
it.each([
{
binding: "runner instance",
diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts
index 5ab150c78f..fe384d7336 100644
--- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts
+++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts
@@ -400,7 +400,7 @@ async function rotateExternalAuthorityEpoch(
}
function rotatedRunAttachPayload(
- state: Record<string, unknown>,
+ state: { commands?: unknown },
desired: DurableRecoveryIdentity,
authorizedTools: Record<string, unknown> | null,
completionContract:
@@ -418,7 +418,22 @@ function rotatedRunAttachPayload(
);
if (!seed)
throw new Error("native_runner_authority_rotation_seed_unavailable");
- const payload = structuredClone(record(seed.payload));
+ return retargetRunAttachPayload(
+ record(seed.payload),
+ desired,
+ authorizedTools,
+ completionContract,
+ );
+}
+
+function retargetRunAttachPayload(
+ seedPayload: Record<string, unknown>,
+ desired: DurableRecoveryIdentity,
+ authorizedTools: Record<string, unknown> | null,
+ completionContract:
+ { revision: string; criterionIds: readonly string[] } | undefined,
+): Record<string, unknown> {
+ const payload = structuredClone(seedPayload);
const provider = record(payload.provider);
if (provider.kind === "acpx" || provider.provider === "acpx") {
provider.runId = desired.runId;
@@ -958,7 +973,10 @@ export interface CapabilityRunnerdCodexTransportOptions {
itemId: string;
};
/** Registers the run-bound PRP authority on Paperclip's shared HTTP server. */
- controlPlaneRegistration?: (authority: DurablePrpControlPlane) => Promise<{
+ controlPlaneRegistration?: (
+ authority: DurablePrpControlPlane,
+ identity?: DurableRecoveryIdentity,
+ ) => Promise<{
connectUrl?: string;
connection?: RunnerProcessConnection;
activate?: () => Promise<void> | void;
@@ -1942,6 +1960,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
#expectedProviderTurnId: string | null = null;
#durableTurnId = "";
#authorizedTools: Record<string, unknown> | null = null;
+ #runAttachTemplate: Record<string, unknown> | null = null;
#closed = false;
#closePromise: Promise<void> | null = null;
#failure: Error | null = null;
@@ -2216,6 +2235,94 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
this.#handler = handler;
}
+ async attachRun(input: {
+ runId: string;
+ turnId: string;
+ itemId: string;
+ }): Promise<void> {
+ const core = this.#core;
+ if (!core || !this.#startupComplete) {
+ throw new Error("native_runner_prp_run_rotation_unavailable");
+ }
+ const prior = core.store.state.identity;
+ const desired: DurableRecoveryIdentity = {
+ ...prior,
+ runId: input.runId,
+ turnId: input.turnId,
+ itemId: input.itemId,
+ };
+ const registration = this.options.controlPlaneRegistration
+ ? await this.options.controlPlaneRegistration(core, desired)
+ : null;
+ const connection: RunnerProcessConnection =
+ registration?.connection ??
+ (registration?.connectUrl
+ ? { mode: "connect", connectUrl: registration.connectUrl }
+ : { mode: "connect", connectUrl: core.connectUrl });
+ const commandId = `command_attach_${createHash("sha256")
+ .update(`${prior.runId}:${desired.runId}:${desired.turnId}`)
+ .digest("hex")
+ .slice(0, 32)}`;
+ const runAttachTemplate = this.#runAttachTemplate
+ ? retargetRunAttachPayload(
+ this.#runAttachTemplate,
+ desired,
+ this.#authorizedTools,
+ this.options.resumeCompletionContract,
+ )
+ : rotatedRunAttachPayload(
+ core.store.state,
+ desired,
+ this.#authorizedTools,
+ this.options.resumeCompletionContract,
+ );
+ this.#runAttachTemplate = structuredClone(runAttachTemplate);
+ const payload = {
+ ...runAttachTemplate,
+ paperclipNextAuthority: { identity: desired, connection },
+ };
+ core.queueCommand("run.attach", payload, commandId, true);
+ await this.#waitCommand("run.attach", commandId);
+ const attached = core.store.state.commands.find(
+ (command) => command.commandId === commandId,
+ );
+ if (attached?.status !== "completed") {
+ await Promise.resolve(registration?.release()).catch(() => undefined);
+ throw new Error("native_runner_prp_run_rotation_failed");
+ }
+
+ const previousRelease = this.#controlPlaneRelease;
+ core.rotateRunIdentity(desired);
+ this.#eventIndex = 0;
+ this.#durableTurnId = desired.turnId;
+ this.#controlPlaneRelease = registration?.release ?? null;
+ let previousReleased = false;
+ try {
+ await registration?.activate?.();
+ if (registration?.failure) {
+ void registration.failure.catch((error: unknown) => {
+ this.#failTransport(
+ error instanceof Error ? error : new Error(String(error)),
+ );
+ });
+ }
+ await previousRelease?.();
+ previousReleased = true;
+ await this.#awaitRegistrationReady(registration?.ready);
+ } catch (error) {
+ const failure = error instanceof Error ? error : new Error(String(error));
+ this.#controlPlaneRelease = null;
+ await Promise.allSettled([
+ Promise.resolve().then(() => registration?.release()),
+ ...(previousReleased
+ ? []
+ : [Promise.resolve().then(() => previousRelease?.())]),
+ ]);
+ this.#failTransport(failure);
+ throw failure;
+ }
+ }
+
async resolveRuntimeRequest(input: {
requestId: string;
turnId: string;
@@ -3298,15 +3405,14 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
});
this.#core = core;
if (rotatedAuthority) {
- core.queueCommand(
- "run.attach",
- rotatedRunAttachPayload(
- controlPlaneState,
- desiredIdentity,
- this.#authorizedTools,
- this.options.resumeCompletionContract,
- ),
+ const runAttachTemplate = rotatedRunAttachPayload(
+ controlPlaneState,
+ desiredIdentity,
+ this.#authorizedTools,
+ this.options.resumeCompletionContract,
);
+ this.#runAttachTemplate = structuredClone(runAttachTemplate);
+ core.queueCommand("run.attach", runAttachTemplate);
}
const committedEvents = core.store.state.committedEvents;
const runAttachment = recoveredRunAttachment(core.store.state);
diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts
index 41da01fb5a..84787dff01 100644
--- a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts
+++ b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts
@@ -1240,6 +1240,8 @@ describe("Daytona sandbox provider plugin", () => {
providerLeaseId: "sandbox-reuse",
metadata: {
resumedLease: true,
+ resumedFromState: "stopped",
+ sandboxState: "started",
workspaceSentinel: {
result: "matched",
token: "sentinel-token",
@@ -4060,6 +4062,35 @@ describe("daytona native file-sync hooks", () => {
});
});
+ it("classifies a deleted sandbox during syncOut with a stable unrecoverable code", async () => {
+ const hostDir = await makeHostDir();
+ mockGet.mockRejectedValue(
+ new MockDaytonaNotFoundError("provider detail must not escape"),
+ );
+
+ await expect(
+ plugin.definition.onEnvironmentSyncOut?.({
+ driverKey: "daytona",
+ companyId: "company-1",
+ environmentId: "env-1",
+ config: { timeoutMs: 300000, reuseLease: true },
+ lease: syncLease(),
+ operations: [
+ {
+ operationId: "sync-op-missing-sandbox",
+ files: [
+ {
+ sourcePath: `${REMOTE_DIR}/out/result.txt`,
+ targetPath: path.join(hostDir, "result.txt"),
+ kind: "file",
+ },
+ ],
+ },
+ ],
+ }),
+ ).rejects.toThrow("daytona_sandbox_not_found");
+ });
+
it("syncOut snapshot guard re-checks the resolved source is a non-symlink regular file immediately before copying (validation→copy TOCTOU)", async () => {
const hostDir = await makeHostDir();
const sandbox = createMockSandbox();
diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.ts
index 52819929b5..afb988d64e 100644
--- a/packages/plugins/sandbox-providers/daytona/src/plugin.ts
+++ b/packages/plugins/sandbox-providers/daytona/src/plugin.ts
@@ -619,6 +619,8 @@ function leaseMetadata(input: {
shellCommand: "bash" | "sh";
remoteCwd: string;
resumedLease: boolean;
+ resumedFromState?: string | null;
+ sandboxState?: string | null;
workspaceSentinel?: WorkspaceSentinelResult;
}) {
return {
@@ -626,7 +628,7 @@ function leaseMetadata(input: {
shellCommand: input.shellCommand,
sandboxId: input.sandbox.id,
sandboxName: input.sandbox.name,
- sandboxState: input.sandbox.state ?? null,
+ sandboxState: input.sandboxState ?? input.sandbox.state ?? null,
image: input.config.image,
snapshot: input.config.snapshot,
target: input.sandbox.target,
@@ -637,6 +639,9 @@ function leaseMetadata(input: {
...(input.config.archiveOnRelease ? { archiveOnRelease: true } : {}),
remoteCwd: input.remoteCwd,
resumedLease: input.resumedLease,
+ ...(input.resumedLease
+ ? { resumedFromState: input.resumedFromState ?? null }
+ : {}),
// Record the resources Paperclip attempted to request so future diagnosis
// can compare requested allocation against what Daytona provisioned.
...(input.config.cpu != null ? { cpu: input.config.cpu } : {}),
@@ -2187,61 +2192,72 @@ const plugin = definePlugin({
return { providerLeaseId: null, metadata: { expired: true } };
}
- // A stopped sandbox loses its session shell, so the stored session id is
- // stale after a real restart. Clear the id only when the sandbox is not
- // already running, and clear it before the restart. A stopped sandbox has
- // no live session, so the clear drops a dead id and a later command opens
- // a fresh session. A running sandbox keeps its live session, so the resume
- // leaves the id in place; a concurrent command still finds it and teardown
- // deletes one session. An unconditional clear would drop the id of a live
- // session and leak its shell until sandbox reaping.
- if (sandbox.state !== "started") {
- sandboxHandleSessionStore.clear(scope);
- // A stopped sandbox loses its pseudo-terminals, so a stored duplex channel
- // is dead after a real restart. Close and drop every channel on this lease
- // before the restart, so no stale channel id survives the resume.
- await closeDaytonaDuplexChannelsForLease(params.providerLeaseId);
- }
- await ensureSandboxStarted(sandbox, toTimeoutSeconds(config.timeoutMs));
- try {
- const remoteCwd = await resolveSandboxWorkingDirectory(sandbox);
- // C3: a resumed lease must clear the workspace sentinel before it is
- // trusted, even when the handle came from the cache. On any non-match we
- // evict the cached handle and expire the lease so a stale/foreign sandbox
- // is never reused on the subsequent (sentinel-skipping) exec path.
- const workspaceSentinel = await verifyWorkspaceSentinel({
- sandbox,
- remoteCwd,
- leaseMetadata: params.leaseMetadata,
- timeoutSeconds: toTimeoutSeconds(config.timeoutMs),
- });
- if (workspaceSentinel.result !== "matched") {
- evictSandboxHandle(scope);
- return { providerLeaseId: null, metadata: { expired: true, workspaceSentinel } };
- }
- const shellCommand = await detectSandboxShellCommand(sandbox, toTimeoutSeconds(config.timeoutMs));
- sandboxHandleCache.markFresh(scope);
- sandboxHandleLeaseAdmissionStates.open(scope);
- return {
- providerLeaseId: sandbox.id,
- metadata: leaseMetadata({
- config,
- sandbox,
- shellCommand,
- remoteCwd,
- resumedLease: true,
- workspaceSentinel,
- }),
- };
- } catch (error) {
- evictSandboxHandle(scope);
- // A timeout, rate limit, or provider 5xx does not prove this sandbox is
- // lost. Preserve the exact resource and let the host retry its recorded
- // lease; replacement is permitted only after an explicit not-found or
- // an immutable workspace identity mismatch.
- throw error;
- }
- }, { allowClosed: true });
+ // A stopped sandbox loses its session shell, so the stored session id is
+ // stale after a real restart. Clear the id only when the sandbox is not
+ // already running, and clear it before the restart. A stopped sandbox has
+ // no live session, so the clear drops a dead id and a later command opens
+ // a fresh session. A running sandbox keeps its live session, so the resume
+ // leaves the id in place; a concurrent command still finds it and teardown
+ // deletes one session. An unconditional clear would drop the id of a live
+ // session and leak its shell until sandbox reaping.
+ if (sandbox.state !== "started") {
+ sandboxHandleSessionStore.clear(scope);
+ // A stopped sandbox loses its pseudo-terminals, so a stored duplex channel
+ // is dead after a real restart. Close and drop every channel on this lease
+ // before the restart, so no stale channel id survives the resume.
+ await closeDaytonaDuplexChannelsForLease(params.providerLeaseId);
+ }
+ const resumedFromState = sandbox.state ?? null;
+ await ensureSandboxStarted(sandbox, toTimeoutSeconds(config.timeoutMs));
+ try {
+ const remoteCwd = await resolveSandboxWorkingDirectory(sandbox);
+ // C3: a resumed lease must clear the workspace sentinel before it is
+ // trusted, even when the handle came from the cache. On any non-match we
+ // evict the cached handle and expire the lease so a stale/foreign sandbox
+ // is never reused on the subsequent (sentinel-skipping) exec path.
+ const workspaceSentinel = await verifyWorkspaceSentinel({
+ sandbox,
+ remoteCwd,
+ leaseMetadata: params.leaseMetadata,
+ timeoutSeconds: toTimeoutSeconds(config.timeoutMs),
+ });
+ if (workspaceSentinel.result !== "matched") {
+ evictSandboxHandle(scope);
+ return {
+ providerLeaseId: null,
+ metadata: { expired: true, workspaceSentinel },
+ };
+ }
+ const shellCommand = await detectSandboxShellCommand(
+ sandbox,
+ toTimeoutSeconds(config.timeoutMs),
+ );
+ sandboxHandleCache.markFresh(scope);
+ sandboxHandleLeaseAdmissionStates.open(scope);
+ return {
+ providerLeaseId: sandbox.id,
+ metadata: leaseMetadata({
+ config,
+ sandbox,
+ shellCommand,
+ remoteCwd,
+ resumedLease: true,
+ resumedFromState,
+ sandboxState: "started",
+ workspaceSentinel,
+ }),
+ };
+ } catch (error) {
+ evictSandboxHandle(scope);
+ // A timeout, rate limit, or provider 5xx does not prove this sandbox is
+ // lost. Preserve the exact resource and let the host retry its recorded
+ // lease; replacement is permitted only after an explicit not-found or
+ // an immutable workspace identity mismatch.
+ throw error;
+ }
+ },
+ { allowClosed: true },
+ );
},
async onEnvironmentReleaseLease(
@@ -2828,18 +2844,28 @@ const plugin = definePlugin({
providerLeaseId: params.lease.providerLeaseId,
config,
};
- return await withSandboxActivityGate(scope, async () => {
- const sandbox = await getSandbox(scope, { bypassTeardownGate: true });
- await ensureSandboxStarted(sandbox, timeoutSeconds);
- const result = await performSyncOut({
- sandbox,
- operations: params.operations,
- remoteDir,
- timeoutSeconds,
+ try {
+ return await withSandboxActivityGate(scope, async () => {
+ const sandbox = await getSandbox(scope, { bypassTeardownGate: true });
+ await ensureSandboxStarted(sandbox, timeoutSeconds);
+ const result = await performSyncOut({
+ sandbox,
+ operations: params.operations,
+ remoteDir,
+ timeoutSeconds,
+ });
+ sandboxHandleCache.markFresh(scope);
+ return result;
});
- sandboxHandleCache.markFresh(scope);
- return result;
- });
+ } catch (error) {
+ // A deleted Daytona sandbox is the one provider failure that proves its
+ // unexported workspace bytes no longer exist. Convert the SDK class to a
+ // stable cross-worker message; every other error remains retryable.
+ if (error instanceof DaytonaNotFoundError) {
+ throw new Error("daytona_sandbox_not_found");
+ }
+ throw error;
+ }
},
// Open one live login pseudo-terminal. Resolve the cached sandbox by the
diff --git a/server/src/__tests__/environment-runtime.test.ts b/server/src/__tests__/environment-runtime.test.ts
index e69e2fa9a9..d24350465f 100644
--- a/server/src/__tests__/environment-runtime.test.ts
+++ b/server/src/__tests__/environment-runtime.test.ts
@@ -561,6 +561,18 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
mode: "shared_workspace",
},
});
+ const workspaceSyncStamp = {
+ schema: "paperclip.native-workspace-stamp/v1",
+ workspaceId: seeded.executionWorkspaceId,
+ providerLeaseId: "sandbox-exact-resume",
+ remoteCwd: "/workspace",
+ hostSha256: "a".repeat(64),
+ finalizedRunId: seeded.runId,
+ };
+ await environmentService(db).updateLeaseMetadata(first.lease.id, {
+ ...(first.lease.metadata ?? {}),
+ nativeWorkspaceSync: workspaceSyncStamp,
+ });
await runtimeWithPlugin.releaseRunLeases(
seeded.runId,
"released",
@@ -592,9 +604,17 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
expect(first.lease.metadata?.sandboxLeaseAcquisition).toEqual({ outcome: "created" });
expect(acquired.lease.providerLeaseId).toBe("sandbox-exact-resume");
- expect(acquired.lease.metadata?.sandboxLeaseAcquisition).toEqual({ outcome: "resumed" });
- expect(workerManager.call.mock.calls.filter((call) => call[1] === "environmentAcquireLease"))
- .toHaveLength(1);
+ expect(acquired.lease.metadata?.sandboxLeaseAcquisition).toEqual({
+ outcome: "resumed",
+ });
+ expect(acquired.lease.metadata?.nativeWorkspaceSync).toEqual(
+ workspaceSyncStamp,
+ );
+ expect(
+ workerManager.call.mock.calls.filter(
+ (call) => call[1] === "environmentAcquireLease",
+ ),
+ ).toHaveLength(1);
});
it("destroys a disposable paperclip_runner sandbox after the turn", async () => {
diff --git a/server/src/__tests__/native-sandbox-lifecycle.test.ts b/server/src/__tests__/native-sandbox-lifecycle.test.ts
index 4cc88c1a20..13b2553b9f 100644
--- a/server/src/__tests__/native-sandbox-lifecycle.test.ts
+++ b/server/src/__tests__/native-sandbox-lifecycle.test.ts
@@ -1,5 +1,9 @@
import { describe, expect, it } from "vitest";
-import { resolveNativeSandboxLifecycle } from "../services/heartbeat.js";
+import {
+ providerResourceDispositionForTerminalRun,
+ resolveNativeSandboxLifecycle,
+ resolveReusableSandboxLifecycle,
+} from "../services/heartbeat.js";
const reusableSandbox = {
kind: "remote" as const,
@@ -21,6 +25,19 @@ describe("paperclip_runner sandbox lifecycle", () => {
});
});
+ it("keeps the same warm reusable sandbox for a legacy adapter", () => {
+ expect(
+ resolveReusableSandboxLifecycle({
+ lifecyclePolicy: { mode: "warm", idleTimeoutMs: 300_000 },
+ target: reusableSandbox,
+ }),
+ ).toEqual({
+ runnerProcess: "warm",
+ sandboxResource: "keep_running",
+ failoverBackup: "verified",
+ });
+ });
+
it("stops and reuses a per-turn reusable sandbox", () => {
expect(resolveNativeSandboxLifecycle({
adapterType: "paperclip_runner",
@@ -71,4 +88,22 @@ describe("paperclip_runner sandbox lifecycle", () => {
target: { kind: "local" },
})).toBeNull();
});
+
+ it("keeps a warm sandbox only after a successful turn", () => {
+ expect(
+ providerResourceDispositionForTerminalRun("keep_running", "succeeded"),
+ ).toBe("keep_running");
+ expect(
+ providerResourceDispositionForTerminalRun("keep_running", "failed"),
+ ).toBe("stop_and_retain");
+ expect(
+ providerResourceDispositionForTerminalRun("keep_running", "cancelled"),
+ ).toBe("stop_and_retain");
+ expect(
+ providerResourceDispositionForTerminalRun("keep_running", "timed_out"),
+ ).toBe("stop_and_retain");
+ expect(providerResourceDispositionForTerminalRun("destroy", "failed")).toBe(
+ "destroy",
+ );
+ });
});
diff --git a/server/src/__tests__/native-workspace-sync.test.ts b/server/src/__tests__/native-workspace-sync.test.ts
new file mode 100644
index 0000000000..a55ee16512
--- /dev/null
+++ b/server/src/__tests__/native-workspace-sync.test.ts
@@ -0,0 +1,297 @@
+import { mkdtemp, readdir, rm } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import {
+ directorySnapshotSha256,
+ serializeDirectorySnapshot,
+} from "@paperclipai/adapter-utils/workspace-restore-merge";
+
+import {
+ classifyNativeWorkspaceInbound,
+ nativeWorkspaceSyncInternals,
+ readNativeWorkspaceSyncReference,
+ resumeNativeWorkspaceSync,
+} from "../services/native-runtime/native-workspace-sync.js";
+
+const digest = "a".repeat(64);
+
+describe("native workspace sync durable metadata", () => {
+ const originalPaperclipHome = process.env.PAPERCLIP_HOME;
+ const originalPaperclipInstanceId = process.env.PAPERCLIP_INSTANCE_ID;
+ const cleanupDirs: string[] = [];
+
+ afterEach(async () => {
+ if (originalPaperclipHome === undefined) delete process.env.PAPERCLIP_HOME;
+ else process.env.PAPERCLIP_HOME = originalPaperclipHome;
+ if (originalPaperclipInstanceId === undefined)
+ delete process.env.PAPERCLIP_INSTANCE_ID;
+ else process.env.PAPERCLIP_INSTANCE_ID = originalPaperclipInstanceId;
+ await Promise.all(
+ cleanupDirs
+ .splice(0)
+ .map((directory) => rm(directory, { recursive: true, force: true })),
+ );
+ });
+
+ it("classifies fresh, warm, replacement, and same-run recovery inputs", () => {
+ expect(
+ classifyNativeWorkspaceInbound({
+ kind: "new_run",
+ acquisition: "created",
+ hasPriorStamp: false,
+ }),
+ ).toBe("host_current");
+ expect(
+ classifyNativeWorkspaceInbound({
+ kind: "new_run",
+ acquisition: "resumed",
+ hasPriorStamp: true,
+ }),
+ ).toBe("adopt_remote");
+ expect(
+ classifyNativeWorkspaceInbound({
+ kind: "new_run",
+ acquisition: "resumed",
+ hasPriorStamp: false,
+ }),
+ ).toBe("host_current");
+ expect(
+ classifyNativeWorkspaceInbound({
+ kind: "new_run",
+ acquisition: "replacement",
+ hasPriorStamp: true,
+ }),
+ ).toBe("host_current");
+ expect(
+ classifyNativeWorkspaceInbound({
+ kind: "existing_run",
+ restartRecovery: true,
+ sameProviderLease: true,
+ }),
+ ).toBe("adopt_remote");
+ expect(
+ classifyNativeWorkspaceInbound({
+ kind: "existing_run",
+ restartRecovery: true,
+ sameProviderLease: false,
+ }),
+ ).toBe("durable_seed");
+ expect(() =>
+ classifyNativeWorkspaceInbound({
+ kind: "existing_run",
+ restartRecovery: false,
+ sameProviderLease: true,
+ }),
+ ).toThrow("native_workspace_sync_unexpected_existing_descriptor");
+ });
+
+ it("reads backward-compatible references and the resource disposition", () => {
+ const base = {
+ schema: "paperclip.native-workspace-sync/v1",
+ state: "prepared",
+ descriptorSha256: digest,
+ baselineSha256: digest,
+ finalHostSha256: null,
+ workspaceId: "workspace-1",
+ leaseId: "lease-1",
+ providerLeaseId: "sandbox-1",
+ remoteCwd: "/workspace",
+ };
+
+ expect(readNativeWorkspaceSyncReference(base)).toEqual({
+ ...base,
+ resourceDisposition: null,
+ });
+ expect(
+ readNativeWorkspaceSyncReference({
+ ...base,
+ resourceDisposition: "keep_running",
+ }),
+ ).toEqual({ ...base, resourceDisposition: "keep_running" });
+ expect(
+ readNativeWorkspaceSyncReference({
+ ...base,
+ resourceDisposition: "delete_everything",
+ }),
+ ).toBeNull();
+ });
+
+ it("rejects traversal before constructing a durable state path", () => {
+ expect(() =>
+ nativeWorkspaceSyncInternals.descriptorPath("../run", digest),
+ ).toThrow("native_workspace_sync_invalid_run_id");
+ expect(() =>
+ nativeWorkspaceSyncInternals.descriptorPath("run-1", "../descriptor"),
+ ).toThrow("native_workspace_sync_descriptor_digest_invalid");
+ });
+
+ it("writes one immutable descriptor when the same state is replayed", async () => {
+ const paperclipHome = await mkdtemp(
+ path.join(os.tmpdir(), "paperclip-native-workspace-sync-"),
+ );
+ cleanupDirs.push(paperclipHome);
+ process.env.PAPERCLIP_HOME = paperclipHome;
+ process.env.PAPERCLIP_INSTANCE_ID = "descriptor-test";
+ const baseline = {
+ exclude: [".paperclip-runtime"],
+ entries: new Map([
+ [
+ "continuity.txt",
+ { kind: "file" as const, mode: 0o644, hash: digest },
+ ],
+ ]),
+ };
+ const descriptor = {
+ schema: "paperclip.native-workspace-sync/v1" as const,
+ binding: {
+ runId: "run-idempotent",
+ companyId: "company-1",
+ workspaceId: "workspace-1",
+ leaseId: "lease-1",
+ providerLeaseId: "sandbox-1",
+ localCwd: path.join(paperclipHome, "workspace"),
+ remoteCwd: "/workspace",
+ },
+ state: "prepared" as const,
+ baselineSha256: directorySnapshotSha256(baseline),
+ baseline: serializeDirectorySnapshot(baseline),
+ gitSnapshot: null,
+ seed: null,
+ createdAt: "2026-01-01T00:00:00.000Z",
+ finalizedAt: null,
+ finalHostSha256: null,
+ resourceDisposition: "keep_running" as const,
+ };
+
+ const first =
+ await nativeWorkspaceSyncInternals.writeDescriptor(descriptor);
+ const second =
+ await nativeWorkspaceSyncInternals.writeDescriptor(descriptor);
+
+ expect(second).toEqual(first);
+ const files = await readdir(
+ path.dirname(
+ nativeWorkspaceSyncInternals.descriptorPath(
+ descriptor.binding.runId,
+ first.descriptorSha256,
+ ),
+ ),
+ );
+ expect(files.filter((file) => file.endsWith(".json"))).toEqual([
+ `descriptor.${first.descriptorSha256}.json`,
+ ]);
+ await expect(
+ nativeWorkspaceSyncInternals.readDescriptor({
+ runId: descriptor.binding.runId,
+ reference: first,
+ }),
+ ).resolves.toMatchObject({ descriptor });
+ });
+
+ it("repairs finalized remote and lease stamps after an interrupted commit", async () => {
+ const paperclipHome = await mkdtemp(
+ path.join(os.tmpdir(), "paperclip-native-workspace-sync-repair-"),
+ );
+ cleanupDirs.push(paperclipHome);
+ process.env.PAPERCLIP_HOME = paperclipHome;
+ process.env.PAPERCLIP_INSTANCE_ID = "descriptor-repair-test";
+ const baseline = {
+ exclude: [".paperclip-runtime"],
+ entries: new Map([
+ [
+ "continuity.txt",
+ { kind: "file" as const, mode: 0o644, hash: digest },
+ ],
+ ]),
+ };
+ const finalHostSha256 = "b".repeat(64);
+ const descriptor = {
+ schema: "paperclip.native-workspace-sync/v1" as const,
+ binding: {
+ runId: "run-finalized-repair",
+ companyId: "company-1",
+ workspaceId: "workspace-1",
+ leaseId: "lease-1",
+ providerLeaseId: "sandbox-1",
+ localCwd: path.join(paperclipHome, "workspace"),
+ remoteCwd: "/workspace",
+ },
+ state: "finalized" as const,
+ baselineSha256: directorySnapshotSha256(baseline),
+ baseline: serializeDirectorySnapshot(baseline),
+ gitSnapshot: null,
+ seed: null,
+ createdAt: "2026-01-01T00:00:00.000Z",
+ finalizedAt: "2026-01-01T00:01:00.000Z",
+ finalHostSha256,
+ resourceDisposition: "keep_running" as const,
+ };
+ const reference =
+ await nativeWorkspaceSyncInternals.writeDescriptor(descriptor);
+ const rows = (values: unknown[]) => {
+ const query = {
+ from: () => query,
+ where: () => query,
+ for: () => query,
+ limit: () => query,
+ then: <TResult1 = unknown, TResult2 = never>(
+ onfulfilled?:
+ ((value: unknown[]) => TResult1 | PromiseLike<TResult1>) | null,
+ onrejected?:
+ ((reason: unknown) => TResult2 | PromiseLike<TResult2>) | null,
+ ) => Promise.resolve(values).then(onfulfilled, onrejected),
+ };
+ return query;
+ };
+ let persistedLeaseMetadata: Record<string, unknown> | null = null;
+ const db = {
+ select: () =>
+ rows([{ runnerProfileJson: { nativeWorkspaceSync: reference } }]),
+ transaction: async (callback: (tx: unknown) => Promise<unknown>) =>
+ callback({
+ select: () => rows([{ metadata: { retained: true } }]),
+ update: () => ({
+ set: (value: { metadata: Record<string, unknown> }) => ({
+ where: async () => {
+ persistedLeaseMetadata = value.metadata;
+ },
+ }),
+ }),
+ }),
+ };
+ const execute = vi
+ .fn()
+ .mockResolvedValueOnce({ timedOut: false, exitCode: 1, stdout: "" })
+ .mockResolvedValueOnce({ timedOut: false, exitCode: 0, stdout: "" });
+
+ await expect(
+ resumeNativeWorkspaceSync({
+ db: db as never,
+ runId: descriptor.binding.runId,
+ target: {
+ kind: "remote",
+ transport: "sandbox",
+ remoteCwd: descriptor.binding.remoteCwd,
+ sandboxLeaseAcquisition: {
+ providerLeaseId: descriptor.binding.providerLeaseId,
+ },
+ runner: { execute },
+ } as never,
+ }),
+ ).resolves.toBe(true);
+
+ expect(execute).toHaveBeenCalledTimes(2);
+ expect(persistedLeaseMetadata).toMatchObject({
+ retained: true,
+ nativeWorkspaceSync: {
+ schema: "paperclip.native-workspace-stamp/v1",
+ workspaceId: descriptor.binding.workspaceId,
+ providerLeaseId: descriptor.binding.providerLeaseId,
+ remoteCwd: descriptor.binding.remoteCwd,
+ hostSha256: finalHostSha256,
+ finalizedRunId: descriptor.binding.runId,
+ },
+ });
+ });
+});
diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts
index 5451d53c0c..f35b16fb40 100644
--- a/server/src/services/environment-runtime.ts
+++ b/server/src/services/environment-runtime.ts
@@ -2060,6 +2060,11 @@ function createSandboxEnvironmentDriver(
...(reusableLease?.metadata?.nativeHarnessBackup
? { nativeHarnessBackup: reusableLease.metadata.nativeHarnessBackup }
: {}),
+ ...(providerLease && reusableLease?.metadata?.nativeWorkspaceSync
+ ? {
+ nativeWorkspaceSync: reusableLease.metadata.nativeWorkspaceSync,
+ }
+ : {}),
...(reusableScope ? { reusableSandboxLease: reusableScope } : {}),
};
try {
@@ -2282,6 +2287,13 @@ function createSandboxEnvironmentDriver(
...(reusableLease?.metadata?.nativeHarnessBackup
? { nativeHarnessBackup: reusableLease.metadata.nativeHarnessBackup }
: {}),
+ ...(reusableLease &&
+ providerLease.providerLeaseId === reusableLease.providerLeaseId &&
+ reusableLease.metadata?.nativeWorkspaceSync
+ ? {
+ nativeWorkspaceSync: reusableLease.metadata.nativeWorkspaceSync,
+ }
+ : {}),
...(reusableScope ? { reusableSandboxLease: reusableScope } : {}),
};
try {
@@ -3154,6 +3166,7 @@ const INTERNAL_PLUGIN_SANDBOX_CONFIG_KEYS = new Set([
"sandboxProviderPlugin",
"sandboxLeaseAcquisition",
"nativeHarnessBackup",
+ "nativeWorkspaceSync",
]);
// Drop the host-internal and per-lease runtime keys from a sandbox config
diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts
index 195ad4810b..94a52139ee 100644
--- a/server/src/services/heartbeat.ts
+++ b/server/src/services/heartbeat.ts
@@ -142,6 +142,9 @@ import {
materializeLegacyQuestionResponseWakeProjection,
materializeNativeInteractionResponses,
NativeCancellationPendingRecoveryError,
+ prepareNativeWorkspaceSync,
+ readNativeWorkspaceSyncReference,
+ recordNativeFinalizationFailure,
type NativeRestartRecoveryClaim,
rebindNativeSessionCheckpoint,
reconcileNativeFinalizations,
@@ -1746,14 +1749,22 @@ export function leaseReleaseStatusForRunStatus(
return status === "failed" || status === "timed_out" ? "failed" : "released";
}
+export function providerResourceDispositionForTerminalRun(
+ desired: ProviderResourceDisposition | undefined,
+ status: string | null | undefined,
+): ProviderResourceDisposition | undefined {
+ return desired === "keep_running" && status !== "succeeded"
+ ? "stop_and_retain"
+ : desired;
+}
+
export interface NativeSandboxLifecycle {
runnerProcess: "per_turn" | "warm";
sandboxResource: "keep_running" | "stop_and_reuse" | "destroy_after_turn";
failoverBackup: "verified";
}
-export function resolveNativeSandboxLifecycle(input: {
- adapterType: string;
+export function resolveReusableSandboxLifecycle(input: {
lifecyclePolicy:
| { mode: "per_turn"; idleTimeoutMs: null }
| { mode: "warm"; idleTimeoutMs: number };
@@ -1764,12 +1775,9 @@ export function resolveNativeSandboxLifecycle(input: {
effectiveCapabilities?: { reusableLeases: boolean } | null;
} | null;
}): NativeSandboxLifecycle | null {
- if (
- input.adapterType !== "paperclip_runner" ||
- input.target?.kind !== "remote" ||
- input.target.transport !== "sandbox"
- )
+ if (input.target?.kind !== "remote" || input.target.transport !== "sandbox") {
return null;
+ }
const reusableLease =
input.target.reusableLeaseConfigured === true &&
input.target.effectiveCapabilities?.reusableLeases === true;
@@ -1788,6 +1796,27 @@ export function resolveNativeSandboxLifecycle(input: {
};
}
+export function resolveNativeSandboxLifecycle(input: {
+ adapterType: string;
+ lifecyclePolicy:
+ | { mode: "per_turn"; idleTimeoutMs: null }
+ | { mode: "warm"; idleTimeoutMs: number };
+ target: {
+ kind: "local" | "remote";
+ transport?: string;
+ reusableLeaseConfigured?: boolean;
+ effectiveCapabilities?: { reusableLeases: boolean } | null;
+ } | null;
+}): NativeSandboxLifecycle | null {
+ if (
+ input.adapterType !== "paperclip_runner" ||
+ input.target?.kind !== "remote" ||
+ input.target.transport !== "sandbox"
+ )
+ return null;
+ return resolveReusableSandboxLifecycle(input);
+}
+
export function applyPersistedExecutionWorkspaceConfig(input: {
config: Record<string, unknown>;
workspaceConfig: ExecutionWorkspaceConfig | null;
@@ -8170,6 +8199,18 @@ class NativeSessionResumeScheduledError extends Error {
}
}
+class NativeWorkspaceFinalizeScheduledError extends Error {
+ constructor(
+ readonly original: unknown,
+ readonly terminalFailure: boolean,
+ readonly reasonCode:
+ "workspace_sync_out_failed" | "workspace_sync_out_unrecoverable",
+ ) {
+ super("Native workspace finalization recovery has been scheduled.");
+ this.name = "NativeWorkspaceFinalizeScheduledError";
+ }
+}
+
type WorkspaceReadyCommentWriter = {
addComment: (
issueId: string,
@@ -12945,7 +12986,10 @@ export function heartbeatService(
// A result committed before the old controller stopped outranks process
// recovery. Finish its durable workspace/status suffix before deciding
// whether any provider authority needs to be reopened.
- await reconcileNativeFinalizations(db);
+ await reconcileNativeFinalizations(db, undefined, {
+ environmentRuntime,
+ onWorkspaceSettled: settleRecoveredNativeWorkspace,
+ });
const intent = await readHotRestartIntent().catch((error) => {
logger.warn(
{ err: error },
@@ -16854,6 +16898,41 @@ export function heartbeatService(
return blocked;
}
+ async function settleRecoveredNativeWorkspace(input: {
+ runId: string;
+ companyId: string;
+ agentId: string;
+ succeeded: boolean;
+ }) {
+ const settledRun = await getRun(input.runId);
+ const workspaceSyncReference = readNativeWorkspaceSyncReference(
+ parseObject(settledRun?.runnerProfileJson).nativeWorkspaceSync,
+ );
+ await releaseEnvironmentLeasesForRun({
+ runId: input.runId,
+ companyId: input.companyId,
+ agentId: input.agentId,
+ status: settledRun?.status,
+ failureReason: settledRun?.error ?? undefined,
+ providerResourceDisposition: input.succeeded
+ ? (workspaceSyncReference?.resourceDisposition ?? "stop_and_retain")
+ : "stop_and_retain",
+ });
+ await releaseRuntimeServicesForRun(input.runId).catch(() => undefined);
+ await finalizeAgentStatus(
+ input.agentId,
+ input.succeeded ? "succeeded" : "failed",
+ input.succeeded
+ ? null
+ : (settledRun?.error ?? "native_workspace_sync_out_failed"),
+ {
+ wasFirstHeartbeat: settledRun
+ ? timerClaimWasFirstHeartbeat(settledRun)
+ : undefined,
+ },
+ ).catch(() => undefined);
+ }
+
async function reapOrphanedRuns(opts?: { staleThresholdMs?: number }) {
const staleThresholdMs = opts?.staleThresholdMs ?? 0;
const now = new Date();
@@ -16861,7 +16940,10 @@ export function heartbeatService(
// Complete persisted native results before generic orphan recovery. The
// reconciler reads the durable workspace barrier and persisted runtime
// mode, never the current feature flag.
- await reconcileNativeFinalizations(db).catch((error) => {
+ await reconcileNativeFinalizations(db, undefined, {
+ environmentRuntime,
+ onWorkspaceSettled: settleRecoveredNativeWorkspace,
+ }).catch((error) => {
logger.warn(
{ err: error },
"failed to reconcile persisted native finalizations before orphan reaping",
@@ -17876,6 +17958,10 @@ export function heartbeatService(
activeRunExecutions.add(run.id);
let runScratch: HeartbeatRunScratch | null = null;
let nativeSessionResumeScheduled = false;
+ let nativeWorkspaceFinalizeScheduled = false;
+ let nativeWorkspaceSync: Awaited<
+ ReturnType<typeof prepareNativeWorkspaceSync>
+ > = null;
let providerResourceDispositionForRun:
ProviderResourceDisposition | undefined;
let nativeLifecycleTelemetryForRun:
@@ -19659,6 +19745,9 @@ export function heartbeatService(
driver: selectedEnvironment.driver,
leaseId: activeEnvironmentLease.lease.id,
workspaceRealization,
+ sandboxLeaseAcquisition:
+ activeEnvironmentLease.lease.metadata?.sandboxLeaseAcquisition ??
+ null,
...(typeof activeEnvironmentLease.lease.metadata?.remoteCwd === "string"
? {
remoteCwd: activeEnvironmentLease.lease.metadata.remoteCwd,
@@ -20363,6 +20452,9 @@ export function heartbeatService(
agentId: heartbeatRuns.agentId,
runnerInstanceId: heartbeatRuns.runnerInstanceId,
nativeSessionId: heartbeatRuns.nativeSessionId,
+ processPid: heartbeatRuns.processPid,
+ processGroupId: heartbeatRuns.processGroupId,
+ processStartedAt: heartbeatRuns.processStartedAt,
runnerProfileJson: heartbeatRuns.runnerProfileJson,
})
.from(heartbeatRuns)
@@ -20718,6 +20810,21 @@ export function heartbeatService(
? previousNativeRun.runnerInstanceId
: (lockedRun.runnerInstanceId ?? nativeRunnerInstanceId),
nativeSessionId: lockedRun.nativeSessionId ?? nativeSessionId,
+ processPid:
+ lockedRun.processPid ??
+ (previousNativeRun?.nativeSessionId === nativeSessionId
+ ? previousNativeRun.processPid
+ : null),
+ processGroupId:
+ lockedRun.processGroupId ??
+ (previousNativeRun?.nativeSessionId === nativeSessionId
+ ? previousNativeRun.processGroupId
+ : null),
+ processStartedAt:
+ lockedRun.processStartedAt ??
+ (previousNativeRun?.nativeSessionId === nativeSessionId
+ ? previousNativeRun.processStartedAt
+ : null),
nativeIssueId: lockedRun.nativeIssueId ?? issueRef.id,
driverKind:
lockedRun.driverKind ??
@@ -20745,7 +20852,30 @@ export function heartbeatService(
})
.onConflictDoNothing();
});
+ nativeWorkspaceSync = await prepareNativeWorkspaceSync({
+ db,
+ runId: run.id,
+ companyId: agent.companyId,
+ workspaceId: nativeExecutionWorkspaceId,
+ workspaceLocalDir: executionWorkspace.cwd,
+ target: executionTarget,
+ lease: activeEnvironmentLease.lease,
+ restartRecovery: runOptions.nativeRestartRecovery,
+ resourceDisposition: providerResourceDispositionForRun,
+ });
} else {
+ const legacyWarmLifecycle =
+ executionTarget?.kind === "remote" &&
+ executionTarget.transport === "sandbox" &&
+ executionTarget.runnerLifecyclePolicy?.mode === "warm"
+ ? resolveReusableSandboxLifecycle({
+ lifecyclePolicy: executionTarget.runnerLifecyclePolicy,
+ target: executionTarget,
+ })
+ : null;
+ if (legacyWarmLifecycle?.sandboxResource === "keep_running") {
+ providerResourceDispositionForRun = "keep_running";
+ }
await db
.update(heartbeatRuns)
.set({
@@ -21315,19 +21445,26 @@ export function heartbeatService(
// If recording the barrier itself fails, propagate as a run failure
// rather than silently leaving dependents stranded behind a missing
// finalize row.
+ if (nativeWorkspaceSync) {
+ await nativeWorkspaceSync.restoreWorkspace();
+ }
await recordWorkspaceFinalize("succeeded");
if (adapterResult.nativeFinalization) {
adapterResult.nativeFinalization.workspaceFinalizeStatus =
"succeeded";
try {
- await finalizeNativeRun({
+ const finalized = await finalizeNativeRun({
db,
runId: run.id,
workspaceFinalizeStatus: "succeeded",
+ preserveProviderAttempt: Boolean(nativeWorkspaceSync),
});
await dispatchPendingNativeStatusWakeups({
companyId: run.companyId,
});
+ if (finalized.phase === "committed") {
+ await nativeWorkspaceSync?.cleanup();
+ }
} catch (finalizeErr) {
logger.warn(
{ err: finalizeErr, runId: run.id },
@@ -21379,6 +21516,40 @@ export function heartbeatService(
);
}
if (nativeRuntimeResolution.kind === "native") {
+ const proposedResult = await db
+ .select({ resultId: nativeRunFinalizations.resultId })
+ .from(nativeRunFinalizations)
+ .where(eq(nativeRunFinalizations.runId, run.id))
+ .limit(1)
+ .then((rows) => rows[0]?.resultId ?? null);
+ if (proposedResult && nativeWorkspaceSync) {
+ const workspaceFailureMessage =
+ adapterErr instanceof Error ? adapterErr.message : "";
+ const unrecoverable =
+ workspaceFailureMessage ===
+ "workspace_sync_out_unrecoverable" ||
+ workspaceFailureMessage.includes("daytona_sandbox_not_found");
+ const failure = await recordNativeFinalizationFailure({
+ db,
+ runId: run.id,
+ error: new Error(
+ unrecoverable
+ ? "native_workspace_sync_out_unrecoverable"
+ : "native_workspace_sync_out_failed",
+ ),
+ projectRunStatus: true,
+ failureScope: "workspace",
+ permanent: unrecoverable,
+ });
+ nativeWorkspaceFinalizeScheduled = true;
+ throw new NativeWorkspaceFinalizeScheduledError(
+ adapterErr,
+ failure.phase === "terminal_failure",
+ unrecoverable
+ ? "workspace_sync_out_unrecoverable"
+ : "workspace_sync_out_failed",
+ );
+ }
try {
await finalizeNativeRun({
db,
@@ -22064,6 +22235,45 @@ export function heartbeatService(
}
return;
}
+ if (err instanceof NativeWorkspaceFinalizeScheduledError) {
+ const coordinator = await db
+ .select({
+ nextAttemptAt: nativeRunFinalizations.nextAttemptAt,
+ attempt: nativeRunFinalizations.attempt,
+ })
+ .from(nativeRunFinalizations)
+ .where(eq(nativeRunFinalizations.runId, run.id))
+ .limit(1)
+ .then((rows) => rows[0] ?? null);
+ await appendRunEvent(run, {
+ eventType: "lifecycle",
+ stream: "system",
+ level: err.terminalFailure ? "error" : "warn",
+ message: err.terminalFailure
+ ? "native result is durable, but the sandbox containing unexported workspace changes is unrecoverable"
+ : "native result is durable; workspace copy-back will retry without another provider turn",
+ payload: {
+ attempt: coordinator?.attempt ?? null,
+ nextAttemptAt: coordinator?.nextAttemptAt?.toISOString() ?? null,
+ fallbackSuppressed: true,
+ retryReasonCode: err.reasonCode,
+ },
+ }).catch(() => undefined);
+ if (err.terminalFailure) {
+ // The durable coordinator already failed the run, blocked the
+ // issue, and cleared its execution lock. Let ordinary teardown
+ // release the now-useless lease and return the agent to service.
+ nativeWorkspaceFinalizeScheduled = false;
+ providerResourceDispositionForRun = "stop_and_retain";
+ await finalizeAgentStatus(
+ run.agentId,
+ "failed",
+ "native_workspace_sync_out_unrecoverable",
+ { wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run) },
+ ).catch(() => undefined);
+ }
+ return;
+ }
const message = redactCurrentUserText(
err instanceof Error ? err.message : "Unknown adapter failure",
await getCurrentUserRedactionOptions(),
@@ -22456,7 +22666,11 @@ export function heartbeatService(
// terminal". When the teardown reaches this point with the run still
// running or queued, force a terminal status before the lease is
// released, so the UI never shows a finished task as "Live".
- if (latestRun && !nativeSessionResumeScheduled) {
+ if (
+ latestRun &&
+ !nativeSessionResumeScheduled &&
+ !nativeWorkspaceFinalizeScheduled
+ ) {
latestRun = await terminalizeRunOnLeaseRelease(latestRun).catch(
(terminalizeErr) => {
logger.error(
@@ -22467,7 +22681,15 @@ export function heartbeatService(
},
);
}
- if (!nativeSessionResumeScheduled) {
+ // Warm retention is earned only by a fully successful turn. A failed,
+ // cancelled, or timed-out run stops the reusable sandbox so the next
+ // acquisition must revalidate and explicitly resume it.
+ providerResourceDispositionForRun =
+ providerResourceDispositionForTerminalRun(
+ providerResourceDispositionForRun,
+ latestRun?.status,
+ );
+ if (!nativeSessionResumeScheduled && !nativeWorkspaceFinalizeScheduled) {
await releaseEnvironmentLeasesForRun({
runId: run.id,
companyId: run.companyId,
@@ -22539,7 +22761,11 @@ export function heartbeatService(
}
}
activeRunExecutions.delete(run.id);
- if (!nativeSessionResumeScheduled && !shutdownInProgress) {
+ if (
+ !nativeSessionResumeScheduled &&
+ !nativeWorkspaceFinalizeScheduled &&
+ !shutdownInProgress
+ ) {
await startNextQueuedRunForAgent(run.agentId);
}
}
diff --git a/server/src/services/native-runtime/index.ts b/server/src/services/native-runtime/index.ts
index f5edaaacb5..643e18b9a9 100644
--- a/server/src/services/native-runtime/index.ts
+++ b/server/src/services/native-runtime/index.ts
@@ -9,4 +9,5 @@ export * from "./paperclip-control-plane-port.js";
export * from "./native-run-finalizer.js";
export * from "./native-finalization-reconciler.js";
export * from "./native-restart-recovery.js";
+export * from "./native-workspace-sync.js";
export * from "./status-arbiter.js";
diff --git a/server/src/services/native-runtime/native-finalization-reconciler.ts b/server/src/services/native-runtime/native-finalization-reconciler.ts
index 4e7073c42c..294a92b985 100644
--- a/server/src/services/native-runtime/native-finalization-reconciler.ts
+++ b/server/src/services/native-runtime/native-finalization-reconciler.ts
@@ -24,6 +24,11 @@ import { issueRecoveryActionService } from "../issue-recovery-actions.js";
import { issueService } from "../issues.js";
import { emitAgentTaskRun } from "../agent-task-run-telemetry.js";
import { resumeNativeWorkspaceFinalization } from "./native-workspace-finalizer.js";
+import {
+ cleanupNativeWorkspaceSync,
+ readNativeWorkspaceSyncReference,
+} from "./native-workspace-sync.js";
+import type { EnvironmentRuntimeService } from "../environment-runtime.js";
import { classifyNativeEvidence } from "./evidence-classifier.js";
import { recordNativeWorkAssessment } from "./work-assessments.js";
import {
@@ -373,19 +378,33 @@ export async function claimNativeSessionResumptions(input: {
}
/** Recovery is keyed only by persisted mode/coordinator state, never the live flag. */
-export async function reconcileNativeFinalizations(db: Db, runIds?: string[]) {
- const rows = await db.select({
- runId: heartbeatRuns.id,
- companyId: heartbeatRuns.companyId,
- agentId: heartbeatRuns.agentId,
- issueId: nativeRunFinalizations.issueId,
- issueStatus: issues.status,
- issueStatusVersion: issues.statusVersion,
- issueDecisionId: issues.lastStatusDecisionId,
- coordinatorPhase: nativeRunFinalizations.phase,
- assessmentId: nativeRunFinalizations.assessmentId,
- decisionId: nativeRunFinalizations.decisionId,
- })
+export async function reconcileNativeFinalizations(
+ db: Db,
+ runIds?: string[],
+ options: {
+ environmentRuntime?: EnvironmentRuntimeService;
+ onWorkspaceSettled?: (input: {
+ runId: string;
+ companyId: string;
+ agentId: string;
+ succeeded: boolean;
+ }) => Promise<void>;
+ } = {},
+) {
+ const rows = await db
+ .select({
+ runId: heartbeatRuns.id,
+ companyId: heartbeatRuns.companyId,
+ agentId: heartbeatRuns.agentId,
+ issueId: nativeRunFinalizations.issueId,
+ issueStatus: issues.status,
+ issueStatusVersion: issues.statusVersion,
+ issueDecisionId: issues.lastStatusDecisionId,
+ coordinatorPhase: nativeRunFinalizations.phase,
+ assessmentId: nativeRunFinalizations.assessmentId,
+ decisionId: nativeRunFinalizations.decisionId,
+ runnerProfileJson: heartbeatRuns.runnerProfileJson,
+ })
.from(heartbeatRuns)
.innerJoin(nativeRunFinalizations, eq(nativeRunFinalizations.runId, heartbeatRuns.id))
.innerJoin(issues, and(
@@ -617,14 +636,59 @@ export async function reconcileNativeFinalizations(db: Db, runIds?: string[]) {
if (!recoveryDecision.effects.some((effect) => effect.kind === "resume_workspace_operation")) {
throw new Error("native_reconciliation_workspace_resume_policy_missing");
}
- const operation = await resumeNativeWorkspaceFinalization({ db, runId: row.runId });
- const workspaceFinalizeStatus = operation.status === "succeeded" ? "succeeded" : "failed";
+ const operation = await resumeNativeWorkspaceFinalization({
+ db,
+ runId: row.runId,
+ environmentRuntime: options.environmentRuntime,
+ });
+ const workspaceFinalizeStatus =
+ operation.status === "succeeded" ? "succeeded" : "failed";
+ if (workspaceFinalizeStatus === "failed") {
+ const unrecoverable = operation.stderrExcerpt?.includes(
+ "workspace_sync_out_unrecoverable",
+ );
+ const failure = await recordNativeFinalizationFailure({
+ db,
+ runId: row.runId,
+ error: new Error(
+ unrecoverable
+ ? "native_workspace_sync_out_unrecoverable"
+ : "native_workspace_sync_out_failed",
+ ),
+ projectRunStatus: true,
+ failureScope: "workspace",
+ permanent: unrecoverable,
+ });
+ results.push({
+ ...failure,
+ reconciliationAction: "resume_workspace_operation" as const,
+ workspaceOperationId: operation.id,
+ workspaceFinalizeStatus,
+ reconciliationDecision: recoveryDecision,
+ });
+ if (failure.phase === "terminal_failure") {
+ await options
+ .onWorkspaceSettled?.({
+ runId: row.runId,
+ companyId: row.companyId,
+ agentId: row.agentId,
+ succeeded: false,
+ })
+ .catch(() => undefined);
+ }
+ continue;
+ }
try {
const finalized = await finalizeNativeRun({
db,
runId: row.runId,
workspaceFinalizeStatus,
projectRunStatus: true,
+ preserveProviderAttempt: Boolean(
+ readNativeWorkspaceSyncReference(
+ record(row.runnerProfileJson).nativeWorkspaceSync,
+ ),
+ ),
});
results.push({
...finalized,
@@ -633,6 +697,20 @@ export async function reconcileNativeFinalizations(db: Db, runIds?: string[]) {
workspaceFinalizeStatus,
reconciliationDecision: recoveryDecision,
});
+ if (
+ workspaceFinalizeStatus === "succeeded" &&
+ finalized.phase === "committed"
+ ) {
+ await cleanupNativeWorkspaceSync(row.runId).catch(() => undefined);
+ await options
+ .onWorkspaceSettled?.({
+ runId: row.runId,
+ companyId: row.companyId,
+ agentId: row.agentId,
+ succeeded: true,
+ })
+ .catch(() => undefined);
+ }
} catch (error) {
const failure = await recordNativeFinalizationFailure({
db,
@@ -659,12 +737,29 @@ export async function reconcileNativeFinalizations(db: Db, runIds?: string[]) {
throw new Error("native_reconciliation_replay_policy_invalid");
}
try {
- results.push(await finalizeNativeRun({
+ const finalized = await finalizeNativeRun({
db,
runId: row.runId,
workspaceFinalizeStatus: barrier.status as "succeeded" | "failed",
projectRunStatus: true,
- }));
+ preserveProviderAttempt: Boolean(
+ readNativeWorkspaceSyncReference(
+ record(row.runnerProfileJson).nativeWorkspaceSync,
+ ),
+ ),
+ });
+ results.push(finalized);
+ if (finalized.phase === "committed") {
+ await cleanupNativeWorkspaceSync(row.runId).catch(() => undefined);
+ await options
+ .onWorkspaceSettled?.({
+ runId: row.runId,
+ companyId: row.companyId,
+ agentId: row.agentId,
+ succeeded: true,
+ })
+ .catch(() => undefined);
+ }
} catch (error) {
results.push(await recordNativeFinalizationFailure({
db,
diff --git a/server/src/services/native-runtime/native-run-finalizer-telemetry.test.ts b/server/src/services/native-runtime/native-run-finalizer-telemetry.test.ts
index ab0446d59a..c06e11c78b 100644
--- a/server/src/services/native-runtime/native-run-finalizer-telemetry.test.ts
+++ b/server/src/services/native-runtime/native-run-finalizer-telemetry.test.ts
@@ -279,6 +279,93 @@ describeEmbeddedPostgres("native run finalizer / status decision committer — a
expect(run?.status).toBe("running");
});
+ it("bounds workspace-only retries without consuming the provider attempt", async () => {
+ const fixture = await seedNativeRun();
+ await db.insert(nativeRunFinalizations).values({
+ runId: fixture.runId,
+ companyId,
+ issueId: fixture.issueId,
+ phase: "observed",
+ attempt: 1,
+ });
+
+ for (
+ let expectedAttempt = 1;
+ expectedAttempt <= 3;
+ expectedAttempt += 1
+ ) {
+ await recordNativeFinalizationFailure({
+ db,
+ runId: fixture.runId,
+ error: new Error("native_workspace_sync_out_failed"),
+ projectRunStatus: true,
+ failureScope: "workspace",
+ });
+ const coordinator = await db
+ .select()
+ .from(nativeRunFinalizations)
+ .where(eq(nativeRunFinalizations.runId, fixture.runId))
+ .then((rows) => rows[0]!);
+ expect(coordinator.attempt).toBe(1);
+ expect(coordinator.failureDetail).toMatchObject({
+ workspaceFinalizeAttempt: expectedAttempt,
+ });
+ expect(coordinator.phase).toBe(
+ expectedAttempt === 3 ? "terminal_failure" : "retryable_failure",
+ );
+ }
+
+ await expect(
+ db
+ .select({ status: heartbeatRuns.status })
+ .from(heartbeatRuns)
+ .where(eq(heartbeatRuns.id, fixture.runId))
+ .then((rows) => rows[0]?.status),
+ ).resolves.toBe("failed");
+ });
+
+ it("blocks immediately when the sandbox with unexported changes is gone", async () => {
+ const fixture = await seedNativeRun();
+ await db.insert(nativeRunFinalizations).values({
+ runId: fixture.runId,
+ companyId,
+ issueId: fixture.issueId,
+ phase: "workspace_finalizing",
+ attempt: 1,
+ resultId: null,
+ });
+
+ const failure = await recordNativeFinalizationFailure({
+ db,
+ runId: fixture.runId,
+ error: new Error("native_workspace_sync_out_unrecoverable"),
+ projectRunStatus: true,
+ failureScope: "workspace",
+ permanent: true,
+ });
+
+ expect(failure).toMatchObject({
+ phase: "terminal_failure",
+ failureCode: "native_workspace_sync_out_unrecoverable",
+ nextAttemptAt: null,
+ attempt: 1,
+ });
+ await expect(
+ db
+ .select({ status: heartbeatRuns.status })
+ .from(heartbeatRuns)
+ .where(eq(heartbeatRuns.id, fixture.runId))
+ .then((rows) => rows[0]?.status),
+ ).resolves.toBe("failed");
+ await expect(
+ db
+ .select({ status: issues.status })
+ .from(issues)
+ .where(eq(issues.id, fixture.issueId))
+ .then((rows) => rows[0]?.status),
+ ).resolves.toBe("blocked");
+ });
+
it("emits exactly one event for a cancel_continuations write (trap 2: :685/:518 overlap)", async () => {
const fixture = await seedNativeRun();
// Build the minimal real rows commitNativeStatusDecision's foreign keys
@@ -329,7 +416,10 @@ describeEmbeddedPostgres("native run finalizer / status decision committer — a
toStatus: "cancelled",
reasonCode: "cancellation_issue_authorized",
unblockDescriptor: null,
- effects: [{ kind: "release_checkout" }, { kind: "cancel_continuations" }],
+ effects: [
+ { kind: "release_checkout" },
+ { kind: "cancel_continuations" },
+ ],
};
const callsBefore = mockTelemetryClient.track.mock.calls.length;
diff --git a/server/src/services/native-runtime/native-run-finalizer.ts b/server/src/services/native-runtime/native-run-finalizer.ts
index 979013c4d5..6042c748fb 100644
--- a/server/src/services/native-runtime/native-run-finalizer.ts
+++ b/server/src/services/native-runtime/native-run-finalizer.ts
@@ -126,7 +126,11 @@ async function acceptedInteractionFromRun(input: {
.then((rows) => rows[0] ?? null);
}
-async function claimCoordinator(input: { db: Db; runId: string }) {
+async function claimCoordinator(input: {
+ db: Db;
+ runId: string;
+ preserveProviderAttempt?: boolean;
+}) {
const leaseOwner = `native-finalizer:${randomUUID()}`;
const now = new Date();
const claimed = await input.db.transaction(async (tx) => {
@@ -152,23 +156,33 @@ async function claimCoordinator(input: { db: Db; runId: string }) {
}
if (coordinator.phase === "terminal_failure") throw new Error("native_finalization_terminal_failure");
if (
- coordinator.leaseOwner
- && coordinator.leaseExpiresAt
- && coordinator.leaseExpiresAt > now
- && coordinator.leaseOwner !== leaseOwner
- ) throw new Error("native_finalization_lease_busy");
- const [updated] = await tx.update(nativeRunFinalizations).set({
- leaseOwner,
- leaseExpiresAt: new Date(now.getTime() + 5 * 60_000),
- attempt: coordinator.attempt + 1,
- phase: coordinator.phase === "retryable_failure"
- ? coordinator.assessmentId ? "arbitrating" : "workspace_finalizing"
- : coordinator.phase,
- failureCode: null,
- failureDetail: null,
- nextAttemptAt: null,
- updatedAt: now,
- }).where(eq(nativeRunFinalizations.runId, input.runId)).returning();
+ coordinator.leaseOwner &&
+ coordinator.leaseExpiresAt &&
+ coordinator.leaseExpiresAt > now &&
+ coordinator.leaseOwner !== leaseOwner
+ )
+ throw new Error("native_finalization_lease_busy");
+ const [updated] = await tx
+ .update(nativeRunFinalizations)
+ .set({
+ leaseOwner,
+ leaseExpiresAt: new Date(now.getTime() + 5 * 60_000),
+ attempt: input.preserveProviderAttempt
+ ? coordinator.attempt
+ : coordinator.attempt + 1,
+ phase:
+ coordinator.phase === "retryable_failure"
+ ? coordinator.assessmentId
+ ? "arbitrating"
+ : "workspace_finalizing"
+ : coordinator.phase,
+ failureCode: null,
+ failureDetail: null,
+ nextAttemptAt: null,
+ updatedAt: now,
+ })
+ .where(eq(nativeRunFinalizations.runId, input.runId))
+ .returning();
if (!updated) throw new Error("native_finalization_claim_failed");
return { coordinator: updated, leaseOwner };
});
@@ -183,6 +197,8 @@ async function recordRetryableFailure(input: {
message: string;
nextAction: string;
projectRunStatus?: boolean;
+ failureScope?: "provider" | "workspace";
+ permanent?: boolean;
}) {
const now = new Date();
const nextAttemptAt = new Date(now.getTime() + 30_000);
@@ -228,50 +244,87 @@ async function recordRetryableFailure(input: {
const supersededByNewerRun = Boolean(
latestDecisionRun && latestDecisionRun.createdAt > input.run.createdAt,
);
- const exhausted = input.coordinator.attempt >= 3;
- const phase = supersededByNewerRun || exhausted
- ? "terminal_failure" as const
- : "retryable_failure" as const;
+ const priorFailureDetail = record(input.coordinator.failureDetail);
+ const workspaceFinalizeAttempt =
+ input.failureScope === "workspace"
+ ? (typeof priorFailureDetail.workspaceFinalizeAttempt === "number" &&
+ Number.isInteger(priorFailureDetail.workspaceFinalizeAttempt) &&
+ priorFailureDetail.workspaceFinalizeAttempt >= 0
+ ? priorFailureDetail.workspaceFinalizeAttempt
+ : 0) + 1
+ : null;
+ const exhausted =
+ input.permanent === true ||
+ (workspaceFinalizeAttempt !== null
+ ? workspaceFinalizeAttempt >= 3
+ : input.coordinator.attempt >= 3);
+ const phase =
+ supersededByNewerRun || exhausted
+ ? ("terminal_failure" as const)
+ : ("retryable_failure" as const);
const failureCode = supersededByNewerRun
? "native_finalization_superseded"
- : exhausted
- ? "native_finalization_retry_exhausted"
- : input.failureCode;
- await tx.update(nativeRunFinalizations).set({
- phase,
- leaseOwner: null,
- leaseExpiresAt: null,
- failureCode,
- failureDetail: {
- message: input.message.slice(0, 2_000),
- originalFailureCode: input.failureCode,
- recoveryOwner: supersededByNewerRun
- ? { kind: "none", reason: "newer_native_decision" }
- : exhausted
- ? { kind: "board" }
- : { kind: "agent", agentId: input.run.agentId },
- nextAction: input.nextAction,
- },
- nextAttemptAt: supersededByNewerRun || exhausted ? null : nextAttemptAt,
- updatedAt: now,
- }).where(eq(nativeRunFinalizations.runId, input.run.id));
- const projectsTerminalStatus = exhausted && !supersededByNewerRun && input.projectRunStatus;
- const [updatedRun] = await tx.update(heartbeatRuns).set({
- ...(projectsTerminalStatus ? {
- status: exhaustedRunStatus,
- finishedAt: input.run.finishedAt ?? now,
- } : {}),
- nativePhase: phase,
- nativePhaseUpdatedAt: now,
- resultJson: {
- ...record(input.run.resultJson),
- finalizationPhase: phase,
+ : input.permanent
+ ? input.failureCode
+ : exhausted
+ ? input.failureScope === "workspace"
+ ? "native_workspace_sync_out_retry_exhausted"
+ : "native_finalization_retry_exhausted"
+ : input.failureCode;
+ await tx
+ .update(nativeRunFinalizations)
+ .set({
+ phase,
+ leaseOwner: null,
+ leaseExpiresAt: null,
failureCode,
- originalFailureCode: input.failureCode,
- nextAttemptAt: supersededByNewerRun || exhausted ? null : nextAttemptAt.toISOString(),
- },
- updatedAt: now,
- }).where(eq(heartbeatRuns.id, input.run.id)).returning();
+ failureDetail: {
+ message: input.message.slice(0, 2_000),
+ originalFailureCode: input.failureCode,
+ ...(workspaceFinalizeAttempt === null
+ ? {}
+ : { workspaceFinalizeAttempt }),
+ recoveryOwner: supersededByNewerRun
+ ? { kind: "none", reason: "newer_native_decision" }
+ : exhausted
+ ? { kind: "board" }
+ : { kind: "agent", agentId: input.run.agentId },
+ nextAction: input.nextAction,
+ },
+ nextAttemptAt: supersededByNewerRun || exhausted ? null : nextAttemptAt,
+ updatedAt: now,
+ })
+ .where(eq(nativeRunFinalizations.runId, input.run.id));
+ const projectsTerminalStatus =
+ exhausted && !supersededByNewerRun && input.projectRunStatus;
+ const [updatedRun] = await tx
+ .update(heartbeatRuns)
+ .set({
+ ...(projectsTerminalStatus
+ ? {
+ status:
+ input.failureScope === "workspace"
+ ? "failed"
+ : exhaustedRunStatus,
+ finishedAt: input.run.finishedAt ?? now,
+ }
+ : {}),
+ nativePhase: phase,
+ nativePhaseUpdatedAt: now,
+ resultJson: {
+ ...record(input.run.resultJson),
+ finalizationPhase: phase,
+ failureCode,
+ originalFailureCode: input.failureCode,
+ nextAttemptAt:
+ supersededByNewerRun || exhausted
+ ? null
+ : nextAttemptAt.toISOString(),
+ },
+ updatedAt: now,
+ })
+ .where(eq(heartbeatRuns.id, input.run.id))
+ .returning();
if (projectsTerminalStatus) terminalRunToEmit = updatedRun ?? null;
if (supersededByNewerRun) {
await issueRecoveryActionService(tx as unknown as Db).resolveActiveForIssue({
@@ -284,9 +337,16 @@ async function recordRetryableFailure(input: {
resolutionNote: "A newer native run already committed the authoritative issue decision; the stale finalizer was retired without changing issue state.",
}, tx);
} else if (exhausted) {
- await issueService(tx as unknown as Db).update(input.coordinator.issueId, {
- status: "in_review",
- }, tx);
+ await issueService(tx as unknown as Db).update(
+ input.coordinator.issueId,
+ {
+ status:
+ input.permanent && input.failureScope === "workspace"
+ ? "blocked"
+ : "in_review",
+ },
+ tx,
+ );
}
if (!supersededByNewerRun) {
await issueRecoveryActionService(tx as unknown as Db).upsertSourceScoped({
@@ -301,10 +361,15 @@ async function recordRetryableFailure(input: {
evidence: {
runId: input.run.id,
coordinatorAttempt: input.coordinator.attempt,
+ ...(workspaceFinalizeAttempt === null
+ ? {}
+ : { workspaceFinalizeAttempt }),
originalFailureCode: input.failureCode,
},
nextAction: exhausted
- ? `Finalization retry budget exhausted. ${input.nextAction}`
+ ? input.permanent
+ ? input.nextAction
+ : `Finalization retry budget exhausted. ${input.nextAction}`
: input.nextAction,
wakePolicy: exhausted
? null
@@ -328,6 +393,8 @@ export async function recordNativeFinalizationFailure(input: {
runId: string;
error: unknown;
projectRunStatus?: boolean;
+ failureScope?: "provider" | "workspace";
+ permanent?: boolean;
}) {
const [run, coordinator] = await Promise.all([
input.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId))
@@ -344,8 +411,15 @@ export async function recordNativeFinalizationFailure(input: {
coordinator,
failureCode,
message,
- nextAction: "Repair the persisted native result or contract discriminator, then resume finalization from the coordinator.",
+ nextAction:
+ input.failureScope === "workspace"
+ ? input.permanent
+ ? "Restore the exact sandbox containing the unexported workspace changes, or resolve the task manually from durable evidence."
+ : "Retry workspace export and merge from the retained sandbox; do not submit another provider turn."
+ : "Repair the persisted native result or contract discriminator, then resume finalization from the coordinator.",
projectRunStatus: input.projectRunStatus,
+ failureScope: input.failureScope,
+ permanent: input.permanent,
});
}
@@ -395,12 +469,23 @@ export async function finalizeNativeRun(input: {
workspaceFinalizeStatus: "succeeded" | "failed";
/** Reconciliation owns terminal run projection; the live heartbeat does it afterward. */
projectRunStatus?: boolean;
+ /** Workspace-only replay must not consume the provider recovery budget. */
+ preserveProviderAttempt?: boolean;
failpoint?: NativeStatusCommitFailpoint;
}) {
- const run = await input.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId))
- .limit(1).then((rows) => rows[0] ?? null);
- if (!run || run.runtimeMode !== "native") throw new Error("native_finalization_run_missing");
- const claim = await claimCoordinator({ db: input.db, runId: input.runId });
+ const run = await input.db
+ .select()
+ .from(heartbeatRuns)
+ .where(eq(heartbeatRuns.id, input.runId))
+ .limit(1)
+ .then((rows) => rows[0] ?? null);
+ if (!run || run.runtimeMode !== "native")
+ throw new Error("native_finalization_run_missing");
+ const claim = await claimCoordinator({
+ db: input.db,
+ runId: input.runId,
+ preserveProviderAttempt: input.preserveProviderAttempt,
+ });
const coordinator = claim.coordinator;
if (!claim.leaseOwner && coordinator.phase === "committed") {
if (input.projectRunStatus) await projectCommittedRun({ db: input.db, run, coordinator });
diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts
index 7a9c857026..e6e70a53d5 100644
--- a/server/src/services/native-runtime/native-session-executor.test.ts
+++ b/server/src/services/native-runtime/native-session-executor.test.ts
@@ -2880,7 +2880,7 @@ describe("native warm session supervision", () => {
);
});
- it("rehydrates a runnerd warm session from its checkpoint under a fresh run authority", async () => {
+ it("reattaches a live runnerd warm session under a fresh run authority", async () => {
const stateBase = await mkdtemp(
join(tmpdir(), "paperclip-runnerd-warm-authority-"),
);
@@ -2889,9 +2889,7 @@ describe("native warm session supervision", () => {
process.env.PAPERCLIP_RUNNER_STATE_DIR = stateBase;
process.env.PAPERCLIP_HOME = stateBase;
const firstClose = vi.fn(async () => undefined);
- const secondClose = vi.fn(async () => undefined);
const firstSession = { close: firstClose };
- const secondSession = { close: secondClose };
const first = {
...execution,
binding: {
@@ -2947,18 +2945,8 @@ describe("native warm session supervision", () => {
return result;
})
.mockImplementationOnce(async (options) => {
- expect(options.existingSession).toBeUndefined();
- expect(options.persistedSession).toEqual(
- expect.objectContaining({
- identity: expect.objectContaining({
- runId: second.binding.runId,
- sessionId: second.session.normalizedSessionId,
- }),
- providerSessionId: "provider-runnerd-warm",
- activeTurnId: null,
- }),
- );
- options.onSession?.(secondSession);
+ expect(options.existingSession).toBe(firstSession);
+ expect(options.persistedSession).toBeUndefined();
return result;
});
@@ -3014,10 +3002,10 @@ describe("native warm session supervision", () => {
runnerInstanceId: "runner-runnerd-warm",
useRunnerd: true,
});
- expect(firstClose).toHaveBeenCalledWith({
- reason: "warm native session authority epoch rotated",
- });
- await vi.waitFor(() => expect(secondClose).toHaveBeenCalled(), {
+ expect(firstClose).not.toHaveBeenCalled();
+ await vi.waitFor(() => expect(firstClose).toHaveBeenCalledWith({
+ reason: "warm native session idle timeout",
+ }), {
timeout: 500,
});
} finally {
@@ -5392,9 +5380,13 @@ describe("runnerd provider runtime wiring", () => {
const firstOptions = state.createBackend.mock.calls[0]![1];
const continuationOptions = state.createBackend.mock.calls[1]![1];
- await expect(firstOptions.dynamicToolHandler!({})).rejects.toThrow(
- "native_tool_authority_epoch_revoked",
- );
+ // The retained runner backend owns one stable callback. After run.attach,
+ // that callback routes through the session-scope authority registry to
+ // the new run; stale provider calls are rejected earlier by runnerd's
+ // turn identity boundary.
+ await expect(firstOptions.dynamicToolHandler!({})).resolves.toEqual({
+ runId: continuation.binding.runId,
+ });
await expect(
continuationOptions.dynamicToolHandler!({}),
).resolves.toEqual({ runId: continuation.binding.runId });
diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts
index f7fc0f2a3e..5af22bd551 100644
--- a/server/src/services/native-runtime/native-session-executor.ts
+++ b/server/src/services/native-runtime/native-session-executor.ts
@@ -2623,7 +2623,13 @@ export async function nativeProviderRecoveryEvidence(input: {
const durableEvents = await input.db
.select({ eventType: heartbeatRunEvents.eventType })
.from(heartbeatRunEvents)
- .where(eq(heartbeatRunEvents.runId, input.runId));
+ .where(
+ and(
+ eq(heartbeatRunEvents.runId, input.runId),
+ inArray(heartbeatRunEvents.eventType, [...PROVIDER_DURABLE_EVENT_TYPES]),
+ ),
+ )
+ .limit(1);
const providerEventsExist = durableEvents.some((event) =>
PROVIDER_DURABLE_EVENT_TYPES.has(event.eventType),
);
@@ -4171,23 +4177,7 @@ async function executePaperclipNativeSessionWithinScope(
entry.busy = true;
if (entry.idleTimer !== null) clearTimeout(entry.idleTimer);
entry.idleTimer = null;
- if (input.useRunnerd) {
- // A retained driver closes over the tool authority from the run that
- // created it. Preserve provider continuity through its durable
- // checkpoint, but rebuild the runnerd backend so the next run gets a
- // fresh, immutable authority epoch. Reusing the live driver would
- // either retain stale authority or require rebinding old callbacks.
- warmNativeSessions.delete(warmSessionId);
- await entry.session.close({
- reason: "warm native session authority epoch rotated",
- });
- persistedWarmSession = loadWarmNativeCheckpoint(
- input.execution,
- warmConfigDigest,
- );
- } else {
- existingWarmSession = entry.session;
- }
+ existingWarmSession = entry.session;
}
} else {
persistedWarmSession = loadWarmNativeCheckpoint(
@@ -7235,6 +7225,13 @@ async function createRunnerdBackendWithinSessionClaim(
input.restartRecovery?.kind === "reattach_existing_runner"
? input.restartRecovery.process
: null;
+ const executeCurrentToolAuthority = (
+ call: Parameters<SessionToolAuthorityEpoch["execute"]>[0],
+ ) => {
+ const current = sessionToolAuthorityEpochs.get(sessionScopeId);
+ if (!current) throw new Error("native_session_tool_authority_unavailable");
+ return current.execute(call);
+ };
const backend = createNativeSessionBackend(runnerExecution, {
runnerInstanceId: input.runnerInstanceId,
environment: effectiveRunnerEnvironment,
@@ -7243,8 +7240,8 @@ async function createRunnerdBackendWithinSessionClaim(
: "local_filesystem",
onSpawn: input.onSpawn,
dynamicTools,
- dynamicToolHandler: (call) => authorityEpoch.execute(call),
- acpxDynamicToolHandler: (call) => authorityEpoch.execute(call),
+ dynamicToolHandler: executeCurrentToolAuthority,
+ acpxDynamicToolHandler: executeCurrentToolAuthority,
opencodeRuntimeDirectory: resolve(
resolvePaperclipInstanceRoot(),
"runtime",
@@ -7419,7 +7416,7 @@ async function createRunnerdBackendWithinSessionClaim(
turnId: `turn-${input.execution.binding.runId}`,
itemId: `item-${input.execution.binding.runId}`,
},
- controlPlaneRegistration: (authority) =>
+ controlPlaneRegistration: (authority, attachmentIdentity) =>
measureNativeRunnerSpan(
input.trace,
"runner.transport.connect",
@@ -7446,7 +7443,9 @@ async function createRunnerdBackendWithinSessionClaim(
() =>
registerRunnerPrpAuthority({
companyId: input.execution.binding.companyId,
- runId: input.execution.binding.runId,
+ runId:
+ attachmentIdentity?.runId ??
+ input.execution.binding.runId,
authority,
}),
);
@@ -7469,7 +7468,9 @@ async function createRunnerdBackendWithinSessionClaim(
() =>
resolvePaperclipRunnerTransport({
target,
- runId: input.execution.binding.runId,
+ runId:
+ attachmentIdentity?.runId ??
+ input.execution.binding.runId,
localConnectUrl: "ws://127.0.0.1/unused",
runnerPublicUrl: input.runnerPublicUrl,
runnerCaBundlePath: input.runnerCaBundlePath,
@@ -7505,7 +7506,9 @@ async function createRunnerdBackendWithinSessionClaim(
() =>
resolvePaperclipRunnerTransport({
target,
- runId: input.execution.binding.runId,
+ runId:
+ attachmentIdentity?.runId ??
+ input.execution.binding.runId,
localConnectUrl: "ws://127.0.0.1/unused",
runnerPublicUrl: input.runnerPublicUrl,
runnerCaBundlePath: input.runnerCaBundlePath,
@@ -7541,7 +7544,9 @@ async function createRunnerdBackendWithinSessionClaim(
() =>
registerRunnerPrpAuthority({
companyId: input.execution.binding.companyId,
- runId: input.execution.binding.runId,
+ runId:
+ attachmentIdentity?.runId ??
+ input.execution.binding.runId,
authority,
}),
{ parentName: "runner.transport.connect" },
diff --git a/server/src/services/native-runtime/native-workspace-finalizer.ts b/server/src/services/native-runtime/native-workspace-finalizer.ts
index 49e87e7795..edec11741c 100644
--- a/server/src/services/native-runtime/native-workspace-finalizer.ts
+++ b/server/src/services/native-runtime/native-workspace-finalizer.ts
@@ -10,6 +10,13 @@ import {
} from "@paperclipai/db";
import { workspaceOperationService } from "../workspace-operations.js";
import { inspectManagedGitWorktreeBranch } from "../workspace-runtime.js";
+import { environmentService } from "../environments.js";
+import type { EnvironmentRuntimeService } from "../environment-runtime.js";
+import { resolveEnvironmentExecutionTarget } from "../environment-execution-target.js";
+import {
+ readNativeWorkspaceSyncReference,
+ resumeNativeWorkspaceSync,
+} from "./native-workspace-sync.js";
function record(value: unknown): Record<string, unknown> {
return value && typeof value === "object" && !Array.isArray(value)
@@ -21,14 +28,27 @@ function readString(value: unknown) {
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
}
+function workspaceSyncFailure(
+ code: "workspace_sync_out_failed" | "workspace_sync_out_unrecoverable",
+) {
+ return {
+ status: "failed" as const,
+ exitCode: 1,
+ stderr: `${code}\n`,
+ metadata: { workspaceSync: { code } },
+ };
+}
+
/**
* Resume the real workspace-finalization action for a result-bearing native
- * run. This service observes the workspace; it never fabricates a successful
- * marker. The caller decides whether a failed observation is retryable.
+ * run. Durable sandbox-backed runs export and merge the remote workspace;
+ * older runs retain the local workspace validation path. The caller owns the
+ * durable retry and failure policy.
*/
export async function resumeNativeWorkspaceFinalization(input: {
db: Db;
runId: string;
+ environmentRuntime?: EnvironmentRuntimeService;
}) {
const bound = await input.db.select({
companyId: heartbeatRuns.companyId,
@@ -57,7 +77,12 @@ export async function resumeNativeWorkspaceFinalization(input: {
eq(workspaceOperations.phase, "workspace_finalize"),
)).orderBy(desc(workspaceOperations.createdAt)).limit(1).then((rows) => rows[0] ?? null);
- const persistedInput = record(record(bound.runnerProfileJson).nativeExecutionInput);
+ const persistedInput = record(
+ record(bound.runnerProfileJson).nativeExecutionInput,
+ );
+ const nativeWorkspaceSync = readNativeWorkspaceSyncReference(
+ record(bound.runnerProfileJson).nativeWorkspaceSync,
+ );
const binding = record(persistedInput.binding);
const workspaceId = previous?.executionWorkspaceId
?? bound.issueWorkspaceId
@@ -87,6 +112,88 @@ export async function resumeNativeWorkspaceFinalization(input: {
: "workspace_directory",
},
run: async () => {
+ if (nativeWorkspaceSync) {
+ if (!input.environmentRuntime) {
+ return {
+ status: "failed",
+ exitCode: 1,
+ stderr:
+ "Native workspace finalization cannot access the environment runtime.\n",
+ };
+ }
+ const environmentsSvc = environmentService(input.db);
+ const lease = await environmentsSvc.getLeaseById(
+ nativeWorkspaceSync.leaseId,
+ );
+ const environment = lease?.environmentId
+ ? await environmentsSvc.getById(lease.environmentId)
+ : null;
+ if (
+ !lease ||
+ !environment ||
+ lease.companyId !== bound.companyId ||
+ environment.id !== lease.environmentId
+ ) {
+ return workspaceSyncFailure("workspace_sync_out_unrecoverable");
+ }
+ if (
+ lease.status === "expired" ||
+ lease.status === "failed" ||
+ lease.status === "pending_cleanup" ||
+ !lease.providerLeaseId ||
+ lease.providerLeaseId !== nativeWorkspaceSync.providerLeaseId
+ ) {
+ return workspaceSyncFailure("workspace_sync_out_unrecoverable");
+ }
+ const target = await resolveEnvironmentExecutionTarget({
+ db: input.db,
+ companyId: bound.companyId,
+ adapterType: "paperclip_runner",
+ environment,
+ leaseId: lease.id,
+ leaseMetadata: lease.metadata,
+ lease,
+ environmentRuntime: input.environmentRuntime,
+ });
+ if (!target) {
+ return {
+ status: "failed",
+ exitCode: 1,
+ stderr: "Native workspace finalization target is unavailable.\n",
+ };
+ }
+ try {
+ const restored = await resumeNativeWorkspaceSync({
+ db: input.db,
+ runId: input.runId,
+ target,
+ });
+ if (!restored) {
+ return workspaceSyncFailure("workspace_sync_out_unrecoverable");
+ }
+ return {
+ status: "succeeded",
+ exitCode: 0,
+ system:
+ "Native workspace finalization restored the remote workspace.\n",
+ metadata: {
+ workspaceSync: {
+ schema: nativeWorkspaceSync.schema,
+ workspaceId: nativeWorkspaceSync.workspaceId,
+ leaseId: nativeWorkspaceSync.leaseId,
+ },
+ },
+ };
+ } catch (error) {
+ const code =
+ error instanceof Error &&
+ (error.message === "workspace_sync_out_unrecoverable" ||
+ error.message.includes("daytona_sandbox_not_found"))
+ ? "workspace_sync_out_unrecoverable"
+ : "workspace_sync_out_failed";
+ return workspaceSyncFailure(code);
+ }
+ }
if (!cwd) {
return {
status: "failed",
diff --git a/server/src/services/native-runtime/native-workspace-sync.ts b/server/src/services/native-runtime/native-workspace-sync.ts
new file mode 100644
index 0000000000..e8f9f95884
--- /dev/null
+++ b/server/src/services/native-runtime/native-workspace-sync.ts
@@ -0,0 +1,1004 @@
+import fs from "node:fs/promises";
+import { createReadStream } from "node:fs";
+import path from "node:path";
+import { createHash, randomUUID } from "node:crypto";
+import { eq } from "drizzle-orm";
+import type { Db } from "@paperclipai/db";
+import { environmentLeases, heartbeatRuns } from "@paperclipai/db";
+import type { EnvironmentLease } from "@paperclipai/shared";
+import {
+ prepareAdapterExecutionTargetRuntime,
+ type AdapterExecutionTarget,
+ type PreparedAdapterExecutionTargetRuntime,
+} from "@paperclipai/adapter-utils/execution-target";
+import type { GitWorkspaceSnapshot } from "@paperclipai/adapter-utils/git-workspace-sync";
+import {
+ directorySnapshotSha256,
+ parseDirectorySnapshot,
+ serializeDirectorySnapshot,
+ type DirectorySnapshot,
+ type SerializedDirectorySnapshot,
+} from "@paperclipai/adapter-utils/workspace-restore-merge";
+import type {
+ WorkspaceDurableSeedPaths,
+ WorkspaceInboundMode,
+} from "@paperclipai/adapter-utils/sandbox-managed-runtime";
+import { resolvePaperclipInstanceRoot } from "../../home-paths.js";
+import { parseObject } from "../../adapters/utils.js";
+import type { NativeRestartRecoveryClaim } from "./native-restart-recovery.js";
+
+const DESCRIPTOR_SCHEMA = "paperclip.native-workspace-sync/v1";
+const STAMP_SCHEMA = "paperclip.native-workspace-stamp/v1";
+const STATE_ROOT_NAME = "native-workspace-sync";
+const DESCRIPTOR_NAME = "descriptor";
+const WORKSPACE_SEED_NAME = "workspace-seed.tar";
+const GIT_SEED_NAME = "git-seed.tar";
+const REMOTE_STAMP_NAME = "workspace-sync-v1.json";
+const MAX_DESCRIPTOR_BYTES = 64 * 1024 * 1024;
+const SAFE_SEGMENT_RE = /^[A-Za-z0-9_-]+$/;
+
+type NativeWorkspaceSyncState = "prepared" | "finalized";
+export type NativeWorkspaceResourceDisposition =
+ "keep_running" | "stop_and_retain" | "destroy";
+
+interface NativeWorkspaceSyncDescriptor {
+ schema: typeof DESCRIPTOR_SCHEMA;
+ binding: {
+ runId: string;
+ companyId: string;
+ workspaceId: string;
+ leaseId: string;
+ providerLeaseId: string;
+ localCwd: string;
+ remoteCwd: string;
+ };
+ state: NativeWorkspaceSyncState;
+ baselineSha256: string;
+ baseline: SerializedDirectorySnapshot;
+ gitSnapshot: GitWorkspaceSnapshot | null;
+ seed: {
+ workspaceArchiveSha256: string;
+ gitArchiveSha256: string | null;
+ } | null;
+ createdAt: string;
+ finalizedAt: string | null;
+ finalHostSha256: string | null;
+ resourceDisposition: NativeWorkspaceResourceDisposition | null;
+}
+
+export interface NativeWorkspaceSyncReference {
+ schema: typeof DESCRIPTOR_SCHEMA;
+ state: NativeWorkspaceSyncState;
+ descriptorSha256: string;
+ baselineSha256: string;
+ finalHostSha256: string | null;
+ workspaceId: string;
+ leaseId: string;
+ providerLeaseId: string;
+ remoteCwd: string;
+ resourceDisposition: NativeWorkspaceResourceDisposition | null;
+}
+
+export interface PreparedNativeWorkspaceSync {
+ mode: WorkspaceInboundMode;
+ reference: NativeWorkspaceSyncReference;
+ restoreWorkspace(): Promise<void>;
+ cleanup(): Promise<void>;
+}
+
+export type NativeWorkspaceInboundEvidence =
+ | {
+ kind: "existing_run";
+ restartRecovery: boolean;
+ sameProviderLease: boolean;
+ }
+ | {
+ kind: "new_run";
+ acquisition: "created" | "resumed" | "replacement" | null;
+ hasPriorStamp: boolean;
+ };
+
+export function classifyNativeWorkspaceInbound(
+ evidence: NativeWorkspaceInboundEvidence,
+): WorkspaceInboundMode {
+ if (evidence.kind === "existing_run") {
+ if (!evidence.restartRecovery) {
+ throw new Error("native_workspace_sync_unexpected_existing_descriptor");
+ }
+ return evidence.sameProviderLease ? "adopt_remote" : "durable_seed";
+ }
+ return evidence.acquisition === "resumed" && evidence.hasPriorStamp
+ ? "adopt_remote"
+ : "host_current";
+}
+
+function canonicalJson(value: unknown): string {
+ if (Array.isArray(value)) {
+ return `[${value.map((entry) => canonicalJson(entry)).join(",")}]`;
+ }
+ if (value && typeof value === "object") {
+ const record = value as Record<string, unknown>;
+ return `{${Object.keys(record)
+ .sort()
+ .map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`)
+ .join(",")}}`;
+ }
+ return JSON.stringify(value);
+}
+
+function sha256(value: unknown): string {
+ return createHash("sha256").update(canonicalJson(value)).digest("hex");
+}
+
+function requireSafeSegment(value: string, label: string): string {
+ if (!SAFE_SEGMENT_RE.test(value)) {
+ throw new Error(`native_workspace_sync_invalid_${label}`);
+ }
+ return value;
+}
+
+function descriptorDirectory(runId: string): string {
+ return path.join(
+ resolvePaperclipInstanceRoot(),
+ STATE_ROOT_NAME,
+ requireSafeSegment(runId, "run_id"),
+ );
+}
+
+function descriptorPath(runId: string, descriptorSha256: string): string {
+ if (!/^[0-9a-f]{64}$/.test(descriptorSha256)) {
+ throw new Error("native_workspace_sync_descriptor_digest_invalid");
+ }
+ return path.join(
+ descriptorDirectory(runId),
+ `${DESCRIPTOR_NAME}.${descriptorSha256}.json`,
+ );
+}
+
+function legacyDescriptorPath(runId: string): string {
+ return path.join(descriptorDirectory(runId), `${DESCRIPTOR_NAME}.json`);
+}
+
+function durableSeedPaths(runId: string): {
+ workspaceArchivePath: string;
+ gitArchivePath: string;
+} {
+ const directory = descriptorDirectory(runId);
+ return {
+ workspaceArchivePath: path.join(directory, WORKSPACE_SEED_NAME),
+ gitArchivePath: path.join(directory, GIT_SEED_NAME),
+ };
+}
+
+async function sha256File(filePath: string): Promise<string> {
+ return await new Promise((resolveDigest, rejectDigest) => {
+ const digest = createHash("sha256");
+ const stream = createReadStream(filePath);
+ stream.on("data", (chunk) => digest.update(chunk));
+ stream.on("error", rejectDigest);
+ stream.on("end", () => resolveDigest(digest.digest("hex")));
+ });
+}
+
+async function verifiedDurableSeed(input: {
+ runId: string;
+ seed: NonNullable<NativeWorkspaceSyncDescriptor["seed"]>;
+ gitSnapshot: GitWorkspaceSnapshot | null;
+}): Promise<WorkspaceDurableSeedPaths> {
+ try {
+ const paths = durableSeedPaths(input.runId);
+ const workspaceStat = await fs.lstat(paths.workspaceArchivePath);
+ if (workspaceStat.isSymbolicLink() || !workspaceStat.isFile()) {
+ throw new Error("workspace_sync_out_unrecoverable");
+ }
+ if (
+ (await sha256File(paths.workspaceArchivePath)) !==
+ input.seed.workspaceArchiveSha256
+ ) {
+ throw new Error("workspace_sync_out_unrecoverable");
+ }
+ if (input.gitSnapshot) {
+ const gitStat = await fs.lstat(paths.gitArchivePath);
+ if (
+ gitStat.isSymbolicLink() ||
+ !gitStat.isFile() ||
+ !input.seed.gitArchiveSha256 ||
+ (await sha256File(paths.gitArchivePath)) !== input.seed.gitArchiveSha256
+ ) {
+ throw new Error("workspace_sync_out_unrecoverable");
+ }
+ } else if (input.seed.gitArchiveSha256 !== null) {
+ throw new Error("workspace_sync_out_unrecoverable");
+ }
+ return {
+ workspaceArchivePath: paths.workspaceArchivePath,
+ workspaceArchiveSha256: input.seed.workspaceArchiveSha256,
+ gitArchivePath: input.gitSnapshot ? paths.gitArchivePath : null,
+ gitArchiveSha256: input.seed.gitArchiveSha256,
+ };
+ } catch (error) {
+ if (
+ error instanceof Error &&
+ error.message === "workspace_sync_out_unrecoverable"
+ ) {
+ throw error;
+ }
+ throw new Error("workspace_sync_out_unrecoverable", { cause: error });
+ }
+}
+
+async function ensurePrivateDirectory(dir: string): Promise<void> {
+ await fs.mkdir(dir, { recursive: true, mode: 0o700 });
+ const stat = await fs.lstat(dir);
+ if (stat.isSymbolicLink() || !stat.isDirectory()) {
+ throw new Error("native_workspace_sync_state_root_invalid");
+ }
+}
+
+async function writeDescriptor(
+ descriptor: NativeWorkspaceSyncDescriptor,
+): Promise<NativeWorkspaceSyncReference> {
+ const dir = descriptorDirectory(descriptor.binding.runId);
+ await ensurePrivateDirectory(dir);
+ const body = `${canonicalJson(descriptor)}\n`;
+ const descriptorSha256 = createHash("sha256").update(body).digest("hex");
+ const finalPath = descriptorPath(descriptor.binding.runId, descriptorSha256);
+ const existingBody = await fs.readFile(finalPath, "utf8").catch(() => null);
+ if (existingBody !== null && existingBody !== body) {
+ throw new Error("native_workspace_sync_descriptor_digest_collision");
+ }
+ const tempPath = path.join(dir, `${DESCRIPTOR_NAME}.${randomUUID()}.tmp`);
+ try {
+ if (existingBody === null) {
+ await fs.writeFile(tempPath, body, { encoding: "utf8", mode: 0o600 });
+ await fs.rename(tempPath, finalPath);
+ }
+ } finally {
+ await fs.rm(tempPath, { force: true }).catch(() => undefined);
+ }
+ return {
+ schema: DESCRIPTOR_SCHEMA,
+ state: descriptor.state,
+ descriptorSha256,
+ baselineSha256: descriptor.baselineSha256,
+ finalHostSha256: descriptor.finalHostSha256,
+ workspaceId: descriptor.binding.workspaceId,
+ leaseId: descriptor.binding.leaseId,
+ providerLeaseId: descriptor.binding.providerLeaseId,
+ remoteCwd: descriptor.binding.remoteCwd,
+ resourceDisposition: descriptor.resourceDisposition,
+ };
+}
+
+export function readNativeWorkspaceSyncReference(
+ value: unknown,
+): NativeWorkspaceSyncReference | null {
+ const candidate = parseObject(value);
+ if (
+ candidate.schema !== DESCRIPTOR_SCHEMA ||
+ (candidate.state !== "prepared" && candidate.state !== "finalized") ||
+ typeof candidate.descriptorSha256 !== "string" ||
+ !/^[0-9a-f]{64}$/.test(candidate.descriptorSha256) ||
+ typeof candidate.baselineSha256 !== "string" ||
+ !/^[0-9a-f]{64}$/.test(candidate.baselineSha256) ||
+ (candidate.finalHostSha256 !== null &&
+ candidate.finalHostSha256 !== undefined &&
+ (typeof candidate.finalHostSha256 !== "string" ||
+ !/^[0-9a-f]{64}$/.test(candidate.finalHostSha256))) ||
+ typeof candidate.workspaceId !== "string" ||
+ typeof candidate.leaseId !== "string" ||
+ typeof candidate.providerLeaseId !== "string" ||
+ typeof candidate.remoteCwd !== "string" ||
+ (candidate.resourceDisposition !== null &&
+ candidate.resourceDisposition !== undefined &&
+ candidate.resourceDisposition !== "keep_running" &&
+ candidate.resourceDisposition !== "stop_and_retain" &&
+ candidate.resourceDisposition !== "destroy")
+ ) {
+ return null;
+ }
+ return {
+ schema: DESCRIPTOR_SCHEMA,
+ state: candidate.state,
+ descriptorSha256: candidate.descriptorSha256,
+ baselineSha256: candidate.baselineSha256,
+ finalHostSha256:
+ typeof candidate.finalHostSha256 === "string"
+ ? candidate.finalHostSha256
+ : null,
+ workspaceId: candidate.workspaceId,
+ leaseId: candidate.leaseId,
+ providerLeaseId: candidate.providerLeaseId,
+ remoteCwd: candidate.remoteCwd,
+ resourceDisposition:
+ candidate.resourceDisposition === "keep_running" ||
+ candidate.resourceDisposition === "stop_and_retain" ||
+ candidate.resourceDisposition === "destroy"
+ ? candidate.resourceDisposition
+ : null,
+ };
+}
+
+function parseGitSnapshot(
+ value: unknown,
+): GitWorkspaceSnapshot | null | undefined {
+ if (value === null) return null;
+ const candidate = parseObject(value);
+ const paths = [
+ candidate.overlayPaths,
+ candidate.deletedPaths,
+ candidate.ignoredPaths,
+ ];
+ if (
+ typeof candidate.headCommit !== "string" ||
+ (candidate.branchName !== null &&
+ typeof candidate.branchName !== "string") ||
+ !paths.every(
+ (entries) =>
+ Array.isArray(entries) &&
+ entries.every(
+ (entry) =>
+ typeof entry === "string" &&
+ !path.posix.isAbsolute(entry) &&
+ !path.win32.isAbsolute(entry) &&
+ !entry.split(/[\\/]/).some((segment) => segment === ".."),
+ ),
+ )
+ ) {
+ return undefined;
+ }
+ return {
+ headCommit: candidate.headCommit,
+ branchName: candidate.branchName as string | null,
+ overlayPaths: [...(candidate.overlayPaths as string[])],
+ deletedPaths: [...(candidate.deletedPaths as string[])],
+ ignoredPaths: [...(candidate.ignoredPaths as string[])],
+ };
+}
+
+async function readDescriptor(input: {
+ runId: string;
+ reference: NativeWorkspaceSyncReference;
+}): Promise<{
+ descriptor: NativeWorkspaceSyncDescriptor;
+ baseline: DirectorySnapshot;
+}> {
+ const contentAddressedPath = descriptorPath(
+ input.runId,
+ input.reference.descriptorSha256,
+ );
+ // Early v1 writers used descriptor.json. Prefer the immutable,
+ // content-addressed name, but keep the digest-verified legacy path readable
+ // so an interrupted upgrade can still finish its already-proposed result.
+ const filePath = await fs
+ .lstat(contentAddressedPath)
+ .then(() => contentAddressedPath)
+ .catch(async (error: NodeJS.ErrnoException) => {
+ if (error.code !== "ENOENT") throw error;
+ const legacyPath = legacyDescriptorPath(input.runId);
+ await fs.lstat(legacyPath);
+ return legacyPath;
+ });
+ const stat = await fs.lstat(filePath);
+ if (
+ stat.isSymbolicLink() ||
+ !stat.isFile() ||
+ stat.size > MAX_DESCRIPTOR_BYTES
+ ) {
+ throw new Error("native_workspace_sync_descriptor_invalid");
+ }
+ const body = await fs.readFile(filePath, "utf8");
+ if (
+ createHash("sha256").update(body).digest("hex") !==
+ input.reference.descriptorSha256
+ ) {
+ throw new Error("native_workspace_sync_descriptor_digest_mismatch");
+ }
+ const parsed = JSON.parse(body) as unknown;
+ const candidate = parseObject(parsed);
+ const binding = parseObject(candidate.binding);
+ const baseline = parseDirectorySnapshot(candidate.baseline);
+ const gitSnapshot = parseGitSnapshot(candidate.gitSnapshot);
+ const rawSeed =
+ candidate.seed === null || candidate.seed === undefined
+ ? null
+ : parseObject(candidate.seed);
+ const seed =
+ rawSeed === null
+ ? null
+ : typeof rawSeed.workspaceArchiveSha256 === "string" &&
+ /^[0-9a-f]{64}$/.test(rawSeed.workspaceArchiveSha256) &&
+ (rawSeed.gitArchiveSha256 === null ||
+ (typeof rawSeed.gitArchiveSha256 === "string" &&
+ /^[0-9a-f]{64}$/.test(rawSeed.gitArchiveSha256)))
+ ? {
+ workspaceArchiveSha256: rawSeed.workspaceArchiveSha256,
+ gitArchiveSha256: rawSeed.gitArchiveSha256 as string | null,
+ }
+ : undefined;
+ const finalHostSha256 =
+ typeof candidate.finalHostSha256 === "string" &&
+ /^[0-9a-f]{64}$/.test(candidate.finalHostSha256)
+ ? candidate.finalHostSha256
+ : candidate.finalHostSha256 === null
+ ? null
+ : undefined;
+ const resourceDisposition =
+ candidate.resourceDisposition === "keep_running" ||
+ candidate.resourceDisposition === "stop_and_retain" ||
+ candidate.resourceDisposition === "destroy"
+ ? candidate.resourceDisposition
+ : candidate.resourceDisposition === null
+ ? null
+ : undefined;
+ if (
+ candidate.schema !== DESCRIPTOR_SCHEMA ||
+ (candidate.state !== "prepared" && candidate.state !== "finalized") ||
+ !baseline ||
+ gitSnapshot === undefined ||
+ seed === undefined ||
+ binding.runId !== input.runId ||
+ binding.workspaceId !== input.reference.workspaceId ||
+ binding.leaseId !== input.reference.leaseId ||
+ binding.providerLeaseId !== input.reference.providerLeaseId ||
+ binding.remoteCwd !== input.reference.remoteCwd ||
+ typeof binding.companyId !== "string" ||
+ typeof binding.localCwd !== "string" ||
+ !path.isAbsolute(binding.localCwd) ||
+ typeof binding.remoteCwd !== "string" ||
+ !path.posix.isAbsolute(binding.remoteCwd) ||
+ typeof candidate.baselineSha256 !== "string" ||
+ candidate.baselineSha256 !== input.reference.baselineSha256 ||
+ directorySnapshotSha256(baseline) !== candidate.baselineSha256 ||
+ typeof candidate.createdAt !== "string" ||
+ !Number.isFinite(Date.parse(candidate.createdAt)) ||
+ candidate.state !== input.reference.state ||
+ finalHostSha256 === undefined ||
+ finalHostSha256 !== input.reference.finalHostSha256 ||
+ resourceDisposition === undefined ||
+ resourceDisposition !== input.reference.resourceDisposition ||
+ (candidate.state === "prepared" &&
+ (candidate.finalizedAt !== null || finalHostSha256 !== null)) ||
+ (candidate.state === "finalized" &&
+ (typeof candidate.finalizedAt !== "string" ||
+ !Number.isFinite(Date.parse(candidate.finalizedAt)) ||
+ finalHostSha256 === null))
+ ) {
+ throw new Error("native_workspace_sync_descriptor_binding_mismatch");
+ }
+ return {
+ descriptor: {
+ schema: DESCRIPTOR_SCHEMA,
+ binding: {
+ runId: binding.runId as string,
+ companyId: binding.companyId,
+ workspaceId: binding.workspaceId as string,
+ leaseId: binding.leaseId as string,
+ providerLeaseId: binding.providerLeaseId as string,
+ localCwd: binding.localCwd,
+ remoteCwd: binding.remoteCwd as string,
+ },
+ state: candidate.state,
+ baselineSha256: candidate.baselineSha256,
+ baseline: candidate.baseline as SerializedDirectorySnapshot,
+ gitSnapshot,
+ seed,
+ createdAt: candidate.createdAt,
+ finalizedAt:
+ typeof candidate.finalizedAt === "string"
+ ? candidate.finalizedAt
+ : null,
+ finalHostSha256,
+ resourceDisposition,
+ },
+ baseline,
+ };
+}
+
+async function persistRunReference(
+ db: Db,
+ runId: string,
+ reference: NativeWorkspaceSyncReference,
+): Promise<void> {
+ await db.transaction(async (tx) => {
+ const run = await tx
+ .select({ runnerProfileJson: heartbeatRuns.runnerProfileJson })
+ .from(heartbeatRuns)
+ .where(eq(heartbeatRuns.id, runId))
+ .for("update")
+ .limit(1)
+ .then((rows) => rows[0] ?? null);
+ if (!run) throw new Error("native_workspace_sync_run_missing");
+ await tx
+ .update(heartbeatRuns)
+ .set({
+ runnerProfileJson: {
+ ...parseObject(run.runnerProfileJson),
+ nativeWorkspaceSync: reference,
+ },
+ updatedAt: new Date(),
+ })
+ .where(eq(heartbeatRuns.id, runId));
+ });
+}
+
+async function persistLeaseStamp(input: {
+ db: Db;
+ leaseId: string;
+ stamp: Record<string, unknown>;
+}): Promise<void> {
+ await input.db.transaction(async (tx) => {
+ const lease = await tx
+ .select({ metadata: environmentLeases.metadata })
+ .from(environmentLeases)
+ .where(eq(environmentLeases.id, input.leaseId))
+ .for("update")
+ .limit(1)
+ .then((rows) => rows[0] ?? null);
+ if (!lease) throw new Error("native_workspace_sync_lease_missing");
+ await tx
+ .update(environmentLeases)
+ .set({
+ metadata: {
+ ...parseObject(lease.metadata),
+ nativeWorkspaceSync: input.stamp,
+ },
+ updatedAt: new Date(),
+ })
+ .where(eq(environmentLeases.id, input.leaseId));
+ });
+}
+
+function providerLeaseIdFor(input: {
+ target: Extract<AdapterExecutionTarget, { transport: "sandbox" }>;
+ lease: EnvironmentLease;
+}): string {
+ const providerLeaseId =
+ input.target.sandboxLeaseAcquisition?.providerLeaseId ??
+ input.lease.providerLeaseId;
+ if (!providerLeaseId) {
+ throw new Error("native_workspace_sync_provider_lease_missing");
+ }
+ return providerLeaseId;
+}
+
+function shellQuote(value: string): string {
+ return `'${value.replaceAll("'", "'\\''")}'`;
+}
+
+async function writeRemoteStamp(input: {
+ target: Extract<AdapterExecutionTarget, { transport: "sandbox" }>;
+ stamp: Record<string, unknown>;
+}): Promise<void> {
+ if (!input.target.runner) {
+ throw new Error("native_workspace_sync_runner_missing");
+ }
+ const runtimeDir = path.posix.join(
+ input.target.remoteCwd,
+ ".paperclip-runtime",
+ "paperclip-runner",
+ );
+ const stampPath = path.posix.join(runtimeDir, REMOTE_STAMP_NAME);
+ const tempPath = `${stampPath}.tmp`;
+ const encoded = Buffer.from(
+ `${canonicalJson(input.stamp)}\n`,
+ "utf8",
+ ).toString("base64");
+ const result = await input.target.runner.execute({
+ command: input.target.shellCommand ?? "sh",
+ args: [
+ "-c",
+ `umask 077; mkdir -p ${shellQuote(runtimeDir)} && printf %s ${shellQuote(encoded)} | base64 -d > ${shellQuote(tempPath)} && mv ${shellQuote(tempPath)} ${shellQuote(stampPath)}`,
+ ],
+ cwd: "/",
+ timeoutMs: 15_000,
+ bypassSession: true,
+ });
+ if (result.timedOut || result.exitCode !== 0) {
+ throw new Error("native_workspace_sync_remote_stamp_failed");
+ }
+}
+
+async function remoteStampMatches(input: {
+ target: Extract<AdapterExecutionTarget, { transport: "sandbox" }>;
+ expected: Record<string, unknown>;
+}): Promise<boolean> {
+ if (!input.target.runner) return false;
+ const stampPath = path.posix.join(
+ input.target.remoteCwd,
+ ".paperclip-runtime",
+ "paperclip-runner",
+ REMOTE_STAMP_NAME,
+ );
+ const result = await input.target.runner.execute({
+ command: input.target.shellCommand ?? "sh",
+ args: [
+ "-c",
+ `test -f ${shellQuote(stampPath)} && cat ${shellQuote(stampPath)}`,
+ ],
+ cwd: "/",
+ timeoutMs: 15_000,
+ bypassSession: true,
+ });
+ if (result.timedOut || result.exitCode !== 0) return false;
+ try {
+ return (
+ canonicalJson(JSON.parse(result.stdout)) === canonicalJson(input.expected)
+ );
+ } catch {
+ return false;
+ }
+}
+
+function leaseStamp(input: {
+ lease: EnvironmentLease;
+ workspaceId: string;
+ providerLeaseId: string;
+ remoteCwd: string;
+}): Record<string, unknown> | null {
+ const stamp = parseObject(
+ parseObject(input.lease.metadata).nativeWorkspaceSync,
+ );
+ if (
+ stamp.schema !== STAMP_SCHEMA ||
+ stamp.workspaceId !== input.workspaceId ||
+ stamp.providerLeaseId !== input.providerLeaseId ||
+ stamp.remoteCwd !== input.remoteCwd ||
+ typeof stamp.hostSha256 !== "string" ||
+ !/^[0-9a-f]{64}$/.test(stamp.hostSha256)
+ ) {
+ return null;
+ }
+ return stamp;
+}
+
+function finalizedWorkspaceStamp(input: {
+ descriptor: NativeWorkspaceSyncDescriptor;
+ hostSha256: string;
+}): Record<string, unknown> {
+ return {
+ schema: STAMP_SCHEMA,
+ workspaceId: input.descriptor.binding.workspaceId,
+ providerLeaseId: input.descriptor.binding.providerLeaseId,
+ remoteCwd: input.descriptor.binding.remoteCwd,
+ hostSha256: input.hostSha256,
+ finalizedRunId: input.descriptor.binding.runId,
+ };
+}
+
+async function prepareRuntime(input: {
+ runId: string;
+ target: Extract<AdapterExecutionTarget, { transport: "sandbox" }>;
+ workspaceLocalDir: string;
+ mode: WorkspaceInboundMode;
+ baseline?: DirectorySnapshot;
+ gitSnapshot?: GitWorkspaceSnapshot | null;
+ durableSeed?: WorkspaceDurableSeedPaths;
+}): Promise<PreparedAdapterExecutionTargetRuntime> {
+ return prepareAdapterExecutionTargetRuntime({
+ runId: input.runId,
+ target: input.target,
+ adapterKey: "paperclip-runner",
+ workspaceLocalDir: input.workspaceLocalDir,
+ workspaceRemoteDir: input.target.remoteCwd,
+ workspaceInboundMode: input.mode,
+ workspaceDurableSeed: input.durableSeed,
+ workspaceBaseline: input.baseline,
+ workspaceGitSnapshot: input.gitSnapshot,
+ });
+}
+
+async function finalizePreparedRuntime(input: {
+ db: Db;
+ runId: string;
+ target: Extract<AdapterExecutionTarget, { transport: "sandbox" }>;
+ runtime: PreparedAdapterExecutionTargetRuntime;
+ descriptor: NativeWorkspaceSyncDescriptor;
+}): Promise<NativeWorkspaceSyncReference> {
+ await input.runtime.restoreWorkspace();
+ const finalSnapshot =
+ await import("@paperclipai/adapter-utils/workspace-restore-merge").then(
+ ({ captureDirectorySnapshot }) =>
+ captureDirectorySnapshot(input.descriptor.binding.localCwd, {
+ exclude: input.runtime.workspaceSyncSnapshot?.baseline.exclude ?? [],
+ }),
+ );
+ const finalHostSha256 = directorySnapshotSha256(finalSnapshot);
+ const stamp = finalizedWorkspaceStamp({
+ descriptor: input.descriptor,
+ hostSha256: finalHostSha256,
+ });
+ await writeRemoteStamp({ target: input.target, stamp });
+ const finalizedDescriptor: NativeWorkspaceSyncDescriptor = {
+ ...input.descriptor,
+ state: "finalized",
+ finalizedAt: new Date().toISOString(),
+ finalHostSha256,
+ };
+ const reference = await writeDescriptor(finalizedDescriptor);
+ await persistRunReference(input.db, input.runId, reference);
+ await persistLeaseStamp({
+ db: input.db,
+ leaseId: input.descriptor.binding.leaseId,
+ stamp,
+ });
+ return reference;
+}
+
+export async function prepareNativeWorkspaceSync(input: {
+ db: Db;
+ runId: string;
+ companyId: string;
+ workspaceId: string;
+ workspaceLocalDir: string;
+ target: AdapterExecutionTarget | null;
+ lease: EnvironmentLease;
+ restartRecovery?: NativeRestartRecoveryClaim;
+ resourceDisposition?: NativeWorkspaceResourceDisposition;
+}): Promise<PreparedNativeWorkspaceSync | null> {
+ if (input.target?.kind !== "remote" || input.target.transport !== "sandbox") {
+ return null;
+ }
+ const target = input.target;
+ const providerLeaseId = providerLeaseIdFor({ target, lease: input.lease });
+ const run = await input.db
+ .select({ runnerProfileJson: heartbeatRuns.runnerProfileJson })
+ .from(heartbeatRuns)
+ .where(eq(heartbeatRuns.id, input.runId))
+ .limit(1)
+ .then((rows) => rows[0] ?? null);
+ if (!run) throw new Error("native_workspace_sync_run_missing");
+ const existingReference = readNativeWorkspaceSyncReference(
+ parseObject(run.runnerProfileJson).nativeWorkspaceSync,
+ );
+
+ let runtime: PreparedAdapterExecutionTargetRuntime;
+ let descriptor: NativeWorkspaceSyncDescriptor;
+ let mode: WorkspaceInboundMode = "host_current";
+ const seedPaths = durableSeedPaths(input.runId);
+ await ensurePrivateDirectory(descriptorDirectory(input.runId));
+
+ if (existingReference) {
+ const existing = await readDescriptor({
+ runId: input.runId,
+ reference: existingReference,
+ });
+ if (
+ existing.descriptor.binding.companyId !== input.companyId ||
+ existing.descriptor.binding.workspaceId !== input.workspaceId ||
+ path.resolve(existing.descriptor.binding.localCwd) !==
+ path.resolve(input.workspaceLocalDir) ||
+ existing.descriptor.binding.remoteCwd !== target.remoteCwd
+ ) {
+ throw new Error("native_workspace_sync_descriptor_binding_mismatch");
+ }
+ const sameProviderLease =
+ existing.descriptor.binding.providerLeaseId === providerLeaseId;
+ mode = classifyNativeWorkspaceInbound({
+ kind: "existing_run",
+ restartRecovery: Boolean(input.restartRecovery),
+ sameProviderLease,
+ });
+ const durableSeed =
+ mode === "durable_seed"
+ ? existing.descriptor.seed
+ ? await verifiedDurableSeed({
+ runId: input.runId,
+ seed: existing.descriptor.seed,
+ gitSnapshot: existing.descriptor.gitSnapshot,
+ })
+ : (() => {
+ throw new Error("workspace_sync_out_unrecoverable");
+ })()
+ : undefined;
+ runtime = await prepareRuntime({
+ runId: input.runId,
+ target,
+ workspaceLocalDir: input.workspaceLocalDir,
+ mode,
+ baseline: existing.baseline,
+ gitSnapshot: existing.descriptor.gitSnapshot,
+ durableSeed,
+ });
+ descriptor = {
+ ...existing.descriptor,
+ binding: {
+ ...existing.descriptor.binding,
+ leaseId: target.leaseId ?? input.lease.id,
+ providerLeaseId,
+ },
+ };
+ } else {
+ const acquisition = target.sandboxLeaseAcquisition?.outcome ?? null;
+ const priorStamp = leaseStamp({
+ lease: input.lease,
+ workspaceId: input.workspaceId,
+ providerLeaseId,
+ remoteCwd: target.remoteCwd,
+ });
+ mode = classifyNativeWorkspaceInbound({
+ kind: "new_run",
+ acquisition,
+ hasPriorStamp: priorStamp !== null,
+ });
+ runtime = await prepareRuntime({
+ runId: input.runId,
+ target,
+ workspaceLocalDir: input.workspaceLocalDir,
+ mode,
+ durableSeed: {
+ workspaceArchivePath: seedPaths.workspaceArchivePath,
+ gitArchivePath: seedPaths.gitArchivePath,
+ },
+ });
+ const currentSnapshot = runtime.workspaceSyncSnapshot;
+ if (!currentSnapshot) {
+ throw new Error("native_workspace_sync_snapshot_missing");
+ }
+ if (acquisition === "resumed" && priorStamp) {
+ const currentHostSha256 = directorySnapshotSha256(
+ currentSnapshot.baseline,
+ );
+ const verifiedWarmAdoption =
+ priorStamp.hostSha256 === currentHostSha256 &&
+ (await remoteStampMatches({ target, expected: priorStamp }));
+ if (verifiedWarmAdoption) {
+ mode = "adopt_remote";
+ } else {
+ mode = "host_current";
+ runtime = await prepareRuntime({
+ runId: input.runId,
+ target,
+ workspaceLocalDir: input.workspaceLocalDir,
+ mode,
+ baseline: currentSnapshot.baseline,
+ gitSnapshot: currentSnapshot.gitSnapshot,
+ durableSeed: {
+ workspaceArchivePath: seedPaths.workspaceArchivePath,
+ gitArchivePath: seedPaths.gitArchivePath,
+ },
+ });
+ }
+ }
+ const snapshot = runtime.workspaceSyncSnapshot;
+ if (!snapshot) throw new Error("native_workspace_sync_snapshot_missing");
+ const now = new Date().toISOString();
+ const workspaceArchiveSha256 = await sha256File(
+ seedPaths.workspaceArchivePath,
+ );
+ const gitArchiveSha256 = snapshot.gitSnapshot
+ ? await sha256File(seedPaths.gitArchivePath)
+ : null;
+ descriptor = {
+ schema: DESCRIPTOR_SCHEMA,
+ binding: {
+ runId: input.runId,
+ companyId: input.companyId,
+ workspaceId: input.workspaceId,
+ leaseId: target.leaseId ?? input.lease.id,
+ providerLeaseId,
+ localCwd: path.resolve(input.workspaceLocalDir),
+ remoteCwd: target.remoteCwd,
+ },
+ state: "prepared",
+ baselineSha256: directorySnapshotSha256(snapshot.baseline),
+ baseline: serializeDirectorySnapshot(snapshot.baseline),
+ gitSnapshot: snapshot.gitSnapshot,
+ seed: { workspaceArchiveSha256, gitArchiveSha256 },
+ createdAt: now,
+ finalizedAt: null,
+ finalHostSha256: null,
+ resourceDisposition: input.resourceDisposition ?? null,
+ };
+ }
+
+ let reference = await writeDescriptor(descriptor);
+ await persistRunReference(input.db, input.runId, reference);
+ let restorePromise: Promise<void> | null = null;
+ return {
+ mode,
+ get reference() {
+ return reference;
+ },
+ restoreWorkspace: async () => {
+ if (!restorePromise) {
+ restorePromise = finalizePreparedRuntime({
+ db: input.db,
+ runId: input.runId,
+ target,
+ runtime,
+ descriptor,
+ })
+ .then((finalizedReference) => {
+ reference = finalizedReference;
+ })
+ .catch((error) => {
+ restorePromise = null;
+ throw error;
+ });
+ }
+ await restorePromise;
+ },
+ cleanup: () => cleanupNativeWorkspaceSync(input.runId),
+ };
+}
+
+export async function resumeNativeWorkspaceSync(input: {
+ db: Db;
+ runId: string;
+ target: AdapterExecutionTarget;
+}): Promise<boolean> {
+ if (input.target.kind !== "remote" || input.target.transport !== "sandbox") {
+ throw new Error("workspace_sync_out_unrecoverable");
+ }
+ const run = await input.db
+ .select({ runnerProfileJson: heartbeatRuns.runnerProfileJson })
+ .from(heartbeatRuns)
+ .where(eq(heartbeatRuns.id, input.runId))
+ .limit(1)
+ .then((rows) => rows[0] ?? null);
+ const reference = readNativeWorkspaceSyncReference(
+ parseObject(run?.runnerProfileJson).nativeWorkspaceSync,
+ );
+ if (!reference) return false;
+ const existing = await readDescriptor({ runId: input.runId, reference });
+ const providerLeaseId =
+ input.target.sandboxLeaseAcquisition?.providerLeaseId ??
+ reference.providerLeaseId;
+ if (
+ providerLeaseId !== reference.providerLeaseId ||
+ input.target.remoteCwd !== reference.remoteCwd
+ ) {
+ throw new Error("workspace_sync_out_unrecoverable");
+ }
+ if (
+ existing.descriptor.state === "finalized" &&
+ existing.descriptor.finalHostSha256
+ ) {
+ const stamp = finalizedWorkspaceStamp({
+ descriptor: existing.descriptor,
+ hostSha256: existing.descriptor.finalHostSha256,
+ });
+ if (
+ !(await remoteStampMatches({ target: input.target, expected: stamp }))
+ ) {
+ await writeRemoteStamp({ target: input.target, stamp });
+ }
+ await persistLeaseStamp({
+ db: input.db,
+ leaseId: existing.descriptor.binding.leaseId,
+ stamp,
+ });
+ return true;
+ }
+ const runtime = await prepareRuntime({
+ runId: input.runId,
+ target: input.target,
+ workspaceLocalDir: existing.descriptor.binding.localCwd,
+ mode: "adopt_remote",
+ baseline: existing.baseline,
+ gitSnapshot: existing.descriptor.gitSnapshot,
+ });
+ await finalizePreparedRuntime({
+ db: input.db,
+ runId: input.runId,
+ target: input.target,
+ runtime,
+ descriptor: existing.descriptor,
+ });
+ return true;
+}
+
+export async function cleanupNativeWorkspaceSync(runId: string): Promise<void> {
+ await fs.rm(descriptorDirectory(runId), { recursive: true, force: true });
+}
+
+export const nativeWorkspaceSyncInternals = {
+ descriptorPath,
+ legacyDescriptorPath,
+ durableSeedPaths,
+ writeDescriptor,
+ readDescriptor,
+ readReference: readNativeWorkspaceSyncReference,
+ sha256,
+};
diff --git a/tests/runner-e2e/FIXTURES.md b/tests/runner-e2e/FIXTURES.md
index 5d533b2d01..01c229242d 100644
--- a/tests/runner-e2e/FIXTURES.md
+++ b/tests/runner-e2e/FIXTURES.md
@@ -58,9 +58,13 @@ registry therefore discovers that row through the public environments API.
This still provides full isolation because every cell starts a new Paperclip
instance and database.
-Daytona creates a sandbox environment through the public API. Keep
-`reuseLease:false`, `runnerLifecycleMode:"per_turn"`, short provider cleanup
-backstops, a Daytona secret reference, and an immutable image digest. Teardown
+Daytona creates sandbox environments through the public API. The core fixture
+keeps `reuseLease:false` and `runnerLifecycleMode:"per_turn"`. The dedicated
+warm-continuity fixture uses `reuseLease:true` and
+`runnerLifecycleMode:"warm"`; its distinct `configurationKey` is part of the
+suite fingerprint even though both fixtures report `environmentId:"daytona"`.
+Keep short provider cleanup backstops, a Daytona secret reference, and an
+immutable image digest. Teardown
must delete the environment with reusable-lease destruction and must fail the
cell if cleanup cannot be confirmed. Keep CPU, memory, and disk explicit: lease
metadata and the per-test public-list-price runtime estimate depend on that
@@ -93,7 +97,10 @@ marker factories. `question_resume_completion` must define the deterministic
browser answer and prove exactly two successful runs with no pending
interaction. `plan_approval_completion` must target the exact two-step
canonical Plan revision, capture its pending UI, approve in the browser, and
-prove exactly two successful runs.
+prove exactly two successful runs. `warm_three_turn` provides exactly two
+browser follow-up messages, preserves one project/execution-workspace scope,
+verifies host file contents after every turn, and finishes within three
+ten-minute turn deadlines.
Every selected case runs in its own isolated Paperclip process, and independent
cases may run concurrently. Follow-up turns inside one case retain their shared
diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md
index ed9fed5caa..b9ffb1dd6c 100644
--- a/tests/runner-e2e/README.md
+++ b/tests/runner-e2e/README.md
@@ -68,10 +68,11 @@ pnpm test:e2e:runner -- --group native --environment local
pnpm test:e2e:runner -- --profile runner-codex --case message-marker
pnpm test:e2e:runner -- --case plan-revise-accept --group local
pnpm test:e2e:runner -- --case ask-question --group native
+pnpm test:e2e:runner -- --suite daytona-warm-continuity
pnpm test:e2e:runner -- --all
```
-The catalog contains three suites. `core-compatibility` (**Core Runner
+The catalog contains four suites. `core-compatibility` (**Core Runner
Compatibility**) is seven major runner profiles × local/Daytona × three
workflows: 42 cells. Its cases are:
@@ -107,7 +108,19 @@ duplicating the final response. The second workflow restarts the isolated
Paperclip server while the interaction is waiting, reloads that state, and
then resumes it. The suite has no Daytona cells.
-The complete catalog is 66 cells (45 local and 21 Daytona) and 114 expected
+`daytona-warm-continuity` (**Daytona Warm Continuity**) is exactly two paid
+cells: legacy Codex and Runner Codex against one reusable warm Daytona
+configuration. Each cell creates a real project with a primary local-path
+workspace through the API, selects it in the browser task dialog, and performs
+three browser-driven turns on one issue. Every turn reads and extends the same
+nonce file, verifies host copy-back, records scheduler/run/end-to-end timing,
+and asserts `created`, `resumed`, `resumed` lease acquisition on one sandbox.
+Runner Codex additionally proves stable native session, provider session,
+runner instance, PID, and process-start identity. Each turn is bounded to ten
+minutes, the cell to thirty minutes, and cleanup explicitly deletes the
+sandbox rather than waiting for Daytona's idle timeout.
+
+The complete catalog is 68 cells (45 local and 23 Daytona) and 120 expected
paid agent turns. Follow-up steps remain ordered within their cell; all other
cells are independent. Narrow selectors are strongly recommended while
developing fixtures.
@@ -351,7 +364,7 @@ Set `RUNNER_E2E_AWS_ENABLED=true` to route paid cells to the repository-scoped
ephemeral AWS RunsOn fleet selected by
`runs-on/fleet=paperclip-public-pr-x64/env=public-ci`. Any other value uses the
proven GitHub-hosted `ubuntu-latest` target. Set `RUNNER_E2E_MAX_PARALLEL` to an
-integer from 1–100 on AWS (default 100); use at least 66 to run the current
+integer from 1–100 on AWS (default 100); use at least 68 to run the current
complete catalog in one wave. The fallback runner retains its 1–57 limit and
default of 32. Multi-turn steps are sequential inside their cell while
independent cells overlap. Artifacts and merged HTML/JUnit/normalized reports
diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts
index 83ecd13b1f..685c18add8 100644
--- a/tests/runner-e2e/catalog.test.ts
+++ b/tests/runner-e2e/catalog.test.ts
@@ -10,7 +10,10 @@ import {
runnerProfiles,
runnerSuites,
runnerTasks,
+ daytonaWarmContinuityTask,
+ daytonaWarmEnvironment,
isImmutableDaytonaImage,
+ suiteDefinitionHash,
validateRunnerCatalog,
} from "./catalog.js";
import {
@@ -21,7 +24,7 @@ import {
} from "./selectors.js";
describe("runner E2E catalog", () => {
- it("validates the core, local-integrity, and breadth suites", () => {
+ it("validates the core, local-integrity, breadth, and warm suites", () => {
expect(runnerProfiles).toHaveLength(7);
expect(openRouterBreadthProfiles).toHaveLength(4);
expect(runnerEnvironments).toHaveLength(2);
@@ -29,10 +32,10 @@ describe("runner E2E catalog", () => {
expect(localIntegrityTasks).toHaveLength(2);
expect(openRouterBreadthTasks).toHaveLength(3);
expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([
- 42, 14, 10,
+ 42, 14, 10, 2,
]);
- expect(validateRunnerCatalog()).toHaveLength(66);
- expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(66);
+ expect(validateRunnerCatalog()).toHaveLength(68);
+ expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(68);
expect(
runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"),
).toHaveLength(42);
@@ -46,18 +49,81 @@ describe("runner E2E catalog", () => {
(entry) => entry.suite.id === "openrouter-model-breadth",
),
).toHaveLength(10);
+ expect(
+ runnerMatrix.filter(
+ (entry) => entry.suite.id === "daytona-warm-continuity",
+ ),
+ ).toHaveLength(2);
expect(
runnerMatrix.reduce(
(total, execution) => total + execution.task.expectedRunCount,
0,
),
- ).toBe(114);
+ ).toBe(120);
expect(
runnerTasks.find((task) => task.id === "plan-revise-accept")
?.attemptTimeoutMs,
).toEqual({ local: 8 * 60_000, daytona: 12 * 60_000 });
});
+ it("defines the warm Daytona continuity fixture as exactly two Codex cells", () => {
+ expect(daytonaWarmEnvironment).toMatchObject({
+ id: "daytona",
+ configurationKey: "warm-reuse-v1",
+ groups: ["daytona", "warm"],
+ });
+ expect(
+ daytonaWarmEnvironment.buildEnvironment({
+ secretRefs: {
+ DAYTONA_API_KEY: {
+ type: "secret_ref",
+ secretId: "22222222-2222-4222-8222-222222222222",
+ version: "latest",
+ },
+ },
+ daytonaImage: `runner@sha256:${"a".repeat(64)}`,
+ executionId: "warm",
+ }),
+ ).toMatchObject({
+ config: {
+ reuseLease: true,
+ runnerLifecycleMode: "warm",
+ autoStopInterval: 5,
+ autoArchiveInterval: 15,
+ autoDeleteInterval: 60,
+ },
+ });
+ expect(daytonaWarmContinuityTask).toMatchObject({
+ flow: "warm_three_turn",
+ expectedRunCount: 3,
+ turnTimeoutMs: 600_000,
+ });
+ expect(
+ daytonaWarmContinuityTask.buildFollowupMessages?.("nonce"),
+ ).toHaveLength(2);
+ const cells = runnerMatrix.filter(
+ (entry) => entry.suite.id === "daytona-warm-continuity",
+ );
+ expect(cells.map((entry) => entry.profile.id)).toEqual([
+ "legacy-codex",
+ "runner-codex",
+ ]);
+ expect(cells.every((entry) => entry.environment.id === "daytona")).toBe(
+ true,
+ );
+ const suite = runnerSuites.find(
+ (candidate) => candidate.id === "daytona-warm-continuity",
+ )!;
+ expect(
+ suiteDefinitionHash({
+ ...suite,
+ environments: [
+ { ...daytonaWarmEnvironment, configurationKey: "changed" },
+ ],
+ }),
+ ).not.toBe(suiteDefinitionHash(suite));
+ });
+
it("derives the qualified local native OpenCode profiles from the ranked snapshot", () => {
expect(openRouterBreadthExcludedModelIds).toEqual(["xiaomi/mimo-v2.5"]);
expect(openRouterBreadthExcludedExecutionIds).toEqual([
@@ -181,9 +247,10 @@ describe("runner E2E catalog", () => {
question?.buildPrompt("nonce"),
...breadthTasks,
]) {
- expect(prompt).toContain("then emit exactly");
+ const terminalTextInstruction = prompt?.match(/then emit (?:exactly|only)/)?.[0];
+ expect(terminalTextInstruction).toBeDefined();
expect(prompt!.indexOf("paperclip_finish exactly once")).toBeLessThan(
- prompt!.indexOf("then emit exactly"),
+ prompt!.indexOf(terminalTextInstruction!),
);
expect(prompt).toContain("Wait for that tool call to succeed");
}
@@ -336,7 +403,7 @@ describe("runner E2E catalog", () => {
"those two tool calls form one indivisible response sequence",
);
expect(task!.buildPrompt("nonce")).toContain(
- "Do not emit assistant text, end the heartbeat, or stop after write_document alone",
+ "Do not emit assistant text, end the response or heartbeat, or stop after write_document alone",
);
expect(task!.buildPrompt("nonce")).toContain(
"one atomic issue PATCH with status `done` and that exact comment",
@@ -433,7 +500,7 @@ describe("runner E2E selectors", () => {
"daytona",
]);
const selected = selectRunnerExecutions(options);
- expect(selected).toHaveLength(12);
+ expect(selected).toHaveLength(13);
expect(
selected.every(
(entry) =>
@@ -443,7 +510,7 @@ describe("runner E2E selectors", () => {
).toBe(true);
});
- it("rejects groups outside the advertised four", () => {
+ it("rejects unknown groups", () => {
const options = parseRunnerSelectors(["--group", "codex"]);
expect(() => selectRunnerExecutions(options)).toThrow("Unknown group");
});
@@ -452,10 +519,10 @@ describe("runner E2E selectors", () => {
const jobs = buildMatrixJobs(
selectRunnerExecutions(parseRunnerSelectors(["--all"])),
);
- expect(jobs).toHaveLength(66);
- expect(jobs.filter((job) => job.needsDaytona)).toHaveLength(21);
+ expect(jobs).toHaveLength(68);
+ expect(jobs.filter((job) => job.needsDaytona)).toHaveLength(23);
expect(jobs.filter((job) => !job.needsDaytona)).toHaveLength(45);
- expect(new Set(jobs.map((job) => job.executionId)).size).toBe(66);
+ expect(new Set(jobs.map((job) => job.executionId)).size).toBe(68);
expect(
jobs.find(
(job) =>
diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts
index 6b887aaec9..586879eace 100644
--- a/tests/runner-e2e/catalog.ts
+++ b/tests/runner-e2e/catalog.ts
@@ -27,6 +27,7 @@ const SELECTABLE_GROUPS = [
"native",
"local",
"daytona",
+ "warm",
"core",
"breadth",
] as const;
@@ -374,6 +375,50 @@ export const runnerEnvironments: readonly EnvironmentFixture[] = [
},
] as const;
+export const daytonaWarmEnvironment: EnvironmentFixture = {
+ id: "daytona",
+ configurationKey: "warm-reuse-v1",
+ label: "Daytona warm reusable sandbox",
+ groups: ["daytona", "warm"],
+ driver: "sandbox",
+ provider: "daytona",
+ credential: "DAYTONA_API_KEY",
+ lifecycle: {
+ setup: "create_via_api",
+ probe: "run_context_via_api",
+ cleanup: "delete_via_api_and_destroy_leases",
+ },
+ expectedExecutionTarget: { kind: "remote", transport: "sandbox" },
+ buildEnvironment(input) {
+ if (!isImmutableDaytonaImage(input.daytonaImage)) {
+ throw new Error(
+ "PAPERCLIP_E2E_DAYTONA_IMAGE must be an immutable image digest",
+ );
+ }
+ return {
+ name: `Runner E2E Daytona warm ${input.executionId}`,
+ description: "Ephemeral reusable Daytona runner E2E environment",
+ driver: "sandbox",
+ config: {
+ provider: "daytona",
+ apiKey: requiredDaytonaSecret(input),
+ image: input.daytonaImage,
+ cpu: 4,
+ memory: 4,
+ disk: 10,
+ reuseLease: true,
+ runnerLifecycleMode: "warm",
+ autoStopInterval: 5,
+ autoArchiveInterval: 15,
+ autoDeleteInterval: 60,
+ timeoutMs: 300_000,
+ livenessTimeoutMs: 30_000,
+ },
+ envVars: {},
+ };
+ },
+};
+
export const runnerTasks: readonly RunnerTaskFixture[] = [
{
id: "message-marker",
@@ -719,6 +764,80 @@ const localEnvironment = runnerEnvironments.find(
(environment) => environment.id === "local",
)!;
+function warmTurnMarker(turn: 1 | 2 | 3, nonce: string) {
+ return `PAPERCLIP_E2E_WARM_T${turn}_${nonce}`;
+}
+
+function warmWorkspaceLine(turn: 1 | 2 | 3, nonce: string) {
+ return `T${turn}_${nonce}`;
+}
+
+function warmTurnInstructions(turn: 1 | 2 | 3, nonce: string) {
+ const file = `daytona-warm-${nonce}.txt`;
+ const lines = Array.from({ length: turn }, (_, index) =>
+ warmWorkspaceLine((index + 1) as 1 | 2 | 3, nonce),
+ );
+ const marker = warmTurnMarker(turn, nonce);
+ const finalTurn = turn === 3;
+ return [
+ `This is warm Daytona continuity turn ${turn} of 3. Work only in the current execution workspace.`,
+ turn === 1
+ ? `Create ${file} with exactly this one line followed by a newline: ${lines[0]}`
+ : `Before changing anything, read ${file} and verify its content is exactly ${lines.slice(0, -1).join("\\n")} followed by a newline. Then append exactly ${lines.at(-1)} followed by a newline.`,
+ `After the write, verify ${file} contains exactly these lines, once each and in order: ${lines.join(" | ")}.`,
+ `In a native runner, call paperclip_finish exactly once with {reportedWorkDisposition:"${finalTurn ? "done" : "needs_review"}",summary:"${marker}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[{commandOrCheck:"read ${file}",status:"passed"}]}. Wait for that tool call to succeed, then emit exactly ${marker} once as the complete user-facing final response.`,
+ `In a legacy runner, make exactly one public-API completion write after verification: PATCH /api/issues/$PAPERCLIP_TASK_ID with {"status":"${finalTurn ? "done" : "in_review"}","comment":"${marker}"}. Include Authorization and X-Paperclip-Run-Id. Do not POST a separate comment.`,
+ `Do not include ${marker} in any other visible response or write. Do not recreate, truncate, reorder, or duplicate prior lines.`,
+ ].join("\n");
+}
+
+export const daytonaWarmContinuityTask: RunnerTaskFixture = {
+ id: "warm-three-turn",
+ label: "Warm three-turn workspace continuity",
+ groups: ["warm"],
+ workMode: "standard",
+ flow: "warm_three_turn",
+ expectedRunCount: 3,
+ attemptTimeoutMs: { local: 30 * 60_000, daytona: 30 * 60_000 },
+ turnTimeoutMs: 10 * 60_000,
+ expectedTerminalState: { issue: "done", run: "succeeded" },
+ buildTitle: (nonce) => `Runner E2E warm Daytona continuity ${nonce}`,
+ buildVisibleMarker: (nonce) => warmTurnMarker(3, nonce),
+ buildPrompt: (nonce) => warmTurnInstructions(1, nonce),
+ buildFollowupMessages: (nonce) => [
+ warmTurnInstructions(2, nonce),
+ warmTurnInstructions(3, nonce),
+ ],
+ buildMatchers(nonce, execution) {
+ const markers = ([1, 2, 3] as const).map((turn) =>
+ warmTurnMarker(turn, nonce),
+ );
+ return [
+ { kind: "message_exact", expected: markers[2] },
+ ...markers.map(
+ (expected) =>
+ ({ kind: "message_occurrences", expected, count: 1 }) as const,
+ ),
+ { kind: "message_ordered", expected: markers },
+ {
+ kind: "file_exact",
+ path: `daytona-warm-${nonce}.txt`,
+ expected: `${([1, 2, 3] as const)
+ .map((turn) => warmWorkspaceLine(turn, nonce))
+ .join("\n")}\n`,
+ },
+ { kind: "issue_status", expected: "done" },
+ { kind: "run_status", expected: "succeeded" },
+ { kind: "runtime_mode", expected: execution.profile.expectedRuntimeMode },
+ { kind: "environment", expected: "daytona" },
+ ];
+ },
+};
+
+const codexContinuityProfiles = runnerProfiles.filter((profile) =>
+ ["legacy-codex", "runner-codex"].includes(profile.id),
+);
+
export const runnerSuites: readonly RunnerSuiteFixture[] = [
{
id: "core-compatibility",
@@ -762,6 +881,17 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [
excludedExecutionIds: openRouterBreadthExcludedExecutionIds,
},
},
+ {
+ id: "daytona-warm-continuity",
+ label: "Daytona Warm Continuity",
+ description:
+ "Three browser-driven turns on one reusable Daytona sandbox for legacy and native Codex.",
+ groups: ["daytona", "warm"],
+ profiles: codexContinuityProfiles,
+ environments: [daytonaWarmEnvironment],
+ tasks: [daytonaWarmContinuityTask],
+ expectedMatrixSize: 2,
+ },
] as const;
export function suiteDefinitionHash(suite: RunnerSuiteFixture) {
@@ -774,7 +904,10 @@ export function suiteDefinitionHash(suite: RunnerSuiteFixture) {
model: profile.model,
qualification: profile.modelQualification,
})),
- environments: suite.environments.map((environment) => environment.id),
+ environments: suite.environments.map((environment) => ({
+ id: environment.id,
+ configurationKey: environment.configurationKey ?? "default",
+ })),
tasks: suite.tasks.map((task) => ({
id: task.id,
flow: task.flow,
@@ -869,6 +1002,7 @@ export function validateRunnerCatalog(): MatrixExecution[] {
...runnerTasks,
...localIntegrityTasks,
...openRouterBreadthTasks,
+ daytonaWarmContinuityTask,
];
for (const [label, values] of [
["suite", runnerSuites],
@@ -888,6 +1022,7 @@ export function validateRunnerCatalog(): MatrixExecution[] {
...runnerSuites,
...allProfiles,
...runnerEnvironments,
+ daytonaWarmEnvironment,
...allTasks,
]) {
const unknownGroups = fixture.groups.filter(
@@ -916,7 +1051,7 @@ export function validateRunnerCatalog(): MatrixExecution[] {
]),
);
- for (const environment of runnerEnvironments) {
+ for (const environment of [...runnerEnvironments, daytonaWarmEnvironment]) {
const payload = environment.buildEnvironment({
secretRefs: sampleRefs,
daytonaImage:
@@ -968,8 +1103,8 @@ export function validateRunnerCatalog(): MatrixExecution[] {
);
}
}
- if (matrix.length !== 66)
- throw new Error(`Expected 66 runner executions; received ${matrix.length}`);
+ if (matrix.length !== 68)
+ throw new Error(`Expected 68 runner executions; received ${matrix.length}`);
return matrix;
}
diff --git a/tests/runner-e2e/dashboard.ts b/tests/runner-e2e/dashboard.ts
index 820615ff92..6fe3f8d782 100644
--- a/tests/runner-e2e/dashboard.ts
+++ b/tests/runner-e2e/dashboard.ts
@@ -196,6 +196,18 @@ function renderCase(
</tr>`,
)
.join("");
+ const turnTimingRows = (entry?.result.turnTimings ?? [])
+ .map(
+ (timing) => `<tr>
+ <td>${timing.turn}</td>
+ <td><code>${html(timing.runId)}</code></td>
+ <td>${html(timing.leaseAcquisitionOutcome)}</td>
+ <td>${html(timing.schedulerLatencyMs === null ? "unavailable" : durationLabel(timing.schedulerLatencyMs))}</td>
+ <td>${html(timing.runDurationMs === null ? "unavailable" : durationLabel(timing.runDurationMs))}</td>
+ <td>${html(timing.responseLatencyMs === null ? "unavailable" : durationLabel(timing.responseLatencyMs))}</td>
+ </tr>`,
+ )
+ .join("");
const gallery = screenshots.length
? `<div class="gallery" aria-label="Screenshots for ${html(execution.id)}">${screenshots
.map(
@@ -268,6 +280,7 @@ function renderCase(
: ""
}
<p class="detail">${html(detail)}</p>
+ ${turnTimingRows ? `<div class="matcher-wrap"><table class="matchers"><thead><tr><th>Turn</th><th>Run</th><th>Lease</th><th>Scheduler</th><th>Run duration</th><th>Response</th></tr></thead><tbody>${turnTimingRows}</tbody></table></div>` : ""}
${matcherRows ? `<div class="matcher-wrap"><table class="matchers"><thead><tr><th>Result</th><th>Matcher</th><th>Expectation</th><th>Detail</th></tr></thead><tbody>${matcherRows}</tbody></table></div>` : `<p class="detail">No matcher result was recorded.</p>`}
${entry ? `<details class="usage"><summary>Usage and billing metadata</summary><pre>${html(JSON.stringify({ billing, rawUsage: entry.result.usage ?? null }, null, 2))}</pre></details>` : ""}
${links}
diff --git a/tests/runner-e2e/history.test.ts b/tests/runner-e2e/history.test.ts
index c11f97e0db..32c6a2bc53 100644
--- a/tests/runner-e2e/history.test.ts
+++ b/tests/runner-e2e/history.test.ts
@@ -195,8 +195,8 @@ describe("runner E2E campaign history", () => {
expect(index).toContain("Runner E2E campaigns");
expect(index).toContain("complete-green");
expect(index).toContain("complete-red");
- expect(index).toContain("66/66 passed");
- expect(index).toContain("65/66 passed");
+ expect(index).toContain("68/68 passed");
+ expect(index).toContain("67/68 passed");
expect(index).toContain("Open report&nbsp;→");
expect(index).toContain(
"campaigns/complete-red/public-images/campaign-summary.png",
diff --git a/tests/runner-e2e/live-fixtures.test.ts b/tests/runner-e2e/live-fixtures.test.ts
index 1fe34be8e0..a919f07c2b 100644
--- a/tests/runner-e2e/live-fixtures.test.ts
+++ b/tests/runner-e2e/live-fixtures.test.ts
@@ -70,4 +70,80 @@ describe("live runner fixtures", () => {
"DELETE /api/environments/environment-1?destroyReusableSandboxLeases=true",
);
});
+
+ it("creates a primary project workspace for reusable Daytona scope", async () => {
+ const calls: string[] = [];
+ const api = {
+ async post(path: string, data?: Record<string, unknown>) {
+ calls.push(`POST ${path}`);
+ if (path === "/api/plugins/install") {
+ return {
+ id: "plugin-daytona",
+ pluginKey: "paperclip.daytona-sandbox-provider",
+ status: "ready",
+ };
+ }
+ if (path === "/api/companies") {
+ return { id: "company-1", name: "Runner E2E" };
+ }
+ if (path.endsWith("/environments")) {
+ return { id: "environment-1", driver: "sandbox" };
+ }
+ if (path.endsWith("/agents")) {
+ return { id: "agent-1", name: "Agent", companyId: "company-1" };
+ }
+ if (path.endsWith("/projects")) {
+ expect(data).toMatchObject({
+ executionWorkspacePolicy: {
+ enabled: true,
+ defaultMode: "shared_workspace",
+ environmentId: "environment-1",
+ },
+ workspace: {
+ sourceType: "local_path",
+ cwd: "/tmp/workspace",
+ isPrimary: true,
+ },
+ });
+ return {
+ id: "project-1",
+ name: data?.name,
+ primaryWorkspace: {
+ id: "project-workspace-1",
+ cwd: "/tmp/workspace",
+ },
+ };
+ }
+ throw new Error(`Unexpected POST ${path}`);
+ },
+ async postSensitive(_path: string, data?: Record<string, unknown>) {
+ return { id: `secret-${String(data?.key).toLowerCase()}` };
+ },
+ async delete() {},
+ } as unknown as RunnerApi;
+ const execution = runnerMatrix.find(
+ (candidate) =>
+ candidate.id ===
+ "daytona-warm-continuity.runner-codex.daytona.warm-three-turn",
+ )!;
+
+ const fixtures = await setupLiveFixtures({
+ api,
+ execution,
+ executionNonce: "nonce",
+ workspacePath: "/tmp/workspace",
+ credentials: {
+ OPENAI_API_KEY: "openai-test-value",
+ DAYTONA_API_KEY: "daytona-test-value",
+ },
+ daytonaImage:
+ "ghcr.io/paperclip/image@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+ });
+
+ expect(fixtures.project?.primaryWorkspace?.id).toBe("project-workspace-1");
+ expect(
+ calls.indexOf("POST /api/companies/company-1/environments"),
+ ).toBeLessThan(calls.indexOf("POST /api/companies/company-1/projects"));
+ await fixtures.teardown();
+ });
});
diff --git a/tests/runner-e2e/live-fixtures.ts b/tests/runner-e2e/live-fixtures.ts
index 4dba281fb7..de9c242a42 100644
--- a/tests/runner-e2e/live-fixtures.ts
+++ b/tests/runner-e2e/live-fixtures.ts
@@ -32,12 +32,21 @@ interface AgentRecord {
name: string;
companyId: string;
}
+interface ProjectRecord {
+ id: string;
+ name: string;
+ primaryWorkspace?: {
+ id: string;
+ cwd?: string | null;
+ } | null;
+}
export interface LiveFixtureValues {
company: CompanyRecord;
secretRefs: SecretReferenceMap;
environment: EnvironmentRecord;
agent: AgentRecord;
+ project?: ProjectRecord;
teardown(): Promise<void>;
}
@@ -210,12 +219,51 @@ export async function setupLiveFixtures(input: {
},
});
+ if (execution.environment.configurationKey === "warm-reuse-v1") {
+ registry.register<ProjectRecord>({
+ id: "project",
+ dependencies: ["company", "environment"],
+ async setup(resolved) {
+ const company = value<CompanyRecord>(resolved, "company");
+ const environment = value<EnvironmentRecord>(resolved, "environment");
+ return api.post<ProjectRecord>(
+ `/api/companies/${company.id}/projects`,
+ {
+ name: `Runner E2E warm project ${input.executionNonce}`,
+ description:
+ "Ephemeral project anchoring a reusable Daytona execution workspace",
+ executionWorkspacePolicy: {
+ enabled: true,
+ defaultMode: "shared_workspace",
+ sharedWorkspaceConcurrency: "serialize",
+ allowIssueOverride: false,
+ environmentId: environment.id,
+ workspaceStrategy: { type: "project_primary" },
+ },
+ workspace: {
+ name: "Primary",
+ sourceType: "local_path",
+ cwd: input.workspacePath,
+ isPrimary: true,
+ },
+ },
+ );
+ },
+ async teardown() {
+ // The isolated instance is deleted after provider resources are gone.
+ },
+ });
+ }
+
const setup = await registry.setupAll();
return {
company: value<CompanyRecord>(setup.values, "company"),
secretRefs: value<SecretReferenceMap>(setup.values, "secrets"),
environment: value<EnvironmentRecord>(setup.values, "environment"),
agent: value<AgentRecord>(setup.values, "agent"),
+ ...(setup.values.has("project")
+ ? { project: value<ProjectRecord>(setup.values, "project") }
+ : {}),
teardown: setup.teardown,
};
}
diff --git a/tests/runner-e2e/matchers.ts b/tests/runner-e2e/matchers.ts
index 8ba474fa15..650fd907f4 100644
--- a/tests/runner-e2e/matchers.ts
+++ b/tests/runner-e2e/matchers.ts
@@ -116,6 +116,7 @@ export async function evaluateMatcher(
passed = actual === matcher.expected;
} else if (
matcher.kind === "file_exists" ||
+ matcher.kind === "file_exact" ||
matcher.kind === "file_contains"
) {
try {
@@ -124,7 +125,9 @@ export async function evaluateMatcher(
(await readFile(matcher.path, "utf8"));
passed =
matcher.kind === "file_exists" ||
- String(actual).includes(matcher.expected);
+ (matcher.kind === "file_exact"
+ ? String(actual) === matcher.expected
+ : String(actual).includes(matcher.expected));
} catch {
actual = undefined;
passed = false;
diff --git a/tests/runner-e2e/report.test.ts b/tests/runner-e2e/report.test.ts
index 2f949b6bc5..4ac6453cf0 100644
--- a/tests/runner-e2e/report.test.ts
+++ b/tests/runner-e2e/report.test.ts
@@ -44,6 +44,19 @@ describe("runner E2E report aggregation", () => {
workflowRunUrl: "https://example.test/actions/runs/forged",
},
runIds: ["run-2"],
+ turnTimings: [
+ {
+ turn: 1,
+ submittedAt: "2026-08-26T00:00:00.000Z",
+ runStartedAt: "2026-08-26T00:00:00.100Z",
+ runFinishedAt: "2026-08-26T00:00:01.000Z",
+ schedulerLatencyMs: 100,
+ runDurationMs: 900,
+ responseLatencyMs: 1_000,
+ runId: "run-2",
+ leaseAcquisitionOutcome: "created",
+ },
+ ],
usage: {
inputTokens: 1_250,
outputTokens: 75,
@@ -201,6 +214,9 @@ describe("runner E2E report aggregation", () => {
);
expect(dashboard).toContain("message_contains");
expect(dashboard).toContain("Matchers and test context");
+ expect(dashboard).toContain("Scheduler");
+ expect(dashboard).toContain("Run duration");
+ expect(dashboard).toContain("100ms");
expect(dashboard).toContain("Campaign billing summary");
expect(dashboard).toContain("LLM reported subtotal");
expect(dashboard).toContain("Agent execution time");
diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts
index a1c2682b4e..f97cd952d6 100644
--- a/tests/runner-e2e/runner.spec.ts
+++ b/tests/runner-e2e/runner.spec.ts
@@ -44,6 +44,9 @@ interface IssueRecord {
status: string;
workMode?: string;
assigneeAgentId?: string | null;
+ projectId?: string | null;
+ projectWorkspaceId?: string | null;
+ executionWorkspaceId?: string | null;
executionRunId?: string | null;
checkoutRunId?: string | null;
}
@@ -66,6 +69,9 @@ interface RunRecord {
continuationAttempt?: number;
retryOfRunId?: string | null;
runnerInstanceId?: string | null;
+ nativeSessionId?: string | null;
+ processPid?: number | null;
+ processStartedAt?: string | null;
contextSnapshot?: Record<string, unknown> | null;
runnerProfileJson?: Record<string, unknown> | null;
usageJson?: Record<string, unknown> | null;
@@ -80,13 +86,17 @@ interface RunRecord {
interface EnvironmentLeaseRecord {
id: string;
+ status?: string;
+ leasePolicy?: string;
+ providerLeaseId?: string | null;
+ executionWorkspaceId?: string | null;
+ metadata?: Record<string, unknown> | null;
issueId?: string | null;
heartbeatRunId?: string | null;
provider?: string | null;
acquiredAt?: string | null;
releasedAt?: string | null;
updatedAt?: string | null;
- metadata?: Record<string, unknown> | null;
}
interface InteractionRecord {
@@ -290,6 +300,7 @@ async function createTaskThroughUi(input: {
title: string;
prompt: string;
workMode: "standard" | "planning" | "ask";
+ projectName?: string;
}) {
const issuesUrl = `/${encodeURIComponent(input.issuePrefix)}/issues`;
const newTask = input.page.getByRole("button", { name: "New Task" }).first();
@@ -336,9 +347,29 @@ async function createTaskThroughUi(input: {
.getByPlaceholder("Search assignees...")
.fill(input.agentName);
await input.page.getByText(input.agentName, { exact: true }).last().click();
+ if (input.projectName) {
+ const dialog = input.page.getByRole("dialog");
+ await dialog.getByRole("button", { name: "Project", exact: true }).click();
+ await dialog.getByPlaceholder("Search projects...").fill(input.projectName);
+ await dialog.getByText(input.projectName, { exact: true }).last().click();
+ }
+ const submittedAtMs = Date.now();
await input.page
.getByRole("button", { name: "Create Task", exact: true })
.click();
+ return submittedAtMs;
+}
+
+async function submitTaskReply(page: Page, body: string): Promise<number> {
+ const composer = page.getByTestId("task-chat-composer-input").last();
+ await expect(composer).toBeVisible({ timeout: 30_000 });
+ await composer
+ .locator('[contenteditable="true"], textarea')
+ .first()
+ .fill(body);
+ const submittedAtMs = Date.now();
+ await page.getByTestId("task-chat-composer-send").last().click();
+ return submittedAtMs;
}
function matchingRuns(runs: RunRecord[], issue: IssueRecord) {
@@ -524,7 +555,11 @@ for (const execution of executions) {
page,
request,
}, testInfo) => {
- test.setTimeout(deadlineMs + 90_000);
+ test.setTimeout(
+ execution.task.flow === "warm_three_turn"
+ ? deadlineMs
+ : deadlineMs + 90_000,
+ );
const startedAtMs = Date.now();
const startedAt = new Date(startedAtMs).toISOString();
const nonce = `${randomBytes(6).toString("hex")}-${attempt}`;
@@ -541,6 +576,8 @@ for (const execution of executions) {
let selectedRuns: RunRecord[] = [];
let runtimeLeases: EnvironmentLeaseRecord[] = [];
let matcherResults: MatcherResult[] = [];
+ let turnTimings: NonNullable<RunnerE2EResult["turnTimings"]> | undefined;
+ const turnSubmissionTimesMs: number[] = [];
const screenshots: NonNullable<RunnerE2EResult["screenshots"]> = [];
let primaryError: unknown;
let failureClassOverride: FailureClass | undefined;
@@ -746,14 +783,17 @@ for (const execution of executions) {
throw new Error(
"Created fixture company did not return an issue prefix",
);
- await createTaskThroughUi({
- page,
- issuePrefix,
- agentName: fixtures.agent.name,
- title,
- prompt,
- workMode: execution.task.workMode,
- });
+ turnSubmissionTimesMs.push(
+ await createTaskThroughUi({
+ page,
+ issuePrefix,
+ agentName: fixtures.agent.name,
+ title,
+ prompt,
+ workMode: execution.task.workMode,
+ projectName: fixtures.project?.name,
+ }),
+ );
const deadlineAt = startedAtMs + deadlineMs;
issue = await pollUntil({
@@ -781,6 +821,16 @@ for (const execution of executions) {
`UI-created issue work mode was ${String(issue.workMode)}; expected ${execution.task.workMode}`,
);
}
+ if (fixtures.project) {
+ if (
+ issue.projectId !== fixtures.project.id ||
+ issue.projectWorkspaceId !== fixtures.project.primaryWorkspace?.id
+ ) {
+ throw new Error(
+ `Warm task did not retain its project/workspace scope: ${JSON.stringify({ projectId: issue.projectId, projectWorkspaceId: issue.projectWorkspaceId })}`,
+ );
+ }
+ }
await page.goto(
`/${encodeURIComponent(issuePrefix)}/issues/${encodeURIComponent(issue.identifier ?? issue.id)}`,
@@ -827,6 +877,7 @@ for (const execution of executions) {
let planLifecycleEvidence: Record<string, unknown> | null = null;
let questionLifecycleEvidence: Record<string, unknown> | null = null;
+ let warmLifecycleEvidence: Record<string, unknown> | null = null;
let expectedQuestionResolution: {
interactionId: string;
optionId: string;
@@ -1214,12 +1265,165 @@ for (const execution of executions) {
.last()
.click();
planLifecycleEvidence = { interaction, plan };
+ } else if (execution.task.flow === "warm_three_turn") {
+ const followups = execution.task.buildFollowupMessages?.(nonce);
+ if (!followups || !fixtures.project?.primaryWorkspace?.id) {
+ throw new Error(
+ `Warm fixture ${execution.task.id} is missing its project or follow-up messages`,
+ );
+ }
+ const workspaceFile = path.join(
+ workspacePath,
+ `daytona-warm-${nonce}.txt`,
+ );
+ const turnEvidence: Array<Record<string, unknown>> = [];
+ for (const completedTurn of [1, 2] as const) {
+ const turnDeadlineAt = Math.min(
+ deadlineAt,
+ turnSubmissionTimesMs[completedTurn - 1]! +
+ (execution.task.turnTimeoutMs ?? 10 * 60_000),
+ );
+ const waitingState = await pollUntil({
+ label: `warm Daytona turn ${completedTurn} review state for issue ${issue.id}`,
+ deadlineAt: turnDeadlineAt,
+ load: loadTaskState,
+ accept: ({ currentIssue, taskRuns }) =>
+ currentIssue.status === "in_review" &&
+ taskRuns.length === completedTurn &&
+ taskRuns.every((run) => run.status === "succeeded"),
+ reject: ({ taskRuns }) =>
+ definitiveRunFailure(taskRuns) ??
+ (taskRuns.length > completedTurn
+ ? `warm turn ${completedTurn} dispatched duplicate runs`
+ : undefined),
+ });
+ const expectedPrefix = `${Array.from(
+ { length: completedTurn },
+ (_, index) => `T${index + 1}_${nonce}`,
+ ).join("\n")}\n`;
+ const hostContent = await readFile(workspaceFile, "utf8");
+ if (hostContent !== expectedPrefix) {
+ throw new Error(
+ `Host workspace was not finalized after warm turn ${completedTurn}: expected ${JSON.stringify(expectedPrefix)}, observed ${JSON.stringify(hostContent)}`,
+ );
+ }
+ if (
+ waitingState.currentIssue.projectId !== fixtures.project.id ||
+ waitingState.currentIssue.projectWorkspaceId !==
+ fixtures.project.primaryWorkspace.id ||
+ !waitingState.currentIssue.executionWorkspaceId
+ ) {
+ throw new Error(
+ `Warm turn ${completedTurn} lost its project execution-workspace scope`,
+ );
+ }
+ const completedRunIds = new Set(
+ waitingState.taskRuns.map((candidate) => candidate.id),
+ );
+ const retainedTurnLeases = await pollUntil({
+ label: `retained Daytona leases after warm turn ${completedTurn}`,
+ deadlineAt: Math.min(turnDeadlineAt, Date.now() + 30_000),
+ intervalMs: 500,
+ load: () =>
+ api.get<EnvironmentLeaseRecord[]>(
+ `/api/environments/${fixtures!.environment.id}/leases`,
+ ),
+ accept: (leases) => {
+ const runOrder = new Map(
+ waitingState.taskRuns.map((candidate, index) => [
+ candidate.id,
+ index,
+ ]),
+ );
+ const completed = leases
+ .filter(
+ (lease) =>
+ lease.heartbeatRunId &&
+ completedRunIds.has(lease.heartbeatRunId),
+ )
+ .sort(
+ (left, right) =>
+ (runOrder.get(left.heartbeatRunId ?? "") ?? 0) -
+ (runOrder.get(right.heartbeatRunId ?? "") ?? 0),
+ );
+ return (
+ completed.length === completedTurn &&
+ completed.every(
+ (lease) =>
+ lease.status === "retained" &&
+ lease.leasePolicy === "reuse_by_environment" &&
+ typeof lease.providerLeaseId === "string" &&
+ record(lease.metadata).sandboxState === "started",
+ ) &&
+ completed
+ .slice(1)
+ .every(
+ (lease) =>
+ record(lease.metadata).resumedFromState === "started",
+ )
+ );
+ },
+ });
+ const turnRunOrder = new Map(
+ waitingState.taskRuns.map((candidate, index) => [
+ candidate.id,
+ index,
+ ]),
+ );
+ const completedLeases = retainedTurnLeases
+ .filter(
+ (lease) =>
+ lease.heartbeatRunId &&
+ completedRunIds.has(lease.heartbeatRunId),
+ )
+ .sort(
+ (left, right) =>
+ (turnRunOrder.get(left.heartbeatRunId ?? "") ?? 0) -
+ (turnRunOrder.get(right.heartbeatRunId ?? "") ?? 0),
+ );
+ if (
+ new Set(completedLeases.map((lease) => lease.providerLeaseId))
+ .size !== 1
+ ) {
+ throw new Error(
+ `Warm turn ${completedTurn} replaced its Daytona sandbox`,
+ );
+ }
+ turnEvidence.push({
+ turn: completedTurn,
+ issue: waitingState.currentIssue,
+ run: waitingState.taskRuns.at(-1),
+ hostContent,
+ leases: completedLeases,
+ });
+ await page.goto(
+ `/${encodeURIComponent(issuePrefix)}/issues/${encodeURIComponent(issue.identifier ?? issue.id)}`,
+ { waitUntil: "domcontentloaded" },
+ );
+ await captureScreenshot(
+ `warm-turn-${completedTurn}`,
+ `Warm Daytona turn ${completedTurn} awaiting review`,
+ `warm-turn-${completedTurn}.png`,
+ );
+ turnSubmissionTimesMs.push(
+ await submitTaskReply(page, followups[completedTurn - 1]),
+ );
+ }
+ warmLifecycleEvidence = { turns: turnEvidence };
}
const taskMatchers = execution.task.buildMatchers(nonce, execution);
+ const terminalDeadlineAt =
+ execution.task.flow === "warm_three_turn"
+ ? Math.min(
+ deadlineAt,
+ turnSubmissionTimesMs.at(-1)! +
+ (execution.task.turnTimeoutMs ?? 10 * 60_000),
+ )
+ : deadlineAt;
let terminal = await pollUntil({
label: `issue ${issue.id} and heartbeat run terminal state`,
- deadlineAt,
+ deadlineAt: terminalDeadlineAt,
load: loadTaskState,
accept: ({ currentIssue, taskRuns }) =>
currentIssue.status === execution.task.expectedTerminalState.issue &&
@@ -1289,6 +1493,45 @@ for (const execution of executions) {
),
);
selectedRuns = sortRunsChronologically(selectedRuns);
+ if (execution.task.flow === "warm_three_turn") {
+ turnTimings = selectedRuns.map((candidate, index) => {
+ const submittedAtMs = turnSubmissionTimesMs[index]!;
+ const runStartedAtMs = candidate.startedAt
+ ? Date.parse(candidate.startedAt)
+ : Number.NaN;
+ const runFinishedAtMs = candidate.finishedAt
+ ? Date.parse(candidate.finishedAt)
+ : Number.NaN;
+ const acquisition = record(
+ record(candidate.contextSnapshot).paperclipEnvironment,
+ ).sandboxLeaseAcquisition;
+ const acquisitionOutcome = record(acquisition).outcome;
+ return {
+ turn: index + 1,
+ submittedAt: new Date(submittedAtMs).toISOString(),
+ runStartedAt: candidate.startedAt ?? null,
+ runFinishedAt: candidate.finishedAt ?? null,
+ schedulerLatencyMs: Number.isFinite(runStartedAtMs)
+ ? Math.max(0, runStartedAtMs - submittedAtMs)
+ : null,
+ runDurationMs:
+ Number.isFinite(runStartedAtMs) &&
+ Number.isFinite(runFinishedAtMs)
+ ? Math.max(0, runFinishedAtMs - runStartedAtMs)
+ : null,
+ responseLatencyMs: Number.isFinite(runFinishedAtMs)
+ ? Math.max(0, runFinishedAtMs - submittedAtMs)
+ : null,
+ runId: candidate.id,
+ leaseAcquisitionOutcome:
+ acquisitionOutcome === "created" ||
+ acquisitionOutcome === "resumed" ||
+ acquisitionOutcome === "replacement"
+ ? acquisitionOutcome
+ : "unknown",
+ };
+ });
+ }
const finalRun = selectedRuns.at(-1)!;
const run =
selectedRuns.find(
@@ -1500,10 +1743,31 @@ for (const execution of executions) {
interactions: terminal.interactions,
},
};
+ const fileObservations = Object.fromEntries(
+ await Promise.all(
+ taskMatchers
+ .filter(
+ (matcher) =>
+ matcher.kind === "file_exists" ||
+ matcher.kind === "file_exact" ||
+ matcher.kind === "file_contains",
+ )
+ .map(async (matcher) => [
+ matcher.path,
+ await readFile(
+ path.isAbsolute(matcher.path)
+ ? matcher.path
+ : path.join(workspacePath, matcher.path),
+ "utf8",
+ ).catch(() => undefined),
+ ]),
+ ),
+ );
matcherResults = await Promise.all(
taskMatchers.map((matcher) =>
evaluateMatcher(matcher, {
...matcherObservation,
+ files: fileObservations,
// Multi-run tasks intentionally retain earlier waiting/revision
// replies. Exact completion text belongs to the chronological
// final run, while occurrence checks still span every agent
@@ -1571,6 +1835,190 @@ for (const execution of executions) {
"expected a Daytona sandbox lease on the run context",
);
}
+ if (execution.task.flow === "warm_three_turn") {
+ const runEnvironmentContexts = selectedRuns.map((candidate) =>
+ record(record(candidate.contextSnapshot).paperclipEnvironment),
+ );
+ const leaseIds = runEnvironmentContexts.map((entry) => entry.leaseId);
+ const acquisitionOutcomes = runEnvironmentContexts.map(
+ (entry) => record(entry.sandboxLeaseAcquisition).outcome,
+ );
+ const projectWorkspaceIds = selectedRuns.map(
+ (candidate) =>
+ record(record(candidate.contextSnapshot).paperclipWorkspace)
+ .workspaceId,
+ );
+ const executionWorkspaceIds = selectedRuns.map(
+ (candidate) => record(candidate.contextSnapshot).executionWorkspaceId,
+ );
+ if (
+ leaseIds.some(
+ (leaseId) => typeof leaseId !== "string" || leaseId.length === 0,
+ )
+ ) {
+ invariantFailures.push(
+ `expected a persisted Daytona lease row for every warm turn; observed ${JSON.stringify(leaseIds)}`,
+ );
+ }
+ if (
+ JSON.stringify(acquisitionOutcomes) !==
+ JSON.stringify(["created", "resumed", "resumed"])
+ ) {
+ invariantFailures.push(
+ `expected warm lease outcomes created,resumed,resumed; observed ${JSON.stringify(acquisitionOutcomes)}`,
+ );
+ }
+ if (
+ !fixtures.project?.primaryWorkspace?.id ||
+ projectWorkspaceIds.some(
+ (workspaceId) =>
+ workspaceId !== fixtures!.project!.primaryWorkspace!.id,
+ ) ||
+ new Set(executionWorkspaceIds).size !== 1 ||
+ executionWorkspaceIds[0] !== issue.executionWorkspaceId ||
+ issue.projectId !== fixtures.project.id ||
+ issue.projectWorkspaceId !== fixtures.project.primaryWorkspace.id ||
+ !issue.executionWorkspaceId
+ ) {
+ invariantFailures.push(
+ `warm task did not preserve project, project-workspace, and execution-workspace identity: ${JSON.stringify({ projectWorkspaceIds, executionWorkspaceIds, projectId: issue.projectId, projectWorkspaceId: issue.projectWorkspaceId, executionWorkspaceId: issue.executionWorkspaceId })}`,
+ );
+ }
+ if (execution.profile.generation === "native") {
+ const stableIdentityFields: Array<{
+ label: string;
+ values: unknown[];
+ }> = [
+ {
+ label: "native session",
+ values: selectedRuns.map(
+ (candidate) => candidate.nativeSessionId,
+ ),
+ },
+ {
+ label: "runner instance",
+ values: selectedRuns.map(
+ (candidate) => candidate.runnerInstanceId,
+ ),
+ },
+ {
+ label: "provider session",
+ values: selectedRuns.map((candidate) => candidate.sessionIdAfter),
+ },
+ {
+ label: "runner pid",
+ values: selectedRuns.map((candidate) => candidate.processPid),
+ },
+ {
+ label: "runner process fingerprint",
+ values: selectedRuns.map(
+ (candidate) => candidate.processStartedAt,
+ ),
+ },
+ ];
+ for (const { label, values } of stableIdentityFields) {
+ if (
+ values.some(
+ (value) =>
+ value === null ||
+ value === undefined ||
+ String(value).length === 0,
+ ) ||
+ new Set(values).size !== 1
+ ) {
+ invariantFailures.push(
+ `expected one stable ${label} across native warm turns; observed ${JSON.stringify(values)}`,
+ );
+ }
+ }
+ }
+ if (
+ turnTimings?.length !== 3 ||
+ turnTimings.some(
+ (timing) =>
+ timing.runStartedAt === null ||
+ timing.runFinishedAt === null ||
+ timing.schedulerLatencyMs === null ||
+ timing.runDurationMs === null ||
+ timing.responseLatencyMs === null ||
+ timing.responseLatencyMs >
+ (execution.task.turnTimeoutMs ?? 10 * 60_000),
+ )
+ ) {
+ invariantFailures.push(
+ `warm turn timing data was incomplete or exceeded its structural deadline: ${JSON.stringify(turnTimings)}`,
+ );
+ }
+ const retainedLeases = await pollUntil({
+ label: `retained warm Daytona lease for issue ${issue.id}`,
+ deadlineAt: Math.min(deadlineAt, Date.now() + 30_000),
+ intervalMs: 500,
+ load: () =>
+ api.get<EnvironmentLeaseRecord[]>(
+ `/api/environments/${fixtures!.environment.id}/leases`,
+ ),
+ accept: (leases) => {
+ const warmLeases = leases.filter(
+ (lease) =>
+ lease.issueId === issue!.id &&
+ selectedRuns.some(
+ (candidate) => candidate.id === lease.heartbeatRunId,
+ ),
+ );
+ return (
+ warmLeases.length === 3 &&
+ warmLeases.every(
+ (lease) =>
+ lease.status === "retained" &&
+ lease.leasePolicy === "reuse_by_environment" &&
+ typeof lease.providerLeaseId === "string" &&
+ record(lease.metadata).sandboxState === "started",
+ )
+ );
+ },
+ });
+ const selectedRunOrder = new Map(
+ selectedRuns.map((candidate, index) => [candidate.id, index]),
+ );
+ const warmLeases = retainedLeases
+ .filter(
+ (lease) =>
+ lease.issueId === issue!.id &&
+ selectedRunOrder.has(lease.heartbeatRunId ?? ""),
+ )
+ .sort(
+ (left, right) =>
+ (selectedRunOrder.get(left.heartbeatRunId ?? "") ?? 0) -
+ (selectedRunOrder.get(right.heartbeatRunId ?? "") ?? 0),
+ );
+ const providerLeaseIds = warmLeases.map(
+ (lease) => lease.providerLeaseId,
+ );
+ const resumedFromStates = warmLeases
+ .slice(1)
+ .map((lease) => record(lease.metadata).resumedFromState);
+ if (
+ new Set(providerLeaseIds).size !== 1 ||
+ typeof providerLeaseIds[0] !== "string" ||
+ JSON.stringify(resumedFromStates) !==
+ JSON.stringify(["started", "started"])
+ ) {
+ invariantFailures.push(
+ `expected one continuously-started Daytona sandbox; observed ${JSON.stringify({ providerLeaseIds, resumedFromStates })}`,
+ );
+ }
+ warmLifecycleEvidence = {
+ ...(warmLifecycleEvidence ?? {}),
+ leaseIds,
+ acquisitionOutcomes,
+ projectWorkspaceIds,
+ executionWorkspaceIds,
+ providerLeaseIds,
+ resumedFromStates,
+ retainedLeases: warmLeases,
+ turnTimings,
+ };
+ }
if (execution.environment.id === "local") {
const runLogContent = String(record(runLog).content ?? "");
// The log endpoint returns NDJSON, so quotes inside each `chunk` are
@@ -1723,6 +2171,7 @@ for (const execution of executions) {
interactions: terminal.interactions,
planLifecycleEvidence,
questionLifecycleEvidence,
+ warmLifecycleEvidence,
matcherResults,
invariantFailures,
runEvents,
@@ -1940,6 +2389,7 @@ for (const execution of executions) {
issueId: issue?.id,
issueIdentifier: issue?.identifier ?? null,
runIds: selectedRuns.map((run) => run.id),
+ ...(turnTimings ? { turnTimings } : {}),
startedAt,
finishedAt: new Date(finishedAtMs).toISOString(),
durationMs: finishedAtMs - startedAtMs,
diff --git a/tests/runner-e2e/support.test.ts b/tests/runner-e2e/support.test.ts
index 79fbe6d3e8..e56d9923db 100644
--- a/tests/runner-e2e/support.test.ts
+++ b/tests/runner-e2e/support.test.ts
@@ -341,6 +341,23 @@ describe("runner E2E matchers", () => {
expect(result?.detail).toContain("observed 2");
});
+ it("matches finalized workspace files byte-for-byte", async () => {
+ const [matched, extraLine] = await evaluateMatchers(
+ [
+ { kind: "file_exact", path: "continuity.txt", expected: "T1\nT2\n" },
+ { kind: "file_exact", path: "duplicate.txt", expected: "T1\nT2\n" },
+ ],
+ {
+ files: {
+ "continuity.txt": "T1\nT2\n",
+ "duplicate.txt": "T1\nT2\nT2\n",
+ },
+ },
+ );
+ expect(matched?.passed).toBe(true);
+ expect(extraLine?.passed).toBe(false);
+ });
+
it("normalizes ordered fragments and evaluates nested JSON Schema", async () => {
const results = await evaluateMatchers(
[
diff --git a/tests/runner-e2e/types.ts b/tests/runner-e2e/types.ts
index d607e60893..fcb5093f7b 100644
--- a/tests/runner-e2e/types.ts
+++ b/tests/runner-e2e/types.ts
@@ -13,7 +13,8 @@ export type RunnerTaskFlow =
| "single_turn"
| "plan_revision_acceptance"
| "question_resume_completion"
- | "plan_approval_completion";
+ | "plan_approval_completion"
+ | "warm_three_turn";
export interface SecretReference {
type: "secret_ref";
@@ -73,6 +74,8 @@ export interface RunnerProfileFixture {
export interface EnvironmentFixture {
id: RunnerEnvironmentId;
+ /** Distinguishes materially different configurations that share a provider ID. */
+ configurationKey?: string;
label: string;
groups: readonly string[];
driver: "local" | "sandbox";
@@ -103,6 +106,7 @@ export type Matcher =
| { kind: "runtime_mode"; expected: RunnerGeneration }
| { kind: "environment"; expected: RunnerEnvironmentId }
| { kind: "file_exists"; path: string }
+ | { kind: "file_exact"; path: string; expected: string }
| { kind: "file_contains"; path: string; expected: string }
| { kind: "artifact_exists"; name: string; mimeType?: string }
| { kind: "json_path"; path: string; expected: unknown }
@@ -124,6 +128,8 @@ export interface RunnerTaskFixture {
buildPrompt(nonce: string): string;
buildVisibleMarker(nonce: string): string;
buildRevisionRequest?(nonce: string): string;
+ buildFollowupMessages?(nonce: string): readonly [string, string];
+ turnTimeoutMs?: number;
buildQuestionAnswer?(nonce: string): {
optionLabel: string;
expectedMarker: string;
@@ -260,6 +266,17 @@ export interface RunnerE2EResult {
issueId?: string;
issueIdentifier?: string | null;
runIds?: string[];
+ turnTimings?: Array<{
+ turn: number;
+ submittedAt: string;
+ runStartedAt: string | null;
+ runFinishedAt: string | null;
+ schedulerLatencyMs: number | null;
+ runDurationMs: number | null;
+ responseLatencyMs: number | null;
+ runId: string;
+ leaseAcquisitionOutcome: "created" | "resumed" | "replacement" | "unknown";
+ }>;
startedAt: string;
finishedAt: string;
durationMs: number;
diff --git a/tests/runner-e2e/workflow-security.test.ts b/tests/runner-e2e/workflow-security.test.ts
index 55997ce9bb..b875add0d3 100644
--- a/tests/runner-e2e/workflow-security.test.ts
+++ b/tests/runner-e2e/workflow-security.test.ts
@@ -4,7 +4,7 @@ import { describe, expect, it } from "vitest";
const repositoryRoot = path.resolve(import.meta.dirname, "../..");
const ordinaryPrTrustedWorkflowRevision =
- "a0a78ee60946a5f79f85b2bd0584fc766fae43bb";
+ "03609aa6ecc9a047ed53d6b6469d8be554fbc46d";
const fullStackTestNeeds =
/needs:\s*\[\s*authorize,\s*target_lock,\s*catalog,\s*daytona_image,\s*build_runner_artifacts,\s*build_remote_provider_pack,?\s*\]/u;
const buildRunnerNeeds =