The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
perf(adapter-utils): content-hash-skip process-session remote script write + lock inbound round-trip count (#10377)
Nicky Leach9ff7e1caf321
diff --git a/packages/adapter-utils/src/execution-target.ts b/packages/adapter-utils/src/execution-target.tsindex 3bf072f552..25777e5732 100644--- a/packages/adapter-utils/src/execution-target.ts+++ b/packages/adapter-utils/src/execution-target.ts@@ -22,6 +22,7 @@ import {sandboxCallbackBridgeDirectories,startSandboxCallbackBridgeServer,startSandboxCallbackBridgeWorker,+ syncRemoteTextFileWithHashSkip,} from "./sandbox-callback-bridge.js";import {createSandboxRunLogTailFactory,@@ -1258,11 +1259,33 @@ async function writeProcessSessionProxyScript(dir: string, port: number, token:return proxyPath;}+// Content-hash-skip the process-session remote script write, mirroring the+// sandbox callback bridge entrypoint sha256 gate. The script is a static+// Paperclip-authored `.mjs` that only changes when the build changes, so on a+// warm start (same sandbox, script already present) the single sha-gate exec+// skips the ~3-exec base64 upload entirely. `syncRemoteTextFileWithHashSkip`+// fails loud on a check error rather than silently re-uploading.async function syncProcessSessionRemoteScript(input: {- client: ReturnType<typeof createCommandManagedSandboxCallbackBridgeQueueClient>;+ runner: CommandManagedRuntimeRunner;+ remoteCwd: string;+ remoteScriptDir: string;remoteScriptPath: string;-}): Promise<void> {- await input.client.writeTextFile(input.remoteScriptPath, getProcessSessionRemoteSource());+ timeoutMs?: number | null;+ shellCommand?: "bash" | "sh" | null;+}): Promise<{ uploaded: boolean }> {+ const { uploaded } = await syncRemoteTextFileWithHashSkip({+ runner: input.runner,+ remoteCwd: input.remoteCwd,+ remoteDir: input.remoteScriptDir,+ remotePath: input.remoteScriptPath,+ body: getProcessSessionRemoteSource(),+ label: "Process session remote script",+ action: "sync process session remote script",+ lockDir: path.posix.join(input.remoteScriptDir, ".paperclip-process-session-script.lock"),+ timeoutMs: input.timeoutMs,+ shellCommand: input.shellCommand,+ });+ return { uploaded };}async function readRemoteJsonFiles(input: {@@ -1342,7 +1365,14 @@ export async function startAdapterExecutionTargetProcessSessionBridge(input: {await client.makeDir(stdinDir);await client.makeDir(eventsDir);- await syncProcessSessionRemoteScript({ client, remoteScriptPath });+ await syncProcessSessionRemoteScript({+ runner,+ remoteCwd: target.remoteCwd,+ remoteScriptDir: bridgeRuntimeDir,+ remoteScriptPath,+ timeoutMs,+ shellCommand,+ });// Resolve the launch env AFTER the env-independent setup above, so a caller// can defer it until an upstream dependency (e.g. the paperclip bridge's env)diff --git a/packages/adapter-utils/src/sandbox-callback-bridge.test.ts b/packages/adapter-utils/src/sandbox-callback-bridge.test.tsindex d47e1dc038..af7ed6c96d 100644--- a/packages/adapter-utils/src/sandbox-callback-bridge.test.ts+++ b/packages/adapter-utils/src/sandbox-callback-bridge.test.ts@@ -13,6 +13,7 @@ import {createSandboxCallbackBridgeAsset,createSandboxCallbackBridgeToken,sandboxCallbackBridgeDirectories,+ syncRemoteTextFileWithHashSkip,syncSandboxCallbackBridgeEntrypoint,startSandboxCallbackBridgeServer,startSandboxCallbackBridgeWorker,@@ -862,6 +863,135 @@ describe("sandbox callback bridge", () => {).resolves.toEqual([]);});+ // The process-session remote script is a static, Paperclip-authored `.mjs`+ // written into the sandbox on every bridge start. `syncRemoteTextFileWithHashSkip`+ // (which now backs that write, mirroring the bridge-entrypoint sha256 gate)+ // content-hash-skips it so a warm start where the remote script already matches+ // costs ZERO write execs instead of the prior ~3 (prepare/append/finalize base64+ // upload).+ it("test_process_session_script_skipped_when_remote_hash_matches: warm start with a matching remote hash writes 0 execs", async () => {+ const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-hashskip-warm-"));+ cleanupDirs.push(rootDir);+ const remoteDir = path.join(rootDir, "runtime", "codex", "process-sessions");+ const remotePath = path.posix.join(remoteDir, "paperclip-process-session-remote.mjs");+ const lockDir = path.posix.join(remoteDir, ".paperclip-process-session-script.lock");+ const body = "console.log('process session remote script v1');\n";++ let execCount = 0;+ const inner = createExecRunner();+ const runner = {+ execute: async (input: Parameters<typeof inner.execute>[0]) => {+ execCount += 1;+ return inner.execute(input);+ },+ };+ const args = {+ runner,+ remoteCwd: rootDir,+ remoteDir,+ remotePath,+ body,+ label: "Process session remote script",+ action: "sync process session remote script",+ lockDir,+ timeoutMs: 30_000,+ } as const;++ // Cold start: the script is uploaded (single sha-gate exec that writes).+ const first = await syncRemoteTextFileWithHashSkip(args);+ expect(first.uploaded).toBe(true);+ await expect(readFile(remotePath, "utf8")).resolves.toBe(body);++ // Warm start: the remote hash matches, so the write is skipped entirely.+ execCount = 0;+ const second = await syncRemoteTextFileWithHashSkip(args);+ expect(second.uploaded).toBe(false);+ // A single hash-gate round-trip that performed 0 writes (down from ~3 execs).+ expect(execCount).toBe(1);+ // sha is still returned on the skip path so callers get a well-formed result.+ expect(second.sha256).toBe(first.sha256);+ // The remote file is unchanged and no upload/partial/lock leftovers remain.+ await expect(readFile(remotePath, "utf8")).resolves.toBe(body);+ await expect(+ readdir(remoteDir).then((entries) =>+ entries.filter(+ (entry) =>+ entry.endsWith(".paperclip-upload.b64") ||+ entry.endsWith(".partial") ||+ entry === ".paperclip-process-session-script.lock",+ ),+ ),+ ).resolves.toEqual([]);+ });++ it("test_process_session_script_rewritten_on_hash_mismatch: a mismatched remote hash still rewrites the script", async () => {+ const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-hashskip-cold-"));+ cleanupDirs.push(rootDir);+ const remoteDir = path.join(rootDir, "runtime", "codex", "process-sessions");+ const remotePath = path.posix.join(remoteDir, "paperclip-process-session-remote.mjs");+ const lockDir = path.posix.join(remoteDir, ".paperclip-process-session-script.lock");+ const body = "console.log('process session remote script v2');\n";++ // Pre-seed the remote with a DIFFERENT script (a prior/stale build).+ await mkdir(remoteDir, { recursive: true });+ await writeFile(remotePath, "console.log('stale remote script');\n", "utf8");++ const result = await syncRemoteTextFileWithHashSkip({+ runner: createExecRunner(),+ remoteCwd: rootDir,+ remoteDir,+ remotePath,+ body,+ label: "Process session remote script",+ action: "sync process session remote script",+ lockDir,+ timeoutMs: 30_000,+ });++ expect(result.uploaded).toBe(true);+ await expect(readFile(remotePath, "utf8")).resolves.toBe(body);+ });++ it("fails loud when the hash-skip sync exec errors instead of silently re-uploading", async () => {+ const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-hashskip-fail-"));+ cleanupDirs.push(rootDir);+ const remoteDir = path.join(rootDir, "runtime", "codex", "process-sessions");+ const remotePath = path.posix.join(remoteDir, "paperclip-process-session-remote.mjs");+ const lockDir = path.posix.join(remoteDir, ".paperclip-process-session-script.lock");++ // A runner whose exec fails: the hash-gate cannot be evaluated. The write+ // must surface the failure, never swallow it and re-upload behind a green+ // return value.+ const runner = {+ execute: async () => ({+ exitCode: 1,+ signal: null,+ timedOut: false,+ stdout: "",+ stderr: "hash gate boom",+ pid: null,+ startedAt: new Date().toISOString(),+ }),+ };++ await expect(+ syncRemoteTextFileWithHashSkip({+ runner,+ remoteCwd: rootDir,+ remoteDir,+ remotePath,+ body: "console.log('never written');\n",+ label: "Process session remote script",+ action: "sync process session remote script",+ lockDir,+ timeoutMs: 30_000,+ }),+ ).rejects.toThrow(/sync process session remote script/i);++ // Nothing was written to the remote path on the failure path.+ await expect(readFile(remotePath, "utf8")).rejects.toThrow();+ });+it("permits the documented heartbeat surface and denies unrelated routes", () => {const allowed: Array<{ method: string; path: string }> = [{ method: "GET", path: "/api/agents/me" },diff --git a/packages/adapter-utils/src/sandbox-callback-bridge.ts b/packages/adapter-utils/src/sandbox-callback-bridge.tsindex 4bc19f08ae..8ba56bc805 100644--- a/packages/adapter-utils/src/sandbox-callback-bridge.ts+++ b/packages/adapter-utils/src/sandbox-callback-bridge.ts@@ -777,34 +777,54 @@ export async function startSandboxCallbackBridgeWorker(input: {};}-export async function syncSandboxCallbackBridgeEntrypoint(input: {+/**+ * Content-hash-skip write of a Paperclip-authored text file into the sandbox, in+ * a SINGLE remote exec. The body's sha256 is computed on the host; the one shell+ * round-trip skips the write entirely when the remote file already hashes to the+ * same value (warm start — 0 write execs), otherwise it uploads (base64 over+ * stdin), verifies the decoded bytes, and atomically renames into place. A+ * PID-liveness lock serializes concurrent writers to the same path and the+ * verify step guards against a torn upload.+ *+ * Fail loudly: a non-zero remote exit (surfaced by `requireSuccessfulResult`) or+ * malformed result JSON throws rather than silently re-uploading and masking a+ * failed check. The only intentional degradation is when the remote has neither+ * `sha256sum` nor `shasum` — then the skip cannot be proven and we conservatively+ * re-upload (and the post-upload verify is best-effort, as noted inline).+ */+export async function syncRemoteTextFileWithHashSkip(input: {runner: CommandManagedRuntimeRunner;remoteCwd: string;- assetRemoteDir: string;- bridgeAsset: SandboxCallbackBridgeAsset;+ remoteDir: string;+ remotePath: string;+ body: string;+ // Human-readable noun phrase used in fail-loud messages, e.g.+ // "Sandbox callback bridge entrypoint" / "Process session remote script".+ label: string;+ // Short action label for `requireSuccessfulResult`, e.g.+ // "sync sandbox callback bridge entrypoint".+ action: string;+ lockDir: string;timeoutMs?: number | null;shellCommand?: "bash" | "sh" | null;-}): Promise<{ remoteEntrypoint: string; sha256: string; uploaded: boolean }> {+}): Promise<{ uploaded: boolean; sha256: string }> {const timeoutMs = normalizeTimeoutMs(input.timeoutMs, DEFAULT_BRIDGE_RESPONSE_TIMEOUT_MS);const shellCommand = preferredShellForSandbox(input.shellCommand);- const remoteEntrypoint = path.posix.join(input.assetRemoteDir, SANDBOX_CALLBACK_BRIDGE_ENTRYPOINT);- const remoteEntrypointPartial = `${remoteEntrypoint}.partial`;- const remoteUploadPath = `${remoteEntrypoint}.paperclip-upload.b64`;- const remoteLockDir = path.posix.join(input.assetRemoteDir, ".paperclip-bridge-upload.lock");- const entrypointSource = await fs.readFile(input.bridgeAsset.entrypoint, "utf8");- const entrypointBase64 = toBuffer(Buffer.from(entrypointSource, "utf8")).toString("base64");- const sha256 = createHash("sha256").update(entrypointSource, "utf8").digest("hex");+ const remotePartial = `${input.remotePath}.partial`;+ const remoteUploadPath = `${input.remotePath}.paperclip-upload.b64`;+ const base64Body = toBuffer(Buffer.from(input.body, "utf8")).toString("base64");+ const sha256 = createHash("sha256").update(input.body, "utf8").digest("hex");const syncResult = await runShell(input.runner,input.remoteCwd,["set -eu",- `remote_dir=${shellQuote(input.assetRemoteDir)}`,- `remote_path=${shellQuote(remoteEntrypoint)}`,- `remote_partial=${shellQuote(remoteEntrypointPartial)}`,+ `remote_dir=${shellQuote(input.remoteDir)}`,+ `remote_path=${shellQuote(input.remotePath)}`,+ `remote_partial=${shellQuote(remotePartial)}`,`remote_upload=${shellQuote(remoteUploadPath)}`,- `lock_dir=${shellQuote(remoteLockDir)}`,+ `lock_dir=${shellQuote(input.lockDir)}`,`expected_sha=${shellQuote(sha256)}`,"hash_file() {"," if command -v sha256sum >/dev/null 2>&1; then",@@ -818,7 +838,7 @@ export async function syncSandboxCallbackBridgeEntrypoint(input: {" return 127","}","mkdir -p \"$remote_dir\"",- ...buildRemotePidLockAcquireScript("\"$lock_dir\"", "Timed out acquiring sandbox callback bridge upload lock."),+ ...buildRemotePidLockAcquireScript("\"$lock_dir\"", `Timed out acquiring ${input.label} upload lock.`),...buildRemotePidLockCleanupScript("\"$lock_dir\"", ["rm -f \"$remote_upload\" \"$remote_partial\"",]),@@ -839,30 +859,56 @@ export async function syncSandboxCallbackBridgeEntrypoint(input: {// best-effort and we trust base64-decode + atomic rename below."if partial_sha=\"$(hash_file \"$remote_partial\" 2>/dev/null)\"; then"," if [ \"$partial_sha\" != \"$expected_sha\" ]; then",- " echo \"Sandbox callback bridge entrypoint upload sha mismatch.\" >&2",+ ` echo ${shellQuote(`${input.label} upload sha mismatch.`)} >&2`," exit 1"," fi","else",- " echo \"Sandbox callback bridge entrypoint sha verify skipped: no sha256sum/shasum on remote.\" >&2",+ ` echo ${shellQuote(`${input.label} sha verify skipped: no sha256sum/shasum on remote.`)} >&2`,"fi","mv \"$remote_partial\" \"$remote_path\"","printf '{\"uploaded\":true}\\n'",].join("\n"),timeoutMs,shellCommand,- entrypointBase64,+ base64Body,);- requireSuccessfulResult("sync sandbox callback bridge entrypoint", syncResult);+ requireSuccessfulResult(input.action, syncResult);let uploaded = false;try {uploaded = JSON.parse(syncResult.stdout.trim())?.uploaded === true;} catch (error) {throw new Error(- `Sandbox callback bridge sync wrote invalid result JSON: ${error instanceof Error ? error.message : String(error)}`,+ `${input.label} sync wrote invalid result JSON: ${error instanceof Error ? error.message : String(error)}`,);}+ return { uploaded, sha256 };+}++export async function syncSandboxCallbackBridgeEntrypoint(input: {+ runner: CommandManagedRuntimeRunner;+ remoteCwd: string;+ assetRemoteDir: string;+ bridgeAsset: SandboxCallbackBridgeAsset;+ timeoutMs?: number | null;+ shellCommand?: "bash" | "sh" | null;+}): Promise<{ remoteEntrypoint: string; sha256: string; uploaded: boolean }> {+ const remoteEntrypoint = path.posix.join(input.assetRemoteDir, SANDBOX_CALLBACK_BRIDGE_ENTRYPOINT);+ const entrypointSource = await fs.readFile(input.bridgeAsset.entrypoint, "utf8");+ const { uploaded, sha256 } = await syncRemoteTextFileWithHashSkip({+ runner: input.runner,+ remoteCwd: input.remoteCwd,+ remoteDir: input.assetRemoteDir,+ remotePath: remoteEntrypoint,+ body: entrypointSource,+ label: "Sandbox callback bridge entrypoint",+ action: "sync sandbox callback bridge entrypoint",+ lockDir: path.posix.join(input.assetRemoteDir, ".paperclip-bridge-upload.lock"),+ timeoutMs: input.timeoutMs,+ shellCommand: input.shellCommand,+ });+return {remoteEntrypoint,sha256,diff --git a/packages/adapter-utils/src/sandbox-managed-runtime.test.ts b/packages/adapter-utils/src/sandbox-managed-runtime.test.tsindex b82ae7301a..8444f05915 100644--- a/packages/adapter-utils/src/sandbox-managed-runtime.test.ts+++ b/packages/adapter-utils/src/sandbox-managed-runtime.test.ts@@ -1363,6 +1363,126 @@ describe("sandbox managed runtime", () => {expect(prepared.workspaceRemoteDir).toBe(remoteWorkspaceDir);});+ // Regression lock: a representative `codex_local` start stages its inbound+ // bytes — git history, workspace overlay, and the managed Codex `home` asset+ // (auth.json merge) — as EXACTLY ONE `syncIn` operation each. Every inbound step+ // is routed through `client.syncIn` (one native `uploadFiles` round-trip per+ // operation, with the extract/merge carried as provider-executed+ // `postUploadCommands`), with no separate custom-provision diversion. Assert+ // the collapsed round-trip count so a future change that re-inlines a+ // `writeFile`+`run` sequence — or fans one staging step across multiple+ // operations — fails loudly here instead of silently regressing the start path.+ it("collapses a representative codex_local start to one syncIn round-trip per inbound staging step", async () => {+ const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-sandbox-codex-roundtrip-"));+ cleanupDirs.push(rootDir);+ const sourceRepoDir = path.join(rootDir, "source-repo");+ const localWorkspaceDir = path.join(rootDir, "local-workspace");+ const remoteWorkspaceDir = path.join(rootDir, "remote-workspace");+ const homeDir = path.join(rootDir, "codex-home");++ // Git-backed workspace → git history + workspace overlay are two staging steps.+ await mkdir(sourceRepoDir, { recursive: true });+ await git(sourceRepoDir, ["init"]);+ await git(sourceRepoDir, ["checkout", "-b", "main"]);+ await git(sourceRepoDir, ["config", "user.name", "Paperclip Test"]);+ await git(sourceRepoDir, ["config", "user.email", "[email protected]"]);+ await writeFile(path.join(sourceRepoDir, "tracked.txt"), "tracked\n", "utf8");+ await git(sourceRepoDir, ["add", "tracked.txt"]);+ await git(sourceRepoDir, ["commit", "-m", "base"]);+ await git(sourceRepoDir, ["worktree", "add", "-b", "work", localWorkspaceDir, "HEAD"]);++ // Managed Codex home with an auth.json that a custom post-upload command+ // merges in-sandbox — the credential path is routed onto native uploadFiles.+ await mkdir(homeDir, { recursive: true });+ await writeFile(path.join(homeDir, "auth.json"), "{\"OPENAI_API_KEY\":\"sk-test\"}\n", "utf8");+ await writeFile(path.join(homeDir, "config.toml"), "model = \"gpt\"\n", "utf8");++ // A native runner delegates every staging step to `syncIn`; ANY direct+ // writeFile/run exec is a collapse regression.+ const directWrites: string[] = [];+ const directRuns: string[] = [];+ const client: SandboxManagedRuntimeClient = {+ makeDir: async (remotePath) => {+ await mkdir(remotePath, { recursive: true });+ },+ writeFile: async (remotePath, bytes) => {+ directWrites.push(remotePath);+ await mkdir(path.dirname(remotePath), { recursive: true });+ await writeFile(remotePath, Buffer.from(bytes));+ },+ readFile: async (remotePath) => await readFile(remotePath),+ listFiles: async () => [],+ remove: async (remotePath) => {+ await rm(remotePath, { recursive: true, force: true });+ },+ run: async (command) => {+ directRuns.push(command);+ await execFile("sh", ["-c", command], { maxBuffer: 32 * 1024 * 1024 });+ },+ };+ const captured: SandboxSyncOperation[] = [];+ attachNativeRecordingSyncIn(client, captured);++ const q = (value: string) => `'${value.replace(/'/g, `'\"'\"'`)}'`;+ const prepared = await prepareSandboxManagedRuntime({+ spec: {+ transport: "sandbox",+ provider: "test",+ sandboxId: "sandbox-1",+ remoteCwd: remoteWorkspaceDir,+ timeoutMs: 30_000,+ apiKey: null,+ },+ adapterKey: "codex",+ client,+ workspaceLocalDir: localWorkspaceDir,+ assets: [{+ key: "home",+ localDir: homeDir,+ provision: {+ stageFiles: [{ name: "home-merge.sh", contents: "#!/bin/sh\ntar -xf \"$2\" -C \"$1\"\n" }],+ postUploadCommand: ({ assetTarPath, assetDir, runtimeRootDir }) =>+ `mkdir -p ${q(assetDir)} && ` ++ `sh ${q(path.posix.join(runtimeRootDir, "home-merge.sh"))} ${q(assetDir)} ${q(assetTarPath)} && ` ++ `rm -f ${q(assetTarPath)}`,+ },+ }],+ });++ // The orchestrator delegated everything to syncIn: no re-inlined writeFile/run.+ expect(directWrites).toEqual([]);+ expect(directRuns).toEqual([]);++ // The collapsed count: exactly three inbound round-trips — git, workspace, home.+ expect(captured).toHaveLength(3);+ const gitOp = captured.find((op) =>+ op.files.some((mapping) => mapping.targetPath.endsWith("git-workspace-upload.tar")),+ );+ const workspaceOp = captured.find((op) =>+ op.files.some((mapping) => mapping.targetPath.endsWith("workspace-upload.tar")),+ );+ const homeOp = captured.find((op) =>+ op.files.some((mapping) => mapping.targetPath.endsWith("home-upload.tar")),+ );+ expect(gitOp).toBeDefined();+ expect(workspaceOp).toBeDefined();+ expect(homeOp).toBeDefined();++ // Every operation is a single native uploadFiles (all `file` mappings) whose+ // extract/merge rides as an ordered provider-executed post-upload command.+ for (const op of captured) {+ expect(op.files.length).toBeGreaterThanOrEqual(1);+ expect(op.files.every((mapping) => mapping.kind === "file")).toBe(true);+ expect(op.postUploadCommands ?? []).not.toHaveLength(0);+ }+ // Operation ids are distinct, so "3 operations" is 3 real round-trips.+ expect(new Set(captured.map((op) => op.operationId)).size).toBe(3);++ // The credential asset actually materialized through the native seam.+ await expect(readFile(path.join(prepared.assetDirs.home, "auth.json"), "utf8"))+ .resolves.toBe("{\"OPENAI_API_KEY\":\"sk-test\"}\n");+ });+it("keeps the sandbox runtime core free of Codex-specific string literals", async () => {const coreSource = await readFile(new URL("./sandbox-managed-runtime.ts", import.meta.url), "utf8");// The seam must be generic: no adapter (Codex) knowledge may live in the core.