Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

PRD-8088 Fix ACP binary installs across filesystems (#3619)

mike-inkeepa95112b86e63
diff --git a/.changeset/acp-binary-cross-device.md b/.changeset/acp-binary-cross-device.md
new file mode 100644
index 000000000..4fc163a45
--- /dev/null
+++ b/.changeset/acp-binary-cross-device.md
@@ -0,0 +1,5 @@
+---
+"@inkeep/open-knowledge": patch
+---
+
+ACP binary agents now install correctly when the temporary directory and OpenKnowledge home are on different filesystems. Binary archives are staged beside their destination for an atomic same-filesystem rename, and concurrent launches adopt an already completed install. Installs also retry the final rename when a Windows antivirus or indexer briefly holds the extracted files open, remember a broken manifest for a day instead of re-downloading the archive on every launch attempt, and sweep lock files left behind by crashed installers.
diff --git a/packages/server/src/acp/archive.ts b/packages/server/src/acp/archive.ts
index c6b597ef1..a5cfb8342 100644
--- a/packages/server/src/acp/archive.ts
+++ b/packages/server/src/acp/archive.ts
@@ -7,7 +7,8 @@
* Both download an archive over HTTPS, verify a publisher-supplied SHA-256,
* extract with the platform's own tar/unzip, and rename the result into a
* per-version cache dir. Kept here so there is one audited copy of the
- * path-escape guard and the streaming-hash download.
+ * path-escape guard and the streaming-hash download. The staging/commit
+ * concurrency machinery both paths share lives in `staged-install.ts`.
*/
import { type ChildProcess, spawn } from 'node:child_process';
diff --git a/packages/server/src/acp/launch.test.ts b/packages/server/src/acp/launch.test.ts
index d16eaa002..9e81aa02c 100644
--- a/packages/server/src/acp/launch.test.ts
+++ b/packages/server/src/acp/launch.test.ts
@@ -8,15 +8,28 @@
* SIGKILL to npx reparents its bin to PID 1, still running).
*/
-import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { execFileSync } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import {
+ existsSync,
+ mkdirSync,
+ mkdtempSync,
+ readdirSync,
+ readFileSync,
+ rmSync,
+ utimesSync,
+ writeFileSync,
+} from 'node:fs';
import { tmpdir } from 'node:os';
import { delimiter, join } from 'node:path';
import { OK_HOSTED_AGENT_ENV } from '@inkeep/open-knowledge-core';
import { afterEach, describe, expect, test } from 'vitest';
+import { getLogger, type PinoLogger } from '../logger.ts';
import {
AgentLaunchError,
brokenInterpreterHint,
declinedRepairHint,
+ ensureBinaryInstalled,
isPathQualified,
mergedEnv,
overlaySetsPath,
@@ -32,6 +45,7 @@ import {
withHostedAgentMarker,
withLoginShellPath,
} from './launch.ts';
+import type { RegistryBinaryTarget } from './registry.ts';
describe('mergedEnv PATH augmentation', () => {
test('an explicit overlay PATH is used verbatim — augmentation repairs only the inherited base', () => {
@@ -186,6 +200,7 @@ async function waitFor(pred: () => boolean, ms: number, what: string): Promise<v
let dirs: string[] = [];
let strayPids: number[] = [];
+const log = getLogger('acp-launch-test');
function tmp(): string {
const d = mkdtempSync(join(tmpdir(), 'acp-launch-test-'));
dirs.push(d);
@@ -204,6 +219,463 @@ afterEach(() => {
dirs = [];
});
+function buildBinaryTarball(
+ dir: string,
+ label = 'agent',
+): { bytes: Buffer; sha: string; target: RegistryBinaryTarget } {
+ const treeRoot = join(dir, `tree-${label}`);
+ const command = join(treeRoot, 'package', 'bin', 'agent');
+ mkdirSync(join(command, '..'), { recursive: true });
+ writeFileSync(command, `#!/bin/sh\necho ${label}\n`, { mode: 0o755 });
+ const tarPath = join(dir, `${label}.tar.gz`);
+ execFileSync('tar', ['-czf', tarPath, '-C', treeRoot, 'package']);
+ const bytes = readFileSync(tarPath);
+ return {
+ bytes,
+ sha: createHash('sha256').update(bytes).digest('hex'),
+ target: {
+ archive: `https://example.com/${label}.tar.gz`,
+ cmd: './package/bin/agent',
+ },
+ };
+}
+
+function binaryFetch(bytes: Buffer): typeof fetch {
+ return (async () =>
+ new Response(new Uint8Array(bytes), {
+ status: 200,
+ headers: { 'content-length': String(bytes.length) },
+ })) as unknown as typeof fetch;
+}
+
+describe('ensureBinaryInstalled', () => {
+ test('stages beside the destination before downloading', async () => {
+ const source = tmp();
+ const cacheDir = tmp();
+ const artifact = buildBinaryTarball(source);
+ const agentDir = join(cacheDir, 'test-agent');
+ let observedStagingDir: string | undefined;
+ const baseFetch = binaryFetch(artifact.bytes);
+ const inspectingFetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
+ observedStagingDir = readdirSync(agentDir).find((name) => name.startsWith('.install-'));
+ return baseFetch(input, init);
+ }) as typeof fetch;
+
+ const installed = await ensureBinaryInstalled(
+ 'test-agent',
+ '1.0.0',
+ { ...artifact.target, sha256: artifact.sha },
+ cacheDir,
+ log,
+ { fetchImpl: inspectingFetch },
+ );
+
+ expect(installed).toBe(join(agentDir, '1.0.0'));
+ expect(observedStagingDir).toBeDefined();
+ expect(readdirSync(agentDir)).not.toContain(observedStagingDir);
+ expect(existsSync(join(installed, 'package', 'bin', 'agent'))).toBe(true);
+ });
+
+ test('removes crash-orphaned staging directories on the next install', async () => {
+ const source = tmp();
+ const cacheDir = tmp();
+ const artifact = buildBinaryTarball(source);
+ const staleDir = join(cacheDir, 'test-agent', '.install-1.0.0-orphaned');
+ mkdirSync(staleDir, { recursive: true });
+ writeFileSync(join(staleDir, 'partial-archive'), 'partial');
+ const staleTime = new Date(Date.now() - 25 * 60 * 60 * 1_000);
+ utimesSync(staleDir, staleTime, staleTime);
+
+ await ensureBinaryInstalled(
+ 'test-agent',
+ '1.0.0',
+ { ...artifact.target, sha256: artifact.sha },
+ cacheDir,
+ log,
+ { fetchImpl: binaryFetch(artifact.bytes) },
+ );
+
+ expect(existsSync(staleDir)).toBe(false);
+ });
+
+ test('concurrent installers adopt the first completed binary', async () => {
+ const firstSource = tmp();
+ const secondSource = tmp();
+ const cacheDir = tmp();
+ const first = buildBinaryTarball(firstSource, 'first');
+ const second = buildBinaryTarball(secondSource, 'second');
+ let arrivals = 0;
+ let releaseCommit!: () => void;
+ const commitGate = new Promise<void>((resolve) => {
+ releaseCommit = resolve;
+ });
+ const beforeCommit = async () => {
+ arrivals += 1;
+ if (arrivals === 2) releaseCommit();
+ await commitGate;
+ };
+ const installMessages: string[] = [];
+ const concurrentLog = {
+ info: (_context: unknown, message?: string) => {
+ if (message !== undefined) installMessages.push(message);
+ },
+ warn: log.warn.bind(log),
+ } as PinoLogger;
+ const install = (artifact: ReturnType<typeof buildBinaryTarball>) =>
+ ensureBinaryInstalled(
+ 'test-agent',
+ '1.0.0',
+ { ...artifact.target, sha256: artifact.sha },
+ cacheDir,
+ concurrentLog,
+ { fetchImpl: binaryFetch(artifact.bytes), beforeCommit },
+ );
+
+ const installed = await Promise.all([install(first), install(second)]);
+
+ expect(arrivals).toBe(2);
+ expect(installed[0]).toBe(installed[1]);
+ expect(
+ installMessages.filter((message) => message === '[acp-launch] agent binary installed'),
+ ).toHaveLength(1);
+ expect(readFileSync(join(installed[0], 'package', 'bin', 'agent'), 'utf8')).toMatch(
+ /echo (first|second)/,
+ );
+ expect(
+ readdirSync(join(cacheDir, 'test-agent')).some((name) => name.startsWith('.install-')),
+ ).toBe(false);
+ });
+
+ test('replaces an incomplete existing directory instead of accepting it', async () => {
+ const source = tmp();
+ const cacheDir = tmp();
+ const artifact = buildBinaryTarball(source);
+ const versionDir = join(cacheDir, 'test-agent', '1.0.0');
+ mkdirSync(versionDir, { recursive: true });
+ writeFileSync(join(versionDir, 'partial'), 'incomplete');
+
+ const installed = await ensureBinaryInstalled(
+ 'test-agent',
+ '1.0.0',
+ { ...artifact.target, sha256: artifact.sha },
+ cacheDir,
+ log,
+ { fetchImpl: binaryFetch(artifact.bytes) },
+ );
+
+ expect(installed).toBe(versionDir);
+ expect(existsSync(join(versionDir, 'partial'))).toBe(false);
+ expect(existsSync(join(versionDir, 'package', 'bin', 'agent'))).toBe(true);
+ });
+
+ test('adopts a completed binary when the commit lock times out', async () => {
+ const source = tmp();
+ const cacheDir = tmp();
+ const artifact = buildBinaryTarball(source);
+ const versionDir = join(cacheDir, 'test-agent', '1.0.0');
+ const command = join(versionDir, 'package', 'bin', 'agent');
+
+ const installed = await ensureBinaryInstalled(
+ 'test-agent',
+ '1.0.0',
+ { ...artifact.target, sha256: artifact.sha },
+ cacheDir,
+ log,
+ {
+ fetchImpl: binaryFetch(artifact.bytes),
+ commitLockTimeoutMs: 100,
+ beforeCommit: async () => {
+ mkdirSync(join(command, '..'), { recursive: true });
+ writeFileSync(command, '#!/bin/sh\necho winner\n', { mode: 0o755 });
+ const lockPath = `${versionDir}.install.lock`;
+ writeFileSync(lockPath, 'held');
+ const future = new Date(Date.now() + 60_000);
+ utimesSync(lockPath, future, future);
+ },
+ },
+ );
+
+ expect(installed).toBe(versionDir);
+ expect(readFileSync(command, 'utf8')).toContain('winner');
+ });
+
+ test('rejects a lock timeout when no completed binary is available', async () => {
+ const source = tmp();
+ const cacheDir = tmp();
+ const artifact = buildBinaryTarball(source);
+ const versionDir = join(cacheDir, 'test-agent', '1.0.0');
+
+ const installation = ensureBinaryInstalled(
+ 'test-agent',
+ '1.0.0',
+ { ...artifact.target, sha256: artifact.sha },
+ cacheDir,
+ log,
+ {
+ fetchImpl: binaryFetch(artifact.bytes),
+ commitLockTimeoutMs: 100,
+ beforeCommit: async () => {
+ const lockPath = `${versionDir}.install.lock`;
+ writeFileSync(lockPath, 'held');
+ const future = new Date(Date.now() + 60_000);
+ utimesSync(lockPath, future, future);
+ },
+ },
+ );
+
+ await expect(installation).rejects.toMatchObject({
+ name: 'AgentLaunchError',
+ code: 'install-failed',
+ });
+ await expect(installation).rejects.toThrow(/could not acquire file lock/i);
+ expect(existsSync(versionDir)).toBe(false);
+ expect(
+ readdirSync(join(cacheDir, 'test-agent')).some((name) => name.startsWith('.install-')),
+ ).toBe(false);
+ });
+
+ test('rejects a checksum mismatch without committing or leaving staging behind', async () => {
+ const source = tmp();
+ const cacheDir = tmp();
+ const artifact = buildBinaryTarball(source);
+ const agentDir = join(cacheDir, 'test-agent');
+ const versionDir = join(agentDir, '1.0.0');
+
+ let caught: unknown;
+ try {
+ await ensureBinaryInstalled(
+ 'test-agent',
+ '1.0.0',
+ { ...artifact.target, sha256: '0'.repeat(64) },
+ cacheDir,
+ log,
+ { fetchImpl: binaryFetch(artifact.bytes) },
+ );
+ } catch (err) {
+ caught = err;
+ }
+
+ expect(caught).toMatchObject({
+ name: 'AgentLaunchError',
+ code: 'install-failed',
+ message: expect.stringContaining('checksum mismatch'),
+ cause: expect.any(Error),
+ });
+ expect(existsSync(versionDir)).toBe(false);
+ expect(readdirSync(agentDir).some((name) => name.startsWith('.install-'))).toBe(false);
+ });
+
+ test('reuses a valid cached binary without downloading again', async () => {
+ const source = tmp();
+ const cacheDir = tmp();
+ const artifact = buildBinaryTarball(source);
+ const target = { ...artifact.target, sha256: artifact.sha };
+ const installed = await ensureBinaryInstalled('test-agent', '1.0.0', target, cacheDir, log, {
+ fetchImpl: binaryFetch(artifact.bytes),
+ });
+ const failFetch = (async () => {
+ throw new Error('cached install attempted an unexpected download');
+ }) as typeof fetch;
+
+ const cached = await ensureBinaryInstalled('test-agent', '1.0.0', target, cacheDir, log, {
+ fetchImpl: failFetch,
+ });
+
+ expect(cached).toBe(installed);
+ });
+
+ test('rejects an archive whose command path is a directory', async () => {
+ const source = tmp();
+ const cacheDir = tmp();
+ const artifact = buildBinaryTarball(source);
+
+ await expect(
+ ensureBinaryInstalled(
+ 'test-agent',
+ '1.0.0',
+ { ...artifact.target, cmd: './package/bin', sha256: artifact.sha },
+ cacheDir,
+ log,
+ { fetchImpl: binaryFetch(artifact.bytes) },
+ ),
+ ).rejects.toThrow(/no usable command/i);
+ });
+
+ test('installs with a loud warning when the manifest omits sha256', async () => {
+ const source = tmp();
+ const cacheDir = tmp();
+ const artifact = buildBinaryTarball(source);
+ const warnMessages: string[] = [];
+ const warnLog = {
+ info: log.info.bind(log),
+ warn: (_context: unknown, message?: string) => {
+ if (message !== undefined) warnMessages.push(message);
+ },
+ } as PinoLogger;
+
+ const installed = await ensureBinaryInstalled(
+ 'test-agent',
+ '1.0.0',
+ artifact.target,
+ cacheDir,
+ warnLog,
+ { fetchImpl: binaryFetch(artifact.bytes) },
+ );
+
+ expect(installed).toBe(join(cacheDir, 'test-agent', '1.0.0'));
+ expect(existsSync(join(installed, 'package', 'bin', 'agent'))).toBe(true);
+ expect(warnMessages.some((message) => message.includes('no sha256'))).toBe(true);
+ });
+
+ test('rejects a manifest cmd that escapes the extracted archive', async () => {
+ const source = tmp();
+ const cacheDir = tmp();
+ const artifact = buildBinaryTarball(source);
+
+ await expect(
+ ensureBinaryInstalled(
+ 'test-agent',
+ '1.0.0',
+ { ...artifact.target, cmd: '../outside-agent', sha256: artifact.sha },
+ cacheDir,
+ log,
+ { fetchImpl: binaryFetch(artifact.bytes) },
+ ),
+ ).rejects.toMatchObject({
+ name: 'AgentLaunchError',
+ code: 'install-failed',
+ message: expect.stringContaining('escapes the extracted archive'),
+ });
+ expect(existsSync(join(cacheDir, 'test-agent', '1.0.0'))).toBe(false);
+ });
+
+ // A manifest whose cmd can never resolve inside the verified archive fails
+ // identically on every attempt — without the marker, every launch of a
+ // broken agent re-streamed the full archive just to fail the same way.
+ test('a deterministic manifest failure is remembered and fails fast without re-downloading', async () => {
+ const source = tmp();
+ const cacheDir = tmp();
+ const artifact = buildBinaryTarball(source);
+ const agentDir = join(cacheDir, 'test-agent');
+ const badTarget = { ...artifact.target, cmd: './package/bin/missing', sha256: artifact.sha };
+
+ await expect(
+ ensureBinaryInstalled('test-agent', '1.0.0', badTarget, cacheDir, log, {
+ fetchImpl: binaryFetch(artifact.bytes),
+ }),
+ ).rejects.toThrow(/no usable command/i);
+ expect(existsSync(join(agentDir, '.install-failed-1.0.0'))).toBe(true);
+
+ let fetched = 0;
+ const countingFetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
+ fetched += 1;
+ return binaryFetch(artifact.bytes)(input, init);
+ }) as typeof fetch;
+ const remembered = await ensureBinaryInstalled(
+ 'test-agent',
+ '1.0.0',
+ badTarget,
+ cacheDir,
+ log,
+ { fetchImpl: countingFetch },
+ ).catch((err: unknown) => err);
+ expect(remembered).toMatchObject({
+ name: 'AgentLaunchError',
+ code: 'install-failed',
+ message: expect.stringContaining('remembered from a recent attempt'),
+ });
+ expect(fetched).toBe(0);
+
+ // A corrected manifest mismatches the marker, so the install retries —
+ // and a successful attempt clears the marker.
+ const installed = await ensureBinaryInstalled(
+ 'test-agent',
+ '1.0.0',
+ { ...artifact.target, sha256: artifact.sha },
+ cacheDir,
+ log,
+ { fetchImpl: countingFetch },
+ );
+ expect(fetched).toBe(1);
+ expect(existsSync(join(installed, 'package', 'bin', 'agent'))).toBe(true);
+ expect(existsSync(join(agentDir, '.install-failed-1.0.0'))).toBe(false);
+ });
+
+ test('a failure marker past its day cap does not block a retry', async () => {
+ const source = tmp();
+ const cacheDir = tmp();
+ const artifact = buildBinaryTarball(source);
+ const target = { ...artifact.target, sha256: artifact.sha };
+ const agentDir = join(cacheDir, 'test-agent');
+ const markerPath = join(agentDir, '.install-failed-1.0.0');
+ mkdirSync(agentDir, { recursive: true });
+ writeFileSync(
+ markerPath,
+ JSON.stringify({
+ archive: target.archive,
+ cmd: target.cmd,
+ sha256: target.sha256,
+ reason: 'old failure',
+ }),
+ );
+ const staleTime = new Date(Date.now() - 25 * 60 * 60 * 1_000);
+ utimesSync(markerPath, staleTime, staleTime);
+
+ const installed = await ensureBinaryInstalled('test-agent', '1.0.0', target, cacheDir, log, {
+ fetchImpl: binaryFetch(artifact.bytes),
+ });
+
+ expect(existsSync(join(installed, 'package', 'bin', 'agent'))).toBe(true);
+ expect(existsSync(markerPath)).toBe(false);
+ });
+
+ test('a malformed archive URL fails through the install-failed contract without orphaning staging', async () => {
+ const cacheDir = tmp();
+ const agentDir = join(cacheDir, 'test-agent');
+
+ await expect(
+ ensureBinaryInstalled(
+ 'test-agent',
+ '1.0.0',
+ { archive: 'not a url', cmd: './bin/agent' },
+ cacheDir,
+ log,
+ {
+ fetchImpl: (async () => {
+ throw new Error('should not fetch');
+ }) as typeof fetch,
+ },
+ ),
+ ).rejects.toMatchObject({ name: 'AgentLaunchError', code: 'install-failed' });
+ expect(readdirSync(agentDir).some((name) => name.startsWith('.install-'))).toBe(false);
+ });
+
+ // Once the install completes via any other path, later launches take the
+ // fast path and never re-enter the commit lock — the sweep is the only
+ // thing left that can collect a lockfile a crashed holder left behind.
+ test('the fast path sweeps a stale lockfile orphaned by a crashed commit', async () => {
+ const source = tmp();
+ const cacheDir = tmp();
+ const artifact = buildBinaryTarball(source);
+ const target = { ...artifact.target, sha256: artifact.sha };
+ await ensureBinaryInstalled('test-agent', '1.0.0', target, cacheDir, log, {
+ fetchImpl: binaryFetch(artifact.bytes),
+ });
+ const orphanedLock = join(cacheDir, 'test-agent', '1.0.0.install.lock');
+ writeFileSync(orphanedLock, 'held by a crashed process');
+ const staleTime = new Date(Date.now() - 25 * 60 * 60 * 1_000);
+ utimesSync(orphanedLock, staleTime, staleTime);
+
+ await ensureBinaryInstalled('test-agent', '1.0.0', target, cacheDir, log, {
+ fetchImpl: (async () => {
+ throw new Error('fast path must not download');
+ }) as typeof fetch,
+ });
+
+ expect(existsSync(orphanedLock)).toBe(false);
+ });
+});
+
describe('hosted-agent marker', () => {
test('sets the marker and leaves the rest of the env alone', () => {
const out = withHostedAgentMarker({ FOO: 'bar' });
diff --git a/packages/server/src/acp/launch.ts b/packages/server/src/acp/launch.ts
index 0a97cebee..ad8738d2d 100644
--- a/packages/server/src/acp/launch.ts
+++ b/packages/server/src/acp/launch.ts
@@ -16,16 +16,17 @@
import { type ChildProcess, spawn } from 'node:child_process';
import { constants, statSync } from 'node:fs';
-import { access, chmod, stat } from 'node:fs/promises';
-import { homedir, tmpdir } from 'node:os';
-import { delimiter, isAbsolute, join, resolve } from 'node:path';
+import { access, chmod, readFile, stat } from 'node:fs/promises';
+import { homedir } from 'node:os';
+import { delimiter, dirname, isAbsolute, join, resolve } from 'node:path';
import { augmentAgentSpawnPath, OK_DIR, OK_HOSTED_AGENT_ENV } from '@inkeep/open-knowledge-core';
-import { tracedMkdir, tracedRename, tracedRm } from '../fs-traced.ts';
+import { tracedMkdir, tracedRm, tracedWriteFile } from '../fs-traced.ts';
import type { PinoLogger } from '../logger.ts';
import { downloadToFileWithSha, extractArchive, isWithin, sanitizeSegment } from './archive.ts';
import { mergeLoginShellPath } from './login-shell-path.ts';
import type { ManagedRuntime } from './managed-runtime.ts';
import type { CustomAgentEntry, RegistryAgent, RegistryBinaryTarget } from './registry.ts';
+import { STALE_INSTALL_ARTIFACT_AGE_MS, stagedInstall } from './staged-install.ts';
export interface ResolvedLaunch {
cmd: string;
@@ -47,8 +48,8 @@ export class AgentLaunchError extends Error {
| 'no-distribution'
| 'install-failed'
| 'command-not-found';
- constructor(code: AgentLaunchError['code'], message: string) {
- super(message);
+ constructor(code: AgentLaunchError['code'], message: string, options?: ErrorOptions) {
+ super(message, options);
this.name = 'AgentLaunchError';
this.code = code;
}
@@ -265,76 +266,195 @@ function prependPath(dir: string, existing: string | undefined): string {
return existing !== undefined && existing !== '' ? `${dir}${delimiter}${existing}` : dir;
}
+export interface EnsureBinaryInstallOptions {
+ /** Test seam — defaults to global `fetch`. */
+ fetchImpl?: typeof fetch;
+ /** Test seam for synchronizing concurrent installers immediately before commit. */
+ beforeCommit?: () => Promise<void>;
+ /** Test seam — defaults to the production commit-lock timeout. */
+ commitLockTimeoutMs?: number;
+}
+
+/** Return the installed root only when the manifest command is present and executable. */
+async function findInstalledBinary(
+ versionDir: string,
+ target: RegistryBinaryTarget,
+ log: PinoLogger,
+): Promise<string | null> {
+ try {
+ if (!(await stat(versionDir)).isDirectory()) return null;
+ } catch (err) {
+ if ((err as NodeJS.ErrnoException).code !== 'ENOENT') {
+ log.warn({ err, versionDir }, '[acp-launch] could not inspect cached agent binary');
+ }
+ return null;
+ }
+
+ const cmdPath = resolve(versionDir, target.cmd.replace(/\\/g, '/'));
+ if (!isWithin(versionDir, cmdPath)) return null;
+ if (!(await isExecutableFile(cmdPath))) {
+ return null;
+ }
+ return versionDir;
+}
+
+/**
+ * A failure whose verdict depends only on the checksum-verified archive bytes
+ * and the manifest `cmd` — re-downloading cannot change it. Remembered in a
+ * failure marker so later launches fail fast from cache instead of
+ * re-streaming the archive on every attempt.
+ */
+class DeterministicInstallError extends Error {}
+
+function installFailureMarkerPath(agentDir: string, version: string): string {
+ return join(agentDir, `.install-failed-${sanitizeSegment(version)}`);
+}
+
+/**
+ * The remembered failure reason when a fresh marker matches the current
+ * manifest, else null. A manifest change (new cmd, sha, or archive URL)
+ * mismatches and clears the way for a retry; so does marker age — the day
+ * cap bounds the wedge for unverified manifests whose archive could be
+ * republished at the same URL.
+ */
+async function readInstallFailureMarker(
+ markerPath: string,
+ target: RegistryBinaryTarget,
+ log: PinoLogger,
+): Promise<string | null> {
+ let mtimeMs: number;
+ let raw: string;
+ try {
+ mtimeMs = (await stat(markerPath)).mtimeMs;
+ raw = await readFile(markerPath, 'utf8');
+ } catch (err) {
+ if ((err as NodeJS.ErrnoException).code !== 'ENOENT') {
+ log.warn({ err, markerPath }, '[acp-launch] could not read the install failure marker');
+ }
+ return null;
+ }
+ if (Date.now() - mtimeMs > STALE_INSTALL_ARTIFACT_AGE_MS) return null;
+ try {
+ const parsed = JSON.parse(raw) as Record<string, unknown>;
+ if (
+ parsed.archive !== target.archive ||
+ parsed.cmd !== target.cmd ||
+ parsed.sha256 !== (target.sha256 ?? null)
+ ) {
+ return null;
+ }
+ return typeof parsed.reason === 'string' ? parsed.reason : null;
+ } catch {
+ log.warn(
+ { markerPath },
+ '[acp-launch] install failure marker is malformed — treating it as absent',
+ );
+ return null;
+ }
+}
+
+async function writeInstallFailureMarker(
+ markerPath: string,
+ target: RegistryBinaryTarget,
+ reason: string,
+ log: PinoLogger,
+): Promise<void> {
+ const payload = JSON.stringify({
+ archive: target.archive,
+ cmd: target.cmd,
+ sha256: target.sha256 ?? null,
+ reason,
+ });
+ await tracedWriteFile(markerPath, payload).catch((err) => {
+ log.warn({ err, markerPath }, '[acp-launch] could not record the failed binary install');
+ });
+}
+
/**
* Download + extract a binary distribution once per (id, version); later
- * launches reuse the extracted tree. Extraction lands in a temp dir first
- * and is renamed into place so a crash mid-extract never leaves a
- * half-populated version dir that would satisfy the fast-path check.
+ * launches reuse the extracted tree. The commit machinery (beside-destination
+ * staging, single-filesystem atomic rename, commit lock, stale sweeps) is the
+ * shared `stagedInstall`.
*/
-async function ensureBinaryInstalled(
+export async function ensureBinaryInstalled(
id: string,
version: string,
target: RegistryBinaryTarget,
cacheDir: string,
log: PinoLogger,
+ opts: EnsureBinaryInstallOptions = {},
): Promise<string> {
const versionDir = join(cacheDir, sanitizeSegment(id), sanitizeSegment(version));
+ const markerPath = installFailureMarkerPath(dirname(versionDir), version);
try {
- const st = await stat(versionDir);
- if (st.isDirectory()) return versionDir;
- } catch {
- // Not installed yet.
- }
-
- log.info({ id, version, archive: target.archive }, '[acp-launch] downloading agent binary');
- const stagingDir = join(tmpdir(), `ok-acp-install-${process.pid}-${Date.now()}`);
- await tracedMkdir(stagingDir, { recursive: true });
- const isZip = /\.zip$/i.test(new URL(target.archive).pathname);
- const archivePath = join(stagingDir, isZip ? 'archive.zip' : 'archive.tar.gz');
- try {
- const sha = await downloadToFileWithSha(target.archive, archivePath, {
- signal: AbortSignal.timeout(120_000),
- });
- // Binary distributions are opaque executables — verify the publisher's
- // checksum whenever the manifest carries one, mirroring the managed-
- // runtime download path. A manifest without one installs with a loud
- // warning rather than silently skipping verification.
- if (target.sha256 !== undefined) {
- if (sha !== target.sha256.toLowerCase()) {
- throw new Error(`archive checksum mismatch: expected ${target.sha256}, got ${sha}`);
- }
- } else {
- log.warn(
- { id, version, archive: target.archive },
- '[acp-launch] manifest carries no sha256 for the binary archive — installing unverified',
- );
- }
+ return await stagedInstall<string>({
+ versionDir,
+ stagingLabel: sanitizeSegment(version),
+ findInstalled: () => findInstalledBinary(versionDir, target, log),
+ prepare: async (stagingDir) => {
+ const remembered = await readInstallFailureMarker(markerPath, target, log);
+ if (remembered !== null) {
+ throw new Error(
+ `${remembered} (remembered from a recent attempt — retried when the manifest changes or after a day)`,
+ );
+ }
+ await tracedRm(markerPath, { force: true });
+ log.info({ id, version, archive: target.archive }, '[acp-launch] downloading agent binary');
+ const isZip = /\.zip$/i.test(new URL(target.archive).pathname);
+ const archivePath = join(stagingDir, isZip ? 'archive.zip' : 'archive.tar.gz');
+ const sha = await downloadToFileWithSha(target.archive, archivePath, {
+ signal: AbortSignal.timeout(120_000),
+ fetchImpl: opts.fetchImpl,
+ });
+ // Binary distributions are opaque executables — verify the publisher's
+ // checksum whenever the manifest carries one, mirroring the managed-
+ // runtime download path. A manifest without one installs with a loud
+ // warning rather than silently skipping verification.
+ if (target.sha256 !== undefined) {
+ if (sha !== target.sha256.toLowerCase()) {
+ throw new Error(`archive checksum mismatch: expected ${target.sha256}, got ${sha}`);
+ }
+ } else {
+ log.warn(
+ { id, version, archive: target.archive },
+ '[acp-launch] manifest carries no sha256 for the binary archive — installing unverified',
+ );
+ }
- const extractDir = join(stagingDir, 'extracted');
- await tracedMkdir(extractDir, { recursive: true });
- await extractArchive(archivePath, extractDir, isZip);
- const cmdPath = resolve(extractDir, target.cmd.replace(/\\/g, '/'));
- if (!isWithin(extractDir, cmdPath)) {
- throw new Error('manifest cmd escapes the extracted archive');
- }
- await chmod(cmdPath, 0o755).catch(() => {
- // Windows / already-executable — non-fatal.
+ const extractDir = join(stagingDir, 'extracted');
+ await tracedMkdir(extractDir, { recursive: true });
+ await extractArchive(archivePath, extractDir, isZip);
+ const cmdPath = resolve(extractDir, target.cmd.replace(/\\/g, '/'));
+ if (!isWithin(extractDir, cmdPath)) {
+ throw new DeterministicInstallError('manifest cmd escapes the extracted archive');
+ }
+ await chmod(cmdPath, 0o755).catch(() => {
+ // Windows / already-executable — non-fatal.
+ });
+ if (!(await isExecutableFile(cmdPath))) {
+ throw new DeterministicInstallError(
+ `extracted agent archive has no usable command at ${target.cmd}`,
+ );
+ }
+ return extractDir;
+ },
+ log,
+ logPrefix: '[acp-launch]',
+ logContext: { id, version },
+ installedMessage: 'agent binary installed',
+ missingAfterCommitMessage: 'installed agent binary not found after extract',
+ beforeCommit: opts.beforeCommit,
+ commitLockTimeoutMs: opts.commitLockTimeoutMs,
});
-
- await tracedMkdir(join(cacheDir, sanitizeSegment(id)), { recursive: true });
- await tracedRm(versionDir, { recursive: true, force: true });
- await tracedRename(extractDir, versionDir);
- log.info({ id, version, versionDir }, '[acp-launch] agent binary installed');
- return versionDir;
} catch (err) {
+ if (err instanceof DeterministicInstallError) {
+ await writeInstallFailureMarker(markerPath, target, err.message, log);
+ }
throw new AgentLaunchError(
'install-failed',
`installing ${id}@${version} failed: ${err instanceof Error ? err.message : String(err)}`,
+ { cause: err },
);
- } finally {
- await tracedRm(stagingDir, { recursive: true, force: true }).catch(() => {
- // Best-effort temp cleanup.
- });
}
}
diff --git a/packages/server/src/acp/managed-runtime.test.ts b/packages/server/src/acp/managed-runtime.test.ts
index 2329eb103..45b1a0bf2 100644
--- a/packages/server/src/acp/managed-runtime.test.ts
+++ b/packages/server/src/acp/managed-runtime.test.ts
@@ -303,6 +303,37 @@ describe('ensureManagedRuntime', () => {
expect(again.kind).toBe('node');
});
+ test('rejects an archive that contains no usable launcher', async () => {
+ const stage = tmp();
+ const root = tmp();
+ const { bytes, sha } = buildTarball(stage, 'node-vTEST', ['bin/notlauncher']);
+
+ const error = await ensureManagedRuntime('node', log, {
+ root,
+ fetchImpl: makeFetch(bytes, sha),
+ }).catch((err: unknown) => err);
+
+ expect(error).toBeInstanceOf(RuntimeInstallError);
+ expect((error as Error).message).toContain('no usable launcher');
+ expect(await findManagedRuntime('node', root)).toBeNull();
+ });
+
+ test('preserves the underlying failure as the error cause', async () => {
+ const root = tmp();
+ const failure = new Error('network unreachable');
+ const failingFetch = (async () => {
+ throw failure;
+ }) as unknown as typeof fetch;
+
+ const error = await ensureManagedRuntime('node', log, {
+ root,
+ fetchImpl: failingFetch,
+ }).catch((err: unknown) => err);
+
+ expect(error).toBeInstanceOf(RuntimeInstallError);
+ expect((error as Error).cause).toBe(failure);
+ });
+
test('rejects a checksum mismatch and installs nothing', async () => {
const stage = tmp();
const root = tmp();
diff --git a/packages/server/src/acp/managed-runtime.ts b/packages/server/src/acp/managed-runtime.ts
index bda2a49cf..bb82cee7d 100644
--- a/packages/server/src/acp/managed-runtime.ts
+++ b/packages/server/src/acp/managed-runtime.ts
@@ -32,7 +32,6 @@ import { access, constants, readdir, stat } from 'node:fs/promises';
import { arch, homedir, platform } from 'node:os';
import { dirname, join } from 'node:path';
import { OK_DIR } from '@inkeep/open-knowledge-core';
-import { FileLockTimeoutError, withFileLock } from '@inkeep/open-knowledge-core/server';
import { tracedMkdir, tracedRename, tracedRm } from '../fs-traced.ts';
import type { PinoLogger } from '../logger.ts';
import {
@@ -44,6 +43,7 @@ import {
sanitizeSegment,
shaForFile,
} from './archive.ts';
+import { cleanupStaleInstallArtifacts, stagedInstall } from './staged-install.ts';
export type ManagedRuntimeKind = 'node' | 'uv';
@@ -61,9 +61,6 @@ const UV_RELEASE_BASE = 'https://github.com/astral-sh/uv/releases/download';
/** Approximate compressed download sizes (MB), for the consent prompt only. */
const APPROX_SIZE_MB: Record<ManagedRuntimeKind, number> = { node: 45, uv: 20 };
-const STALE_STAGING_AGE_MS = 24 * 60 * 60 * 1_000;
-const INSTALL_COMMIT_LOCK_TIMEOUT_MS = 60_000;
-
interface ManagedNode {
kind: 'node';
/** Prepended to PATH so `npx`/`npm` resolve their sibling `node`. */
@@ -285,43 +282,18 @@ export interface EnsureRuntimeOptions {
}
export class RuntimeInstallError extends Error {
- constructor(message: string) {
- super(message);
+ constructor(message: string, options?: ErrorOptions) {
+ super(message, options);
this.name = 'RuntimeInstallError';
}
}
-async function cleanupStaleStagingDirs(kindDir: string, log: PinoLogger): Promise<void> {
- let entries: import('node:fs').Dirent[];
- try {
- entries = await readdir(kindDir, { withFileTypes: true });
- } catch (err) {
- if ((err as NodeJS.ErrnoException).code !== 'ENOENT') {
- log.warn({ err }, '[managed-runtime] could not read staging dir for cleanup');
- }
- return;
- }
-
- const cutoff = Date.now() - STALE_STAGING_AGE_MS;
- for (const entry of entries) {
- if (!entry.isDirectory() || !entry.name.startsWith('.install-')) continue;
- const path = join(kindDir, entry.name);
- try {
- if ((await stat(path)).mtimeMs >= cutoff) continue;
- await tracedRm(path, { recursive: true, force: true });
- } catch (err) {
- if ((err as NodeJS.ErrnoException).code === 'ENOENT') continue;
- log.warn({ err }, '[managed-runtime] stale staging cleanup failed');
- }
- }
-}
-
export async function cleanupManagedRuntimeStaging(
kind: ManagedRuntimeKind,
log: PinoLogger,
root: string = defaultRuntimeRoot(),
): Promise<void> {
- await cleanupStaleStagingDirs(join(root, kind), log);
+ await cleanupStaleInstallArtifacts(join(root, kind), log, '[managed-runtime]');
}
/**
@@ -364,10 +336,9 @@ export async function quarantineManagedRuntime(
/**
* Download + verify + install a managed runtime once per pinned version;
* later launches reuse the extracted tree. Verified against the publisher's
- * SHA-256 before it is trusted. Extraction lands beside the destination and
- * is renamed into place so the atomic commit stays on one filesystem and a
- * crash mid-install never leaves a half-tree that the fast-path check would
- * accept.
+ * SHA-256 before it is trusted. The commit machinery (beside-destination
+ * staging, single-filesystem atomic rename, commit lock, stale sweeps) is the
+ * shared `stagedInstall`.
*
* The CALLER is responsible for consent — this only runs once told to.
*/
@@ -378,111 +349,61 @@ export async function ensureManagedRuntime(
): Promise<ManagedRuntime> {
const root = opts.root ?? defaultRuntimeRoot();
const versionDir = join(root, kind, sanitizeSegment(versionOf(kind)));
- const kindDir = dirname(versionDir);
- const existing = await findManagedRuntime(kind, root);
- if (existing !== null) {
- await cleanupStaleStagingDirs(kindDir, log);
- return existing;
- }
-
- const spec = artifactFor(kind);
- if (spec === null) {
- throw new RuntimeInstallError(`no ${kind} build for ${platform()}-${arch()}`);
- }
-
- await tracedMkdir(kindDir, { recursive: true });
- await cleanupStaleStagingDirs(kindDir, log);
- const stagingDir = join(
- kindDir,
- `.install-${sanitizeSegment(versionOf(kind))}-${process.pid}-${randomUUID()}`,
- );
- await tracedMkdir(stagingDir, { recursive: true });
- const archivePath = join(stagingDir, spec.archiveName);
- log.info(
- { kind, version: versionOf(kind), url: spec.archiveUrl },
- '[managed-runtime] downloading runtime',
- );
try {
- const [actualSha, checksums] = await Promise.all([
- downloadToFileWithSha(spec.archiveUrl, archivePath, {
- signal: opts.signal,
- onProgress: opts.onProgress,
- fetchImpl: opts.fetchImpl,
- }),
- fetchText(spec.checksumUrl, { signal: opts.signal, fetchImpl: opts.fetchImpl }),
- ]);
- const expectedSha = shaForFile(checksums, spec.archiveName);
- if (expectedSha === null) {
- throw new RuntimeInstallError(`no published checksum for ${spec.archiveName}`);
- }
- if (actualSha !== expectedSha) {
- throw new RuntimeInstallError(
- `checksum mismatch for ${spec.archiveName}: got ${actualSha}, expected ${expectedSha}`,
- );
- }
-
- const extractDir = join(stagingDir, 'extracted');
- await tracedMkdir(extractDir, { recursive: true });
- await extractArchive(archivePath, extractDir, spec.isZip);
- const launcher = await findLauncher(extractDir, launcherNames(kind), 3);
- if (launcher === null || !isWithin(extractDir, launcher)) {
- throw new RuntimeInstallError(`extracted ${kind} archive has no usable launcher`);
- }
-
- // A concurrent launch (two npm agents started on a bare machine) may have
- // finished installing the same version while we downloaded — adopt it and
- // drop our copy rather than racing on the rename.
- const raced = await findManagedRuntime(kind, root);
- if (raced !== null) return raced;
- await opts.beforeCommit?.();
-
- try {
- return await withFileLock(
- `${versionDir}.install.lock`,
- async () => {
- // Only the commit is serialized: concurrent launches may download in
- // parallel, but none can remove a complete tree installed by another.
- const winner = await findManagedRuntime(kind, root);
- if (winner !== null) return winner;
-
- await tracedRm(versionDir, { recursive: true, force: true });
- await tracedRename(extractDir, versionDir);
- log.info(
- { kind, version: versionOf(kind), versionDir },
- '[managed-runtime] runtime installed',
- );
- const installed = await findManagedRuntime(kind, root);
- if (installed === null) {
- throw new RuntimeInstallError(`installed ${kind} runtime not found after extract`);
- }
- return installed;
- },
- {
- timeoutMs: opts.commitLockTimeoutMs ?? INSTALL_COMMIT_LOCK_TIMEOUT_MS,
- onWarn: (message, context) => {
- log.warn({ ...context, kind }, `[managed-runtime] ${message}`);
- },
- },
- );
- } catch (err) {
- if (err instanceof FileLockTimeoutError) {
- const winner = await findManagedRuntime(kind, root);
- if (winner !== null) return winner;
- log.warn(
- { kind, lockPath: err.lockPath, timeoutMs: err.timeoutMs },
- '[managed-runtime] commit lock timed out with no winner installed',
+ return await stagedInstall<ManagedRuntime>({
+ versionDir,
+ stagingLabel: sanitizeSegment(versionOf(kind)),
+ findInstalled: () => findManagedRuntime(kind, root),
+ prepare: async (stagingDir) => {
+ const spec = artifactFor(kind);
+ if (spec === null) {
+ throw new RuntimeInstallError(`no ${kind} build for ${platform()}-${arch()}`);
+ }
+ const archivePath = join(stagingDir, spec.archiveName);
+ log.info(
+ { kind, version: versionOf(kind), url: spec.archiveUrl },
+ '[managed-runtime] downloading runtime',
);
- }
- throw err;
- }
+ const [actualSha, checksums] = await Promise.all([
+ downloadToFileWithSha(spec.archiveUrl, archivePath, {
+ signal: opts.signal,
+ onProgress: opts.onProgress,
+ fetchImpl: opts.fetchImpl,
+ }),
+ fetchText(spec.checksumUrl, { signal: opts.signal, fetchImpl: opts.fetchImpl }),
+ ]);
+ const expectedSha = shaForFile(checksums, spec.archiveName);
+ if (expectedSha === null) {
+ throw new RuntimeInstallError(`no published checksum for ${spec.archiveName}`);
+ }
+ if (actualSha !== expectedSha) {
+ throw new RuntimeInstallError(
+ `checksum mismatch for ${spec.archiveName}: got ${actualSha}, expected ${expectedSha}`,
+ );
+ }
+
+ const extractDir = join(stagingDir, 'extracted');
+ await tracedMkdir(extractDir, { recursive: true });
+ await extractArchive(archivePath, extractDir, spec.isZip);
+ const launcher = await findLauncher(extractDir, launcherNames(kind), 3);
+ if (launcher === null || !isWithin(extractDir, launcher)) {
+ throw new RuntimeInstallError(`extracted ${kind} archive has no usable launcher`);
+ }
+ return extractDir;
+ },
+ log,
+ logPrefix: '[managed-runtime]',
+ logContext: { kind, version: versionOf(kind) },
+ installedMessage: 'runtime installed',
+ missingAfterCommitMessage: `installed ${kind} runtime not found after extract`,
+ beforeCommit: opts.beforeCommit,
+ commitLockTimeoutMs: opts.commitLockTimeoutMs,
+ });
} catch (err) {
if (err instanceof RuntimeInstallError) throw err;
throw new RuntimeInstallError(
`installing ${kind} ${versionOf(kind)} failed: ${err instanceof Error ? err.message : String(err)}`,
+ { cause: err },
);
- } finally {
- await tracedRm(stagingDir, { recursive: true, force: true }).catch(() => {
- // Best-effort temp cleanup.
- });
}
}
diff --git a/packages/server/src/acp/staged-install.test.ts b/packages/server/src/acp/staged-install.test.ts
new file mode 100644
index 000000000..31a37e86a
--- /dev/null
+++ b/packages/server/src/acp/staged-install.test.ts
@@ -0,0 +1,114 @@
+/**
+ * Shared staged-install primitives: the bounded rename retry (Windows
+ * share-violation resilience) and the stale-artifact sweep both install
+ * paths rely on. The end-to-end commit flow is covered through its callers
+ * in `launch.test.ts` and `managed-runtime.test.ts`.
+ */
+
+import { existsSync, mkdirSync, mkdtempSync, rmSync, utimesSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, describe, expect, test } from 'vitest';
+import { getLogger } from '../logger.ts';
+import { cleanupStaleInstallArtifacts, renameWithRetries } from './staged-install.ts';
+
+const log = getLogger('staged-install-test');
+
+let dirs: string[] = [];
+function tmp(): string {
+ const d = mkdtempSync(join(tmpdir(), 'staged-install-test-'));
+ dirs.push(d);
+ return d;
+}
+afterEach(() => {
+ for (const d of dirs) rmSync(d, { recursive: true, force: true });
+ dirs = [];
+});
+
+function errnoError(code: string): NodeJS.ErrnoException {
+ return Object.assign(new Error(`${code}: simulated`), { code });
+}
+
+describe('renameWithRetries', () => {
+ test('retries transient share-violation codes until the handle is released', async () => {
+ let calls = 0;
+ await renameWithRetries('from', 'to', {
+ baseDelayMs: 1,
+ renameImpl: async () => {
+ calls += 1;
+ if (calls < 3) throw errnoError('EPERM');
+ },
+ });
+ expect(calls).toBe(3);
+ });
+
+ test('rethrows non-transient codes without retrying', async () => {
+ let calls = 0;
+ await expect(
+ renameWithRetries('from', 'to', {
+ baseDelayMs: 1,
+ renameImpl: async () => {
+ calls += 1;
+ throw errnoError('ENOENT');
+ },
+ }),
+ ).rejects.toThrow('ENOENT');
+ expect(calls).toBe(1);
+ });
+
+ test('gives up once the retry budget is spent', async () => {
+ let calls = 0;
+ await expect(
+ renameWithRetries('from', 'to', {
+ attempts: 3,
+ baseDelayMs: 1,
+ renameImpl: async () => {
+ calls += 1;
+ throw errnoError('EBUSY');
+ },
+ }),
+ ).rejects.toThrow('EBUSY');
+ expect(calls).toBe(3);
+ });
+});
+
+describe('cleanupStaleInstallArtifacts', () => {
+ test('sweeps stale staging dirs, lockfiles, and failure markers; keeps everything else', async () => {
+ const dir = tmp();
+ const staleTime = new Date(Date.now() - 25 * 60 * 60 * 1_000);
+ const age = (path: string) => utimesSync(path, staleTime, staleTime);
+
+ const staleStaging = join(dir, '.install-1.0.0-dead');
+ mkdirSync(staleStaging);
+ age(staleStaging);
+ const staleLock = join(dir, '1.0.0.install.lock');
+ writeFileSync(staleLock, 'held');
+ age(staleLock);
+ const staleMarker = join(dir, '.install-failed-1.0.0');
+ writeFileSync(staleMarker, '{}');
+ age(staleMarker);
+
+ const freshLock = join(dir, '2.0.0.install.lock');
+ writeFileSync(freshLock, 'held');
+ const versionDir = join(dir, '1.0.0');
+ mkdirSync(versionDir);
+ const unrelated = join(dir, 'unrelated.txt');
+ writeFileSync(unrelated, 'keep');
+ age(unrelated);
+
+ await cleanupStaleInstallArtifacts(dir, log, '[test]');
+
+ expect(existsSync(staleStaging)).toBe(false);
+ expect(existsSync(staleLock)).toBe(false);
+ expect(existsSync(staleMarker)).toBe(false);
+ expect(existsSync(freshLock)).toBe(true);
+ expect(existsSync(versionDir)).toBe(true);
+ expect(existsSync(unrelated)).toBe(true);
+ });
+
+ test('a missing parent dir is a no-op', async () => {
+ await expect(
+ cleanupStaleInstallArtifacts(join(tmp(), 'never-created'), log, '[test]'),
+ ).resolves.toBeUndefined();
+ });
+});
diff --git a/packages/server/src/acp/staged-install.ts b/packages/server/src/acp/staged-install.ts
new file mode 100644
index 000000000..92edbeeba
--- /dev/null
+++ b/packages/server/src/acp/staged-install.ts
@@ -0,0 +1,229 @@
+/**
+ * Staged-install commit machinery shared by the two ACP install paths:
+ * proprietary agent binaries (`launch.ts` → `ensureBinaryInstalled`) and the
+ * managed language runtimes (`managed-runtime.ts` → `ensureManagedRuntime`).
+ * One audited copy of the concurrency contract, so a crash-safety or
+ * Windows-robustness fix lands in both paths at once:
+ *
+ * - Extraction lands in a staging dir BESIDE the destination and is renamed
+ * into place, so the atomic commit stays on one filesystem and a crash
+ * mid-install never leaves a half-populated version dir that would satisfy
+ * the caller's fast-path check.
+ * - Only the commit is serialized (file lock): concurrent installers
+ * download in parallel, and every loser adopts the winner's tree instead
+ * of racing on the rename.
+ * - Stale artifacts from crashed installers — `.install-*` staging dirs,
+ * `*.install.lock` commit locks, `.install-failed-*` markers — are swept
+ * on the next install once they age past a day.
+ */
+
+import { randomUUID } from 'node:crypto';
+import { readdir, stat } from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+import { FileLockTimeoutError, withFileLock } from '@inkeep/open-knowledge-core/server';
+import { tracedMkdir, tracedRename, tracedRm } from '../fs-traced.ts';
+import type { PinoLogger } from '../logger.ts';
+
+/** Age past which a crashed installer's leftovers are fair game for the sweep. */
+export const STALE_INSTALL_ARTIFACT_AGE_MS = 24 * 60 * 60 * 1_000;
+const INSTALL_COMMIT_LOCK_TIMEOUT_MS = 60_000;
+
+/**
+ * Windows share-violation codes: an antivirus scanner or search indexer
+ * briefly holding a handle inside a freshly extracted tree fails rename with
+ * one of these even though nothing is wrong with the install. On POSIX the
+ * same codes are genuine permission errors — the bounded retry merely delays
+ * that verdict by a few seconds.
+ */
+const RETRIABLE_RENAME_CODES = new Set(['EPERM', 'EACCES', 'EBUSY']);
+const RENAME_RETRY_ATTEMPTS = 6;
+const RENAME_RETRY_BASE_DELAY_MS = 100;
+
+export interface RenameRetryOptions {
+ attempts?: number;
+ /** Delay before the first retry; doubles on each subsequent one. */
+ baseDelayMs?: number;
+ /** Test seam — defaults to `tracedRename`. */
+ renameImpl?: (from: string, to: string) => Promise<void>;
+}
+
+/**
+ * `rename` with a bounded retry on transient Windows share violations (see
+ * {@link RETRIABLE_RENAME_CODES}) — the failure a freshly extracted `.exe`
+ * under real-time scanning hits at commit time. Everything else (ENOENT,
+ * EXDEV, ENOTEMPTY, …) rethrows immediately.
+ */
+export async function renameWithRetries(
+ from: string,
+ to: string,
+ opts: RenameRetryOptions = {},
+): Promise<void> {
+ const attempts = opts.attempts ?? RENAME_RETRY_ATTEMPTS;
+ const rename = opts.renameImpl ?? tracedRename;
+ let delayMs = opts.baseDelayMs ?? RENAME_RETRY_BASE_DELAY_MS;
+ for (let attempt = 1; ; attempt += 1) {
+ try {
+ return await rename(from, to);
+ } catch (err) {
+ const code = (err as NodeJS.ErrnoException).code;
+ if (attempt >= attempts || code === undefined || !RETRIABLE_RENAME_CODES.has(code)) {
+ throw err;
+ }
+ await new Promise((resolvePromise) => setTimeout(resolvePromise, delayMs));
+ delayMs *= 2;
+ }
+ }
+}
+
+/**
+ * Remove stale install leftovers in `parentDir` (the dir holding the
+ * per-version trees): crashed installers orphan `.install-*` staging dirs and
+ * `.install-failed-*` markers, and a process that dies while holding
+ * `<version>.install.lock` orphans the lockfile forever — once any other path
+ * completes the install, every later launch takes the fast path and nothing
+ * re-enters the lock to clean it up. The age cutoff keeps live installs safe:
+ * an active lock or staging dir is minutes old at most.
+ */
+export async function cleanupStaleInstallArtifacts(
+ parentDir: string,
+ log: PinoLogger,
+ logPrefix: string,
+): Promise<void> {
+ let entries: import('node:fs').Dirent[];
+ try {
+ entries = await readdir(parentDir, { withFileTypes: true });
+ } catch (err) {
+ if ((err as NodeJS.ErrnoException).code !== 'ENOENT') {
+ log.warn({ err, parentDir }, `${logPrefix} could not read install dir for cleanup`);
+ }
+ return;
+ }
+
+ const cutoff = Date.now() - STALE_INSTALL_ARTIFACT_AGE_MS;
+ for (const entry of entries) {
+ const staleStagingDir = entry.isDirectory() && entry.name.startsWith('.install-');
+ const straggler =
+ entry.isFile() &&
+ (entry.name.endsWith('.install.lock') || entry.name.startsWith('.install-'));
+ if (!staleStagingDir && !straggler) continue;
+ const path = join(parentDir, entry.name);
+ try {
+ if ((await stat(path)).mtimeMs >= cutoff) continue;
+ await tracedRm(path, { recursive: true, force: true });
+ } catch (err) {
+ if ((err as NodeJS.ErrnoException).code === 'ENOENT') continue;
+ log.warn({ err, path }, `${logPrefix} stale install artifact cleanup failed`);
+ }
+ }
+}
+
+export interface StagedInstallOptions<T> {
+ /** Final destination; committed atomically via rename from staging. */
+ versionDir: string;
+ /** Sanitized version label used in the staging dir name. */
+ stagingLabel: string;
+ /** The installed artifact when a complete tree exists at `versionDir`, else null. */
+ findInstalled: () => Promise<T | null>;
+ /**
+ * Download + verify + extract under `stagingDir` and return the tree to
+ * commit. Runs before the commit lock — concurrent installers prepare in
+ * parallel. Whatever it throws propagates to the caller unwrapped.
+ */
+ prepare: (stagingDir: string) => Promise<string>;
+ log: PinoLogger;
+ /** `[acp-launch]` / `[managed-runtime]` — tags every log line. */
+ logPrefix: string;
+ /** Stable identifiers (`{id, version}` / `{kind, version}`) carried on every log line. */
+ logContext: Record<string, unknown>;
+ /** Log message emitted on a successful fresh commit. */
+ installedMessage: string;
+ /** Error message for a committed tree that still fails `findInstalled`. */
+ missingAfterCommitMessage: string;
+ /** Test seam for synchronizing concurrent installers immediately before commit. */
+ beforeCommit?: () => Promise<void>;
+ /** Test seam — defaults to the production commit-lock timeout. */
+ commitLockTimeoutMs?: number;
+}
+
+/**
+ * Install once per version dir; later calls reuse the committed tree via the
+ * caller's `findInstalled` fast path. Errors propagate unwrapped so each
+ * caller keeps its own error contract.
+ */
+export async function stagedInstall<T>(opts: StagedInstallOptions<T>): Promise<T> {
+ const parentDir = dirname(opts.versionDir);
+ const existing = await opts.findInstalled();
+ if (existing !== null) {
+ await cleanupStaleInstallArtifacts(parentDir, opts.log, opts.logPrefix);
+ return existing;
+ }
+
+ await tracedMkdir(parentDir, { recursive: true });
+ await cleanupStaleInstallArtifacts(parentDir, opts.log, opts.logPrefix);
+ const stagingDir = join(
+ parentDir,
+ `.install-${opts.stagingLabel}-${process.pid}-${randomUUID()}`,
+ );
+ await tracedMkdir(stagingDir, { recursive: true });
+ try {
+ const extractDir = await opts.prepare(stagingDir);
+
+ // A concurrent installer may have finished the same version while we
+ // downloaded — adopt it and drop our copy rather than racing on the rename.
+ const raced = await opts.findInstalled();
+ if (raced !== null) return raced;
+ await opts.beforeCommit?.();
+
+ try {
+ return await withFileLock(
+ `${opts.versionDir}.install.lock`,
+ async () => {
+ // Only the commit is serialized: concurrent installers download in
+ // parallel, but none can remove a complete tree installed by another.
+ const winner = await opts.findInstalled();
+ if (winner !== null) return winner;
+
+ await tracedRm(opts.versionDir, {
+ recursive: true,
+ force: true,
+ maxRetries: 5,
+ retryDelay: 100,
+ });
+ await renameWithRetries(extractDir, opts.versionDir);
+ opts.log.info(
+ { ...opts.logContext, versionDir: opts.versionDir },
+ `${opts.logPrefix} ${opts.installedMessage}`,
+ );
+ const installed = await opts.findInstalled();
+ if (installed === null) {
+ throw new Error(opts.missingAfterCommitMessage);
+ }
+ return installed;
+ },
+ {
+ timeoutMs: opts.commitLockTimeoutMs ?? INSTALL_COMMIT_LOCK_TIMEOUT_MS,
+ onWarn: (message, context) => {
+ opts.log.warn({ ...context, ...opts.logContext }, `${opts.logPrefix} ${message}`);
+ },
+ },
+ );
+ } catch (err) {
+ if (err instanceof FileLockTimeoutError) {
+ const winner = await opts.findInstalled();
+ if (winner !== null) return winner;
+ opts.log.warn(
+ { ...opts.logContext, lockPath: err.lockPath, timeoutMs: err.timeoutMs },
+ `${opts.logPrefix} install commit lock timed out with no winner installed`,
+ );
+ }
+ throw err;
+ }
+ } finally {
+ await tracedRm(stagingDir, { recursive: true, force: true }).catch((err) => {
+ opts.log.warn(
+ { err, ...opts.logContext, stagingDir },
+ `${opts.logPrefix} install staging cleanup failed`,
+ );
+ });
+ }
+}