Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
Let an npx agent install outlive the server that started it (PRD-9262) (#6155)
Andrew Mikofalvyfb237a71fdf7
diff --git a/.changeset/prd-9262-install-survives-server-exit.md b/.changeset/prd-9262-install-survives-server-exit.mdnew file mode 100644index 000000000..9480f1505--- /dev/null+++ b/.changeset/prd-9262-install-survives-server-exit.md@@ -0,0 +1,5 @@+---+"@inkeep/open-knowledge": patch+---++On macOS and Linux, stopping, restarting or updating OpenKnowledge while it installs a catalog-managed agent no longer leaves a broken npm cache entry. Custom agent commands are not covered, and a supervisor that stops everything OpenKnowledge started, such as systemd or Docker, still stops the install.diff --git a/packages/server/src/acp/held-install-registry.test-helper.ts b/packages/server/src/acp/held-install-registry.test-helper.tsindex 036bcfb32..d85f4a7a6 100644--- a/packages/server/src/acp/held-install-registry.test-helper.ts+++ b/packages/server/src/acp/held-install-registry.test-helper.ts@@ -137,7 +137,10 @@ function packument(export async function withHeldInstallRegistry(run: (registry: HeldInstallRegistry) => Promise<void>,- { dependencyPublished = PUBLISHED }: { dependencyPublished?: string } = {},+ {+ dependencyPublished = PUBLISHED,+ dependencyDeprecated,+ }: { dependencyPublished?: string; dependencyDeprecated?: string } = {},): Promise<void> {await withAcquisitionHome(async (home) => {const sources = join(home, 'held-install-sources');@@ -229,7 +232,15 @@ export async function withHeldInstallRegistry();} else if (path === `/${DEPENDENCY_NAME}`) {send(- Buffer.from(packument(host, DEPENDENCY_NAME, dependencyTarball, {}, dependencyPublished)),+ Buffer.from(+ packument(+ host,+ DEPENDENCY_NAME,+ dependencyTarball,+ dependencyDeprecated === undefined ? {} : { deprecated: dependencyDeprecated },+ dependencyPublished,+ ),+ ),'application/json',);} else if (path === `/${AGENT_NAME}/-/${AGENT_NAME}-${VERSION}.tgz`) {diff --git a/packages/server/src/acp/install-server-process.test-helper.ts b/packages/server/src/acp/install-server-process.test-helper.tsnew file mode 100644index 000000000..b1a30cfaf--- /dev/null+++ b/packages/server/src/acp/install-server-process.test-helper.ts@@ -0,0 +1,31 @@+import { inspect } from 'node:util';+import { bootCompositionRig } from '../composition-rig.test-helper.ts';+import type { RegistryAgent } from './registry.ts';++process.once('disconnect', () => process.exit(0));+setTimeout(() => process.exit(2), 120_000).unref();++const [contentDir, agentJson] = process.argv.slice(2);+if (contentDir === undefined || agentJson === undefined) {+ throw new Error('The install server process requires a content directory and an agent.');+}+const agent = JSON.parse(agentJson) as RegistryAgent;++const server = await bootCompositionRig(contentDir, {+ acpRegistryFetchImpl: (async () =>+ new Response(JSON.stringify({ agents: [agent] }))) as typeof fetch,+});+await server.ready;++process.once('SIGTERM', () => {+ void server.destroy('external-signal').then(+ () => process.exit(0),+ (err: unknown) => {+ process.stderr.write(`${inspect(err)}\n`, () => process.exit(1));+ },+ );+});++const manager = server.acpThreadManager;+if (manager === null) throw new Error('The booted server has no ACP thread manager.');+await manager.createThread({ agent: { source: 'registry', id: agent.id } });diff --git a/packages/server/src/acp/launch-package-acquisition.test.ts b/packages/server/src/acp/launch-package-acquisition.test.tsindex 607197faf..21a70c19c 100644--- a/packages/server/src/acp/launch-package-acquisition.test.ts+++ b/packages/server/src/acp/launch-package-acquisition.test.ts@@ -1,7 +1,15 @@import { once } from 'node:events';-import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';+import {+ chmodSync,+ existsSync,+ mkdirSync,+ readdirSync,+ readFileSync,+ writeFileSync,+} from 'node:fs';import { createServer } from 'node:http';import { join } from 'node:path';+import { okUserHomeDir } from '@inkeep/open-knowledge-core/server';import { describe, expect, test } from 'vitest';import {executableFacts,@@ -294,6 +302,83 @@ describe('hermetic package acquisition and native dispatch', () => {});});+ test.skipIf(process.platform === 'win32')(+ 'an install reports the output it wrote without leaving an output file in its working directory',+ async () => {+ await withAcquisitionHome(async (home) => {+ const bin = join(home, 'bin');+ mkdirSync(bin);+ installNodeFixture(bin);+ const listing = join(home, 'npx-cwd-listing.json');+ writeExecutable(+ join(bin, 'npx'),+ `const fs = require('node:fs');+ fs.writeFileSync(${JSON.stringify(listing)}, JSON.stringify(fs.readdirSync(process.cwd())));+ process.stderr.write('npm error fixture install output\\n');+ process.exit(1);`,+ );+ writeExecutable(+ join(bin, 'npm'),+ `process.stdout.write(${JSON.stringify(JSON.stringify([{ name: 'is-number', version: '7.0.0' }]))});`,+ );+ await expect(+ resolveRegistryLaunch(+ registryPackage('[email protected]', 'npx', { PATH: bin }),+ null,+ log,+ undefined,+ async (launch) => launch,+ { install: true },+ ),+ ).rejects.toMatchObject({+ code: 'install-failed',+ message: 'Could not install [email protected].',+ machineDetail: 'npm error fixture install output',+ });+ expect(JSON.parse(readFileSync(listing, 'utf8'))).toEqual(['package.json']);+ expect(readdirSync(join(okUserHomeDir(home), 'acp-npx-cwd'))).toEqual(['package.json']);+ });+ },+ );++ test.skipIf(process.platform === 'win32' || process.getuid?.() === 0)(+ 'an install whose working directory refuses its output file fails as an install naming the cause',+ async () => {+ await withAcquisitionHome(async (home) => {+ const bin = join(home, 'bin');+ mkdirSync(bin);+ installNodeFixture(bin);+ writeExecutable(join(bin, 'npx'), 'process.exit(0);');+ writeExecutable(+ join(bin, 'npm'),+ `process.stdout.write(${JSON.stringify(JSON.stringify([{ name: 'is-number', version: '7.0.0' }]))});`,+ );+ const npxCwd = join(okUserHomeDir(home), 'acp-npx-cwd');+ mkdirSync(npxCwd, { recursive: true });+ writeFileSync(join(npxCwd, 'package.json'), '{}\n');+ chmodSync(npxCwd, 0o500);+ try {+ const result = await resolveRegistryLaunch(+ registryPackage('[email protected]', 'npx', { PATH: bin }),+ null,+ log,+ undefined,+ async (launch) => launch,+ { install: true },+ ).catch((error: unknown) => error);+ expect(result).toBeInstanceOf(AgentLaunchError);+ expect(result).toMatchObject({+ code: 'install-failed',+ message: 'Could not install [email protected].',+ machineDetail: expect.stringContaining('EACCES'),+ });+ } finally {+ chmodSync(npxCwd, 0o700);+ }+ });+ },+ );+test.each(['malformed', 'overflow', 'timeout'] as const)('native npm %s output fails acquisition and releases the owned process',async (mode) => {diff --git a/packages/server/src/acp/launch.ts b/packages/server/src/acp/launch.tsindex 2775d9d19..ee9879c27 100644--- a/packages/server/src/acp/launch.ts+++ b/packages/server/src/acp/launch.ts@@ -1,5 +1,6 @@-import { type ChildProcess, spawn } from 'node:child_process';-import { constants, existsSync, statSync } from 'node:fs';+import { type ChildProcess, type StdioOptions, spawn } from 'node:child_process';+import { randomUUID } from 'node:crypto';+import { closeSync, constants, createReadStream, existsSync, statSync } from 'node:fs';import { access, chmod, readFile, stat } from 'node:fs/promises';import { homedir } from 'node:os';import { basename, delimiter, dirname, isAbsolute, join, resolve } from 'node:path';@@ -9,7 +10,9 @@ import { z } from 'zod';import {tracedMkdir,tracedMkdirSync,+ tracedOpenSync,tracedRm,+ tracedUnlinkSync,tracedWriteFile,tracedWriteFileSync,} from '../fs-traced.ts';@@ -441,17 +444,13 @@ async function installNpmPackage(log: PinoLogger,): Promise<void> {const startedAt = Date.now();- const child = spawnAcpAgent({+ const installLaunch = {...launch,- kind: 'npx',+ kind: 'npx' as const,args: ['--yes', `--package=${acquired}`, '--call=exit 0'],- });+ };let stderr = '';let rawStderr = '';- let spawnFailure: string | undefined;- child.stdin?.end();- child.stdout?.resume();- child.stderr?.setEncoding('utf8');const stderrCapture = createDiagnosticStderrCapture((line) => {stderr = `${stderr}${line}\n`.slice(-ACQUISITION_DETAIL_MAX_CHARS);@@ -460,14 +459,10 @@ async function installNpmPackage(rawStderr = `${rawStderr}${rawLine}\n`.slice(-ACQUISITION_DETAIL_MAX_CHARS);},);- child.stderr?.on('data', stderrCapture.write);- child.stderr?.on('end', stderrCapture.end);- child.once('error', (err) => {- spawnFailure = err.message;- });- const [code, signal] = await new Promise<[number | null, NodeJS.Signals | null]>(- (resolvePromise) => child.once('close', (...exit) => resolvePromise(exit)),- );+ const { code, signal, spawnFailure } =+ process.platform === 'win32'+ ? await runPipedInstall(installLaunch, acquired, stderrCapture, log)+ : await runFileBackedInstall(installLaunch, acquired, stderrCapture, log);if (spawnFailure === undefined && code === 0) {log.info({ package: acquired, elapsedMs: Date.now() - startedAt },@@ -487,6 +482,108 @@ async function installNpmPackage(});}+interface InstallExit {+ code: number | null;+ signal: NodeJS.Signals | null;+ spawnFailure: string | undefined;+}++type DiagnosticStderrCapture = ReturnType<typeof createDiagnosticStderrCapture>;++function awaitInstallExit(+ child: ChildProcess,+ acquired: string,+ log: PinoLogger,+): Promise<InstallExit> {+ let spawnFailure: string | undefined;+ child.once('spawn', () => {+ log.info({ package: acquired, installPid: child.pid }, '[acp-launch] installing agent package');+ });+ child.once('error', (err) => {+ spawnFailure = err.message;+ });+ return new Promise((resolvePromise) =>+ child.once('close', (code, signal) => resolvePromise({ code, signal, spawnFailure })),+ );+}++async function runFileBackedInstall(+ launch: ResolvedLaunch & { kind: 'npx' },+ acquired: string,+ capture: DiagnosticStderrCapture,+ log: PinoLogger,+): Promise<InstallExit> {+ const stderrPath = join(acpNpxIsolatedCwd(), `.install-stderr-${process.pid}-${randomUUID()}`);+ let fd: number;+ try {+ fd = openUnlinkedFile(stderrPath);+ } catch (err) {+ return {+ code: null,+ signal: null,+ spawnFailure: err instanceof Error ? err.message : String(err),+ };+ }+ try {+ return await awaitInstallExit(+ spawnLaunch(launch, undefined, ['ignore', 'ignore', fd]),+ acquired,+ log,+ );+ } finally {+ await replayInstallStderr(fd, capture).catch((err: unknown) => {+ log.warn(+ { err, package: acquired },+ '[acp-launch] could not read the agent package install output',+ );+ });+ }+}++function openUnlinkedFile(path: string): number {+ const fd = tracedOpenSync(path, 'wx+', 0o600);+ try {+ tracedUnlinkSync(path);+ return fd;+ } catch (err) {+ closeSync(fd);+ throw err;+ }+}++async function replayInstallStderr(fd: number, capture: DiagnosticStderrCapture): Promise<void> {+ try {+ for await (const chunk of createReadStream('', {+ fd,+ start: 0,+ encoding: 'utf8',+ autoClose: false,+ })) {+ capture.write(chunk);+ }+ } finally {+ capture.end();+ closeSync(fd);+ }+}++/* UPSTREAM(libuv/libuv#1659, microsoft/terminal#12464): an inherited stdio descriptor drops+ CREATE_NO_WINDOW and Windows Terminal may show the console cmd.exe then opens. */+function runPipedInstall(+ launch: ResolvedLaunch & { kind: 'npx' },+ acquired: string,+ capture: DiagnosticStderrCapture,+ log: PinoLogger,+): Promise<InstallExit> {+ const child = spawnAcpAgent(launch);+ child.stdin?.end();+ child.stdout?.resume();+ child.stderr?.setEncoding('utf8');+ child.stderr?.on('data', capture.write);+ child.stderr?.on('end', capture.end);+ return awaitInstallExit(child, acquired, log);+}+function heldNpmPackage(entry: AcquiredNpmPackage | undefined): HeldNpmPackage | null {return entry === undefined || entry.state === 'probing' ? null : entry;}@@ -1178,6 +1275,14 @@ export function spawnAcpAgent(projectCwd: string,): ChildProcess;export function spawnAcpAgent(launch: ResolvedLaunch, projectCwd?: string): ChildProcess {+ return spawnLaunch(launch, projectCwd, ['pipe', 'pipe', 'pipe']);+}++function spawnLaunch(+ launch: ResolvedLaunch,+ projectCwd: string | undefined,+ stdio: StdioOptions,+): ChildProcess {if (launch.kind === 'npx' && projectCwd !== undefined) {throw new Error('spawnAcpAgent does not accept a project cwd for npx launches.');}@@ -1194,7 +1299,7 @@ export function spawnAcpAgent(launch: ResolvedLaunch, projectCwd?: string): Chilreturn spawn(cmd, args, {cwd,env: withHostedAgentMarker(launch.env),- stdio: ['pipe', 'pipe', 'pipe'],+ stdio,shell: false,detached: !win,windowsHide: true,diff --git a/packages/server/src/acp/npx-install-server-exit.integration.test.ts b/packages/server/src/acp/npx-install-server-exit.integration.test.tsnew file mode 100644index 000000000..d1f88f08e--- /dev/null+++ b/packages/server/src/acp/npx-install-server-exit.integration.test.ts@@ -0,0 +1,230 @@+import { type ChildProcess, execFileSync, fork } from 'node:child_process';+import { existsSync, mkdtempSync, readdirSync, readFileSync } from 'node:fs';+import { join } from 'node:path';+import { fileURLToPath } from 'node:url';+import { describe, expect, test } from 'vitest';+import { readProcessState } from '../process-scan.ts';+import { waitWithinTestBudget } from '../wait-within-test-budget.test-helper.ts';+import {+ type HeldInstallRegistry,+ withHeldInstallRegistry,+} from './held-install-registry.test-helper.ts';+import type { RegistryAgent } from './registry.ts';++const SERVER_PROCESS = fileURLToPath(+ new URL('./install-server-process.test-helper.ts', import.meta.url),+);++const WAIT = { timeoutMs: 60_000 };++type ExitStatus = { code: number | null; signal: NodeJS.Signals | null };++type ServerProcess = {+ child: ChildProcess;+ exited: () => boolean;+ exitStatus: () => ExitStatus | undefined;+ closed: () => boolean;+ output: () => string;+};++function startServerProcess(held: HeldInstallRegistry): ServerProcess {+ const agent: RegistryAgent = {+ id: 'held-install',+ name: 'Held install agent',+ version: '1.0.0',+ distribution: { npx: { package: held.npmPackage } },+ };+ const contentDir = mkdtempSync(join(held.home, 'content-'));+ const child = fork(SERVER_PROCESS, [contentDir, JSON.stringify(agent)], {+ execArgv: ['--import', 'tsx', '--conditions=@inkeep/source'],+ silent: true,+ });+ let output = '';+ let exitStatus: ExitStatus | undefined;+ let closed = false;+ child.stdout?.on('data', (chunk) => {+ output += String(chunk);+ });+ child.stderr?.on('data', (chunk) => {+ output += String(chunk);+ });+ child.once('exit', (code, signal) => {+ exitStatus = { code, signal };+ });+ child.once('close', () => {+ closed = true;+ });+ return {+ child,+ exited: () => exitStatus !== undefined,+ exitStatus: () => exitStatus,+ closed: () => closed,+ output: () => output,+ };+}++function descendantsOf(root: number): number[] {+ const table = execFileSync('ps', ['-A', '-o', 'pid=,ppid='], { encoding: 'utf8' });+ const children = new Map<number, number[]>();+ for (const line of table.split('\n')) {+ const [pid, ppid] = line.trim().split(/\s+/).map(Number);+ if (pid === undefined || ppid === undefined || !Number.isInteger(pid)) continue;+ children.set(ppid, [...(children.get(ppid) ?? []), pid]);+ }+ const found: number[] = [];+ const queue = [root];+ for (let next = queue.shift(); next !== undefined; next = queue.shift()) {+ for (const kid of children.get(next) ?? []) {+ found.push(kid);+ queue.push(kid);+ }+ }+ return found;+}++function processEnded(pid: number): boolean {+ const state = readProcessState(pid, {+ onProbeFailure: ({ reason }) => {+ throw new Error(`could not read the state of process ${pid}: ${reason}`);+ },+ });+ return state.status === 'gone' || state.status === 'defunct';+}++function npxEntries(home: string): string[] {+ const npxCache = join(home, 'npm-cache', '_npx');+ return existsSync(npxCache) ? readdirSync(npxCache).map((name) => join(npxCache, name)) : [];+}++function installedPackages(entry: string): string[] {+ const modules = join(entry, 'node_modules');+ return existsSync(modules) ? readdirSync(modules).filter((name) => !name.startsWith('.')) : [];+}++function npmDebugLogs(home: string): string {+ const logs = join(home, 'npm-cache', '_logs');+ return existsSync(logs)+ ? readdirSync(logs)+ .filter((name) => name.endsWith('.log'))+ .map((name) => readFileSync(join(logs, name), 'utf8'))+ .join('\n')+ : '';+}++function hasManifest(dir: string): boolean {+ return existsSync(join(dir, 'package.json'));+}++function entryOutcome(entry: string, agentName: string) {+ const packages = installedPackages(entry);+ const modules = join(entry, 'node_modules');+ const agentDir = join(modules, agentName);+ const agentDependencies = hasManifest(agentDir)+ ? Object.keys(+ (+ JSON.parse(readFileSync(join(agentDir, 'package.json'), 'utf8')) as {+ dependencies?: Record<string, string>;+ }+ ).dependencies ?? {},+ )+ : [];+ const snapshot = {+ rootManifest: hasManifest(entry),+ lock: existsSync(join(entry, 'concurrency.lock')),+ packages,+ withoutManifest: packages.filter((name) => !hasManifest(join(modules, name))),+ };+ const complete =+ snapshot.rootManifest &&+ !snapshot.lock &&+ hasManifest(agentDir) &&+ agentDependencies.every((name) => hasManifest(join(modules, name))) &&+ snapshot.withoutManifest.length === 0;+ const rolledBack = !snapshot.rootManifest && !snapshot.lock && packages.length === 0;+ const state = complete ? 'complete' : rolledBack ? 'rolled-back' : 'partial';+ return { state, ...snapshot };+}++const SERVER_ENDINGS = [+ { ending: 'shuts down on SIGTERM', signal: 'SIGTERM', exit: { code: 0, signal: null } },+ { ending: 'is killed', signal: 'SIGKILL', exit: { code: null, signal: 'SIGKILL' } },+] as const;++describe.skipIf(process.platform === 'win32')(+ 'an npx install that outlives the server process that started it',+ () => {+ test.each(SERVER_ENDINGS)(+ 'the npx cache entry ends complete or rolled back when the server $ending mid-install',+ async ({ signal, exit }) => {+ expect(+ () => descendantsOf(process.pid),+ 'this test reads the process table with ps',+ ).not.toThrow();+ await withHeldInstallRegistry(+ async (held) => {+ const agentName = held.npmPackage.slice(0, held.npmPackage.lastIndexOf('@'));+ const server = startServerProcess(held);+ let outliving: number[] = [];+ try {+ await waitWithinTestBudget(+ 'the server to start installing the agent package',+ () => {+ if (server.exited()) {+ throw new Error(`the server process exited early: ${server.output()}`);+ }+ return held.requested('npm-dependency') > 0;+ },+ WAIT,+ );+ const entries = npxEntries(held.home);+ expect(entries).toHaveLength(1);+ const entry = String(entries[0]);+ expect(hasManifest(entry)).toBe(false);+ const serverPid = server.child.pid;+ if (serverPid === undefined) throw new Error('the server process has no pid');+ outliving = descendantsOf(serverPid);+ expect(outliving.length).toBeGreaterThan(0);++ server.child.kill(signal);+ await waitWithinTestBudget('the server process to exit', server.exited, WAIT);+ held.release('npm-dependency');+ await waitWithinTestBudget(+ 'every process the server started to exit',+ () => outliving.every(processEnded),+ WAIT,+ );+ await waitWithinTestBudget('the server process output to close', server.closed, WAIT);+ expect(server.exitStatus(), server.output()).toEqual(exit);++ expect(+ npmDebugLogs(held.home),+ 'npm wrote its deprecation warning after the server ended',+ ).toMatch(/warn deprecated \S*ok-held-install-dependency/);+ const outcome = entryOutcome(entry, agentName);+ expect(['complete', 'rolled-back'], JSON.stringify(outcome)).toContain(outcome.state);+ } finally {+ try {+ if (!server.exited() && outliving.length === 0 && server.child.pid !== undefined) {+ outliving = descendantsOf(server.child.pid);+ }+ } finally {+ if (!server.exited()) {+ server.child.kill('SIGKILL');+ await waitWithinTestBudget('the server process to exit', server.exited, WAIT);+ }+ held.release('npm-dependency');+ await waitWithinTestBudget(+ 'every process the server started to exit',+ () => outliving.every(processEnded),+ WAIT,+ );+ }+ }+ },+ { dependencyDeprecated: 'this dependency is no longer maintained' },+ );+ },+ 150_000,+ );+ },+);diff --git a/packages/server/src/fs-traced.ts b/packages/server/src/fs-traced.tsindex 3ed07c858..af58f2fc0 100644--- a/packages/server/src/fs-traced.ts+++ b/packages/server/src/fs-traced.ts@@ -5,6 +5,7 @@ import {cpSync,linkSync,mkdirSync,+ openSync,renameSync,rmdirSync,rmSync,@@ -177,6 +178,12 @@ export function tracedCpSync();}+export function tracedOpenSync(path: string, flags: string, mode: number): number {+ return withSpanSync('fs.openSync', { attributes: buildAttrs('openSync', path) }, () =>+ openSync(path, flags, mode),+ );+}+export function tracedUnlinkSync(path: string): void {withSpanSync('fs.unlinkSync', { attributes: buildAttrs('unlinkSync', path) }, () => {unlinkSync(path);diff --git a/packages/server/tsconfig.test-helper-callers.json b/packages/server/tsconfig.test-helper-callers.jsonindex 5b96c4d91..d810ab925 100644--- a/packages/server/tsconfig.test-helper-callers.json+++ b/packages/server/tsconfig.test-helper-callers.json@@ -2,6 +2,7 @@"extends": "./tsconfig.json","include": [],"files": [+ "src/acp/npx-install-server-exit.integration.test.ts","src/config-file-watcher-readiness.test.ts","src/expect-stable.test.ts","src/file-watcher-chokidar-fallback.test.ts",