Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

test: settle completion capture fixtures before cancellation teardown (#13464)

[._.]/ Adam Eivyd44da2c74786
diff --git a/scripts/completionCaptureCleanup.test.js b/scripts/completionCaptureCleanup.test.js
new file mode 100644
index 0000000000..921a2db9e5
--- /dev/null
+++ b/scripts/completionCaptureCleanup.test.js
@@ -0,0 +1,99 @@
+import { afterAll, expect, it as vitestIt } from 'vitest';
+import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { ownTestBodies } from '../server/lib/mockPathsDataRoot.js';
+import { createVitestTempFixture, fixtureReadyFailureSource, runFixtureToCompletion } from './lib/vitestTempRootFixture.js';
+
+const RUN_CEILING_MS = 120000;
+const owned = ownTestBodies(vitestIt);
+afterAll(() => owned.drain());
+
+// Run the real seed/restart scenario against a neighboring worker's bail. The
+// negative control removes only the ownership drain, leaving every assertion.
+for (const drainBodies of [false, true]) {
+ owned.it(`completion capture sibling bail: ${drainBodies ? 'settles before teardown' : 'detects unfinished seed body'}`, async ({ signal }) => {
+ const host = mkdtempSync(join(tmpdir(), 'completion-cancel-'));
+ let closeConfirmed = true;
+ try {
+ const fixture = join(host, 'fixture');
+ const ready = join(fixture, 'ready');
+ const events = join(fixture, 'events');
+ const fixtureRun = createVitestTempFixture(host, 'server', '');
+ // Real setup owns Git templates after per-suite teardown has settled.
+ writeFileSync(join(fixture, 'vitest.setup.js'), `import ${JSON.stringify(new URL('../server/vitest.setup.js', import.meta.url).href)};\n`);
+ const sourcePath = fileURLToPath(new URL('../server/services/cosAgentLifecycle.test.js', import.meta.url));
+ const input = readFileSync(sourcePath, 'utf8');
+ const header = 'seedBodies.it(`recovers only the original completion physical seed after restart, scenario=${scenario}`, async () => {';
+ const gitRead = "expect((await execGit(['rev-parse', 'HEAD'], checkout)).stdout.trim()).toBe(retainedHead);";
+ for (const marker of [header, gitRead, 'await seedBodies.drain();']) expect(input).toContain(marker);
+ const source = (drainBodies ? input : input.replace('await seedBodies.drain();', '/* negative control: no body settlement */'))
+ .replace('afterAll(cleanupTempDataRoots);', `
+ const { appendFileSync, writeFileSync } = await import('node:fs');
+ const record = event => appendFileSync(${JSON.stringify(events)}, event + '\\n');
+ let seedRoot;
+ let pending = false;
+ let release;
+ let settled;
+ const bodySettled = new Promise(resolve => { settled = resolve; });
+ // Registered before real cleanup: reverse hook order observes teardown
+ // first, then safely joins the negative control's orphaned body.
+ afterAll(async () => {
+ await bodySettled;
+ record('observer-complete:exists=' + existsSync(seedRoot));
+ });
+ afterAll(() => {
+ record('outer-cleanup:pending=' + pending + ':exists=' + existsSync(seedRoot));
+ cleanupTempDataRoots();
+ });`)
+ .replace(header, header.replace('async () =>', 'async ({ signal }) =>'))
+ .replace("let failFirstRead;", `let failFirstRead;
+ seedRoot = scratch;
+ record('allocated');
+ const cancelled = new Promise(resolve => { release = resolve; });
+ signal.addEventListener('abort', () => {
+ record('aborted');
+ // Let Vitest enter teardown while this body still owns Git work.
+ setImmediate(release);
+ }, { once: true });`)
+ .replace(gitRead, `pending = true;
+ record('git-ready');
+ writeFileSync(${JSON.stringify(ready)}, 'ready');
+ await cancelled;
+ record('git-resume');
+ ${gitRead}
+ pending = false;
+ record('git-settled');`)
+ .replace('failFirstRead?.mockRestore();', "record('finally-enter');\n failFirstRead?.mockRestore();")
+ .replace('await rm(scratch, { recursive: true, force: true, maxRetries: 3 });', `await rm(scratch, { recursive: true, force: true, maxRetries: 3 });
+ record('seed-removed:exists=' + existsSync(scratch));
+ settled();`)
+ .replace('fsFailures.dateBucketWritesFail = false;', `record('teardown:pending=' + pending + ':exists=' + existsSync(seedRoot));
+ fsFailures.dateBucketWritesFail = false;`)
+ .replace(/(['"])(\.\.?\/[^'"]+)\1/g, (_, quote, specifier) => JSON.stringify(resolve(dirname(sourcePath), specifier)));
+ writeFileSync(join(fixture, 'lifecycle.test.js'), source);
+ writeFileSync(join(fixture, 'failure.test.js'), fixtureReadyFailureSource(ready));
+ fixtureRun.args[fixtureRun.args.indexOf('--maxWorkers') + 1] = '2';
+ fixtureRun.args.push('--bail=1', '--testNamePattern', 'scenario=unchanged$|controlled sibling failure');
+ closeConfirmed = false;
+ const result = await runFixtureToCompletion(fixtureRun, RUN_CEILING_MS, { signal });
+ closeConfirmed = result.closeConfirmed;
+ if (signal.aborted && closeConfirmed) return;
+ const observed = existsSync(events) ? readFileSync(events, 'utf8') : '';
+ const output = `${result.stdout}\n${result.stderr}\nlifecycle: ${observed}`;
+ expect(result.error, output).toBeUndefined();
+ expect(result.status, output).toBe(1);
+ expect(output).toContain('controlled sibling failure');
+ const outerCleanup = observed.split('\n').find(event => event.startsWith('outer-cleanup:'));
+ if (drainBodies) expect(outerCleanup, output).toBe('outer-cleanup:pending=false:exists=false');
+ expect(observed.replace(/^outer-cleanup:.*\n/m, ''), output).toBe(drainBodies
+ ? 'allocated\ngit-ready\naborted\ngit-resume\ngit-settled\nfinally-enter\nseed-removed:exists=false\nteardown:pending=false:exists=false\nobserver-complete:exists=false\n'
+ : 'allocated\ngit-ready\naborted\nteardown:pending=true:exists=true\ngit-resume\ngit-settled\nfinally-enter\nseed-removed:exists=false\nobserver-complete:exists=false\n');
+ expect(output).not.toMatch(/ENOTEMPTY|ENOENT|Unhandled|test temp leak:/);
+ } finally {
+ // Preserve an unconfirmed host instead of removing beneath a child.
+ if (closeConfirmed) rmSync(host, { recursive: true, force: true });
+ }
+ }, RUN_CEILING_MS + 15000);
+}
diff --git a/server/services/cosAgentLifecycle.test.js b/server/services/cosAgentLifecycle.test.js
index 3e1a6c3a1b..b533456d74 100644
--- a/server/services/cosAgentLifecycle.test.js
+++ b/server/services/cosAgentLifecycle.test.js
@@ -4,7 +4,7 @@ import { mkdir, rm, writeFile, readFile, mkdtemp, rename } from 'fs/promises';
import { tmpdir } from 'node:os';
import { existsSync } from 'fs';
import { join } from 'path';
-import { cleanupTempDataRoots, lazyTempDataRoot, makePathsProxy } from '../lib/mockPathsDataRoot.js';
+import { cleanupTempDataRoots, lazyTempDataRoot, makePathsProxy, ownTestBodies } from '../lib/mockPathsDataRoot.js';
const mockCosState = vi.hoisted(() => ({
// Use $TMPDIR (falls back to /tmp) rather than a hardcoded /private/tmp — the
@@ -556,6 +556,7 @@ describe('completeAgent budget-ledger ordering (#1683)', () => {
describe('completeAgent idempotence (#3384)', () => {
const emittedCompletions = [];
+ const seedBodies = ownTestBodies(it);
beforeEach(async () => {
await rm(mockCosState.agentsDir, { recursive: true, force: true });
@@ -569,9 +570,15 @@ describe('completeAgent idempotence (#3384)', () => {
});
afterEach(async () => {
- fsFailures.dateBucketWritesFail = false;
- await rm(mockCosState.agentsDir, { recursive: true, force: true });
- cosEvents.removeAllListeners('agent:completed');
+ // Bail settles Vitest's wrapper before its body: retain fixtures and spies
+ // until the seed's Git work, completion/retry and finally have all finished.
+ try {
+ await seedBodies.drain();
+ } finally {
+ fsFailures.dateBucketWritesFail = false;
+ await rm(mockCosState.agentsDir, { recursive: true, force: true });
+ cosEvents.removeAllListeners('agent:completed');
+ }
});
it.each([{ success: true, pinned: false }, { success: false, pinned: false }, { success: true, pinned: true }, { success: false, pinned: true }])('archives checkout evidence before releasing a delivered or cleanup-pending claim ($success, pinned=$pinned)', async ({ success, pinned }) => {
@@ -646,71 +653,73 @@ describe('completeAgent idempotence (#3384)', () => {
expect(persisted.metadata.completedCheckouts).toEqual([evidence]);
});
- it.each(['unchanged', 'physical-generation', 'new-owner', 'later-head', 'orphan-empty', 'orphan-physical-generation', 'orphan-new-owner', 'orphan-later-head'])('recovers only the original completion physical seed after restart, scenario=%s', async scenario => {
- const orphan = scenario.startsWith('orphan-');
- const changed = !['unchanged', 'orphan-empty'].includes(scenario);
- const scratch = await mkdtemp(join(tmpdir(), 'portos-completion-capture-seed-'));
- let failFirstRead;
- try {
- const repo = join(scratch, 'repo');
- const checkout = join(scratch, 'checkout');
- await materializeGitRepo(repo);
- await execGit(['worktree', 'add', '-b', 'feature/seeded', checkout, 'main'], repo);
- const actual = await vi.importActual('./completedCheckoutEvidence.js');
- captureCheckoutSeeds.mockImplementationOnce(actual.captureCheckoutSeeds);
- captureCompletedCheckouts.mockImplementationOnce(actual.captureCompletedCheckouts);
- const agentId = `seeded-${changed}`;
- await registerAgent(agentId, 'seeded-task', { isWorktree: true, sourceWorkspace: repo,
- workspacePath: checkout, worktreeBranch: 'feature/seeded' });
- const seed = mockCosState.state.agents[agentId].metadata.checkoutSeeds[0];
- expect(seed).toMatchObject({ agentId, branch: 'feature/seeded' });
- await writeFile(join(checkout, 'delivery.txt'), 'wanted original work');
- await execGit(['add', 'delivery.txt'], checkout);
- await execGit(['-c', 'user.name=Test', '-c', '[email protected]', 'commit', '-m', 'final delivered head'], checkout);
- const head = (await execGit(['rev-parse', 'HEAD'], checkout)).stdout.trim();
- if (orphan) captureCompletedCheckouts.mockReset().mockResolvedValueOnce([]); // Historical failed-run empty capture.
- else failFirstRead = vi.spyOn(await import('./worktreeManager.js'), 'listWorktrees').mockRejectedValueOnce(new Error('first completion Git read unavailable'));
- const completed = await completeAgent(agentId, orphan ? { success: false, executionSuccess: false, orphaned: true } : { success: true });
- if (orphan) expect(completed).toMatchObject({ result: { success: false, executionSuccess: false, orphaned: true },
- metadata: { completedCheckouts: [], checkoutCapture: { status: 'captured' } } });
- else expect(completed).toMatchObject({ status: 'completed', result: { cleanupPending: true, executionSuccess: true },
- metadata: { checkoutCapture: { status: 'pending', unknown: false, targets: [expect.objectContaining({ head })] } } });
- expect(cosEvents.listenerCount('agent:completed')).toBeGreaterThan(0);
- delete mockCosState.state.agents[agentId]; // Restart: only the real archived metadata remains.
- let retainedHead = head;
- if (scenario.endsWith('later-head')) {
- await execGit(['-c', 'user.name=Test', '-c', '[email protected]', 'commit', '--allow-empty', '-m', 'later unrelated writer'], checkout);
- retainedHead = (await execGit(['rev-parse', 'HEAD'], checkout)).stdout.trim();
+ for (const scenario of ['unchanged', 'physical-generation', 'new-owner', 'later-head', 'orphan-empty', 'orphan-physical-generation', 'orphan-new-owner', 'orphan-later-head']) {
+ seedBodies.it(`recovers only the original completion physical seed after restart, scenario=${scenario}`, async () => {
+ const orphan = scenario.startsWith('orphan-');
+ const changed = !['unchanged', 'orphan-empty'].includes(scenario);
+ const scratch = await mkdtemp(join(tmpdir(), 'portos-completion-capture-seed-'));
+ let failFirstRead;
+ try {
+ const repo = join(scratch, 'repo');
+ const checkout = join(scratch, 'checkout');
+ await materializeGitRepo(repo);
+ await execGit(['worktree', 'add', '-b', 'feature/seeded', checkout, 'main'], repo);
+ const actual = await vi.importActual('./completedCheckoutEvidence.js');
+ captureCheckoutSeeds.mockImplementationOnce(actual.captureCheckoutSeeds);
+ captureCompletedCheckouts.mockImplementationOnce(actual.captureCompletedCheckouts);
+ const agentId = `seeded-${changed}`;
+ await registerAgent(agentId, 'seeded-task', { isWorktree: true, sourceWorkspace: repo,
+ workspacePath: checkout, worktreeBranch: 'feature/seeded' });
+ const seed = mockCosState.state.agents[agentId].metadata.checkoutSeeds[0];
+ expect(seed).toMatchObject({ agentId, branch: 'feature/seeded' });
+ await writeFile(join(checkout, 'delivery.txt'), 'wanted original work');
+ await execGit(['add', 'delivery.txt'], checkout);
+ await execGit(['-c', 'user.name=Test', '-c', '[email protected]', 'commit', '-m', 'final delivered head'], checkout);
+ const head = (await execGit(['rev-parse', 'HEAD'], checkout)).stdout.trim();
+ if (orphan) captureCompletedCheckouts.mockReset().mockResolvedValueOnce([]); // Historical failed-run empty capture.
+ else failFirstRead = vi.spyOn(await import('./worktreeManager.js'), 'listWorktrees').mockRejectedValueOnce(new Error('first completion Git read unavailable'));
+ const completed = await completeAgent(agentId, orphan ? { success: false, executionSuccess: false, orphaned: true } : { success: true });
+ if (orphan) expect(completed).toMatchObject({ result: { success: false, executionSuccess: false, orphaned: true },
+ metadata: { completedCheckouts: [], checkoutCapture: { status: 'captured' } } });
+ else expect(completed).toMatchObject({ status: 'completed', result: { cleanupPending: true, executionSuccess: true },
+ metadata: { checkoutCapture: { status: 'pending', unknown: false, targets: [expect.objectContaining({ head })] } } });
+ expect(cosEvents.listenerCount('agent:completed')).toBeGreaterThan(0);
+ delete mockCosState.state.agents[agentId]; // Restart: only the real archived metadata remains.
+ let retainedHead = head;
+ if (scenario.endsWith('later-head')) {
+ await execGit(['-c', 'user.name=Test', '-c', '[email protected]', 'commit', '--allow-empty', '-m', 'later unrelated writer'], checkout);
+ retainedHead = (await execGit(['rev-parse', 'HEAD'], checkout)).stdout.trim();
+ }
+ if (scenario.endsWith('new-owner')) {
+ mockCosState.state.agents['new-owner'] = { id: 'new-owner', status: 'completed', startedAt: new Date().toISOString(),
+ metadata: { sourceWorkspace: repo, workspacePath: checkout, worktreeBranch: 'feature/seeded' } };
+ }
+ if (scenario.endsWith('physical-generation')) {
+ const pointer = join(checkout, '.git');
+ const bytes = await readFile(pointer);
+ await rename(pointer, join(scratch, 'original-git-pointer'));
+ await writeFile(pointer, bytes);
+ }
+ const options = orphan ? { retirement: { repoPath: repo, branch: 'feature/seeded', path: checkout, head } } : {};
+ expect(await retryCompletionCheckoutCapture(agentId, options)).toBe(!changed);
+ const recovered = await getAgent(agentId);
+ if (changed) expect(recovered.metadata.checkoutCapture.status).toBe(orphan ? 'captured' : 'pending');
+ else {
+ const receipt = recovered.metadata.completedCheckouts[0];
+ expect(receipt).toMatchObject({ version: 2, head, seededAt: seed.seededAt });
+ expect(Date.parse(receipt.capturedAt)).toBeGreaterThanOrEqual(Date.parse(completed.completedAt));
+ expect(completedCheckoutReceiptValid(recovered, receipt)).toBe(true);
+ }
+ if (orphan) expect(recovered.result).toEqual(completed.result);
+ expect(await readFile(join(checkout, 'delivery.txt'), 'utf8')).toBe('wanted original work');
+ expect((await execGit(['rev-parse', 'HEAD'], checkout)).stdout.trim()).toBe(retainedHead);
+ expect(emittedCompletions).toHaveLength(1);
+ } finally {
+ failFirstRead?.mockRestore();
+ await rm(scratch, { recursive: true, force: true, maxRetries: 3 });
}
- if (scenario.endsWith('new-owner')) {
- mockCosState.state.agents['new-owner'] = { id: 'new-owner', status: 'completed', startedAt: new Date().toISOString(),
- metadata: { sourceWorkspace: repo, workspacePath: checkout, worktreeBranch: 'feature/seeded' } };
- }
- if (scenario.endsWith('physical-generation')) {
- const pointer = join(checkout, '.git');
- const bytes = await readFile(pointer);
- await rename(pointer, join(scratch, 'original-git-pointer'));
- await writeFile(pointer, bytes);
- }
- const options = orphan ? { retirement: { repoPath: repo, branch: 'feature/seeded', path: checkout, head } } : {};
- expect(await retryCompletionCheckoutCapture(agentId, options)).toBe(!changed);
- const recovered = await getAgent(agentId);
- if (changed) expect(recovered.metadata.checkoutCapture.status).toBe(orphan ? 'captured' : 'pending');
- else {
- const receipt = recovered.metadata.completedCheckouts[0];
- expect(receipt).toMatchObject({ version: 2, head, seededAt: seed.seededAt });
- expect(Date.parse(receipt.capturedAt)).toBeGreaterThanOrEqual(Date.parse(completed.completedAt));
- expect(completedCheckoutReceiptValid(recovered, receipt)).toBe(true);
- }
- if (orphan) expect(recovered.result).toEqual(completed.result);
- expect(await readFile(join(checkout, 'delivery.txt'), 'utf8')).toBe('wanted original work');
- expect((await execGit(['rev-parse', 'HEAD'], checkout)).stdout.trim()).toBe(retainedHead);
- expect(emittedCompletions).toHaveLength(1);
- } finally {
- failFirstRead?.mockRestore();
- await rm(scratch, { recursive: true, force: true, maxRetries: 3 });
- }
- });
+ });
+ }
it('keeps the first verdict when a duplicate completion arrives', async () => {
// Regression: a stray runner `agent:completed` for an agent that had already