Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
test: settle completion capture fixtures before cancellation teardown (#13464)
[._.]/ Adam Eivyd44da2c74786
diff --git a/scripts/completionCaptureCleanup.test.js b/scripts/completionCaptureCleanup.test.jsnew file mode 100644index 0000000000..921a2db9e5--- /dev/null+++ b/scripts/completionCaptureCleanup.test.js@@ -0,0 +1,99 @@+import { afterAll, expect, it as vitestIt } from 'vitest';+import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { dirname, join, resolve } from 'node:path';+import { fileURLToPath } from 'node:url';+import { ownTestBodies } from '../server/lib/mockPathsDataRoot.js';+import { createVitestTempFixture, fixtureReadyFailureSource, runFixtureToCompletion } from './lib/vitestTempRootFixture.js';++const RUN_CEILING_MS = 120000;+const owned = ownTestBodies(vitestIt);+afterAll(() => owned.drain());++// Run the real seed/restart scenario against a neighboring worker's bail. The+// negative control removes only the ownership drain, leaving every assertion.+for (const drainBodies of [false, true]) {+ owned.it(`completion capture sibling bail: ${drainBodies ? 'settles before teardown' : 'detects unfinished seed body'}`, async ({ signal }) => {+ const host = mkdtempSync(join(tmpdir(), 'completion-cancel-'));+ let closeConfirmed = true;+ try {+ const fixture = join(host, 'fixture');+ const ready = join(fixture, 'ready');+ const events = join(fixture, 'events');+ const fixtureRun = createVitestTempFixture(host, 'server', '');+ // Real setup owns Git templates after per-suite teardown has settled.+ writeFileSync(join(fixture, 'vitest.setup.js'), `import ${JSON.stringify(new URL('../server/vitest.setup.js', import.meta.url).href)};\n`);+ const sourcePath = fileURLToPath(new URL('../server/services/cosAgentLifecycle.test.js', import.meta.url));+ const input = readFileSync(sourcePath, 'utf8');+ const header = 'seedBodies.it(`recovers only the original completion physical seed after restart, scenario=${scenario}`, async () => {';+ const gitRead = "expect((await execGit(['rev-parse', 'HEAD'], checkout)).stdout.trim()).toBe(retainedHead);";+ for (const marker of [header, gitRead, 'await seedBodies.drain();']) expect(input).toContain(marker);+ const source = (drainBodies ? input : input.replace('await seedBodies.drain();', '/* negative control: no body settlement */'))+ .replace('afterAll(cleanupTempDataRoots);', `+ const { appendFileSync, writeFileSync } = await import('node:fs');+ const record = event => appendFileSync(${JSON.stringify(events)}, event + '\\n');+ let seedRoot;+ let pending = false;+ let release;+ let settled;+ const bodySettled = new Promise(resolve => { settled = resolve; });+ // Registered before real cleanup: reverse hook order observes teardown+ // first, then safely joins the negative control's orphaned body.+ afterAll(async () => {+ await bodySettled;+ record('observer-complete:exists=' + existsSync(seedRoot));+ });+ afterAll(() => {+ record('outer-cleanup:pending=' + pending + ':exists=' + existsSync(seedRoot));+ cleanupTempDataRoots();+ });`)+ .replace(header, header.replace('async () =>', 'async ({ signal }) =>'))+ .replace("let failFirstRead;", `let failFirstRead;+ seedRoot = scratch;+ record('allocated');+ const cancelled = new Promise(resolve => { release = resolve; });+ signal.addEventListener('abort', () => {+ record('aborted');+ // Let Vitest enter teardown while this body still owns Git work.+ setImmediate(release);+ }, { once: true });`)+ .replace(gitRead, `pending = true;+ record('git-ready');+ writeFileSync(${JSON.stringify(ready)}, 'ready');+ await cancelled;+ record('git-resume');+ ${gitRead}+ pending = false;+ record('git-settled');`)+ .replace('failFirstRead?.mockRestore();', "record('finally-enter');\n failFirstRead?.mockRestore();")+ .replace('await rm(scratch, { recursive: true, force: true, maxRetries: 3 });', `await rm(scratch, { recursive: true, force: true, maxRetries: 3 });+ record('seed-removed:exists=' + existsSync(scratch));+ settled();`)+ .replace('fsFailures.dateBucketWritesFail = false;', `record('teardown:pending=' + pending + ':exists=' + existsSync(seedRoot));+ fsFailures.dateBucketWritesFail = false;`)+ .replace(/(['"])(\.\.?\/[^'"]+)\1/g, (_, quote, specifier) => JSON.stringify(resolve(dirname(sourcePath), specifier)));+ writeFileSync(join(fixture, 'lifecycle.test.js'), source);+ writeFileSync(join(fixture, 'failure.test.js'), fixtureReadyFailureSource(ready));+ fixtureRun.args[fixtureRun.args.indexOf('--maxWorkers') + 1] = '2';+ fixtureRun.args.push('--bail=1', '--testNamePattern', 'scenario=unchanged$|controlled sibling failure');+ closeConfirmed = false;+ const result = await runFixtureToCompletion(fixtureRun, RUN_CEILING_MS, { signal });+ closeConfirmed = result.closeConfirmed;+ if (signal.aborted && closeConfirmed) return;+ const observed = existsSync(events) ? readFileSync(events, 'utf8') : '';+ const output = `${result.stdout}\n${result.stderr}\nlifecycle: ${observed}`;+ expect(result.error, output).toBeUndefined();+ expect(result.status, output).toBe(1);+ expect(output).toContain('controlled sibling failure');+ const outerCleanup = observed.split('\n').find(event => event.startsWith('outer-cleanup:'));+ if (drainBodies) expect(outerCleanup, output).toBe('outer-cleanup:pending=false:exists=false');+ expect(observed.replace(/^outer-cleanup:.*\n/m, ''), output).toBe(drainBodies+ ? 'allocated\ngit-ready\naborted\ngit-resume\ngit-settled\nfinally-enter\nseed-removed:exists=false\nteardown:pending=false:exists=false\nobserver-complete:exists=false\n'+ : 'allocated\ngit-ready\naborted\nteardown:pending=true:exists=true\ngit-resume\ngit-settled\nfinally-enter\nseed-removed:exists=false\nobserver-complete:exists=false\n');+ expect(output).not.toMatch(/ENOTEMPTY|ENOENT|Unhandled|test temp leak:/);+ } finally {+ // Preserve an unconfirmed host instead of removing beneath a child.+ if (closeConfirmed) rmSync(host, { recursive: true, force: true });+ }+ }, RUN_CEILING_MS + 15000);+}diff --git a/server/services/cosAgentLifecycle.test.js b/server/services/cosAgentLifecycle.test.jsindex 3e1a6c3a1b..b533456d74 100644--- a/server/services/cosAgentLifecycle.test.js+++ b/server/services/cosAgentLifecycle.test.js@@ -4,7 +4,7 @@ import { mkdir, rm, writeFile, readFile, mkdtemp, rename } from 'fs/promises';import { tmpdir } from 'node:os';import { existsSync } from 'fs';import { join } from 'path';-import { cleanupTempDataRoots, lazyTempDataRoot, makePathsProxy } from '../lib/mockPathsDataRoot.js';+import { cleanupTempDataRoots, lazyTempDataRoot, makePathsProxy, ownTestBodies } from '../lib/mockPathsDataRoot.js';const mockCosState = vi.hoisted(() => ({// Use $TMPDIR (falls back to /tmp) rather than a hardcoded /private/tmp — the@@ -556,6 +556,7 @@ describe('completeAgent budget-ledger ordering (#1683)', () => {describe('completeAgent idempotence (#3384)', () => {const emittedCompletions = [];+ const seedBodies = ownTestBodies(it);beforeEach(async () => {await rm(mockCosState.agentsDir, { recursive: true, force: true });@@ -569,9 +570,15 @@ describe('completeAgent idempotence (#3384)', () => {});afterEach(async () => {- fsFailures.dateBucketWritesFail = false;- await rm(mockCosState.agentsDir, { recursive: true, force: true });- cosEvents.removeAllListeners('agent:completed');+ // Bail settles Vitest's wrapper before its body: retain fixtures and spies+ // until the seed's Git work, completion/retry and finally have all finished.+ try {+ await seedBodies.drain();+ } finally {+ fsFailures.dateBucketWritesFail = false;+ await rm(mockCosState.agentsDir, { recursive: true, force: true });+ cosEvents.removeAllListeners('agent:completed');+ }});it.each([{ success: true, pinned: false }, { success: false, pinned: false }, { success: true, pinned: true }, { success: false, pinned: true }])('archives checkout evidence before releasing a delivered or cleanup-pending claim ($success, pinned=$pinned)', async ({ success, pinned }) => {@@ -646,71 +653,73 @@ describe('completeAgent idempotence (#3384)', () => {expect(persisted.metadata.completedCheckouts).toEqual([evidence]);});- it.each(['unchanged', 'physical-generation', 'new-owner', 'later-head', 'orphan-empty', 'orphan-physical-generation', 'orphan-new-owner', 'orphan-later-head'])('recovers only the original completion physical seed after restart, scenario=%s', async scenario => {- const orphan = scenario.startsWith('orphan-');- const changed = !['unchanged', 'orphan-empty'].includes(scenario);- const scratch = await mkdtemp(join(tmpdir(), 'portos-completion-capture-seed-'));- let failFirstRead;- try {- const repo = join(scratch, 'repo');- const checkout = join(scratch, 'checkout');- await materializeGitRepo(repo);- await execGit(['worktree', 'add', '-b', 'feature/seeded', checkout, 'main'], repo);- const actual = await vi.importActual('./completedCheckoutEvidence.js');- captureCheckoutSeeds.mockImplementationOnce(actual.captureCheckoutSeeds);- captureCompletedCheckouts.mockImplementationOnce(actual.captureCompletedCheckouts);- const agentId = `seeded-${changed}`;- await registerAgent(agentId, 'seeded-task', { isWorktree: true, sourceWorkspace: repo,- workspacePath: checkout, worktreeBranch: 'feature/seeded' });- const seed = mockCosState.state.agents[agentId].metadata.checkoutSeeds[0];- expect(seed).toMatchObject({ agentId, branch: 'feature/seeded' });- await writeFile(join(checkout, 'delivery.txt'), 'wanted original work');- await execGit(['add', 'delivery.txt'], checkout);- await execGit(['-c', 'user.name=Test', '-c', '[email protected]', 'commit', '-m', 'final delivered head'], checkout);- const head = (await execGit(['rev-parse', 'HEAD'], checkout)).stdout.trim();- if (orphan) captureCompletedCheckouts.mockReset().mockResolvedValueOnce([]); // Historical failed-run empty capture.- else failFirstRead = vi.spyOn(await import('./worktreeManager.js'), 'listWorktrees').mockRejectedValueOnce(new Error('first completion Git read unavailable'));- const completed = await completeAgent(agentId, orphan ? { success: false, executionSuccess: false, orphaned: true } : { success: true });- if (orphan) expect(completed).toMatchObject({ result: { success: false, executionSuccess: false, orphaned: true },- metadata: { completedCheckouts: [], checkoutCapture: { status: 'captured' } } });- else expect(completed).toMatchObject({ status: 'completed', result: { cleanupPending: true, executionSuccess: true },- metadata: { checkoutCapture: { status: 'pending', unknown: false, targets: [expect.objectContaining({ head })] } } });- expect(cosEvents.listenerCount('agent:completed')).toBeGreaterThan(0);- delete mockCosState.state.agents[agentId]; // Restart: only the real archived metadata remains.- let retainedHead = head;- if (scenario.endsWith('later-head')) {- await execGit(['-c', 'user.name=Test', '-c', '[email protected]', 'commit', '--allow-empty', '-m', 'later unrelated writer'], checkout);- retainedHead = (await execGit(['rev-parse', 'HEAD'], checkout)).stdout.trim();+ for (const scenario of ['unchanged', 'physical-generation', 'new-owner', 'later-head', 'orphan-empty', 'orphan-physical-generation', 'orphan-new-owner', 'orphan-later-head']) {+ seedBodies.it(`recovers only the original completion physical seed after restart, scenario=${scenario}`, async () => {+ const orphan = scenario.startsWith('orphan-');+ const changed = !['unchanged', 'orphan-empty'].includes(scenario);+ const scratch = await mkdtemp(join(tmpdir(), 'portos-completion-capture-seed-'));+ let failFirstRead;+ try {+ const repo = join(scratch, 'repo');+ const checkout = join(scratch, 'checkout');+ await materializeGitRepo(repo);+ await execGit(['worktree', 'add', '-b', 'feature/seeded', checkout, 'main'], repo);+ const actual = await vi.importActual('./completedCheckoutEvidence.js');+ captureCheckoutSeeds.mockImplementationOnce(actual.captureCheckoutSeeds);+ captureCompletedCheckouts.mockImplementationOnce(actual.captureCompletedCheckouts);+ const agentId = `seeded-${changed}`;+ await registerAgent(agentId, 'seeded-task', { isWorktree: true, sourceWorkspace: repo,+ workspacePath: checkout, worktreeBranch: 'feature/seeded' });+ const seed = mockCosState.state.agents[agentId].metadata.checkoutSeeds[0];+ expect(seed).toMatchObject({ agentId, branch: 'feature/seeded' });+ await writeFile(join(checkout, 'delivery.txt'), 'wanted original work');+ await execGit(['add', 'delivery.txt'], checkout);+ await execGit(['-c', 'user.name=Test', '-c', '[email protected]', 'commit', '-m', 'final delivered head'], checkout);+ const head = (await execGit(['rev-parse', 'HEAD'], checkout)).stdout.trim();+ if (orphan) captureCompletedCheckouts.mockReset().mockResolvedValueOnce([]); // Historical failed-run empty capture.+ else failFirstRead = vi.spyOn(await import('./worktreeManager.js'), 'listWorktrees').mockRejectedValueOnce(new Error('first completion Git read unavailable'));+ const completed = await completeAgent(agentId, orphan ? { success: false, executionSuccess: false, orphaned: true } : { success: true });+ if (orphan) expect(completed).toMatchObject({ result: { success: false, executionSuccess: false, orphaned: true },+ metadata: { completedCheckouts: [], checkoutCapture: { status: 'captured' } } });+ else expect(completed).toMatchObject({ status: 'completed', result: { cleanupPending: true, executionSuccess: true },+ metadata: { checkoutCapture: { status: 'pending', unknown: false, targets: [expect.objectContaining({ head })] } } });+ expect(cosEvents.listenerCount('agent:completed')).toBeGreaterThan(0);+ delete mockCosState.state.agents[agentId]; // Restart: only the real archived metadata remains.+ let retainedHead = head;+ if (scenario.endsWith('later-head')) {+ await execGit(['-c', 'user.name=Test', '-c', '[email protected]', 'commit', '--allow-empty', '-m', 'later unrelated writer'], checkout);+ retainedHead = (await execGit(['rev-parse', 'HEAD'], checkout)).stdout.trim();+ }+ if (scenario.endsWith('new-owner')) {+ mockCosState.state.agents['new-owner'] = { id: 'new-owner', status: 'completed', startedAt: new Date().toISOString(),+ metadata: { sourceWorkspace: repo, workspacePath: checkout, worktreeBranch: 'feature/seeded' } };+ }+ if (scenario.endsWith('physical-generation')) {+ const pointer = join(checkout, '.git');+ const bytes = await readFile(pointer);+ await rename(pointer, join(scratch, 'original-git-pointer'));+ await writeFile(pointer, bytes);+ }+ const options = orphan ? { retirement: { repoPath: repo, branch: 'feature/seeded', path: checkout, head } } : {};+ expect(await retryCompletionCheckoutCapture(agentId, options)).toBe(!changed);+ const recovered = await getAgent(agentId);+ if (changed) expect(recovered.metadata.checkoutCapture.status).toBe(orphan ? 'captured' : 'pending');+ else {+ const receipt = recovered.metadata.completedCheckouts[0];+ expect(receipt).toMatchObject({ version: 2, head, seededAt: seed.seededAt });+ expect(Date.parse(receipt.capturedAt)).toBeGreaterThanOrEqual(Date.parse(completed.completedAt));+ expect(completedCheckoutReceiptValid(recovered, receipt)).toBe(true);+ }+ if (orphan) expect(recovered.result).toEqual(completed.result);+ expect(await readFile(join(checkout, 'delivery.txt'), 'utf8')).toBe('wanted original work');+ expect((await execGit(['rev-parse', 'HEAD'], checkout)).stdout.trim()).toBe(retainedHead);+ expect(emittedCompletions).toHaveLength(1);+ } finally {+ failFirstRead?.mockRestore();+ await rm(scratch, { recursive: true, force: true, maxRetries: 3 });}- if (scenario.endsWith('new-owner')) {- mockCosState.state.agents['new-owner'] = { id: 'new-owner', status: 'completed', startedAt: new Date().toISOString(),- metadata: { sourceWorkspace: repo, workspacePath: checkout, worktreeBranch: 'feature/seeded' } };- }- if (scenario.endsWith('physical-generation')) {- const pointer = join(checkout, '.git');- const bytes = await readFile(pointer);- await rename(pointer, join(scratch, 'original-git-pointer'));- await writeFile(pointer, bytes);- }- const options = orphan ? { retirement: { repoPath: repo, branch: 'feature/seeded', path: checkout, head } } : {};- expect(await retryCompletionCheckoutCapture(agentId, options)).toBe(!changed);- const recovered = await getAgent(agentId);- if (changed) expect(recovered.metadata.checkoutCapture.status).toBe(orphan ? 'captured' : 'pending');- else {- const receipt = recovered.metadata.completedCheckouts[0];- expect(receipt).toMatchObject({ version: 2, head, seededAt: seed.seededAt });- expect(Date.parse(receipt.capturedAt)).toBeGreaterThanOrEqual(Date.parse(completed.completedAt));- expect(completedCheckoutReceiptValid(recovered, receipt)).toBe(true);- }- if (orphan) expect(recovered.result).toEqual(completed.result);- expect(await readFile(join(checkout, 'delivery.txt'), 'utf8')).toBe('wanted original work');- expect((await execGit(['rev-parse', 'HEAD'], checkout)).stdout.trim()).toBe(retainedHead);- expect(emittedCompletions).toHaveLength(1);- } finally {- failFirstRead?.mockRestore();- await rm(scratch, { recursive: true, force: true, maxRetries: 3 });- }- });+ });+ }it('keeps the first verdict when a duplicate completion arrives', async () => {// Regression: a stray runner `agent:completed` for an agent that had already