Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/** Real archive trees, primary index and state snapshots under synthetic cuts. */
import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';
import { mkdir, readFile, readdir, rm, writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import { createFileWriteQueue } from '../lib/fileWriteQueue.js';
const fixture = await vi.hoisted(async () => {
const { mkdtemp } = await import('node:fs/promises');
const { tmpdir } = await import('node:os');
const { join } = await import('node:path');
const { createFileWriteQueue } = await import('../lib/fileWriteQueue.js');
const base = await mkdtemp(join(tmpdir(), 'cos-archive-backup-'));
return { base, root: join(base, 'agents'), state: null, beforeIndex: null, bypass: false, withStateLock: createFileWriteQueue() };
});
vi.mock('./cosState.js', () => ({
AGENTS_DIR: fixture.root,
loadState: async () => structuredClone(fixture.state),
saveState: async state => { fixture.state = structuredClone(state); await writeFile(join(fixture.root, 'state.json'), JSON.stringify(state)); },
withStateLock: fn => fixture.withStateLock(fn),
readAgentsStateForSafetyCheck: async () => ({ trusted: true, agents: fixture.state.agents }),
}));
vi.mock('../lib/fileUtils.js', async importOriginal => {
const actual = await importOriginal();
return { ...actual, PATHS: { ...actual.PATHS, cos: fixture.base }, atomicWrite: async (path, data) => {
if (path === join(fixture.root, 'index.json')) await fixture.beforeIndex?.();
return actual.atomicWrite(path, data);
} };
});
vi.mock('../lib/backupSnapshotBoundary.js', async importOriginal => {
const actual = await importOriginal();
return { ...actual, withBackupAssetPublication: work => fixture.bypass ? work() : actual.withBackupAssetPublication(work) };
});
vi.mock('./domainUsage.js', () => ({ recordDomainUsage: async () => {} }));
vi.mock('./codexSummaryRepair.js', () => ({ repairCodexTaskSummary: async () => null }));
vi.mock('./cosRunnerClient.js', () => ({ getActiveAgentsFromRunner: async () => [] }));
const deferred = () => {
let resolve; const promise = new Promise(done => { resolve = done; });
return { promise, resolve };
};
const settle = () => new Promise(resolve => setImmediate(resolve));
let api;
let index;
let acquireCut;
const date = '2001-01-01';
const id = 'agent-example';
let taskStore;
let taskRepo;
const readIndex = async () => JSON.parse(await readFile(join(fixture.root, 'index.json'), 'utf8'));
const assertIndexedTrees = async () => {
for (const [agentId, bucket] of Object.entries(await readIndex())) {
expect(await readFile(join(fixture.root, bucket, agentId, 'output.txt'), 'utf8')).toBe('Example preserved output');
expect(JSON.parse(await readFile(join(fixture.root, bucket, agentId, 'metadata.json'), 'utf8')).id).toBe(agentId);
}
};
async function seedAgent(agentId, { completed = false, indexed = false, resident = true, bucketDate = date, result = { success: true } } = {}) {
const record = {
id: agentId,
status: completed ? 'completed' : 'running',
startedAt: `${bucketDate}T01:00:00Z`,
...(completed ? { completedAt: `${bucketDate}T02:00:00Z`, result } : {}),
metadata: {},
};
if (resident) fixture.state.agents[agentId] = record;
const dir = indexed ? join(fixture.root, bucketDate, agentId) : join(fixture.root, agentId);
await mkdir(dir, { recursive: true });
await writeFile(join(dir, 'output.txt'), 'Example preserved output');
await writeFile(join(dir, 'metadata.json'), JSON.stringify(record));
if (indexed) { (await index.loadAgentIndex()).set(agentId, bucketDate); await index.saveAgentIndex(); }
}
async function seed({ completed = false, indexed = false } = {}) {
await seedAgent(id, { completed, indexed });
}
beforeEach(async () => {
vi.resetModules(); fixture.beforeIndex = null; fixture.bypass = false;
fixture.state = { agents: {}, config: { completedAgentRetentionMs: 1 }, stats: { tasksCompleted: 0, errors: 0 } };
await rm(fixture.root, { recursive: true, force: true }); await mkdir(fixture.root);
await writeFile(join(fixture.root, 'index.json'), '{}');
({ acquireBackupSnapshotCut: acquireCut } = await import('../lib/backupSnapshotBoundary.js'));
taskRepo = await import('./cosTaskRepository.js');
taskRepo.__resetCosTaskRepository();
taskStore = await import('./cosTaskStore.js');
taskStore.__resetTaskCache();
index = await import('./cosAgentIndex.js');
api = { ...await import('./cosAgentLifecycle.js'), ...await import('./cosAgentArchive.js') };
});
afterAll(() => rm(fixture.base, { recursive: true, force: true }));
it('preserves pending verification before the task hold is written and after archival', async () => {
await seed({ completed: true, indexed: true });
const agent = fixture.state.agents[id];
agent.result.cleanupPending = true;
agent.metadata.repoStateVerification = { status: 'pending' };
const metadataPath = join(fixture.root, date, id, 'metadata.json');
await writeFile(metadataPath, JSON.stringify(agent));
expect(await api.clearCompletedAgents()).toEqual({ cleared: 0, retained: 1 });
expect(fixture.state.agents[id]).toEqual(agent);
delete fixture.state.agents[id];
expect(await api.clearCompletedAgents()).toEqual({ cleared: 0, retained: 1 });
await assertIndexedTrees();
agent.metadata.repoStateVerification.status = 'completed';
await writeFile(metadataPath, JSON.stringify(agent));
expect(await api.clearCompletedAgents()).toEqual({ cleared: 1, retained: 0 });
});
it.each(['user', 'cos'])('preserves cleanup-owner evidence in the %s queue until delivery settles', async source => {
await seed({ completed: true, indexed: true });
// An archived owner is absent from resident state but still required after restart.
delete fixture.state.agents[id];
const taskType = source === 'cos' ? 'internal' : 'user';
await taskStore.addTask({
id: 'task-example',
status: 'in_progress',
priority: 'MEDIUM',
priorityValue: 2,
description: 'Cleanup task',
metadata: { deliveryRecovery: 'cleanup', deliveryRecoveryAgentId: id },
}, taskType, { raw: true });
const beforeState = structuredClone(fixture.state);
const beforeIndex = await readIndex();
expect(await api.clearCompletedAgents()).toEqual({ cleared: 0, retained: 1 });
expect(fixture.state).toEqual(beforeState);
expect(await readIndex()).toEqual(beforeIndex);
await assertIndexedTrees();
await taskStore.updateTask('task-example', { status: 'completed' }, taskType);
expect(await api.clearCompletedAgents()).toEqual({ cleared: 1, retained: 0 });
expect(await readIndex()).toEqual({});
expect(await readdir(fixture.root)).not.toContain(date);
});
const cases = [
{ name: 'completion', setup: () => seed(), run: () => api.completeAgent(id, { success: true }) },
{ name: 'zombie archive', setup: () => seed(), run: () => api.cleanupZombieAgents() },
{ name: 'stale archive', setup: () => seed({ completed: true }), run: () => api.archiveStaleAgents() },
{ name: 'single deletion', setup: () => seed({ completed: true, indexed: true }), run: () => api.deleteAgent(id) },
{ name: 'clear completed', setup: () => seed({ completed: true, indexed: true }), run: () => api.clearCompletedAgents() },
];
describe.each(cases)('$name', ({ setup, run }) => {
it('keeps archive trees and the primary index behind an open cut', async () => {
await setup();
const beforeIndex = await readIndex();
const beforeState = structuredClone(fixture.state);
const beforeNames = await readdir(fixture.root);
const release = await acquireCut();
const work = run();
try { await settle(); await settle(); expect(await readIndex()).toEqual(beforeIndex); expect(fixture.state).toEqual(beforeState); expect(await readdir(fixture.root)).toEqual(beforeNames); }
finally { release(); }
await work; await assertIndexedTrees();
});
it('drains directory changes through the last primary-index commit', async () => {
await setup();
const reached = deferred(); const commit = deferred();
fixture.beforeIndex = async () => { reached.resolve(); await commit.promise; };
const work = run(); await Promise.race([reached.promise, work.then(() => { throw new Error('workflow missed index seam'); })]);
let ready = false;
const cut = acquireCut().then(release => { ready = true; return release; });
try { await settle(); expect(ready).toBe(false); }
finally { commit.resolve(); }
await work;
const release = await cut;
try { await assertIndexedTrees(); } finally { release(); }
});
});
it('legacy migration holds moved directories and its new index in one publication', async () => {
await seed({ completed: true });
await rm(join(fixture.root, 'index.json'));
const reached = deferred(); const commit = deferred();
fixture.beforeIndex = async () => { reached.resolve(); await commit.promise; };
const work = index.loadAgentIndex(); await Promise.race([reached.promise, work.then(() => { throw new Error('migration missed index seam'); })]);
let ready = false;
const cut = acquireCut().then(release => { ready = true; return release; });
try { await settle(); expect(ready).toBe(false); } finally { commit.resolve(); }
await work;
const release = await cut;
try { await assertIndexedTrees(); } finally { release(); }
});
it('index publication failure remains retryable for peer reconciliation', async () => {
const pairs = [{ agentId: id, date }];
fixture.beforeIndex = async () => { throw new Error('synthetic index write failure'); };
await expect(index.addAgentArchivesToIndex(pairs)).rejects.toThrow('synthetic index write failure');
expect(await readIndex()).toEqual({});
fixture.beforeIndex = null;
expect(await index.addAgentArchivesToIndex(pairs)).toBe(0);
expect(await readIndex()).toEqual({ [id]: date });
});
it('negative control exposes an index naming a relocated tree missing from the earlier copy', async () => {
await seed(); fixture.bypass = true;
const copiedNames = await readdir(fixture.root);
const release = await acquireCut();
try {
await api.completeAgent(id, { success: true });
const bucket = (await readIndex())[id];
expect(bucket).toBeTruthy(); expect(copiedNames).not.toContain(bucket);
} finally { release(); }
});
describe.each([
{ name: 'single deletion', run: () => api.deleteAgent(id) },
{ name: 'clear completed', run: () => api.clearCompletedAgents() },
])('$name failure', ({ run }) => {
it('retains indexed archive bytes and retries after a primary-index write failure', async () => {
await seed({ completed: true, indexed: true });
fixture.beforeIndex = async () => { throw new Error('synthetic index write failure'); };
await expect(run()).rejects.toThrow('synthetic index write failure');
await assertIndexedTrees();
expect((await index.loadAgentIndex()).get(id)).toBe(date);
fixture.beforeIndex = null;
await run();
expect(await readIndex()).toEqual({});
});
});
it('does not let a cold reader queue its shared initialization ahead of an admitted publisher', async () => {
await seed({ completed: true });
await rm(join(fixture.root, 'index.json'));
const { withBackupAssetPublication } = await import('../lib/backupSnapshotBoundary.js');
const entered = deferred(); const proceed = deferred();
const publisher = withBackupAssetPublication(async () => {
entered.resolve(); await proceed.promise;
return index.loadAgentIndex();
});
await entered.promise;
const cut = acquireCut();
const reader = index.loadAgentIndex();
proceed.resolve();
await publisher;
const release = await cut;
try { await assertIndexedTrees(); } finally { release(); }
await reader;
});
describe('active cleanup and retry owner preservation (#13430)', () => {
async function seedCleanupTask(taskId, metadata, taskType = 'internal') {
return taskStore.addTask({
id: taskId,
status: 'in_progress',
priority: 'MEDIUM',
priorityValue: 2,
description: `Cleanup task ${taskId}`,
metadata,
}, taskType, { raw: true });
}
it.each([
{ label: 'resident record', indexed: false, resident: true },
{ label: 'indexed archive', indexed: true, resident: false },
])('refuses single delete and retains $label while task holds deliveryRecovery cleanup', async ({ indexed, resident }) => {
await seedAgent(id, { completed: true, indexed, resident });
await seedCleanupTask('sys-rl-example', {
deliveryRecovery: 'cleanup',
deliveryRecoveryAgentId: id,
});
await expect(api.deleteAgent(id)).rejects.toMatchObject({
status: 409,
code: 'AGENT_CLEANUP_OWNED',
});
const record = await api.getAgent(id);
expect(record?.id).toBe(id);
if (resident) expect(fixture.state.agents[id]?.id).toBe(id);
if (indexed) await assertIndexedTrees();
await taskStore.updateTask('sys-rl-example', {
status: 'completed',
metadata: { deliveryRecovery: undefined, deliveryRecoveryAgentId: undefined },
}, 'internal');
await expect(api.deleteAgent(id)).resolves.toEqual({ success: true, agentId: id });
expect(await api.getAgent(id)).toBeNull();
});
it('retains protected resident and archived runs across mixed buckets during clearCompletedAgents', async () => {
const protectedResident = 'agent-protected-resident';
const protectedArchived = 'agent-protected-archived';
const unprotectedSameBucket = 'agent-unprotected-same-bucket';
const unprotectedOtherBucket = 'agent-unprotected-other-bucket';
const otherDate = '2001-01-02';
await seedAgent(protectedResident, { completed: true, indexed: false, resident: true });
await seedAgent(protectedArchived, { completed: true, indexed: true, resident: false, bucketDate: date });
await seedAgent(unprotectedSameBucket, { completed: true, indexed: true, resident: true, bucketDate: date });
await seedAgent(unprotectedOtherBucket, { completed: true, indexed: true, resident: false, bucketDate: otherDate });
await seedCleanupTask('sys-rl-example', {
deliveryRecovery: 'cleanup',
deliveryRecoveryAgentId: protectedResident,
}, 'internal');
await seedCleanupTask('task-retry-example', {
retryPendingCleanup: protectedArchived,
retryPendingSince: `${date}T02:00:00.000Z`,
}, 'user');
const cleared = await api.clearCompletedAgents();
expect(cleared).toEqual({ cleared: 2, retained: 2 });
expect(fixture.state.agents[protectedResident]?.id).toBe(protectedResident);
expect(fixture.state.agents[unprotectedSameBucket]).toBeUndefined();
expect(await readIndex()).toEqual({ [protectedArchived]: date });
await assertIndexedTrees();
await expect(readFile(join(fixture.root, date, unprotectedSameBucket, 'metadata.json'), 'utf8')).rejects.toThrow();
await expect(readdir(join(fixture.root, otherDate))).rejects.toThrow();
await taskStore.updateTask('sys-rl-example', { status: 'completed' }, 'internal');
await taskStore.updateTask('task-retry-example', { status: 'pending' }, 'user');
const secondClear = await api.clearCompletedAgents();
expect(secondClear).toEqual({ cleared: 2, retained: 0 });
expect(fixture.state.agents).toEqual({});
expect(await readIndex()).toEqual({});
});
it('does not retain completed history solely because a stale agent result still carries cleanupPending', async () => {
await seedAgent(id, {
completed: true,
indexed: true,
resident: true,
result: { success: true, cleanupPending: true },
});
await taskStore.addTask({
id: 'sys-rl-settled',
status: 'completed',
priority: 'MEDIUM',
priorityValue: 2,
description: 'Already settled task',
metadata: {},
}, 'internal', { raw: true });
await expect(api.clearCompletedAgents()).resolves.toEqual({ cleared: 1, retained: 0 });
expect(await api.getAgent(id)).toBeNull();
});
it('revalidates current task ownership under the state lock so a concurrent reassignment is not deleted', async () => {
const oldOwner = 'agent-old-owner';
const newOwner = 'agent-new-owner';
await seedAgent(oldOwner, { completed: true, indexed: true, resident: true });
await seedAgent(newOwner, { completed: true, indexed: true, resident: true });
await seedCleanupTask('sys-rl-race', {
deliveryRecovery: 'cleanup',
deliveryRecoveryAgentId: oldOwner,
}, 'internal');
// Start clearCompletedAgents while an earlier task reassignment is holding the shared
// state lock; clearCompletedAgents must read task ownership AFTER acquiring the lock,
// not from a stale pre-lock snapshot.
const entered = deferred();
const release = deferred();
const blocker = fixture.withStateLock(async () => {
entered.resolve();
await release.promise;
});
await entered.promise;
const reassign = taskStore.updateTask('sys-rl-race', {
status: 'in_progress',
metadata: { deliveryRecovery: 'cleanup', deliveryRecoveryAgentId: newOwner },
}, 'internal');
await settle();
const clear = api.clearCompletedAgents();
await settle();
release.resolve();
await blocker;
const [, clearResult] = await Promise.all([reassign, clear]);
expect(clearResult).toEqual({ cleared: 1, retained: 1 });
expect((await api.getAgent(newOwner))?.id).toBe(newOwner);
expect(await api.getAgent(oldOwner)).toBeNull();
});
it('preserves evidence when task cleanup ownership cannot be verified', async () => {
await seedAgent(id, { completed: true, indexed: true, resident: true });
const spy = vi.spyOn(taskStore, 'getTaskView').mockRejectedValueOnce(new Error('task store unavailable'));
try {
await expect(api.deleteAgent(id)).rejects.toMatchObject({
status: 503,
code: 'CLEANUP_OWNERSHIP_UNAVAILABLE',
});
expect((await api.getAgent(id))?.id).toBe(id);
await assertIndexedTrees();
} finally {
spy.mockRestore();
}
});
});