Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
import { afterEach, describe, expect, it, vi } from 'vitest';
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
import * as fs from 'node:fs';
import { randomUUID } from 'node:crypto';
import { z } from 'zod';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { createMaintenanceAdmission } from './maintenanceAdmission.js';
const roots = [];
// Frozen pre-stamp v1 operation-reader fixture: unknown fields were rejected.
const legacyOperation = z.object({ id: z.string().uuid(), kind: z.string().min(1), resource: z.string().max(256),
pid: z.number().int().positive(), startedAt: z.string().datetime(), unsettled: z.boolean().optional() }).strict();
const gate = () => {
const root = mkdtempSync(join(tmpdir(), 'workflow-admission-'));
roots.push(root);
return { root, admission: createMaintenanceAdmission(root) };
};
afterEach(() => { vi.useRealTimers(); vi.restoreAllMocks(); for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); });
describe('durable graceful maintenance', () => {
it('names an interrupted transaction and its lock time without modifying evidence', () => {
const { root, admission } = gate();
const hold = admission.begin({ reason: 'Drain', owner: 'Operator' }).hold;
const lock = join(admission.directory, 'transaction');
mkdirSync(lock);
const since = fs.lstatSync(lock).mtime.toISOString();
expect(createMaintenanceAdmission(root).status()).toMatchObject({ state: 'unavailable', cause: 'interrupted-transaction', since });
expect(fs.lstatSync(lock).mtime.toISOString()).toBe(since);
rmSync(lock, { recursive: true });
expect(admission.status()).toMatchObject({ state: 'ready', hold });
});
it('logs a failed ENOSPC publication once and remains uncertain even after lock removal', () => {
const { root } = gate();
const log = vi.spyOn(console, 'error').mockImplementation(() => {});
const admission = createMaintenanceAdmission(root, { io: { ...fs, writeFileSync: () => {
throw Object.assign(new Error('fixture disk full'), { code: 'ENOSPC' });
} } });
expect(() => admission.admit('agent', 'fixture')).toThrow(/needs recovery/);
expect(admission.status()).toMatchObject({ state: 'unavailable', cause: 'interrupted-transaction' });
expect(admission.tryAdmit('agent', 'again')).toBeNull();
expect(log).toHaveBeenCalledExactlyOnceWith('❌ Maintenance transaction failed (ENOSPC): fixture disk full');
rmSync(join(admission.directory, 'transaction'), { recursive: true });
expect(admission.status()).toMatchObject({ state: 'unavailable', cause: 'process-uncertain' });
expect(admission.tryAdmit('agent', 'still held')).toBeNull();
});
it('logs the original lock acquisition failure once', () => {
const { root } = gate();
const log = vi.spyOn(console, 'error').mockImplementation(() => {});
const admission = createMaintenanceAdmission(root, { io: { ...fs, mkdirSync: () => {
throw Object.assign(new Error('fixture denied'), { code: 'EACCES' });
} } });
expect(admission.tryAdmit('agent', 'fixture')).toBeNull();
expect(admission.tryAdmit('agent', 'again')).toBeNull();
expect(log).toHaveBeenCalledExactlyOnceWith('❌ Maintenance transaction failed (EACCES): fixture denied');
});
it('binds operator publication reconciliation to the exact hold and owner and publishes before retirement', async () => {
const { root, admission } = gate();
const owner = admission.admit('settlement', 'Output publication');
owner.markUnsettled();
const other = admission.admit('agent', 'unrelated');
const hold = admission.begin({ reason: 'Inspect refused publication', owner: 'Operator' }).hold;
const expected = JSON.parse(fs.readFileSync(join(root, 'workflow-maintenance', 'state.json'))).operations.find(op => op.id === owner.id);
let receipt;
const publish = vi.fn(() => {
expect(JSON.parse(fs.readFileSync(join(root, 'workflow-maintenance', 'state.json'))).operations)
.toContainEqual(expect.objectContaining({ id: owner.id, kind: 'settlement' }));
receipt ??= { operation: expected, disposition: 'operator-reconciled' };
return receipt;
});
const replay = vi.fn(() => receipt);
expect(() => admission.reconcilePublicationRefusal({ hold: { ...hold, revision: hold.revision + 1 }, expected, publish, replay })).toThrow('hold changed');
expect(() => admission.reconcilePublicationRefusal({ hold, expected: { ...expected, uncertaintyStamp: 'changed' }, publish, replay })).toThrow('reservation changed');
expect(publish).not.toHaveBeenCalled();
expect(admission.reconcilePublicationRefusal({ hold, expected, publish, replay })).toBe(receipt);
expect(admission.status()).toMatchObject({ hold, blockers: [{ resource: 'unrelated' }] });
expect(admission.reconcilePublicationRefusal({ hold, expected, publish, replay })).toBe(receipt);
expect(publish).toHaveBeenCalledOnce();
expect(replay).toHaveBeenCalledOnce();
await other.finish();
});
it('mints completion only when trusted recovery reuses an existing operation', () => {
const { admission } = gate();
expect(admission.admit('media', 'new')).not.toHaveProperty('completeRecovery');
expect(admission.recoverOwned('media', 'observed-new')).not.toHaveProperty('completeRecovery');
expect(admission.admit('media', 'reconnect-new', { reconnect: true })).not.toHaveProperty('completeRecovery');
expect(admission.recoverOwned('media', 'new')).toHaveProperty('completeRecovery');
});
it('completes only the captured recovered operation and preserves the hold and failed settlement child', async () => {
const { admission } = gate();
const original = admission.admit('media', 'peer-render');
await original.run(async () => admission.markCurrentUnsettled());
const other = admission.admit('provider', 'unrelated');
const hold = admission.begin({ reason: 'Drain', owner: 'Operator' }).hold;
await expect(original.run(() => admission.continueSettlement(async () => { throw new Error('save failed'); }))).rejects.toThrow('save failed');
await original.finish();
const recovered = admission.recoverOwned('media', 'peer-render');
other.markUnsettled();
const blockers = admission.status().blockers.filter(entry => entry.resource !== 'peer-render');
expect(recovered.completeRecovery()).toBe(true);
expect(admission.status()).toMatchObject({ state: 'draining', hold, blockers });
expect(blockers).toContainEqual(expect.objectContaining({ kind: 'settlement', unsettled: true }));
expect(blockers).toContainEqual(expect.objectContaining({ resource: 'unrelated' }));
expect(recovered.completeRecovery()).toBe(false);
await other.finish();
expect(admission.status().state).toBe('draining');
});
it('rotates every uncertainty stamp and refuses a captured capability after another mark', async () => {
const { admission } = gate();
const work = admission.admit('media', 'peer-render');
const readOperation = () => JSON.parse(fs.readFileSync(join(admission.directory, 'state.json'), 'utf8')).operations[0];
await work.run(async () => admission.markCurrentUnsettled());
const first = readOperation().uncertaintyStamp;
const recovered = admission.recoverOwned('media', 'peer-render');
await recovered.run(async () => admission.markCurrentUnsettled());
expect(readOperation().uncertaintyStamp).not.toBe(first);
expect(recovered.completeRecovery()).toBe(false);
await recovered.finish();
expect(admission.status().blockers).toContainEqual(expect.objectContaining({ resource: 'peer-render', unsettled: true }));
});
it('refuses a replacement operation with the same resource or a changed bound resource', async () => {
const { admission } = gate();
const work = admission.admit('media', 'peer-render');
work.markUnsettled();
const recovered = admission.recoverOwned('media', 'peer-render');
const file = join(admission.directory, 'state.json');
const state = JSON.parse(fs.readFileSync(file, 'utf8'));
state.operations[0].id = randomUUID();
fs.writeFileSync(file, JSON.stringify(state));
expect(recovered.completeRecovery()).toBe(false);
const replacement = admission.recoverOwned('media', 'peer-render');
replacement.identify('changed-resource');
expect(replacement.completeRecovery()).toBe(false);
expect(admission.status().blockers).toContainEqual(expect.objectContaining({ resource: 'changed-resource', unsettled: true }));
});
it('requires fresh recovery after restart and never infers completion from old owner PID or age', async () => {
const { root, admission } = gate();
const original = admission.admit('media', 'peer-render');
original.markUnsettled();
const hold = admission.begin({ reason: 'Drain', owner: 'Operator' }).hold;
const restarted = createMaintenanceAdmission(root);
expect(restarted.status()).toMatchObject({ state: 'draining', hold, blockers: [{ unsettled: true }] });
const recovered = restarted.recoverOwned('media', 'peer-render');
expect(recovered.id).toBe(original.id);
await recovered.finish();
expect(restarted.status().state).toBe('draining');
expect(recovered.completeRecovery()).toBe(true);
expect(restarted.status()).toMatchObject({ state: 'ready', hold, blockers: [] });
});
it('reads legacy records but refuses to complete legacy uncertainty without a stamp', async () => {
const { root, admission } = gate();
const work = admission.admit('media', 'legacy');
work.markUnsettled();
const file = join(admission.directory, 'state.json');
const state = JSON.parse(fs.readFileSync(file, 'utf8'));
delete state.operations[0].uncertaintyStamp;
fs.writeFileSync(file, JSON.stringify(state));
const restarted = createMaintenanceAdmission(root);
const recovered = restarted.recoverOwned('media', 'legacy');
expect(recovered.completeRecovery()).toBe(false);
await recovered.finish();
expect(restarted.status().blockers).toContainEqual(expect.objectContaining({ resource: 'legacy', unsettled: true }));
});
it('accounts for strict old-reader rejection instead of treating stamped v1 journals as rollback compatible', () => {
const { admission } = gate();
const work = admission.admit('media', 'peer-render');
const file = join(admission.directory, 'state.json');
const readOperation = () => JSON.parse(fs.readFileSync(file, 'utf8')).operations[0];
expect(legacyOperation.safeParse(readOperation()).success).toBe(true);
work.markUnsettled();
expect(legacyOperation.safeParse(readOperation()).success).toBe(false);
expect(admission.status().blockers).toContainEqual(expect.objectContaining({ resource: 'peer-render', unsettled: true }));
});
it('preserves an old writer transaction lock and refuses admission or completion after a compatible restart', () => {
const { root, admission } = gate();
const work = admission.admit('media', 'peer-render');
work.markUnsettled();
const recovered = admission.recoverOwned('media', 'peer-render');
const file = join(admission.directory, 'state.json');
const before = fs.readFileSync(file, 'utf8');
// The older writer acquired its lock before parsing. A parse failure kept
// the lock, just as an uncertain publication did; do not repair it by age.
fs.mkdirSync(join(admission.directory, 'transaction'));
expect(() => legacyOperation.parse(JSON.parse(before).operations[0])).toThrow();
const restarted = createMaintenanceAdmission(root);
expect(restarted.status()).toMatchObject({ state: 'unavailable', cause: 'interrupted-transaction' });
expect(restarted.tryAdmit('media', 'new')).toBeNull();
expect(() => recovered.completeRecovery()).toThrow(/needs recovery/);
expect(fs.existsSync(join(admission.directory, 'transaction'))).toBe(true);
expect(fs.readFileSync(file, 'utf8')).toBe(before);
});
it('serves status from one parse until the journal file changes', () => {
const { root } = gate();
let reads = 0;
const io = { ...fs, readFileSync: (...args) => { reads++; return fs.readFileSync(...args); } };
const admission = createMaintenanceAdmission(root, { io });
admission.admit('media', 'render-a');
const before = reads;
for (let i = 0; i < 5; i++) expect(admission.held()).toBe(false);
expect(reads - before).toBeLessThanOrEqual(1);
const settled = reads;
admission.status();
expect(reads).toBe(settled);
// A second process publishing (here: a second admission over the same root) is seen at once.
const other = createMaintenanceAdmission(root);
const hold = other.begin({ reason: 'Drain', owner: 'Operator' }).hold;
expect(admission.held()).toBe(true);
// Callers cannot corrupt the cached journal through the projection they receive.
admission.status().hold.reason = 'tampered';
expect(admission.status().hold.reason).toBe('Drain');
other.resume({ id: hold.id, revision: hold.revision });
expect(admission.held()).toBe(false);
});
it('keeps journal write failure unavailable without claiming recovered completion', () => {
const { root } = gate();
let fail = false;
const io = { ...fs, renameSync: (...args) => {
if (fail) throw Object.assign(new Error('fixture disk failure'), { code: 'EIO' });
return fs.renameSync(...args);
} };
const admission = createMaintenanceAdmission(root, { io });
const work = admission.admit('media', 'peer-render');
work.markUnsettled();
const recovered = admission.recoverOwned('media', 'peer-render');
fail = true;
expect(() => recovered.completeRecovery()).toThrow(/needs recovery/);
expect(admission.status().state).toBe('unavailable');
const persisted = JSON.parse(fs.readFileSync(join(admission.directory, 'state.json'), 'utf8'));
expect(persisted.operations.map(op => op.id)).toContain(work.id);
});
it('reuses remote reconciliation ownership without bypassing a hold', async () => {
const { admission } = gate();
const existing = admission.admit('media', 'peer-render');
const hold = admission.begin({ reason: 'Drain', owner: 'Operator' }).hold;
expect(admission.tryAdmit('media', 'peer-render', { reconnect: true })).toBeNull();
admission.resume({ id: hold.id, revision: hold.revision });
const reconnected = admission.admit('media', 'peer-render', { reconnect: true });
expect(reconnected.id).toBe(existing.id);
expect(admission.status().blockers).toHaveLength(1);
await reconnected.finish();
expect(admission.status().blockers).toEqual([]);
});
it('fences every new admission while an admitted thought can finish its next provider call and saving', async () => {
const { admission } = gate();
const turn = admission.admit('mind-turn', 'thought');
const hold = admission.begin({ reason: 'Update renderer', owner: 'Local operator' });
expect(hold.state).toBe('draining');
for (const kind of ['agent', 'media', 'mind-turn', 'runner', 'shell-job']) {
expect(() => admission.admit(kind, 'new')).toThrowError(/holding new work/);
}
let releaseSave;
const saving = new Promise(resolve => { releaseSave = resolve; });
const completion = turn.run(() => admission.run('provider', 'next call', () => saving, { continuation: true }));
await turn.finish();
expect(admission.status().state).toBe('draining');
releaseSave();
await completion;
expect(admission.status().state).toBe('ready');
expect(admission.status().hold.id).toBe(hold.hold.id);
expect(admission.status().blockers).toEqual([]);
});
it('runs required completion callbacks after a failed save while retaining the original blocker', async () => {
const { admission } = gate();
const provider = admission.admit('provider', 'example');
admission.begin({ reason: 'Work', owner: 'Operator' });
provider.markUnsettled();
const callback = vi.fn(async () => 'completed');
expect(await provider.run(() => admission.continueSettlement(callback))).toBe('completed');
await provider.finish();
expect(callback).toHaveBeenCalledTimes(1);
expect(admission.status()).toMatchObject({ state: 'draining', blockers: [{ unsettled: true }] });
});
it('reconnects observed existing work under its original durable identity during a hold', async () => {
const { root, admission } = gate();
const original = admission.admit('media', 'training');
admission.begin({ reason: 'Work', owner: 'Operator' });
const restarted = createMaintenanceAdmission(root);
const recovered = restarted.recoverOwned('media', 'training');
expect(recovered.id).toBe(original.id);
expect(restarted.status().blockers).toHaveLength(1);
await recovered.finish();
expect(admission.status().state).toBe('ready');
});
it('a second process sees the same fence and restart never discards old ownership', async () => {
const { root, admission } = gate();
const owner = admission.admit('agent', 'agent-fixture');
const restarted = createMaintenanceAdmission(root);
const held = restarted.begin({ reason: 'Service work', owner: 'Operator session' });
expect(admission.tryAdmit('runner', 'new')).toBeNull();
expect(createMaintenanceAdmission(root).status()).toMatchObject({ state: 'draining', hold: held.hold });
await owner.finish();
expect(restarted.status().state).toBe('ready');
});
it('rejects stale resume without clearing the current owner hold', () => {
const { admission } = gate();
const first = admission.begin({ reason: 'First', owner: 'Operator' }).hold;
admission.resume({ id: first.id, revision: first.revision });
const second = admission.begin({ reason: 'Second', owner: 'Operator' }).hold;
expect(() => admission.resume({ id: first.id, revision: first.revision })).toThrowError(/changed/);
expect(admission.status().hold).toEqual(second);
expect(admission.resume({ id: second.id, revision: second.revision }).state).toBe('normal');
});
it('does not let a detached callback reuse a settled parent capability', async () => {
const { admission } = gate();
const parent = admission.admit('media', 'old');
await parent.finish();
admission.begin({ reason: 'Drain', owner: 'Operator' });
expect(() => parent.run(() => admission.admit('provider', 'late', { continuation: true }))).toThrow();
expect(admission.status().state).toBe('ready');
});
it('retries completion lock contention and keeps readiness closed until the write succeeds', async () => {
vi.useFakeTimers();
const { admission } = gate();
const work = admission.admit('media', 'render');
admission.begin({ reason: 'Drain', owner: 'Operator' });
mkdirSync(join(admission.directory, 'transaction'));
const finishing = work.finish();
expect(admission.status().state).toBe('unavailable');
rmSync(join(admission.directory, 'transaction'), { recursive: true });
expect(admission.status().state).toBe('draining');
await vi.advanceTimersByTimeAsync(50);
expect(await finishing).toBe(true);
expect(admission.status().state).toBe('ready');
});
it('keeps corrupt/future journals and failed terminal persistence fail-closed', async () => {
const { root, admission } = gate();
const render = admission.admit('media', 'render');
render.markUnsettled();
await render.finish();
admission.begin({ reason: 'Drain', owner: 'Operator' });
expect(admission.status()).toMatchObject({ state: 'draining', blockers: [{ unsettled: true }] });
writeFileSync(join(admission.directory, 'state.json'), '{"version":999}');
const restarted = createMaintenanceAdmission(root);
expect(restarted.status()).toMatchObject({ state: 'unavailable', cause: 'unreadable-journal' });
expect(restarted.tryAdmit('agent', 'new')).toBeNull();
expect(() => restarted.resume({ id: '00000000-0000-4000-8000-000000000000', revision: 1 })).toThrow();
});
});