Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: recover interrupted maintenance transactions without reopening work (#13332)

Adam Eivy6f984639d281
diff --git a/client/src/components/MaintenanceBanner.jsx b/client/src/components/MaintenanceBanner.jsx
index f3b9acbe19..35a04b9405 100644
--- a/client/src/components/MaintenanceBanner.jsx
+++ b/client/src/components/MaintenanceBanner.jsx
@@ -6,6 +6,7 @@ export default function MaintenanceBanner() {
if (!status || status.state === 'normal') return null;
const label = status.state === 'ready' ? 'Maintenance ready — new work is held.'
: status.state === 'draining' ? `Maintenance draining — ${formatCount(status.blockerCount)} operation(s) finishing.`
+ : status.state === 'unavailable' ? 'Maintenance recovery needed — new work is held.'
: 'Maintenance readiness unknown.';
return <div role="status" className="px-4 py-2 text-sm bg-port-card border-b border-port-border">{label} <Link className="underline" to="/settings/general">View maintenance</Link></div>;
}
diff --git a/client/src/components/MaintenanceBanner.test.jsx b/client/src/components/MaintenanceBanner.test.jsx
index 1c3117032f..aa3f675f34 100644
--- a/client/src/components/MaintenanceBanner.test.jsx
+++ b/client/src/components/MaintenanceBanner.test.jsx
@@ -18,6 +18,7 @@ describe('MaintenanceBanner', () => {
expect(container).toBeEmptyDOMElement();
unmount();
show({ state: 'unavailable', blockerCount: 0 });
- expect(screen.getByRole('status')).toHaveTextContent('readiness unknown');
+ expect(screen.getByRole('status')).toHaveTextContent('recovery needed');
+ expect(screen.getByRole('link', { name: 'View maintenance' })).toHaveAttribute('href', '/settings/general');
});
});
diff --git a/client/src/components/settings/MaintenancePanel.jsx b/client/src/components/settings/MaintenancePanel.jsx
index 04aa3b4628..31dbd69c27 100644
--- a/client/src/components/settings/MaintenancePanel.jsx
+++ b/client/src/components/settings/MaintenancePanel.jsx
@@ -3,6 +3,18 @@ import { formatDateTime, formatTimeOfDay } from '../../utils/formatters.js';
import { useMaintenance } from '../../hooks/useMaintenance.js';
import useMounted from '../../hooks/useMounted.js';
import { beginMaintenance, resumeMaintenance } from '../../services/apiSystem.js';
+import { copyToClipboard } from '../../lib/clipboard.js';
+
+const recoveryCommands = [
+ 'pm2 stop portos-server portos-cos',
+ 'node scripts/recover-maintenance-transaction.mjs --confirm "Reviewed interrupted maintenance transaction"',
+ 'pm2 start portos-server portos-cos',
+].join('\n');
+const recoveryCauses = {
+ 'interrupted-transaction': 'An interrupted journal transaction is holding new work.',
+ 'unreadable-journal': 'The maintenance journal is unreadable or incompatible. Restore compatible code or repair the journal before recovery; the command refuses invalid state.',
+ 'process-uncertain': 'This process could not confirm a journal write. Both PortOS processes must restart after recovery.',
+};
export default function MaintenancePanel() {
const { status, refresh } = useMaintenance();
@@ -23,11 +35,19 @@ export default function MaintenancePanel() {
<h3 id="maintenance-heading" className="font-semibold text-port-text">Graceful maintenance</h3>
<p role="status" className="text-sm">{status ? ({ normal: 'Normal — new work can start', draining: 'Draining — letting active work finish', ready: 'Ready — maintenance hold is active', unavailable: 'Recovery needed — new work is held', unknown: 'Readiness unknown' }[status.state]) : 'Loading maintenance status…'}</p>
<p className="text-sm text-port-text-muted">New agents, mind turns and renders wait. Active work finishes, including saving and cleanup. Queues and existing scheduling policies are preserved.</p>
- {status?.hold ? <p className="text-sm">{status.hold.reason} · {status.hold.owner} · {formatDateTime(status.hold.requestedAt)}</p> : <label htmlFor="maintenance-reason" className="block text-sm">Reason<input id="maintenance-reason" className="block w-full mt-1 p-2 rounded bg-port-bg border border-port-border" value={reason} maxLength={500} onChange={e => setReason(e.target.value)} disabled={busy || unknown} /></label>}
+ {status?.cause && <div className="rounded border border-port-border p-3 space-y-2 text-sm">
+ <p>{recoveryCauses[status.cause] || 'The maintenance journal needs recovery.'}</p>
+ {status.since && <p>Held since {formatDateTime(status.since)}</p>}
+ <p>Inspect the server logs for the original error and resolve disk space or I/O problems first. From the PortOS install directory, stop both PM2 apps and any standalone journal writers, run this command with your recovery reason, then start both apps:</p>
+ <pre className="whitespace-pre-wrap break-words rounded bg-port-bg p-3">{recoveryCommands}</pre>
+ <button className="underline" onClick={() => copyToClipboard(recoveryCommands, 'Recovery steps copied')}>Copy recovery steps</button>
+ <p>The command preserves transaction evidence and keeps a maintenance hold. Inspect any remaining blockers after restarting, then use Resume previous policies when ready. Never delete the journal or lock by hand.</p>
+ </div>}
+ {status?.hold ? <p className="text-sm">{status.hold.reason} · {status.hold.owner} · {formatDateTime(status.hold.requestedAt)}</p> : !status?.cause && <label htmlFor="maintenance-reason" className="block text-sm">Reason<input id="maintenance-reason" className="block w-full mt-1 p-2 rounded bg-port-bg border border-port-border" value={reason} maxLength={500} onChange={e => setReason(e.target.value)} disabled={busy || unknown} /></label>}
{status?.blockers?.length > 0 && <ul className="text-sm space-y-1 max-h-56 overflow-auto" aria-label="Maintenance blockers">{status.blockers.map((b, i) => <li key={`${b.kind}:${b.resource}:${i}`}>{b.kind}: {b.resource || 'Preparing'}{b.unsettled ? ' — saving or cleanup needs recovery' : ''} · since {formatTimeOfDay(b.startedAt)}</li>)}</ul>}
<p className="text-xs text-port-text-muted">{status?.scope || 'Readiness covers admitted PortOS automation and media work. Unrelated applications on this computer are outside this scope.'} A restart never clears a hold or unresolved work. Resuming cancels this hold; it does not certify readiness.</p>
{(error || status?.error) && <p role="alert" className="text-sm text-port-error">{error || status.error}</p>}
- <button className="px-4 py-2 rounded bg-port-accent text-white disabled:opacity-50" disabled={busy || unknown || (!status?.hold && !reason.trim())} onClick={act}>{busy ? 'Saving…' : status?.hold ? 'Resume previous policies' : 'Enter maintenance'}</button>
+ {!status?.cause && <button className="px-4 py-2 rounded bg-port-accent text-white disabled:opacity-50" disabled={busy || unknown || (!status?.hold && !reason.trim())} onClick={act}>{busy ? 'Saving…' : status?.hold ? 'Resume previous policies' : 'Enter maintenance'}</button>}
{unknown && <button className="ml-3 text-sm underline" onClick={refresh}>Refresh status</button>}
</section>;
}
diff --git a/client/src/components/settings/MaintenancePanel.test.jsx b/client/src/components/settings/MaintenancePanel.test.jsx
index 16d92171ee..d1a784ff2b 100644
--- a/client/src/components/settings/MaintenancePanel.test.jsx
+++ b/client/src/components/settings/MaintenancePanel.test.jsx
@@ -3,6 +3,8 @@ import { fireEvent, render, screen, waitFor } from '@testing-library/react';
const fixture = vi.hoisted(() => ({ status: null, refresh: vi.fn(), begin: vi.fn(), resume: vi.fn() }));
vi.mock('../../hooks/useMaintenance.js', () => ({ useMaintenance: () => fixture }));
vi.mock('../../services/apiSystem.js', () => ({ beginMaintenance: fixture.begin, resumeMaintenance: fixture.resume }));
+vi.mock('../../lib/clipboard.js', () => ({ copyToClipboard: vi.fn() }));
+import { copyToClipboard } from '../../lib/clipboard.js';
import MaintenancePanel from './MaintenancePanel.jsx';
beforeEach(() => {
@@ -40,4 +42,25 @@ describe('maintenance controls', () => {
expect(screen.getByRole('status')).toHaveTextContent('Readiness unknown');
expect(fixture.begin).not.toHaveBeenCalled();
});
+
+ it('shows copyable offline recovery steps and the lock time instead of a disabled maintenance action', () => {
+ fixture.status = { state: 'unavailable', cause: 'interrupted-transaction', since: '2026-10-04T12:00:00.000Z' };
+ render(<MaintenancePanel />);
+ expect(screen.getByText(/interrupted journal transaction/)).toBeInTheDocument();
+ expect(screen.getByText(/Held since/)).toBeInTheDocument();
+ expect(screen.queryByRole('button', { name: 'Enter maintenance' })).not.toBeInTheDocument();
+ expect(screen.queryByLabelText('Reason')).not.toBeInTheDocument();
+ fireEvent.click(screen.getByRole('button', { name: 'Copy recovery steps' }));
+ expect(copyToClipboard).toHaveBeenCalledWith(expect.stringMatching(/pm2 stop portos-server portos-cos\nnode scripts\/recover-maintenance-transaction\.mjs --confirm .+\npm2 start portos-server portos-cos/), 'Recovery steps copied');
+ expect(screen.getByRole('button', { name: 'Refresh status' })).toBeEnabled();
+ expect(fixture.begin).not.toHaveBeenCalled();
+ });
+
+ it.each(['unreadable-journal', 'process-uncertain'])('explains %s recovery without offering mutations', cause => {
+ fixture.status = { state: 'unavailable', cause };
+ render(<MaintenancePanel />);
+ expect(screen.getByRole('button', { name: 'Copy recovery steps' })).toBeEnabled();
+ expect(screen.queryByRole('button', { name: 'Enter maintenance' })).not.toBeInTheDocument();
+ expect(screen.getByText(cause === 'unreadable-journal' ? /command refuses invalid state/ : /could not confirm a journal write/)).toBeInTheDocument();
+ });
});
diff --git a/docs/STORAGE.md b/docs/STORAGE.md
index 78138c951b..5428cbd9f9 100644
--- a/docs/STORAGE.md
+++ b/docs/STORAGE.md
@@ -69,6 +69,47 @@ retires that exact owner under one lock. Missing ownership can only replay the
existing receipt, never create an owner. This does not certify a complete
transcript, clear transcript warnings, or change the original task outcome.
+### Interrupted maintenance transactions
+
+An interrupted journal write stays fail-closed across restarts. Settings → General
+→ Graceful maintenance reports `interrupted-transaction` and the lock time,
+`unreadable-journal` for damaged/incompatible state, or `process-uncertain` when
+this process could not confirm a write. The first transaction failure logs its
+original error code and message; repeated refusals do not flood that diagnostic.
+
+Inspect the logs and resolve disk-space or I/O problems first. From the install
+directory, stop both PM2 apps and any standalone processes using this journal:
+
+```bash
+pm2 stop portos-server portos-cos
+node scripts/recover-maintenance-transaction.mjs --confirm "Inspected interrupted transaction and resolved disk problem"
+pm2 start portos-server portos-cos
+```
+
+Use your actual recovery reason. From a reviewed isolated checkout, set
+`PORTOS_DATA_ROOT` to the install root (the parent of `data/`). The command refuses
+live/transitioning PM2 ownership, an unreadable PM2 inventory, an invalid/future
+`state.json`, or linked/non-regular journal evidence. Restore compatible code or
+recover damaged journal data separately before retrying; never delete valid
+operations or replace damaged state with an empty journal to bypass this check.
+
+Recovery preserves the original state, `transaction/`, every `pending-*.json`,
+and a receipt recording the operator reason, lock mtime, pending sizes, original
+state hash and retained hold under `data/workflow-maintenance/recovered-<timestamp>-<id>/`.
+It publishes a **Transaction recovery** hold before moving the lock when no hold
+exists; an existing hold and every operation/exclusive owner remain intact.
+A failed repair retains admission fencing and evidence; after addressing its
+cause, repeat the confirmed command. There is no automatic expiry or PID-based
+lock removal. Starting both apps clears their in-memory uncertainty, but does
+not clear the hold or certify that unresolved work finished.
+
+After restarting, refresh maintenance status, inspect/reconcile any remaining
+blockers through their owning workflows, and choose **Resume previous policies**
+deliberately. With no unresolved operations the status is Ready; otherwise it is
+Draining. Recovery archives stay in the same backed-up, machine-local subtree:
+snapshots retain them as evidence and restores preserve the current subtree
+byte-for-byte, including archives, rather than installing a snapshot's authority.
+
### Explicitly abandoned duplicate agents
A local operator may reconcile a user-killed duplicate that saved no output with
diff --git a/docs/decisions/2026-10-04-music-video-render-pools.md b/docs/decisions/2026-10-04-music-video-render-pools.md
index 0b2bfb704c..94478a4765 100644
--- a/docs/decisions/2026-10-04-music-video-render-pools.md
+++ b/docs/decisions/2026-10-04-music-video-render-pools.md
@@ -80,8 +80,10 @@ rollback therefore require an explicit compatibility plan before deployment:
prove that existing work has settled or authorize resuming admission.
- Do not roll back to a strict older binary against a stamped journal. Preserve
the journal and any transaction lock if maintenance becomes unavailable;
- restore compatible code and obtain a separately reviewed recovery procedure.
- This feature supplies no automatic lock repair or rollback migration.
+ restore compatible code, then use the explicitly confirmed offline
+ [transaction recovery procedure](../STORAGE.md#interrupted-maintenance-transactions).
+ It archives the lock and pending evidence and retains a hold and unresolved
+ ownership. There is no automatic lock repair or rollback migration.
- Never manually delete uncertainty, operation records or a transaction lock,
or infer completion from process age/PID. Legacy unsettled records without a
stamp remain blockers; they cannot use the new recovery completion capability.
diff --git a/scripts/recover-maintenance-transaction.mjs b/scripts/recover-maintenance-transaction.mjs
new file mode 100644
index 0000000000..c6a6d6baf1
--- /dev/null
+++ b/scripts/recover-maintenance-transaction.mjs
@@ -0,0 +1,95 @@
+#!/usr/bin/env node
+import * as fs from 'node:fs';
+import { createHash, randomUUID } from 'node:crypto';
+import { join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { maintenanceBeginSchema, maintenanceJournalSchema, syncMaintenanceDirectory } from '../server/lib/maintenanceAdmission.js';
+import { PATHS } from '../server/lib/paths.js';
+import { assertNotRealDataWrite } from '../server/lib/testDataIsolation.js';
+
+const listProcesses = () => import('../server/services/pm2.js').then(pm2 => pm2.listMaintenanceProcesses());
+const refuse = message => { throw new Error(message); };
+
+/** Offline operator disposition of a transaction, never settlement of its work. */
+export async function recoverMaintenanceTransaction(reason, { data = PATHS.data, io = fs, processes = listProcesses } = {}) {
+ const { reason: confirmedReason } = maintenanceBeginSchema.parse({ reason });
+ const offline = async () => {
+ const inventory = await processes();
+ if (!Array.isArray(inventory) || inventory.some(proc => !proc || typeof proc.name !== 'string' || typeof proc.status !== 'string'))
+ refuse('PM2 process state is unreadable; recovery refused.');
+ if (inventory.some(proc => ['portos-server', 'portos-cos'].includes(proc.name)
+ && (!['stopped', 'errored'].includes(proc.status) || proc.pid !== 0)))
+ refuse('Stop portos-server and portos-cos in PM2 before recovery.');
+ };
+ await offline();
+ const directory = join(data, 'workflow-maintenance');
+ const file = join(directory, 'state.json');
+ const lock = join(directory, 'transaction');
+ const stat = path => {
+ try { return io.lstatSync(path); }
+ catch (err) { if (err.code === 'ENOENT') return null; throw err; }
+ };
+ const dirStat = stat(directory);
+ if (dirStat && !dirStat.isDirectory()) refuse('The maintenance journal must be a real directory.');
+ const originalStat = stat(file);
+ if (originalStat && !originalStat.isFile()) refuse('state.json must be a regular file, not a link or directory.');
+ const original = originalStat ? io.readFileSync(file) : null;
+ // Use exactly the runtime schema; future or damaged journals require compatible
+ // code/data recovery, never replacement with an empty journal.
+ const state = original ? maintenanceJournalSchema.parse(JSON.parse(original.toString('utf8')))
+ : { version: 1, revision: 0, hold: null, operations: [] };
+ const lockStat = stat(lock);
+ if (lockStat && !lockStat.isDirectory()) refuse('The transaction lock must be a real directory.');
+ const pending = dirStat ? io.readdirSync(directory).filter(name => /^pending-.*\.json$/.test(name)).sort().map(name => {
+ const stats = stat(join(directory, name));
+ if (!stats?.isFile()) refuse('Pending journal evidence must be regular files.');
+ return { name, size: stats.size };
+ }) : [];
+ await offline();
+ const identity = stats => stats && [stats.dev, stats.ino, stats.size, stats.mtimeMs, stats.ctimeMs].join(':');
+ const current = stat(file);
+ if (identity(current) !== identity(originalStat) || identity(stat(lock)) !== identity(lockStat)
+ || (original && !io.readFileSync(file).equals(original)))
+ refuse('The maintenance journal changed during inspection; inspect again before recovery.');
+ assertNotRealDataWrite(directory, 'maintenance transaction recovery');
+ io.mkdirSync(directory, { recursive: true, mode: 0o700 });
+ if (!lockStat) io.mkdirSync(lock, { mode: 0o700 });
+ const at = new Date().toISOString();
+ const archiveDir = join(directory, `recovered-${at.replace(/:/g, '-')}-${randomUUID()}`);
+ io.mkdirSync(archiveDir, { mode: 0o700 });
+ const durableWrite = (path, contents) => {
+ const fd = io.openSync(path, 'wx', 0o600);
+ try { io.writeFileSync(fd, contents); io.fsyncSync(fd); } finally { io.closeSync(fd); }
+ };
+ if (original) durableWrite(join(archiveDir, 'state.json'), original);
+ if (!state.hold) {
+ state.revision++;
+ state.hold = { id: randomUUID(), revision: state.revision, owner: 'Transaction recovery',
+ reason: confirmedReason, requestedAt: at };
+ }
+ maintenanceJournalSchema.parse(state);
+ const receipt = { version: 1, reason: confirmedReason, at, lockMtime: lockStat?.mtime.toISOString() ?? null,
+ pending, originalStateSha256: original ? createHash('sha256').update(original).digest('hex') : null,
+ hold: state.hold };
+ durableWrite(join(archiveDir, 'receipt.json'), `${JSON.stringify(receipt)}\n`);
+ for (const { name } of pending) io.renameSync(join(directory, name), join(archiveDir, name));
+ syncMaintenanceDirectory(archiveDir, io);
+ // Publish and fsync a hold BEFORE moving the admission lock. A crash at any
+ // step retains either the original fence or the deliberate recovery hold.
+ const staged = join(directory, `pending-${randomUUID()}.json`);
+ durableWrite(staged, `${JSON.stringify(state)}\n`);
+ io.renameSync(staged, file);
+ syncMaintenanceDirectory(directory, io);
+ io.renameSync(lock, join(archiveDir, 'transaction'));
+ syncMaintenanceDirectory(archiveDir, io);
+ syncMaintenanceDirectory(directory, io);
+ return { archiveDir, hold: state.hold, pendingCount: pending.length };
+}
+
+if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1])) {
+ try {
+ const [confirmation, ...reason] = process.argv.slice(2);
+ if (confirmation !== '--confirm') throw new Error('Usage: node scripts/recover-maintenance-transaction.mjs --confirm <reason>');
+ console.log(JSON.stringify(await recoverMaintenanceTransaction(reason.join(' '))));
+ } catch (err) { console.error(`❌ Maintenance recovery refused: ${err.message}`); process.exitCode = 1; }
+}
diff --git a/scripts/recover-maintenance-transaction.test.js b/scripts/recover-maintenance-transaction.test.js
new file mode 100644
index 0000000000..f14dc109bc
--- /dev/null
+++ b/scripts/recover-maintenance-transaction.test.js
@@ -0,0 +1,117 @@
+import { afterEach, describe, expect, it } from 'vitest';
+import * as fs from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { randomUUID } from 'node:crypto';
+import { recoverMaintenanceTransaction } from './recover-maintenance-transaction.mjs';
+import { createMaintenanceAdmission } from '../server/lib/maintenanceAdmission.js';
+
+const roots = [];
+const reason = 'Inspected interrupted transaction evidence';
+const stopped = async () => ['portos-server', 'portos-cos'].map(name => ({ name, status: 'stopped', pid: 0 }));
+function fixture() {
+ const data = fs.mkdtempSync(join(tmpdir(), 'maintenance-recovery-'));
+ roots.push(data);
+ const admission = createMaintenanceAdmission(data);
+ fs.mkdirSync(admission.directory, { recursive: true });
+ return { data, admission, directory: admission.directory };
+}
+const interrupt = directory => {
+ fs.mkdirSync(join(directory, 'transaction'));
+ fs.writeFileSync(join(directory, 'transaction', 'evidence'), 'retained lock evidence');
+ fs.writeFileSync(join(directory, 'pending-fixture.json'), '');
+ fs.writeFileSync(join(directory, 'pending-other.json'), '{"partial":true}');
+};
+afterEach(() => { for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); });
+
+describe('offline maintenance transaction recovery', () => {
+ it('preserves interrupted bytes and the original state, then holds admission until deliberate resume', async () => {
+ const { data, directory, admission } = fixture();
+ // A previously published journal must be preserved byte-for-byte as evidence.
+ const work = admission.admit('agent', 'fixture');
+ await work.finish();
+ const before = fs.readFileSync(join(directory, 'state.json'));
+ interrupt(directory);
+ const since = fs.lstatSync(join(directory, 'transaction')).mtime.toISOString();
+ const result = await recoverMaintenanceTransaction(reason, { data, processes: stopped });
+ expect(fs.readFileSync(join(result.archiveDir, 'state.json'))).toEqual(before);
+ expect(fs.readFileSync(join(result.archiveDir, 'transaction', 'evidence'), 'utf8')).toBe('retained lock evidence');
+ expect(fs.readFileSync(join(result.archiveDir, 'pending-fixture.json'))).toHaveLength(0);
+ expect(fs.readFileSync(join(result.archiveDir, 'pending-other.json'), 'utf8')).toBe('{"partial":true}');
+ expect(JSON.parse(fs.readFileSync(join(result.archiveDir, 'receipt.json'), 'utf8'))).toMatchObject({
+ reason, lockMtime: since, pending: [{ name: 'pending-fixture.json', size: 0 }, { name: 'pending-other.json', size: 16 }],
+ hold: { owner: 'Transaction recovery' },
+ });
+ expect(fs.readdirSync(directory).sort()).toEqual([result.archiveDir.split(/[\\/]/).at(-1), 'state.json'].sort());
+ const restarted = createMaintenanceAdmission(data);
+ expect(restarted.status()).toMatchObject({ state: 'ready', hold: result.hold });
+ expect(restarted.tryAdmit('agent', 'new')).toBeNull();
+ expect(restarted.resume({ id: result.hold.id, revision: result.hold.revision }).state).toBe('normal');
+ });
+
+ it('recovers an absent state conservatively and preserves existing holds and unresolved operations', async () => {
+ const absent = fixture();
+ interrupt(absent.directory);
+ await recoverMaintenanceTransaction(reason, { data: absent.data, processes: stopped });
+ expect(createMaintenanceAdmission(absent.data).status()).toMatchObject({ state: 'ready', hold: { owner: 'Transaction recovery' } });
+
+ const { data, directory, admission } = fixture();
+ const work = admission.admit('settlement', 'Unfinished save');
+ work.markUnsettled();
+ const hold = admission.begin({ reason: 'Existing hold', owner: 'Operator' }).hold;
+ const before = JSON.parse(fs.readFileSync(join(directory, 'state.json'), 'utf8'));
+ interrupt(directory);
+ await recoverMaintenanceTransaction(reason, { data, processes: stopped });
+ expect(JSON.parse(fs.readFileSync(join(directory, 'state.json'), 'utf8'))).toEqual(before);
+ expect(createMaintenanceAdmission(data).status()).toMatchObject({ state: 'draining', hold, blockers: [{ unsettled: true }] });
+ });
+
+ it('refuses live, transitioning or unreadable PM2 ownership and invalid journals without changing evidence', async () => {
+ const { data, directory } = fixture();
+ interrupt(directory);
+ for (const inventory of [null, [{}], [{ name: 'portos-server', status: 'online', pid: 123 }],
+ [{ name: 'portos-cos', status: 'launching', pid: 0 }], [{ name: 'portos-cos', status: 'stopped', pid: 123 }]]) {
+ await expect(recoverMaintenanceTransaction(reason, { data, processes: async () => inventory })).rejects.toThrow(/PM2/);
+ }
+ for (const text of ['', '{bad json', '{"version":999}']) {
+ fs.writeFileSync(join(directory, 'state.json'), text);
+ await expect(recoverMaintenanceTransaction(reason, { data, processes: stopped })).rejects.toThrow();
+ expect(fs.readFileSync(join(directory, 'state.json'), 'utf8')).toBe(text);
+ }
+ expect(fs.readdirSync(directory).sort()).toEqual(['pending-fixture.json', 'pending-other.json', 'state.json', 'transaction']);
+ expect(fs.readFileSync(join(directory, 'transaction', 'evidence'), 'utf8')).toBe('retained lock evidence');
+ });
+
+ it('preserves an exclusive owner and its resume fence', async () => {
+ const { data, directory, admission } = fixture();
+ const hold = admission.begin({ reason: 'Exclusive work', owner: 'Operator' }).hold;
+ admission.claimReady({ id: hold.id, revision: hold.revision, operation: {
+ operationId: randomUUID(), requestId: randomUUID(), peerInstanceId: randomUUID(),
+ hostInstanceId: randomUUID(), grantId: randomUUID(), grantGeneration: 1, scope: 'execution-v1',
+ pairBinding: 'a'.repeat(64), intent: { action: 'portos.update' }, receiverVersion: '1.2.3', evidenceDigest: 'b'.repeat(64),
+ } }, admission.observeIdle());
+ const before = JSON.parse(fs.readFileSync(join(directory, 'state.json'), 'utf8'));
+ interrupt(directory);
+ await recoverMaintenanceTransaction(reason, { data, processes: stopped });
+ expect(JSON.parse(fs.readFileSync(join(directory, 'state.json'), 'utf8'))).toEqual(before);
+ const restarted = createMaintenanceAdmission(data);
+ expect(restarted.status()).toMatchObject({ state: 'draining', hold });
+ expect(() => restarted.resume({ id: hold.id, revision: hold.revision })).toThrow(/exclusive maintenance/);
+ });
+
+ it('keeps admission closed if recovery publication fails, and a confirmed retry preserves partial recovery', async () => {
+ const { data, directory, admission } = fixture();
+ interrupt(directory);
+ const io = { ...fs, renameSync: (source, destination) => {
+ if (destination === join(directory, 'state.json')) throw Object.assign(new Error('fixture disk full'), { code: 'ENOSPC' });
+ return fs.renameSync(source, destination);
+ } };
+ await expect(recoverMaintenanceTransaction(reason, { data, processes: stopped, io })).rejects.toThrow('fixture disk full');
+ expect(admission.status()).toMatchObject({ state: 'unavailable', cause: 'interrupted-transaction' });
+ const partial = fs.readdirSync(directory).find(name => name.startsWith('recovered-'));
+ expect(fs.readFileSync(join(directory, partial, 'pending-fixture.json'))).toHaveLength(0);
+ await recoverMaintenanceTransaction(reason, { data, processes: stopped });
+ expect(createMaintenanceAdmission(data).status()).toMatchObject({ state: 'ready', hold: { owner: 'Transaction recovery' } });
+ expect(fs.existsSync(join(directory, partial, 'receipt.json'))).toBe(true);
+ });
+});
diff --git a/server/lib/maintenanceAdmission.js b/server/lib/maintenanceAdmission.js
index 00378a6518..144b558852 100644
--- a/server/lib/maintenanceAdmission.js
+++ b/server/lib/maintenanceAdmission.js
@@ -14,7 +14,7 @@ const operationSchema = z.object({
pid: z.number().int().positive(), startedAt: z.string().datetime(), unsettled: z.boolean().optional(),
uncertaintyStamp: z.string().uuid().optional(),
}).strict();
-const schema = z.object({
+export const maintenanceJournalSchema = z.object({
version: z.literal(1), revision: z.number().int().nonnegative(),
hold: z.object({ id: z.string().uuid(), revision: z.number().int().positive(),
reason: z.string().min(1).max(500), owner: z.string().min(1).max(128), requestedAt: z.string().datetime(),
@@ -31,7 +31,7 @@ const error = (code = 'MAINTENANCE_HELD', message = 'Maintenance is holding new
Object.assign(new Error(message), { status: code === 'MAINTENANCE_STALE' ? 409 : 503, code });
export const isMaintenanceHold = err => ['MAINTENANCE_HELD', 'MAINTENANCE_UNAVAILABLE'].includes(err?.code);
-function syncDirectory(path, io) {
+export function syncMaintenanceDirectory(path, io = fs) {
if (process.platform === 'win32') return;
const fd = io.openSync(path, 'r');
try { io.fsyncSync(fd); } finally { io.closeSync(fd); }
@@ -47,7 +47,14 @@ export function createMaintenanceAdmission(dataDir = PATHS.data, { io = fs, asse
// A failed publication must not reopen admission in this process, even if
// an operator removes an interrupted transaction before restarting it.
let uncertain = false;
- const unavailable = () => error('MAINTENANCE_UNAVAILABLE', 'Maintenance state needs recovery; new work remains held.');
+ let failureReported = false;
+ const reportFailure = err => {
+ if (failureReported) return;
+ failureReported = true;
+ while (err.cause instanceof Error) err = err.cause;
+ console.error(`❌ Maintenance transaction failed (${err.code ?? 'UNKNOWN'}): ${String(err.message).replace(/\s+/g, ' ')}`);
+ };
+ const unavailable = cause => Object.assign(error('MAINTENANCE_UNAVAILABLE', 'Maintenance state needs recovery; new work remains held.'), cause ? { cause } : {});
const exists = path => {
try { io.lstatSync(path); return true; } catch (err) { if (err.code === 'ENOENT') return false; throw err; }
};
@@ -56,8 +63,8 @@ export function createMaintenanceAdmission(dataDir = PATHS.data, { io = fs, asse
try {
if (!exists(file)) return empty();
if (!io.lstatSync(file).isFile()) throw unavailable();
- return schema.parse(JSON.parse(io.readFileSync(file, 'utf8')));
- } catch { throw unavailable(); }
+ return maintenanceJournalSchema.parse(JSON.parse(io.readFileSync(file, 'utf8')));
+ } catch (err) { throw unavailable(err); }
};
// status()/held() are polled several times a second (media drain loop, spawn
// gates); re-parsing a journal that carries every settled operation allocated
@@ -70,7 +77,7 @@ export function createMaintenanceAdmission(dataDir = PATHS.data, { io = fs, asse
if (uncertain) throw unavailable();
let stats;
try { stats = io.lstatSync(file); }
- catch (err) { if (err.code === 'ENOENT') return empty(); throw unavailable(); }
+ catch (err) { if (err.code === 'ENOENT') return empty(); throw unavailable(err); }
if (!stats.isFile()) throw unavailable();
const key = `${stats.ino}:${stats.size}:${stats.mtimeMs}:${stats.ctimeMs}`;
if (statusCache?.key !== key) statusCache = { key, state: read() };
@@ -85,11 +92,19 @@ export function createMaintenanceAdmission(dataDir = PATHS.data, { io = fs, asse
scope: 'PortOS agents, mind turns, provider runs, media renders and scheduled shell/script work, through cleanup and saving. Unrelated host applications are outside this scope.',
});
const status = () => {
+ let cause = 'unreadable-journal';
+ let since;
try {
- if (exists(lock)) throw unavailable();
+ if (exists(lock)) {
+ cause = 'interrupted-transaction';
+ since = io.lstatSync(lock).mtime.toISOString();
+ throw unavailable();
+ }
+ if (uncertain) { cause = 'process-uncertain'; throw unavailable(); }
return project(readForStatus());
} catch {
return { state: 'unavailable', revision: null, hold: null, blockers: [],
+ cause, ...(since ? { since } : {}),
error: 'Maintenance state needs recovery. New work is held; readiness is unknown.' };
}
};
@@ -110,28 +125,29 @@ export function createMaintenanceAdmission(dataDir = PATHS.data, { io = fs, asse
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 5);
}
}
- } catch { throw unavailable(); }
+ } catch (err) { reportFailure(err); throw unavailable(); }
let published = false;
let result;
try {
const state = read();
result = change(state);
state.revision++;
- schema.parse(state);
+ maintenanceJournalSchema.parse(state);
const pending = join(directory, `pending-${makeId()}.json`);
const fd = io.openSync(pending, 'wx', 0o600);
try { io.writeFileSync(fd, JSON.stringify(state) + '\n'); io.fsyncSync(fd); } finally { io.closeSync(fd); }
io.renameSync(pending, file);
- syncDirectory(directory, io);
+ syncMaintenanceDirectory(directory, io);
published = true;
} catch (err) {
// Refused admission/stale requests didn't attempt a publication.
if (['MAINTENANCE_HELD', 'MAINTENANCE_STALE'].includes(err.code)) published = true;
- else uncertain = true;
+ else { uncertain = true; reportFailure(err); }
throw err.code?.startsWith('MAINTENANCE_') ? err : unavailable();
} finally {
if (published) {
- try { io.rmdirSync(lock); syncDirectory(directory, io); } catch { uncertain = true; }
+ try { io.rmdirSync(lock); syncMaintenanceDirectory(directory, io); }
+ catch (err) { uncertain = true; reportFailure(err); }
}
}
if (uncertain) throw unavailable();
diff --git a/server/lib/maintenanceAdmission.test.js b/server/lib/maintenanceAdmission.test.js
index a3ae761ae3..68acc04dc0 100644
--- a/server/lib/maintenanceAdmission.test.js
+++ b/server/lib/maintenanceAdmission.test.js
@@ -16,9 +16,47 @@ const gate = () => {
roots.push(root);
return { root, admission: createMaintenanceAdmission(root) };
};
-afterEach(() => { vi.useRealTimers(); for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); });
+afterEach(() => { vi.useRealTimers(); vi.restoreAllMocks(); for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); });
describe('durable graceful maintenance', () => {
+ it('names an interrupted transaction and its lock time without modifying evidence', () => {
+ const { root, admission } = gate();
+ const hold = admission.begin({ reason: 'Drain', owner: 'Operator' }).hold;
+ const lock = join(admission.directory, 'transaction');
+ mkdirSync(lock);
+ const since = fs.lstatSync(lock).mtime.toISOString();
+ expect(createMaintenanceAdmission(root).status()).toMatchObject({ state: 'unavailable', cause: 'interrupted-transaction', since });
+ expect(fs.lstatSync(lock).mtime.toISOString()).toBe(since);
+ rmSync(lock, { recursive: true });
+ expect(admission.status()).toMatchObject({ state: 'ready', hold });
+ });
+
+ it('logs a failed ENOSPC publication once and remains uncertain even after lock removal', () => {
+ const { root } = gate();
+ const log = vi.spyOn(console, 'error').mockImplementation(() => {});
+ const admission = createMaintenanceAdmission(root, { io: { ...fs, writeFileSync: () => {
+ throw Object.assign(new Error('fixture disk full'), { code: 'ENOSPC' });
+ } } });
+ expect(() => admission.admit('agent', 'fixture')).toThrow(/needs recovery/);
+ expect(admission.status()).toMatchObject({ state: 'unavailable', cause: 'interrupted-transaction' });
+ expect(admission.tryAdmit('agent', 'again')).toBeNull();
+ expect(log).toHaveBeenCalledExactlyOnceWith('❌ Maintenance transaction failed (ENOSPC): fixture disk full');
+ rmSync(join(admission.directory, 'transaction'), { recursive: true });
+ expect(admission.status()).toMatchObject({ state: 'unavailable', cause: 'process-uncertain' });
+ expect(admission.tryAdmit('agent', 'still held')).toBeNull();
+ });
+
+ it('logs the original lock acquisition failure once', () => {
+ const { root } = gate();
+ const log = vi.spyOn(console, 'error').mockImplementation(() => {});
+ const admission = createMaintenanceAdmission(root, { io: { ...fs, mkdirSync: () => {
+ throw Object.assign(new Error('fixture denied'), { code: 'EACCES' });
+ } } });
+ expect(admission.tryAdmit('agent', 'fixture')).toBeNull();
+ expect(admission.tryAdmit('agent', 'again')).toBeNull();
+ expect(log).toHaveBeenCalledExactlyOnceWith('❌ Maintenance transaction failed (EACCES): fixture denied');
+ });
+
it('binds operator publication reconciliation to the exact hold and owner and publishes before retirement', async () => {
const { root, admission } = gate();
const owner = admission.admit('settlement', 'Output publication');
@@ -156,7 +194,7 @@ describe('durable graceful maintenance', () => {
fs.mkdirSync(join(admission.directory, 'transaction'));
expect(() => legacyOperation.parse(JSON.parse(before).operations[0])).toThrow();
const restarted = createMaintenanceAdmission(root);
- expect(restarted.status().state).toBe('unavailable');
+ expect(restarted.status()).toMatchObject({ state: 'unavailable', cause: 'interrupted-transaction' });
expect(restarted.tryAdmit('media', 'new')).toBeNull();
expect(() => recovered.completeRecovery()).toThrow(/needs recovery/);
expect(fs.existsSync(join(admission.directory, 'transaction'))).toBe(true);
@@ -316,7 +354,7 @@ describe('durable graceful maintenance', () => {
expect(admission.status()).toMatchObject({ state: 'draining', blockers: [{ unsettled: true }] });
writeFileSync(join(admission.directory, 'state.json'), '{"version":999}');
const restarted = createMaintenanceAdmission(root);
- expect(restarted.status().state).toBe('unavailable');
+ expect(restarted.status()).toMatchObject({ state: 'unavailable', cause: 'unreadable-journal' });
expect(restarted.tryAdmit('agent', 'new')).toBeNull();
expect(() => restarted.resume({ id: '00000000-0000-4000-8000-000000000000', revision: 1 })).toThrow();
});
diff --git a/server/services/backup.test.js b/server/services/backup.test.js
index 7a06e219e0..8381aeade1 100644
--- a/server/services/backup.test.js
+++ b/server/services/backup.test.js
@@ -3190,7 +3190,7 @@ describe('restoreSnapshot manifest verification', () => {
});
});
- it.for(['peer-execution-grants.json', 'peer-execution-catalog.json', 'peer-execution/fixture-operation/evidence.json', 'peer-execution-authority.json', 'peer-execution-recovery.jsonl', 'workflow-maintenance/state.json', 'Workflow-Maintenance/state.json', 'backup-admission/cut/owner.json', 'Backup-Admission/publications/owner.json'])
+ it.for(['peer-execution-grants.json', 'peer-execution-catalog.json', 'peer-execution/fixture-operation/evidence.json', 'peer-execution-authority.json', 'peer-execution-recovery.jsonl', 'workflow-maintenance/state.json', 'workflow-maintenance/recovered-fixture/receipt.json', 'Workflow-Maintenance/state.json', 'backup-admission/cut/owner.json', 'Backup-Admission/publications/owner.json'])
('preserves non-rewound execution authority and ownership at %s (#10127)', async (path, context) => {
await withRealRsync(context, async () => {
const localPath = joinPath(PATHS.data, path);