Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: recover interrupted maintenance transactions without reopening work (#13332)
Adam Eivy6f984639d281
diff --git a/client/src/components/MaintenanceBanner.jsx b/client/src/components/MaintenanceBanner.jsxindex f3b9acbe19..35a04b9405 100644--- a/client/src/components/MaintenanceBanner.jsx+++ b/client/src/components/MaintenanceBanner.jsx@@ -6,6 +6,7 @@ export default function MaintenanceBanner() {if (!status || status.state === 'normal') return null;const label = status.state === 'ready' ? 'Maintenance ready — new work is held.': status.state === 'draining' ? `Maintenance draining — ${formatCount(status.blockerCount)} operation(s) finishing.`+ : status.state === 'unavailable' ? 'Maintenance recovery needed — new work is held.': 'Maintenance readiness unknown.';return <div role="status" className="px-4 py-2 text-sm bg-port-card border-b border-port-border">{label} <Link className="underline" to="/settings/general">View maintenance</Link></div>;}diff --git a/client/src/components/MaintenanceBanner.test.jsx b/client/src/components/MaintenanceBanner.test.jsxindex 1c3117032f..aa3f675f34 100644--- a/client/src/components/MaintenanceBanner.test.jsx+++ b/client/src/components/MaintenanceBanner.test.jsx@@ -18,6 +18,7 @@ describe('MaintenanceBanner', () => {expect(container).toBeEmptyDOMElement();unmount();show({ state: 'unavailable', blockerCount: 0 });- expect(screen.getByRole('status')).toHaveTextContent('readiness unknown');+ expect(screen.getByRole('status')).toHaveTextContent('recovery needed');+ expect(screen.getByRole('link', { name: 'View maintenance' })).toHaveAttribute('href', '/settings/general');});});diff --git a/client/src/components/settings/MaintenancePanel.jsx b/client/src/components/settings/MaintenancePanel.jsxindex 04aa3b4628..31dbd69c27 100644--- a/client/src/components/settings/MaintenancePanel.jsx+++ b/client/src/components/settings/MaintenancePanel.jsx@@ -3,6 +3,18 @@ import { formatDateTime, formatTimeOfDay } from '../../utils/formatters.js';import { useMaintenance } from '../../hooks/useMaintenance.js';import useMounted from '../../hooks/useMounted.js';import { beginMaintenance, resumeMaintenance } from '../../services/apiSystem.js';+import { copyToClipboard } from '../../lib/clipboard.js';++const recoveryCommands = [+ 'pm2 stop portos-server portos-cos',+ 'node scripts/recover-maintenance-transaction.mjs --confirm "Reviewed interrupted maintenance transaction"',+ 'pm2 start portos-server portos-cos',+].join('\n');+const recoveryCauses = {+ 'interrupted-transaction': 'An interrupted journal transaction is holding new work.',+ 'unreadable-journal': 'The maintenance journal is unreadable or incompatible. Restore compatible code or repair the journal before recovery; the command refuses invalid state.',+ 'process-uncertain': 'This process could not confirm a journal write. Both PortOS processes must restart after recovery.',+};export default function MaintenancePanel() {const { status, refresh } = useMaintenance();@@ -23,11 +35,19 @@ export default function MaintenancePanel() {<h3 id="maintenance-heading" className="font-semibold text-port-text">Graceful maintenance</h3><p role="status" className="text-sm">{status ? ({ normal: 'Normal — new work can start', draining: 'Draining — letting active work finish', ready: 'Ready — maintenance hold is active', unavailable: 'Recovery needed — new work is held', unknown: 'Readiness unknown' }[status.state]) : 'Loading maintenance status…'}</p><p className="text-sm text-port-text-muted">New agents, mind turns and renders wait. Active work finishes, including saving and cleanup. Queues and existing scheduling policies are preserved.</p>- {status?.hold ? <p className="text-sm">{status.hold.reason} · {status.hold.owner} · {formatDateTime(status.hold.requestedAt)}</p> : <label htmlFor="maintenance-reason" className="block text-sm">Reason<input id="maintenance-reason" className="block w-full mt-1 p-2 rounded bg-port-bg border border-port-border" value={reason} maxLength={500} onChange={e => setReason(e.target.value)} disabled={busy || unknown} /></label>}+ {status?.cause && <div className="rounded border border-port-border p-3 space-y-2 text-sm">+ <p>{recoveryCauses[status.cause] || 'The maintenance journal needs recovery.'}</p>+ {status.since && <p>Held since {formatDateTime(status.since)}</p>}+ <p>Inspect the server logs for the original error and resolve disk space or I/O problems first. From the PortOS install directory, stop both PM2 apps and any standalone journal writers, run this command with your recovery reason, then start both apps:</p>+ <pre className="whitespace-pre-wrap break-words rounded bg-port-bg p-3">{recoveryCommands}</pre>+ <button className="underline" onClick={() => copyToClipboard(recoveryCommands, 'Recovery steps copied')}>Copy recovery steps</button>+ <p>The command preserves transaction evidence and keeps a maintenance hold. Inspect any remaining blockers after restarting, then use Resume previous policies when ready. Never delete the journal or lock by hand.</p>+ </div>}+ {status?.hold ? <p className="text-sm">{status.hold.reason} · {status.hold.owner} · {formatDateTime(status.hold.requestedAt)}</p> : !status?.cause && <label htmlFor="maintenance-reason" className="block text-sm">Reason<input id="maintenance-reason" className="block w-full mt-1 p-2 rounded bg-port-bg border border-port-border" value={reason} maxLength={500} onChange={e => setReason(e.target.value)} disabled={busy || unknown} /></label>}{status?.blockers?.length > 0 && <ul className="text-sm space-y-1 max-h-56 overflow-auto" aria-label="Maintenance blockers">{status.blockers.map((b, i) => <li key={`${b.kind}:${b.resource}:${i}`}>{b.kind}: {b.resource || 'Preparing'}{b.unsettled ? ' — saving or cleanup needs recovery' : ''} · since {formatTimeOfDay(b.startedAt)}</li>)}</ul>}<p className="text-xs text-port-text-muted">{status?.scope || 'Readiness covers admitted PortOS automation and media work. Unrelated applications on this computer are outside this scope.'} A restart never clears a hold or unresolved work. Resuming cancels this hold; it does not certify readiness.</p>{(error || status?.error) && <p role="alert" className="text-sm text-port-error">{error || status.error}</p>}- <button className="px-4 py-2 rounded bg-port-accent text-white disabled:opacity-50" disabled={busy || unknown || (!status?.hold && !reason.trim())} onClick={act}>{busy ? 'Saving…' : status?.hold ? 'Resume previous policies' : 'Enter maintenance'}</button>+ {!status?.cause && <button className="px-4 py-2 rounded bg-port-accent text-white disabled:opacity-50" disabled={busy || unknown || (!status?.hold && !reason.trim())} onClick={act}>{busy ? 'Saving…' : status?.hold ? 'Resume previous policies' : 'Enter maintenance'}</button>}{unknown && <button className="ml-3 text-sm underline" onClick={refresh}>Refresh status</button>}</section>;}diff --git a/client/src/components/settings/MaintenancePanel.test.jsx b/client/src/components/settings/MaintenancePanel.test.jsxindex 16d92171ee..d1a784ff2b 100644--- a/client/src/components/settings/MaintenancePanel.test.jsx+++ b/client/src/components/settings/MaintenancePanel.test.jsx@@ -3,6 +3,8 @@ import { fireEvent, render, screen, waitFor } from '@testing-library/react';const fixture = vi.hoisted(() => ({ status: null, refresh: vi.fn(), begin: vi.fn(), resume: vi.fn() }));vi.mock('../../hooks/useMaintenance.js', () => ({ useMaintenance: () => fixture }));vi.mock('../../services/apiSystem.js', () => ({ beginMaintenance: fixture.begin, resumeMaintenance: fixture.resume }));+vi.mock('../../lib/clipboard.js', () => ({ copyToClipboard: vi.fn() }));+import { copyToClipboard } from '../../lib/clipboard.js';import MaintenancePanel from './MaintenancePanel.jsx';beforeEach(() => {@@ -40,4 +42,25 @@ describe('maintenance controls', () => {expect(screen.getByRole('status')).toHaveTextContent('Readiness unknown');expect(fixture.begin).not.toHaveBeenCalled();});++ it('shows copyable offline recovery steps and the lock time instead of a disabled maintenance action', () => {+ fixture.status = { state: 'unavailable', cause: 'interrupted-transaction', since: '2026-10-04T12:00:00.000Z' };+ render(<MaintenancePanel />);+ expect(screen.getByText(/interrupted journal transaction/)).toBeInTheDocument();+ expect(screen.getByText(/Held since/)).toBeInTheDocument();+ expect(screen.queryByRole('button', { name: 'Enter maintenance' })).not.toBeInTheDocument();+ expect(screen.queryByLabelText('Reason')).not.toBeInTheDocument();+ fireEvent.click(screen.getByRole('button', { name: 'Copy recovery steps' }));+ expect(copyToClipboard).toHaveBeenCalledWith(expect.stringMatching(/pm2 stop portos-server portos-cos\nnode scripts\/recover-maintenance-transaction\.mjs --confirm .+\npm2 start portos-server portos-cos/), 'Recovery steps copied');+ expect(screen.getByRole('button', { name: 'Refresh status' })).toBeEnabled();+ expect(fixture.begin).not.toHaveBeenCalled();+ });++ it.each(['unreadable-journal', 'process-uncertain'])('explains %s recovery without offering mutations', cause => {+ fixture.status = { state: 'unavailable', cause };+ render(<MaintenancePanel />);+ expect(screen.getByRole('button', { name: 'Copy recovery steps' })).toBeEnabled();+ expect(screen.queryByRole('button', { name: 'Enter maintenance' })).not.toBeInTheDocument();+ expect(screen.getByText(cause === 'unreadable-journal' ? /command refuses invalid state/ : /could not confirm a journal write/)).toBeInTheDocument();+ });});diff --git a/docs/STORAGE.md b/docs/STORAGE.mdindex 78138c951b..5428cbd9f9 100644--- a/docs/STORAGE.md+++ b/docs/STORAGE.md@@ -69,6 +69,47 @@ retires that exact owner under one lock. Missing ownership can only replay theexisting receipt, never create an owner. This does not certify a completetranscript, clear transcript warnings, or change the original task outcome.+### Interrupted maintenance transactions++An interrupted journal write stays fail-closed across restarts. Settings → General+→ Graceful maintenance reports `interrupted-transaction` and the lock time,+`unreadable-journal` for damaged/incompatible state, or `process-uncertain` when+this process could not confirm a write. The first transaction failure logs its+original error code and message; repeated refusals do not flood that diagnostic.++Inspect the logs and resolve disk-space or I/O problems first. From the install+directory, stop both PM2 apps and any standalone processes using this journal:++```bash+pm2 stop portos-server portos-cos+node scripts/recover-maintenance-transaction.mjs --confirm "Inspected interrupted transaction and resolved disk problem"+pm2 start portos-server portos-cos+```++Use your actual recovery reason. From a reviewed isolated checkout, set+`PORTOS_DATA_ROOT` to the install root (the parent of `data/`). The command refuses+live/transitioning PM2 ownership, an unreadable PM2 inventory, an invalid/future+`state.json`, or linked/non-regular journal evidence. Restore compatible code or+recover damaged journal data separately before retrying; never delete valid+operations or replace damaged state with an empty journal to bypass this check.++Recovery preserves the original state, `transaction/`, every `pending-*.json`,+and a receipt recording the operator reason, lock mtime, pending sizes, original+state hash and retained hold under `data/workflow-maintenance/recovered-<timestamp>-<id>/`.+It publishes a **Transaction recovery** hold before moving the lock when no hold+exists; an existing hold and every operation/exclusive owner remain intact.+A failed repair retains admission fencing and evidence; after addressing its+cause, repeat the confirmed command. There is no automatic expiry or PID-based+lock removal. Starting both apps clears their in-memory uncertainty, but does+not clear the hold or certify that unresolved work finished.++After restarting, refresh maintenance status, inspect/reconcile any remaining+blockers through their owning workflows, and choose **Resume previous policies**+deliberately. With no unresolved operations the status is Ready; otherwise it is+Draining. Recovery archives stay in the same backed-up, machine-local subtree:+snapshots retain them as evidence and restores preserve the current subtree+byte-for-byte, including archives, rather than installing a snapshot's authority.+### Explicitly abandoned duplicate agentsA local operator may reconcile a user-killed duplicate that saved no output withdiff --git a/docs/decisions/2026-10-04-music-video-render-pools.md b/docs/decisions/2026-10-04-music-video-render-pools.mdindex 0b2bfb704c..94478a4765 100644--- a/docs/decisions/2026-10-04-music-video-render-pools.md+++ b/docs/decisions/2026-10-04-music-video-render-pools.md@@ -80,8 +80,10 @@ rollback therefore require an explicit compatibility plan before deployment:prove that existing work has settled or authorize resuming admission.- Do not roll back to a strict older binary against a stamped journal. Preservethe journal and any transaction lock if maintenance becomes unavailable;- restore compatible code and obtain a separately reviewed recovery procedure.- This feature supplies no automatic lock repair or rollback migration.+ restore compatible code, then use the explicitly confirmed offline+ [transaction recovery procedure](../STORAGE.md#interrupted-maintenance-transactions).+ It archives the lock and pending evidence and retains a hold and unresolved+ ownership. There is no automatic lock repair or rollback migration.- Never manually delete uncertainty, operation records or a transaction lock,or infer completion from process age/PID. Legacy unsettled records without astamp remain blockers; they cannot use the new recovery completion capability.diff --git a/scripts/recover-maintenance-transaction.mjs b/scripts/recover-maintenance-transaction.mjsnew file mode 100644index 0000000000..c6a6d6baf1--- /dev/null+++ b/scripts/recover-maintenance-transaction.mjs@@ -0,0 +1,95 @@+#!/usr/bin/env node+import * as fs from 'node:fs';+import { createHash, randomUUID } from 'node:crypto';+import { join, resolve } from 'node:path';+import { fileURLToPath } from 'node:url';+import { maintenanceBeginSchema, maintenanceJournalSchema, syncMaintenanceDirectory } from '../server/lib/maintenanceAdmission.js';+import { PATHS } from '../server/lib/paths.js';+import { assertNotRealDataWrite } from '../server/lib/testDataIsolation.js';++const listProcesses = () => import('../server/services/pm2.js').then(pm2 => pm2.listMaintenanceProcesses());+const refuse = message => { throw new Error(message); };++/** Offline operator disposition of a transaction, never settlement of its work. */+export async function recoverMaintenanceTransaction(reason, { data = PATHS.data, io = fs, processes = listProcesses } = {}) {+ const { reason: confirmedReason } = maintenanceBeginSchema.parse({ reason });+ const offline = async () => {+ const inventory = await processes();+ if (!Array.isArray(inventory) || inventory.some(proc => !proc || typeof proc.name !== 'string' || typeof proc.status !== 'string'))+ refuse('PM2 process state is unreadable; recovery refused.');+ if (inventory.some(proc => ['portos-server', 'portos-cos'].includes(proc.name)+ && (!['stopped', 'errored'].includes(proc.status) || proc.pid !== 0)))+ refuse('Stop portos-server and portos-cos in PM2 before recovery.');+ };+ await offline();+ const directory = join(data, 'workflow-maintenance');+ const file = join(directory, 'state.json');+ const lock = join(directory, 'transaction');+ const stat = path => {+ try { return io.lstatSync(path); }+ catch (err) { if (err.code === 'ENOENT') return null; throw err; }+ };+ const dirStat = stat(directory);+ if (dirStat && !dirStat.isDirectory()) refuse('The maintenance journal must be a real directory.');+ const originalStat = stat(file);+ if (originalStat && !originalStat.isFile()) refuse('state.json must be a regular file, not a link or directory.');+ const original = originalStat ? io.readFileSync(file) : null;+ // Use exactly the runtime schema; future or damaged journals require compatible+ // code/data recovery, never replacement with an empty journal.+ const state = original ? maintenanceJournalSchema.parse(JSON.parse(original.toString('utf8')))+ : { version: 1, revision: 0, hold: null, operations: [] };+ const lockStat = stat(lock);+ if (lockStat && !lockStat.isDirectory()) refuse('The transaction lock must be a real directory.');+ const pending = dirStat ? io.readdirSync(directory).filter(name => /^pending-.*\.json$/.test(name)).sort().map(name => {+ const stats = stat(join(directory, name));+ if (!stats?.isFile()) refuse('Pending journal evidence must be regular files.');+ return { name, size: stats.size };+ }) : [];+ await offline();+ const identity = stats => stats && [stats.dev, stats.ino, stats.size, stats.mtimeMs, stats.ctimeMs].join(':');+ const current = stat(file);+ if (identity(current) !== identity(originalStat) || identity(stat(lock)) !== identity(lockStat)+ || (original && !io.readFileSync(file).equals(original)))+ refuse('The maintenance journal changed during inspection; inspect again before recovery.');+ assertNotRealDataWrite(directory, 'maintenance transaction recovery');+ io.mkdirSync(directory, { recursive: true, mode: 0o700 });+ if (!lockStat) io.mkdirSync(lock, { mode: 0o700 });+ const at = new Date().toISOString();+ const archiveDir = join(directory, `recovered-${at.replace(/:/g, '-')}-${randomUUID()}`);+ io.mkdirSync(archiveDir, { mode: 0o700 });+ const durableWrite = (path, contents) => {+ const fd = io.openSync(path, 'wx', 0o600);+ try { io.writeFileSync(fd, contents); io.fsyncSync(fd); } finally { io.closeSync(fd); }+ };+ if (original) durableWrite(join(archiveDir, 'state.json'), original);+ if (!state.hold) {+ state.revision++;+ state.hold = { id: randomUUID(), revision: state.revision, owner: 'Transaction recovery',+ reason: confirmedReason, requestedAt: at };+ }+ maintenanceJournalSchema.parse(state);+ const receipt = { version: 1, reason: confirmedReason, at, lockMtime: lockStat?.mtime.toISOString() ?? null,+ pending, originalStateSha256: original ? createHash('sha256').update(original).digest('hex') : null,+ hold: state.hold };+ durableWrite(join(archiveDir, 'receipt.json'), `${JSON.stringify(receipt)}\n`);+ for (const { name } of pending) io.renameSync(join(directory, name), join(archiveDir, name));+ syncMaintenanceDirectory(archiveDir, io);+ // Publish and fsync a hold BEFORE moving the admission lock. A crash at any+ // step retains either the original fence or the deliberate recovery hold.+ const staged = join(directory, `pending-${randomUUID()}.json`);+ durableWrite(staged, `${JSON.stringify(state)}\n`);+ io.renameSync(staged, file);+ syncMaintenanceDirectory(directory, io);+ io.renameSync(lock, join(archiveDir, 'transaction'));+ syncMaintenanceDirectory(archiveDir, io);+ syncMaintenanceDirectory(directory, io);+ return { archiveDir, hold: state.hold, pendingCount: pending.length };+}++if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1])) {+ try {+ const [confirmation, ...reason] = process.argv.slice(2);+ if (confirmation !== '--confirm') throw new Error('Usage: node scripts/recover-maintenance-transaction.mjs --confirm <reason>');+ console.log(JSON.stringify(await recoverMaintenanceTransaction(reason.join(' '))));+ } catch (err) { console.error(`❌ Maintenance recovery refused: ${err.message}`); process.exitCode = 1; }+}diff --git a/scripts/recover-maintenance-transaction.test.js b/scripts/recover-maintenance-transaction.test.jsnew file mode 100644index 0000000000..f14dc109bc--- /dev/null+++ b/scripts/recover-maintenance-transaction.test.js@@ -0,0 +1,117 @@+import { afterEach, describe, expect, it } from 'vitest';+import * as fs from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { randomUUID } from 'node:crypto';+import { recoverMaintenanceTransaction } from './recover-maintenance-transaction.mjs';+import { createMaintenanceAdmission } from '../server/lib/maintenanceAdmission.js';++const roots = [];+const reason = 'Inspected interrupted transaction evidence';+const stopped = async () => ['portos-server', 'portos-cos'].map(name => ({ name, status: 'stopped', pid: 0 }));+function fixture() {+ const data = fs.mkdtempSync(join(tmpdir(), 'maintenance-recovery-'));+ roots.push(data);+ const admission = createMaintenanceAdmission(data);+ fs.mkdirSync(admission.directory, { recursive: true });+ return { data, admission, directory: admission.directory };+}+const interrupt = directory => {+ fs.mkdirSync(join(directory, 'transaction'));+ fs.writeFileSync(join(directory, 'transaction', 'evidence'), 'retained lock evidence');+ fs.writeFileSync(join(directory, 'pending-fixture.json'), '');+ fs.writeFileSync(join(directory, 'pending-other.json'), '{"partial":true}');+};+afterEach(() => { for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); });++describe('offline maintenance transaction recovery', () => {+ it('preserves interrupted bytes and the original state, then holds admission until deliberate resume', async () => {+ const { data, directory, admission } = fixture();+ // A previously published journal must be preserved byte-for-byte as evidence.+ const work = admission.admit('agent', 'fixture');+ await work.finish();+ const before = fs.readFileSync(join(directory, 'state.json'));+ interrupt(directory);+ const since = fs.lstatSync(join(directory, 'transaction')).mtime.toISOString();+ const result = await recoverMaintenanceTransaction(reason, { data, processes: stopped });+ expect(fs.readFileSync(join(result.archiveDir, 'state.json'))).toEqual(before);+ expect(fs.readFileSync(join(result.archiveDir, 'transaction', 'evidence'), 'utf8')).toBe('retained lock evidence');+ expect(fs.readFileSync(join(result.archiveDir, 'pending-fixture.json'))).toHaveLength(0);+ expect(fs.readFileSync(join(result.archiveDir, 'pending-other.json'), 'utf8')).toBe('{"partial":true}');+ expect(JSON.parse(fs.readFileSync(join(result.archiveDir, 'receipt.json'), 'utf8'))).toMatchObject({+ reason, lockMtime: since, pending: [{ name: 'pending-fixture.json', size: 0 }, { name: 'pending-other.json', size: 16 }],+ hold: { owner: 'Transaction recovery' },+ });+ expect(fs.readdirSync(directory).sort()).toEqual([result.archiveDir.split(/[\\/]/).at(-1), 'state.json'].sort());+ const restarted = createMaintenanceAdmission(data);+ expect(restarted.status()).toMatchObject({ state: 'ready', hold: result.hold });+ expect(restarted.tryAdmit('agent', 'new')).toBeNull();+ expect(restarted.resume({ id: result.hold.id, revision: result.hold.revision }).state).toBe('normal');+ });++ it('recovers an absent state conservatively and preserves existing holds and unresolved operations', async () => {+ const absent = fixture();+ interrupt(absent.directory);+ await recoverMaintenanceTransaction(reason, { data: absent.data, processes: stopped });+ expect(createMaintenanceAdmission(absent.data).status()).toMatchObject({ state: 'ready', hold: { owner: 'Transaction recovery' } });++ const { data, directory, admission } = fixture();+ const work = admission.admit('settlement', 'Unfinished save');+ work.markUnsettled();+ const hold = admission.begin({ reason: 'Existing hold', owner: 'Operator' }).hold;+ const before = JSON.parse(fs.readFileSync(join(directory, 'state.json'), 'utf8'));+ interrupt(directory);+ await recoverMaintenanceTransaction(reason, { data, processes: stopped });+ expect(JSON.parse(fs.readFileSync(join(directory, 'state.json'), 'utf8'))).toEqual(before);+ expect(createMaintenanceAdmission(data).status()).toMatchObject({ state: 'draining', hold, blockers: [{ unsettled: true }] });+ });++ it('refuses live, transitioning or unreadable PM2 ownership and invalid journals without changing evidence', async () => {+ const { data, directory } = fixture();+ interrupt(directory);+ for (const inventory of [null, [{}], [{ name: 'portos-server', status: 'online', pid: 123 }],+ [{ name: 'portos-cos', status: 'launching', pid: 0 }], [{ name: 'portos-cos', status: 'stopped', pid: 123 }]]) {+ await expect(recoverMaintenanceTransaction(reason, { data, processes: async () => inventory })).rejects.toThrow(/PM2/);+ }+ for (const text of ['', '{bad json', '{"version":999}']) {+ fs.writeFileSync(join(directory, 'state.json'), text);+ await expect(recoverMaintenanceTransaction(reason, { data, processes: stopped })).rejects.toThrow();+ expect(fs.readFileSync(join(directory, 'state.json'), 'utf8')).toBe(text);+ }+ expect(fs.readdirSync(directory).sort()).toEqual(['pending-fixture.json', 'pending-other.json', 'state.json', 'transaction']);+ expect(fs.readFileSync(join(directory, 'transaction', 'evidence'), 'utf8')).toBe('retained lock evidence');+ });++ it('preserves an exclusive owner and its resume fence', async () => {+ const { data, directory, admission } = fixture();+ const hold = admission.begin({ reason: 'Exclusive work', owner: 'Operator' }).hold;+ admission.claimReady({ id: hold.id, revision: hold.revision, operation: {+ operationId: randomUUID(), requestId: randomUUID(), peerInstanceId: randomUUID(),+ hostInstanceId: randomUUID(), grantId: randomUUID(), grantGeneration: 1, scope: 'execution-v1',+ pairBinding: 'a'.repeat(64), intent: { action: 'portos.update' }, receiverVersion: '1.2.3', evidenceDigest: 'b'.repeat(64),+ } }, admission.observeIdle());+ const before = JSON.parse(fs.readFileSync(join(directory, 'state.json'), 'utf8'));+ interrupt(directory);+ await recoverMaintenanceTransaction(reason, { data, processes: stopped });+ expect(JSON.parse(fs.readFileSync(join(directory, 'state.json'), 'utf8'))).toEqual(before);+ const restarted = createMaintenanceAdmission(data);+ expect(restarted.status()).toMatchObject({ state: 'draining', hold });+ expect(() => restarted.resume({ id: hold.id, revision: hold.revision })).toThrow(/exclusive maintenance/);+ });++ it('keeps admission closed if recovery publication fails, and a confirmed retry preserves partial recovery', async () => {+ const { data, directory, admission } = fixture();+ interrupt(directory);+ const io = { ...fs, renameSync: (source, destination) => {+ if (destination === join(directory, 'state.json')) throw Object.assign(new Error('fixture disk full'), { code: 'ENOSPC' });+ return fs.renameSync(source, destination);+ } };+ await expect(recoverMaintenanceTransaction(reason, { data, processes: stopped, io })).rejects.toThrow('fixture disk full');+ expect(admission.status()).toMatchObject({ state: 'unavailable', cause: 'interrupted-transaction' });+ const partial = fs.readdirSync(directory).find(name => name.startsWith('recovered-'));+ expect(fs.readFileSync(join(directory, partial, 'pending-fixture.json'))).toHaveLength(0);+ await recoverMaintenanceTransaction(reason, { data, processes: stopped });+ expect(createMaintenanceAdmission(data).status()).toMatchObject({ state: 'ready', hold: { owner: 'Transaction recovery' } });+ expect(fs.existsSync(join(directory, partial, 'receipt.json'))).toBe(true);+ });+});diff --git a/server/lib/maintenanceAdmission.js b/server/lib/maintenanceAdmission.jsindex 00378a6518..144b558852 100644--- a/server/lib/maintenanceAdmission.js+++ b/server/lib/maintenanceAdmission.js@@ -14,7 +14,7 @@ const operationSchema = z.object({pid: z.number().int().positive(), startedAt: z.string().datetime(), unsettled: z.boolean().optional(),uncertaintyStamp: z.string().uuid().optional(),}).strict();-const schema = z.object({+export const maintenanceJournalSchema = z.object({version: z.literal(1), revision: z.number().int().nonnegative(),hold: z.object({ id: z.string().uuid(), revision: z.number().int().positive(),reason: z.string().min(1).max(500), owner: z.string().min(1).max(128), requestedAt: z.string().datetime(),@@ -31,7 +31,7 @@ const error = (code = 'MAINTENANCE_HELD', message = 'Maintenance is holding newObject.assign(new Error(message), { status: code === 'MAINTENANCE_STALE' ? 409 : 503, code });export const isMaintenanceHold = err => ['MAINTENANCE_HELD', 'MAINTENANCE_UNAVAILABLE'].includes(err?.code);-function syncDirectory(path, io) {+export function syncMaintenanceDirectory(path, io = fs) {if (process.platform === 'win32') return;const fd = io.openSync(path, 'r');try { io.fsyncSync(fd); } finally { io.closeSync(fd); }@@ -47,7 +47,14 @@ export function createMaintenanceAdmission(dataDir = PATHS.data, { io = fs, asse// A failed publication must not reopen admission in this process, even if// an operator removes an interrupted transaction before restarting it.let uncertain = false;- const unavailable = () => error('MAINTENANCE_UNAVAILABLE', 'Maintenance state needs recovery; new work remains held.');+ let failureReported = false;+ const reportFailure = err => {+ if (failureReported) return;+ failureReported = true;+ while (err.cause instanceof Error) err = err.cause;+ console.error(`❌ Maintenance transaction failed (${err.code ?? 'UNKNOWN'}): ${String(err.message).replace(/\s+/g, ' ')}`);+ };+ const unavailable = cause => Object.assign(error('MAINTENANCE_UNAVAILABLE', 'Maintenance state needs recovery; new work remains held.'), cause ? { cause } : {});const exists = path => {try { io.lstatSync(path); return true; } catch (err) { if (err.code === 'ENOENT') return false; throw err; }};@@ -56,8 +63,8 @@ export function createMaintenanceAdmission(dataDir = PATHS.data, { io = fs, assetry {if (!exists(file)) return empty();if (!io.lstatSync(file).isFile()) throw unavailable();- return schema.parse(JSON.parse(io.readFileSync(file, 'utf8')));- } catch { throw unavailable(); }+ return maintenanceJournalSchema.parse(JSON.parse(io.readFileSync(file, 'utf8')));+ } catch (err) { throw unavailable(err); }};// status()/held() are polled several times a second (media drain loop, spawn// gates); re-parsing a journal that carries every settled operation allocated@@ -70,7 +77,7 @@ export function createMaintenanceAdmission(dataDir = PATHS.data, { io = fs, asseif (uncertain) throw unavailable();let stats;try { stats = io.lstatSync(file); }- catch (err) { if (err.code === 'ENOENT') return empty(); throw unavailable(); }+ catch (err) { if (err.code === 'ENOENT') return empty(); throw unavailable(err); }if (!stats.isFile()) throw unavailable();const key = `${stats.ino}:${stats.size}:${stats.mtimeMs}:${stats.ctimeMs}`;if (statusCache?.key !== key) statusCache = { key, state: read() };@@ -85,11 +92,19 @@ export function createMaintenanceAdmission(dataDir = PATHS.data, { io = fs, assescope: 'PortOS agents, mind turns, provider runs, media renders and scheduled shell/script work, through cleanup and saving. Unrelated host applications are outside this scope.',});const status = () => {+ let cause = 'unreadable-journal';+ let since;try {- if (exists(lock)) throw unavailable();+ if (exists(lock)) {+ cause = 'interrupted-transaction';+ since = io.lstatSync(lock).mtime.toISOString();+ throw unavailable();+ }+ if (uncertain) { cause = 'process-uncertain'; throw unavailable(); }return project(readForStatus());} catch {return { state: 'unavailable', revision: null, hold: null, blockers: [],+ cause, ...(since ? { since } : {}),error: 'Maintenance state needs recovery. New work is held; readiness is unknown.' };}};@@ -110,28 +125,29 @@ export function createMaintenanceAdmission(dataDir = PATHS.data, { io = fs, asseAtomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 5);}}- } catch { throw unavailable(); }+ } catch (err) { reportFailure(err); throw unavailable(); }let published = false;let result;try {const state = read();result = change(state);state.revision++;- schema.parse(state);+ maintenanceJournalSchema.parse(state);const pending = join(directory, `pending-${makeId()}.json`);const fd = io.openSync(pending, 'wx', 0o600);try { io.writeFileSync(fd, JSON.stringify(state) + '\n'); io.fsyncSync(fd); } finally { io.closeSync(fd); }io.renameSync(pending, file);- syncDirectory(directory, io);+ syncMaintenanceDirectory(directory, io);published = true;} catch (err) {// Refused admission/stale requests didn't attempt a publication.if (['MAINTENANCE_HELD', 'MAINTENANCE_STALE'].includes(err.code)) published = true;- else uncertain = true;+ else { uncertain = true; reportFailure(err); }throw err.code?.startsWith('MAINTENANCE_') ? err : unavailable();} finally {if (published) {- try { io.rmdirSync(lock); syncDirectory(directory, io); } catch { uncertain = true; }+ try { io.rmdirSync(lock); syncMaintenanceDirectory(directory, io); }+ catch (err) { uncertain = true; reportFailure(err); }}}if (uncertain) throw unavailable();diff --git a/server/lib/maintenanceAdmission.test.js b/server/lib/maintenanceAdmission.test.jsindex a3ae761ae3..68acc04dc0 100644--- a/server/lib/maintenanceAdmission.test.js+++ b/server/lib/maintenanceAdmission.test.js@@ -16,9 +16,47 @@ const gate = () => {roots.push(root);return { root, admission: createMaintenanceAdmission(root) };};-afterEach(() => { vi.useRealTimers(); for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); });+afterEach(() => { vi.useRealTimers(); vi.restoreAllMocks(); for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); });describe('durable graceful maintenance', () => {+ it('names an interrupted transaction and its lock time without modifying evidence', () => {+ const { root, admission } = gate();+ const hold = admission.begin({ reason: 'Drain', owner: 'Operator' }).hold;+ const lock = join(admission.directory, 'transaction');+ mkdirSync(lock);+ const since = fs.lstatSync(lock).mtime.toISOString();+ expect(createMaintenanceAdmission(root).status()).toMatchObject({ state: 'unavailable', cause: 'interrupted-transaction', since });+ expect(fs.lstatSync(lock).mtime.toISOString()).toBe(since);+ rmSync(lock, { recursive: true });+ expect(admission.status()).toMatchObject({ state: 'ready', hold });+ });++ it('logs a failed ENOSPC publication once and remains uncertain even after lock removal', () => {+ const { root } = gate();+ const log = vi.spyOn(console, 'error').mockImplementation(() => {});+ const admission = createMaintenanceAdmission(root, { io: { ...fs, writeFileSync: () => {+ throw Object.assign(new Error('fixture disk full'), { code: 'ENOSPC' });+ } } });+ expect(() => admission.admit('agent', 'fixture')).toThrow(/needs recovery/);+ expect(admission.status()).toMatchObject({ state: 'unavailable', cause: 'interrupted-transaction' });+ expect(admission.tryAdmit('agent', 'again')).toBeNull();+ expect(log).toHaveBeenCalledExactlyOnceWith('❌ Maintenance transaction failed (ENOSPC): fixture disk full');+ rmSync(join(admission.directory, 'transaction'), { recursive: true });+ expect(admission.status()).toMatchObject({ state: 'unavailable', cause: 'process-uncertain' });+ expect(admission.tryAdmit('agent', 'still held')).toBeNull();+ });++ it('logs the original lock acquisition failure once', () => {+ const { root } = gate();+ const log = vi.spyOn(console, 'error').mockImplementation(() => {});+ const admission = createMaintenanceAdmission(root, { io: { ...fs, mkdirSync: () => {+ throw Object.assign(new Error('fixture denied'), { code: 'EACCES' });+ } } });+ expect(admission.tryAdmit('agent', 'fixture')).toBeNull();+ expect(admission.tryAdmit('agent', 'again')).toBeNull();+ expect(log).toHaveBeenCalledExactlyOnceWith('❌ Maintenance transaction failed (EACCES): fixture denied');+ });+it('binds operator publication reconciliation to the exact hold and owner and publishes before retirement', async () => {const { root, admission } = gate();const owner = admission.admit('settlement', 'Output publication');@@ -156,7 +194,7 @@ describe('durable graceful maintenance', () => {fs.mkdirSync(join(admission.directory, 'transaction'));expect(() => legacyOperation.parse(JSON.parse(before).operations[0])).toThrow();const restarted = createMaintenanceAdmission(root);- expect(restarted.status().state).toBe('unavailable');+ expect(restarted.status()).toMatchObject({ state: 'unavailable', cause: 'interrupted-transaction' });expect(restarted.tryAdmit('media', 'new')).toBeNull();expect(() => recovered.completeRecovery()).toThrow(/needs recovery/);expect(fs.existsSync(join(admission.directory, 'transaction'))).toBe(true);@@ -316,7 +354,7 @@ describe('durable graceful maintenance', () => {expect(admission.status()).toMatchObject({ state: 'draining', blockers: [{ unsettled: true }] });writeFileSync(join(admission.directory, 'state.json'), '{"version":999}');const restarted = createMaintenanceAdmission(root);- expect(restarted.status().state).toBe('unavailable');+ expect(restarted.status()).toMatchObject({ state: 'unavailable', cause: 'unreadable-journal' });expect(restarted.tryAdmit('agent', 'new')).toBeNull();expect(() => restarted.resume({ id: '00000000-0000-4000-8000-000000000000', revision: 1 })).toThrow();});diff --git a/server/services/backup.test.js b/server/services/backup.test.jsindex 7a06e219e0..8381aeade1 100644--- a/server/services/backup.test.js+++ b/server/services/backup.test.js@@ -3190,7 +3190,7 @@ describe('restoreSnapshot manifest verification', () => {});});- it.for(['peer-execution-grants.json', 'peer-execution-catalog.json', 'peer-execution/fixture-operation/evidence.json', 'peer-execution-authority.json', 'peer-execution-recovery.jsonl', 'workflow-maintenance/state.json', 'Workflow-Maintenance/state.json', 'backup-admission/cut/owner.json', 'Backup-Admission/publications/owner.json'])+ it.for(['peer-execution-grants.json', 'peer-execution-catalog.json', 'peer-execution/fixture-operation/evidence.json', 'peer-execution-authority.json', 'peer-execution-recovery.jsonl', 'workflow-maintenance/state.json', 'workflow-maintenance/recovered-fixture/receipt.json', 'Workflow-Maintenance/state.json', 'backup-admission/cut/owner.json', 'Backup-Admission/publications/owner.json'])('preserves non-rewound execution authority and ownership at %s (#10127)', async (path, context) => {await withRealRsync(context, async () => {const localPath = joinPath(PATHS.data, path);