Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: keep competing maintenance recoveries from blocking the winning owner

[._.]/ Adam Eivy053a0d30d21c
diff --git a/scripts/recover-maintenance-transaction.mjs b/scripts/recover-maintenance-transaction.mjs
index e6a2b09a4d..9e62ace007 100644
--- a/scripts/recover-maintenance-transaction.mjs
+++ b/scripts/recover-maintenance-transaction.mjs
@@ -15,6 +15,11 @@ const refuse = message => { throw new Error(message); };
// closed for explicit operator inspection, as with the migration runner lock.
function claimRecoveryFence(directory, io, stat) {
const lock = join(directory, 'recovery');
+ // A takeover marker must not live inside a fence that a winning recovery
+ // could be releasing concurrently. Keep its shared name stable instead.
+ const transition = join(directory, 'recovery-transition');
+ const legacyTransition = join(lock, '.transition');
+ const unfinished = () => refuse('Maintenance recovery has an unfinished transition; stop all recovery processes and inspect its fence.');
const physical = value => value && [value.dev, value.ino, value.birthtimeMs].join(':');
const readOwner = path => {
const file = join(path, 'owner.json');
@@ -34,6 +39,7 @@ function claimRecoveryFence(directory, io, stat) {
const owner = { version: 1, pid: process.pid, token: randomUUID() };
const bytes = Buffer.from(`${JSON.stringify(owner)}\n`);
for (let attempt = 0; attempt < 2; attempt++) {
+ if (stat(transition)) unfinished();
if (!stat(lock)) {
const staged = join(directory, `recovery-acquire-${randomUUID()}`);
io.mkdirSync(staged, { mode: 0o700 });
@@ -70,19 +76,25 @@ function claimRecoveryFence(directory, io, stat) {
const holder = observed?.isDirectory() ? readOwner(lock) : null;
if (!holder) refuse('Maintenance recovery ownership is unverified; preserve its fence for inspection.');
if (!dead(holder.pid)) refuse('Maintenance recovery is already owned by a live or unreadable process.');
- const transition = join(lock, '.transition');
- try { io.mkdirSync(transition, { mode: 0o700 }); }
- catch { refuse('Maintenance recovery has an unfinished transition; stop all recovery processes and inspect its fence.'); }
- const transitionStat = physical(stat(transition));
- let retired = false;
+ if (stat(legacyTransition)) unfinished();
+ let transitionFd;
+ try { transitionFd = io.openSync(transition, 'wx', 0o600); }
+ catch { unfinished(); }
+ let transitionStat;
try {
+ // The descriptor identifies our creation even if its parent or pathname
+ // was replaced before this read; a path stat could identify another owner.
+ transitionStat = physical(io.fstatSync(transitionFd));
+ if (stat(legacyTransition)) unfinished();
if (physical(stat(lock)) !== physical(observed) || readOwner(lock)?.bytes.equals(holder.bytes) !== true || !dead(holder.pid))
refuse('Maintenance recovery ownership changed during inspection.');
io.renameSync(lock, join(directory, `recovery-interrupted-${randomUUID()}`));
- retired = true;
syncMaintenanceDirectory(directory, io);
} finally {
- if (!retired && physical(stat(lock)) === physical(observed) && physical(stat(transition)) === transitionStat) io.rmdirSync(transition);
+ try {
+ // Release only our exact marker, including after successful retirement.
+ if (transitionStat && physical(stat(transition)) === transitionStat) io.unlinkSync(transition);
+ } finally { io.closeSync(transitionFd); }
}
}
refuse('Maintenance recovery is already owned; retry after its owner finishes.');
diff --git a/scripts/recover-maintenance-transaction.test.js b/scripts/recover-maintenance-transaction.test.js
index d14775f8e5..9424c11984 100644
--- a/scripts/recover-maintenance-transaction.test.js
+++ b/scripts/recover-maintenance-transaction.test.js
@@ -125,6 +125,153 @@ describe('offline maintenance transaction recovery', () => {
expect(createMaintenanceAdmission(data).status()).toMatchObject({ state: 'ready', hold: recovered.hold });
});
+ it('lets the winning recovery release while a stale contender inspects its takeover marker', async () => {
+ const { data, directory } = fixture();
+ const dead = spawnSync(process.execPath, ['-e', ''], { timeout: 5000 });
+ expect(dead.status).toBe(0);
+ const fence = join(directory, 'recovery');
+ fs.mkdirSync(fence);
+ const originalOwner = JSON.stringify({ version: 1, pid: dead.pid, token: randomUUID() });
+ fs.writeFileSync(join(fence, 'owner.json'), originalOwner);
+ const barrier = join(data, 'barrier');
+ fs.mkdirSync(barrier);
+ const module = new URL('./recover-maintenance-transaction.mjs', import.meta.url).href;
+ const source = `
+ import * as fs from 'node:fs';
+ import { join, basename } from 'node:path';
+ import { recoverMaintenanceTransaction } from ${JSON.stringify(module)};
+ const [data, barrier] = process.argv.slice(1);
+ const wait = name => {
+ const until = Date.now() + 10000;
+ while (!fs.existsSync(join(barrier, name))) {
+ if (Date.now() >= until) throw new Error('Recovery fixture barrier timed out');
+ Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10);
+ }
+ };
+ const io = { ...fs, mkdirSync(path, options) {
+ if (basename(path).startsWith('recovered-')) {
+ fs.writeFileSync(join(barrier, 'ready'), '');
+ wait('release');
+ }
+ return fs.mkdirSync(path, options);
+ } };
+ try {
+ wait('start');
+ const result = await recoverMaintenanceTransaction('Winning recovery fixture', { data, io, processes: async () => [] });
+ process.stdout.write(JSON.stringify(result));
+ } catch (error) { process.stdout.write(JSON.stringify({ error: error.message })); }
+ finally { fs.writeFileSync(join(barrier, 'finished'), ''); }
+ `;
+ const child = spawn(process.execPath, ['--input-type=module', '-e', source, data, barrier], { stdio: ['ignore', 'pipe', 'pipe'] });
+ let output = '';
+ let errors = '';
+ child.stdout.on('data', bytes => { output += bytes; });
+ child.stderr.on('data', bytes => { errors += bytes; });
+ const done = new Promise((resolve, reject) => {
+ child.once('error', reject);
+ child.once('close', code => {
+ try { resolve(JSON.parse(output)); }
+ catch { reject(new Error(`Recovery fixture exited ${code}: ${errors}`)); }
+ });
+ });
+ let replacementOwner;
+ let transitionFd;
+ const io = { ...fs, openSync(path, ...args) {
+ const transition = path === join(directory, 'recovery-transition');
+ if (transition) {
+ // The other process takes ownership after our inspection, immediately
+ // before our takeover marker is created.
+ fs.writeFileSync(join(barrier, 'start'), '');
+ const until = Date.now() + 10000;
+ while (!fs.existsSync(join(barrier, 'ready'))) {
+ if (Date.now() >= until) throw new Error('Winning recovery did not take ownership');
+ Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10);
+ }
+ replacementOwner = fs.readFileSync(join(fence, 'owner.json'));
+ }
+ const fd = fs.openSync(path, ...args);
+ if (transition) transitionFd = fd;
+ return fd;
+ }, fstatSync(fd) {
+ if (fd === transitionFd) {
+ // Let the winner release after our marker is created but before our
+ // descriptor inspection and cleanup. Its fence must remain removable.
+ fs.writeFileSync(join(barrier, 'release'), '');
+ const until = Date.now() + 10000;
+ while (!fs.existsSync(join(barrier, 'finished'))) {
+ if (Date.now() >= until) throw new Error('Winning recovery did not finish');
+ Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10);
+ }
+ }
+ return fs.fstatSync(fd);
+ } };
+ try {
+ await expect(recoverMaintenanceTransaction(reason, { data, io, processes: stopped }))
+ .rejects.toThrow('Maintenance recovery ownership changed during inspection.');
+ expect(JSON.parse(replacementOwner).pid).toBe(child.pid);
+ fs.writeFileSync(join(barrier, 'release'), '');
+ const result = await done;
+ expect(result.error).toBeUndefined();
+ expect(fs.existsSync(fence)).toBe(false);
+ expect(fs.existsSync(join(directory, 'recovery-transition'))).toBe(false);
+ expect(createMaintenanceAdmission(data).status()).toMatchObject({ state: 'ready', hold: result.hold });
+ const retired = fs.readdirSync(directory).find(name => name.startsWith('recovery-interrupted-'));
+ expect(fs.readFileSync(join(directory, retired, 'owner.json'), 'utf8')).toBe(originalOwner);
+ } finally {
+ fs.writeFileSync(join(barrier, 'start'), '');
+ fs.writeFileSync(join(barrier, 'release'), '');
+ await done;
+ }
+ });
+
+ it('preserves a foreign transition that replaces its marker before descriptor inspection', async () => {
+ const { data, directory } = fixture();
+ const dead = spawnSync(process.execPath, ['-e', ''], { timeout: 5000 });
+ expect(dead.status).toBe(0);
+ const fence = join(directory, 'recovery');
+ fs.mkdirSync(fence);
+ fs.writeFileSync(join(fence, 'owner.json'), JSON.stringify({ version: 1, pid: dead.pid, token: randomUUID() }));
+ const transition = join(directory, 'recovery-transition');
+ const displaced = join(data, 'displaced-transition');
+ const replacement = JSON.stringify({ version: 1, pid: process.pid, token: randomUUID() });
+ const io = { ...fs, openSync(path, ...args) {
+ const fd = fs.openSync(path, ...args);
+ if (path === transition) {
+ fs.renameSync(transition, displaced);
+ fs.writeFileSync(transition, 'foreign transition evidence');
+ fs.writeFileSync(join(fence, 'owner.json'), replacement);
+ }
+ return fd;
+ } };
+ await expect(recoverMaintenanceTransaction(reason, { data, io, processes: stopped }))
+ .rejects.toThrow('Maintenance recovery ownership changed during inspection.');
+ expect(fs.readFileSync(transition, 'utf8')).toBe('foreign transition evidence');
+ expect(fs.readFileSync(join(fence, 'owner.json'), 'utf8')).toBe(replacement);
+ expect(fs.readFileSync(displaced)).toHaveLength(0);
+ expect(fs.existsSync(join(directory, 'state.json'))).toBe(false);
+ });
+
+ it('keeps legacy and standalone unfinished transitions closed without changing their evidence', async () => {
+ const dead = spawnSync(process.execPath, ['-e', ''], { timeout: 5000 });
+ expect(dead.status).toBe(0);
+ for (const kind of ['legacy-directory', 'legacy-file', 'standalone-file']) {
+ const { data, directory } = fixture();
+ const fence = join(directory, 'recovery');
+ const legacy = kind.startsWith('legacy-');
+ if (legacy) {
+ fs.mkdirSync(fence);
+ fs.writeFileSync(join(fence, 'owner.json'), JSON.stringify({ version: 1, pid: dead.pid, token: randomUUID() }));
+ }
+ const marker = legacy ? join(fence, '.transition') : join(directory, 'recovery-transition');
+ if (kind === 'legacy-directory') fs.mkdirSync(marker);
+ else fs.writeFileSync(marker, 'interrupted transition evidence');
+ await expect(recoverMaintenanceTransaction(reason, { data, processes: stopped })).rejects.toThrow(/unfinished transition/);
+ if (kind === 'legacy-directory') expect(fs.lstatSync(marker).isDirectory()).toBe(true);
+ else expect(fs.readFileSync(marker, 'utf8')).toBe('interrupted transition evidence');
+ expect(fs.existsSync(join(directory, 'state.json'))).toBe(false);
+ }
+ });
+
it('preserves unknown recovery ownership and refuses journal symlinks before any mutation', async () => {
const { data, directory } = fixture();
fs.mkdirSync(join(directory, 'recovery'));