Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: preserve migration lock ownership during stale recovery

Adam Eivyd7155aa58bb0
diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md
index 65b2a47ad9..bbd718d91a 100644
--- a/docs/TROUBLESHOOTING.md
+++ b/docs/TROUBLESHOOTING.md
@@ -518,7 +518,9 @@ The split never overwrites a record directory that already exists. If a later Po
**Symptom**: `npm run migrations` or server boot exits with `Another migration run (pid N) holds …/data/migrations.lock`.
-Only one migration runner may touch `data/migrations.applied.json` at a time: the booting server and a manual `npm run migrations` would otherwise both read the ledger before either persists it and apply the same migration twice. The second runner waits up to a minute for the holder to finish, then gives up with this message. Let the running server finish booting (or stop it with `pm2 stop portos`) and retry. A runner that died mid-run (power loss, `kill -9`) leaves the directory behind; the next run reclaims it on its own once the recorded pid is gone or the lock is over 30 minutes old, so only remove `data/migrations.lock` by hand when you are certain no migration runner is still alive.
+Only one migration runner may touch `data/migrations.applied.json` at a time: the booting server and a manual `npm run migrations` would otherwise both read the ledger before either persists it and apply the same migration twice. The second runner waits up to a minute for the holder to finish, then gives up with this message. Let the running server finish booting (or stop it with `pm2 stop portos`) and retry. A live holder never expires, even during a long upgrade. A runner that died mid-run (power loss, `kill -9`) leaves the directory behind; the next run reclaims it once the recorded pid is gone. An acquisition with a missing or malformed owner can be reclaimed after 30 minutes; other owner-read errors refuse recovery.
+
+Release and stale recovery serialize through `data/migrations.lock/.transition`. If a process dies during that brief transition, the lock stays closed rather than risking duplicate migrations. Stop all migration runners before removing `data/migrations.lock` by hand, then retry. A crash after atomic retirement can also leave a `data/migrations.lock.retired-*` directory; it is no longer an active lock and can be removed once runners have stopped.
### Lost App Registrations
diff --git a/scripts/run-migrations.js b/scripts/run-migrations.js
index 722d0d435c..26c6db1bd5 100644
--- a/scripts/run-migrations.js
+++ b/scripts/run-migrations.js
@@ -2,6 +2,7 @@
import { readFile, readdir, mkdir, rm, stat, writeFile, rename } from 'fs/promises';
import { basename, join, dirname } from 'path';
import { pathToFileURL } from 'url';
+import { randomUUID } from 'crypto';
import { isWorktreeRoot } from '../server/lib/dataRoot.js';
import { isDirectlyInvoked } from './lib/directInvocation.js';
import {
@@ -69,7 +70,10 @@ const readLockOwner = (lockDir) => readFile(join(lockDir, LOCK_OWNER_FILE), 'utf
// A reader can land between the holder's mkdir and a complete owner write;
// an unparseable file means "owner unknown", same as a missing one.
try { return JSON.parse(raw); } catch { return null; }
- }, () => null)
+ }, (err) => {
+ if (err.code === 'ENOENT') return null;
+ throw err;
+ })
.then((owner) => (owner && typeof owner === 'object' ? owner : null));
/**
@@ -79,17 +83,61 @@ const readLockOwner = (lockDir) => readFile(join(lockDir, LOCK_OWNER_FILE), 'utf
* window between the holder's mkdir and its owner.json write — is judged by
* the directory's age alone.
*/
-async function lockStaleReason(lockDir) {
+async function lockStaleReason(lockDir, unknownStartedAt) {
const owner = await readLockOwner(lockDir);
- const pid = Number.isInteger(owner?.pid) ? owner.pid : null;
- if (pid !== null && pid !== process.pid && !pidAlive(pid)) return `holder pid ${pid} is no longer running`;
+ const pid = Number.isInteger(owner?.pid) && owner.pid > 0 ? owner.pid : null;
+ // Age cannot revoke a live runner: large upgrades can outlast the timeout.
+ if (pid !== null) return pidAlive(pid) ? null : `holder pid ${pid} is no longer running`;
const startedAt = Number.isFinite(owner?.startedAt)
? owner.startedAt
- : (await stat(lockDir).then((s) => s.mtimeMs, () => null));
+ : (unknownStartedAt ?? await stat(lockDir).then((s) => s.mtimeMs, () => null));
if (startedAt !== null && Date.now() - startedAt > LOCK_STALE_MS) return `held for over ${LOCK_STALE_MS / 60000} minutes`;
return null;
}
+/** Serialize release and stale recovery, including their ownership recheck. */
+async function removeMigrationLock(lockDir, token) {
+ const observed = await stat(lockDir).catch(err => {
+ if (err.code === 'ENOENT') return null;
+ throw err;
+ });
+ if (!observed) return false;
+ const transition = join(lockDir, '.transition');
+ const taken = await mkdir(transition).then(() => true, err => {
+ if (err.code === 'EEXIST' || err.code === 'ENOENT') return false;
+ throw err;
+ });
+ if (!taken) return false;
+ let removed = false;
+ try {
+ const current = await stat(lockDir);
+ if (current.dev !== observed.dev || current.ino !== observed.ino) return false;
+ if (token) {
+ if ((await readLockOwner(lockDir))?.token !== token) {
+ throw new Error('Migration lock ownership changed; refusing to release another runner');
+ }
+ } else {
+ // mkdir(.transition) changed the directory mtime. Use its age BEFORE
+ // that mkdir when an interrupted acquisition left no readable owner.
+ const stale = await lockStaleReason(lockDir, observed.mtimeMs);
+ if (!stale) return false;
+ console.warn(`⚠️ Reclaiming stale migration lock ${lockDir}: ${stale}`);
+ }
+ // Keep the transition present until the shared path disappears atomically:
+ // recursive rm would remove it first and let another waiter enter mid-rm.
+ const retired = `${lockDir}.retired-${randomUUID()}`;
+ await rename(lockDir, retired);
+ removed = true;
+ await rm(retired, { recursive: true, force: true });
+ return true;
+ } finally {
+ // After deletion a new runner may already own this path. Never clean a
+ // transition in that replacement directory. An interrupted transition
+ // stays fail-closed and uses the runner's explicit manual-recovery error.
+ if (!removed) await rm(transition, { recursive: true, force: true });
+ }
+}
+
/**
* Take `data/migrations.lock`; resolves to the release function. A second
* runner waits up to `waitMs` for the holder to finish, reclaims a stale lock
@@ -107,21 +155,28 @@ async function acquireMigrationLock(lockDir, { waitMs = LOCK_WAIT_MS, pollMs = L
if (taken) {
// temp + rename so a concurrent runner sees no owner file or a whole one.
const ownerPath = join(lockDir, LOCK_OWNER_FILE);
- await writeFile(`${ownerPath}.tmp`, JSON.stringify({ pid: process.pid, startedAt: Date.now() }));
+ const token = randomUUID();
+ await writeFile(`${ownerPath}.tmp`, JSON.stringify({ pid: process.pid, startedAt: Date.now(), token }));
await rename(`${ownerPath}.tmp`, ownerPath);
- return () => rm(lockDir, { recursive: true, force: true });
+ return async () => {
+ const releaseDeadline = Date.now() + waitMs;
+ while (!await removeMigrationLock(lockDir, token)) {
+ if (Date.now() >= releaseDeadline) {
+ throw new Error('Migration lock release blocked by an unfinished transition; stop all migration runners before recovering the lock');
+ }
+ await sleep(pollMs);
+ }
+ };
}
const stale = await lockStaleReason(lockDir);
- if (stale) {
- console.warn(`⚠️ Reclaiming stale migration lock ${lockDir}: ${stale}`);
- await rm(lockDir, { recursive: true, force: true });
+ if (stale && await removeMigrationLock(lockDir)) {
continue;
}
if (Date.now() >= deadline) {
const owner = await readLockOwner(lockDir);
throw new Error(
`Another migration run (pid ${owner?.pid ?? 'unknown'}) holds ${lockDir} — wait for it to finish (or stop the PortOS server) and retry; `
- + `remove the directory by hand only if no migration runner is still alive`,
+ + `an interrupted .transition also keeps the lock closed; remove the directory by hand only if no migration runner is still alive`,
);
}
if (!announced) {
diff --git a/scripts/run-migrations.test.js b/scripts/run-migrations.test.js
index 46aa585af3..dd86d86208 100644
--- a/scripts/run-migrations.test.js
+++ b/scripts/run-migrations.test.js
@@ -1,8 +1,9 @@
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
-import { mkdtempSync, rmSync, writeFileSync, readFileSync, mkdirSync, readdirSync, existsSync } from 'fs';
+import { mkdtempSync, rmSync, writeFileSync, readFileSync, mkdirSync, readdirSync, existsSync, utimesSync } from 'fs';
import * as fsPromises from 'fs/promises';
import { tmpdir } from 'os';
import { join } from 'path';
+import { pathToFileURL } from 'url';
import { runMigrations, listPendingMigrations, sourceDeclaresPurge } from './run-migrations.js';
@@ -491,6 +492,95 @@ export default {
expect(existsSync(lockDir)).toBe(true); // never steals a live holder's lock
});
+ it('never expires a live holder, even after the stale timeout', async () => {
+ mkdirSync(lockDir);
+ writeFileSync(join(lockDir, 'owner.json'), JSON.stringify({ pid: process.pid, startedAt: Date.now() - 31 * 60 * 1000 }));
+
+ await expect(runMigrations({ rootDir, migrationsDir, lockWaitMs: 0 }))
+ .rejects.toThrow(/Another migration run/);
+
+ expect(appliedCount()).toBe(0);
+ expect(existsSync(lockDir)).toBe(true);
+ expect(warnSpy).not.toHaveBeenCalled();
+ });
+
+ it('refuses an owner read failure instead of reclaiming an aged live lock', async () => {
+ mkdirSync(lockDir);
+ const ownerPath = join(lockDir, 'owner.json');
+ const old = new Date(Date.now() - 31 * 60 * 1000);
+ writeFileSync(ownerPath, JSON.stringify({ pid: process.pid, startedAt: old.getTime() }));
+ utimesSync(lockDir, old, old);
+ fsMocks.readFailures.set(ownerPath, Object.assign(new Error('owner read failed'), { code: 'EIO' }));
+ try {
+ await expect(runMigrations({ rootDir, migrationsDir, lockWaitMs: 0 }))
+ .rejects.toMatchObject({ code: 'EIO' });
+ expect(appliedCount()).toBe(0);
+ expect(existsSync(lockDir)).toBe(true);
+ } finally {
+ fsMocks.readFailures.clear();
+ }
+ });
+
+ it('rechecks stale evidence before removing a replacement holder', async () => {
+ mkdirSync(lockDir);
+ const ownerPath = join(lockDir, 'owner.json');
+ writeFileSync(ownerPath, JSON.stringify({ pid: 2147483646, startedAt: Date.now() }));
+ const fixturePath = join(migrationsDir, '001-count.js');
+ writeFileSync(fixturePath, `
+import { appendFileSync } from 'fs';
+import { join } from 'path';
+let announceStart, announceDuplicate, release;
+export const started = new Promise(resolve => { announceStart = resolve; });
+export const duplicate = new Promise(resolve => { announceDuplicate = resolve; });
+const finish = new Promise(resolve => { release = resolve; });
+export { release };
+export let entries = 0;
+export default { async up({ rootDir }) {
+ entries++;
+ announceStart();
+ if (entries > 1) announceDuplicate();
+ await finish;
+ appendFileSync(join(rootDir, 'data', 'applied-count.txt'), 'x');
+} };
+`);
+ const fixture = await import(pathToFileURL(fixturePath).href);
+ const readFile = vi.mocked(fsPromises.readFile);
+ const originalRead = readFile.getMockImplementation();
+ let announceRead, resumeRead;
+ const readStarted = new Promise(resolve => { announceRead = resolve; });
+ const resume = new Promise(resolve => { resumeRead = resolve; });
+ let paused = false;
+ readFile.mockImplementation(async (path, ...args) => {
+ const raw = await originalRead(path, ...args);
+ if (String(path) === ownerPath && !paused) {
+ paused = true;
+ announceRead();
+ await resume;
+ }
+ return raw;
+ });
+ const staleRun = runMigrations({ rootDir, migrationsDir, lockWaitMs: 0 });
+ const staleResult = staleRun.then(value => ({ value }), error => ({ error }));
+ let holder;
+ try {
+ await readStarted;
+ holder = runMigrations({ rootDir, migrationsDir, lockWaitMs: 0 });
+ await fixture.started;
+ resumeRead();
+ await Promise.race([staleResult, fixture.duplicate]);
+ } finally {
+ resumeRead();
+ fixture.release();
+ await Promise.allSettled([staleRun, holder]);
+ readFile.mockImplementation(originalRead);
+ }
+
+ expect(fixture.entries).toBe(1);
+ expect((await staleResult).error?.message).toMatch(/Another migration run/);
+ expect(appliedCount()).toBe(1);
+ expect(existsSync(lockDir)).toBe(false);
+ });
+
it('reclaims a lock whose holder process is gone', async () => {
mkdirSync(lockDir);
// Above every platform's pid ceiling, so the liveness probe reports ESRCH.
@@ -520,6 +610,26 @@ export default {
expect(ran).toBe(1);
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('held for over 30 minutes'));
});
+
+ it('recovers an old ownerless acquisition using its age before the transition mkdir', async () => {
+ mkdirSync(lockDir);
+ const old = new Date(Date.now() - 31 * 60 * 1000);
+ utimesSync(lockDir, old, old);
+
+ expect(await runMigrations({ rootDir, migrationsDir, lockWaitMs: 0 })).toBe(1);
+ expect(appliedCount()).toBe(1);
+ expect(existsSync(lockDir)).toBe(false);
+ });
+
+ it('keeps an interrupted transition closed instead of racing its recovery', async () => {
+ mkdirSync(join(lockDir, '.transition'), { recursive: true });
+ writeFileSync(join(lockDir, 'owner.json'), JSON.stringify({ pid: 2147483646, startedAt: Date.now() }));
+
+ await expect(runMigrations({ rootDir, migrationsDir, lockWaitMs: 0 }))
+ .rejects.toThrow(/interrupted .transition/);
+ expect(appliedCount()).toBe(0);
+ expect(existsSync(join(lockDir, '.transition'))).toBe(true);
+ });
});
describe('sourceDeclaresPurge', () => {