Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: serialize migration runners with a lock on data/migrations.lock

[._.]/ Adam Eivy093a207249e9
diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md
index 2253f89a0d..25c06650a0 100644
--- a/docs/TROUBLESHOOTING.md
+++ b/docs/TROUBLESHOOTING.md
@@ -476,6 +476,12 @@ npm run pm2:start
The split never overwrites a record directory that already exists. If a later Postgres import (`server/scripts/migrateUniversesToDB.js` and similar) already ran, re-run that explicit legacy-import tool after the split rather than resetting import markers or schema versions.
+### "Another migration run holds data/migrations.lock"
+
+**Symptom**: `npm run migrations` or server boot exits with `Another migration run (pid N) holds …/data/migrations.lock`.
+
+Only one migration runner may touch `data/migrations.applied.json` at a time: the booting server and a manual `npm run migrations` would otherwise both read the ledger before either persists it and apply the same migration twice. The second runner waits up to a minute for the holder to finish, then gives up with this message. Let the running server finish booting (or stop it with `pm2 stop portos`) and retry. A runner that died mid-run (power loss, `kill -9`) leaves the directory behind; the next run reclaims it on its own once the recorded pid is gone or the lock is over 30 minutes old, so only remove `data/migrations.lock` by hand when you are certain no migration runner is still alive.
+
### Lost App Registrations
**Symptom**: Apps disappear after restart.
diff --git a/scripts/run-migrations.js b/scripts/run-migrations.js
index 356c722896..e3e7ae345e 100644
--- a/scripts/run-migrations.js
+++ b/scripts/run-migrations.js
@@ -1,5 +1,5 @@
#!/usr/bin/env node
-import { readdir, readFile, mkdir, rename } from 'fs/promises';
+import { readdir, readFile, mkdir, rename, rm, stat, writeFile } from 'fs/promises';
import { join, dirname } from 'path';
import { fileURLToPath, pathToFileURL } from 'url';
import { resolveInstallRoot, isWorktreeRoot } from '../server/lib/dataRoot.js';
@@ -17,6 +17,91 @@ const DEFAULT_MIGRATIONS_DIR = join(__dirname, 'migrations');
const appliedFilePath = (rootDir) => join(rootDir, 'data', 'migrations.applied.json');
+// ---- runner lock (#12046) ----
+//
+// The ledger is read-then-written by every runner: the booting server
+// (bootstrap.js) and the documented manual `npm run migrations` both call
+// runMigrations, and a PM2 restart can start a second server mid-run. Two
+// runners that both read the ledger before either persists it each apply the
+// same non-idempotent migration. A `mkdir` is atomic on every filesystem, so
+// the lock is a directory; `owner.json` inside it carries the holder's pid and
+// start time so a runner that died (SIGKILL, power loss) can be recognised and
+// its lock reclaimed instead of blocking every later boot.
+const lockDirPath = (rootDir) => join(rootDir, 'data', 'migrations.lock');
+const LOCK_OWNER_FILE = 'owner.json';
+const LOCK_STALE_MS = 30 * 60 * 1000;
+const LOCK_WAIT_MS = 60 * 1000;
+const LOCK_POLL_MS = 250;
+
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
+// Signal 0 probes existence without delivering anything. EPERM means the pid
+// exists but belongs to another user, which still counts as alive.
+const pidAlive = (pid) => {
+ try { process.kill(pid, 0); return true; } catch (err) { return err.code === 'EPERM'; }
+};
+
+const readLockOwner = (lockDir) => readFile(join(lockDir, LOCK_OWNER_FILE), 'utf-8')
+ .then((raw) => JSON.parse(raw), () => null)
+ .then((owner) => (owner && typeof owner === 'object' ? owner : null));
+
+/**
+ * Why a held lock can be reclaimed, or null while its holder may still be
+ * running. The holder's own pid is never judged dead (an in-process second
+ * caller must wait, not steal), and a lock with no readable owner yet — the
+ * window between the holder's mkdir and its owner.json write — is judged by
+ * the directory's age alone.
+ */
+async function lockStaleReason(lockDir) {
+ const owner = await readLockOwner(lockDir);
+ const pid = Number.isInteger(owner?.pid) ? owner.pid : null;
+ if (pid !== null && pid !== process.pid && !pidAlive(pid)) return `holder pid ${pid} is no longer running`;
+ const startedAt = Number.isFinite(owner?.startedAt)
+ ? owner.startedAt
+ : (await stat(lockDir).then((s) => s.mtimeMs, () => null));
+ if (startedAt !== null && Date.now() - startedAt > LOCK_STALE_MS) return `held for over ${LOCK_STALE_MS / 60000} minutes`;
+ return null;
+}
+
+/**
+ * Take `data/migrations.lock`; resolves to the release function. A second
+ * runner waits up to `waitMs` for the holder to finish, reclaims a stale lock
+ * on sight, and otherwise rejects with the holder's identity so the operator
+ * knows what to wait for rather than racing it.
+ */
+async function acquireMigrationLock(lockDir, { waitMs = LOCK_WAIT_MS, pollMs = LOCK_POLL_MS } = {}) {
+ const deadline = Date.now() + waitMs;
+ let announced = false;
+ for (;;) {
+ const taken = await mkdir(lockDir).then(() => true, (err) => {
+ if (err.code === 'EEXIST') return false;
+ throw err;
+ });
+ if (taken) {
+ await writeFile(join(lockDir, LOCK_OWNER_FILE), JSON.stringify({ pid: process.pid, startedAt: Date.now() }));
+ return () => rm(lockDir, { recursive: true, force: true });
+ }
+ const stale = await lockStaleReason(lockDir);
+ if (stale) {
+ console.warn(`⚠️ Reclaiming stale migration lock ${lockDir}: ${stale}`);
+ await rm(lockDir, { recursive: true, force: true });
+ continue;
+ }
+ if (Date.now() >= deadline) {
+ const owner = await readLockOwner(lockDir);
+ throw new Error(
+ `Another migration run (pid ${owner?.pid ?? 'unknown'}) holds ${lockDir} — wait for it to finish (or stop the PortOS server) and retry; `
+ + `remove the directory by hand only if no migration runner is still alive`,
+ );
+ }
+ if (!announced) {
+ console.log(`⏳ Waiting for another migration run holding ${lockDir}`);
+ announced = true;
+ }
+ await sleep(pollMs);
+ }
+}
+
/**
* Scan for migration files (*.js, sorted by filename). Co-located vitest
* files (*.test.js) are excluded — they don't export `up()` and would
@@ -88,6 +173,8 @@ export async function listPendingMigrations({
export async function runMigrations({
rootDir = DEFAULT_ROOT_DIR,
migrationsDir = DEFAULT_MIGRATIONS_DIR,
+ lockWaitMs = LOCK_WAIT_MS,
+ lockPollMs = LOCK_POLL_MS,
} = {}) {
// Backstop for #1947: a process booted from inside a CoS agent git worktree
// (data/cos/worktrees/agent-*) has no gitignored data/ runtime tree, so
@@ -105,6 +192,17 @@ export async function runMigrations({
// ships in the repo and always exists).
await mkdir(dirname(appliedFile), { recursive: true });
+ // Everything from the ledger read to its last persist runs under the lock:
+ // the read is only trustworthy while no other runner can append to it.
+ const releaseLock = await acquireMigrationLock(lockDirPath(rootDir), { waitMs: lockWaitMs, pollMs: lockPollMs });
+ try {
+ return await runPendingMigrations({ rootDir, migrationsDir, appliedFile });
+ } finally {
+ await releaseLock();
+ }
+}
+
+async function runPendingMigrations({ rootDir, migrationsDir, appliedFile }) {
// Load applied migrations list (repairing a corrupt file aside — see
// readAppliedList).
const applied = await readAppliedList(appliedFile, { repair: true });
diff --git a/scripts/run-migrations.test.js b/scripts/run-migrations.test.js
index e2db407beb..92f55f28fd 100644
--- a/scripts/run-migrations.test.js
+++ b/scripts/run-migrations.test.js
@@ -340,6 +340,116 @@ export default {
});
});
+describe('runMigrations lock (#12046)', () => {
+ let rootDir;
+ let dataDir;
+ let migrationsDir;
+ let appliedFile;
+ let lockDir;
+ let warnSpy;
+ let logSpy;
+
+ // Deliberately NOT idempotent: every execution appends one byte, so a
+ // double-apply shows up as a two-byte marker rather than hiding behind an
+ // existsSync guard.
+ const COUNTING_FIXTURE = `
+import { appendFileSync } from 'fs';
+import { join } from 'path';
+export default {
+ async up({ rootDir }) {
+ await new Promise((resolve) => setTimeout(resolve, 20));
+ appendFileSync(join(rootDir, 'data', 'applied-count.txt'), 'x');
+ }
+};
+`;
+
+ beforeEach(() => {
+ rootDir = mkdtempSync(join(tmpdir(), 'run-migrations-lock-'));
+ dataDir = join(rootDir, 'data');
+ migrationsDir = join(rootDir, 'migrations');
+ mkdirSync(dataDir, { recursive: true });
+ mkdirSync(migrationsDir, { recursive: true });
+ appliedFile = join(dataDir, 'migrations.applied.json');
+ lockDir = join(dataDir, 'migrations.lock');
+ writeFileSync(appliedFile, JSON.stringify(['000-earlier.js']) + '\n');
+ writeFileSync(join(migrationsDir, '001-count.js'), COUNTING_FIXTURE);
+ warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
+ logSpy = vi.spyOn(console, 'log').mockImplementation(() => {});
+ });
+
+ afterEach(() => {
+ rmSync(rootDir, { recursive: true, force: true });
+ warnSpy.mockRestore();
+ logSpy.mockRestore();
+ });
+
+ const appliedCount = () => (existsSync(join(dataDir, 'applied-count.txt'))
+ ? readFileSync(join(dataDir, 'applied-count.txt'), 'utf-8').length
+ : 0);
+
+ it('two concurrent runs apply each pending migration exactly once', async () => {
+ // The race: the manual `npm run migrations` and the booting server both
+ // read the ledger before either persists it. The second runner must wait
+ // for the first and then find nothing pending.
+ const results = await Promise.all([
+ runMigrations({ rootDir, migrationsDir }),
+ runMigrations({ rootDir, migrationsDir }),
+ ]);
+
+ expect(results.sort()).toEqual([0, 1]);
+ expect(appliedCount()).toBe(1);
+ expect(JSON.parse(readFileSync(appliedFile, 'utf-8'))).toEqual(['000-earlier.js', '001-count.js']);
+ expect(existsSync(lockDir)).toBe(false);
+ expect(logSpy).toHaveBeenCalledWith(expect.stringContaining('Waiting for another migration run'));
+ });
+
+ it('releases the lock when a migration throws', async () => {
+ writeFileSync(join(migrationsDir, '002-throws.js'), 'export default { async up() { throw new Error("boom"); } };');
+
+ await expect(runMigrations({ rootDir, migrationsDir })).rejects.toThrow('boom');
+
+ expect(existsSync(lockDir)).toBe(false);
+ // The retry reaches the migration again (rejecting with its error, not the
+ // lock's) and does not re-apply 001, which the first run recorded.
+ await expect(runMigrations({ rootDir, migrationsDir, lockWaitMs: 50, lockPollMs: 10 })).rejects.toThrow('boom');
+ expect(appliedCount()).toBe(1);
+ });
+
+ it('gives up after the bounded wait while a live holder keeps the lock', async () => {
+ mkdirSync(lockDir);
+ writeFileSync(join(lockDir, 'owner.json'), JSON.stringify({ pid: process.pid, startedAt: Date.now() }));
+
+ await expect(runMigrations({ rootDir, migrationsDir, lockWaitMs: 60, lockPollMs: 10 }))
+ .rejects.toThrow(/Another migration run \(pid \d+\) holds/);
+
+ expect(appliedCount()).toBe(0);
+ expect(JSON.parse(readFileSync(appliedFile, 'utf-8'))).toEqual(['000-earlier.js']);
+ expect(existsSync(lockDir)).toBe(true); // never steals a live holder's lock
+ });
+
+ it('reclaims a lock whose holder process is gone', async () => {
+ mkdirSync(lockDir);
+ // Above every platform's pid ceiling, so the liveness probe reports ESRCH.
+ writeFileSync(join(lockDir, 'owner.json'), JSON.stringify({ pid: 2147483646, startedAt: Date.now() }));
+
+ const ran = await runMigrations({ rootDir, migrationsDir, lockWaitMs: 60, lockPollMs: 10 });
+
+ expect(ran).toBe(1);
+ expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('no longer running'));
+ expect(existsSync(lockDir)).toBe(false);
+ });
+
+ it('reclaims a lock held past the stale timeout even when its pid is unknown', async () => {
+ mkdirSync(lockDir);
+ writeFileSync(join(lockDir, 'owner.json'), JSON.stringify({ startedAt: Date.now() - 31 * 60 * 1000 }));
+
+ const ran = await runMigrations({ rootDir, migrationsDir, lockWaitMs: 60, lockPollMs: 10 });
+
+ expect(ran).toBe(1);
+ expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('held for over 30 minutes'));
+ });
+});
+
describe('runMigrations worktree backstop (#1947)', () => {
let baseDir;
let migrationsDir;