Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: serialize migration runners with a lock on data/migrations.lock
[._.]/ Adam Eivy093a207249e9
diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.mdindex 2253f89a0d..25c06650a0 100644--- a/docs/TROUBLESHOOTING.md+++ b/docs/TROUBLESHOOTING.md@@ -476,6 +476,12 @@ npm run pm2:startThe split never overwrites a record directory that already exists. If a later Postgres import (`server/scripts/migrateUniversesToDB.js` and similar) already ran, re-run that explicit legacy-import tool after the split rather than resetting import markers or schema versions.+### "Another migration run holds data/migrations.lock"++**Symptom**: `npm run migrations` or server boot exits with `Another migration run (pid N) holds …/data/migrations.lock`.++Only one migration runner may touch `data/migrations.applied.json` at a time: the booting server and a manual `npm run migrations` would otherwise both read the ledger before either persists it and apply the same migration twice. The second runner waits up to a minute for the holder to finish, then gives up with this message. Let the running server finish booting (or stop it with `pm2 stop portos`) and retry. A runner that died mid-run (power loss, `kill -9`) leaves the directory behind; the next run reclaims it on its own once the recorded pid is gone or the lock is over 30 minutes old, so only remove `data/migrations.lock` by hand when you are certain no migration runner is still alive.+### Lost App Registrations**Symptom**: Apps disappear after restart.diff --git a/scripts/run-migrations.js b/scripts/run-migrations.jsindex 356c722896..e3e7ae345e 100644--- a/scripts/run-migrations.js+++ b/scripts/run-migrations.js@@ -1,5 +1,5 @@#!/usr/bin/env node-import { readdir, readFile, mkdir, rename } from 'fs/promises';+import { readdir, readFile, mkdir, rename, rm, stat, writeFile } from 'fs/promises';import { join, dirname } from 'path';import { fileURLToPath, pathToFileURL } from 'url';import { resolveInstallRoot, isWorktreeRoot } from '../server/lib/dataRoot.js';@@ -17,6 +17,91 @@ const DEFAULT_MIGRATIONS_DIR = join(__dirname, 'migrations');const appliedFilePath = (rootDir) => join(rootDir, 'data', 'migrations.applied.json');+// ---- runner lock (#12046) ----+//+// The ledger is read-then-written by every runner: the booting server+// (bootstrap.js) and the documented manual `npm run migrations` both call+// runMigrations, and a PM2 restart can start a second server mid-run. Two+// runners that both read the ledger before either persists it each apply the+// same non-idempotent migration. A `mkdir` is atomic on every filesystem, so+// the lock is a directory; `owner.json` inside it carries the holder's pid and+// start time so a runner that died (SIGKILL, power loss) can be recognised and+// its lock reclaimed instead of blocking every later boot.+const lockDirPath = (rootDir) => join(rootDir, 'data', 'migrations.lock');+const LOCK_OWNER_FILE = 'owner.json';+const LOCK_STALE_MS = 30 * 60 * 1000;+const LOCK_WAIT_MS = 60 * 1000;+const LOCK_POLL_MS = 250;++const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));++// Signal 0 probes existence without delivering anything. EPERM means the pid+// exists but belongs to another user, which still counts as alive.+const pidAlive = (pid) => {+ try { process.kill(pid, 0); return true; } catch (err) { return err.code === 'EPERM'; }+};++const readLockOwner = (lockDir) => readFile(join(lockDir, LOCK_OWNER_FILE), 'utf-8')+ .then((raw) => JSON.parse(raw), () => null)+ .then((owner) => (owner && typeof owner === 'object' ? owner : null));++/**+ * Why a held lock can be reclaimed, or null while its holder may still be+ * running. The holder's own pid is never judged dead (an in-process second+ * caller must wait, not steal), and a lock with no readable owner yet — the+ * window between the holder's mkdir and its owner.json write — is judged by+ * the directory's age alone.+ */+async function lockStaleReason(lockDir) {+ const owner = await readLockOwner(lockDir);+ const pid = Number.isInteger(owner?.pid) ? owner.pid : null;+ if (pid !== null && pid !== process.pid && !pidAlive(pid)) return `holder pid ${pid} is no longer running`;+ const startedAt = Number.isFinite(owner?.startedAt)+ ? owner.startedAt+ : (await stat(lockDir).then((s) => s.mtimeMs, () => null));+ if (startedAt !== null && Date.now() - startedAt > LOCK_STALE_MS) return `held for over ${LOCK_STALE_MS / 60000} minutes`;+ return null;+}++/**+ * Take `data/migrations.lock`; resolves to the release function. A second+ * runner waits up to `waitMs` for the holder to finish, reclaims a stale lock+ * on sight, and otherwise rejects with the holder's identity so the operator+ * knows what to wait for rather than racing it.+ */+async function acquireMigrationLock(lockDir, { waitMs = LOCK_WAIT_MS, pollMs = LOCK_POLL_MS } = {}) {+ const deadline = Date.now() + waitMs;+ let announced = false;+ for (;;) {+ const taken = await mkdir(lockDir).then(() => true, (err) => {+ if (err.code === 'EEXIST') return false;+ throw err;+ });+ if (taken) {+ await writeFile(join(lockDir, LOCK_OWNER_FILE), JSON.stringify({ pid: process.pid, startedAt: Date.now() }));+ return () => rm(lockDir, { recursive: true, force: true });+ }+ const stale = await lockStaleReason(lockDir);+ if (stale) {+ console.warn(`⚠️ Reclaiming stale migration lock ${lockDir}: ${stale}`);+ await rm(lockDir, { recursive: true, force: true });+ continue;+ }+ if (Date.now() >= deadline) {+ const owner = await readLockOwner(lockDir);+ throw new Error(+ `Another migration run (pid ${owner?.pid ?? 'unknown'}) holds ${lockDir} — wait for it to finish (or stop the PortOS server) and retry; `+ + `remove the directory by hand only if no migration runner is still alive`,+ );+ }+ if (!announced) {+ console.log(`⏳ Waiting for another migration run holding ${lockDir}`);+ announced = true;+ }+ await sleep(pollMs);+ }+}+/*** Scan for migration files (*.js, sorted by filename). Co-located vitest* files (*.test.js) are excluded — they don't export `up()` and would@@ -88,6 +173,8 @@ export async function listPendingMigrations({export async function runMigrations({rootDir = DEFAULT_ROOT_DIR,migrationsDir = DEFAULT_MIGRATIONS_DIR,+ lockWaitMs = LOCK_WAIT_MS,+ lockPollMs = LOCK_POLL_MS,} = {}) {// Backstop for #1947: a process booted from inside a CoS agent git worktree// (data/cos/worktrees/agent-*) has no gitignored data/ runtime tree, so@@ -105,6 +192,17 @@ export async function runMigrations({// ships in the repo and always exists).await mkdir(dirname(appliedFile), { recursive: true });+ // Everything from the ledger read to its last persist runs under the lock:+ // the read is only trustworthy while no other runner can append to it.+ const releaseLock = await acquireMigrationLock(lockDirPath(rootDir), { waitMs: lockWaitMs, pollMs: lockPollMs });+ try {+ return await runPendingMigrations({ rootDir, migrationsDir, appliedFile });+ } finally {+ await releaseLock();+ }+}++async function runPendingMigrations({ rootDir, migrationsDir, appliedFile }) {// Load applied migrations list (repairing a corrupt file aside — see// readAppliedList).const applied = await readAppliedList(appliedFile, { repair: true });diff --git a/scripts/run-migrations.test.js b/scripts/run-migrations.test.jsindex e2db407beb..92f55f28fd 100644--- a/scripts/run-migrations.test.js+++ b/scripts/run-migrations.test.js@@ -340,6 +340,116 @@ export default {});});+describe('runMigrations lock (#12046)', () => {+ let rootDir;+ let dataDir;+ let migrationsDir;+ let appliedFile;+ let lockDir;+ let warnSpy;+ let logSpy;++ // Deliberately NOT idempotent: every execution appends one byte, so a+ // double-apply shows up as a two-byte marker rather than hiding behind an+ // existsSync guard.+ const COUNTING_FIXTURE = `+import { appendFileSync } from 'fs';+import { join } from 'path';+export default {+ async up({ rootDir }) {+ await new Promise((resolve) => setTimeout(resolve, 20));+ appendFileSync(join(rootDir, 'data', 'applied-count.txt'), 'x');+ }+};+`;++ beforeEach(() => {+ rootDir = mkdtempSync(join(tmpdir(), 'run-migrations-lock-'));+ dataDir = join(rootDir, 'data');+ migrationsDir = join(rootDir, 'migrations');+ mkdirSync(dataDir, { recursive: true });+ mkdirSync(migrationsDir, { recursive: true });+ appliedFile = join(dataDir, 'migrations.applied.json');+ lockDir = join(dataDir, 'migrations.lock');+ writeFileSync(appliedFile, JSON.stringify(['000-earlier.js']) + '\n');+ writeFileSync(join(migrationsDir, '001-count.js'), COUNTING_FIXTURE);+ warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});+ logSpy = vi.spyOn(console, 'log').mockImplementation(() => {});+ });++ afterEach(() => {+ rmSync(rootDir, { recursive: true, force: true });+ warnSpy.mockRestore();+ logSpy.mockRestore();+ });++ const appliedCount = () => (existsSync(join(dataDir, 'applied-count.txt'))+ ? readFileSync(join(dataDir, 'applied-count.txt'), 'utf-8').length+ : 0);++ it('two concurrent runs apply each pending migration exactly once', async () => {+ // The race: the manual `npm run migrations` and the booting server both+ // read the ledger before either persists it. The second runner must wait+ // for the first and then find nothing pending.+ const results = await Promise.all([+ runMigrations({ rootDir, migrationsDir }),+ runMigrations({ rootDir, migrationsDir }),+ ]);++ expect(results.sort()).toEqual([0, 1]);+ expect(appliedCount()).toBe(1);+ expect(JSON.parse(readFileSync(appliedFile, 'utf-8'))).toEqual(['000-earlier.js', '001-count.js']);+ expect(existsSync(lockDir)).toBe(false);+ expect(logSpy).toHaveBeenCalledWith(expect.stringContaining('Waiting for another migration run'));+ });++ it('releases the lock when a migration throws', async () => {+ writeFileSync(join(migrationsDir, '002-throws.js'), 'export default { async up() { throw new Error("boom"); } };');++ await expect(runMigrations({ rootDir, migrationsDir })).rejects.toThrow('boom');++ expect(existsSync(lockDir)).toBe(false);+ // The retry reaches the migration again (rejecting with its error, not the+ // lock's) and does not re-apply 001, which the first run recorded.+ await expect(runMigrations({ rootDir, migrationsDir, lockWaitMs: 50, lockPollMs: 10 })).rejects.toThrow('boom');+ expect(appliedCount()).toBe(1);+ });++ it('gives up after the bounded wait while a live holder keeps the lock', async () => {+ mkdirSync(lockDir);+ writeFileSync(join(lockDir, 'owner.json'), JSON.stringify({ pid: process.pid, startedAt: Date.now() }));++ await expect(runMigrations({ rootDir, migrationsDir, lockWaitMs: 60, lockPollMs: 10 }))+ .rejects.toThrow(/Another migration run \(pid \d+\) holds/);++ expect(appliedCount()).toBe(0);+ expect(JSON.parse(readFileSync(appliedFile, 'utf-8'))).toEqual(['000-earlier.js']);+ expect(existsSync(lockDir)).toBe(true); // never steals a live holder's lock+ });++ it('reclaims a lock whose holder process is gone', async () => {+ mkdirSync(lockDir);+ // Above every platform's pid ceiling, so the liveness probe reports ESRCH.+ writeFileSync(join(lockDir, 'owner.json'), JSON.stringify({ pid: 2147483646, startedAt: Date.now() }));++ const ran = await runMigrations({ rootDir, migrationsDir, lockWaitMs: 60, lockPollMs: 10 });++ expect(ran).toBe(1);+ expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('no longer running'));+ expect(existsSync(lockDir)).toBe(false);+ });++ it('reclaims a lock held past the stale timeout even when its pid is unknown', async () => {+ mkdirSync(lockDir);+ writeFileSync(join(lockDir, 'owner.json'), JSON.stringify({ startedAt: Date.now() - 31 * 60 * 1000 }));++ const ran = await runMigrations({ rootDir, migrationsDir, lockWaitMs: 60, lockPollMs: 10 });++ expect(ran).toBe(1);+ expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('held for over 30 minutes'));+ });+});+describe('runMigrations worktree backstop (#1947)', () => {let baseDir;let migrationsDir;