Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: preserve migration lock ownership during stale recovery
Adam Eivyd7155aa58bb0
diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.mdindex 65b2a47ad9..bbd718d91a 100644--- a/docs/TROUBLESHOOTING.md+++ b/docs/TROUBLESHOOTING.md@@ -518,7 +518,9 @@ The split never overwrites a record directory that already exists. If a later Po**Symptom**: `npm run migrations` or server boot exits with `Another migration run (pid N) holds …/data/migrations.lock`.-Only one migration runner may touch `data/migrations.applied.json` at a time: the booting server and a manual `npm run migrations` would otherwise both read the ledger before either persists it and apply the same migration twice. The second runner waits up to a minute for the holder to finish, then gives up with this message. Let the running server finish booting (or stop it with `pm2 stop portos`) and retry. A runner that died mid-run (power loss, `kill -9`) leaves the directory behind; the next run reclaims it on its own once the recorded pid is gone or the lock is over 30 minutes old, so only remove `data/migrations.lock` by hand when you are certain no migration runner is still alive.+Only one migration runner may touch `data/migrations.applied.json` at a time: the booting server and a manual `npm run migrations` would otherwise both read the ledger before either persists it and apply the same migration twice. The second runner waits up to a minute for the holder to finish, then gives up with this message. Let the running server finish booting (or stop it with `pm2 stop portos`) and retry. A live holder never expires, even during a long upgrade. A runner that died mid-run (power loss, `kill -9`) leaves the directory behind; the next run reclaims it once the recorded pid is gone. An acquisition with a missing or malformed owner can be reclaimed after 30 minutes; other owner-read errors refuse recovery.++Release and stale recovery serialize through `data/migrations.lock/.transition`. If a process dies during that brief transition, the lock stays closed rather than risking duplicate migrations. Stop all migration runners before removing `data/migrations.lock` by hand, then retry. A crash after atomic retirement can also leave a `data/migrations.lock.retired-*` directory; it is no longer an active lock and can be removed once runners have stopped.### Lost App Registrationsdiff --git a/scripts/run-migrations.js b/scripts/run-migrations.jsindex 722d0d435c..26c6db1bd5 100644--- a/scripts/run-migrations.js+++ b/scripts/run-migrations.js@@ -2,6 +2,7 @@import { readFile, readdir, mkdir, rm, stat, writeFile, rename } from 'fs/promises';import { basename, join, dirname } from 'path';import { pathToFileURL } from 'url';+import { randomUUID } from 'crypto';import { isWorktreeRoot } from '../server/lib/dataRoot.js';import { isDirectlyInvoked } from './lib/directInvocation.js';import {@@ -69,7 +70,10 @@ const readLockOwner = (lockDir) => readFile(join(lockDir, LOCK_OWNER_FILE), 'utf// A reader can land between the holder's mkdir and a complete owner write;// an unparseable file means "owner unknown", same as a missing one.try { return JSON.parse(raw); } catch { return null; }- }, () => null)+ }, (err) => {+ if (err.code === 'ENOENT') return null;+ throw err;+ }).then((owner) => (owner && typeof owner === 'object' ? owner : null));/**@@ -79,17 +83,61 @@ const readLockOwner = (lockDir) => readFile(join(lockDir, LOCK_OWNER_FILE), 'utf* window between the holder's mkdir and its owner.json write — is judged by* the directory's age alone.*/-async function lockStaleReason(lockDir) {+async function lockStaleReason(lockDir, unknownStartedAt) {const owner = await readLockOwner(lockDir);- const pid = Number.isInteger(owner?.pid) ? owner.pid : null;- if (pid !== null && pid !== process.pid && !pidAlive(pid)) return `holder pid ${pid} is no longer running`;+ const pid = Number.isInteger(owner?.pid) && owner.pid > 0 ? owner.pid : null;+ // Age cannot revoke a live runner: large upgrades can outlast the timeout.+ if (pid !== null) return pidAlive(pid) ? null : `holder pid ${pid} is no longer running`;const startedAt = Number.isFinite(owner?.startedAt)? owner.startedAt- : (await stat(lockDir).then((s) => s.mtimeMs, () => null));+ : (unknownStartedAt ?? await stat(lockDir).then((s) => s.mtimeMs, () => null));if (startedAt !== null && Date.now() - startedAt > LOCK_STALE_MS) return `held for over ${LOCK_STALE_MS / 60000} minutes`;return null;}+/** Serialize release and stale recovery, including their ownership recheck. */+async function removeMigrationLock(lockDir, token) {+ const observed = await stat(lockDir).catch(err => {+ if (err.code === 'ENOENT') return null;+ throw err;+ });+ if (!observed) return false;+ const transition = join(lockDir, '.transition');+ const taken = await mkdir(transition).then(() => true, err => {+ if (err.code === 'EEXIST' || err.code === 'ENOENT') return false;+ throw err;+ });+ if (!taken) return false;+ let removed = false;+ try {+ const current = await stat(lockDir);+ if (current.dev !== observed.dev || current.ino !== observed.ino) return false;+ if (token) {+ if ((await readLockOwner(lockDir))?.token !== token) {+ throw new Error('Migration lock ownership changed; refusing to release another runner');+ }+ } else {+ // mkdir(.transition) changed the directory mtime. Use its age BEFORE+ // that mkdir when an interrupted acquisition left no readable owner.+ const stale = await lockStaleReason(lockDir, observed.mtimeMs);+ if (!stale) return false;+ console.warn(`⚠️ Reclaiming stale migration lock ${lockDir}: ${stale}`);+ }+ // Keep the transition present until the shared path disappears atomically:+ // recursive rm would remove it first and let another waiter enter mid-rm.+ const retired = `${lockDir}.retired-${randomUUID()}`;+ await rename(lockDir, retired);+ removed = true;+ await rm(retired, { recursive: true, force: true });+ return true;+ } finally {+ // After deletion a new runner may already own this path. Never clean a+ // transition in that replacement directory. An interrupted transition+ // stays fail-closed and uses the runner's explicit manual-recovery error.+ if (!removed) await rm(transition, { recursive: true, force: true });+ }+}+/*** Take `data/migrations.lock`; resolves to the release function. A second* runner waits up to `waitMs` for the holder to finish, reclaims a stale lock@@ -107,21 +155,28 @@ async function acquireMigrationLock(lockDir, { waitMs = LOCK_WAIT_MS, pollMs = Lif (taken) {// temp + rename so a concurrent runner sees no owner file or a whole one.const ownerPath = join(lockDir, LOCK_OWNER_FILE);- await writeFile(`${ownerPath}.tmp`, JSON.stringify({ pid: process.pid, startedAt: Date.now() }));+ const token = randomUUID();+ await writeFile(`${ownerPath}.tmp`, JSON.stringify({ pid: process.pid, startedAt: Date.now(), token }));await rename(`${ownerPath}.tmp`, ownerPath);- return () => rm(lockDir, { recursive: true, force: true });+ return async () => {+ const releaseDeadline = Date.now() + waitMs;+ while (!await removeMigrationLock(lockDir, token)) {+ if (Date.now() >= releaseDeadline) {+ throw new Error('Migration lock release blocked by an unfinished transition; stop all migration runners before recovering the lock');+ }+ await sleep(pollMs);+ }+ };}const stale = await lockStaleReason(lockDir);- if (stale) {- console.warn(`⚠️ Reclaiming stale migration lock ${lockDir}: ${stale}`);- await rm(lockDir, { recursive: true, force: true });+ if (stale && await removeMigrationLock(lockDir)) {continue;}if (Date.now() >= deadline) {const owner = await readLockOwner(lockDir);throw new Error(`Another migration run (pid ${owner?.pid ?? 'unknown'}) holds ${lockDir} — wait for it to finish (or stop the PortOS server) and retry; `- + `remove the directory by hand only if no migration runner is still alive`,+ + `an interrupted .transition also keeps the lock closed; remove the directory by hand only if no migration runner is still alive`,);}if (!announced) {diff --git a/scripts/run-migrations.test.js b/scripts/run-migrations.test.jsindex 46aa585af3..dd86d86208 100644--- a/scripts/run-migrations.test.js+++ b/scripts/run-migrations.test.js@@ -1,8 +1,9 @@import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';-import { mkdtempSync, rmSync, writeFileSync, readFileSync, mkdirSync, readdirSync, existsSync } from 'fs';+import { mkdtempSync, rmSync, writeFileSync, readFileSync, mkdirSync, readdirSync, existsSync, utimesSync } from 'fs';import * as fsPromises from 'fs/promises';import { tmpdir } from 'os';import { join } from 'path';+import { pathToFileURL } from 'url';import { runMigrations, listPendingMigrations, sourceDeclaresPurge } from './run-migrations.js';@@ -491,6 +492,95 @@ export default {expect(existsSync(lockDir)).toBe(true); // never steals a live holder's lock});+ it('never expires a live holder, even after the stale timeout', async () => {+ mkdirSync(lockDir);+ writeFileSync(join(lockDir, 'owner.json'), JSON.stringify({ pid: process.pid, startedAt: Date.now() - 31 * 60 * 1000 }));++ await expect(runMigrations({ rootDir, migrationsDir, lockWaitMs: 0 }))+ .rejects.toThrow(/Another migration run/);++ expect(appliedCount()).toBe(0);+ expect(existsSync(lockDir)).toBe(true);+ expect(warnSpy).not.toHaveBeenCalled();+ });++ it('refuses an owner read failure instead of reclaiming an aged live lock', async () => {+ mkdirSync(lockDir);+ const ownerPath = join(lockDir, 'owner.json');+ const old = new Date(Date.now() - 31 * 60 * 1000);+ writeFileSync(ownerPath, JSON.stringify({ pid: process.pid, startedAt: old.getTime() }));+ utimesSync(lockDir, old, old);+ fsMocks.readFailures.set(ownerPath, Object.assign(new Error('owner read failed'), { code: 'EIO' }));+ try {+ await expect(runMigrations({ rootDir, migrationsDir, lockWaitMs: 0 }))+ .rejects.toMatchObject({ code: 'EIO' });+ expect(appliedCount()).toBe(0);+ expect(existsSync(lockDir)).toBe(true);+ } finally {+ fsMocks.readFailures.clear();+ }+ });++ it('rechecks stale evidence before removing a replacement holder', async () => {+ mkdirSync(lockDir);+ const ownerPath = join(lockDir, 'owner.json');+ writeFileSync(ownerPath, JSON.stringify({ pid: 2147483646, startedAt: Date.now() }));+ const fixturePath = join(migrationsDir, '001-count.js');+ writeFileSync(fixturePath, `+import { appendFileSync } from 'fs';+import { join } from 'path';+let announceStart, announceDuplicate, release;+export const started = new Promise(resolve => { announceStart = resolve; });+export const duplicate = new Promise(resolve => { announceDuplicate = resolve; });+const finish = new Promise(resolve => { release = resolve; });+export { release };+export let entries = 0;+export default { async up({ rootDir }) {+ entries++;+ announceStart();+ if (entries > 1) announceDuplicate();+ await finish;+ appendFileSync(join(rootDir, 'data', 'applied-count.txt'), 'x');+} };+`);+ const fixture = await import(pathToFileURL(fixturePath).href);+ const readFile = vi.mocked(fsPromises.readFile);+ const originalRead = readFile.getMockImplementation();+ let announceRead, resumeRead;+ const readStarted = new Promise(resolve => { announceRead = resolve; });+ const resume = new Promise(resolve => { resumeRead = resolve; });+ let paused = false;+ readFile.mockImplementation(async (path, ...args) => {+ const raw = await originalRead(path, ...args);+ if (String(path) === ownerPath && !paused) {+ paused = true;+ announceRead();+ await resume;+ }+ return raw;+ });+ const staleRun = runMigrations({ rootDir, migrationsDir, lockWaitMs: 0 });+ const staleResult = staleRun.then(value => ({ value }), error => ({ error }));+ let holder;+ try {+ await readStarted;+ holder = runMigrations({ rootDir, migrationsDir, lockWaitMs: 0 });+ await fixture.started;+ resumeRead();+ await Promise.race([staleResult, fixture.duplicate]);+ } finally {+ resumeRead();+ fixture.release();+ await Promise.allSettled([staleRun, holder]);+ readFile.mockImplementation(originalRead);+ }++ expect(fixture.entries).toBe(1);+ expect((await staleResult).error?.message).toMatch(/Another migration run/);+ expect(appliedCount()).toBe(1);+ expect(existsSync(lockDir)).toBe(false);+ });+it('reclaims a lock whose holder process is gone', async () => {mkdirSync(lockDir);// Above every platform's pid ceiling, so the liveness probe reports ESRCH.@@ -520,6 +610,26 @@ export default {expect(ran).toBe(1);expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('held for over 30 minutes'));});++ it('recovers an old ownerless acquisition using its age before the transition mkdir', async () => {+ mkdirSync(lockDir);+ const old = new Date(Date.now() - 31 * 60 * 1000);+ utimesSync(lockDir, old, old);++ expect(await runMigrations({ rootDir, migrationsDir, lockWaitMs: 0 })).toBe(1);+ expect(appliedCount()).toBe(1);+ expect(existsSync(lockDir)).toBe(false);+ });++ it('keeps an interrupted transition closed instead of racing its recovery', async () => {+ mkdirSync(join(lockDir, '.transition'), { recursive: true });+ writeFileSync(join(lockDir, 'owner.json'), JSON.stringify({ pid: 2147483646, startedAt: Date.now() }));++ await expect(runMigrations({ rootDir, migrationsDir, lockWaitMs: 0 }))+ .rejects.toThrow(/interrupted .transition/);+ expect(appliedCount()).toBe(0);+ expect(existsSync(join(lockDir, '.transition'))).toBe(true);+ });});describe('sourceDeclaresPurge', () => {