Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
#!/usr/bin/env node
import { readFile, readdir, mkdir, rm, stat, writeFile, rename } from 'fs/promises';
import { basename, join, dirname } from 'path';
import { pathToFileURL } from 'url';
import { randomUUID } from 'crypto';
import { isWorktreeRoot } from '../server/lib/dataRoot.js';
import { isDirectlyInvoked } from './lib/directInvocation.js';
import {
DEFAULT_ROOT_DIR,
DEFAULT_MIGRATIONS_DIR,
adoptLegacyAppliedList,
appliedFilePath,
listPendingMigrations,
readAppliedList,
scanMigrationFiles,
} from './lib/migrationLedger.js';
// ---- purge-class probe (#12049) ----
//
// The empty-ledger guard below has to know which PENDING migrations are purge
// migrations before any of them runs. Importing each module to read its
// `purge` export loaded every pending migration's import graph up front — on a
// fresh install that is every migration, dozens of which pull in live server
// services before the toolkit has booted. The marker is read from the source
// text instead: a `purge: true` property at the two-space indentation of the
// default export's own properties (how every purge migration writes it), or
// an `export const purge = true`. Comments are stripped first so the rationale
// block every purge migration carries cannot match. A `purge: true` nested
// deeper inside a data literal is data, not the marker.
// `migrationConventions.guard.test.js` checks every shipped migration's module
// against this probe so the two cannot drift.
const stripComments = (source) => source
.replace(/\/\*[\s\S]*?\*\//g, '')
.replace(/(^|[^:'"`\\])\/\/.*$/gm, '$1');
const PURGE_PROPERTY = /^ {2}purge\s*:\s*true\s*,?\s*$/m;
const PURGE_EXPORT = /^export\s+const\s+purge\s*=\s*true\b/m;
/** Whether a migration's source text opts into the runner's purge class. */
export const sourceDeclaresPurge = (source) => {
const code = stripComments(source);
return PURGE_PROPERTY.test(code) || PURGE_EXPORT.test(code);
};
// ---- runner lock (#12046) ----
//
// The ledger is read-then-written by every runner: the booting server
// (bootstrap.js) and the documented manual `npm run migrations` both call
// runMigrations, and a PM2 restart can start a second server mid-run. Two
// runners that both read the ledger before either persists it each apply the
// same non-idempotent migration. A `mkdir` is atomic on every filesystem, so
// the lock is a directory; `owner.json` inside it carries the holder's pid and
// start time so a runner that died (SIGKILL, power loss) can be recognised and
// its lock reclaimed instead of blocking every later boot.
const lockDirPath = (rootDir) => join(rootDir, 'data', 'migrations.lock');
const LOCK_OWNER_FILE = 'owner.json';
const LOCK_STALE_MS = 30 * 60 * 1000;
const LOCK_WAIT_MS = 60 * 1000;
const LOCK_POLL_MS = 250;
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
// Signal 0 probes existence without delivering anything. EPERM means the pid
// exists but belongs to another user, which still counts as alive.
const pidAlive = (pid) => {
try { process.kill(pid, 0); return true; } catch (err) { return err.code === 'EPERM'; }
};
const readLockOwner = (lockDir) => readFile(join(lockDir, LOCK_OWNER_FILE), 'utf-8')
.then((raw) => {
// A reader can land between the holder's mkdir and a complete owner write;
// an unparseable file means "owner unknown", same as a missing one.
try { return JSON.parse(raw); } catch { return null; }
}, (err) => {
if (err.code === 'ENOENT') return null;
throw err;
})
.then((owner) => (owner && typeof owner === 'object' ? owner : null));
/**
* Why a held lock can be reclaimed, or null while its holder may still be
* running. The holder's own pid is never judged dead (an in-process second
* caller must wait, not steal), and a lock with no readable owner yet — the
* window between the holder's mkdir and its owner.json write — is judged by
* the directory's age alone.
*/
async function lockStaleReason(lockDir, unknownStartedAt) {
const owner = await readLockOwner(lockDir);
const pid = Number.isInteger(owner?.pid) && owner.pid > 0 ? owner.pid : null;
// Age cannot revoke a live runner: large upgrades can outlast the timeout.
if (pid !== null) return pidAlive(pid) ? null : `holder pid ${pid} is no longer running`;
const startedAt = Number.isFinite(owner?.startedAt)
? owner.startedAt
: (unknownStartedAt ?? await stat(lockDir).then((s) => s.mtimeMs, () => null));
if (startedAt !== null && Date.now() - startedAt > LOCK_STALE_MS) return `held for over ${LOCK_STALE_MS / 60000} minutes`;
return null;
}
/** Serialize release and stale recovery, including their ownership recheck. */
async function removeMigrationLock(lockDir, token) {
const observed = await stat(lockDir).catch(err => {
if (err.code === 'ENOENT') return null;
throw err;
});
if (!observed) return false;
const transition = join(lockDir, '.transition');
const taken = await mkdir(transition).then(() => true, err => {
if (err.code === 'EEXIST' || err.code === 'ENOENT') return false;
throw err;
});
if (!taken) return false;
let removed = false;
try {
const current = await stat(lockDir);
if (current.dev !== observed.dev || current.ino !== observed.ino) return false;
if (token) {
if ((await readLockOwner(lockDir))?.token !== token) {
throw new Error('Migration lock ownership changed; refusing to release another runner');
}
} else {
// mkdir(.transition) changed the directory mtime. Use its age BEFORE
// that mkdir when an interrupted acquisition left no readable owner.
const stale = await lockStaleReason(lockDir, observed.mtimeMs);
if (!stale) return false;
console.warn(`⚠️ Reclaiming stale migration lock ${lockDir}: ${stale}`);
}
// Keep the transition present until the shared path disappears atomically:
// recursive rm would remove it first and let another waiter enter mid-rm.
const retired = `${lockDir}.retired-${randomUUID()}`;
await rename(lockDir, retired);
removed = true;
await rm(retired, { recursive: true, force: true });
return true;
} finally {
// After deletion a new runner may already own this path. Never clean a
// transition in that replacement directory. An interrupted transition
// stays fail-closed and uses the runner's explicit manual-recovery error.
if (!removed) await rm(transition, { recursive: true, force: true });
}
}
/**
* Take `data/migrations.lock`; resolves to the release function. A second
* runner waits up to `waitMs` for the holder to finish, reclaims a stale lock
* on sight, and otherwise rejects with the holder's identity so the operator
* knows what to wait for rather than racing it.
*/
async function acquireMigrationLock(lockDir, { waitMs = LOCK_WAIT_MS, pollMs = LOCK_POLL_MS } = {}) {
const deadline = Date.now() + waitMs;
let announced = false;
for (;;) {
const taken = await mkdir(lockDir).then(() => true, (err) => {
if (err.code === 'EEXIST') return false;
throw err;
});
if (taken) {
// temp + rename so a concurrent runner sees no owner file or a whole one.
const ownerPath = join(lockDir, LOCK_OWNER_FILE);
const token = randomUUID();
await writeFile(`${ownerPath}.tmp`, JSON.stringify({ pid: process.pid, startedAt: Date.now(), token }));
await rename(`${ownerPath}.tmp`, ownerPath);
return async () => {
const releaseDeadline = Date.now() + waitMs;
while (!await removeMigrationLock(lockDir, token)) {
if (Date.now() >= releaseDeadline) {
throw new Error('Migration lock release blocked by an unfinished transition; stop all migration runners before recovering the lock');
}
await sleep(pollMs);
}
};
}
const stale = await lockStaleReason(lockDir);
if (stale && await removeMigrationLock(lockDir)) {
continue;
}
if (Date.now() >= deadline) {
const owner = await readLockOwner(lockDir);
throw new Error(
`Another migration run (pid ${owner?.pid ?? 'unknown'}) holds ${lockDir} — wait for it to finish (or stop the PortOS server) and retry; `
+ `an interrupted .transition also keeps the lock closed; remove the directory by hand only if no migration runner is still alive`,
);
}
if (!announced) {
console.log(`⏳ Waiting for another migration run holding ${lockDir}`);
announced = true;
}
await sleep(pollMs);
}
}
// The ledger read side lives in scripts/lib/migrationLedger.js; re-exported
// so existing `listPendingMigrations` callers keep this specifier.
export { listPendingMigrations };
export async function runMigrations({
rootDir = DEFAULT_ROOT_DIR,
migrationsDir = DEFAULT_MIGRATIONS_DIR,
lockWaitMs = LOCK_WAIT_MS,
lockPollMs = LOCK_POLL_MS,
// Stop after this migration file, leaving later ones pending. A restore
// replays a pre-split snapshot's pending migrations through its split this
// way, so the split sees what an in-order upgrade would have given it (#12813).
through = null,
} = {}) {
// Backstop for #1947: a process booted from inside a CoS agent git worktree
// (data/cos/worktrees/agent-*) has no gitignored data/ runtime tree, so
// migrations that read/write data/ or data.reference/ crash on ENOENT. A
// worktree checkout has no business running its own migration pass — skip it
// with a clear warning instead of crashing boot.
if (isWorktreeRoot(rootDir)) {
console.warn(`⚠️ Skipping migrations: rootDir is a CoS agent worktree checkout (${rootDir}) — no data/ tree to migrate`);
return 0;
}
const appliedFile = appliedFilePath(rootDir);
// Ensure data/ exists so we can persist applied state (migrationsDir
// ships in the repo and always exists).
await mkdir(dirname(appliedFile), { recursive: true });
// Everything from the ledger read to its last persist runs under the lock:
// the read is only trustworthy while no other runner can append to it.
const releaseLock = await acquireMigrationLock(lockDirPath(rootDir), { waitMs: lockWaitMs, pollMs: lockPollMs });
try {
return await runPendingMigrations({ rootDir, migrationsDir, appliedFile, through });
} finally {
await releaseLock();
}
}
async function runPendingMigrations({ rootDir, migrationsDir, appliedFile, through }) {
if (await adoptLegacyAppliedList(rootDir)) {
console.log('🧹 Moved data/migrations/.applied.json → data/migrations.applied.json');
}
// A runner killed between writing the ledger's temp file and renaming it
// leaves `migrations.applied.json.tmp-<pid>` behind, and no later run (a new
// pid) would ever reuse it. Remove only a dead writer's file: a lock reclaimed
// by age can still have a live holder mid-write.
const tmpPrefix = `${basename(appliedFile)}.tmp-`;
for (const name of await readdir(dirname(appliedFile))) {
if (!name.startsWith(tmpPrefix)) continue;
const pid = Number(name.slice(tmpPrefix.length));
if (!Number.isInteger(pid) || pid <= 0 || pid === process.pid || pidAlive(pid)) continue;
await rm(join(dirname(appliedFile), name), { force: true });
}
// Load applied migrations list (repairing a corrupt file aside — see
// readAppliedList).
const applied = await readAppliedList(appliedFile, { repair: true });
// Guard for #2770: when the applied-list started EMPTY — missing, corrupt and
// rebuilt from [], or a genuinely fresh install — we cannot prove that a
// non-idempotent PURGE migration hasn't already run. Rerunning one (they
// identify their targets by PRESENCE, e.g. a learning bucket) would delete
// legitimately-earned data accumulated since the purge first ran. So a
// migration that opts into the purge class (`export default { purge: true }`)
// is recorded as applied WITHOUT executing whenever the ledger started empty:
// on a fresh install there is nothing to purge anyway, and on a lost/corrupt
// ledger this protects post-fix data. The rare install that has genuine
// poisoned data AND lost its ledger before the purge ran keeps that stale data
// rather than losing real data — the deliberately safer trade. This is the
// single shared guard for every purge migration (197, 198, …) — no per-migration
// marker to drift out of sync.
const ledgerStartedEmpty = applied.length === 0;
const files = await scanMigrationFiles(migrationsDir);
if (through !== null && !files.includes(through)) {
throw new Error(`Cannot run migrations through "${through}": no such migration in ${migrationsDir}`);
}
// The migration helper also carries prompt-drift and provider-seed
// machinery. Boot/status callers import this runner widely, but only an
// actual migration write needs the atomic writer, so keep that subtree off
// the runner's normal static closure.
const persistAppliedList = async () => {
const { writeJsonAtomic } = await import('./migrations/_lib.js');
await writeJsonAtomic(appliedFile, applied);
};
// Retire this frozen transform before ANY pending migration executes, even
// with a rebuilt ledger or a later failure. Local absence cannot prove image
// deletion: host-mode peers intentionally retain metadata without bytes.
let disarmed = 0;
const retired = '428-prune-dangling-collection-images.js';
if (files.includes(retired) && !applied.includes(retired)) {
applied.push(retired);
disarmed++;
console.warn(`⏭️ Retired migration ${retired}: local image absence is not deletion evidence; recorded without running (#12398)`);
}
// Disarm EVERY pending purge migration up front, not as each is reached in
// the run loop: if this rebuilt-from-empty run aborted on an earlier
// throwing migration (the documented repair-and-reboot flow), the partial
// ledger would no longer "start empty" on the next boot — and the purge
// would then execute destructively against the rebuilt install. The probe
// reads source text, never importing a module before its turn in the loop.
if (ledgerStartedEmpty) {
for (const file of files) {
if (applied.includes(file)) continue;
if (!sourceDeclaresPurge(await readFile(join(migrationsDir, file), 'utf-8'))) continue;
console.warn(`⏭️ Skipping purge migration ${file}: applied-list started empty/rebuilt — recorded as applied without running to avoid a destructive rerun (#2770)`);
applied.push(file);
disarmed++;
}
}
// Publish both policies together: persisting retirement first would leave a
// nonempty ledger after a crash, re-arming pending purge-class migrations.
if (disarmed > 0) await persistAppliedList();
let ran = 0;
for (const file of through === null ? files : files.slice(0, files.indexOf(through) + 1)) {
if (applied.includes(file)) continue;
const mod = await import(pathToFileURL(join(migrationsDir, file)).href);
const migration = (mod?.default && typeof mod.default.up === 'function') ? mod.default : mod;
if (!migration || typeof migration.up !== 'function') {
throw new Error(`Migration "${file}" does not export an up() function`);
}
console.log(`🔄 Running migration: ${file}`);
await migration.up({ rootDir, migrationsDir });
applied.push(file);
await persistAppliedList();
ran++;
console.log(`✅ Migration applied: ${file}`);
}
if (ran === 0) {
console.log('✅ No pending migrations');
} else {
console.log(`✅ ${ran} migration(s) applied`);
}
return ran;
}
if (isDirectlyInvoked(import.meta.url)) {
runMigrations().catch(err => {
console.error(`❌ Migration failed: ${err?.stack ?? err}`);
process.exit(1);
});
}