Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
Split the migration ledger reader out of the runner
[._.]/ Adam Eivy5171cb42d5ed
diff --git a/scripts/lib/migrationLedger.js b/scripts/lib/migrationLedger.jsnew file mode 100644index 0000000000..60948eec52--- /dev/null+++ b/scripts/lib/migrationLedger.js@@ -0,0 +1,88 @@+import { readdir, readFile, rename } from 'fs/promises';+import { join, dirname } from 'path';+import { fileURLToPath } from 'url';+import { resolveInstallRoot } from '../../server/lib/dataRoot.js';++// The read side of the migration ledger, split out of scripts/run-migrations.js+// so status callers (server/services/installState.js, reached by ~1,100 suites+// through peerExecutionAdapters' lazy import) do not put the runner in their+// `vitest related` graph: a runner-only change would otherwise select them all.++const __dirname = dirname(fileURLToPath(import.meta.url));+// Prefer an explicit PORTOS_DATA_ROOT env var over the executing-file location+// so a process booted from inside a CoS agent git worktree still resolves to+// the real install (#1947). resolveInstallRoot refuses the pin when the derived+// path is itself a worktree, so a worktree launch falls through to its own path+// and the runner's isWorktreeRoot backstop skips. Falls back to the derived+// path when the env var is unset.+export const DEFAULT_ROOT_DIR = resolveInstallRoot(join(__dirname, '..', '..'));+export const DEFAULT_MIGRATIONS_DIR = join(__dirname, '..', 'migrations');++export const appliedFilePath = (rootDir) => join(rootDir, 'data', 'migrations.applied.json');++/**+ * Scan for numbered migration files (NNN-name.js, sorted by filename). Co-located vitest+ * files (*.test.js) are excluded — they don't export `up()` and would+ * throw the runner if imported as migrations. The vitest config picks+ * them up via its own scripts test glob in server/vitest.config.js.+ * Unnumbered modules include shared helpers and executable tooling; importing+ * them here can run unrelated checks or exit the process before migrations finish.+ */+export async function scanMigrationFiles(migrationsDir) {+ return (await readdir(migrationsDir))+ .filter(f => /^\d+-.*\.js$/.test(f) && !f.endsWith('.test.js'))+ .sort();+}++/**+ * Read the applied-migrations list. A missing file defaults to [].+ *+ * When `repair` is true and the file is corrupt (mid-write truncation, bad+ * JSON, wrong shape), rename it aside and rebuild the ledger from scratch so+ * corruption does not brick future boots. Non-ENOENT read errors+ * propagate in repair mode: treating an unreadable ledger as empty could rerun+ * migrations against live data. When `repair` is false (read-only callers like+ * listPendingMigrations), read errors and corrupt content are treated as `[]`+ * WITHOUT mutating anything on disk.+ */+export async function readAppliedList(appliedFile, { repair = false } = {}) {+ const raw = await readFile(appliedFile, 'utf-8').catch(err => {+ if (err.code !== 'ENOENT' && repair) throw err;+ return null;+ });+ if (raw === null) return [];++ let parsed;+ let corruptReason = null;+ try { parsed = JSON.parse(raw); } catch (err) {+ corruptReason = `invalid JSON: ${err.message}`;+ }+ if (corruptReason === null && !Array.isArray(parsed)) {+ corruptReason = `expected array, got ${typeof parsed}`;+ }+ if (corruptReason !== null) {+ if (repair) {+ const aside = `${appliedFile}.corrupt-${new Date().toISOString().replace(/[:.]/g, '-')}`;+ await rename(appliedFile, aside);+ console.warn(`⚠️ Corrupt migrations file ${appliedFile} (${corruptReason}); renamed to ${aside} and rebuilding from scratch`);+ }+ return [];+ }+ return parsed;+}++/**+ * List migration files present on disk that are NOT yet in the applied-list —+ * i.e. migrations a boot would run. Pure read: never applies, renames, or+ * writes anything (so it's safe to call from a status endpoint). Returns the+ * pending filenames in sorted (apply) order.+ */+export async function listPendingMigrations({+ rootDir = DEFAULT_ROOT_DIR,+ migrationsDir = DEFAULT_MIGRATIONS_DIR,+} = {}) {+ const applied = await readAppliedList(appliedFilePath(rootDir), { repair: false });+ const appliedSet = new Set(applied);+ const files = await scanMigrationFiles(migrationsDir).catch(() => []);+ return files.filter(f => !appliedSet.has(f));+}diff --git a/scripts/run-migrations.js b/scripts/run-migrations.jsindex fdbf956121..8cad1346fb 100644--- a/scripts/run-migrations.js+++ b/scripts/run-migrations.js@@ -1,21 +1,17 @@#!/usr/bin/env node-import { readdir, readFile, mkdir, rename, rm, stat, writeFile } from 'fs/promises';+import { readFile, mkdir, rm, stat, writeFile } from 'fs/promises';import { join, dirname } from 'path';-import { fileURLToPath, pathToFileURL } from 'url';-import { resolveInstallRoot, isWorktreeRoot } from '../server/lib/dataRoot.js';+import { pathToFileURL } from 'url';+import { isWorktreeRoot } from '../server/lib/dataRoot.js';import { isDirectlyInvoked } from './lib/directInvocation.js';--const __dirname = dirname(fileURLToPath(import.meta.url));-// Prefer an explicit PORTOS_DATA_ROOT env var over the executing-file location-// so a process booted from inside a CoS agent git worktree still resolves to-// the real install (#1947). resolveInstallRoot refuses the pin when the derived-// path is itself a worktree, so a worktree launch falls through to its own path-// and the isWorktreeRoot backstop below skips. Falls back to the derived path-// when the env var is unset.-const DEFAULT_ROOT_DIR = resolveInstallRoot(join(__dirname, '..'));-const DEFAULT_MIGRATIONS_DIR = join(__dirname, 'migrations');--const appliedFilePath = (rootDir) => join(rootDir, 'data', 'migrations.applied.json');+import {+ DEFAULT_ROOT_DIR,+ DEFAULT_MIGRATIONS_DIR,+ appliedFilePath,+ listPendingMigrations,+ readAppliedList,+ scanMigrationFiles,+} from './lib/migrationLedger.js';// ---- purge-class probe (#12049) ----//@@ -128,72 +124,9 @@ async function acquireMigrationLock(lockDir, { waitMs = LOCK_WAIT_MS, pollMs = L}}-/**- * Scan for numbered migration files (NNN-name.js, sorted by filename). Co-located vitest- * files (*.test.js) are excluded — they don't export `up()` and would- * throw the runner if imported as migrations. The vitest config picks- * them up via its own scripts test glob in server/vitest.config.js.- * Unnumbered modules include shared helpers and executable tooling; importing- * them here can run unrelated checks or exit the process before migrations finish.- */-async function scanMigrationFiles(migrationsDir) {- return (await readdir(migrationsDir))- .filter(f => /^\d+-.*\.js$/.test(f) && !f.endsWith('.test.js'))- .sort();-}--/**- * Read the applied-migrations list. A missing file defaults to [].- *- * When `repair` is true and the file is corrupt (mid-write truncation, bad- * JSON, wrong shape), rename it aside and rebuild the ledger from scratch so- * corruption does not brick future boots. Non-ENOENT read errors- * propagate in repair mode: treating an unreadable ledger as empty could rerun- * migrations against live data. When `repair` is false (read-only callers like- * listPendingMigrations), read errors and corrupt content are treated as `[]`- * WITHOUT mutating anything on disk.- */-async function readAppliedList(appliedFile, { repair = false } = {}) {- const raw = await readFile(appliedFile, 'utf-8').catch(err => {- if (err.code !== 'ENOENT' && repair) throw err;- return null;- });- if (raw === null) return [];-- let parsed;- let corruptReason = null;- try { parsed = JSON.parse(raw); } catch (err) {- corruptReason = `invalid JSON: ${err.message}`;- }- if (corruptReason === null && !Array.isArray(parsed)) {- corruptReason = `expected array, got ${typeof parsed}`;- }- if (corruptReason !== null) {- if (repair) {- const aside = `${appliedFile}.corrupt-${new Date().toISOString().replace(/[:.]/g, '-')}`;- await rename(appliedFile, aside);- console.warn(`⚠️ Corrupt migrations file ${appliedFile} (${corruptReason}); renamed to ${aside} and rebuilding from scratch`);- }- return [];- }- return parsed;-}--/**- * List migration files present on disk that are NOT yet in the applied-list —- * i.e. migrations a boot would run. Pure read: never applies, renames, or- * writes anything (so it's safe to call from a status endpoint). Returns the- * pending filenames in sorted (apply) order.- */-export async function listPendingMigrations({- rootDir = DEFAULT_ROOT_DIR,- migrationsDir = DEFAULT_MIGRATIONS_DIR,-} = {}) {- const applied = await readAppliedList(appliedFilePath(rootDir), { repair: false });- const appliedSet = new Set(applied);- const files = await scanMigrationFiles(migrationsDir).catch(() => []);- return files.filter(f => !appliedSet.has(f));-}+// The ledger read side lives in scripts/lib/migrationLedger.js; re-exported+// so existing `listPendingMigrations` callers keep this specifier.+export { listPendingMigrations };export async function runMigrations({rootDir = DEFAULT_ROOT_DIR,diff --git a/server/lib/importScoping.test.js b/server/lib/importScoping.test.jsindex 80cf8c6bfb..f225440c35 100644--- a/server/lib/importScoping.test.js+++ b/server/lib/importScoping.test.js@@ -140,6 +140,10 @@ const NARROWED = ['needs isStr / trimTo, which textUtils.js declares'],['services/review.js', 'services/reviewActionAdapters.js','owns source-owned completion policy without the queue action adapters'],+ // A runner-only change selected ~1,100 suites through vitest related, which+ // also walks peerExecutionAdapters' lazy import of installState.js.+ ['services/installState.js', '../scripts/run-migrations.js',+ 'reads pending migrations through the scripts/lib/migrationLedger.js leaf'],];describe('narrowed imports stay narrow (#6009)', () => {@@ -174,6 +178,8 @@ describe('narrowed imports stay narrow (#6009)', () => {expect(reaches('services/voice/tools/pipeline.js', 'lib/pipelineStages.js')).toBe(true);expect(reaches('services/instances.js', 'services/instanceIdentity.js')).toBe(true);expect(reaches('lib/storyBible.js', 'lib/textUtils.js')).toBe(true);+ expect(reaches('services/installState.js', '../scripts/lib/migrationLedger.js')).toBe(true);+ expect(reaches('../scripts/run-migrations.js', '../scripts/lib/migrationLedger.js')).toBe(true);});it('keeps the process-type vocabulary dependency-free', () => {diff --git a/server/services/installState.js b/server/services/installState.jsindex 8d687b9a0a..d6c12aae27 100644--- a/server/services/installState.js+++ b/server/services/installState.js@@ -33,7 +33,7 @@ import { stat, readdir } from 'fs/promises';import { PATHS } from '../lib/fileUtils.js';import { execGit } from '../lib/execGit.js';import { parseSubmoduleStatusLine } from '../lib/gitOutputParsers.js';-import { listPendingMigrations } from '../../scripts/run-migrations.js';+import { listPendingMigrations } from '../../scripts/lib/migrationLedger.js';import { isWorktreeRoot } from '../lib/dataRoot.js';// The commit the running process was launched at. Captured once at boot (seediff --git a/server/services/installState.test.js b/server/services/installState.test.jsindex a84bef0222..2bb4dcc9bb 100644--- a/server/services/installState.test.js+++ b/server/services/installState.test.js@@ -6,7 +6,7 @@ import { join } from 'path';// Mock the migration-status reader so the migration-ledger-root test below can// assert which rootDir the DEFAULT wiring passes (the rest of the suite injects// its own `listPending`, so the mock is inert for them).-vi.mock('../../scripts/run-migrations.js', () => ({+vi.mock('../../scripts/lib/migrationLedger.js', () => ({listPendingMigrations: vi.fn(async () => [])}));@@ -30,7 +30,7 @@ import {__setBootCommitForTest,__internal} from './installState.js';-import { listPendingMigrations } from '../../scripts/run-migrations.js';+import { listPendingMigrations } from '../../scripts/lib/migrationLedger.js';import { PATHS } from '../lib/fileUtils.js';// All external effects are injected, so these tests never touch real git/fs.