Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
refactor: export the migration readJson and dashboard-grid helpers from _lib.js
[._.]/ Adam Eivy217d86c87f92
diff --git a/scripts/migrations/_lib.js b/scripts/migrations/_lib.jsindex 47715512dc..daf412e60d 100644--- a/scripts/migrations/_lib.js+++ b/scripts/migrations/_lib.js@@ -40,6 +40,11 @@* migration needs to honour an install-specific path, across both the* pre- and post-339 layouts (`data/cos/state.json`'s `config` slice and* `data/cos/config.json`).+ * 9. Plain document reads — `readJsonOrNull` is the "absent or unparseable+ * → null, anything else throws" reader that thirty-odd applied migrations+ * each declared as a local `readJson`. Those stay frozen;+ * `migrationConventions.guard.test.js` stops a new migration from+ * declaring its own copy of any helper exported here.** Families 5, 7, and 7b all target `data/providers.json` and share its* read → parse → shape-guard preamble via `readProvidersDoc`; each still owns@@ -142,6 +147,23 @@ export const writeTextAtomic = async (path, content) => {export const writeJsonAtomic = (path, value) =>writeTextAtomic(path, JSON.stringify(value, null, 2) + '\n');+/**+ * Read a JSON document, or null when it is absent (ENOENT) or not valid JSON.+ * Any other read error (EACCES, EISDIR, …) propagates: a store that exists+ * but cannot be read is not "nothing to migrate", and treating it so would+ * let a later write replace it. A migration that must tell "absent" from+ * "unreadable" apart (to skip one and warn on the other) reads through+ * `readJSONFileStrict` from `server/lib/fileUtils.js` instead.+ */+export const readJsonOrNull = async (path) => {+ const raw = await readFile(path, 'utf-8').catch((err) => {+ if (err.code === 'ENOENT') return null;+ throw err;+ });+ if (raw == null) return null;+ try { return JSON.parse(raw); } catch { return null; }+};+// Reserved keys a legacy `map`-shape doc can carry as own properties (JSON.parse// surfaces a literal `"__proto__"` key as an own property). They pass a typical// `idPattern`, so a map split must drop them explicitly rather than write a@@ -574,6 +596,61 @@ export async function writeLayoutsDoc(path, doc) {await writeTextAtomic(path, JSON.stringify(doc, null, 2));}+// ---- dashboard-grid helpers ----+//+// Two grid shapes exist in `data/dashboard-layouts.json` history. Before+// migration 269 every grid entry was a rectangle (`{ id, x, y, w, h }`) and a+// seed had to find a free slot; 269 converted grids to a packing sequence+// (`{ id, x, w, h, order }`) where a seed appends at `max(order) + 1`. The+// rectangle helpers are kept for a migration that has to read or repair a+// legacy rectangle grid; a new widget seed uses `applyToLayout` (or the+// `makeWidgetSeedMigration` factory around it).++/** Axis-aligned overlap of two `{ x, y, w, h }` rectangles. */+export function rectsOverlap(a, b) {+ return a.x < b.x + b.w && b.x < a.x + a.w && a.y < b.y + b.h && b.y < a.y + a.h;+}++/** Whether `candidate` overlaps any entry of a rectangle grid. */+export function collidesWith(grid, candidate) {+ return grid.some((item) => rectsOverlap(item, candidate));+}++/**+ * Place `widgetId` at the preferred rectangle `slot` when it is free, else on+ * a clean row below everything in the grid (a user rearranged the layout but+ * kept the built-in id).+ */+export function pickGridEntry(grid, widgetId, slot) {+ const candidate = { id: widgetId, ...slot };+ if (!collidesWith(grid, candidate)) return candidate;+ const bottom = grid.reduce((max, it) => Math.max(max, (it.y ?? 0) + (it.h ?? 0)), 0);+ return { id: widgetId, x: 0, y: bottom, w: slot.w, h: slot.h };+}++/**+ * Seed `widgetId` into one layout of the post-269 (`order`-packed) shape:+ * append it to `widgets` when absent and to the end of the packing sequence+ * when it has no grid entry. Returns whether the layout changed. `targets`+ * (a Set of layout ids) limits the seed to built-in layouts; custom/user+ * layouts are the layout editor's to manage. Idempotent — a rerun heals+ * whichever of the two arrays is missing the widget.+ */+export function applyToLayout(layout, { widgetId, cell, targets }) {+ if (!layout || !targets.has(layout.id) || !Array.isArray(layout.widgets)) return false;+ const hasWidget = layout.widgets.includes(widgetId);+ const hasGridEntry = Array.isArray(layout.grid) && layout.grid.some((item) => item?.id === widgetId);+ if (hasWidget && hasGridEntry) return false;++ if (!hasWidget) layout.widgets = [...layout.widgets, widgetId];+ if (!hasGridEntry) {+ const grid = Array.isArray(layout.grid) ? layout.grid : [];+ const maxOrder = grid.reduce((max, item) => Math.max(max, Number.isFinite(item?.order) ? item.order : -1), -1);+ layout.grid = [...grid, { id: widgetId, x: 0, w: cell.w, order: maxOrder + 1, h: cell.h }];+ }+ return true;+}+/*** Factory for the "seed a new dashboard widget into built-in layouts"* migration family (327 seeded today-agenda by hand; 329 onward use this).@@ -597,21 +674,6 @@ export async function writeLayoutsDoc(path, doc) {export function makeWidgetSeedMigration({ label, widgetId, layoutIds = ['default', 'morning-review'], cell, logLine }) {const targets = new Set(layoutIds);- const applyToLayout = (layout) => {- if (!layout || !targets.has(layout.id) || !Array.isArray(layout.widgets)) return false;- const hasWidget = layout.widgets.includes(widgetId);- const hasGridEntry = Array.isArray(layout.grid) && layout.grid.some((item) => item?.id === widgetId);- if (hasWidget && hasGridEntry) return false;-- if (!hasWidget) layout.widgets = [...layout.widgets, widgetId];- if (!hasGridEntry) {- const grid = Array.isArray(layout.grid) ? layout.grid : [];- const maxOrder = grid.reduce((max, item) => Math.max(max, Number.isFinite(item?.order) ? item.order : -1), -1);- layout.grid = [...grid, { id: widgetId, x: 0, w: cell.w, order: maxOrder + 1, h: cell.h }];- }- return true;- };-return {async up({ rootDir }) {const result = await readLayoutsDoc({ rootDir, label });@@ -619,7 +681,7 @@ export function makeWidgetSeedMigration({ label, widgetId, layoutIds = ['defaultconst { doc, path } = result;let updated = 0;for (const layout of doc.layouts) {- if (applyToLayout(layout)) updated += 1;+ if (applyToLayout(layout, { widgetId, cell, targets })) updated += 1;}if (updated === 0) return { updated: 0, reason: 'already-applied' };await writeLayoutsDoc(path, doc);diff --git a/scripts/migrations/_lib.test.js b/scripts/migrations/_lib.test.jsindex e95ec669d4..3004703e01 100644--- a/scripts/migrations/_lib.test.js+++ b/scripts/migrations/_lib.test.js@@ -21,7 +21,7 @@ import { mkdtempSync, rmSync, writeFileSync, readFileSync, readdirSync, mkdirSynimport { tmpdir } from 'os';import { join } from 'path';-import { applyPromptReplaceMigration, md5, readLayoutsDoc, writeLayoutsDoc, makeSplitMigration, makeProviderSeedMigration, makeSeededProviderTierMigration, seededProviderTierModels } from './_lib.js';+import { applyPromptReplaceMigration, md5, readLayoutsDoc, writeLayoutsDoc, makeSplitMigration, makeProviderSeedMigration, makeSeededProviderTierMigration, seededProviderTierModels, readJsonOrNull, pickGridEntry, makeWidgetSeedMigration } from './_lib.js';import { runSeededProviderTierMigrationTests } from './_testHelpers.js';import sonnet5Migration from './153-claude-default-sonnet-5.js';import opus5Migration from './206-claude-default-opus-5.js';@@ -264,6 +264,74 @@ describe('readLayoutsDoc / writeLayoutsDoc', () => {// end-to-end in their own suites; these tests pin the DISTINGUISHING FLAGS in// isolation so a future edit to makeSplitMigration that breaks one flag fails// loudly here even if the per-migration suites still pass by coincidence.+describe('readJsonOrNull', () => {+ let dir;+ beforeEach(() => { dir = mkdtempSync(join(tmpdir(), 'migration-readjson-')); });+ afterEach(() => { rmSync(dir, { recursive: true, force: true }); });++ it('returns null for an absent or unparseable document and the value otherwise', async () => {+ writeFileSync(join(dir, 'bad.json'), '{ truncated');+ writeFileSync(join(dir, 'ok.json'), '{"a":1}');+ expect(await readJsonOrNull(join(dir, 'missing.json'))).toBeNull();+ expect(await readJsonOrNull(join(dir, 'bad.json'))).toBeNull();+ expect(await readJsonOrNull(join(dir, 'ok.json'))).toEqual({ a: 1 });+ });++ it('propagates a read error that is not ENOENT instead of reporting "absent"', async () => {+ // A directory at the path exists but cannot be read as a file (EISDIR);+ // collapsing that into null would let a migration overwrite it.+ await expect(readJsonOrNull(dir)).rejects.toMatchObject({ code: expect.stringMatching(/EISDIR|EPERM|EACCES/) });+ });+});++describe('dashboard-grid helpers', () => {+ it('pickGridEntry keeps a free slot and otherwise drops below the occupied grid', () => {+ const grid = [{ id: 'a', x: 0, y: 0, w: 4, h: 2 }, { id: 'b', x: 4, y: 0, w: 4, h: 3 }];+ expect(pickGridEntry(grid, 'c', { x: 8, y: 0, w: 4, h: 2 })).toEqual({ id: 'c', x: 8, y: 0, w: 4, h: 2 });+ expect(pickGridEntry(grid, 'c', { x: 2, y: 1, w: 4, h: 2 })).toEqual({ id: 'c', x: 0, y: 3, w: 4, h: 2 });+ });++ describe('makeWidgetSeedMigration', () => {+ let rootDir;+ let layoutsPath;+ const migration = makeWidgetSeedMigration({ label: 'migration test', widgetId: 'new-widget', cell: { w: 4, h: 3 }, logLine: 'seeded in' });+ const seedDoc = () => writeFileSync(layoutsPath, JSON.stringify({+ layouts: [+ { id: 'default', widgets: ['clock'], grid: [{ id: 'clock', x: 0, w: 4, h: 2, order: 0 }] },+ { id: 'morning-review', widgets: ['new-widget'], grid: [] },+ { id: 'custom-1', widgets: ['clock'], grid: [{ id: 'clock', x: 0, w: 4, h: 2, order: 0 }] },+ ],+ }));+ const readDoc = () => JSON.parse(readFileSync(layoutsPath, 'utf-8'));++ beforeEach(() => {+ rootDir = mkdtempSync(join(tmpdir(), 'migration-widget-seed-'));+ mkdirSync(join(rootDir, 'data'), { recursive: true });+ layoutsPath = join(rootDir, 'data', 'dashboard-layouts.json');+ seedDoc();+ });+ afterEach(() => { rmSync(rootDir, { recursive: true, force: true }); });++ it('appends the widget to the built-in layouts only, after the last packed order, and heals a widgets-only entry', async () => {+ expect(await migration.up({ rootDir })).toEqual({ updated: 2 });+ const [dflt, morning, custom] = readDoc().layouts;+ expect(dflt.widgets).toEqual(['clock', 'new-widget']);+ expect(dflt.grid.at(-1)).toEqual({ id: 'new-widget', x: 0, w: 4, order: 1, h: 3 });+ expect(morning.widgets).toEqual(['new-widget']);+ expect(morning.grid).toEqual([{ id: 'new-widget', x: 0, w: 4, order: 0, h: 3 }]);+ expect(custom.widgets).toEqual(['clock']);+ expect(custom.grid).toHaveLength(1);+ });++ it('reports already-applied without rewriting on a rerun', async () => {+ await migration.up({ rootDir });+ const before = readFileSync(layoutsPath, 'utf-8');+ expect(await migration.up({ rootDir })).toEqual({ updated: 0, reason: 'already-applied' });+ expect(readFileSync(layoutsPath, 'utf-8')).toBe(before);+ });+ });+});+describe('makeSplitMigration flags', () => {let rootDir;let dataDir;diff --git a/scripts/migrations/migrationConventions.guard.test.js b/scripts/migrations/migrationConventions.guard.test.jsnew file mode 100644index 0000000000..4529c7544e--- /dev/null+++ b/scripts/migrations/migrationConventions.guard.test.js@@ -0,0 +1,109 @@+/**+ * Conventions for NEW migrations — files numbered at or above `GUARDED_FROM`.+ * Applied migrations are frozen (other installs have already run them), so+ * everything below that number is left exactly as shipped, duplicated helpers+ * and all; the guard only stops the next migration from repeating the shape.+ *+ * Rules:+ *+ * 1. No local copy of a `_lib.js` helper (#12050). `readJson` was declared+ * 35 times, `applyToLayout` 9, `pickGridEntry` 6, `rectsOverlap` /+ * `collidesWith` 5 each before `_lib.js` exported them. A declaration+ * whose name matches a `_lib.js` export — or one of the local spellings+ * in `LOCAL_ALIASES` — must import the export instead.+ */+import { describe, it, expect } from 'vitest';+import { readdirSync, readFileSync } from 'node:fs';+import { resolve } from 'node:path';+import { fileURLToPath } from 'node:url';+import * as lib from './_lib.js';++const HERE = resolve(fileURLToPath(import.meta.url), '..');++// First migration number the conventions apply to. Raise it only when a+// migration below it is retired from the tree — never to exempt a new one.+export const GUARDED_FROM = 429;++const LIB_EXPORTS = new Set(Object.keys(lib));++// Local names applied migrations used for a helper `_lib.js` now exports+// under another name.+const LOCAL_ALIASES = {+ readJson: 'readJsonOrNull',+ readJsonFile: 'readJsonOrNull',+ readJsonSafe: 'readJsonOrNull',+ writeJson: 'writeJsonAtomic',+ writeJsonFile: 'writeJsonAtomic',+ atomicWriteJson: 'writeJsonAtomic',+ writeText: 'writeTextAtomic',+ writeAtomic: 'writeTextAtomic',+};++const DECLARATION = /^\s*(?:export\s+)?(?:async\s+)?(?:const|let|var|function)\s+([A-Za-z_$][\w$]*)/gm;++const stripComments = (source) => source+ .replace(/\/\*[\s\S]*?\*\//g, '')+ .replace(/(^|[^:'"`\\])\/\/.*$/gm, '$1');++/** `{ name, use }` for every top-level or nested declaration that shadows a `_lib.js` helper. */+const redeclaredLibHelpers = (source) => [...stripComments(source).matchAll(DECLARATION)]+ .map(([, name]) => name)+ .filter((name) => LIB_EXPORTS.has(name) || name in LOCAL_ALIASES)+ .map((name) => ({ name, use: LOCAL_ALIASES[name] ?? name }));++const migrationNumber = (file) => Number.parseInt(file.slice(0, 3), 10);++const migrationFiles = () => readdirSync(HERE)+ .filter((file) => /^\d{3}-.*\.js$/.test(file) && !file.endsWith('.test.js'))+ .sort();++const guardedSources = () => migrationFiles()+ .filter((file) => migrationNumber(file) >= GUARDED_FROM)+ .map((file) => [file, readFileSync(resolve(HERE, file), 'utf8')]);++describe('new-migration conventions', () => {+ it('starts guarding no later than the next migration number', () => {+ const head = Math.max(...migrationFiles().map(migrationNumber));+ expect(GUARDED_FROM).toBeLessThanOrEqual(head + 1);+ });++ it('no guarded migration declares its own copy of a _lib.js helper', () => {+ const offenders = guardedSources()+ .flatMap(([file, source]) => redeclaredLibHelpers(source).map(({ name, use }) => `${file}: ${name} — import ${use} from ./_lib.js`));+ expect(offenders).toEqual([]);+ });++ describe('helper redeclaration matching', () => {+ it('flags the local readJson spelling and points at readJsonOrNull', () => {+ const source = [+ "import { readFile } from 'fs/promises';",+ 'async function readJson(path) {',+ " const raw = await readFile(path, 'utf-8').catch(() => null);",+ ' return raw == null ? null : JSON.parse(raw);',+ '}',+ ].join('\n');+ expect(redeclaredLibHelpers(source)).toEqual([{ name: 'readJson', use: 'readJsonOrNull' }]);+ });++ it('flags dashboard-grid helpers whichever declaration form they take', () => {+ const source = [+ 'const rectsOverlap = (a, b) => a.x < b.x + b.w;',+ 'export function collidesWith(grid, candidate) { return false; }',+ 'function pickGridEntry(grid, widgetId, slot) { return slot; }',+ 'function applyToLayout(layout) { return false; }',+ ].join('\n');+ expect(redeclaredLibHelpers(source).map(({ name }) => name))+ .toEqual(['rectsOverlap', 'collidesWith', 'pickGridEntry', 'applyToLayout']);+ });++ it('passes a migration that imports the helpers and declares only its own data', () => {+ const source = [+ "import { readJsonOrNull, writeJsonAtomic, applyToLayout } from './_lib.js';",+ "const SEEDS = [{ widgetId: 'example', cell: { w: 4, h: 4 } }];",+ '// a local readJson would be flagged here',+ 'export default { async up({ rootDir }) { const doc = await readJsonOrNull(rootDir); return doc; } };',+ ].join('\n');+ expect(redeclaredLibHelpers(source)).toEqual([]);+ });+ });+});