Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

refactor: export the migration readJson and dashboard-grid helpers from _lib.js

[._.]/ Adam Eivy217d86c87f92
diff --git a/scripts/migrations/_lib.js b/scripts/migrations/_lib.js
index 47715512dc..daf412e60d 100644
--- a/scripts/migrations/_lib.js
+++ b/scripts/migrations/_lib.js
@@ -40,6 +40,11 @@
* migration needs to honour an install-specific path, across both the
* pre- and post-339 layouts (`data/cos/state.json`'s `config` slice and
* `data/cos/config.json`).
+ * 9. Plain document reads — `readJsonOrNull` is the "absent or unparseable
+ * → null, anything else throws" reader that thirty-odd applied migrations
+ * each declared as a local `readJson`. Those stay frozen;
+ * `migrationConventions.guard.test.js` stops a new migration from
+ * declaring its own copy of any helper exported here.
*
* Families 5, 7, and 7b all target `data/providers.json` and share its
* read → parse → shape-guard preamble via `readProvidersDoc`; each still owns
@@ -142,6 +147,23 @@ export const writeTextAtomic = async (path, content) => {
export const writeJsonAtomic = (path, value) =>
writeTextAtomic(path, JSON.stringify(value, null, 2) + '\n');
+/**
+ * Read a JSON document, or null when it is absent (ENOENT) or not valid JSON.
+ * Any other read error (EACCES, EISDIR, …) propagates: a store that exists
+ * but cannot be read is not "nothing to migrate", and treating it so would
+ * let a later write replace it. A migration that must tell "absent" from
+ * "unreadable" apart (to skip one and warn on the other) reads through
+ * `readJSONFileStrict` from `server/lib/fileUtils.js` instead.
+ */
+export const readJsonOrNull = async (path) => {
+ const raw = await readFile(path, 'utf-8').catch((err) => {
+ if (err.code === 'ENOENT') return null;
+ throw err;
+ });
+ if (raw == null) return null;
+ try { return JSON.parse(raw); } catch { return null; }
+};
+
// Reserved keys a legacy `map`-shape doc can carry as own properties (JSON.parse
// surfaces a literal `"__proto__"` key as an own property). They pass a typical
// `idPattern`, so a map split must drop them explicitly rather than write a
@@ -574,6 +596,61 @@ export async function writeLayoutsDoc(path, doc) {
await writeTextAtomic(path, JSON.stringify(doc, null, 2));
}
+// ---- dashboard-grid helpers ----
+//
+// Two grid shapes exist in `data/dashboard-layouts.json` history. Before
+// migration 269 every grid entry was a rectangle (`{ id, x, y, w, h }`) and a
+// seed had to find a free slot; 269 converted grids to a packing sequence
+// (`{ id, x, w, h, order }`) where a seed appends at `max(order) + 1`. The
+// rectangle helpers are kept for a migration that has to read or repair a
+// legacy rectangle grid; a new widget seed uses `applyToLayout` (or the
+// `makeWidgetSeedMigration` factory around it).
+
+/** Axis-aligned overlap of two `{ x, y, w, h }` rectangles. */
+export function rectsOverlap(a, b) {
+ return a.x < b.x + b.w && b.x < a.x + a.w && a.y < b.y + b.h && b.y < a.y + a.h;
+}
+
+/** Whether `candidate` overlaps any entry of a rectangle grid. */
+export function collidesWith(grid, candidate) {
+ return grid.some((item) => rectsOverlap(item, candidate));
+}
+
+/**
+ * Place `widgetId` at the preferred rectangle `slot` when it is free, else on
+ * a clean row below everything in the grid (a user rearranged the layout but
+ * kept the built-in id).
+ */
+export function pickGridEntry(grid, widgetId, slot) {
+ const candidate = { id: widgetId, ...slot };
+ if (!collidesWith(grid, candidate)) return candidate;
+ const bottom = grid.reduce((max, it) => Math.max(max, (it.y ?? 0) + (it.h ?? 0)), 0);
+ return { id: widgetId, x: 0, y: bottom, w: slot.w, h: slot.h };
+}
+
+/**
+ * Seed `widgetId` into one layout of the post-269 (`order`-packed) shape:
+ * append it to `widgets` when absent and to the end of the packing sequence
+ * when it has no grid entry. Returns whether the layout changed. `targets`
+ * (a Set of layout ids) limits the seed to built-in layouts; custom/user
+ * layouts are the layout editor's to manage. Idempotent — a rerun heals
+ * whichever of the two arrays is missing the widget.
+ */
+export function applyToLayout(layout, { widgetId, cell, targets }) {
+ if (!layout || !targets.has(layout.id) || !Array.isArray(layout.widgets)) return false;
+ const hasWidget = layout.widgets.includes(widgetId);
+ const hasGridEntry = Array.isArray(layout.grid) && layout.grid.some((item) => item?.id === widgetId);
+ if (hasWidget && hasGridEntry) return false;
+
+ if (!hasWidget) layout.widgets = [...layout.widgets, widgetId];
+ if (!hasGridEntry) {
+ const grid = Array.isArray(layout.grid) ? layout.grid : [];
+ const maxOrder = grid.reduce((max, item) => Math.max(max, Number.isFinite(item?.order) ? item.order : -1), -1);
+ layout.grid = [...grid, { id: widgetId, x: 0, w: cell.w, order: maxOrder + 1, h: cell.h }];
+ }
+ return true;
+}
+
/**
* Factory for the "seed a new dashboard widget into built-in layouts"
* migration family (327 seeded today-agenda by hand; 329 onward use this).
@@ -597,21 +674,6 @@ export async function writeLayoutsDoc(path, doc) {
export function makeWidgetSeedMigration({ label, widgetId, layoutIds = ['default', 'morning-review'], cell, logLine }) {
const targets = new Set(layoutIds);
- const applyToLayout = (layout) => {
- if (!layout || !targets.has(layout.id) || !Array.isArray(layout.widgets)) return false;
- const hasWidget = layout.widgets.includes(widgetId);
- const hasGridEntry = Array.isArray(layout.grid) && layout.grid.some((item) => item?.id === widgetId);
- if (hasWidget && hasGridEntry) return false;
-
- if (!hasWidget) layout.widgets = [...layout.widgets, widgetId];
- if (!hasGridEntry) {
- const grid = Array.isArray(layout.grid) ? layout.grid : [];
- const maxOrder = grid.reduce((max, item) => Math.max(max, Number.isFinite(item?.order) ? item.order : -1), -1);
- layout.grid = [...grid, { id: widgetId, x: 0, w: cell.w, order: maxOrder + 1, h: cell.h }];
- }
- return true;
- };
-
return {
async up({ rootDir }) {
const result = await readLayoutsDoc({ rootDir, label });
@@ -619,7 +681,7 @@ export function makeWidgetSeedMigration({ label, widgetId, layoutIds = ['default
const { doc, path } = result;
let updated = 0;
for (const layout of doc.layouts) {
- if (applyToLayout(layout)) updated += 1;
+ if (applyToLayout(layout, { widgetId, cell, targets })) updated += 1;
}
if (updated === 0) return { updated: 0, reason: 'already-applied' };
await writeLayoutsDoc(path, doc);
diff --git a/scripts/migrations/_lib.test.js b/scripts/migrations/_lib.test.js
index e95ec669d4..3004703e01 100644
--- a/scripts/migrations/_lib.test.js
+++ b/scripts/migrations/_lib.test.js
@@ -21,7 +21,7 @@ import { mkdtempSync, rmSync, writeFileSync, readFileSync, readdirSync, mkdirSyn
import { tmpdir } from 'os';
import { join } from 'path';
-import { applyPromptReplaceMigration, md5, readLayoutsDoc, writeLayoutsDoc, makeSplitMigration, makeProviderSeedMigration, makeSeededProviderTierMigration, seededProviderTierModels } from './_lib.js';
+import { applyPromptReplaceMigration, md5, readLayoutsDoc, writeLayoutsDoc, makeSplitMigration, makeProviderSeedMigration, makeSeededProviderTierMigration, seededProviderTierModels, readJsonOrNull, pickGridEntry, makeWidgetSeedMigration } from './_lib.js';
import { runSeededProviderTierMigrationTests } from './_testHelpers.js';
import sonnet5Migration from './153-claude-default-sonnet-5.js';
import opus5Migration from './206-claude-default-opus-5.js';
@@ -264,6 +264,74 @@ describe('readLayoutsDoc / writeLayoutsDoc', () => {
// end-to-end in their own suites; these tests pin the DISTINGUISHING FLAGS in
// isolation so a future edit to makeSplitMigration that breaks one flag fails
// loudly here even if the per-migration suites still pass by coincidence.
+describe('readJsonOrNull', () => {
+ let dir;
+ beforeEach(() => { dir = mkdtempSync(join(tmpdir(), 'migration-readjson-')); });
+ afterEach(() => { rmSync(dir, { recursive: true, force: true }); });
+
+ it('returns null for an absent or unparseable document and the value otherwise', async () => {
+ writeFileSync(join(dir, 'bad.json'), '{ truncated');
+ writeFileSync(join(dir, 'ok.json'), '{"a":1}');
+ expect(await readJsonOrNull(join(dir, 'missing.json'))).toBeNull();
+ expect(await readJsonOrNull(join(dir, 'bad.json'))).toBeNull();
+ expect(await readJsonOrNull(join(dir, 'ok.json'))).toEqual({ a: 1 });
+ });
+
+ it('propagates a read error that is not ENOENT instead of reporting "absent"', async () => {
+ // A directory at the path exists but cannot be read as a file (EISDIR);
+ // collapsing that into null would let a migration overwrite it.
+ await expect(readJsonOrNull(dir)).rejects.toMatchObject({ code: expect.stringMatching(/EISDIR|EPERM|EACCES/) });
+ });
+});
+
+describe('dashboard-grid helpers', () => {
+ it('pickGridEntry keeps a free slot and otherwise drops below the occupied grid', () => {
+ const grid = [{ id: 'a', x: 0, y: 0, w: 4, h: 2 }, { id: 'b', x: 4, y: 0, w: 4, h: 3 }];
+ expect(pickGridEntry(grid, 'c', { x: 8, y: 0, w: 4, h: 2 })).toEqual({ id: 'c', x: 8, y: 0, w: 4, h: 2 });
+ expect(pickGridEntry(grid, 'c', { x: 2, y: 1, w: 4, h: 2 })).toEqual({ id: 'c', x: 0, y: 3, w: 4, h: 2 });
+ });
+
+ describe('makeWidgetSeedMigration', () => {
+ let rootDir;
+ let layoutsPath;
+ const migration = makeWidgetSeedMigration({ label: 'migration test', widgetId: 'new-widget', cell: { w: 4, h: 3 }, logLine: 'seeded in' });
+ const seedDoc = () => writeFileSync(layoutsPath, JSON.stringify({
+ layouts: [
+ { id: 'default', widgets: ['clock'], grid: [{ id: 'clock', x: 0, w: 4, h: 2, order: 0 }] },
+ { id: 'morning-review', widgets: ['new-widget'], grid: [] },
+ { id: 'custom-1', widgets: ['clock'], grid: [{ id: 'clock', x: 0, w: 4, h: 2, order: 0 }] },
+ ],
+ }));
+ const readDoc = () => JSON.parse(readFileSync(layoutsPath, 'utf-8'));
+
+ beforeEach(() => {
+ rootDir = mkdtempSync(join(tmpdir(), 'migration-widget-seed-'));
+ mkdirSync(join(rootDir, 'data'), { recursive: true });
+ layoutsPath = join(rootDir, 'data', 'dashboard-layouts.json');
+ seedDoc();
+ });
+ afterEach(() => { rmSync(rootDir, { recursive: true, force: true }); });
+
+ it('appends the widget to the built-in layouts only, after the last packed order, and heals a widgets-only entry', async () => {
+ expect(await migration.up({ rootDir })).toEqual({ updated: 2 });
+ const [dflt, morning, custom] = readDoc().layouts;
+ expect(dflt.widgets).toEqual(['clock', 'new-widget']);
+ expect(dflt.grid.at(-1)).toEqual({ id: 'new-widget', x: 0, w: 4, order: 1, h: 3 });
+ expect(morning.widgets).toEqual(['new-widget']);
+ expect(morning.grid).toEqual([{ id: 'new-widget', x: 0, w: 4, order: 0, h: 3 }]);
+ expect(custom.widgets).toEqual(['clock']);
+ expect(custom.grid).toHaveLength(1);
+ });
+
+ it('reports already-applied without rewriting on a rerun', async () => {
+ await migration.up({ rootDir });
+ const before = readFileSync(layoutsPath, 'utf-8');
+ expect(await migration.up({ rootDir })).toEqual({ updated: 0, reason: 'already-applied' });
+ expect(readFileSync(layoutsPath, 'utf-8')).toBe(before);
+ });
+ });
+});
+
describe('makeSplitMigration flags', () => {
let rootDir;
let dataDir;
diff --git a/scripts/migrations/migrationConventions.guard.test.js b/scripts/migrations/migrationConventions.guard.test.js
new file mode 100644
index 0000000000..4529c7544e
--- /dev/null
+++ b/scripts/migrations/migrationConventions.guard.test.js
@@ -0,0 +1,109 @@
+/**
+ * Conventions for NEW migrations — files numbered at or above `GUARDED_FROM`.
+ * Applied migrations are frozen (other installs have already run them), so
+ * everything below that number is left exactly as shipped, duplicated helpers
+ * and all; the guard only stops the next migration from repeating the shape.
+ *
+ * Rules:
+ *
+ * 1. No local copy of a `_lib.js` helper (#12050). `readJson` was declared
+ * 35 times, `applyToLayout` 9, `pickGridEntry` 6, `rectsOverlap` /
+ * `collidesWith` 5 each before `_lib.js` exported them. A declaration
+ * whose name matches a `_lib.js` export — or one of the local spellings
+ * in `LOCAL_ALIASES` — must import the export instead.
+ */
+import { describe, it, expect } from 'vitest';
+import { readdirSync, readFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import * as lib from './_lib.js';
+
+const HERE = resolve(fileURLToPath(import.meta.url), '..');
+
+// First migration number the conventions apply to. Raise it only when a
+// migration below it is retired from the tree — never to exempt a new one.
+export const GUARDED_FROM = 429;
+
+const LIB_EXPORTS = new Set(Object.keys(lib));
+
+// Local names applied migrations used for a helper `_lib.js` now exports
+// under another name.
+const LOCAL_ALIASES = {
+ readJson: 'readJsonOrNull',
+ readJsonFile: 'readJsonOrNull',
+ readJsonSafe: 'readJsonOrNull',
+ writeJson: 'writeJsonAtomic',
+ writeJsonFile: 'writeJsonAtomic',
+ atomicWriteJson: 'writeJsonAtomic',
+ writeText: 'writeTextAtomic',
+ writeAtomic: 'writeTextAtomic',
+};
+
+const DECLARATION = /^\s*(?:export\s+)?(?:async\s+)?(?:const|let|var|function)\s+([A-Za-z_$][\w$]*)/gm;
+
+const stripComments = (source) => source
+ .replace(/\/\*[\s\S]*?\*\//g, '')
+ .replace(/(^|[^:'"`\\])\/\/.*$/gm, '$1');
+
+/** `{ name, use }` for every top-level or nested declaration that shadows a `_lib.js` helper. */
+const redeclaredLibHelpers = (source) => [...stripComments(source).matchAll(DECLARATION)]
+ .map(([, name]) => name)
+ .filter((name) => LIB_EXPORTS.has(name) || name in LOCAL_ALIASES)
+ .map((name) => ({ name, use: LOCAL_ALIASES[name] ?? name }));
+
+const migrationNumber = (file) => Number.parseInt(file.slice(0, 3), 10);
+
+const migrationFiles = () => readdirSync(HERE)
+ .filter((file) => /^\d{3}-.*\.js$/.test(file) && !file.endsWith('.test.js'))
+ .sort();
+
+const guardedSources = () => migrationFiles()
+ .filter((file) => migrationNumber(file) >= GUARDED_FROM)
+ .map((file) => [file, readFileSync(resolve(HERE, file), 'utf8')]);
+
+describe('new-migration conventions', () => {
+ it('starts guarding no later than the next migration number', () => {
+ const head = Math.max(...migrationFiles().map(migrationNumber));
+ expect(GUARDED_FROM).toBeLessThanOrEqual(head + 1);
+ });
+
+ it('no guarded migration declares its own copy of a _lib.js helper', () => {
+ const offenders = guardedSources()
+ .flatMap(([file, source]) => redeclaredLibHelpers(source).map(({ name, use }) => `${file}: ${name} — import ${use} from ./_lib.js`));
+ expect(offenders).toEqual([]);
+ });
+
+ describe('helper redeclaration matching', () => {
+ it('flags the local readJson spelling and points at readJsonOrNull', () => {
+ const source = [
+ "import { readFile } from 'fs/promises';",
+ 'async function readJson(path) {',
+ " const raw = await readFile(path, 'utf-8').catch(() => null);",
+ ' return raw == null ? null : JSON.parse(raw);',
+ '}',
+ ].join('\n');
+ expect(redeclaredLibHelpers(source)).toEqual([{ name: 'readJson', use: 'readJsonOrNull' }]);
+ });
+
+ it('flags dashboard-grid helpers whichever declaration form they take', () => {
+ const source = [
+ 'const rectsOverlap = (a, b) => a.x < b.x + b.w;',
+ 'export function collidesWith(grid, candidate) { return false; }',
+ 'function pickGridEntry(grid, widgetId, slot) { return slot; }',
+ 'function applyToLayout(layout) { return false; }',
+ ].join('\n');
+ expect(redeclaredLibHelpers(source).map(({ name }) => name))
+ .toEqual(['rectsOverlap', 'collidesWith', 'pickGridEntry', 'applyToLayout']);
+ });
+
+ it('passes a migration that imports the helpers and declares only its own data', () => {
+ const source = [
+ "import { readJsonOrNull, writeJsonAtomic, applyToLayout } from './_lib.js';",
+ "const SEEDS = [{ widgetId: 'example', cell: { w: 4, h: 4 } }];",
+ '// a local readJson would be flagged here',
+ 'export default { async up({ rootDir }) { const doc = await readJsonOrNull(rootDir); return doc; } };',
+ ].join('\n');
+ expect(redeclaredLibHelpers(source)).toEqual([]);
+ });
+ });
+});