Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: repair health files written as bare arrays by MortalLoom import (#13185)
Adam Eivyef93ba34443f
diff --git a/scripts/migrations/438-repair-legacy-health-array-files.js b/scripts/migrations/438-repair-legacy-health-array-files.jsnew file mode 100644index 0000000000..3d105dba1f--- /dev/null+++ b/scripts/migrations/438-repair-legacy-health-array-files.js@@ -0,0 +1,118 @@+/**+ * Repair MortalLoom-import health files that were written as bare arrays.+ *+ * Earlier `importToPortOS` builds created a missing destination as a bare JSON+ * array. The local readers (`meatspaceHealth.js`) expect object wrappers, so+ * such a file parsed fine yet read as zero records:+ *+ * blood-tests.json -> { tests: [...], referenceRanges: {} }+ * epigenetic-tests.json -> { tests: [...] }+ * eyes.json -> { exams: [...] }+ * health-metrics.json -> { entries: [...] }+ *+ * Blood rows from that import also kept MortalLoom's nested `markers`; the+ * local reader consumes flat marker fields, so each is flattened (row `id`,+ * `date` and every other field are kept).+ *+ * Safety contract:+ * - Only a file whose whole content is a JSON array of plain objects is+ * touched. Wrappers, absent files and any other shape are left byte-for-byte.+ * - Invalid JSON, a non-object row, or ambiguous blood `markers` (not an+ * object, or one that collides with a row field) refuses that file and+ * logs; nothing is replaced with defaults.+ * - The original bytes are copied to `<file>.pre-438.json` before the+ * wrapper replaces the file (atomic rename). An existing backup is kept+ * only when it is byte-identical to the current file; otherwise the file+ * is refused rather than losing either copy.+ * - An unreadable file (EACCES, EISDIR, ...) fails the migration after the+ * other files are handled, so it is retried rather than marked applied.+ * - Idempotent and interruption-safe: a repaired file is a wrapper (skipped);+ * a crash after the backup leaves the array in place for the next run.+ *+ * This is a frozen transform: it does not import the live services. The repair+ * is derived from the install's own rows, so it ships no `data.reference` seed.+ */+import { copyFile, readFile } from 'fs/promises';+import { join } from 'path';+import { writeJsonAtomic } from './_lib.js';++const TARGETS = [+ { file: 'blood-tests.json', key: 'tests', extra: () => ({ referenceRanges: {} }), flattenMarkers: true },+ { file: 'epigenetic-tests.json', key: 'tests', extra: () => ({}) },+ { file: 'eyes.json', key: 'exams', extra: () => ({}) },+ { file: 'health-metrics.json', key: 'entries', extra: () => ({}) },+];++const BACKUP_SUFFIX = '.pre-438.json';+// A marker named like a row field would silently overwrite or be overwritten.+const RESERVED_ROW_KEYS = ['id', 'date', 'createdAt', 'updatedAt', 'aliasIds', 'markers'];++const isObject = (value) => value !== null && typeof value === 'object' && !Array.isArray(value);++/** Flat blood row, or null when `markers` is ambiguous. */+function flattenBloodRow(row) {+ if (!Object.hasOwn(row, 'markers') || row.markers == null) {+ const { markers: _drop, ...rest } = row;+ return rest;+ }+ const { markers, ...rest } = row;+ if (!isObject(markers) || RESERVED_ROW_KEYS.some((key) => Object.hasOwn(markers, key))) return null;+ return { ...markers, ...rest };+}++async function readBytes(path) {+ try { return await readFile(path); } catch (err) {+ if (err.code === 'ENOENT') return null;+ throw err;+ }+}++async function repairFile(dir, { file, key, extra, flattenMarkers }) {+ const path = join(dir, file);+ const bytes = await readBytes(path);+ if (bytes === null) return;++ let parsed;+ try { parsed = JSON.parse(bytes.toString('utf-8')); } catch {+ console.log(`⏭️ ${file}: not valid JSON; left untouched`);+ return;+ }+ if (!Array.isArray(parsed)) return;++ if (!parsed.every(isObject)) {+ console.log(`⏭️ ${file}: array holds non-object rows; left untouched`);+ return;+ }+ const rows = flattenMarkers ? parsed.map(flattenBloodRow) : parsed;+ if (rows.includes(null)) {+ console.log(`⏭️ ${file}: ambiguous blood markers; left untouched`);+ return;+ }++ const backupPath = `${path}${BACKUP_SUFFIX}`;+ const existingBackup = await readBytes(backupPath);+ if (existingBackup === null) {+ await copyFile(path, backupPath);+ } else if (!existingBackup.equals(bytes)) {+ console.log(`⏭️ ${file}: a different ${file}${BACKUP_SUFFIX} already exists; left untouched`);+ return;+ }++ await writeJsonAtomic(path, { [key]: rows, ...extra() });+ console.log(`📝 ${file}: wrapped ${rows.length} legacy array row(s) as { ${key} } (original kept as ${file}${BACKUP_SUFFIX})`);+}++export default {+ async up({ rootDir }) {+ const dir = join(rootDir, 'data', 'meatspace');+ const failures = [];+ for (const target of TARGETS) {+ try { await repairFile(dir, target); } catch (err) {+ failures.push(`${target.file}: ${err.message}`);+ }+ }+ if (failures.length) {+ throw new Error(`Legacy health array repair incomplete (will retry): ${failures.join('; ')}`);+ }+ },+};diff --git a/scripts/migrations/438-repair-legacy-health-array-files.test.js b/scripts/migrations/438-repair-legacy-health-array-files.test.jsnew file mode 100644index 0000000000..48b6991790--- /dev/null+++ b/scripts/migrations/438-repair-legacy-health-array-files.test.js@@ -0,0 +1,117 @@+import { describe, it, expect, beforeEach, afterEach } from 'vitest';+import { mkdtempSync, rmSync, writeFileSync, readFileSync, mkdirSync, existsSync, chmodSync } from 'fs';+import { tmpdir } from 'os';+import { join } from 'path';++import migration from './438-repair-legacy-health-array-files.js';++describe('migration 438 — legacy bare-array health files', () => {+ let rootDir, dir;+ const p = (name) => join(dir, name);+ const put = (name, value) => writeFileSync(p(name), typeof value === 'string' ? value : JSON.stringify(value));+ const read = (name) => JSON.parse(readFileSync(p(name), 'utf-8'));++ beforeEach(() => {+ rootDir = mkdtempSync(join(tmpdir(), 'migration-438-'));+ dir = join(rootDir, 'data', 'meatspace');+ mkdirSync(dir, { recursive: true });+ });+ afterEach(() => rmSync(rootDir, { recursive: true, force: true }));++ it('does nothing when no health files exist', async () => {+ await migration.up({ rootDir });+ expect(existsSync(p('eyes.json'))).toBe(false);+ });++ it('wraps every known legacy array in its reader contract, keeping rows and a backup', async () => {+ const eyes = [{ id: 'e1', date: '2024-01-02', od: -1 }, { id: 'e2', date: '2024-06-01', od: -1.25 }];+ const epi = [{ id: 'p1', date: '2024-03-01', biologicalAge: 40 }];+ const metrics = [{ date: '2024-02-01', bloodPressureSystolic: 120, bloodPressureDiastolic: 80 }];+ put('eyes.json', eyes);+ put('epigenetic-tests.json', epi);+ put('health-metrics.json', metrics);+ const original = readFileSync(p('eyes.json'));++ await migration.up({ rootDir });++ expect(read('eyes.json')).toEqual({ exams: eyes });+ expect(read('epigenetic-tests.json')).toEqual({ tests: epi });+ expect(read('health-metrics.json')).toEqual({ entries: metrics });+ expect(readFileSync(p('eyes.json.pre-438.json')).equals(original)).toBe(true);+ });++ it('flattens nested blood markers, keeping id, date and other fields', async () => {+ put('blood-tests.json', [+ { id: 'b1', date: '2024-01-01', lab: 'Acme', createdAt: 't', markers: { glucose: 90, ldl: 100 } },+ { id: 'b2', date: '2024-02-01', hdl: 55 },+ { id: 'b3', date: '2024-03-01', markers: null },+ ]);+ await migration.up({ rootDir });+ expect(read('blood-tests.json')).toEqual({+ tests: [+ { id: 'b1', date: '2024-01-01', lab: 'Acme', createdAt: 't', glucose: 90, ldl: 100 },+ { id: 'b2', date: '2024-02-01', hdl: 55 },+ { id: 'b3', date: '2024-03-01' },+ ],+ referenceRanges: {},+ });+ });++ it('leaves valid wrappers (and their unknown metadata) byte-for-byte unchanged', async () => {+ const wrapped = '{"exams":[{"id":"e1","date":"2024-01-01"}],"_rowTombstones":{"x":1}}';+ put('eyes.json', wrapped);+ put('blood-tests.json', { tests: [], referenceRanges: { glucose: [70, 99] } });+ const before = readFileSync(p('blood-tests.json'));+ await migration.up({ rootDir });+ expect(readFileSync(p('eyes.json'), 'utf-8')).toBe(wrapped);+ expect(readFileSync(p('blood-tests.json')).equals(before)).toBe(true);+ expect(existsSync(p('eyes.json.pre-438.json'))).toBe(false);+ });++ it('refuses malformed JSON, non-object rows and ambiguous markers without overwriting', async () => {+ put('eyes.json', '[{"id":"e1"');+ put('epigenetic-tests.json', [{ id: 'p1' }, 'stray']);+ put('blood-tests.json', [{ id: 'b1', markers: { id: 'clash' } }]);+ put('health-metrics.json', '"just a string"');+ const before = ['eyes.json', 'epigenetic-tests.json', 'blood-tests.json', 'health-metrics.json']+ .map((name) => [name, readFileSync(p(name), 'utf-8')]);++ await migration.up({ rootDir });++ for (const [name, text] of before) expect(readFileSync(p(name), 'utf-8')).toBe(text);+ expect(existsSync(p('eyes.json.pre-438.json'))).toBe(false);+ expect(existsSync(p('blood-tests.json.pre-438.json'))).toBe(false);+ });++ it('is safe to repeat and resumes after an interruption that left only the backup', async () => {+ const rows = [{ id: 'e1', date: '2024-01-01' }];+ put('eyes.json', rows);+ writeFileSync(p('eyes.json.pre-438.json'), readFileSync(p('eyes.json'))); // crash after backup, before swap++ await migration.up({ rootDir });+ const once = readFileSync(p('eyes.json'), 'utf-8');+ expect(JSON.parse(once)).toEqual({ exams: rows });++ await migration.up({ rootDir });+ expect(readFileSync(p('eyes.json'), 'utf-8')).toBe(once);+ });++ it('refuses when a different backup already occupies the backup path', async () => {+ put('eyes.json', [{ id: 'e1' }]);+ writeFileSync(p('eyes.json.pre-438.json'), 'someone else');+ await migration.up({ rootDir });+ expect(read('eyes.json')).toEqual([{ id: 'e1' }]);+ expect(readFileSync(p('eyes.json.pre-438.json'), 'utf-8')).toBe('someone else');+ });++ it.skipIf(process.getuid?.() === 0)('fails (to be retried) on an unreadable file while still repairing the others', async () => {+ put('eyes.json', [{ id: 'e1' }]);+ put('health-metrics.json', [{ date: '2024-01-01' }]);+ chmodSync(p('eyes.json'), 0o000);+ try {+ await expect(migration.up({ rootDir })).rejects.toThrow(/eyes\.json/);+ } finally { chmodSync(p('eyes.json'), 0o600); }+ expect(read('health-metrics.json')).toEqual({ entries: [{ date: '2024-01-01' }] });+ expect(read('eyes.json')).toEqual([{ id: 'e1' }]);+ });+});