Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: repair health files written as bare arrays by MortalLoom import (#13185)

Adam Eivyef93ba34443f
diff --git a/scripts/migrations/438-repair-legacy-health-array-files.js b/scripts/migrations/438-repair-legacy-health-array-files.js
new file mode 100644
index 0000000000..3d105dba1f
--- /dev/null
+++ b/scripts/migrations/438-repair-legacy-health-array-files.js
@@ -0,0 +1,118 @@
+/**
+ * Repair MortalLoom-import health files that were written as bare arrays.
+ *
+ * Earlier `importToPortOS` builds created a missing destination as a bare JSON
+ * array. The local readers (`meatspaceHealth.js`) expect object wrappers, so
+ * such a file parsed fine yet read as zero records:
+ *
+ * blood-tests.json -> { tests: [...], referenceRanges: {} }
+ * epigenetic-tests.json -> { tests: [...] }
+ * eyes.json -> { exams: [...] }
+ * health-metrics.json -> { entries: [...] }
+ *
+ * Blood rows from that import also kept MortalLoom's nested `markers`; the
+ * local reader consumes flat marker fields, so each is flattened (row `id`,
+ * `date` and every other field are kept).
+ *
+ * Safety contract:
+ * - Only a file whose whole content is a JSON array of plain objects is
+ * touched. Wrappers, absent files and any other shape are left byte-for-byte.
+ * - Invalid JSON, a non-object row, or ambiguous blood `markers` (not an
+ * object, or one that collides with a row field) refuses that file and
+ * logs; nothing is replaced with defaults.
+ * - The original bytes are copied to `<file>.pre-438.json` before the
+ * wrapper replaces the file (atomic rename). An existing backup is kept
+ * only when it is byte-identical to the current file; otherwise the file
+ * is refused rather than losing either copy.
+ * - An unreadable file (EACCES, EISDIR, ...) fails the migration after the
+ * other files are handled, so it is retried rather than marked applied.
+ * - Idempotent and interruption-safe: a repaired file is a wrapper (skipped);
+ * a crash after the backup leaves the array in place for the next run.
+ *
+ * This is a frozen transform: it does not import the live services. The repair
+ * is derived from the install's own rows, so it ships no `data.reference` seed.
+ */
+import { copyFile, readFile } from 'fs/promises';
+import { join } from 'path';
+import { writeJsonAtomic } from './_lib.js';
+
+const TARGETS = [
+ { file: 'blood-tests.json', key: 'tests', extra: () => ({ referenceRanges: {} }), flattenMarkers: true },
+ { file: 'epigenetic-tests.json', key: 'tests', extra: () => ({}) },
+ { file: 'eyes.json', key: 'exams', extra: () => ({}) },
+ { file: 'health-metrics.json', key: 'entries', extra: () => ({}) },
+];
+
+const BACKUP_SUFFIX = '.pre-438.json';
+// A marker named like a row field would silently overwrite or be overwritten.
+const RESERVED_ROW_KEYS = ['id', 'date', 'createdAt', 'updatedAt', 'aliasIds', 'markers'];
+
+const isObject = (value) => value !== null && typeof value === 'object' && !Array.isArray(value);
+
+/** Flat blood row, or null when `markers` is ambiguous. */
+function flattenBloodRow(row) {
+ if (!Object.hasOwn(row, 'markers') || row.markers == null) {
+ const { markers: _drop, ...rest } = row;
+ return rest;
+ }
+ const { markers, ...rest } = row;
+ if (!isObject(markers) || RESERVED_ROW_KEYS.some((key) => Object.hasOwn(markers, key))) return null;
+ return { ...markers, ...rest };
+}
+
+async function readBytes(path) {
+ try { return await readFile(path); } catch (err) {
+ if (err.code === 'ENOENT') return null;
+ throw err;
+ }
+}
+
+async function repairFile(dir, { file, key, extra, flattenMarkers }) {
+ const path = join(dir, file);
+ const bytes = await readBytes(path);
+ if (bytes === null) return;
+
+ let parsed;
+ try { parsed = JSON.parse(bytes.toString('utf-8')); } catch {
+ console.log(`⏭️ ${file}: not valid JSON; left untouched`);
+ return;
+ }
+ if (!Array.isArray(parsed)) return;
+
+ if (!parsed.every(isObject)) {
+ console.log(`⏭️ ${file}: array holds non-object rows; left untouched`);
+ return;
+ }
+ const rows = flattenMarkers ? parsed.map(flattenBloodRow) : parsed;
+ if (rows.includes(null)) {
+ console.log(`⏭️ ${file}: ambiguous blood markers; left untouched`);
+ return;
+ }
+
+ const backupPath = `${path}${BACKUP_SUFFIX}`;
+ const existingBackup = await readBytes(backupPath);
+ if (existingBackup === null) {
+ await copyFile(path, backupPath);
+ } else if (!existingBackup.equals(bytes)) {
+ console.log(`⏭️ ${file}: a different ${file}${BACKUP_SUFFIX} already exists; left untouched`);
+ return;
+ }
+
+ await writeJsonAtomic(path, { [key]: rows, ...extra() });
+ console.log(`📝 ${file}: wrapped ${rows.length} legacy array row(s) as { ${key} } (original kept as ${file}${BACKUP_SUFFIX})`);
+}
+
+export default {
+ async up({ rootDir }) {
+ const dir = join(rootDir, 'data', 'meatspace');
+ const failures = [];
+ for (const target of TARGETS) {
+ try { await repairFile(dir, target); } catch (err) {
+ failures.push(`${target.file}: ${err.message}`);
+ }
+ }
+ if (failures.length) {
+ throw new Error(`Legacy health array repair incomplete (will retry): ${failures.join('; ')}`);
+ }
+ },
+};
diff --git a/scripts/migrations/438-repair-legacy-health-array-files.test.js b/scripts/migrations/438-repair-legacy-health-array-files.test.js
new file mode 100644
index 0000000000..48b6991790
--- /dev/null
+++ b/scripts/migrations/438-repair-legacy-health-array-files.test.js
@@ -0,0 +1,117 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { mkdtempSync, rmSync, writeFileSync, readFileSync, mkdirSync, existsSync, chmodSync } from 'fs';
+import { tmpdir } from 'os';
+import { join } from 'path';
+
+import migration from './438-repair-legacy-health-array-files.js';
+
+describe('migration 438 — legacy bare-array health files', () => {
+ let rootDir, dir;
+ const p = (name) => join(dir, name);
+ const put = (name, value) => writeFileSync(p(name), typeof value === 'string' ? value : JSON.stringify(value));
+ const read = (name) => JSON.parse(readFileSync(p(name), 'utf-8'));
+
+ beforeEach(() => {
+ rootDir = mkdtempSync(join(tmpdir(), 'migration-438-'));
+ dir = join(rootDir, 'data', 'meatspace');
+ mkdirSync(dir, { recursive: true });
+ });
+ afterEach(() => rmSync(rootDir, { recursive: true, force: true }));
+
+ it('does nothing when no health files exist', async () => {
+ await migration.up({ rootDir });
+ expect(existsSync(p('eyes.json'))).toBe(false);
+ });
+
+ it('wraps every known legacy array in its reader contract, keeping rows and a backup', async () => {
+ const eyes = [{ id: 'e1', date: '2024-01-02', od: -1 }, { id: 'e2', date: '2024-06-01', od: -1.25 }];
+ const epi = [{ id: 'p1', date: '2024-03-01', biologicalAge: 40 }];
+ const metrics = [{ date: '2024-02-01', bloodPressureSystolic: 120, bloodPressureDiastolic: 80 }];
+ put('eyes.json', eyes);
+ put('epigenetic-tests.json', epi);
+ put('health-metrics.json', metrics);
+ const original = readFileSync(p('eyes.json'));
+
+ await migration.up({ rootDir });
+
+ expect(read('eyes.json')).toEqual({ exams: eyes });
+ expect(read('epigenetic-tests.json')).toEqual({ tests: epi });
+ expect(read('health-metrics.json')).toEqual({ entries: metrics });
+ expect(readFileSync(p('eyes.json.pre-438.json')).equals(original)).toBe(true);
+ });
+
+ it('flattens nested blood markers, keeping id, date and other fields', async () => {
+ put('blood-tests.json', [
+ { id: 'b1', date: '2024-01-01', lab: 'Acme', createdAt: 't', markers: { glucose: 90, ldl: 100 } },
+ { id: 'b2', date: '2024-02-01', hdl: 55 },
+ { id: 'b3', date: '2024-03-01', markers: null },
+ ]);
+ await migration.up({ rootDir });
+ expect(read('blood-tests.json')).toEqual({
+ tests: [
+ { id: 'b1', date: '2024-01-01', lab: 'Acme', createdAt: 't', glucose: 90, ldl: 100 },
+ { id: 'b2', date: '2024-02-01', hdl: 55 },
+ { id: 'b3', date: '2024-03-01' },
+ ],
+ referenceRanges: {},
+ });
+ });
+
+ it('leaves valid wrappers (and their unknown metadata) byte-for-byte unchanged', async () => {
+ const wrapped = '{"exams":[{"id":"e1","date":"2024-01-01"}],"_rowTombstones":{"x":1}}';
+ put('eyes.json', wrapped);
+ put('blood-tests.json', { tests: [], referenceRanges: { glucose: [70, 99] } });
+ const before = readFileSync(p('blood-tests.json'));
+ await migration.up({ rootDir });
+ expect(readFileSync(p('eyes.json'), 'utf-8')).toBe(wrapped);
+ expect(readFileSync(p('blood-tests.json')).equals(before)).toBe(true);
+ expect(existsSync(p('eyes.json.pre-438.json'))).toBe(false);
+ });
+
+ it('refuses malformed JSON, non-object rows and ambiguous markers without overwriting', async () => {
+ put('eyes.json', '[{"id":"e1"');
+ put('epigenetic-tests.json', [{ id: 'p1' }, 'stray']);
+ put('blood-tests.json', [{ id: 'b1', markers: { id: 'clash' } }]);
+ put('health-metrics.json', '"just a string"');
+ const before = ['eyes.json', 'epigenetic-tests.json', 'blood-tests.json', 'health-metrics.json']
+ .map((name) => [name, readFileSync(p(name), 'utf-8')]);
+
+ await migration.up({ rootDir });
+
+ for (const [name, text] of before) expect(readFileSync(p(name), 'utf-8')).toBe(text);
+ expect(existsSync(p('eyes.json.pre-438.json'))).toBe(false);
+ expect(existsSync(p('blood-tests.json.pre-438.json'))).toBe(false);
+ });
+
+ it('is safe to repeat and resumes after an interruption that left only the backup', async () => {
+ const rows = [{ id: 'e1', date: '2024-01-01' }];
+ put('eyes.json', rows);
+ writeFileSync(p('eyes.json.pre-438.json'), readFileSync(p('eyes.json'))); // crash after backup, before swap
+
+ await migration.up({ rootDir });
+ const once = readFileSync(p('eyes.json'), 'utf-8');
+ expect(JSON.parse(once)).toEqual({ exams: rows });
+
+ await migration.up({ rootDir });
+ expect(readFileSync(p('eyes.json'), 'utf-8')).toBe(once);
+ });
+
+ it('refuses when a different backup already occupies the backup path', async () => {
+ put('eyes.json', [{ id: 'e1' }]);
+ writeFileSync(p('eyes.json.pre-438.json'), 'someone else');
+ await migration.up({ rootDir });
+ expect(read('eyes.json')).toEqual([{ id: 'e1' }]);
+ expect(readFileSync(p('eyes.json.pre-438.json'), 'utf-8')).toBe('someone else');
+ });
+
+ it.skipIf(process.getuid?.() === 0)('fails (to be retried) on an unreadable file while still repairing the others', async () => {
+ put('eyes.json', [{ id: 'e1' }]);
+ put('health-metrics.json', [{ date: '2024-01-01' }]);
+ chmodSync(p('eyes.json'), 0o000);
+ try {
+ await expect(migration.up({ rootDir })).rejects.toThrow(/eyes\.json/);
+ } finally { chmodSync(p('eyes.json'), 0o600); }
+ expect(read('health-metrics.json')).toEqual({ entries: [{ date: '2024-01-01' }] });
+ expect(read('eyes.json')).toEqual([{ id: 'e1' }]);
+ });
+});