Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: preserve health records during imports and concurrent saves

[._.]/ Adam Eivya9d598452dd7
diff --git a/server/services/meatspaceHealth.js b/server/services/meatspaceHealth.js
index 470eaed27f..b2e039aaf5 100644
--- a/server/services/meatspaceHealth.js
+++ b/server/services/meatspaceHealth.js
@@ -38,7 +38,7 @@ const WORKOUTS_FILE = join(MEATSPACE_DIR, 'workouts.json');
const byDate = (a, b) => (a.date || '').localeCompare(b.date || '');
-// Every read-modify-write of the three files below runs inside `queueHealthWrite`
+// Every read-modify-write of the local health files runs inside `queueHealthWrite`
// — the queue the federation snapshot apply (`dataSync.js`) also uses — so a peer
// snapshot cannot overwrite a record added between its read and its write.
// `writeLocal` itself is NOT queued: callers already hold the queue.
@@ -294,11 +294,13 @@ export async function addWorkout({ date, type, durationMinutes, intensity, notes
intensity: typeof intensity === 'string' && intensity.trim() ? intensity.trim() : null,
notes: typeof notes === 'string' && notes.trim() ? notes.trim() : null,
};
- const data = await readJSONFile(WORKOUTS_FILE, { workouts: [] });
- if (!Array.isArray(data.workouts)) data.workouts = [];
- data.workouts.push(entry);
- data.workouts.sort(byDate);
- await writeLocal(WORKOUTS_FILE, data);
+ await queueHealthWrite(async () => {
+ const data = await readJSONFile(WORKOUTS_FILE, { workouts: [] }, { strict: true });
+ if (!Array.isArray(data.workouts)) throw new Error('Workouts malformed');
+ data.workouts.push(entry);
+ data.workouts.sort(byDate);
+ await writeLocal(WORKOUTS_FILE, data);
+ });
console.log(`🏋️ Workout logged: ${trimmedType}${entry.durationMinutes ? ` (${entry.durationMinutes}min)` : ''} for ${targetDate}`);
return entry;
}
@@ -339,12 +341,15 @@ export async function addBloodPressureReading({ date, systolic, diastolic }) {
return { date: targetDate, systolic, diastolic };
}
- const log = await readJSONFile(HEALTH_METRICS_FILE, { entries: [] });
- let entry = log.entries.find(e => e.date === targetDate);
- if (!entry) { entry = { date: targetDate }; log.entries.push(entry); }
- Object.assign(entry, patch);
- log.entries.sort(byDate);
- await writeLocal(HEALTH_METRICS_FILE, log);
+ await queueHealthWrite(async () => {
+ const log = await readJSONFile(HEALTH_METRICS_FILE, { entries: [] }, { strict: true });
+ if (!Array.isArray(log.entries)) throw new Error('Health metrics malformed');
+ let entry = log.entries.find(e => e.date === targetDate);
+ if (!entry) { entry = { date: targetDate }; log.entries.push(entry); }
+ Object.assign(entry, patch);
+ log.entries.sort(byDate);
+ await writeLocal(HEALTH_METRICS_FILE, log);
+ });
console.log(`🩺 Blood pressure ${systolic}/${diastolic} logged for ${targetDate}`);
return { date: targetDate, systolic, diastolic };
}
diff --git a/server/services/meatspaceHealthIntegrity.test.js b/server/services/meatspaceHealthIntegrity.test.js
new file mode 100644
index 0000000000..1adb1cbf04
--- /dev/null
+++ b/server/services/meatspaceHealthIntegrity.test.js
@@ -0,0 +1,130 @@
+import { afterAll, beforeEach, expect, it, vi } from 'vitest';
+import { mkdir, readFile, rm, writeFile } from 'fs/promises';
+import { dirname, join } from 'path';
+import { cleanupTempDataRoots, lazyTempDataRoot, makePathsProxy } from '../lib/mockPathsDataRoot.js';
+
+const hooks = vi.hoisted(() => ({ afterRead: null }));
+vi.mock('../lib/fileUtils.js', async (original) => {
+ const actual = await original();
+ return makePathsProxy(actual, {
+ dataRoot: () => lazyTempDataRoot('portos-mortalloom-import-'),
+ overrides: {
+ dataPath: (...parts) => join(lazyTempDataRoot('portos-mortalloom-import-'), ...parts),
+ readJSONFile: async (...args) => {
+ const value = await actual.readJSONFile(...args);
+ await hooks.afterRead?.(args[0]);
+ return value;
+ },
+ },
+ });
+});
+vi.mock('../lib/paths.js', async (original) => makePathsProxy(await original(), {
+ dataRoot: () => lazyTempDataRoot('portos-mortalloom-import-'),
+}));
+vi.mock('./settings.js', async () => {
+ const { EventEmitter } = await import('events');
+ return { settingsEvents: new EventEmitter(), getSettings: async () => ({
+ mortalloom: { enabled: false, path: join(lazyTempDataRoot('portos-mortalloom-import-'), 'source.json') },
+ }) };
+});
+
+import { PATHS } from '../lib/fileUtils.js';
+import { importToPortOS } from './mortalLoomStore.js';
+import { addBloodTest, addBloodPressureReading, addWorkout, getBloodTests, getEpigeneticTests, getEyeExams, getBloodPressureHistory, getWorkouts } from './meatspaceHealth.js';
+
+const localPath = (name) => join(PATHS.meatspace, name);
+async function seed(path, value) {
+ await mkdir(dirname(path), { recursive: true });
+ await writeFile(path, JSON.stringify(value));
+}
+const source = (value) => seed(join(PATHS.data, 'source.json'), value);
+const read = async (name) => JSON.parse(await readFile(localPath(name), 'utf8'));
+
+beforeEach(async () => {
+ hooks.afterRead = null;
+ await rm(PATHS.data, { recursive: true, force: true });
+ await mkdir(PATHS.data, { recursive: true });
+});
+afterAll(cleanupTempDataRoots);
+
+it('imports into the health reader contracts while preserving local rows and wrapper metadata', async () => {
+ await source({
+ bloodTests: [{ id: 'incoming-blood', date: '2026-01-02', markers: { homocysteine: 8 } }],
+ epigeneticTests: [{ id: 'incoming-epigenetic', date: '2026-01-02', biologicalAge: 30 }],
+ eyeExams: [{ id: 'incoming-eye', date: '2026-01-02', notes: 'Example exam' }],
+ healthMetrics: [{ id: 'incoming-metric', date: '2026-01-02', bloodPressureSystolic: 120, bloodPressureDiastolic: 80 }],
+ });
+ await seed(localPath('blood-tests.json'), { tests: [{ id: 'local-blood', date: '2026-01-01' }], referenceRanges: { homocysteine: [5, 15] } });
+ await seed(localPath('epigenetic-tests.json'), { tests: [{ id: 'local-epigenetic', date: '2026-01-01' }] });
+ await seed(localPath('eyes.json'), { exams: [{ id: 'local-eye', date: '2026-01-01' }], rowTombstones: [{ id: 'deleted-eye', deletedAt: '2026-01-01' }] });
+ await seed(localPath('health-metrics.json'), { entries: [{ date: '2026-01-01', bloodPressureSystolic: 110, bloodPressureDiastolic: 70 }] });
+
+ expect(await importToPortOS()).toMatchObject({ ok: true, added: { bloodTests: 1, epigeneticTests: 1, eyeExams: 1, healthMetrics: 1 } });
+ expect(await getBloodTests()).toEqual({ tests: [{ id: 'local-blood', date: '2026-01-01' }, { id: 'incoming-blood', date: '2026-01-02', homocysteine: 8 }], referenceRanges: { homocysteine: [5, 15] } });
+ expect((await getEpigeneticTests()).tests.map(row => row.id)).toEqual(['local-epigenetic', 'incoming-epigenetic']);
+ expect((await getEyeExams()).exams.map(row => row.id)).toEqual(['local-eye', 'incoming-eye']);
+ expect((await read('eyes.json')).rowTombstones).toEqual([{ id: 'deleted-eye', deletedAt: '2026-01-01' }]);
+ expect(await getBloodPressureHistory()).toEqual([{ date: '2026-01-01', systolic: 110, diastolic: 70 }, { date: '2026-01-02', systolic: 120, diastolic: 80 }]);
+});
+
+it('initializes absent health stores and coalesces overlapping imports', async () => {
+ await source({
+ bloodTests: [{ id: 'incoming-blood', date: '2026-01-02', markers: { homocysteine: 8 } }],
+ healthMetrics: [{ id: 'incoming-metric', date: '2026-01-02', bloodPressureSystolic: 120, bloodPressureDiastolic: 80 }],
+ });
+ const reports = await Promise.all([importToPortOS(), importToPortOS()]);
+ expect(reports.reduce((sum, report) => sum + report.added.bloodTests, 0)).toBe(1);
+ expect((await getBloodTests()).tests).toEqual([{ id: 'incoming-blood', date: '2026-01-02', homocysteine: 8 }]);
+ expect(await getBloodPressureHistory()).toEqual([{ date: '2026-01-02', systolic: 120, diastolic: 80 }]);
+});
+
+it('deduplicates within a batch and on retry without replacing local edits or reviving deletions', async () => {
+ const incoming = { id: 'incoming-eye', date: '2026-01-02', notes: 'Original' };
+ await source({ eyeExams: [incoming, incoming, { id: 'deleted-eye', date: '2026-01-01' }] });
+ await seed(localPath('eyes.json'), { exams: [], rowTombstones: [{ id: 'deleted-eye', deletedAt: '2026-01-03' }] });
+ expect(await importToPortOS()).toMatchObject({ added: { eyeExams: 1 }, skipped: { eyeExams: 2 } });
+ const data = await read('eyes.json');
+ data.exams[0].notes = 'Local edit';
+ await seed(localPath('eyes.json'), data);
+ expect(await importToPortOS()).toMatchObject({ added: { eyeExams: 0 }, skipped: { eyeExams: 3 } });
+ expect((await getEyeExams()).exams).toEqual([{ ...incoming, notes: 'Local edit' }]);
+});
+
+it('merges against a local health write committed after import preflight', async () => {
+ await source({ bloodTests: [{ id: 'imported', date: '2026-01-03', markers: {} }] });
+ await seed(localPath('blood-tests.json'), { tests: [{ id: 'retained', date: '2026-01-01' }] });
+ hooks.afterRead = async (path) => {
+ if (path !== localPath('blood-tests.json')) return;
+ hooks.afterRead = null;
+ await addBloodTest({ id: 'concurrent', date: '2026-01-02' });
+ };
+ await importToPortOS();
+ expect((await getBloodTests()).tests.map(row => row.id)).toEqual(['retained', 'concurrent', 'imported']);
+});
+
+it('retains all concurrently submitted local blood-pressure readings', async () => {
+ await Promise.all(Array.from({ length: 8 }, (_, index) => addBloodPressureReading({
+ date: `2026-01-0${index + 1}`, systolic: 120 + index, diastolic: 80,
+ })));
+ expect(await getBloodPressureHistory()).toHaveLength(8);
+});
+
+it('retains all concurrently submitted local workouts', async () => {
+ await Promise.all(Array.from({ length: 8 }, (_, index) => addWorkout({
+ date: `2026-01-0${index + 1}`, type: 'Example exercise', durationMinutes: 30,
+ })));
+ expect(await getWorkouts()).toHaveLength(8);
+});
+
+it.each([
+ ['source collection', { bloodTests: {} }, { tests: [] }],
+ ['destination wrapper', { bloodTests: [{ id: 'incoming', date: '2026-01-02' }] }, { tests: 'invalid' }],
+])('refuses a malformed %s before importing any goals or health records', async (_label, incoming, local) => {
+ const goals = { goals: [{ id: 'retained-goal' }] };
+ await source({ ...incoming, goals: [{ id: 'incoming-goal' }] });
+ await seed(join(PATHS.digitalTwin, 'goals.json'), goals);
+ await seed(localPath('blood-tests.json'), local);
+ await expect(importToPortOS()).rejects.toThrow(/malformed/i);
+ expect(JSON.parse(await readFile(join(PATHS.digitalTwin, 'goals.json'), 'utf8'))).toEqual(goals);
+ expect(await read('blood-tests.json')).toEqual(local);
+});
diff --git a/server/services/meatspaceWriteQueues.js b/server/services/meatspaceWriteQueues.js
index e15b739d3f..dec694007e 100644
--- a/server/services/meatspaceWriteQueues.js
+++ b/server/services/meatspaceWriteQueues.js
@@ -17,8 +17,8 @@ import { createFileWriteQueue } from '../lib/fileWriteQueue.js';
export const queueConfigWrite = createFileWriteQueue();
/**
- * Serializes read-modify-write of `blood-tests.json`, `epigenetic-tests.json`
- * and `eyes.json`. One shared tail (not one per file): the three are tiny and
- * rarely written, so the simpler single queue costs nothing.
+ * Serializes local, import and sync read-modify-writes of `blood-tests.json`,
+ * `epigenetic-tests.json`, `eyes.json`, `health-metrics.json` and `workouts.json`.
+ * These are tiny and rarely written, so one shared tail costs nothing.
*/
export const queueHealthWrite = createFileWriteQueue();
diff --git a/server/services/mortalLoomStore.js b/server/services/mortalLoomStore.js
index fd87512af2..7a36b8de43 100644
--- a/server/services/mortalLoomStore.js
+++ b/server/services/mortalLoomStore.js
@@ -17,8 +17,11 @@ import { invalidateMeatspace, invalidateMortalLoomChanges } from './meatspaceEve
import { randomUUID } from 'crypto';
import { atomicWrite, safeJSONParse, readJSONFile, dataPath, ensureDir, sleep } from '../lib/fileUtils.js';
import { ICLOUD_NOT_MATERIALIZED, isEvictedStats, materializeAndWait, readIfMaterialized, requestMaterialization } from '../lib/icloudFile.js';
-import { isPlainObject } from '../lib/objects.js';
+import { canonicalStringify, isPlainObject } from '../lib/objects.js';
import { createFileWriteQueue } from '../lib/fileWriteQueue.js';
+import { rowIdentityKeys, ROW_TOMBSTONES_KEY } from '../lib/meatspaceRowIdentity.js';
+import { isTombstoned } from '../lib/tombstones.js';
+import { queueHealthWrite } from './meatspaceWriteQueues.js';
import { getSettings, settingsEvents } from './settings.js';
const DEFAULT_ICLOUD_PATH = join(
@@ -810,36 +813,79 @@ export async function importToPortOS() {
const collectionFiles = [
['alcoholDrinks', 'alcohol-drinks.json'],
['nicotineEntries', 'nicotine-entries.json'],
- ['bloodTests', 'blood-tests.json'],
+ ['bloodTests', 'blood-tests.json', 'tests'],
['bodyEntries', 'body-entries.json'],
- ['epigeneticTests', 'epigenetic-tests.json'],
- ['eyeExams', 'eyes.json'],
+ ['epigeneticTests', 'epigenetic-tests.json', 'tests'],
+ ['eyeExams', 'eyes.json', 'exams'],
['saunaSessions', 'sauna-sessions.json'],
['habits', 'habits.json'],
- ['healthMetrics', 'health-metrics.json']
+ ['healthMetrics', 'health-metrics.json', 'entries']
];
+ const assertRows = (rows, label) => {
+ if (!Array.isArray(rows) || rows.some(row => !isPlainObject(row)
+ || (row.date != null && typeof row.date !== 'string')
+ || (row.id != null && typeof row.id !== 'string'))) {
+ throw new Error(`MortalLoom import ${label} malformed`);
+ }
+ };
+ // Validate the whole input before even goals can be committed.
+ for (const key of ['goals', ...collectionFiles.map(([key]) => key)]) {
+ if (store[key] !== undefined) assertRows(store[key], key);
+ }
+ for (const row of store.bloodTests || []) {
+ if (row.markers != null && !isPlainObject(row.markers)) {
+ throw new Error('MortalLoom import blood markers malformed');
+ }
+ }
+ if ((store.healthMetrics || []).some(row => !row.date)) {
+ throw new Error('MortalLoom import health metric date malformed');
+ }
+ const readDestination = async ({ localPath, fileName, arrayKey }) => {
+ const document = await readJSONFile(localPath, arrayKey ? { [arrayKey]: [] } : [], { strict: true });
+ if (arrayKey && !isPlainObject(document)) {
+ throw new Error(`MortalLoom import ${fileName} malformed`);
+ }
+ const rows = arrayKey ? document[arrayKey] : document;
+ assertRows(rows, fileName);
+ return { document, rows };
+ };
// Preflight all destinations before the first write; one unreadable local file
- // must not turn the import into an unreported partially applied batch.
- const collections = await Promise.all(collectionFiles.map(async ([mlKey, fileName]) => {
+ // must not turn the import into an unreported partially applied batch. These
+ // snapshots are validation only: the queued write re-reads the current rows.
+ const collections = await Promise.all(collectionFiles.map(async ([mlKey, fileName, arrayKey]) => {
const mlArr = store[mlKey] || [];
- const localPath = dataPath('meatspace', fileName);
- const local = mlArr.length ? await readJSONFile(localPath, [], { strict: true }) : [];
- return { mlKey, mlArr, localPath, local };
+ const collection = { mlKey, mlArr, fileName, arrayKey, localPath: dataPath('meatspace', fileName) };
+ if (mlArr.length) await readDestination(collection);
+ return collection;
}));
- const mergeById = async (mlArr, localPath, pathDir, local) => {
- const localArr = Array.isArray(local) ? local : [];
- const seen = new Set(localArr.map(x => x.id).filter(Boolean));
+ const mergeById = async (collection) => queueHealthWrite(async () => {
+ const { mlKey, mlArr, localPath, arrayKey } = collection;
+ const { document, rows } = await readDestination(collection);
+ const keysFor = row => arrayKey
+ ? (mlKey === 'healthMetrics' ? [row.date] : rowIdentityKeys(row))
+ : [row.id || canonicalStringify(row)];
+ const seen = new Set(rows.flatMap(keysFor));
let added = 0, skipped = 0;
for (const item of mlArr) {
- if (item?.id && seen.has(item.id)) { skipped++; continue; }
- localArr.push(item); added++;
+ // MortalLoom nests blood markers; the local reader consumes flat fields.
+ const { markers, ...rest } = item;
+ const row = mlKey === 'bloodTests' ? { ...rest, ...(markers || {}) } : item;
+ const keys = keysFor(row);
+ if (keys.some(key => seen.has(key) || isTombstoned(document[ROW_TOMBSTONES_KEY], key, undefined, 'id'))) {
+ skipped++;
+ continue;
+ }
+ rows.push(row);
+ keys.forEach(key => seen.add(key));
+ added++;
}
if (added > 0) {
- await ensureDir(pathDir);
- await atomicWrite(localPath, localArr);
+ rows.sort((a, b) => (a.date || '').localeCompare(b.date || ''));
+ await ensureDir(dataPath('meatspace'));
+ await atomicWrite(localPath, arrayKey ? { ...document, [arrayKey]: rows } : rows);
}
return { added, skipped };
- };
+ });
// Goals live in a wrapper object, not a bare array
const { editGoals } = await import('./identity/store.js');
@@ -857,12 +903,15 @@ export async function importToPortOS() {
report.skipped.goals = skipped;
}, { localOnly: true });
- for (const { mlKey, mlArr, localPath, local } of collections) {
+ for (const collection of collections) {
+ const { mlKey, mlArr } = collection;
if (mlArr.length === 0) { report.added[mlKey] = 0; report.skipped[mlKey] = 0; continue; }
- const { added, skipped } = await mergeById(mlArr, localPath, dataPath('meatspace'), local);
+ const { added, skipped } = await mergeById(collection);
report.added[mlKey] = added; report.skipped[mlKey] = skipped;
}
+ invalidateMeatspace(['overview', 'blood', 'epigenetic', 'eyes', 'body', 'alcohol']);
+
if (store.profile && typeof store.profile === 'object') {
const profilePath = dataPath('meatspace', 'profile.json');
if (!existsSync(profilePath)) {
diff --git a/server/services/personalStoreReadSafety.test.js b/server/services/personalStoreReadSafety.test.js
index 6b5739c828..f355eb4a35 100644
--- a/server/services/personalStoreReadSafety.test.js
+++ b/server/services/personalStoreReadSafety.test.js
@@ -99,6 +99,14 @@ beforeAll(async () => { ({ importToPortOS } = await import('./mortalLoomStore.js
const cases = [
['alcohol daily log', () => join(PATHS.meatspace, 'daily-log.json'), () => logDrink({ name: 'Example', oz: 12, abv: 5, date: '2026-01-02' })],
['nicotine daily log', () => join(PATHS.meatspace, 'daily-log.json'), () => logNicotine({ product: 'Example', mgPerUnit: 1, date: '2026-01-02' })],
+ ['blood pressure', () => join(PATHS.meatspace, 'health-metrics.json'), async () => {
+ const { addBloodPressureReading } = await import('./meatspaceHealth.js');
+ return addBloodPressureReading({ date: '2026-01-02', systolic: 120, diastolic: 80 });
+ }],
+ ['workouts', () => join(PATHS.meatspace, 'workouts.json'), async () => {
+ const { addWorkout } = await import('./meatspaceHealth.js');
+ return addWorkout({ date: '2026-01-02', type: 'Example exercise' });
+ }],
['POST config', () => join(PATHS.meatspace, 'post-config.json'), () => updatePostConfig({ enabled: true })],
['memory items', () => join(PATHS.meatspace, 'post-memory-items.json'), () => createMemoryItem({ title: 'Example', lines: ['Example line'] })],
['review schedule', () => join(PATHS.meatspace, 'post-review-schedule.json'), () => applySessionToReviewSchedule({ masteredSkills: [{ skillId: 'example' }] })],