Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: preserve health records during imports and concurrent saves
[._.]/ Adam Eivya9d598452dd7
diff --git a/server/services/meatspaceHealth.js b/server/services/meatspaceHealth.jsindex 470eaed27f..b2e039aaf5 100644--- a/server/services/meatspaceHealth.js+++ b/server/services/meatspaceHealth.js@@ -38,7 +38,7 @@ const WORKOUTS_FILE = join(MEATSPACE_DIR, 'workouts.json');const byDate = (a, b) => (a.date || '').localeCompare(b.date || '');-// Every read-modify-write of the three files below runs inside `queueHealthWrite`+// Every read-modify-write of the local health files runs inside `queueHealthWrite`// — the queue the federation snapshot apply (`dataSync.js`) also uses — so a peer// snapshot cannot overwrite a record added between its read and its write.// `writeLocal` itself is NOT queued: callers already hold the queue.@@ -294,11 +294,13 @@ export async function addWorkout({ date, type, durationMinutes, intensity, notesintensity: typeof intensity === 'string' && intensity.trim() ? intensity.trim() : null,notes: typeof notes === 'string' && notes.trim() ? notes.trim() : null,};- const data = await readJSONFile(WORKOUTS_FILE, { workouts: [] });- if (!Array.isArray(data.workouts)) data.workouts = [];- data.workouts.push(entry);- data.workouts.sort(byDate);- await writeLocal(WORKOUTS_FILE, data);+ await queueHealthWrite(async () => {+ const data = await readJSONFile(WORKOUTS_FILE, { workouts: [] }, { strict: true });+ if (!Array.isArray(data.workouts)) throw new Error('Workouts malformed');+ data.workouts.push(entry);+ data.workouts.sort(byDate);+ await writeLocal(WORKOUTS_FILE, data);+ });console.log(`🏋️ Workout logged: ${trimmedType}${entry.durationMinutes ? ` (${entry.durationMinutes}min)` : ''} for ${targetDate}`);return entry;}@@ -339,12 +341,15 @@ export async function addBloodPressureReading({ date, systolic, diastolic }) {return { date: targetDate, systolic, diastolic };}- const log = await readJSONFile(HEALTH_METRICS_FILE, { entries: [] });- let entry = log.entries.find(e => e.date === targetDate);- if (!entry) { entry = { date: targetDate }; log.entries.push(entry); }- Object.assign(entry, patch);- log.entries.sort(byDate);- await writeLocal(HEALTH_METRICS_FILE, log);+ await queueHealthWrite(async () => {+ const log = await readJSONFile(HEALTH_METRICS_FILE, { entries: [] }, { strict: true });+ if (!Array.isArray(log.entries)) throw new Error('Health metrics malformed');+ let entry = log.entries.find(e => e.date === targetDate);+ if (!entry) { entry = { date: targetDate }; log.entries.push(entry); }+ Object.assign(entry, patch);+ log.entries.sort(byDate);+ await writeLocal(HEALTH_METRICS_FILE, log);+ });console.log(`🩺 Blood pressure ${systolic}/${diastolic} logged for ${targetDate}`);return { date: targetDate, systolic, diastolic };}diff --git a/server/services/meatspaceHealthIntegrity.test.js b/server/services/meatspaceHealthIntegrity.test.jsnew file mode 100644index 0000000000..1adb1cbf04--- /dev/null+++ b/server/services/meatspaceHealthIntegrity.test.js@@ -0,0 +1,130 @@+import { afterAll, beforeEach, expect, it, vi } from 'vitest';+import { mkdir, readFile, rm, writeFile } from 'fs/promises';+import { dirname, join } from 'path';+import { cleanupTempDataRoots, lazyTempDataRoot, makePathsProxy } from '../lib/mockPathsDataRoot.js';++const hooks = vi.hoisted(() => ({ afterRead: null }));+vi.mock('../lib/fileUtils.js', async (original) => {+ const actual = await original();+ return makePathsProxy(actual, {+ dataRoot: () => lazyTempDataRoot('portos-mortalloom-import-'),+ overrides: {+ dataPath: (...parts) => join(lazyTempDataRoot('portos-mortalloom-import-'), ...parts),+ readJSONFile: async (...args) => {+ const value = await actual.readJSONFile(...args);+ await hooks.afterRead?.(args[0]);+ return value;+ },+ },+ });+});+vi.mock('../lib/paths.js', async (original) => makePathsProxy(await original(), {+ dataRoot: () => lazyTempDataRoot('portos-mortalloom-import-'),+}));+vi.mock('./settings.js', async () => {+ const { EventEmitter } = await import('events');+ return { settingsEvents: new EventEmitter(), getSettings: async () => ({+ mortalloom: { enabled: false, path: join(lazyTempDataRoot('portos-mortalloom-import-'), 'source.json') },+ }) };+});++import { PATHS } from '../lib/fileUtils.js';+import { importToPortOS } from './mortalLoomStore.js';+import { addBloodTest, addBloodPressureReading, addWorkout, getBloodTests, getEpigeneticTests, getEyeExams, getBloodPressureHistory, getWorkouts } from './meatspaceHealth.js';++const localPath = (name) => join(PATHS.meatspace, name);+async function seed(path, value) {+ await mkdir(dirname(path), { recursive: true });+ await writeFile(path, JSON.stringify(value));+}+const source = (value) => seed(join(PATHS.data, 'source.json'), value);+const read = async (name) => JSON.parse(await readFile(localPath(name), 'utf8'));++beforeEach(async () => {+ hooks.afterRead = null;+ await rm(PATHS.data, { recursive: true, force: true });+ await mkdir(PATHS.data, { recursive: true });+});+afterAll(cleanupTempDataRoots);++it('imports into the health reader contracts while preserving local rows and wrapper metadata', async () => {+ await source({+ bloodTests: [{ id: 'incoming-blood', date: '2026-01-02', markers: { homocysteine: 8 } }],+ epigeneticTests: [{ id: 'incoming-epigenetic', date: '2026-01-02', biologicalAge: 30 }],+ eyeExams: [{ id: 'incoming-eye', date: '2026-01-02', notes: 'Example exam' }],+ healthMetrics: [{ id: 'incoming-metric', date: '2026-01-02', bloodPressureSystolic: 120, bloodPressureDiastolic: 80 }],+ });+ await seed(localPath('blood-tests.json'), { tests: [{ id: 'local-blood', date: '2026-01-01' }], referenceRanges: { homocysteine: [5, 15] } });+ await seed(localPath('epigenetic-tests.json'), { tests: [{ id: 'local-epigenetic', date: '2026-01-01' }] });+ await seed(localPath('eyes.json'), { exams: [{ id: 'local-eye', date: '2026-01-01' }], rowTombstones: [{ id: 'deleted-eye', deletedAt: '2026-01-01' }] });+ await seed(localPath('health-metrics.json'), { entries: [{ date: '2026-01-01', bloodPressureSystolic: 110, bloodPressureDiastolic: 70 }] });++ expect(await importToPortOS()).toMatchObject({ ok: true, added: { bloodTests: 1, epigeneticTests: 1, eyeExams: 1, healthMetrics: 1 } });+ expect(await getBloodTests()).toEqual({ tests: [{ id: 'local-blood', date: '2026-01-01' }, { id: 'incoming-blood', date: '2026-01-02', homocysteine: 8 }], referenceRanges: { homocysteine: [5, 15] } });+ expect((await getEpigeneticTests()).tests.map(row => row.id)).toEqual(['local-epigenetic', 'incoming-epigenetic']);+ expect((await getEyeExams()).exams.map(row => row.id)).toEqual(['local-eye', 'incoming-eye']);+ expect((await read('eyes.json')).rowTombstones).toEqual([{ id: 'deleted-eye', deletedAt: '2026-01-01' }]);+ expect(await getBloodPressureHistory()).toEqual([{ date: '2026-01-01', systolic: 110, diastolic: 70 }, { date: '2026-01-02', systolic: 120, diastolic: 80 }]);+});++it('initializes absent health stores and coalesces overlapping imports', async () => {+ await source({+ bloodTests: [{ id: 'incoming-blood', date: '2026-01-02', markers: { homocysteine: 8 } }],+ healthMetrics: [{ id: 'incoming-metric', date: '2026-01-02', bloodPressureSystolic: 120, bloodPressureDiastolic: 80 }],+ });+ const reports = await Promise.all([importToPortOS(), importToPortOS()]);+ expect(reports.reduce((sum, report) => sum + report.added.bloodTests, 0)).toBe(1);+ expect((await getBloodTests()).tests).toEqual([{ id: 'incoming-blood', date: '2026-01-02', homocysteine: 8 }]);+ expect(await getBloodPressureHistory()).toEqual([{ date: '2026-01-02', systolic: 120, diastolic: 80 }]);+});++it('deduplicates within a batch and on retry without replacing local edits or reviving deletions', async () => {+ const incoming = { id: 'incoming-eye', date: '2026-01-02', notes: 'Original' };+ await source({ eyeExams: [incoming, incoming, { id: 'deleted-eye', date: '2026-01-01' }] });+ await seed(localPath('eyes.json'), { exams: [], rowTombstones: [{ id: 'deleted-eye', deletedAt: '2026-01-03' }] });+ expect(await importToPortOS()).toMatchObject({ added: { eyeExams: 1 }, skipped: { eyeExams: 2 } });+ const data = await read('eyes.json');+ data.exams[0].notes = 'Local edit';+ await seed(localPath('eyes.json'), data);+ expect(await importToPortOS()).toMatchObject({ added: { eyeExams: 0 }, skipped: { eyeExams: 3 } });+ expect((await getEyeExams()).exams).toEqual([{ ...incoming, notes: 'Local edit' }]);+});++it('merges against a local health write committed after import preflight', async () => {+ await source({ bloodTests: [{ id: 'imported', date: '2026-01-03', markers: {} }] });+ await seed(localPath('blood-tests.json'), { tests: [{ id: 'retained', date: '2026-01-01' }] });+ hooks.afterRead = async (path) => {+ if (path !== localPath('blood-tests.json')) return;+ hooks.afterRead = null;+ await addBloodTest({ id: 'concurrent', date: '2026-01-02' });+ };+ await importToPortOS();+ expect((await getBloodTests()).tests.map(row => row.id)).toEqual(['retained', 'concurrent', 'imported']);+});++it('retains all concurrently submitted local blood-pressure readings', async () => {+ await Promise.all(Array.from({ length: 8 }, (_, index) => addBloodPressureReading({+ date: `2026-01-0${index + 1}`, systolic: 120 + index, diastolic: 80,+ })));+ expect(await getBloodPressureHistory()).toHaveLength(8);+});++it('retains all concurrently submitted local workouts', async () => {+ await Promise.all(Array.from({ length: 8 }, (_, index) => addWorkout({+ date: `2026-01-0${index + 1}`, type: 'Example exercise', durationMinutes: 30,+ })));+ expect(await getWorkouts()).toHaveLength(8);+});++it.each([+ ['source collection', { bloodTests: {} }, { tests: [] }],+ ['destination wrapper', { bloodTests: [{ id: 'incoming', date: '2026-01-02' }] }, { tests: 'invalid' }],+])('refuses a malformed %s before importing any goals or health records', async (_label, incoming, local) => {+ const goals = { goals: [{ id: 'retained-goal' }] };+ await source({ ...incoming, goals: [{ id: 'incoming-goal' }] });+ await seed(join(PATHS.digitalTwin, 'goals.json'), goals);+ await seed(localPath('blood-tests.json'), local);+ await expect(importToPortOS()).rejects.toThrow(/malformed/i);+ expect(JSON.parse(await readFile(join(PATHS.digitalTwin, 'goals.json'), 'utf8'))).toEqual(goals);+ expect(await read('blood-tests.json')).toEqual(local);+});diff --git a/server/services/meatspaceWriteQueues.js b/server/services/meatspaceWriteQueues.jsindex e15b739d3f..dec694007e 100644--- a/server/services/meatspaceWriteQueues.js+++ b/server/services/meatspaceWriteQueues.js@@ -17,8 +17,8 @@ import { createFileWriteQueue } from '../lib/fileWriteQueue.js';export const queueConfigWrite = createFileWriteQueue();/**- * Serializes read-modify-write of `blood-tests.json`, `epigenetic-tests.json`- * and `eyes.json`. One shared tail (not one per file): the three are tiny and- * rarely written, so the simpler single queue costs nothing.+ * Serializes local, import and sync read-modify-writes of `blood-tests.json`,+ * `epigenetic-tests.json`, `eyes.json`, `health-metrics.json` and `workouts.json`.+ * These are tiny and rarely written, so one shared tail costs nothing.*/export const queueHealthWrite = createFileWriteQueue();diff --git a/server/services/mortalLoomStore.js b/server/services/mortalLoomStore.jsindex fd87512af2..7a36b8de43 100644--- a/server/services/mortalLoomStore.js+++ b/server/services/mortalLoomStore.js@@ -17,8 +17,11 @@ import { invalidateMeatspace, invalidateMortalLoomChanges } from './meatspaceEveimport { randomUUID } from 'crypto';import { atomicWrite, safeJSONParse, readJSONFile, dataPath, ensureDir, sleep } from '../lib/fileUtils.js';import { ICLOUD_NOT_MATERIALIZED, isEvictedStats, materializeAndWait, readIfMaterialized, requestMaterialization } from '../lib/icloudFile.js';-import { isPlainObject } from '../lib/objects.js';+import { canonicalStringify, isPlainObject } from '../lib/objects.js';import { createFileWriteQueue } from '../lib/fileWriteQueue.js';+import { rowIdentityKeys, ROW_TOMBSTONES_KEY } from '../lib/meatspaceRowIdentity.js';+import { isTombstoned } from '../lib/tombstones.js';+import { queueHealthWrite } from './meatspaceWriteQueues.js';import { getSettings, settingsEvents } from './settings.js';const DEFAULT_ICLOUD_PATH = join(@@ -810,36 +813,79 @@ export async function importToPortOS() {const collectionFiles = [['alcoholDrinks', 'alcohol-drinks.json'],['nicotineEntries', 'nicotine-entries.json'],- ['bloodTests', 'blood-tests.json'],+ ['bloodTests', 'blood-tests.json', 'tests'],['bodyEntries', 'body-entries.json'],- ['epigeneticTests', 'epigenetic-tests.json'],- ['eyeExams', 'eyes.json'],+ ['epigeneticTests', 'epigenetic-tests.json', 'tests'],+ ['eyeExams', 'eyes.json', 'exams'],['saunaSessions', 'sauna-sessions.json'],['habits', 'habits.json'],- ['healthMetrics', 'health-metrics.json']+ ['healthMetrics', 'health-metrics.json', 'entries']];+ const assertRows = (rows, label) => {+ if (!Array.isArray(rows) || rows.some(row => !isPlainObject(row)+ || (row.date != null && typeof row.date !== 'string')+ || (row.id != null && typeof row.id !== 'string'))) {+ throw new Error(`MortalLoom import ${label} malformed`);+ }+ };+ // Validate the whole input before even goals can be committed.+ for (const key of ['goals', ...collectionFiles.map(([key]) => key)]) {+ if (store[key] !== undefined) assertRows(store[key], key);+ }+ for (const row of store.bloodTests || []) {+ if (row.markers != null && !isPlainObject(row.markers)) {+ throw new Error('MortalLoom import blood markers malformed');+ }+ }+ if ((store.healthMetrics || []).some(row => !row.date)) {+ throw new Error('MortalLoom import health metric date malformed');+ }+ const readDestination = async ({ localPath, fileName, arrayKey }) => {+ const document = await readJSONFile(localPath, arrayKey ? { [arrayKey]: [] } : [], { strict: true });+ if (arrayKey && !isPlainObject(document)) {+ throw new Error(`MortalLoom import ${fileName} malformed`);+ }+ const rows = arrayKey ? document[arrayKey] : document;+ assertRows(rows, fileName);+ return { document, rows };+ };// Preflight all destinations before the first write; one unreadable local file- // must not turn the import into an unreported partially applied batch.- const collections = await Promise.all(collectionFiles.map(async ([mlKey, fileName]) => {+ // must not turn the import into an unreported partially applied batch. These+ // snapshots are validation only: the queued write re-reads the current rows.+ const collections = await Promise.all(collectionFiles.map(async ([mlKey, fileName, arrayKey]) => {const mlArr = store[mlKey] || [];- const localPath = dataPath('meatspace', fileName);- const local = mlArr.length ? await readJSONFile(localPath, [], { strict: true }) : [];- return { mlKey, mlArr, localPath, local };+ const collection = { mlKey, mlArr, fileName, arrayKey, localPath: dataPath('meatspace', fileName) };+ if (mlArr.length) await readDestination(collection);+ return collection;}));- const mergeById = async (mlArr, localPath, pathDir, local) => {- const localArr = Array.isArray(local) ? local : [];- const seen = new Set(localArr.map(x => x.id).filter(Boolean));+ const mergeById = async (collection) => queueHealthWrite(async () => {+ const { mlKey, mlArr, localPath, arrayKey } = collection;+ const { document, rows } = await readDestination(collection);+ const keysFor = row => arrayKey+ ? (mlKey === 'healthMetrics' ? [row.date] : rowIdentityKeys(row))+ : [row.id || canonicalStringify(row)];+ const seen = new Set(rows.flatMap(keysFor));let added = 0, skipped = 0;for (const item of mlArr) {- if (item?.id && seen.has(item.id)) { skipped++; continue; }- localArr.push(item); added++;+ // MortalLoom nests blood markers; the local reader consumes flat fields.+ const { markers, ...rest } = item;+ const row = mlKey === 'bloodTests' ? { ...rest, ...(markers || {}) } : item;+ const keys = keysFor(row);+ if (keys.some(key => seen.has(key) || isTombstoned(document[ROW_TOMBSTONES_KEY], key, undefined, 'id'))) {+ skipped++;+ continue;+ }+ rows.push(row);+ keys.forEach(key => seen.add(key));+ added++;}if (added > 0) {- await ensureDir(pathDir);- await atomicWrite(localPath, localArr);+ rows.sort((a, b) => (a.date || '').localeCompare(b.date || ''));+ await ensureDir(dataPath('meatspace'));+ await atomicWrite(localPath, arrayKey ? { ...document, [arrayKey]: rows } : rows);}return { added, skipped };- };+ });// Goals live in a wrapper object, not a bare arrayconst { editGoals } = await import('./identity/store.js');@@ -857,12 +903,15 @@ export async function importToPortOS() {report.skipped.goals = skipped;}, { localOnly: true });- for (const { mlKey, mlArr, localPath, local } of collections) {+ for (const collection of collections) {+ const { mlKey, mlArr } = collection;if (mlArr.length === 0) { report.added[mlKey] = 0; report.skipped[mlKey] = 0; continue; }- const { added, skipped } = await mergeById(mlArr, localPath, dataPath('meatspace'), local);+ const { added, skipped } = await mergeById(collection);report.added[mlKey] = added; report.skipped[mlKey] = skipped;}+ invalidateMeatspace(['overview', 'blood', 'epigenetic', 'eyes', 'body', 'alcohol']);+if (store.profile && typeof store.profile === 'object') {const profilePath = dataPath('meatspace', 'profile.json');if (!existsSync(profilePath)) {diff --git a/server/services/personalStoreReadSafety.test.js b/server/services/personalStoreReadSafety.test.jsindex 6b5739c828..f355eb4a35 100644--- a/server/services/personalStoreReadSafety.test.js+++ b/server/services/personalStoreReadSafety.test.js@@ -99,6 +99,14 @@ beforeAll(async () => { ({ importToPortOS } = await import('./mortalLoomStore.jsconst cases = [['alcohol daily log', () => join(PATHS.meatspace, 'daily-log.json'), () => logDrink({ name: 'Example', oz: 12, abv: 5, date: '2026-01-02' })],['nicotine daily log', () => join(PATHS.meatspace, 'daily-log.json'), () => logNicotine({ product: 'Example', mgPerUnit: 1, date: '2026-01-02' })],+ ['blood pressure', () => join(PATHS.meatspace, 'health-metrics.json'), async () => {+ const { addBloodPressureReading } = await import('./meatspaceHealth.js');+ return addBloodPressureReading({ date: '2026-01-02', systolic: 120, diastolic: 80 });+ }],+ ['workouts', () => join(PATHS.meatspace, 'workouts.json'), async () => {+ const { addWorkout } = await import('./meatspaceHealth.js');+ return addWorkout({ date: '2026-01-02', type: 'Example exercise' });+ }],['POST config', () => join(PATHS.meatspace, 'post-config.json'), () => updatePostConfig({ enabled: true })],['memory items', () => join(PATHS.meatspace, 'post-memory-items.json'), () => createMemoryItem({ title: 'Example', lines: ['Example line'] })],['review schedule', () => join(PATHS.meatspace, 'post-review-schedule.json'), () => applySessionToReviewSchedule({ masteredSkills: [{ skillId: 'example' }] })],