Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: stop peer sync from dropping same-date blood, epigenetic and eye records (#10910)
Adam Eivy2cc7543a1e2a
diff --git a/server/services/dataSync.js b/server/services/dataSync.jsindex c2ad8cfe4f..7d6c0a39d2 100644--- a/server/services/dataSync.js+++ b/server/services/dataSync.js@@ -22,6 +22,7 @@ import {dailyLogEventLiveStamp,queueDailyLogWrite,} from './meatspaceDailyLog.js';+import { queueConfigWrite, queueHealthWrite } from './meatspaceWriteQueues.js';import {PORTOS_SCHEMA_VERSIONS,RECORD_KIND_SCHEMA_CATEGORIES,@@ -125,11 +126,20 @@ const STORY_BUILDER_DIR = join(PATHS.data, 'story-builder');// checksum cache.const STORY_BUILDER_EPOCH_KEY = '__storyBuilderEpoch';+// Blood/epigenetic/eye records merge by identity-preserving union+// (`unionPreservingRows`): `date` is NOT unique for them (two labs on one day,+// a retest, left/right exams entered separately), so keying by date would+// collapse same-day rows (#10910). Known limitation, intentionally not fixed+// here: the union is add-only, so a row edited on one machine never reaches a+// peer that already holds the old copy, and a deleted row (e.g. an eye exam+// removed with `removeEyeExam`) is re-added by the next snapshot from any peer+// that still has it. Propagating edits/deletes needs tombstones plus a+// `PORTOS_SCHEMA_VERSIONS.meatspace` bump.const MEATSPACE_FILES = {'daily-log.json': { type: 'daily-log' }, // mergeDailyLogFile- 'blood-tests.json': { arrayKey: 'tests', idField: 'date' },- 'epigenetic-tests.json': { arrayKey: 'tests', idField: 'date' },- 'eyes.json': { arrayKey: 'exams', idField: 'date' },+ 'blood-tests.json': { type: 'row-union', arrayKey: 'tests' },+ 'epigenetic-tests.json': { type: 'row-union', arrayKey: 'tests' },+ 'eyes.json': { type: 'row-union', arrayKey: 'exams' },'config.json': { type: 'object-lww' }};@@ -396,6 +406,55 @@ function mergeArraysByKey(localArr, remoteArr, idField, timestampField, mergeRecreturn { merged: Array.from(localMap.values()), changed };}+/**+ * Union two arrays of records WITHOUT collapsing any local row: every local row+ * is kept verbatim and in order, and a remote row is appended only when its+ * identity is not already present locally. For records whose date (or any other+ * field) is not unique, so `mergeArraysByKey` would silently drop one of two+ * same-key local rows.+ *+ * Identity is the string `id` when the row has one, else the row's content+ * (`canonicalStringify`, the same content-signature approach as+ * `digital-twin-sync.js#unionKeyFor`). A row stamped with an id after the fact+ * (legacy eye exams get one on first read) still matches its id-less copy on a+ * peer, so a content match ignores `id` — but an id-bearing remote row is only+ * content-matched against local rows that have no id, so two distinct records+ * that happen to hold identical values are both kept.+ *+ * Same-identity rows are not reconciled: the local copy stands (see the+ * `MEATSPACE_FILES` note on edits/deletes).+ */+function unionPreservingRows(localArr, remoteArr) {+ const contentKey = (row) => {+ const { id: _id, ...rest } = row;+ return canonicalStringify(rest);+ };+ const localIds = new Set();+ const idlessLocalContent = new Set();+ const allLocalContent = new Set();+ const track = (row) => {+ const hasId = typeof row.id === 'string' && row.id !== '';+ if (hasId) localIds.add(row.id);+ const content = contentKey(row);+ allLocalContent.add(content);+ if (!hasId) idlessLocalContent.add(content);+ };+ for (const row of localArr) if (isPlainObject(row)) track(row);++ const merged = [...localArr];+ for (const remoteRow of remoteArr) {+ if (!isPlainObject(remoteRow)) continue;+ const hasId = typeof remoteRow.id === 'string' && remoteRow.id !== '';+ const present = hasId+ ? localIds.has(remoteRow.id) || idlessLocalContent.has(contentKey(remoteRow))+ : allLocalContent.has(contentKey(remoteRow));+ if (present) continue;+ merged.push(remoteRow);+ track(remoteRow);+ }+ return { merged, changed: merged.length !== localArr.length };+}+/*** LWW merge for single objects. Remote wins if its updatedAt is newer.*/@@ -550,28 +609,33 @@ async function applyMeatspaceRemote(remoteData) {continue;}- const local = await readJSONFile(filePath, null, { strict: true });-+ // Both branches below read-modify-write a file whose local writers run on+ // a shared queue, so the sync apply must take the same one.if (config.type === 'object-lww') {- const { merged, changed } = mergeObjectLWW(local, remoteFile, 'updatedAt');- if (changed) {+ const written = await queueConfigWrite(async () => {+ const local = await readJSONFile(filePath, null, { strict: true });+ const { merged, changed } = mergeObjectLWW(local, remoteFile, 'updatedAt');+ if (!changed) return false;await atomicWrite(filePath, merged);+ return true;+ });+ if (written) {if (filename === 'config.json') meatspaceEvents.emit('death-clock:changed', {});totalApplied++;}} else {- // Array merge- const localArr = local?.[config.arrayKey] || [];- const remoteArr = remoteFile[config.arrayKey] || [];- const { merged, changed } = mergeArraysByKey(localArr, remoteArr, config.idField);-- if (changed) {- // Sort by idField (usually date)- merged.sort((a, b) => (a[config.idField] || '').localeCompare(b[config.idField] || ''));- const mergedFile = { ...(remoteFile || {}), ...(local || {}), [config.arrayKey]: merged };- await atomicWrite(filePath, mergedFile);- totalApplied++;- }+ const written = await queueHealthWrite(async () => {+ const local = await readJSONFile(filePath, null, { strict: true });+ const localArr = Array.isArray(local?.[config.arrayKey]) ? local[config.arrayKey] : [];+ const remoteArr = Array.isArray(remoteFile[config.arrayKey]) ? remoteFile[config.arrayKey] : [];+ const { merged, changed } = unionPreservingRows(localArr, remoteArr);+ if (!changed) return false;+ // Stable by date; same-date rows keep local-then-remote order.+ merged.sort((a, b) => (a?.date || '').localeCompare(b?.date || ''));+ await atomicWrite(filePath, { ...remoteFile, ...local, [config.arrayKey]: merged });+ return true;+ });+ if (written) totalApplied++;}}diff --git a/server/services/dataSync.meatspace.test.js b/server/services/dataSync.meatspace.test.jsnew file mode 100644index 0000000000..b96b1296db--- /dev/null+++ b/server/services/dataSync.meatspace.test.js@@ -0,0 +1,161 @@+/**+ * Meatspace federation apply — blood / epigenetic / eye records (#10910).+ *+ * `date` is not a unique identity for these records (two labs on one day, a retest,+ * left/right exams entered separately), so the snapshot merge must never collapse+ * same-date rows. And because a sync apply is a read -> merge -> write cycle on the+ * same files the local writers rewrite, it must share their write queues.+ *+ * Everything here drives the real boundary — `applyRemote('meatspace', …)` against a+ * throwaway data root — so the merge, the queueing and the persisted bytes are the+ * production code. All fixture values are invented.+ */++import { describe, it, expect, beforeEach, afterAll, vi } from 'vitest';+import { readFile, writeFile, mkdir, rm } from 'fs/promises';+import { join } from 'path';+import { mockPathsDataRoot } from '../lib/mockPathsDataRoot.js';++const { makeProxy, cleanup, tempRoot } = mockPathsDataRoot({ prefix: 'portos-datasync-meatspace-' });++vi.mock('../lib/fileUtils.js', async () => {+ const actual = await vi.importActual('../lib/fileUtils.js');+ return makeProxy(actual);+});++const dataSync = await import('./dataSync.js');+const health = await import('./meatspaceHealth.js');+const { queueConfigWrite, queueHealthWrite } = await import('./meatspaceWriteQueues.js');++const MEATSPACE_DIR = join(tempRoot, 'meatspace');+const fileOf = (name) => join(MEATSPACE_DIR, name);++afterAll(cleanup);++beforeEach(async () => {+ await rm(MEATSPACE_DIR, { recursive: true, force: true });+ await mkdir(MEATSPACE_DIR, { recursive: true });+});++const persist = (name, doc) => writeFile(fileOf(name), JSON.stringify(doc, null, 2));+const readBytes = (name) => readFile(fileOf(name), 'utf8');+const readDoc = async (name) => JSON.parse(await readBytes(name));+const apply = (name, doc) => dataSync.applyRemote('meatspace', { [name]: doc });++// One fixture family per file: two same-date local rows, a duplicate-of-local, a+// same-date-but-different remote row, and a brand-new-date remote row.+const FAMILIES = {+ 'blood-tests.json': {+ arrayKey: 'tests',+ addLocal: (row) => health.addBloodTest(row),+ sameDateA: { date: '2026-03-01', glucose: 90 },+ sameDateB: { date: '2026-03-01', ldl: 110 },+ sameDateRemote: { date: '2026-03-01', hdl: 55 },+ newDate: { date: '2026-04-01', glucose: 92 },+ },+ 'epigenetic-tests.json': {+ arrayKey: 'tests',+ addLocal: (row) => health.addEpigeneticTest(row),+ sameDateA: { date: '2026-03-01', biologicalAge: 40.1 },+ sameDateB: { date: '2026-03-01', biologicalAge: 40.9 },+ sameDateRemote: { date: '2026-03-01', biologicalAge: 41.5 },+ newDate: { date: '2026-04-01', biologicalAge: 41.0 },+ },+ 'eyes.json': {+ arrayKey: 'exams',+ addLocal: (row) => health.addEyeExam(row),+ sameDateA: { id: 'exam-a', date: '2026-03-01', leftSphere: -1.0 },+ sameDateB: { id: 'exam-b', date: '2026-03-01', rightSphere: -1.25 },+ sameDateRemote: { id: 'exam-c', date: '2026-03-01', rightSphere: -1.5 },+ newDate: { id: 'exam-d', date: '2026-04-01', leftSphere: -1.0 },+ },+};++describe.each(Object.entries(FAMILIES))('applyRemote meatspace %s', (filename, fx) => {+ it('keeps every same-date local row, dedupes identical rows, and is a byte-level no-op when re-applied', async () => {+ await persist(filename, { [fx.arrayKey]: [fx.sameDateA, fx.sameDateB] });++ // Remote carries a copy of a local row, a different row on the same date, and a new date.+ const remote = { [fx.arrayKey]: [fx.sameDateA, fx.sameDateRemote, fx.newDate] };+ const first = await apply(filename, remote);++ expect(first.applied).toBe(true);+ const rows = (await readDoc(filename))[fx.arrayKey];+ expect(rows).toHaveLength(4);+ // Local rows survive verbatim (the pre-fix map-by-date merge kept one per date).+ expect(rows).toEqual(expect.arrayContaining([fx.sameDateA, fx.sameDateB, fx.sameDateRemote, fx.newDate]));+ expect(rows.map((r) => r.date)).toEqual(['2026-03-01', '2026-03-01', '2026-03-01', '2026-04-01']);++ const bytes = await readBytes(filename);+ const second = await apply(filename, remote);+ expect(second.applied).toBe(false);+ expect(await readBytes(filename)).toBe(bytes);+ });+});++describe('applyRemote meatspace eyes.json identity', () => {+ it('matches an id-less local exam to its stamped remote copy, but keeps distinct ids with identical values', async () => {+ const values = { date: '2026-03-01', leftSphere: -2, rightSphere: -2 };+ await persist('eyes.json', { exams: [values] }); // legacy row, no id yet++ const result = await apply('eyes.json', {+ exams: [+ { id: 'stamped-copy', ...values }, // the same exam after the peer stamped an id+ { id: 'twin-1', date: '2026-05-01', leftSphere: -3 },+ { id: 'twin-2', date: '2026-05-01', leftSphere: -3 }, // distinct record, identical values+ ],+ });++ expect(result.applied).toBe(true);+ const exams = (await readDoc('eyes.json')).exams;+ expect(exams.map((e) => e.id ?? null)).toEqual([null, 'twin-1', 'twin-2']);+ });+});++describe('applyRemote meatspace write queues', () => {+ // Hold the shared queue, start an apply, and prove it did not touch the file until+ // the queue drained — i.e. it is serialized with the local writers, not beside them.+ const holdQueue = async (queue, name, run) => {+ let release;+ const held = queue(() => new Promise((resolve) => { release = resolve; }));+ const before = await readBytes(name);+ const pending = run();+ await new Promise((resolve) => setTimeout(resolve, 50));+ expect(await readBytes(name)).toBe(before);+ release();+ await held;+ const result = await pending;+ expect(await readBytes(name)).not.toBe(before);+ return result;+ };++ it('writes config.json inside the queue the local config writers use', async () => {+ await persist('config.json', { updatedAt: '2025-01-01T00:00:00.000Z', sex: 'unspecified' });+ const result = await holdQueue(queueConfigWrite, 'config.json', () =>+ apply('config.json', { updatedAt: '2026-01-01T00:00:00.000Z', sex: 'other' }));+ expect(result.applied).toBe(true);+ });++ it.each(Object.keys(FAMILIES))('writes %s inside the queue the local health writers use', async (filename) => {+ const fx = FAMILIES[filename];+ await persist(filename, { [fx.arrayKey]: [fx.sameDateA] });+ const result = await holdQueue(queueHealthWrite, filename, () =>+ apply(filename, { [fx.arrayKey]: [fx.newDate] }));+ expect(result.applied).toBe(true);+ });++ it.each(Object.keys(FAMILIES))('does not lose a local add that races a snapshot apply of %s', async (filename) => {+ const fx = FAMILIES[filename];+ await persist(filename, { [fx.arrayKey]: [fx.sameDateA] });++ await Promise.all([+ fx.addLocal(fx.sameDateB),+ apply(filename, { [fx.arrayKey]: [fx.newDate] }),+ ]);++ const rows = (await readDoc(filename))[fx.arrayKey];+ expect(rows).toHaveLength(3);+ expect(rows.filter((r) => r.date === '2026-03-01')).toHaveLength(2);+ expect(rows.filter((r) => r.date === '2026-04-01')).toHaveLength(1);+ });+});diff --git a/server/services/meatspace.js b/server/services/meatspace.jsindex c83d940024..554f529e54 100644--- a/server/services/meatspace.js+++ b/server/services/meatspace.js@@ -9,7 +9,7 @@ import { meatspaceEvents, invalidateMeatspace } from './meatspaceEvents.js';import { readFile } from 'fs/promises';import { join } from 'path';import { atomicWrite, PATHS, ensureDir, readJSONFile, readJSONFileStrict } from '../lib/fileUtils.js';-import { createFileWriteQueue } from '../lib/fileWriteQueue.js';+import { queueConfigWrite } from './meatspaceWriteQueues.js';import { isPlainObject } from '../lib/objects.js';import { readLocalDailyLog } from './meatspaceDailyLog.js';import { getSnpIndex } from './genome.js';@@ -33,7 +33,6 @@ function pickMortalLoomLifestyle(lifestyle) {const MEATSPACE_DIR = PATHS.meatspace;const CONFIG_FILE = join(MEATSPACE_DIR, 'config.json');-const queueConfigWrite = createFileWriteQueue();// Digital Twin paths (read-only)const LONGEVITY_FILE = join(PATHS.digitalTwin, 'longevity.json');diff --git a/server/services/meatspaceHealth.js b/server/services/meatspaceHealth.jsindex 2af55f76df..9f3ceacfad 100644--- a/server/services/meatspaceHealth.js+++ b/server/services/meatspaceHealth.js@@ -11,6 +11,7 @@ import { join } from 'path';import { randomUUID } from 'crypto';import { atomicWrite, PATHS, ensureDir, readJSONFile, getDateString } from '../lib/fileUtils.js';import { readLocalDailyLog, mutateDailyLog } from './meatspaceDailyLog.js';+import { queueHealthWrite } from './meatspaceWriteQueues.js';import {isMortalLoomEnabled,mlArrayIfEnabled,@@ -29,6 +30,10 @@ const WORKOUTS_FILE = join(MEATSPACE_DIR, 'workouts.json');const byDate = (a, b) => (a.date || '').localeCompare(b.date || '');+// Every read-modify-write of the three files below runs inside `queueHealthWrite`+// — the queue the federation snapshot apply (`dataSync.js`) also uses — so a peer+// snapshot cannot overwrite a record added between its read and its write.+// `writeLocal` itself is NOT queued: callers already hold the queue.async function writeLocal(file, data) {await ensureDir(MEATSPACE_DIR);await atomicWrite(file, data);@@ -57,10 +62,12 @@ export async function addBloodTest(test) {const stored = await mlPush('bloodTests', { date, markers: { ...(markers || {}), ...flat } });return { date: stored.date, ...(stored.markers || {}) };}- const data = await getBloodTests();- data.tests.push(test);- data.tests.sort(byDate);- await writeLocal(BLOOD_TESTS_FILE, data);+ await queueHealthWrite(async () => {+ const data = await getBloodTests();+ data.tests.push(test);+ data.tests.sort(byDate);+ await writeLocal(BLOOD_TESTS_FILE, data);+ });console.log(`🩸 Blood test added for ${test.date}`);return test;}@@ -122,21 +129,29 @@ export async function addEpigeneticTest(test) {console.log(`🧬 Epigenetic test added for ${stored.date} (MortalLoom)`);return stored;}- const data = await getEpigeneticTests();- data.tests.push(test);- data.tests.sort(byDate);- await writeLocal(EPIGENETIC_TESTS_FILE, data);+ await queueHealthWrite(async () => {+ const data = await getEpigeneticTests();+ data.tests.push(test);+ data.tests.sort(byDate);+ await writeLocal(EPIGENETIC_TESTS_FILE, data);+ });console.log(`🧬 Epigenetic test added for ${test.date}`);return test;}// === Eyes ===-export async function getEyeExams() {- const ml = await mlArrayIfEnabled('eyeExams');- if (ml) return { exams: [...ml].sort(byDate) };+const readLocalEyes = async () => {const data = await readJSONFile(EYES_FILE, { exams: [] });- const exams = Array.isArray(data?.exams) ? data.exams : [];+ return { data, exams: Array.isArray(data?.exams) ? data.exams : [] };+};++/**+ * Local eyes.json with every row id-stamped. MUST run inside `queueHealthWrite`:+ * legacy rows have no id, so stamping rewrites the file.+ */+async function loadLocalEyeExamsQueued() {+ const { data, exams } = await readLocalEyes();// Legacy eyes.json rows have no id. Stamp and persist them once so the ids// the client reads back are the ones later edits/deletes resolve.if (exams.some(e => !e.id)) {@@ -147,6 +162,15 @@ export async function getEyeExams() {return { ...data, exams };}+export async function getEyeExams() {+ const ml = await mlArrayIfEnabled('eyeExams');+ if (ml) return { exams: [...ml].sort(byDate) };+ const { data, exams } = await readLocalEyes();+ // Only a legacy file needs the (queued) stamping write; the common read stays lock-free.+ if (exams.some(e => !e.id)) return queueHealthWrite(loadLocalEyeExamsQueued);+ return { ...data, exams };+}+/** Resolve an eye exam id, or a legacy numeric index into the date-sorted list, to an id. */export async function resolveEyeExamId(ref) {if (typeof ref === 'string' && /^\d+$/.test(ref)) {@@ -162,11 +186,13 @@ export async function addEyeExam(exam) {console.log(`👁️ Eye exam added for ${stored.date} (MortalLoom)`);return stored;}- const data = await getEyeExams();const stored = { ...exam, id: exam.id || randomUUID() };- data.exams.push(stored);- data.exams.sort(byDate);- await writeLocal(EYES_FILE, data);+ await queueHealthWrite(async () => {+ const data = await loadLocalEyeExamsQueued();+ data.exams.push(stored);+ data.exams.sort(byDate);+ await writeLocal(EYES_FILE, data);+ });console.log(`👁️ Eye exam added for ${stored.date}`);return stored;}@@ -174,38 +200,47 @@ export async function addEyeExam(exam) {const EYE_FIELDS = ['date', 'leftSphere', 'leftCylinder', 'leftAxis', 'rightSphere', 'rightCylinder', 'rightAxis'];export async function updateEyeExam(id, updates) {- const data = await getEyeExams();- const exam = data.exams.find(e => e.id === id);- if (!exam) return null;const patch = Object.fromEntries(EYE_FIELDS.filter(k => updates[k] !== undefined).map(k => [k, updates[k]]));if (await isMortalLoomEnabled()) {+ const exam = (await getEyeExams()).exams.find(e => e.id === id);+ if (!exam) return null;const updated = await mlPatchById('eyeExams', exam.id, patch);console.log(`👁️ Eye exam updated: ${updated?.date} (MortalLoom)`);return updated;}- Object.assign(exam, patch);- data.exams.sort(byDate);- await writeLocal(EYES_FILE, data);- console.log(`👁️ Eye exam updated ${id}: ${exam.date}`);+ const exam = await queueHealthWrite(async () => {+ const data = await loadLocalEyeExamsQueued();+ const target = data.exams.find(e => e.id === id);+ if (!target) return null;+ Object.assign(target, patch);+ data.exams.sort(byDate);+ await writeLocal(EYES_FILE, data);+ return target;+ });+ if (exam) console.log(`👁️ Eye exam updated ${id}: ${exam.date}`);return exam;}export async function removeEyeExam(id) {- const data = await getEyeExams();- const target = data.exams.find(e => e.id === id);- if (!target) return null;-if (await isMortalLoomEnabled()) {+ const target = (await getEyeExams()).exams.find(e => e.id === id);+ if (!target) return null;const removed = await mlRemoveById('eyeExams', target.id);console.log(`👁️ Eye exam removed: ${removed?.date} (MortalLoom)`);return removed;}- data.exams = data.exams.filter(e => e.id !== id);- await writeLocal(EYES_FILE, data);- console.log(`👁️ Eye exam removed: ${target.date}`);+ const target = await queueHealthWrite(async () => {+ const data = await loadLocalEyeExamsQueued();+ const found = data.exams.find(e => e.id === id);+ if (!found) return null;+ data.exams = data.exams.filter(e => e.id !== id);+ await writeLocal(EYES_FILE, data);+ return found;+ });+ if (target) console.log(`👁️ Eye exam removed: ${target.date}`);return target;}diff --git a/server/services/meatspaceWriteQueues.js b/server/services/meatspaceWriteQueues.jsnew file mode 100644index 0000000000..e15b739d3f--- /dev/null+++ b/server/services/meatspaceWriteQueues.js@@ -0,0 +1,24 @@+/**+ * Write queues shared between the local MeatSpace writers and the federation+ * snapshot apply (`dataSync.js#applyMeatspaceRemote`).+ *+ * A sync apply is a read -> merge -> atomicWrite cycle on the same file a local+ * add/edit is rewriting, so both sides must run on the SAME queue or a peer+ * snapshot can overwrite a record added between the apply's read and write+ * (and vice versa). The queues live in their own leaf module so dataSync does+ * not have to import the full meatspace / meatspaceHealth service graphs just+ * to reach them. The daily log's queue lives with its store+ * (`queueDailyLogWrite` in `meatspaceDailyLog.js`).+ */++import { createFileWriteQueue } from '../lib/fileWriteQueue.js';++/** Serializes every read-modify-write of `meatspace/config.json`. */+export const queueConfigWrite = createFileWriteQueue();++/**+ * Serializes read-modify-write of `blood-tests.json`, `epigenetic-tests.json`+ * and `eyes.json`. One shared tail (not one per file): the three are tiny and+ * rarely written, so the simpler single queue costs nothing.+ */+export const queueHealthWrite = createFileWriteQueue();