Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: import MortalLoom daily records and profile into the stores PortOS reads (#13183)
Adam Eivy55da0fc1c144
diff --git a/server/services/meatspace.js b/server/services/meatspace.jsindex 554f529e54..ac452996f9 100644--- a/server/services/meatspace.js+++ b/server/services/meatspace.js@@ -277,6 +277,55 @@ export async function updateLifestyle(updates) {return config;}+/**+ * Strict read of the stored `config.json` with no defaults applied: `null` when+ * absent, a throw when present-but-unreadable or not an object.+ */+export async function readStoredConfig() {+ const stored = await readJSONFile(CONFIG_FILE, null, { allowArray: false, strict: true });+ if (stored !== null && !isPlainObject(stored)) throw new Error(`MeatSpace config malformed: ${CONFIG_FILE}`);+ return stored;+}++const presentString = value => typeof value === 'string' && value !== '';++/**+ * Explicit MortalLoom import of a native `HealthProfile` into `config.json`,+ * fill-only: a field the stored config already holds is a local edit and wins.+ * With no stored config the shipped lifestyle defaults are not edits, so the+ * profile's lifestyle values replace them.+ * @returns {Promise<{ filled: string[] }>}+ */+export async function fillConfigFromMortalLoomProfile(profile) {+ return queueConfigWrite(async () => {+ const stored = await readStoredConfig();+ const config = stored ?? structuredClone(DEFAULT_CONFIG);+ const storedLifestyle = isPlainObject(stored?.lifestyle) ? stored.lifestyle : {};+ const filled = [];+ if (config.sex == null && presentString(profile.biologicalSex)) {+ config.sex = profile.biologicalSex;+ config.sexSource = 'mortalloom';+ filled.push('sex');+ }+ if (config.birthDate == null && presentString(profile.birthDate)) {+ config.birthDate = profile.birthDate;+ filled.push('birthDate');+ }+ if (isPlainObject(profile.lifestyle)) {+ if (!isPlainObject(config.lifestyle)) config.lifestyle = {};+ for (const [key, value] of Object.entries(profile.lifestyle)) {+ if (value == null || storedLifestyle[key] !== undefined) continue;+ config.lifestyle[key] = value;+ filled.push(`lifestyle.${key}`);+ }+ }+ if (filled.length === 0) return { filled };+ config.updatedAt = new Date().toISOString();+ await saveConfig(config);+ return { filled };+ });+}+// === Birth Date (canonical source: meatspace/config.json) ===async function migrateBirthDateFromGoals(config) {diff --git a/server/services/mortalLoomImport.test.js b/server/services/mortalLoomImport.test.jsnew file mode 100644index 0000000000..f532948918--- /dev/null+++ b/server/services/mortalLoomImport.test.js@@ -0,0 +1,125 @@+import { afterAll, beforeEach, expect, it, vi } from 'vitest';+import { mkdir, readFile, rm, writeFile } from 'fs/promises';+import { dirname, join } from 'path';+import { cleanupTempDataRoots, lazyTempDataRoot, makePathsProxy } from '../lib/mockPathsDataRoot.js';++const hooks = vi.hoisted(() => ({ afterRead: null }));+vi.mock('../lib/fileUtils.js', async (original) => {+ const actual = await original();+ return makePathsProxy(actual, {+ dataRoot: () => lazyTempDataRoot('portos-mortalloom-daily-import-'),+ overrides: {+ dataPath: (...parts) => join(lazyTempDataRoot('portos-mortalloom-daily-import-'), ...parts),+ readJSONFile: async (...args) => {+ const value = await actual.readJSONFile(...args);+ await hooks.afterRead?.(args[0]);+ return value;+ },+ },+ });+});+vi.mock('../lib/paths.js', async (original) => makePathsProxy(await original(), {+ dataRoot: () => lazyTempDataRoot('portos-mortalloom-daily-import-'),+}));+// iCloud sync stays OFF: every assertion reads the local stores the import must fill.+vi.mock('./settings.js', async () => {+ const { EventEmitter } = await import('events');+ return { settingsEvents: new EventEmitter(), getSettings: async () => ({+ mortalloom: { enabled: false, path: join(lazyTempDataRoot('portos-mortalloom-daily-import-'), 'source.json') },+ }) };+});+vi.mock('./genome.js', () => ({ getSnpIndex: async () => null }));++import { PATHS } from '../lib/fileUtils.js';+import { importToPortOS } from './mortalLoomStore.js';+import { getDailyAlcohol, logDrink, removeDrink } from './meatspaceAlcohol.js';+import { getDailyNicotine } from './meatspaceNicotine.js';+import { getBodyHistory } from './meatspaceHealth.js';+import { getConfig } from './meatspace.js';++const localPath = (name) => join(PATHS.meatspace, name);+async function seed(path, value) {+ await mkdir(dirname(path), { recursive: true });+ await writeFile(path, JSON.stringify(value));+}+const source = (value) => seed(join(PATHS.data, 'source.json'), value);+const read = async (name) => JSON.parse(await readFile(localPath(name), 'utf8'));+const beer = (id, date = '2026-01-02') => ({ id, date, name: 'Example beer', oz: 12, abv: 5, count: 1 });+const drinkIds = async () => (await getDailyAlcohol()).flatMap(entry => entry.alcohol.drinks.map(drink => drink.id));++beforeEach(async () => {+ hooks.afterRead = null;+ await rm(PATHS.data, { recursive: true, force: true });+ await mkdir(PATHS.data, { recursive: true });+});+afterAll(cleanupTempDataRoots);++it('imports daily records and the profile into the stores the normal readers use', async () => {+ await source({+ alcoholDrinks: [beer('ML-DRINK')],+ nicotineEntries: [{ id: 'ML-NIC', date: '2026-01-02', product: 'Example pouch', mgPerUnit: 6, count: 2 }],+ bodyEntries: [{ id: 'ML-BODY', date: '2026-01-02', weight: 180, bodyFat: 20 }],+ profile: { biologicalSex: 'female', birthDate: '1990-05-05', lifestyle: { sleepHoursPerNight: 8, bmi: 22 } },+ });+ await seed(localPath('daily-log.json'), {+ entries: [{ date: '2026-01-02', alcohol: { drinks: [{ id: 'local-drink', name: 'Example wine', oz: 5, abv: 12, count: 1 }], standardDrinks: 1 },+ body: { weight: 175 }, notes: 'Example tenant' }],+ lastEntryDate: '2026-01-02',+ eventTombstones: [{ id: 'gone', deletedAt: '2026-01-01T00:00:00.000Z' }],+ });+ await seed(localPath('config.json'), { birthDate: null, sex: null, lifestyle: { sleepHoursPerNight: 6 }, updatedAt: null });++ expect(await importToPortOS()).toMatchObject({+ ok: true,+ added: { alcoholDrinks: 1, nicotineEntries: 1, bodyEntries: 1, profile: 1 },+ });++ const [day] = await getDailyAlcohol();+ expect(day.alcohol.drinks.map(drink => drink.id)).toEqual(['local-drink', 'ML-DRINK']);+ // 5oz@12% = 1.0 + 12oz@5% = 1.0, recomputed from the retained list.+ expect(day.alcohol.standardDrinks).toBe(2);+ expect(day.notes).toBe('Example tenant');+ const [nicotineDay] = await getDailyNicotine();+ expect(nicotineDay.nicotine).toEqual({ items: [{ id: 'ML-NIC', product: 'Example pouch', mgPerUnit: 6, count: 2 }], totalMg: 12 });+ // Fill-only body: the local weight edit wins, the missing field is filled.+ expect(await getBodyHistory()).toEqual([{ date: '2026-01-02', weight: 175, bodyFat: 20 }]);+ const config = await getConfig();+ expect(config).toMatchObject({ sex: 'female', sexSource: 'mortalloom', birthDate: '1990-05-05' });+ expect(config.lifestyle).toEqual({ sleepHoursPerNight: 6, bmi: 22 });+ const log = await read('daily-log.json');+ expect(log.eventTombstones).toEqual([{ id: 'gone', deletedAt: '2026-01-01T00:00:00.000Z' }]);+ for (const legacy of ['alcohol-drinks.json', 'nicotine-entries.json', 'body-entries.json', 'profile.json']) {+ await expect(readFile(localPath(legacy), 'utf8')).rejects.toMatchObject({ code: 'ENOENT' });+ }+});++it('keeps native identity on retry: no duplicates, no resurrected deletions, equal content stays distinct', async () => {+ const idless = { date: '2026-01-03', name: 'Example cider', oz: 12, abv: 6, count: 1 };+ await source({ alcoholDrinks: [beer('ML-A'), beer('ML-B'), beer('ML-C'), idless, idless] });+ expect(await importToPortOS()).toMatchObject({ added: { alcoholDrinks: 5 }, skipped: { alcoholDrinks: 0 } });+ expect(await removeDrink('2026-01-02', 2)).toMatchObject({ id: 'ML-C' });++ expect(await importToPortOS()).toMatchObject({ added: { alcoholDrinks: 0 }, skipped: { alcoholDrinks: 5 } });+ const days = await getDailyAlcohol();+ expect(days.map(entry => [entry.date, entry.alcohol.drinks.length])).toEqual([['2026-01-03', 2], ['2026-01-02', 2]]);+ expect(await drinkIds()).toEqual([undefined, undefined, 'ML-A', 'ML-B']);+ expect(days.find(entry => entry.date === '2026-01-02').alcohol.standardDrinks).toBe(2);+});++it('keeps local writes and overlapping imports made after preflight', async () => {+ await source({ alcoholDrinks: [beer('ML-A')], profile: { biologicalSex: 'male' } });+ await seed(localPath('config.json'), { birthDate: '1980-01-01', sex: null, lifestyle: {} });+ let raced = false;+ hooks.afterRead = async (path) => {+ if (raced || path !== localPath('daily-log.json')) return;+ raced = true;+ await logDrink({ name: 'Example local', oz: 12, abv: 5, date: '2026-01-02' });+ };+ const reports = await Promise.all([importToPortOS(), importToPortOS()]);+ expect(reports.reduce((sum, report) => sum + report.added.alcoholDrinks, 0)).toBe(1);+ expect(reports.reduce((sum, report) => sum + report.added.profile, 0)).toBe(1);+ const [day] = await getDailyAlcohol();+ expect(day.alcohol.drinks.map(drink => drink.name).sort()).toEqual(['Example beer', 'Example local']);+ expect(day.alcohol.standardDrinks).toBe(2);+ expect(await getConfig()).toMatchObject({ sex: 'male', birthDate: '1980-01-01' });+});diff --git a/server/services/mortalLoomStore.js b/server/services/mortalLoomStore.jsindex 0ff2c56455..d6385acd6e 100644--- a/server/services/mortalLoomStore.js+++ b/server/services/mortalLoomStore.js@@ -799,7 +799,92 @@ export async function getStatus() {};}-/** Non-destructive import: append MortalLoom records missing from PortOS local files. */+// Native MortalLoom substance arrays → the `daily-log.json` tenants the local+// alcohol/nicotine readers consume. Item normalization mirrors+// buildDailyLogFromMortalLoom so an imported day reads the same as the iCloud view.+const IMPORT_SUBSTANCES = [+ {+ mlKey: 'alcoholDrinks', key: 'alcohol', itemsField: 'drinks', totalField: 'standardDrinks',+ toItem: ({ name, oz, abv, count }) => ({ name: name || '', oz: Number(oz) || 0, abv: Number(abv) || 0, count: Number(count) || 1 }),+ },+ {+ mlKey: 'nicotineEntries', key: 'nicotine', itemsField: 'items', totalField: 'totalMg',+ toItem: ({ product, mgPerUnit, count }) => ({ product: product || '', mgPerUnit: Number(mgPerUnit) || 0, count: Number(count) || 1 }),+ },+];++/**+ * Merge native alcohol/nicotine/body rows into a local daily log in place.+ * Identity is the native event id (two ids with equal content stay distinct); a+ * locally tombstoned id is never re-added. Id-less rows match id-less local items+ * of the same day by content as a multiset, so a retry adds nothing while two+ * identical id-less drinks still import as two. Body fields are fill-only per+ * date: a field the local day already holds is a local edit and wins.+ */+function mergeNativeDailyRecords(log, store, { tombstones, computeTotals }) {+ const report = { added: {}, skipped: {} };+ const entryFor = (date) => {+ let entry = log.entries.find(e => e.date === date);+ if (!entry) { entry = { date }; log.entries.push(entry); }+ return entry;+ };+ const touched = new Set();+ for (const { mlKey, key, itemsField, toItem } of IMPORT_SUBSTANCES) {+ const rows = store[mlKey] || [];+ let added = 0, skipped = 0;+ const localItems = log.entries.flatMap(e => (Array.isArray(e[key]?.[itemsField]) ? e[key][itemsField] : []).map(item => [e.date, item]));+ const seenIds = new Set(localItems.map(([, item]) => item?.id).filter(id => typeof id === 'string' && id));+ const unmatchedContent = new Map();+ for (const [date, item] of localItems) {+ if (item?.id) continue;+ const contentKey = `${date}\u0000${canonicalStringify(toItem(item || {}))}`;+ unmatchedContent.set(contentKey, (unmatchedContent.get(contentKey) || 0) + 1);+ }+ for (const row of rows) {+ if (!row.date) { skipped++; continue; }+ const { id, date, ...fields } = row;+ const item = toItem(fields);+ if (id) {+ if (seenIds.has(id) || isTombstoned(tombstones, id, undefined, 'id')) { skipped++; continue; }+ seenIds.add(id);+ } else {+ const contentKey = `${date}\u0000${canonicalStringify(item)}`;+ const available = unmatchedContent.get(contentKey) || 0;+ if (available > 0) { unmatchedContent.set(contentKey, available - 1); skipped++; continue; }+ }+ const entry = entryFor(date);+ if (!isPlainObject(entry[key]) || !Array.isArray(entry[key][itemsField])) entry[key] = { ...(isPlainObject(entry[key]) ? entry[key] : {}), [itemsField]: [] };+ // Keep native fields the local shape does not name (stamps, app-only metadata).+ entry[key][itemsField].push({ ...fields, ...item, ...(id ? { id } : {}) });+ touched.add(entry);+ added++;+ }+ report.added[mlKey] = added; report.skipped[mlKey] = skipped;+ }+ for (const entry of touched) computeTotals(entry);++ let bodyAdded = 0, bodySkipped = 0;+ for (const row of store.bodyEntries || []) {+ const { id: _id, date, ...fields } = row;+ if (!date) { bodySkipped++; continue; }+ const existing = log.entries.find(e => e.date === date);+ const body = isPlainObject(existing?.body) ? existing.body : {};+ const fill = Object.entries(fields).filter(([field, value]) => value != null && body[field] === undefined);+ if (fill.length === 0) { bodySkipped++; continue; }+ const entry = existing || entryFor(date);+ entry.body = { ...body, ...Object.fromEntries(fill) };+ bodyAdded++;+ }+ report.added.bodyEntries = bodyAdded; report.skipped.bodyEntries = bodySkipped;+ return report;+}++/**+ * Non-destructive import: add MortalLoom records missing from the local stores+ * the normal MeatSpace readers consume. Alcohol, nicotine and body rows land in+ * `daily-log.json`; the profile fills `config.json`. Legacy per-domain copies an+ * older import wrote are left untouched.+ */export async function importToPortOS() {// Use the {present, ok} result, not readStore()'s store-or-null: an evicted// store is present-but-unreadable, and reporting it as "file not found" is the@@ -810,11 +895,9 @@ export async function importToPortOS() {if (!store) return { ok: false, reason: 'mortalloom-file-not-found' };const report = { added: {}, skipped: {} };+ const dailyKeys = ['alcoholDrinks', 'nicotineEntries', 'bodyEntries'];const collectionFiles = [- ['alcoholDrinks', 'alcohol-drinks.json'],- ['nicotineEntries', 'nicotine-entries.json'],['bloodTests', 'blood-tests.json', 'tests'],- ['bodyEntries', 'body-entries.json'],['epigeneticTests', 'epigenetic-tests.json', 'tests'],['eyeExams', 'eyes.json', 'exams'],['saunaSessions', 'sauna-sessions.json'],@@ -829,7 +912,7 @@ export async function importToPortOS() {}};// Validate the whole input before even goals can be committed.- for (const key of ['goals', ...collectionFiles.map(([key]) => key)]) {+ for (const key of ['goals', ...dailyKeys, ...collectionFiles.map(([key]) => key)]) {if (store[key] !== undefined) assertRows(store[key], key);}for (const row of store.bloodTests || []) {@@ -842,6 +925,12 @@ export async function importToPortOS() {if ((store.healthMetrics || []).some(row => !row.date)) {throw new Error('MortalLoom import health metric date malformed');}+ const profile = isPlainObject(store.profile) ? store.profile : null;+ const hasDaily = dailyKeys.some(key => store[key]?.length);+ const [dailyLog, meatspaceConfig] = await Promise.all([+ import('./meatspaceDailyLog.js'),+ import('./meatspace.js'),+ ]);const readDestination = async ({ localPath, fileName, arrayKey }) => {const document = await readJSONFile(localPath, arrayKey ? { [arrayKey]: [] } : [], { strict: true });if (arrayKey && !isPlainObject(document)) {@@ -860,6 +949,8 @@ export async function importToPortOS() {if (mlArr.length) await readDestination(collection);return collection;}));+ if (hasDaily) await dailyLog.readLocalDailyLog({ strict: true, label: 'MortalLoom import' });+ if (profile) await meatspaceConfig.readStoredConfig();const mergeById = async (collection) => queueHealthWrite(async () => {const { mlKey, mlArr, localPath, arrayKey } = collection;const { document, rows } = await readDestination(collection);@@ -912,17 +1003,41 @@ export async function importToPortOS() {report.added[mlKey] = added; report.skipped[mlKey] = skipped;}+ if (hasDaily) {+ const { computeStandardDrinks } = await import('./meatspaceAlcohol.js');+ // Day totals are recomputed from the full retained item list, with the same+ // per-item formulas the alcohol/nicotine services use on a local write.+ const itemTotals = {+ alcohol: item => computeStandardDrinks((item.oz || 0) * (item.count || 1), item.abv || 0),+ nicotine: item => (item.mgPerUnit ?? 0) * (item.count ?? 1),+ };+ const computeTotals = (entry) => {+ for (const { key, itemsField, totalField } of IMPORT_SUBSTANCES) {+ const items = entry[key]?.[itemsField];+ if (Array.isArray(items)) entry[key][totalField] = Math.round(items.reduce((sum, item) => sum + itemTotals[key](item), 0) * 100) / 100;+ }+ };+ let merged;+ // Merge under the daily-log queue against a fresh read, so local writes and+ // overlapping imports committed after preflight are kept; a no-op retry skips the write.+ await dailyLog.mutateDailyLog((log) => {+ merged = mergeNativeDailyRecords(log, store, {+ tombstones: log[dailyLog.DAILY_LOG_TOMBSTONES_KEY], computeTotals,+ });+ return Object.values(merged.added).some(n => n > 0) ? merged : null;+ }, { label: 'MortalLoom import' });+ Object.assign(report.added, merged.added);+ Object.assign(report.skipped, merged.skipped);+ } else {+ for (const key of dailyKeys) { report.added[key] = 0; report.skipped[key] = 0; }+ }+invalidateMeatspace(['overview', 'blood', 'epigenetic', 'eyes', 'body', 'alcohol']);- if (store.profile && typeof store.profile === 'object') {- const profilePath = dataPath('meatspace', 'profile.json');- if (!existsSync(profilePath)) {- await ensureDir(dataPath('meatspace'));- await atomicWrite(profilePath, store.profile);- report.added.profile = 1; report.skipped.profile = 0;- } else {- report.added.profile = 0; report.skipped.profile = 1;- }+ if (profile) {+ const { filled } = await meatspaceConfig.fillConfigFromMortalLoomProfile(profile);+ report.added.profile = filled.length > 0 ? 1 : 0;+ report.skipped.profile = filled.length > 0 ? 0 : 1;}report.ok = true;diff --git a/server/services/personalStoreReadSafety.test.js b/server/services/personalStoreReadSafety.test.jsindex f355eb4a35..46e8565bf1 100644--- a/server/services/personalStoreReadSafety.test.js+++ b/server/services/personalStoreReadSafety.test.js@@ -215,18 +215,18 @@ it('checks the legacy goals mirror before changing the canonical birth date', asit('preflights MortalLoom import destinations and preserves repaired records on retry', async () => {const source = join(PATHS.data, 'mortal-source.json');const goalsPath = join(PATHS.digitalTwin, 'goals.json');- const alcoholPath = join(PATHS.meatspace, 'alcohol-drinks.json');+ const dailyLogPath = join(PATHS.meatspace, 'daily-log.json');const goals = JSON.stringify({ goals: [{ id: 'retained-goal' }] });- await seed(source, JSON.stringify({ goals: [{ id: 'incoming-goal' }], alcoholDrinks: [{ id: 'incoming-drink' }] }));+ await seed(source, JSON.stringify({ goals: [{ id: 'incoming-goal' }], alcoholDrinks: [{ id: 'incoming-drink', date: '2026-01-02', name: 'Example', oz: 12, abv: 5 }] }));await seed(goalsPath, goals);- await seed(alcoholPath, '{');+ await seed(dailyLogPath, '{');await expect(importToPortOS()).rejects.toThrow(/Unreadable JSON file/);expect(await readFile(goalsPath, 'utf8')).toBe(goals);- expect(await readFile(alcoholPath, 'utf8')).toBe('{');- await seed(alcoholPath, JSON.stringify([{ id: 'retained-drink' }]));+ expect(await readFile(dailyLogPath, 'utf8')).toBe('{');+ await seed(dailyLogPath, JSON.stringify({ entries: [{ date: '2026-01-01', alcohol: { drinks: [{ id: 'retained-drink', name: 'Example', oz: 12, abv: 5, count: 1 }], standardDrinks: 1 } }] }));await expect(importToPortOS()).resolves.toMatchObject({ ok: true });expect(JSON.parse(await readFile(goalsPath, 'utf8')).goals.map(goal => goal.id)).toEqual(['retained-goal', 'incoming-goal']);- expect(JSON.parse(await readFile(alcoholPath, 'utf8')).map(drink => drink.id)).toEqual(['retained-drink', 'incoming-drink']);+ expect(JSON.parse(await readFile(dailyLogPath, 'utf8')).entries.flatMap(entry => entry.alcohol.drinks.map(drink => drink.id))).toEqual(['retained-drink', 'incoming-drink']);});it('keeps goal edits while importing MortalLoom records and updating the birth-date mirror', async () => {