Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: stop legacy eye exams duplicating across synced machines and sync eye exam edits and deletes (#12432)
[._.]/ Adam Eivy7a05c81b3ec6
diff --git a/scripts/migrations/429-eye-exam-legacy-row-ids.js b/scripts/migrations/429-eye-exam-legacy-row-ids.jsnew file mode 100644index 0000000000..fd8a185cde--- /dev/null+++ b/scripts/migrations/429-eye-exam-legacy-row-ids.js@@ -0,0 +1,25 @@+/** Re-key legacy eye exams to content-derived ids and collapse their duplicates+ * (#12432). Before this, each machine stamped an id-less eyes.json row with its+ * own `randomUUID()`, so syncing peers appended each other's copy of the same+ * exam. Every row without `createdAt` predates minted ids, so its random stamp+ * is replaced by the `legacy-` id every peer derives from the row's content, and+ * rows that land on the same id are one exam. Rows created since keep their id.+ * Input-gated on eyes.json; ships no seed. Idempotent: re-keyed rows keep their id. */+import { join } from 'node:path';+import { readJsonOrNull, writeJsonAtomic } from './_lib.js';+import { normalizeLegacyRows } from '../../server/lib/meatspaceRowIdentity.js';++export default {+ async up({ rootDir }) {+ const path = join(rootDir, 'data', 'meatspace', 'eyes.json');+ const doc = await readJsonOrNull(path);+ if (!doc || !Array.isArray(doc.exams)) return { rekeyed: 0, collapsed: 0 };+ const { rows, changed } = normalizeLegacyRows(doc.exams);+ if (!changed) return { rekeyed: 0, collapsed: 0 };+ const kept = new Set(doc.exams);+ const rekeyed = rows.filter((row) => !kept.has(row)).length;+ rows.sort((a, b) => (a?.date || '').localeCompare(b?.date || ''));+ await writeJsonAtomic(path, { ...doc, exams: rows });+ return { rekeyed, collapsed: doc.exams.length - rows.length };+ }+};diff --git a/scripts/migrations/429-eye-exam-legacy-row-ids.test.js b/scripts/migrations/429-eye-exam-legacy-row-ids.test.jsnew file mode 100644index 0000000000..d38b39e458--- /dev/null+++ b/scripts/migrations/429-eye-exam-legacy-row-ids.test.js@@ -0,0 +1,53 @@+import { mkdtemp, mkdir, readFile, writeFile, rm } from 'node:fs/promises';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { afterEach, beforeEach, expect, it } from 'vitest';+import migration from './429-eye-exam-legacy-row-ids.js';+import { legacyRowId } from '../../server/lib/meatspaceRowIdentity.js';++let rootDir;+beforeEach(async () => { rootDir = await mkdtemp(join(tmpdir(), 'portos-eye-legacy-ids-')); });+afterEach(() => rm(rootDir, { recursive: true, force: true }));++const eyesPath = () => join(rootDir, 'data', 'meatspace', 'eyes.json');+const seed = async (doc) => {+ await mkdir(join(rootDir, 'data', 'meatspace'), { recursive: true });+ await writeFile(eyesPath(), JSON.stringify(doc));+};+const read = async () => JSON.parse(await readFile(eyesPath(), 'utf8'));++it('collapses random-stamped copies of one legacy exam, re-keys singles, keeps minted rows, and is idempotent', async () => {+ const legacy = { date: '2024-05-01', leftSphere: -1, rightSphere: -1.25 };+ const single = { date: '2023-02-01', leftSphere: -0.5 };+ const minted = { id: 'exam-minted', createdAt: '2026-01-01T00:00:00.000Z', date: '2026-01-01', leftSphere: -2 };+ await seed({+ exams: [+ { id: '11111111-0000-4000-8000-000000000001', ...legacy }, // stamped on this machine+ { id: '22222222-0000-4000-8000-000000000002', ...single },+ { id: '33333333-0000-4000-8000-000000000003', ...legacy }, // the peer's stamp, appended by a sync+ minted,+ ],+ note: 'kept',+ });++ expect(await migration.up({ rootDir })).toEqual({ rekeyed: 2, collapsed: 1 });+ const doc = await read();+ expect(doc.note).toBe('kept');+ expect(doc.exams).toEqual([+ { id: legacyRowId(single), ...single },+ { id: legacyRowId(legacy), ...legacy },+ minted,+ ]);++ const bytes = await readFile(eyesPath(), 'utf8');+ expect(await migration.up({ rootDir })).toEqual({ rekeyed: 0, collapsed: 0 });+ expect(await readFile(eyesPath(), 'utf8')).toBe(bytes);+});++it('does nothing without an eyes.json or with an unreadable one', async () => {+ expect(await migration.up({ rootDir })).toEqual({ rekeyed: 0, collapsed: 0 });+ await mkdir(join(rootDir, 'data', 'meatspace'), { recursive: true });+ await writeFile(eyesPath(), '{not json');+ expect(await migration.up({ rootDir })).toEqual({ rekeyed: 0, collapsed: 0 });+ expect(await readFile(eyesPath(), 'utf8')).toBe('{not json');+});diff --git a/server/lib/README.md b/server/lib/README.mdindex c007e0e83e..baefe57814 100644--- a/server/lib/README.md+++ b/server/lib/README.md@@ -250,6 +250,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub| `modelInventory.js` | Identity AND durable shape of a downloaded-model inventory row. `modelInventoryId(backend, key)` mints the `hf:` / `lora:` / `ollama:` / `lmstudio:` id and `parseModelInventoryId` splits it back on the FIRST colon (an Ollama tag like `qwen3:8b` carries one of its own). `hfInventoryRow` / `loraInventoryRow` / `localModelInventoryRow` build the row a backend contributes, and `presentInventoryRow` re-derives the live fields (residency, inventory-verified) that a STORED row can never answer. Shared by the scan that mints rows from disk (`services/systemResources.js`) and the manifest that records them at the install chokepoints (`services/modelManifest.js`) — a drifted copy of a row's risk, manage path or delete action is a delete button pointed at the wrong place. Pure and dependency-free, so the install services reach the builders without pulling the scan's module graph. || `videoFailure.js` | `normalizeVideoFailure` classifies and scrubs local video causes; `createVideoDiagnosticTail` retains bounded stdout/stderr diagnostics without grouping bare exit codes. || `maintenanceSequence.js` | The maintenance ladder shared by both runners: `MAINTENANCE_TASK_ORDER` / `MAINTENANCE_ORDER_GUIDANCE`, the interleaved `MAINTENANCE_SEQUENCE_TYPES` (a perpetual `claim-issue` drain between every pair of audits), `maintenanceStepParams` (audits file issues, documentation does the work), and `buildMaintenanceSteps` (fix mode runs audits directly with one final drain; issue filing can omit claim jobs) which emits quota-burn-shaped run-once steps so `quotaBurnInvoke.js` can run a step from the Schedule tab's manual run or a Quota Burn family plan alike. Pure. |+| `meatspaceRowIdentity.js` | Cross-machine identity for MeatSpace health rows (#12432): `legacyRowId` derives a `legacy-<sha256>` id from a legacy row's content, `legacyRowIdentity` / `contentRowIdentity` key rows for the `row-union` snapshot merge, `preferRowCopy` is the total per-row `updatedAt` LWW order, `normalizeLegacyRows` re-keys and collapses legacy eye exams (also migration 429), and `ROW_TOMBSTONES_KEY` names the per-file deletion markers. Pure. || `meetingUrl.js` | Selects the one cacheable join URL (`selectMeetingUrl`) out of a raw Google Calendar event — first http(s) `video` entry point, else `hangoutLink` — and owns the shared `MEETING_URL_MAX` bound. Three-state: `undefined` = the producer never described conferencing, `null` = described with none usable. || `minimaxH3Memory.js` | The declared weight-placement table for the three MiniMax H3 entries (#5420) — H3 is the one video model family whose components fit nowhere unassisted, so "does this box have enough" is a render gate, not a UI fact. `MINIMAX_H3_MEMORY_PROFILES` maps entry id → `{ shippedRepo, shippedRevision, profiles }`, each profile carrying an honest `minMemoryGb` host floor and (CUDA only) a `minVramGb` device floor, ordered best-first. Every capacity number is HOISTED from what already existed — the CUDA tiers are `resolve_offload_profile()`'s own thresholds in `scripts/generate_minimax_h3_cuda.py`, the host floors are the entries' `memoryGb` — the sole new number being `MINIMAX_H3_HOST_RESERVE_GB`, a policy reserve held back for the OS. `applyMiniMaxH3MemoryProfiles(list)` is the load-time backfill (twin of migration 317) and guards BOTH `repo` and `revision`, like the speed-profile decorator. `selectMiniMaxH3MemoryProfile({ model, totalMemoryGb })` picks the best profile the HOST can hold (VRAM is the runner's call — the server has no synchronous device view); `miniMaxH3MemoryDeclineReason()` RETURNS the fail-closed reason so the render path can 400 it and a status surface can show it, and returns `null` on an UNMEASURED host — "not measured" is a deferral to the runner, never the same as zero. `validateMiniMaxH3MemoryProfileTable` / `sanitizeMiniMaxH3MemoryProfiles` warn + strip a hand-edited table (NaN floor, duplicate/reserved id, mis-ordered tiers) at load. || `videoContinuity.js` | How chunk N+1 of a chained video render is conditioned on chunk N. `resolveContextFrames(requested)` normalizes the tail-window size (absent → `DEFAULT_CONTEXT_FRAMES` = 22 ≈ 1s @ 24fps; an explicit `0` is preserved as "last frame only", NOT collapsed into the default) and clamps to `MIN_CONTEXT_FRAMES`..`MAX_CONTEXT_FRAMES`. `resolveContinuityStrategy({model, contextFrames})` picks `'window'` (LTX-2 `extend_from_video` conditioned on the prior chunk's last N frames — motion, not just a pose) or `'frame'` (extract the last frame, run i2v), degrading to `'frame'` on any runtime outside `CONTEXT_WINDOW_RUNTIMES` rather than rejecting. `extendLatentFrames` / `extendedPixelFrames` convert across the VAE's `LATENT_FRAME_STRIDE` (8 pixel frames per latent), `contextPrefixFrames({totalFrames, extendLatents})` measures how much of an extend render is echoed context to trim back off before stitching (0 = leave it alone), `tailWindowStartFrame` gives the cut index for the window itself, and `planExtendSplice({sourceFrames, outputFrames, extendLatents})` decides where a plain native Extend joins the ORIGINAL source to the render's new frames (`null` = layout doesn't verify, keep the model output). Pure — importable from `prepareParams.js` without dragging in `local.js`. Mirrored for the picker in `client/src/lib/videoGenParams.js`, pinned by `videoContinuity.parity.test.js`. |diff --git a/server/lib/index.js b/server/lib/index.jsindex c9cfdd8b28..50c337031e 100644--- a/server/lib/index.js+++ b/server/lib/index.js@@ -227,6 +227,7 @@ export * from './mediaModelBuckets.js';export * from './mediaModels.js';export * from './modelInput.js';export * from './modelInventory.js';+export * from './meatspaceRowIdentity.js';export * from './meetingUrl.js';export * from './minimaxH3Memory.js';export * from './videoContinuity.js';diff --git a/server/lib/meatspaceRowIdentity.js b/server/lib/meatspaceRowIdentity.jsnew file mode 100644index 0000000000..7242d60138--- /dev/null+++ b/server/lib/meatspaceRowIdentity.js@@ -0,0 +1,99 @@+/**+ * Cross-machine identity for MeatSpace health rows (blood / epigenetic / eye+ * records) that federate through the `row-union` snapshot merge (#12432).+ *+ * A row created before rows carried identity has no `id`, or an `id` a machine+ * stamped with `randomUUID()` on first read — two machines holding the same+ * legacy row then gave it two different ids, and every sync duplicated it. A+ * legacy row's identity is therefore derived from its content+ * (`legacy-<sha256>`), so every machine names the same legacy row the same way.+ * Rows created since carry a minted id plus `createdAt`; the id is their+ * identity and two identical rows with different ids stay two records.+ *+ * Pure. Migration 429 imports `normalizeLegacyRows`, so the derivation here IS+ * the on-disk id: changing it would re-key every peer's rows differently.+ */++import { createHash } from 'crypto';+import { canonicalStringify, isPlainObject } from './objects.js';+import { parseTsMs } from './lwwTimestamp.js';++export const LEGACY_ROW_ID_PREFIX = 'legacy-';++// Per-row deletion markers (`{ id, deletedAt }`) kept at the top level of a+// row-union file, unioned in both directions by the snapshot merge.+export const ROW_TOMBSTONES_KEY = 'rowTombstones';++const hasStringId = (row) => typeof row?.id === 'string' && row.id !== '';++export const isLegacyRowId = (id) => typeof id === 'string' && id.startsWith(LEGACY_ROW_ID_PREFIX);++/** Deterministic id for a legacy row: a hash of its content without identity or clock fields. */+export function legacyRowId(row) {+ const { id: _id, createdAt: _createdAt, updatedAt: _updatedAt, ...content } = row;+ const digest = createHash('sha256').update(canonicalStringify(content)).digest('hex');+ return `${LEGACY_ROW_ID_PREFIX}${digest.slice(0, 24)}`;+}++/**+ * The identity of a row whose file once stamped random ids (eyes.json). A row+ * without `createdAt` predates minted identity, so a non-legacy id on it is a+ * per-machine random stamp and is replaced by the content-derived id. An+ * edited legacy row keeps its `legacy-` id even though its content moved on.+ */+export function legacyRowIdentity(row) {+ if (hasStringId(row) && (row.createdAt !== undefined || isLegacyRowId(row.id))) return row.id;+ return legacyRowId(row);+}++/** Identity for rows that never had random stamps: their own id, else their content. */+export function contentRowIdentity(row) {+ return hasStringId(row) ? row.id : legacyRowId(row);+}++/** A row's newest clock: `updatedAt`, else `createdAt` (string, or undefined). */+export const rowLiveStamp = (row) => {+ const updated = parseTsMs(row?.updatedAt);+ const created = parseTsMs(row?.createdAt);+ if (updated === null) return created === null ? undefined : row.createdAt;+ if (created === null || updated >= created) return row.updatedAt;+ return row.createdAt;+};++/**+ * Pick one of two copies of the same row: newer live stamp wins, then the+ * larger canonical text. The order is total, so every machine converges on the+ * same copy whichever side is local.+ */+export function preferRowCopy(a, b) {+ const am = parseTsMs(rowLiveStamp(a)) ?? -Infinity;+ const bm = parseTsMs(rowLiveStamp(b)) ?? -Infinity;+ if (am !== bm) return am > bm ? a : b;+ return canonicalStringify(a) >= canonicalStringify(b) ? a : b;+}++/**+ * Re-key every row of an id-stamping file to `legacyRowIdentity` and collapse+ * rows that share an identity to their preferred copy (content-identical legacy+ * rows are one record). Order follows each identity's first occurrence.+ * Non-object rows are kept as they are.+ */+export function normalizeLegacyRows(rows) {+ const out = [];+ const indexById = new Map();+ for (const row of Array.isArray(rows) ? rows : []) {+ if (!isPlainObject(row)) { out.push(row); continue; }+ const id = legacyRowIdentity(row);+ const keyed = row.id === id ? row : { ...row, id };+ const index = indexById.get(id);+ if (index === undefined) {+ indexById.set(id, out.length);+ out.push(keyed);+ } else {+ out[index] = preferRowCopy(out[index], keyed);+ }+ }+ const changed = out.length !== (Array.isArray(rows) ? rows.length : 0)+ || out.some((row, i) => row !== rows[i]);+ return { rows: out, changed };+}diff --git a/server/lib/schemaVersions.js b/server/lib/schemaVersions.jsindex 2737b0fa4c..9a6daedfdc 100644--- a/server/lib/schemaVersions.js+++ b/server/lib/schemaVersions.js@@ -922,9 +922,21 @@ export const PORTOS_SCHEMA_VERSIONS = Object.freeze({// send a stale subset back, which a union ignores.// Rows without an `id` (logged before #8143) are still add-only: deleting// or moving one does not reach peers.- // The key exists so the NEXT incompatible daily-log change can bump to 2 and- // have v1 receivers reject it.- meatspace: 1,+ //+ // v2 (#12432): blood/epigenetic/eye rows merge by identity. A legacy row (no+ // id, or the random id a pre-v2 reader stamped on it) is keyed by a+ // content-derived `legacy-` id every machine computes alike, copies of one+ // row resolve by per-row `updatedAt`, and a delete rides a top-level+ // `rowTombstones: [{ id, deletedAt }]` list in the row's file. A v1 receiver+ // must refuse that: its union keys a legacy row by its own random stamp, so+ // every `legacy-` id would append a second copy of a row it already holds,+ // and it cannot read tombstones or edits. v1 → v2 stays accepted: a v2+ // receiver re-keys the v1 sender's random-stamped legacy rows (they carry no+ // `createdAt`) to the same content id, so they match instead of duplicating.+ // An edit made on a v1 peer still arrives as a second row, as it always did.+ // Until a v1 peer upgrades it refuses the whole meatspace snapshot from a v2+ // peer, daily log included; the gap shows on the Instances page.+ meatspace: 2,// v1 = the paired-peer credential (#8356): `X-PortOS-Peer-Auth`, an HMAC of the// pair `syncSecret` bound to the sender's instance id, replacing the instance// password over HTTP Basic between paired peers. Not a record or snapshotdiff --git a/server/services/dataSync.js b/server/services/dataSync.jsindex 7d6c0a39d2..e4073ee5d8 100644--- a/server/services/dataSync.js+++ b/server/services/dataSync.js@@ -17,6 +17,13 @@ import { snapshotChecksum } from '../lib/snapshotChecksum.js';import { parseTsMs } from '../lib/lwwTimestamp.js';import { isTombstoned, mergeTombstones, pruneTombstones } from '../lib/tombstones.js';import { GOAL_TOMBSTONES_KEY, reconcileGoalTombstones } from './identity/goalTombstones.js';+import {+ ROW_TOMBSTONES_KEY,+ contentRowIdentity,+ legacyRowIdentity,+ preferRowCopy,+ rowLiveStamp,+} from '../lib/meatspaceRowIdentity.js';import {DAILY_LOG_TOMBSTONES_KEY,dailyLogEventLiveStamp,@@ -126,20 +133,19 @@ const STORY_BUILDER_DIR = join(PATHS.data, 'story-builder');// checksum cache.const STORY_BUILDER_EPOCH_KEY = '__storyBuilderEpoch';-// Blood/epigenetic/eye records merge by identity-preserving union-// (`unionPreservingRows`): `date` is NOT unique for them (two labs on one day,-// a retest, left/right exams entered separately), so keying by date would-// collapse same-day rows (#10910). Known limitation, intentionally not fixed-// here: the union is add-only, so a row edited on one machine never reaches a-// peer that already holds the old copy, and a deleted row (e.g. an eye exam-// removed with `removeEyeExam`) is re-added by the next snapshot from any peer-// that still has it. Propagating edits/deletes needs tombstones plus a-// `PORTOS_SCHEMA_VERSIONS.meatspace` bump.+// Blood/epigenetic/eye records merge by row identity (`mergeRowUnionFile`), not+// by `date`: two labs on one day, a retest, or left/right exams entered+// separately share a date (#10910). Identity is the row's minted id, or for a+// legacy row a content-derived `legacy-` id every machine computes alike+// (`lib/meatspaceRowIdentity.js`, #12432). Copies of one row resolve by per-row+// `updatedAt` LWW, and a delete travels as a `rowTombstones` entry, so edits and+// deletes reach peers at `PORTOS_SCHEMA_VERSIONS.meatspace` 2. `legacyIds` marks+// the file whose pre-#12432 readers stamped random ids on legacy rows.const MEATSPACE_FILES = {'daily-log.json': { type: 'daily-log' }, // mergeDailyLogFile'blood-tests.json': { type: 'row-union', arrayKey: 'tests' },'epigenetic-tests.json': { type: 'row-union', arrayKey: 'tests' },- 'eyes.json': { type: 'row-union', arrayKey: 'exams' },+ 'eyes.json': { type: 'row-union', arrayKey: 'exams', legacyIds: true },'config.json': { type: 'object-lww' }};@@ -407,52 +413,66 @@ function mergeArraysByKey(localArr, remoteArr, idField, timestampField, mergeRec}/**- * Union two arrays of records WITHOUT collapsing any local row: every local row- * is kept verbatim and in order, and a remote row is appended only when its- * identity is not already present locally. For records whose date (or any other- * field) is not unique, so `mergeArraysByKey` would silently drop one of two- * same-key local rows.- *- * Identity is the string `id` when the row has one, else the row's content- * (`canonicalStringify`, the same content-signature approach as- * `digital-twin-sync.js#unionKeyFor`). A row stamped with an id after the fact- * (legacy eye exams get one on first read) still matches its id-less copy on a- * peer, so a content match ignores `id` — but an id-bearing remote row is only- * content-matched against local rows that have no id, so two distinct records- * that happen to hold identical values are both kept.+ * Merge a peer's row-union file (blood / epigenetic / eye records) into the+ * local one. Pure: returns the file to persist.*- * Same-identity rows are not reconciled: the local copy stands (see the- * `MEATSPACE_FILES` note on edits/deletes).+ * Rows match by identity (`legacyRowIdentity` for an id-stamping file, else+ * `contentRowIdentity`); copies of one row resolve to `preferRowCopy`, so an+ * edit stamped with `updatedAt` wins on every machine. Local rows that share an+ * identity in a non-stamping file are kept as they are — only a peer's copy is+ * deduped against them — so the merge never drops a local row it cannot tell+ * apart. Tombstones union both ways and drop a row unless it was edited after+ * the delete; a row that outlives its tombstone prunes it.*/-function unionPreservingRows(localArr, remoteArr) {- const contentKey = (row) => {- const { id: _id, ...rest } = row;- return canonicalStringify(rest);- };- const localIds = new Set();- const idlessLocalContent = new Set();- const allLocalContent = new Set();- const track = (row) => {- const hasId = typeof row.id === 'string' && row.id !== '';- if (hasId) localIds.add(row.id);- const content = contentKey(row);- allLocalContent.add(content);- if (!hasId) idlessLocalContent.add(content);+function mergeRowUnionFile(localFile, remoteFile, { arrayKey, legacyIds = false }) {+ const local = isPlainObject(localFile) ? localFile : {};+ const remote = isPlainObject(remoteFile) ? remoteFile : {};+ const identity = legacyIds ? legacyRowIdentity : contentRowIdentity;+ const rows = [];+ const keys = [];+ const indexByKey = new Map();+ const add = (row, fromLocal) => {+ if (!isPlainObject(row)) {+ if (fromLocal) { rows.push(row); keys.push(null); }+ return;+ }+ const key = identity(row);+ const keyed = legacyIds && row.id !== key ? { ...row, id: key } : row;+ const index = indexByKey.get(key);+ if (index !== undefined && (legacyIds || !fromLocal)) {+ rows[index] = preferRowCopy(rows[index], keyed);+ return;+ }+ if (index === undefined) indexByKey.set(key, rows.length);+ rows.push(keyed);+ keys.push(key);};- for (const row of localArr) if (isPlainObject(row)) track(row);-- const merged = [...localArr];- for (const remoteRow of remoteArr) {- if (!isPlainObject(remoteRow)) continue;- const hasId = typeof remoteRow.id === 'string' && remoteRow.id !== '';- const present = hasId- ? localIds.has(remoteRow.id) || idlessLocalContent.has(contentKey(remoteRow))- : allLocalContent.has(contentKey(remoteRow));- if (present) continue;- merged.push(remoteRow);- track(remoteRow);- }- return { merged, changed: merged.length !== localArr.length };+ for (const row of Array.isArray(local[arrayKey]) ? local[arrayKey] : []) add(row, true);+ for (const row of Array.isArray(remote[arrayKey]) ? remote[arrayKey] : []) add(row, false);++ const { merged: unionTombstones } = mergeTombstones(+ local[ROW_TOMBSTONES_KEY],+ remote[ROW_TOMBSTONES_KEY],+ { keyField: 'id' },+ );+ const kept = [];+ const liveStamps = [];+ rows.forEach((row, i) => {+ if (keys[i] !== null) {+ const stamp = rowLiveStamp(row);+ if (isTombstoned(unionTombstones, keys[i], stamp, 'id')) return;+ liveStamps.push({ id: keys[i], stamp });+ }+ kept.push(row);+ });+ const tombstones = pruneTombstones(unionTombstones, liveStamps, { keyField: 'id', timestampField: 'stamp' });+ // Stable by date; same-date rows keep local-then-remote order.+ kept.sort((a, b) => (a?.date || '').localeCompare(b?.date || ''));++ const mergedFile = { ...remote, ...local, [arrayKey]: kept };+ if (tombstones.length > 0) mergedFile[ROW_TOMBSTONES_KEY] = tombstones;+ else delete mergedFile[ROW_TOMBSTONES_KEY];+ return mergedFile;}/**@@ -626,13 +646,11 @@ async function applyMeatspaceRemote(remoteData) {} else {const written = await queueHealthWrite(async () => {const local = await readJSONFile(filePath, null, { strict: true });- const localArr = Array.isArray(local?.[config.arrayKey]) ? local[config.arrayKey] : [];- const remoteArr = Array.isArray(remoteFile[config.arrayKey]) ? remoteFile[config.arrayKey] : [];- const { merged, changed } = unionPreservingRows(localArr, remoteArr);- if (!changed) return false;- // Stable by date; same-date rows keep local-then-remote order.- merged.sort((a, b) => (a?.date || '').localeCompare(b?.date || ''));- await atomicWrite(filePath, { ...remoteFile, ...local, [config.arrayKey]: merged });+ const merged = mergeRowUnionFile(local, remoteFile, config);+ // A peer with nothing to add must not create the file here.+ if (local === null && merged[config.arrayKey].length === 0 && !merged[ROW_TOMBSTONES_KEY]) return false;+ if (canonicalStringify(merged) === canonicalStringify(local)) return false;+ await atomicWrite(filePath, merged);return true;});if (written) totalApplied++;diff --git a/server/services/dataSync.meatspace.test.js b/server/services/dataSync.meatspace.test.jsindex b96b1296db..5587f9c91e 100644--- a/server/services/dataSync.meatspace.test.js+++ b/server/services/dataSync.meatspace.test.js@@ -25,6 +25,7 @@ vi.mock('../lib/fileUtils.js', async () => {const dataSync = await import('./dataSync.js');const health = await import('./meatspaceHealth.js');+const { legacyRowId } = await import('../lib/meatspaceRowIdentity.js');const { queueConfigWrite, queueHealthWrite } = await import('./meatspaceWriteQueues.js');const MEATSPACE_DIR = join(tempRoot, 'meatspace');@@ -64,10 +65,10 @@ const FAMILIES = {'eyes.json': {arrayKey: 'exams',addLocal: (row) => health.addEyeExam(row),- sameDateA: { id: 'exam-a', date: '2026-03-01', leftSphere: -1.0 },- sameDateB: { id: 'exam-b', date: '2026-03-01', rightSphere: -1.25 },- sameDateRemote: { id: 'exam-c', date: '2026-03-01', rightSphere: -1.5 },- newDate: { id: 'exam-d', date: '2026-04-01', leftSphere: -1.0 },+ sameDateA: { id: 'exam-a', createdAt: '2026-03-01T10:00:00.000Z', date: '2026-03-01', leftSphere: -1.0 },+ sameDateB: { id: 'exam-b', createdAt: '2026-03-01T11:00:00.000Z', date: '2026-03-01', rightSphere: -1.25 },+ sameDateRemote: { id: 'exam-c', createdAt: '2026-03-01T12:00:00.000Z', date: '2026-03-01', rightSphere: -1.5 },+ newDate: { id: 'exam-d', createdAt: '2026-04-01T10:00:00.000Z', date: '2026-04-01', leftSphere: -1.0 },},};@@ -94,21 +95,83 @@ describe.each(Object.entries(FAMILIES))('applyRemote meatspace %s', (filename, f});describe('applyRemote meatspace eyes.json identity', () => {- it('matches an id-less local exam to its stamped remote copy, but keeps distinct ids with identical values', async () => {+ const minted = '2026-05-01T09:00:00.000Z';++ it('matches an id-less local exam to its stamped remote copy, but keeps distinct minted ids with identical values', async () => {const values = { date: '2026-03-01', leftSphere: -2, rightSphere: -2 };await persist('eyes.json', { exams: [values] }); // legacy row, no id yetconst result = await apply('eyes.json', {exams: [- { id: 'stamped-copy', ...values }, // the same exam after the peer stamped an id- { id: 'twin-1', date: '2026-05-01', leftSphere: -3 },- { id: 'twin-2', date: '2026-05-01', leftSphere: -3 }, // distinct record, identical values+ { id: 'stamped-copy', ...values }, // the same exam after a pre-#12432 peer stamped a random id+ { id: 'twin-1', createdAt: minted, date: '2026-05-01', leftSphere: -3 },+ { id: 'twin-2', createdAt: minted, date: '2026-05-01', leftSphere: -3 }, // distinct record, identical values],});expect(result.applied).toBe(true);const exams = (await readDoc('eyes.json')).exams;- expect(exams.map((e) => e.id ?? null)).toEqual([null, 'twin-1', 'twin-2']);+ expect(exams.map((e) => e.id)).toEqual([legacyRowId(values), 'twin-1', 'twin-2']);+ });++ // #12432: each machine used to stamp a legacy row with its own random id, so the+ // second sync (after both had stamped) appended the peer's copy on both sides.+ it('does not duplicate a legacy exam when re-syncing after both machines stamped it', async () => {+ const legacy = { date: '2025-02-01', leftSphere: -1.25, rightSphere: -1 };+ await persist('eyes.json', { exams: [{ ...legacy }] });+ await health.getEyeExams(); // this machine stamps its copy on first read++ // The peer stamped the same row too — with a random id before upgrading, or+ // the content id after.+ await apply('eyes.json', { exams: [{ id: '0d9b4c1e-0000-4000-8000-000000000001', ...legacy }] });+ await apply('eyes.json', { exams: [{ id: legacyRowId(legacy), ...legacy }] });++ const exams = (await readDoc('eyes.json')).exams;+ expect(exams).toEqual([{ id: legacyRowId(legacy), ...legacy }]);+ });++ it('resolves two copies of one exam to the newer edit on either side', async () => {+ const original = { id: 'exam-x', createdAt: minted, date: '2026-05-01', leftSphere: -1 };+ const edited = { ...original, leftSphere: -1.5, updatedAt: '2026-06-01T00:00:00.000Z' };++ await persist('eyes.json', { exams: [original] });+ expect((await apply('eyes.json', { exams: [edited] })).applied).toBe(true);+ expect((await readDoc('eyes.json')).exams).toEqual([edited]);++ // The stale copy from a peer that has not seen the edit changes nothing.+ expect((await apply('eyes.json', { exams: [original] })).applied).toBe(false);+ expect((await readDoc('eyes.json')).exams).toEqual([edited]);+ });++ it('keeps a deleted exam deleted when a peer that still holds it syncs, and carries the delete to that peer', async () => {+ const kept = { id: 'exam-keep', createdAt: minted, date: '2026-05-01', leftSphere: -1 };+ const gone = { id: 'exam-gone', createdAt: minted, date: '2026-05-02', leftSphere: -2 };+ await persist('eyes.json', { exams: [kept, gone] });+ await health.removeEyeExam('exam-gone');+ const afterDelete = await readDoc('eyes.json');++ // The peer still holds the row: applying its snapshot must not bring it back.+ expect((await apply('eyes.json', { exams: [kept, gone] })).applied).toBe(false);+ expect((await readDoc('eyes.json')).exams).toEqual([kept]);++ // The other direction: a machine still holding the row drops it on receipt.+ await persist('eyes.json', { exams: [kept, gone] });+ expect((await apply('eyes.json', afterDelete)).applied).toBe(true);+ const peer = await readDoc('eyes.json');+ expect(peer.exams).toEqual([kept]);+ expect(peer.rowTombstones).toEqual(afterDelete.rowTombstones);+ });++ it('keeps a row edited after its deletion and drops the outlived tombstone', async () => {+ const row = { id: 'exam-back', createdAt: minted, date: '2026-05-01', leftSphere: -1 };+ const tombstone = { id: 'exam-back', deletedAt: '2026-05-10T00:00:00.000Z' };+ await persist('eyes.json', { exams: [], rowTombstones: [tombstone] });++ const edited = { ...row, leftSphere: -3, updatedAt: '2026-05-11T00:00:00.000Z' };+ expect((await apply('eyes.json', { exams: [edited] })).applied).toBe(true);+ const doc = await readDoc('eyes.json');+ expect(doc.exams).toEqual([edited]);+ expect(doc).not.toHaveProperty('rowTombstones');});});diff --git a/server/services/meatspaceHealth.js b/server/services/meatspaceHealth.jsindex 9f3ceacfad..4c9dff0fa7 100644--- a/server/services/meatspaceHealth.js+++ b/server/services/meatspaceHealth.js@@ -12,6 +12,8 @@ import { randomUUID } from 'crypto';import { atomicWrite, PATHS, ensureDir, readJSONFile, getDateString } from '../lib/fileUtils.js';import { readLocalDailyLog, mutateDailyLog } from './meatspaceDailyLog.js';import { queueHealthWrite } from './meatspaceWriteQueues.js';+import { normalizeLegacyRows, ROW_TOMBSTONES_KEY } from '../lib/meatspaceRowIdentity.js';+import { recordTombstone } from '../lib/tombstones.js';import {isMortalLoomEnabled,mlArrayIfEnabled,@@ -147,28 +149,31 @@ const readLocalEyes = async () => {};/**- * Local eyes.json with every row id-stamped. MUST run inside `queueHealthWrite`:- * legacy rows have no id, so stamping rewrites the file.+ * Local eyes.json with every row carrying its identity. MUST run inside+ * `queueHealthWrite`: a legacy row (no id, or a pre-#12432 random stamp) is+ * re-keyed to its content-derived id and persisted once, so the ids the client+ * reads back match the ones every peer computes for the same row.*/async function loadLocalEyeExamsQueued() {const { data, exams } = await readLocalEyes();- // Legacy eyes.json rows have no id. Stamp and persist them once so the ids- // the client reads back are the ones later edits/deletes resolve.- if (exams.some(e => !e.id)) {- for (const e of exams) e.id ||= randomUUID();- exams.sort(byDate);- await writeLocal(EYES_FILE, { ...data, exams });- }- return { ...data, exams };+ const normalized = normalizeLegacyRows(exams);+ if (!normalized.changed) return { ...data, exams };+ normalized.rows.sort(byDate);+ const next = { ...data, exams: normalized.rows };+ await writeLocal(EYES_FILE, next);+ return next;}+// Deletion markers are sync metadata, not part of the exam list callers read.+const withoutTombstones = ({ [ROW_TOMBSTONES_KEY]: _tombstones, ...rest }) => rest;+export async function getEyeExams() {const ml = await mlArrayIfEnabled('eyeExams');if (ml) return { exams: [...ml].sort(byDate) };const { data, exams } = await readLocalEyes();- // Only a legacy file needs the (queued) stamping write; the common read stays lock-free.- if (exams.some(e => !e.id)) return queueHealthWrite(loadLocalEyeExamsQueued);- return { ...data, exams };+ // Only a legacy file needs the (queued) re-keying write; the common read stays lock-free.+ if (normalizeLegacyRows(exams).changed) return withoutTombstones(await queueHealthWrite(loadLocalEyeExamsQueued));+ return withoutTombstones({ ...data, exams });}/** Resolve an eye exam id, or a legacy numeric index into the date-sorted list, to an id. */@@ -186,7 +191,8 @@ export async function addEyeExam(exam) {console.log(`👁️ Eye exam added for ${stored.date} (MortalLoom)`);return stored;}- const stored = { ...exam, id: exam.id || randomUUID() };+ // `createdAt` marks a minted id: without it the row reads as legacy and is re-keyed by content.+ const stored = { ...exam, id: exam.id || randomUUID(), createdAt: new Date().toISOString() };await queueHealthWrite(async () => {const data = await loadLocalEyeExamsQueued();data.exams.push(stored);@@ -214,7 +220,8 @@ export async function updateEyeExam(id, updates) {const data = await loadLocalEyeExamsQueued();const target = data.exams.find(e => e.id === id);if (!target) return null;- Object.assign(target, patch);+ // `updatedAt` is the per-row clock the federation merge resolves copies by.+ Object.assign(target, patch, { updatedAt: new Date().toISOString() });data.exams.sort(byDate);await writeLocal(EYES_FILE, data);return target;@@ -237,6 +244,8 @@ export async function removeEyeExam(id) {const found = data.exams.find(e => e.id === id);if (!found) return null;data.exams = data.exams.filter(e => e.id !== id);+ // The marker keeps a peer that still holds the row from syncing it back.+ data[ROW_TOMBSTONES_KEY] = recordTombstone(data[ROW_TOMBSTONES_KEY], id, { keyField: 'id' });await writeLocal(EYES_FILE, data);return found;});diff --git a/server/services/meatspaceHealth.test.js b/server/services/meatspaceHealth.test.jsindex bed66df504..1ffcc2ba52 100644--- a/server/services/meatspaceHealth.test.js+++ b/server/services/meatspaceHealth.test.js@@ -279,7 +279,32 @@ describe('eye exams by id', () => {expect(disk.exams.every(e => e.id)).toBe(true);expect(await resolveEyeExamId('0')).toBe(exams[0].id);expect(await resolveEyeExamId('9')).toBeNull();- await removeEyeExam(await resolveEyeExamId('0'));+ const removedId = await resolveEyeExamId('0');+ await removeEyeExam(removedId);expect(disk.exams.map(e => e.date)).toEqual(['2026-03-01']);+ // The delete leaves a marker for peers; readers never see it.+ expect(disk.rowTombstones.map(t => t.id)).toEqual([removedId]);+ expect(await getEyeExams()).not.toHaveProperty('rowTombstones');+ });++ it('gives the same legacy exam the same id on every machine (#12432)', async () => {+ const legacy = { date: '2025-01-01', leftSphere: -1.5 };+ disk = { exams: [{ ...legacy }] };+ const first = (await getEyeExams()).exams[0].id;+ // A second machine holding the same row, or one this machine stamped with a+ // random id before #12432 (no createdAt), lands on the same id.+ disk = { exams: [{ ...legacy, id: '4f1c2a9e-0000-4000-8000-000000000000' }] };+ const second = (await getEyeExams()).exams[0].id;+ expect(first).toMatch(/^legacy-/);+ expect(second).toBe(first);+ });++ it('stamps createdAt on a new exam and updatedAt on an edit, so peers resolve copies by clock', async () => {+ const exam = await addEyeExam({ date: '2025-01-01' });+ expect(exam.createdAt).toEqual(expect.any(String));+ // A minted exam keeps its id through the legacy re-keying read.+ expect((await getEyeExams()).exams[0].id).toBe(exam.id);+ const updated = await updateEyeExam(exam.id, { leftSphere: -2 });+ expect(updated.updatedAt).toEqual(expect.any(String));});});diff --git a/server/services/sharingDurability.test.js b/server/services/sharingDurability.test.jsindex bcfe5a4d7a..ac7211d2f6 100644--- a/server/services/sharingDurability.test.js+++ b/server/services/sharingDurability.test.js@@ -44,6 +44,7 @@ const annotations = await import('./sharing/annotationsSync.js');const alcohol = await import('./meatspaceAlcohol.js');const nicotine = await import('./meatspaceNicotine.js');const { PATHS } = await import('../lib/fileUtils.js');+const { PORTOS_SCHEMA_VERSIONS } = await import('../lib/schemaVersions.js');const bucketPath = join(tempRoot, 'bucket');let bucket;const seed = async (path, value) => {@@ -260,10 +261,12 @@ describe('daily-log event identity (#8143)', () => {await seed(path(), { entries: [day([lager('drink-a')])] });const before = await readFile(path(), 'utf8');const result = await dataSync.applyRemote('meatspace', snapshot(day([lager('drink-b')])), {- portosMeta: { portosVersion: '99.0.0', schemaVersions: { meatspace: 2 } },+ portosMeta: { portosVersion: '99.0.0', schemaVersions: { meatspace: PORTOS_SCHEMA_VERSIONS.meatspace + 1 } },});expect(result.applied).toBe(false);- expect(result.blockedBySchema.ahead).toEqual([{ category: 'meatspace', senderV: 2, receiverV: 1 }]);+ expect(result.blockedBySchema.ahead).toEqual([+ { category: 'meatspace', senderV: PORTOS_SCHEMA_VERSIONS.meatspace + 1, receiverV: PORTOS_SCHEMA_VERSIONS.meatspace },+ ]);expect(await readFile(path(), 'utf8')).toBe(before);});});