Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: stop legacy eye exams duplicating across synced machines and sync eye exam edits and deletes (#12432)

[._.]/ Adam Eivy7a05c81b3ec6
diff --git a/scripts/migrations/429-eye-exam-legacy-row-ids.js b/scripts/migrations/429-eye-exam-legacy-row-ids.js
new file mode 100644
index 0000000000..fd8a185cde
--- /dev/null
+++ b/scripts/migrations/429-eye-exam-legacy-row-ids.js
@@ -0,0 +1,25 @@
+/** Re-key legacy eye exams to content-derived ids and collapse their duplicates
+ * (#12432). Before this, each machine stamped an id-less eyes.json row with its
+ * own `randomUUID()`, so syncing peers appended each other's copy of the same
+ * exam. Every row without `createdAt` predates minted ids, so its random stamp
+ * is replaced by the `legacy-` id every peer derives from the row's content, and
+ * rows that land on the same id are one exam. Rows created since keep their id.
+ * Input-gated on eyes.json; ships no seed. Idempotent: re-keyed rows keep their id. */
+import { join } from 'node:path';
+import { readJsonOrNull, writeJsonAtomic } from './_lib.js';
+import { normalizeLegacyRows } from '../../server/lib/meatspaceRowIdentity.js';
+
+export default {
+ async up({ rootDir }) {
+ const path = join(rootDir, 'data', 'meatspace', 'eyes.json');
+ const doc = await readJsonOrNull(path);
+ if (!doc || !Array.isArray(doc.exams)) return { rekeyed: 0, collapsed: 0 };
+ const { rows, changed } = normalizeLegacyRows(doc.exams);
+ if (!changed) return { rekeyed: 0, collapsed: 0 };
+ const kept = new Set(doc.exams);
+ const rekeyed = rows.filter((row) => !kept.has(row)).length;
+ rows.sort((a, b) => (a?.date || '').localeCompare(b?.date || ''));
+ await writeJsonAtomic(path, { ...doc, exams: rows });
+ return { rekeyed, collapsed: doc.exams.length - rows.length };
+ }
+};
diff --git a/scripts/migrations/429-eye-exam-legacy-row-ids.test.js b/scripts/migrations/429-eye-exam-legacy-row-ids.test.js
new file mode 100644
index 0000000000..d38b39e458
--- /dev/null
+++ b/scripts/migrations/429-eye-exam-legacy-row-ids.test.js
@@ -0,0 +1,53 @@
+import { mkdtemp, mkdir, readFile, writeFile, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, beforeEach, expect, it } from 'vitest';
+import migration from './429-eye-exam-legacy-row-ids.js';
+import { legacyRowId } from '../../server/lib/meatspaceRowIdentity.js';
+
+let rootDir;
+beforeEach(async () => { rootDir = await mkdtemp(join(tmpdir(), 'portos-eye-legacy-ids-')); });
+afterEach(() => rm(rootDir, { recursive: true, force: true }));
+
+const eyesPath = () => join(rootDir, 'data', 'meatspace', 'eyes.json');
+const seed = async (doc) => {
+ await mkdir(join(rootDir, 'data', 'meatspace'), { recursive: true });
+ await writeFile(eyesPath(), JSON.stringify(doc));
+};
+const read = async () => JSON.parse(await readFile(eyesPath(), 'utf8'));
+
+it('collapses random-stamped copies of one legacy exam, re-keys singles, keeps minted rows, and is idempotent', async () => {
+ const legacy = { date: '2024-05-01', leftSphere: -1, rightSphere: -1.25 };
+ const single = { date: '2023-02-01', leftSphere: -0.5 };
+ const minted = { id: 'exam-minted', createdAt: '2026-01-01T00:00:00.000Z', date: '2026-01-01', leftSphere: -2 };
+ await seed({
+ exams: [
+ { id: '11111111-0000-4000-8000-000000000001', ...legacy }, // stamped on this machine
+ { id: '22222222-0000-4000-8000-000000000002', ...single },
+ { id: '33333333-0000-4000-8000-000000000003', ...legacy }, // the peer's stamp, appended by a sync
+ minted,
+ ],
+ note: 'kept',
+ });
+
+ expect(await migration.up({ rootDir })).toEqual({ rekeyed: 2, collapsed: 1 });
+ const doc = await read();
+ expect(doc.note).toBe('kept');
+ expect(doc.exams).toEqual([
+ { id: legacyRowId(single), ...single },
+ { id: legacyRowId(legacy), ...legacy },
+ minted,
+ ]);
+
+ const bytes = await readFile(eyesPath(), 'utf8');
+ expect(await migration.up({ rootDir })).toEqual({ rekeyed: 0, collapsed: 0 });
+ expect(await readFile(eyesPath(), 'utf8')).toBe(bytes);
+});
+
+it('does nothing without an eyes.json or with an unreadable one', async () => {
+ expect(await migration.up({ rootDir })).toEqual({ rekeyed: 0, collapsed: 0 });
+ await mkdir(join(rootDir, 'data', 'meatspace'), { recursive: true });
+ await writeFile(eyesPath(), '{not json');
+ expect(await migration.up({ rootDir })).toEqual({ rekeyed: 0, collapsed: 0 });
+ expect(await readFile(eyesPath(), 'utf8')).toBe('{not json');
+});
diff --git a/server/lib/README.md b/server/lib/README.md
index c007e0e83e..baefe57814 100644
--- a/server/lib/README.md
+++ b/server/lib/README.md
@@ -250,6 +250,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub
| `modelInventory.js` | Identity AND durable shape of a downloaded-model inventory row. `modelInventoryId(backend, key)` mints the `hf:` / `lora:` / `ollama:` / `lmstudio:` id and `parseModelInventoryId` splits it back on the FIRST colon (an Ollama tag like `qwen3:8b` carries one of its own). `hfInventoryRow` / `loraInventoryRow` / `localModelInventoryRow` build the row a backend contributes, and `presentInventoryRow` re-derives the live fields (residency, inventory-verified) that a STORED row can never answer. Shared by the scan that mints rows from disk (`services/systemResources.js`) and the manifest that records them at the install chokepoints (`services/modelManifest.js`) — a drifted copy of a row's risk, manage path or delete action is a delete button pointed at the wrong place. Pure and dependency-free, so the install services reach the builders without pulling the scan's module graph. |
| `videoFailure.js` | `normalizeVideoFailure` classifies and scrubs local video causes; `createVideoDiagnosticTail` retains bounded stdout/stderr diagnostics without grouping bare exit codes. |
| `maintenanceSequence.js` | The maintenance ladder shared by both runners: `MAINTENANCE_TASK_ORDER` / `MAINTENANCE_ORDER_GUIDANCE`, the interleaved `MAINTENANCE_SEQUENCE_TYPES` (a perpetual `claim-issue` drain between every pair of audits), `maintenanceStepParams` (audits file issues, documentation does the work), and `buildMaintenanceSteps` (fix mode runs audits directly with one final drain; issue filing can omit claim jobs) which emits quota-burn-shaped run-once steps so `quotaBurnInvoke.js` can run a step from the Schedule tab's manual run or a Quota Burn family plan alike. Pure. |
+| `meatspaceRowIdentity.js` | Cross-machine identity for MeatSpace health rows (#12432): `legacyRowId` derives a `legacy-<sha256>` id from a legacy row's content, `legacyRowIdentity` / `contentRowIdentity` key rows for the `row-union` snapshot merge, `preferRowCopy` is the total per-row `updatedAt` LWW order, `normalizeLegacyRows` re-keys and collapses legacy eye exams (also migration 429), and `ROW_TOMBSTONES_KEY` names the per-file deletion markers. Pure. |
| `meetingUrl.js` | Selects the one cacheable join URL (`selectMeetingUrl`) out of a raw Google Calendar event — first http(s) `video` entry point, else `hangoutLink` — and owns the shared `MEETING_URL_MAX` bound. Three-state: `undefined` = the producer never described conferencing, `null` = described with none usable. |
| `minimaxH3Memory.js` | The declared weight-placement table for the three MiniMax H3 entries (#5420) — H3 is the one video model family whose components fit nowhere unassisted, so "does this box have enough" is a render gate, not a UI fact. `MINIMAX_H3_MEMORY_PROFILES` maps entry id → `{ shippedRepo, shippedRevision, profiles }`, each profile carrying an honest `minMemoryGb` host floor and (CUDA only) a `minVramGb` device floor, ordered best-first. Every capacity number is HOISTED from what already existed — the CUDA tiers are `resolve_offload_profile()`'s own thresholds in `scripts/generate_minimax_h3_cuda.py`, the host floors are the entries' `memoryGb` — the sole new number being `MINIMAX_H3_HOST_RESERVE_GB`, a policy reserve held back for the OS. `applyMiniMaxH3MemoryProfiles(list)` is the load-time backfill (twin of migration 317) and guards BOTH `repo` and `revision`, like the speed-profile decorator. `selectMiniMaxH3MemoryProfile({ model, totalMemoryGb })` picks the best profile the HOST can hold (VRAM is the runner's call — the server has no synchronous device view); `miniMaxH3MemoryDeclineReason()` RETURNS the fail-closed reason so the render path can 400 it and a status surface can show it, and returns `null` on an UNMEASURED host — "not measured" is a deferral to the runner, never the same as zero. `validateMiniMaxH3MemoryProfileTable` / `sanitizeMiniMaxH3MemoryProfiles` warn + strip a hand-edited table (NaN floor, duplicate/reserved id, mis-ordered tiers) at load. |
| `videoContinuity.js` | How chunk N+1 of a chained video render is conditioned on chunk N. `resolveContextFrames(requested)` normalizes the tail-window size (absent → `DEFAULT_CONTEXT_FRAMES` = 22 ≈ 1s @ 24fps; an explicit `0` is preserved as "last frame only", NOT collapsed into the default) and clamps to `MIN_CONTEXT_FRAMES`..`MAX_CONTEXT_FRAMES`. `resolveContinuityStrategy({model, contextFrames})` picks `'window'` (LTX-2 `extend_from_video` conditioned on the prior chunk's last N frames — motion, not just a pose) or `'frame'` (extract the last frame, run i2v), degrading to `'frame'` on any runtime outside `CONTEXT_WINDOW_RUNTIMES` rather than rejecting. `extendLatentFrames` / `extendedPixelFrames` convert across the VAE's `LATENT_FRAME_STRIDE` (8 pixel frames per latent), `contextPrefixFrames({totalFrames, extendLatents})` measures how much of an extend render is echoed context to trim back off before stitching (0 = leave it alone), `tailWindowStartFrame` gives the cut index for the window itself, and `planExtendSplice({sourceFrames, outputFrames, extendLatents})` decides where a plain native Extend joins the ORIGINAL source to the render's new frames (`null` = layout doesn't verify, keep the model output). Pure — importable from `prepareParams.js` without dragging in `local.js`. Mirrored for the picker in `client/src/lib/videoGenParams.js`, pinned by `videoContinuity.parity.test.js`. |
diff --git a/server/lib/index.js b/server/lib/index.js
index c9cfdd8b28..50c337031e 100644
--- a/server/lib/index.js
+++ b/server/lib/index.js
@@ -227,6 +227,7 @@ export * from './mediaModelBuckets.js';
export * from './mediaModels.js';
export * from './modelInput.js';
export * from './modelInventory.js';
+export * from './meatspaceRowIdentity.js';
export * from './meetingUrl.js';
export * from './minimaxH3Memory.js';
export * from './videoContinuity.js';
diff --git a/server/lib/meatspaceRowIdentity.js b/server/lib/meatspaceRowIdentity.js
new file mode 100644
index 0000000000..7242d60138
--- /dev/null
+++ b/server/lib/meatspaceRowIdentity.js
@@ -0,0 +1,99 @@
+/**
+ * Cross-machine identity for MeatSpace health rows (blood / epigenetic / eye
+ * records) that federate through the `row-union` snapshot merge (#12432).
+ *
+ * A row created before rows carried identity has no `id`, or an `id` a machine
+ * stamped with `randomUUID()` on first read — two machines holding the same
+ * legacy row then gave it two different ids, and every sync duplicated it. A
+ * legacy row's identity is therefore derived from its content
+ * (`legacy-<sha256>`), so every machine names the same legacy row the same way.
+ * Rows created since carry a minted id plus `createdAt`; the id is their
+ * identity and two identical rows with different ids stay two records.
+ *
+ * Pure. Migration 429 imports `normalizeLegacyRows`, so the derivation here IS
+ * the on-disk id: changing it would re-key every peer's rows differently.
+ */
+
+import { createHash } from 'crypto';
+import { canonicalStringify, isPlainObject } from './objects.js';
+import { parseTsMs } from './lwwTimestamp.js';
+
+export const LEGACY_ROW_ID_PREFIX = 'legacy-';
+
+// Per-row deletion markers (`{ id, deletedAt }`) kept at the top level of a
+// row-union file, unioned in both directions by the snapshot merge.
+export const ROW_TOMBSTONES_KEY = 'rowTombstones';
+
+const hasStringId = (row) => typeof row?.id === 'string' && row.id !== '';
+
+export const isLegacyRowId = (id) => typeof id === 'string' && id.startsWith(LEGACY_ROW_ID_PREFIX);
+
+/** Deterministic id for a legacy row: a hash of its content without identity or clock fields. */
+export function legacyRowId(row) {
+ const { id: _id, createdAt: _createdAt, updatedAt: _updatedAt, ...content } = row;
+ const digest = createHash('sha256').update(canonicalStringify(content)).digest('hex');
+ return `${LEGACY_ROW_ID_PREFIX}${digest.slice(0, 24)}`;
+}
+
+/**
+ * The identity of a row whose file once stamped random ids (eyes.json). A row
+ * without `createdAt` predates minted identity, so a non-legacy id on it is a
+ * per-machine random stamp and is replaced by the content-derived id. An
+ * edited legacy row keeps its `legacy-` id even though its content moved on.
+ */
+export function legacyRowIdentity(row) {
+ if (hasStringId(row) && (row.createdAt !== undefined || isLegacyRowId(row.id))) return row.id;
+ return legacyRowId(row);
+}
+
+/** Identity for rows that never had random stamps: their own id, else their content. */
+export function contentRowIdentity(row) {
+ return hasStringId(row) ? row.id : legacyRowId(row);
+}
+
+/** A row's newest clock: `updatedAt`, else `createdAt` (string, or undefined). */
+export const rowLiveStamp = (row) => {
+ const updated = parseTsMs(row?.updatedAt);
+ const created = parseTsMs(row?.createdAt);
+ if (updated === null) return created === null ? undefined : row.createdAt;
+ if (created === null || updated >= created) return row.updatedAt;
+ return row.createdAt;
+};
+
+/**
+ * Pick one of two copies of the same row: newer live stamp wins, then the
+ * larger canonical text. The order is total, so every machine converges on the
+ * same copy whichever side is local.
+ */
+export function preferRowCopy(a, b) {
+ const am = parseTsMs(rowLiveStamp(a)) ?? -Infinity;
+ const bm = parseTsMs(rowLiveStamp(b)) ?? -Infinity;
+ if (am !== bm) return am > bm ? a : b;
+ return canonicalStringify(a) >= canonicalStringify(b) ? a : b;
+}
+
+/**
+ * Re-key every row of an id-stamping file to `legacyRowIdentity` and collapse
+ * rows that share an identity to their preferred copy (content-identical legacy
+ * rows are one record). Order follows each identity's first occurrence.
+ * Non-object rows are kept as they are.
+ */
+export function normalizeLegacyRows(rows) {
+ const out = [];
+ const indexById = new Map();
+ for (const row of Array.isArray(rows) ? rows : []) {
+ if (!isPlainObject(row)) { out.push(row); continue; }
+ const id = legacyRowIdentity(row);
+ const keyed = row.id === id ? row : { ...row, id };
+ const index = indexById.get(id);
+ if (index === undefined) {
+ indexById.set(id, out.length);
+ out.push(keyed);
+ } else {
+ out[index] = preferRowCopy(out[index], keyed);
+ }
+ }
+ const changed = out.length !== (Array.isArray(rows) ? rows.length : 0)
+ || out.some((row, i) => row !== rows[i]);
+ return { rows: out, changed };
+}
diff --git a/server/lib/schemaVersions.js b/server/lib/schemaVersions.js
index 2737b0fa4c..9a6daedfdc 100644
--- a/server/lib/schemaVersions.js
+++ b/server/lib/schemaVersions.js
@@ -922,9 +922,21 @@ export const PORTOS_SCHEMA_VERSIONS = Object.freeze({
// send a stale subset back, which a union ignores.
// Rows without an `id` (logged before #8143) are still add-only: deleting
// or moving one does not reach peers.
- // The key exists so the NEXT incompatible daily-log change can bump to 2 and
- // have v1 receivers reject it.
- meatspace: 1,
+ //
+ // v2 (#12432): blood/epigenetic/eye rows merge by identity. A legacy row (no
+ // id, or the random id a pre-v2 reader stamped on it) is keyed by a
+ // content-derived `legacy-` id every machine computes alike, copies of one
+ // row resolve by per-row `updatedAt`, and a delete rides a top-level
+ // `rowTombstones: [{ id, deletedAt }]` list in the row's file. A v1 receiver
+ // must refuse that: its union keys a legacy row by its own random stamp, so
+ // every `legacy-` id would append a second copy of a row it already holds,
+ // and it cannot read tombstones or edits. v1 → v2 stays accepted: a v2
+ // receiver re-keys the v1 sender's random-stamped legacy rows (they carry no
+ // `createdAt`) to the same content id, so they match instead of duplicating.
+ // An edit made on a v1 peer still arrives as a second row, as it always did.
+ // Until a v1 peer upgrades it refuses the whole meatspace snapshot from a v2
+ // peer, daily log included; the gap shows on the Instances page.
+ meatspace: 2,
// v1 = the paired-peer credential (#8356): `X-PortOS-Peer-Auth`, an HMAC of the
// pair `syncSecret` bound to the sender's instance id, replacing the instance
// password over HTTP Basic between paired peers. Not a record or snapshot
diff --git a/server/services/dataSync.js b/server/services/dataSync.js
index 7d6c0a39d2..e4073ee5d8 100644
--- a/server/services/dataSync.js
+++ b/server/services/dataSync.js
@@ -17,6 +17,13 @@ import { snapshotChecksum } from '../lib/snapshotChecksum.js';
import { parseTsMs } from '../lib/lwwTimestamp.js';
import { isTombstoned, mergeTombstones, pruneTombstones } from '../lib/tombstones.js';
import { GOAL_TOMBSTONES_KEY, reconcileGoalTombstones } from './identity/goalTombstones.js';
+import {
+ ROW_TOMBSTONES_KEY,
+ contentRowIdentity,
+ legacyRowIdentity,
+ preferRowCopy,
+ rowLiveStamp,
+} from '../lib/meatspaceRowIdentity.js';
import {
DAILY_LOG_TOMBSTONES_KEY,
dailyLogEventLiveStamp,
@@ -126,20 +133,19 @@ const STORY_BUILDER_DIR = join(PATHS.data, 'story-builder');
// checksum cache.
const STORY_BUILDER_EPOCH_KEY = '__storyBuilderEpoch';
-// Blood/epigenetic/eye records merge by identity-preserving union
-// (`unionPreservingRows`): `date` is NOT unique for them (two labs on one day,
-// a retest, left/right exams entered separately), so keying by date would
-// collapse same-day rows (#10910). Known limitation, intentionally not fixed
-// here: the union is add-only, so a row edited on one machine never reaches a
-// peer that already holds the old copy, and a deleted row (e.g. an eye exam
-// removed with `removeEyeExam`) is re-added by the next snapshot from any peer
-// that still has it. Propagating edits/deletes needs tombstones plus a
-// `PORTOS_SCHEMA_VERSIONS.meatspace` bump.
+// Blood/epigenetic/eye records merge by row identity (`mergeRowUnionFile`), not
+// by `date`: two labs on one day, a retest, or left/right exams entered
+// separately share a date (#10910). Identity is the row's minted id, or for a
+// legacy row a content-derived `legacy-` id every machine computes alike
+// (`lib/meatspaceRowIdentity.js`, #12432). Copies of one row resolve by per-row
+// `updatedAt` LWW, and a delete travels as a `rowTombstones` entry, so edits and
+// deletes reach peers at `PORTOS_SCHEMA_VERSIONS.meatspace` 2. `legacyIds` marks
+// the file whose pre-#12432 readers stamped random ids on legacy rows.
const MEATSPACE_FILES = {
'daily-log.json': { type: 'daily-log' }, // mergeDailyLogFile
'blood-tests.json': { type: 'row-union', arrayKey: 'tests' },
'epigenetic-tests.json': { type: 'row-union', arrayKey: 'tests' },
- 'eyes.json': { type: 'row-union', arrayKey: 'exams' },
+ 'eyes.json': { type: 'row-union', arrayKey: 'exams', legacyIds: true },
'config.json': { type: 'object-lww' }
};
@@ -407,52 +413,66 @@ function mergeArraysByKey(localArr, remoteArr, idField, timestampField, mergeRec
}
/**
- * Union two arrays of records WITHOUT collapsing any local row: every local row
- * is kept verbatim and in order, and a remote row is appended only when its
- * identity is not already present locally. For records whose date (or any other
- * field) is not unique, so `mergeArraysByKey` would silently drop one of two
- * same-key local rows.
- *
- * Identity is the string `id` when the row has one, else the row's content
- * (`canonicalStringify`, the same content-signature approach as
- * `digital-twin-sync.js#unionKeyFor`). A row stamped with an id after the fact
- * (legacy eye exams get one on first read) still matches its id-less copy on a
- * peer, so a content match ignores `id` — but an id-bearing remote row is only
- * content-matched against local rows that have no id, so two distinct records
- * that happen to hold identical values are both kept.
+ * Merge a peer's row-union file (blood / epigenetic / eye records) into the
+ * local one. Pure: returns the file to persist.
*
- * Same-identity rows are not reconciled: the local copy stands (see the
- * `MEATSPACE_FILES` note on edits/deletes).
+ * Rows match by identity (`legacyRowIdentity` for an id-stamping file, else
+ * `contentRowIdentity`); copies of one row resolve to `preferRowCopy`, so an
+ * edit stamped with `updatedAt` wins on every machine. Local rows that share an
+ * identity in a non-stamping file are kept as they are — only a peer's copy is
+ * deduped against them — so the merge never drops a local row it cannot tell
+ * apart. Tombstones union both ways and drop a row unless it was edited after
+ * the delete; a row that outlives its tombstone prunes it.
*/
-function unionPreservingRows(localArr, remoteArr) {
- const contentKey = (row) => {
- const { id: _id, ...rest } = row;
- return canonicalStringify(rest);
- };
- const localIds = new Set();
- const idlessLocalContent = new Set();
- const allLocalContent = new Set();
- const track = (row) => {
- const hasId = typeof row.id === 'string' && row.id !== '';
- if (hasId) localIds.add(row.id);
- const content = contentKey(row);
- allLocalContent.add(content);
- if (!hasId) idlessLocalContent.add(content);
+function mergeRowUnionFile(localFile, remoteFile, { arrayKey, legacyIds = false }) {
+ const local = isPlainObject(localFile) ? localFile : {};
+ const remote = isPlainObject(remoteFile) ? remoteFile : {};
+ const identity = legacyIds ? legacyRowIdentity : contentRowIdentity;
+ const rows = [];
+ const keys = [];
+ const indexByKey = new Map();
+ const add = (row, fromLocal) => {
+ if (!isPlainObject(row)) {
+ if (fromLocal) { rows.push(row); keys.push(null); }
+ return;
+ }
+ const key = identity(row);
+ const keyed = legacyIds && row.id !== key ? { ...row, id: key } : row;
+ const index = indexByKey.get(key);
+ if (index !== undefined && (legacyIds || !fromLocal)) {
+ rows[index] = preferRowCopy(rows[index], keyed);
+ return;
+ }
+ if (index === undefined) indexByKey.set(key, rows.length);
+ rows.push(keyed);
+ keys.push(key);
};
- for (const row of localArr) if (isPlainObject(row)) track(row);
-
- const merged = [...localArr];
- for (const remoteRow of remoteArr) {
- if (!isPlainObject(remoteRow)) continue;
- const hasId = typeof remoteRow.id === 'string' && remoteRow.id !== '';
- const present = hasId
- ? localIds.has(remoteRow.id) || idlessLocalContent.has(contentKey(remoteRow))
- : allLocalContent.has(contentKey(remoteRow));
- if (present) continue;
- merged.push(remoteRow);
- track(remoteRow);
- }
- return { merged, changed: merged.length !== localArr.length };
+ for (const row of Array.isArray(local[arrayKey]) ? local[arrayKey] : []) add(row, true);
+ for (const row of Array.isArray(remote[arrayKey]) ? remote[arrayKey] : []) add(row, false);
+
+ const { merged: unionTombstones } = mergeTombstones(
+ local[ROW_TOMBSTONES_KEY],
+ remote[ROW_TOMBSTONES_KEY],
+ { keyField: 'id' },
+ );
+ const kept = [];
+ const liveStamps = [];
+ rows.forEach((row, i) => {
+ if (keys[i] !== null) {
+ const stamp = rowLiveStamp(row);
+ if (isTombstoned(unionTombstones, keys[i], stamp, 'id')) return;
+ liveStamps.push({ id: keys[i], stamp });
+ }
+ kept.push(row);
+ });
+ const tombstones = pruneTombstones(unionTombstones, liveStamps, { keyField: 'id', timestampField: 'stamp' });
+ // Stable by date; same-date rows keep local-then-remote order.
+ kept.sort((a, b) => (a?.date || '').localeCompare(b?.date || ''));
+
+ const mergedFile = { ...remote, ...local, [arrayKey]: kept };
+ if (tombstones.length > 0) mergedFile[ROW_TOMBSTONES_KEY] = tombstones;
+ else delete mergedFile[ROW_TOMBSTONES_KEY];
+ return mergedFile;
}
/**
@@ -626,13 +646,11 @@ async function applyMeatspaceRemote(remoteData) {
} else {
const written = await queueHealthWrite(async () => {
const local = await readJSONFile(filePath, null, { strict: true });
- const localArr = Array.isArray(local?.[config.arrayKey]) ? local[config.arrayKey] : [];
- const remoteArr = Array.isArray(remoteFile[config.arrayKey]) ? remoteFile[config.arrayKey] : [];
- const { merged, changed } = unionPreservingRows(localArr, remoteArr);
- if (!changed) return false;
- // Stable by date; same-date rows keep local-then-remote order.
- merged.sort((a, b) => (a?.date || '').localeCompare(b?.date || ''));
- await atomicWrite(filePath, { ...remoteFile, ...local, [config.arrayKey]: merged });
+ const merged = mergeRowUnionFile(local, remoteFile, config);
+ // A peer with nothing to add must not create the file here.
+ if (local === null && merged[config.arrayKey].length === 0 && !merged[ROW_TOMBSTONES_KEY]) return false;
+ if (canonicalStringify(merged) === canonicalStringify(local)) return false;
+ await atomicWrite(filePath, merged);
return true;
});
if (written) totalApplied++;
diff --git a/server/services/dataSync.meatspace.test.js b/server/services/dataSync.meatspace.test.js
index b96b1296db..5587f9c91e 100644
--- a/server/services/dataSync.meatspace.test.js
+++ b/server/services/dataSync.meatspace.test.js
@@ -25,6 +25,7 @@ vi.mock('../lib/fileUtils.js', async () => {
const dataSync = await import('./dataSync.js');
const health = await import('./meatspaceHealth.js');
+const { legacyRowId } = await import('../lib/meatspaceRowIdentity.js');
const { queueConfigWrite, queueHealthWrite } = await import('./meatspaceWriteQueues.js');
const MEATSPACE_DIR = join(tempRoot, 'meatspace');
@@ -64,10 +65,10 @@ const FAMILIES = {
'eyes.json': {
arrayKey: 'exams',
addLocal: (row) => health.addEyeExam(row),
- sameDateA: { id: 'exam-a', date: '2026-03-01', leftSphere: -1.0 },
- sameDateB: { id: 'exam-b', date: '2026-03-01', rightSphere: -1.25 },
- sameDateRemote: { id: 'exam-c', date: '2026-03-01', rightSphere: -1.5 },
- newDate: { id: 'exam-d', date: '2026-04-01', leftSphere: -1.0 },
+ sameDateA: { id: 'exam-a', createdAt: '2026-03-01T10:00:00.000Z', date: '2026-03-01', leftSphere: -1.0 },
+ sameDateB: { id: 'exam-b', createdAt: '2026-03-01T11:00:00.000Z', date: '2026-03-01', rightSphere: -1.25 },
+ sameDateRemote: { id: 'exam-c', createdAt: '2026-03-01T12:00:00.000Z', date: '2026-03-01', rightSphere: -1.5 },
+ newDate: { id: 'exam-d', createdAt: '2026-04-01T10:00:00.000Z', date: '2026-04-01', leftSphere: -1.0 },
},
};
@@ -94,21 +95,83 @@ describe.each(Object.entries(FAMILIES))('applyRemote meatspace %s', (filename, f
});
describe('applyRemote meatspace eyes.json identity', () => {
- it('matches an id-less local exam to its stamped remote copy, but keeps distinct ids with identical values', async () => {
+ const minted = '2026-05-01T09:00:00.000Z';
+
+ it('matches an id-less local exam to its stamped remote copy, but keeps distinct minted ids with identical values', async () => {
const values = { date: '2026-03-01', leftSphere: -2, rightSphere: -2 };
await persist('eyes.json', { exams: [values] }); // legacy row, no id yet
const result = await apply('eyes.json', {
exams: [
- { id: 'stamped-copy', ...values }, // the same exam after the peer stamped an id
- { id: 'twin-1', date: '2026-05-01', leftSphere: -3 },
- { id: 'twin-2', date: '2026-05-01', leftSphere: -3 }, // distinct record, identical values
+ { id: 'stamped-copy', ...values }, // the same exam after a pre-#12432 peer stamped a random id
+ { id: 'twin-1', createdAt: minted, date: '2026-05-01', leftSphere: -3 },
+ { id: 'twin-2', createdAt: minted, date: '2026-05-01', leftSphere: -3 }, // distinct record, identical values
],
});
expect(result.applied).toBe(true);
const exams = (await readDoc('eyes.json')).exams;
- expect(exams.map((e) => e.id ?? null)).toEqual([null, 'twin-1', 'twin-2']);
+ expect(exams.map((e) => e.id)).toEqual([legacyRowId(values), 'twin-1', 'twin-2']);
+ });
+
+ // #12432: each machine used to stamp a legacy row with its own random id, so the
+ // second sync (after both had stamped) appended the peer's copy on both sides.
+ it('does not duplicate a legacy exam when re-syncing after both machines stamped it', async () => {
+ const legacy = { date: '2025-02-01', leftSphere: -1.25, rightSphere: -1 };
+ await persist('eyes.json', { exams: [{ ...legacy }] });
+ await health.getEyeExams(); // this machine stamps its copy on first read
+
+ // The peer stamped the same row too — with a random id before upgrading, or
+ // the content id after.
+ await apply('eyes.json', { exams: [{ id: '0d9b4c1e-0000-4000-8000-000000000001', ...legacy }] });
+ await apply('eyes.json', { exams: [{ id: legacyRowId(legacy), ...legacy }] });
+
+ const exams = (await readDoc('eyes.json')).exams;
+ expect(exams).toEqual([{ id: legacyRowId(legacy), ...legacy }]);
+ });
+
+ it('resolves two copies of one exam to the newer edit on either side', async () => {
+ const original = { id: 'exam-x', createdAt: minted, date: '2026-05-01', leftSphere: -1 };
+ const edited = { ...original, leftSphere: -1.5, updatedAt: '2026-06-01T00:00:00.000Z' };
+
+ await persist('eyes.json', { exams: [original] });
+ expect((await apply('eyes.json', { exams: [edited] })).applied).toBe(true);
+ expect((await readDoc('eyes.json')).exams).toEqual([edited]);
+
+ // The stale copy from a peer that has not seen the edit changes nothing.
+ expect((await apply('eyes.json', { exams: [original] })).applied).toBe(false);
+ expect((await readDoc('eyes.json')).exams).toEqual([edited]);
+ });
+
+ it('keeps a deleted exam deleted when a peer that still holds it syncs, and carries the delete to that peer', async () => {
+ const kept = { id: 'exam-keep', createdAt: minted, date: '2026-05-01', leftSphere: -1 };
+ const gone = { id: 'exam-gone', createdAt: minted, date: '2026-05-02', leftSphere: -2 };
+ await persist('eyes.json', { exams: [kept, gone] });
+ await health.removeEyeExam('exam-gone');
+ const afterDelete = await readDoc('eyes.json');
+
+ // The peer still holds the row: applying its snapshot must not bring it back.
+ expect((await apply('eyes.json', { exams: [kept, gone] })).applied).toBe(false);
+ expect((await readDoc('eyes.json')).exams).toEqual([kept]);
+
+ // The other direction: a machine still holding the row drops it on receipt.
+ await persist('eyes.json', { exams: [kept, gone] });
+ expect((await apply('eyes.json', afterDelete)).applied).toBe(true);
+ const peer = await readDoc('eyes.json');
+ expect(peer.exams).toEqual([kept]);
+ expect(peer.rowTombstones).toEqual(afterDelete.rowTombstones);
+ });
+
+ it('keeps a row edited after its deletion and drops the outlived tombstone', async () => {
+ const row = { id: 'exam-back', createdAt: minted, date: '2026-05-01', leftSphere: -1 };
+ const tombstone = { id: 'exam-back', deletedAt: '2026-05-10T00:00:00.000Z' };
+ await persist('eyes.json', { exams: [], rowTombstones: [tombstone] });
+
+ const edited = { ...row, leftSphere: -3, updatedAt: '2026-05-11T00:00:00.000Z' };
+ expect((await apply('eyes.json', { exams: [edited] })).applied).toBe(true);
+ const doc = await readDoc('eyes.json');
+ expect(doc.exams).toEqual([edited]);
+ expect(doc).not.toHaveProperty('rowTombstones');
});
});
diff --git a/server/services/meatspaceHealth.js b/server/services/meatspaceHealth.js
index 9f3ceacfad..4c9dff0fa7 100644
--- a/server/services/meatspaceHealth.js
+++ b/server/services/meatspaceHealth.js
@@ -12,6 +12,8 @@ import { randomUUID } from 'crypto';
import { atomicWrite, PATHS, ensureDir, readJSONFile, getDateString } from '../lib/fileUtils.js';
import { readLocalDailyLog, mutateDailyLog } from './meatspaceDailyLog.js';
import { queueHealthWrite } from './meatspaceWriteQueues.js';
+import { normalizeLegacyRows, ROW_TOMBSTONES_KEY } from '../lib/meatspaceRowIdentity.js';
+import { recordTombstone } from '../lib/tombstones.js';
import {
isMortalLoomEnabled,
mlArrayIfEnabled,
@@ -147,28 +149,31 @@ const readLocalEyes = async () => {
};
/**
- * Local eyes.json with every row id-stamped. MUST run inside `queueHealthWrite`:
- * legacy rows have no id, so stamping rewrites the file.
+ * Local eyes.json with every row carrying its identity. MUST run inside
+ * `queueHealthWrite`: a legacy row (no id, or a pre-#12432 random stamp) is
+ * re-keyed to its content-derived id and persisted once, so the ids the client
+ * reads back match the ones every peer computes for the same row.
*/
async function loadLocalEyeExamsQueued() {
const { data, exams } = await readLocalEyes();
- // Legacy eyes.json rows have no id. Stamp and persist them once so the ids
- // the client reads back are the ones later edits/deletes resolve.
- if (exams.some(e => !e.id)) {
- for (const e of exams) e.id ||= randomUUID();
- exams.sort(byDate);
- await writeLocal(EYES_FILE, { ...data, exams });
- }
- return { ...data, exams };
+ const normalized = normalizeLegacyRows(exams);
+ if (!normalized.changed) return { ...data, exams };
+ normalized.rows.sort(byDate);
+ const next = { ...data, exams: normalized.rows };
+ await writeLocal(EYES_FILE, next);
+ return next;
}
+// Deletion markers are sync metadata, not part of the exam list callers read.
+const withoutTombstones = ({ [ROW_TOMBSTONES_KEY]: _tombstones, ...rest }) => rest;
+
export async function getEyeExams() {
const ml = await mlArrayIfEnabled('eyeExams');
if (ml) return { exams: [...ml].sort(byDate) };
const { data, exams } = await readLocalEyes();
- // Only a legacy file needs the (queued) stamping write; the common read stays lock-free.
- if (exams.some(e => !e.id)) return queueHealthWrite(loadLocalEyeExamsQueued);
- return { ...data, exams };
+ // Only a legacy file needs the (queued) re-keying write; the common read stays lock-free.
+ if (normalizeLegacyRows(exams).changed) return withoutTombstones(await queueHealthWrite(loadLocalEyeExamsQueued));
+ return withoutTombstones({ ...data, exams });
}
/** Resolve an eye exam id, or a legacy numeric index into the date-sorted list, to an id. */
@@ -186,7 +191,8 @@ export async function addEyeExam(exam) {
console.log(`👁️ Eye exam added for ${stored.date} (MortalLoom)`);
return stored;
}
- const stored = { ...exam, id: exam.id || randomUUID() };
+ // `createdAt` marks a minted id: without it the row reads as legacy and is re-keyed by content.
+ const stored = { ...exam, id: exam.id || randomUUID(), createdAt: new Date().toISOString() };
await queueHealthWrite(async () => {
const data = await loadLocalEyeExamsQueued();
data.exams.push(stored);
@@ -214,7 +220,8 @@ export async function updateEyeExam(id, updates) {
const data = await loadLocalEyeExamsQueued();
const target = data.exams.find(e => e.id === id);
if (!target) return null;
- Object.assign(target, patch);
+ // `updatedAt` is the per-row clock the federation merge resolves copies by.
+ Object.assign(target, patch, { updatedAt: new Date().toISOString() });
data.exams.sort(byDate);
await writeLocal(EYES_FILE, data);
return target;
@@ -237,6 +244,8 @@ export async function removeEyeExam(id) {
const found = data.exams.find(e => e.id === id);
if (!found) return null;
data.exams = data.exams.filter(e => e.id !== id);
+ // The marker keeps a peer that still holds the row from syncing it back.
+ data[ROW_TOMBSTONES_KEY] = recordTombstone(data[ROW_TOMBSTONES_KEY], id, { keyField: 'id' });
await writeLocal(EYES_FILE, data);
return found;
});
diff --git a/server/services/meatspaceHealth.test.js b/server/services/meatspaceHealth.test.js
index bed66df504..1ffcc2ba52 100644
--- a/server/services/meatspaceHealth.test.js
+++ b/server/services/meatspaceHealth.test.js
@@ -279,7 +279,32 @@ describe('eye exams by id', () => {
expect(disk.exams.every(e => e.id)).toBe(true);
expect(await resolveEyeExamId('0')).toBe(exams[0].id);
expect(await resolveEyeExamId('9')).toBeNull();
- await removeEyeExam(await resolveEyeExamId('0'));
+ const removedId = await resolveEyeExamId('0');
+ await removeEyeExam(removedId);
expect(disk.exams.map(e => e.date)).toEqual(['2026-03-01']);
+ // The delete leaves a marker for peers; readers never see it.
+ expect(disk.rowTombstones.map(t => t.id)).toEqual([removedId]);
+ expect(await getEyeExams()).not.toHaveProperty('rowTombstones');
+ });
+
+ it('gives the same legacy exam the same id on every machine (#12432)', async () => {
+ const legacy = { date: '2025-01-01', leftSphere: -1.5 };
+ disk = { exams: [{ ...legacy }] };
+ const first = (await getEyeExams()).exams[0].id;
+ // A second machine holding the same row, or one this machine stamped with a
+ // random id before #12432 (no createdAt), lands on the same id.
+ disk = { exams: [{ ...legacy, id: '4f1c2a9e-0000-4000-8000-000000000000' }] };
+ const second = (await getEyeExams()).exams[0].id;
+ expect(first).toMatch(/^legacy-/);
+ expect(second).toBe(first);
+ });
+
+ it('stamps createdAt on a new exam and updatedAt on an edit, so peers resolve copies by clock', async () => {
+ const exam = await addEyeExam({ date: '2025-01-01' });
+ expect(exam.createdAt).toEqual(expect.any(String));
+ // A minted exam keeps its id through the legacy re-keying read.
+ expect((await getEyeExams()).exams[0].id).toBe(exam.id);
+ const updated = await updateEyeExam(exam.id, { leftSphere: -2 });
+ expect(updated.updatedAt).toEqual(expect.any(String));
});
});
diff --git a/server/services/sharingDurability.test.js b/server/services/sharingDurability.test.js
index bcfe5a4d7a..ac7211d2f6 100644
--- a/server/services/sharingDurability.test.js
+++ b/server/services/sharingDurability.test.js
@@ -44,6 +44,7 @@ const annotations = await import('./sharing/annotationsSync.js');
const alcohol = await import('./meatspaceAlcohol.js');
const nicotine = await import('./meatspaceNicotine.js');
const { PATHS } = await import('../lib/fileUtils.js');
+const { PORTOS_SCHEMA_VERSIONS } = await import('../lib/schemaVersions.js');
const bucketPath = join(tempRoot, 'bucket');
let bucket;
const seed = async (path, value) => {
@@ -260,10 +261,12 @@ describe('daily-log event identity (#8143)', () => {
await seed(path(), { entries: [day([lager('drink-a')])] });
const before = await readFile(path(), 'utf8');
const result = await dataSync.applyRemote('meatspace', snapshot(day([lager('drink-b')])), {
- portosMeta: { portosVersion: '99.0.0', schemaVersions: { meatspace: 2 } },
+ portosMeta: { portosVersion: '99.0.0', schemaVersions: { meatspace: PORTOS_SCHEMA_VERSIONS.meatspace + 1 } },
});
expect(result.applied).toBe(false);
- expect(result.blockedBySchema.ahead).toEqual([{ category: 'meatspace', senderV: 2, receiverV: 1 }]);
+ expect(result.blockedBySchema.ahead).toEqual([
+ { category: 'meatspace', senderV: PORTOS_SCHEMA_VERSIONS.meatspace + 1, receiverV: PORTOS_SCHEMA_VERSIONS.meatspace },
+ ]);
expect(await readFile(path(), 'utf8')).toBe(before);
});
});