Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: keep source version clocks when share buckets merge records (#10761)
Adam Eivyb184ae91f03a
diff --git a/server/services/pipeline/issueCrud.js b/server/services/pipeline/issueCrud.jsindex 03bd84f73c..c71794fc64 100644--- a/server/services/pipeline/issueCrud.js+++ b/server/services/pipeline/issueCrud.js@@ -21,7 +21,7 @@ import {ERR_NOT_FOUND, ERR_VALIDATION, ERR_DUPLICATE, ERR_SEASON_LOCKED,TITLE_MAX, SERIES_ID_MAX, ISSUES_PER_RESPONSE_MAX,} from './issuesShared.js';-import { isStr, trimTo } from '../../lib/textUtils.js';+import { isNonBlankStr, isStr, trimTo } from '../../lib/textUtils.js';import { codedError } from '../../lib/codedError.js';export async function listIssues({@@ -356,7 +356,7 @@ export function insertIssueWithId(input = {}) {});}-function mergeIssuePatch(cur, patch = {}) {+function mergeIssuePatch(cur, patch = {}, updatedAt = null) {// Per-stage merge: a stage patch carries only the fields the caller is// changing (e.g. `{ genConfig }` or `{ cover }`). Without this, the top-level// spread would replace the entire stage object and silently drop sibling@@ -415,7 +415,7 @@ function mergeIssuePatch(cur, patch = {}) {// series push payload via sanitizeRecordForWire returning null....('ephemeral' in patch ? { ephemeral: patch.ephemeral } : {}),stages: mergedStages,- updatedAt: new Date().toISOString(),+ updatedAt: isNonBlankStr(updatedAt) ? updatedAt : new Date().toISOString(),});if (!merged) throw codedError('Invalid issue payload', ERR_VALIDATION);return merged;@@ -434,8 +434,30 @@ async function returnIssueAfterCoverRefresh({ issue, frontCoverChanged }) {return issue;}-export function updateIssue(id, patch = {}, { skipRenumber = false } = {}) {- const needsRenumber = !skipRenumber && ('seasonId' in patch || 'arcPosition' in patch);+// `patchOrMutator` overloads (mirrors updateSeries / updateUniverse):+// - Plain object: patch is applied directly inside the queue.+// - `async (latest) => patch | null`: runs INSIDE the series write queue+// against the freshest persisted issue; `null`/`undefined` skips the write+// (no clock bump, no emit) and resolves with the unchanged issue. A mutator+// always takes the full-state path, since whether it renumbers isn't known+// until its patch is.+// `updatedAt` stamps that clock instead of receipt time — only the share-bucket+// importer's remote LWW apply passes it, so the winning source revision's clock+// survives and the next one can still win (#10761).+export function updateIssue(id, patchOrMutator = {}, { skipRenumber = false, updatedAt = null } = {}) {+ const isMutator = typeof patchOrMutator === 'function';+ const resolvePatch = async (cur) => {+ if (!isMutator) return patchOrMutator;+ const patch = await patchOrMutator(cur);+ if (patch === null || patch === undefined) return null;+ if (Array.isArray(patch) || typeof patch !== 'object') {+ throw codedError('updateIssue mutator must return a plain object or null', ERR_VALIDATION);+ }+ return patch;+ };+ const skippedResult = (cur) => ({ issue: cur, frontCoverChanged: false });+ const needsRenumber = !skipRenumber+ && (isMutator || 'seasonId' in patchOrMutator || 'arcPosition' in patchOrMutator);if (!needsRenumber) {// Route through the SERIES tail (see updateStageWithLatest) so a plain// field update can't race a concurrent series-wide renumber rewriting this@@ -446,7 +468,9 @@ export function updateIssue(id, patch = {}, { skipRenumber = false } = {}) {const cur = await store().loadOne(id);if (!cur) throw codedError(`Issue not found: ${id}`, ERR_NOT_FOUND);if (cur.deleted) throw codedError(`Issue not found: ${id}`, ERR_NOT_FOUND);- const merged = mergeIssuePatch(cur, patch);+ const patch = await resolvePatch(cur);+ if (!patch) return skippedResult(cur);+ const merged = mergeIssuePatch(cur, patch, updatedAt);await saveIssueNow(merged);emitRecordUpdated('series', merged.seriesId);return {@@ -464,7 +488,9 @@ export function updateIssue(id, patch = {}, { skipRenumber = false } = {}) {if (idx < 0) throw codedError(`Issue not found: ${id}`, ERR_NOT_FOUND);const cur = state.issues[idx];if (cur.deleted) throw codedError(`Issue not found: ${id}`, ERR_NOT_FOUND);- const merged = mergeIssuePatch(cur, patch);+ const patch = await resolvePatch(cur);+ if (!patch) return skippedResult(cur);+ const merged = mergeIssuePatch(cur, patch, updatedAt);state.issues[idx] = merged;// A seasonId move affects both source and destination volumes, so full// renumber. An arcPosition change only reorders within the current volume.diff --git a/server/services/pipeline/series.js b/server/services/pipeline/series.jsindex 3c91b917f2..ca7648ccf3 100644--- a/server/services/pipeline/series.js+++ b/server/services/pipeline/series.js@@ -37,7 +37,7 @@ import {autoSubscribeRecordToAllPeers, unsubscribeAllForRecord,} from '../sharing/recordEvents.js';import { renameCollectionForSeries, unlinkCollectionsForSeries } from '../mediaCollections.js';-import { isStr, trimTo } from '../../lib/textUtils.js';+import { isNonBlankStr, isStr, trimTo } from '../../lib/textUtils.js';import { codedError } from '../../lib/codedError.js';// Storage backend dispatcher (#1015). Series records moved from per-record@@ -893,7 +893,7 @@ async function reconcileDraftParentUniverse(universeId) {});}-export async function updateSeries(id, patchOrMutator = {}) {+export async function updateSeries(id, patchOrMutator = {}, { updatedAt: incomingUpdatedAt = null } = {}) {// `patchOrMutator` overloads (mirrors updateUniverse in// server/services/universeBuilder/crud.js):// - Plain object: patch is applied directly inside the queue (legacy).@@ -903,6 +903,11 @@ export async function updateSeries(id, patchOrMutator = {}) {// pass) can't clobber a concurrent edit to the same series (issue// #8453). Returning `null`/`undefined` short-circuits the write and// resolves with the unchanged record.+ //+ // `options.updatedAt` stamps that clock instead of receipt time. Only a+ // remote LWW apply (the share-bucket importer) passes it: the winning source+ // record's clock must survive, or a receipt-time stamp would out-rank the+ // NEXT source revision and silently drop it (#10761).const isMutator = typeof patchOrMutator === 'function';// Pre-B.4 canon (characters/settings/objects) lives on the universe, not the// series — but a stale browser tab can still POST a legacy series shape and@@ -1033,7 +1038,7 @@ export async function updateSeries(id, patchOrMutator = {}) {...('severityWeights' in patch ? { severityWeights: patch.severityWeights } : {}),...('blockingSeverities' in patch ? { blockingSeverities: patch.blockingSeverities } : {}),llm: mergedLlm,- updatedAt: new Date().toISOString(),+ updatedAt: isNonBlankStr(incomingUpdatedAt) ? incomingUpdatedAt : new Date().toISOString(),});if (!next) throw codedError('Invalid series payload', ERR_VALIDATION);await store().saveOneNow(next.id, next);diff --git a/server/services/sharing/importer.js b/server/services/sharing/importer.jsindex faf30084cb..51aee6e082 100644--- a/server/services/sharing/importer.js+++ b/server/services/sharing/importer.js@@ -584,52 +584,67 @@ async function applyAutoMerge(bucket, manifest, records, { availableAssetKeys =}return;}- if (remoteWins(existing.updatedAt, record.updatedAt)) {+ // Cheap out-of-queue early-out: a local copy already at or past the remote+ // clock can only move further ahead, so there is nothing to apply. The+ // authoritative comparison re-runs below INSIDE the owning write queue.+ if (!remoteWins(existing.updatedAt, record.updatedAt)) return;+ // Re-read, compare, journal, and persist inside the domain write queue so a+ // local edit that lands after the read above can't be clobbered by this+ // older snapshot (#10761). `updatedAt: record.updatedAt` keeps the winning+ // SOURCE clock — a receipt-time stamp would out-rank the next delayed source+ // revision and silently drop it once its manifest is marked processed.+ let wrote = false;+ await withReexportSuppressed(suppressKind, suppressId, () => updateFn(existing.id, async (latest) => {+ if (!remoteWins(latest.updatedAt, record.updatedAt)) return null;+ let incoming = record;// A remote series that arrived without a universe link (an orphan on the// sender — older peer, or a record whose link was cleared before the- // hierarchy rule shipped) must NOT clear the local link: updateSeries now+ // hierarchy rule shipped) must NOT clear the local link: updateSeries// refuses to unlink a linked series and would throw, aborting the whole// manifest. Preserve the existing link (a *move* to a different non-empty- // universe still applies). Mirrors the legacy-canon re-stamp above, but+ // universe still applies). Mirrors the legacy-canon re-stamp below, but// for every series — not just ones carrying legacy canon arrays.- if (kind === 'series' && !record.universeId && existing.universeId) {- record = { ...record, universeId: existing.universeId };+ if (kind === 'series' && !incoming.universeId && latest.universeId) {+ incoming = { ...incoming, universeId: latest.universeId };}if (kind === 'issue' && senderCannotRepresentClimax- && existing.arcRole === 'climax' && record.arcRole !== 'climax') {- record = { ...record, arcRole: 'climax' };+ && latest.arcRole === 'climax' && incoming.arcRole !== 'climax') {+ incoming = { ...incoming, arcRole: 'climax' };}if (kind === 'universe') {- record = preserveLegacyUniverseFields(record, existing, senderUniversesVersion);+ incoming = preserveLegacyUniverseFields(incoming, latest, senderUniversesVersion);}// Non-blocking conflict journal for every synced record kind — a// share-bucket import that LWW-overwrites a locally-diverged record- // archives the losing local version first. Issues are included now- // (they previously rode LWW silently).+ // archives the losing local version first.if (JOURNALED_KINDS.has(kind)) {await maybeJournalBeforeOverwrite({- kind, id: record.id, local: existing, remote: record,+ kind, id: incoming.id, local: latest, remote: incoming,source: { via: 'share-bucket', bucketId: bucket.id, peerId: manifest.senderInstanceId ?? null },});}- await withReexportSuppressed(suppressKind, suppressId, () => updateFn(existing.id, record));+ wrote = true;+ return incoming;+ }, { updatedAt: record.updatedAt }));+ // Count only a write the queue actually performed — a newer local edit that+ // landed first turns this merge into a no-op.+ if (wrote) {overridden.push({ kind, id: record.id, label });applied++;}};// Universes first — series may reference them via universeId.- // Wrap updateUniverse in a mutator form so the service-side- // `referenceSheetImageRef` preservation guard (gated on `!isMutator`) is- // skipped: sync's intent is that the remote's newer record wins LWW,+ // updateUniverse's mutator form (which mergeOne always uses) also skips the+ // service-side `referenceSheetImageRef` preservation guard (gated on+ // `!isMutator`): sync's intent is that the remote's newer record wins LWW,// including server-owned operational pointers like the rendered sheet// filename. A literal-patch call would let the local stale pointer// overwrite the remote's freshly-rendered one.for (const uni of records.universes) {await mergeOne({kind: 'universe', record: uni, label: uni.name,- getFn: getUniverse, insertFn: insertUniverseWithId,- updateFn: (id, record) => updateUniverse(id, () => record),+ getFn: getUniverse, insertFn: insertUniverseWithId, updateFn: updateUniverse,});}// Pre-B.4 peers ship series records with legacy canon arrays (`characters /diff --git a/server/services/sharing/integration.test.js b/server/services/sharing/integration.test.jsindex 4c05ca7070..018300db21 100644--- a/server/services/sharing/integration.test.js+++ b/server/services/sharing/integration.test.js@@ -2675,4 +2675,162 @@ describe('sharing round-trip', () => {expect(hasBeenProcessed(cursor, exp.filename, exp.manifestId)).toBe(true);});});++ // ---------------------------------------------------------------------------+ // Remote LWW applies keep the SOURCE clock and compare inside the owning+ // write queue (#10761). A receipt-time stamp let a delayed second source+ // revision lose to the first one's manufactured clock, and the out-of-queue+ // compare let an older snapshot overwrite a local edit made after the read.+ // Clocks sit in 2020 so any real "now" a local edit stamps is newer.+ // ---------------------------------------------------------------------------+ describe('remote LWW keeps source clocks (#10761)', () => {+ const CLOCK = {+ initial: '2020-01-01T00:00:00.000Z',+ v2: '2020-01-02T00:00:00.000Z',+ v3: '2020-01-03T00:00:00.000Z',+ };+ const KINDS = [+ {+ kind: 'universe', recordDir: 'universes', field: 'name',+ async seed(bucketId) {+ const u = await universeSvc.createUniverse({ name: 'Initial' });+ await universeSvc.updateUniverse(u.id, { name: 'Initial' }, { updatedAt: CLOCK.initial });+ return { id: u.id, exportOnce: () => exporter.exportUniverse(u.id, bucketId) };+ },+ get: (id) => universeSvc.getUniverse(id),+ edit: (id, value) => universeSvc.updateUniverse(id, { name: value }),+ store: () => universeSvc.store(),+ },+ {+ kind: 'series', recordDir: 'series', field: 'name',+ async seed(bucketId) {+ const s = await series.createSeries({ name: 'Initial' });+ await series.updateSeries(s.id, { name: 'Initial' }, { updatedAt: CLOCK.initial });+ return { id: s.id, exportOnce: () => exporter.exportSeries(s.id, bucketId) };+ },+ get: (id) => series.getSeries(id),+ edit: (id, value) => series.updateSeries(id, { name: value }),+ store: () => series.seriesStore(),+ },+ {+ kind: 'issue', recordDir: 'issues', field: 'title',+ async seed(bucketId) {+ const s = await series.createSeries({ name: 'Parent Series' });+ const iss = await issues.createIssue({ seriesId: s.id, title: 'Initial' });+ await issues.updateIssue(iss.id, { title: 'Initial' }, { updatedAt: CLOCK.initial });+ return { id: iss.id, exportOnce: () => exporter.exportSeries(s.id, bucketId) };+ },+ get: (id) => issues.getIssue(id),+ edit: (id, value) => issues.updateIssue(id, { title: value }),+ store: () => issues.store(),+ },+ ];++ // Export N manifests up front (as if the sender shared N times before any+ // of them was delivered), each attributed to a remote peer.+ async function exportManifests(seeded, count) {+ const exps = [];+ for (let i = 0; i < count; i++) exps.push(await seeded.exportOnce());+ for (const exp of exps) simulateRemoteSender(tempBucket, exp.filename);+ expect(new Set(exps.map((e) => e.filename)).size).toBe(count);+ return exps;+ }++ // Overwrite the bucket's record file with one source revision — the+ // record-file content is what the next processed manifest delivers.+ function deliverRevision(fixture, id, value, updatedAt) {+ const p = join(tempBucket, 'records', fixture.recordDir, `${id}.json`);+ const rec = JSON.parse(readFileSync(p, 'utf-8'));+ writeFileSync(p, JSON.stringify({ ...rec, [fixture.field]: value, updatedAt }, null, 2));+ }++ const overriddenFor = (result, fixture, id) =>+ result.outcome.overridden.filter((o) => o.kind === fixture.kind && o.id === id);++ it.each(KINDS)('applies two delayed source revisions in order and stores the winning source clock ($kind)', async (fixture) => {+ const bucket = await buckets.createBucket({ name: 'ClockBucket', path: tempBucket, mode: 'auto-merge' });+ const seeded = await fixture.seed(bucket.id);+ const [first, second] = await exportManifests(seeded, 2);++ deliverRevision(fixture, seeded.id, 'Remote Version 2', CLOCK.v2);+ const r1 = await importer.processManifest(bucket.id, first.filename);+ expect(overriddenFor(r1, fixture, seeded.id)).toHaveLength(1);+ expect(await fixture.get(seeded.id)).toMatchObject({ [fixture.field]: 'Remote Version 2', updatedAt: CLOCK.v2 });++ deliverRevision(fixture, seeded.id, 'Remote Version 3', CLOCK.v3);+ const r2 = await importer.processManifest(bucket.id, second.filename);+ expect(overriddenFor(r2, fixture, seeded.id)).toHaveLength(1);+ expect(await fixture.get(seeded.id)).toMatchObject({ [fixture.field]: 'Remote Version 3', updatedAt: CLOCK.v3 });+ });++ it.each(KINDS)('converges on reverse-order and tied deliveries without bumping the clock ($kind)', async (fixture) => {+ const bucket = await buckets.createBucket({ name: 'ReverseBucket', path: tempBucket, mode: 'auto-merge' });+ const seeded = await fixture.seed(bucket.id);+ const [first, second, third] = await exportManifests(seeded, 3);++ deliverRevision(fixture, seeded.id, 'Remote Version 3', CLOCK.v3);+ await importer.processManifest(bucket.id, second.filename);++ // The older revision arrives late, then a different body at the SAME+ // clock — both are no-ops that must neither apply nor manufacture a clock.+ deliverRevision(fixture, seeded.id, 'Remote Version 2', CLOCK.v2);+ const late = await importer.processManifest(bucket.id, first.filename);+ expect(overriddenFor(late, fixture, seeded.id)).toEqual([]);+ deliverRevision(fixture, seeded.id, 'Tied Rival', CLOCK.v3);+ const tied = await importer.processManifest(bucket.id, third.filename);+ expect(overriddenFor(tied, fixture, seeded.id)).toEqual([]);++ expect(await fixture.get(seeded.id)).toMatchObject({ [fixture.field]: 'Remote Version 3', updatedAt: CLOCK.v3 });+ });++ it.each(KINDS)('keeps a local edit made after the importer read but before its queued write ($kind)', async (fixture) => {+ const bucket = await buckets.createBucket({ name: 'RaceBucket', path: tempBucket, mode: 'auto-merge' });+ const seeded = await fixture.seed(bucket.id);+ const [exp] = await exportManifests(seeded, 1);+ deliverRevision(fixture, seeded.id, 'Remote Version 2', CLOCK.v2);++ // The importer's first read of this record returns the stale snapshot,+ // but only after a local edit has fully committed — exactly the window+ // the old out-of-queue compare could not see.+ const st = fixture.store();+ const realLoadOne = st.loadOne;+ let armed = true;+ const spy = vi.spyOn(st, 'loadOne').mockImplementation(async (id, ...rest) => {+ const snapshot = await realLoadOne(id, ...rest);+ if (armed && id === seeded.id) {+ armed = false;+ await fixture.edit(seeded.id, 'Local Newer');+ }+ return snapshot;+ });+ try {+ const r = await importer.processManifest(bucket.id, exp.filename);+ expect(armed).toBe(false);+ expect(overriddenFor(r, fixture, seeded.id)).toEqual([]);+ } finally {+ spy.mockRestore();+ }+ const stored = await fixture.get(seeded.id);+ expect(stored[fixture.field]).toBe('Local Newer');+ expect(stored.updatedAt > CLOCK.v2).toBe(true);+ });++ it('inbox promotion applies a remote revision with its source clock and origin', async () => {+ const bucket = await buckets.createBucket({ name: 'PromoteClockBucket', path: tempBucket, mode: 'inbox' });+ const fixture = KINDS.find((k) => k.kind === 'series');+ const seeded = await fixture.seed(bucket.id);+ const [exp] = await exportManifests(seeded, 1);+ deliverRevision(fixture, seeded.id, 'Remote Version 2', CLOCK.v2);++ const queued = await importer.processManifest(bucket.id, exp.filename);+ expect(queued.outcome.queued).toBe(true);+ await importer.promoteInboxItem(bucket.id, exp.manifestId);++ expect(await series.getSeries(seeded.id)).toMatchObject({+ name: 'Remote Version 2',+ updatedAt: CLOCK.v2,+ origin: { bucketId: bucket.id, manifestId: exp.manifestId },+ });+ });+ });});diff --git a/server/services/universeBuilder/crud.js b/server/services/universeBuilder/crud.jsindex 23a34e9826..b38f52d381 100644--- a/server/services/universeBuilder/crud.js+++ b/server/services/universeBuilder/crud.js@@ -33,7 +33,7 @@ import {// series.js does NOT import universeBuilder (one-directional), so this static// import is cycle-safe — unlike canonUsage.js, which back-imports this module.import { listSeries } from '../pipeline/series.js';-import { isStr, trimTo } from '../../lib/textUtils.js';+import { isNonBlankStr, isStr, trimTo } from '../../lib/textUtils.js';import { codedError } from '../../lib/codedError.js';// Once-per-process flag for the canon-backfill log — readState() runs in both@@ -438,9 +438,12 @@ export async function updateUniverse(id, patchOrMutator = {}, options = {}) {// provably carries no canon key (see addStyleReference / removeStyleReference); a// literal-object PATCH doesn't need it, its own `'characters' in patch` checks are// already precise.+ // `options.updatedAt` stamps that record clock instead of receipt time — only+ // the share-bucket importer's remote LWW apply passes it, so the winning+ // source revision's clock survives and the next one can still win (#10761).const {silent = false, canonProjectionGuard = null, replaceCategories = false,- touchesCanon = true,+ touchesCanon = true, updatedAt: incomingUpdatedAt = null,} = options;const s = store();const { merged, nameChanged, skipped, removedCharacterIds, prevEphemeral, nextEphemeral } = await s.queueRecordWrite(id, async () => {@@ -627,7 +630,7 @@ export async function updateUniverse(id, patchOrMutator = {}, options = {}) {influences: mergedInfluences,locked: mergedLocked,llm: mergedLlm,- updatedAt: new Date().toISOString(),+ updatedAt: isNonBlankStr(incomingUpdatedAt) ? incomingUpdatedAt : new Date().toISOString(),});if (!mergedRecord) throw codedError('Invalid universe payload', ERR_VALIDATION);// Stamp `updatedAt = now` on every canon entry whose CONTENT changed vs the