Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: serialize peer schema-gap updates so parallel categories don't overwrite each other (#11778)
[._.]/ Adam Eivyc35318573362
diff --git a/server/services/instances.js b/server/services/instances.jsindex 01e940d974..199b206276 100644--- a/server/services/instances.js+++ b/server/services/instances.js@@ -786,6 +786,30 @@ async function pairPeerSyncSecretNow(id) {return updated;}+/**+ * Set (gap object) or clear (null) one category's schema gap on the peer with+ * the given remote instanceId. The read-modify-write runs inside the serialized+ * withData write so parallel category syncs cannot overwrite each other's gaps+ * (updatePeer replaces the whole map). An empty map is stored as no key.+ */+export async function setPeerSchemaGap(instanceId, category, gapOrNull) {+ return withData(async (data) => {+ const peer = data.peers.find((p) => p.instanceId === instanceId);+ if (!peer) return null;+ const gaps = peer.schemaGaps && typeof peer.schemaGaps === 'object' && !Array.isArray(peer.schemaGaps) ? { ...peer.schemaGaps } : {};+ if (gapOrNull === null) {+ if (!(category in gaps)) return peer;+ delete gaps[category];+ } else {+ gaps[category] = gapOrNull;+ }+ if (Object.keys(gaps).length > 0) peer.schemaGaps = gaps;+ else delete peer.schemaGaps;+ instanceEvents.emit('peers:updated', data.peers);+ return peer;+ });+}+/** Save a pair secret only for the already-registered peer named by its identity header. */export async function acceptPeerSyncSecretFromPeer(instanceId, syncSecret) {let changed = false;diff --git a/server/services/instances.test.js b/server/services/instances.test.jsindex 2ae4b872d7..ea03535645 100644--- a/server/services/instances.test.js+++ b/server/services/instances.test.js@@ -70,6 +70,7 @@ import {updatePeer,pairPeerSyncSecret,acceptPeerSyncSecretFromPeer,+ setPeerSchemaGap,probePeer,probeAllPeers,queryPeer,@@ -2108,4 +2109,38 @@ describe('instances.js', () => {expect(await acceptPeerSyncSecretFromPeer('unknown-peer', 'synthetic-pair-secret-32-characters-long')).toBeNull();});});++ describe('setPeerSchemaGap', () => {+ const gap = (n) => ({ detectedAt: '2026-01-01T00:00:00.000Z', ahead: [n], behind: [], senderPortosVersion: '9.9.9' });+ const seed = (schemaGaps) => {+ const peer = { id: 'p1', instanceId: 'remote-1', ...(schemaGaps ? { schemaGaps } : {}) };+ readJSONFile.mockResolvedValue({ self: { instanceId: 'self-id' }, peers: [peer] });+ return peer;+ };++ it('keeps both gaps when two categories are recorded concurrently', async () => {+ const peer = seed();+ await Promise.all([+ setPeerSchemaGap('remote-1', 'universe', gap('universe')),+ setPeerSchemaGap('remote-1', 'pipeline', gap('pipeline'))+ ]);+ expect(Object.keys(peer.schemaGaps).sort()).toEqual(['pipeline', 'universe']);+ });++ it('leaves exactly the recorded category when another is cleared concurrently', async () => {+ const peer = seed({ B: gap('B') });+ await Promise.all([+ setPeerSchemaGap('remote-1', 'A', gap('A')),+ setPeerSchemaGap('remote-1', 'B', null)+ ]);+ expect(Object.keys(peer.schemaGaps)).toEqual(['A']);+ });++ it('removes the schemaGaps key when the last gap clears and ignores unknown peers', async () => {+ const peer = seed({ A: gap('A') });+ await setPeerSchemaGap('remote-1', 'A', null);+ expect(peer).not.toHaveProperty('schemaGaps');+ expect(await setPeerSchemaGap('nobody', 'A', gap('A'))).toBeNull();+ });+ });});diff --git a/server/services/syncOrchestrator.js b/server/services/syncOrchestrator.jsindex 2b6943a89e..facaf3cf1c 100644--- a/server/services/syncOrchestrator.js+++ b/server/services/syncOrchestrator.js@@ -12,7 +12,7 @@ import { readJSONFile, ensureDir, PATHS, dataPath, atomicWrite, writeFileGuardedimport { createMutex } from '../lib/asyncMutex.js';import { logFailureWithStack } from '../lib/failureLogging.js';import { instanceEvents } from './instanceEvents.js';-import { getPeers, peerLogLabel, resolveEffectiveCategories, updatePeer } from './instances.js';+import { getPeers, peerLogLabel, resolveEffectiveCategories, setPeerSchemaGap } from './instances.js';import { getInstanceId, UNKNOWN_INSTANCE_ID } from './instanceIdentity.js';import { peerBaseUrl } from '../lib/peerUrl.js';import { peerFetch, readPeerBody, PEER_BODY_IDLE_TIMEOUT, PEER_BODY_DEFAULT_MAX_BYTES } from '../lib/peerHttpClient.js';@@ -661,39 +661,20 @@ const slotsQsOf = (manifest, forPeerQs) => (manifest* peers.json stays a pure config surface. For now we co-locate on the peer* record because that's the entity the Instances page already renders.*/-// Look up the local peer row by remote instanceId (the only id we have at-// the orchestrator level) and resolve to the LOCAL `peer.id` that updatePeer-// keys by. Passing `peer.instanceId` straight to updatePeer would silently-// return null because instances.js matches on `p.id === id`.+// Writes go through setPeerSchemaGap, which resolves the peer by remote+// instanceId and merges this one category inside the serialized peers write —+// categories sync in parallel and must not overwrite each other's gaps.async function recordPeerSchemaGap(peerId, category, gap) {- const peers = await getPeers().catch(() => []);- const peer = peers.find((p) => p.instanceId === peerId);- if (!peer) return;- const existingGaps = isPlainObjectShallow(peer.schemaGaps) ? peer.schemaGaps : {};- await updatePeer(peer.id, {- schemaGaps: {- ...existingGaps,- [category]: {- detectedAt: new Date().toISOString(),- ahead: Array.isArray(gap.ahead) ? gap.ahead : [],- behind: Array.isArray(gap.behind) ? gap.behind : [],- senderPortosVersion: typeof gap.senderPortosVersion === 'string' ? gap.senderPortosVersion : null,- },- },+ await setPeerSchemaGap(peerId, category, {+ detectedAt: new Date().toISOString(),+ ahead: Array.isArray(gap.ahead) ? gap.ahead : [],+ behind: Array.isArray(gap.behind) ? gap.behind : [],+ senderPortosVersion: typeof gap.senderPortosVersion === 'string' ? gap.senderPortosVersion : null,});}async function clearPeerSchemaGap(peerId, category) {- const peers = await getPeers().catch(() => []);- const peer = peers.find((p) => p.instanceId === peerId);- if (!peer) return;- const existingGaps = isPlainObjectShallow(peer.schemaGaps) ? peer.schemaGaps : null;- if (!existingGaps || !(category in existingGaps)) return;- const next = { ...existingGaps };- delete next[category];- await updatePeer(peer.id, {- schemaGaps: Object.keys(next).length > 0 ? next : null,- });+ await setPeerSchemaGap(peerId, category, null);}const isPlainObjectShallow = (v) => v !== null && typeof v === 'object' && !Array.isArray(v);diff --git a/server/services/syncOrchestrator.test.js b/server/services/syncOrchestrator.test.jsindex 48348bd174..82d55f5339 100644--- a/server/services/syncOrchestrator.test.js+++ b/server/services/syncOrchestrator.test.js@@ -9,7 +9,7 @@ vi.mock('./instances.js', async () => ({// what these tests assert the sync loop does with them....(await vi.importActual('./instances.js')),getPeers: vi.fn(),- updatePeer: vi.fn().mockResolvedValue(undefined),+ setPeerSchemaGap: vi.fn().mockResolvedValue(undefined),}));vi.mock('./instanceIdentity.js', () => ({UNKNOWN_INSTANCE_ID: 'unknown',@@ -167,10 +167,6 @@ describe('syncOrchestrator', () => {// this reset a later test inherits a prior test's (mutated) cursor and the// catalog rebuild/reset detection fires on it.readJSONFile.mockImplementation(async () => ({}));- // Default so clearPeerSchemaGap's `await getPeers()` (a best-effort lookup- // on every successful, non-blocked catalog drain) resolves to an array- // instead of throwing on undefined; tests that care about the peer lookup- // override this explicitly.getPeers.mockResolvedValue([]);vi.useFakeTimers();mockFetch.mockReset();@@ -694,22 +690,12 @@ describe('syncOrchestrator', () => {sources: { applied: 0 }, refs: { applied: 0 }, relations: { applied: 0 },tags: { inserted: 0, updated: 0 }, media: { applied: 0 }, errors: [],});- const { updatePeer } = await import('./instances.js');- getPeers.mockResolvedValue([{- ...catalogPeer,- id: 'local-peer-row',- instanceId: catalogPeer.instanceId,- schemaGaps: {- catalog: { detectedAt: '2026-01-01T00:00:00.000Z', ahead: [{ category: 'catalog' }], behind: [], senderPortosVersion: '99.0.0' },- },- }]);+ const { setPeerSchemaGap } = await import('./instances.js');await syncWithPeer(catalogPeer);- // A successful, non-blocked drain clears the gap it previously recorded —- // this fails if clearPeerSchemaGap early-returns (e.g. on an unmocked- // getPeers, or a peer lookup that never matches).- expect(updatePeer).toHaveBeenCalledWith('local-peer-row', { schemaGaps: null });+ // A successful, non-blocked drain clears the gap it previously recorded.+ expect(setPeerSchemaGap).toHaveBeenCalledWith(catalogPeer.instanceId, 'catalog', null);});it.each(['ahead', 'behind'])('records a %s catalog gap without advancing cursors, then retries', async (direction) => {@@ -728,8 +714,7 @@ describe('syncOrchestrator', () => {mismatch.diff = { ahead: [], behind: [], [direction]: [gap] };applyCatalogChanges.mockRejectedValueOnce(mismatch);- const { updatePeer } = await import('./instances.js');- getPeers.mockResolvedValue([{ ...catalogPeer, id: 'local-peer-row', instanceId: catalogPeer.instanceId }]);+ const { setPeerSchemaGap } = await import('./instances.js');const result = await syncWithPeer(catalogPeer);@@ -738,15 +723,10 @@ describe('syncOrchestrator', () => {expect(result.catalog.blockedBySchema).toBeTruthy();expect(result.catalog.catalogSeqs.ingredients).toBeUndefined();// Gap persisted on the local peer row under schemaGaps.catalog.- expect(updatePeer).toHaveBeenCalledWith(- 'local-peer-row',- expect.objectContaining({- schemaGaps: expect.objectContaining({- catalog: expect.objectContaining({- [direction]: [gap],- }),- }),- }),+ expect(setPeerSchemaGap).toHaveBeenCalledWith(+ catalogPeer.instanceId,+ 'catalog',+ expect.objectContaining({ [direction]: [gap] }),);// The next cycle retries the same rows after peers upgrade.mockFetch.mockResolvedValue({ ok: true, json: async () => ({