Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: stop swallowing peer-sync push read failures (#11455)
[._.]/ Adam Eivy4a61a6ac8eff
diff --git a/server/services/sharing/peerSync.test.js b/server/services/sharing/peerSync.test.jsindex 78569bf5ef..987b049d0b 100644--- a/server/services/sharing/peerSync.test.js+++ b/server/services/sharing/peerSync.test.js@@ -256,6 +256,7 @@ import {pullRecordFromPeer,syncNowForPeer,collectSubscriptionsForUpdate,+ triggerPushForRecord,buildMediaLibraryManifest,libraryKindsExcludedByPatterns,syncMediaLibraryFromPeer,@@ -266,6 +267,7 @@ import {} from './peerSync.js';import { buildPushPayload, __resetPushFailureLogForTests } from './peerSyncPush.js';+import { DEBOUNCE_MS } from './peerSyncShared.js';import { ENVELOPE_EXTENSIONS, ENVELOPE_PENDING_KEYS } from './peerSyncShared.js';import { getPeers } from '../instances.js';import { getInstanceId } from '../instanceIdentity.js';@@ -1989,6 +1991,86 @@ describe('peerSync', () => {expect(result.reason).toBe('record-not-found');});+ // #11455 — a store read FAILURE must not read as "record does not exist"+ // or ship a truncated payload that then advances lastPushedHash.+ describe('push read failures (#11455)', () => {+ const notFound = () => Object.assign(new Error('gone'), { code: 'PIPELINE_SERIES_NOT_FOUND' });+ const dbDown = () => Object.assign(new Error('connection refused'), { code: 'ECONNREFUSED' });++ it('maps a not-found loader error to record-not-found with no network call', async () => {+ vi.mocked(getUniverse).mockRejectedValue(Object.assign(new Error('nope'), { code: 'NOT_FOUND' }));+ const result = await pushRecordToPeer({+ id: 's', peerId: 'peer-a', recordKind: 'universe', recordId: 'gone',+ });+ expect(result).toMatchObject({ pushed: false, reason: 'record-not-found' });+ expect(peerFetch).not.toHaveBeenCalled();+ });++ it('rejects, posts nothing and keeps lastPushedHash when the loader fails for another reason', async () => {+ vi.mocked(getUniverse).mockResolvedValue({ id: 'u1', name: 'Foo' });+ vi.mocked(peerFetch).mockResolvedValue({ ok: true, json: async () => ({ missingAssets: [] }) });+ await subscribePeer({ peerId: 'peer-a', recordKind: 'universe', recordId: 'u1' });+ let sub;+ await vi.waitFor(async () => {+ sub = await findPeerSubscription('peer-a', 'universe', 'u1');+ expect(sub?.lastPushedHash).toBeTruthy();+ });+ const priorHash = sub.lastPushedHash;+ vi.mocked(peerFetch).mockClear();+ vi.mocked(getUniverse).mockRejectedValue(dbDown());+ await expect(pushRecordToPeer(sub)).rejects.toThrow('connection refused');+ expect(peerFetch).not.toHaveBeenCalled();+ expect((await findPeerSubscription('peer-a', 'universe', 'u1')).lastPushedHash).toBe(priorHash);+ });++ it('series push: a failing listIssuesForSeries rejects instead of pushing issues: []', async () => {+ vi.mocked(getSeries).mockResolvedValue({ id: 'ser-1', name: 'Series' });+ vi.mocked(listIssuesForSeries).mockRejectedValue(dbDown());+ const sub = await subscribePeer({+ peerId: 'peer-a', recordKind: 'series', recordId: 'ser-1',+ }, { adoptedFromReverse: true });+ await expect(pushRecordToPeer(sub)).rejects.toThrow('connection refused');+ expect(peerFetch).not.toHaveBeenCalled();+ expect((await findPeerSubscription('peer-a', 'series', 'ser-1')).lastPushedHash).toBeFalsy();+ });++ it('series push: a failing review read rejects, but a not-found review is simply omitted', async () => {+ vi.mocked(getSeries).mockResolvedValue({ id: 'ser-1', name: 'Series' });+ vi.mocked(getReview).mockRejectedValue(dbDown());+ const sub = await subscribePeer({+ peerId: 'peer-a', recordKind: 'series', recordId: 'ser-1',+ }, { adoptedFromReverse: true });+ await expect(pushRecordToPeer(sub)).rejects.toThrow('connection refused');+ expect(peerFetch).not.toHaveBeenCalled();+ vi.mocked(getReview).mockRejectedValue(notFound());+ vi.mocked(peerFetch).mockResolvedValue({ ok: true, json: async () => ({ missingAssets: [] }) });+ const ok = await pushRecordToPeer(sub);+ expect(ok.pushed).toBe(true);+ });++ it('debounced push logs the failure with the subscription id', async () => {+ vi.mocked(getUniverse).mockResolvedValue({ id: 'u1', name: 'Foo' });+ const sub = await subscribePeer({+ peerId: 'peer-a', recordKind: 'universe', recordId: 'u1',+ }, { adoptedFromReverse: true });+ vi.mocked(getUniverse).mockRejectedValue(dbDown());+ const errSpy = vi.spyOn(console, 'error').mockImplementation(() => {});+ vi.useFakeTimers({ toFake: ['setTimeout'] });+ try {+ await triggerPushForRecord('universe', 'u1');+ await vi.advanceTimersByTimeAsync(DEBOUNCE_MS + 1);+ } finally {+ vi.useRealTimers();+ }+ await vi.waitFor(() => {+ expect(errSpy.mock.calls.some(([msg]) => String(msg).includes(`scheduled push failed for ${sub.id}`)+ && String(msg).includes('connection refused'))).toBe(true);+ });+ errSpy.mockRestore();+ expect(peerFetch).not.toHaveBeenCalled();+ });+ });+it('short-circuits when the record hashes match the last push (no-op edits dont round-trip)', async () => {vi.mocked(getUniverse).mockResolvedValue({ id: 'u1', name: 'Foo' });vi.mocked(peerFetch).mockResolvedValue({ ok: true, json: async () => ({}) });diff --git a/server/services/sharing/peerSyncPush.js b/server/services/sharing/peerSyncPush.jsindex 84e6a502b8..fa24b48ae5 100644--- a/server/services/sharing/peerSyncPush.js+++ b/server/services/sharing/peerSyncPush.js@@ -106,10 +106,29 @@ export function __resetPushFailureLogForTests() {loggedPushFailures.clear();}+// A store read that fails is NOT "record does not exist": only a not-found+// error (ServerError 404 / a `*NOT_FOUND` code, as the record loaders throw)+// maps to null. Anything else (DB down, a sanitizer throwing) rethrows so the+// push rejects before any network call or lastPushedHash write, and the caller+// logs it (#11455).+function isNotFoundError(err) {+ const code = err?.code;+ return err?.status === 404 || (typeof code === 'string' && (code === 'NOT_FOUND' || code.endsWith('_NOT_FOUND')));+}++async function loadOrNotFound(fn) {+ try {+ return await fn();+ } catch (err) {+ if (isNotFoundError(err)) return null;+ throw err;+ }+}+async function isSubscriptionRecordTombstone(sub) {const desc = RECORD_KINDS[sub.recordKind];if (!desc) return false;- const record = await desc.load(sub.recordId).catch(() => null);+ const record = await loadOrNotFound(() => desc.load(sub.recordId));return record?.deleted === true;}@@ -582,7 +601,7 @@ async function clearSchemaVersionBlock(subId) {* the two genuinely-special kinds don't inflate the dispatcher's complexity.*/async function buildUniversePushPayload(sub, sourceInstanceId, portosMeta) {- const record = await RECORD_KINDS.universe.load(sub.recordId).catch(() => null);+ const record = await loadOrNotFound(() => RECORD_KINDS.universe.load(sub.recordId));if (!record) return null;const sanitized = sanitizeRecordForWire('universe', record);if (!sanitized) return null;@@ -596,7 +615,7 @@ async function buildUniversePushPayload(sub, sourceInstanceId, portosMeta) {// it would re-create an empty bucket on the receiver.const linkedCollection = record.deleted === true? null- : await findCollectionByUniverseId(sub.recordId).catch(() => null);+ : await loadOrNotFound(() => findCollectionByUniverseId(sub.recordId));// Tombstone push: deleted records carry no on-disk assets the receiver// should pull. Sending an empty manifest avoids triggering// pullMissingAssetsFromPeer for a record we're telling the peer to@@ -641,14 +660,14 @@ async function buildUniversePushPayload(sub, sourceInstanceId, portosMeta) {* but nothing else. Extracted out of `buildPushPayload`'s own body (#6843).*/async function buildSeriesPushPayload(sub, sourceInstanceId, portosMeta) {- const record = await RECORD_KINDS.series.load(sub.recordId).catch(() => null);+ const record = await loadOrNotFound(() => RECORD_KINDS.series.load(sub.recordId));if (!record) return null;const sanitized = sanitizeRecordForWire('series', record);if (!sanitized) return null;// Bundle child issues — the series + its issues form one unit of edit// for downstream consumers (panels, comic pages), so the receiver// applies them atomically per merge cycle.- const childIssues = await listIssuesForSeries(sub.recordId, { includeDeleted: true }).catch(() => []);+ const childIssues = await listIssuesForSeries(sub.recordId, { includeDeleted: true });const sanitizedIssues = childIssues.map((i) => sanitizeRecordForWire('issue', i)).filter(Boolean);@@ -676,7 +695,7 @@ async function buildSeriesPushPayload(sub, sourceInstanceId, portosMeta) {// and the receiver's collection would diverge.const linkedCollection = record.deleted === true? null- : await findCollectionBySeriesId(sub.recordId).catch(() => null);+ : await loadOrNotFound(() => findCollectionBySeriesId(sub.recordId));// Tombstone push at the series level: same reasoning as universe above.// When the series itself is deleted, send an empty asset manifest so// the receiver doesn't pull bytes for a record it's about to tombstone.@@ -693,8 +712,8 @@ async function buildSeriesPushPayload(sub, sourceInstanceId, portosMeta) {const manuscriptReview = record.deleted === true? null: await import('../pipeline/manuscriptReview.js')- .then(({ getReview }) => getReview(sub.recordId))- .catch(() => null);++ .then(({ getReview }) => loadOrNotFound(() => getReview(sub.recordId)));// Bundle the reverse-outline sibling doc (the scene-by-scene segmentation)// on the same terms as the review above: a regenerate-only change doesn't// move the series record, so without bundling it the per-record push would@@ -704,8 +723,8 @@ async function buildSeriesPushPayload(sub, sourceInstanceId, portosMeta) {const reverseOutline = record.deleted === true? null: await import('../pipeline/reverseOutline.js')- .then(({ getStoredOutline }) => getStoredOutline(sub.recordId))- .catch(() => null);++ .then(({ getStoredOutline }) => loadOrNotFound(() => getStoredOutline(sub.recordId)));// Same wire projection as the universe branch — see the note there.const bundledCollection = sanitizeRecordForWire('mediaCollection', linkedCollection);return {@@ -736,7 +755,7 @@ export async function buildPushPayload(sub, sourceInstanceId) {if (!desc) return null;if (sub.recordKind === 'universe') return buildUniversePushPayload(sub, sourceInstanceId, portosMeta);if (sub.recordKind === 'series') return buildSeriesPushPayload(sub, sourceInstanceId, portosMeta);- const record = await desc.load(sub.recordId).catch(() => null);+ const record = await loadOrNotFound(() => desc.load(sub.recordId));if (!record) return null;const sanitized = sanitizeRecordForWire(sub.recordKind, record);if (!sanitized) return null;@@ -757,7 +776,7 @@ export async function buildPushPayload(sub, sourceInstanceId) {// it (same as before), so no schema-version bump (mirrors `linkedCollection`).let linkedTrack = null;if (record.deleted !== true && isStr(record.trackId)) {- const track = await getTrack(record.trackId, { includeDeleted: true }).catch(() => null);+ const track = await loadOrNotFound(() => getTrack(record.trackId, { includeDeleted: true }));// Ship the linked track whether LIVE or a TOMBSTONE. A track delete fans// out to its linked music-video projects (collectSubscriptionsForUpdate),// and a musicVideoProjects-only peer needs the tombstone to converge