Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix([issue-12620]): retry a failed peer push while the peer stays online (#12745)

[._.]/48f7e5b3f02e
diff --git a/server/services/sharing/peerSync.js b/server/services/sharing/peerSync.js
index 7b190012e4..a7e5c3c3af 100644
--- a/server/services/sharing/peerSync.js
+++ b/server/services/sharing/peerSync.js
@@ -57,7 +57,18 @@ import {
peerHasCategory,
findPeerById,
} from './peerSyncShared.js';
-import { pushRecordToPeer, buildPushPayload } from './peerSyncPush.js';
+import {
+ pushRecordToPeer,
+ buildPushPayload,
+ beginPushRetryWalk,
+ finishPushRetryWalk,
+ clearPushRetryDue,
+ retainPushRetryDue,
+ isPushRetryDue,
+ listPushRetryDue,
+ registerPushRetryScheduler,
+ resetPushRetryBookkeeping,
+} from './peerSyncPush.js';
import { applyIncomingPush } from './peerSyncReceive.js';
import {
pendingTimers,
@@ -575,43 +586,135 @@ async function getIssueSeriesId(issueId) {
* set. The unconditional retry + hash short-circuit covers both the
* "initial push" and "out-of-band drift" cases with the same code path.
*
- * Failures stay non-fatal — the next `peer:online` (or the user's next
- * edit) gets another attempt.
+ * Failures stay non-fatal. A retryable outcome schedules this walk again
+ * (60s, then 5 min, 15 min, 30 min cap). A refusal or a settled walk waits
+ * for the next `peer:online` or a later edit.
*/
+// First failure waits a minute; continued failures back off to a 30 min cap
+// so a peer that stays up but keeps 5xx-ing does not get a push storm.
+const PUSH_RETRY_DELAYS_MS = Object.freeze([60_000, 5 * 60_000, 15 * 60_000, 30 * 60_000]);
+const pushRetryState = new Map(); // peerId → { step, timer, consumeAdvance }
+const retryWalksInFlight = new Set();
+
+function armPushRetry(peerId) {
+ if (stopping || !isNonBlankStr(peerId)) return;
+ let state = pushRetryState.get(peerId);
+ if (!state) {
+ state = { step: 0, timer: null, consumeAdvance: false };
+ pushRetryState.set(peerId, state);
+ }
+ if (state.timer) return;
+ const delay = PUSH_RETRY_DELAYS_MS[Math.min(state.step, PUSH_RETRY_DELAYS_MS.length - 1)];
+ const timer = setTimeout(() => {
+ if (state.timer === timer) state.timer = null;
+ trackBackgroundOperation(runScheduledPushRetry(peerId).catch((err) => {
+ console.error(`❌ peerSync: scheduled push retry failed for ${peerId}: ${err?.message || err}`);
+ }));
+ }, delay);
+ if (typeof timer.unref === 'function') timer.unref();
+ state.timer = timer;
+}
+
+function cancelPushRetry(peerId) {
+ const state = pushRetryState.get(peerId);
+ if (state?.timer) clearTimeout(state.timer);
+ pushRetryState.delete(peerId);
+}
+
+function cancelAllPushRetries() {
+ for (const peerId of [...pushRetryState.keys()]) cancelPushRetry(peerId);
+ resetPushRetryBookkeeping({ keepWalks: retryWalksInFlight.size > 0 });
+}
+
+function scheduleNextPushRetry(peerId) {
+ if (stopping || !isNonBlankStr(peerId)) return;
+ let state = pushRetryState.get(peerId);
+ if (!state) {
+ state = { step: 0, timer: null, consumeAdvance: false };
+ pushRetryState.set(peerId, state);
+ }
+ // The tick that entered this walk consumed its delay. A peer:online walk
+ // does not, so the first failure still waits 60s instead of skipping to 5 min.
+ if (state.consumeAdvance) {
+ state.step = Math.min(state.step + 1, PUSH_RETRY_DELAYS_MS.length - 1);
+ state.consumeAdvance = false;
+ }
+ retainPushRetryDue(peerId);
+ if (!state.timer) armPushRetry(peerId);
+}
+
+async function runScheduledPushRetry(peerId) {
+ if (stopping || !isPushRetryDue(peerId)) return;
+ const peer = await findPeerById(peerId);
+ if (stopping || !isPushRetryDue(peerId)) return;
+ // An offline peer is the probe's problem: `peer:online` runs the walk on
+ // the transition. Rescheduling here would only duplicate that, and a tick
+ // that finds a walk already running must not count as a failed attempt.
+ if (peer?.status !== 'online') return;
+ if (retryWalksInFlight.has(peerId)) return;
+ const state = pushRetryState.get(peerId);
+ if (!state || state.timer) return;
+ state.consumeAdvance = true;
+ await retryPendingPushesForPeer(peerId);
+}
+
export async function retryPendingPushesForPeer(peerId) {
if (!isNonBlankStr(peerId) || stopping) return { walked: 0, pushed: 0 };
- const subs = await listPeerSubscriptions({ peerId });
- if (subs.length === 0 || stopping) return { walked: 0, pushed: 0 };
- // Separate counter for the log line — only count subs that were never
- // pushed (genuine retries) so steady-state convergence runs stay quiet.
- const neverPushedCount = subs.filter(s => !s.lastPushedAt).length;
- if (neverPushedCount > 0) {
- console.log(`🔄 peerSync: retrying ${neverPushedCount} pending push${neverPushedCount === 1 ? '' : 'es'} → ${peerId}`);
- }
- // Track `walked` (subs we iterated) and `pushed` (HTTP call landed)
- // separately. `walked === pushed` would be misleading at steady state
- // because the lastPushedHash short-circuit inside pushRecordToPeer skips
- // the network call for any sub whose content is unchanged — we still
- // "walked" the sub but never pushed it.
+ if (retryWalksInFlight.has(peerId)) return { walked: 0, pushed: 0, skipped: 'in-flight' };
+ let started = false;
+ let walked = 0;
let pushed = 0;
- // Coalesce the per-push base-hash flushes: this loop pushes every subscribed
- // record in sequence, and each push would otherwise rewrite
- // sync_base_hashes.json once. The flush batch defers them into a single
- // terminal write covering all N records' stamps.
- await withBaseHashFlushBatch(async () => {
- for (const sub of subs) {
- if (stopping) break;
- // peer:online fired — re-probe schema-blocked subs immediately (peer
- // may have upgraded since the last 409). Edit-triggered pushes still
- // respect the cooldown.
- const result = await pushRecordToPeer(sub, { bypassSchemaCooldown: true }).catch((err) => {
- logFailureWithStack(`⚠️ peerSync: retry push failed for ${sub.id}`, err);
- return null;
- });
- if (result?.pushed) pushed += 1;
+ let walkFailed = false;
+ try {
+ retryWalksInFlight.add(peerId);
+ beginPushRetryWalk(peerId);
+ started = true;
+ const subs = await listPeerSubscriptions({ peerId });
+ if (subs.length === 0 || stopping) return { walked: 0, pushed: 0 };
+ // Separate counter for the log line — only count subs that were never
+ // pushed (genuine retries) so steady-state convergence runs stay quiet.
+ const neverPushedCount = subs.filter(s => !s.lastPushedAt).length;
+ if (neverPushedCount > 0) {
+ console.log(`🔄 peerSync: retrying ${neverPushedCount} pending push${neverPushedCount === 1 ? '' : 'es'} → ${peerId}`);
}
- });
- return { walked: subs.length, pushed };
+ // Track `walked` (subs we iterated) and `pushed` (HTTP call landed)
+ // separately. `walked === pushed` would be misleading at steady state
+ // because the lastPushedHash short-circuit inside pushRecordToPeer skips
+ // the network call for any sub whose content is unchanged — we still
+ // "walked" the sub but never pushed it.
+ // Coalesce the per-push base-hash flushes: this loop pushes every subscribed
+ // record in sequence, and each push would otherwise rewrite
+ // sync_base_hashes.json once. The flush batch defers them into a single
+ // terminal write covering all N records' stamps.
+ await withBaseHashFlushBatch(async () => {
+ for (const sub of subs) {
+ if (stopping) break;
+ // peer:online fired — re-probe schema-blocked subs immediately (peer
+ // may have upgraded since the last 409). Edit-triggered pushes still
+ // respect the cooldown.
+ const result = await pushRecordToPeer(sub, { bypassSchemaCooldown: true }).catch((err) => {
+ logFailureWithStack(`⚠️ peerSync: retry push failed for ${sub.id}`, err);
+ return null;
+ });
+ if (result?.pushed) pushed += 1;
+ }
+ });
+ walked = subs.length;
+ return { walked, pushed };
+ } catch (err) {
+ walkFailed = true;
+ throw err;
+ } finally {
+ if (started) {
+ retryWalksInFlight.delete(peerId);
+ const retryable = finishPushRetryWalk(peerId);
+ // A thrown walk did not finish its pushes; keep the chain armed.
+ // Otherwise a clean pass cancels it — including a pass that only
+ // short-circuited — and a retryable pass takes the next backoff step.
+ if (!stopping && (retryable || walkFailed)) scheduleNextPushRetry(peerId);
+ else if (!stopping) clearPushRetryDue(peerId);
+ }
+ }
}
/**
@@ -836,6 +939,9 @@ export function installPeerSyncListener() {
})().catch(() => {}));
};
instanceEvents.on('peer:online', onPeerOnline);
+ // A failure that landed while stopping (or before this listener existed)
+ // left the due set without a timer. Arm it now that walks are allowed.
+ for (const peerId of listPushRetryDue()) armPushRetry(peerId);
}
/**
@@ -849,6 +955,10 @@ export function uninstallPeerSyncListener() {
stopping = true;
for (const t of pendingTimers.values()) clearTimeout(t);
pendingTimers.clear();
+ // shutdownSharing and test reset both come through here. Drop the retry
+ // chain so a timer cannot fire into a closing DB pool; an in-flight walk
+ // keeps its depth until its own finally sees `stopping` and does not re-arm.
+ cancelAllPushRetries();
if (onUpdated) recordEvents.off('updated', onUpdated);
if (onDeleted) recordEvents.off('deleted', onDeleted);
if (onPeerOnline) instanceEvents.off('peer:online', onPeerOnline);
@@ -896,6 +1006,11 @@ export async function __drainForTests() {
// in the other direction (even a dynamic one) formed a load-order-sensitive
// cycle. Module-load registration is safe: sharing/index.js imports this file
// during server boot, before any HTTP write can fire an adapter call.
+registerPushRetryScheduler({
+ onDue: armPushRetry,
+ onClear: cancelPushRetry,
+});
+
registerSubscriptionAdapter({
autoSubscribeRecordToAllPeers,
unsubscribeAllForRecord,
diff --git a/server/services/sharing/peerSync.test.js b/server/services/sharing/peerSync.test.js
index 27cc87d750..015d5f9df5 100644
--- a/server/services/sharing/peerSync.test.js
+++ b/server/services/sharing/peerSync.test.js
@@ -265,6 +265,7 @@ import {
peerSyncEvents,
__resetForTests,
__drainForTests,
+ installPeerSyncListener,
uninstallPeerSyncListener,
} from './peerSync.js';
@@ -1519,6 +1520,227 @@ describe('peerSync', () => {
});
});
+ describe('scheduled push retry while the peer stays online', () => {
+ const MINUTE = 60_000;
+ const BACKOFF = [MINUTE, 5 * MINUTE, 15 * MINUTE, 30 * MINUTE];
+ const examplePeer = (status) => ({
+ instanceId: 'peer-a',
+ name: 'Example Peer',
+ host: null,
+ address: '192.0.2.10',
+ port: 5555,
+ enabled: true,
+ syncEnabled: true,
+ status,
+ directions: ['outbound', 'inbound'],
+ syncCategories: { universe: true, pipeline: true },
+ });
+ const httpStatus = (status, body = { error: 'no' }) => {
+ const res = { ok: false, status, json: async () => body };
+ res.clone = () => res;
+ return res;
+ };
+ const settled = () => ({ ok: true, status: 200, json: async () => ({ missingAssets: [] }) });
+
+ beforeEach(() => {
+ vi.mocked(getPeers).mockResolvedValue([examplePeer('online')]);
+ vi.mocked(getUniverse).mockResolvedValue({ id: 'u1', name: 'Foo', updatedAt: '2026-01-01T00:00:00Z' });
+ });
+
+ afterEach(() => {
+ vi.useRealTimers();
+ });
+
+ async function subscribed(recordId = 'u1') {
+ return subscribePeer(
+ { peerId: 'peer-a', recordKind: 'universe', recordId },
+ { adoptedFromReverse: true },
+ );
+ }
+
+ async function advance(ms) {
+ await vi.advanceTimersByTimeAsync(ms);
+ await __drainForTests();
+ }
+
+ async function withFakeTimers(run) {
+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] });
+ try {
+ await run();
+ } finally {
+ vi.useRealTimers();
+ }
+ }
+
+ it('delivers a network failure on the scheduled retry without a peer:online event', async () => {
+ await withFakeTimers(async () => {
+ const sub = await subscribed();
+ vi.mocked(peerFetch)
+ .mockRejectedValueOnce(new Error('ECONNRESET'))
+ .mockResolvedValue(settled());
+ expect(await pushRecordToPeer(sub)).toMatchObject({ pushed: false, reason: 'network' });
+ expect(peerFetch).toHaveBeenCalledTimes(1);
+
+ await advance(MINUTE - 1);
+ expect(peerFetch).toHaveBeenCalledTimes(1);
+ await advance(1);
+
+ expect(peerFetch).toHaveBeenCalledTimes(2);
+ expect((await findPeerSubscription('peer-a', 'universe', 'u1')).lastPushedAt).toBeTruthy();
+ await advance(30 * MINUTE);
+ expect(peerFetch).toHaveBeenCalledTimes(2);
+ });
+ });
+
+ it.each([500, 408, 429])('retries HTTP %s on the same schedule', async (status) => {
+ await withFakeTimers(async () => {
+ const sub = await subscribed();
+ vi.mocked(peerFetch)
+ .mockResolvedValueOnce(httpStatus(status, { error: 'unavailable' }))
+ .mockResolvedValue(settled());
+ expect(await pushRecordToPeer(sub)).toMatchObject({ pushed: false, reason: `http-${status}` });
+ await advance(MINUTE);
+ expect(peerFetch).toHaveBeenCalledTimes(2);
+ expect((await pushRecordToPeer(await findPeerSubscription('peer-a', 'universe', 'u1'))).reason).toBe('unchanged');
+ });
+ });
+
+ it('does not retry a 400 or a schema 409', async () => {
+ await withFakeTimers(async () => {
+ const rejected = await subscribed('u-400');
+ vi.mocked(getUniverse).mockResolvedValue({ id: 'u-400', name: 'Foo', updatedAt: '2026-01-01T00:00:00Z' });
+ vi.mocked(peerFetch).mockResolvedValue(httpStatus(400, { error: 'bad record' }));
+ expect(await pushRecordToPeer(rejected)).toMatchObject({ pushed: false, reason: 'http-400' });
+
+ vi.mocked(getUniverse).mockResolvedValue({ id: 'u-409', name: 'Foo', updatedAt: '2026-01-01T00:00:00Z' });
+ const blocked = await subscribed('u-409');
+ vi.mocked(peerFetch).mockResolvedValue(httpStatus(409, {
+ code: 'PEER_SYNC_SCHEMA_VERSION_AHEAD',
+ context: {
+ details: {
+ ahead: [{ category: 'universes', senderV: 5, receiverV: 4 }],
+ behind: [],
+ receiverPortosVersion: '0.0.0',
+ },
+ },
+ }));
+ expect(await pushRecordToPeer(blocked)).toMatchObject({ pushed: false, reason: 'peer-schema-behind' });
+
+ await advance(MINUTE);
+ await advance(30 * MINUTE);
+ expect(peerFetch).toHaveBeenCalledTimes(2);
+ });
+ });
+
+ it('backs off through 5 min and 15 min to the 30 min cap', async () => {
+ await withFakeTimers(async () => {
+ const sub = await subscribed();
+ vi.mocked(peerFetch).mockRejectedValue(new Error('ECONNRESET'));
+ await pushRecordToPeer(sub);
+ for (const delay of BACKOFF) {
+ const before = vi.mocked(peerFetch).mock.calls.length;
+ await advance(delay - 1);
+ expect(vi.mocked(peerFetch).mock.calls.length).toBe(before);
+ await advance(1);
+ expect(vi.mocked(peerFetch).mock.calls.length).toBe(before + 1);
+ }
+ const atCap = vi.mocked(peerFetch).mock.calls.length;
+ await advance(15 * MINUTE);
+ expect(vi.mocked(peerFetch).mock.calls.length).toBe(atCap);
+ await advance(15 * MINUTE);
+ expect(vi.mocked(peerFetch).mock.calls.length).toBe(atCap + 1);
+ });
+ });
+
+ it('a landed push cancels further retries for that peer', async () => {
+ await withFakeTimers(async () => {
+ const sub = await subscribed();
+ vi.mocked(peerFetch)
+ .mockRejectedValueOnce(new Error('ECONNRESET'))
+ .mockResolvedValue(settled());
+ await pushRecordToPeer(sub);
+ expect(await pushRecordToPeer(await findPeerSubscription('peer-a', 'universe', 'u1'))).toMatchObject({ pushed: true });
+ vi.mocked(getUniverse).mockResolvedValue({ id: 'u1', name: 'Edited after delivery', updatedAt: '2026-01-02T00:00:00Z' });
+ vi.mocked(peerFetch).mockClear();
+ await advance(MINUTE);
+ await advance(30 * MINUTE);
+ expect(peerFetch).not.toHaveBeenCalled();
+ });
+ });
+
+ it('skips the retry tick while the peer is offline', async () => {
+ await withFakeTimers(async () => {
+ const sub = await subscribed();
+ vi.mocked(peerFetch).mockRejectedValue(new Error('ECONNRESET'));
+ await pushRecordToPeer(sub);
+ vi.mocked(getPeers).mockResolvedValue([examplePeer('offline')]);
+ await advance(MINUTE);
+ await advance(30 * MINUTE);
+ expect(peerFetch).toHaveBeenCalledTimes(1);
+ });
+ });
+
+ it('skips the retry tick while a walk for that peer is already running', async () => {
+ await withFakeTimers(async () => {
+ const sub = await subscribed();
+ vi.mocked(peerFetch).mockRejectedValueOnce(new Error('ECONNRESET'));
+ await pushRecordToPeer(sub);
+ vi.mocked(peerFetch).mockClear();
+
+ let resolveFetch = () => {};
+ vi.mocked(peerFetch).mockImplementation(() => new Promise((resolve) => {
+ resolveFetch = () => resolve(settled());
+ }));
+ const walk = retryPendingPushesForPeer('peer-a');
+ const tick = () => new Promise((resolve) => { setImmediate(resolve); });
+ for (let i = 0; i < 50 && vi.mocked(peerFetch).mock.calls.length === 0; i += 1) await tick();
+ expect(peerFetch).toHaveBeenCalledTimes(1);
+ expect(await retryPendingPushesForPeer('peer-a')).toEqual({ walked: 0, pushed: 0, skipped: 'in-flight' });
+
+ await advance(MINUTE);
+ expect(peerFetch).toHaveBeenCalledTimes(1);
+ resolveFetch();
+ await walk;
+ });
+ });
+
+ it('retries a missing-asset response without a new edit', async () => {
+ await withFakeTimers(async () => {
+ const sub = await subscribed();
+ vi.mocked(peerFetch).mockResolvedValue({
+ ok: true,
+ status: 200,
+ json: async () => ({ missingAssets: [{ filename: 'example.png' }] }),
+ });
+ const first = await pushRecordToPeer(sub);
+ expect(first.pushed).toBe(true);
+ expect(first.missingAssets).toHaveLength(1);
+ expect((await findPeerSubscription('peer-a', 'universe', 'u1')).lastPushedHash).toBeFalsy();
+ vi.mocked(peerFetch).mockClear();
+
+ await advance(MINUTE - 1);
+ expect(peerFetch).not.toHaveBeenCalled();
+ await advance(1);
+ expect(peerFetch).toHaveBeenCalledTimes(1);
+ const body = JSON.parse(vi.mocked(peerFetch).mock.calls[0][1].body);
+ expect(body.record.id).toBe('u1');
+ });
+ });
+
+ it('uninstall cancels a pending retry', async () => {
+ await withFakeTimers(async () => {
+ const sub = await subscribed();
+ vi.mocked(peerFetch).mockRejectedValue(new Error('ECONNRESET'));
+ await pushRecordToPeer(sub);
+ uninstallPeerSyncListener();
+ installPeerSyncListener();
+ await advance(MINUTE);
+ await advance(30 * MINUTE);
+ expect(peerFetch).toHaveBeenCalledTimes(1);
+ });
+ });
+ });
+
describe('forcePushRecord', () => {
beforeEach(() => {
vi.mocked(getUniverse).mockResolvedValue({ id: 'u1', name: 'Forced', updatedAt: '2026-01-01T00:00:00Z' });
diff --git a/server/services/sharing/peerSyncPush.js b/server/services/sharing/peerSyncPush.js
index 100d4e4f83..441a887917 100644
--- a/server/services/sharing/peerSyncPush.js
+++ b/server/services/sharing/peerSyncPush.js
@@ -83,15 +83,104 @@ const SCHEMA_BLOCK_RETRY_COOLDOWN_MS = 5 * 60_000;
// hundreds of subscriptions from repeating the same line hundreds of times.
const loggedPushFailures = new Map();
+// Instance ids that still owe a retry walk. A retryable failure marks the
+// peer; a settled push, or a walk that saw no retryable outcome, clears it.
+// `peer:online` only runs on an offline→online transition, so a peer that
+// stays "online" through a blip would otherwise never be retried.
+const pushRetryDue = new Set();
+// While a walk is on the stack, per-push mark/clear only records whether
+// THAT walk saw a retryable outcome. Deciding at the end keeps one landed
+// record from cancelling a sibling that failed in the same pass.
+const pushRetryWalks = new Map();
+const pushRetryWalkSaw = new Map();
+let pushRetryScheduler = null;
+
const flattenLogText = (value) => String(value ?? '').replace(/\s+/g, ' ').trim().slice(0, 200);
+function isRetryablePushReason(reason) {
+ if (reason === 'network') return true;
+ const match = /^http-(\d{3})$/.exec(reason || '');
+ if (!match) return false;
+ const status = Number(match[1]);
+ // 4xx is a rejection, not a blip — except the two statuses that mean
+ // "try again" (408 timeout, 429 backoff). Schema 409 never reaches here.
+ return status === 408 || status === 429 || status >= 500;
+}
+
+function inPushRetryWalk(peerId) {
+ return (pushRetryWalks.get(peerId) || 0) > 0;
+}
+
+function markPushRetryDue(peerId) {
+ if (!isNonBlankStr(peerId)) return;
+ if (inPushRetryWalk(peerId)) {
+ pushRetryWalkSaw.set(peerId, true);
+ return;
+ }
+ pushRetryDue.add(peerId);
+ pushRetryScheduler?.onDue(peerId);
+}
+
+export function clearPushRetryDue(peerId) {
+ if (!isNonBlankStr(peerId) || inPushRetryWalk(peerId)) return;
+ pushRetryDue.delete(peerId);
+ pushRetryScheduler?.onClear(peerId);
+}
+
+export function retainPushRetryDue(peerId) {
+ if (!isNonBlankStr(peerId)) return;
+ pushRetryDue.add(peerId);
+}
+
+export function isPushRetryDue(peerId) {
+ return pushRetryDue.has(peerId);
+}
+
+export function listPushRetryDue() {
+ return [...pushRetryDue];
+}
+
+export function registerPushRetryScheduler(scheduler) {
+ pushRetryScheduler = scheduler;
+}
+
+export function beginPushRetryWalk(peerId) {
+ if (!isNonBlankStr(peerId)) return;
+ const depth = (pushRetryWalks.get(peerId) || 0) + 1;
+ pushRetryWalks.set(peerId, depth);
+ if (depth === 1) pushRetryWalkSaw.set(peerId, false);
+}
+
+// Outermost walk returns whether any push in it was retryable.
+export function finishPushRetryWalk(peerId) {
+ const depth = (pushRetryWalks.get(peerId) || 1) - 1;
+ if (depth > 0) {
+ pushRetryWalks.set(peerId, depth);
+ return false;
+ }
+ pushRetryWalks.delete(peerId);
+ const saw = pushRetryWalkSaw.get(peerId) === true;
+ pushRetryWalkSaw.delete(peerId);
+ return saw;
+}
+
+export function resetPushRetryBookkeeping({ keepWalks = false } = {}) {
+ pushRetryDue.clear();
+ if (!keepWalks) {
+ pushRetryWalks.clear();
+ pushRetryWalkSaw.clear();
+ }
+}
+
function reportPushFailure(sub, peer, reason, detail) {
+ if (isRetryablePushReason(reason)) markPushRetryDue(sub.peerId);
if (loggedPushFailures.get(sub.peerId) === reason) return;
loggedPushFailures.set(sub.peerId, reason);
const target = `${sub.recordKind}/${sub.recordId} (sub ${sub.id}) to ${flattenLogText(peer.name || peer.instanceId)}`;
const suffix = `${detail ? `: ${flattenLogText(detail)}` : ''} — record not delivered; repeat ${reason} failures for this peer are logged once until a push lands`;
- // An unreachable peer is routine (asleep, off the tailnet) and `peer:online`
- // retries it; a peer that answered and refused the push is a real failure.
+ // An unreachable peer is routine (asleep, off the tailnet). Retryable
+ // failures schedule the bounded walk; a peer the probe has marked offline
+ // is left for `peer:online`. A peer that answered and refused is not retried.
if (reason === 'network') console.warn(`⚠️ peerSync: push ${target} unreachable${suffix}`);
else console.error(`❌ peerSync: push ${target} rejected with ${reason}${suffix}`);
}
@@ -299,7 +388,7 @@ export async function pushRecordToPeer(sub, options = {}) {
// reported missing assets. The receiver pulls them asynchronously,
// and any pull failure (transient Tailnet flake, rejected
// Content-Length, receiver restart mid-pull) would otherwise be
- // permanently masked — the next `peer:online` retry would short-
+ // permanently masked — a later retry would short-
// circuit on `unchanged` and never re-POST the manifest, so the
// receiver never gets a fresh asset list to retry against. The
// record itself still landed (mergeXxxFromSync ran on the receiver),
@@ -405,6 +494,14 @@ export async function pushRecordToPeer(sub, options = {}) {
// a batch it fires async exactly as before.
flushBaseHashes().catch((err) => console.log(`⚠️ peerSync: base-hash flush after push failed: ${err?.message || err}`));
}
+ // HTTP 2xx is not "settled" when lastPushedHash was withheld: assets, draft
+ // bodies, or a receiver-pending sidecar are still owed, and `peer:online`
+ // will not run again while the probe still says online. A legacy strip is
+ // settled — lastPushedLegacyHash holds that cycle until the peer upgrades.
+ // The failure log was already cleared in reportPushRecovered; the retry
+ // mark follows this decision so a withheld hash is not treated as recovered.
+ if (pushedHash == null && legacyStrippedHash == null) markPushRetryDue(sub.peerId);
+ else clearPushRetryDue(sub.peerId);
return {
pushed: true,
hash,