Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix([issue-12621]): wake probe on peer announce or inbound auth and cap backoff at 1h (#12746)

[._.]/3e50b094aee1
diff --git a/server/services/authGate.js b/server/services/authGate.js
index 06c1ae4ad7..f0188e44ca 100644
--- a/server/services/authGate.js
+++ b/server/services/authGate.js
@@ -106,7 +106,16 @@ const refusePeerScope = (req, res, path, peer) => {
}));
};
-export const __testing = { verifyBasicPassword };
+const wakePeer = (peer) => {
+ if (!peer?.id) return;
+ import('./instances.js').then(({ wakePeerProbe }) => {
+ wakePeerProbe(peer);
+ }).catch((err) => {
+ console.error(`❌ Failed to wake peer probe: ${err?.message || err}`);
+ });
+};
+
+export const __testing = { verifyBasicPassword, wakePeer };
const BROWSER_REFUSAL_MESSAGES = {
CROSS_ORIGIN_BLOCKED: 'Cross-origin request rejected',
@@ -165,7 +174,10 @@ export const authGate = async (req, res, next) => {
// as anonymous as any other request to a password-free install.
const peer = isPeerApiRequestAllowed(req.method, req.path.toLowerCase())
? await verifyPeerToken(req.headers) : null;
- if (peer) req.portosAuthContext = { enabled: false, authenticated: true, method: 'peer', peerId: peer.id };
+ if (peer) {
+ req.portosAuthContext = { enabled: false, authenticated: true, method: 'peer', peerId: peer.id };
+ wakePeer(peer);
+ }
return next();
}
// Express mounts match case-insensitively. Use the same casing for every
@@ -201,6 +213,7 @@ export const authGate = async (req, res, next) => {
return;
}
req.portosAuthContext = { enabled: true, authenticated: true, method: 'peer', peerId: peer.id };
+ wakePeer(peer);
return next();
}
// Also accept HTTP Basic auth — the legacy peer credential, kept so unpaired
diff --git a/server/services/authGate.test.js b/server/services/authGate.test.js
index 00e427f922..122503025d 100644
--- a/server/services/authGate.test.js
+++ b/server/services/authGate.test.js
@@ -1318,6 +1318,41 @@ describe('paired peer credential (#8356)', () => {
expect(await handshake(headers)).toBeUndefined();
expect((await handshake({ ...headers, 'x-portos-peer-auth': 'f'.repeat(64) }))?.data).toEqual({ code: 'AUTH_REQUIRED' });
});
+
+ it('wakes probe on inbound peer-authenticated request at most once per 30 s per peer (#12621)', async () => {
+ const instances = await import('./instances.js');
+ instances.__resetPeerWakesForTests();
+ const wakeSpy = vi.spyOn(instances, 'wakePeerProbe');
+
+ const auth = await import('./auth.js');
+ await auth.setPassword({ newPassword: 'example-password' });
+ const peerRecord = pairedPeer({ status: 'offline', nextProbeAt: new Date(Date.now() + 3_600_000).toISOString() });
+ writePeers([peerRecord]);
+ const app = await buildApp();
+ const headers = await peerHeaders();
+
+ // 1st inbound request: wakes probe
+ const res1 = await withHeaders(request(app).get('/api/peer-sync/manifest'), headers);
+ expect(res1.status).toBe(200);
+ expect(res1.body.auth.method).toBe('peer');
+ await vi.waitFor(() => expect(wakeSpy).toHaveBeenCalledTimes(1));
+ expect(wakeSpy).toHaveBeenLastCalledWith(expect.objectContaining({ id: 'peer-record' }));
+
+ // 2nd inbound request immediately: auth succeeds, calls wakePeerProbe which rate-limits
+ const res2 = await withHeaders(request(app).get('/api/peer-sync/manifest'), headers);
+ expect(res2.status).toBe(200);
+ expect(res2.body.auth.method).toBe('peer');
+ await vi.waitFor(() => expect(wakeSpy).toHaveBeenCalledTimes(2));
+
+ // Verify rate limiter directly rejects duplicate wake within 30s
+ expect(instances.wakePeerProbe(peerRecord)).toBe(false);
+
+ // After 30s window (simulated by resetting wake timer), wake is allowed again
+ instances.__resetPeerWakesForTests();
+ expect(instances.wakePeerProbe(peerRecord)).toBe(true);
+
+ wakeSpy.mockRestore();
+ });
});
// #8707: without a password, the browser-relay guard is the only thing between
diff --git a/server/services/instances.js b/server/services/instances.js
index 7e101fd5c9..11dc6f1913 100644
--- a/server/services/instances.js
+++ b/server/services/instances.js
@@ -46,16 +46,51 @@ const INITIAL_PROBE_DELAY_MS = 2000;
const TAILSCALE_RECHECK_MS = 60_000;
// Backoff tiers for consecutive probe failures (in ms)
-// 30s → 1m → 5m → 15m → 1h → 24h
-const BACKOFF_TIERS_MS = [
+// 30s → 1m → 5m → 15m → 1h (max)
+export const BACKOFF_TIERS_MS = [
30_000, // tier 0: normal (1 failure)
60_000, // tier 1: 1 minute
300_000, // tier 2: 5 minutes
900_000, // tier 3: 15 minutes
- 3_600_000, // tier 4: 1 hour
- 86_400_000 // tier 5: 24 hours (max)
+ 3_600_000 // tier 4: 1 hour (max)
];
+export const PEER_WAKE_RATE_LIMIT_MS = 30_000;
+const lastWakeAtByPeerId = new Map();
+
+function shouldWakePeer(peerId, now = Date.now()) {
+ if (!peerId) return false;
+ const lastWake = lastWakeAtByPeerId.get(peerId) || 0;
+ if (now - lastWake < PEER_WAKE_RATE_LIMIT_MS) return false;
+ lastWakeAtByPeerId.set(peerId, now);
+ return true;
+}
+
+export function __resetPeerWakesForTests() {
+ lastWakeAtByPeerId.clear();
+}
+
+export function wakePeerProbe(peer) {
+ if (!peer?.id || peer.enabled === false) return false;
+ if (peer.status !== 'offline' && !peer.nextProbeAt) return false;
+ if (!shouldWakePeer(peer.id)) return false;
+
+ withData(async (data) => {
+ const entry = data.peers.find((p) => p.id === peer.id);
+ if (entry && entry.nextProbeAt) {
+ entry.nextProbeAt = null;
+ }
+ }).catch((err) => {
+ console.error(`❌ Failed to clear nextProbeAt on peer wake: ${err.message}`);
+ });
+
+ probePeer(peer).catch((err) => {
+ console.log(`⚠️ Wake probe failed for ${peer.name || peer.id}: ${err.message}`);
+ });
+
+ return true;
+}
+
let pollTimer = null;
// Set while we're waiting for Tailscale to connect before starting the real
// probe loop (see startPolling). Kept separate from pollTimer so stopPolling can
@@ -638,7 +673,7 @@ export async function updatePeer(id, updates, { probe = true } = {}) {
// When sameAuth was true (credential unchanged), authChanged is still
// false. If the peer is stuck in authRequired+backoff the user is
// explicitly asking for a retry — clear backoff so the immediate probe
- // below fires instead of waiting up to 24h.
+ // below fires instead of waiting up to 1h.
if (!authChanged && normalizedAuth !== null && peer.authRequired) {
peer.consecutiveFailures = 0;
peer.nextProbeAt = null;
@@ -691,7 +726,7 @@ export async function updatePeer(id, updates, { probe = true } = {}) {
// writes no longer disrupt an already-healthy connection.
if (updates.enabled === false || hostChanged || authChanged) disconnectFromPeer(id);
// A credential edit is almost always the fix for a 401, but the peer may be
- // deep in exponential probe backoff (up to 24h) from those failures — so the
+ // deep in exponential probe backoff (up to 1h) from those failures — so the
// regular poll loop would skip it for hours. Retry immediately: probePeer
// bypasses the nextProbeAt gate and, on success, resets backoff + reconnects
// the relay with the new credential. Fire-and-forget; failure just re-arms
@@ -1154,6 +1189,7 @@ export async function handleAnnounce({ address, port, instanceId, name, host })
// Set only when this announce marked the peer online (not the forged-address
// branch, which records a sighting without trusting it).
let markedOnlinePeer = null;
+ let peerToWake = null;
const result = await withData(async (data) => {
// Check for existing peer by instanceId
let existing = data.peers.find(p => p.instanceId === instanceId);
@@ -1169,6 +1205,14 @@ export async function handleAnnounce({ address, port, instanceId, name, host })
matchedByAddress = true;
}
+ const instanceIdMatches = !existing || !existing.instanceId || existing.instanceId === instanceId;
+ if (existing && existing.enabled !== false && instanceIdMatches) {
+ if (shouldWakePeer(existing.id)) {
+ if (existing.nextProbeAt) existing.nextProbeAt = null;
+ peerToWake = existing;
+ }
+ }
+
const normalizedHost = announcedHost(host);
if (existing?.syncSecret) return { created: false, peer: existing };
@@ -1192,6 +1236,8 @@ export async function handleAnnounce({ address, port, instanceId, name, host })
if (existing) {
existing.lastSeen = new Date().toISOString();
existing.status = 'online';
+ existing.nextProbeAt = null;
+ existing.consecutiveFailures = 0;
markedOnlinePeer = existing;
existing.instanceId = instanceId;
// A managed forward uses local coordinates, not the remote listen port.
@@ -1260,6 +1306,10 @@ export async function handleAnnounce({ address, port, instanceId, name, host })
probePeer(result.peer).catch(err => {
console.log(`⚠️ Initial probe failed for announced peer ${result.peer.name}: ${err.message}`);
});
+ } else if (peerToWake) {
+ probePeer(peerToWake).catch(err => {
+ console.log(`⚠️ Wake probe failed for ${peerToWake.name || peerToWake.id}: ${err.message}`);
+ });
}
return result;
diff --git a/server/services/instances.test.js b/server/services/instances.test.js
index 3ef7587ba2..78b2b417e8 100644
--- a/server/services/instances.test.js
+++ b/server/services/instances.test.js
@@ -86,15 +86,22 @@ import {
DEFAULT_SYNC_CATEGORIES,
startPolling,
stopPolling,
- __resetPeersSeenOnlineForTests
+ __resetPeersSeenOnlineForTests,
+ __resetPeerWakesForTests,
+ BACKOFF_TIERS_MS,
+ PEER_WAKE_RATE_LIMIT_MS,
+ shouldWakePeer,
+ wakePeerProbe
} from './instances.js';
describe('instances.js', () => {
beforeEach(() => {
vi.clearAllMocks();
__resetPeersSeenOnlineForTests();
+ __resetPeerWakesForTests();
vi.useFakeTimers();
vi.stubGlobal('fetch', vi.fn());
+ dns.lookup.mockResolvedValue([]);
readJSONFile.mockResolvedValue({ self: null, peers: [] });
getTailscaleStatus.mockResolvedValue({ available: true, running: true, state: 'Running', reason: 'running' });
});
@@ -1967,6 +1974,210 @@ describe('instances.js', () => {
expect(result.peer.host).toBeNull();
});
+
+ it('probes a paired peer on the top backoff tier immediately when it announces, emits peer:online, and preserves host/port/name (#12621)', async () => {
+ const peer = {
+ id: 'peer-paired',
+ instanceId: 'remote-paired',
+ address: '192.0.2.10',
+ port: 5555,
+ host: null,
+ name: 'Stored Box Name',
+ status: 'offline',
+ consecutiveFailures: 5,
+ nextProbeAt: new Date(Date.now() + 3_600_000).toISOString(),
+ syncSecret: 'synthetic-pair-secret-32-characters-long',
+ enabled: true,
+ };
+ readJSONFile.mockResolvedValue({ self: { instanceId: 'local-instance' }, peers: [peer] });
+
+ fetch.mockImplementation(async (url) => {
+ if (url.includes('/api/system/health/details')) {
+ return {
+ ok: true,
+ status: 200,
+ json: async () => ({ instanceId: 'remote-paired', version: '1.7.0' }),
+ };
+ }
+ return { ok: false, status: 404 };
+ });
+
+ const onlineEmits = () => instanceEvents.emit.mock.calls.filter(([name]) => name === 'peer:online');
+
+ const result = await handleAnnounce({
+ instanceId: 'remote-paired',
+ address: '192.0.2.10',
+ port: 5555,
+ host: 'attacker-host.example.com',
+ name: 'Attacker Attempt Name',
+ });
+
+ expect(result.created).toBe(false);
+ expect(result.peer.host).toBeNull();
+ expect(result.peer.port).toBe(5555);
+ expect(result.peer.name).toBe('Stored Box Name');
+ expect(result.peer.nextProbeAt).toBeNull();
+
+ await vi.waitFor(() => expect(onlineEmits().length).toBe(1));
+ expect(onlineEmits()[0][1]).toMatchObject({
+ id: 'peer-paired',
+ status: 'online',
+ host: null,
+ port: 5555,
+ name: 'Stored Box Name',
+ consecutiveFailures: 0,
+ nextProbeAt: null,
+ });
+ });
+
+ it('triggers probe to stored address only when a forged announce with mismatched address arrives (#12621)', async () => {
+ const peer = {
+ id: 'peer-paired',
+ instanceId: 'remote-paired',
+ address: '192.0.2.10',
+ port: 5555,
+ host: null,
+ name: 'Stored Box Name',
+ status: 'offline',
+ consecutiveFailures: 5,
+ nextProbeAt: new Date(Date.now() + 3_600_000).toISOString(),
+ syncSecret: 'synthetic-pair-secret-32-characters-long',
+ enabled: true,
+ };
+ readJSONFile.mockResolvedValue({ self: { instanceId: 'local-instance' }, peers: [peer] });
+
+ const fetchedUrls = [];
+ fetch.mockImplementation(async (url) => {
+ fetchedUrls.push(url);
+ return { ok: false, status: 500 };
+ });
+
+ const result = await handleAnnounce({
+ instanceId: 'remote-paired',
+ address: '198.51.100.99', // forged IP
+ port: 9999,
+ host: 'attacker-host.example.com',
+ name: 'Attacker Attempt Name',
+ });
+
+ expect(result.created).toBe(false);
+ expect(result.peer.address).toBe('192.0.2.10');
+ expect(result.peer.port).toBe(5555);
+ expect(result.peer.host).toBeNull();
+ expect(result.peer.name).toBe('Stored Box Name');
+
+ await vi.waitFor(() => expect(fetchedUrls.length).toBeGreaterThan(0));
+ for (const url of fetchedUrls) {
+ expect(url).not.toContain('198.51.100.99');
+ expect(url).not.toContain('9999');
+ expect(url).not.toContain('attacker-host.example.com');
+ expect(url).toContain('192.0.2.10');
+ }
+ });
+
+ it('rate-limits announce wakes to at most once per 30 seconds per peer (#12621)', async () => {
+ const peer = {
+ id: 'peer-paired',
+ instanceId: 'remote-paired',
+ address: '192.0.2.10',
+ port: 5555,
+ status: 'offline',
+ syncSecret: 'synthetic-pair-secret-32-characters-long',
+ enabled: true,
+ };
+ readJSONFile.mockResolvedValue({ self: { instanceId: 'local-instance' }, peers: [peer] });
+
+ let probeCount = 0;
+ fetch.mockImplementation(async (url) => {
+ if (url.includes('/api/system/health/details')) probeCount++;
+ return { ok: false, status: 500 };
+ });
+
+ // First announce wakes the probe
+ await handleAnnounce({ instanceId: 'remote-paired', address: '192.0.2.10', port: 5555 });
+ await vi.waitFor(() => expect(probeCount).toBe(1));
+
+ // Second announce 5s later does not wake
+ vi.advanceTimersByTime(5_000);
+ await handleAnnounce({ instanceId: 'remote-paired', address: '192.0.2.10', port: 5555 });
+ expect(probeCount).toBe(1);
+
+ // Third announce after 30s interval wakes again
+ vi.advanceTimersByTime(26_000); // 31s total
+ await handleAnnounce({ instanceId: 'remote-paired', address: '192.0.2.10', port: 5555 });
+ await vi.waitFor(() => expect(probeCount).toBe(2));
+ });
+ });
+
+ describe('probe backoff and wake (#12621)', () => {
+ it('pins the tier sequence and caps backoff at 1 hour (#12621)', () => {
+ expect(BACKOFF_TIERS_MS).toEqual([
+ 30_000,
+ 60_000,
+ 300_000,
+ 900_000,
+ 3_600_000,
+ ]);
+ expect(BACKOFF_TIERS_MS.at(-1)).toBe(3_600_000);
+ });
+
+ it('caps backoff tier at 1h (3_600_000 ms) on repeated consecutive failures', async () => {
+ const peer = {
+ id: 'peer-failing',
+ address: '192.0.2.20',
+ port: 5555,
+ name: 'Failing peer',
+ status: 'offline',
+ consecutiveFailures: 10,
+ enabled: true,
+ };
+ readJSONFile.mockResolvedValue({ self: null, peers: [peer] });
+ fetch.mockRejectedValue(new Error('offline'));
+
+ const updated = await probePeer(peer);
+ expect(updated.consecutiveFailures).toBe(11);
+ const expectedTime = new Date(Date.now() + 3_600_000).toISOString();
+ expect(updated.nextProbeAt).toBe(expectedTime);
+ });
+
+ it('wakePeerProbe clears nextProbeAt and fires probePeer rate-limited to once per 30s', async () => {
+ const peer = {
+ id: 'peer-sleepy',
+ address: '192.0.2.30',
+ port: 5555,
+ name: 'Sleepy peer',
+ status: 'offline',
+ nextProbeAt: new Date(Date.now() + 3_600_000).toISOString(),
+ enabled: true,
+ };
+ readJSONFile.mockResolvedValue({ self: null, peers: [peer] });
+
+ let probeCalled = 0;
+ fetch.mockImplementation(async (url) => {
+ if (url.includes('/api/system/health/details')) probeCalled++;
+ return { ok: false, status: 500 };
+ });
+
+ const first = wakePeerProbe(peer);
+ expect(first).toBe(true);
+ await vi.waitFor(() => expect(probeCalled).toBe(1));
+
+ // Second call immediately is rate-limited
+ const second = wakePeerProbe(peer);
+ expect(second).toBe(false);
+ expect(probeCalled).toBe(1);
+
+ // After 30s passes, it wakes again
+ vi.advanceTimersByTime(PEER_WAKE_RATE_LIMIT_MS);
+ const third = wakePeerProbe(peer);
+ expect(third).toBe(true);
+ await vi.waitFor(() => expect(probeCalled).toBe(2));
+ });
+
+ it('wakePeerProbe ignores disabled peers', () => {
+ const disabled = { id: 'peer-off', enabled: false };
+ expect(wakePeerProbe(disabled)).toBe(false);
+ });
});
// --- Polling ---