Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix([issue-12621]): wake probe on peer announce or inbound auth and cap backoff at 1h (#12746)
[._.]/3e50b094aee1
diff --git a/server/services/authGate.js b/server/services/authGate.jsindex 06c1ae4ad7..f0188e44ca 100644--- a/server/services/authGate.js+++ b/server/services/authGate.js@@ -106,7 +106,16 @@ const refusePeerScope = (req, res, path, peer) => {}));};-export const __testing = { verifyBasicPassword };+const wakePeer = (peer) => {+ if (!peer?.id) return;+ import('./instances.js').then(({ wakePeerProbe }) => {+ wakePeerProbe(peer);+ }).catch((err) => {+ console.error(`❌ Failed to wake peer probe: ${err?.message || err}`);+ });+};++export const __testing = { verifyBasicPassword, wakePeer };const BROWSER_REFUSAL_MESSAGES = {CROSS_ORIGIN_BLOCKED: 'Cross-origin request rejected',@@ -165,7 +174,10 @@ export const authGate = async (req, res, next) => {// as anonymous as any other request to a password-free install.const peer = isPeerApiRequestAllowed(req.method, req.path.toLowerCase())? await verifyPeerToken(req.headers) : null;- if (peer) req.portosAuthContext = { enabled: false, authenticated: true, method: 'peer', peerId: peer.id };+ if (peer) {+ req.portosAuthContext = { enabled: false, authenticated: true, method: 'peer', peerId: peer.id };+ wakePeer(peer);+ }return next();}// Express mounts match case-insensitively. Use the same casing for every@@ -201,6 +213,7 @@ export const authGate = async (req, res, next) => {return;}req.portosAuthContext = { enabled: true, authenticated: true, method: 'peer', peerId: peer.id };+ wakePeer(peer);return next();}// Also accept HTTP Basic auth — the legacy peer credential, kept so unpaireddiff --git a/server/services/authGate.test.js b/server/services/authGate.test.jsindex 00e427f922..122503025d 100644--- a/server/services/authGate.test.js+++ b/server/services/authGate.test.js@@ -1318,6 +1318,41 @@ describe('paired peer credential (#8356)', () => {expect(await handshake(headers)).toBeUndefined();expect((await handshake({ ...headers, 'x-portos-peer-auth': 'f'.repeat(64) }))?.data).toEqual({ code: 'AUTH_REQUIRED' });});++ it('wakes probe on inbound peer-authenticated request at most once per 30 s per peer (#12621)', async () => {+ const instances = await import('./instances.js');+ instances.__resetPeerWakesForTests();+ const wakeSpy = vi.spyOn(instances, 'wakePeerProbe');++ const auth = await import('./auth.js');+ await auth.setPassword({ newPassword: 'example-password' });+ const peerRecord = pairedPeer({ status: 'offline', nextProbeAt: new Date(Date.now() + 3_600_000).toISOString() });+ writePeers([peerRecord]);+ const app = await buildApp();+ const headers = await peerHeaders();++ // 1st inbound request: wakes probe+ const res1 = await withHeaders(request(app).get('/api/peer-sync/manifest'), headers);+ expect(res1.status).toBe(200);+ expect(res1.body.auth.method).toBe('peer');+ await vi.waitFor(() => expect(wakeSpy).toHaveBeenCalledTimes(1));+ expect(wakeSpy).toHaveBeenLastCalledWith(expect.objectContaining({ id: 'peer-record' }));++ // 2nd inbound request immediately: auth succeeds, calls wakePeerProbe which rate-limits+ const res2 = await withHeaders(request(app).get('/api/peer-sync/manifest'), headers);+ expect(res2.status).toBe(200);+ expect(res2.body.auth.method).toBe('peer');+ await vi.waitFor(() => expect(wakeSpy).toHaveBeenCalledTimes(2));++ // Verify rate limiter directly rejects duplicate wake within 30s+ expect(instances.wakePeerProbe(peerRecord)).toBe(false);++ // After 30s window (simulated by resetting wake timer), wake is allowed again+ instances.__resetPeerWakesForTests();+ expect(instances.wakePeerProbe(peerRecord)).toBe(true);++ wakeSpy.mockRestore();+ });});// #8707: without a password, the browser-relay guard is the only thing betweendiff --git a/server/services/instances.js b/server/services/instances.jsindex 7e101fd5c9..11dc6f1913 100644--- a/server/services/instances.js+++ b/server/services/instances.js@@ -46,16 +46,51 @@ const INITIAL_PROBE_DELAY_MS = 2000;const TAILSCALE_RECHECK_MS = 60_000;// Backoff tiers for consecutive probe failures (in ms)-// 30s → 1m → 5m → 15m → 1h → 24h-const BACKOFF_TIERS_MS = [+// 30s → 1m → 5m → 15m → 1h (max)+export const BACKOFF_TIERS_MS = [30_000, // tier 0: normal (1 failure)60_000, // tier 1: 1 minute300_000, // tier 2: 5 minutes900_000, // tier 3: 15 minutes- 3_600_000, // tier 4: 1 hour- 86_400_000 // tier 5: 24 hours (max)+ 3_600_000 // tier 4: 1 hour (max)];+export const PEER_WAKE_RATE_LIMIT_MS = 30_000;+const lastWakeAtByPeerId = new Map();++function shouldWakePeer(peerId, now = Date.now()) {+ if (!peerId) return false;+ const lastWake = lastWakeAtByPeerId.get(peerId) || 0;+ if (now - lastWake < PEER_WAKE_RATE_LIMIT_MS) return false;+ lastWakeAtByPeerId.set(peerId, now);+ return true;+}++export function __resetPeerWakesForTests() {+ lastWakeAtByPeerId.clear();+}++export function wakePeerProbe(peer) {+ if (!peer?.id || peer.enabled === false) return false;+ if (peer.status !== 'offline' && !peer.nextProbeAt) return false;+ if (!shouldWakePeer(peer.id)) return false;++ withData(async (data) => {+ const entry = data.peers.find((p) => p.id === peer.id);+ if (entry && entry.nextProbeAt) {+ entry.nextProbeAt = null;+ }+ }).catch((err) => {+ console.error(`❌ Failed to clear nextProbeAt on peer wake: ${err.message}`);+ });++ probePeer(peer).catch((err) => {+ console.log(`⚠️ Wake probe failed for ${peer.name || peer.id}: ${err.message}`);+ });++ return true;+}+let pollTimer = null;// Set while we're waiting for Tailscale to connect before starting the real// probe loop (see startPolling). Kept separate from pollTimer so stopPolling can@@ -638,7 +673,7 @@ export async function updatePeer(id, updates, { probe = true } = {}) {// When sameAuth was true (credential unchanged), authChanged is still// false. If the peer is stuck in authRequired+backoff the user is// explicitly asking for a retry — clear backoff so the immediate probe- // below fires instead of waiting up to 24h.+ // below fires instead of waiting up to 1h.if (!authChanged && normalizedAuth !== null && peer.authRequired) {peer.consecutiveFailures = 0;peer.nextProbeAt = null;@@ -691,7 +726,7 @@ export async function updatePeer(id, updates, { probe = true } = {}) {// writes no longer disrupt an already-healthy connection.if (updates.enabled === false || hostChanged || authChanged) disconnectFromPeer(id);// A credential edit is almost always the fix for a 401, but the peer may be- // deep in exponential probe backoff (up to 24h) from those failures — so the+ // deep in exponential probe backoff (up to 1h) from those failures — so the// regular poll loop would skip it for hours. Retry immediately: probePeer// bypasses the nextProbeAt gate and, on success, resets backoff + reconnects// the relay with the new credential. Fire-and-forget; failure just re-arms@@ -1154,6 +1189,7 @@ export async function handleAnnounce({ address, port, instanceId, name, host })// Set only when this announce marked the peer online (not the forged-address// branch, which records a sighting without trusting it).let markedOnlinePeer = null;+ let peerToWake = null;const result = await withData(async (data) => {// Check for existing peer by instanceIdlet existing = data.peers.find(p => p.instanceId === instanceId);@@ -1169,6 +1205,14 @@ export async function handleAnnounce({ address, port, instanceId, name, host })matchedByAddress = true;}+ const instanceIdMatches = !existing || !existing.instanceId || existing.instanceId === instanceId;+ if (existing && existing.enabled !== false && instanceIdMatches) {+ if (shouldWakePeer(existing.id)) {+ if (existing.nextProbeAt) existing.nextProbeAt = null;+ peerToWake = existing;+ }+ }+const normalizedHost = announcedHost(host);if (existing?.syncSecret) return { created: false, peer: existing };@@ -1192,6 +1236,8 @@ export async function handleAnnounce({ address, port, instanceId, name, host })if (existing) {existing.lastSeen = new Date().toISOString();existing.status = 'online';+ existing.nextProbeAt = null;+ existing.consecutiveFailures = 0;markedOnlinePeer = existing;existing.instanceId = instanceId;// A managed forward uses local coordinates, not the remote listen port.@@ -1260,6 +1306,10 @@ export async function handleAnnounce({ address, port, instanceId, name, host })probePeer(result.peer).catch(err => {console.log(`⚠️ Initial probe failed for announced peer ${result.peer.name}: ${err.message}`);});+ } else if (peerToWake) {+ probePeer(peerToWake).catch(err => {+ console.log(`⚠️ Wake probe failed for ${peerToWake.name || peerToWake.id}: ${err.message}`);+ });}return result;diff --git a/server/services/instances.test.js b/server/services/instances.test.jsindex 3ef7587ba2..78b2b417e8 100644--- a/server/services/instances.test.js+++ b/server/services/instances.test.js@@ -86,15 +86,22 @@ import {DEFAULT_SYNC_CATEGORIES,startPolling,stopPolling,- __resetPeersSeenOnlineForTests+ __resetPeersSeenOnlineForTests,+ __resetPeerWakesForTests,+ BACKOFF_TIERS_MS,+ PEER_WAKE_RATE_LIMIT_MS,+ shouldWakePeer,+ wakePeerProbe} from './instances.js';describe('instances.js', () => {beforeEach(() => {vi.clearAllMocks();__resetPeersSeenOnlineForTests();+ __resetPeerWakesForTests();vi.useFakeTimers();vi.stubGlobal('fetch', vi.fn());+ dns.lookup.mockResolvedValue([]);readJSONFile.mockResolvedValue({ self: null, peers: [] });getTailscaleStatus.mockResolvedValue({ available: true, running: true, state: 'Running', reason: 'running' });});@@ -1967,6 +1974,210 @@ describe('instances.js', () => {expect(result.peer.host).toBeNull();});++ it('probes a paired peer on the top backoff tier immediately when it announces, emits peer:online, and preserves host/port/name (#12621)', async () => {+ const peer = {+ id: 'peer-paired',+ instanceId: 'remote-paired',+ address: '192.0.2.10',+ port: 5555,+ host: null,+ name: 'Stored Box Name',+ status: 'offline',+ consecutiveFailures: 5,+ nextProbeAt: new Date(Date.now() + 3_600_000).toISOString(),+ syncSecret: 'synthetic-pair-secret-32-characters-long',+ enabled: true,+ };+ readJSONFile.mockResolvedValue({ self: { instanceId: 'local-instance' }, peers: [peer] });++ fetch.mockImplementation(async (url) => {+ if (url.includes('/api/system/health/details')) {+ return {+ ok: true,+ status: 200,+ json: async () => ({ instanceId: 'remote-paired', version: '1.7.0' }),+ };+ }+ return { ok: false, status: 404 };+ });++ const onlineEmits = () => instanceEvents.emit.mock.calls.filter(([name]) => name === 'peer:online');++ const result = await handleAnnounce({+ instanceId: 'remote-paired',+ address: '192.0.2.10',+ port: 5555,+ host: 'attacker-host.example.com',+ name: 'Attacker Attempt Name',+ });++ expect(result.created).toBe(false);+ expect(result.peer.host).toBeNull();+ expect(result.peer.port).toBe(5555);+ expect(result.peer.name).toBe('Stored Box Name');+ expect(result.peer.nextProbeAt).toBeNull();++ await vi.waitFor(() => expect(onlineEmits().length).toBe(1));+ expect(onlineEmits()[0][1]).toMatchObject({+ id: 'peer-paired',+ status: 'online',+ host: null,+ port: 5555,+ name: 'Stored Box Name',+ consecutiveFailures: 0,+ nextProbeAt: null,+ });+ });++ it('triggers probe to stored address only when a forged announce with mismatched address arrives (#12621)', async () => {+ const peer = {+ id: 'peer-paired',+ instanceId: 'remote-paired',+ address: '192.0.2.10',+ port: 5555,+ host: null,+ name: 'Stored Box Name',+ status: 'offline',+ consecutiveFailures: 5,+ nextProbeAt: new Date(Date.now() + 3_600_000).toISOString(),+ syncSecret: 'synthetic-pair-secret-32-characters-long',+ enabled: true,+ };+ readJSONFile.mockResolvedValue({ self: { instanceId: 'local-instance' }, peers: [peer] });++ const fetchedUrls = [];+ fetch.mockImplementation(async (url) => {+ fetchedUrls.push(url);+ return { ok: false, status: 500 };+ });++ const result = await handleAnnounce({+ instanceId: 'remote-paired',+ address: '198.51.100.99', // forged IP+ port: 9999,+ host: 'attacker-host.example.com',+ name: 'Attacker Attempt Name',+ });++ expect(result.created).toBe(false);+ expect(result.peer.address).toBe('192.0.2.10');+ expect(result.peer.port).toBe(5555);+ expect(result.peer.host).toBeNull();+ expect(result.peer.name).toBe('Stored Box Name');++ await vi.waitFor(() => expect(fetchedUrls.length).toBeGreaterThan(0));+ for (const url of fetchedUrls) {+ expect(url).not.toContain('198.51.100.99');+ expect(url).not.toContain('9999');+ expect(url).not.toContain('attacker-host.example.com');+ expect(url).toContain('192.0.2.10');+ }+ });++ it('rate-limits announce wakes to at most once per 30 seconds per peer (#12621)', async () => {+ const peer = {+ id: 'peer-paired',+ instanceId: 'remote-paired',+ address: '192.0.2.10',+ port: 5555,+ status: 'offline',+ syncSecret: 'synthetic-pair-secret-32-characters-long',+ enabled: true,+ };+ readJSONFile.mockResolvedValue({ self: { instanceId: 'local-instance' }, peers: [peer] });++ let probeCount = 0;+ fetch.mockImplementation(async (url) => {+ if (url.includes('/api/system/health/details')) probeCount++;+ return { ok: false, status: 500 };+ });++ // First announce wakes the probe+ await handleAnnounce({ instanceId: 'remote-paired', address: '192.0.2.10', port: 5555 });+ await vi.waitFor(() => expect(probeCount).toBe(1));++ // Second announce 5s later does not wake+ vi.advanceTimersByTime(5_000);+ await handleAnnounce({ instanceId: 'remote-paired', address: '192.0.2.10', port: 5555 });+ expect(probeCount).toBe(1);++ // Third announce after 30s interval wakes again+ vi.advanceTimersByTime(26_000); // 31s total+ await handleAnnounce({ instanceId: 'remote-paired', address: '192.0.2.10', port: 5555 });+ await vi.waitFor(() => expect(probeCount).toBe(2));+ });+ });++ describe('probe backoff and wake (#12621)', () => {+ it('pins the tier sequence and caps backoff at 1 hour (#12621)', () => {+ expect(BACKOFF_TIERS_MS).toEqual([+ 30_000,+ 60_000,+ 300_000,+ 900_000,+ 3_600_000,+ ]);+ expect(BACKOFF_TIERS_MS.at(-1)).toBe(3_600_000);+ });++ it('caps backoff tier at 1h (3_600_000 ms) on repeated consecutive failures', async () => {+ const peer = {+ id: 'peer-failing',+ address: '192.0.2.20',+ port: 5555,+ name: 'Failing peer',+ status: 'offline',+ consecutiveFailures: 10,+ enabled: true,+ };+ readJSONFile.mockResolvedValue({ self: null, peers: [peer] });+ fetch.mockRejectedValue(new Error('offline'));++ const updated = await probePeer(peer);+ expect(updated.consecutiveFailures).toBe(11);+ const expectedTime = new Date(Date.now() + 3_600_000).toISOString();+ expect(updated.nextProbeAt).toBe(expectedTime);+ });++ it('wakePeerProbe clears nextProbeAt and fires probePeer rate-limited to once per 30s', async () => {+ const peer = {+ id: 'peer-sleepy',+ address: '192.0.2.30',+ port: 5555,+ name: 'Sleepy peer',+ status: 'offline',+ nextProbeAt: new Date(Date.now() + 3_600_000).toISOString(),+ enabled: true,+ };+ readJSONFile.mockResolvedValue({ self: null, peers: [peer] });++ let probeCalled = 0;+ fetch.mockImplementation(async (url) => {+ if (url.includes('/api/system/health/details')) probeCalled++;+ return { ok: false, status: 500 };+ });++ const first = wakePeerProbe(peer);+ expect(first).toBe(true);+ await vi.waitFor(() => expect(probeCalled).toBe(1));++ // Second call immediately is rate-limited+ const second = wakePeerProbe(peer);+ expect(second).toBe(false);+ expect(probeCalled).toBe(1);++ // After 30s passes, it wakes again+ vi.advanceTimersByTime(PEER_WAKE_RATE_LIMIT_MS);+ const third = wakePeerProbe(peer);+ expect(third).toBe(true);+ await vi.waitFor(() => expect(probeCalled).toBe(2));+ });++ it('wakePeerProbe ignores disabled peers', () => {+ const disabled = { id: 'peer-off', enabled: false };+ expect(wakePeerProbe(disabled)).toBe(false);+ });});// --- Polling ---