Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Instances Service
*
* Manages PortOS peer orchestration — peer registration, health probing,
* query proxying, and reciprocal sync. This install's own federation identity
* (instanceId/name) and the data/instances.json file I/O + mutex live in
* services/instanceIdentity.js; this module is a consumer of that leaf.
* Data persists to data/instances.json.
*/
import net from 'net';
import dns from 'dns/promises';
import { isBlockedIngestHost } from '../lib/catalogValidation.js';
import crypto from 'crypto';
import { loadData, withData, UNKNOWN_INSTANCE_ID, getInstanceId, getSelf } from './instanceIdentity.js';
import { createKeyCachedQueue } from '../lib/createKeyCachedQueue.js';
import { canonicalStringify } from '../lib/objects.js';
import { instanceEvents } from './instanceEvents.js';
import { connectToPeer, disconnectFromPeer } from './peerSocketRelay.js';
import { DEFAULT_PEER_PORT } from '../lib/ports.js';
import { peerBaseUrl } from '../lib/peerUrl.js';
import { isPeerApiRequestAllowed } from '../lib/apiAccessPolicy.js';
import { peerFetch, peerAuthHeaders, PEER_AUTH_HEADER } from '../lib/peerHttpClient.js';
import { withAbortTimeout } from '../lib/abortTimeout.js';
import { ServerError } from '../lib/errorHandler.js';
import { getSelfHost } from '../lib/peerSelfHost.js';
import { getTailscaleStatus } from '../lib/tailscale.js';
import { autoSubscribePeerToAllRecords } from './sharing/recordEvents.js';
import {
mergePeerMediaProviderConfig,
normalizePeerMediaProviderConfig,
probeFederatedMediaProvider,
} from './federatedMediaConsumer.js';
import {
classifyPeerProbeFailure,
buildLastProbeRecord,
formatProbeDiagnosticLog,
} from '../lib/peerProbeDiagnostics.js';
const PROBE_TIMEOUT_MS = 5000;
const POLL_INTERVAL_MS = 30000;
const INITIAL_PROBE_DELAY_MS = 2000;
// While peer probing is deferred (Tailscale not connected at boot), re-check
// Tailscale on this cadence and start polling the moment it comes up — so the
// user just has to connect Tailscale, no manual "sync now" required.
const TAILSCALE_RECHECK_MS = 60_000;
// Backoff tiers for consecutive probe failures (in ms)
// 30s → 1m → 5m → 15m → 1h (max)
export const BACKOFF_TIERS_MS = [
30_000, // tier 0: normal (1 failure)
60_000, // tier 1: 1 minute
300_000, // tier 2: 5 minutes
900_000, // tier 3: 15 minutes
3_600_000 // tier 4: 1 hour (max)
];
export const PEER_WAKE_RATE_LIMIT_MS = 30_000;
const lastWakeAtByPeerId = new Map();
function shouldWakePeer(peerId, now = Date.now()) {
if (!peerId) return false;
const lastWake = lastWakeAtByPeerId.get(peerId) || 0;
if (now - lastWake < PEER_WAKE_RATE_LIMIT_MS) return false;
lastWakeAtByPeerId.set(peerId, now);
return true;
}
export function __resetPeerWakesForTests() {
lastWakeAtByPeerId.clear();
}
export function wakePeerProbe(peer) {
if (!peer?.id || peer.enabled === false) return false;
if (peer.status !== 'offline' && !peer.nextProbeAt) return false;
if (!shouldWakePeer(peer.id)) return false;
withData(async (data) => {
const entry = data.peers.find((p) => p.id === peer.id);
if (entry && entry.nextProbeAt) {
entry.nextProbeAt = null;
}
}).catch((err) => {
console.error(`❌ Failed to clear nextProbeAt on peer wake: ${err.message}`);
});
probePeer(peer).catch((err) => {
console.log(`⚠️ Wake probe failed for ${peer.name || peer.id}: ${err.message}`);
});
return true;
}
let pollTimer = null;
// Set while we're waiting for Tailscale to connect before starting the real
// probe loop (see startPolling). Kept separate from pollTimer so stopPolling can
// tear down either state, and so the boot gate is idempotent under double-start.
let tailscaleWatchTimer = null;
let pollingStartPending = false;
// Bumped by stopPolling() to cancel any in-flight startPolling gate or deferred
// watcher callback that is mid-await when the stop lands — without it, a stop
// during the async window is a silent no-op and a timer gets created after stop
// was requested.
let pollingGeneration = 0;
function classifyProbeError(err, peer, tunnelError = null) {
const classified = classifyPeerProbeFailure(err, {
peer,
tunnelError,
probeTimeoutMs: PROBE_TIMEOUT_MS,
});
// Keep emoji prefixes for classic log grepping; structured class is on lastProbe.
if (classified.class === 'dns') return `🌐 ❌ ${classified.message}`;
if (classified.class === 'local_refused') return `🌐 ❌ ${classified.message}`;
if (classified.class === 'host_unreachable') return `🌐 ❌ ${classified.message}`;
if (classified.class === 'probe_timeout') return `🌐 ⏱️ ${classified.message}`;
if (classified.class === 'tunnel_dial') return `🐈 ❌ ${classified.message}`;
if (classified.class === 'auth_required') return `🔒 ${classified.message}`;
if (classified.class === 'probe_http') return `🌐 ❌ ${classified.message}`;
return classified.message;
}
/** Best-effort live forward hint for a tailcat peer (dynamic import avoids a cycle). */
async function lookupTailcatForwardHint(peer) {
if (peer?.transport !== 'tailcat') return null;
try {
const { listTailcatForwards } = await import('./tailcatPeer.js');
const forwards = await listTailcatForwards();
return forwards.find((f) => f.peerId === peer.id) || null;
} catch {
return null;
}
}
// Normalize a credential object off a peer add/update payload. Returns:
// - `undefined` for absent / malformed input, or a username-only payload
// (caller leaves the field as-is — see below)
// - `null` for an explicit clear (both fields blank)
// - `{ username, password }` for a credential to store
// The password is the secret that defines the credential; username defaults to
// '' so a password-only credential is valid Basic auth. A payload with a
// username but no password is treated as "ignore", NOT a blank-password store:
// the client only ever receives a redacted peer (`{ username, hasPassword }`),
// so round-tripping that shape back into a PATCH must not silently wipe a
// working password. An explicit clear always goes through `auth: null`.
function sanitizePeerAuth(auth) {
if (auth === null) return null;
if (!auth || typeof auth !== 'object' || Array.isArray(auth)) return undefined;
const username = typeof auth.username === 'string' ? auth.username.trim() : '';
const password = typeof auth.password === 'string' ? auth.password : '';
if (!password) return username ? undefined : null;
return { username, password };
}
// True when two credential values (null or { username, password }) are
// equivalent — used to skip a needless socket-relay reconnect on a no-op
// credential write.
function sameAuth(a, b) {
if (!a && !b) return true;
if (!a || !b) return false;
return a.username === b.username && a.password === b.password;
}
// Strip the locally-stored proxy credential before a peer record crosses the
// wire (e.g. the announce response echoes the matched local peer back to the
// announcing instance). The password is OUR secret for reaching THEM and must
// never leak to the peer itself. Consumer routing policy and cached capacity
// are local-only too, so peers cannot discover or influence our assignments.
export function redactPeerForWire(peer) {
if (!peer || typeof peer !== 'object') return peer;
if (!('auth' in peer) && !('mediaProvider' in peer) && !('mediaProviderStatus' in peer)
&& !('tcAddress' in peer) && !('syncSecret' in peer)) {
return peer;
}
const {
auth: _auth,
syncSecret: _syncSecret,
mediaProvider: _mediaProvider,
mediaProviderStatus: _mediaProviderStatus,
tcAddress: _tcAddress,
...rest
} = peer;
return rest;
}
// Redact the stored credential before a peer record is returned to the local
// client (API responses + socket broadcasts). Keeps the non-secret username
// and a `hasPassword` marker so the UI can show "credential set" and prefill
// the username, but never ships the password to the browser — mirrors the
// `hasApiKey` pattern in server/routes/providers.js. The password stays only
// on the server-side record, where peerAuthHeaders reads it.
export function sanitizePeerForClient(peer) {
if (!peer || typeof peer !== 'object') return peer;
const auth = peer.auth && typeof peer.auth === 'object'
? { username: peer.auth.username ?? '', hasPassword: Boolean(peer.auth.password) }
: peer.auth;
// EFFECTIVE categories, not the raw stored map — the SAME resolution the sync
// loop runs. Shipping the raw map would render a default-ON category's
// checkbox off while the server is actively syncing it (and make turning it
// off take two clicks), and a second, subtly different resolver here would
// put the UI and the sync loop back out of step on the legacy no-map peer.
// `masterSwitch: false` — show what the user has CONFIGURED, not what the
// master switch currently lets through. The UI renders `peer.syncEnabled`
// separately, so masking here would hide the stored selection: every box on a
// `syncEnabled: false` peer would read unchecked, and ticking one would
// silently reactivate every other category still true underneath it.
const { tcAddress: _tcAddress, syncSecret: _syncSecret, ...safePeer } = peer;
return { ...safePeer, auth, hasSyncSecret: Boolean(peer.syncSecret), syncCategories: resolveEffectiveCategories(peer, { masterSwitch: false }) };
}
/**
* The instances this install can direct a CoS task at (#4520): this machine
* plus every peer that can actually RECEIVE the task. Two filters, both of
* which exist to stop a pin from stranding work:
*
* - `instanceId` must be set. A peer that has never completed a probe/connect
* has no addressable federation identity yet, so nothing could ever match a
* pin naming it.
* - The peer must be an enabled full-sync peer. CoS task replication only
* sweeps `fullSync === true && enabled !== false` peers (see
* sharing/peerCosSync.js) — pin a task to any other peer and it never leaves
* this machine, while this machine skips it for being pinned elsewhere, so it
* sits pending forever.
*
* Returns only the id/name/isSelf triple the picker needs; no addresses, no
* credentials, no sync state.
*/
export async function getAssignableInstances() {
const data = await loadData();
const assignable = [];
if (data.self?.instanceId) {
assignable.push({ instanceId: data.self.instanceId, name: data.self.name || 'This instance', isSelf: true });
}
for (const peer of data.peers || []) {
if (!peer.instanceId || peer.fullSync !== true || peer.enabled === false) continue;
assignable.push({ instanceId: peer.instanceId, name: peer.name || peer.address || peer.instanceId, isSelf: false });
}
return assignable;
}
// --- Peer CRUD ---
export async function getPeers() {
const data = await loadData();
return data.peers;
}
function validName(name, fallback) {
if (!name || typeof name !== 'string') return fallback;
if (!name.trim()) return fallback;
return name.trim();
}
// "NaN" and "undefined" are what a missing number or a missing value stringify
/**
* Peer display names, hosts, addresses and instance IDs can identify a machine.
* Keep server logs generic so runtime sync events do not persist that data.
*/
export function peerLogLabel(ordinal = null) {
return Number.isInteger(ordinal) && ordinal >= 0 ? `peer ${ordinal + 1}` : 'peer';
}
function isIPAddress(str) {
return net.isIP(str) !== 0;
}
// Returns: null = explicit clear, undefined = invalid input (callers should
// ignore), string = valid lowercased hostname. Three-state distinction lets
// callers choose between "noisy/optional input" (use undefined) vs "user
// asked to clear" (use null).
function validHost(str) {
if (str === '' || str === null) return null;
if (typeof str !== 'string') return undefined;
const trimmed = str.trim();
if (!trimmed) return null;
// Accept DNS names: letters, digits, hyphens, dots. No scheme, no port, no path.
if (!/^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)*$/i.test(trimmed)) {
return undefined;
}
return trimmed.toLowerCase();
}
export { validHost };
// An announce is unauthenticated, so unlike an operator-entered host it must not
// name this machine or a link-local/metadata service (probePeer would GET it).
function announcedHost(str) {
const host = validHost(str);
return host && !isBlockedIngestHost(host) ? host : undefined;
}
// A DNS host that answers loopback/link-local would turn the fixed-path probe
// and relay into an SSRF read. LAN and tailnet answers stay allowed; a lookup
// failure is left to the fetch itself to report as unreachable.
async function assertPeerHostNotLocal(peer) {
if (peer.transport === 'tailcat' || !peer.host || net.isIP(peer.host)) return;
const answers = await dns.lookup(peer.host, { all: true }).catch(() => []);
if (answers.some(({ address }) => isBlockedIngestHost(address))) {
throw new Error(`peer host ${peer.host} resolves to a loopback or link-local address`);
}
}
// Default sync categories. Everything that replicates the user's own CONTENT is
// off until explicitly enabled per-peer; the exception is `usage`, which is
// default-ON (see DEFAULT_ON_SYNC_CATEGORIES below).
const DEFAULT_SYNC_CATEGORIES = {
brain: false,
memory: false,
goals: false,
character: false,
digitalTwin: false,
meatspace: false,
universe: false,
pipeline: false,
mediaCollections: false,
videoHistory: false,
storyBuilder: false,
// ON by default — the only category that is. Rationale + the rules that
// follow from it: docs/decisions/2026-09-01-federated-usage-metrics.md.
usage: true,
fableLoom: false,
authors: false,
artists: false,
albums: false,
tracks: false,
creativeDirectorProjects: false,
moodBoards: false,
writersRoomWorks: false,
writersRoomFolders: false,
writersRoomExercises: false,
musicVideoProjects: false,
commissionFeedback: false,
creativeCommissions: false,
decks: false,
storyboards: false,
catalog: false
};
export { DEFAULT_SYNC_CATEGORIES };
// How this install obtains the image/video BYTES behind a synced Media
// Collection from a peer: `copy` downloads them into local `data/` (default —
// the historical behavior), `host` keeps only the references and streams each
// file from the peer on demand (see services/peerHostedMedia.js).
export const MEDIA_SYNC_MODES = Object.freeze(['copy', 'host']);
export const DEFAULT_MEDIA_SYNC_MODE = 'copy';
export const peerMediaSyncMode = (peer) => (MEDIA_SYNC_MODES.includes(peer?.mediaSyncMode) ? peer.mediaSyncMode : DEFAULT_MEDIA_SYNC_MODE);
// Categories that stay on for a peer even when the user's master `syncEnabled`
// switch is off — that switch predates them and means "don't replicate my
// content to this peer", so it must not silently retract a default-ON one.
// Turning one off is an explicit per-category act; disabling the PEER
// (`enabled: false`) still stops everything.
export const DEFAULT_ON_SYNC_CATEGORIES = Object.freeze(
Object.entries(DEFAULT_SYNC_CATEGORIES).filter(([, on]) => on).map(([key]) => key)
);
const DEFAULT_ON_SET = new Set(DEFAULT_ON_SYNC_CATEGORIES);
/** Is this category one that ships ON? The single definition of that policy. */
export const isDefaultOnCategory = (key) => DEFAULT_ON_SET.has(key);
// Does this category map enable anything the user actually opted into?
//
// `syncEnabled` is derived from this, and it is NOT a cosmetic flag: it also
// gates per-record OUTBOUND pushes (`peerAllowsOutbound`). A default-ON category
// must therefore never flip it on by itself, or adding one would silently widen
// what every existing peer is allowed to receive.
function hasOptedInCategory(categories) {
return Object.entries(categories || {}).some(([key, on]) => on === true && !isDefaultOnCategory(key));
}
/**
* Did the USER establish this peer relationship from this machine? Mirrors
* `peerAllowsOutbound`'s reading of `directions`: an absent/empty array is a
* legacy record (permissive), `['outbound']` is a peer the user added here, and
* `['inbound']` is one that merely ANNOUNCED itself to us and was auto-created
* by `handleAnnounce` — never approved by anyone.
*/
function peerUserEstablished(peer) {
const directions = Array.isArray(peer?.directions) ? peer.directions : [];
return directions.length === 0 || directions.includes('outbound');
}
/**
* The effective sync-category map for a peer — the single resolution of "what
* syncs with this peer", used by BOTH the sync loop (syncOrchestrator) and the
* client-facing peer payload (sanitizePeerForClient). Two resolvers would drift,
* and the UI would then show something the loop doesn't do.
*
* The rules, in order:
* - A full-sync ("mirror everything") peer implies every current AND future
* category on, whatever its stored map says.
* - The shipped defaults sit UNDER the stored map, so a category added after
* this peer record was written picks up its default instead of reading as
* `undefined` (= off) forever. No migration needed; a key the user actually
* toggled is present in the stored map and always wins.
* - **A default-ON category defaults on only for a peer the user established.**
* `handleAnnounce` auto-creates an inbound-only record for any host that can
* reach the port, with no approval step — so letting a default reach those
* would start pulling an unknown machine's data into this install the moment
* it announced itself. An explicit stored `true` still wins there: if the
* user ticked the box for an inbound peer, that is a decision, not a default.
* - `masterSwitch` (default true) applies `syncEnabled: false` — the user's
* "don't replicate my content to this peer" switch — by masking the result
* down to the default-ON categories rather than silencing the peer outright.
* Pass `false` to read what the user has CONFIGURED, independent of that
* switch (what the settings UI renders). Disabling the PEER
* (`enabled: false`) always stops everything, in either mode.
*/
export function resolveEffectiveCategories(peer, { masterSwitch = true } = {}) {
if (peer?.fullSync === true) return allSyncCategoriesOn();
const defaults = { ...DEFAULT_SYNC_CATEGORIES };
if (!peerUserEstablished(peer)) {
for (const key of DEFAULT_ON_SYNC_CATEGORIES) defaults[key] = false;
}
const storedRaw = peer?.syncCategories
? { ...defaults, ...peer.syncCategories }
// Legacy fallback: a peer with no stored map but sync on gets brain+memory,
// the pre-per-category behavior.
: { ...defaults, ...(peer?.syncEnabled !== false ? { brain: true, memory: true } : {}) };
// Video-history rows are the metadata half of Media Collections: a synced
// collection's `{ kind:'video' }` items only render once the matching row
// exists. There is no separate user-facing toggle, so `mediaCollections`
// implies `videoHistory`; a legacy peer that only had `videoHistory` on keeps
// it (OR, never AND), so upgrading stops nothing that was flowing.
const stored = storedRaw.mediaCollections ? { ...storedRaw, videoHistory: true } : storedRaw;
if (!masterSwitch || peer?.syncEnabled !== false) return stored;
return Object.fromEntries(DEFAULT_ON_SYNC_CATEGORIES.map((key) => [key, stored[key] === true]));
}
// A category map with every key forced on — the effective view a full-sync
// ("mirror everything") peer presents. Derived from DEFAULT_SYNC_CATEGORIES so a
// newly added category is covered by full-sync peers with no per-peer change.
// Used by the snapshot orchestrator (what a full-sync peer pulls) and the
// reciprocation send (the all-on map a full-sync peer asks its mirror to adopt).
export function allSyncCategoriesOn() {
return Object.fromEntries(Object.keys(DEFAULT_SYNC_CATEGORIES).map((k) => [k, true]));
}
// Sync categories whose records ride the PER-RECORD push pipeline (not the 60s
// snapshot loop), paired with the record kind autoSubscribePeerToAllRecords
// backfills. A false→true toggle on any of these triggers an inline backfill of
// existing local records so the user doesn't have to wait for the next
// `peer:online` / manual sync-now. Mirror of peerSync's KIND_TO_CATEGORY
// (inverted). `pipeline → series` bundles child issues at push time; authors,
// music records, and mediaCollections are standalone per-record kinds with no
// snapshot category, so toggle-time backfill is the main path that subscribes
// existing records short of a reconnect.
const PER_RECORD_CATEGORY_KINDS = Object.freeze([
['universe', 'universe'],
['pipeline', 'series'],
['mediaCollections', 'mediaCollection'],
['authors', 'author'],
['artists', 'artist'],
['albums', 'album'],
['tracks', 'track'],
['creativeDirectorProjects', 'creativeDirectorProject'],
['moodBoards', 'moodBoard'],
['fableLoom', 'fableLoom'],
['writersRoomWorks', 'writersRoomWork'],
['writersRoomFolders', 'writersRoomFolder'],
['writersRoomExercises', 'writersRoomExercise'],
['musicVideoProjects', 'musicVideoProject'],
['commissionFeedback', 'commissionFeedback'],
['creativeCommissions', 'creativeCommission'],
['decks', 'deck'],
['storyboards', 'storyboard'],
]);
export async function addPeer({ address, port = DEFAULT_PEER_PORT, name, host, auth, transport, protocol = 'http' }) {
const peer = await withData(async (data) => {
const normalizedHost = validHost(host);
const normalizedAuth = sanitizePeerAuth(auth);
const entry = {
id: crypto.randomUUID(),
address,
host: normalizedHost || null,
// Optional HTTP Basic credential for peers gated behind an auth proxy.
// null when unset; `peerAuthHeaders` no-ops on it. Stays local — never
// synced or announced (see redactPeerForWire).
auth: normalizedAuth || null,
// Set to true once the user explicitly chooses a host (set/clear via UI).
// Once true, handleAnnounce never auto-overwrites — it's the only way to
// honor "the user explicitly cleared this; stay on IP" against a peer
// that keeps announcing its DNS name.
hostManual: transport === 'tailcat' || !!normalizedHost,
port,
name: validName(name, normalizedHost || address),
// Optional transport marker (e.g. 'tailcat'). Never carries the tc address —
// that capability lives only in data/tailcat-forwards.json.
...(transport === 'tailcat' ? { transport: 'tailcat', protocol: protocol === 'https' ? 'https' : 'http' } : {}),
instanceId: null,
addedAt: new Date().toISOString(),
lastSeen: null,
lastHealth: null,
status: 'unknown',
enabled: true,
// Full-sync ("mirror everything") mode. When true, every current and
// future sync category is implied on for this peer and a back-subscribe
// sweep keeps all subscribable records mirrored. New peers inherit the
// self-side default so a user who runs full-mirror node pairs doesn't have
// to flip it on every peer by hand.
fullSync: data.self?.defaultPeerFullSync === true,
syncEnabled: data.self?.defaultPeerFullSync === true,
syncCategories: { ...DEFAULT_SYNC_CATEGORIES },
// Consumer-side routing is explicit and machine-local. Enabling a peer
// here never changes that peer's provider settings.
mediaProvider: { enabled: false, audioModels: [], imageModels: [], videoModels: [] },
consecutiveFailures: 0,
nextProbeAt: null,
directions: ['outbound']
};
data.peers.push(entry);
console.log(`🌐 Peer added: ${entry.name} (${peerBaseUrl(entry)})`);
instanceEvents.emit('peers:updated', data.peers);
return entry;
});
void announceSelf(peer);
return peer;
}
/**
* Repoint a tailcat peer at a new loopback port.
*
* The public updatePeer deliberately owns no `port` field — a peer's address is
* how the operator reached it, not something the server rewrites. A managed
* tailcat forward is the exception: PortOS chose that loopback port itself, so
* when a retry has to bind a different one (the old port got taken while the
* forward was down) the peer record has to follow or every request keeps dialing
* a dead port. Restricted to `transport: 'tailcat'` for exactly that reason.
*/
export async function setTailcatPeerPort(id, port) {
if (!Number.isInteger(port) || port < 1 || port > 65535) return null;
const updated = await withData(async (data) => {
const peer = data.peers.find(p => p.id === id && p.transport === 'tailcat');
if (!peer || peer.port === port) return peer || null;
peer.port = port;
console.log(`🐈 Tailcat peer ${peer.name} repointed to 127.0.0.1:${port}`);
instanceEvents.emit('peers:updated', data.peers);
return peer;
});
// The relay pins the peer URL at connect time, so it has to redial the new port.
if (updated) disconnectFromPeer(id);
return updated;
}
export async function removePeer(id, { stopTransport = true } = {}) {
disconnectFromPeer(id);
// Retire the transport outside the data lock: its lifecycle may itself need
// that lock when rolling back a failed add. Keep the peer on retirement errors.
if (stopTransport && (await getPeers()).some(peer => peer.id === id && peer.transport === 'tailcat')) {
const { stopForwardForPeer } = await import('./tailcatPeer.js');
await stopForwardForPeer(id);
}
const removed = await withData(async (data) => {
const idx = data.peers.findIndex(p => p.id === id);
if (idx === -1) return null;
const [entry] = data.peers.splice(idx, 1);
console.log(`🌐 Peer removed: ${entry.name}`);
instanceEvents.emit('peers:updated', data.peers);
return entry;
});
// Retire that instance's federated usage digest. Without this the row stays in
// the fleet report forever: our own snapshot forwards every digest we hold, so
// a surviving peer would hand it straight back on the next cycle. Dynamic
// import keeps peerUsage (and its usage.js graph) off this module's load path.
if (removed?.instanceId) {
const { forgetInstanceUsage } = await import('./peerUsage.js');
await forgetInstanceUsage(removed.instanceId)
.catch((err) => console.log(`⚠️ instances: retiring usage digest failed: ${err.message}`));
}
return removed;
}
export async function updatePeer(id, updates, { probe = true } = {}) {
let hostChanged = false;
// Credential edits must reconnect the live socket relay (it pins the
// Basic-auth header into extraHeaders at connect time), not just the next
// probe cycle — otherwise an online relayed peer keeps the stale credential
// until a natural reconnect. Tracked like hostChanged, consumed below.
let authChanged = false;
// Track false→true transitions for the per-record-subscribable categories
// so we can backfill-subscribe existing local records after the data write
// settles. Set inside withData (where we have the merged before/after
// peer object) and consumed after the lock releases.
const turnedOnKinds = [];
let backfillPeerInstanceId = null;
// Set when a syncCategories change should be mirrored back to the peer so the
// sync is bidirectional (the user owns all machines). The actual send reads
// the freshest persisted map at send time via the serialized queue below.
let reciprocate = false;
const result = await withData(async (data) => {
const peer = data.peers.find(p => p.id === id);
if (!peer) return null;
if (updates.syncSecret !== undefined) {
if (updates.syncSecret !== null && (typeof updates.syncSecret !== 'string' || updates.syncSecret.length < 32 || updates.syncSecret.length > 256)) {
throw new Error('Peer sync secret must contain 32 to 256 characters');
}
if (updates.syncSecret !== peer.syncSecret) {
peer.syncSecret = updates.syncSecret;
// The pair token is derived from this secret, so the receiver's earlier
// confirmation no longer applies: present Basic again (when stored)
// until the next probe re-confirms, and reconnect the relay with the
// new headers (#8356).
peer.peerAuthAccepted = false;
authChanged = true;
}
// Entering the pair secret is the local inbound-admission action.
if (peer.syncSecret && !peer.directions?.includes('inbound')) {
const directions = Array.isArray(peer.directions) && peer.directions.length ? peer.directions : ['outbound'];
peer.directions = [...directions, 'inbound'];
}
}
if (updates.name !== undefined) peer.name = validName(updates.name, peer.name);
if (updates.mediaSyncMode !== undefined && MEDIA_SYNC_MODES.includes(updates.mediaSyncMode)) {
peer.mediaSyncMode = updates.mediaSyncMode;
}
if (updates.enabled !== undefined) peer.enabled = updates.enabled;
if (updates.syncEnabled !== undefined) peer.syncEnabled = updates.syncEnabled;
// Full-sync ("mirror everything") toggle. A false→true flip implies every
// subscribable category, so back-subscribe ALL record kinds (not just the
// ones whose individual syncCategories bit flipped) and reciprocate so the
// peer mirrors us back. Enabling full-sync also implies the peer is sync-
// enabled — otherwise peerAllowsOutbound would gate every push off.
if (updates.fullSync !== undefined && updates.fullSync !== peer.fullSync) {
peer.fullSync = updates.fullSync === true;
if (peer.fullSync) {
peer.syncEnabled = true;
for (const [, kind] of PER_RECORD_CATEGORY_KINDS) turnedOnKinds.push(kind);
backfillPeerInstanceId = peer.instanceId || null;
} else {
// Turning full mirror off restores the per-category selection preserved
// underneath — recompute syncEnabled from it so a peer with no remaining
// enabled categories doesn't read as "sync on, nothing to sync".
peer.syncEnabled = hasOptedInCategory(peer.syncCategories);
}
// Reciprocate the full-sync intent (and its all-on category view) to the
// peer whenever we know its identity, so a node pair converges to a mutual
// mirror in one round-trip.
if (peer.instanceId) reciprocate = true;
}
if (updates.syncCategories !== undefined) {
const prev = peer.syncCategories || DEFAULT_SYNC_CATEGORIES;
const incoming = updates.syncCategories;
// Detect false→true flips for kinds the per-record push pipeline owns
// (PER_RECORD_CATEGORY_KINDS). enabled + outbound-allowed gating is
// enforced inside peerSync.autoSubscribePeerToAllRecords.
for (const [cat, kind] of PER_RECORD_CATEGORY_KINDS) {
if (prev[cat] !== true && incoming[cat] === true) turnedOnKinds.push(kind);
}
peer.syncCategories = { ...prev, ...incoming };
// `syncEnabled` is DERIVED from the category map — it gates per-record
// outbound pushes (peerAllowsOutbound), so it must reflect what the user
// opted into, not what a caller computed. Recomputing here (rather than
// honoring `updates.syncEnabled` on this path) is what keeps a default-ON
// category from widening outbound consent: a client that sums the whole
// map would see `usage: true` and send `syncEnabled: true`.
peer.syncEnabled = hasOptedInCategory(peer.syncCategories);
if (turnedOnKinds.length > 0) backfillPeerInstanceId = peer.instanceId || null;
// Reciprocate so the peer enables the same categories toward us. Only when
// we know the peer's identity (post-handshake).
if (peer.instanceId) reciprocate = true;
}
// Optional proxy credential. `null` (or both fields blank) clears it; a
// valid object replaces it; malformed input (sanitizePeerAuth → undefined)
// is ignored so a stray payload can't wipe a working credential.
if (updates.auth !== undefined) {
const normalizedAuth = sanitizePeerAuth(updates.auth);
if (normalizedAuth !== undefined) {
if (!sameAuth(peer.auth, normalizedAuth)) {
peer.auth = normalizedAuth; // null clears, object sets
authChanged = true;
console.log(`🌐 Peer credential ${peer.auth ? 'set' : 'cleared'}: ${peer.name}`);
}
// When sameAuth was true (credential unchanged), authChanged is still
// false. If the peer is stuck in authRequired+backoff the user is
// explicitly asking for a retry — clear backoff so the immediate probe
// below fires instead of waiting up to 1h.
if (!authChanged && normalizedAuth !== null && peer.authRequired) {
peer.consecutiveFailures = 0;
peer.nextProbeAt = null;
peer.authRequired = false;
authChanged = true; // ensure probe fires
}
}
}
if (updates.mediaProvider !== undefined
&& updates.mediaProvider && typeof updates.mediaProvider === 'object'
&& !Array.isArray(updates.mediaProvider)) {
const previous = peer.mediaProvider;
const next = mergePeerMediaProviderConfig(previous, updates.mediaProvider);
if (canonicalStringify(previous ?? null) !== canonicalStringify(next)) {
peer.mediaProvider = next;
if (!next.enabled) delete peer.mediaProviderStatus;
console.log(`🌐 Remote media provider ${next.enabled ? 'enabled' : 'disabled'}: ${peer.name}`);
}
}
if (peer.transport !== 'tailcat' && updates.host !== undefined) {
const normalized = validHost(updates.host);
if (normalized !== undefined && normalized !== peer.host) {
peer.host = normalized; // null clears, string sets
// Latch manual mode so handleAnnounce stops auto-learning (esp.
// important for clears — without this the next inbound announce
// re-adopts the DNS name and the user can't revert to IP).
peer.hostManual = true;
hostChanged = true;
console.log(`🌐 Peer host ${peer.host ? `set to ${peer.host}` : 'cleared'}: ${peer.name}`);
}
}
// Per-(peer, category) schema-version gaps, populated by syncOrchestrator
// when a remote snapshot is rejected because the sender's schemaVersions
// are ahead of local. Stored on the peer record so the Instances UI's
// SchemaGapBadge can read it via the standard peers payload. Accept
// either a plain object (set/replace the map) or null (clear all gaps).
// Any other value is silently ignored.
if (updates.schemaGaps !== undefined) {
if (updates.schemaGaps === null) {
delete peer.schemaGaps;
} else if (updates.schemaGaps && typeof updates.schemaGaps === 'object' && !Array.isArray(updates.schemaGaps)) {
peer.schemaGaps = updates.schemaGaps;
}
}
instanceEvents.emit('peers:updated', data.peers);
return peer;
});
// Tear down the socket relay only after a real state transition so it can
// reconnect using the new URL on the next probe cycle. Invalid/no-op host
// writes no longer disrupt an already-healthy connection.
if (updates.enabled === false || hostChanged || authChanged) disconnectFromPeer(id);
// A credential edit is almost always the fix for a 401, but the peer may be
// deep in exponential probe backoff (up to 1h) from those failures — so the
// regular poll loop would skip it for hours. Retry immediately: probePeer
// bypasses the nextProbeAt gate and, on success, resets backoff + reconnects
// the relay with the new credential. Fire-and-forget; failure just re-arms
// backoff as before.
if (probe && authChanged && result && result.enabled !== false) {
probePeer(result).catch((err) => console.log(`⚠️ Probe after credential change failed: ${err.message}`));
}
// Backfill-subscribe every local record of any kind whose category just
// flipped on. Fire-and-forget — `autoSubscribePeerToAllRecords` is
// idempotent + per-record-error tolerant, and we don't want to block the
// PATCH response on a slow peer's initial-push round-trip. Goes through the
// recordEvents subscription adapter (peerSync statically imports getPeers
// from us, so importing it back would close a cycle). Per-kind catch so a
// transient failure in one kind's backfill (e.g. universe) doesn't abort
// the loop and leave the peer with no series subscriptions either; the next
// category-toggle PATCH or peer-online event re-fires any kind that didn't
// land.
if (turnedOnKinds.length > 0 && backfillPeerInstanceId) {
(async () => {
for (const kind of turnedOnKinds) {
await autoSubscribePeerToAllRecords(backfillPeerInstanceId, kind).catch((err) => {
console.log(`⚠️ peer: backfill-subscribe ${kind} after category toggle failed: ${err.message}`);
});
}
})();
}
// Mirror the category change back to the peer so sync is bidirectional.
// Fire-and-forget, but SERIALIZED per peer (see enqueueReciprocalSync): two
// rapid toggles must not race — an earlier send arriving after a later one
// would push a stale full map and undo the newer change on the peer.
if (reciprocate && result?.instanceId) {
enqueueReciprocalSync(result.id);
}
return result;
}
/**
* Generate and provision the same pair secret on both ends of a peer link.
* The peer's saved instance password is used only for this one setup request;
* subsequent federation requests use the scoped pair token.
*/
const peerPairingQueue = createKeyCachedQueue();
export function pairPeerSyncSecret(id) {
return peerPairingQueue(id, () => pairPeerSyncSecretNow(id));
}
async function pairPeerSyncSecretNow(id) {
const peer = (await getPeers()).find((item) => item.id === id);
if (!peer) throw new ServerError('Peer not found', { status: 404 });
if (!peer.instanceId || peer.instanceId === UNKNOWN_INSTANCE_ID) {
throw new ServerError('Probe or connect this peer before pairing sync.', { status: 409, code: 'PEER_IDENTITY_REQUIRED' });
}
if (typeof peer.auth?.password !== 'string' || !peer.auth.password) {
throw new ServerError('Save this peer’s instance password before pairing automatically.', {
status: 400, code: 'PEER_PAIR_CREDENTIAL_REQUIRED',
});
}
const selfInstanceId = await getInstanceId();
if (!selfInstanceId || selfInstanceId === UNKNOWN_INSTANCE_ID) {
throw new ServerError('This instance identity is not ready for peer pairing.', { status: 409, code: 'INSTANCE_IDENTITY_REQUIRED' });
}
const syncSecret = typeof peer.syncSecret === 'string' && peer.syncSecret.length >= 32 && peer.peerAuthAccepted !== true
? peer.syncSecret
: crypto.randomBytes(32).toString('base64url');
const headers = {
'Content-Type': 'application/json',
// Force this one-time bootstrap to use the saved instance password, even
// if a stale pair token is cached locally. The receiver rejects peer-token
// authority on this operator-scoped setup route.
[PEER_AUTH_HEADER]: '',
...peerAuthHeaders(peer),
};
let response;
try {
response = await withAbortTimeout(PROBE_TIMEOUT_MS, (signal) => peerFetch(
`${peerBaseUrl(peer)}/api/instances/peers/pair-secret`,
{
method: 'POST',
headers,
body: JSON.stringify({ syncSecret }),
redirect: 'error',
signal,
},
peer,
));
} catch {
throw new ServerError('Couldn’t reach this peer to pair the sync secret. Retry when it is online.', {
status: 502, code: 'PEER_PAIR_UNREACHABLE',
});
}
if (!response.ok) {
const message = response.status === 401
? 'The peer rejected its saved instance password. Update the peer credential, then retry.'
: response.status === 404
? 'The peer could not pair this instance. Upgrade both peers and make sure they are connected.'
: response.status === 403
? 'The peer requires its instance password for automatic pairing. Save that password on this peer, then retry.'
: `The peer could not pair the sync secret (HTTP ${response.status}).`;
throw new ServerError(message, { status: 502, code: 'PEER_PAIR_REJECTED' });
}
const body = await response.json().catch(() => null);
if (body?.paired !== true) {
throw new ServerError('The peer did not confirm sync-secret pairing.', { status: 502, code: 'PEER_PAIR_UNCONFIRMED' });
}
const updated = await updatePeer(id, { syncSecret }, { probe: false });
if (!updated) throw new ServerError('Peer not found', { status: 404 });
if (updated.enabled !== false) return (await probePeer(updated).catch(() => null)) || updated;
return updated;
}
/**
* Set (gap object) or clear (null) one category's schema gap on the peer with
* the given remote instanceId. The read-modify-write runs inside the serialized
* withData write so parallel category syncs cannot overwrite each other's gaps
* (updatePeer replaces the whole map). An empty map is stored as no key.
*/
export async function setPeerSchemaGap(instanceId, category, gapOrNull) {
return withData(async (data) => {
const peer = data.peers.find((p) => p.instanceId === instanceId);
if (!peer) return null;
const gaps = peer.schemaGaps && typeof peer.schemaGaps === 'object' && !Array.isArray(peer.schemaGaps) ? { ...peer.schemaGaps } : {};
if (gapOrNull === null) {
if (!(category in gaps)) return peer;
delete gaps[category];
} else {
gaps[category] = gapOrNull;
}
if (Object.keys(gaps).length > 0) peer.schemaGaps = gaps;
else delete peer.schemaGaps;
instanceEvents.emit('peers:updated', data.peers);
return peer;
});
}
/** Save a pair secret only for the already-registered peer named by its identity header. */
export async function acceptPeerSyncSecretFromPeer(instanceId, syncSecret) {
let changed = false;
const peer = await withData(async (data) => {
const matches = data.peers.filter((item) => item.instanceId === instanceId);
if (matches.length !== 1) return null;
const entry = matches[0];
if (entry.syncSecret !== syncSecret) {
entry.syncSecret = syncSecret;
entry.peerAuthAccepted = false;
changed = true;
}
// Pairing also records this machine's choice to accept inbound pushes;
// sync/category flags still decide which records may be written.
if (entry.syncSecret && !entry.directions?.includes('inbound')) {
const directions = Array.isArray(entry.directions) && entry.directions.length ? entry.directions : ['outbound'];
entry.directions = [...directions, 'inbound'];
changed = true;
}
instanceEvents.emit('peers:updated', data.peers);
return entry;
});
if (changed && peer) disconnectFromPeer(peer.id);
return peer;
}
const reciprocalSyncQueue = createKeyCachedQueue();
// Per-peer tail that serializes reciprocal-sync sends. Two rapid category
// toggles for the same peer each want to push the full resulting map; if their
// fire-and-forget requests raced, the earlier (staler) map could land last on
// the peer and undo the newer change. Chaining per peer.id guarantees in-order
// delivery, and each send re-reads the FRESHEST persisted category map (not a
// value captured at enqueue time) so the final send always carries final state.
// Per-peer serialization (silence prior + self-pruning tail) via the shared
// helper — `work` runs on both fulfil and reject of the prior send, so a failed
// send can't break the chain.
export function enqueueReciprocalSync(peerId) {
return reciprocalSyncQueue(peerId, async () => {
// Re-read the peer fresh at send time — the last enqueued send wins with
// the latest persisted syncCategories, regardless of enqueue order.
const data = await loadData();
const peer = data.peers.find(p => p.id === peerId);
if (!peer?.instanceId) return { ok: false, reason: 'no-peer-identity' };
// For a full-sync peer send the all-on category view (its stored
// syncCategories can be all-false underneath — sending that raw would tell
// the peer to DISABLE everything, since reciprocal-sync apply is an
// authoritative overlay; the all-on map also makes a peer too old to
// understand the fullSync flag still mirror every category) plus the
// fullSync flag. A regular peer keeps sending its raw partial map so a
// category it never set stays untouched on the peer (absent ≠ false).
const reciprocalCategories = peer.fullSync === true ? allSyncCategoriesOn() : (peer.syncCategories || {});
return requestReciprocalSync(peer, reciprocalCategories, { fullSync: peer.fullSync === true }).catch((err) => {
console.log(`⚠️ peer: reciprocal sync request failed: ${err.message}`);
return { ok: false, reason: err.message };
});
});
}
// --- Probing ---
// Peers observed online in THIS process. The persisted `status` survives a
// restart, so a peer that was online before the restart would never produce a
// status transition and `peer:online` listeners (push retry, sync) would not
// run for edits stranded by the restart (#11750). The first online sighting
// per process emits regardless of the persisted status; later sightings keep
// the transition-only rule.
const peersSeenOnlineThisProcess = new Set();
/** Test-only: forget which peers were seen online so a test can simulate a fresh boot. */
export function __resetPeersSeenOnlineForTests() {
peersSeenOnlineThisProcess.clear();
}
function isFirstOnlineSighting(peerId) {
if (peersSeenOnlineThisProcess.has(peerId)) return false;
peersSeenOnlineThisProcess.add(peerId);
return true;
}
export async function probePeer(peer) {
const baseUrl = peerBaseUrl(peer);
const previousStatus = peer.status;
let status, lastHealth, lastSeen, remoteInstanceId, remoteVersion, remoteApps, remoteSyncSeqs;
let remoteMediaProviderStatus = null;
// Latches when the peer answers 401/403 — a reachable-but-auth-gated peer, as
// opposed to an unreachable one. The Instances UI reads this to prompt for a
// credential instead of showing a generic offline state.
let authRequired = false;
let probeFailure = null;
const probeStartedAt = Date.now();
// For tailcat peers, read the live forward's tunnelError up front so a probe
// that times out / resets can be classified as tunnel dial vs local refuse vs
// PortOS health/details failure — the three operator-visible cases.
const forwardHint = await lookupTailcatForwardHint(peer);
const tunnelError = forwardHint?.tunnelError || null;
// One shared abort signal spans the instanceId read, all three parallel
// fetches, AND their body reads, so a single PROBE_TIMEOUT_MS bounds the whole
// probe (the instanceId read is inside the budget, as it was before).
await withAbortTimeout(PROBE_TIMEOUT_MS, async (signal) => {
await assertPeerHostNotLocal(peer);
// Pass our own instanceId as `forPeer` so the peer also returns ITS cursor
// into our data (`cursorForYou`) — our push-frontier toward it. Best-effort:
// an unresolved/unknown id just omits the param and we get the inbound-only
// shape (push count renders "unknown" client-side, never a misleading 0).
const ourInstanceId = await getInstanceId().catch(() => null);
const forPeerQs = typeof ourInstanceId === 'string' && ourInstanceId && ourInstanceId !== UNKNOWN_INSTANCE_ID
? `?forPeer=${encodeURIComponent(ourInstanceId)}`
: '';
// Fetch health details, the PM2-backed app probe projection, sync status, and opted-in media capacity in
// parallel under the same bounded probe budget. The first three paths are
// the frozen peer-probe contract documented in docs/API.md: deployed peers
// call them across independently upgraded installs, so remove/rename or
// incompatible response changes require a versioned replacement rather
// than a silent change here. Older/unconfigured peers incur no fourth
// request.
const [healthRes, appsRes, syncRes, mediaProviderStatus] = await Promise.all([
peerFetch(`${baseUrl}/api/system/health/details`, { signal }, peer),
peerFetch(`${baseUrl}/api/apps?view=probe`, { signal }, peer).catch(() => null),
peerFetch(`${baseUrl}/api/instances/sync-status${forPeerQs}`, { signal }, peer).catch(() => null),
normalizePeerMediaProviderConfig(peer).enabled
? probeFederatedMediaProvider(peer, { signal })
: Promise.resolve(null),
]);
remoteMediaProviderStatus = mediaProviderStatus;
if (!healthRes.ok) {
const err = new Error(`HTTP ${healthRes.status}`);
err.httpStatus = healthRes.status;
throw err;
}
const json = await healthRes.json();
status = 'online';
lastHealth = json;
lastSeen = new Date().toISOString();
remoteInstanceId = json.instanceId ?? null;
remoteVersion = json.version ?? null;
if (appsRes?.ok) {
const appsJson = await appsRes.json().catch(() => null);
const appsList = Array.isArray(appsJson) ? appsJson : appsJson?.apps;
remoteApps = appsList?.map(a => ({
id: a.id, name: a.name, icon: a.icon,
overallStatus: a.overallStatus, uiPort: a.uiPort, apiPort: a.apiPort, type: a.type
})) ?? null;
}
if (syncRes?.ok) {
remoteSyncSeqs = await syncRes.json().catch(() => null);
}
}).catch((err) => {
probeFailure = err;
status = 'offline';
authRequired = err?.httpStatus === 401 || err?.httpStatus === 403;
lastHealth = peer.lastHealth; // preserve last known
lastSeen = peer.lastSeen;
if (normalizePeerMediaProviderConfig(peer).enabled && !remoteMediaProviderStatus) {
remoteMediaProviderStatus = {
checkedAt: new Date().toISOString(),
state: 'unreachable',
reason: 'peer-offline',
freshUntil: null,
snapshot: null,
};
}
});
const latencyMs = Date.now() - probeStartedAt;
const probedAt = new Date().toISOString();
const classification = probeFailure
? classifyPeerProbeFailure(probeFailure, {
peer,
tunnelError,
probeTimeoutMs: PROBE_TIMEOUT_MS,
})
: { class: 'ok', message: null, httpStatus: 200 };
const lastProbe = buildLastProbeRecord({
ok: status === 'online',
classification,
latencyMs,
at: probedAt,
});
if (status !== 'online') {
console.log(`⚠️ Probe failed for ${baseUrl}: ${classifyProbeError(probeFailure, peer, tunnelError)} ${formatProbeDiagnosticLog(lastProbe)}`);
}
const stored = await withData(async (data) => {
const entry = data.peers.find(p => p.id === peer.id);
if (!entry) return null;
entry.status = status;
entry.lastSeen = lastSeen;
entry.lastHealth = lastHealth;
entry.lastProbe = lastProbe;
// Surface "reachable but needs a credential" distinctly from plain offline.
// Cleared on any successful probe (including after the user adds the password).
entry.authRequired = authRequired;
// Peer credential handshake (#8356). Latch on the receiver's confirmation
// that this probe's pair token verified; after that peerFetch stops sending
// the stored Basic password. A 401/403 unlatches, so a receiver that loses
// its side of the pairing (or is downgraded) gets Basic again next probe.
// Other failures (timeouts, offline) keep the last answer.
if (status === 'online') entry.peerAuthAccepted = lastHealth?.peerAuth?.accepted === true;
else if (authRequired) entry.peerAuthAccepted = false;
entry.lastApps = remoteApps ?? entry.lastApps ?? null;
entry.remoteSyncSeqs = remoteSyncSeqs ?? entry.remoteSyncSeqs ?? null;
if (normalizePeerMediaProviderConfig(entry).enabled) {
if (remoteMediaProviderStatus) entry.mediaProviderStatus = remoteMediaProviderStatus;
} else {
delete entry.mediaProviderStatus;
}
if (remoteInstanceId && (!entry.syncSecret || !entry.instanceId)) entry.instanceId = remoteInstanceId;
if (status === 'online') entry.version = remoteVersion;
// Auto-update name from hostname if current name is just an IP address
const remoteHostname = validName(lastHealth?.hostname, null);
if (remoteHostname && isIPAddress(entry.name)) {
entry.name = remoteHostname;
}
// Backoff tracking for failed probes
if (status === 'online') {
if (entry.consecutiveFailures > 0) {
console.log(`🌐 Peer ${entry.name} recovered after ${entry.consecutiveFailures} consecutive failures`);
}
entry.consecutiveFailures = 0;
entry.nextProbeAt = null;
} else {
entry.consecutiveFailures = (entry.consecutiveFailures ?? 0) + 1;
const tier = Math.min(entry.consecutiveFailures - 1, BACKOFF_TIERS_MS.length - 1);
const backoffMs = BACKOFF_TIERS_MS[tier];
entry.nextProbeAt = new Date(Date.now() + backoffMs).toISOString();
console.log(`⏳ Peer ${entry.name} backoff tier ${tier} (${backoffMs / 1000}s), failures: ${entry.consecutiveFailures}`);
}
return entry;
});
// Manage peer socket relay based on status
if (status === 'online') {
connectToPeer(peer);
} else {
disconnectFromPeer(peer.id);
}
// Announce ourselves only when peer transitions to online (not every poll cycle)
if (status === 'online') {
const firstSighting = isFirstOnlineSighting(peer.id);
const transitioned = previousStatus !== 'online';
if (stored) {
if (transitioned) void announceSelf(stored);
if (transitioned || firstSighting) instanceEvents.emit('peer:online', stored);
}
}
return stored;
}
export async function probeAllPeers() {
const data = await loadData();
const now = Date.now();
const enabled = data.peers.filter(p => {
if (!p.enabled) return false;
// Respect backoff: skip peers whose next probe time hasn't arrived
if (p.nextProbeAt && new Date(p.nextProbeAt).getTime() > now) return false;
return true;
});
if (enabled.length === 0) return;
await Promise.allSettled(enabled.map(p => probePeer(p)));
// Re-read to get updated state and emit
const updated = await loadData();
instanceEvents.emit('peers:updated', updated.peers);
}
// --- Query Proxy ---
const PEER_PLACEHOLDER_ORIGIN = 'http://peer.invalid';
// Resolve a caller-supplied proxy path to "pathname + query" that is guaranteed
// to stay on the peer origin and on the documented federation surface
// (PEER_API_SURFACE), or null. The allowlist runs on the path exactly as it will
// be sent, so a segment a URL parser would normalize away (`..`, tabs, `\`) is
// refused rather than argued past.
function resolvePeerQueryTarget(apiPath) {
if (typeof apiPath !== 'string' || !apiPath.startsWith('/') || /[\s\\#]/.test(apiPath)) return null;
const queryAt = apiPath.indexOf('?');
const rawPath = queryAt === -1 ? apiPath : apiPath.slice(0, queryAt);
const search = queryAt === -1 ? '' : apiPath.slice(queryAt);
let parsed;
try {
parsed = new URL(rawPath, PEER_PLACEHOLDER_ORIGIN);
} catch {
return null;
}
if (parsed.origin !== PEER_PLACEHOLDER_ORIGIN || parsed.username || parsed.password) return null;
if (parsed.pathname !== rawPath) return null;
const lower = rawPath.toLowerCase();
// The `/api/peer-sync/` read prefix also spans the POST-only push endpoint,
// which carries the sync secret; a GET proxy never has a reason to name it.
if (lower.replace(/\/$/, '') === '/api/peer-sync/push') return null;
if (!isPeerApiRequestAllowed('GET', lower)) return null;
return `${rawPath}${search}`;
}
export async function queryPeer(id, apiPath) {
const data = await loadData();
const peer = data.peers.find(p => p.id === id);
if (!peer) return { error: 'Peer not found' };
const target = resolvePeerQueryTarget(apiPath);
if (!target) return { error: 'Path is not on the peer federation surface', status: 400 };
const url = `${peerBaseUrl(peer)}${target}`;
return withAbortTimeout(PROBE_TIMEOUT_MS, async (signal) => {
const res = await peerFetch(url, { signal }, peer);
return { success: true, data: await res.json() };
}).catch((err) => ({ error: `Failed to query peer: ${err.message}` }));
}
// --- Announce (Bidirectional Registration) ---
export async function handleAnnounce({ address, port, instanceId, name, host }) {
// Set only when this announce marked the peer online (not the forged-address
// branch, which records a sighting without trusting it).
let markedOnlinePeer = null;
let peerToWake = null;
const result = await withData(async (data) => {
// Check for existing peer by instanceId
let existing = data.peers.find(p => p.instanceId === instanceId);
// instanceId is public (GET /api/system/health returns it unauthenticated),
// so a match on instanceId alone does not prove the caller IS that peer —
// only that it knows a public value. Track whether the match instead came
// through the address+port fallback below, where the caller's address is
// already part of the match.
let matchedByAddress = false;
// Fallback: check by address + port
if (!existing) {
existing = data.peers.find(p => p.address === address && p.port === port);
matchedByAddress = true;
}
const instanceIdMatches = !existing || !existing.instanceId || existing.instanceId === instanceId;
if (existing && existing.enabled !== false && instanceIdMatches) {
if (shouldWakePeer(existing.id)) {
if (existing.nextProbeAt) existing.nextProbeAt = null;
peerToWake = existing;
}
}
const normalizedHost = announcedHost(host);
if (existing?.syncSecret) return { created: false, peer: existing };
// A forged announce: instanceId matches a stored peer, but the caller's
// source address does not match the stored one. Record the sighting only
// — never let it teach host/port/name/status, or an attacker who merely
// reads a peer's public instanceId could redirect our future requests
// (and the peer's Basic credential) to a host they control.
// Exempt tailcat: its stored `address` is intentionally the LOCAL forward's
// loopback address, never the remote caller's — a mismatch there is the
// normal case, not a forgery signal, and its host/port are already pinned
// to the local forward by the `transport !== 'tailcat'` guards below.
if (existing && !matchedByAddress && existing.transport !== 'tailcat' && existing.address !== address) {
console.warn(`⚠️ Peer announce instanceId matched but address mismatched (expected ${existing.address}, got ${address}) — ignoring host/port/name update`);
existing.lastSeen = new Date().toISOString();
instanceEvents.emit('peers:updated', data.peers);
return { created: false, peer: existing };
}
if (existing) {
existing.lastSeen = new Date().toISOString();
existing.status = 'online';
existing.nextProbeAt = null;
existing.consecutiveFailures = 0;
markedOnlinePeer = existing;
existing.instanceId = instanceId;
// A managed forward uses local coordinates, not the remote listen port.
if (existing.transport !== 'tailcat') existing.port = port;
// Only auto-update name if still an IP address (preserve user-set names)
const sanitized = validName(name, null);
if (sanitized && isIPAddress(existing.name)) {
existing.name = sanitized;
}
// Adopt host from inbound announce only when we don't already have one
// AND the user hasn't manually intervened. The hostManual flag covers
// the "user explicitly cleared this — stay on IP" case that the
// existing.host check alone can't distinguish from "never set".
if (existing.transport !== 'tailcat' && normalizedHost && !existing.host && !existing.hostManual) {
existing.host = normalizedHost;
console.log(`🌐 Peer host learned via announce: ${existing.name} → ${normalizedHost}`);
}
// Mark that this peer has announced to us (inbound connection)
existing.directions = existing.directions || [];
if (!existing.directions.includes('inbound')) existing.directions.push('inbound');
console.log(`🌐 Peer announced (existing): ${existing.name} (${address}:${port})`);
instanceEvents.emit('peers:updated', data.peers);
return { created: false, peer: existing };
}
// Create new peer entry from remote announcement
const peer = {
id: crypto.randomUUID(),
address,
host: normalizedHost || null,
// A credential can only be entered locally, never learned from an
// announce — start null. If this peer needs auth, our probe to it will
// 401 and the user sets the password from the Instances UI.
auth: null,
// The host came from the peer's announce, not from a user — leave
// hostManual false so subsequent updates from the peer can still refine.
hostManual: false,
port,
name: validName(name, normalizedHost || address),
instanceId,
addedAt: new Date().toISOString(),
lastSeen: new Date().toISOString(),
lastHealth: null,
status: 'online',
enabled: true,
syncEnabled: false,
syncCategories: { ...DEFAULT_SYNC_CATEGORIES },
consecutiveFailures: 0,
nextProbeAt: null,
directions: ['inbound']
};
data.peers.push(peer);
console.log(`🌐 Peer announced (new): ${peer.name} (${peerBaseUrl(peer)})`);
instanceEvents.emit('peers:updated', data.peers);
return { created: true, peer };
});
// The announce path sets status = 'online' without a probe, so it needs the
// same once-per-process convergence trigger as probePeer.
if (markedOnlinePeer && isFirstOnlineSighting(markedOnlinePeer.id)) {
instanceEvents.emit('peer:online', markedOnlinePeer);
}
// Immediately probe newly announced peers to populate health data
if (result.created) {
probePeer(result.peer).catch(err => {
console.log(`⚠️ Initial probe failed for announced peer ${result.peer.name}: ${err.message}`);
});
} else if (peerToWake) {
probePeer(peerToWake).catch(err => {
console.log(`⚠️ Wake probe failed for ${peerToWake.name || peerToWake.id}: ${err.message}`);
});
}
return result;
}
// Owns every rejection: callers in addPeer/probePeer fire it without awaiting.
async function announceSelf(peer) {
let data;
try {
data = await loadData();
} catch (err) {
console.log(`🌐 Announce skipped: instances data unreadable: ${err.message}`);
return;
}
if (!data.self) return;
const selfPort = parseInt(process.env.PORT, 10) || DEFAULT_PEER_PORT;
const selfHost = getSelfHost();
const url = `${peerBaseUrl(peer)}/api/instances/peers/announce`;
await withAbortTimeout(PROBE_TIMEOUT_MS, async (signal) => {
const res = await peerFetch(url, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
port: selfPort,
instanceId: data.self.instanceId,
name: data.self.name,
host: selfHost
}),
signal
}, peer);
if (res.ok) {
console.log(`🌐 Announced self to ${url}`);
await markDirection(peer.id, 'outbound');
} else {
console.log(`🌐 Announce to ${url} failed: HTTP ${res.status}`);
}
}).catch((err) => {
console.log(`🌐 Announce to ${url} unreachable: ${err.message}`);
});
}
export async function connectPeer(id) {
const data = await loadData();
const peer = data.peers.find(p => p.id === id);
if (!peer) return null;
await announceSelf(peer);
const probed = await probePeer(peer);
return probed;
}
// --- Bidirectional sync reciprocation ---
//
// A single user commonly owns every federated machine and expects enabling a
// sync category toward a peer to make it two-way without also toggling it on
// the peer by hand. Sync is pull-based per direction, so enabling category C
// toward peer B only makes US pull B's C. For B to pull OUR C, B must enable C
// toward US. `requestReciprocalSync` asks B to do exactly that; the peer side
// is `applyReciprocalSync`, invoked by the POST /sync-categories route.
//
// SEMANTICS — full mirror, NOT enable-only (resolved, issue #1094). A peer's
// announced keys are applied as an authoritative overlay onto our existing map
// (`{ ...prev, ...sanitized }`) — announced `false` values are authoritative
// (they DISABLE), while categories the peer omits are preserved untouched. So a
// peer sending `C:false` DISABLES C on our record for that peer even when we had
// independently enabled C toward them. This is intentional, not a bug:
// - The 'Make mutual' UI deliberately pushes the sender's *current* set
// (including its disables) so a previously-offline / one-directional peer
// converges to the sender's view in one round-trip.
// - It's the recovery path for clearing a stale enabled category after a peer
// was offline during a disable (the all-false / offline-disable test below).
// Treating sync categories as a symmetric mirror (rather than two strictly
// independent per-direction switches) is the chosen model. If a future change
// wants per-direction-independent semantics, 'Make mutual' must grow a separate
// explicit-disable path so reciprocation never clobbers an independent enable —
// don't quietly switch this to enable-only, which would regress the two paths
// above. The full-mirror behavior is pinned by tests in instances.test.js.
// Keep only the keys DEFAULT_SYNC_CATEGORIES defines, coerced to booleans, so a
// peer can never inject unknown/garbage category flags onto our peer record.
function sanitizeSyncCategories(input) {
if (!input || typeof input !== 'object' || Array.isArray(input)) return null;
const out = {};
for (const key of Object.keys(DEFAULT_SYNC_CATEGORIES)) {
if (typeof input[key] === 'boolean') out[key] = input[key];
}
return Object.keys(out).length > 0 ? out : null;
}
/**
* Receiver side of reciprocation: a peer is telling us it enabled `categories`
* toward us and is asking us to mirror them so the sync is bidirectional.
* Find the local peer record for the announcing instance and apply the same
* category flags. Returns `{ changed, peer }`; `changed` is false when our
* record already matched (the echo guard — the caller never re-reciprocates,
* but this keeps a misbehaving peer from churning our state).
*/
export async function applyReciprocalSync(instanceId, categories, { fullSync } = {}) {
// The admitted peer's direction and categories are machine-local consent.
// Anonymous reciprocal callbacks must never broaden or revoke them.
const sanitized = sanitizeSyncCategories(categories);
// A fullSync-only signal (peer asking us to mirror everything) is valid even
// if the category map didn't sanitize to anything actionable. An explicit
// `fullSync === false` is the peer telling us it STOPPED mirroring us, so we
// drop our mirror toward it too — keeping enable AND disable symmetric (a peer
// too old to send the field leaves it undefined, which touches nothing).
const wantsFullSync = fullSync === true;
const wantsFullSyncOff = fullSync === false;
if (!sanitized && !wantsFullSync && !wantsFullSyncOff) return { changed: false, peer: null };
const turnedOnKinds = [];
let backfillInstanceId = null;
let changed = false;
const peer = await withData(async (data) => {
const entry = data.peers.find(p => p.instanceId === instanceId);
if (!entry || entry.syncSecret) return null;
// Default the baseline so a partial stored map doesn't make absent-vs-false
// diverge — otherwise sanitized adding `goals:false` to a `prev` missing
// `goals` reads as a change (`false !== undefined`) and defeats the guard.
const prev = { ...DEFAULT_SYNC_CATEGORIES, ...(entry.syncCategories || {}) };
const next = sanitized ? { ...prev, ...sanitized } : prev;
const fullSyncFlips = wantsFullSync && entry.fullSync !== true;
const fullSyncOffFlips = wantsFullSyncOff && entry.fullSync === true;
const categoriesFlip = !Object.keys(next).every(k => next[k] === prev[k]);
// Does THIS reciprocal request itself ask us to push per-record data back —
// full mirror, or at least one per-record category carried IN the request?
// Scope the consent decision to what the peer asked for this time (wantsFullSync
// / sanitized), NOT the preserved `next` map: a request that only reciprocates a
// snapshot category (or a per-record kind we merely happen to already have on
// locally) must not silently widen us to push every locally-enabled kind back.
// The request is the consent signal, so it also scopes what that consent covers.
const requestEnablesPushable = wantsFullSync
|| (sanitized ? PER_RECORD_CATEGORY_KINDS.some(([cat]) => sanitized[cat] === true) : false);
const directions = Array.isArray(entry.directions) ? entry.directions : [];
// An `/announce`-created peer record is inbound-only (it announced to us; the
// user here never added it back), and `peerAllowsOutbound` then refuses our
// pushes — so the backfill below, every future per-record push, and reverse-
// subscription creation on incoming pushes would all silently no-op, leaving
// the mirror one-directional. An explicit reciprocal-sync request IS the peer
// asking us to mirror our data back — its consent to receive our pushes — so
// adopt `outbound`. Crucially this must hold even when nothing flips: a record
// that adopted fullSync/categories BEFORE this fix (#1636) but stayed inbound-
// only never back-fills, and the echo guard below would return first — so the
// missing direction has to participate in change detection to heal it.
const needsOutboundAdopt = requestEnablesPushable && directions.length > 0 && !directions.includes('outbound');
// No-op only when nothing flips AND there's no outbound to adopt — the echo guard.
if (!fullSyncFlips && !fullSyncOffFlips && !categoriesFlip && !needsOutboundAdopt) return entry;
if (fullSyncFlips) {
// Adopting full mirror. fullSync alone drives gating, so the stored
// per-category map is PRESERVED untouched — we do NOT apply the sender's
// all-on compat overlay. That keeps the user's own selection intact so a
// later disable restores it, exactly like the local toggle path. (Legacy
// receivers that don't understand fullSync fall into the category branch
// below and mirror via the all-on overlay instead — that's its purpose.)
entry.fullSync = true;
} else if (fullSyncOffFlips) {
// Dropping full mirror: clear the flag. If this SAME reciprocal request
// also carries a changed per-category selection, adopt it too — otherwise
// (the old combined branch fell through without touching categories) the
// peer's intended post-full-sync set would be ignored and the stale
// preserved map would keep mirroring the wrong categories. With no
// category change in the request, the preserved map stands (restore path).
entry.fullSync = false;
if (categoriesFlip) entry.syncCategories = next;
} else if (categoriesFlip) {
entry.syncCategories = next;
}
// Adopt outbound consent from the reciprocal request (see needsOutboundAdopt).
// Inline rather than via markDirection — that helper opens its own withData and
// would deadlock on this same lock; idempotent (a peer already pushing has
// outbound and never reaches here, so directions.push is never a duplicate).
if (needsOutboundAdopt) entry.directions = [...directions, 'outbound'];
// Recompute from the (possibly preserved) stored map — a full-sync peer is
// always sync-enabled; otherwise it follows the per-category selection.
entry.syncEnabled = entry.fullSync === true || hasOptedInCategory(entry.syncCategories);
// Backfill every per-record kind whose push toward this peer just became
// viable: a kind that flipped false→true, all kinds when full mirror was just
// adopted, and — when we just adopted outbound on a previously inbound-only
// record — every kind already enabled (it was on but never pushed while we
// couldn't push outbound). autoSubscribePeerToAllRecords is idempotent, so a
// kind already subscribed is a harmless no-op.
for (const [cat, kind] of PER_RECORD_CATEGORY_KINDS) {
const enabledNow = entry.fullSync === true || !!(entry.syncCategories && entry.syncCategories[cat] === true);
const flippedOn = prev[cat] !== true && next[cat] === true;
// On a heal (outbound just adopted, no flip) only back-fill kinds THIS request
// enabled — full mirror, or the specific per-record category it carried — so a
// heal never pushes a kind the current request didn't authorize.
const requestEnablesKind = wantsFullSync || !!(sanitized && sanitized[cat] === true);
if ((fullSyncFlips && enabledNow) || flippedOn || (needsOutboundAdopt && enabledNow && requestEnablesKind)) {
turnedOnKinds.push(kind);
}
}
if (turnedOnKinds.length > 0) backfillInstanceId = entry.instanceId || null;
changed = true;
instanceEvents.emit('peers:updated', data.peers);
return entry;
});
if (!peer) return { changed: false, peer: null };
// Mirror updatePeer's backfill-subscribe so a reciprocally-enabled
// per-record category (universe/pipeline) pushes our existing records too.
if (changed && turnedOnKinds.length > 0 && backfillInstanceId) {
(async () => {
for (const kind of turnedOnKinds) {
await autoSubscribePeerToAllRecords(backfillInstanceId, kind).catch((err) => {
console.log(`⚠️ peer: reciprocal backfill-subscribe ${kind} failed: ${err.message}`);
});
}
})();
}
if (changed) {
const enabledList = sanitized ? Object.keys(sanitized).filter(k => sanitized[k]).join(',') : '';
console.log(`🔁 Reciprocal sync applied for peer ${peer.name}: ${peer.fullSync ? 'full-mirror' : (enabledList || 'none')}`);
}
return { changed, peer };
}
/**
* Sender side: ask `peer` to enable `categories` toward us (mirror what we just
* enabled toward them). Best-effort and fire-and-forget at the call site — a
* peer that's offline or on an older version (404s the endpoint) just stays
* one-directional until the next toggle. Returns `{ ok, reason }`.
*/
export async function requestReciprocalSync(peer, categories, { fullSync } = {}) {
const self = await getSelf();
if (!self?.instanceId) return { ok: false, reason: 'no-self-identity' };
const sanitized = sanitizeSyncCategories(categories);
// Bail only when there's genuinely nothing to say: no actionable categories
// AND no full-sync state to communicate. A full-sync DISABLE (fullSync:false)
// on a peer whose stored category map is empty must still send — otherwise the
// remote never learns to drop its mirror. `fullSync === undefined` means the
// caller didn't communicate a full-sync state at all (legacy direct callers).
if (!sanitized && fullSync === undefined) return { ok: false, reason: 'no-categories' };
const url = `${peerBaseUrl(peer)}/api/instances/peers/sync-categories`;
return withAbortTimeout(PROBE_TIMEOUT_MS, async (signal) => {
const res = await peerFetch(url, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
// Always carry our current full-sync state (true OR false) so a peer new
// enough to understand it adopts mirror mode on enable AND drops it on
// disable. Older peers ignore the field; on enable they still mirror via
// the all-on syncCategories map we sent. syncCategories defaults to {} so
// the receiver's required-object schema still parses a fullSync-only send.
body: JSON.stringify({ instanceId: self.instanceId, syncCategories: sanitized || {}, fullSync: fullSync === true }),
signal
}, peer);
if (res.ok) {
const enabledList = sanitized ? Object.keys(sanitized).filter(k => sanitized[k]).join(',') : '';
console.log(`🔁 Requested reciprocal sync from ${peer.name}: ${fullSync === false ? 'full-mirror-off' : (enabledList || 'none')}`);
return { ok: true };
}
// 404 = peer predates this endpoint; not an error worth surfacing loudly.
return { ok: false, reason: `http-${res.status}` };
}).catch((err) => ({ ok: false, reason: err.message }));
}
async function markDirection(peerId, direction) {
await withData(async (data) => {
const peer = data.peers.find(p => p.id === peerId);
if (!peer) return;
peer.directions = peer.directions || [];
if (!peer.directions.includes(direction)) {
peer.directions.push(direction);
instanceEvents.emit('peers:updated', data.peers);
}
});
}
// --- Polling ---
// Actually create the probe schedule (backoff clear + initial probe + interval).
// Idempotent: a no-op if the loop is already running.
function beginPolling() {
if (pollTimer) return;
console.log(`🌐 Instance polling started (${POLL_INTERVAL_MS / 1000}s interval)`);
// Backoff is a rate limit on the polling loop, not a durable judgment about
// the peer — boot (or a fresh Tailscale connect) may itself be the event that
// fixes connectivity, so clear it.
withData(async (data) => {
let cleared = 0;
for (const peer of data.peers) {
if (peer.nextProbeAt) {
peer.nextProbeAt = null;
peer.consecutiveFailures = 0;
cleared++;
}
}
if (cleared > 0) console.log(`🌐 Cleared backoff on ${cleared} peer(s) for fresh probe after boot`);
}).catch(err => console.error(`❌ Failed to clear peer backoff on boot: ${err.message}`));
// Initial probe after a short delay. probeAllPeers() awaits fs/network work
// that can reject; setTimeout/setInterval don't await the callback, so catch
// here to avoid a process-killing unhandled rejection.
setTimeout(() => {
probeAllPeers().catch(err => console.error(`❌ Initial peer probe failed: ${err?.message || String(err)}`));
}, INITIAL_PROBE_DELAY_MS);
pollTimer = setInterval(() => {
probeAllPeers().catch(err => console.error(`❌ Peer probe failed: ${err?.message || String(err)}`));
}, POLL_INTERVAL_MS);
}
// A peer we can only reach over the tailnet: its probe URL (see peerBaseUrl)
// resolves to a MagicDNS name (*.ts.net) or a Tailscale CGNAT address
// (100.64.0.0/10 for IPv4, fd7a:115c:a1e0::/48 for IPv6). A peer addressed by a
// plain LAN/routable IP or a non-tailnet DNS host is reachable with Tailscale
// down, so it must NOT be deferred by the gate below.
function peerRequiresTailscale(peer) {
if (peer.host) return /\.ts\.net$/i.test(peer.host.trim());
const addr = (peer.address || '').trim();
const v4 = addr.match(/^100\.(\d{1,3})\./);
if (v4) {
const second = Number(v4[1]);
if (second >= 64 && second <= 127) return true; // 100.64.0.0/10 CGNAT
}
if (/^fd7a:115c:a1e0:/i.test(addr)) return true; // Tailscale IPv6 ULA
return false;
}
// Decide whether to start probing now or defer until Tailscale connects.
// Probing a tailnet-only peer's MagicDNS hostname while Tailscale is down just
// spams DNS-failure + backoff logs (a laptop booted off-tailnet). So we defer
// ONLY when every enabled peer is tailnet-dependent; a LAN/IP-reachable peer is
// probed immediately (it doesn't need Tailscale). While deferred, a slow watcher
// starts polling automatically the moment Tailscale connects. `gen` is captured
// at startPolling time so a stopPolling() during either await aborts cleanly.
async function evaluatePollingGate(gen) {
if (pollTimer || tailscaleWatchTimer) return;
const data = await loadData();
if (gen !== pollingGeneration) return; // stopPolling() landed during loadData
const enabledPeers = Array.isArray(data.peers) ? data.peers.filter(p => p.enabled) : [];
// No peers, or at least one peer reachable without Tailscale → run the loop
// now (silent no-op when there are no peers). Only an all-tailnet peer set
// gets deferred, so solo installs and LAN peers are unaffected.
const needsTailscale = enabledPeers.length > 0 && enabledPeers.every(peerRequiresTailscale);
if (!needsTailscale) {
beginPolling();
return;
}
const status = await getTailscaleStatus();
if (gen !== pollingGeneration) return; // stopPolling() landed during status check
if (status.running) {
beginPolling();
return;
}
console.log(`🌐 Peer sync deferred — Tailscale not connected (${status.state || status.reason}); polling starts automatically once it's up`);
// Capture the handle locally so a stale in-flight callback can't clear/null a
// watcher installed by a later startPolling() (identity guard below).
const myTimer = setInterval(() => {
getTailscaleStatus().then((s) => {
if (!s.running || tailscaleWatchTimer !== myTimer) return;
clearInterval(myTimer);
tailscaleWatchTimer = null;
console.log('🌐 Tailscale connected — starting peer sync polling');
beginPolling();
}).catch(() => {});
}, TAILSCALE_RECHECK_MS);
tailscaleWatchTimer = myTimer;
// Don't let the recheck timer keep the process alive on its own.
if (typeof myTimer.unref === 'function') myTimer.unref();
}
export function startPolling() {
// pollingStartPending guards the async gap in evaluatePollingGate so two
// synchronous startPolling() calls can't each spin up a schedule/watcher.
if (pollTimer || tailscaleWatchTimer || pollingStartPending) return;
pollingStartPending = true;
const gen = pollingGeneration;
evaluatePollingGate(gen)
.catch(err => console.error(`❌ Failed to start peer polling: ${err.message}`))
.finally(() => { pollingStartPending = false; });
}
export function stopPolling() {
// Invalidate any in-flight startPolling gate / deferred watcher callback.
pollingGeneration++;
if (tailscaleWatchTimer) {
clearInterval(tailscaleWatchTimer);
tailscaleWatchTimer = null;
}
if (pollTimer) {
clearInterval(pollTimer);
pollTimer = null;
console.log('🌐 Instance polling stopped');
}
}