Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: converge peers and share buckets after a restart (#11750)
[._.]/ Adam Eivy829879f8d489
diff --git a/server/services/instances.js b/server/services/instances.jsindex 199b206276..7bc8fac261 100644--- a/server/services/instances.js+++ b/server/services/instances.js@@ -871,6 +871,25 @@ export function enqueueReciprocalSync(peerId) {// --- Probing ---+// Peers observed online in THIS process. The persisted `status` survives a+// restart, so a peer that was online before the restart would never produce a+// status transition and `peer:online` listeners (push retry, sync) would not+// run for edits stranded by the restart (#11750). The first online sighting+// per process emits regardless of the persisted status; later sightings keep+// the transition-only rule.+const peersSeenOnlineThisProcess = new Set();++/** Test-only: forget which peers were seen online so a test can simulate a fresh boot. */+export function __resetPeersSeenOnlineForTests() {+ peersSeenOnlineThisProcess.clear();+}++function isFirstOnlineSighting(peerId) {+ if (peersSeenOnlineThisProcess.has(peerId)) return false;+ peersSeenOnlineThisProcess.add(peerId);+ return true;+}+export async function probePeer(peer) {const baseUrl = peerBaseUrl(peer);@@ -1033,10 +1052,12 @@ export async function probePeer(peer) {}// Announce ourselves only when peer transitions to online (not every poll cycle)- if (status === 'online' && previousStatus !== 'online') {+ if (status === 'online') {+ const firstSighting = isFirstOnlineSighting(peer.id);+ const transitioned = previousStatus !== 'online';if (stored) {- announceSelf(stored);- instanceEvents.emit('peer:online', stored);+ if (transitioned) announceSelf(stored);+ if (transitioned || firstSighting) instanceEvents.emit('peer:online', stored);}}@@ -1109,6 +1130,9 @@ export async function queryPeer(id, apiPath) {// --- Announce (Bidirectional Registration) ---export async function handleAnnounce({ address, port, instanceId, name, host }) {+ // Set only when this announce marked the peer online (not the forged-address+ // branch, which records a sighting without trusting it).+ let markedOnlinePeer = null;const result = await withData(async (data) => {// Check for existing peer by instanceIdlet existing = data.peers.find(p => p.instanceId === instanceId);@@ -1147,6 +1171,7 @@ export async function handleAnnounce({ address, port, instanceId, name, host })if (existing) {existing.lastSeen = new Date().toISOString();existing.status = 'online';+ markedOnlinePeer = existing;existing.instanceId = instanceId;// A managed forward uses local coordinates, not the remote listen port.if (existing.transport !== 'tailcat') existing.port = port;@@ -1203,6 +1228,12 @@ export async function handleAnnounce({ address, port, instanceId, name, host })return { created: true, peer };});+ // The announce path sets status = 'online' without a probe, so it needs the+ // same once-per-process convergence trigger as probePeer.+ if (markedOnlinePeer && isFirstOnlineSighting(markedOnlinePeer.id)) {+ instanceEvents.emit('peer:online', markedOnlinePeer);+ }+// Immediately probe newly announced peers to populate health dataif (result.created) {probePeer(result.peer).catch(err => {diff --git a/server/services/instances.test.js b/server/services/instances.test.jsindex ea03535645..3e44c00da2 100644--- a/server/services/instances.test.js+++ b/server/services/instances.test.js@@ -83,12 +83,14 @@ import {enqueueReciprocalSync,DEFAULT_SYNC_CATEGORIES,startPolling,- stopPolling+ stopPolling,+ __resetPeersSeenOnlineForTests} from './instances.js';describe('instances.js', () => {beforeEach(() => {vi.clearAllMocks();+ __resetPeersSeenOnlineForTests();vi.useFakeTimers();vi.stubGlobal('fetch', vi.fn());readJSONFile.mockResolvedValue({ self: null, peers: [] });@@ -1421,20 +1423,20 @@ describe('instances.js', () => {expect(instanceEvents.emit).toHaveBeenCalledWith('peer:online', expect.any(Object));});- it('should not emit peer:online if already online', async () => {+ it('emits peer:online on the first probe after boot for a peer persisted as online, then only on transitions (#11750)', async () => {+ // The persisted status survives a restart, so without the per-process+ // first-sighting rule an edit stranded by the restart is never retried.const peer = makePeer({ status: 'online' });- const peers = [peer];- readJSONFile.mockResolvedValue({ self: null, peers });-+ readJSONFile.mockResolvedValue({ self: null, peers: [peer] });const healthData = { instanceId: 'r-id' };- fetch- .mockResolvedValueOnce({ ok: true, json: () => Promise.resolve(healthData) })- .mockResolvedValueOnce({ ok: false })- .mockResolvedValueOnce({ ok: false });+ fetch.mockImplementation(async () => ({ ok: true, json: () => Promise.resolve(healthData) }));+ const onlineEmits = () => instanceEvents.emit.mock.calls.filter(([name]) => name === 'peer:online').length;await probePeer(peer);+ expect(onlineEmits()).toBe(1);- expect(instanceEvents.emit).not.toHaveBeenCalledWith('peer:online', expect.anything());+ await probePeer(peer);+ expect(onlineEmits()).toBe(1);});it('should return null if peer is removed during probe', async () => {@@ -1565,6 +1567,27 @@ describe('instances.js', () => {});describe('handleAnnounce', () => {+ it('emits peer:online once per process when an announce marks a known peer online (#11750)', async () => {+ const peer = { id: 'peer-local', instanceId: 'peer-a', address: '192.0.2.10', port: 5555,+ name: 'Example peer', status: 'online', directions: ['outbound'] };+ readJSONFile.mockResolvedValue({ self: null, peers: [peer] });+ const onlineEmits = () => instanceEvents.emit.mock.calls.filter(([name]) => name === 'peer:online').length;++ await handleAnnounce({ instanceId: 'peer-a', address: '192.0.2.10', port: 5555 });+ expect(onlineEmits()).toBe(1);++ await handleAnnounce({ instanceId: 'peer-a', address: '192.0.2.10', port: 5555 });+ expect(onlineEmits()).toBe(1);+ });++ it('does not emit peer:online for a forged announce that was not trusted', async () => {+ const peer = { id: 'peer-local', instanceId: 'peer-a', address: '192.0.2.10', port: 5555,+ name: 'Example peer', status: 'online', directions: ['outbound'] };+ readJSONFile.mockResolvedValue({ self: null, peers: [peer] });+ await handleAnnounce({ instanceId: 'peer-a', address: '198.51.100.99', port: 5555 });+ expect(instanceEvents.emit).not.toHaveBeenCalledWith('peer:online', expect.anything());+ });+it('keeps an admitted peer identity, callback endpoint and consent unchanged after forged discovery', async () => {const peer = { id: 'peer-local', instanceId: 'peer-a', address: '192.0.2.10', port: 5555,host: 'configured.example.com', directions: ['inbound'], syncSecret: 'synthetic-pair-secret-32-characters-long',diff --git a/server/services/sharing/peerSync.test.js b/server/services/sharing/peerSync.test.jsindex a91992d2d5..242f882430 100644--- a/server/services/sharing/peerSync.test.js+++ b/server/services/sharing/peerSync.test.js@@ -1425,8 +1425,11 @@ describe('peerSync', () => {// (the lastPushedHash short-circuit inside pushRecordToPeer is what// skips the actual HTTP call for unchanged records). This is the// mechanism that lets out-of-band file edits (e.g., a cleanup script- // that wrote tombstones directly to disk + a server restart)- // re-propagate via peer:online without needing a per-record edit.+ // that wrote tombstones directly to disk) re-propagate via peer:online+ // without needing a per-record edit. A server restart reaches this path+ // too: instances.js emits peer:online on the first probe of each peer per+ // process even when the persisted status was already online (#11750; see+ // the boot-convergence test below).vi.mocked(peerFetch).mockClear();const second = await retryPendingPushesForPeer('peer-a');expect(second.walked).toBe(1); // walked, not skipped by the helper@@ -1436,6 +1439,40 @@ describe('peerSync', () => {expect(vi.mocked(peerFetch)).not.toHaveBeenCalled();});+ it('boot convergence: the first peer:online after a restart pushes an edit stranded by the restart exactly once, a converged peer costs zero HTTP calls (#11750)', async () => {+ const { instanceEvents } = await import('../instanceEvents.js');+ const { installPeerSyncListener } = await import('./peerSync.js');+ installPeerSyncListener();+ const peer = {+ id: 'local-a', instanceId: 'peer-a', enabled: true, syncEnabled: true, status: 'online',+ directions: ['outbound'], syncCategories: { universe: true },+ };+ vi.mocked(listUniverses).mockResolvedValue([{ id: 'u1' }]);+ vi.mocked(peerFetch).mockResolvedValue({ ok: true, json: async () => ({ missingAssets: [] }) });+ await subscribePeer({ peerId: 'peer-a', recordKind: 'universe', recordId: 'u1' });+ await __drainForTests();++ // Converged peer: the boot-time event walks the sub but sends nothing.+ vi.mocked(peerFetch).mockClear();+ instanceEvents.emit('peer:online', peer);+ await __drainForTests();+ expect(vi.mocked(peerFetch)).not.toHaveBeenCalled();++ // The record changed after the last push but the debounce never fired+ // (the process restarted); the first boot event delivers it once.+ vi.mocked(getUniverse).mockResolvedValue({ id: 'u1', name: 'Edited just before restart' });+ instanceEvents.emit('peer:online', peer);+ await __drainForTests();+ const pushCalls = () => vi.mocked(peerFetch).mock.calls.length;+ expect(pushCalls()).toBeGreaterThan(0);++ // A repeat event with the record unchanged since that push sends nothing.+ const afterFirst = pushCalls();+ instanceEvents.emit('peer:online', peer);+ await __drainForTests();+ expect(pushCalls()).toBe(afterFirst);+ });+it('returns {walked: 0, pushed: 0} when the peer has no subscriptions', async () => {const result = await retryPendingPushesForPeer('peer-without-subs');expect(result).toEqual({ walked: 0, pushed: 0 });diff --git a/server/services/sharing/subscriptions.js b/server/services/sharing/subscriptions.jsindex 8b98781df6..fa50738267 100644--- a/server/services/sharing/subscriptions.js+++ b/server/services/sharing/subscriptions.js@@ -25,6 +25,8 @@ import { existsSync } from 'fs';import { PATHS, atomicWrite, readJSONFile, ensureDir } from '../../lib/fileUtils.js';import { createFileWriteQueue } from '../../lib/fileWriteQueue.js';import { getBucket } from './buckets.js';+import { getSeries } from '../pipeline/series.js';+import { getUniverse } from '../universeBuilder.js';import { exportSeries, exportUniverse } from './exporter.js';import {recordEvents,@@ -323,6 +325,41 @@ export async function reexportSubscribedRecord(recordKind, recordId) {}}+async function loadRecordUpdatedAt(recordKind, recordId) {+ const record = recordKind === 'series' ? await getSeries(recordId)+ : recordKind === 'universe' ? await getUniverse(recordId)+ : null;+ return isStr(record?.updatedAt) ? record.updatedAt : null;+}++/**+ * Boot convergence for share buckets. The 3 s re-export debounce lives only in+ * memory, so an edit made just before a restart/crash never reaches the bucket.+ * Re-export every subscription whose record changed after its last export (or+ * was never exported) through the same runExport + stampExport path the+ * debounce uses. A subscription adopted from an import has no export yet; its+ * baseline is the adoption time so an unedited imported record is not written+ * straight back to the bucket it came from.+ */+export async function reconcileSubscriptionsOnBoot() {+ const subscriptions = await listSubscriptions();+ let reexported = 0;+ for (const sub of subscriptions) {+ const baseline = sub.lastExportedAt || (sub.adoptedFromImport ? sub.updatedAt : null);+ const recordUpdatedAt = await loadRecordUpdatedAt(sub.recordKind, sub.recordId).catch(() => undefined);+ // undefined = record missing/unreadable (nothing to export); null = no timestamp.+ if (recordUpdatedAt === undefined) continue;+ const stale = baseline+ ? recordUpdatedAt !== null && recordUpdatedAt > baseline+ : true;+ if (!stale) continue;+ await reexportNow(sub);+ reexported += 1;+ }+ if (reexported > 0) console.log(`🔄 sharing.subscriptions: boot reconcile re-exported ${reexported} stale share${reexported === 1 ? '' : 's'}`);+ return { checked: subscriptions.length, reexported };+}+// Track the listener handles we install so __resetForTests can remove just// ours — `removeAllListeners('updated')` would also kill listeners installed// by neighboring test files that share the recordEvents bus.diff --git a/server/services/sharing/subscriptions.test.js b/server/services/sharing/subscriptions.test.jsindex 3257d89cf9..612bb89cdf 100644--- a/server/services/sharing/subscriptions.test.js+++ b/server/services/sharing/subscriptions.test.js@@ -323,4 +323,58 @@ describe('sharing/subscriptions', () => {return onDisk.id === live.lastManifestId;});});++ describe('reconcileSubscriptionsOnBoot (#11750)', () => {+ // Rewrite persisted lastExportedAt to simulate "exported before the edit"+ // (past) or "exported after the last edit" (future) without sleeping.+ async function setLastExportedAt(subId, iso) {+ const fs = await import('fs');+ const path = join(tempData, 'sharing', 'subscriptions.json');+ const state = JSON.parse(fs.readFileSync(path, 'utf8'));+ state.subscriptions.find((x) => x.id === subId).lastExportedAt = iso;+ fs.writeFileSync(path, JSON.stringify(state));+ }++ it('re-exports a share whose record changed after lastExportedAt exactly once, and leaves an unchanged one alone', async () => {+ const bucket = await buckets.createBucket({ name: 'B', path: tempBucket });+ const stale = await universeBuilder.createUniverse({ name: 'U-stale' });+ const fresh = await universeBuilder.createUniverse({ name: 'U-fresh' });+ const staleSub = await subs.subscribe({ bucketId: bucket.id, recordKind: 'universe', recordId: stale.id });+ const freshSub = await subs.subscribe({ bucketId: bucket.id, recordKind: 'universe', recordId: fresh.id });+ await setLastExportedAt(staleSub.id, '2000-01-01T00:00:00.000Z');+ await setLastExportedAt(freshSub.id, '2999-01-01T00:00:00.000Z');++ const first = await subs.reconcileSubscriptionsOnBoot();+ expect(first.reexported).toBe(1);+ const [afterStale] = await subs.listSubscriptions({ recordId: stale.id });+ const [afterFresh] = await subs.listSubscriptions({ recordId: fresh.id });+ expect(afterStale.lastManifestId).not.toBe(staleSub.lastManifestId);+ expect(afterFresh.lastManifestId).toBe(freshSub.lastManifestId);++ // The stamp moved lastExportedAt past the record, so a second pass is a no-op.+ expect((await subs.reconcileSubscriptionsOnBoot()).reexported).toBe(0);+ });++ it('exports a never-exported subscription, but not an import-adopted one whose record is unchanged', async () => {+ const bucket = await buckets.createBucket({ name: 'B', path: tempBucket });+ const never = await universeBuilder.createUniverse({ name: 'U-never' });+ const adopted = await universeBuilder.createUniverse({ name: 'U-adopted' });+ const neverSub = await subs.subscribe({ bucketId: bucket.id, recordKind: 'universe', recordId: never.id });+ await setLastExportedAt(neverSub.id, null);+ await subs.adoptImportedSubscription({ bucketId: bucket.id, recordKind: 'universe', recordId: adopted.id });++ const result = await subs.reconcileSubscriptionsOnBoot();+ expect(result.reexported).toBe(1);+ const [reexported] = await subs.listSubscriptions({ recordId: never.id });+ expect(reexported.lastExportedAt).toBeTruthy();+ const [stillAdopted] = await subs.listSubscriptions({ recordId: adopted.id });+ expect(stillAdopted.lastExportedAt).toBeNull();+ });++ it('skips a subscription whose record no longer exists', async () => {+ const bucket = await buckets.createBucket({ name: 'B', path: tempBucket });+ await subs.adoptImportedSubscription({ bucketId: bucket.id, recordKind: 'universe', recordId: 'missing-universe' });+ await expect(subs.reconcileSubscriptionsOnBoot()).resolves.toMatchObject({ reexported: 0 });+ });+ });});diff --git a/server/services/sharing/watcher.js b/server/services/sharing/watcher.jsindex 408132a447..537aac75c8 100644--- a/server/services/sharing/watcher.js+++ b/server/services/sharing/watcher.js@@ -19,6 +19,7 @@ import { join, basename, sep } from 'path';import { processManifest, processBacklog, handleUnshare, sharingEvents } from './importer.js';import { getBucket, listBuckets, ensureBucketLayout } from './buckets.js';import { isManifestPruning, pruneBucketManifests } from './manifest.js';+import { reconcileSubscriptionsOnBoot } from './subscriptions.js';import { getInstanceId } from '../instanceIdentity.js';import { createFileWriteQueue } from '../../lib/fileWriteQueue.js';@@ -187,6 +188,11 @@ async function attachAllWatchersNow() {});}}+ // Re-export shares whose record changed after the last export — the in-memory+ // re-export debounce does not survive a restart (#11750).+ await reconcileSubscriptionsOnBoot().catch((err) => {+ console.error(`❌ sharing.watcher: subscription reconcile failed: ${err.message}`);+ });return { attached: watchers.size };}diff --git a/server/services/sharing/watcher.test.js b/server/services/sharing/watcher.test.jsindex fe537e93c8..9d4139f07b 100644--- a/server/services/sharing/watcher.test.js+++ b/server/services/sharing/watcher.test.js@@ -13,12 +13,14 @@ vi.mock('./buckets.js', () => ({}));vi.mock('./manifest.js', () => ({ isManifestPruning: vi.fn(), pruneBucketManifests: vi.fn() }));vi.mock('../instanceIdentity.js', () => ({ getInstanceId: vi.fn() }));+vi.mock('./subscriptions.js', () => ({ reconcileSubscriptionsOnBoot: vi.fn() }));import { watch } from 'chokidar';import { processBacklog, processManifest, handleUnshare } from './importer.js';import { getBucket, listBuckets, ensureBucketLayout } from './buckets.js';import { isManifestPruning } from './manifest.js';import { getInstanceId } from '../instanceIdentity.js';+import { reconcileSubscriptionsOnBoot } from './subscriptions.js';let shutdown;function deferred() {@@ -54,6 +56,7 @@ beforeEach(() => {return watcher;});getInstanceId.mockReset().mockResolvedValue(null);+ reconcileSubscriptionsOnBoot.mockReset().mockResolvedValue({ checked: 0, reexported: 0 });listBuckets.mockReset().mockResolvedValue([]);ensureBucketLayout.mockReset().mockResolvedValue(undefined);processBacklog.mockReset().mockResolvedValue(undefined);@@ -232,6 +235,16 @@ describe('share-bucket watcher attachment lifecycle', () => {expect(liveHandles()).toEqual([]);});+ it('boot attach reconciles stale share subscriptions, and a reconcile failure does not fail the attach', async () => {+ const { attachAllWatchers, listAttachedWatchers } = await loadWatcher();+ await attachAllWatchers();+ expect(reconcileSubscriptionsOnBoot).toHaveBeenCalledTimes(1);++ reconcileSubscriptionsOnBoot.mockRejectedValueOnce(new Error('export exploded'));+ await expect(attachAllWatchers()).resolves.toEqual({ attached: listAttachedWatchers().length });+ expect(reconcileSubscriptionsOnBoot).toHaveBeenCalledTimes(2);+ });+it('shutdown drains overlapping admitted attachments and synchronously rejects later ones', async () => {const { attachWatcher, listAttachedWatchers, attachAllWatchers } = await loadWatcher();const layout = pauseLayout();