Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: converge peers and share buckets after a restart (#11750)

[._.]/ Adam Eivy829879f8d489
diff --git a/server/services/instances.js b/server/services/instances.js
index 199b206276..7bc8fac261 100644
--- a/server/services/instances.js
+++ b/server/services/instances.js
@@ -871,6 +871,25 @@ export function enqueueReciprocalSync(peerId) {
// --- Probing ---
+// Peers observed online in THIS process. The persisted `status` survives a
+// restart, so a peer that was online before the restart would never produce a
+// status transition and `peer:online` listeners (push retry, sync) would not
+// run for edits stranded by the restart (#11750). The first online sighting
+// per process emits regardless of the persisted status; later sightings keep
+// the transition-only rule.
+const peersSeenOnlineThisProcess = new Set();
+
+/** Test-only: forget which peers were seen online so a test can simulate a fresh boot. */
+export function __resetPeersSeenOnlineForTests() {
+ peersSeenOnlineThisProcess.clear();
+}
+
+function isFirstOnlineSighting(peerId) {
+ if (peersSeenOnlineThisProcess.has(peerId)) return false;
+ peersSeenOnlineThisProcess.add(peerId);
+ return true;
+}
+
export async function probePeer(peer) {
const baseUrl = peerBaseUrl(peer);
@@ -1033,10 +1052,12 @@ export async function probePeer(peer) {
}
// Announce ourselves only when peer transitions to online (not every poll cycle)
- if (status === 'online' && previousStatus !== 'online') {
+ if (status === 'online') {
+ const firstSighting = isFirstOnlineSighting(peer.id);
+ const transitioned = previousStatus !== 'online';
if (stored) {
- announceSelf(stored);
- instanceEvents.emit('peer:online', stored);
+ if (transitioned) announceSelf(stored);
+ if (transitioned || firstSighting) instanceEvents.emit('peer:online', stored);
}
}
@@ -1109,6 +1130,9 @@ export async function queryPeer(id, apiPath) {
// --- Announce (Bidirectional Registration) ---
export async function handleAnnounce({ address, port, instanceId, name, host }) {
+ // Set only when this announce marked the peer online (not the forged-address
+ // branch, which records a sighting without trusting it).
+ let markedOnlinePeer = null;
const result = await withData(async (data) => {
// Check for existing peer by instanceId
let existing = data.peers.find(p => p.instanceId === instanceId);
@@ -1147,6 +1171,7 @@ export async function handleAnnounce({ address, port, instanceId, name, host })
if (existing) {
existing.lastSeen = new Date().toISOString();
existing.status = 'online';
+ markedOnlinePeer = existing;
existing.instanceId = instanceId;
// A managed forward uses local coordinates, not the remote listen port.
if (existing.transport !== 'tailcat') existing.port = port;
@@ -1203,6 +1228,12 @@ export async function handleAnnounce({ address, port, instanceId, name, host })
return { created: true, peer };
});
+ // The announce path sets status = 'online' without a probe, so it needs the
+ // same once-per-process convergence trigger as probePeer.
+ if (markedOnlinePeer && isFirstOnlineSighting(markedOnlinePeer.id)) {
+ instanceEvents.emit('peer:online', markedOnlinePeer);
+ }
+
// Immediately probe newly announced peers to populate health data
if (result.created) {
probePeer(result.peer).catch(err => {
diff --git a/server/services/instances.test.js b/server/services/instances.test.js
index ea03535645..3e44c00da2 100644
--- a/server/services/instances.test.js
+++ b/server/services/instances.test.js
@@ -83,12 +83,14 @@ import {
enqueueReciprocalSync,
DEFAULT_SYNC_CATEGORIES,
startPolling,
- stopPolling
+ stopPolling,
+ __resetPeersSeenOnlineForTests
} from './instances.js';
describe('instances.js', () => {
beforeEach(() => {
vi.clearAllMocks();
+ __resetPeersSeenOnlineForTests();
vi.useFakeTimers();
vi.stubGlobal('fetch', vi.fn());
readJSONFile.mockResolvedValue({ self: null, peers: [] });
@@ -1421,20 +1423,20 @@ describe('instances.js', () => {
expect(instanceEvents.emit).toHaveBeenCalledWith('peer:online', expect.any(Object));
});
- it('should not emit peer:online if already online', async () => {
+ it('emits peer:online on the first probe after boot for a peer persisted as online, then only on transitions (#11750)', async () => {
+ // The persisted status survives a restart, so without the per-process
+ // first-sighting rule an edit stranded by the restart is never retried.
const peer = makePeer({ status: 'online' });
- const peers = [peer];
- readJSONFile.mockResolvedValue({ self: null, peers });
-
+ readJSONFile.mockResolvedValue({ self: null, peers: [peer] });
const healthData = { instanceId: 'r-id' };
- fetch
- .mockResolvedValueOnce({ ok: true, json: () => Promise.resolve(healthData) })
- .mockResolvedValueOnce({ ok: false })
- .mockResolvedValueOnce({ ok: false });
+ fetch.mockImplementation(async () => ({ ok: true, json: () => Promise.resolve(healthData) }));
+ const onlineEmits = () => instanceEvents.emit.mock.calls.filter(([name]) => name === 'peer:online').length;
await probePeer(peer);
+ expect(onlineEmits()).toBe(1);
- expect(instanceEvents.emit).not.toHaveBeenCalledWith('peer:online', expect.anything());
+ await probePeer(peer);
+ expect(onlineEmits()).toBe(1);
});
it('should return null if peer is removed during probe', async () => {
@@ -1565,6 +1567,27 @@ describe('instances.js', () => {
});
describe('handleAnnounce', () => {
+ it('emits peer:online once per process when an announce marks a known peer online (#11750)', async () => {
+ const peer = { id: 'peer-local', instanceId: 'peer-a', address: '192.0.2.10', port: 5555,
+ name: 'Example peer', status: 'online', directions: ['outbound'] };
+ readJSONFile.mockResolvedValue({ self: null, peers: [peer] });
+ const onlineEmits = () => instanceEvents.emit.mock.calls.filter(([name]) => name === 'peer:online').length;
+
+ await handleAnnounce({ instanceId: 'peer-a', address: '192.0.2.10', port: 5555 });
+ expect(onlineEmits()).toBe(1);
+
+ await handleAnnounce({ instanceId: 'peer-a', address: '192.0.2.10', port: 5555 });
+ expect(onlineEmits()).toBe(1);
+ });
+
+ it('does not emit peer:online for a forged announce that was not trusted', async () => {
+ const peer = { id: 'peer-local', instanceId: 'peer-a', address: '192.0.2.10', port: 5555,
+ name: 'Example peer', status: 'online', directions: ['outbound'] };
+ readJSONFile.mockResolvedValue({ self: null, peers: [peer] });
+ await handleAnnounce({ instanceId: 'peer-a', address: '198.51.100.99', port: 5555 });
+ expect(instanceEvents.emit).not.toHaveBeenCalledWith('peer:online', expect.anything());
+ });
+
it('keeps an admitted peer identity, callback endpoint and consent unchanged after forged discovery', async () => {
const peer = { id: 'peer-local', instanceId: 'peer-a', address: '192.0.2.10', port: 5555,
host: 'configured.example.com', directions: ['inbound'], syncSecret: 'synthetic-pair-secret-32-characters-long',
diff --git a/server/services/sharing/peerSync.test.js b/server/services/sharing/peerSync.test.js
index a91992d2d5..242f882430 100644
--- a/server/services/sharing/peerSync.test.js
+++ b/server/services/sharing/peerSync.test.js
@@ -1425,8 +1425,11 @@ describe('peerSync', () => {
// (the lastPushedHash short-circuit inside pushRecordToPeer is what
// skips the actual HTTP call for unchanged records). This is the
// mechanism that lets out-of-band file edits (e.g., a cleanup script
- // that wrote tombstones directly to disk + a server restart)
- // re-propagate via peer:online without needing a per-record edit.
+ // that wrote tombstones directly to disk) re-propagate via peer:online
+ // without needing a per-record edit. A server restart reaches this path
+ // too: instances.js emits peer:online on the first probe of each peer per
+ // process even when the persisted status was already online (#11750; see
+ // the boot-convergence test below).
vi.mocked(peerFetch).mockClear();
const second = await retryPendingPushesForPeer('peer-a');
expect(second.walked).toBe(1); // walked, not skipped by the helper
@@ -1436,6 +1439,40 @@ describe('peerSync', () => {
expect(vi.mocked(peerFetch)).not.toHaveBeenCalled();
});
+ it('boot convergence: the first peer:online after a restart pushes an edit stranded by the restart exactly once, a converged peer costs zero HTTP calls (#11750)', async () => {
+ const { instanceEvents } = await import('../instanceEvents.js');
+ const { installPeerSyncListener } = await import('./peerSync.js');
+ installPeerSyncListener();
+ const peer = {
+ id: 'local-a', instanceId: 'peer-a', enabled: true, syncEnabled: true, status: 'online',
+ directions: ['outbound'], syncCategories: { universe: true },
+ };
+ vi.mocked(listUniverses).mockResolvedValue([{ id: 'u1' }]);
+ vi.mocked(peerFetch).mockResolvedValue({ ok: true, json: async () => ({ missingAssets: [] }) });
+ await subscribePeer({ peerId: 'peer-a', recordKind: 'universe', recordId: 'u1' });
+ await __drainForTests();
+
+ // Converged peer: the boot-time event walks the sub but sends nothing.
+ vi.mocked(peerFetch).mockClear();
+ instanceEvents.emit('peer:online', peer);
+ await __drainForTests();
+ expect(vi.mocked(peerFetch)).not.toHaveBeenCalled();
+
+ // The record changed after the last push but the debounce never fired
+ // (the process restarted); the first boot event delivers it once.
+ vi.mocked(getUniverse).mockResolvedValue({ id: 'u1', name: 'Edited just before restart' });
+ instanceEvents.emit('peer:online', peer);
+ await __drainForTests();
+ const pushCalls = () => vi.mocked(peerFetch).mock.calls.length;
+ expect(pushCalls()).toBeGreaterThan(0);
+
+ // A repeat event with the record unchanged since that push sends nothing.
+ const afterFirst = pushCalls();
+ instanceEvents.emit('peer:online', peer);
+ await __drainForTests();
+ expect(pushCalls()).toBe(afterFirst);
+ });
+
it('returns {walked: 0, pushed: 0} when the peer has no subscriptions', async () => {
const result = await retryPendingPushesForPeer('peer-without-subs');
expect(result).toEqual({ walked: 0, pushed: 0 });
diff --git a/server/services/sharing/subscriptions.js b/server/services/sharing/subscriptions.js
index 8b98781df6..fa50738267 100644
--- a/server/services/sharing/subscriptions.js
+++ b/server/services/sharing/subscriptions.js
@@ -25,6 +25,8 @@ import { existsSync } from 'fs';
import { PATHS, atomicWrite, readJSONFile, ensureDir } from '../../lib/fileUtils.js';
import { createFileWriteQueue } from '../../lib/fileWriteQueue.js';
import { getBucket } from './buckets.js';
+import { getSeries } from '../pipeline/series.js';
+import { getUniverse } from '../universeBuilder.js';
import { exportSeries, exportUniverse } from './exporter.js';
import {
recordEvents,
@@ -323,6 +325,41 @@ export async function reexportSubscribedRecord(recordKind, recordId) {
}
}
+async function loadRecordUpdatedAt(recordKind, recordId) {
+ const record = recordKind === 'series' ? await getSeries(recordId)
+ : recordKind === 'universe' ? await getUniverse(recordId)
+ : null;
+ return isStr(record?.updatedAt) ? record.updatedAt : null;
+}
+
+/**
+ * Boot convergence for share buckets. The 3 s re-export debounce lives only in
+ * memory, so an edit made just before a restart/crash never reaches the bucket.
+ * Re-export every subscription whose record changed after its last export (or
+ * was never exported) through the same runExport + stampExport path the
+ * debounce uses. A subscription adopted from an import has no export yet; its
+ * baseline is the adoption time so an unedited imported record is not written
+ * straight back to the bucket it came from.
+ */
+export async function reconcileSubscriptionsOnBoot() {
+ const subscriptions = await listSubscriptions();
+ let reexported = 0;
+ for (const sub of subscriptions) {
+ const baseline = sub.lastExportedAt || (sub.adoptedFromImport ? sub.updatedAt : null);
+ const recordUpdatedAt = await loadRecordUpdatedAt(sub.recordKind, sub.recordId).catch(() => undefined);
+ // undefined = record missing/unreadable (nothing to export); null = no timestamp.
+ if (recordUpdatedAt === undefined) continue;
+ const stale = baseline
+ ? recordUpdatedAt !== null && recordUpdatedAt > baseline
+ : true;
+ if (!stale) continue;
+ await reexportNow(sub);
+ reexported += 1;
+ }
+ if (reexported > 0) console.log(`🔄 sharing.subscriptions: boot reconcile re-exported ${reexported} stale share${reexported === 1 ? '' : 's'}`);
+ return { checked: subscriptions.length, reexported };
+}
+
// Track the listener handles we install so __resetForTests can remove just
// ours — `removeAllListeners('updated')` would also kill listeners installed
// by neighboring test files that share the recordEvents bus.
diff --git a/server/services/sharing/subscriptions.test.js b/server/services/sharing/subscriptions.test.js
index 3257d89cf9..612bb89cdf 100644
--- a/server/services/sharing/subscriptions.test.js
+++ b/server/services/sharing/subscriptions.test.js
@@ -323,4 +323,58 @@ describe('sharing/subscriptions', () => {
return onDisk.id === live.lastManifestId;
});
});
+
+ describe('reconcileSubscriptionsOnBoot (#11750)', () => {
+ // Rewrite persisted lastExportedAt to simulate "exported before the edit"
+ // (past) or "exported after the last edit" (future) without sleeping.
+ async function setLastExportedAt(subId, iso) {
+ const fs = await import('fs');
+ const path = join(tempData, 'sharing', 'subscriptions.json');
+ const state = JSON.parse(fs.readFileSync(path, 'utf8'));
+ state.subscriptions.find((x) => x.id === subId).lastExportedAt = iso;
+ fs.writeFileSync(path, JSON.stringify(state));
+ }
+
+ it('re-exports a share whose record changed after lastExportedAt exactly once, and leaves an unchanged one alone', async () => {
+ const bucket = await buckets.createBucket({ name: 'B', path: tempBucket });
+ const stale = await universeBuilder.createUniverse({ name: 'U-stale' });
+ const fresh = await universeBuilder.createUniverse({ name: 'U-fresh' });
+ const staleSub = await subs.subscribe({ bucketId: bucket.id, recordKind: 'universe', recordId: stale.id });
+ const freshSub = await subs.subscribe({ bucketId: bucket.id, recordKind: 'universe', recordId: fresh.id });
+ await setLastExportedAt(staleSub.id, '2000-01-01T00:00:00.000Z');
+ await setLastExportedAt(freshSub.id, '2999-01-01T00:00:00.000Z');
+
+ const first = await subs.reconcileSubscriptionsOnBoot();
+ expect(first.reexported).toBe(1);
+ const [afterStale] = await subs.listSubscriptions({ recordId: stale.id });
+ const [afterFresh] = await subs.listSubscriptions({ recordId: fresh.id });
+ expect(afterStale.lastManifestId).not.toBe(staleSub.lastManifestId);
+ expect(afterFresh.lastManifestId).toBe(freshSub.lastManifestId);
+
+ // The stamp moved lastExportedAt past the record, so a second pass is a no-op.
+ expect((await subs.reconcileSubscriptionsOnBoot()).reexported).toBe(0);
+ });
+
+ it('exports a never-exported subscription, but not an import-adopted one whose record is unchanged', async () => {
+ const bucket = await buckets.createBucket({ name: 'B', path: tempBucket });
+ const never = await universeBuilder.createUniverse({ name: 'U-never' });
+ const adopted = await universeBuilder.createUniverse({ name: 'U-adopted' });
+ const neverSub = await subs.subscribe({ bucketId: bucket.id, recordKind: 'universe', recordId: never.id });
+ await setLastExportedAt(neverSub.id, null);
+ await subs.adoptImportedSubscription({ bucketId: bucket.id, recordKind: 'universe', recordId: adopted.id });
+
+ const result = await subs.reconcileSubscriptionsOnBoot();
+ expect(result.reexported).toBe(1);
+ const [reexported] = await subs.listSubscriptions({ recordId: never.id });
+ expect(reexported.lastExportedAt).toBeTruthy();
+ const [stillAdopted] = await subs.listSubscriptions({ recordId: adopted.id });
+ expect(stillAdopted.lastExportedAt).toBeNull();
+ });
+
+ it('skips a subscription whose record no longer exists', async () => {
+ const bucket = await buckets.createBucket({ name: 'B', path: tempBucket });
+ await subs.adoptImportedSubscription({ bucketId: bucket.id, recordKind: 'universe', recordId: 'missing-universe' });
+ await expect(subs.reconcileSubscriptionsOnBoot()).resolves.toMatchObject({ reexported: 0 });
+ });
+ });
});
diff --git a/server/services/sharing/watcher.js b/server/services/sharing/watcher.js
index 408132a447..537aac75c8 100644
--- a/server/services/sharing/watcher.js
+++ b/server/services/sharing/watcher.js
@@ -19,6 +19,7 @@ import { join, basename, sep } from 'path';
import { processManifest, processBacklog, handleUnshare, sharingEvents } from './importer.js';
import { getBucket, listBuckets, ensureBucketLayout } from './buckets.js';
import { isManifestPruning, pruneBucketManifests } from './manifest.js';
+import { reconcileSubscriptionsOnBoot } from './subscriptions.js';
import { getInstanceId } from '../instanceIdentity.js';
import { createFileWriteQueue } from '../../lib/fileWriteQueue.js';
@@ -187,6 +188,11 @@ async function attachAllWatchersNow() {
});
}
}
+ // Re-export shares whose record changed after the last export — the in-memory
+ // re-export debounce does not survive a restart (#11750).
+ await reconcileSubscriptionsOnBoot().catch((err) => {
+ console.error(`❌ sharing.watcher: subscription reconcile failed: ${err.message}`);
+ });
return { attached: watchers.size };
}
diff --git a/server/services/sharing/watcher.test.js b/server/services/sharing/watcher.test.js
index fe537e93c8..9d4139f07b 100644
--- a/server/services/sharing/watcher.test.js
+++ b/server/services/sharing/watcher.test.js
@@ -13,12 +13,14 @@ vi.mock('./buckets.js', () => ({
}));
vi.mock('./manifest.js', () => ({ isManifestPruning: vi.fn(), pruneBucketManifests: vi.fn() }));
vi.mock('../instanceIdentity.js', () => ({ getInstanceId: vi.fn() }));
+vi.mock('./subscriptions.js', () => ({ reconcileSubscriptionsOnBoot: vi.fn() }));
import { watch } from 'chokidar';
import { processBacklog, processManifest, handleUnshare } from './importer.js';
import { getBucket, listBuckets, ensureBucketLayout } from './buckets.js';
import { isManifestPruning } from './manifest.js';
import { getInstanceId } from '../instanceIdentity.js';
+import { reconcileSubscriptionsOnBoot } from './subscriptions.js';
let shutdown;
function deferred() {
@@ -54,6 +56,7 @@ beforeEach(() => {
return watcher;
});
getInstanceId.mockReset().mockResolvedValue(null);
+ reconcileSubscriptionsOnBoot.mockReset().mockResolvedValue({ checked: 0, reexported: 0 });
listBuckets.mockReset().mockResolvedValue([]);
ensureBucketLayout.mockReset().mockResolvedValue(undefined);
processBacklog.mockReset().mockResolvedValue(undefined);
@@ -232,6 +235,16 @@ describe('share-bucket watcher attachment lifecycle', () => {
expect(liveHandles()).toEqual([]);
});
+ it('boot attach reconciles stale share subscriptions, and a reconcile failure does not fail the attach', async () => {
+ const { attachAllWatchers, listAttachedWatchers } = await loadWatcher();
+ await attachAllWatchers();
+ expect(reconcileSubscriptionsOnBoot).toHaveBeenCalledTimes(1);
+
+ reconcileSubscriptionsOnBoot.mockRejectedValueOnce(new Error('export exploded'));
+ await expect(attachAllWatchers()).resolves.toEqual({ attached: listAttachedWatchers().length });
+ expect(reconcileSubscriptionsOnBoot).toHaveBeenCalledTimes(2);
+ });
+
it('shutdown drains overlapping admitted attachments and synchronously rejects later ones', async () => {
const { attachWatcher, listAttachedWatchers, attachAllWatchers } = await loadWatcher();
const layout = pauseLayout();