Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

fix(ok): stop a server restart from stranding open documents (#2998)

Nick Gomez4e533556e27e
diff --git a/.changeset/epoch-rotation-recycle.md b/.changeset/epoch-rotation-recycle.md
new file mode 100644
index 000000000..dc3a5a2d1
--- /dev/null
+++ b/.changeset/epoch-rotation-recycle.md
@@ -0,0 +1,5 @@
+---
+"@inkeep/open-knowledge": minor
+---
+
+Documents no longer get permanently stuck offline after the server restarts. Previously, if the restart took more than a couple of seconds, an open document could land in a state where it reconnected forever without ever syncing: the banner said "Connection lost, keep this tab open, your edits will sync when reconnected", but nothing it promised was going to happen. Switching to another document and back did not help, and anything typed while stuck was lost on reload. The client now treats a restarted server the same way whichever way it finds out about it, so the affected documents recover on their own and the edits typed during the outage are replayed instead of dropped.
diff --git a/packages/app/src/components/SystemDocSubscriber.tsx b/packages/app/src/components/SystemDocSubscriber.tsx
index 04ca6ccb4..48dc5c24a 100644
--- a/packages/app/src/components/SystemDocSubscriber.tsx
+++ b/packages/app/src/components/SystemDocSubscriber.tsx
@@ -50,6 +50,14 @@ export function SystemDocSubscriber() {
useEffect(() => {
if (collabUrl === null) return;
const doc = new Y.Doc();
+ // STOP: this provider must stay token-less. It is the only channel by
+ // which a live client learns that the server rotated its instance id,
+ // and it can only be that channel because it carries no epoch claim for
+ // the server to reject. Giving it a claim-bearing token makes it fail
+ // authentication after every restart; it has no `authenticationFailed`
+ // handler and this effect does not re-run, so `refreshServerInfo` would
+ // never fire and the pool would never learn the new epoch — turning an
+ // intermittent single-doc stall into a deterministic total one.
const provider = new HocuspocusProvider({
url: collabUrl,
name: SYSTEM_DOC_NAME,
diff --git a/packages/app/src/editor/branch-invalidation.test.ts b/packages/app/src/editor/branch-invalidation.test.ts
index 2be98ce54..53a1050eb 100644
--- a/packages/app/src/editor/branch-invalidation.test.ts
+++ b/packages/app/src/editor/branch-invalidation.test.ts
@@ -216,6 +216,107 @@ describe('handleBranchSwitched', () => {
expect(pool.__test_hasBufferedUpdate(d1)).toBe(false);
expect(pool.__test_hasBufferedUpdate(d2)).toBe(false);
});
+
+ // One `/api/server-info` response can report a rotated epoch AND a changed
+ // branch — the late-join case the refresher exists for. The refresher
+ // observes the epoch first, so by the time the branch decision resolves a
+ // recycle is already in flight over the same entries.
+ test('stands down while a server-instance-mismatch recycle owns the pool', async () => {
+ const infoSpy = vi.spyOn(console, 'info').mockImplementation(() => undefined);
+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined);
+ try {
+ pool = new ProviderPool(3, DUMMY_WS);
+ pool.setExpectedServerInstanceId(TEST_SERVER_INSTANCE_ID);
+ pool.setObservedBranch('main');
+ const d1 = docName('combined');
+ const entry = pool.open(d1);
+ if (!entry) throw new Error('expected entry');
+ pool.setActive(d1);
+ await awaitAttachedPersistence(entry);
+
+ // Same order `refreshServerInfo` dispatches in.
+ pool.setExpectedServerInstanceId('server-rotated');
+ expect(pool.isMismatchRecycleInFlight()).toBe(true);
+ await handleBranchSwitched(pool, 'feature');
+
+ const deferred = warnSpy.mock.calls.filter(([first]) => {
+ if (typeof first !== 'string') return false;
+ try {
+ return (
+ (JSON.parse(first) as { event?: string }).event ===
+ 'ok-branch-switched-deferred-to-mismatch-recycle'
+ );
+ } catch {
+ return false;
+ }
+ });
+ expect(deferred.length).toBe(1);
+
+ await pool.awaitMismatchSettled();
+
+ // Exactly one whole-pool invalidation ran, not two.
+ const recycleAlls = infoSpy.mock.calls.filter(([first]) => {
+ if (typeof first !== 'string') return false;
+ try {
+ return (JSON.parse(first) as { event?: string }).event === 'ok-pool-recycle-all';
+ } catch {
+ return false;
+ }
+ });
+ expect(recycleAlls.length).toBe(1);
+ } finally {
+ warnSpy.mockRestore();
+ infoSpy.mockRestore();
+ }
+ });
+
+ // The stand-down is only safe because the discard policy is enforced where
+ // the buffer is consumed rather than only at this entry point.
+ test('a buffer captured on another branch is discarded rather than replayed', async () => {
+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined);
+ try {
+ pool = new ProviderPool(3, DUMMY_WS);
+ pool.setExpectedServerInstanceId(TEST_SERVER_INSTANCE_ID);
+ pool.setObservedBranch('main');
+ const d1 = docName('provenance');
+ const entry = pool.open(d1);
+ if (!entry) throw new Error('expected entry');
+ pool.setActive(d1);
+ await awaitAttachedPersistence(entry);
+
+ // A recycle installs the replay listener on the fresh provider.
+ entry.provider.emit('authenticationFailed', { reason: 'server-instance-mismatch' });
+ await pool.awaitMismatchSettled();
+ const fresh = pool.entries.get(d1);
+ if (!fresh || fresh.kind !== 'active') throw new Error('expected a recycled entry');
+
+ // Stage a buffer whose provenance is the branch we are leaving, then
+ // move the pool to the new branch before the fresh provider syncs.
+ pool.__test_seedBufferedUpdate(d1, new Uint8Array([0x01, 0x02]), { branch: 'main' });
+ pool.setObservedBranch('feature');
+ fresh.provider.emit('synced', { state: true });
+
+ const deadline = Date.now() + 2_000;
+ while (pool.__test_hasBufferedUpdate(d1) && Date.now() < deadline) {
+ await new Promise<void>((resolve) => setTimeout(resolve, 10));
+ }
+ expect(pool.__test_hasBufferedUpdate(d1)).toBe(false);
+
+ const fenced = warnSpy.mock.calls.filter(([first]) => {
+ if (typeof first !== 'string') return false;
+ try {
+ return (
+ (JSON.parse(first) as { event?: string }).event === 'ok-buffer-replay-branch-mismatch'
+ );
+ } catch {
+ return false;
+ }
+ });
+ expect(fenced.length).toBe(1);
+ } finally {
+ warnSpy.mockRestore();
+ }
+ });
});
describe('ProviderPool.close drains bufferedUpdates', () => {
diff --git a/packages/app/src/editor/branch-invalidation.ts b/packages/app/src/editor/branch-invalidation.ts
index 9fd6ce82c..4d7f8c70a 100644
--- a/packages/app/src/editor/branch-invalidation.ts
+++ b/packages/app/src/editor/branch-invalidation.ts
@@ -48,7 +48,11 @@ type BranchSwitchedClearFailedLog = z.infer<typeof BranchSwitchedClearFailedLogS
* `clearBufferedUpdates()` those bytes (captured against branch A's Y.Doc)
* would replay onto branch B the next time the user opened the affected
* doc. The branch-switch policy is "discard, don't preserve" — apply it
- * to the in-memory buffer slot, not just the IDB layer.
+ * to the in-memory buffer slot, not just the IDB layer. The pool's replay
+ * drain fences on the same policy independently (it declines any buffer
+ * whose capture branch differs from the live one), which is what makes the
+ * stand-down below safe; this eager drop stays because it releases the
+ * durable mirror rather than waiting for a reopen to discover it.
*
* `clearData` failures are caught per-entry and logged as structured
* `ok-branch-switched-clear-failed` warn events so the recycle still
@@ -56,6 +60,25 @@ type BranchSwitchedClearFailedLog = z.infer<typeof BranchSwitchedClearFailedLogS
* the pool stranded on branch A.
*/
export async function handleBranchSwitched(pool: ProviderPool, branch: string): Promise<void> {
+ // Stand down while a server-instance-mismatch recycle owns the pool. One
+ // `/api/server-info` response can report both a rotated epoch and a changed
+ // branch, and the CC1 push has the same composition; running a second
+ // invalidation over the same entries would tear down providers the recycle
+ // just re-opened and advance the restart-recovery state machine off a
+ // snapshot this pass had already replaced. The recycle does everything this
+ // function would: it clears every entry's IDB and re-opens all of them
+ // against the branch the pool now holds. The buffer policy still differs and
+ // is the stricter one, but it no longer needs forcing from here — the replay
+ // drain fences on the buffer's own capture branch, so edits authored against
+ // the previous branch are discarded wherever the recycle came from. Same
+ // stand-down `flushOnHide` and `resyncOnVisible` take.
+ if (pool.isMismatchRecycleInFlight()) {
+ console.warn(
+ JSON.stringify({ event: 'ok-branch-switched-deferred-to-mismatch-recycle', branch }),
+ );
+ return;
+ }
+
const clears: Promise<void>[] = [];
for (const [docName, entry] of pool.entries) {
// TearingDown entries are transient and don't carry persistence.
diff --git a/packages/app/src/editor/provider-pool.test.ts b/packages/app/src/editor/provider-pool.test.ts
index 6ebbf2ae4..4693a9668 100644
--- a/packages/app/src/editor/provider-pool.test.ts
+++ b/packages/app/src/editor/provider-pool.test.ts
@@ -2205,6 +2205,235 @@ describe("ProviderPool authenticationFailed handling (US-002 / 'server-instance-
});
});
+// ---------------------------------------------------------------------------
+// Epoch observation as a transition, not a field write.
+//
+// `cachedServerInstanceId` serves two masters: it is the live-epoch cache
+// (auth claim + IDB name) AND it used to be the recycle-dedupe latch. Any
+// independent writer of the field could therefore disarm the latch. The
+// token-less `__system__` provider is exactly such a writer: it carries no
+// epoch claim, so the server never rejects it, and its post-restart refresh
+// reaches `setExpectedServerInstanceId` while per-doc providers are still
+// retrying a frozen stale claim. These tests pin the ordering both ways.
+// ---------------------------------------------------------------------------
+describe('ProviderPool epoch-transition observation', () => {
+ /** Count of `handleServerInstanceMismatch` entries that reached the recycle. */
+ function countRecycles(infoSpy: ReturnType<typeof vi.spyOn>): number {
+ return infoSpy.mock.calls.filter(([first]) => {
+ if (typeof first !== 'string') return false;
+ try {
+ return (JSON.parse(first) as { event?: string }).event === 'ok-pool-mismatch-recycle-begin';
+ } catch {
+ return false;
+ }
+ }).length;
+ }
+
+ test('observing a rotated epoch over HTTP recycles the pool', async () => {
+ const infoSpy = vi.spyOn(console, 'info').mockImplementation(() => undefined);
+ try {
+ pool = new ProviderPool(3, DUMMY_WS, { storage: null });
+ pool.setExpectedServerInstanceId('server-old');
+ const entry = pool.open('doc1');
+ if (!entry) throw new Error('expected entry');
+ pool.setActive('doc1');
+ const originalProvider = entry.provider;
+
+ // The refresher's sole pool call. Learning of a rotation here is
+ // semantically identical to learning of it from a rejected claim.
+ pool.setExpectedServerInstanceId('server-new');
+ await pool.awaitMismatchSettled();
+
+ expect(countRecycles(infoSpy)).toBe(1);
+ expect(pool.entries.get('doc1')?.provider).not.toBe(originalProvider);
+ // The transition must end with the observed epoch adopted, not merely
+ // cleared — otherwise the re-opened providers carry no claim at all.
+ expect(await pool.whenServerInstanceKnown()).toBe('server-new');
+ } finally {
+ infoSpy.mockRestore();
+ }
+ });
+
+ test('a stale claim rejected after the refresh landed the new epoch still recycles exactly once', async () => {
+ const infoSpy = vi.spyOn(console, 'info').mockImplementation(() => undefined);
+ try {
+ pool = new ProviderPool(3, DUMMY_WS, { storage: null });
+ pool.setExpectedServerInstanceId('server-old');
+ const entry = pool.open('doc1');
+ if (!entry) throw new Error('expected entry');
+ pool.setActive('doc1');
+ const originalProvider = entry.provider;
+
+ // The suppression interleaving: the token-less refresh wins the race
+ // to the pool, then the per-doc provider's frozen stale claim is
+ // rejected. Dedupe keys on the claim VALUE, so the epoch the refresh
+ // just cached cannot decide whether the transition already ran.
+ pool.setExpectedServerInstanceId('server-new');
+ originalProvider.emit('authenticationFailed', { reason: 'server-instance-mismatch' });
+ await pool.awaitMismatchSettled();
+
+ expect(countRecycles(infoSpy)).toBe(1);
+ expect(pool.entries.get('doc1')?.provider).not.toBe(originalProvider);
+ // Without the adoption the pool would recycle onto a null epoch and the
+ // re-opened providers would claim nothing, silently dropping the fence.
+ expect(await pool.whenServerInstanceKnown()).toBe('server-new');
+ } finally {
+ infoSpy.mockRestore();
+ }
+ });
+
+ test('a stale claim rejected before the refresh lands still recycles exactly once', async () => {
+ const infoSpy = vi.spyOn(console, 'info').mockImplementation(() => undefined);
+ try {
+ pool = new ProviderPool(3, DUMMY_WS, { storage: null });
+ pool.setExpectedServerInstanceId('server-old');
+ const entry = pool.open('doc1');
+ if (!entry) throw new Error('expected entry');
+ pool.setActive('doc1');
+
+ entry.provider.emit('authenticationFailed', { reason: 'server-instance-mismatch' });
+ // The refresh that follows the recycle is an adoption, not a second
+ // transition — the pool no longer holds the dead epoch.
+ pool.setExpectedServerInstanceId('server-new');
+ await pool.awaitMismatchSettled();
+
+ expect(countRecycles(infoSpy)).toBe(1);
+ } finally {
+ infoSpy.mockRestore();
+ }
+ });
+
+ test('re-observing the same epoch is a no-op', async () => {
+ const infoSpy = vi.spyOn(console, 'info').mockImplementation(() => undefined);
+ try {
+ pool = new ProviderPool(3, DUMMY_WS, { storage: null });
+ pool.setExpectedServerInstanceId('server-old');
+ const entry = pool.open('doc1');
+ if (!entry) throw new Error('expected entry');
+ pool.setActive('doc1');
+ const originalProvider = entry.provider;
+
+ pool.setExpectedServerInstanceId('server-old');
+ await pool.awaitMismatchSettled();
+
+ expect(countRecycles(infoSpy)).toBe(0);
+ expect(pool.entries.get('doc1')?.provider).toBe(originalProvider);
+ expect(await pool.whenServerInstanceKnown()).toBe('server-old');
+ } finally {
+ infoSpy.mockRestore();
+ }
+ });
+
+ test('the first epoch observed after boot is adopted without a recycle', async () => {
+ const infoSpy = vi.spyOn(console, 'info').mockImplementation(() => undefined);
+ try {
+ pool = new ProviderPool(3, DUMMY_WS, { storage: null });
+ const entry = pool.open('doc1');
+ if (!entry) throw new Error('expected entry');
+ pool.setActive('doc1');
+ const originalProvider = entry.provider;
+
+ pool.setExpectedServerInstanceId('server-first');
+ await pool.awaitMismatchSettled();
+
+ expect(countRecycles(infoSpy)).toBe(0);
+ expect(pool.entries.get('doc1')?.provider).toBe(originalProvider);
+ } finally {
+ infoSpy.mockRestore();
+ }
+ });
+
+ test('an epoch re-observed after a recycle is adopted without a second recycle', async () => {
+ const infoSpy = vi.spyOn(console, 'info').mockImplementation(() => undefined);
+ try {
+ pool = new ProviderPool(3, DUMMY_WS, { storage: null });
+ pool.setExpectedServerInstanceId('server-old');
+ const entry = pool.open('doc1');
+ if (!entry) throw new Error('expected entry');
+ pool.setActive('doc1');
+
+ pool.setExpectedServerInstanceId('server-new');
+ await pool.awaitMismatchSettled();
+ // A second refresh observing the epoch the pool already adopted.
+ pool.setExpectedServerInstanceId('server-new');
+ await pool.awaitMismatchSettled();
+
+ expect(countRecycles(infoSpy)).toBe(1);
+ } finally {
+ infoSpy.mockRestore();
+ }
+ });
+
+ test('two providers holding distinct stale claims collapse into one recycle', async () => {
+ const infoSpy = vi.spyOn(console, 'info').mockImplementation(() => undefined);
+ try {
+ pool = new ProviderPool(3, DUMMY_WS, { storage: null });
+ pool.setExpectedServerInstanceId('server-a');
+ const e1 = pool.open('doc1');
+ if (!e1) throw new Error('expected entry');
+ pool.setActive('doc1');
+ // Adopt a second epoch without a transition — the null-then-id shape the
+ // auth-rejection arm leaves behind mid-recovery. The entry admitted
+ // afterwards freezes a different claim, so both providers are live and
+ // claim-keyed dedupe alone cannot collapse them.
+ pool.setExpectedServerInstanceId(null);
+ pool.setExpectedServerInstanceId('server-b');
+ const e2 = pool.open('doc2');
+ if (!e2) throw new Error('expected entry');
+
+ const recyclesBefore = countRecycles(infoSpy);
+ e2.provider.emit('authenticationFailed', { reason: 'server-instance-mismatch' });
+ // e1 froze its claim under the earlier epoch, so claim-keyed dedupe
+ // does not cover it — only the in-flight latch collapses this one.
+ expect(pool.isMismatchRecycleInFlight()).toBe(true);
+ e1.provider.emit('authenticationFailed', { reason: 'server-instance-mismatch' });
+ await pool.awaitMismatchSettled();
+
+ expect(countRecycles(infoSpy) - recyclesBefore).toBe(1);
+ const collapsed = infoSpy.mock.calls.filter(([first]) => {
+ if (typeof first !== 'string') return false;
+ try {
+ const parsed = JSON.parse(first) as { event?: string; reason?: string };
+ return (
+ parsed.event === 'ok-pool-mismatch-transition-deduped' &&
+ parsed.reason === 'recycle-in-flight'
+ );
+ } catch {
+ return false;
+ }
+ });
+ expect(collapsed.length).toBe(1);
+ // Both halves: the claim that was collapsed, and the one the running
+ // recycle belongs to. Correlating a burst needs the pair.
+ const collapsedFields = JSON.parse(String(collapsed[0]?.[0])) as {
+ staleClaim?: string;
+ inflightForClaim?: string;
+ };
+ expect(collapsedFields.staleClaim).toBe('server-a');
+ expect(collapsedFields.inflightForClaim).toBe('server-b');
+ } finally {
+ infoSpy.mockRestore();
+ }
+ });
+
+ test('a disposed pool no longer reports a recycle in flight', async () => {
+ pool = new ProviderPool(3, DUMMY_WS, { storage: null });
+ pool.setExpectedServerInstanceId('server-old');
+ const entry = pool.open('doc1');
+ if (!entry) throw new Error('expected entry');
+ pool.setActive('doc1');
+
+ entry.provider.emit('authenticationFailed', { reason: 'server-instance-mismatch' });
+ expect(pool.isMismatchRecycleInFlight()).toBe(true);
+ pool.dispose();
+
+ // A latched flag would make every later whole-pool invalidation stand
+ // down against a recycle that can never settle.
+ expect(pool.isMismatchRecycleInFlight()).toBe(false);
+ pool = new ProviderPool(3, DUMMY_WS, { storage: null });
+ });
+});
+
describe('ProviderPool syncPromise lifecycle integration (F15)', () => {
beforeEach(() => {
__resetSyncPromiseCache();
@@ -4144,7 +4373,7 @@ describe('ProviderPool provider-open gating', () => {
}
});
- test('setExpectedServerInstanceId is a no-op for entries that already have persistence attached', async () => {
+ test('re-observing the live epoch leaves already-attached persistence alone', async () => {
pool = new ProviderPool(3, DUMMY_WS);
pool.setExpectedServerInstanceId('server-warm');
const docName = uniqueDocName('pp-warm');
@@ -4152,12 +4381,33 @@ describe('ProviderPool provider-open gating', () => {
if (!entry) throw new Error('expected entry');
const persistenceBefore = await awaitAttachedPersistence(entry);
- pool.setExpectedServerInstanceId('server-warm-update');
+ pool.setExpectedServerInstanceId('server-warm');
await wait(20);
if (entry.kind !== 'active') throw new Error('expected entry to remain active');
expect(entry.persistence).toBe(persistenceBefore);
});
+ test('observing a rotated epoch recycles the entry onto fresh persistence', async () => {
+ pool = new ProviderPool(3, DUMMY_WS);
+ pool.setExpectedServerInstanceId('server-warm');
+ const docName = uniqueDocName('pp-rotate');
+ const entry = pool.open(docName);
+ if (!entry) throw new Error('expected entry');
+ const persistenceBefore = await awaitAttachedPersistence(entry);
+
+ pool.setExpectedServerInstanceId('server-warm-update');
+ await pool.awaitMismatchSettled();
+
+ // The pre-rotation entry cannot keep serving: its Y.Doc merged items
+ // under the previous server's clientID and its IDB is epoch-scoped to
+ // the dead epoch.
+ const replaced = pool.entries.get(docName);
+ expect(replaced?.provider).not.toBe(entry.provider);
+ if (replaced !== undefined && replaced.kind === 'active' && replaced.persistence !== null) {
+ expect(replaced.persistence).not.toBe(persistenceBefore);
+ }
+ });
+
test('setExpectedServerInstanceId(null) does not detach already-attached persistence', async () => {
pool = new ProviderPool(3, DUMMY_WS);
pool.setExpectedServerInstanceId('server-warm');
diff --git a/packages/app/src/editor/provider-pool.ts b/packages/app/src/editor/provider-pool.ts
index 0dbee910e..a900c9a14 100644
--- a/packages/app/src/editor/provider-pool.ts
+++ b/packages/app/src/editor/provider-pool.ts
@@ -494,9 +494,16 @@ export class ProviderPool {
/**
* Live server instance ID observed from `/api/server-info` or CC1
* `server-info`. Drives the auth-token claim and the
- * `serverInstanceId` segment of the IndexedDB DB name. Cleared on
- * mismatch so the next epoch cleanly transitions through
- * `whenServerInstanceKnown()` + `attachDeferredPersistence`.
+ * `serverInstanceId` segment of the IndexedDB DB name.
+ *
+ * An epoch transition clears it only while it still holds the epoch just
+ * proven dead, so a fresher one observed on another channel survives to be
+ * claimed by the entries the recycle re-opens. What follows the clear
+ * depends on how the rotation was learned: the auth-rejection arm leaves
+ * the field null until a later observation adopts an epoch through
+ * `whenServerInstanceKnown()` + `attachDeferredPersistence`, while an
+ * observation that already carries the new epoch adopts it inline and
+ * leaves re-attachment to the recycle's own re-opens.
*/
private cachedServerInstanceId: string | null = null;
/**
@@ -507,10 +514,11 @@ export class ProviderPool {
* allocate a fresh handle bound to the next epoch transition.
*
* `null` arg to `setExpectedServerInstanceId` does NOT reject — the
- * pending handle stays alive until a real epoch lands. This matches the
- * mismatch-recycle path: the handler clears `cachedServerInstanceId` to
- * null mid-recovery, then the boot/refresh fetch races the new id back
- * into place.
+ * pending handle stays alive until a real epoch lands. That matters on the
+ * auth-rejection arm, where the transition clears the epoch mid-recovery
+ * and a later boot/refresh fetch races the new one back into place. An
+ * observation that reports the rotation directly has no such window: it
+ * resolves the handle inline, in the same call that starts the transition.
*/
private pendingServerInstanceKnown: {
promise: Promise<string>;
@@ -522,6 +530,24 @@ export class ProviderPool {
* structured client telemetry alongside `docName` / `branch`.
*/
private recoveryMismatchStaleClaim: string | undefined;
+ /**
+ * Dead epochs whose transition has already been routed through
+ * `handleServerInstanceMismatch`.
+ *
+ * Dedupe keys on the stale claim VALUE rather than on
+ * `cachedServerInstanceId` equality. That field is the live-epoch cache
+ * (auth claim + IDB name) and has more than one legitimate writer, so
+ * using it as the latch let an unrelated writer decide whether a
+ * transition had run: an epoch observed over HTTP while per-doc providers
+ * were still retrying a frozen stale claim used to suppress their recycle
+ * entirely. The claim value is immutable per provider and cannot be
+ * disturbed that way.
+ *
+ * Bounded by construction — a provider freezes exactly one claim at
+ * admission, so the set holds at most one entry per pool slot within a
+ * recycle window, and it is dropped when the recycle settles.
+ */
+ private readonly handledStaleClaims = new Set<string>();
/**
* Unsynced-edit buffer captured per-doc during a `server-instance-mismatch`
* recycle. Populated right before `clearData()` wipes IDB; drained at the
@@ -651,29 +677,74 @@ export class ProviderPool {
}
/**
- * Update the live server instance ID observed from `/api/server-info` or CC1
- * `server-info`. Does NOT overwrite the storage-backed IDB-associated ID:
- * a fast boot fetch after server restart must not mask stale IDB contents
- * before the first document provider opens.
+ * Observe the live server epoch reported by `/api/server-info` or CC1
+ * `server-info`. This is an observation entry point, not a field setter:
+ * the pool owns the epoch-transition contract, and every channel that can
+ * learn of a rotation (HTTP refresh, CC1 push, a rejected auth claim) is a
+ * reporter routed through the same handling.
*
- * On a non-null id this also (a) resolves any pending
- * `whenServerInstanceKnown()` handle and (b) retroactively attaches
- * persistence to entries opened during the cold-boot window before the
- * epoch was known. Persistence is `IndexeddbPersistence`-backed and the
- * DB-name shape `ok-ydoc:${branch}:${serverInstanceId}:${docName}`
- * carries the epoch as a structural correctness signal; opening a
- * provider before the live epoch is known means the DB cannot be
- * attached at admission time without picking the wrong epoch.
+ * Three observations, three outcomes:
+ *
+ * - **Same epoch** — no-op. The common case on every `__system__`
+ * reconnect when the server did not restart.
+ * - **First epoch after boot or after a recycle** (`null` → id) — adopt.
+ * Resolves any pending `whenServerInstanceKnown()` handle and
+ * retroactively attaches persistence to entries opened during the
+ * window before the epoch was known. Persistence is
+ * `IndexeddbPersistence`-backed and the DB-name shape
+ * `ok-ydoc:${branch}:${serverInstanceId}:${docName}` carries the epoch
+ * as a structural correctness signal, so a provider opened before the
+ * live epoch is known cannot attach at admission time without picking
+ * the wrong epoch.
+ * - **Rotation** (idA → idB) — route through the same transition handler
+ * the auth-rejection path uses, then adopt idB. Learning that the
+ * server rotated from an HTTP response is semantically identical to
+ * learning it from a rejected claim, and the per-doc providers holding
+ * frozen claims on idA cannot deliver that news themselves once the
+ * token-less `__system__` channel has already reported it.
+ *
+ * Does NOT overwrite the storage-backed IDB-associated ID: a fast boot
+ * fetch after a server restart must not mask stale IDB contents before
+ * the first document provider opens.
*/
setExpectedServerInstanceId(id: string | null): void {
- this.cachedServerInstanceId = id;
- if (id === null || id.length === 0) return;
- if (this.pendingServerInstanceKnown !== null) {
- const pending = this.pendingServerInstanceKnown;
- this.pendingServerInstanceKnown = null;
- pending.resolve(id);
+ const observed = id !== null && id.length > 0 ? id : null;
+ const known =
+ this.cachedServerInstanceId !== null && this.cachedServerInstanceId.length > 0
+ ? this.cachedServerInstanceId
+ : null;
+
+ if (known !== null && observed === known) return;
+
+ if (known !== null && observed !== null) {
+ // The handler drops the dead epoch from the cache before it runs, so
+ // it sees the same "no live epoch" state the auth-rejection path
+ // hands it. Adoption of the new epoch happens below, after the
+ // transition is under way, so the entries the recycle re-opens claim
+ // the epoch that is actually live.
+ this.handleServerInstanceMismatch(known);
+ this.cachedServerInstanceId = observed;
+ this.resolvePendingServerInstanceKnown(observed);
+ // Deliberately no `attachDeferredPersistence` here. The recycle
+ // re-opens every entry and attaches persistence at admission under
+ // the new epoch; attaching mid-transition would instead bind a
+ // pre-restart Y.Doc to a fresh-epoch IDB and seed it with the exact
+ // stale state the recycle exists to discard.
+ return;
}
- this.attachDeferredPersistence(id);
+
+ this.cachedServerInstanceId = id;
+ if (observed === null) return;
+ this.resolvePendingServerInstanceKnown(observed);
+ this.attachDeferredPersistence(observed);
+ }
+
+ /** Hand a newly-adopted epoch to whoever is waiting on one. */
+ private resolvePendingServerInstanceKnown(id: string): void {
+ if (this.pendingServerInstanceKnown === null) return;
+ const pending = this.pendingServerInstanceKnown;
+ this.pendingServerInstanceKnown = null;
+ pending.resolve(id);
}
/**
@@ -1926,9 +1997,12 @@ export class ProviderPool {
// bytes back onto the Y.Doc so the user's unsynced edits survive.
//
// Idempotence: after a server restart, every open provider fires
- // authenticationFailed in quick succession. The first call clears the
- // IDB-associated claim; sibling failures with the same stale claim then
- // short-circuit while preserving any already-observed fresh server ID.
+ // authenticationFailed in quick succession. The transition handler
+ // dedupes on the stale claim value, so siblings reporting the same dead
+ // epoch short-circuit there, and an in-flight recycle collapses siblings
+ // reporting different ones. Clearing the cached epoch is a consequence of
+ // the transition, not the dedupe mechanism, and is conditional so a
+ // fresher epoch already observed on another channel survives.
const onAuthenticationFailed = ({ reason }: { reason: string }): void => {
// Trust-boundary narrow: `reason` is a wire-foreign string from
// Hocuspocus. Inlined (not imported from the server's runtime
@@ -1969,20 +2043,14 @@ export class ProviderPool {
const payload: string | undefined = rawPayload.length > 0 ? rawPayload : undefined;
const typed = candidateKind as HocuspocusAuthRejectionReason;
if (typed === 'server-instance-mismatch') {
- // `expectedServerInstanceId` is the claim this provider sent at
- // construction time. Idempotence: the first authenticationFailed
- // for this epoch transition clears `cachedServerInstanceId`;
- // every later sibling event observes `cached !== expected` and
- // short-circuits so the recycle path runs exactly once.
+ // `expectedServerInstanceId` is the claim this provider froze at
+ // construction time, and the rejection proves that epoch is dead.
+ // Idempotence lives in the transition handler, keyed by the claim
+ // value: a provider admitted without a claim has nothing to report.
if (expectedServerInstanceId === null) {
return;
}
- if (this.cachedServerInstanceId !== expectedServerInstanceId) {
- return;
- }
- const staleClaimFromToken = expectedServerInstanceId;
- this.cachedServerInstanceId = null;
- this.handleServerInstanceMismatch(staleClaimFromToken);
+ this.handleServerInstanceMismatch(expectedServerInstanceId);
return;
}
// Branch-mismatch is the late-join backstop for the cross-branch
@@ -2169,6 +2237,22 @@ export class ProviderPool {
let tokenBacked: boolean;
const buffered = this.bufferedUpdates.get(docName);
if (buffered !== undefined) {
+ if (buffered.branch !== branch) {
+ // Provenance fence. The buffer was captured against a different
+ // branch's content, and edits authored on one branch are not valid
+ // against another's — the same "discard, don't preserve" policy the
+ // cross-branch invalidation applies, enforced here so it holds no
+ // matter which channel triggered this recycle. Discard drops the
+ // RAM copy and its durable mirror together, so the branch this
+ // buffer belonged to keeps no resurrectable record.
+ this.discardBufferedUpdate(docName);
+ this.emitStructuredClientRecoveryEvent({
+ event: 'ok-buffer-replay-branch-mismatch',
+ ...this.recoveryTelemetryBase(docName, staleClaimAtReplayInstall),
+ reason: 'buffer-captured-on-another-branch',
+ });
+ return;
+ }
this.bufferedUpdates.delete(docName);
source = buffered;
tokenBranch = buffered.branch;
@@ -2400,6 +2484,38 @@ export class ProviderPool {
* never rethrown into Hocuspocus's event emitter.
*/
private handleServerInstanceMismatch(staleClaimedServerInstanceId: string): void {
+ if (this.handledStaleClaims.has(staleClaimedServerInstanceId)) {
+ // The claim rides along because these breadcrumbs are the diagnostic
+ // for this race, and "dedupe fired" without "for which epoch" cannot be
+ // correlated across a burst. Server instance ids are server-generated
+ // UUIDs, so the cardinality discipline is satisfied.
+ this.emitStructuredClientBreadcrumb({
+ event: 'ok-pool-mismatch-transition-deduped',
+ reason: 'claim-already-handled',
+ staleClaim: staleClaimedServerInstanceId,
+ });
+ return;
+ }
+ this.handledStaleClaims.add(staleClaimedServerInstanceId);
+ // Drop the cache only while it still holds the epoch just proven dead.
+ // A fresher epoch already observed on another channel must survive so
+ // the entries this recycle re-opens claim the one that is live.
+ if (this.cachedServerInstanceId === staleClaimedServerInstanceId) {
+ this.cachedServerInstanceId = null;
+ }
+ if (this.mismatchInFlight !== null) {
+ // A recycle already covers every pool entry, so a sibling transition
+ // reported under a different stale claim must not start a second one.
+ // Claim-keyed dedupe alone would miss this collapse.
+ this.emitStructuredClientBreadcrumb({
+ event: 'ok-pool-mismatch-transition-deduped',
+ reason: 'recycle-in-flight',
+ staleClaim: staleClaimedServerInstanceId,
+ inflightForClaim: this.recoveryMismatchStaleClaim ?? '',
+ });
+ return;
+ }
+
this.recoveryMismatchStaleClaim =
staleClaimedServerInstanceId.length > 0 ? staleClaimedServerInstanceId : undefined;
@@ -2654,6 +2770,13 @@ export class ProviderPool {
).finally(() => {
if (this.mismatchInFlight === inflight) {
this.mismatchInFlight = null;
+ // Every entry has been re-opened under a fresh claim, so nothing
+ // in the pool can still report these epochs. Dropping them keeps
+ // the set bounded and leaves the retry path open for an entry the
+ // recycle left inert on a failed clear: its next rejection carries
+ // the same claim and gets a real transition rather than a silent
+ // bail.
+ this.handledStaleClaims.clear();
}
});
this.mismatchInFlight = inflight;
@@ -2764,6 +2887,19 @@ export class ProviderPool {
return this.mismatchInFlight ?? Promise.resolve();
}
+ /**
+ * True while a `server-instance-mismatch` recycle owns the pool.
+ *
+ * The stand-down signal for any other whole-pool invalidation. `flushOnHide`
+ * and `resyncOnVisible` read the private field directly; the cross-branch
+ * invalidation lives outside this class and needs the same answer, because
+ * one `/api/server-info` dispatch can report a rotated epoch and a changed
+ * branch together and must still run a single invalidation.
+ */
+ isMismatchRecycleInFlight(): boolean {
+ return this.mismatchInFlight !== null;
+ }
+
/**
* Recycle every pool entry by calling `recycleDisconnectedEntry` for each.
* Called by the `authenticationFailed` handler on `server-instance-mismatch`
@@ -3326,6 +3462,9 @@ export class ProviderPool {
// harness) starts from a clean slate. Leaving the flag set would
// cause the next pool to retry a clear we've already abandoned.
this.clearFailures.clear();
+ // Epoch-transition dedupe is per-pool: a fresh pool holds no providers
+ // that could still report these claims.
+ this.handledStaleClaims.clear();
// In-memory lineage records die with the pool; the storage envelope
// deliberately survives — a fresh pool (new tab, HMR remount) re-reads
// and instance-validates it, which is the cross-tab channel the
@@ -3334,6 +3473,9 @@ export class ProviderPool {
this.docLineageEpochsEnvelopeConsumed = false;
this.onBranchMismatch = null;
this.branchMismatchInFlight = null;
+ // Same reset as its branch-side sibling: a disposed pool must not report
+ // a recycle as in flight, or the stand-down would latch on forever.
+ this.mismatchInFlight = null;
this.onRenameRedirect = null;
this.onDocDeleted = null;
this.evictListeners.clear();
diff --git a/packages/app/src/lib/server-info-refresh.ts b/packages/app/src/lib/server-info-refresh.ts
index 7669ee9d7..d05c1c6c8 100644
--- a/packages/app/src/lib/server-info-refresh.ts
+++ b/packages/app/src/lib/server-info-refresh.ts
@@ -18,15 +18,16 @@
* `lastDiskAckedSV` watermark so the mismatch-recycle baseline-
* selection always operates on fresh data.
*
- * Mostly idempotent: `compareAndUpdateObservedBranch` returns false
- * unless the branch actually changed, and `observeDiskAckBatch`
- * overwrites in-place. `setExpectedServerInstanceId` is NOT — the pool
- * assigns unconditionally (see provider-pool.ts), so a refresh that
- * observes a rotated server id overwrites the cached epoch even while
- * per-doc providers are still learning of the rotation through auth
- * rejection, and their recycle dedupe then compares against the
- * already-updated value. Do not rely on this path to no-op on
- * unchanged ids until the pool owns epoch transitions.
+ * Idempotent on an unchanged server: `compareAndUpdateObservedBranch`
+ * returns false unless the branch actually changed,
+ * `observeDiskAckBatch` overwrites in-place, and
+ * `setExpectedServerInstanceId` early-returns on a matching id. When the
+ * id does NOT match, this path is a rotation report rather than a cache
+ * write — the pool routes it through the same epoch transition a
+ * rejected auth claim takes, and recycles. That matters here because
+ * `__system__` carries no epoch claim of its own, so this refresher is
+ * often the first channel to observe a rotation while per-doc providers
+ * are still retrying tokens frozen with the dead epoch.
*
* Silent on failure: endpoint unavailability falls back to the
* existing recovery paths (auth-token-claim mismatch on next provider
diff --git a/packages/app/tests/integration/system-refresh-epoch-rotation.test.ts b/packages/app/tests/integration/system-refresh-epoch-rotation.test.ts
new file mode 100644
index 000000000..155abc435
--- /dev/null
+++ b/packages/app/tests/integration/system-refresh-epoch-rotation.test.ts
@@ -0,0 +1,127 @@
+/**
+ * The token-less `__system__` refresh must not suppress per-doc recycle.
+ *
+ * `__system__` is constructed without an auth token, so it carries no epoch
+ * claim and the server's stale-claim rejection never applies to it. After a
+ * server restart it therefore re-syncs freely and its reconnect refresh
+ * reaches the pool over HTTP, while per-doc providers are still retrying
+ * tokens frozen with the dead epoch. If that refresh is allowed to be a
+ * plain field write, the pool caches the new epoch and every subsequent
+ * per-doc rejection is discarded as "already handled" — the docs stay
+ * offline for the rest of the session, showing a reconnect toast that
+ * promises edits will sync.
+ *
+ * Both halves have to be wired at once for the composition to be observable:
+ * the refresher alone recycles nothing, and a restart without a `__system__`
+ * subscriber never produces the suppressing write. The ordering is forced
+ * rather than raced, so this is a falsifier and not a coin flip.
+ */
+
+import './idb-preload';
+import { writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { setTimeout as wait } from 'node:timers/promises';
+import { afterEach, describe, expect, test, vi } from 'vitest';
+import { ProviderPool } from '../../src/editor/provider-pool';
+import { refreshServerInfo } from '../../src/lib/server-info-refresh';
+import {
+ attachSystemDocSubscriber,
+ createRestartableServer,
+ pollUntil,
+ seedPoolServerInstanceId,
+} from './test-harness';
+
+const FIXTURE = `# Rotation Fixture
+
+Body paragraph that must survive the recycle.
+`;
+
+const cleanups: Array<() => Promise<void> | void> = [];
+
+afterEach(async () => {
+ while (cleanups.length > 0) {
+ await cleanups.pop()?.();
+ }
+}, 30_000);
+
+/** Recycle entries observed on the structured breadcrumb channel. */
+function countRecycleBegins(calls: readonly unknown[][]): number {
+ return calls.filter((call) => {
+ const first = call[0];
+ if (typeof first !== 'string') return false;
+ try {
+ return (JSON.parse(first) as { event?: string }).event === 'ok-pool-mismatch-recycle-begin';
+ } catch {
+ return false;
+ }
+ }).length;
+}
+
+describe('__system__ refresh across a server epoch rotation', () => {
+ test('a refresh that observes the rotated epoch first still recycles the per-doc pool', async () => {
+ const infoSpy = vi.spyOn(console, 'info').mockImplementation(() => undefined);
+ cleanups.push(() => {
+ infoSpy.mockRestore();
+ });
+
+ let server = await createRestartableServer();
+ cleanups.push(() => server.shutdown());
+
+ const docName = 'system-refresh-rotation-doc';
+ writeFileSync(join(server.contentDir, `${docName}.md`), FIXTURE, 'utf-8');
+
+ const pool = new ProviderPool(3, `ws://127.0.0.1:${server.port}/collab`, { storage: null });
+ cleanups.push(() => pool.dispose());
+ const firstEpoch = await seedPoolServerInstanceId(server, pool);
+
+ pool.open(docName);
+ pool.setActive(docName);
+ await pollUntil(() => pool.getActive()?.provider.isSynced === true, 10_000, 50);
+ await pollUntil(() => pool.getActive()?.provider.unsyncedChanges === 0, 10_000, 50);
+ const staleProvider = pool.getActive()?.provider;
+ if (!staleProvider) throw new Error('expected an active provider before the restart');
+
+ // The half that was never wired alongside a recycle assertion.
+ const systemSub = attachSystemDocSubscriber(pool, server.port);
+ cleanups.push(() => systemSub.dispose());
+ await wait(200);
+
+ // Short downtime keeps the pool's debounced disconnect-recycle out of the
+ // picture, so any recycle observed below is attributable to the epoch
+ // transition and nothing else.
+ server = await server.killAndRestartOnSamePort({ downtimeMs: 300 });
+ cleanups.unshift(() => server.shutdown());
+
+ const recyclesBeforeRefresh = countRecycleBegins(infoSpy.mock.calls);
+
+ // Force the arm rather than race it. Hocuspocus's first reconnect attempt
+ // is a second out, so the per-doc provider cannot have been rejected yet:
+ // whatever this observes is the refresh reaching the pool alone. This is
+ // the same call the `__system__` reconnect gate makes in production.
+ await refreshServerInfo(pool, `http://127.0.0.1:${server.port}`);
+ const recyclesAfterRefresh = countRecycleBegins(infoSpy.mock.calls);
+
+ expect(recyclesAfterRefresh).toBeGreaterThan(recyclesBeforeRefresh);
+
+ // The epoch really did rotate, so the suppression would have been real.
+ const secondEpoch = await pool.whenServerInstanceKnown();
+ expect(secondEpoch).not.toBe(firstEpoch);
+
+ // User-visible outcome: the doc comes back rather than retrying a frozen
+ // claim forever.
+ await pool.awaitMismatchSettled();
+ await pollUntil(
+ () => {
+ const entry = pool.entries.get(docName);
+ return entry !== undefined && entry.provider !== staleProvider;
+ },
+ 10_000,
+ 50,
+ );
+ expect(pool.entries.get(docName)?.provider).not.toBe(staleProvider);
+
+ await pollUntil(() => pool.getActive()?.provider.isSynced === true, 15_000, 50);
+ const body = pool.getActive()?.provider.document.getText('source').toString() ?? '';
+ expect(body).toContain('Body paragraph that must survive the recycle.');
+ }, 60_000);
+});