Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(storage): recover closed IndexedDB connections (#7828)
Peter Chinman087df8a42f8c
diff --git a/apps/web/src/lib/persistence/recoverable-idb.test.ts b/apps/web/src/lib/persistence/recoverable-idb.test.tsnew file mode 100644index 0000000000..074aa6a48d--- /dev/null+++ b/apps/web/src/lib/persistence/recoverable-idb.test.ts@@ -0,0 +1,117 @@+import 'fake-indexeddb/auto';+import { forceCloseDatabase } from 'fake-indexeddb';+import { afterEach, describe, expect, it, vi } from 'vitest';+import { RecoverableIDB } from './recoverable-idb';++const databases: RecoverableIDB[] = [];+function database() {+ const db = new RecoverableIDB(crypto.randomUUID(), 1, (db) =>+ db.createObjectStore('items')+ );+ databases.push(db);+ return db;+}+afterEach(async () => {+ vi.restoreAllMocks();+ for (const db of databases.splice(0)) await db.close();+});++describe('recoverable native IndexedDB connections', () => {+ it.each(['closed', 'terminated'])(+ 'shares one replacement after a connection is %s',+ async (reason) => {+ const db = database();+ const original = await db.open();+ const open = vi.spyOn(indexedDB, 'open');+ if (reason === 'closed') original.close();+ else {+ const closed = new Promise<void>((resolve) =>+ original.addEventListener('close', () => resolve())+ );+ // The library accepts an instance but declares a constructor type.+ forceCloseDatabase(+ original as unknown as Parameters<typeof forceCloseDatabase>[0]+ );+ await closed;+ }+ const [first, second] = await Promise.all([+ db.transaction('items', 'readonly'),+ db.transaction('items', 'readonly'),+ ]);+ expect(first.db).toBe(second.db);+ expect(first.db).not.toBe(original);+ expect(open).toHaveBeenCalledOnce();+ original.onclose?.call(original, new Event('close'));+ expect(await db.open()).toBe(first.db);+ }+ );++ it('bounds transaction creation retries and permits a later recovery', async () => {+ const db = database();+ await db.open();+ const error = new DOMException('Closing', 'InvalidStateError');+ const transaction = vi+ .spyOn(IDBDatabase.prototype, 'transaction')+ .mockImplementation(() => {+ throw error;+ });+ await expect(db.transaction('items', 'readwrite')).rejects.toBe(error);+ expect(transaction).toHaveBeenCalledTimes(2);+ transaction.mockRestore();+ await expect(db.transaction('items', 'readwrite')).resolves.toBeDefined();+ });++ it('does not retry unrelated errors', async () => {+ const db = database();+ const native = await db.open();+ const error = new DOMException('Quota exhausted', 'QuotaExceededError');+ const transaction = vi+ .spyOn(native, 'transaction')+ .mockImplementationOnce(() => {+ throw error;+ });+ await expect(db.transaction('items', 'readwrite')).rejects.toBe(error);+ expect(transaction).toHaveBeenCalledOnce();+ expect(await db.open()).toBe(native);+ });++ it('releases connections for version changes', async () => {+ const db = database();+ const native = await db.open();+ const upgraded = await new Promise<IDBDatabase>((resolve, reject) => {+ const request = indexedDB.open(native.name, 2);+ request.onsuccess = () => resolve(request.result);+ request.onerror = () => reject(request.error);+ });+ upgraded.close();+ expect(() => native.transaction('items')).toThrow();+ const open = vi.spyOn(indexedDB, 'open');+ await expect(db.open()).rejects.toMatchObject({ name: 'VersionError' });+ await expect(db.open()).rejects.toMatchObject({ name: 'VersionError' });+ expect(open).toHaveBeenCalledTimes(2);+ });++ it('does not cache synchronous opening failures', async () => {+ const db = database();+ vi.spyOn(indexedDB, 'open').mockImplementationOnce(() => {+ throw new Error('Unavailable');+ });+ await expect(db.transaction('items', 'readonly')).rejects.toThrow(+ 'Unavailable'+ );+ await expect(db.transaction('items', 'readonly')).resolves.toBeDefined();+ });++ it('closes an in-flight open and never revives a disposed owner', async () => {+ const db = database();+ const pending = db.transaction('items', 'readonly');+ const failure = expect(pending).rejects.toMatchObject({+ name: 'InvalidStateError',+ });+ await db.close();+ await failure;+ await expect(db.transaction('items', 'readonly')).rejects.toMatchObject({+ name: 'InvalidStateError',+ });+ });+});diff --git a/apps/web/src/lib/persistence/recoverable-idb.ts b/apps/web/src/lib/persistence/recoverable-idb.tsnew file mode 100644index 0000000000..7a85371c91--- /dev/null+++ b/apps/web/src/lib/persistence/recoverable-idb.ts@@ -0,0 +1,93 @@+/** Reopens stale connections without replaying requests or started transactions. */+export class RecoverableIDB {+ private connection?: Promise<IDBDatabase>;+ private disposed = false;++ constructor(+ private readonly name: string,+ private readonly version: number,+ private readonly upgrade: (db: IDBDatabase, tx: IDBTransaction) => void,+ private readonly onVersionChange?: () => void+ ) {}++ open(): Promise<IDBDatabase> {+ if (this.disposed)+ throw new DOMException(+ 'Database connection disposed',+ 'InvalidStateError'+ );+ if (this.connection) return this.connection;+ const request = indexedDB.open(this.name, this.version);+ const opening = new Promise<IDBDatabase>((resolve, reject) => {+ request.onupgradeneeded = () =>+ this.upgrade(request.result, request.transaction!);+ request.onerror = () => {+ if (this.connection === opening) this.connection = undefined;+ reject(request.error);+ };+ request.onsuccess = () => {+ const db = request.result;+ const invalidate = () => {+ if (this.connection === opening) this.connection = undefined;+ };+ db.onclose = invalidate;+ db.onversionchange = () => {+ db.close();+ invalidate();+ this.onVersionChange?.();+ };+ if (this.disposed) {+ db.close();+ invalidate();+ reject(+ new DOMException(+ 'Database connection disposed',+ 'InvalidStateError'+ )+ );+ return;+ }+ resolve(db);+ };+ });+ this.connection = opening;+ return opening;+ }++ async transaction(stores: string | string[], mode: IDBTransactionMode) {+ for (let attempt = 0; ; attempt++) {+ const opening = this.open();+ const db = await opening;+ if (this.disposed)+ throw new DOMException(+ 'Database connection disposed',+ 'InvalidStateError'+ );+ try {+ return db.transaction(stores, mode);+ } catch (error) {+ if (+ !(error instanceof DOMException) ||+ error.name !== 'InvalidStateError'+ )+ throw error;+ if (this.connection === opening) this.connection = undefined;+ db.close();+ // Creation failed before any requests ran, so a single retry is safe.+ if (attempt === 1) throw error;+ }+ }+ }++ /** Terminal close: pending opens cannot revive a disposed owner. */+ async close(): Promise<void> {+ this.disposed = true;+ const opening = this.connection;+ this.connection = undefined;+ try {+ (await opening)?.close();+ } catch {+ // A failed open has no connection to close; its caller observes the error.+ }+ }+}diff --git a/apps/web/src/lib/queries/email/local-draft-store.test.ts b/apps/web/src/lib/queries/email/local-draft-store.test.tsindex 9d081d47f3..1ebde171cb 100644--- a/apps/web/src/lib/queries/email/local-draft-store.test.ts+++ b/apps/web/src/lib/queries/email/local-draft-store.test.ts@@ -1,7 +1,10 @@import { Blob as NodeBlob } from 'node:buffer';import 'fake-indexeddb/auto';import type { LocalDraft } from '@app/features/email-compose/core/local-draft';-import { afterEach, describe, expect, it, vi } from 'vitest';+import { createDraftAutosave } from '@app/features/email-compose/primitives/draft-autosave';+import { forceCloseDatabase } from 'fake-indexeddb';+import { createRoot } from 'solid-js';+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';import { createLocalDraftStore } from './local-draft-store';const stores: ReturnType<typeof createLocalDraftStore>[] = [];@@ -32,6 +35,197 @@ afterEach(async () => {vi.restoreAllMocks();});+describe('draft connection recovery', () => {+ let connections: IDBDatabase[];+ beforeEach(() => {+ connections = [];+ const original = indexedDB.open.bind(indexedDB);+ vi.spyOn(indexedDB, 'open').mockImplementation((...args) => {+ const request = original(...args);+ request.addEventListener('success', () =>+ connections.push(request.result)+ );+ return request;+ });+ });++ it.each(['closed', 'terminated'])(+ 'saves the latest edit and continues server autosave after a connection is %s',+ async (reason) => {+ const store = open();+ const session = await store.activate('owner');+ await store.save(session, snapshot());+ if (reason === 'closed') connections[0].close();+ else {+ const db = connections[0];+ const closed = new Promise<void>((resolve) =>+ db.addEventListener('close', () => resolve(), { once: true })+ );+ // fake-indexeddb accepts an instance but declares a constructor type.+ forceCloseDatabase(+ db as unknown as Parameters<typeof forceCloseDatabase>[0]+ );+ await closed;+ }+ const latest = snapshot('After resuming', 1);+ const persist = vi.fn(async () => {+ expect(await store.read(session, 'local')).toMatchObject({+ revision: 2,+ content: latest.content,+ });+ });+ const onLocalError = vi.fn();+ let dispose = () => {};+ const autosave = createRoot((cleanup) => {+ dispose = cleanup;+ return createDraftAutosave({+ capture: () => latest,+ saveLocalSnapshot: async (value) => {+ await store.save(session, value);+ },+ persist,+ paused: () => false,+ onLocalError,+ });+ });+ try {+ await autosave.save();+ expect(persist).toHaveBeenCalledExactlyOnceWith(latest);+ expect(onLocalError).not.toHaveBeenCalled();+ expect(autosave.localSaveState()).toBe('saved');+ expect(connections).toHaveLength(2);+ } finally {+ dispose();+ }+ }+ );++ it('shares a reopened connection across concurrent operations without duplicate revisions', async () => {+ const store = open();+ const session = await store.activate('owner');+ await store.save(session, snapshot());+ connections[0].close();+ const notify = vi.fn();+ store.subscribe(notify);+ await Promise.all([+ store.save(session, snapshot('New revision', 1)),+ store.list(session),+ store.read(session, 'local'),+ ]);+ expect(connections).toHaveLength(2);+ expect(notify).toHaveBeenCalledOnce();+ expect(await store.read(session, 'local')).toMatchObject({ revision: 2 });+ });++ it('bounds retries, reports one failure per episode and allows a later recovery', async () => {+ const report = vi.fn();+ const store = createLocalDraftStore(crypto.randomUUID(), report);+ stores.push(store);+ const session = await store.activate('owner');+ const error = new DOMException(+ 'Connection is closing',+ 'InvalidStateError'+ );+ const transaction = vi+ .spyOn(IDBDatabase.prototype, 'transaction')+ .mockImplementation(() => {+ throw error;+ });+ await expect(store.save(session, snapshot())).rejects.toBe(error);+ expect(transaction).toHaveBeenCalledTimes(2);+ await expect(store.save(session, snapshot())).rejects.toBe(error);+ expect(transaction).toHaveBeenCalledTimes(4);+ expect(report).toHaveBeenCalledExactlyOnceWith({ type: 'failed', error });+ transaction.mockRestore();+ await store.save(session, snapshot());+ expect(report).toHaveBeenLastCalledWith({ type: 'recovered' });+ expect(await store.read(session, 'local')).toMatchObject({ revision: 1 });+ });++ it('does not retry unrelated transaction errors', async () => {+ const store = open();+ const session = await store.activate('owner');+ const error = new DOMException('Quota exhausted', 'QuotaExceededError');+ const transaction = vi+ .spyOn(connections[0], 'transaction')+ .mockImplementationOnce(() => {+ throw error;+ });+ await expect(store.save(session, snapshot())).rejects.toBe(error);+ expect(transaction).toHaveBeenCalledOnce();+ expect(connections).toHaveLength(1);+ await store.save(session, snapshot());+ });++ it('never replays a started transaction when a request throws InvalidStateError', async () => {+ const store = open();+ const session = await store.activate('owner');+ const error = new DOMException('Request failed', 'InvalidStateError');+ const get = vi+ .spyOn(IDBObjectStore.prototype, 'get')+ .mockImplementationOnce(() => {+ throw error;+ });+ await expect(store.save(session, snapshot())).rejects.toBe(error);+ expect(get).toHaveBeenCalledOnce();+ expect(connections).toHaveLength(1);+ await store.save(session, snapshot());+ expect(await store.read(session, 'local')).toMatchObject({ revision: 1 });+ });++ it('keeps account fences after reopening', async () => {+ const name = crypto.randomUUID();+ const store = open(name);+ const session = await store.activate('owner');+ await store.save(session, snapshot());+ connections[0].close();+ const other = open(name);+ await other.clear(true);+ await other.activate('other-owner');+ await expect(store.save(session, snapshot('Late edit', 1))).rejects.toThrow(+ 'no longer active'+ );+ expect(await store.list(session)).toEqual([]);+ });++ it('retries a later open after a synchronous opening failure', async () => {+ const store = open();+ vi.mocked(indexedDB.open).mockImplementationOnce(() => {+ throw new DOMException('Storage unavailable', 'UnknownError');+ });+ await expect(store.activate('owner')).rejects.toThrow(+ 'Storage unavailable'+ );+ const session = await store.activate('owner');+ await store.save(session, snapshot());+ expect(await store.list(session)).toHaveLength(1);+ });++ it('does not let an old close event invalidate the replacement connection', async () => {+ const store = open();+ const session = await store.activate('owner');+ const old = connections[0];+ old.close();+ await store.save(session, snapshot());+ old.onclose?.call(old, new Event('close'));+ await store.list(session);+ expect(connections).toHaveLength(2);+ });++ it('keeps persistence working when recovery reporting throws', async () => {+ const report = vi.fn(() => {+ throw new Error('Telemetry unavailable');+ });+ const store = createLocalDraftStore(crypto.randomUUID(), report);+ stores.push(store);+ const session = await store.activate('owner');+ connections[0].close();+ await store.save(session, snapshot());+ expect(report).toHaveBeenCalledExactlyOnceWith({ type: 'recovered' });+ expect(await store.list(session)).toHaveLength(1);+ });+});+describe('durable email working copies', () => {it('retains rejected content and further edits across reopening', async () => {const name = crypto.randomUUID();diff --git a/apps/web/src/lib/queries/email/local-draft-store.ts b/apps/web/src/lib/queries/email/local-draft-store.tsindex 275c7d446c..9c1345a0bd 100644--- a/apps/web/src/lib/queries/email/local-draft-store.ts+++ b/apps/web/src/lib/queries/email/local-draft-store.ts@@ -12,11 +12,25 @@ type Session = { accountId: string; epoch: string };type StoredDraft = LocalDraft & { epoch: string };const STORES = ['drafts', 'files', 'attempts', 'meta'] as const;+export type DraftStorageEvent =+ | { type: 'closed' | 'recovered' }+ | { type: 'failed'; error: unknown };+/** Transactional working-copy storage. Unlike a query cache, it never evicts edits. */export function createLocalDraftStore(- dbName = 'macro-email-working-copies-v1'+ dbName = 'macro-email-working-copies-v1',+ onStorageEvent?: (event: DraftStorageEvent) => void) {let connection: Promise<IDBDatabase> | undefined;+ let recovering = false;+ let failed = false;+ const report = (event: DraftStorageEvent) => {+ try {+ onStorageEvent?.(event);+ } catch {+ /* Observability must not interrupt draft persistence. */+ }+ };const listeners = new Set<() => void>();let channel: BroadcastChannel | undefined;const notify = () => {@@ -36,21 +50,28 @@ export function createLocalDraftStore(/* Durable state remains readable without notifications. */}};- const open = () =>- (connection ??= new Promise<IDBDatabase>((resolve, reject) => {- const request = indexedDB.open(dbName, 1);+ const open = () => {+ if (connection) return connection;+ const request = indexedDB.open(dbName, 1);+ const opening = new Promise<IDBDatabase>((resolve, reject) => {request.onupgradeneeded = () => {for (const name of STORES) request.result.createObjectStore(name);};request.onerror = () => {- connection = undefined;+ if (connection === opening) connection = undefined;reject(request.error);};request.onsuccess = () => {const db = request.result;+ db.onclose = () => {+ if (connection !== opening) return;+ connection = undefined;+ recovering = true;+ report({ type: 'closed' });+ };db.onversionchange = () => {db.close();- connection = undefined;+ if (connection === opening) connection = undefined;};resolve(db);try {@@ -62,33 +83,65 @@ export function createLocalDraftStore(/* Embedded browsers may expose an unavailable channel. */}};- }));+ });+ connection = opening;+ return opening;+ };+ async function beginTransaction(mode: IDBTransactionMode) {+ for (let attempt = 0; ; attempt++) {+ const opening = open();+ const db = await opening;+ try {+ return db.transaction([...STORES], mode);+ } catch (error) {+ if (+ !(error instanceof DOMException) ||+ error.name !== 'InvalidStateError'+ )+ throw error;+ if (connection === opening) connection = undefined;+ db.close();+ recovering = true;+ // No requests have run yet. Never replay a started draft transaction.+ if (attempt === 1) throw error;+ }+ }+ }async function transaction<T>(mode: IDBTransactionMode,run: (tx: IDBTransaction, finish: (value: T) => void) => void,publish = true): Promise<T> {- const db = await open();- return await new Promise<T>((resolve, reject) => {- const tx = db.transaction([...STORES], mode);- let value: T;- tx.oncomplete = () => {- resolve(value);- if (mode === 'readwrite' && publish) changed();- };- tx.onabort = () =>- reject(tx.error ?? new Error('Draft storage transaction aborted'));- tx.onerror = () =>- reject(tx.error ?? new Error('Unable to save draft on this device'));- try {- run(tx, (next) => {- value = next;- });- } catch (error) {- tx.abort();- reject(error);- }- });+ try {+ const tx = await beginTransaction(mode);+ const result = await new Promise<T>((resolve, reject) => {+ let value: T;+ tx.oncomplete = () => {+ resolve(value);+ if (mode === 'readwrite' && publish) changed();+ };+ tx.onabort = () =>+ reject(tx.error ?? new Error('Draft storage transaction aborted'));+ tx.onerror = () =>+ reject(tx.error ?? new Error('Unable to save draft on this device'));+ try {+ run(tx, (next) => {+ value = next;+ });+ } catch (error) {+ tx.abort();+ reject(error);+ }+ });+ if (recovering || failed) report({ type: 'recovered' });+ recovering = false;+ failed = false;+ return result;+ } catch (error) {+ if (!failed) report({ type: 'failed', error });+ failed = true;+ throw error;+ }}function withSession<T>(tx: IDBTransaction,diff --git a/apps/web/src/lib/queries/email/local-drafts.test.ts b/apps/web/src/lib/queries/email/local-drafts.test.tsindex 45850cc382..15c36cfc3a 100644--- a/apps/web/src/lib/queries/email/local-drafts.test.ts+++ b/apps/web/src/lib/queries/email/local-drafts.test.ts@@ -8,6 +8,11 @@ import type { OperationResult } from '@urql/core';import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';const auth = vi.hoisted(() => ({ user: { authenticated: true, id: 'owner' } }));+const telemetry = vi.hoisted(() => ({+ track: vi.fn(),+ posthog: { captureException: vi.fn() },+}));+vi.mock('@app/lib/analytics', () => ({ analytics: telemetry }));const stagedUpload = vi.hoisted(() =>vi.fn<() => { previewSrc: string; size: number } | undefined>());@@ -50,15 +55,43 @@ beforeEach(async () => {queue.length = 0;auth.user = { authenticated: true, id: 'owner' };stagedUpload.mockReset();+ telemetry.track.mockClear();+ telemetry.posthog.captureException.mockClear();runtime = await import('./local-drafts');await runtime.localDraftStore.clear();});afterEach(async () => {vi.unstubAllGlobals();await runtime.localDraftStore.close();+ vi.restoreAllMocks();});describe('draft queue recovery', () => {+ it('reports caught storage failures and recovery with lifecycle context, without draft content', async () => {+ const error = new DOMException('Storage unavailable', 'UnknownError');+ vi.spyOn(IDBDatabase.prototype, 'transaction').mockImplementationOnce(+ () => {+ throw error;+ }+ );+ await expect(runtime.saveLocalDraft(input('Private content'))).rejects.toBe(+ error+ );+ const context = {+ storage: 'email-working-copies',+ visibility: document.visibilityState,+ online: navigator.onLine,+ };+ expect(telemetry.posthog.captureException).toHaveBeenCalledExactlyOnceWith(+ error,+ context+ );+ await runtime.saveLocalDraft(input('Private content'));+ expect(telemetry.track).toHaveBeenCalledExactlyOnceWith(+ 'email_draft_storage_recovered',+ context+ );+ });it('rejects an old editor generation after discard and Undo reuse the server identity', async () => {const serverInput = (subject: string) => ({draft: { db_id: 'server', subject },diff --git a/apps/web/src/lib/queries/email/local-drafts.ts b/apps/web/src/lib/queries/email/local-drafts.tsindex f7f6acb13e..b752734eb4 100644--- a/apps/web/src/lib/queries/email/local-drafts.ts+++ b/apps/web/src/lib/queries/email/local-drafts.ts@@ -6,6 +6,7 @@ import type {} from '@app/features/email-compose/core/local-draft';import type { DraftFormAttachment } from '@app/features/email-compose/primitives/email-form-state';import type { EmailMessage } from '@app/features/email-message/core/email-message';+import { analytics } from '@app/lib/analytics';import { getNativeStagedUpload } from '@core/mobile/nativeStagedUpload';import type { NormalizedCacheExchangeOptions } from '@graphql-cache/exchange/normalized-cache-exchange';import type { CacheHost } from '@graphql-cache/host/types';@@ -19,7 +20,18 @@ import type { UserInfoData } from '../auth/user-info';import { queryClient } from '../client';import { createLocalDraftStore } from './local-draft-store';-export const localDraftStore = createLocalDraftStore();+export const localDraftStore = createLocalDraftStore(undefined, (event) => {+ const context = {+ storage: 'email-working-copies',+ visibility: document.visibilityState,+ online: navigator.onLine,+ };+ if (event.type === 'failed') {+ analytics.posthog.captureException(event.error, context);+ } else {+ analytics.track('email_draft_storage_' + event.type, context);+ }+});const fileIds = new WeakMap<File, string>();const fileCopies = new WeakMap<File, Promise<Blob>>();const pendingWrites = new Set<Promise<LocalDraft>>();diff --git a/apps/web/src/lib/queries/persistence/per-query-idb.test.ts b/apps/web/src/lib/queries/persistence/per-query-idb.test.tsindex f830dd29d2..6c703b38ad 100644--- a/apps/web/src/lib/queries/persistence/per-query-idb.test.ts+++ b/apps/web/src/lib/queries/persistence/per-query-idb.test.ts@@ -1,6 +1,6 @@import 'fake-indexeddb/auto';import { IDBObjectStore } from 'fake-indexeddb';-import { afterEach, describe, expect, it, vi } from 'vitest';+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';import {createPerQueryIDBStore,type PersistedQueryEntry,@@ -20,7 +20,92 @@ const store = () =>debounceMs: 60_000,});-afterEach(() => vi.restoreAllMocks());+let connections: IDBDatabase[];+beforeEach(() => {+ connections = [];+ const open = indexedDB.open.bind(indexedDB);+ vi.spyOn(indexedDB, 'open').mockImplementation((...args) => {+ const request = open(...args);+ request.addEventListener('success', () => connections.push(request.result));+ return request;+ });+});+afterEach(() => {+ vi.restoreAllMocks();+ for (const db of connections) db.close();+});++describe('query persistence connection recovery', () => {+ it('recovers reads, writes, deletes and clear without losing cached entries', async () => {+ const db = store();+ db.set(entry('original'));+ await db.flush({ throwOnError: true });+ connections.at(-1)!.close();+ expect(await db.get('original')).toEqual(entry('original'));+ connections.at(-1)!.close();+ db.set(entry('next'));+ await db.flush({ throwOnError: true });+ expect(await db.get('original')).toEqual(entry('original'));+ connections.at(-1)!.close();+ db.remove('original');+ await db.flush({ throwOnError: true });+ expect(await db.get('original')).toBeUndefined();+ expect(await db.get('next')).toEqual(entry('next'));+ connections.at(-1)!.close();+ await db.clear();+ expect(await db.get('next')).toBeUndefined();+ expect(connections).toHaveLength(5);+ });++ it('shares the replacement across stores for the same database', async () => {+ const name = crypto.randomUUID();+ const first = createPerQueryIDBStore({ dbName: name });+ const second = createPerQueryIDBStore({ dbName: name });+ await Promise.all([first.get('missing'), second.get('missing')]);+ expect(connections).toHaveLength(1);+ connections[0].close();+ await Promise.all([first.get('missing'), second.get('missing')]);+ expect(connections).toHaveLength(2);+ });++ it('does not write an old generation when clear runs during recovery', async () => {+ const db = store();+ await db.clear();+ let clearing: Promise<void> | undefined;+ vi.spyOn(connections[0], 'transaction').mockImplementationOnce(() => {+ clearing = db.clear();+ throw new DOMException('Closing', 'InvalidStateError');+ });+ const put = vi.spyOn(IDBObjectStore.prototype, 'put');+ db.set(entry('old'));+ await db.flush({ throwOnError: true });+ await clearing;+ expect(put).not.toHaveBeenCalled();+ db.set(entry('new'));+ await db.flush({ throwOnError: true });+ expect(await db.get('old')).toBeUndefined();+ expect(await db.get('new')).toEqual(entry('new'));+ });++ it('retains pending writes after recovery is exhausted', async () => {+ const db = store();+ await db.clear();+ vi.spyOn(console, 'error').mockImplementation(() => {});+ const transaction = vi+ .spyOn(IDBDatabase.prototype, 'transaction')+ .mockImplementation(() => {+ throw new DOMException('Closing', 'InvalidStateError');+ });+ db.set(entry('pending'));+ await expect(db.flush({ throwOnError: true })).rejects.toMatchObject({+ name: 'InvalidStateError',+ });+ expect(transaction).toHaveBeenCalledTimes(2);+ transaction.mockRestore();+ await db.flush({ throwOnError: true });+ expect(await db.get('pending')).toEqual(entry('pending'));+ });+});describe('clearable query persistence', () => {it('rejects a strict flush on storage failure and retries the pending data', async () => {diff --git a/apps/web/src/lib/queries/persistence/per-query-idb.ts b/apps/web/src/lib/queries/persistence/per-query-idb.tsindex 55ae3bb707..fc68221ce5 100644--- a/apps/web/src/lib/queries/persistence/per-query-idb.ts+++ b/apps/web/src/lib/queries/persistence/per-query-idb.ts@@ -1,4 +1,5 @@import type { QueryKey } from '@tanstack/query-core';+import { RecoverableIDB } from '../../persistence/recoverable-idb';export type PersistedQueryEntry = Readonly<{queryHash: string;@@ -29,27 +30,16 @@ type PerQueryPersistenceOptions = Readonly<{const STORE_NAME = 'queries';const DEFAULT_DEBOUNCE_MS = 1000;-const dbCache = new Map<string, IDBDatabase>();+const dbCache = new Map<string, RecoverableIDB>();-function openDB(dbName: string): Promise<IDBDatabase> {+function database(dbName: string): RecoverableIDB {const cached = dbCache.get(dbName);- if (cached) return Promise.resolve(cached);- return new Promise((resolve, reject) => {- const req = indexedDB.open(dbName, 1);- req.onupgradeneeded = () => {- const db = req.result;- if (!db.objectStoreNames.contains(STORE_NAME)) {- db.createObjectStore(STORE_NAME);- }- };- req.onsuccess = () => {- const db = req.result;- db.onclose = () => dbCache.delete(dbName);- dbCache.set(dbName, db);- resolve(db);- };- req.onerror = () => reject(req.error);+ if (cached) return cached;+ const db = new RecoverableIDB(dbName, 1, (db) => {+ db.createObjectStore(STORE_NAME);});+ dbCache.set(dbName, db);+ return db;}async function withStore<T>(@@ -57,9 +47,8 @@ async function withStore<T>(mode: IDBTransactionMode,fn: (store: IDBObjectStore) => IDBRequest<T>): Promise<T> {- const db = await openDB(dbName);+ const tx = await database(dbName).transaction(STORE_NAME, mode);return new Promise<T>((resolve, reject) => {- const tx = db.transaction(STORE_NAME, mode);const store = tx.objectStore(STORE_NAME);const req = fn(store);req.onsuccess = () => resolve(req.result);@@ -109,9 +98,8 @@ export function createPerQueryIDBStore(try {await enqueueWrite(async () => {- const db = await openDB(dbName);+ const tx = await database(dbName).transaction(STORE_NAME, 'readwrite');if (generation !== currentGeneration) return;- const tx = db.transaction(STORE_NAME, 'readwrite');const store = tx.objectStore(STORE_NAME);for (const [hash, entry] of puts) {@@ -188,8 +176,7 @@ export function createPerQueryIDBStore(pendingPuts.clear();pendingDeletes.clear();await enqueueWrite(async () => {- const db = await openDB(dbName);- const tx = db.transaction(STORE_NAME, 'readwrite');+ const tx = await database(dbName).transaction(STORE_NAME, 'readwrite');tx.objectStore(STORE_NAME).clear();await new Promise<void>((resolve, reject) => {tx.oncomplete = () => resolve();diff --git a/apps/web/src/lib/workers/slack-import/message-store.test.ts b/apps/web/src/lib/workers/slack-import/message-store.test.tsindex fa84d7e623..c7d81353eb 100644--- a/apps/web/src/lib/workers/slack-import/message-store.test.ts+++ b/apps/web/src/lib/workers/slack-import/message-store.test.ts@@ -1,7 +1,8 @@// @vitest-environment nodeimport 'fake-indexeddb/auto';import { readFileSync } from 'node:fs';-import { afterEach, describe, expect, it, vi } from 'vitest';+import { forceCloseDatabase } from 'fake-indexeddb';+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';import {exactTimestamp,MessageStore,@@ -28,6 +29,140 @@ afterEach(async () => {for (const store of stores.splice(0)) await store.dispose();});+describe('scratch database recovery', () => {+ let connections: IDBDatabase[];+ beforeEach(() => {+ connections = [];+ const original = indexedDB.open.bind(indexedDB);+ vi.spyOn(indexedDB, 'open').mockImplementation((...args) => {+ const request = original(...args);+ request.addEventListener('success', () =>+ connections.push(request.result)+ );+ return request;+ });+ });++ it.each(['closed', 'terminated'])(+ 'preserves staged history and byte limits after %s',+ async (reason) => {+ const first = { ts: '1.0', text: 'original' };+ const second = { ts: '2.0', text: 'next' };+ const bytes =+ normalizeMessage('C100', first)!.byteLength ++ normalizeMessage('C100', second)!.byteLength;+ const store = await open(bytes);+ await store.stage('C100', [first]);+ if (reason === 'closed') connections[0].close();+ else {+ const closed = new Promise<void>((resolve) =>+ connections[0].addEventListener('close', () => resolve())+ );+ forceCloseDatabase(+ connections[0] as unknown as Parameters<typeof forceCloseDatabase>[0]+ );+ await closed;+ }+ await store.stage('C100', [first, second]);+ expect((await lines(store)).map((line) => JSON.parse(line).text)).toEqual(+ ['original', 'next']+ );+ expect(connections).toHaveLength(2);+ await expect(store.stage('C100', [{ ts: '3.0' }])).rejects.toMatchObject({+ code: 'selected_limit',+ });+ expect(await lines(store)).toHaveLength(2);+ }+ );++ it('continues paginated reads after the connection closes between pages', async () => {+ const store = await open();+ await store.stage(+ 'C100',+ Array.from({ length: 300 }, (_, n) => ({ ts: n + 1 + '.0' }))+ );+ const result: string[] = [];+ for await (const message of store.ordered('C100')) {+ result.push(message.line);+ if (result.length === 128) connections[0].close();+ }+ expect(result).toHaveLength(300);+ expect(new Set(result).size).toBe(300);+ expect(connections).toHaveLength(2);+ });++ it.each([false, true])(+ 'fences a replaced session and leaves its cleanup to the new owner (closed=%s)',+ async (closed) => {+ const id = crypto.randomUUID();+ const first = await MessageStore.open(id, 10000);+ await first.stage('C100', [{ ts: '1.0' }]);+ if (closed) connections[0].close();+ const replacement = await MessageStore.open(id, 10000);+ stores.push(replacement);+ await replacement.stage('C100', [{ ts: '2.0' }]);+ await expect(first.stage('C100', [{ ts: '3.0' }])).rejects.toMatchObject({+ code: 'storage_unavailable',+ });+ await first.dispose();+ expect(+ (await lines(replacement)).map((line) => JSON.parse(line).ts)+ ).toEqual(['2.000000']);+ }+ );++ it('does not recreate lost history or reopen after disposal', async () => {+ const store = await open();+ await store.stage('C100', [{ ts: '1.0' }]);+ const native = connections[0];+ native.close();+ await new Promise<void>((resolve, reject) => {+ const request = indexedDB.deleteDatabase(native.name);+ request.onsuccess = () => resolve();+ request.onerror = () => reject(request.error);+ });+ await expect(lines(store)).rejects.toMatchObject({+ code: 'storage_unavailable',+ });+ await store.dispose();+ await expect(store.stage('C100', [{ ts: '2.0' }])).rejects.toMatchObject({+ code: 'storage_unavailable',+ });+ expect(+ (await indexedDB.databases()).some((db) => db.name === native.name)+ ).toBe(false);+ });++ it('does not replay a batch that fails after its transaction starts', async () => {+ const store = await open();+ await store.stage('C100', [{ ts: '1.0' }]);+ const put = vi+ .spyOn(IDBObjectStore.prototype, 'put')+ .mockImplementationOnce(() => {+ throw new DOMException('Interrupted request', 'InvalidStateError');+ });+ await expect(store.stage('C100', [{ ts: '2.0' }])).rejects.toMatchObject({+ code: 'storage_unavailable',+ });+ expect(put).toHaveBeenCalledOnce();+ expect(await lines(store)).toHaveLength(1);+ expect(connections).toHaveLength(1);+ });++ it('cancels a batch while its connection is reopening', async () => {+ const store = await open();+ const abort = new AbortController();+ vi.spyOn(connections[0], 'transaction').mockImplementationOnce(() => {+ abort.abort();+ throw new DOMException('Closing', 'InvalidStateError');+ });+ await expect(+ store.stage('C100', [{ ts: '1.0' }], abort.signal)+ ).rejects.toMatchObject({ code: 'cancelled' });+ expect(await lines(store)).toEqual([]);+ });+});+describe('exact normalization and disk-backed dedupe', () => {it('normalizes the shared Rust events fixture, strips ignored payloads and skips nonmessages/deletions', async () => {const records = readFileSync(diff --git a/apps/web/src/lib/workers/slack-import/message-store.ts b/apps/web/src/lib/workers/slack-import/message-store.tsindex 83da8cccb7..4c58f3922c 100644--- a/apps/web/src/lib/workers/slack-import/message-store.ts+++ b/apps/web/src/lib/workers/slack-import/message-store.ts@@ -1,5 +1,6 @@import { z } from 'zod';import { ArchiveError } from '../../../features/slack-import/core/export';+import { RecoverableIDB } from '../../persistence/recoverable-idb';export const MAX_RECORD_BYTES = 1024 * 1024;const PAGE_BYTES = 1024 * 1024;@@ -123,12 +124,54 @@ export async function deleteMessageStore(sessionId: string): Promise<void> {/** One writer per session. Transactions and cursor pages are byte- and count-bounded. */export class MessageStore {private totalBytes = 0;+ private initialized = false;+ private retired = false;+ private readonly ownerStore = `owner:${crypto.randomUUID()}`;+ private readonly database: RecoverableIDB;private constructor(- private readonly db: IDBDatabase,private readonly sessionId: string,private readonly maxBytes: number- ) {}+ ) {+ this.database = new RecoverableIDB(+ databaseName(sessionId),+ 1,+ (db, tx) => {+ // Recovery cannot silently replace staged history with an empty database.+ if (this.initialized) {+ tx.abort();+ return;+ }+ const messages = db.createObjectStore('messages', {+ keyPath: ['conversation', 'time'],+ });+ messages.createIndex('conversation', 'conversation');+ // Including this empty store in transactions fences a replacement even+ // when the old connection was already closed and missed versionchange.+ db.createObjectStore(this.ownerStore);+ },+ () => {+ // Another importer deleted/replaced this session. Never revive the old owner.+ this.retired = true;+ void this.database.close();+ }+ );+ }++ private async transaction(mode: IDBTransactionMode) {+ try {+ return await this.database.transaction(+ ['messages', this.ownerStore],+ mode+ );+ } catch (error) {+ if (error instanceof DOMException && error.name === 'NotFoundError') {+ this.retired = true;+ await this.database.close();+ }+ throw error;+ }+ }static async open(sessionId: string,@@ -137,19 +180,10 @@ export class MessageStore {// Reusing a caller's session after termination must never reuse partial history.await deleteMessageStore(sessionId);try {- const db = await new Promise<IDBDatabase>((resolve, reject) => {- const request = indexedDB.open(databaseName(sessionId), 1);- request.onupgradeneeded = () => {- const messages = request.result.createObjectStore('messages', {- keyPath: ['conversation', 'time'],- });- messages.createIndex('conversation', 'conversation');- };- request.onsuccess = () => resolve(request.result);- request.onerror = () => reject(request.error);- });- db.onversionchange = () => db.close();- return new MessageStore(db, sessionId, maxBytes);+ const store = new MessageStore(sessionId, maxBytes);+ await store.database.open();+ store.initialized = true;+ return store;} catch (error) {throw storageError(error);}@@ -187,8 +221,8 @@ export class MessageStore {): Promise<void> {if (signal?.aborted) throw new ArchiveError('cancelled');try {+ const tx = await this.transaction('readwrite');await new Promise<void>((resolve, reject) => {- const tx = this.db.transaction('messages', 'readwrite');const store = tx.objectStore('messages');let bytes = this.totalBytes;let failure: ArchiveError | undefined;@@ -240,7 +274,8 @@ export class MessageStore {}};};- next();+ if (signal?.aborted) tx.abort();+ else next();});} catch (error) {throw storageError(error);@@ -252,8 +287,8 @@ export class MessageStore {after?: string): Promise<StoredMessage[]> {try {+ const tx = await this.transaction('readonly');return await new Promise<StoredMessage[]>((resolve, reject) => {- const tx = this.db.transaction('messages', 'readonly');const range = IDBKeyRange.bound([conversation, after ?? ''],[conversation, '\uffff'],@@ -302,7 +337,14 @@ export class MessageStore {}async dispose(): Promise<void> {- this.db.close();- await deleteMessageStore(this.sessionId);+ let ownsDatabase = false;+ try {+ const db = await this.database.open();+ ownsDatabase = db.objectStoreNames.contains(this.ownerStore);+ } catch {+ // A retired or missing database has no scratch history for us to remove.+ }+ await this.database.close();+ if (ownsDatabase && !this.retired) await deleteMessageStore(this.sessionId);}}diff --git a/bun.lock b/bun.lockindex 7daea6d200..5861f97107 100644--- a/bun.lock+++ b/bun.lock@@ -249,6 +249,7 @@"@types/node": "^24.8.0","@types/ws": "^8.5.14","bebop-tools": "3.1.3",+ "fake-indexeddb": "^6.2.5","jsdom": "^26.1.0","typescript": "^5.9.3","typescript-native": "npm:[email protected]",diff --git a/docs/AGENT_GUIDE/surfaces.md b/docs/AGENT_GUIDE/surfaces.mdindex 9643554226..7458c324f3 100644--- a/docs/AGENT_GUIDE/surfaces.md+++ b/docs/AGENT_GUIDE/surfaces.md@@ -584,6 +584,12 @@ native offline archive support therefore requires a full app build, not just OTA### Cached Mail filtering+The per-query caches for email threads, channels, and native offline document+context reopen closed IndexedDB connections. Verify a cached read and a subsequent+write after closing the connection, then clear/logout during recovery: old-account+entries must not return. Only transaction creation retries automatically; persistent+storage failures still follow the existing cache fallback behavior.+Performance check: switch Signal → All twice against a large synchronized cache.Dense local pages use bounded sort-index candidates rather than sorting the entiremailbox. Sparse filters and large timestamp ties retain the exact fallback plan.@@ -735,6 +741,15 @@ when delivery fails. Later edits can show it again after the usual pause.Verify the label does not cycle through saving/syncing text on each edit andthat retry remains accessible beside the actions on mobile. Editing while failedcontinues saving locally without repeatedly submitting the rejected request.+Local draft storage reopens unexpectedly closed IndexedDB connections. If creating+a transaction fails because the connection is closing, it retries once before+running any writes. Verify by closing the draft database connection, then editing:+the latest content must persist locally and server autosave must continue without+reloading. Repeat across background/resume on an iPhone. A transaction that has+already started is never automatically replayed. Persistent failures still block+server autosave; copy unsaved content before restarting. Storage failures and+recovery report current page visibility and network state without draft content.+Repeated local-save failures show one persistent warning per composer; recoveryor closing that composer dismisses it. A later failure shows a new warning.Retry preserves the original draft handle. An already-sent rejection drops the@@ -2211,6 +2226,13 @@ afterwards in **Settings → Billing**, where Guest users can buy Premium or Max### Slack archive import+Temporary message storage recovers closed IndexedDB connections without restarting+the import or deleting staged history. Verify closure between staging batches and+between cursor pages: all messages must remain ordered, deduplicated, and within+the original byte limit. Cancellation and cleanup still work during recovery.+Deleted or replaced scratch databases fail the old import instead of silently+continuing with incomplete history; its cleanup must leave a replacement intact.+In **Team → Connections**, **Import from Slack** appears only for team adminsand owners with `enable-slack-archive-import` enabled. Production rollout staysoff until explicitly enabled. Select an export ZIP, filter the grouped conversationdiff --git a/packages/collaboration/package.json b/packages/collaboration/package.jsonindex 1575915198..cbeb144e63 100644--- a/packages/collaboration/package.json+++ b/packages/collaboration/package.json@@ -37,6 +37,7 @@"@types/node": "^24.8.0","@types/ws": "^8.5.14","bebop-tools": "3.1.3",+ "fake-indexeddb": "^6.2.5","jsdom": "^26.1.0","typescript": "^5.9.3","typescript-native": "npm:[email protected]",diff --git a/packages/collaboration/src/collab/indexeddb-recovery.test.ts b/packages/collaboration/src/collab/indexeddb-recovery.test.tsnew file mode 100644index 0000000000..753bbc678f--- /dev/null+++ b/packages/collaboration/src/collab/indexeddb-recovery.test.ts@@ -0,0 +1,279 @@+import 'fake-indexeddb/auto';+import { forceCloseDatabase, IDBFactory } from 'fake-indexeddb';+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';+import { IDBSnapshotStore } from './snapshot-store';+import { BrowserWALStore, WALSyncer } from './wal';++let connections: IDBDatabase[];++beforeEach(() => {+ connections = [];+ const factory = new IDBFactory();+ vi.stubGlobal('indexedDB', factory);+ const open = factory.open.bind(factory);+ vi.spyOn(factory, 'open').mockImplementation((...args) => {+ const request = open(...args);+ request.addEventListener('success', () => connections.push(request.result));+ return request;+ });+});++afterEach(() => {+ for (const db of connections) db.close();+ vi.restoreAllMocks();+ vi.unstubAllGlobals();+});++describe('collaboration IndexedDB recovery', () => {+ it.each(['success', 'cursor failure'])(+ 'closes the scope-listing connection after %s',+ async (outcome) => {+ const first = new BrowserWALStore<number>('wal', 'first');+ const second = new BrowserWALStore<number>('wal', 'second');+ await first.append(1);+ await first.append(2);+ await second.append(3);++ if (outcome === 'cursor failure') {+ const error = new Error('Cursor iteration failed');+ vi.spyOn(IDBCursor.prototype, 'continue').mockImplementationOnce(() => {+ throw error;+ });+ await expect(BrowserWALStore.listScopeIds('wal')).rejects.toBe(error);+ } else {+ await expect(BrowserWALStore.listScopeIds('wal')).resolves.toEqual([+ 'first',+ 'second',+ ]);+ }++ expect(() => connections.at(-1)!.transaction('updates')).toThrow(+ expect.objectContaining({ name: 'InvalidStateError' })+ );+ expect(await first.count()).toBe(2);+ expect(await second.count()).toBe(1);+ }+ );++ it('flushes persisted edits after the WAL connection closes', async () => {+ const store = new BrowserWALStore<Uint8Array>('wal', 'doc');+ const push = vi.fn(async (_updates: Uint8Array[]) => true);+ const wal = new WALSyncer(store, push);+ await wal.ready();+ await store.append(new Uint8Array([1, 2, 3]));+ connections[0]!.close();++ await wal.flush();+ await wal.flush();++ expect(push).toHaveBeenCalledOnce();+ // IndexedDB clones into the host realm rather than jsdom's typed-array realm.+ expect(push.mock.calls[0]![0].map((update) => Array.from(update))).toEqual([+ [1, 2, 3],+ ]);+ expect(await store.getAll()).toEqual([+ expect.objectContaining({ id: 1, delivered: true }),+ ]);+ expect(connections).toHaveLength(2);+ });++ it('shares a reopened connection across concurrent appends without duplicates', async () => {+ const store = new BrowserWALStore<number>('wal', 'doc');+ await store.append(1);+ connections[0]!.close();++ await Promise.all([store.append(2), store.append(3), store.count()]);++ expect((await store.getAll()).map((entry) => entry.update)).toEqual([+ 1, 2, 3,+ ]);+ expect(connections).toHaveLength(2);+ });++ it('recovers after the browser forcibly terminates a connection', async () => {+ const store = new BrowserWALStore<number>('wal', 'doc');+ await store.append(1);+ const db = connections[0]!;+ const closed = new Promise<void>((resolve) =>+ db.addEventListener('close', () => resolve())+ );+ // fake-indexeddb declares a constructor here, but accepts a database instance.+ forceCloseDatabase(+ db as unknown as Parameters<typeof forceCloseDatabase>[0]+ );+ await closed;++ expect(await store.count()).toBe(1);+ expect(connections).toHaveLength(2);+ });++ it('recovers snapshot reads and writes while keeping document scopes separate', async () => {+ const first = new IDBSnapshotStore<number>('snapshots', 'first');+ const second = new IDBSnapshotStore<number>('snapshots', 'second');+ await first.save(1);+ await second.save(2);+ connections[0]!.close();+ expect(await first.load()).toBe(1);+ connections.at(-1)!.close();+ await first.save(3);++ expect(await first.load()).toBe(3);+ expect(await second.load()).toBe(2);+ });++ it('bounds recovery attempts and permits a later retry', async () => {+ const store = new BrowserWALStore<number>('wal', 'doc');+ await store.append(1);+ const transaction = vi+ .spyOn(IDBDatabase.prototype, 'transaction')+ .mockImplementation(() => {+ throw new DOMException('Connection is closing', 'InvalidStateError');+ });++ await expect(store.count()).rejects.toMatchObject({+ name: 'InvalidStateError',+ });+ expect(transaction).toHaveBeenCalledTimes(2);+ transaction.mockRestore();++ expect(await store.count()).toBe(1);+ });++ it('does not retry unrelated transaction failures', async () => {+ const store = new BrowserWALStore<number>('wal', 'doc');+ await store.append(1);+ const transaction = vi+ .spyOn(IDBDatabase.prototype, 'transaction')+ .mockImplementation(() => {+ throw new DOMException('Storage unavailable', 'UnknownError');+ });++ await expect(store.append(2)).rejects.toMatchObject({+ name: 'UnknownError',+ });+ expect(transaction).toHaveBeenCalledOnce();+ transaction.mockRestore();+ expect(await store.count()).toBe(1);+ });++ it('retries once after an interrupted append has rolled back', async () => {+ const store = new BrowserWALStore<number>('wal', 'doc');+ await store.append(1);+ const db = connections[0]!;+ const closed = new Promise<void>((resolve) =>+ db.addEventListener('close', () => resolve())+ );+ const add = IDBObjectStore.prototype.add;+ const write = vi+ .spyOn(IDBObjectStore.prototype, 'add')+ .mockImplementationOnce(function (value, key) {+ const request = add.call(this, value, key);+ // Simulate an interrupted write as well as the connection close event.+ this.transaction.abort();+ forceCloseDatabase(+ db as unknown as Parameters<typeof forceCloseDatabase>[0]+ );+ return request;+ });++ await store.append(2);+ await closed;+ expect(write).toHaveBeenCalledTimes(2);+ write.mockRestore();++ expect((await store.getAll()).map((entry) => entry.update)).toEqual([1, 2]);+ });++ it('retries an abort after request success without duplicating the entry', async () => {+ const store = new BrowserWALStore<number>('wal', 'doc');+ await store.append(1);+ const add = IDBObjectStore.prototype.add;+ const write = vi+ .spyOn(IDBObjectStore.prototype, 'add')+ .mockImplementationOnce(function (value, key) {+ const request = add.call(this, value, key);+ request.addEventListener('success', () => this.transaction.abort());+ return request;+ });++ await store.append(2);++ expect(write).toHaveBeenCalledTimes(2);+ expect((await store.getAll()).map((entry) => entry.update)).toEqual([1, 2]);+ });++ it('does not retry an error when the transaction actually commits', async () => {+ const store = new BrowserWALStore<number>('wal', 'doc');+ await store.append(1);+ const add = IDBObjectStore.prototype.add;+ const error = new DOMException('Ambiguous request failure', 'AbortError');+ const write = vi+ .spyOn(IDBObjectStore.prototype, 'add')+ .mockImplementationOnce(function (value, key) {+ add.call(this, value, key);+ throw error;+ });++ await expect(store.append(2)).rejects.toBe(error);++ expect(write).toHaveBeenCalledOnce();+ expect((await store.getAll()).map((entry) => entry.update)).toEqual([1, 2]);+ });++ it('bounds aborted append retries and leaves existing entries intact', async () => {+ const store = new BrowserWALStore<number>('wal', 'doc');+ await store.append(1);+ const add = IDBObjectStore.prototype.add;+ const write = vi+ .spyOn(IDBObjectStore.prototype, 'add')+ .mockImplementation(function (value, key) {+ const request = add.call(this, value, key);+ this.transaction.abort();+ return request;+ });++ await expect(store.append(2)).rejects.toMatchObject({ name: 'AbortError' });+ expect(write).toHaveBeenCalledTimes(2);+ expect((await store.getAll()).map((entry) => entry.update)).toEqual([1]);+ });++ it('does not retry a transaction aborted by a constraint failure', async () => {+ const store = new BrowserWALStore<number>('wal', 'doc');+ await store.append(1);+ const add = IDBObjectStore.prototype.add;+ const write = vi+ .spyOn(IDBObjectStore.prototype, 'add')+ .mockImplementationOnce(function (value) {+ return add.call(this, { ...value, id: 1 });+ });++ await expect(store.append(2)).rejects.toMatchObject({+ name: 'ConstraintError',+ });+ expect(write).toHaveBeenCalledOnce();+ expect((await store.getAll()).map((entry) => entry.update)).toEqual([1]);+ });++ it('releases the connection when another client upgrades the database', async () => {+ const store = new BrowserWALStore<number>('wal', 'doc');+ await store.append(1);+ const request = indexedDB.open('wal', 2);+ await new Promise<void>((resolve, reject) => {+ request.onsuccess = () => resolve();+ request.onerror = () => reject(request.error);+ request.onblocked = () => reject(new Error('Upgrade was blocked'));+ });++ // The old client must not bypass a schema version change or delete data.+ await expect(store.count()).rejects.toMatchObject({ name: 'VersionError' });+ const read = request.result+ .transaction('updates')+ .objectStore('updates')+ .getAll();+ const entries = await new Promise<unknown[]>((resolve, reject) => {+ read.onsuccess = () => resolve(read.result);+ read.onerror = () => reject(read.error);+ });+ expect(entries).toHaveLength(1);+ });+});diff --git a/packages/collaboration/src/collab/recoverable-database.ts b/packages/collaboration/src/collab/recoverable-database.tsnew file mode 100644index 0000000000..4066215844--- /dev/null+++ b/packages/collaboration/src/collab/recoverable-database.ts@@ -0,0 +1,72 @@+import {+ type DBSchema,+ type IDBPDatabase,+ type OpenDBCallbacks,+ openDB,+ type StoreNames,+} from 'idb';++/** Keeps a connection only while it is usable. Recovery never replays writes. */+export class RecoverableDatabase<Schema extends DBSchema> {+ private connection?: Promise<IDBPDatabase<Schema>>;++ constructor(+ private readonly name: string,+ private readonly version: number,+ private readonly upgrade: OpenDBCallbacks<Schema>['upgrade']+ ) {}++ private async open(): Promise<IDBPDatabase<Schema>> {+ if (this.connection) return this.connection;++ const connection = openDB<Schema>(this.name, this.version, {+ upgrade: this.upgrade,+ terminated: () => {+ if (this.connection === connection) this.connection = undefined;+ },+ });+ this.connection = connection;+ try {+ const db = await connection;+ db.addEventListener('versionchange', () => {+ db.close();+ if (this.connection === connection) this.connection = undefined;+ });+ return db;+ } catch (error) {+ if (this.connection === connection) this.connection = undefined;+ throw error;+ }+ }++ async transaction<+ Name extends StoreNames<Schema>,+ Mode extends IDBTransactionMode,+ >(store: Name, mode: Mode) {+ for (let attempt = 0; ; attempt++) {+ const db = await this.open();+ try {+ return db.transaction(store, mode);+ } catch (error) {+ if (+ !(error instanceof DOMException) ||+ error.name !== 'InvalidStateError'+ ) {+ throw error;+ }+ // Only transaction creation is retried: no request has run yet, so an+ // auto-increment WAL append cannot be duplicated by recovery.+ const connection = this.connection;+ if (+ connection &&+ (await connection) === db &&+ this.connection === connection+ ) {+ this.connection = undefined;+ }+ db.close();+ if (attempt === 1) throw error;+ }+ }+ }+}diff --git a/packages/collaboration/src/collab/snapshot-store.ts b/packages/collaboration/src/collab/snapshot-store.tsindex 2881dba137..c823e42a75 100644--- a/packages/collaboration/src/collab/snapshot-store.ts+++ b/packages/collaboration/src/collab/snapshot-store.ts@@ -1,7 +1,8 @@-import { type DBSchema, type IDBPDatabase, openDB as idbOpen } from 'idb';+import type { DBSchema } from 'idb';import { logSyncService } from './logger';import type { LoroManager } from './manager';import { LoroManagerError } from './manager-error';+import { RecoverableDatabase } from './recoverable-database';import type { GenericRootSchema, RawUpdate } from './shared';import type { WALStore } from './wal';@@ -25,22 +26,23 @@ interface SnapshotSchema<T> extends DBSchema {}export class IDBSnapshotStore<T> implements SnapshotStore<T> {- private db: Promise<IDBPDatabase<SnapshotSchema<T>>>;+ private readonly database: RecoverableDatabase<SnapshotSchema<T>>;constructor(dbName: string,private readonly scopeId: string) {- this.db = idbOpen<SnapshotSchema<T>>(dbName, DB_VERSION, {- upgrade(db) {- db.createObjectStore(STORE, { keyPath: 'scopeId' });- },+ this.database = new RecoverableDatabase(dbName, DB_VERSION, (db) => {+ db.createObjectStore(STORE, { keyPath: 'scopeId' });});}public async save(snapshot: T): Promise<void> {- const db = await this.db;- await db.put(STORE, { scopeId: this.scopeId, snapshot });+ const tx = await this.database.transaction(STORE, 'readwrite');+ await Promise.all([+ tx.store.put({ scopeId: this.scopeId, snapshot }),+ tx.done,+ ]);logSyncService({documentId: this.scopeId,level: 'debug',@@ -50,8 +52,8 @@ export class IDBSnapshotStore<T> implements SnapshotStore<T> {}public async load(): Promise<T | null> {- const db = await this.db;- const row = await db.get(STORE, this.scopeId);+ const tx = await this.database.transaction(STORE, 'readonly');+ const row = await tx.store.get(this.scopeId);const found = row?.snapshot ?? null;logSyncService({documentId: this.scopeId,@@ -65,8 +67,8 @@ export class IDBSnapshotStore<T> implements SnapshotStore<T> {}public async delete(): Promise<void> {- const db = await this.db;- await db.delete(STORE, this.scopeId);+ const tx = await this.database.transaction(STORE, 'readwrite');+ await Promise.all([tx.store.delete(this.scopeId), tx.done]);}}diff --git a/packages/collaboration/src/collab/wal.test.ts b/packages/collaboration/src/collab/wal.test.tsindex a272890590..b316f576b3 100644--- a/packages/collaboration/src/collab/wal.test.ts+++ b/packages/collaboration/src/collab/wal.test.ts@@ -1,5 +1,6 @@import { type Span, Telemetry } from '@macro-inc/observability';import { afterEach, describe, expect, it, vi } from 'vitest';+import * as logger from './logger';import type { RawUpdate } from './shared';import { disposeTelemetryFor } from './telemetry';import { MockLiveSyncSource, MockWALStore, makeTestWAL } from './testing';@@ -35,6 +36,119 @@ async function undeliveredCount(// pruneDelivered() is called by the snapshot tick after a durable snapshot save.describe('WALSyncer', () => {+ it('retries failed initialization on demand and shares the retry across callers', async () => {+ vi.spyOn(logger, 'logSyncService').mockImplementation(() => {});+ const store = new MockWALStore<RawUpdate>();+ const error = new DOMException('Temporarily unavailable', 'UnknownError');+ const retry = Promise.withResolvers<number>();+ const prune = vi+ .spyOn(store, 'pruneExpired')+ .mockRejectedValueOnce(error)+ .mockReturnValueOnce(retry.promise);+ const push = vi.fn(async () => true);+ const wal = new WALSyncer(store, push);++ await expect(wal.ready()).rejects.toBe(error);+ expect(prune).toHaveBeenCalledOnce();+ const ready = wal.ready();+ expect(wal.ready()).toBe(ready);+ const append = wal.append(new Uint8Array([1]));+ expect(prune).toHaveBeenCalledTimes(2);+ expect(await store.count()).toBe(0);++ retry.resolve(0);+ await append;+ await wal.pendingFlush;+ await wal.ready();+ expect(prune).toHaveBeenCalledTimes(2);+ expect(push).toHaveBeenCalledExactlyOnceWith([new Uint8Array([1])]);+ });++ it('handles constructor setup failure even when no caller immediately awaits ready', async () => {+ const log = vi.spyOn(logger, 'logSyncService').mockImplementation(() => {});+ const store = new MockWALStore<RawUpdate>();+ const error = new Error('Initial prune failed');+ const prune = vi.spyOn(store, 'pruneExpired').mockRejectedValueOnce(error);+ const wal = new WALSyncer(store, async () => true);++ await vi.waitFor(() =>+ expect(log).toHaveBeenCalledWith(+ expect.objectContaining({+ context: {+ misc: expect.objectContaining({+ errStack: error.stack,+ 'wal.stage': 'initialization',+ }),+ },+ })+ )+ );+ expect(prune).toHaveBeenCalledOnce();+ await wal.ready();+ expect(prune).toHaveBeenCalledTimes(2);+ });++ it('reports distinct flush failures with their stacks during the same outage', async () => {+ const log = vi.spyOn(logger, 'logSyncService').mockImplementation(() => {});+ const store = new MockWALStore<RawUpdate>();+ const wal = new WALSyncer(store, async () => true, 'wal-telemetry-test');+ await wal.ready();+ const first = new Error('Storage failed');+ const second = new Error('Storage failed');+ first.stack = 'Error: Storage failed\n at firstOperation';+ second.stack = 'Error: Storage failed\n at secondOperation';+ const getAll = vi.spyOn(store, 'getAll').mockRejectedValue(first);+ await expect(wal.flush()).rejects.toBe(first);+ await expect(wal.flush()).rejects.toBe(first);+ getAll.mockRejectedValue(second);+ await expect(wal.flush()).rejects.toBe(second);+ await expect(wal.flush()).rejects.toBe(second);++ const errors = log.mock.calls.filter(([entry]) => entry.level === 'error');+ expect(errors.map(([entry]) => entry.context.misc?.errStack)).toEqual([+ first.stack,+ second.stack,+ ]);+ });++ it('handles background storage failures once per outage and retries on the next trigger', async () => {+ const log = vi.spyOn(logger, 'logSyncService').mockImplementation(() => {});+ const store = new MockWALStore<RawUpdate>();+ const push = vi.fn(async () => true);+ const wal = new WALSyncer(store, push, 'wal-telemetry-test');+ await wal.ready();+ await store.append(new Uint8Array([1]));+ const getAll = vi+ .spyOn(store, 'getAll')+ .mockRejectedValue(+ new DOMException('Storage unavailable', 'UnknownError')+ );++ void wal.flush();+ await vi.waitFor(() =>+ expect(+ log.mock.calls.filter(([entry]) => entry.level === 'error')+ ).toHaveLength(1)+ );+ await expect(wal.flush()).rejects.toMatchObject({ name: 'UnknownError' });+ expect(+ log.mock.calls.filter(([entry]) => entry.level === 'error')+ ).toHaveLength(1);+ expect(push).not.toHaveBeenCalled();++ getAll.mockRestore();+ await wal.flush();+ expect(push).toHaveBeenCalledExactlyOnceWith([new Uint8Array([1])]);++ vi.spyOn(store, 'getAll').mockRejectedValue(+ new DOMException('Storage unavailable', 'UnknownError')+ );+ await expect(wal.flush()).rejects.toMatchObject({ name: 'UnknownError' });+ expect(+ log.mock.calls.filter(([entry]) => entry.level === 'error')+ ).toHaveLength(2);+ });+it('persists before delivering, marks delivered on ack', async () => {const live = new MockLiveSyncSource();const { wal, walStore } = makeWAL(live);diff --git a/packages/collaboration/src/collab/wal.ts b/packages/collaboration/src/collab/wal.tsindex 49a8c25d7c..7752d08813 100644--- a/packages/collaboration/src/collab/wal.ts+++ b/packages/collaboration/src/collab/wal.ts@@ -1,6 +1,7 @@import type { Span } from '@macro-inc/observability';import { type DBSchema, type IDBPDatabase, openDB as idbOpen } from 'idb';import { logSyncService, type WalContext } from './logger';+import { RecoverableDatabase } from './recoverable-database';import type { RawUpdate } from './shared';import type { LiveSyncSource } from './source';import { telemetrySpan } from './telemetry';@@ -60,72 +61,91 @@ interface WALSchema<T> extends DBSchema {};}-export class BrowserWALStore<T> implements WALStore<T> {- /** Resolves to the open IDB database, shared across all operations. */- private _db: Promise<IDBPDatabase<WALSchema<T>>>;+function upgradeWAL<T>(db: IDBPDatabase<WALSchema<T>>): void {+ const store = db.createObjectStore('updates', {+ keyPath: 'id',+ autoIncrement: true,+ });+ store.createIndex('scopeId', 'scopeId');+}- private db(): Promise<IDBPDatabase<WALSchema<T>>> {- return this._db;- }+export class BrowserWALStore<T> implements WALStore<T> {+ private readonly database: RecoverableDatabase<WALSchema<T>>;constructor(dbName: string,private readonly scopeId: string) {- this._db = BrowserWALStore.openDb<T>(dbName);- }-- private static openDb<U>(- dbName: string- ): Promise<IDBPDatabase<WALSchema<U>>> {- return idbOpen<WALSchema<U>>(dbName, DB_VERSION, {- upgrade(db) {- const store = db.createObjectStore('updates', {- keyPath: 'id',- autoIncrement: true,- });- store.createIndex('scopeId', 'scopeId');- },- });+ this.database = new RecoverableDatabase(dbName, DB_VERSION, upgradeWAL);}/** List every scopeId that currently has at least one entry. Uses a* unique-key cursor on the `scopeId` index, so it doesn't load entries. */static async listScopeIds(dbName: string): Promise<string[]> {- const db = await BrowserWALStore.openDb<unknown>(dbName);- const scopeIds: string[] = [];- let cursor = await db- .transaction('updates')- .store.index('scopeId')- .openKeyCursor(null, 'nextunique');- while (cursor) {- scopeIds.push(cursor.key);- cursor = await cursor.continue();+ const db = await idbOpen<WALSchema<unknown>>(dbName, DB_VERSION, {+ upgrade: upgradeWAL,+ });+ try {+ const scopeIds: string[] = [];+ let cursor = await db+ .transaction('updates')+ .store.index('scopeId')+ .openKeyCursor(null, 'nextunique');+ while (cursor) {+ scopeIds.push(cursor.key);+ cursor = await cursor.continue();+ }+ return scopeIds;+ } finally {+ db.close();}- return scopeIds;}public async append(update: T): Promise<void> {- const db = await this.db();- await db.add('updates', {+ const entry = {scopeId: this.scopeId,update,delivered: false,createdAt: Date.now(),- });+ };+ for (let attempt = 0; ; attempt++) {+ const tx = await this.database.transaction('updates', 'readwrite');+ // A rejected request alone does not prove rollback. Wait for the native+ // transaction outcome before deciding whether replay is safe.+ const aborted = new Promise<boolean>((resolve) => {+ tx.addEventListener('abort', () => resolve(true), { once: true });+ tx.addEventListener('complete', () => resolve(false), { once: true });+ });+ try {+ // Wrap synchronous request errors too, so tx.done is always observed.+ const write = async () => tx.store.add(entry);+ await Promise.all([write(), tx.done]);+ return;+ } catch (error) {+ const rolledBack = await aborted;+ if (+ attempt === 1 ||+ !rolledBack ||+ (tx.error !== null && tx.error.name !== 'AbortError') ||+ !(error instanceof DOMException) ||+ error.name !== 'AbortError'+ ) {+ throw tx.error ?? error;+ }+ }+ }}public async getAll(): Promise<WALEntry<T>[]> {- const db = await this.db();- return db.getAllFromIndex('updates', 'scopeId', this.scopeId) as Promise<+ const tx = await this.database.transaction('updates', 'readonly');+ return tx.store.index('scopeId').getAll(this.scopeId) as Promise<WALEntry<T>[]>;}public async markDelivered(ids: number[]): Promise<void> {if (ids.length === 0) return;- const db = await this.db();- const tx = db.transaction('updates', 'readwrite');+ const tx = await this.database.transaction('updates', 'readwrite');const store = tx.objectStore('updates');for (const id of ids) {const row = await store.get(id);@@ -135,13 +155,8 @@ export class BrowserWALStore<T> implements WALStore<T> {}public async pruneDelivered(): Promise<void> {- const db = await this.db();- const entries = await db.getAllFromIndex(- 'updates',- 'scopeId',- this.scopeId- );- const tx = db.transaction('updates', 'readwrite');+ const entries = await this.getAll();+ const tx = await this.database.transaction('updates', 'readwrite');const store = tx.objectStore('updates');for (const row of entries) {if (row.delivered && row.id !== undefined) {@@ -152,14 +167,9 @@ export class BrowserWALStore<T> implements WALStore<T> {}public async pruneExpired(ttlMs: number): Promise<number> {- const db = await this.db();- const entries = await db.getAllFromIndex(- 'updates',- 'scopeId',- this.scopeId- );+ const entries = await this.getAll();const cutoff = Date.now() - ttlMs;- const tx = db.transaction('updates', 'readwrite');+ const tx = await this.database.transaction('updates', 'readwrite');const store = tx.objectStore('updates');let deleted = 0;for (const row of entries) {@@ -173,9 +183,7 @@ export class BrowserWALStore<T> implements WALStore<T> {}public async count(): Promise<number> {- const db = await this.db();- return (await db.getAllFromIndex('updates', 'scopeId', this.scopeId))- .length;+ return (await this.getAll()).length;}}@@ -229,6 +237,7 @@ export class WALSyncer<T> {/** True if append was called while a flush was in progress. Causes flush* to re-run after completing so those entries aren't stranded. */private hasNewPending = false;+ private reportedFailures = new Map<'initialization' | 'flush', string>();public pendingFlush: Promise<void> = Promise.resolve();private cleanupFns: Array<() => void> = [];private deliveryOutage:@@ -245,14 +254,14 @@ export class WALSyncer<T> {* Exists so that we can not "do stuff" until we have pruned expired entries* (like when snapshot loading occurs right after construction for example).*/- private readonly readyPromise: Promise<void>;+ private readyPromise?: Promise<void>;constructor(private readonly store: WALStore<T>,private readonly push: (items: T[]) => Promise<boolean>,private readonly label?: string) {- this.readyPromise = this.setup();+ void this.ready();}/* Right now just drops expired entries. */@@ -269,9 +278,24 @@ export class WALSyncer<T> {}public ready(): Promise<void> {+ if (!this.readyPromise) {+ this.readyPromise = this.setup();+ void this.observeSetup(this.readyPromise);+ }return this.readyPromise;}+ private async observeSetup(setup: Promise<void>): Promise<void> {+ try {+ await setup;+ this.reportedFailures.delete('initialization');+ } catch (error) {+ // Retry only on the next caller's request, sharing any in-flight setup.+ if (this.readyPromise === setup) this.readyPromise = undefined;+ this.reportFailure('initialization', error);+ }+ }+public async append(item: T): Promise<void> {await this.ready();await this.store.append(item);@@ -295,10 +319,43 @@ export class WALSyncer<T> {message: `WAL flush: triggered (isFlushing=${this.isFlushing})`,});if (this.isFlushing) return this.pendingFlush;+ this.isFlushing = true;this.pendingFlush = this.doFlush();+ // Observe the same promise we return: awaited callers still see failures,+ // while timer/reconnect callers cannot create unhandled rejections.+ void this.observeFlush(this.pendingFlush);return this.pendingFlush;}+ private async observeFlush(flush: Promise<void>): Promise<void> {+ try {+ await flush;+ this.reportedFailures.delete('flush');+ } catch (error) {+ this.reportFailure('flush', error);+ }+ }++ private reportFailure(+ stage: 'initialization' | 'flush',+ error: unknown+ ): void {+ const details = {+ errName: error instanceof Error ? error.name : undefined,+ errMessage: error instanceof Error ? error.message : String(error),+ errStack: error instanceof Error ? error.stack : undefined,+ };+ const fingerprint = JSON.stringify(details);+ if (this.reportedFailures.get(stage) === fingerprint) return;+ this.reportedFailures.set(stage, fingerprint);+ logSyncService({+ documentId: this.label ?? 'unknown',+ level: 'error',+ context: { misc: { ...details, 'wal.stage': stage } },+ message: `WAL ${stage} failed; retrying on the next trigger`,+ });+ }+public pruneDelivered(): Promise<void> {return this.store.pruneDelivered();}@@ -327,13 +384,11 @@ export class WALSyncer<T> {}private async doFlush(): Promise<void> {- await this.ready();-- this.isFlushing = true;this.hasNewPending = false;let succeeded = true;try {+ await this.ready();const entries = await this.store.getAll();const undelivered = entries.filter((e) => !e.delivered);if (undelivered.length === 0) return; // nothing to do@@ -373,6 +428,7 @@ export class WALSyncer<T> {}if (succeeded && this.hasNewPending) {+ this.isFlushing = true;this.pendingFlush = this.doFlush();return this.pendingFlush;}