Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(storage): recover closed IndexedDB connections (#7828)

Peter Chinman087df8a42f8c
diff --git a/apps/web/src/lib/persistence/recoverable-idb.test.ts b/apps/web/src/lib/persistence/recoverable-idb.test.ts
new file mode 100644
index 0000000000..074aa6a48d
--- /dev/null
+++ b/apps/web/src/lib/persistence/recoverable-idb.test.ts
@@ -0,0 +1,117 @@
+import 'fake-indexeddb/auto';
+import { forceCloseDatabase } from 'fake-indexeddb';
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import { RecoverableIDB } from './recoverable-idb';
+
+const databases: RecoverableIDB[] = [];
+function database() {
+ const db = new RecoverableIDB(crypto.randomUUID(), 1, (db) =>
+ db.createObjectStore('items')
+ );
+ databases.push(db);
+ return db;
+}
+afterEach(async () => {
+ vi.restoreAllMocks();
+ for (const db of databases.splice(0)) await db.close();
+});
+
+describe('recoverable native IndexedDB connections', () => {
+ it.each(['closed', 'terminated'])(
+ 'shares one replacement after a connection is %s',
+ async (reason) => {
+ const db = database();
+ const original = await db.open();
+ const open = vi.spyOn(indexedDB, 'open');
+ if (reason === 'closed') original.close();
+ else {
+ const closed = new Promise<void>((resolve) =>
+ original.addEventListener('close', () => resolve())
+ );
+ // The library accepts an instance but declares a constructor type.
+ forceCloseDatabase(
+ original as unknown as Parameters<typeof forceCloseDatabase>[0]
+ );
+ await closed;
+ }
+ const [first, second] = await Promise.all([
+ db.transaction('items', 'readonly'),
+ db.transaction('items', 'readonly'),
+ ]);
+ expect(first.db).toBe(second.db);
+ expect(first.db).not.toBe(original);
+ expect(open).toHaveBeenCalledOnce();
+ original.onclose?.call(original, new Event('close'));
+ expect(await db.open()).toBe(first.db);
+ }
+ );
+
+ it('bounds transaction creation retries and permits a later recovery', async () => {
+ const db = database();
+ await db.open();
+ const error = new DOMException('Closing', 'InvalidStateError');
+ const transaction = vi
+ .spyOn(IDBDatabase.prototype, 'transaction')
+ .mockImplementation(() => {
+ throw error;
+ });
+ await expect(db.transaction('items', 'readwrite')).rejects.toBe(error);
+ expect(transaction).toHaveBeenCalledTimes(2);
+ transaction.mockRestore();
+ await expect(db.transaction('items', 'readwrite')).resolves.toBeDefined();
+ });
+
+ it('does not retry unrelated errors', async () => {
+ const db = database();
+ const native = await db.open();
+ const error = new DOMException('Quota exhausted', 'QuotaExceededError');
+ const transaction = vi
+ .spyOn(native, 'transaction')
+ .mockImplementationOnce(() => {
+ throw error;
+ });
+ await expect(db.transaction('items', 'readwrite')).rejects.toBe(error);
+ expect(transaction).toHaveBeenCalledOnce();
+ expect(await db.open()).toBe(native);
+ });
+
+ it('releases connections for version changes', async () => {
+ const db = database();
+ const native = await db.open();
+ const upgraded = await new Promise<IDBDatabase>((resolve, reject) => {
+ const request = indexedDB.open(native.name, 2);
+ request.onsuccess = () => resolve(request.result);
+ request.onerror = () => reject(request.error);
+ });
+ upgraded.close();
+ expect(() => native.transaction('items')).toThrow();
+ const open = vi.spyOn(indexedDB, 'open');
+ await expect(db.open()).rejects.toMatchObject({ name: 'VersionError' });
+ await expect(db.open()).rejects.toMatchObject({ name: 'VersionError' });
+ expect(open).toHaveBeenCalledTimes(2);
+ });
+
+ it('does not cache synchronous opening failures', async () => {
+ const db = database();
+ vi.spyOn(indexedDB, 'open').mockImplementationOnce(() => {
+ throw new Error('Unavailable');
+ });
+ await expect(db.transaction('items', 'readonly')).rejects.toThrow(
+ 'Unavailable'
+ );
+ await expect(db.transaction('items', 'readonly')).resolves.toBeDefined();
+ });
+
+ it('closes an in-flight open and never revives a disposed owner', async () => {
+ const db = database();
+ const pending = db.transaction('items', 'readonly');
+ const failure = expect(pending).rejects.toMatchObject({
+ name: 'InvalidStateError',
+ });
+ await db.close();
+ await failure;
+ await expect(db.transaction('items', 'readonly')).rejects.toMatchObject({
+ name: 'InvalidStateError',
+ });
+ });
+});
diff --git a/apps/web/src/lib/persistence/recoverable-idb.ts b/apps/web/src/lib/persistence/recoverable-idb.ts
new file mode 100644
index 0000000000..7a85371c91
--- /dev/null
+++ b/apps/web/src/lib/persistence/recoverable-idb.ts
@@ -0,0 +1,93 @@
+/** Reopens stale connections without replaying requests or started transactions. */
+export class RecoverableIDB {
+ private connection?: Promise<IDBDatabase>;
+ private disposed = false;
+
+ constructor(
+ private readonly name: string,
+ private readonly version: number,
+ private readonly upgrade: (db: IDBDatabase, tx: IDBTransaction) => void,
+ private readonly onVersionChange?: () => void
+ ) {}
+
+ open(): Promise<IDBDatabase> {
+ if (this.disposed)
+ throw new DOMException(
+ 'Database connection disposed',
+ 'InvalidStateError'
+ );
+ if (this.connection) return this.connection;
+ const request = indexedDB.open(this.name, this.version);
+ const opening = new Promise<IDBDatabase>((resolve, reject) => {
+ request.onupgradeneeded = () =>
+ this.upgrade(request.result, request.transaction!);
+ request.onerror = () => {
+ if (this.connection === opening) this.connection = undefined;
+ reject(request.error);
+ };
+ request.onsuccess = () => {
+ const db = request.result;
+ const invalidate = () => {
+ if (this.connection === opening) this.connection = undefined;
+ };
+ db.onclose = invalidate;
+ db.onversionchange = () => {
+ db.close();
+ invalidate();
+ this.onVersionChange?.();
+ };
+ if (this.disposed) {
+ db.close();
+ invalidate();
+ reject(
+ new DOMException(
+ 'Database connection disposed',
+ 'InvalidStateError'
+ )
+ );
+ return;
+ }
+ resolve(db);
+ };
+ });
+ this.connection = opening;
+ return opening;
+ }
+
+ async transaction(stores: string | string[], mode: IDBTransactionMode) {
+ for (let attempt = 0; ; attempt++) {
+ const opening = this.open();
+ const db = await opening;
+ if (this.disposed)
+ throw new DOMException(
+ 'Database connection disposed',
+ 'InvalidStateError'
+ );
+ try {
+ return db.transaction(stores, mode);
+ } catch (error) {
+ if (
+ !(error instanceof DOMException) ||
+ error.name !== 'InvalidStateError'
+ )
+ throw error;
+ if (this.connection === opening) this.connection = undefined;
+ db.close();
+ // Creation failed before any requests ran, so a single retry is safe.
+ if (attempt === 1) throw error;
+ }
+ }
+ }
+
+ /** Terminal close: pending opens cannot revive a disposed owner. */
+ async close(): Promise<void> {
+ this.disposed = true;
+ const opening = this.connection;
+ this.connection = undefined;
+ try {
+ (await opening)?.close();
+ } catch {
+ // A failed open has no connection to close; its caller observes the error.
+ }
+ }
+}
diff --git a/apps/web/src/lib/queries/email/local-draft-store.test.ts b/apps/web/src/lib/queries/email/local-draft-store.test.ts
index 9d081d47f3..1ebde171cb 100644
--- a/apps/web/src/lib/queries/email/local-draft-store.test.ts
+++ b/apps/web/src/lib/queries/email/local-draft-store.test.ts
@@ -1,7 +1,10 @@
import { Blob as NodeBlob } from 'node:buffer';
import 'fake-indexeddb/auto';
import type { LocalDraft } from '@app/features/email-compose/core/local-draft';
-import { afterEach, describe, expect, it, vi } from 'vitest';
+import { createDraftAutosave } from '@app/features/email-compose/primitives/draft-autosave';
+import { forceCloseDatabase } from 'fake-indexeddb';
+import { createRoot } from 'solid-js';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { createLocalDraftStore } from './local-draft-store';
const stores: ReturnType<typeof createLocalDraftStore>[] = [];
@@ -32,6 +35,197 @@ afterEach(async () => {
vi.restoreAllMocks();
});
+describe('draft connection recovery', () => {
+ let connections: IDBDatabase[];
+ beforeEach(() => {
+ connections = [];
+ const original = indexedDB.open.bind(indexedDB);
+ vi.spyOn(indexedDB, 'open').mockImplementation((...args) => {
+ const request = original(...args);
+ request.addEventListener('success', () =>
+ connections.push(request.result)
+ );
+ return request;
+ });
+ });
+
+ it.each(['closed', 'terminated'])(
+ 'saves the latest edit and continues server autosave after a connection is %s',
+ async (reason) => {
+ const store = open();
+ const session = await store.activate('owner');
+ await store.save(session, snapshot());
+ if (reason === 'closed') connections[0].close();
+ else {
+ const db = connections[0];
+ const closed = new Promise<void>((resolve) =>
+ db.addEventListener('close', () => resolve(), { once: true })
+ );
+ // fake-indexeddb accepts an instance but declares a constructor type.
+ forceCloseDatabase(
+ db as unknown as Parameters<typeof forceCloseDatabase>[0]
+ );
+ await closed;
+ }
+ const latest = snapshot('After resuming', 1);
+ const persist = vi.fn(async () => {
+ expect(await store.read(session, 'local')).toMatchObject({
+ revision: 2,
+ content: latest.content,
+ });
+ });
+ const onLocalError = vi.fn();
+ let dispose = () => {};
+ const autosave = createRoot((cleanup) => {
+ dispose = cleanup;
+ return createDraftAutosave({
+ capture: () => latest,
+ saveLocalSnapshot: async (value) => {
+ await store.save(session, value);
+ },
+ persist,
+ paused: () => false,
+ onLocalError,
+ });
+ });
+ try {
+ await autosave.save();
+ expect(persist).toHaveBeenCalledExactlyOnceWith(latest);
+ expect(onLocalError).not.toHaveBeenCalled();
+ expect(autosave.localSaveState()).toBe('saved');
+ expect(connections).toHaveLength(2);
+ } finally {
+ dispose();
+ }
+ }
+ );
+
+ it('shares a reopened connection across concurrent operations without duplicate revisions', async () => {
+ const store = open();
+ const session = await store.activate('owner');
+ await store.save(session, snapshot());
+ connections[0].close();
+ const notify = vi.fn();
+ store.subscribe(notify);
+ await Promise.all([
+ store.save(session, snapshot('New revision', 1)),
+ store.list(session),
+ store.read(session, 'local'),
+ ]);
+ expect(connections).toHaveLength(2);
+ expect(notify).toHaveBeenCalledOnce();
+ expect(await store.read(session, 'local')).toMatchObject({ revision: 2 });
+ });
+
+ it('bounds retries, reports one failure per episode and allows a later recovery', async () => {
+ const report = vi.fn();
+ const store = createLocalDraftStore(crypto.randomUUID(), report);
+ stores.push(store);
+ const session = await store.activate('owner');
+ const error = new DOMException(
+ 'Connection is closing',
+ 'InvalidStateError'
+ );
+ const transaction = vi
+ .spyOn(IDBDatabase.prototype, 'transaction')
+ .mockImplementation(() => {
+ throw error;
+ });
+ await expect(store.save(session, snapshot())).rejects.toBe(error);
+ expect(transaction).toHaveBeenCalledTimes(2);
+ await expect(store.save(session, snapshot())).rejects.toBe(error);
+ expect(transaction).toHaveBeenCalledTimes(4);
+ expect(report).toHaveBeenCalledExactlyOnceWith({ type: 'failed', error });
+ transaction.mockRestore();
+ await store.save(session, snapshot());
+ expect(report).toHaveBeenLastCalledWith({ type: 'recovered' });
+ expect(await store.read(session, 'local')).toMatchObject({ revision: 1 });
+ });
+
+ it('does not retry unrelated transaction errors', async () => {
+ const store = open();
+ const session = await store.activate('owner');
+ const error = new DOMException('Quota exhausted', 'QuotaExceededError');
+ const transaction = vi
+ .spyOn(connections[0], 'transaction')
+ .mockImplementationOnce(() => {
+ throw error;
+ });
+ await expect(store.save(session, snapshot())).rejects.toBe(error);
+ expect(transaction).toHaveBeenCalledOnce();
+ expect(connections).toHaveLength(1);
+ await store.save(session, snapshot());
+ });
+
+ it('never replays a started transaction when a request throws InvalidStateError', async () => {
+ const store = open();
+ const session = await store.activate('owner');
+ const error = new DOMException('Request failed', 'InvalidStateError');
+ const get = vi
+ .spyOn(IDBObjectStore.prototype, 'get')
+ .mockImplementationOnce(() => {
+ throw error;
+ });
+ await expect(store.save(session, snapshot())).rejects.toBe(error);
+ expect(get).toHaveBeenCalledOnce();
+ expect(connections).toHaveLength(1);
+ await store.save(session, snapshot());
+ expect(await store.read(session, 'local')).toMatchObject({ revision: 1 });
+ });
+
+ it('keeps account fences after reopening', async () => {
+ const name = crypto.randomUUID();
+ const store = open(name);
+ const session = await store.activate('owner');
+ await store.save(session, snapshot());
+ connections[0].close();
+ const other = open(name);
+ await other.clear(true);
+ await other.activate('other-owner');
+ await expect(store.save(session, snapshot('Late edit', 1))).rejects.toThrow(
+ 'no longer active'
+ );
+ expect(await store.list(session)).toEqual([]);
+ });
+
+ it('retries a later open after a synchronous opening failure', async () => {
+ const store = open();
+ vi.mocked(indexedDB.open).mockImplementationOnce(() => {
+ throw new DOMException('Storage unavailable', 'UnknownError');
+ });
+ await expect(store.activate('owner')).rejects.toThrow(
+ 'Storage unavailable'
+ );
+ const session = await store.activate('owner');
+ await store.save(session, snapshot());
+ expect(await store.list(session)).toHaveLength(1);
+ });
+
+ it('does not let an old close event invalidate the replacement connection', async () => {
+ const store = open();
+ const session = await store.activate('owner');
+ const old = connections[0];
+ old.close();
+ await store.save(session, snapshot());
+ old.onclose?.call(old, new Event('close'));
+ await store.list(session);
+ expect(connections).toHaveLength(2);
+ });
+
+ it('keeps persistence working when recovery reporting throws', async () => {
+ const report = vi.fn(() => {
+ throw new Error('Telemetry unavailable');
+ });
+ const store = createLocalDraftStore(crypto.randomUUID(), report);
+ stores.push(store);
+ const session = await store.activate('owner');
+ connections[0].close();
+ await store.save(session, snapshot());
+ expect(report).toHaveBeenCalledExactlyOnceWith({ type: 'recovered' });
+ expect(await store.list(session)).toHaveLength(1);
+ });
+});
+
describe('durable email working copies', () => {
it('retains rejected content and further edits across reopening', async () => {
const name = crypto.randomUUID();
diff --git a/apps/web/src/lib/queries/email/local-draft-store.ts b/apps/web/src/lib/queries/email/local-draft-store.ts
index 275c7d446c..9c1345a0bd 100644
--- a/apps/web/src/lib/queries/email/local-draft-store.ts
+++ b/apps/web/src/lib/queries/email/local-draft-store.ts
@@ -12,11 +12,25 @@ type Session = { accountId: string; epoch: string };
type StoredDraft = LocalDraft & { epoch: string };
const STORES = ['drafts', 'files', 'attempts', 'meta'] as const;
+export type DraftStorageEvent =
+ | { type: 'closed' | 'recovered' }
+ | { type: 'failed'; error: unknown };
+
/** Transactional working-copy storage. Unlike a query cache, it never evicts edits. */
export function createLocalDraftStore(
- dbName = 'macro-email-working-copies-v1'
+ dbName = 'macro-email-working-copies-v1',
+ onStorageEvent?: (event: DraftStorageEvent) => void
) {
let connection: Promise<IDBDatabase> | undefined;
+ let recovering = false;
+ let failed = false;
+ const report = (event: DraftStorageEvent) => {
+ try {
+ onStorageEvent?.(event);
+ } catch {
+ /* Observability must not interrupt draft persistence. */
+ }
+ };
const listeners = new Set<() => void>();
let channel: BroadcastChannel | undefined;
const notify = () => {
@@ -36,21 +50,28 @@ export function createLocalDraftStore(
/* Durable state remains readable without notifications. */
}
};
- const open = () =>
- (connection ??= new Promise<IDBDatabase>((resolve, reject) => {
- const request = indexedDB.open(dbName, 1);
+ const open = () => {
+ if (connection) return connection;
+ const request = indexedDB.open(dbName, 1);
+ const opening = new Promise<IDBDatabase>((resolve, reject) => {
request.onupgradeneeded = () => {
for (const name of STORES) request.result.createObjectStore(name);
};
request.onerror = () => {
- connection = undefined;
+ if (connection === opening) connection = undefined;
reject(request.error);
};
request.onsuccess = () => {
const db = request.result;
+ db.onclose = () => {
+ if (connection !== opening) return;
+ connection = undefined;
+ recovering = true;
+ report({ type: 'closed' });
+ };
db.onversionchange = () => {
db.close();
- connection = undefined;
+ if (connection === opening) connection = undefined;
};
resolve(db);
try {
@@ -62,33 +83,65 @@ export function createLocalDraftStore(
/* Embedded browsers may expose an unavailable channel. */
}
};
- }));
+ });
+ connection = opening;
+ return opening;
+ };
+ async function beginTransaction(mode: IDBTransactionMode) {
+ for (let attempt = 0; ; attempt++) {
+ const opening = open();
+ const db = await opening;
+ try {
+ return db.transaction([...STORES], mode);
+ } catch (error) {
+ if (
+ !(error instanceof DOMException) ||
+ error.name !== 'InvalidStateError'
+ )
+ throw error;
+ if (connection === opening) connection = undefined;
+ db.close();
+ recovering = true;
+ // No requests have run yet. Never replay a started draft transaction.
+ if (attempt === 1) throw error;
+ }
+ }
+ }
async function transaction<T>(
mode: IDBTransactionMode,
run: (tx: IDBTransaction, finish: (value: T) => void) => void,
publish = true
): Promise<T> {
- const db = await open();
- return await new Promise<T>((resolve, reject) => {
- const tx = db.transaction([...STORES], mode);
- let value: T;
- tx.oncomplete = () => {
- resolve(value);
- if (mode === 'readwrite' && publish) changed();
- };
- tx.onabort = () =>
- reject(tx.error ?? new Error('Draft storage transaction aborted'));
- tx.onerror = () =>
- reject(tx.error ?? new Error('Unable to save draft on this device'));
- try {
- run(tx, (next) => {
- value = next;
- });
- } catch (error) {
- tx.abort();
- reject(error);
- }
- });
+ try {
+ const tx = await beginTransaction(mode);
+ const result = await new Promise<T>((resolve, reject) => {
+ let value: T;
+ tx.oncomplete = () => {
+ resolve(value);
+ if (mode === 'readwrite' && publish) changed();
+ };
+ tx.onabort = () =>
+ reject(tx.error ?? new Error('Draft storage transaction aborted'));
+ tx.onerror = () =>
+ reject(tx.error ?? new Error('Unable to save draft on this device'));
+ try {
+ run(tx, (next) => {
+ value = next;
+ });
+ } catch (error) {
+ tx.abort();
+ reject(error);
+ }
+ });
+ if (recovering || failed) report({ type: 'recovered' });
+ recovering = false;
+ failed = false;
+ return result;
+ } catch (error) {
+ if (!failed) report({ type: 'failed', error });
+ failed = true;
+ throw error;
+ }
}
function withSession<T>(
tx: IDBTransaction,
diff --git a/apps/web/src/lib/queries/email/local-drafts.test.ts b/apps/web/src/lib/queries/email/local-drafts.test.ts
index 45850cc382..15c36cfc3a 100644
--- a/apps/web/src/lib/queries/email/local-drafts.test.ts
+++ b/apps/web/src/lib/queries/email/local-drafts.test.ts
@@ -8,6 +8,11 @@ import type { OperationResult } from '@urql/core';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
const auth = vi.hoisted(() => ({ user: { authenticated: true, id: 'owner' } }));
+const telemetry = vi.hoisted(() => ({
+ track: vi.fn(),
+ posthog: { captureException: vi.fn() },
+}));
+vi.mock('@app/lib/analytics', () => ({ analytics: telemetry }));
const stagedUpload = vi.hoisted(() =>
vi.fn<() => { previewSrc: string; size: number } | undefined>()
);
@@ -50,15 +55,43 @@ beforeEach(async () => {
queue.length = 0;
auth.user = { authenticated: true, id: 'owner' };
stagedUpload.mockReset();
+ telemetry.track.mockClear();
+ telemetry.posthog.captureException.mockClear();
runtime = await import('./local-drafts');
await runtime.localDraftStore.clear();
});
afterEach(async () => {
vi.unstubAllGlobals();
await runtime.localDraftStore.close();
+ vi.restoreAllMocks();
});
describe('draft queue recovery', () => {
+ it('reports caught storage failures and recovery with lifecycle context, without draft content', async () => {
+ const error = new DOMException('Storage unavailable', 'UnknownError');
+ vi.spyOn(IDBDatabase.prototype, 'transaction').mockImplementationOnce(
+ () => {
+ throw error;
+ }
+ );
+ await expect(runtime.saveLocalDraft(input('Private content'))).rejects.toBe(
+ error
+ );
+ const context = {
+ storage: 'email-working-copies',
+ visibility: document.visibilityState,
+ online: navigator.onLine,
+ };
+ expect(telemetry.posthog.captureException).toHaveBeenCalledExactlyOnceWith(
+ error,
+ context
+ );
+ await runtime.saveLocalDraft(input('Private content'));
+ expect(telemetry.track).toHaveBeenCalledExactlyOnceWith(
+ 'email_draft_storage_recovered',
+ context
+ );
+ });
it('rejects an old editor generation after discard and Undo reuse the server identity', async () => {
const serverInput = (subject: string) => ({
draft: { db_id: 'server', subject },
diff --git a/apps/web/src/lib/queries/email/local-drafts.ts b/apps/web/src/lib/queries/email/local-drafts.ts
index f7f6acb13e..b752734eb4 100644
--- a/apps/web/src/lib/queries/email/local-drafts.ts
+++ b/apps/web/src/lib/queries/email/local-drafts.ts
@@ -6,6 +6,7 @@ import type {
} from '@app/features/email-compose/core/local-draft';
import type { DraftFormAttachment } from '@app/features/email-compose/primitives/email-form-state';
import type { EmailMessage } from '@app/features/email-message/core/email-message';
+import { analytics } from '@app/lib/analytics';
import { getNativeStagedUpload } from '@core/mobile/nativeStagedUpload';
import type { NormalizedCacheExchangeOptions } from '@graphql-cache/exchange/normalized-cache-exchange';
import type { CacheHost } from '@graphql-cache/host/types';
@@ -19,7 +20,18 @@ import type { UserInfoData } from '../auth/user-info';
import { queryClient } from '../client';
import { createLocalDraftStore } from './local-draft-store';
-export const localDraftStore = createLocalDraftStore();
+export const localDraftStore = createLocalDraftStore(undefined, (event) => {
+ const context = {
+ storage: 'email-working-copies',
+ visibility: document.visibilityState,
+ online: navigator.onLine,
+ };
+ if (event.type === 'failed') {
+ analytics.posthog.captureException(event.error, context);
+ } else {
+ analytics.track('email_draft_storage_' + event.type, context);
+ }
+});
const fileIds = new WeakMap<File, string>();
const fileCopies = new WeakMap<File, Promise<Blob>>();
const pendingWrites = new Set<Promise<LocalDraft>>();
diff --git a/apps/web/src/lib/queries/persistence/per-query-idb.test.ts b/apps/web/src/lib/queries/persistence/per-query-idb.test.ts
index f830dd29d2..6c703b38ad 100644
--- a/apps/web/src/lib/queries/persistence/per-query-idb.test.ts
+++ b/apps/web/src/lib/queries/persistence/per-query-idb.test.ts
@@ -1,6 +1,6 @@
import 'fake-indexeddb/auto';
import { IDBObjectStore } from 'fake-indexeddb';
-import { afterEach, describe, expect, it, vi } from 'vitest';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import {
createPerQueryIDBStore,
type PersistedQueryEntry,
@@ -20,7 +20,92 @@ const store = () =>
debounceMs: 60_000,
});
-afterEach(() => vi.restoreAllMocks());
+let connections: IDBDatabase[];
+beforeEach(() => {
+ connections = [];
+ const open = indexedDB.open.bind(indexedDB);
+ vi.spyOn(indexedDB, 'open').mockImplementation((...args) => {
+ const request = open(...args);
+ request.addEventListener('success', () => connections.push(request.result));
+ return request;
+ });
+});
+afterEach(() => {
+ vi.restoreAllMocks();
+ for (const db of connections) db.close();
+});
+
+describe('query persistence connection recovery', () => {
+ it('recovers reads, writes, deletes and clear without losing cached entries', async () => {
+ const db = store();
+ db.set(entry('original'));
+ await db.flush({ throwOnError: true });
+ connections.at(-1)!.close();
+ expect(await db.get('original')).toEqual(entry('original'));
+ connections.at(-1)!.close();
+ db.set(entry('next'));
+ await db.flush({ throwOnError: true });
+ expect(await db.get('original')).toEqual(entry('original'));
+ connections.at(-1)!.close();
+ db.remove('original');
+ await db.flush({ throwOnError: true });
+ expect(await db.get('original')).toBeUndefined();
+ expect(await db.get('next')).toEqual(entry('next'));
+ connections.at(-1)!.close();
+ await db.clear();
+ expect(await db.get('next')).toBeUndefined();
+ expect(connections).toHaveLength(5);
+ });
+
+ it('shares the replacement across stores for the same database', async () => {
+ const name = crypto.randomUUID();
+ const first = createPerQueryIDBStore({ dbName: name });
+ const second = createPerQueryIDBStore({ dbName: name });
+ await Promise.all([first.get('missing'), second.get('missing')]);
+ expect(connections).toHaveLength(1);
+ connections[0].close();
+ await Promise.all([first.get('missing'), second.get('missing')]);
+ expect(connections).toHaveLength(2);
+ });
+
+ it('does not write an old generation when clear runs during recovery', async () => {
+ const db = store();
+ await db.clear();
+ let clearing: Promise<void> | undefined;
+ vi.spyOn(connections[0], 'transaction').mockImplementationOnce(() => {
+ clearing = db.clear();
+ throw new DOMException('Closing', 'InvalidStateError');
+ });
+ const put = vi.spyOn(IDBObjectStore.prototype, 'put');
+ db.set(entry('old'));
+ await db.flush({ throwOnError: true });
+ await clearing;
+ expect(put).not.toHaveBeenCalled();
+ db.set(entry('new'));
+ await db.flush({ throwOnError: true });
+ expect(await db.get('old')).toBeUndefined();
+ expect(await db.get('new')).toEqual(entry('new'));
+ });
+
+ it('retains pending writes after recovery is exhausted', async () => {
+ const db = store();
+ await db.clear();
+ vi.spyOn(console, 'error').mockImplementation(() => {});
+ const transaction = vi
+ .spyOn(IDBDatabase.prototype, 'transaction')
+ .mockImplementation(() => {
+ throw new DOMException('Closing', 'InvalidStateError');
+ });
+ db.set(entry('pending'));
+ await expect(db.flush({ throwOnError: true })).rejects.toMatchObject({
+ name: 'InvalidStateError',
+ });
+ expect(transaction).toHaveBeenCalledTimes(2);
+ transaction.mockRestore();
+ await db.flush({ throwOnError: true });
+ expect(await db.get('pending')).toEqual(entry('pending'));
+ });
+});
describe('clearable query persistence', () => {
it('rejects a strict flush on storage failure and retries the pending data', async () => {
diff --git a/apps/web/src/lib/queries/persistence/per-query-idb.ts b/apps/web/src/lib/queries/persistence/per-query-idb.ts
index 55ae3bb707..fc68221ce5 100644
--- a/apps/web/src/lib/queries/persistence/per-query-idb.ts
+++ b/apps/web/src/lib/queries/persistence/per-query-idb.ts
@@ -1,4 +1,5 @@
import type { QueryKey } from '@tanstack/query-core';
+import { RecoverableIDB } from '../../persistence/recoverable-idb';
export type PersistedQueryEntry = Readonly<{
queryHash: string;
@@ -29,27 +30,16 @@ type PerQueryPersistenceOptions = Readonly<{
const STORE_NAME = 'queries';
const DEFAULT_DEBOUNCE_MS = 1000;
-const dbCache = new Map<string, IDBDatabase>();
+const dbCache = new Map<string, RecoverableIDB>();
-function openDB(dbName: string): Promise<IDBDatabase> {
+function database(dbName: string): RecoverableIDB {
const cached = dbCache.get(dbName);
- if (cached) return Promise.resolve(cached);
- return new Promise((resolve, reject) => {
- const req = indexedDB.open(dbName, 1);
- req.onupgradeneeded = () => {
- const db = req.result;
- if (!db.objectStoreNames.contains(STORE_NAME)) {
- db.createObjectStore(STORE_NAME);
- }
- };
- req.onsuccess = () => {
- const db = req.result;
- db.onclose = () => dbCache.delete(dbName);
- dbCache.set(dbName, db);
- resolve(db);
- };
- req.onerror = () => reject(req.error);
+ if (cached) return cached;
+ const db = new RecoverableIDB(dbName, 1, (db) => {
+ db.createObjectStore(STORE_NAME);
});
+ dbCache.set(dbName, db);
+ return db;
}
async function withStore<T>(
@@ -57,9 +47,8 @@ async function withStore<T>(
mode: IDBTransactionMode,
fn: (store: IDBObjectStore) => IDBRequest<T>
): Promise<T> {
- const db = await openDB(dbName);
+ const tx = await database(dbName).transaction(STORE_NAME, mode);
return new Promise<T>((resolve, reject) => {
- const tx = db.transaction(STORE_NAME, mode);
const store = tx.objectStore(STORE_NAME);
const req = fn(store);
req.onsuccess = () => resolve(req.result);
@@ -109,9 +98,8 @@ export function createPerQueryIDBStore(
try {
await enqueueWrite(async () => {
- const db = await openDB(dbName);
+ const tx = await database(dbName).transaction(STORE_NAME, 'readwrite');
if (generation !== currentGeneration) return;
- const tx = db.transaction(STORE_NAME, 'readwrite');
const store = tx.objectStore(STORE_NAME);
for (const [hash, entry] of puts) {
@@ -188,8 +176,7 @@ export function createPerQueryIDBStore(
pendingPuts.clear();
pendingDeletes.clear();
await enqueueWrite(async () => {
- const db = await openDB(dbName);
- const tx = db.transaction(STORE_NAME, 'readwrite');
+ const tx = await database(dbName).transaction(STORE_NAME, 'readwrite');
tx.objectStore(STORE_NAME).clear();
await new Promise<void>((resolve, reject) => {
tx.oncomplete = () => resolve();
diff --git a/apps/web/src/lib/workers/slack-import/message-store.test.ts b/apps/web/src/lib/workers/slack-import/message-store.test.ts
index fa84d7e623..c7d81353eb 100644
--- a/apps/web/src/lib/workers/slack-import/message-store.test.ts
+++ b/apps/web/src/lib/workers/slack-import/message-store.test.ts
@@ -1,7 +1,8 @@
// @vitest-environment node
import 'fake-indexeddb/auto';
import { readFileSync } from 'node:fs';
-import { afterEach, describe, expect, it, vi } from 'vitest';
+import { forceCloseDatabase } from 'fake-indexeddb';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import {
exactTimestamp,
MessageStore,
@@ -28,6 +29,140 @@ afterEach(async () => {
for (const store of stores.splice(0)) await store.dispose();
});
+describe('scratch database recovery', () => {
+ let connections: IDBDatabase[];
+ beforeEach(() => {
+ connections = [];
+ const original = indexedDB.open.bind(indexedDB);
+ vi.spyOn(indexedDB, 'open').mockImplementation((...args) => {
+ const request = original(...args);
+ request.addEventListener('success', () =>
+ connections.push(request.result)
+ );
+ return request;
+ });
+ });
+
+ it.each(['closed', 'terminated'])(
+ 'preserves staged history and byte limits after %s',
+ async (reason) => {
+ const first = { ts: '1.0', text: 'original' };
+ const second = { ts: '2.0', text: 'next' };
+ const bytes =
+ normalizeMessage('C100', first)!.byteLength +
+ normalizeMessage('C100', second)!.byteLength;
+ const store = await open(bytes);
+ await store.stage('C100', [first]);
+ if (reason === 'closed') connections[0].close();
+ else {
+ const closed = new Promise<void>((resolve) =>
+ connections[0].addEventListener('close', () => resolve())
+ );
+ forceCloseDatabase(
+ connections[0] as unknown as Parameters<typeof forceCloseDatabase>[0]
+ );
+ await closed;
+ }
+ await store.stage('C100', [first, second]);
+ expect((await lines(store)).map((line) => JSON.parse(line).text)).toEqual(
+ ['original', 'next']
+ );
+ expect(connections).toHaveLength(2);
+ await expect(store.stage('C100', [{ ts: '3.0' }])).rejects.toMatchObject({
+ code: 'selected_limit',
+ });
+ expect(await lines(store)).toHaveLength(2);
+ }
+ );
+
+ it('continues paginated reads after the connection closes between pages', async () => {
+ const store = await open();
+ await store.stage(
+ 'C100',
+ Array.from({ length: 300 }, (_, n) => ({ ts: n + 1 + '.0' }))
+ );
+ const result: string[] = [];
+ for await (const message of store.ordered('C100')) {
+ result.push(message.line);
+ if (result.length === 128) connections[0].close();
+ }
+ expect(result).toHaveLength(300);
+ expect(new Set(result).size).toBe(300);
+ expect(connections).toHaveLength(2);
+ });
+
+ it.each([false, true])(
+ 'fences a replaced session and leaves its cleanup to the new owner (closed=%s)',
+ async (closed) => {
+ const id = crypto.randomUUID();
+ const first = await MessageStore.open(id, 10000);
+ await first.stage('C100', [{ ts: '1.0' }]);
+ if (closed) connections[0].close();
+ const replacement = await MessageStore.open(id, 10000);
+ stores.push(replacement);
+ await replacement.stage('C100', [{ ts: '2.0' }]);
+ await expect(first.stage('C100', [{ ts: '3.0' }])).rejects.toMatchObject({
+ code: 'storage_unavailable',
+ });
+ await first.dispose();
+ expect(
+ (await lines(replacement)).map((line) => JSON.parse(line).ts)
+ ).toEqual(['2.000000']);
+ }
+ );
+
+ it('does not recreate lost history or reopen after disposal', async () => {
+ const store = await open();
+ await store.stage('C100', [{ ts: '1.0' }]);
+ const native = connections[0];
+ native.close();
+ await new Promise<void>((resolve, reject) => {
+ const request = indexedDB.deleteDatabase(native.name);
+ request.onsuccess = () => resolve();
+ request.onerror = () => reject(request.error);
+ });
+ await expect(lines(store)).rejects.toMatchObject({
+ code: 'storage_unavailable',
+ });
+ await store.dispose();
+ await expect(store.stage('C100', [{ ts: '2.0' }])).rejects.toMatchObject({
+ code: 'storage_unavailable',
+ });
+ expect(
+ (await indexedDB.databases()).some((db) => db.name === native.name)
+ ).toBe(false);
+ });
+
+ it('does not replay a batch that fails after its transaction starts', async () => {
+ const store = await open();
+ await store.stage('C100', [{ ts: '1.0' }]);
+ const put = vi
+ .spyOn(IDBObjectStore.prototype, 'put')
+ .mockImplementationOnce(() => {
+ throw new DOMException('Interrupted request', 'InvalidStateError');
+ });
+ await expect(store.stage('C100', [{ ts: '2.0' }])).rejects.toMatchObject({
+ code: 'storage_unavailable',
+ });
+ expect(put).toHaveBeenCalledOnce();
+ expect(await lines(store)).toHaveLength(1);
+ expect(connections).toHaveLength(1);
+ });
+
+ it('cancels a batch while its connection is reopening', async () => {
+ const store = await open();
+ const abort = new AbortController();
+ vi.spyOn(connections[0], 'transaction').mockImplementationOnce(() => {
+ abort.abort();
+ throw new DOMException('Closing', 'InvalidStateError');
+ });
+ await expect(
+ store.stage('C100', [{ ts: '1.0' }], abort.signal)
+ ).rejects.toMatchObject({ code: 'cancelled' });
+ expect(await lines(store)).toEqual([]);
+ });
+});
+
describe('exact normalization and disk-backed dedupe', () => {
it('normalizes the shared Rust events fixture, strips ignored payloads and skips nonmessages/deletions', async () => {
const records = readFileSync(
diff --git a/apps/web/src/lib/workers/slack-import/message-store.ts b/apps/web/src/lib/workers/slack-import/message-store.ts
index 83da8cccb7..4c58f3922c 100644
--- a/apps/web/src/lib/workers/slack-import/message-store.ts
+++ b/apps/web/src/lib/workers/slack-import/message-store.ts
@@ -1,5 +1,6 @@
import { z } from 'zod';
import { ArchiveError } from '../../../features/slack-import/core/export';
+import { RecoverableIDB } from '../../persistence/recoverable-idb';
export const MAX_RECORD_BYTES = 1024 * 1024;
const PAGE_BYTES = 1024 * 1024;
@@ -123,12 +124,54 @@ export async function deleteMessageStore(sessionId: string): Promise<void> {
/** One writer per session. Transactions and cursor pages are byte- and count-bounded. */
export class MessageStore {
private totalBytes = 0;
+ private initialized = false;
+ private retired = false;
+ private readonly ownerStore = `owner:${crypto.randomUUID()}`;
+ private readonly database: RecoverableIDB;
private constructor(
- private readonly db: IDBDatabase,
private readonly sessionId: string,
private readonly maxBytes: number
- ) {}
+ ) {
+ this.database = new RecoverableIDB(
+ databaseName(sessionId),
+ 1,
+ (db, tx) => {
+ // Recovery cannot silently replace staged history with an empty database.
+ if (this.initialized) {
+ tx.abort();
+ return;
+ }
+ const messages = db.createObjectStore('messages', {
+ keyPath: ['conversation', 'time'],
+ });
+ messages.createIndex('conversation', 'conversation');
+ // Including this empty store in transactions fences a replacement even
+ // when the old connection was already closed and missed versionchange.
+ db.createObjectStore(this.ownerStore);
+ },
+ () => {
+ // Another importer deleted/replaced this session. Never revive the old owner.
+ this.retired = true;
+ void this.database.close();
+ }
+ );
+ }
+
+ private async transaction(mode: IDBTransactionMode) {
+ try {
+ return await this.database.transaction(
+ ['messages', this.ownerStore],
+ mode
+ );
+ } catch (error) {
+ if (error instanceof DOMException && error.name === 'NotFoundError') {
+ this.retired = true;
+ await this.database.close();
+ }
+ throw error;
+ }
+ }
static async open(
sessionId: string,
@@ -137,19 +180,10 @@ export class MessageStore {
// Reusing a caller's session after termination must never reuse partial history.
await deleteMessageStore(sessionId);
try {
- const db = await new Promise<IDBDatabase>((resolve, reject) => {
- const request = indexedDB.open(databaseName(sessionId), 1);
- request.onupgradeneeded = () => {
- const messages = request.result.createObjectStore('messages', {
- keyPath: ['conversation', 'time'],
- });
- messages.createIndex('conversation', 'conversation');
- };
- request.onsuccess = () => resolve(request.result);
- request.onerror = () => reject(request.error);
- });
- db.onversionchange = () => db.close();
- return new MessageStore(db, sessionId, maxBytes);
+ const store = new MessageStore(sessionId, maxBytes);
+ await store.database.open();
+ store.initialized = true;
+ return store;
} catch (error) {
throw storageError(error);
}
@@ -187,8 +221,8 @@ export class MessageStore {
): Promise<void> {
if (signal?.aborted) throw new ArchiveError('cancelled');
try {
+ const tx = await this.transaction('readwrite');
await new Promise<void>((resolve, reject) => {
- const tx = this.db.transaction('messages', 'readwrite');
const store = tx.objectStore('messages');
let bytes = this.totalBytes;
let failure: ArchiveError | undefined;
@@ -240,7 +274,8 @@ export class MessageStore {
}
};
};
- next();
+ if (signal?.aborted) tx.abort();
+ else next();
});
} catch (error) {
throw storageError(error);
@@ -252,8 +287,8 @@ export class MessageStore {
after?: string
): Promise<StoredMessage[]> {
try {
+ const tx = await this.transaction('readonly');
return await new Promise<StoredMessage[]>((resolve, reject) => {
- const tx = this.db.transaction('messages', 'readonly');
const range = IDBKeyRange.bound(
[conversation, after ?? ''],
[conversation, '\uffff'],
@@ -302,7 +337,14 @@ export class MessageStore {
}
async dispose(): Promise<void> {
- this.db.close();
- await deleteMessageStore(this.sessionId);
+ let ownsDatabase = false;
+ try {
+ const db = await this.database.open();
+ ownsDatabase = db.objectStoreNames.contains(this.ownerStore);
+ } catch {
+ // A retired or missing database has no scratch history for us to remove.
+ }
+ await this.database.close();
+ if (ownsDatabase && !this.retired) await deleteMessageStore(this.sessionId);
}
}
diff --git a/bun.lock b/bun.lock
index 7daea6d200..5861f97107 100644
--- a/bun.lock
+++ b/bun.lock
@@ -249,6 +249,7 @@
"@types/node": "^24.8.0",
"@types/ws": "^8.5.14",
"bebop-tools": "3.1.3",
+ "fake-indexeddb": "^6.2.5",
"jsdom": "^26.1.0",
"typescript": "^5.9.3",
"typescript-native": "npm:[email protected]",
diff --git a/docs/AGENT_GUIDE/surfaces.md b/docs/AGENT_GUIDE/surfaces.md
index 9643554226..7458c324f3 100644
--- a/docs/AGENT_GUIDE/surfaces.md
+++ b/docs/AGENT_GUIDE/surfaces.md
@@ -584,6 +584,12 @@ native offline archive support therefore requires a full app build, not just OTA
### Cached Mail filtering
+The per-query caches for email threads, channels, and native offline document
+context reopen closed IndexedDB connections. Verify a cached read and a subsequent
+write after closing the connection, then clear/logout during recovery: old-account
+entries must not return. Only transaction creation retries automatically; persistent
+storage failures still follow the existing cache fallback behavior.
+
Performance check: switch Signal → All twice against a large synchronized cache.
Dense local pages use bounded sort-index candidates rather than sorting the entire
mailbox. Sparse filters and large timestamp ties retain the exact fallback plan.
@@ -735,6 +741,15 @@ when delivery fails. Later edits can show it again after the usual pause.
Verify the label does not cycle through saving/syncing text on each edit and
that retry remains accessible beside the actions on mobile. Editing while failed
continues saving locally without repeatedly submitting the rejected request.
+Local draft storage reopens unexpectedly closed IndexedDB connections. If creating
+a transaction fails because the connection is closing, it retries once before
+running any writes. Verify by closing the draft database connection, then editing:
+the latest content must persist locally and server autosave must continue without
+reloading. Repeat across background/resume on an iPhone. A transaction that has
+already started is never automatically replayed. Persistent failures still block
+server autosave; copy unsaved content before restarting. Storage failures and
+recovery report current page visibility and network state without draft content.
+
Repeated local-save failures show one persistent warning per composer; recovery
or closing that composer dismisses it. A later failure shows a new warning.
Retry preserves the original draft handle. An already-sent rejection drops the
@@ -2211,6 +2226,13 @@ afterwards in **Settings → Billing**, where Guest users can buy Premium or Max
### Slack archive import
+Temporary message storage recovers closed IndexedDB connections without restarting
+the import or deleting staged history. Verify closure between staging batches and
+between cursor pages: all messages must remain ordered, deduplicated, and within
+the original byte limit. Cancellation and cleanup still work during recovery.
+Deleted or replaced scratch databases fail the old import instead of silently
+continuing with incomplete history; its cleanup must leave a replacement intact.
+
In **Team → Connections**, **Import from Slack** appears only for team admins
and owners with `enable-slack-archive-import` enabled. Production rollout stays
off until explicitly enabled. Select an export ZIP, filter the grouped conversation
diff --git a/packages/collaboration/package.json b/packages/collaboration/package.json
index 1575915198..cbeb144e63 100644
--- a/packages/collaboration/package.json
+++ b/packages/collaboration/package.json
@@ -37,6 +37,7 @@
"@types/node": "^24.8.0",
"@types/ws": "^8.5.14",
"bebop-tools": "3.1.3",
+ "fake-indexeddb": "^6.2.5",
"jsdom": "^26.1.0",
"typescript": "^5.9.3",
"typescript-native": "npm:[email protected]",
diff --git a/packages/collaboration/src/collab/indexeddb-recovery.test.ts b/packages/collaboration/src/collab/indexeddb-recovery.test.ts
new file mode 100644
index 0000000000..753bbc678f
--- /dev/null
+++ b/packages/collaboration/src/collab/indexeddb-recovery.test.ts
@@ -0,0 +1,279 @@
+import 'fake-indexeddb/auto';
+import { forceCloseDatabase, IDBFactory } from 'fake-indexeddb';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { IDBSnapshotStore } from './snapshot-store';
+import { BrowserWALStore, WALSyncer } from './wal';
+
+let connections: IDBDatabase[];
+
+beforeEach(() => {
+ connections = [];
+ const factory = new IDBFactory();
+ vi.stubGlobal('indexedDB', factory);
+ const open = factory.open.bind(factory);
+ vi.spyOn(factory, 'open').mockImplementation((...args) => {
+ const request = open(...args);
+ request.addEventListener('success', () => connections.push(request.result));
+ return request;
+ });
+});
+
+afterEach(() => {
+ for (const db of connections) db.close();
+ vi.restoreAllMocks();
+ vi.unstubAllGlobals();
+});
+
+describe('collaboration IndexedDB recovery', () => {
+ it.each(['success', 'cursor failure'])(
+ 'closes the scope-listing connection after %s',
+ async (outcome) => {
+ const first = new BrowserWALStore<number>('wal', 'first');
+ const second = new BrowserWALStore<number>('wal', 'second');
+ await first.append(1);
+ await first.append(2);
+ await second.append(3);
+
+ if (outcome === 'cursor failure') {
+ const error = new Error('Cursor iteration failed');
+ vi.spyOn(IDBCursor.prototype, 'continue').mockImplementationOnce(() => {
+ throw error;
+ });
+ await expect(BrowserWALStore.listScopeIds('wal')).rejects.toBe(error);
+ } else {
+ await expect(BrowserWALStore.listScopeIds('wal')).resolves.toEqual([
+ 'first',
+ 'second',
+ ]);
+ }
+
+ expect(() => connections.at(-1)!.transaction('updates')).toThrow(
+ expect.objectContaining({ name: 'InvalidStateError' })
+ );
+ expect(await first.count()).toBe(2);
+ expect(await second.count()).toBe(1);
+ }
+ );
+
+ it('flushes persisted edits after the WAL connection closes', async () => {
+ const store = new BrowserWALStore<Uint8Array>('wal', 'doc');
+ const push = vi.fn(async (_updates: Uint8Array[]) => true);
+ const wal = new WALSyncer(store, push);
+ await wal.ready();
+ await store.append(new Uint8Array([1, 2, 3]));
+ connections[0]!.close();
+
+ await wal.flush();
+ await wal.flush();
+
+ expect(push).toHaveBeenCalledOnce();
+ // IndexedDB clones into the host realm rather than jsdom's typed-array realm.
+ expect(push.mock.calls[0]![0].map((update) => Array.from(update))).toEqual([
+ [1, 2, 3],
+ ]);
+ expect(await store.getAll()).toEqual([
+ expect.objectContaining({ id: 1, delivered: true }),
+ ]);
+ expect(connections).toHaveLength(2);
+ });
+
+ it('shares a reopened connection across concurrent appends without duplicates', async () => {
+ const store = new BrowserWALStore<number>('wal', 'doc');
+ await store.append(1);
+ connections[0]!.close();
+
+ await Promise.all([store.append(2), store.append(3), store.count()]);
+
+ expect((await store.getAll()).map((entry) => entry.update)).toEqual([
+ 1, 2, 3,
+ ]);
+ expect(connections).toHaveLength(2);
+ });
+
+ it('recovers after the browser forcibly terminates a connection', async () => {
+ const store = new BrowserWALStore<number>('wal', 'doc');
+ await store.append(1);
+ const db = connections[0]!;
+ const closed = new Promise<void>((resolve) =>
+ db.addEventListener('close', () => resolve())
+ );
+ // fake-indexeddb declares a constructor here, but accepts a database instance.
+ forceCloseDatabase(
+ db as unknown as Parameters<typeof forceCloseDatabase>[0]
+ );
+ await closed;
+
+ expect(await store.count()).toBe(1);
+ expect(connections).toHaveLength(2);
+ });
+
+ it('recovers snapshot reads and writes while keeping document scopes separate', async () => {
+ const first = new IDBSnapshotStore<number>('snapshots', 'first');
+ const second = new IDBSnapshotStore<number>('snapshots', 'second');
+ await first.save(1);
+ await second.save(2);
+ connections[0]!.close();
+ expect(await first.load()).toBe(1);
+ connections.at(-1)!.close();
+ await first.save(3);
+
+ expect(await first.load()).toBe(3);
+ expect(await second.load()).toBe(2);
+ });
+
+ it('bounds recovery attempts and permits a later retry', async () => {
+ const store = new BrowserWALStore<number>('wal', 'doc');
+ await store.append(1);
+ const transaction = vi
+ .spyOn(IDBDatabase.prototype, 'transaction')
+ .mockImplementation(() => {
+ throw new DOMException('Connection is closing', 'InvalidStateError');
+ });
+
+ await expect(store.count()).rejects.toMatchObject({
+ name: 'InvalidStateError',
+ });
+ expect(transaction).toHaveBeenCalledTimes(2);
+ transaction.mockRestore();
+
+ expect(await store.count()).toBe(1);
+ });
+
+ it('does not retry unrelated transaction failures', async () => {
+ const store = new BrowserWALStore<number>('wal', 'doc');
+ await store.append(1);
+ const transaction = vi
+ .spyOn(IDBDatabase.prototype, 'transaction')
+ .mockImplementation(() => {
+ throw new DOMException('Storage unavailable', 'UnknownError');
+ });
+
+ await expect(store.append(2)).rejects.toMatchObject({
+ name: 'UnknownError',
+ });
+ expect(transaction).toHaveBeenCalledOnce();
+ transaction.mockRestore();
+ expect(await store.count()).toBe(1);
+ });
+
+ it('retries once after an interrupted append has rolled back', async () => {
+ const store = new BrowserWALStore<number>('wal', 'doc');
+ await store.append(1);
+ const db = connections[0]!;
+ const closed = new Promise<void>((resolve) =>
+ db.addEventListener('close', () => resolve())
+ );
+ const add = IDBObjectStore.prototype.add;
+ const write = vi
+ .spyOn(IDBObjectStore.prototype, 'add')
+ .mockImplementationOnce(function (value, key) {
+ const request = add.call(this, value, key);
+ // Simulate an interrupted write as well as the connection close event.
+ this.transaction.abort();
+ forceCloseDatabase(
+ db as unknown as Parameters<typeof forceCloseDatabase>[0]
+ );
+ return request;
+ });
+
+ await store.append(2);
+ await closed;
+ expect(write).toHaveBeenCalledTimes(2);
+ write.mockRestore();
+
+ expect((await store.getAll()).map((entry) => entry.update)).toEqual([1, 2]);
+ });
+
+ it('retries an abort after request success without duplicating the entry', async () => {
+ const store = new BrowserWALStore<number>('wal', 'doc');
+ await store.append(1);
+ const add = IDBObjectStore.prototype.add;
+ const write = vi
+ .spyOn(IDBObjectStore.prototype, 'add')
+ .mockImplementationOnce(function (value, key) {
+ const request = add.call(this, value, key);
+ request.addEventListener('success', () => this.transaction.abort());
+ return request;
+ });
+
+ await store.append(2);
+
+ expect(write).toHaveBeenCalledTimes(2);
+ expect((await store.getAll()).map((entry) => entry.update)).toEqual([1, 2]);
+ });
+
+ it('does not retry an error when the transaction actually commits', async () => {
+ const store = new BrowserWALStore<number>('wal', 'doc');
+ await store.append(1);
+ const add = IDBObjectStore.prototype.add;
+ const error = new DOMException('Ambiguous request failure', 'AbortError');
+ const write = vi
+ .spyOn(IDBObjectStore.prototype, 'add')
+ .mockImplementationOnce(function (value, key) {
+ add.call(this, value, key);
+ throw error;
+ });
+
+ await expect(store.append(2)).rejects.toBe(error);
+
+ expect(write).toHaveBeenCalledOnce();
+ expect((await store.getAll()).map((entry) => entry.update)).toEqual([1, 2]);
+ });
+
+ it('bounds aborted append retries and leaves existing entries intact', async () => {
+ const store = new BrowserWALStore<number>('wal', 'doc');
+ await store.append(1);
+ const add = IDBObjectStore.prototype.add;
+ const write = vi
+ .spyOn(IDBObjectStore.prototype, 'add')
+ .mockImplementation(function (value, key) {
+ const request = add.call(this, value, key);
+ this.transaction.abort();
+ return request;
+ });
+
+ await expect(store.append(2)).rejects.toMatchObject({ name: 'AbortError' });
+ expect(write).toHaveBeenCalledTimes(2);
+ expect((await store.getAll()).map((entry) => entry.update)).toEqual([1]);
+ });
+
+ it('does not retry a transaction aborted by a constraint failure', async () => {
+ const store = new BrowserWALStore<number>('wal', 'doc');
+ await store.append(1);
+ const add = IDBObjectStore.prototype.add;
+ const write = vi
+ .spyOn(IDBObjectStore.prototype, 'add')
+ .mockImplementationOnce(function (value) {
+ return add.call(this, { ...value, id: 1 });
+ });
+
+ await expect(store.append(2)).rejects.toMatchObject({
+ name: 'ConstraintError',
+ });
+ expect(write).toHaveBeenCalledOnce();
+ expect((await store.getAll()).map((entry) => entry.update)).toEqual([1]);
+ });
+
+ it('releases the connection when another client upgrades the database', async () => {
+ const store = new BrowserWALStore<number>('wal', 'doc');
+ await store.append(1);
+ const request = indexedDB.open('wal', 2);
+ await new Promise<void>((resolve, reject) => {
+ request.onsuccess = () => resolve();
+ request.onerror = () => reject(request.error);
+ request.onblocked = () => reject(new Error('Upgrade was blocked'));
+ });
+
+ // The old client must not bypass a schema version change or delete data.
+ await expect(store.count()).rejects.toMatchObject({ name: 'VersionError' });
+ const read = request.result
+ .transaction('updates')
+ .objectStore('updates')
+ .getAll();
+ const entries = await new Promise<unknown[]>((resolve, reject) => {
+ read.onsuccess = () => resolve(read.result);
+ read.onerror = () => reject(read.error);
+ });
+ expect(entries).toHaveLength(1);
+ });
+});
diff --git a/packages/collaboration/src/collab/recoverable-database.ts b/packages/collaboration/src/collab/recoverable-database.ts
new file mode 100644
index 0000000000..4066215844
--- /dev/null
+++ b/packages/collaboration/src/collab/recoverable-database.ts
@@ -0,0 +1,72 @@
+import {
+ type DBSchema,
+ type IDBPDatabase,
+ type OpenDBCallbacks,
+ openDB,
+ type StoreNames,
+} from 'idb';
+
+/** Keeps a connection only while it is usable. Recovery never replays writes. */
+export class RecoverableDatabase<Schema extends DBSchema> {
+ private connection?: Promise<IDBPDatabase<Schema>>;
+
+ constructor(
+ private readonly name: string,
+ private readonly version: number,
+ private readonly upgrade: OpenDBCallbacks<Schema>['upgrade']
+ ) {}
+
+ private async open(): Promise<IDBPDatabase<Schema>> {
+ if (this.connection) return this.connection;
+
+ const connection = openDB<Schema>(this.name, this.version, {
+ upgrade: this.upgrade,
+ terminated: () => {
+ if (this.connection === connection) this.connection = undefined;
+ },
+ });
+ this.connection = connection;
+ try {
+ const db = await connection;
+ db.addEventListener('versionchange', () => {
+ db.close();
+ if (this.connection === connection) this.connection = undefined;
+ });
+ return db;
+ } catch (error) {
+ if (this.connection === connection) this.connection = undefined;
+ throw error;
+ }
+ }
+
+ async transaction<
+ Name extends StoreNames<Schema>,
+ Mode extends IDBTransactionMode,
+ >(store: Name, mode: Mode) {
+ for (let attempt = 0; ; attempt++) {
+ const db = await this.open();
+ try {
+ return db.transaction(store, mode);
+ } catch (error) {
+ if (
+ !(error instanceof DOMException) ||
+ error.name !== 'InvalidStateError'
+ ) {
+ throw error;
+ }
+ // Only transaction creation is retried: no request has run yet, so an
+ // auto-increment WAL append cannot be duplicated by recovery.
+ const connection = this.connection;
+ if (
+ connection &&
+ (await connection) === db &&
+ this.connection === connection
+ ) {
+ this.connection = undefined;
+ }
+ db.close();
+ if (attempt === 1) throw error;
+ }
+ }
+ }
+}
diff --git a/packages/collaboration/src/collab/snapshot-store.ts b/packages/collaboration/src/collab/snapshot-store.ts
index 2881dba137..c823e42a75 100644
--- a/packages/collaboration/src/collab/snapshot-store.ts
+++ b/packages/collaboration/src/collab/snapshot-store.ts
@@ -1,7 +1,8 @@
-import { type DBSchema, type IDBPDatabase, openDB as idbOpen } from 'idb';
+import type { DBSchema } from 'idb';
import { logSyncService } from './logger';
import type { LoroManager } from './manager';
import { LoroManagerError } from './manager-error';
+import { RecoverableDatabase } from './recoverable-database';
import type { GenericRootSchema, RawUpdate } from './shared';
import type { WALStore } from './wal';
@@ -25,22 +26,23 @@ interface SnapshotSchema<T> extends DBSchema {
}
export class IDBSnapshotStore<T> implements SnapshotStore<T> {
- private db: Promise<IDBPDatabase<SnapshotSchema<T>>>;
+ private readonly database: RecoverableDatabase<SnapshotSchema<T>>;
constructor(
dbName: string,
private readonly scopeId: string
) {
- this.db = idbOpen<SnapshotSchema<T>>(dbName, DB_VERSION, {
- upgrade(db) {
- db.createObjectStore(STORE, { keyPath: 'scopeId' });
- },
+ this.database = new RecoverableDatabase(dbName, DB_VERSION, (db) => {
+ db.createObjectStore(STORE, { keyPath: 'scopeId' });
});
}
public async save(snapshot: T): Promise<void> {
- const db = await this.db;
- await db.put(STORE, { scopeId: this.scopeId, snapshot });
+ const tx = await this.database.transaction(STORE, 'readwrite');
+ await Promise.all([
+ tx.store.put({ scopeId: this.scopeId, snapshot }),
+ tx.done,
+ ]);
logSyncService({
documentId: this.scopeId,
level: 'debug',
@@ -50,8 +52,8 @@ export class IDBSnapshotStore<T> implements SnapshotStore<T> {
}
public async load(): Promise<T | null> {
- const db = await this.db;
- const row = await db.get(STORE, this.scopeId);
+ const tx = await this.database.transaction(STORE, 'readonly');
+ const row = await tx.store.get(this.scopeId);
const found = row?.snapshot ?? null;
logSyncService({
documentId: this.scopeId,
@@ -65,8 +67,8 @@ export class IDBSnapshotStore<T> implements SnapshotStore<T> {
}
public async delete(): Promise<void> {
- const db = await this.db;
- await db.delete(STORE, this.scopeId);
+ const tx = await this.database.transaction(STORE, 'readwrite');
+ await Promise.all([tx.store.delete(this.scopeId), tx.done]);
}
}
diff --git a/packages/collaboration/src/collab/wal.test.ts b/packages/collaboration/src/collab/wal.test.ts
index a272890590..b316f576b3 100644
--- a/packages/collaboration/src/collab/wal.test.ts
+++ b/packages/collaboration/src/collab/wal.test.ts
@@ -1,5 +1,6 @@
import { type Span, Telemetry } from '@macro-inc/observability';
import { afterEach, describe, expect, it, vi } from 'vitest';
+import * as logger from './logger';
import type { RawUpdate } from './shared';
import { disposeTelemetryFor } from './telemetry';
import { MockLiveSyncSource, MockWALStore, makeTestWAL } from './testing';
@@ -35,6 +36,119 @@ async function undeliveredCount(
// pruneDelivered() is called by the snapshot tick after a durable snapshot save.
describe('WALSyncer', () => {
+ it('retries failed initialization on demand and shares the retry across callers', async () => {
+ vi.spyOn(logger, 'logSyncService').mockImplementation(() => {});
+ const store = new MockWALStore<RawUpdate>();
+ const error = new DOMException('Temporarily unavailable', 'UnknownError');
+ const retry = Promise.withResolvers<number>();
+ const prune = vi
+ .spyOn(store, 'pruneExpired')
+ .mockRejectedValueOnce(error)
+ .mockReturnValueOnce(retry.promise);
+ const push = vi.fn(async () => true);
+ const wal = new WALSyncer(store, push);
+
+ await expect(wal.ready()).rejects.toBe(error);
+ expect(prune).toHaveBeenCalledOnce();
+ const ready = wal.ready();
+ expect(wal.ready()).toBe(ready);
+ const append = wal.append(new Uint8Array([1]));
+ expect(prune).toHaveBeenCalledTimes(2);
+ expect(await store.count()).toBe(0);
+
+ retry.resolve(0);
+ await append;
+ await wal.pendingFlush;
+ await wal.ready();
+ expect(prune).toHaveBeenCalledTimes(2);
+ expect(push).toHaveBeenCalledExactlyOnceWith([new Uint8Array([1])]);
+ });
+
+ it('handles constructor setup failure even when no caller immediately awaits ready', async () => {
+ const log = vi.spyOn(logger, 'logSyncService').mockImplementation(() => {});
+ const store = new MockWALStore<RawUpdate>();
+ const error = new Error('Initial prune failed');
+ const prune = vi.spyOn(store, 'pruneExpired').mockRejectedValueOnce(error);
+ const wal = new WALSyncer(store, async () => true);
+
+ await vi.waitFor(() =>
+ expect(log).toHaveBeenCalledWith(
+ expect.objectContaining({
+ context: {
+ misc: expect.objectContaining({
+ errStack: error.stack,
+ 'wal.stage': 'initialization',
+ }),
+ },
+ })
+ )
+ );
+ expect(prune).toHaveBeenCalledOnce();
+ await wal.ready();
+ expect(prune).toHaveBeenCalledTimes(2);
+ });
+
+ it('reports distinct flush failures with their stacks during the same outage', async () => {
+ const log = vi.spyOn(logger, 'logSyncService').mockImplementation(() => {});
+ const store = new MockWALStore<RawUpdate>();
+ const wal = new WALSyncer(store, async () => true, 'wal-telemetry-test');
+ await wal.ready();
+ const first = new Error('Storage failed');
+ const second = new Error('Storage failed');
+ first.stack = 'Error: Storage failed\n at firstOperation';
+ second.stack = 'Error: Storage failed\n at secondOperation';
+ const getAll = vi.spyOn(store, 'getAll').mockRejectedValue(first);
+ await expect(wal.flush()).rejects.toBe(first);
+ await expect(wal.flush()).rejects.toBe(first);
+ getAll.mockRejectedValue(second);
+ await expect(wal.flush()).rejects.toBe(second);
+ await expect(wal.flush()).rejects.toBe(second);
+
+ const errors = log.mock.calls.filter(([entry]) => entry.level === 'error');
+ expect(errors.map(([entry]) => entry.context.misc?.errStack)).toEqual([
+ first.stack,
+ second.stack,
+ ]);
+ });
+
+ it('handles background storage failures once per outage and retries on the next trigger', async () => {
+ const log = vi.spyOn(logger, 'logSyncService').mockImplementation(() => {});
+ const store = new MockWALStore<RawUpdate>();
+ const push = vi.fn(async () => true);
+ const wal = new WALSyncer(store, push, 'wal-telemetry-test');
+ await wal.ready();
+ await store.append(new Uint8Array([1]));
+ const getAll = vi
+ .spyOn(store, 'getAll')
+ .mockRejectedValue(
+ new DOMException('Storage unavailable', 'UnknownError')
+ );
+
+ void wal.flush();
+ await vi.waitFor(() =>
+ expect(
+ log.mock.calls.filter(([entry]) => entry.level === 'error')
+ ).toHaveLength(1)
+ );
+ await expect(wal.flush()).rejects.toMatchObject({ name: 'UnknownError' });
+ expect(
+ log.mock.calls.filter(([entry]) => entry.level === 'error')
+ ).toHaveLength(1);
+ expect(push).not.toHaveBeenCalled();
+
+ getAll.mockRestore();
+ await wal.flush();
+ expect(push).toHaveBeenCalledExactlyOnceWith([new Uint8Array([1])]);
+
+ vi.spyOn(store, 'getAll').mockRejectedValue(
+ new DOMException('Storage unavailable', 'UnknownError')
+ );
+ await expect(wal.flush()).rejects.toMatchObject({ name: 'UnknownError' });
+ expect(
+ log.mock.calls.filter(([entry]) => entry.level === 'error')
+ ).toHaveLength(2);
+ });
+
it('persists before delivering, marks delivered on ack', async () => {
const live = new MockLiveSyncSource();
const { wal, walStore } = makeWAL(live);
diff --git a/packages/collaboration/src/collab/wal.ts b/packages/collaboration/src/collab/wal.ts
index 49a8c25d7c..7752d08813 100644
--- a/packages/collaboration/src/collab/wal.ts
+++ b/packages/collaboration/src/collab/wal.ts
@@ -1,6 +1,7 @@
import type { Span } from '@macro-inc/observability';
import { type DBSchema, type IDBPDatabase, openDB as idbOpen } from 'idb';
import { logSyncService, type WalContext } from './logger';
+import { RecoverableDatabase } from './recoverable-database';
import type { RawUpdate } from './shared';
import type { LiveSyncSource } from './source';
import { telemetrySpan } from './telemetry';
@@ -60,72 +61,91 @@ interface WALSchema<T> extends DBSchema {
};
}
-export class BrowserWALStore<T> implements WALStore<T> {
- /** Resolves to the open IDB database, shared across all operations. */
- private _db: Promise<IDBPDatabase<WALSchema<T>>>;
+function upgradeWAL<T>(db: IDBPDatabase<WALSchema<T>>): void {
+ const store = db.createObjectStore('updates', {
+ keyPath: 'id',
+ autoIncrement: true,
+ });
+ store.createIndex('scopeId', 'scopeId');
+}
- private db(): Promise<IDBPDatabase<WALSchema<T>>> {
- return this._db;
- }
+export class BrowserWALStore<T> implements WALStore<T> {
+ private readonly database: RecoverableDatabase<WALSchema<T>>;
constructor(
dbName: string,
private readonly scopeId: string
) {
- this._db = BrowserWALStore.openDb<T>(dbName);
- }
-
- private static openDb<U>(
- dbName: string
- ): Promise<IDBPDatabase<WALSchema<U>>> {
- return idbOpen<WALSchema<U>>(dbName, DB_VERSION, {
- upgrade(db) {
- const store = db.createObjectStore('updates', {
- keyPath: 'id',
- autoIncrement: true,
- });
- store.createIndex('scopeId', 'scopeId');
- },
- });
+ this.database = new RecoverableDatabase(dbName, DB_VERSION, upgradeWAL);
}
/** List every scopeId that currently has at least one entry. Uses a
* unique-key cursor on the `scopeId` index, so it doesn't load entries. */
static async listScopeIds(dbName: string): Promise<string[]> {
- const db = await BrowserWALStore.openDb<unknown>(dbName);
- const scopeIds: string[] = [];
- let cursor = await db
- .transaction('updates')
- .store.index('scopeId')
- .openKeyCursor(null, 'nextunique');
- while (cursor) {
- scopeIds.push(cursor.key);
- cursor = await cursor.continue();
+ const db = await idbOpen<WALSchema<unknown>>(dbName, DB_VERSION, {
+ upgrade: upgradeWAL,
+ });
+ try {
+ const scopeIds: string[] = [];
+ let cursor = await db
+ .transaction('updates')
+ .store.index('scopeId')
+ .openKeyCursor(null, 'nextunique');
+ while (cursor) {
+ scopeIds.push(cursor.key);
+ cursor = await cursor.continue();
+ }
+ return scopeIds;
+ } finally {
+ db.close();
}
- return scopeIds;
}
public async append(update: T): Promise<void> {
- const db = await this.db();
- await db.add('updates', {
+ const entry = {
scopeId: this.scopeId,
update,
delivered: false,
createdAt: Date.now(),
- });
+ };
+ for (let attempt = 0; ; attempt++) {
+ const tx = await this.database.transaction('updates', 'readwrite');
+ // A rejected request alone does not prove rollback. Wait for the native
+ // transaction outcome before deciding whether replay is safe.
+ const aborted = new Promise<boolean>((resolve) => {
+ tx.addEventListener('abort', () => resolve(true), { once: true });
+ tx.addEventListener('complete', () => resolve(false), { once: true });
+ });
+ try {
+ // Wrap synchronous request errors too, so tx.done is always observed.
+ const write = async () => tx.store.add(entry);
+ await Promise.all([write(), tx.done]);
+ return;
+ } catch (error) {
+ const rolledBack = await aborted;
+ if (
+ attempt === 1 ||
+ !rolledBack ||
+ (tx.error !== null && tx.error.name !== 'AbortError') ||
+ !(error instanceof DOMException) ||
+ error.name !== 'AbortError'
+ ) {
+ throw tx.error ?? error;
+ }
+ }
+ }
}
public async getAll(): Promise<WALEntry<T>[]> {
- const db = await this.db();
- return db.getAllFromIndex('updates', 'scopeId', this.scopeId) as Promise<
+ const tx = await this.database.transaction('updates', 'readonly');
+ return tx.store.index('scopeId').getAll(this.scopeId) as Promise<
WALEntry<T>[]
>;
}
public async markDelivered(ids: number[]): Promise<void> {
if (ids.length === 0) return;
- const db = await this.db();
- const tx = db.transaction('updates', 'readwrite');
+ const tx = await this.database.transaction('updates', 'readwrite');
const store = tx.objectStore('updates');
for (const id of ids) {
const row = await store.get(id);
@@ -135,13 +155,8 @@ export class BrowserWALStore<T> implements WALStore<T> {
}
public async pruneDelivered(): Promise<void> {
- const db = await this.db();
- const entries = await db.getAllFromIndex(
- 'updates',
- 'scopeId',
- this.scopeId
- );
- const tx = db.transaction('updates', 'readwrite');
+ const entries = await this.getAll();
+ const tx = await this.database.transaction('updates', 'readwrite');
const store = tx.objectStore('updates');
for (const row of entries) {
if (row.delivered && row.id !== undefined) {
@@ -152,14 +167,9 @@ export class BrowserWALStore<T> implements WALStore<T> {
}
public async pruneExpired(ttlMs: number): Promise<number> {
- const db = await this.db();
- const entries = await db.getAllFromIndex(
- 'updates',
- 'scopeId',
- this.scopeId
- );
+ const entries = await this.getAll();
const cutoff = Date.now() - ttlMs;
- const tx = db.transaction('updates', 'readwrite');
+ const tx = await this.database.transaction('updates', 'readwrite');
const store = tx.objectStore('updates');
let deleted = 0;
for (const row of entries) {
@@ -173,9 +183,7 @@ export class BrowserWALStore<T> implements WALStore<T> {
}
public async count(): Promise<number> {
- const db = await this.db();
- return (await db.getAllFromIndex('updates', 'scopeId', this.scopeId))
- .length;
+ return (await this.getAll()).length;
}
}
@@ -229,6 +237,7 @@ export class WALSyncer<T> {
/** True if append was called while a flush was in progress. Causes flush
* to re-run after completing so those entries aren't stranded. */
private hasNewPending = false;
+ private reportedFailures = new Map<'initialization' | 'flush', string>();
public pendingFlush: Promise<void> = Promise.resolve();
private cleanupFns: Array<() => void> = [];
private deliveryOutage:
@@ -245,14 +254,14 @@ export class WALSyncer<T> {
* Exists so that we can not "do stuff" until we have pruned expired entries
* (like when snapshot loading occurs right after construction for example).
*/
- private readonly readyPromise: Promise<void>;
+ private readyPromise?: Promise<void>;
constructor(
private readonly store: WALStore<T>,
private readonly push: (items: T[]) => Promise<boolean>,
private readonly label?: string
) {
- this.readyPromise = this.setup();
+ void this.ready();
}
/* Right now just drops expired entries. */
@@ -269,9 +278,24 @@ export class WALSyncer<T> {
}
public ready(): Promise<void> {
+ if (!this.readyPromise) {
+ this.readyPromise = this.setup();
+ void this.observeSetup(this.readyPromise);
+ }
return this.readyPromise;
}
+ private async observeSetup(setup: Promise<void>): Promise<void> {
+ try {
+ await setup;
+ this.reportedFailures.delete('initialization');
+ } catch (error) {
+ // Retry only on the next caller's request, sharing any in-flight setup.
+ if (this.readyPromise === setup) this.readyPromise = undefined;
+ this.reportFailure('initialization', error);
+ }
+ }
+
public async append(item: T): Promise<void> {
await this.ready();
await this.store.append(item);
@@ -295,10 +319,43 @@ export class WALSyncer<T> {
message: `WAL flush: triggered (isFlushing=${this.isFlushing})`,
});
if (this.isFlushing) return this.pendingFlush;
+ this.isFlushing = true;
this.pendingFlush = this.doFlush();
+ // Observe the same promise we return: awaited callers still see failures,
+ // while timer/reconnect callers cannot create unhandled rejections.
+ void this.observeFlush(this.pendingFlush);
return this.pendingFlush;
}
+ private async observeFlush(flush: Promise<void>): Promise<void> {
+ try {
+ await flush;
+ this.reportedFailures.delete('flush');
+ } catch (error) {
+ this.reportFailure('flush', error);
+ }
+ }
+
+ private reportFailure(
+ stage: 'initialization' | 'flush',
+ error: unknown
+ ): void {
+ const details = {
+ errName: error instanceof Error ? error.name : undefined,
+ errMessage: error instanceof Error ? error.message : String(error),
+ errStack: error instanceof Error ? error.stack : undefined,
+ };
+ const fingerprint = JSON.stringify(details);
+ if (this.reportedFailures.get(stage) === fingerprint) return;
+ this.reportedFailures.set(stage, fingerprint);
+ logSyncService({
+ documentId: this.label ?? 'unknown',
+ level: 'error',
+ context: { misc: { ...details, 'wal.stage': stage } },
+ message: `WAL ${stage} failed; retrying on the next trigger`,
+ });
+ }
+
public pruneDelivered(): Promise<void> {
return this.store.pruneDelivered();
}
@@ -327,13 +384,11 @@ export class WALSyncer<T> {
}
private async doFlush(): Promise<void> {
- await this.ready();
-
- this.isFlushing = true;
this.hasNewPending = false;
let succeeded = true;
try {
+ await this.ready();
const entries = await this.store.getAll();
const undelivered = entries.filter((e) => !e.delivered);
if (undelivered.length === 0) return; // nothing to do
@@ -373,6 +428,7 @@ export class WALSyncer<T> {
}
if (succeeded && this.hasNewPending) {
+ this.isFlushing = true;
this.pendingFlush = this.doFlush();
return this.pendingFlush;
}