Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(web): bound GraphQL cache recovery retries (#6173)
Sean Aye5b540d7b7e40
diff --git a/apps/web/src/lib/graphql-cache/worker/coordinator-core.test.ts b/apps/web/src/lib/graphql-cache/worker/coordinator-core.test.tsindex c624b82af1..3887c24ec5 100644--- a/apps/web/src/lib/graphql-cache/worker/coordinator-core.test.ts+++ b/apps/web/src/lib/graphql-cache/worker/coordinator-core.test.ts@@ -424,4 +424,34 @@ describe('CoordinatorCore', () => {},]);});++ it('makes retry exhaustion terminal for current and future tabs', () => {+ const core = new CoordinatorCore('scope');+ core.registerTab('tab-a');+ core.registerTab('tab-b');+ ready(core, 'tab-a', 1, 'opened-existing');+ core.ownerLost('tab-a', 1, 'OPFS recovery failed');+ core.request('tab-b', { id: 7, kind: 'clear' });++ expect(core.terminalFailure('cache recovery exhausted')).toEqual([+ { kind: 'terminal-failure', error: 'cache recovery exhausted' },+ ]);+ expect(core.state).toEqual({+ kind: 'failed',+ reason: 'cache recovery exhausted',+ });+ expect(core.snapshot().queuedRequestCount).toBe(0);+ expect(core.request('tab-b', { id: 8, kind: 'clear' })).toEqual([+ {+ kind: 'reject-request',+ tabId: 'tab-b',+ requestId: 8,+ error: 'cache recovery exhausted',+ },+ ]);+ expect(core.registerTab('tab-c')).toEqual([+ { kind: 'terminal-failure', error: 'cache recovery exhausted' },+ ]);+ expect(core.resumeAfterLoss()).toEqual([]);+ });});diff --git a/apps/web/src/lib/graphql-cache/worker/coordinator-core.ts b/apps/web/src/lib/graphql-cache/worker/coordinator-core.tsindex 7e118eb584..5b807389f7 100644--- a/apps/web/src/lib/graphql-cache/worker/coordinator-core.ts+++ b/apps/web/src/lib/graphql-cache/worker/coordinator-core.ts@@ -29,7 +29,8 @@ export type CoordinatorState =previousEpoch: OwnerEpoch;nextEpoch: OwnerEpoch;reason: string;- };+ }+ | { kind: 'failed'; reason: string };export type CoordinatorSnapshot = {scope: string;@@ -89,7 +90,8 @@ export type CoordinatorAction =routeId?: RouteId;reason: 'stale-epoch' | 'unknown-route' | 'inactive-engine';}- | { kind: 'protocol-violation'; error: string };+ | { kind: 'protocol-violation'; error: string }+ | { kind: 'terminal-failure'; error: string };export type EngineReady = {tabId: string;@@ -144,6 +146,9 @@ export class CoordinatorCore {registerTab(tabId: string): CoordinatorAction[] {if (this.tabs.includes(tabId)) return [];this.tabs.push(tabId);+ if (this.stateValue.kind === 'failed') {+ return [{ kind: 'terminal-failure', error: this.stateValue.reason }];+ }if (this.stateValue.kind !== 'waiting-for-tab') return [];return this.activateNext(this.stateValue.nextDatabaseAction);}@@ -157,6 +162,9 @@ export class CoordinatorCore {if (this.retiringTabs.has(tabId)) {return [this.reject(tabId, request.id, 'requester tab is retiring')];}+ if (this.stateValue.kind === 'failed') {+ return [this.reject(tabId, request.id, this.stateValue.reason)];+ }const queued = { tabId, request };if (this.stateValue.kind !== 'active') {@@ -303,6 +311,16 @@ export class CoordinatorCore {return this.transitionToAbruptLoss(tabId, ownerEpoch, reason);}+ /** Stops owner recovery and fails every current or future page connection. */+ terminalFailure(reason: string): CoordinatorAction[] {+ if (this.stateValue.kind === 'failed') return [];+ this.queuedRequests.length = 0;+ this.inFlight.clear();+ this.stateValue = { kind: 'failed', reason };+ this.assertInvariants();+ return [{ kind: 'terminal-failure', error: reason }];+ }+departForNavigation(tabId: string,ownerEpoch: OwnerEpoch,diff --git a/apps/web/src/lib/graphql-cache/worker/coordinator-page-adapter.test.ts b/apps/web/src/lib/graphql-cache/worker/coordinator-page-adapter.test.tsindex 1cce85333a..547f155a29 100644--- a/apps/web/src/lib/graphql-cache/worker/coordinator-page-adapter.test.ts+++ b/apps/web/src/lib/graphql-cache/worker/coordinator-page-adapter.test.ts@@ -391,6 +391,34 @@ describe('CacheCoordinatorPageAdapter', () => {await adapter.dispose();});+ it('treats retry exhaustion from the coordinator as terminal', async () => {+ const coordinatorPort = new FakeCoordinatorPort();+ const terminalErrors: string[] = [];+ const adapter = createCacheCoordinatorPageAdapter({+ scope: 'scope',+ tabId: 'tab-a',+ lockManager: heldLockManager(),+ createSharedWorker: () => ({+ port: coordinatorPort as unknown as MessagePort,+ }),+ createDedicatedWorker: () => new FakeWorker(),+ onTerminalError: (error) => terminalErrors.push(error.message),+ });+ const started = adapter.start();+ await vi.waitFor(() => expect(coordinatorPort.messages).toHaveLength(1));+ coordinatorPort.receive({ ...version, kind: 'registered', tabId: 'tab-a' });+ await started;++ coordinatorPort.receive({+ ...version,+ kind: 'terminal-error',+ error: 'cache recovery failed after 5 attempts',+ });++ expect(terminalErrors).toEqual(['cache recovery failed after 5 attempts']);+ expect(coordinatorPort.closed).toBe(true);+ });+it('terminates an owned engine before closing a failed SharedWorker transport', async () => {const order: string[] = [];const coordinatorPort = new FakeCoordinatorPort();diff --git a/apps/web/src/lib/graphql-cache/worker/coordinator-page-adapter.ts b/apps/web/src/lib/graphql-cache/worker/coordinator-page-adapter.tsindex e165f8ada3..0b755a1011 100644--- a/apps/web/src/lib/graphql-cache/worker/coordinator-page-adapter.ts+++ b/apps/web/src/lib/graphql-cache/worker/coordinator-page-adapter.ts@@ -550,6 +550,9 @@ export class CacheCoordinatorPageAdapter {else this.options.onProtocolError?.(error);break;}+ case 'terminal-error':+ this.failTerminal(new Error(message.error));+ break;}}diff --git a/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.test.ts b/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.test.tsindex b26eec73ca..17238e950c 100644--- a/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.test.ts+++ b/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.test.ts@@ -264,6 +264,7 @@ describe('coordinator runtime protocol', () => {},{ ...version, kind: 'engine-replaced', ownerEpoch: 2 },{ ...version, kind: 'protocol-error', error: 'bad envelope' },+ { ...version, kind: 'terminal-error', error: 'recovery exhausted' },])('accepts coordinator-to-tab envelope $kind', (message) => {expect(validateCoordinatorToTabEnvelope(message).ok).toBe(true);});diff --git a/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.ts b/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.tsindex 38339fcf84..3494990bf0 100644--- a/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.ts+++ b/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.ts@@ -128,6 +128,11 @@ export type CoordinatorToTabEnvelope =coordinatorVersion: 2;kind: 'protocol-error';error: string;+ }+ | {+ coordinatorVersion: 2;+ kind: 'terminal-error';+ error: string;};export type PageToEngineEnvelope = {@@ -797,6 +802,7 @@ export function validateCoordinatorToTabEnvelope(}break;case 'protocol-error':+ case 'terminal-error':if (hasOnlyKeys(value, ['coordinatorVersion', 'kind', 'error']) &&isNonEmptyString(value.error)diff --git a/apps/web/src/lib/graphql-cache/worker/coordinator-router.test.ts b/apps/web/src/lib/graphql-cache/worker/coordinator-router.test.tsindex bb62975579..d8d83efb8e 100644--- a/apps/web/src/lib/graphql-cache/worker/coordinator-router.test.ts+++ b/apps/web/src/lib/graphql-cache/worker/coordinator-router.test.ts@@ -207,6 +207,80 @@ describe('CoordinatorRouter', () => {]);});+ it('backs off repeated recovery attempts and terminal-fails at the retry limit', async () => {+ vi.useFakeTimers();+ const router = new CoordinatorRouter({+ verifyTabLockHeld: async () => true,+ watchTabLock: () => () => {},+ });+ const tabA = new FakePort();+ const tabB = new FakePort();+ await register(router, tabA, 'tab-a');+ await register(router, tabB, 'tab-b');+ const initialEngine = new FakePort();+ await attach(router, tabA, 'tab-a', 1, initialEngine);++ initialEngine.receive({+ ...version,+ kind: 'activation-failed',+ tabId: 'tab-a',+ ownerEpoch: 1,+ reason: 'initial OPFS open failed',+ failureCode: 'initialization-failed',+ });+ await vi.advanceTimersByTimeAsync(0);++ const backoffDelays = [100, 200, 400, 800];+ for (let attempt = 1; attempt <= 5; attempt += 1) {+ const state = router.snapshot()?.state;+ expect(state).toMatchObject({+ kind: 'activating',+ ownerEpoch: attempt + 1,+ databaseAction: 'wipe-before-open',+ });+ if (state?.kind !== 'activating') {+ throw new Error('expected an activating recovery owner');+ }+ const tabId = state.tabId;+ const tab = tabId === 'tab-a' ? tabA : tabB;+ const engine = new FakePort();+ await attach(router, tab, tabId, state.ownerEpoch, engine);+ engine.receive({+ ...version,+ kind: 'activation-failed',+ tabId,+ ownerEpoch: state.ownerEpoch,+ reason: 'OPFS path remove failed (NoModificationAllowedError)',+ failureCode: 'recovery-open-failed',+ });++ if (attempt === 5) break;+ expect(router.snapshot()?.state).toMatchObject({+ kind: 'resetting-after-loss',+ nextEpoch: attempt + 2,+ });+ const delay = backoffDelays[attempt - 1] ?? 0;+ await vi.advanceTimersByTimeAsync(delay - 1);+ expect(router.snapshot()?.state.kind).toBe('resetting-after-loss');+ await vi.advanceTimersByTimeAsync(1);+ }++ expect(router.snapshot()?.state).toEqual({+ kind: 'failed',+ reason: expect.stringContaining('cache recovery failed after 5 attempts'),+ });+ expect(messagesOfKind(tabA, 'terminal-error')).toEqual([+ expect.objectContaining({+ error: expect.stringContaining(+ 'OPFS path remove failed (NoModificationAllowedError)'+ ),+ }),+ ]);+ expect(messagesOfKind(tabB, 'terminal-error')).toHaveLength(1);+ await vi.advanceTimersByTimeAsync(10_000);+ expect(router.snapshot()?.state.kind).toBe('failed');+ });+it('registers only after independent liveness-lock contention succeeds', async () => {let releaseVerification: ((held: boolean) => void) | undefined;const verification = new Promise<boolean>((resolve) => {diff --git a/apps/web/src/lib/graphql-cache/worker/coordinator-router.ts b/apps/web/src/lib/graphql-cache/worker/coordinator-router.tsindex 5af2a9862a..98d1b905d8 100644--- a/apps/web/src/lib/graphql-cache/worker/coordinator-router.ts+++ b/apps/web/src/lib/graphql-cache/worker/coordinator-router.ts@@ -1,4 +1,6 @@+import * as Duration from 'effect/Duration';import * as Effect from 'effect/Effect';+import * as Schedule from 'effect/Schedule';import { match } from 'ts-pattern';import type { CacheResponse } from '../protocol';import {@@ -75,6 +77,22 @@ type EngineRoute = {const DEFAULT_ACTIVATION_TIMEOUT_MS = 20_000;const DEFAULT_HEARTBEAT_INTERVAL_MS = 2_000;const DEFAULT_HEARTBEAT_TIMEOUT_MS = 5_000;+const RECOVERY_RETRY_LIMIT = 5;+const RECOVERY_RETRY_BASE_DELAY_MS = 100;++const recoveryRetryDelayMs = (failureCount: number): number =>+ Effect.runSync(+ Effect.gen(function* () {+ const step = yield* Schedule.toStep(+ Schedule.exponential(Duration.millis(RECOVERY_RETRY_BASE_DELAY_MS))+ );+ let delay = Duration.zero;+ for (let index = 0; index < failureCount; index += 1) {+ [, delay] = yield* step(index, undefined);+ }+ return Duration.toMillis(delay);+ })+ );const errorMessage = (error: unknown): string =>error instanceof Error ? error.message : String(error);@@ -189,6 +207,8 @@ export class CoordinatorRouter {>();private nextRecoveryResetReason: CacheResetReason | undefined;private readonly resetRequiredEpochs = new Set<number>();+ private recoveryAttemptCount = 0;+ private recoveryRetryTimer: ReturnType<typeof setTimeout> | undefined;private readonly now = (): number =>globalThis.performance?.now() ?? Date.now();@@ -568,6 +588,7 @@ export class CoordinatorRouter {startedAt === undefined ? undefined : this.now() - startedAt,});this.clearActivationTimer();+ this.resetRecoveryRetries();this.scheduleHeartbeat(message.ownerEpoch);}break;@@ -791,11 +812,7 @@ export class CoordinatorRouter {this.removeTabConnection(action.tabId);break;case 'schedule-reset-activation':- this.queueMicrotaskFn(() => {- if (this.coreValue) {- this.applyActions(this.coreValue.resumeAfterLoss());- }- });+ this.scheduleResetActivation();break;case 'broadcast-engine-replaced':this.telemetry.record({@@ -839,10 +856,54 @@ export class CoordinatorRouter {}));break;+ case 'terminal-failure':+ this.broadcast(+ envelope<CoordinatorToTabEnvelope>({+ kind: 'terminal-error',+ error: action.error,+ })+ );+ break;}}}+ private scheduleResetActivation(): void {+ const core = this.coreValue;+ if (!core || core.state.kind !== 'resetting-after-loss') return;+ if (this.recoveryAttemptCount >= RECOVERY_RETRY_LIMIT) {+ const reason = `cache recovery failed after ${RECOVERY_RETRY_LIMIT} attempts: ${core.state.reason}`;+ this.clearRecoveryRetryTimer();+ this.applyActions(core.terminalFailure(reason));+ return;+ }++ this.recoveryAttemptCount += 1;+ const activate = () => {+ this.recoveryRetryTimer = undefined;+ if (this.coreValue) {+ this.applyActions(this.coreValue.resumeAfterLoss());+ }+ };+ if (this.recoveryAttemptCount === 1) {+ this.queueMicrotaskFn(activate);+ return;+ }+ const delayMs = recoveryRetryDelayMs(this.recoveryAttemptCount - 1);+ this.recoveryRetryTimer = this.setTimeoutFn(activate, delayMs);+ }++ private resetRecoveryRetries(): void {+ this.recoveryAttemptCount = 0;+ this.clearRecoveryRetryTimer();+ }++ private clearRecoveryRetryTimer(): void {+ if (this.recoveryRetryTimer === undefined) return;+ this.clearTimeoutFn(this.recoveryRetryTimer);+ this.recoveryRetryTimer = undefined;+ }+private failOwner(tabId: string,ownerEpoch: number,@@ -906,6 +967,7 @@ export class CoordinatorRouter {const isCurrentOwner =state.kind !== 'waiting-for-tab' &&state.kind !== 'resetting-after-loss' &&+ state.kind !== 'failed' &&state.tabId === tabId &&state.ownerEpoch === ownerEpoch;const actions = core.departForNavigation(tabId, ownerEpoch, reason);@@ -951,6 +1013,7 @@ export class CoordinatorRouter {if (state.kind !== 'waiting-for-tab' &&state.kind !== 'resetting-after-loss' &&+ state.kind !== 'failed' &&state.tabId === tabId) {this.activationStarted.delete(state.ownerEpoch);@@ -1063,6 +1126,7 @@ export class CoordinatorRouter {state &&state.kind !== 'waiting-for-tab' &&state.kind !== 'resetting-after-loss' &&+ state.kind !== 'failed' &&state.tabId === tabId &&state.ownerEpoch === ownerEpoch);diff --git a/apps/web/src/lib/service-clients/service-storage/graphql-soup.test.ts b/apps/web/src/lib/service-clients/service-storage/graphql-soup.test.tsindex 63174e0161..3f1ec17545 100644--- a/apps/web/src/lib/service-clients/service-storage/graphql-soup.test.ts+++ b/apps/web/src/lib/service-clients/service-storage/graphql-soup.test.ts@@ -42,6 +42,7 @@ const mocks = vi.hoisted(() => {const realtimeClient = { kind: 'realtime' };const replaceSubscriptions = vi.fn();const platformFetch = vi.fn();+ const toastFailure = vi.fn();return {get enabled() {return enabled;@@ -81,6 +82,7 @@ const mocks = vi.hoisted(() => {realtimeClient,replaceSubscriptions,platformFetch,+ toastFailure,createWorkerCacheHost: vi.fn((options: { onInitializationError?: (error: Error) => void }) => {initializationErrorHandler = options.onInitializationError;@@ -91,6 +93,9 @@ const mocks = vi.hoisted(() => {};});+vi.mock('@core/component/Toast/Toast', () => ({+ toast: { failure: mocks.toastFailure },+}));vi.mock('@core/constant/featureFlags', () => ({ENABLE_BEARER_TOKEN_AUTH: false,enableGraphqlSoup: { key: 'enable-graphql-soup' },@@ -335,6 +340,9 @@ describe('GraphQL Soup browser cache session gate', () => {expect(soup.getGraphqlSoupClient()).toBe(mocks.realtimeClient);expect(soup.graphqlCacheEnabled()).toBe(false);expect(mocks.cleanupOrder()).toEqual(['subscriptions', 'host']);+ expect(mocks.toastFailure).toHaveBeenCalledWith('Local cache unavailable', {+ subtext: 'Macro will continue without local caching for this session.',+ });expect(warn).toHaveBeenCalledWith('graphql cache async init failed; using uncached client',expect.objectContaining({ message: 'injected initialization failure' })diff --git a/apps/web/src/lib/service-clients/service-storage/graphql-soup.ts b/apps/web/src/lib/service-clients/service-storage/graphql-soup.tsindex cec390c377..06832c6ceb 100644--- a/apps/web/src/lib/service-clients/service-storage/graphql-soup.ts+++ b/apps/web/src/lib/service-clients/service-storage/graphql-soup.ts@@ -1,3 +1,4 @@+import { toast } from '@core/component/Toast/Toast';import {ENABLE_BEARER_TOKEN_AUTH,enableGraphqlSoup,@@ -408,6 +409,9 @@ export function getGraphqlSoupClient(): Client {const onInitializationError = (error: Error) => {if (!host || cachedCacheHost !== host) return;fallbackAfterInitializationFailure();+ toast.failure('Local cache unavailable', {+ subtext: 'Macro will continue without local caching for this session.',+ });console.warn('graphql cache async init failed; using uncached client',error