Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(web): bound GraphQL cache recovery retries (#6173)

Sean Aye5b540d7b7e40
diff --git a/apps/web/src/lib/graphql-cache/worker/coordinator-core.test.ts b/apps/web/src/lib/graphql-cache/worker/coordinator-core.test.ts
index c624b82af1..3887c24ec5 100644
--- a/apps/web/src/lib/graphql-cache/worker/coordinator-core.test.ts
+++ b/apps/web/src/lib/graphql-cache/worker/coordinator-core.test.ts
@@ -424,4 +424,34 @@ describe('CoordinatorCore', () => {
},
]);
});
+
+ it('makes retry exhaustion terminal for current and future tabs', () => {
+ const core = new CoordinatorCore('scope');
+ core.registerTab('tab-a');
+ core.registerTab('tab-b');
+ ready(core, 'tab-a', 1, 'opened-existing');
+ core.ownerLost('tab-a', 1, 'OPFS recovery failed');
+ core.request('tab-b', { id: 7, kind: 'clear' });
+
+ expect(core.terminalFailure('cache recovery exhausted')).toEqual([
+ { kind: 'terminal-failure', error: 'cache recovery exhausted' },
+ ]);
+ expect(core.state).toEqual({
+ kind: 'failed',
+ reason: 'cache recovery exhausted',
+ });
+ expect(core.snapshot().queuedRequestCount).toBe(0);
+ expect(core.request('tab-b', { id: 8, kind: 'clear' })).toEqual([
+ {
+ kind: 'reject-request',
+ tabId: 'tab-b',
+ requestId: 8,
+ error: 'cache recovery exhausted',
+ },
+ ]);
+ expect(core.registerTab('tab-c')).toEqual([
+ { kind: 'terminal-failure', error: 'cache recovery exhausted' },
+ ]);
+ expect(core.resumeAfterLoss()).toEqual([]);
+ });
});
diff --git a/apps/web/src/lib/graphql-cache/worker/coordinator-core.ts b/apps/web/src/lib/graphql-cache/worker/coordinator-core.ts
index 7e118eb584..5b807389f7 100644
--- a/apps/web/src/lib/graphql-cache/worker/coordinator-core.ts
+++ b/apps/web/src/lib/graphql-cache/worker/coordinator-core.ts
@@ -29,7 +29,8 @@ export type CoordinatorState =
previousEpoch: OwnerEpoch;
nextEpoch: OwnerEpoch;
reason: string;
- };
+ }
+ | { kind: 'failed'; reason: string };
export type CoordinatorSnapshot = {
scope: string;
@@ -89,7 +90,8 @@ export type CoordinatorAction =
routeId?: RouteId;
reason: 'stale-epoch' | 'unknown-route' | 'inactive-engine';
}
- | { kind: 'protocol-violation'; error: string };
+ | { kind: 'protocol-violation'; error: string }
+ | { kind: 'terminal-failure'; error: string };
export type EngineReady = {
tabId: string;
@@ -144,6 +146,9 @@ export class CoordinatorCore {
registerTab(tabId: string): CoordinatorAction[] {
if (this.tabs.includes(tabId)) return [];
this.tabs.push(tabId);
+ if (this.stateValue.kind === 'failed') {
+ return [{ kind: 'terminal-failure', error: this.stateValue.reason }];
+ }
if (this.stateValue.kind !== 'waiting-for-tab') return [];
return this.activateNext(this.stateValue.nextDatabaseAction);
}
@@ -157,6 +162,9 @@ export class CoordinatorCore {
if (this.retiringTabs.has(tabId)) {
return [this.reject(tabId, request.id, 'requester tab is retiring')];
}
+ if (this.stateValue.kind === 'failed') {
+ return [this.reject(tabId, request.id, this.stateValue.reason)];
+ }
const queued = { tabId, request };
if (this.stateValue.kind !== 'active') {
@@ -303,6 +311,16 @@ export class CoordinatorCore {
return this.transitionToAbruptLoss(tabId, ownerEpoch, reason);
}
+ /** Stops owner recovery and fails every current or future page connection. */
+ terminalFailure(reason: string): CoordinatorAction[] {
+ if (this.stateValue.kind === 'failed') return [];
+ this.queuedRequests.length = 0;
+ this.inFlight.clear();
+ this.stateValue = { kind: 'failed', reason };
+ this.assertInvariants();
+ return [{ kind: 'terminal-failure', error: reason }];
+ }
+
departForNavigation(
tabId: string,
ownerEpoch: OwnerEpoch,
diff --git a/apps/web/src/lib/graphql-cache/worker/coordinator-page-adapter.test.ts b/apps/web/src/lib/graphql-cache/worker/coordinator-page-adapter.test.ts
index 1cce85333a..547f155a29 100644
--- a/apps/web/src/lib/graphql-cache/worker/coordinator-page-adapter.test.ts
+++ b/apps/web/src/lib/graphql-cache/worker/coordinator-page-adapter.test.ts
@@ -391,6 +391,34 @@ describe('CacheCoordinatorPageAdapter', () => {
await adapter.dispose();
});
+ it('treats retry exhaustion from the coordinator as terminal', async () => {
+ const coordinatorPort = new FakeCoordinatorPort();
+ const terminalErrors: string[] = [];
+ const adapter = createCacheCoordinatorPageAdapter({
+ scope: 'scope',
+ tabId: 'tab-a',
+ lockManager: heldLockManager(),
+ createSharedWorker: () => ({
+ port: coordinatorPort as unknown as MessagePort,
+ }),
+ createDedicatedWorker: () => new FakeWorker(),
+ onTerminalError: (error) => terminalErrors.push(error.message),
+ });
+ const started = adapter.start();
+ await vi.waitFor(() => expect(coordinatorPort.messages).toHaveLength(1));
+ coordinatorPort.receive({ ...version, kind: 'registered', tabId: 'tab-a' });
+ await started;
+
+ coordinatorPort.receive({
+ ...version,
+ kind: 'terminal-error',
+ error: 'cache recovery failed after 5 attempts',
+ });
+
+ expect(terminalErrors).toEqual(['cache recovery failed after 5 attempts']);
+ expect(coordinatorPort.closed).toBe(true);
+ });
+
it('terminates an owned engine before closing a failed SharedWorker transport', async () => {
const order: string[] = [];
const coordinatorPort = new FakeCoordinatorPort();
diff --git a/apps/web/src/lib/graphql-cache/worker/coordinator-page-adapter.ts b/apps/web/src/lib/graphql-cache/worker/coordinator-page-adapter.ts
index e165f8ada3..0b755a1011 100644
--- a/apps/web/src/lib/graphql-cache/worker/coordinator-page-adapter.ts
+++ b/apps/web/src/lib/graphql-cache/worker/coordinator-page-adapter.ts
@@ -550,6 +550,9 @@ export class CacheCoordinatorPageAdapter {
else this.options.onProtocolError?.(error);
break;
}
+ case 'terminal-error':
+ this.failTerminal(new Error(message.error));
+ break;
}
}
diff --git a/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.test.ts b/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.test.ts
index b26eec73ca..17238e950c 100644
--- a/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.test.ts
+++ b/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.test.ts
@@ -264,6 +264,7 @@ describe('coordinator runtime protocol', () => {
},
{ ...version, kind: 'engine-replaced', ownerEpoch: 2 },
{ ...version, kind: 'protocol-error', error: 'bad envelope' },
+ { ...version, kind: 'terminal-error', error: 'recovery exhausted' },
])('accepts coordinator-to-tab envelope $kind', (message) => {
expect(validateCoordinatorToTabEnvelope(message).ok).toBe(true);
});
diff --git a/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.ts b/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.ts
index 38339fcf84..3494990bf0 100644
--- a/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.ts
+++ b/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.ts
@@ -128,6 +128,11 @@ export type CoordinatorToTabEnvelope =
coordinatorVersion: 2;
kind: 'protocol-error';
error: string;
+ }
+ | {
+ coordinatorVersion: 2;
+ kind: 'terminal-error';
+ error: string;
};
export type PageToEngineEnvelope = {
@@ -797,6 +802,7 @@ export function validateCoordinatorToTabEnvelope(
}
break;
case 'protocol-error':
+ case 'terminal-error':
if (
hasOnlyKeys(value, ['coordinatorVersion', 'kind', 'error']) &&
isNonEmptyString(value.error)
diff --git a/apps/web/src/lib/graphql-cache/worker/coordinator-router.test.ts b/apps/web/src/lib/graphql-cache/worker/coordinator-router.test.ts
index bb62975579..d8d83efb8e 100644
--- a/apps/web/src/lib/graphql-cache/worker/coordinator-router.test.ts
+++ b/apps/web/src/lib/graphql-cache/worker/coordinator-router.test.ts
@@ -207,6 +207,80 @@ describe('CoordinatorRouter', () => {
]);
});
+ it('backs off repeated recovery attempts and terminal-fails at the retry limit', async () => {
+ vi.useFakeTimers();
+ const router = new CoordinatorRouter({
+ verifyTabLockHeld: async () => true,
+ watchTabLock: () => () => {},
+ });
+ const tabA = new FakePort();
+ const tabB = new FakePort();
+ await register(router, tabA, 'tab-a');
+ await register(router, tabB, 'tab-b');
+ const initialEngine = new FakePort();
+ await attach(router, tabA, 'tab-a', 1, initialEngine);
+
+ initialEngine.receive({
+ ...version,
+ kind: 'activation-failed',
+ tabId: 'tab-a',
+ ownerEpoch: 1,
+ reason: 'initial OPFS open failed',
+ failureCode: 'initialization-failed',
+ });
+ await vi.advanceTimersByTimeAsync(0);
+
+ const backoffDelays = [100, 200, 400, 800];
+ for (let attempt = 1; attempt <= 5; attempt += 1) {
+ const state = router.snapshot()?.state;
+ expect(state).toMatchObject({
+ kind: 'activating',
+ ownerEpoch: attempt + 1,
+ databaseAction: 'wipe-before-open',
+ });
+ if (state?.kind !== 'activating') {
+ throw new Error('expected an activating recovery owner');
+ }
+ const tabId = state.tabId;
+ const tab = tabId === 'tab-a' ? tabA : tabB;
+ const engine = new FakePort();
+ await attach(router, tab, tabId, state.ownerEpoch, engine);
+ engine.receive({
+ ...version,
+ kind: 'activation-failed',
+ tabId,
+ ownerEpoch: state.ownerEpoch,
+ reason: 'OPFS path remove failed (NoModificationAllowedError)',
+ failureCode: 'recovery-open-failed',
+ });
+
+ if (attempt === 5) break;
+ expect(router.snapshot()?.state).toMatchObject({
+ kind: 'resetting-after-loss',
+ nextEpoch: attempt + 2,
+ });
+ const delay = backoffDelays[attempt - 1] ?? 0;
+ await vi.advanceTimersByTimeAsync(delay - 1);
+ expect(router.snapshot()?.state.kind).toBe('resetting-after-loss');
+ await vi.advanceTimersByTimeAsync(1);
+ }
+
+ expect(router.snapshot()?.state).toEqual({
+ kind: 'failed',
+ reason: expect.stringContaining('cache recovery failed after 5 attempts'),
+ });
+ expect(messagesOfKind(tabA, 'terminal-error')).toEqual([
+ expect.objectContaining({
+ error: expect.stringContaining(
+ 'OPFS path remove failed (NoModificationAllowedError)'
+ ),
+ }),
+ ]);
+ expect(messagesOfKind(tabB, 'terminal-error')).toHaveLength(1);
+ await vi.advanceTimersByTimeAsync(10_000);
+ expect(router.snapshot()?.state.kind).toBe('failed');
+ });
+
it('registers only after independent liveness-lock contention succeeds', async () => {
let releaseVerification: ((held: boolean) => void) | undefined;
const verification = new Promise<boolean>((resolve) => {
diff --git a/apps/web/src/lib/graphql-cache/worker/coordinator-router.ts b/apps/web/src/lib/graphql-cache/worker/coordinator-router.ts
index 5af2a9862a..98d1b905d8 100644
--- a/apps/web/src/lib/graphql-cache/worker/coordinator-router.ts
+++ b/apps/web/src/lib/graphql-cache/worker/coordinator-router.ts
@@ -1,4 +1,6 @@
+import * as Duration from 'effect/Duration';
import * as Effect from 'effect/Effect';
+import * as Schedule from 'effect/Schedule';
import { match } from 'ts-pattern';
import type { CacheResponse } from '../protocol';
import {
@@ -75,6 +77,22 @@ type EngineRoute = {
const DEFAULT_ACTIVATION_TIMEOUT_MS = 20_000;
const DEFAULT_HEARTBEAT_INTERVAL_MS = 2_000;
const DEFAULT_HEARTBEAT_TIMEOUT_MS = 5_000;
+const RECOVERY_RETRY_LIMIT = 5;
+const RECOVERY_RETRY_BASE_DELAY_MS = 100;
+
+const recoveryRetryDelayMs = (failureCount: number): number =>
+ Effect.runSync(
+ Effect.gen(function* () {
+ const step = yield* Schedule.toStep(
+ Schedule.exponential(Duration.millis(RECOVERY_RETRY_BASE_DELAY_MS))
+ );
+ let delay = Duration.zero;
+ for (let index = 0; index < failureCount; index += 1) {
+ [, delay] = yield* step(index, undefined);
+ }
+ return Duration.toMillis(delay);
+ })
+ );
const errorMessage = (error: unknown): string =>
error instanceof Error ? error.message : String(error);
@@ -189,6 +207,8 @@ export class CoordinatorRouter {
>();
private nextRecoveryResetReason: CacheResetReason | undefined;
private readonly resetRequiredEpochs = new Set<number>();
+ private recoveryAttemptCount = 0;
+ private recoveryRetryTimer: ReturnType<typeof setTimeout> | undefined;
private readonly now = (): number =>
globalThis.performance?.now() ?? Date.now();
@@ -568,6 +588,7 @@ export class CoordinatorRouter {
startedAt === undefined ? undefined : this.now() - startedAt,
});
this.clearActivationTimer();
+ this.resetRecoveryRetries();
this.scheduleHeartbeat(message.ownerEpoch);
}
break;
@@ -791,11 +812,7 @@ export class CoordinatorRouter {
this.removeTabConnection(action.tabId);
break;
case 'schedule-reset-activation':
- this.queueMicrotaskFn(() => {
- if (this.coreValue) {
- this.applyActions(this.coreValue.resumeAfterLoss());
- }
- });
+ this.scheduleResetActivation();
break;
case 'broadcast-engine-replaced':
this.telemetry.record({
@@ -839,10 +856,54 @@ export class CoordinatorRouter {
})
);
break;
+ case 'terminal-failure':
+ this.broadcast(
+ envelope<CoordinatorToTabEnvelope>({
+ kind: 'terminal-error',
+ error: action.error,
+ })
+ );
+ break;
}
}
}
+ private scheduleResetActivation(): void {
+ const core = this.coreValue;
+ if (!core || core.state.kind !== 'resetting-after-loss') return;
+ if (this.recoveryAttemptCount >= RECOVERY_RETRY_LIMIT) {
+ const reason = `cache recovery failed after ${RECOVERY_RETRY_LIMIT} attempts: ${core.state.reason}`;
+ this.clearRecoveryRetryTimer();
+ this.applyActions(core.terminalFailure(reason));
+ return;
+ }
+
+ this.recoveryAttemptCount += 1;
+ const activate = () => {
+ this.recoveryRetryTimer = undefined;
+ if (this.coreValue) {
+ this.applyActions(this.coreValue.resumeAfterLoss());
+ }
+ };
+ if (this.recoveryAttemptCount === 1) {
+ this.queueMicrotaskFn(activate);
+ return;
+ }
+ const delayMs = recoveryRetryDelayMs(this.recoveryAttemptCount - 1);
+ this.recoveryRetryTimer = this.setTimeoutFn(activate, delayMs);
+ }
+
+ private resetRecoveryRetries(): void {
+ this.recoveryAttemptCount = 0;
+ this.clearRecoveryRetryTimer();
+ }
+
+ private clearRecoveryRetryTimer(): void {
+ if (this.recoveryRetryTimer === undefined) return;
+ this.clearTimeoutFn(this.recoveryRetryTimer);
+ this.recoveryRetryTimer = undefined;
+ }
+
private failOwner(
tabId: string,
ownerEpoch: number,
@@ -906,6 +967,7 @@ export class CoordinatorRouter {
const isCurrentOwner =
state.kind !== 'waiting-for-tab' &&
state.kind !== 'resetting-after-loss' &&
+ state.kind !== 'failed' &&
state.tabId === tabId &&
state.ownerEpoch === ownerEpoch;
const actions = core.departForNavigation(tabId, ownerEpoch, reason);
@@ -951,6 +1013,7 @@ export class CoordinatorRouter {
if (
state.kind !== 'waiting-for-tab' &&
state.kind !== 'resetting-after-loss' &&
+ state.kind !== 'failed' &&
state.tabId === tabId
) {
this.activationStarted.delete(state.ownerEpoch);
@@ -1063,6 +1126,7 @@ export class CoordinatorRouter {
state &&
state.kind !== 'waiting-for-tab' &&
state.kind !== 'resetting-after-loss' &&
+ state.kind !== 'failed' &&
state.tabId === tabId &&
state.ownerEpoch === ownerEpoch
);
diff --git a/apps/web/src/lib/service-clients/service-storage/graphql-soup.test.ts b/apps/web/src/lib/service-clients/service-storage/graphql-soup.test.ts
index 63174e0161..3f1ec17545 100644
--- a/apps/web/src/lib/service-clients/service-storage/graphql-soup.test.ts
+++ b/apps/web/src/lib/service-clients/service-storage/graphql-soup.test.ts
@@ -42,6 +42,7 @@ const mocks = vi.hoisted(() => {
const realtimeClient = { kind: 'realtime' };
const replaceSubscriptions = vi.fn();
const platformFetch = vi.fn();
+ const toastFailure = vi.fn();
return {
get enabled() {
return enabled;
@@ -81,6 +82,7 @@ const mocks = vi.hoisted(() => {
realtimeClient,
replaceSubscriptions,
platformFetch,
+ toastFailure,
createWorkerCacheHost: vi.fn(
(options: { onInitializationError?: (error: Error) => void }) => {
initializationErrorHandler = options.onInitializationError;
@@ -91,6 +93,9 @@ const mocks = vi.hoisted(() => {
};
});
+vi.mock('@core/component/Toast/Toast', () => ({
+ toast: { failure: mocks.toastFailure },
+}));
vi.mock('@core/constant/featureFlags', () => ({
ENABLE_BEARER_TOKEN_AUTH: false,
enableGraphqlSoup: { key: 'enable-graphql-soup' },
@@ -335,6 +340,9 @@ describe('GraphQL Soup browser cache session gate', () => {
expect(soup.getGraphqlSoupClient()).toBe(mocks.realtimeClient);
expect(soup.graphqlCacheEnabled()).toBe(false);
expect(mocks.cleanupOrder()).toEqual(['subscriptions', 'host']);
+ expect(mocks.toastFailure).toHaveBeenCalledWith('Local cache unavailable', {
+ subtext: 'Macro will continue without local caching for this session.',
+ });
expect(warn).toHaveBeenCalledWith(
'graphql cache async init failed; using uncached client',
expect.objectContaining({ message: 'injected initialization failure' })
diff --git a/apps/web/src/lib/service-clients/service-storage/graphql-soup.ts b/apps/web/src/lib/service-clients/service-storage/graphql-soup.ts
index cec390c377..06832c6ceb 100644
--- a/apps/web/src/lib/service-clients/service-storage/graphql-soup.ts
+++ b/apps/web/src/lib/service-clients/service-storage/graphql-soup.ts
@@ -1,3 +1,4 @@
+import { toast } from '@core/component/Toast/Toast';
import {
ENABLE_BEARER_TOKEN_AUTH,
enableGraphqlSoup,
@@ -408,6 +409,9 @@ export function getGraphqlSoupClient(): Client {
const onInitializationError = (error: Error) => {
if (!host || cachedCacheHost !== host) return;
fallbackAfterInitializationFailure();
+ toast.failure('Local cache unavailable', {
+ subtext: 'Macro will continue without local caching for this session.',
+ });
console.warn(
'graphql cache async init failed; using uncached client',
error