Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(cache): confirm owner lock loss before heartbeat recovery (#6694)

Sean Aye763395880dcf
diff --git a/apps/web/docs/graphql-normalized-cache-plan.md b/apps/web/docs/graphql-normalized-cache-plan.md
index 97fe764407..f598dda6d2 100644
--- a/apps/web/docs/graphql-normalized-cache-plan.md
+++ b/apps/web/docs/graphql-normalized-cache-plan.md
@@ -189,8 +189,14 @@ The cache's validated envelopes, request ids, Web Locks, owner epochs,
heartbeats, and replay policy remain application-owned. A normal `pagehide`
uses a fenced navigation-departure message: the page terminates its dedicated
worker, but the next owner opens the existing OPFS database. Only an
-unannounced liveness, heartbeat, transport, or engine failure **after the database
-open grant** selects `wipe-before-open`.
+unannounced liveness, transport, or engine failure **after the database
+open grant** selects `wipe-before-open`. A missed heartbeat alone is not owner
+loss: background tabs and their dedicated workers may be suspended while the
+SharedWorker still runs. The coordinator probes the physical owner Web Lock
+without waiting or stealing it; a held lock (or a failed probe) retries the
+heartbeat without changing epochs or touching storage. A released lock confirms
+silent worker termination and triggers the usual fenced recovery. Read deadlines
+still bound callers waiting on an unresponsive engine.
Protocol v3 separates startup into asset loading and database opening. The engine
loads/compiles WASM without touching OPFS, reports `engine-assets-ready`, and waits
diff --git a/apps/web/src/lib/graphql-cache/worker/browser-test/coordinator.browser.e2e.ts b/apps/web/src/lib/graphql-cache/worker/browser-test/coordinator.browser.e2e.ts
index b1aa8898e8..9eafdf52a6 100644
--- a/apps/web/src/lib/graphql-cache/worker/browser-test/coordinator.browser.e2e.ts
+++ b/apps/web/src/lib/graphql-cache/worker/browser-test/coordinator.browser.e2e.ts
@@ -160,6 +160,115 @@ test('direct cutover lazily deletes only the former normalized-cache IDB', async
expect(browserErrors).toEqual([]);
});
+test('a suspended owner resumes without replacement or cache loss', async ({
+ context,
+ page,
+ browserName,
+}, testInfo) => {
+ test.skip(
+ browserName !== 'chromium',
+ 'CDP lifecycle control is Chromium-only'
+ );
+ const scope = `cache-suspension-${crypto.randomUUID()}`;
+ const path = `${harnessPath(testInfo.project.name, 'cache-lifecycle.html')}?treatment=true&scope=${scope}`;
+ await page.goto(path);
+ await page.evaluate(async () => {
+ await window.cacheLifecycleHarness.startSingle();
+ await window.cacheLifecycleHarness.write('preserve-through-suspension');
+ });
+ const before = await page.evaluate(() => window.cacheLifecycleHarness.read());
+ expect(before).toMatchObject({ kind: 'hit' });
+ const follower = await context.newPage();
+ await follower.goto(path);
+ await follower.evaluate(() => window.cacheLifecycleHarness.startSingle());
+
+ const cdp = await context.newCDPSession(page);
+ const { targetInfos } = await cdp.send('Target.getTargets');
+ const engine = targetInfos.find(
+ (target) =>
+ target.type === 'worker' && target.url.includes('cache.engine-worker')
+ );
+ if (!engine) throw new Error('missing cache engine target');
+ const { sessionId } = await cdp.send('Target.attachToTarget', {
+ targetId: engine.targetId,
+ flatten: false,
+ });
+ const paused = new Promise<void>((resolve) => {
+ cdp.on('Target.receivedMessageFromTarget', (event) => {
+ if (
+ event.sessionId === sessionId &&
+ JSON.parse(event.message).method === 'Debugger.paused'
+ )
+ resolve();
+ });
+ });
+ const debug = async (id: number, method: string) => {
+ await cdp.send('Target.sendMessageToTarget', {
+ sessionId,
+ message: JSON.stringify({ id, method }),
+ });
+ };
+ await debug(1, 'Debugger.enable');
+ await debug(2, 'Debugger.pause');
+ await paused;
+ try {
+ await cdp.send('Page.setWebLifecycleState', { state: 'frozen' });
+ // CDP page freezing alone need not pause a lock-holding worker in Chrome.
+ // Explicitly pause its event loop, keeping its lock held, while the other
+ // page keeps the coordinator running beyond multiple heartbeat deadlines.
+ await new Promise((resolve) => setTimeout(resolve, 16_000));
+ const { targetInfos: suspendedTargets } =
+ await cdp.send('Target.getTargets');
+ expect(
+ suspendedTargets.some((target) => target.targetId === engine.targetId)
+ ).toBe(true);
+ } finally {
+ const { targetInfos: remainingTargets } =
+ await cdp.send('Target.getTargets');
+ if (
+ remainingTargets.some((target) => target.targetId === engine.targetId)
+ ) {
+ await debug(3, 'Debugger.resume');
+ }
+ await cdp.send('Page.setWebLifecycleState', { state: 'active' });
+ await cdp.detach();
+ }
+
+ expect(
+ await page.evaluate(() => window.cacheLifecycleHarness.read())
+ ).toEqual(before);
+ expect(
+ await page.evaluate(() => window.cacheLifecycleHarness.engineWorkerCount())
+ ).toBe(1);
+ expect(
+ await follower.evaluate(() =>
+ window.cacheLifecycleHarness.engineWorkerCount()
+ )
+ ).toBe(0);
+ expect(
+ await follower.evaluate(() => window.cacheLifecycleHarness.read())
+ ).toEqual(before);
+});
+
+test('a silently terminated engine still recovers after releasing its owner lock', async ({
+ page,
+}, testInfo) => {
+ await page.goto(
+ `${harnessPath(testInfo.project.name, 'cache-lifecycle.html')}?treatment=true`
+ );
+ await page.evaluate(() => window.cacheLifecycleHarness.startSingle());
+ const result = await page.evaluate(() =>
+ window.cacheLifecycleHarness.abruptOwnerLoss()
+ );
+ expect(result).toMatchObject({
+ oldRequestRejected: true,
+ replacement: { kind: 'miss' },
+ });
+ expect(
+ await page.evaluate(() => window.cacheLifecycleHarness.engineWorkerCount())
+ ).toBe(2);
+});
+
test('production cache-wasm Turso engine preserves graceful data and atomically recovers abrupt loss', async ({
context,
page,
diff --git a/apps/web/src/lib/graphql-cache/worker/coordinator-router.test.ts b/apps/web/src/lib/graphql-cache/worker/coordinator-router.test.ts
index 3beb7d4155..d9839d0a4b 100644
--- a/apps/web/src/lib/graphql-cache/worker/coordinator-router.test.ts
+++ b/apps/web/src/lib/graphql-cache/worker/coordinator-router.test.ts
@@ -799,13 +799,160 @@ describe('CoordinatorRouter', () => {
]);
});
- it('uses activation and heartbeat watchdogs to terminate and wipe', async () => {
+ it('preserves the owner and in-flight work through repeated missed heartbeats', async () => {
+ vi.useFakeTimers();
+ const verifyLockHeld = vi.fn(async () => true);
+ const router = new CoordinatorRouter({
+ heartbeatIntervalMs: 5,
+ heartbeatTimeoutMs: 7,
+ verifyTabLockHeld: async () => true,
+ verifyOwnerLockHeld: verifyLockHeld,
+ watchTabLock: () => () => {},
+ });
+ const tab = new FakePort();
+ await register(router, tab, 'tab-a');
+ const engine = new FakePort();
+ await attach(router, tab, 'tab-a', 1, engine);
+ ready(engine, 'tab-a', 1, 'opened-existing');
+ await router.handleTabMessage(tab, {
+ ...version,
+ kind: 'cache-request',
+ tabId: 'tab-a',
+ request: { id: 42, kind: 'clear' },
+ });
+ const request = messagesOfKind(engine, 'engine-request')[0]!;
+
+ // Background suspension can outlast every recovery attempt. A held
+ // physical lock is still authoritative; silence is not owner loss.
+ await vi.advanceTimersByTimeAsync(120_000);
+ expect(router.snapshot()?.state).toMatchObject({
+ kind: 'active',
+ ownerEpoch: 1,
+ });
+ expect(messagesOfKind(tab, 'terminate-engine')).toHaveLength(0);
+ expect(messagesOfKind(tab, 'terminal-error')).toHaveLength(0);
+ expect(messagesOfKind(tab, 'cache-message')).toHaveLength(0);
+ expect(verifyLockHeld).toHaveBeenCalledWith(databaseOwnerLockName('scope'));
+ expect(messagesOfKind(engine, 'heartbeat').length).toBeGreaterThan(5);
+
+ const heartbeat = messagesOfKind(engine, 'heartbeat').at(-1)!;
+ engine.receive({
+ ...version,
+ kind: 'heartbeat-ack',
+ ownerEpoch: 1,
+ heartbeatId: heartbeat.heartbeatId,
+ });
+ engine.receive({
+ ...version,
+ kind: 'engine-response',
+ ownerEpoch: 1,
+ routeId: request.routeId,
+ response: {
+ id: request.routeId,
+ ok: true,
+ result: INITIAL_CACHE_REVISION,
+ },
+ });
+ expect(messagesOfKind(tab, 'cache-message')).toEqual([
+ expect.objectContaining({
+ message: { id: 42, ok: true, result: INITIAL_CACHE_REVISION },
+ }),
+ ]);
+ expect(vi.getTimerCount()).toBe(1);
+ });
+
+ it.each(['ack', 'drain', 'replacement'] as const)(
+ 'ignores a late lock probe after %s',
+ async (action) => {
+ vi.useFakeTimers();
+ let resolveProbe!: (held: boolean) => void;
+ const probe = new Promise<boolean>((resolve) => {
+ resolveProbe = resolve;
+ });
+ const verifyOwnerLockHeld = vi.fn(() => probe);
+ const router = new CoordinatorRouter({
+ heartbeatIntervalMs: 5,
+ heartbeatTimeoutMs: 7,
+ verifyTabLockHeld: async () => true,
+ verifyOwnerLockHeld,
+ watchTabLock: () => () => {},
+ });
+ const tab = new FakePort();
+ await register(router, tab, 'tab-a');
+ const engine = new FakePort();
+ await attach(router, tab, 'tab-a', 1, engine);
+ ready(engine, 'tab-a', 1, 'opened-existing');
+ await vi.advanceTimersByTimeAsync(12);
+ expect(verifyOwnerLockHeld).toHaveBeenCalledOnce();
+
+ if (action === 'ack') {
+ engine.receive({
+ ...version,
+ kind: 'heartbeat-ack',
+ ownerEpoch: 1,
+ heartbeatId: messagesOfKind(engine, 'heartbeat')[0]!.heartbeatId,
+ });
+ } else {
+ await router.handleTabMessage(tab, {
+ ...version,
+ kind: action === 'drain' ? 'graceful-departure' : 'engine-lost',
+ tabId: 'tab-a',
+ ownerEpoch: 1,
+ ...(action === 'replacement' ? { reason: 'worker failed' } : {}),
+ });
+ if (action === 'replacement') {
+ await vi.advanceTimersByTimeAsync(0);
+ const replacement = new FakePort();
+ await attach(router, tab, 'tab-a', 2, replacement);
+ ready(replacement, 'tab-a', 2, 'wiped-before-open');
+ }
+ }
+ const before = router.snapshot();
+ const terminations = messagesOfKind(tab, 'terminate-engine').length;
+ resolveProbe(false);
+ await vi.advanceTimersByTimeAsync(0);
+ expect(router.snapshot()).toEqual(before);
+ expect(messagesOfKind(tab, 'terminate-engine')).toHaveLength(
+ terminations
+ );
+ }
+ );
+
+ it('retries failed lock probes without losing the owner', async () => {
+ vi.useFakeTimers();
+ const verifyOwnerLockHeld = vi.fn(async () => {
+ throw new Error('lock service unavailable');
+ });
+ const router = new CoordinatorRouter({
+ heartbeatIntervalMs: 5,
+ heartbeatTimeoutMs: 7,
+ verifyTabLockHeld: async () => true,
+ verifyOwnerLockHeld,
+ watchTabLock: () => () => {},
+ });
+ const tab = new FakePort();
+ await register(router, tab, 'tab-a');
+ const engine = new FakePort();
+ await attach(router, tab, 'tab-a', 1, engine);
+ ready(engine, 'tab-a', 1, 'opened-existing');
+ await vi.advanceTimersByTimeAsync(36);
+ expect(verifyOwnerLockHeld).toHaveBeenCalledTimes(3);
+ expect(router.snapshot()?.state).toMatchObject({
+ kind: 'active',
+ ownerEpoch: 1,
+ });
+ expect(messagesOfKind(tab, 'terminate-engine')).toHaveLength(0);
+ expect(vi.getTimerCount()).toBe(1);
+ });
+
+ it('uses activation deadlines and confirmed engine lock loss to terminate and wipe', async () => {
vi.useFakeTimers();
const router = new CoordinatorRouter({
activationTimeoutMs: 10,
heartbeatIntervalMs: 5,
heartbeatTimeoutMs: 7,
verifyTabLockHeld: async () => true,
+ verifyOwnerLockHeld: async () => false,
watchTabLock: () => () => {},
});
const tabA = new FakePort();
@@ -844,7 +991,7 @@ describe('CoordinatorRouter', () => {
expect(messagesOfKind(tabB, 'terminate-engine')).toContainEqual(
expect.objectContaining({
ownerEpoch: 2,
- reason: 'engine heartbeat watchdog timed out',
+ reason: 'engine owner lock was released',
})
);
expect(router.snapshot()?.state.kind).toBe('activating');
diff --git a/apps/web/src/lib/graphql-cache/worker/coordinator-router.ts b/apps/web/src/lib/graphql-cache/worker/coordinator-router.ts
index 251d4095a4..6d4b306047 100644
--- a/apps/web/src/lib/graphql-cache/worker/coordinator-router.ts
+++ b/apps/web/src/lib/graphql-cache/worker/coordinator-router.ts
@@ -56,6 +56,7 @@ export interface CoordinatorRouterOptions {
heartbeatIntervalMs?: number;
heartbeatTimeoutMs?: number;
verifyTabLockHeld?: (lockName: string) => Promise<boolean>;
+ verifyOwnerLockHeld?: (lockName: string) => Promise<boolean>;
watchTabLock?: (
lockName: string,
onReleased: () => void
@@ -129,8 +130,8 @@ const envelope = <T extends { coordinatorVersion: 3 }>(
...value,
}) as unknown as T;
-/** Independently checks that registration cannot acquire the page-held lock. */
-export async function verifyTabLivenessLockHeld(
+/** Probes lock ownership without queuing behind or stealing the held lock. */
+export async function verifyExclusiveLockHeld(
lockName: string
): Promise<boolean> {
return await navigator.locks.request(
@@ -189,6 +190,7 @@ export class CoordinatorRouter {
private readonly heartbeatIntervalMs: number;
private readonly heartbeatTimeoutMs: number;
private readonly verifyTabLockHeld: (lockName: string) => Promise<boolean>;
+ private readonly verifyOwnerLockHeld: (lockName: string) => Promise<boolean>;
private readonly watchTabLock: (
lockName: string,
onReleased: () => void
@@ -227,7 +229,9 @@ export class CoordinatorRouter {
this.heartbeatTimeoutMs =
options.heartbeatTimeoutMs ?? DEFAULT_HEARTBEAT_TIMEOUT_MS;
this.verifyTabLockHeld =
- options.verifyTabLockHeld ?? verifyTabLivenessLockHeld;
+ options.verifyTabLockHeld ?? verifyExclusiveLockHeld;
+ this.verifyOwnerLockHeld =
+ options.verifyOwnerLockHeld ?? verifyExclusiveLockHeld;
this.watchTabLock = options.watchTabLock ?? watchTabLivenessLock;
this.setTimeoutFn =
options.setTimeout ?? globalThis.setTimeout.bind(globalThis);
@@ -1302,20 +1306,52 @@ export class CoordinatorRouter {
return;
}
this.heartbeatTimeoutTimer = this.setTimeoutFn(() => {
- if (
- this.pendingHeartbeat?.ownerEpoch === ownerEpoch &&
- this.pendingHeartbeat.heartbeatId === heartbeatId
- ) {
- this.failOwner(
- route.tabId,
- ownerEpoch,
- 'engine heartbeat watchdog timed out'
- );
- }
+ void this.checkHeartbeatOwner(route, heartbeatId);
}, this.heartbeatTimeoutMs);
}, this.heartbeatIntervalMs);
}
+ private async checkHeartbeatOwner(
+ route: EngineRoute,
+ heartbeatId: number
+ ): Promise<void> {
+ const core = this.coreValue;
+ if (!core || this.engineRoute !== route) return;
+
+ // A backgrounded/frozen page can suspend its DedicatedWorker while the
+ // SharedWorker keeps running. A missed heartbeat is not evidence of a
+ // crash: revoking ownership here wipes healthy storage and can exhaust
+ // recovery while the page is still suspended. Only a released physical
+ // owner lock confirms silent worker death (terminate() emits no error).
+ let lockHeld: boolean;
+ try {
+ lockHeld = await this.verifyOwnerLockHeld(
+ databaseOwnerLockName(core.scope)
+ );
+ } catch {
+ // A failed probe is not proof of owner loss either. Retry later.
+ lockHeld = true;
+ }
+ // The ack, a graceful drain, or another failure may have won the race
+ // while the lock probe was pending. Never act on that stale observation.
+ if (
+ this.engineRoute !== route ||
+ this.pendingHeartbeat?.ownerEpoch !== route.ownerEpoch ||
+ this.pendingHeartbeat.heartbeatId !== heartbeatId
+ ) {
+ return;
+ }
+ if (!lockHeld) {
+ this.failOwner(
+ route.tabId,
+ route.ownerEpoch,
+ 'engine owner lock was released'
+ );
+ return;
+ }
+ this.scheduleHeartbeat(route.ownerEpoch);
+ }
+
private acceptHeartbeat(ownerEpoch: number, heartbeatId: number): void {
if (
this.pendingHeartbeat?.ownerEpoch !== ownerEpoch ||