Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(cache): confirm owner lock loss before heartbeat recovery (#6694)
Sean Aye763395880dcf
diff --git a/apps/web/docs/graphql-normalized-cache-plan.md b/apps/web/docs/graphql-normalized-cache-plan.mdindex 97fe764407..f598dda6d2 100644--- a/apps/web/docs/graphql-normalized-cache-plan.md+++ b/apps/web/docs/graphql-normalized-cache-plan.md@@ -189,8 +189,14 @@ The cache's validated envelopes, request ids, Web Locks, owner epochs,heartbeats, and replay policy remain application-owned. A normal `pagehide`uses a fenced navigation-departure message: the page terminates its dedicatedworker, but the next owner opens the existing OPFS database. Only an-unannounced liveness, heartbeat, transport, or engine failure **after the database-open grant** selects `wipe-before-open`.+unannounced liveness, transport, or engine failure **after the database+open grant** selects `wipe-before-open`. A missed heartbeat alone is not owner+loss: background tabs and their dedicated workers may be suspended while the+SharedWorker still runs. The coordinator probes the physical owner Web Lock+without waiting or stealing it; a held lock (or a failed probe) retries the+heartbeat without changing epochs or touching storage. A released lock confirms+silent worker termination and triggers the usual fenced recovery. Read deadlines+still bound callers waiting on an unresponsive engine.Protocol v3 separates startup into asset loading and database opening. The engineloads/compiles WASM without touching OPFS, reports `engine-assets-ready`, and waitsdiff --git a/apps/web/src/lib/graphql-cache/worker/browser-test/coordinator.browser.e2e.ts b/apps/web/src/lib/graphql-cache/worker/browser-test/coordinator.browser.e2e.tsindex b1aa8898e8..9eafdf52a6 100644--- a/apps/web/src/lib/graphql-cache/worker/browser-test/coordinator.browser.e2e.ts+++ b/apps/web/src/lib/graphql-cache/worker/browser-test/coordinator.browser.e2e.ts@@ -160,6 +160,115 @@ test('direct cutover lazily deletes only the former normalized-cache IDB', asyncexpect(browserErrors).toEqual([]);});+test('a suspended owner resumes without replacement or cache loss', async ({+ context,+ page,+ browserName,+}, testInfo) => {+ test.skip(+ browserName !== 'chromium',+ 'CDP lifecycle control is Chromium-only'+ );+ const scope = `cache-suspension-${crypto.randomUUID()}`;+ const path = `${harnessPath(testInfo.project.name, 'cache-lifecycle.html')}?treatment=true&scope=${scope}`;+ await page.goto(path);+ await page.evaluate(async () => {+ await window.cacheLifecycleHarness.startSingle();+ await window.cacheLifecycleHarness.write('preserve-through-suspension');+ });+ const before = await page.evaluate(() => window.cacheLifecycleHarness.read());+ expect(before).toMatchObject({ kind: 'hit' });+ const follower = await context.newPage();+ await follower.goto(path);+ await follower.evaluate(() => window.cacheLifecycleHarness.startSingle());++ const cdp = await context.newCDPSession(page);+ const { targetInfos } = await cdp.send('Target.getTargets');+ const engine = targetInfos.find(+ (target) =>+ target.type === 'worker' && target.url.includes('cache.engine-worker')+ );+ if (!engine) throw new Error('missing cache engine target');+ const { sessionId } = await cdp.send('Target.attachToTarget', {+ targetId: engine.targetId,+ flatten: false,+ });+ const paused = new Promise<void>((resolve) => {+ cdp.on('Target.receivedMessageFromTarget', (event) => {+ if (+ event.sessionId === sessionId &&+ JSON.parse(event.message).method === 'Debugger.paused'+ )+ resolve();+ });+ });+ const debug = async (id: number, method: string) => {+ await cdp.send('Target.sendMessageToTarget', {+ sessionId,+ message: JSON.stringify({ id, method }),+ });+ };+ await debug(1, 'Debugger.enable');+ await debug(2, 'Debugger.pause');+ await paused;+ try {+ await cdp.send('Page.setWebLifecycleState', { state: 'frozen' });+ // CDP page freezing alone need not pause a lock-holding worker in Chrome.+ // Explicitly pause its event loop, keeping its lock held, while the other+ // page keeps the coordinator running beyond multiple heartbeat deadlines.+ await new Promise((resolve) => setTimeout(resolve, 16_000));+ const { targetInfos: suspendedTargets } =+ await cdp.send('Target.getTargets');+ expect(+ suspendedTargets.some((target) => target.targetId === engine.targetId)+ ).toBe(true);+ } finally {+ const { targetInfos: remainingTargets } =+ await cdp.send('Target.getTargets');+ if (+ remainingTargets.some((target) => target.targetId === engine.targetId)+ ) {+ await debug(3, 'Debugger.resume');+ }+ await cdp.send('Page.setWebLifecycleState', { state: 'active' });+ await cdp.detach();+ }++ expect(+ await page.evaluate(() => window.cacheLifecycleHarness.read())+ ).toEqual(before);+ expect(+ await page.evaluate(() => window.cacheLifecycleHarness.engineWorkerCount())+ ).toBe(1);+ expect(+ await follower.evaluate(() =>+ window.cacheLifecycleHarness.engineWorkerCount()+ )+ ).toBe(0);+ expect(+ await follower.evaluate(() => window.cacheLifecycleHarness.read())+ ).toEqual(before);+});++test('a silently terminated engine still recovers after releasing its owner lock', async ({+ page,+}, testInfo) => {+ await page.goto(+ `${harnessPath(testInfo.project.name, 'cache-lifecycle.html')}?treatment=true`+ );+ await page.evaluate(() => window.cacheLifecycleHarness.startSingle());+ const result = await page.evaluate(() =>+ window.cacheLifecycleHarness.abruptOwnerLoss()+ );+ expect(result).toMatchObject({+ oldRequestRejected: true,+ replacement: { kind: 'miss' },+ });+ expect(+ await page.evaluate(() => window.cacheLifecycleHarness.engineWorkerCount())+ ).toBe(2);+});+test('production cache-wasm Turso engine preserves graceful data and atomically recovers abrupt loss', async ({context,page,diff --git a/apps/web/src/lib/graphql-cache/worker/coordinator-router.test.ts b/apps/web/src/lib/graphql-cache/worker/coordinator-router.test.tsindex 3beb7d4155..d9839d0a4b 100644--- a/apps/web/src/lib/graphql-cache/worker/coordinator-router.test.ts+++ b/apps/web/src/lib/graphql-cache/worker/coordinator-router.test.ts@@ -799,13 +799,160 @@ describe('CoordinatorRouter', () => {]);});- it('uses activation and heartbeat watchdogs to terminate and wipe', async () => {+ it('preserves the owner and in-flight work through repeated missed heartbeats', async () => {+ vi.useFakeTimers();+ const verifyLockHeld = vi.fn(async () => true);+ const router = new CoordinatorRouter({+ heartbeatIntervalMs: 5,+ heartbeatTimeoutMs: 7,+ verifyTabLockHeld: async () => true,+ verifyOwnerLockHeld: verifyLockHeld,+ watchTabLock: () => () => {},+ });+ const tab = new FakePort();+ await register(router, tab, 'tab-a');+ const engine = new FakePort();+ await attach(router, tab, 'tab-a', 1, engine);+ ready(engine, 'tab-a', 1, 'opened-existing');+ await router.handleTabMessage(tab, {+ ...version,+ kind: 'cache-request',+ tabId: 'tab-a',+ request: { id: 42, kind: 'clear' },+ });+ const request = messagesOfKind(engine, 'engine-request')[0]!;++ // Background suspension can outlast every recovery attempt. A held+ // physical lock is still authoritative; silence is not owner loss.+ await vi.advanceTimersByTimeAsync(120_000);+ expect(router.snapshot()?.state).toMatchObject({+ kind: 'active',+ ownerEpoch: 1,+ });+ expect(messagesOfKind(tab, 'terminate-engine')).toHaveLength(0);+ expect(messagesOfKind(tab, 'terminal-error')).toHaveLength(0);+ expect(messagesOfKind(tab, 'cache-message')).toHaveLength(0);+ expect(verifyLockHeld).toHaveBeenCalledWith(databaseOwnerLockName('scope'));+ expect(messagesOfKind(engine, 'heartbeat').length).toBeGreaterThan(5);++ const heartbeat = messagesOfKind(engine, 'heartbeat').at(-1)!;+ engine.receive({+ ...version,+ kind: 'heartbeat-ack',+ ownerEpoch: 1,+ heartbeatId: heartbeat.heartbeatId,+ });+ engine.receive({+ ...version,+ kind: 'engine-response',+ ownerEpoch: 1,+ routeId: request.routeId,+ response: {+ id: request.routeId,+ ok: true,+ result: INITIAL_CACHE_REVISION,+ },+ });+ expect(messagesOfKind(tab, 'cache-message')).toEqual([+ expect.objectContaining({+ message: { id: 42, ok: true, result: INITIAL_CACHE_REVISION },+ }),+ ]);+ expect(vi.getTimerCount()).toBe(1);+ });++ it.each(['ack', 'drain', 'replacement'] as const)(+ 'ignores a late lock probe after %s',+ async (action) => {+ vi.useFakeTimers();+ let resolveProbe!: (held: boolean) => void;+ const probe = new Promise<boolean>((resolve) => {+ resolveProbe = resolve;+ });+ const verifyOwnerLockHeld = vi.fn(() => probe);+ const router = new CoordinatorRouter({+ heartbeatIntervalMs: 5,+ heartbeatTimeoutMs: 7,+ verifyTabLockHeld: async () => true,+ verifyOwnerLockHeld,+ watchTabLock: () => () => {},+ });+ const tab = new FakePort();+ await register(router, tab, 'tab-a');+ const engine = new FakePort();+ await attach(router, tab, 'tab-a', 1, engine);+ ready(engine, 'tab-a', 1, 'opened-existing');+ await vi.advanceTimersByTimeAsync(12);+ expect(verifyOwnerLockHeld).toHaveBeenCalledOnce();++ if (action === 'ack') {+ engine.receive({+ ...version,+ kind: 'heartbeat-ack',+ ownerEpoch: 1,+ heartbeatId: messagesOfKind(engine, 'heartbeat')[0]!.heartbeatId,+ });+ } else {+ await router.handleTabMessage(tab, {+ ...version,+ kind: action === 'drain' ? 'graceful-departure' : 'engine-lost',+ tabId: 'tab-a',+ ownerEpoch: 1,+ ...(action === 'replacement' ? { reason: 'worker failed' } : {}),+ });+ if (action === 'replacement') {+ await vi.advanceTimersByTimeAsync(0);+ const replacement = new FakePort();+ await attach(router, tab, 'tab-a', 2, replacement);+ ready(replacement, 'tab-a', 2, 'wiped-before-open');+ }+ }+ const before = router.snapshot();+ const terminations = messagesOfKind(tab, 'terminate-engine').length;+ resolveProbe(false);+ await vi.advanceTimersByTimeAsync(0);+ expect(router.snapshot()).toEqual(before);+ expect(messagesOfKind(tab, 'terminate-engine')).toHaveLength(+ terminations+ );+ }+ );++ it('retries failed lock probes without losing the owner', async () => {+ vi.useFakeTimers();+ const verifyOwnerLockHeld = vi.fn(async () => {+ throw new Error('lock service unavailable');+ });+ const router = new CoordinatorRouter({+ heartbeatIntervalMs: 5,+ heartbeatTimeoutMs: 7,+ verifyTabLockHeld: async () => true,+ verifyOwnerLockHeld,+ watchTabLock: () => () => {},+ });+ const tab = new FakePort();+ await register(router, tab, 'tab-a');+ const engine = new FakePort();+ await attach(router, tab, 'tab-a', 1, engine);+ ready(engine, 'tab-a', 1, 'opened-existing');+ await vi.advanceTimersByTimeAsync(36);+ expect(verifyOwnerLockHeld).toHaveBeenCalledTimes(3);+ expect(router.snapshot()?.state).toMatchObject({+ kind: 'active',+ ownerEpoch: 1,+ });+ expect(messagesOfKind(tab, 'terminate-engine')).toHaveLength(0);+ expect(vi.getTimerCount()).toBe(1);+ });++ it('uses activation deadlines and confirmed engine lock loss to terminate and wipe', async () => {vi.useFakeTimers();const router = new CoordinatorRouter({activationTimeoutMs: 10,heartbeatIntervalMs: 5,heartbeatTimeoutMs: 7,verifyTabLockHeld: async () => true,+ verifyOwnerLockHeld: async () => false,watchTabLock: () => () => {},});const tabA = new FakePort();@@ -844,7 +991,7 @@ describe('CoordinatorRouter', () => {expect(messagesOfKind(tabB, 'terminate-engine')).toContainEqual(expect.objectContaining({ownerEpoch: 2,- reason: 'engine heartbeat watchdog timed out',+ reason: 'engine owner lock was released',}));expect(router.snapshot()?.state.kind).toBe('activating');diff --git a/apps/web/src/lib/graphql-cache/worker/coordinator-router.ts b/apps/web/src/lib/graphql-cache/worker/coordinator-router.tsindex 251d4095a4..6d4b306047 100644--- a/apps/web/src/lib/graphql-cache/worker/coordinator-router.ts+++ b/apps/web/src/lib/graphql-cache/worker/coordinator-router.ts@@ -56,6 +56,7 @@ export interface CoordinatorRouterOptions {heartbeatIntervalMs?: number;heartbeatTimeoutMs?: number;verifyTabLockHeld?: (lockName: string) => Promise<boolean>;+ verifyOwnerLockHeld?: (lockName: string) => Promise<boolean>;watchTabLock?: (lockName: string,onReleased: () => void@@ -129,8 +130,8 @@ const envelope = <T extends { coordinatorVersion: 3 }>(...value,}) as unknown as T;-/** Independently checks that registration cannot acquire the page-held lock. */-export async function verifyTabLivenessLockHeld(+/** Probes lock ownership without queuing behind or stealing the held lock. */+export async function verifyExclusiveLockHeld(lockName: string): Promise<boolean> {return await navigator.locks.request(@@ -189,6 +190,7 @@ export class CoordinatorRouter {private readonly heartbeatIntervalMs: number;private readonly heartbeatTimeoutMs: number;private readonly verifyTabLockHeld: (lockName: string) => Promise<boolean>;+ private readonly verifyOwnerLockHeld: (lockName: string) => Promise<boolean>;private readonly watchTabLock: (lockName: string,onReleased: () => void@@ -227,7 +229,9 @@ export class CoordinatorRouter {this.heartbeatTimeoutMs =options.heartbeatTimeoutMs ?? DEFAULT_HEARTBEAT_TIMEOUT_MS;this.verifyTabLockHeld =- options.verifyTabLockHeld ?? verifyTabLivenessLockHeld;+ options.verifyTabLockHeld ?? verifyExclusiveLockHeld;+ this.verifyOwnerLockHeld =+ options.verifyOwnerLockHeld ?? verifyExclusiveLockHeld;this.watchTabLock = options.watchTabLock ?? watchTabLivenessLock;this.setTimeoutFn =options.setTimeout ?? globalThis.setTimeout.bind(globalThis);@@ -1302,20 +1306,52 @@ export class CoordinatorRouter {return;}this.heartbeatTimeoutTimer = this.setTimeoutFn(() => {- if (- this.pendingHeartbeat?.ownerEpoch === ownerEpoch &&- this.pendingHeartbeat.heartbeatId === heartbeatId- ) {- this.failOwner(- route.tabId,- ownerEpoch,- 'engine heartbeat watchdog timed out'- );- }+ void this.checkHeartbeatOwner(route, heartbeatId);}, this.heartbeatTimeoutMs);}, this.heartbeatIntervalMs);}+ private async checkHeartbeatOwner(+ route: EngineRoute,+ heartbeatId: number+ ): Promise<void> {+ const core = this.coreValue;+ if (!core || this.engineRoute !== route) return;++ // A backgrounded/frozen page can suspend its DedicatedWorker while the+ // SharedWorker keeps running. A missed heartbeat is not evidence of a+ // crash: revoking ownership here wipes healthy storage and can exhaust+ // recovery while the page is still suspended. Only a released physical+ // owner lock confirms silent worker death (terminate() emits no error).+ let lockHeld: boolean;+ try {+ lockHeld = await this.verifyOwnerLockHeld(+ databaseOwnerLockName(core.scope)+ );+ } catch {+ // A failed probe is not proof of owner loss either. Retry later.+ lockHeld = true;+ }+ // The ack, a graceful drain, or another failure may have won the race+ // while the lock probe was pending. Never act on that stale observation.+ if (+ this.engineRoute !== route ||+ this.pendingHeartbeat?.ownerEpoch !== route.ownerEpoch ||+ this.pendingHeartbeat.heartbeatId !== heartbeatId+ ) {+ return;+ }+ if (!lockHeld) {+ this.failOwner(+ route.tabId,+ route.ownerEpoch,+ 'engine owner lock was released'+ );+ return;+ }+ this.scheduleHeartbeat(route.ownerEpoch);+ }+private acceptHeartbeat(ownerEpoch: number, heartbeatId: number): void {if (this.pendingHeartbeat?.ownerEpoch !== ownerEpoch ||