Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

Seanaye/fix/thread content backfill (#6143)

Sean Aye78aea7f67037
diff --git a/apps/web/src/lib/graphql-cache/host/types.ts b/apps/web/src/lib/graphql-cache/host/types.ts
index 3e5e79a3db..a8ec1746f2 100644
--- a/apps/web/src/lib/graphql-cache/host/types.ts
+++ b/apps/web/src/lib/graphql-cache/host/types.ts
@@ -102,7 +102,11 @@ export interface CacheHost {
/** Evaluates an exact initial Soup filter page over complete local projections. */
entityFilter(args: EntityFilterCacheArgs): Promise<EntityFilterCacheResult>;
writeQuery(args: CacheWriteArgs): Promise<WriteResult>;
- /** Stores a query response and returns only fields not marked `@cacheOnly`. */
+ /**
+ * Stores a background query response and returns only fields not marked
+ * `@cacheOnly`. Advances the internal revision for coherent reads without
+ * notifying foreground subscribers unless the write resets cache identity.
+ */
hydrateQuery(args: Omit<CacheWriteArgs, 'opKey'>): Promise<HydrationResult>;
/** Durably queues an optimistic mutation and claims the strict head. */
enqueueOptimisticMutation(
diff --git a/apps/web/src/lib/graphql-cache/lifecycle.test.ts b/apps/web/src/lib/graphql-cache/lifecycle.test.ts
index 1ab7803968..0f8a6f76f8 100644
--- a/apps/web/src/lib/graphql-cache/lifecycle.test.ts
+++ b/apps/web/src/lib/graphql-cache/lifecycle.test.ts
@@ -16,8 +16,11 @@ describe('clearRegisteredCaches', () => {
vi.unstubAllGlobals();
});
- it('preserves the scope when every cache clears', async () => {
+ it('preserves the scope and clears external cursors when every cache clears', async () => {
localStorage.setItem('graphql-cache:scope', 'current-scope');
+ localStorage.setItem('graphql-soup-backfill:v6:user-1:core', '{}');
+ localStorage.setItem('graphql-soup-backfill:v7:user-1:email', '{}');
+ localStorage.setItem('unrelated', 'keep');
const clear = vi.fn().mockResolvedValue(undefined);
const { clearRegisteredCaches, registerCacheHost } = await import(
'./lifecycle'
@@ -28,6 +31,13 @@ describe('clearRegisteredCaches', () => {
expect(clear).toHaveBeenCalledOnce();
expect(localStorage.getItem('graphql-cache:scope')).toBe('current-scope');
+ expect(localStorage.getItem('graphql-soup-backfill:v6:user-1:core')).toBe(
+ null
+ );
+ expect(localStorage.getItem('graphql-soup-backfill:v7:user-1:email')).toBe(
+ null
+ );
+ expect(localStorage.getItem('unrelated')).toBe('keep');
});
it('awaits the serialized scope rotation before logout clearing resolves', async () => {
diff --git a/apps/web/src/lib/graphql-cache/lifecycle.ts b/apps/web/src/lib/graphql-cache/lifecycle.ts
index c4c6ee31e1..ecca90be82 100644
--- a/apps/web/src/lib/graphql-cache/lifecycle.ts
+++ b/apps/web/src/lib/graphql-cache/lifecycle.ts
@@ -3,6 +3,19 @@ import { rotateCacheScope } from './scope';
const hosts = new Set<CacheHost>();
+function clearExternalCacheState(): void {
+ try {
+ for (let index = localStorage.length - 1; index >= 0; index -= 1) {
+ const key = localStorage.key(index);
+ if (key?.startsWith('graphql-soup-backfill:')) {
+ localStorage.removeItem(key);
+ }
+ }
+ } catch {
+ // Storage can be unavailable in restricted browser contexts.
+ }
+}
+
/** Registers a cache host for account-lifecycle clearing. */
export function registerCacheHost(host: CacheHost): () => void {
hosts.add(host);
@@ -11,6 +24,9 @@ export function registerCacheHost(host: CacheHost): () => void {
/** Best-effort reset of each active cache database during logout. */
export async function clearRegisteredCaches(): Promise<void> {
+ // Soup cursors live outside the normalized cache. Reset them in the same
+ // lifecycle operation so no later login resumes past records wiped below.
+ clearExternalCacheState();
const results = await Promise.allSettled(
[...hosts].map((host) => host.clear())
);
diff --git a/apps/web/src/lib/graphql-cache/protocol.ts b/apps/web/src/lib/graphql-cache/protocol.ts
index 6f5615c0aa..4980f5f72d 100644
--- a/apps/web/src/lib/graphql-cache/protocol.ts
+++ b/apps/web/src/lib/graphql-cache/protocol.ts
@@ -440,6 +440,7 @@ export type CacheRequest = { id: number } & (
identity?: string;
}
| {
+ /** Background cache warming; ordinary writes do not publish pushes. */
kind: 'hydrate';
query: string;
operationName?: string;
diff --git a/apps/web/src/lib/graphql-cache/worker/worker-core.test.ts b/apps/web/src/lib/graphql-cache/worker/worker-core.test.ts
index 3535fb6148..5732a0ca91 100644
--- a/apps/web/src/lib/graphql-cache/worker/worker-core.test.ts
+++ b/apps/web/src/lib/graphql-cache/worker/worker-core.test.ts
@@ -923,6 +923,53 @@ describe('CacheWorkerCore', () => {
});
});
+ it('keeps revision-advancing background hydration silent', async () => {
+ const hydrateQuery = vi.fn().mockResolvedValue({
+ revision: INITIAL_CACHE_REVISION,
+ revisionAdvanced: true,
+ changed: ['GraphqlSoupDocument:doc-1'],
+ affectedOps: ['client:7'],
+ reset: false,
+ data: { cursor: 'next' },
+ });
+ loadCacheWasmMock.mockResolvedValue({
+ openCache: vi.fn().mockResolvedValue({ hydrateQuery }),
+ });
+ const messages: unknown[] = [];
+ const port = { postMessage: (message: unknown) => messages.push(message) };
+ const core = new CacheWorkerCore();
+ core.addPort(port);
+
+ await core.handleRequest(port, {
+ id: 1,
+ kind: 'init',
+ scope: 'scope-1',
+ });
+ await core.handleRequest(port, {
+ id: 2,
+ kind: 'hydrate',
+ query: 'query Backfill { cursor }',
+ data: { cursor: 'next' },
+ });
+
+ expect(messages).not.toContainEqual(
+ expect.objectContaining({ kind: 'ops-affected' })
+ );
+ expect(messages).not.toContainEqual(
+ expect.objectContaining({ kind: 'cache-changed' })
+ );
+ expect(messages.at(-1)).toEqual({
+ id: 2,
+ ok: true,
+ result: {
+ kind: 'data',
+ data: { cursor: 'next' },
+ revision: INITIAL_CACHE_REVISION,
+ reset: false,
+ },
+ });
+ });
+
it('drains earlier request responses before consuming close and rejects later admission', async () => {
const order: string[] = [];
let releaseRead!: () => void;
@@ -1351,11 +1398,12 @@ describe('CacheWorkerCore', () => {
expect(onStorageResetRequired).not.toHaveBeenCalled();
});
- it('records an identity-changing hydration as a logical reset', async () => {
+ it('records and broadcasts identity-changing hydration as a logical reset', async () => {
const hydrateQuery = vi.fn().mockResolvedValue({
revision: INITIAL_CACHE_REVISION,
- changed: [],
- affectedOps: [],
+ revisionAdvanced: true,
+ changed: ['GraphqlUser:user-1'],
+ affectedOps: ['client:7'],
reset: true,
data: { cursor: 'next' },
});
@@ -1370,6 +1418,7 @@ describe('CacheWorkerCore', () => {
flush: vi.fn(),
},
});
+ core.addPort(port);
await core.handleRequest(port, { id: 1, kind: 'init', scope: 'scope-1' });
await core.handleRequest(port, {
@@ -1386,6 +1435,15 @@ describe('CacheWorkerCore', () => {
resetReason: 'identity-change',
})
);
+ expect(port.postMessage).toHaveBeenCalledWith({
+ kind: 'ops-affected',
+ opIds: ['client:7'],
+ keys: ['GraphqlUser:user-1'],
+ });
+ expect(port.postMessage).toHaveBeenCalledWith({
+ kind: 'cache-changed',
+ revision: INITIAL_CACHE_REVISION,
+ });
expect(port.postMessage).toHaveBeenLastCalledWith({
id: 2,
ok: true,
diff --git a/apps/web/src/lib/graphql-cache/worker/worker-core.ts b/apps/web/src/lib/graphql-cache/worker/worker-core.ts
index b35a1b21ab..9d4c7553da 100644
--- a/apps/web/src/lib/graphql-cache/worker/worker-core.ts
+++ b/apps/web/src/lib/graphql-cache/worker/worker-core.ts
@@ -490,7 +490,11 @@ export class CacheWorkerCore {
request.identity
);
result.revision = parseCacheRevision(result.revision);
- this.fanOut(result, true);
+ // Hydration is background cache warming. Keep its revision advancement
+ // for coherent reads, but do not publish foreground invalidations that
+ // would make mounted Soup views switch authority mid-backfill. An
+ // identity change is a real cache reset and must still be broadcast.
+ if (result.reset) this.fanOut(result, true);
const hydration: HydrationResult & Pick<WriteResult, 'reset'> =
result.data === null
? { kind: 'void', revision: result.revision, reset: result.reset }
diff --git a/apps/web/src/lib/queries/soup/backfill.test.tsx b/apps/web/src/lib/queries/soup/backfill.test.tsx
index 57df9be045..90a18f6c83 100644
--- a/apps/web/src/lib/queries/soup/backfill.test.tsx
+++ b/apps/web/src/lib/queries/soup/backfill.test.tsx
@@ -1,5 +1,6 @@
import { render } from '@solidjs/testing-library';
import * as Effect from 'effect/Effect';
+import * as Fiber from 'effect/Fiber';
import { createSignal, Show } from 'solid-js';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import {
@@ -27,6 +28,14 @@ const telemetryMocks = vi.hoisted(() => ({
anonymousSpan: vi.fn(),
}));
+const cacheGenerationCallbacks = new Set<() => void>();
+const mockCacheHost = {
+ onCacheGenerationChanged: vi.fn((callback: () => void) => {
+ cacheGenerationCallbacks.add(callback);
+ return () => cacheGenerationCallbacks.delete(callback);
+ }),
+};
+
vi.mock('@app/lib/analytics/posthog', () => ({
useFeatureFlag: featureFlagMocks.useFeatureFlag,
}));
@@ -78,7 +87,11 @@ describe('runSoupBackfills', () => {
featureFlagMocks.useFeatureFlag
.mockReset()
.mockReturnValue(() => ({ enabled: true, payload: undefined }));
- graphqlMocks.getGraphqlSoupCacheHost.mockReset().mockReturnValue({});
+ cacheGenerationCallbacks.clear();
+ mockCacheHost.onCacheGenerationChanged.mockClear();
+ graphqlMocks.getGraphqlSoupCacheHost
+ .mockReset()
+ .mockReturnValue(mockCacheHost);
graphqlMocks.hydrateGraphqlSoup.mockReset();
leaderMocks.createTabLeaderSignal.mockReset().mockReturnValue(() => true);
telemetryMocks.anonymousSpan.mockReset().mockImplementation(() => ({
@@ -131,6 +144,136 @@ describe('runSoupBackfills', () => {
);
});
+ it('immediately catches up email changes that moved ahead of the initial cursor', async () => {
+ vi.useFakeTimers();
+ vi.setSystemTime('2026-09-02T12:00:00.000Z');
+ const fetchPage = vi.fn(
+ async (
+ _input: Parameters<NonNullable<SoupBackfillParams['fetchPage']>>[0]
+ ) => {
+ if (fetchPage.mock.calls.length === 1) {
+ vi.setSystemTime('2026-09-02T12:01:00.000Z');
+ }
+ return { nextCursor: null };
+ }
+ );
+
+ await Effect.runPromise(
+ runSoupBackfill('user-1', {
+ ...lane('email-thread-pages', fetchPage),
+ catchUpAfterInitialPass: true,
+ })
+ );
+
+ expect(fetchPage).toHaveBeenCalledTimes(2);
+ expect(fetchPage.mock.calls[0]?.[0]).toMatchObject({
+ initial: { sortMethod: 'VIEWED_UPDATED' },
+ });
+ expect(fetchPage.mock.calls[1]?.[0]).toMatchObject({
+ initial: {
+ filters: {
+ emailFilter: {
+ tree: {
+ literal: {
+ updatedAt: { gte: '2026-09-02T12:00:00.000Z' },
+ },
+ },
+ },
+ },
+ },
+ });
+ expect(
+ loadSoupBackfillCheckpoint('user-1', 'email-thread-pages')
+ ).toMatchObject({
+ completed: true,
+ scanStartedAt: null,
+ updatedSince: '2026-09-02T12:01:00.000Z',
+ });
+ });
+
+ it('atomically checkpoints and resumes an interrupted email catch-up pass', async () => {
+ vi.useFakeTimers();
+ vi.setSystemTime('2026-09-02T12:00:00.000Z');
+ let holdCatchUp = true;
+ let markCatchUpStarted!: () => void;
+ const catchUpStarted = new Promise<void>((resolve) => {
+ markCatchUpStarted = resolve;
+ });
+ const fetchPage = vi.fn(
+ (
+ _input: Parameters<NonNullable<SoupBackfillParams['fetchPage']>>[0],
+ options: Parameters<NonNullable<SoupBackfillParams['fetchPage']>>[1]
+ ) => {
+ if (fetchPage.mock.calls.length === 1) {
+ vi.setSystemTime('2026-09-02T12:01:00.000Z');
+ return Promise.resolve({ nextCursor: null });
+ }
+ if (holdCatchUp) {
+ markCatchUpStarted();
+ const signal = options?.signal;
+ if (!signal) throw new Error('expected a backfill abort signal');
+ return new Promise<{ nextCursor: string | null }>(
+ (_resolve, reject) =>
+ signal.addEventListener(
+ 'abort',
+ () =>
+ reject(
+ signal.reason ?? new DOMException('Aborted', 'AbortError')
+ ),
+ { once: true }
+ )
+ );
+ }
+ return Promise.resolve({ nextCursor: null });
+ }
+ );
+ const params = {
+ ...lane('email-thread-pages', fetchPage),
+ catchUpAfterInitialPass: true,
+ };
+
+ const fiber = Effect.runFork(runSoupBackfill('user-1', params));
+ await catchUpStarted;
+
+ expect(
+ loadSoupBackfillCheckpoint('user-1', 'email-thread-pages')
+ ).toMatchObject({
+ nextCursor: null,
+ pagesFetched: 1,
+ completed: false,
+ completedAt: null,
+ scanStartedAt: '2026-09-02T12:01:00.000Z',
+ updatedSince: '2026-09-02T12:00:00.000Z',
+ });
+
+ await Effect.runPromise(Fiber.interrupt(fiber));
+ holdCatchUp = false;
+ await Effect.runPromise(runSoupBackfill('user-1', params));
+
+ expect(fetchPage).toHaveBeenCalledTimes(3);
+ expect(fetchPage.mock.calls[2]?.[0]).toMatchObject({
+ initial: {
+ filters: {
+ emailFilter: {
+ tree: {
+ literal: {
+ updatedAt: { gte: '2026-09-02T12:00:00.000Z' },
+ },
+ },
+ },
+ },
+ },
+ });
+ expect(
+ loadSoupBackfillCheckpoint('user-1', 'email-thread-pages')
+ ).toMatchObject({
+ pagesFetched: 2,
+ completed: true,
+ scanStartedAt: null,
+ updatedSince: '2026-09-02T12:01:00.000Z',
+ });
+ });
+
it('does not checkpoint a page whose required v2 capsule ingestion fails', async () => {
const projectionFailure = new Error(
'SoupBackfill page contains an incomplete required cache projection'
@@ -215,7 +358,7 @@ describe('runSoupBackfills', () => {
it('starts when only the cache host becomes available', async () => {
vi.useFakeTimers();
- let cacheHost: object | undefined;
+ let cacheHost: typeof mockCacheHost | undefined;
graphqlMocks.getGraphqlSoupCacheHost.mockImplementation(() => cacheHost);
graphqlMocks.hydrateGraphqlSoup.mockResolvedValue({ nextCursor: null });
@@ -223,7 +366,7 @@ describe('runSoupBackfills', () => {
expect(graphqlMocks.getGraphqlSoupCacheHost).toHaveBeenCalledOnce();
expect(graphqlMocks.hydrateGraphqlSoup).not.toHaveBeenCalled();
- cacheHost = {};
+ cacheHost = mockCacheHost;
await vi.advanceTimersByTimeAsync(100);
expect(graphqlMocks.getGraphqlSoupCacheHost).toHaveBeenCalledTimes(2);
@@ -231,6 +374,62 @@ describe('runSoupBackfills', () => {
rendered.unmount();
});
+ it('restarts from the beginning when the cache generation is replaced', async () => {
+ localStorage.setItem(
+ 'graphql-soup-backfill:v7:user-1:core-entities',
+ JSON.stringify({
+ userId: 'user-1',
+ nextCursor: 'stale-cursor',
+ pagesFetched: 12,
+ completed: false,
+ scanStartedAt: '2026-09-01T00:00:00.000Z',
+ updatedSince: null,
+ completedAt: null,
+ })
+ );
+ const fetchInputs: unknown[] = [];
+ const fetchSignals: AbortSignal[] = [];
+ graphqlMocks.hydrateGraphqlSoup.mockImplementation(
+ (
+ _document,
+ variables: { input: unknown },
+ options: { signal: AbortSignal }
+ ) =>
+ new Promise((_resolve, reject) => {
+ fetchInputs.push(variables.input);
+ fetchSignals.push(options.signal);
+ options.signal.addEventListener(
+ 'abort',
+ () => reject(options.signal.reason),
+ { once: true }
+ );
+ })
+ );
+
+ const rendered = render(() => <BackfillRunner userId="user-1" />);
+ await vi.waitFor(() =>
+ expect(graphqlMocks.hydrateGraphqlSoup).toHaveBeenCalledOnce()
+ );
+ expect(fetchInputs[0]).toMatchObject({
+ continuation: { cursor: 'stale-cursor' },
+ });
+
+ for (const callback of cacheGenerationCallbacks) callback();
+
+ await vi.waitFor(() =>
+ expect(graphqlMocks.hydrateGraphqlSoup).toHaveBeenCalledTimes(2)
+ );
+ expect(fetchSignals[0]?.aborted).toBe(true);
+ expect(fetchInputs[1]).toMatchObject({ initial: { limit: 100 } });
+ expect(loadSoupBackfillCheckpoint('user-1', 'core-entities')).toMatchObject(
+ {
+ nextCursor: null,
+ pagesFetched: 0,
+ }
+ );
+ rendered.unmount();
+ });
+
it('cancels the current backfill and starts a new one when the user changes', async () => {
const fetchSignals: AbortSignal[] = [];
graphqlMocks.hydrateGraphqlSoup.mockImplementation(
diff --git a/apps/web/src/lib/queries/soup/backfill.ts b/apps/web/src/lib/queries/soup/backfill.ts
index 7d6dd1e167..9ddca3263c 100644
--- a/apps/web/src/lib/queries/soup/backfill.ts
+++ b/apps/web/src/lib/queries/soup/backfill.ts
@@ -5,6 +5,7 @@ import {
ENABLE_GRAPHQL_SOUP_OVERRIDE,
} from '@core/constant/featureFlags';
import { createTabLeaderSignal } from '@core/cross-tab/tab-leader';
+import type { CacheHost } from '@graphql-cache/host/types';
import { Telemetry } from '@macro-inc/observability';
import { SoupBackfillDocument } from '@service-storage/graphql/generated/graphql';
import {
@@ -18,11 +19,11 @@ import {
import * as Effect from 'effect/Effect';
import * as Fiber from 'effect/Fiber';
import * as Schedule from 'effect/Schedule';
-import { createEffect, onCleanup } from 'solid-js';
+import { createEffect, createSignal, onCleanup } from 'solid-js';
-// Bump when a default backfill input changes so persisted opaque cursors
-// cannot retain the previous server-side filters.
-const BACKFILL_VERSION = 6;
+// Bump when a default backfill input or completion guarantee changes so
+// persisted cursors cannot retain an older hydration contract.
+const BACKFILL_VERSION = 7;
const PAGE_LIMIT = 100;
// Five threads × twenty messages reaches the backend's 100-message cap.
const EMAIL_CONTENT_PAGE_LIMIT = 5;
@@ -54,6 +55,8 @@ export type SoupBackfillParams = {
input: GraphqlSoupInitialInput;
/** Delay between successful pages. Defaults to two seconds. */
pageDelayMs?: number;
+ /** Immediately follows the initial full scan with its watermark pass. */
+ catchUpAfterInitialPass?: boolean;
};
/** Backfills the entities used most often by Quick Access and primary views. */
@@ -80,9 +83,12 @@ export const CORE_SOUP_BACKFILL_LANE: SoupBackfillParams = {
* while excluding every other entity variant with an impossible id filter.
*/
export const EMAIL_SOUP_BACKFILL_LANE: SoupBackfillParams = {
- // Restart completed legacy newest-message checkpoints with the new shape.
checkpointId: 'email-thread-pages',
fetchPage: fetchEmailContentPage,
+ // A long full scan can skip threads created, viewed, or updated after its
+ // VIEWED_UPDATED cursor has passed them. Consume the recorded watermark
+ // before reporting this lane complete.
+ catchUpAfterInitialPass: true,
input: {
limit: EMAIL_CONTENT_PAGE_LIMIT,
expand: true,
@@ -243,6 +249,12 @@ export function resetSoupBackfillCheckpoint(
}
}
+function resetDefaultSoupBackfillCheckpoints(userId: string): void {
+ for (const lane of DEFAULT_SOUP_BACKFILL_LANES) {
+ resetSoupBackfillCheckpoint(userId, lane.checkpointId);
+ }
+}
+
type SoupBackfillTelemetryState =
| 'started'
| 'progress'
@@ -326,66 +338,92 @@ export const runSoupBackfill = Effect.fn('runSoupBackfill')(function* (
let checkpoint = yield* Effect.sync(() =>
loadSoupBackfillCheckpoint(userId, params.checkpointId)
);
+ // Only a never-completed full scan needs the additional watermark pass. An
+ // interrupted catch-up already has updatedSince and resumes normally.
+ let catchUpPassPending =
+ params.catchUpAfterInitialPass === true && checkpoint.updatedSince === null;
- // `completed` only marks the end of one pass. A later invocation resets
- // pagination so it can run another pass narrowed by the stored updatedSince.
- // An unfinished checkpoint without scanStartedAt keeps its cursor and only
- // initializes the timestamp needed for the next watermark.
- if (checkpoint.completed || checkpoint.scanStartedAt === null) {
+ const startPass = () => {
checkpoint = {
...checkpoint,
nextCursor: checkpoint.completed ? null : checkpoint.nextCursor,
completed: false,
scanStartedAt: new Date().toISOString(),
};
- yield* Effect.sync(() =>
- saveSoupBackfillCheckpoint(checkpoint, params.checkpointId)
- );
+ saveSoupBackfillCheckpoint(checkpoint, params.checkpointId);
+ };
+
+ // `completed` marks the end of one pass. Start a fresh pass from its stored
+ // watermark, while an unfinished checkpoint keeps its cursor.
+ if (checkpoint.completed || checkpoint.scanStartedAt === null) {
+ yield* Effect.sync(startPass);
}
- const passInput = withUpdatedSince(params.input, checkpoint.updatedSince);
const fetchPage = params.fetchPage ?? fetchSoupPage;
while (true) {
- const input: GraphqlSoupInput = checkpoint.nextCursor
- ? {
- continuation: {
- cursor: checkpoint.nextCursor,
- expand: passInput.expand,
- emailView: passInput.emailView,
- },
- }
- : { initial: passInput };
- // Hydration returns only the cursor projection. Cache-only entity payloads
- // are persisted without being materialized back into this page.
- const page = yield* Effect.tryPromise((signal) =>
- fetchPage(input, { signal })
- );
+ const passInput = withUpdatedSince(params.input, checkpoint.updatedSince);
- const completed = page.nextCursor == null;
- checkpoint = {
- ...checkpoint,
- nextCursor: page.nextCursor ?? null,
- pagesFetched: checkpoint.pagesFetched + 1,
- completed,
- ...(completed
+ while (true) {
+ const input: GraphqlSoupInput = checkpoint.nextCursor
? {
- // Use the pass start rather than its completion time so updates
- // made while this pass was running are included next time.
- updatedSince: checkpoint.scanStartedAt ?? checkpoint.updatedSince,
- completedAt: new Date().toISOString(),
- scanStartedAt: null,
+ continuation: {
+ cursor: checkpoint.nextCursor,
+ expand: passInput.expand,
+ emailView: passInput.emailView,
+ },
}
- : {}),
- };
- yield* Effect.sync(() => {
- saveSoupBackfillCheckpoint(checkpoint, params.checkpointId);
- onCheckpoint?.(checkpoint);
- });
+ : { initial: passInput };
+ // Hydration returns only the cursor projection. Cache-only entity
+ // payloads are persisted without being materialized back into this page.
+ const page = yield* Effect.tryPromise((signal) =>
+ fetchPage(input, { signal })
+ );
+
+ const passCompleted = page.nextCursor == null;
+ const transitionToCatchUp = passCompleted && catchUpPassPending;
+ const passCompletedAt = passCompleted ? new Date().toISOString() : null;
+ checkpoint = {
+ ...checkpoint,
+ nextCursor: page.nextCursor ?? null,
+ pagesFetched: checkpoint.pagesFetched + 1,
+ // Atomically persist the required catch-up as in progress instead of
+ // exposing a completed lane between the two passes.
+ completed: passCompleted && !transitionToCatchUp,
+ ...(passCompleted
+ ? transitionToCatchUp
+ ? {
+ // Filter from the full pass start, while using its completion
+ // as the resumable catch-up pass watermark.
+ updatedSince:
+ checkpoint.scanStartedAt ?? checkpoint.updatedSince,
+ completedAt: null,
+ scanStartedAt: passCompletedAt,
+ }
+ : {
+ // Use the pass start rather than its completion time so updates
+ // made while this pass was running are included next time.
+ updatedSince:
+ checkpoint.scanStartedAt ?? checkpoint.updatedSince,
+ completedAt: passCompletedAt,
+ scanStartedAt: null,
+ }
+ : {}),
+ };
+ if (transitionToCatchUp) catchUpPassPending = false;
+ yield* Effect.sync(() => {
+ saveSoupBackfillCheckpoint(checkpoint, params.checkpointId);
+ onCheckpoint?.(checkpoint);
+ });
+
+ if (passCompleted) break;
+
+ yield* Effect.sleep(params.pageDelayMs ?? PAGE_DELAY_MS);
+ }
if (checkpoint.completed) return;
-
- yield* Effect.sleep(params.pageDelayMs ?? PAGE_DELAY_MS);
+ // The only incomplete terminal-page state is the atomic transition above.
+ // Continue directly into its narrowed catch-up pass.
}
});
@@ -471,11 +509,12 @@ export const runSoupBackfills = Effect.fn('runSoupBackfills')(function* (
);
});
-const waitForGraphqlSoupCacheHost = Effect.suspend(() =>
- getGraphqlSoupCacheHost() === undefined
+const waitForGraphqlSoupCacheHost = Effect.suspend(() => {
+ const host = getGraphqlSoupCacheHost();
+ return host === undefined
? Effect.fail('cache-host-unavailable' as const)
- : Effect.void
-).pipe(
+ : Effect.succeed(host);
+}).pipe(
Effect.retry({
times: CACHE_HOST_RETRY_COUNT,
schedule: CACHE_HOST_RETRY_SCHEDULE,
@@ -485,7 +524,8 @@ const waitForGraphqlSoupCacheHost = Effect.suspend(() =>
/**
* Runs the checkpointed backfill Effect while this tab owns leadership.
* Interrupting the fiber cancels cache readiness waits, active fetches, and
- * inter-page sleeps.
+ * inter-page sleeps. A replacement cache generation resets the external
+ * cursors before restarting so they can never point past wiped cache data.
*/
export function useSoupBackfills(userId: string): void {
const graphqlSoupFlag = useFeatureFlag(ENABLE_GRAPHQL_SOUP_FLAG, {
@@ -494,15 +534,18 @@ export function useSoupBackfills(userId: string): void {
const isLeader = createTabLeaderSignal(
`graphql-soup-backfill:v${BACKFILL_VERSION}:coordinator`
);
+ const [cacheHost, setCacheHost] = createSignal<CacheHost>();
+ const [cacheGeneration, setCacheGeneration] = createSignal(0);
createEffect(() => {
if (!ENABLE_GRAPHQL_BACKFILL || !graphqlSoupFlag().enabled || !isLeader()) {
+ setCacheHost(undefined);
return;
}
const fiber = Effect.runFork(
waitForGraphqlSoupCacheHost.pipe(
- Effect.flatMap(() => runSoupBackfills(userId)),
+ Effect.tap((host) => Effect.sync(() => setCacheHost(host))),
Effect.ignore
)
);
@@ -510,4 +553,25 @@ export function useSoupBackfills(userId: string): void {
Effect.runFork(Fiber.interrupt(fiber));
});
});
+
+ createEffect(() => {
+ const host = cacheHost();
+ if (!host) return;
+
+ const unsubscribe = host.onCacheGenerationChanged(() => {
+ resetDefaultSoupBackfillCheckpoints(userId);
+ setCacheGeneration((generation) => generation + 1);
+ });
+ onCleanup(unsubscribe);
+ });
+
+ createEffect(() => {
+ if (!cacheHost()) return;
+ cacheGeneration();
+
+ const fiber = Effect.runFork(runSoupBackfills(userId).pipe(Effect.ignore));
+ onCleanup(() => {
+ Effect.runFork(Fiber.interrupt(fiber));
+ });
+ });
}
diff --git a/apps/web/tauri/graphql_cache_plugin/src/commands.rs b/apps/web/tauri/graphql_cache_plugin/src/commands.rs
index 85f1451468..51465a904d 100644
--- a/apps/web/tauri/graphql_cache_plugin/src/commands.rs
+++ b/apps/web/tauri/graphql_cache_plugin/src/commands.rs
@@ -193,8 +193,9 @@ pub enum HydrationResultWire {
},
}
-/// Normalizes and stores a network response, broadcasts affected operations,
-/// and returns only fields not marked `@cacheOnly`.
+/// Normalizes and stores a background hydration response and returns only
+/// fields not marked `@cacheOnly`. Ordinary hydration remains silent to
+/// foreground subscribers; identity-changing resets are still broadcast.
#[tauri::command]
pub async fn graphql_cache_hydrate<R: Runtime>(
app: AppHandle<R>,
@@ -214,13 +215,15 @@ pub async fn graphql_cache_hydrate<R: Runtime>(
identity,
)
.await?;
- emit_ops_affected(
- &app,
- &result.write_result.affected_ops,
- &result.write_result.changed,
- );
- if result.write_result.revision_advanced {
- emit_cache_changed(&app, &result.write_result.revision);
+ if result.write_result.reset {
+ emit_ops_affected(
+ &app,
+ &result.write_result.affected_ops,
+ &result.write_result.changed,
+ );
+ if result.write_result.revision_advanced {
+ emit_cache_changed(&app, &result.write_result.revision);
+ }
}
Ok(match result.data {
Some(data) => HydrationResultWire::Data {