Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(web): reliably start GraphQL soup backfill (#6096)
Sean Aye3db689e44bb6
diff --git a/apps/web/src/lib/queries/soup/backfill.test.tsx b/apps/web/src/lib/queries/soup/backfill.test.tsxindex 1b1bef01a8..13b7d29dbc 100644--- a/apps/web/src/lib/queries/soup/backfill.test.tsx+++ b/apps/web/src/lib/queries/soup/backfill.test.tsx@@ -10,16 +10,39 @@ import {useSoupBackfills,} from './backfill';+const featureFlagMocks = vi.hoisted(() => ({+ useFeatureFlag: vi.fn(),+}));+const graphqlMocks = vi.hoisted(() => ({+ getGraphqlSoupCacheHost: vi.fn(),hydrateGraphqlSoup: vi.fn(),}));+const leaderMocks = vi.hoisted(() => ({+ createTabLeaderSignal: vi.fn(),+}));++const telemetryMocks = vi.hoisted(() => ({+ anonymousSpan: vi.fn(),+}));++vi.mock('@app/lib/analytics/posthog', () => ({+ useFeatureFlag: featureFlagMocks.useFeatureFlag,+}));+vi.mock('@core/constant/featureFlags', () => ({ENABLE_GRAPHQL_BACKFILL: true,+ ENABLE_GRAPHQL_SOUP_FLAG: 'enable-graphql-soup',+ ENABLE_GRAPHQL_SOUP_OVERRIDE: undefined,}));vi.mock('@core/cross-tab/tab-leader', () => ({- createTabLeaderSignal: () => () => true,+ createTabLeaderSignal: leaderMocks.createTabLeaderSignal,+}));++vi.mock('@macro-inc/observability', () => ({+ Telemetry: { anonymousSpan: telemetryMocks.anonymousSpan },}));vi.mock('@service-storage/graphql/generated/graphql', () => ({@@ -27,7 +50,7 @@ vi.mock('@service-storage/graphql/generated/graphql', () => ({}));vi.mock('@service-storage/graphql-soup', () => ({- getGraphqlSoupCacheHost: () => ({}),+ getGraphqlSoupCacheHost: graphqlMocks.getGraphqlSoupCacheHost,hydrateGraphqlSoup: graphqlMocks.hydrateGraphqlSoup,}));@@ -52,11 +75,21 @@ function BackfillRunner(props: { userId: string }) {describe('runSoupBackfills', () => {beforeEach(() => {localStorage.clear();+ featureFlagMocks.useFeatureFlag+ .mockReset()+ .mockReturnValue(() => ({ enabled: true, payload: undefined }));+ graphqlMocks.getGraphqlSoupCacheHost.mockReset().mockReturnValue({});graphqlMocks.hydrateGraphqlSoup.mockReset();+ leaderMocks.createTabLeaderSignal.mockReset().mockReturnValue(() => true);+ telemetryMocks.anonymousSpan.mockReset().mockImplementation(() => ({+ setAttr: vi.fn(),+ end: vi.fn(),+ }));});afterEach(() => {vi.useRealTimers();+ vi.restoreAllMocks();});it('runs each lane to completion before starting the next lane', async () => {@@ -124,6 +157,7 @@ describe('runSoupBackfills', () => {it('continues to later lanes after a lane exhausts its retries', async () => {vi.useFakeTimers();+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined);const failedFetch = vi.fn(async () => {throw new Error('failed lane');});@@ -147,6 +181,57 @@ describe('runSoupBackfills', () => {expect(loadSoupBackfillCheckpoint('user-1', 'later-lane').completed).toBe(true);+ expect(warn).toHaveBeenCalledWith(+ '[graphql-soup-backfill] lane failed after retries',+ expect.objectContaining({ checkpointId: 'failed-lane' })+ );+ const telemetryAttributes = telemetryMocks.anonymousSpan.mock.results+ .flatMap((result) => result.value.setAttr.mock.calls)+ .filter(([name]) => name === 'cache.backfill_state');+ expect(telemetryAttributes).toContainEqual([+ 'cache.backfill_state',+ 'failed',+ ]);+ });++ it('starts after the GraphQL flag becomes enabled', async () => {+ const [enabled, setEnabled] = createSignal(false);+ featureFlagMocks.useFeatureFlag.mockReturnValue(() => ({+ enabled: enabled(),+ payload: undefined,+ }));+ graphqlMocks.hydrateGraphqlSoup.mockResolvedValue({ nextCursor: null });++ const rendered = render(() => <BackfillRunner userId="user-1" />);+ expect(graphqlMocks.hydrateGraphqlSoup).not.toHaveBeenCalled();++ setEnabled(true);++ await vi.waitFor(() =>+ expect(graphqlMocks.hydrateGraphqlSoup).toHaveBeenCalled()+ );+ rendered.unmount();+ });++ it('retries cache-host lookup after it was initially unavailable', async () => {+ const [isLeader, setIsLeader] = createSignal(true);+ leaderMocks.createTabLeaderSignal.mockReturnValue(isLeader);+ let cacheHost: object | undefined;+ graphqlMocks.getGraphqlSoupCacheHost.mockImplementation(() => cacheHost);+ graphqlMocks.hydrateGraphqlSoup.mockResolvedValue({ nextCursor: null });++ const rendered = render(() => <BackfillRunner userId="user-1" />);+ expect(graphqlMocks.getGraphqlSoupCacheHost).toHaveBeenCalledOnce();+ expect(graphqlMocks.hydrateGraphqlSoup).not.toHaveBeenCalled();++ cacheHost = {};+ setIsLeader(false);+ setIsLeader(true);++ await vi.waitFor(() =>+ expect(graphqlMocks.hydrateGraphqlSoup).toHaveBeenCalled()+ );+ rendered.unmount();});it('cancels the current backfill and starts a new one when the user changes', async () => {diff --git a/apps/web/src/lib/queries/soup/backfill.ts b/apps/web/src/lib/queries/soup/backfill.tsindex 4a8f923a33..9ff65ae524 100644--- a/apps/web/src/lib/queries/soup/backfill.ts+++ b/apps/web/src/lib/queries/soup/backfill.ts@@ -1,5 +1,11 @@-import { ENABLE_GRAPHQL_BACKFILL } from '@core/constant/featureFlags';+import { useFeatureFlag } from '@app/lib/analytics/posthog';+import {+ ENABLE_GRAPHQL_BACKFILL,+ ENABLE_GRAPHQL_SOUP_FLAG,+ ENABLE_GRAPHQL_SOUP_OVERRIDE,+} from '@core/constant/featureFlags';import { createTabLeaderSignal } from '@core/cross-tab/tab-leader';+import { Telemetry } from '@macro-inc/observability';import { SoupBackfillDocument } from '@service-storage/graphql/generated/graphql';import {type FetchGraphqlSoupOptions,@@ -23,6 +29,7 @@ const EMAIL_CONTENT_PAGE_LIMIT = 5;const PAGE_DELAY_MS = 2_000;const BACKFILL_RETRY_COUNT = 5;const BACKFILL_RETRY_SCHEDULE = Schedule.exponential('1 second');+const BACKFILL_PROGRESS_PAGE_INTERVAL = 10;const EXCLUDED_ENTITY_ID = '00000000-0000-0000-0000-000000000000';type SoupBackfillFetchPage = (@@ -234,6 +241,35 @@ export function resetSoupBackfillCheckpoint(}}+type SoupBackfillTelemetryState =+ | 'started'+ | 'progress'+ | 'completed'+ | 'failed';++function recordSoupBackfillTelemetry(input: {+ checkpointId: string;+ durationMs?: number;+ pagesFetched: number;+ state: SoupBackfillTelemetryState;+ totalPagesFetched: number;+}): void {+ try {+ const span = Telemetry.anonymousSpan('graphql_cache.backfill');+ span.setAttr('cache.backfill_lane', input.checkpointId);+ span.setAttr('cache.backfill_version', BACKFILL_VERSION);+ span.setAttr('cache.backfill_state', input.state);+ span.setAttr('cache.backfill_pages_fetched', input.pagesFetched);+ span.setAttr('cache.backfill_total_pages_fetched', input.totalPagesFetched);+ if (input.durationMs !== undefined) {+ span.setAttr('cache.duration_ms', input.durationMs);+ }+ span.end();+ } catch {+ // Observability must never affect cache hydration.+ }+}+function and<T>(left: T | null | undefined, right: T): T {return left ? ({ and: { left, right } } as T) : right;}@@ -282,7 +318,8 @@ export function withUpdatedSince(export const runSoupBackfill = Effect.fn('runSoupBackfill')(function* (userId: string,- params: SoupBackfillParams+ params: SoupBackfillParams,+ onCheckpoint?: (checkpoint: SoupBackfillCheckpoint) => void) {let checkpoint = yield* Effect.sync(() =>loadSoupBackfillCheckpoint(userId, params.checkpointId)@@ -339,9 +376,10 @@ export const runSoupBackfill = Effect.fn('runSoupBackfill')(function* (}: {}),};- yield* Effect.sync(() =>- saveSoupBackfillCheckpoint(checkpoint, params.checkpointId)- );+ yield* Effect.sync(() => {+ saveSoupBackfillCheckpoint(checkpoint, params.checkpointId);+ onCheckpoint?.(checkpoint);+ });if (checkpoint.completed) return;@@ -357,13 +395,76 @@ export const runSoupBackfills = Effect.fn('runSoupBackfills')(function* (yield* Effect.forEach(lanes,(lane) =>- runSoupBackfill(userId, lane).pipe(- Effect.retry({- times: BACKFILL_RETRY_COUNT,- schedule: BACKFILL_RETRY_SCHEDULE,- }),- Effect.ignore- ),+ Effect.gen(function* () {+ const startedAt = Date.now();+ const initialCheckpoint = yield* Effect.sync(() =>+ loadSoupBackfillCheckpoint(userId, lane.checkpointId)+ );+ const initialPagesFetched = initialCheckpoint.pagesFetched;+ yield* Effect.sync(() =>+ recordSoupBackfillTelemetry({+ checkpointId: lane.checkpointId,+ pagesFetched: 0,+ state: 'started',+ totalPagesFetched: initialPagesFetched,+ })+ );++ yield* runSoupBackfill(userId, lane, (checkpoint) => {+ const pagesFetched = checkpoint.pagesFetched - initialPagesFetched;+ if (+ !checkpoint.completed &&+ pagesFetched % BACKFILL_PROGRESS_PAGE_INTERVAL === 0+ ) {+ recordSoupBackfillTelemetry({+ checkpointId: lane.checkpointId,+ durationMs: Date.now() - startedAt,+ pagesFetched,+ state: 'progress',+ totalPagesFetched: checkpoint.pagesFetched,+ });+ }+ }).pipe(+ Effect.retry({+ times: BACKFILL_RETRY_COUNT,+ schedule: BACKFILL_RETRY_SCHEDULE,+ }),+ Effect.matchEffect({+ onFailure: (error) =>+ Effect.sync(() => {+ const checkpoint = loadSoupBackfillCheckpoint(+ userId,+ lane.checkpointId+ );+ recordSoupBackfillTelemetry({+ checkpointId: lane.checkpointId,+ durationMs: Date.now() - startedAt,+ pagesFetched: checkpoint.pagesFetched - initialPagesFetched,+ state: 'failed',+ totalPagesFetched: checkpoint.pagesFetched,+ });+ console.warn(+ '[graphql-soup-backfill] lane failed after retries',+ { checkpointId: lane.checkpointId, error }+ );+ }),+ onSuccess: () =>+ Effect.sync(() => {+ const checkpoint = loadSoupBackfillCheckpoint(+ userId,+ lane.checkpointId+ );+ recordSoupBackfillTelemetry({+ checkpointId: lane.checkpointId,+ durationMs: Date.now() - startedAt,+ pagesFetched: checkpoint.pagesFetched - initialPagesFetched,+ state: 'completed',+ totalPagesFetched: checkpoint.pagesFetched,+ });+ }),+ })+ );+ }),{ concurrency: 1, discard: true });});@@ -373,15 +474,21 @@ export const runSoupBackfills = Effect.fn('runSoupBackfills')(function* (* Interrupting the fiber cancels the active fetch or inter-page sleep.*/export function useSoupBackfills(userId: string): void {+ const graphqlSoupFlag = useFeatureFlag(ENABLE_GRAPHQL_SOUP_FLAG, {+ enabledOverride: ENABLE_GRAPHQL_SOUP_OVERRIDE,+ });const isLeader = createTabLeaderSignal(`graphql-soup-backfill:v${BACKFILL_VERSION}:coordinator`);createEffect(() => {+ // Read reactive gates before the non-reactive cache-host lookup so a host+ // that becomes available after flag loading or leader election is retried.if (!ENABLE_GRAPHQL_BACKFILL ||- getGraphqlSoupCacheHost() === undefined ||- !isLeader()+ !graphqlSoupFlag().enabled ||+ !isLeader() ||+ getGraphqlSoupCacheHost() === undefined) {return;}