Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(web): reliably start GraphQL soup backfill (#6096)

Sean Aye3db689e44bb6
diff --git a/apps/web/src/lib/queries/soup/backfill.test.tsx b/apps/web/src/lib/queries/soup/backfill.test.tsx
index 1b1bef01a8..13b7d29dbc 100644
--- a/apps/web/src/lib/queries/soup/backfill.test.tsx
+++ b/apps/web/src/lib/queries/soup/backfill.test.tsx
@@ -10,16 +10,39 @@ import {
useSoupBackfills,
} from './backfill';
+const featureFlagMocks = vi.hoisted(() => ({
+ useFeatureFlag: vi.fn(),
+}));
+
const graphqlMocks = vi.hoisted(() => ({
+ getGraphqlSoupCacheHost: vi.fn(),
hydrateGraphqlSoup: vi.fn(),
}));
+const leaderMocks = vi.hoisted(() => ({
+ createTabLeaderSignal: vi.fn(),
+}));
+
+const telemetryMocks = vi.hoisted(() => ({
+ anonymousSpan: vi.fn(),
+}));
+
+vi.mock('@app/lib/analytics/posthog', () => ({
+ useFeatureFlag: featureFlagMocks.useFeatureFlag,
+}));
+
vi.mock('@core/constant/featureFlags', () => ({
ENABLE_GRAPHQL_BACKFILL: true,
+ ENABLE_GRAPHQL_SOUP_FLAG: 'enable-graphql-soup',
+ ENABLE_GRAPHQL_SOUP_OVERRIDE: undefined,
}));
vi.mock('@core/cross-tab/tab-leader', () => ({
- createTabLeaderSignal: () => () => true,
+ createTabLeaderSignal: leaderMocks.createTabLeaderSignal,
+}));
+
+vi.mock('@macro-inc/observability', () => ({
+ Telemetry: { anonymousSpan: telemetryMocks.anonymousSpan },
}));
vi.mock('@service-storage/graphql/generated/graphql', () => ({
@@ -27,7 +50,7 @@ vi.mock('@service-storage/graphql/generated/graphql', () => ({
}));
vi.mock('@service-storage/graphql-soup', () => ({
- getGraphqlSoupCacheHost: () => ({}),
+ getGraphqlSoupCacheHost: graphqlMocks.getGraphqlSoupCacheHost,
hydrateGraphqlSoup: graphqlMocks.hydrateGraphqlSoup,
}));
@@ -52,11 +75,21 @@ function BackfillRunner(props: { userId: string }) {
describe('runSoupBackfills', () => {
beforeEach(() => {
localStorage.clear();
+ featureFlagMocks.useFeatureFlag
+ .mockReset()
+ .mockReturnValue(() => ({ enabled: true, payload: undefined }));
+ graphqlMocks.getGraphqlSoupCacheHost.mockReset().mockReturnValue({});
graphqlMocks.hydrateGraphqlSoup.mockReset();
+ leaderMocks.createTabLeaderSignal.mockReset().mockReturnValue(() => true);
+ telemetryMocks.anonymousSpan.mockReset().mockImplementation(() => ({
+ setAttr: vi.fn(),
+ end: vi.fn(),
+ }));
});
afterEach(() => {
vi.useRealTimers();
+ vi.restoreAllMocks();
});
it('runs each lane to completion before starting the next lane', async () => {
@@ -124,6 +157,7 @@ describe('runSoupBackfills', () => {
it('continues to later lanes after a lane exhausts its retries', async () => {
vi.useFakeTimers();
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined);
const failedFetch = vi.fn(async () => {
throw new Error('failed lane');
});
@@ -147,6 +181,57 @@ describe('runSoupBackfills', () => {
expect(loadSoupBackfillCheckpoint('user-1', 'later-lane').completed).toBe(
true
);
+ expect(warn).toHaveBeenCalledWith(
+ '[graphql-soup-backfill] lane failed after retries',
+ expect.objectContaining({ checkpointId: 'failed-lane' })
+ );
+ const telemetryAttributes = telemetryMocks.anonymousSpan.mock.results
+ .flatMap((result) => result.value.setAttr.mock.calls)
+ .filter(([name]) => name === 'cache.backfill_state');
+ expect(telemetryAttributes).toContainEqual([
+ 'cache.backfill_state',
+ 'failed',
+ ]);
+ });
+
+ it('starts after the GraphQL flag becomes enabled', async () => {
+ const [enabled, setEnabled] = createSignal(false);
+ featureFlagMocks.useFeatureFlag.mockReturnValue(() => ({
+ enabled: enabled(),
+ payload: undefined,
+ }));
+ graphqlMocks.hydrateGraphqlSoup.mockResolvedValue({ nextCursor: null });
+
+ const rendered = render(() => <BackfillRunner userId="user-1" />);
+ expect(graphqlMocks.hydrateGraphqlSoup).not.toHaveBeenCalled();
+
+ setEnabled(true);
+
+ await vi.waitFor(() =>
+ expect(graphqlMocks.hydrateGraphqlSoup).toHaveBeenCalled()
+ );
+ rendered.unmount();
+ });
+
+ it('retries cache-host lookup after it was initially unavailable', async () => {
+ const [isLeader, setIsLeader] = createSignal(true);
+ leaderMocks.createTabLeaderSignal.mockReturnValue(isLeader);
+ let cacheHost: object | undefined;
+ graphqlMocks.getGraphqlSoupCacheHost.mockImplementation(() => cacheHost);
+ graphqlMocks.hydrateGraphqlSoup.mockResolvedValue({ nextCursor: null });
+
+ const rendered = render(() => <BackfillRunner userId="user-1" />);
+ expect(graphqlMocks.getGraphqlSoupCacheHost).toHaveBeenCalledOnce();
+ expect(graphqlMocks.hydrateGraphqlSoup).not.toHaveBeenCalled();
+
+ cacheHost = {};
+ setIsLeader(false);
+ setIsLeader(true);
+
+ await vi.waitFor(() =>
+ expect(graphqlMocks.hydrateGraphqlSoup).toHaveBeenCalled()
+ );
+ rendered.unmount();
});
it('cancels the current backfill and starts a new one when the user changes', async () => {
diff --git a/apps/web/src/lib/queries/soup/backfill.ts b/apps/web/src/lib/queries/soup/backfill.ts
index 4a8f923a33..9ff65ae524 100644
--- a/apps/web/src/lib/queries/soup/backfill.ts
+++ b/apps/web/src/lib/queries/soup/backfill.ts
@@ -1,5 +1,11 @@
-import { ENABLE_GRAPHQL_BACKFILL } from '@core/constant/featureFlags';
+import { useFeatureFlag } from '@app/lib/analytics/posthog';
+import {
+ ENABLE_GRAPHQL_BACKFILL,
+ ENABLE_GRAPHQL_SOUP_FLAG,
+ ENABLE_GRAPHQL_SOUP_OVERRIDE,
+} from '@core/constant/featureFlags';
import { createTabLeaderSignal } from '@core/cross-tab/tab-leader';
+import { Telemetry } from '@macro-inc/observability';
import { SoupBackfillDocument } from '@service-storage/graphql/generated/graphql';
import {
type FetchGraphqlSoupOptions,
@@ -23,6 +29,7 @@ const EMAIL_CONTENT_PAGE_LIMIT = 5;
const PAGE_DELAY_MS = 2_000;
const BACKFILL_RETRY_COUNT = 5;
const BACKFILL_RETRY_SCHEDULE = Schedule.exponential('1 second');
+const BACKFILL_PROGRESS_PAGE_INTERVAL = 10;
const EXCLUDED_ENTITY_ID = '00000000-0000-0000-0000-000000000000';
type SoupBackfillFetchPage = (
@@ -234,6 +241,35 @@ export function resetSoupBackfillCheckpoint(
}
}
+type SoupBackfillTelemetryState =
+ | 'started'
+ | 'progress'
+ | 'completed'
+ | 'failed';
+
+function recordSoupBackfillTelemetry(input: {
+ checkpointId: string;
+ durationMs?: number;
+ pagesFetched: number;
+ state: SoupBackfillTelemetryState;
+ totalPagesFetched: number;
+}): void {
+ try {
+ const span = Telemetry.anonymousSpan('graphql_cache.backfill');
+ span.setAttr('cache.backfill_lane', input.checkpointId);
+ span.setAttr('cache.backfill_version', BACKFILL_VERSION);
+ span.setAttr('cache.backfill_state', input.state);
+ span.setAttr('cache.backfill_pages_fetched', input.pagesFetched);
+ span.setAttr('cache.backfill_total_pages_fetched', input.totalPagesFetched);
+ if (input.durationMs !== undefined) {
+ span.setAttr('cache.duration_ms', input.durationMs);
+ }
+ span.end();
+ } catch {
+ // Observability must never affect cache hydration.
+ }
+}
+
function and<T>(left: T | null | undefined, right: T): T {
return left ? ({ and: { left, right } } as T) : right;
}
@@ -282,7 +318,8 @@ export function withUpdatedSince(
export const runSoupBackfill = Effect.fn('runSoupBackfill')(function* (
userId: string,
- params: SoupBackfillParams
+ params: SoupBackfillParams,
+ onCheckpoint?: (checkpoint: SoupBackfillCheckpoint) => void
) {
let checkpoint = yield* Effect.sync(() =>
loadSoupBackfillCheckpoint(userId, params.checkpointId)
@@ -339,9 +376,10 @@ export const runSoupBackfill = Effect.fn('runSoupBackfill')(function* (
}
: {}),
};
- yield* Effect.sync(() =>
- saveSoupBackfillCheckpoint(checkpoint, params.checkpointId)
- );
+ yield* Effect.sync(() => {
+ saveSoupBackfillCheckpoint(checkpoint, params.checkpointId);
+ onCheckpoint?.(checkpoint);
+ });
if (checkpoint.completed) return;
@@ -357,13 +395,76 @@ export const runSoupBackfills = Effect.fn('runSoupBackfills')(function* (
yield* Effect.forEach(
lanes,
(lane) =>
- runSoupBackfill(userId, lane).pipe(
- Effect.retry({
- times: BACKFILL_RETRY_COUNT,
- schedule: BACKFILL_RETRY_SCHEDULE,
- }),
- Effect.ignore
- ),
+ Effect.gen(function* () {
+ const startedAt = Date.now();
+ const initialCheckpoint = yield* Effect.sync(() =>
+ loadSoupBackfillCheckpoint(userId, lane.checkpointId)
+ );
+ const initialPagesFetched = initialCheckpoint.pagesFetched;
+ yield* Effect.sync(() =>
+ recordSoupBackfillTelemetry({
+ checkpointId: lane.checkpointId,
+ pagesFetched: 0,
+ state: 'started',
+ totalPagesFetched: initialPagesFetched,
+ })
+ );
+
+ yield* runSoupBackfill(userId, lane, (checkpoint) => {
+ const pagesFetched = checkpoint.pagesFetched - initialPagesFetched;
+ if (
+ !checkpoint.completed &&
+ pagesFetched % BACKFILL_PROGRESS_PAGE_INTERVAL === 0
+ ) {
+ recordSoupBackfillTelemetry({
+ checkpointId: lane.checkpointId,
+ durationMs: Date.now() - startedAt,
+ pagesFetched,
+ state: 'progress',
+ totalPagesFetched: checkpoint.pagesFetched,
+ });
+ }
+ }).pipe(
+ Effect.retry({
+ times: BACKFILL_RETRY_COUNT,
+ schedule: BACKFILL_RETRY_SCHEDULE,
+ }),
+ Effect.matchEffect({
+ onFailure: (error) =>
+ Effect.sync(() => {
+ const checkpoint = loadSoupBackfillCheckpoint(
+ userId,
+ lane.checkpointId
+ );
+ recordSoupBackfillTelemetry({
+ checkpointId: lane.checkpointId,
+ durationMs: Date.now() - startedAt,
+ pagesFetched: checkpoint.pagesFetched - initialPagesFetched,
+ state: 'failed',
+ totalPagesFetched: checkpoint.pagesFetched,
+ });
+ console.warn(
+ '[graphql-soup-backfill] lane failed after retries',
+ { checkpointId: lane.checkpointId, error }
+ );
+ }),
+ onSuccess: () =>
+ Effect.sync(() => {
+ const checkpoint = loadSoupBackfillCheckpoint(
+ userId,
+ lane.checkpointId
+ );
+ recordSoupBackfillTelemetry({
+ checkpointId: lane.checkpointId,
+ durationMs: Date.now() - startedAt,
+ pagesFetched: checkpoint.pagesFetched - initialPagesFetched,
+ state: 'completed',
+ totalPagesFetched: checkpoint.pagesFetched,
+ });
+ }),
+ })
+ );
+ }),
{ concurrency: 1, discard: true }
);
});
@@ -373,15 +474,21 @@ export const runSoupBackfills = Effect.fn('runSoupBackfills')(function* (
* Interrupting the fiber cancels the active fetch or inter-page sleep.
*/
export function useSoupBackfills(userId: string): void {
+ const graphqlSoupFlag = useFeatureFlag(ENABLE_GRAPHQL_SOUP_FLAG, {
+ enabledOverride: ENABLE_GRAPHQL_SOUP_OVERRIDE,
+ });
const isLeader = createTabLeaderSignal(
`graphql-soup-backfill:v${BACKFILL_VERSION}:coordinator`
);
createEffect(() => {
+ // Read reactive gates before the non-reactive cache-host lookup so a host
+ // that becomes available after flag loading or leader election is retried.
if (
!ENABLE_GRAPHQL_BACKFILL ||
- getGraphqlSoupCacheHost() === undefined ||
- !isLeader()
+ !graphqlSoupFlag().enabled ||
+ !isLeader() ||
+ getGraphqlSoupCacheHost() === undefined
) {
return;
}