Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(email): backfill not working and block not loading from cache data (#5592)
dev-rb0f429369b3a9
diff --git a/apps/web/src/features/block-email/component/Block.tsx b/apps/web/src/features/block-email/component/Block.tsxindex 127b00d485..936e6d1636 100644--- a/apps/web/src/features/block-email/component/Block.tsx+++ b/apps/web/src/features/block-email/component/Block.tsx@@ -1,6 +1,7 @@import { useBlockEntityCommands } from '@app/features/next-soup/actions';import { useGlobalNotificationSource } from '@components/app/GlobalAppState';import { useSplitPanel } from '@components/app/split-layout/layoutUtils';+import { useBlockId } from '@core/block';import { DocumentBlockContainer } from '@core/component/DocumentBlockContainer';import { buildEntityData } from '@entity';import { EmailDebouncedReadMarker } from '@notifications';@@ -12,6 +13,7 @@ import { EmailView } from './Email';export default function BlockEmail() {const blockData = blockDataSignal.get;+ const blockId = useBlockId();// Email threads are absent from quick access, so the entity the block-level// commands act on has to come from here. Built off the block's loaded data@@ -35,7 +37,7 @@ export default function BlockEmail() {// marking it seen so scanning/previewing doesn't clear unread state.const isPreview = !!useSplitPanel()?.handle.isViewerSplit();- const threadId = createMemo(() => blockData()?.thread?.db_id ?? '');+ const threadId = () => blockId;const threadQuery = useThreadQuery(threadId, () => ({enabled: !!threadId(),diff --git a/apps/web/src/lib/queries/email/thread.ts b/apps/web/src/lib/queries/email/thread.tsindex ca9727300c..91d259bafa 100644--- a/apps/web/src/lib/queries/email/thread.ts+++ b/apps/web/src/lib/queries/email/thread.ts@@ -2,6 +2,7 @@ import { useAnalytics } from '@app/lib/analytics/analytics-context';import { useFeatureFlag } from '@app/lib/analytics/posthog';import { toast } from '@core/component/Toast/Toast';import {+ ENABLE_GRAPHQL_SOUP,ENABLE_GRAPHQL_SOUP_FLAG,ENABLE_GRAPHQL_SOUP_OVERRIDE,} from '@core/constant/featureFlags';@@ -19,6 +20,15 @@ import type {ApiThread as Thread,UpsertScheduledResponse,} from '@service-email/generated/schemas';+import {+ EmailThreadPageDocument,+ type EmailThreadPageQuery,+ type EmailThreadPageQueryVariables,+} from '@service-storage/graphql/generated/graphql';+import {+ getGraphqlSoupClient,+ graphqlCacheEnabled,+} from '@service-storage/graphql-soup';import {type InfiniteData,useInfiniteQuery,@@ -32,6 +42,7 @@ import { optimisticUpdateSoupEntity, refetchSoupEntity } from '../soup/cache';import { invalidateAllSoup } from '../soup/normalized-cache';import { type UndoHandle, useUndoableMutation } from '../undo';import { type MutationCallbacks, withCallbacks } from '../utils';+import { mapGraphqlEmailThreadPage } from './graphql/mapper';import { createGraphqlEmailThreadQuery } from './graphql/thread';import { emailKeys } from './keys';@@ -79,30 +90,67 @@ function flattenThreadPages(}/**- * Imperatively fetch a thread (for use outside of components).- * Returns cached data if fresh, otherwise fetches from server.- *- * TODO: Most of the time we have the updated_at timestamp of an email before we fetch it.- * Would be nice to accept that as a parameter and only fetch if it's stale.+ * Imperatively fetch a thread through GraphQL and merge it into the normalized+ * cache. Network failures fall back to a complete cached first page.*/export async function fetchAndCacheThread(threadId: string): ReturnType<typeof emailClient.getThread> {- let data: InfiniteData<Thread, number> | undefined;-- const result = await catchToResult(- async () =>- await queryClient.fetchInfiniteQuery(threadQueryOptions(threadId))- );+ if (!ENABLE_GRAPHQL_SOUP()) {+ const result = await catchToResult(() =>+ queryClient.fetchInfiniteQuery(threadQueryOptions(threadId))+ );+ if (result.isErr()) return err(result.error as any);- if (result.isErr()) {- return err(result.error as any);+ const thread = flattenThreadPages(result.value);+ if (!thread) {+ return err([{ code: 'NOT_FOUND', message: 'Email thread not found' }]);+ }+ return ok({ thread });}- data = result.value;+ const result = await catchToResult(async () => {+ const client = getGraphqlSoupClient();+ const variables: EmailThreadPageQueryVariables = {+ threadId,+ offset: 0,+ limit: DEFAULT_THREAD_MESSAGES_LIMIT,+ };+ let queryResult = await client+ .query<EmailThreadPageQuery, EmailThreadPageQueryVariables>(+ EmailThreadPageDocument,+ variables,+ { requestPolicy: 'cache-and-network' }+ )+ .toPromise();++ if (queryResult.error?.networkError && graphqlCacheEnabled()) {+ const cached = await client+ .query<EmailThreadPageQuery, EmailThreadPageQueryVariables>(+ EmailThreadPageDocument,+ variables,+ { requestPolicy: 'cache-only' }+ )+ .toPromise();+ if (cached.data) queryResult = cached;+ }++ if (queryResult.error) {+ throw mapGraphqlThreadError(queryResult.error);+ }++ const thread = queryResult.data?.user.emailThread;+ if (!thread) {+ throw new ThrownResultError([+ { code: 'NOT_FOUND', message: 'Email thread not found' },+ ]);+ }++ return mapGraphqlEmailThreadPage(thread);+ });- const thread = flattenThreadPages(data);- return ok({ thread: thread! });+ if (result.isErr()) return err(result.error as any);+ return ok({ thread: result.value });}/**diff --git a/apps/web/src/lib/queries/soup/backfill.ts b/apps/web/src/lib/queries/soup/backfill.tsindex 3fa6ca766e..c6c573d95e 100644--- a/apps/web/src/lib/queries/soup/backfill.ts+++ b/apps/web/src/lib/queries/soup/backfill.ts@@ -1,6 +1,12 @@import { createTabLeaderSignal } from '@notifications/notification-election';+import type { SoupPage } from '@service-storage/generated/schemas/soupPage';import {- fetchGraphqlSoupBackfill,+ SoupBackfillDocument,+ SoupDocument,+} from '@service-storage/graphql/generated/graphql';+import {+ type FetchGraphqlSoupOptions,+ fetchGraphqlSoup,type GraphqlSoupInitialInput,type GraphqlSoupInput,graphqlCacheEnabled,@@ -16,18 +22,31 @@ import {// Bump when a default backfill input changes so persisted opaque cursors// cannot retain the previous server-side filters.-const BACKFILL_VERSION = 4;+const BACKFILL_VERSION = 5;const PAGE_LIMIT = 250;-// Five threads × twenty messages reaches the DataLoader's 100-message cap.+// Five threads × twenty messages reaches the backend's 100-message cap.const EMAIL_CONTENT_PAGE_LIMIT = 5;const PAGE_DELAY_MS = 2_000;const INITIAL_RETRY_DELAY_MS = 1_000;const MAX_RETRY_DELAY_MS = 30_000;const EXCLUDED_ENTITY_ID = '00000000-0000-0000-0000-000000000000';+type SoupBackfillFetchPage = (+ input: GraphqlSoupInput,+ options?: FetchGraphqlSoupOptions+) => Promise<SoupPage>;++const fetchSoupPage: SoupBackfillFetchPage = (input, options) =>+ fetchGraphqlSoup(SoupDocument, { input }, options);++const fetchEmailContentPage: SoupBackfillFetchPage = (input, options) =>+ fetchGraphqlSoup(SoupBackfillDocument, { input }, options);+export type SoupBackfillParams = {/** Stable checkpoint namespace. Change it when the input changes. */checkpointId: string;+ /** Optional network fetcher; defaults to the standard Soup operation. */+ fetchPage?: SoupBackfillFetchPage;/** Soup input shared by every page. The backfill manages the cursor. */input: GraphqlSoupInitialInput;/** Delay between successful pages. Defaults to two seconds. */@@ -60,6 +79,7 @@ export const CORE_SOUP_BACKFILL_LANE: SoupBackfillParams = {export const EMAIL_SOUP_BACKFILL_LANE: SoupBackfillParams = {// Restart completed legacy newest-message checkpoints with the new shape.checkpointId: 'email-thread-pages',+ fetchPage: fetchEmailContentPage,input: {limit: EMAIL_CONTENT_PAGE_LIMIT,expand: true,@@ -329,8 +349,14 @@ export async function runSoupBackfill(}: { initial: passInput };// Network-only fetching leaves the persisted cursor unchanged on failure,- // so TanStack retries from the same page.- const page = await fetchGraphqlSoupBackfill(input, { signal });+ // so TanStack retries from the same page. Only the email lane overrides the+ // standard content-free Soup operation.+ const fetchPage = params.fetchPage ?? fetchSoupPage;+ const page = await fetchPage(input, {+ signal,+ allowOfflineFallback: false,+ requestPolicy: 'network-only',+ });const completed = page.next_cursor == null;checkpoint = {diff --git a/apps/web/src/lib/service-clients/service-storage/graphql-soup.ts b/apps/web/src/lib/service-clients/service-storage/graphql-soup.tsindex 694c651b2a..db7adcbcff 100644--- a/apps/web/src/lib/service-clients/service-storage/graphql-soup.ts+++ b/apps/web/src/lib/service-clients/service-storage/graphql-soup.ts@@ -18,9 +18,12 @@ import { getOrCreateCacheScope } from '@graphql-cache/scope';import { getMacroApiToken } from '@service-auth/fetch';import type { ApiUserNotification } from '@service-notification/generated/schemas/apiUserNotification';import {+ type AnyVariables,type Client,createClient,+ type DocumentInput,fetchExchange,+ type RequestPolicy,subscriptionExchange,} from '@urql/core';import { print } from 'graphql';@@ -42,16 +45,11 @@ import {type GroupSoupQuery,GroupSoupDocument as GroupSoupQueryDocument,type GroupSoupQueryVariables,- SoupBackfillDocument,- type SoupBackfillQuery,- type SoupBackfillQueryVariables,type SoupInitialInput,type SoupInput,type SoupNotificationFieldsFragment,type SoupPropertyFieldsFragment,type SoupQuery,- SoupDocument as SoupQueryDocument,- type SoupQueryVariables,} from './graphql/generated/graphql';import {createGraphqlSoupWebSocketUrlResolver,@@ -790,6 +788,17 @@ export type FetchGraphqlSoupOptions = {signal?: AbortSignal;/** Defaults to true for normal foreground Soup reads. */allowOfflineFallback?: boolean;+ /** Overrides cache selection for callers such as durable backfills. */+ requestPolicy?: RequestPolicy;+};++type GraphqlSoupPageData = {+ user: {+ soup: {+ items: GraphqlSoupItem[];+ nextCursor: string | null;+ };+ };};/**@@ -797,7 +806,7 @@ export type FetchGraphqlSoupOptions = {* the existing soup pipeline (both the imperative fetch below and the* reactive urql subscriptions).*/-export function mapGraphqlSoupPage(data: SoupQuery): SoupPage {+export function mapGraphqlSoupPage(data: GraphqlSoupPageData): SoupPage {return {items: data.user.soup.items.map(mapGraphqlSoupItem)@@ -832,21 +841,22 @@ export function mapGraphqlGroupedSoupPage(return { items, groups };}-export async function fetchGraphqlSoup(- input: GraphqlSoupInput,+/** Executes any Soup-shaped query and maps its result to the shared page type. */+export async function fetchGraphqlSoup<+ Data extends GraphqlSoupPageData,+ Variables extends AnyVariables,+>(+ document: DocumentInput<Data, Variables>,+ variables: Variables,options: FetchGraphqlSoupOptions = {}): Promise<SoupPage> {const client = getGraphqlSoupClient();const useCache = graphqlCacheEnabled();- const variables: SoupQueryVariables = { input };-- // `cache-and-network` writes responses through the normalized cache;- // `.toPromise()` skips the stale cache emission, so callers keep- // network-fresh semantics. Reactive urql consumers will see the- // stale-then-fresh stream once components migrate.+ const requestPolicy =+ options.requestPolicy ?? (useCache ? 'cache-and-network' : undefined);const result = await client- .query<SoupQuery, SoupQueryVariables>(SoupQueryDocument, variables, {- ...(useCache ? { requestPolicy: 'cache-and-network' as const } : {}),+ .query<Data, Variables>(document, variables, {+ ...(requestPolicy ? { requestPolicy } : {}),...(options.signal ? { fetchOptions: { signal: options.signal } } : {}),}).toPromise();@@ -859,44 +869,17 @@ export async function fetchGraphqlSoup(result.error.networkError) {const cached = await client- .query<SoupQuery, SoupQueryVariables>(SoupQueryDocument, variables, {+ .query<Data, Variables>(document, variables, {requestPolicy: 'cache-only',}).toPromise();- if (cached.data) {- return mapGraphqlSoupPage(cached.data);- }+ if (cached.data) return mapGraphqlSoupPage(cached.data);}throw result.error;}- const data = result.data;- if (!data) {- throw new Error('GraphQL Soup query returned no data');- }-- return mapGraphqlSoupPage(data);-}--/** Fetches one network-only page for normalized-cache backfills. */-export async function fetchGraphqlSoupBackfill(- input: GraphqlSoupInput,- options: Pick<FetchGraphqlSoupOptions, 'signal'> = {}-): Promise<SoupPage> {- const result = await getGraphqlSoupClient()- .query<SoupBackfillQuery, SoupBackfillQueryVariables>(- SoupBackfillDocument,- { input },- {- requestPolicy: 'network-only',- ...(options.signal ? { fetchOptions: { signal: options.signal } } : {}),- }- )- .toPromise();-- if (result.error) throw result.error;if (!result.data) {- throw new Error('GraphQL Soup backfill query returned no data');+ throw new Error('GraphQL Soup query returned no data');}return mapGraphqlSoupPage(result.data);diff --git a/crates/email/src/domain/service/previews.rs b/crates/email/src/domain/service/previews.rsindex 6c3142e872..e300e5c607 100644--- a/crates/email/src/domain/service/previews.rs+++ b/crates/email/src/domain/service/previews.rs@@ -70,7 +70,7 @@ wherelet sort_method = *query.sort_method();- const MIN_PAGE: u32 = 20;+ const MIN_PAGE: u32 = 1;const MAX_PAGE: u32 = 500;let limit = limit.unwrap_or_default().clamp(MIN_PAGE, MAX_PAGE);diff --git a/crates/soup/src/domain/service.rs b/crates/soup/src/domain/service.rsindex d8505326b8..7eed1728cb 100644--- a/crates/soup/src/domain/service.rs+++ b/crates/soup/src/domain/service.rs@@ -805,7 +805,7 @@ where{let entity_filter = req.filters().clone();let req = req.into_ast()?;- let limit = req.limit.clamp(20, 500);+ let limit = req.limit.clamp(1, 500);// CRM-scoped visibility (team-wide email scope or hidden CRM// companies) requires a team receipt. Without this check the CRMdiff --git a/crates/soup/src/domain/service/tests.rs b/crates/soup/src/domain/service/tests.rsindex 55229a5804..981154b218 100644--- a/crates/soup/src/domain/service/tests.rs+++ b/crates/soup/src/domain/service/tests.rs@@ -1090,7 +1090,7 @@ async fn it_should_not_query_frecency() {&& assert_matches!(a,SimpleSortRequest {- limit: 20,+ limit: 1,user_id,cursor: SimpleSortQuery::NoFilter(Query::Sort(SimpleSortMethod::ViewedUpdated, ())),} => {@@ -1145,7 +1145,7 @@ async fn it_should_not_query_frecency() {dbg!(&res);- assert_eq!(res.items.len(), 10)+ assert_eq!(res.items.len(), 1)}#[tokio::test]@@ -2001,7 +2001,7 @@ async fn cursor_should_return_simple_sort() {&& assert_matches!(a,SimpleSortRequest {- limit: 20,+ limit: 1,user_id,cursor: SimpleSortQuery::NoFilter(Query::Sort(SimpleSortMethod::ViewedUpdated, ())),} => {@@ -2052,8 +2052,8 @@ async fn cursor_should_return_simple_sort() {let simple_cursor = res.unwrap_left();let cursor_decoded: CursorWithValAndFilter<String, SimpleSortMethod, EntityFilters> =simple_cursor.next_cursor.unwrap().decode_json().unwrap();- assert_matches!(cursor_decoded, CursorWithValAndFilter { id, limit: 20, val: CursorVal { sort_type: SimpleSortMethod::ViewedUpdated, last_val }, filter: _ } => {- let expected_uuid_str = Uuid::from_u128(19).to_string(); // "my-document-19" -> 19+ assert_matches!(cursor_decoded, CursorWithValAndFilter { id, limit: 1, val: CursorVal { sort_type: SimpleSortMethod::ViewedUpdated, last_val }, filter: _ } => {+ let expected_uuid_str = Uuid::from_u128(0).to_string(); // "my-document-0" -> 0assert_eq!(id, expected_uuid_str);let date: DateTime<Utc> = Default::default();assert_eq!(last_val, date);@@ -2337,7 +2337,7 @@ async fn it_should_preserve_is_completed_for_mixed_items() {),link_ids: vec![Uuid::new_v4()],soup_type: SoupType::UnExpanded,- limit: 0,+ limit: 3,cursor: SoupQuery::new_sort_simple(SimpleSortMethod::ViewedUpdated,EntityFilters::default(),@@ -2361,10 +2361,10 @@ async fn it_should_preserve_is_completed_in_by_ids_queries() {let mut frecency = MockFrecencyQueryService::new();frecency.expect_get_frecency_page()- .withf(|params| assert_matches!(params, FrecencyPageRequest { limit: 20, .. } => true))+ .withf(|params| assert_matches!(params, FrecencyPageRequest { limit: 3, .. } => true)).times(1).returning(|params| {- // Return 20 items to match the limit and avoid fallback+ // Return 3 items to match the requested limit and avoid fallbacklet iter = (1..=params.limit).map(|v| {AggregateFrecency::new_mock(EntityType::Document@@ -2432,8 +2432,8 @@ async fn it_should_preserve_is_completed_in_by_ids_queries() {.unwrap().unwrap_right();- // Should have 20 items, verify is_completed values are preserved- assert_eq!(res.items.len(), 20);+ // Should have 3 items, verify is_completed values are preserved+ assert_eq!(res.items.len(), 3);let is_completed_values: Vec<Option<bool>> = res.items.iter().map(get_is_completed).collect();// Verify that all three is_completed values (true, false, None) are presentassert!(