Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(email): backfill not working and block not loading from cache data (#5592)

dev-rb0f429369b3a9
diff --git a/apps/web/src/features/block-email/component/Block.tsx b/apps/web/src/features/block-email/component/Block.tsx
index 127b00d485..936e6d1636 100644
--- a/apps/web/src/features/block-email/component/Block.tsx
+++ b/apps/web/src/features/block-email/component/Block.tsx
@@ -1,6 +1,7 @@
import { useBlockEntityCommands } from '@app/features/next-soup/actions';
import { useGlobalNotificationSource } from '@components/app/GlobalAppState';
import { useSplitPanel } from '@components/app/split-layout/layoutUtils';
+import { useBlockId } from '@core/block';
import { DocumentBlockContainer } from '@core/component/DocumentBlockContainer';
import { buildEntityData } from '@entity';
import { EmailDebouncedReadMarker } from '@notifications';
@@ -12,6 +13,7 @@ import { EmailView } from './Email';
export default function BlockEmail() {
const blockData = blockDataSignal.get;
+ const blockId = useBlockId();
// Email threads are absent from quick access, so the entity the block-level
// commands act on has to come from here. Built off the block's loaded data
@@ -35,7 +37,7 @@ export default function BlockEmail() {
// marking it seen so scanning/previewing doesn't clear unread state.
const isPreview = !!useSplitPanel()?.handle.isViewerSplit();
- const threadId = createMemo(() => blockData()?.thread?.db_id ?? '');
+ const threadId = () => blockId;
const threadQuery = useThreadQuery(threadId, () => ({
enabled: !!threadId(),
diff --git a/apps/web/src/lib/queries/email/thread.ts b/apps/web/src/lib/queries/email/thread.ts
index ca9727300c..91d259bafa 100644
--- a/apps/web/src/lib/queries/email/thread.ts
+++ b/apps/web/src/lib/queries/email/thread.ts
@@ -2,6 +2,7 @@ import { useAnalytics } from '@app/lib/analytics/analytics-context';
import { useFeatureFlag } from '@app/lib/analytics/posthog';
import { toast } from '@core/component/Toast/Toast';
import {
+ ENABLE_GRAPHQL_SOUP,
ENABLE_GRAPHQL_SOUP_FLAG,
ENABLE_GRAPHQL_SOUP_OVERRIDE,
} from '@core/constant/featureFlags';
@@ -19,6 +20,15 @@ import type {
ApiThread as Thread,
UpsertScheduledResponse,
} from '@service-email/generated/schemas';
+import {
+ EmailThreadPageDocument,
+ type EmailThreadPageQuery,
+ type EmailThreadPageQueryVariables,
+} from '@service-storage/graphql/generated/graphql';
+import {
+ getGraphqlSoupClient,
+ graphqlCacheEnabled,
+} from '@service-storage/graphql-soup';
import {
type InfiniteData,
useInfiniteQuery,
@@ -32,6 +42,7 @@ import { optimisticUpdateSoupEntity, refetchSoupEntity } from '../soup/cache';
import { invalidateAllSoup } from '../soup/normalized-cache';
import { type UndoHandle, useUndoableMutation } from '../undo';
import { type MutationCallbacks, withCallbacks } from '../utils';
+import { mapGraphqlEmailThreadPage } from './graphql/mapper';
import { createGraphqlEmailThreadQuery } from './graphql/thread';
import { emailKeys } from './keys';
@@ -79,30 +90,67 @@ function flattenThreadPages(
}
/**
- * Imperatively fetch a thread (for use outside of components).
- * Returns cached data if fresh, otherwise fetches from server.
- *
- * TODO: Most of the time we have the updated_at timestamp of an email before we fetch it.
- * Would be nice to accept that as a parameter and only fetch if it's stale.
+ * Imperatively fetch a thread through GraphQL and merge it into the normalized
+ * cache. Network failures fall back to a complete cached first page.
*/
export async function fetchAndCacheThread(
threadId: string
): ReturnType<typeof emailClient.getThread> {
- let data: InfiniteData<Thread, number> | undefined;
-
- const result = await catchToResult(
- async () =>
- await queryClient.fetchInfiniteQuery(threadQueryOptions(threadId))
- );
+ if (!ENABLE_GRAPHQL_SOUP()) {
+ const result = await catchToResult(() =>
+ queryClient.fetchInfiniteQuery(threadQueryOptions(threadId))
+ );
+ if (result.isErr()) return err(result.error as any);
- if (result.isErr()) {
- return err(result.error as any);
+ const thread = flattenThreadPages(result.value);
+ if (!thread) {
+ return err([{ code: 'NOT_FOUND', message: 'Email thread not found' }]);
+ }
+ return ok({ thread });
}
- data = result.value;
+ const result = await catchToResult(async () => {
+ const client = getGraphqlSoupClient();
+ const variables: EmailThreadPageQueryVariables = {
+ threadId,
+ offset: 0,
+ limit: DEFAULT_THREAD_MESSAGES_LIMIT,
+ };
+ let queryResult = await client
+ .query<EmailThreadPageQuery, EmailThreadPageQueryVariables>(
+ EmailThreadPageDocument,
+ variables,
+ { requestPolicy: 'cache-and-network' }
+ )
+ .toPromise();
+
+ if (queryResult.error?.networkError && graphqlCacheEnabled()) {
+ const cached = await client
+ .query<EmailThreadPageQuery, EmailThreadPageQueryVariables>(
+ EmailThreadPageDocument,
+ variables,
+ { requestPolicy: 'cache-only' }
+ )
+ .toPromise();
+ if (cached.data) queryResult = cached;
+ }
+
+ if (queryResult.error) {
+ throw mapGraphqlThreadError(queryResult.error);
+ }
+
+ const thread = queryResult.data?.user.emailThread;
+ if (!thread) {
+ throw new ThrownResultError([
+ { code: 'NOT_FOUND', message: 'Email thread not found' },
+ ]);
+ }
+
+ return mapGraphqlEmailThreadPage(thread);
+ });
- const thread = flattenThreadPages(data);
- return ok({ thread: thread! });
+ if (result.isErr()) return err(result.error as any);
+ return ok({ thread: result.value });
}
/**
diff --git a/apps/web/src/lib/queries/soup/backfill.ts b/apps/web/src/lib/queries/soup/backfill.ts
index 3fa6ca766e..c6c573d95e 100644
--- a/apps/web/src/lib/queries/soup/backfill.ts
+++ b/apps/web/src/lib/queries/soup/backfill.ts
@@ -1,6 +1,12 @@
import { createTabLeaderSignal } from '@notifications/notification-election';
+import type { SoupPage } from '@service-storage/generated/schemas/soupPage';
import {
- fetchGraphqlSoupBackfill,
+ SoupBackfillDocument,
+ SoupDocument,
+} from '@service-storage/graphql/generated/graphql';
+import {
+ type FetchGraphqlSoupOptions,
+ fetchGraphqlSoup,
type GraphqlSoupInitialInput,
type GraphqlSoupInput,
graphqlCacheEnabled,
@@ -16,18 +22,31 @@ import {
// Bump when a default backfill input changes so persisted opaque cursors
// cannot retain the previous server-side filters.
-const BACKFILL_VERSION = 4;
+const BACKFILL_VERSION = 5;
const PAGE_LIMIT = 250;
-// Five threads × twenty messages reaches the DataLoader's 100-message cap.
+// Five threads × twenty messages reaches the backend's 100-message cap.
const EMAIL_CONTENT_PAGE_LIMIT = 5;
const PAGE_DELAY_MS = 2_000;
const INITIAL_RETRY_DELAY_MS = 1_000;
const MAX_RETRY_DELAY_MS = 30_000;
const EXCLUDED_ENTITY_ID = '00000000-0000-0000-0000-000000000000';
+type SoupBackfillFetchPage = (
+ input: GraphqlSoupInput,
+ options?: FetchGraphqlSoupOptions
+) => Promise<SoupPage>;
+
+const fetchSoupPage: SoupBackfillFetchPage = (input, options) =>
+ fetchGraphqlSoup(SoupDocument, { input }, options);
+
+const fetchEmailContentPage: SoupBackfillFetchPage = (input, options) =>
+ fetchGraphqlSoup(SoupBackfillDocument, { input }, options);
+
export type SoupBackfillParams = {
/** Stable checkpoint namespace. Change it when the input changes. */
checkpointId: string;
+ /** Optional network fetcher; defaults to the standard Soup operation. */
+ fetchPage?: SoupBackfillFetchPage;
/** Soup input shared by every page. The backfill manages the cursor. */
input: GraphqlSoupInitialInput;
/** Delay between successful pages. Defaults to two seconds. */
@@ -60,6 +79,7 @@ export const CORE_SOUP_BACKFILL_LANE: SoupBackfillParams = {
export const EMAIL_SOUP_BACKFILL_LANE: SoupBackfillParams = {
// Restart completed legacy newest-message checkpoints with the new shape.
checkpointId: 'email-thread-pages',
+ fetchPage: fetchEmailContentPage,
input: {
limit: EMAIL_CONTENT_PAGE_LIMIT,
expand: true,
@@ -329,8 +349,14 @@ export async function runSoupBackfill(
}
: { initial: passInput };
// Network-only fetching leaves the persisted cursor unchanged on failure,
- // so TanStack retries from the same page.
- const page = await fetchGraphqlSoupBackfill(input, { signal });
+ // so TanStack retries from the same page. Only the email lane overrides the
+ // standard content-free Soup operation.
+ const fetchPage = params.fetchPage ?? fetchSoupPage;
+ const page = await fetchPage(input, {
+ signal,
+ allowOfflineFallback: false,
+ requestPolicy: 'network-only',
+ });
const completed = page.next_cursor == null;
checkpoint = {
diff --git a/apps/web/src/lib/service-clients/service-storage/graphql-soup.ts b/apps/web/src/lib/service-clients/service-storage/graphql-soup.ts
index 694c651b2a..db7adcbcff 100644
--- a/apps/web/src/lib/service-clients/service-storage/graphql-soup.ts
+++ b/apps/web/src/lib/service-clients/service-storage/graphql-soup.ts
@@ -18,9 +18,12 @@ import { getOrCreateCacheScope } from '@graphql-cache/scope';
import { getMacroApiToken } from '@service-auth/fetch';
import type { ApiUserNotification } from '@service-notification/generated/schemas/apiUserNotification';
import {
+ type AnyVariables,
type Client,
createClient,
+ type DocumentInput,
fetchExchange,
+ type RequestPolicy,
subscriptionExchange,
} from '@urql/core';
import { print } from 'graphql';
@@ -42,16 +45,11 @@ import {
type GroupSoupQuery,
GroupSoupDocument as GroupSoupQueryDocument,
type GroupSoupQueryVariables,
- SoupBackfillDocument,
- type SoupBackfillQuery,
- type SoupBackfillQueryVariables,
type SoupInitialInput,
type SoupInput,
type SoupNotificationFieldsFragment,
type SoupPropertyFieldsFragment,
type SoupQuery,
- SoupDocument as SoupQueryDocument,
- type SoupQueryVariables,
} from './graphql/generated/graphql';
import {
createGraphqlSoupWebSocketUrlResolver,
@@ -790,6 +788,17 @@ export type FetchGraphqlSoupOptions = {
signal?: AbortSignal;
/** Defaults to true for normal foreground Soup reads. */
allowOfflineFallback?: boolean;
+ /** Overrides cache selection for callers such as durable backfills. */
+ requestPolicy?: RequestPolicy;
+};
+
+type GraphqlSoupPageData = {
+ user: {
+ soup: {
+ items: GraphqlSoupItem[];
+ nextCursor: string | null;
+ };
+ };
};
/**
@@ -797,7 +806,7 @@ export type FetchGraphqlSoupOptions = {
* the existing soup pipeline (both the imperative fetch below and the
* reactive urql subscriptions).
*/
-export function mapGraphqlSoupPage(data: SoupQuery): SoupPage {
+export function mapGraphqlSoupPage(data: GraphqlSoupPageData): SoupPage {
return {
items: data.user.soup.items
.map(mapGraphqlSoupItem)
@@ -832,21 +841,22 @@ export function mapGraphqlGroupedSoupPage(
return { items, groups };
}
-export async function fetchGraphqlSoup(
- input: GraphqlSoupInput,
+/** Executes any Soup-shaped query and maps its result to the shared page type. */
+export async function fetchGraphqlSoup<
+ Data extends GraphqlSoupPageData,
+ Variables extends AnyVariables,
+>(
+ document: DocumentInput<Data, Variables>,
+ variables: Variables,
options: FetchGraphqlSoupOptions = {}
): Promise<SoupPage> {
const client = getGraphqlSoupClient();
const useCache = graphqlCacheEnabled();
- const variables: SoupQueryVariables = { input };
-
- // `cache-and-network` writes responses through the normalized cache;
- // `.toPromise()` skips the stale cache emission, so callers keep
- // network-fresh semantics. Reactive urql consumers will see the
- // stale-then-fresh stream once components migrate.
+ const requestPolicy =
+ options.requestPolicy ?? (useCache ? 'cache-and-network' : undefined);
const result = await client
- .query<SoupQuery, SoupQueryVariables>(SoupQueryDocument, variables, {
- ...(useCache ? { requestPolicy: 'cache-and-network' as const } : {}),
+ .query<Data, Variables>(document, variables, {
+ ...(requestPolicy ? { requestPolicy } : {}),
...(options.signal ? { fetchOptions: { signal: options.signal } } : {}),
})
.toPromise();
@@ -859,44 +869,17 @@ export async function fetchGraphqlSoup(
result.error.networkError
) {
const cached = await client
- .query<SoupQuery, SoupQueryVariables>(SoupQueryDocument, variables, {
+ .query<Data, Variables>(document, variables, {
requestPolicy: 'cache-only',
})
.toPromise();
- if (cached.data) {
- return mapGraphqlSoupPage(cached.data);
- }
+ if (cached.data) return mapGraphqlSoupPage(cached.data);
}
throw result.error;
}
- const data = result.data;
- if (!data) {
- throw new Error('GraphQL Soup query returned no data');
- }
-
- return mapGraphqlSoupPage(data);
-}
-
-/** Fetches one network-only page for normalized-cache backfills. */
-export async function fetchGraphqlSoupBackfill(
- input: GraphqlSoupInput,
- options: Pick<FetchGraphqlSoupOptions, 'signal'> = {}
-): Promise<SoupPage> {
- const result = await getGraphqlSoupClient()
- .query<SoupBackfillQuery, SoupBackfillQueryVariables>(
- SoupBackfillDocument,
- { input },
- {
- requestPolicy: 'network-only',
- ...(options.signal ? { fetchOptions: { signal: options.signal } } : {}),
- }
- )
- .toPromise();
-
- if (result.error) throw result.error;
if (!result.data) {
- throw new Error('GraphQL Soup backfill query returned no data');
+ throw new Error('GraphQL Soup query returned no data');
}
return mapGraphqlSoupPage(result.data);
diff --git a/crates/email/src/domain/service/previews.rs b/crates/email/src/domain/service/previews.rs
index 6c3142e872..e300e5c607 100644
--- a/crates/email/src/domain/service/previews.rs
+++ b/crates/email/src/domain/service/previews.rs
@@ -70,7 +70,7 @@ where
let sort_method = *query.sort_method();
- const MIN_PAGE: u32 = 20;
+ const MIN_PAGE: u32 = 1;
const MAX_PAGE: u32 = 500;
let limit = limit.unwrap_or_default().clamp(MIN_PAGE, MAX_PAGE);
diff --git a/crates/soup/src/domain/service.rs b/crates/soup/src/domain/service.rs
index d8505326b8..7eed1728cb 100644
--- a/crates/soup/src/domain/service.rs
+++ b/crates/soup/src/domain/service.rs
@@ -805,7 +805,7 @@ where
{
let entity_filter = req.filters().clone();
let req = req.into_ast()?;
- let limit = req.limit.clamp(20, 500);
+ let limit = req.limit.clamp(1, 500);
// CRM-scoped visibility (team-wide email scope or hidden CRM
// companies) requires a team receipt. Without this check the CRM
diff --git a/crates/soup/src/domain/service/tests.rs b/crates/soup/src/domain/service/tests.rs
index 55229a5804..981154b218 100644
--- a/crates/soup/src/domain/service/tests.rs
+++ b/crates/soup/src/domain/service/tests.rs
@@ -1090,7 +1090,7 @@ async fn it_should_not_query_frecency() {
&& assert_matches!(
a,
SimpleSortRequest {
- limit: 20,
+ limit: 1,
user_id,
cursor: SimpleSortQuery::NoFilter(Query::Sort(SimpleSortMethod::ViewedUpdated, ())),
} => {
@@ -1145,7 +1145,7 @@ async fn it_should_not_query_frecency() {
dbg!(&res);
- assert_eq!(res.items.len(), 10)
+ assert_eq!(res.items.len(), 1)
}
#[tokio::test]
@@ -2001,7 +2001,7 @@ async fn cursor_should_return_simple_sort() {
&& assert_matches!(
a,
SimpleSortRequest {
- limit: 20,
+ limit: 1,
user_id,
cursor: SimpleSortQuery::NoFilter(Query::Sort(SimpleSortMethod::ViewedUpdated, ())),
} => {
@@ -2052,8 +2052,8 @@ async fn cursor_should_return_simple_sort() {
let simple_cursor = res.unwrap_left();
let cursor_decoded: CursorWithValAndFilter<String, SimpleSortMethod, EntityFilters> =
simple_cursor.next_cursor.unwrap().decode_json().unwrap();
- assert_matches!(cursor_decoded, CursorWithValAndFilter { id, limit: 20, val: CursorVal { sort_type: SimpleSortMethod::ViewedUpdated, last_val }, filter: _ } => {
- let expected_uuid_str = Uuid::from_u128(19).to_string(); // "my-document-19" -> 19
+ assert_matches!(cursor_decoded, CursorWithValAndFilter { id, limit: 1, val: CursorVal { sort_type: SimpleSortMethod::ViewedUpdated, last_val }, filter: _ } => {
+ let expected_uuid_str = Uuid::from_u128(0).to_string(); // "my-document-0" -> 0
assert_eq!(id, expected_uuid_str);
let date: DateTime<Utc> = Default::default();
assert_eq!(last_val, date);
@@ -2337,7 +2337,7 @@ async fn it_should_preserve_is_completed_for_mixed_items() {
),
link_ids: vec![Uuid::new_v4()],
soup_type: SoupType::UnExpanded,
- limit: 0,
+ limit: 3,
cursor: SoupQuery::new_sort_simple(
SimpleSortMethod::ViewedUpdated,
EntityFilters::default(),
@@ -2361,10 +2361,10 @@ async fn it_should_preserve_is_completed_in_by_ids_queries() {
let mut frecency = MockFrecencyQueryService::new();
frecency
.expect_get_frecency_page()
- .withf(|params| assert_matches!(params, FrecencyPageRequest { limit: 20, .. } => true))
+ .withf(|params| assert_matches!(params, FrecencyPageRequest { limit: 3, .. } => true))
.times(1)
.returning(|params| {
- // Return 20 items to match the limit and avoid fallback
+ // Return 3 items to match the requested limit and avoid fallback
let iter = (1..=params.limit).map(|v| {
AggregateFrecency::new_mock(
EntityType::Document
@@ -2432,8 +2432,8 @@ async fn it_should_preserve_is_completed_in_by_ids_queries() {
.unwrap()
.unwrap_right();
- // Should have 20 items, verify is_completed values are preserved
- assert_eq!(res.items.len(), 20);
+ // Should have 3 items, verify is_completed values are preserved
+ assert_eq!(res.items.len(), 3);
let is_completed_values: Vec<Option<bool>> = res.items.iter().map(get_is_completed).collect();
// Verify that all three is_completed values (true, false, None) are present
assert!(