Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

Seanaye/fix/long email queries (#6695)

Sean Ayedd0455f5fca6
diff --git a/Cargo.lock b/Cargo.lock
index 6c1515ac08..92b59a0fc5 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3118,7 +3118,7 @@ dependencies = [
[[package]]
name = "cache-wasm"
-version = "0.6.16"
+version = "0.6.17"
dependencies = [
"async-lock",
"cache-core",
diff --git a/apps/web/src/features/email-view/queries/use-email-query.test.ts b/apps/web/src/features/email-view/queries/use-email-query.test.ts
new file mode 100644
index 0000000000..9e919214b9
--- /dev/null
+++ b/apps/web/src/features/email-view/queries/use-email-query.test.ts
@@ -0,0 +1,245 @@
+import type { EmailEntity } from '@entity/types/entity';
+import {
+ type SoupAstItemsQuery,
+ useSoupAstItemsQuery,
+} from '@queries/soup/items';
+import type { useSearchSoupQuery } from '@queries/soup/search';
+import { batch, createRoot, createSignal } from 'solid-js';
+import { createStore } from 'solid-js/store';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import type { EmailTab, EmailViewState } from '../types';
+import { type EmailDataSource, useEmailDataSource } from './use-email-query';
+
+const searchQueryMock = vi.hoisted(() => vi.fn());
+
+vi.mock('@app/features/soup', async () => ({
+ ...(await import('@app/features/soup/filters')),
+ ...(await import('@app/features/soup/collection/rows')),
+ ...(await import('@app/features/soup/search/create-search-state')),
+}));
+vi.mock('@queries/soup/search', async (importOriginal) => ({
+ ...(await importOriginal<typeof import('@queries/soup/search')>()),
+ useSearchSoupQuery: searchQueryMock,
+}));
+vi.mock('@components/app/GlobalAppState', () => ({
+ useGlobalNotificationSource: () => ({ notificationsByEntity: () => ({}) }),
+}));
+vi.mock('@core/context/user', () => ({ useUserId: () => () => 'alice' }));
+vi.mock('@queries/soup/items', () => ({ useSoupAstItemsQuery: vi.fn() }));
+vi.mock('@service-storage/websocket', () => ({
+ storageWS: { reconnectIfDisconnected: vi.fn() },
+ createWebSocketJob: vi.fn(),
+}));
+vi.mock('@service-connection/websocket', () => ({
+ ws: { addEventListener: vi.fn(), send: vi.fn() },
+ state: () => 'closed',
+ createConnectionBlockWebsocketEffect: vi.fn(),
+ createConnectionWebsocketEffect: vi.fn(),
+}));
+
+function email(id: string): EmailEntity {
+ return {
+ type: 'email',
+ id,
+ name: id,
+ ownerId: 'alice',
+ linkId: 'inbox-a',
+ isRead: false,
+ isDraft: false,
+ isImportant: true,
+ done: false,
+ updatedAt: new Date('2026-09-21T12:00:00Z'),
+ };
+}
+
+const ids = (source: EmailDataSource) =>
+ source
+ .items()
+ .flatMap((row) => (row.kind === 'entity' ? [row.entity.id] : []));
+
+let dispose: (() => void) | undefined;
+function mount(search = '') {
+ return createRoot((cleanup) => {
+ dispose = cleanup;
+ const [state, setState] = createStore<EmailViewState>({
+ tab: 'noise',
+ search,
+ inboxIds: ['inbox-a'],
+ facets: {},
+ collapsedSidebarSectionIds: [],
+ });
+ const [entities, setEntities] = createSignal([email('noise')]);
+ const [loading, setLoading] = createSignal(false);
+ const [placeholder, setPlaceholder] = createSignal(false);
+ const [fetching, setFetching] = createSignal(false);
+ const [tagSetsReady, setTagSetsReady] = createSignal(true);
+ const [searchEntities, setSearchEntities] = createSignal([email('search')]);
+ searchQueryMock.mockImplementation(
+ (
+ _args: Parameters<typeof useSearchSoupQuery>[0],
+ options: Parameters<typeof useSearchSoupQuery>[1]
+ ) => ({
+ // A disabled search keeps its previous data, just like placeholderData.
+ get data() {
+ return searchEntities();
+ },
+ get isEnabled() {
+ return options?.().enabled ?? true;
+ },
+ isFetching: false,
+ isFetchingNextPage: false,
+ hasNextPage: true,
+ error: null,
+ })
+ );
+ const query: SoupAstItemsQuery = {
+ get data() {
+ if (loading()) throw new Error('Read pending query data');
+ return { entities: entities(), groups: undefined };
+ },
+ get isLoading() {
+ return loading();
+ },
+ get isPlaceholderData() {
+ return placeholder();
+ },
+ get isFetching() {
+ return fetching();
+ },
+ error: null,
+ hasNextPage: true,
+ isFetchingNextPage: false,
+ isEnabled: true,
+ transport: 'graphql',
+ fetchNextPage: vi.fn(async () => {}),
+ refetch: vi.fn(async () => {}),
+ refresh: vi.fn(async () => {}),
+ resetToInitialPage: vi.fn(),
+ };
+ vi.mocked(useSoupAstItemsQuery).mockReturnValue(query);
+ const source = useEmailDataSource(state, {
+ tagSets: () => [],
+ tagSetsReady,
+ });
+ return {
+ source,
+ setState,
+ setEntities,
+ setLoading,
+ setPlaceholder,
+ setFetching,
+ setTagSetsReady,
+ searchEntities,
+ setSearchEntities,
+ };
+ });
+}
+
+describe('Email list query transitions', () => {
+ beforeEach(() => vi.clearAllMocks());
+ afterEach(() => dispose?.());
+
+ it('does not show Noise rows under other tabs while their cache reads are pending', () => {
+ const { source, setState, setEntities, setLoading } = mount();
+ expect(ids(source)).toEqual(['noise']);
+
+ for (const tab of ['important', 'sent', 'all'] satisfies EmailTab[]) {
+ batch(() => {
+ setState('tab', tab);
+ setLoading(true);
+ });
+ expect(ids(source)).toEqual([]);
+ expect(source.isLoading()).toBe(true);
+ expect(source.hasMore()).toBe(false);
+ expect(source.items()).toEqual([]);
+ }
+
+ batch(() => {
+ setEntities([email('all')]);
+ setLoading(false);
+ });
+ expect(ids(source)).toEqual(['all']);
+ expect(source.isLoading()).toBe(false);
+ expect(source.hasMore()).toBe(true);
+ });
+
+ it('does not keep rows from the previous inbox while the new inbox loads', () => {
+ const { source, setState, setLoading, setEntities } = mount();
+ expect(ids(source)).toEqual(['noise']);
+ batch(() => {
+ setState('inboxIds', ['inbox-b']);
+ setLoading(true);
+ });
+ expect(ids(source)).toEqual([]);
+ expect(source.isLoading()).toBe(true);
+ batch(() => {
+ setEntities([]);
+ setLoading(false);
+ });
+ expect(ids(source)).toEqual([]);
+ expect(source.isLoading()).toBe(false);
+ });
+
+ it('treats REST previous-query placeholders as loading, not current results', () => {
+ const { source, setState, setPlaceholder } = mount();
+ expect(ids(source)).toEqual(['noise']);
+ batch(() => {
+ setState('tab', 'sent');
+ setPlaceholder(true);
+ });
+ expect(ids(source)).toEqual([]);
+ expect(source.isLoading()).toBe(true);
+ expect(source.hasMore()).toBe(false);
+ });
+
+ it('keeps current-query cached results visible during a background refresh', () => {
+ const { source, setFetching } = mount();
+ setFetching(true);
+ expect(ids(source)).toEqual(['noise']);
+ expect(source.isLoading()).toBe(false);
+ expect(source.isFetching()).toBe(true);
+ });
+
+ it('clears retained active-search rows and reports loading while tag sets are pending', () => {
+ const {
+ source,
+ setState,
+ setTagSetsReady,
+ searchEntities,
+ setSearchEntities,
+ } = mount('invoice');
+ expect(ids(source)).toEqual(['search']);
+ expect(source.isLoading()).toBe(false);
+ expect(source.hasMore()).toBe(true);
+
+ batch(() => {
+ setState('facets', { tags: ['pending-tag'] });
+ setTagSetsReady(false);
+ });
+ // The shared search still holds the previous service result.
+ expect(searchEntities().map((entity) => entity.id)).toEqual(['search']);
+ expect(ids(source)).toEqual([]);
+ expect(source.items()).toEqual([]);
+ expect(source.isLoading()).toBe(true);
+ expect(source.hasMore()).toBe(false);
+
+ batch(() => {
+ setSearchEntities([email('refined-search')]);
+ setTagSetsReady(true);
+ });
+ expect(ids(source)).toEqual(['refined-search']);
+ expect(source.isLoading()).toBe(false);
+ });
+
+ it('does not retain unfiltered rows while a tag selection waits for tag sets', () => {
+ const { source, setState, setTagSetsReady } = mount();
+ expect(ids(source)).toEqual(['noise']);
+ batch(() => {
+ setState('facets', { tags: ['pending-tag'] });
+ setTagSetsReady(false);
+ });
+ expect(ids(source)).toEqual([]);
+ expect(source.isLoading()).toBe(true);
+ expect(source.hasMore()).toBe(false);
+ });
+});
diff --git a/apps/web/src/features/email-view/queries/use-email-query.ts b/apps/web/src/features/email-view/queries/use-email-query.ts
index 7b540ff46a..caa4977169 100644
--- a/apps/web/src/features/email-view/queries/use-email-query.ts
+++ b/apps/web/src/features/email-view/queries/use-email-query.ts
@@ -97,6 +97,8 @@ export function useEmailDataSource(
const query = useSoupAstItemsQuery(queryArgs, () => ({
enabled: facetsReady(),
}));
+ const isListPending = () =>
+ !facetsReady() || query.isLoading || query.isPlaceholderData;
const selectEmails = (entities: EntityData[]): EmailEntity[] => {
const context = queryContext();
@@ -122,8 +124,14 @@ export function useEmailDataSource(
});
const rawEntities = createMemo<EntityData[]>((previous) => {
+ // Disabled searches can retain placeholder data for the previous facets.
+ if (!facetsReady()) return [];
+
if (!search.isSearching()) {
- if (query.isLoading) return previous;
+ // Previous-tab/inbox rows are not valid results for the new query.
+ // REST placeholders need the same treatment as pending GraphQL reads.
+ // Background refreshes still retain usable current-query cache results.
+ if (isListPending()) return [];
return query.data?.entities ?? [];
}
@@ -200,7 +208,7 @@ export function useEmailDataSource(
const hasMore = () => {
if (usesServiceSearch()) return search.hasNextPage();
- return query.hasNextPage;
+ return !isListPending() && query.hasNextPage;
};
const isLoadingMore = () => {
@@ -228,9 +236,10 @@ export function useEmailDataSource(
});
const isLoading = () => {
+ if (!facetsReady()) return true;
if (!search.isSearching()) {
// A query held back for the tag sets is loading, not empty.
- return (query.isLoading || !facetsReady()) && rawEntities().length === 0;
+ return isListPending();
}
if (entities().items.length > 0) return false;
if (usesServiceSearch()) return search.isLoading();
diff --git a/crates/client/cache-turso/src/storage.rs b/crates/client/cache-turso/src/storage.rs
index 7514977c70..77b47520fa 100644
--- a/crates/client/cache-turso/src/storage.rs
+++ b/crates/client/cache-turso/src/storage.rs
@@ -2546,10 +2546,12 @@ fn compile_predicate_selection(
compiler
.parameters
.push(text(descriptor.sort_attribute.as_str()));
- // Look up identity and sort facts by their real primary keys. Joining three
- // materialized sets loses those indexes and makes even small queries quadratic.
+ // Drive the join from matching IDs and point-probe the sort-fact primary key.
+ // Without both constraints Turso prefers the (attribute, value, document_id)
+ // lookup index using only attribute, rescanning all sort facts for each match.
+ // The composite primary keys are part of the validated storage schema.
compiler.ctes.push(format!(
- "{hits}(record_key, value) AS MATERIALIZED (SELECT d.record_key, s.value FROM {matches} AS m JOIN index_documents AS d ON m.source = 0 AND d.id = m.document_id JOIN sort_facts AS s ON s.document_id = m.document_id AND s.attribute = ? UNION ALL SELECT d.record_key, s.value FROM {matches} AS m JOIN optimistic_index_documents AS d ON m.source = 1 AND d.id = m.document_id JOIN optimistic_sort_facts AS s ON s.document_id = m.document_id AND s.attribute = ?)"
+ "{hits}(record_key, value) AS MATERIALIZED (SELECT d.record_key, s.value FROM {matches} AS m CROSS JOIN index_documents AS d ON m.source = 0 AND d.id = m.document_id CROSS JOIN sort_facts AS s INDEXED BY sqlite_autoindex_sort_facts_1 ON s.document_id = m.document_id AND s.attribute = ? UNION ALL SELECT d.record_key, s.value FROM {matches} AS m CROSS JOIN optimistic_index_documents AS d ON m.source = 1 AND d.id = m.document_id CROSS JOIN optimistic_sort_facts AS s INDEXED BY sqlite_autoindex_optimistic_sort_facts_1 ON s.document_id = m.document_id AND s.attribute = ?)"
));
compiler
.parameters
diff --git a/crates/client/cache-turso/src/storage/test.rs b/crates/client/cache-turso/src/storage/test.rs
index dbea75a792..82b602d924 100644
--- a/crates/client/cache-turso/src/storage/test.rs
+++ b/crates/client/cache-turso/src/storage/test.rs
@@ -2225,14 +2225,23 @@ fn predicate_query_plan_uses_fact_indexes_and_never_scans_record_blobs() {
for index in [
"optimistic_exact_facts_lookup_idx",
"optimistic_integer_facts_lookup_idx",
- "sort_facts_lookup_idx",
- "optimistic_sort_facts_lookup_idx",
] {
assert!(
details.iter().any(|detail| detail.contains(index)),
"missing {index}: {details:#?}"
);
}
+ for index in [
+ "sqlite_autoindex_sort_facts_1",
+ "sqlite_autoindex_optimistic_sort_facts_1",
+ ] {
+ assert!(
+ details.iter().any(|detail| {
+ detail.contains(index) && detail.contains("(document_id=? AND attribute=?)")
+ }),
+ "missing composite-key sort lookup {index}: {details:#?}"
+ );
+ }
assert!(details.iter().all(|detail| !detail.contains("records")));
assert!(
details
diff --git a/crates/client/cache-turso/src/storage/test/predicate_cost.rs b/crates/client/cache-turso/src/storage/test/predicate_cost.rs
index 38f1d5598c..0a310273ca 100644
--- a/crates/client/cache-turso/src/storage/test/predicate_cost.rs
+++ b/crates/client/cache-turso/src/storage/test/predicate_cost.rs
@@ -61,6 +61,106 @@ fn compound_predicates_materialize_sets_once_instead_of_reexecuting_for_each_joi
});
}
+/// A bounded page must use point lookups for sort facts, not repeatedly scan
+/// every row with the requested sort attribute. Exercise each SQL source leg
+/// independently so the usually small optimistic set cannot hide a bad plan.
+#[test]
+fn sort_fact_lookups_stay_linear_at_thousands_of_matches() {
+ block_on(async {
+ for optimistic in [false, true] {
+ for count in [1_000_u32, 3_000] {
+ let mut storage = TursoStorage::open_in_memory("sort-lookup-cost").unwrap();
+ let documents = (0..count)
+ .map(|n| {
+ let mut document =
+ authoritative_projection(&format!("Thing:{n:05}"), "owner-1");
+ // Include ties and rows lacking this sort attribute.
+ document.sort_facts = if n % 7 == 0 {
+ Vec::new()
+ } else {
+ vec![predicate_index::IntegerFact {
+ attribute: Token::new("updated-at").unwrap(),
+ value: i64::from(n / 3),
+ }]
+ };
+ document
+ })
+ .collect::<Vec<_>>();
+ if optimistic {
+ storage
+ .enqueue_mutation_with_shadow(
+ queued("SortLookupCost"),
+ documents
+ .into_iter()
+ .map(|document| PendingOptimisticProjection {
+ state: OptimisticProjectionState::Complete(document),
+ uncertainty: OptimisticUncertainty::Attributes(BTreeSet::new()),
+ })
+ .collect(),
+ )
+ .await
+ .unwrap();
+ } else {
+ storage
+ .put_batch_with_projections(
+ Vec::new(),
+ documents
+ .into_iter()
+ .map(ProjectionMutation::Replace)
+ .collect(),
+ )
+ .await
+ .unwrap();
+ }
+
+ for direction in [SortDirection::Asc, SortDirection::Desc] {
+ let query = ValidatedIndexQuery::new(IndexQuery {
+ profile: Profile::new(Token::new("profile-v1").unwrap()),
+ partitions: vec![PartitionPredicate {
+ partition: Token::new("thing").unwrap(),
+ predicate: PredicateExpr::All,
+ }],
+ sort_attribute: Token::new("updated-at").unwrap(),
+ sort_direction: direction,
+ tie_break_direction: direction,
+ limit: 100,
+ })
+ .unwrap();
+ let (sql, parameters) = compile_predicate_selection(&query, &[], true);
+ let mut statement = driver::prepare(&storage.connection(), &sql).unwrap();
+ let rows = driver::query_prepared(&mut statement, parameters).unwrap();
+ let actual = rows
+ .iter()
+ .map(|row| {
+ (
+ required_text(row, 0).unwrap(),
+ required_i64(row, 1).unwrap(),
+ )
+ })
+ .collect::<Vec<_>>();
+ let mut expected = (0..count).filter(|n| n % 7 != 0).collect::<Vec<_>>();
+ if direction == SortDirection::Desc {
+ expected.reverse();
+ }
+ let expected = expected
+ .into_iter()
+ .take(100)
+ .map(|n| (format!("Thing:{n:05}"), i64::from(n / 3)))
+ .collect::<Vec<_>>();
+ assert_eq!(actual, expected);
+ let steps = statement.metrics().vm_steps;
+ let budget = u64::from(count) * 300;
+ assert!(
+ steps < budget,
+ "sort lookup: {count} rows, optimistic={optimistic}, {direction:?}: \
+ {steps} VM steps exceeds linear budget {budget}"
+ );
+ }
+ }
+ }
+ });
+}
+
#[test]
fn exact_alternatives_use_one_lookup_without_widening_nested_boolean_filters() {
block_on(async {
diff --git a/crates/client/cache-wasm/Cargo.toml b/crates/client/cache-wasm/Cargo.toml
index 29bcba1214..fa0acf0459 100644
--- a/crates/client/cache-wasm/Cargo.toml
+++ b/crates/client/cache-wasm/Cargo.toml
@@ -2,8 +2,8 @@
edition = "2024"
name = "cache-wasm"
publish = false
-# Rebuild for raw file-type projections and SQL-compatible nullable filters.
-version = "0.6.16"
+# Rebuild for indexed predicate sort-fact lookups on large cached mailboxes.
+version = "0.6.17"
[features]
default = []
diff --git a/docs/AGENT_GUIDE/surfaces.md b/docs/AGENT_GUIDE/surfaces.md
index 35370d6c11..3bb287936b 100644
--- a/docs/AGENT_GUIDE/surfaces.md
+++ b/docs/AGENT_GUIDE/surfaces.md
@@ -248,6 +248,13 @@ Full email client. Tabs: `Signal` / `Noise` / `Sent` / `Calendar` / `Drafts` / `
shows `Connect your email` (Gmail/Google Workspace OAuth) — most functionality needs a
connected account. Search is `Ctrl+F` within the surface.
+When switching email tabs or inboxes, the list shows current-query cached results
+or a scoped `Loading email` spinner until they arrive—not the previous tab's rows
+under the new heading. Active searches also hide retained results and show loading
+while selected tag sets are pending. Background refreshes retain the current list. To check this,
+rapidly alternate Signal, Noise, and Sent, then change inboxes; a delayed cache or
+network read must not leave the old rows visible or expose their Load more action.
+
The new views reuse the legacy filter option rows and searchable submenus.
Their triggers are icon-only buttons matching the surrounding view controls;
Clear/Reset filters and the mobile Clear all action use destructive text styling.