Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(web): bound email soup backfill to recent signal and noise threads (#7614)

dev-rb3b95c07bf5a6
diff --git a/apps/web/src/lib/queries/soup/backfill.test.tsx b/apps/web/src/lib/queries/soup/backfill.test.tsx
index 94910aa8c5..0fb75fca6c 100644
--- a/apps/web/src/lib/queries/soup/backfill.test.tsx
+++ b/apps/web/src/lib/queries/soup/backfill.test.tsx
@@ -5,14 +5,22 @@ import * as Fiber from 'effect/Fiber';
import { createSignal, Show } from 'solid-js';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import {
+ DEFAULT_SOUP_BACKFILL_LANES,
loadSoupBackfillCheckpoint,
+ NOISE_EMAIL_FILTER_BACKFILL_LANE,
+ NOISE_EMAIL_SOUP_BACKFILL_LANE,
runSoupBackfill,
runSoupBackfills,
+ SHARED_EMAIL_FILTER_BACKFILL_LANE,
+ SIGNAL_EMAIL_FILTER_BACKFILL_LANE,
+ SIGNAL_EMAIL_SOUP_BACKFILL_LANE,
type SoupBackfillCheckpoint,
type SoupBackfillParams,
useSoupBackfills,
} from './backfill';
+type FetchInput = Parameters<NonNullable<SoupBackfillParams['fetchPage']>>[0];
+
const featureFlagMocks = vi.hoisted(() => ({
useFeatureFlag: vi.fn(),
}));
@@ -95,11 +103,18 @@ function seedCheckpoint(overrides: Partial<SoupBackfillCheckpoint> = {}) {
...overrides,
};
localStorage.setItem(
- 'graphql-soup-backfill:v15:user-1:core-entities',
+ 'graphql-soup-backfill:v16:user-1:core-entities',
JSON.stringify(checkpoint)
);
}
+const recencyTree = (signal: boolean, gte: string) => ({
+ and: {
+ left: { literal: { importance: signal } },
+ right: { literal: { updatedAt: { gte } } },
+ },
+});
+
function BackfillRunner(props: { userId: string }) {
useSoupBackfills(props.userId);
return null;
@@ -304,6 +319,148 @@ describe('runSoupBackfills', () => {
expect(fetchPage.mock.calls[1]?.[0]).toEqual(fetchPage.mock.calls[0]?.[0]);
});
+ it('runs signal email lanes before their noise counterparts', () => {
+ expect(
+ DEFAULT_SOUP_BACKFILL_LANES.map((lane) => lane.checkpointId)
+ ).toEqual([
+ 'core-entities',
+ 'email-signal-filter-metadata',
+ 'email-noise-filter-metadata',
+ 'shared-email-filter-metadata',
+ 'email-signal-thread-pages',
+ 'email-noise-thread-pages',
+ 'auxiliary-entities',
+ ]);
+ });
+
+ it.each([
+ {
+ checkpointId: 'email-signal-filter-metadata',
+ params: SIGNAL_EMAIL_FILTER_BACKFILL_LANE,
+ tree: recencyTree(true, '2026-07-08T12:00:00.000Z'),
+ },
+ {
+ checkpointId: 'email-noise-filter-metadata',
+ params: NOISE_EMAIL_FILTER_BACKFILL_LANE,
+ tree: recencyTree(false, '2026-09-06T12:00:00.000Z'),
+ },
+ {
+ checkpointId: 'email-signal-thread-pages',
+ params: SIGNAL_EMAIL_SOUP_BACKFILL_LANE,
+ tree: recencyTree(true, '2026-07-08T12:00:00.000Z'),
+ },
+ {
+ checkpointId: 'email-noise-thread-pages',
+ params: NOISE_EMAIL_SOUP_BACKFILL_LANE,
+ tree: recencyTree(false, '2026-09-06T12:00:00.000Z'),
+ },
+ ])(
+ 'bounds $checkpointId by latest message recency and resumes without new filters',
+ async ({ params, tree }) => {
+ vi.useFakeTimers({ toFake: ['Date'] });
+ vi.setSystemTime('2026-10-06T12:00:00.000Z');
+ const fetchPage = vi.fn(async (_input: FetchInput) => {
+ vi.setSystemTime('2026-10-07T12:00:00.000Z');
+ return {
+ nextCursor: fetchPage.mock.calls.length === 1 ? 'page-2' : null,
+ };
+ });
+ await Effect.runPromise(
+ runSoupBackfill('user-1', mockCacheHost, {
+ ...params,
+ fetchPage,
+ pageDelayMs: 0,
+ })
+ );
+ expect(fetchPage.mock.calls[0]?.[0]).toMatchObject({
+ initial: {
+ sortMethod: 'UPDATED_AT',
+ emailView: 'ALL',
+ filters: { emailFilter: { tree } },
+ },
+ });
+ expect(fetchPage.mock.calls[1]?.[0]).toEqual({
+ continuation: { cursor: 'page-2', expand: true, emailView: 'ALL' },
+ });
+ }
+ );
+
+ it('bounds Shared mail to either recency window in one scan', async () => {
+ vi.useFakeTimers({ toFake: ['Date'] });
+ vi.setSystemTime('2026-10-06T12:00:00.000Z');
+ const fetchPage = vi.fn(async (_input: FetchInput) => ({
+ nextCursor: null,
+ }));
+ await Effect.runPromise(
+ runSoupBackfill('user-1', mockCacheHost, {
+ ...SHARED_EMAIL_FILTER_BACKFILL_LANE,
+ createFetchPage: async () => fetchPage,
+ })
+ );
+ expect(fetchPage.mock.calls[0]?.[0]).toMatchObject({
+ initial: {
+ filters: {
+ emailFilter: {
+ tree: {
+ and: {
+ left: { literal: { shared: 'ONLY' } },
+ right: {
+ or: {
+ left: recencyTree(true, '2026-07-08T12:00:00.000Z'),
+ right: recencyTree(false, '2026-09-06T12:00:00.000Z'),
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ });
+ });
+
+ it('recomputes the recency cutoff for the catch-up pass and keeps its watermark', async () => {
+ vi.useFakeTimers({ toFake: ['Date'] });
+ vi.setSystemTime('2026-10-06T12:00:00.000Z');
+ const fetchPage = vi.fn(async (_input: FetchInput) => {
+ vi.setSystemTime('2026-10-07T12:00:00.000Z');
+ return { nextCursor: null };
+ });
+ await Effect.runPromise(
+ runSoupBackfill('user-1', mockCacheHost, {
+ ...SIGNAL_EMAIL_SOUP_BACKFILL_LANE,
+ fetchPage,
+ })
+ );
+ expect(fetchPage).toHaveBeenCalledTimes(2);
+ expect(fetchPage.mock.calls[1]?.[0]).toMatchObject({
+ initial: {
+ filters: {
+ emailFilter: {
+ tree: {
+ and: {
+ left: recencyTree(true, '2026-07-09T12:00:00.000Z'),
+ right: {
+ or: {
+ left: {
+ literal: {
+ updatedAt: { gte: '2026-10-06T12:00:00.000Z' },
+ },
+ },
+ right: {
+ literal: {
+ viewedAt: { gte: '2026-10-06T12:00:00.000Z' },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ });
+ });
+
it('runs each lane to completion before starting the next lane', async () => {
const order: string[] = [];
let finishFirstLane!: () => void;
@@ -633,33 +790,36 @@ describe('runSoupBackfills', () => {
rendered.unmount();
});
- it.each([13, 14])('does not reuse v%i backfill checkpoints', (version) => {
- localStorage.setItem(
- `graphql-soup-backfill:v${version}:user-1:email-filter-metadata`,
- JSON.stringify({
- userId: 'user-1',
- nextCursor: 'old-page',
- pagesFetched: 3,
+ it.each([13, 14, 15])(
+ 'does not reuse v%i backfill checkpoints',
+ (version) => {
+ localStorage.setItem(
+ `graphql-soup-backfill:v${version}:user-1:email-filter-metadata`,
+ JSON.stringify({
+ userId: 'user-1',
+ nextCursor: 'old-page',
+ pagesFetched: 3,
+ completed: false,
+ scanStartedAt: '2026-09-01T00:00:00.000Z',
+ updatedSince: null,
+ completedAt: null,
+ })
+ );
+ expect(
+ loadSoupBackfillCheckpoint('user-1', 'email-filter-metadata')
+ ).toMatchObject({
+ nextCursor: null,
+ pagesFetched: 0,
completed: false,
- scanStartedAt: '2026-09-01T00:00:00.000Z',
- updatedSince: null,
- completedAt: null,
- })
- );
- expect(
- loadSoupBackfillCheckpoint('user-1', 'email-filter-metadata')
- ).toMatchObject({
- nextCursor: null,
- pagesFetched: 0,
- completed: false,
- });
- });
+ });
+ }
+ );
it.each(['preserved', 'reset'] as const)(
'restarts the runner with %s storage semantics and ignores obsolete completions',
async (storage) => {
localStorage.setItem(
- 'graphql-soup-backfill:v15:user-1:core-entities',
+ 'graphql-soup-backfill:v16:user-1:core-entities',
JSON.stringify({
userId: 'user-1',
storageGeneration: 'storage-1',
@@ -733,7 +893,7 @@ describe('runSoupBackfills', () => {
const watermark = '2026-09-01T00:00:00.000Z';
for (const checkpointId of ['core-entities', 'email-thread-pages']) {
localStorage.setItem(
- `graphql-soup-backfill:v15:user-1:${checkpointId}`,
+ `graphql-soup-backfill:v16:user-1:${checkpointId}`,
JSON.stringify({
userId: 'user-1',
storageGeneration: 'storage-1',
diff --git a/apps/web/src/lib/queries/soup/backfill.ts b/apps/web/src/lib/queries/soup/backfill.ts
index aa0bc7762f..73f15282e5 100644
--- a/apps/web/src/lib/queries/soup/backfill.ts
+++ b/apps/web/src/lib/queries/soup/backfill.ts
@@ -7,6 +7,7 @@ import { createTabLeaderSignal } from '@core/cross-tab/tab-leader';
import type { CacheHost } from '@graphql-cache/host/types';
import { Telemetry } from '@macro-inc/observability';
import {
+ type GraphqlEmailExpr,
SoupBackfillDocument,
SoupMailBackfillDocument,
} from '@service-storage/graphql/generated/graphql';
@@ -28,7 +29,7 @@ import { createEffect, createSignal, onCleanup } from 'solid-js';
// persisted cursors cannot retain an older hydration contract.
// Rehydrate raw file-type projections after retiring enum-normalized facts.
// Old cursors must not skip records when the cache compatibility epoch changes.
-const BACKFILL_VERSION = 15;
+const BACKFILL_VERSION = 16;
const PAGE_LIMIT = 100;
// Five threads × twenty messages reaches the backend's 100-message cap.
const EMAIL_CONTENT_PAGE_LIMIT = 5;
@@ -39,6 +40,14 @@ const CACHE_HOST_RETRY_COUNT = 6;
const CACHE_HOST_RETRY_SCHEDULE = Schedule.exponential('100 millis');
const BACKFILL_PROGRESS_PAGE_INTERVAL = 10;
const EXCLUDED_ENTITY_ID = '00000000-0000-0000-0000-000000000000';
+const DAY_MS = 24 * 60 * 60 * 1000;
+
+/** Email threads of one signal classification whose latest message is at most
+ * `days` old. */
+type EmailRecencyWindow = { signal: boolean; days: number };
+
+const SIGNAL_EMAIL_WINDOW: EmailRecencyWindow = { signal: true, days: 90 };
+const NOISE_EMAIL_WINDOW: EmailRecencyWindow = { signal: false, days: 30 };
type SoupBackfillFetchPage = (
input: GraphqlSoupInput,
@@ -69,6 +78,9 @@ export type SoupBackfillParams = {
/** Refresh the whole metadata corpus: message-time watermarks do not capture
* read/archive changes to old email threads. Interrupted scans still resume. */
refreshAll?: boolean;
+ /** Email threads matching any window. The cutoff is fixed when a pass starts;
+ * resumed cursors keep the cutoff they were issued with. */
+ emailRecency?: readonly EmailRecencyWindow[];
};
/** Backfills the entities used most often by Quick Access and primary views. */
@@ -91,56 +103,85 @@ export const CORE_SOUP_BACKFILL_LANE: SoupBackfillParams = {
};
/**
- * Backfills email threads and the first message page used by the thread view
- * while excluding every other entity variant with an impossible id filter.
+ * Email threads and the first message page used by the thread view, excluding
+ * every other entity variant with an impossible id filter.
*/
-export const EMAIL_SOUP_BACKFILL_LANE: SoupBackfillParams = {
- checkpointId: 'email-thread-pages',
+const EMAIL_THREAD_PAGES_INPUT: GraphqlSoupInitialInput = {
+ limit: EMAIL_CONTENT_PAGE_LIMIT,
+ expand: true,
+ // Orders by the latest-message timestamp that emailRecency bounds, so each
+ // page is an ordered index range ending at the cutoff.
+ sortMethod: 'UPDATED_AT',
+ emailView: 'ALL',
+ filters: {
+ calendarEventFilter: { literal: { id: EXCLUDED_ENTITY_ID } },
+ documentFilter: { literal: { id: EXCLUDED_ENTITY_ID } },
+ projectFilter: { literal: { projectIdSelf: EXCLUDED_ENTITY_ID } },
+ chatFilter: { literal: { chatId: EXCLUDED_ENTITY_ID } },
+ channelFilter: { literal: { channelId: EXCLUDED_ENTITY_ID } },
+ channelThreadFilter: { literal: { threadId: EXCLUDED_ENTITY_ID } },
+ callFilter: { literal: { callId: EXCLUDED_ENTITY_ID } },
+ crmCompanyFilter: { literal: { id: EXCLUDED_ENTITY_ID } },
+ foreignEntityFilter: { literal: { id: EXCLUDED_ENTITY_ID } },
+ },
+};
+
+const EMAIL_THREAD_PAGES_LANE = {
fetchPage: fetchEmailContentPage,
- // A long full scan can skip threads created, viewed, or updated after its
- // VIEWED_UPDATED cursor has passed them. Consume the recorded watermark
- // before reporting this lane complete.
+ // A long full scan can skip threads that receive messages after its cursor
+ // has passed them. Consume the recorded watermark before reporting the lane
+ // complete.
catchUpAfterInitialPass: true,
- input: {
- limit: EMAIL_CONTENT_PAGE_LIMIT,
- expand: true,
- sortMethod: 'VIEWED_UPDATED',
- emailView: 'ALL',
- filters: {
- calendarEventFilter: { literal: { id: EXCLUDED_ENTITY_ID } },
- documentFilter: { literal: { id: EXCLUDED_ENTITY_ID } },
- projectFilter: { literal: { projectIdSelf: EXCLUDED_ENTITY_ID } },
- chatFilter: { literal: { chatId: EXCLUDED_ENTITY_ID } },
- channelFilter: { literal: { channelId: EXCLUDED_ENTITY_ID } },
- channelThreadFilter: { literal: { threadId: EXCLUDED_ENTITY_ID } },
- callFilter: { literal: { callId: EXCLUDED_ENTITY_ID } },
- crmCompanyFilter: { literal: { id: EXCLUDED_ENTITY_ID } },
- foreignEntityFilter: { literal: { id: EXCLUDED_ENTITY_ID } },
- },
- },
+ input: EMAIL_THREAD_PAGES_INPUT,
+} satisfies Omit<SoupBackfillParams, 'checkpointId'>;
+
+export const SIGNAL_EMAIL_SOUP_BACKFILL_LANE: SoupBackfillParams = {
+ ...EMAIL_THREAD_PAGES_LANE,
+ checkpointId: 'email-signal-thread-pages',
+ emailRecency: [SIGNAL_EMAIL_WINDOW],
+};
+
+export const NOISE_EMAIL_SOUP_BACKFILL_LANE: SoupBackfillParams = {
+ ...EMAIL_THREAD_PAGES_LANE,
+ checkpointId: 'email-noise-thread-pages',
+ emailRecency: [NOISE_EMAIL_WINDOW],
};
/** Filter/row metadata is synchronized before the independently bounded body cache.
* ALL covers the first Mail slice across every readable owned/delegated inbox. */
-export const EMAIL_FILTER_BACKFILL_LANE: SoupBackfillParams = {
- checkpointId: 'email-filter-metadata',
+const EMAIL_FILTER_LANE = {
fetchPage: (input, options) =>
hydrateGraphqlSoup(SoupMailBackfillDocument, { input }, options),
refreshAll: true,
- input: { ...EMAIL_SOUP_BACKFILL_LANE.input, limit: PAGE_LIMIT },
+ input: { ...EMAIL_THREAD_PAGES_INPUT, limit: PAGE_LIMIT },
+} satisfies Omit<SoupBackfillParams, 'checkpointId'>;
+
+export const SIGNAL_EMAIL_FILTER_BACKFILL_LANE: SoupBackfillParams = {
+ ...EMAIL_FILTER_LANE,
+ checkpointId: 'email-signal-filter-metadata',
+ emailRecency: [SIGNAL_EMAIL_WINDOW],
+};
+
+export const NOISE_EMAIL_FILTER_BACKFILL_LANE: SoupBackfillParams = {
+ ...EMAIL_FILTER_LANE,
+ checkpointId: 'email-noise-filter-metadata',
+ emailRecency: [NOISE_EMAIL_WINDOW],
};
/** Shared grants are separate from owned/delegated inbox scope. A complete scan
- * invalidates omitted old proof; interrupted scans preserve last-known evidence. */
+ * invalidates omitted old proof; interrupted scans preserve last-known evidence.
+ * One lane covers both windows because each scan invalidates every cached
+ * Shared thread it did not return, including threads outside the windows. */
export const SHARED_EMAIL_FILTER_BACKFILL_LANE: SoupBackfillParams = {
checkpointId: 'shared-email-filter-metadata',
createFetchPage: createSharedMailBackfillFetcher,
refreshAll: true,
restartOnRun: true,
+ emailRecency: [SIGNAL_EMAIL_WINDOW, NOISE_EMAIL_WINDOW],
input: {
- ...EMAIL_FILTER_BACKFILL_LANE.input,
+ ...EMAIL_FILTER_LANE.input,
filters: {
- ...EMAIL_FILTER_BACKFILL_LANE.input.filters,
+ ...EMAIL_FILTER_LANE.input.filters,
emailFilter: { tree: { literal: { shared: 'ONLY' } } },
},
},
@@ -170,9 +211,11 @@ export const AUXILIARY_SOUP_BACKFILL_LANE: SoupBackfillParams = {
/** Independently checkpointed backfills run serially in priority order. */
export const DEFAULT_SOUP_BACKFILL_LANES = [
CORE_SOUP_BACKFILL_LANE,
- EMAIL_FILTER_BACKFILL_LANE,
+ SIGNAL_EMAIL_FILTER_BACKFILL_LANE,
+ NOISE_EMAIL_FILTER_BACKFILL_LANE,
SHARED_EMAIL_FILTER_BACKFILL_LANE,
- EMAIL_SOUP_BACKFILL_LANE,
+ SIGNAL_EMAIL_SOUP_BACKFILL_LANE,
+ NOISE_EMAIL_SOUP_BACKFILL_LANE,
AUXILIARY_SOUP_BACKFILL_LANE,
] as const satisfies readonly SoupBackfillParams[];
@@ -342,8 +385,8 @@ export function withUpdatedSince(
literal: { updatedAt: { gte: updatedSince } },
};
- // VIEWED_UPDATED can move a row ahead of the scan cursor through either
- // the thread timestamp or this viewer's history timestamp. Cover both.
+ // A cached email row changes through either the thread timestamp or this
+ // viewer's history timestamp, and VIEWED_UPDATED sorts on both. Cover both.
const emailSortWatermark = {
or: {
left: { literal: { updatedAt: { gte: updatedSince } } },
@@ -366,6 +409,44 @@ export function withUpdatedSince(
};
}
+/**
+ * Restricts email threads to those matching any window. In the ALL view the
+ * email updatedAt literal compares the latest non-spam message timestamp.
+ */
+function withEmailRecency(
+ input: GraphqlSoupInitialInput,
+ windows: readonly EmailRecencyWindow[] | undefined,
+ now: number
+): GraphqlSoupInitialInput {
+ if (!windows?.length) return input;
+
+ const recency = windows
+ .map(
+ ({ signal, days }): GraphqlEmailExpr => ({
+ and: {
+ left: { literal: { importance: signal } },
+ right: {
+ literal: {
+ updatedAt: { gte: new Date(now - days * DAY_MS).toISOString() },
+ },
+ },
+ },
+ })
+ )
+ .reduce((left, right) => ({ or: { left, right } }));
+ const filters = input.filters ?? {};
+ return {
+ ...input,
+ filters: {
+ ...filters,
+ emailFilter: {
+ ...(filters.emailFilter ?? {}),
+ tree: and(filters.emailFilter?.tree, recency),
+ },
+ },
+ };
+}
+
type SoupBackfillCacheHost = Pick<CacheHost, 'currentStorageGeneration'>;
export const runSoupBackfill = Effect.fn('runSoupBackfill')(function* (
@@ -430,7 +511,7 @@ export const runSoupBackfill = Effect.fn('runSoupBackfill')(function* (
continue;
}
const passInput = withUpdatedSince(
- params.input,
+ withEmailRecency(params.input, params.emailRecency, Date.now()),
params.refreshAll ? null : checkpoint.updatedSince
);
const input: GraphqlSoupInput = checkpoint.nextCursor
diff --git a/apps/web/tests/native/README.md b/apps/web/tests/native/README.md
index 53b006aa6e..a55b729dd5 100644
--- a/apps/web/tests/native/README.md
+++ b/apps/web/tests/native/README.md
@@ -46,8 +46,10 @@ file-backed native Turso cache.
Fixtures replace the **remote services**, not the client cache. `fixtures/server.ts`
serves the normal `/dss/items/soup/graphql` path, executes the app's generated
-queries against `static_assets/schema.graphql`, and returns three pages of Mail
-metadata. Projection capsules reuse the existing Rust-encoded fixtures.
+queries against `static_assets/schema.graphql`, and returns two pages each of
+signal and noise Mail metadata. Metadata requests must carry a recency cutoff,
+but fixture timestamps are fixed, so the server does not apply it. Projection
+capsules reuse the existing Rust-encoded fixtures.
Unrelated app-chrome reads return explicit empty resources. Unexpected requests,
unknown cursors and GraphQL validation errors are recorded and fail the smoke
suite; nothing forwards to hosted services. This does not test the Rust HTTP
@@ -62,11 +64,11 @@ The six threads are IDs ending in 4, 6, 8, 9, 10, and 12:
| Offline All | 4, 6, 8, 9, 10, 12 |
Only the initial Signal view may receive an online Mail response. The other rows
-arrive via metadata backfill. Tests wait for the real three-page completion
-checkpoint, then read the six normalized records through native IPC before
-changing filters. Tab changes use the app's public numeric hotkeys (2 / 7) and
-verify `aria-current` before checking results. Assertions compare exact row
-identities, not just counts.
+arrive via metadata backfill. Tests wait for the real signal and noise
+completion checkpoints (four pages), then read the six normalized records
+through native IPC before changing filters. Tab changes use the app's public
+numeric hotkeys (2 / 7) and verify `aria-current` before checking results.
+Assertions compare exact row identities, not just counts.
## Exhaustive filter-selection matrix
@@ -87,7 +89,7 @@ The corpus has 75 entities and fits within one initial page: this suite does
not claim coverage of non-Mail offline pagination or every possible real-world
user/tag/file value.
-After all five real backfill lanes finish, the fixture API and WebSockets are
+After all seven real backfill lanes finish, the fixture API and WebSockets are
closed. The browser-side runner calls the real Tauri cache host in bounded
read-only batches and asserts exact matching IDs against fixture truth. No
query-specific network baselines are supplied. Mail attachment selections also
diff --git a/apps/web/tests/native/driver.ts b/apps/web/tests/native/driver.ts
index aed9703926..55716fa569 100644
--- a/apps/web/tests/native/driver.ts
+++ b/apps/web/tests/native/driver.ts
@@ -47,15 +47,23 @@ export async function checkpoints(browser: Browser) {
);
}
-export async function waitForBackfill(browser: Browser, pages = 3) {
+const METADATA_LANES = [
+ ':email-signal-filter-metadata',
+ ':email-noise-filter-metadata',
+];
+
+export async function waitForBackfill(browser: Browser, pages: number) {
await browser.waitUntil(
- async () =>
- (await checkpoints(browser)).some(
- (checkpoint) =>
- checkpoint.key.endsWith(':email-filter-metadata') &&
- checkpoint.completed &&
- checkpoint.pagesFetched === pages
- ),
+ async () => {
+ const lanes = (await checkpoints(browser)).filter((checkpoint) =>
+ METADATA_LANES.some((suffix) => checkpoint.key.endsWith(suffix))
+ );
+ return (
+ lanes.length === METADATA_LANES.length &&
+ lanes.every((lane) => lane.completed) &&
+ lanes.reduce((total, lane) => total + lane.pagesFetched, 0) === pages
+ );
+ },
{
timeout: 120_000,
timeoutMsg: `Real metadata backfill did not checkpoint all ${pages} pages`,
diff --git a/apps/web/tests/native/fixtures/server.test.ts b/apps/web/tests/native/fixtures/server.test.ts
index 652eb64ad6..fe2b047abf 100644
--- a/apps/web/tests/native/fixtures/server.test.ts
+++ b/apps/web/tests/native/fixtures/server.test.ts
@@ -44,52 +44,86 @@ async function requestSoup(
return await response.json();
}
-test('real generated backfill document validates, pages, and includes projection/preview metadata', async () => {
- server = startFixtureServer();
- const ids: string[] = [];
- let cursor: string | null = null;
- for (let page = 0; page < 3; page++) {
- const response = await fetch(`${server.origin}/dss/items/soup/graphql`, {
- method: 'POST',
- body: JSON.stringify({
- query,
- operationName: 'SoupMailBackfill',
- variables: {
- input: cursor
- ? { continuation: { cursor, emailView: 'ALL' } }
- : { initial: { limit: 100, emailView: 'ALL' } },
- },
- }),
- });
- const result: {
- errors?: unknown;
- data: {
- user: {
- soup: {
- items: {
- id: string;
- cacheProjection: string;
- mailAllPreview: { subject: string };
- }[];
- nextCursor: string | null;
- };
+const recencyTree = (importance: boolean) => ({
+ and: {
+ left: { literal: { importance } },
+ right: { literal: { updatedAt: { gte: '2026-01-01T00:00:00.000Z' } } },
+ },
+});
+
+async function requestMetadata(input: SoupInput) {
+ if (!server) throw new Error('Fixture server must be started first');
+ const response = await fetch(`${server.origin}/dss/items/soup/graphql`, {
+ method: 'POST',
+ body: JSON.stringify({
+ query,
+ operationName: 'SoupMailBackfill',
+ variables: { input },
+ }),
+ });
+ const result: {
+ errors?: { message: string }[];
+ data: {
+ user: {
+ soup: {
+ items: {
+ id: string;
+ cacheProjection: string;
+ mailAllPreview: { subject: string };
+ }[];
+ nextCursor: string | null;
};
};
- } = await response.json();
- expect(result.errors).toBeUndefined();
- const soup = result.data.user.soup;
- expect(soup.items).toHaveLength(2);
- for (const item of soup.items) {
- ids.push(item.id);
- expect(item.cacheProjection).toBeString();
- expect(item.mailAllPreview.subject).toStartWith('Email ');
+ } | null;
+ } = await response.json();
+ return result;
+}
+
+test('real generated backfill document validates, pages each signal class, and includes projection/preview metadata', async () => {
+ server = startFixtureServer();
+ for (const [importance, expected] of [
+ [true, [6, 9, 12]],
+ [false, [4, 8, 10]],
+ ] as const) {
+ const ids: string[] = [];
+ let cursor: string | null = null;
+ for (let page = 0; page < 2; page++) {
+ const result = await requestMetadata(
+ cursor
+ ? { continuation: { cursor, emailView: 'ALL' } }
+ : {
+ initial: {
+ limit: 100,
+ emailView: 'ALL',
+ filters: { emailFilter: { tree: recencyTree(importance) } },
+ },
+ }
+ );
+ expect(result.errors).toBeUndefined();
+ const soup = result.data!.user.soup;
+ for (const item of soup.items) {
+ ids.push(item.id);
+ expect(item.cacheProjection).toBeString();
+ expect(item.mailAllPreview.subject).toStartWith('Email ');
+ }
+ cursor = soup.nextCursor;
+ if (page < 1) expect(cursor).toBeString();
}
- cursor = soup.nextCursor;
- if (page < 2) expect(cursor).toBeString();
+ expect(cursor).toBeNull();
+ expect(ids).toEqual(expected.map(fixtureId));
}
- expect(cursor).toBeNull();
- expect(ids).toEqual([4, 6, 8, 9, 10, 12].map(fixtureId));
- expect(server.metadataPagesServed).toBe(3);
+ expect(server.metadataPagesServed).toBe(4);
+ expect(server.expectedMetadataPages).toBe(4);
+});
+
+test('metadata backfill without a recency-bounded signal class fails', async () => {
+ server = startFixtureServer();
+ const result = await requestMetadata({
+ initial: { limit: 100, emailView: 'ALL' },
+ });
+ expect(result.errors?.map((error) => error.message)).toEqual([
+ 'Metadata backfill must bound one signal class by recency',
+ ]);
});
const signalTree = {
diff --git a/apps/web/tests/native/fixtures/server.ts b/apps/web/tests/native/fixtures/server.ts
index 7293d8ac6e..f932e7c53a 100644
--- a/apps/web/tests/native/fixtures/server.ts
+++ b/apps/web/tests/native/fixtures/server.ts
@@ -24,6 +24,17 @@ function excludesMail(input: SoupInput) {
);
}
+type MetadataLane = 'signal' | 'noise';
+
+/** Fixture timestamps are fixed, so the recency cutoff is required but not applied. */
+function metadataLane(input: SoupInput): MetadataLane {
+ const tree = input.initial?.filters?.emailFilter?.tree;
+ const signal = tree?.and?.left?.literal?.importance;
+ if (typeof signal !== 'boolean' || !tree?.and?.right?.literal?.updatedAt?.gte)
+ throw new Error('Metadata backfill must bound one signal class by recency');
+ return signal ? 'signal' : 'noise';
+}
+
export type RequestRecord = {
method: string;
path: string;
@@ -37,13 +48,17 @@ export type RequestRecord = {
* No forwarding to a hosted backend, cache seeding, or filter-result injection. */
export function startFixtureServer(port = 0, filterMatrix = false) {
const corpus = filterMatrix ? filterCorpus(capsules) : undefined;
- const metadataMail = corpus
+ const metadataRows = corpus
? corpus
.filter(
(row) => row.kind === 'email' && LINKS.includes(row.linkId ?? '')
)
- .map((row) => row.api)
- : mail;
+ .map((row) => ({ signal: row.signal === true, api: row.api }))
+ : mail.map((api) => ({ signal: api.isSignal, api }));
+ const metadataMail: Record<MetadataLane, unknown[]> = {
+ signal: metadataRows.filter((row) => row.signal).map((row) => row.api),
+ noise: metadataRows.filter((row) => !row.signal).map((row) => row.api),
+ };
const sharedMail =
corpus
?.filter((row) => row.kind === 'email' && row.shared)
@@ -63,26 +78,32 @@ export function startFixtureServer(port = 0, filterMatrix = false) {
: [mail[1], mail[5]];
const requests: RequestRecord[] = [];
const pageSize = 2;
- const cursors = new Map<string, number>();
+ const cursors = new Map<string, { lane: MetadataLane; offset: number }>();
let pagesServed = 0;
- function soupPage(input: SoupInput, operation: string) {
- if (operation === 'SoupMailBackfill') {
- let offset = 0;
- if (input.continuation) {
- const saved = cursors.get(input.continuation.cursor);
- if (saved === undefined) throw new Error('Unknown metadata cursor');
- offset = saved;
- } else if (input.initial?.emailView !== 'ALL') {
- throw new Error('Metadata backfill must request ALL');
- }
- const nextOffset = offset + pageSize;
- const nextCursor =
- nextOffset < metadataMail.length ? `metadata-${nextOffset}` : null;
- if (nextCursor) cursors.set(nextCursor, nextOffset);
- pagesServed += 1;
- return { items: metadataMail.slice(offset, nextOffset), nextCursor };
+ function metadataPage(input: SoupInput) {
+ let lane: MetadataLane;
+ let offset = 0;
+ if (input.continuation) {
+ const saved = cursors.get(input.continuation.cursor);
+ if (saved === undefined) throw new Error('Unknown metadata cursor');
+ ({ lane, offset } = saved);
+ } else if (input.initial?.emailView !== 'ALL') {
+ throw new Error('Metadata backfill must request ALL');
+ } else {
+ lane = metadataLane(input);
}
+ const rows = metadataMail[lane];
+ const nextOffset = offset + pageSize;
+ const nextCursor =
+ nextOffset < rows.length ? `metadata-${lane}-${nextOffset}` : null;
+ if (nextCursor) cursors.set(nextCursor, { lane, offset: nextOffset });
+ pagesServed += 1;
+ return { items: rows.slice(offset, nextOffset), nextCursor };
+ }
+
+ function soupPage(input: SoupInput, operation: string) {
+ if (operation === 'SoupMailBackfill') return metadataPage(input);
if (operation === 'SoupSharedMailBackfill')
return { items: sharedMail, nextCursor: null };
if (operation === 'SoupBackfill') {
@@ -229,7 +250,10 @@ export function startFixtureServer(port = 0, filterMatrix = false) {
get socketCount() {
return sockets.size;
},
- expectedMetadataPages: Math.ceil(metadataMail.length / pageSize),
+ expectedMetadataPages: Object.values(metadataMail).reduce(
+ (pages, rows) => pages + Math.max(1, Math.ceil(rows.length / pageSize)),
+ 0
+ ),
initialSignalIds: signalMail.map((row) => String(row.id)),
get metadataPagesServed() {
return pagesServed;
diff --git a/apps/web/tests/native/runner.ts b/apps/web/tests/native/runner.ts
index 19f82868f6..6f5fcdb258 100644
--- a/apps/web/tests/native/runner.ts
+++ b/apps/web/tests/native/runner.ts
@@ -181,7 +181,7 @@ try {
await browser.waitUntil(
async () => {
const lanes = await checkpoints(browser!);
- return lanes.length === 5 && lanes.every((lane) => lane.completed);
+ return lanes.length === 7 && lanes.every((lane) => lane.completed);
},
{ timeout: 60_000 }
);