Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(web): bound email soup backfill to recent signal and noise threads (#7614)
dev-rb3b95c07bf5a6
diff --git a/apps/web/src/lib/queries/soup/backfill.test.tsx b/apps/web/src/lib/queries/soup/backfill.test.tsxindex 94910aa8c5..0fb75fca6c 100644--- a/apps/web/src/lib/queries/soup/backfill.test.tsx+++ b/apps/web/src/lib/queries/soup/backfill.test.tsx@@ -5,14 +5,22 @@ import * as Fiber from 'effect/Fiber';import { createSignal, Show } from 'solid-js';import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';import {+ DEFAULT_SOUP_BACKFILL_LANES,loadSoupBackfillCheckpoint,+ NOISE_EMAIL_FILTER_BACKFILL_LANE,+ NOISE_EMAIL_SOUP_BACKFILL_LANE,runSoupBackfill,runSoupBackfills,+ SHARED_EMAIL_FILTER_BACKFILL_LANE,+ SIGNAL_EMAIL_FILTER_BACKFILL_LANE,+ SIGNAL_EMAIL_SOUP_BACKFILL_LANE,type SoupBackfillCheckpoint,type SoupBackfillParams,useSoupBackfills,} from './backfill';+type FetchInput = Parameters<NonNullable<SoupBackfillParams['fetchPage']>>[0];+const featureFlagMocks = vi.hoisted(() => ({useFeatureFlag: vi.fn(),}));@@ -95,11 +103,18 @@ function seedCheckpoint(overrides: Partial<SoupBackfillCheckpoint> = {}) {...overrides,};localStorage.setItem(- 'graphql-soup-backfill:v15:user-1:core-entities',+ 'graphql-soup-backfill:v16:user-1:core-entities',JSON.stringify(checkpoint));}+const recencyTree = (signal: boolean, gte: string) => ({+ and: {+ left: { literal: { importance: signal } },+ right: { literal: { updatedAt: { gte } } },+ },+});+function BackfillRunner(props: { userId: string }) {useSoupBackfills(props.userId);return null;@@ -304,6 +319,148 @@ describe('runSoupBackfills', () => {expect(fetchPage.mock.calls[1]?.[0]).toEqual(fetchPage.mock.calls[0]?.[0]);});+ it('runs signal email lanes before their noise counterparts', () => {+ expect(+ DEFAULT_SOUP_BACKFILL_LANES.map((lane) => lane.checkpointId)+ ).toEqual([+ 'core-entities',+ 'email-signal-filter-metadata',+ 'email-noise-filter-metadata',+ 'shared-email-filter-metadata',+ 'email-signal-thread-pages',+ 'email-noise-thread-pages',+ 'auxiliary-entities',+ ]);+ });++ it.each([+ {+ checkpointId: 'email-signal-filter-metadata',+ params: SIGNAL_EMAIL_FILTER_BACKFILL_LANE,+ tree: recencyTree(true, '2026-07-08T12:00:00.000Z'),+ },+ {+ checkpointId: 'email-noise-filter-metadata',+ params: NOISE_EMAIL_FILTER_BACKFILL_LANE,+ tree: recencyTree(false, '2026-09-06T12:00:00.000Z'),+ },+ {+ checkpointId: 'email-signal-thread-pages',+ params: SIGNAL_EMAIL_SOUP_BACKFILL_LANE,+ tree: recencyTree(true, '2026-07-08T12:00:00.000Z'),+ },+ {+ checkpointId: 'email-noise-thread-pages',+ params: NOISE_EMAIL_SOUP_BACKFILL_LANE,+ tree: recencyTree(false, '2026-09-06T12:00:00.000Z'),+ },+ ])(+ 'bounds $checkpointId by latest message recency and resumes without new filters',+ async ({ params, tree }) => {+ vi.useFakeTimers({ toFake: ['Date'] });+ vi.setSystemTime('2026-10-06T12:00:00.000Z');+ const fetchPage = vi.fn(async (_input: FetchInput) => {+ vi.setSystemTime('2026-10-07T12:00:00.000Z');+ return {+ nextCursor: fetchPage.mock.calls.length === 1 ? 'page-2' : null,+ };+ });+ await Effect.runPromise(+ runSoupBackfill('user-1', mockCacheHost, {+ ...params,+ fetchPage,+ pageDelayMs: 0,+ })+ );+ expect(fetchPage.mock.calls[0]?.[0]).toMatchObject({+ initial: {+ sortMethod: 'UPDATED_AT',+ emailView: 'ALL',+ filters: { emailFilter: { tree } },+ },+ });+ expect(fetchPage.mock.calls[1]?.[0]).toEqual({+ continuation: { cursor: 'page-2', expand: true, emailView: 'ALL' },+ });+ }+ );++ it('bounds Shared mail to either recency window in one scan', async () => {+ vi.useFakeTimers({ toFake: ['Date'] });+ vi.setSystemTime('2026-10-06T12:00:00.000Z');+ const fetchPage = vi.fn(async (_input: FetchInput) => ({+ nextCursor: null,+ }));+ await Effect.runPromise(+ runSoupBackfill('user-1', mockCacheHost, {+ ...SHARED_EMAIL_FILTER_BACKFILL_LANE,+ createFetchPage: async () => fetchPage,+ })+ );+ expect(fetchPage.mock.calls[0]?.[0]).toMatchObject({+ initial: {+ filters: {+ emailFilter: {+ tree: {+ and: {+ left: { literal: { shared: 'ONLY' } },+ right: {+ or: {+ left: recencyTree(true, '2026-07-08T12:00:00.000Z'),+ right: recencyTree(false, '2026-09-06T12:00:00.000Z'),+ },+ },+ },+ },+ },+ },+ },+ });+ });++ it('recomputes the recency cutoff for the catch-up pass and keeps its watermark', async () => {+ vi.useFakeTimers({ toFake: ['Date'] });+ vi.setSystemTime('2026-10-06T12:00:00.000Z');+ const fetchPage = vi.fn(async (_input: FetchInput) => {+ vi.setSystemTime('2026-10-07T12:00:00.000Z');+ return { nextCursor: null };+ });+ await Effect.runPromise(+ runSoupBackfill('user-1', mockCacheHost, {+ ...SIGNAL_EMAIL_SOUP_BACKFILL_LANE,+ fetchPage,+ })+ );+ expect(fetchPage).toHaveBeenCalledTimes(2);+ expect(fetchPage.mock.calls[1]?.[0]).toMatchObject({+ initial: {+ filters: {+ emailFilter: {+ tree: {+ and: {+ left: recencyTree(true, '2026-07-09T12:00:00.000Z'),+ right: {+ or: {+ left: {+ literal: {+ updatedAt: { gte: '2026-10-06T12:00:00.000Z' },+ },+ },+ right: {+ literal: {+ viewedAt: { gte: '2026-10-06T12:00:00.000Z' },+ },+ },+ },+ },+ },+ },+ },+ },+ },+ });+ });+it('runs each lane to completion before starting the next lane', async () => {const order: string[] = [];let finishFirstLane!: () => void;@@ -633,33 +790,36 @@ describe('runSoupBackfills', () => {rendered.unmount();});- it.each([13, 14])('does not reuse v%i backfill checkpoints', (version) => {- localStorage.setItem(- `graphql-soup-backfill:v${version}:user-1:email-filter-metadata`,- JSON.stringify({- userId: 'user-1',- nextCursor: 'old-page',- pagesFetched: 3,+ it.each([13, 14, 15])(+ 'does not reuse v%i backfill checkpoints',+ (version) => {+ localStorage.setItem(+ `graphql-soup-backfill:v${version}:user-1:email-filter-metadata`,+ JSON.stringify({+ userId: 'user-1',+ nextCursor: 'old-page',+ pagesFetched: 3,+ completed: false,+ scanStartedAt: '2026-09-01T00:00:00.000Z',+ updatedSince: null,+ completedAt: null,+ })+ );+ expect(+ loadSoupBackfillCheckpoint('user-1', 'email-filter-metadata')+ ).toMatchObject({+ nextCursor: null,+ pagesFetched: 0,completed: false,- scanStartedAt: '2026-09-01T00:00:00.000Z',- updatedSince: null,- completedAt: null,- })- );- expect(- loadSoupBackfillCheckpoint('user-1', 'email-filter-metadata')- ).toMatchObject({- nextCursor: null,- pagesFetched: 0,- completed: false,- });- });+ });+ }+ );it.each(['preserved', 'reset'] as const)('restarts the runner with %s storage semantics and ignores obsolete completions',async (storage) => {localStorage.setItem(- 'graphql-soup-backfill:v15:user-1:core-entities',+ 'graphql-soup-backfill:v16:user-1:core-entities',JSON.stringify({userId: 'user-1',storageGeneration: 'storage-1',@@ -733,7 +893,7 @@ describe('runSoupBackfills', () => {const watermark = '2026-09-01T00:00:00.000Z';for (const checkpointId of ['core-entities', 'email-thread-pages']) {localStorage.setItem(- `graphql-soup-backfill:v15:user-1:${checkpointId}`,+ `graphql-soup-backfill:v16:user-1:${checkpointId}`,JSON.stringify({userId: 'user-1',storageGeneration: 'storage-1',diff --git a/apps/web/src/lib/queries/soup/backfill.ts b/apps/web/src/lib/queries/soup/backfill.tsindex aa0bc7762f..73f15282e5 100644--- a/apps/web/src/lib/queries/soup/backfill.ts+++ b/apps/web/src/lib/queries/soup/backfill.ts@@ -7,6 +7,7 @@ import { createTabLeaderSignal } from '@core/cross-tab/tab-leader';import type { CacheHost } from '@graphql-cache/host/types';import { Telemetry } from '@macro-inc/observability';import {+ type GraphqlEmailExpr,SoupBackfillDocument,SoupMailBackfillDocument,} from '@service-storage/graphql/generated/graphql';@@ -28,7 +29,7 @@ import { createEffect, createSignal, onCleanup } from 'solid-js';// persisted cursors cannot retain an older hydration contract.// Rehydrate raw file-type projections after retiring enum-normalized facts.// Old cursors must not skip records when the cache compatibility epoch changes.-const BACKFILL_VERSION = 15;+const BACKFILL_VERSION = 16;const PAGE_LIMIT = 100;// Five threads × twenty messages reaches the backend's 100-message cap.const EMAIL_CONTENT_PAGE_LIMIT = 5;@@ -39,6 +40,14 @@ const CACHE_HOST_RETRY_COUNT = 6;const CACHE_HOST_RETRY_SCHEDULE = Schedule.exponential('100 millis');const BACKFILL_PROGRESS_PAGE_INTERVAL = 10;const EXCLUDED_ENTITY_ID = '00000000-0000-0000-0000-000000000000';+const DAY_MS = 24 * 60 * 60 * 1000;++/** Email threads of one signal classification whose latest message is at most+ * `days` old. */+type EmailRecencyWindow = { signal: boolean; days: number };++const SIGNAL_EMAIL_WINDOW: EmailRecencyWindow = { signal: true, days: 90 };+const NOISE_EMAIL_WINDOW: EmailRecencyWindow = { signal: false, days: 30 };type SoupBackfillFetchPage = (input: GraphqlSoupInput,@@ -69,6 +78,9 @@ export type SoupBackfillParams = {/** Refresh the whole metadata corpus: message-time watermarks do not capture* read/archive changes to old email threads. Interrupted scans still resume. */refreshAll?: boolean;+ /** Email threads matching any window. The cutoff is fixed when a pass starts;+ * resumed cursors keep the cutoff they were issued with. */+ emailRecency?: readonly EmailRecencyWindow[];};/** Backfills the entities used most often by Quick Access and primary views. */@@ -91,56 +103,85 @@ export const CORE_SOUP_BACKFILL_LANE: SoupBackfillParams = {};/**- * Backfills email threads and the first message page used by the thread view- * while excluding every other entity variant with an impossible id filter.+ * Email threads and the first message page used by the thread view, excluding+ * every other entity variant with an impossible id filter.*/-export const EMAIL_SOUP_BACKFILL_LANE: SoupBackfillParams = {- checkpointId: 'email-thread-pages',+const EMAIL_THREAD_PAGES_INPUT: GraphqlSoupInitialInput = {+ limit: EMAIL_CONTENT_PAGE_LIMIT,+ expand: true,+ // Orders by the latest-message timestamp that emailRecency bounds, so each+ // page is an ordered index range ending at the cutoff.+ sortMethod: 'UPDATED_AT',+ emailView: 'ALL',+ filters: {+ calendarEventFilter: { literal: { id: EXCLUDED_ENTITY_ID } },+ documentFilter: { literal: { id: EXCLUDED_ENTITY_ID } },+ projectFilter: { literal: { projectIdSelf: EXCLUDED_ENTITY_ID } },+ chatFilter: { literal: { chatId: EXCLUDED_ENTITY_ID } },+ channelFilter: { literal: { channelId: EXCLUDED_ENTITY_ID } },+ channelThreadFilter: { literal: { threadId: EXCLUDED_ENTITY_ID } },+ callFilter: { literal: { callId: EXCLUDED_ENTITY_ID } },+ crmCompanyFilter: { literal: { id: EXCLUDED_ENTITY_ID } },+ foreignEntityFilter: { literal: { id: EXCLUDED_ENTITY_ID } },+ },+};++const EMAIL_THREAD_PAGES_LANE = {fetchPage: fetchEmailContentPage,- // A long full scan can skip threads created, viewed, or updated after its- // VIEWED_UPDATED cursor has passed them. Consume the recorded watermark- // before reporting this lane complete.+ // A long full scan can skip threads that receive messages after its cursor+ // has passed them. Consume the recorded watermark before reporting the lane+ // complete.catchUpAfterInitialPass: true,- input: {- limit: EMAIL_CONTENT_PAGE_LIMIT,- expand: true,- sortMethod: 'VIEWED_UPDATED',- emailView: 'ALL',- filters: {- calendarEventFilter: { literal: { id: EXCLUDED_ENTITY_ID } },- documentFilter: { literal: { id: EXCLUDED_ENTITY_ID } },- projectFilter: { literal: { projectIdSelf: EXCLUDED_ENTITY_ID } },- chatFilter: { literal: { chatId: EXCLUDED_ENTITY_ID } },- channelFilter: { literal: { channelId: EXCLUDED_ENTITY_ID } },- channelThreadFilter: { literal: { threadId: EXCLUDED_ENTITY_ID } },- callFilter: { literal: { callId: EXCLUDED_ENTITY_ID } },- crmCompanyFilter: { literal: { id: EXCLUDED_ENTITY_ID } },- foreignEntityFilter: { literal: { id: EXCLUDED_ENTITY_ID } },- },- },+ input: EMAIL_THREAD_PAGES_INPUT,+} satisfies Omit<SoupBackfillParams, 'checkpointId'>;++export const SIGNAL_EMAIL_SOUP_BACKFILL_LANE: SoupBackfillParams = {+ ...EMAIL_THREAD_PAGES_LANE,+ checkpointId: 'email-signal-thread-pages',+ emailRecency: [SIGNAL_EMAIL_WINDOW],+};++export const NOISE_EMAIL_SOUP_BACKFILL_LANE: SoupBackfillParams = {+ ...EMAIL_THREAD_PAGES_LANE,+ checkpointId: 'email-noise-thread-pages',+ emailRecency: [NOISE_EMAIL_WINDOW],};/** Filter/row metadata is synchronized before the independently bounded body cache.* ALL covers the first Mail slice across every readable owned/delegated inbox. */-export const EMAIL_FILTER_BACKFILL_LANE: SoupBackfillParams = {- checkpointId: 'email-filter-metadata',+const EMAIL_FILTER_LANE = {fetchPage: (input, options) =>hydrateGraphqlSoup(SoupMailBackfillDocument, { input }, options),refreshAll: true,- input: { ...EMAIL_SOUP_BACKFILL_LANE.input, limit: PAGE_LIMIT },+ input: { ...EMAIL_THREAD_PAGES_INPUT, limit: PAGE_LIMIT },+} satisfies Omit<SoupBackfillParams, 'checkpointId'>;++export const SIGNAL_EMAIL_FILTER_BACKFILL_LANE: SoupBackfillParams = {+ ...EMAIL_FILTER_LANE,+ checkpointId: 'email-signal-filter-metadata',+ emailRecency: [SIGNAL_EMAIL_WINDOW],+};++export const NOISE_EMAIL_FILTER_BACKFILL_LANE: SoupBackfillParams = {+ ...EMAIL_FILTER_LANE,+ checkpointId: 'email-noise-filter-metadata',+ emailRecency: [NOISE_EMAIL_WINDOW],};/** Shared grants are separate from owned/delegated inbox scope. A complete scan- * invalidates omitted old proof; interrupted scans preserve last-known evidence. */+ * invalidates omitted old proof; interrupted scans preserve last-known evidence.+ * One lane covers both windows because each scan invalidates every cached+ * Shared thread it did not return, including threads outside the windows. */export const SHARED_EMAIL_FILTER_BACKFILL_LANE: SoupBackfillParams = {checkpointId: 'shared-email-filter-metadata',createFetchPage: createSharedMailBackfillFetcher,refreshAll: true,restartOnRun: true,+ emailRecency: [SIGNAL_EMAIL_WINDOW, NOISE_EMAIL_WINDOW],input: {- ...EMAIL_FILTER_BACKFILL_LANE.input,+ ...EMAIL_FILTER_LANE.input,filters: {- ...EMAIL_FILTER_BACKFILL_LANE.input.filters,+ ...EMAIL_FILTER_LANE.input.filters,emailFilter: { tree: { literal: { shared: 'ONLY' } } },},},@@ -170,9 +211,11 @@ export const AUXILIARY_SOUP_BACKFILL_LANE: SoupBackfillParams = {/** Independently checkpointed backfills run serially in priority order. */export const DEFAULT_SOUP_BACKFILL_LANES = [CORE_SOUP_BACKFILL_LANE,- EMAIL_FILTER_BACKFILL_LANE,+ SIGNAL_EMAIL_FILTER_BACKFILL_LANE,+ NOISE_EMAIL_FILTER_BACKFILL_LANE,SHARED_EMAIL_FILTER_BACKFILL_LANE,- EMAIL_SOUP_BACKFILL_LANE,+ SIGNAL_EMAIL_SOUP_BACKFILL_LANE,+ NOISE_EMAIL_SOUP_BACKFILL_LANE,AUXILIARY_SOUP_BACKFILL_LANE,] as const satisfies readonly SoupBackfillParams[];@@ -342,8 +385,8 @@ export function withUpdatedSince(literal: { updatedAt: { gte: updatedSince } },};- // VIEWED_UPDATED can move a row ahead of the scan cursor through either- // the thread timestamp or this viewer's history timestamp. Cover both.+ // A cached email row changes through either the thread timestamp or this+ // viewer's history timestamp, and VIEWED_UPDATED sorts on both. Cover both.const emailSortWatermark = {or: {left: { literal: { updatedAt: { gte: updatedSince } } },@@ -366,6 +409,44 @@ export function withUpdatedSince(};}+/**+ * Restricts email threads to those matching any window. In the ALL view the+ * email updatedAt literal compares the latest non-spam message timestamp.+ */+function withEmailRecency(+ input: GraphqlSoupInitialInput,+ windows: readonly EmailRecencyWindow[] | undefined,+ now: number+): GraphqlSoupInitialInput {+ if (!windows?.length) return input;++ const recency = windows+ .map(+ ({ signal, days }): GraphqlEmailExpr => ({+ and: {+ left: { literal: { importance: signal } },+ right: {+ literal: {+ updatedAt: { gte: new Date(now - days * DAY_MS).toISOString() },+ },+ },+ },+ })+ )+ .reduce((left, right) => ({ or: { left, right } }));+ const filters = input.filters ?? {};+ return {+ ...input,+ filters: {+ ...filters,+ emailFilter: {+ ...(filters.emailFilter ?? {}),+ tree: and(filters.emailFilter?.tree, recency),+ },+ },+ };+}+type SoupBackfillCacheHost = Pick<CacheHost, 'currentStorageGeneration'>;export const runSoupBackfill = Effect.fn('runSoupBackfill')(function* (@@ -430,7 +511,7 @@ export const runSoupBackfill = Effect.fn('runSoupBackfill')(function* (continue;}const passInput = withUpdatedSince(- params.input,+ withEmailRecency(params.input, params.emailRecency, Date.now()),params.refreshAll ? null : checkpoint.updatedSince);const input: GraphqlSoupInput = checkpoint.nextCursordiff --git a/apps/web/tests/native/README.md b/apps/web/tests/native/README.mdindex 53b006aa6e..a55b729dd5 100644--- a/apps/web/tests/native/README.md+++ b/apps/web/tests/native/README.md@@ -46,8 +46,10 @@ file-backed native Turso cache.Fixtures replace the **remote services**, not the client cache. `fixtures/server.ts`serves the normal `/dss/items/soup/graphql` path, executes the app's generated-queries against `static_assets/schema.graphql`, and returns three pages of Mail-metadata. Projection capsules reuse the existing Rust-encoded fixtures.+queries against `static_assets/schema.graphql`, and returns two pages each of+signal and noise Mail metadata. Metadata requests must carry a recency cutoff,+but fixture timestamps are fixed, so the server does not apply it. Projection+capsules reuse the existing Rust-encoded fixtures.Unrelated app-chrome reads return explicit empty resources. Unexpected requests,unknown cursors and GraphQL validation errors are recorded and fail the smokesuite; nothing forwards to hosted services. This does not test the Rust HTTP@@ -62,11 +64,11 @@ The six threads are IDs ending in 4, 6, 8, 9, 10, and 12:| Offline All | 4, 6, 8, 9, 10, 12 |Only the initial Signal view may receive an online Mail response. The other rows-arrive via metadata backfill. Tests wait for the real three-page completion-checkpoint, then read the six normalized records through native IPC before-changing filters. Tab changes use the app's public numeric hotkeys (2 / 7) and-verify `aria-current` before checking results. Assertions compare exact row-identities, not just counts.+arrive via metadata backfill. Tests wait for the real signal and noise+completion checkpoints (four pages), then read the six normalized records+through native IPC before changing filters. Tab changes use the app's public+numeric hotkeys (2 / 7) and verify `aria-current` before checking results.+Assertions compare exact row identities, not just counts.## Exhaustive filter-selection matrix@@ -87,7 +89,7 @@ The corpus has 75 entities and fits within one initial page: this suite doesnot claim coverage of non-Mail offline pagination or every possible real-worlduser/tag/file value.-After all five real backfill lanes finish, the fixture API and WebSockets are+After all seven real backfill lanes finish, the fixture API and WebSockets areclosed. The browser-side runner calls the real Tauri cache host in boundedread-only batches and asserts exact matching IDs against fixture truth. Noquery-specific network baselines are supplied. Mail attachment selections alsodiff --git a/apps/web/tests/native/driver.ts b/apps/web/tests/native/driver.tsindex aed9703926..55716fa569 100644--- a/apps/web/tests/native/driver.ts+++ b/apps/web/tests/native/driver.ts@@ -47,15 +47,23 @@ export async function checkpoints(browser: Browser) {);}-export async function waitForBackfill(browser: Browser, pages = 3) {+const METADATA_LANES = [+ ':email-signal-filter-metadata',+ ':email-noise-filter-metadata',+];++export async function waitForBackfill(browser: Browser, pages: number) {await browser.waitUntil(- async () =>- (await checkpoints(browser)).some(- (checkpoint) =>- checkpoint.key.endsWith(':email-filter-metadata') &&- checkpoint.completed &&- checkpoint.pagesFetched === pages- ),+ async () => {+ const lanes = (await checkpoints(browser)).filter((checkpoint) =>+ METADATA_LANES.some((suffix) => checkpoint.key.endsWith(suffix))+ );+ return (+ lanes.length === METADATA_LANES.length &&+ lanes.every((lane) => lane.completed) &&+ lanes.reduce((total, lane) => total + lane.pagesFetched, 0) === pages+ );+ },{timeout: 120_000,timeoutMsg: `Real metadata backfill did not checkpoint all ${pages} pages`,diff --git a/apps/web/tests/native/fixtures/server.test.ts b/apps/web/tests/native/fixtures/server.test.tsindex 652eb64ad6..fe2b047abf 100644--- a/apps/web/tests/native/fixtures/server.test.ts+++ b/apps/web/tests/native/fixtures/server.test.ts@@ -44,52 +44,86 @@ async function requestSoup(return await response.json();}-test('real generated backfill document validates, pages, and includes projection/preview metadata', async () => {- server = startFixtureServer();- const ids: string[] = [];- let cursor: string | null = null;- for (let page = 0; page < 3; page++) {- const response = await fetch(`${server.origin}/dss/items/soup/graphql`, {- method: 'POST',- body: JSON.stringify({- query,- operationName: 'SoupMailBackfill',- variables: {- input: cursor- ? { continuation: { cursor, emailView: 'ALL' } }- : { initial: { limit: 100, emailView: 'ALL' } },- },- }),- });- const result: {- errors?: unknown;- data: {- user: {- soup: {- items: {- id: string;- cacheProjection: string;- mailAllPreview: { subject: string };- }[];- nextCursor: string | null;- };+const recencyTree = (importance: boolean) => ({+ and: {+ left: { literal: { importance } },+ right: { literal: { updatedAt: { gte: '2026-01-01T00:00:00.000Z' } } },+ },+});++async function requestMetadata(input: SoupInput) {+ if (!server) throw new Error('Fixture server must be started first');+ const response = await fetch(`${server.origin}/dss/items/soup/graphql`, {+ method: 'POST',+ body: JSON.stringify({+ query,+ operationName: 'SoupMailBackfill',+ variables: { input },+ }),+ });+ const result: {+ errors?: { message: string }[];+ data: {+ user: {+ soup: {+ items: {+ id: string;+ cacheProjection: string;+ mailAllPreview: { subject: string };+ }[];+ nextCursor: string | null;};};- } = await response.json();- expect(result.errors).toBeUndefined();- const soup = result.data.user.soup;- expect(soup.items).toHaveLength(2);- for (const item of soup.items) {- ids.push(item.id);- expect(item.cacheProjection).toBeString();- expect(item.mailAllPreview.subject).toStartWith('Email ');+ } | null;+ } = await response.json();+ return result;+}++test('real generated backfill document validates, pages each signal class, and includes projection/preview metadata', async () => {+ server = startFixtureServer();+ for (const [importance, expected] of [+ [true, [6, 9, 12]],+ [false, [4, 8, 10]],+ ] as const) {+ const ids: string[] = [];+ let cursor: string | null = null;+ for (let page = 0; page < 2; page++) {+ const result = await requestMetadata(+ cursor+ ? { continuation: { cursor, emailView: 'ALL' } }+ : {+ initial: {+ limit: 100,+ emailView: 'ALL',+ filters: { emailFilter: { tree: recencyTree(importance) } },+ },+ }+ );+ expect(result.errors).toBeUndefined();+ const soup = result.data!.user.soup;+ for (const item of soup.items) {+ ids.push(item.id);+ expect(item.cacheProjection).toBeString();+ expect(item.mailAllPreview.subject).toStartWith('Email ');+ }+ cursor = soup.nextCursor;+ if (page < 1) expect(cursor).toBeString();}- cursor = soup.nextCursor;- if (page < 2) expect(cursor).toBeString();+ expect(cursor).toBeNull();+ expect(ids).toEqual(expected.map(fixtureId));}- expect(cursor).toBeNull();- expect(ids).toEqual([4, 6, 8, 9, 10, 12].map(fixtureId));- expect(server.metadataPagesServed).toBe(3);+ expect(server.metadataPagesServed).toBe(4);+ expect(server.expectedMetadataPages).toBe(4);+});++test('metadata backfill without a recency-bounded signal class fails', async () => {+ server = startFixtureServer();+ const result = await requestMetadata({+ initial: { limit: 100, emailView: 'ALL' },+ });+ expect(result.errors?.map((error) => error.message)).toEqual([+ 'Metadata backfill must bound one signal class by recency',+ ]);});const signalTree = {diff --git a/apps/web/tests/native/fixtures/server.ts b/apps/web/tests/native/fixtures/server.tsindex 7293d8ac6e..f932e7c53a 100644--- a/apps/web/tests/native/fixtures/server.ts+++ b/apps/web/tests/native/fixtures/server.ts@@ -24,6 +24,17 @@ function excludesMail(input: SoupInput) {);}+type MetadataLane = 'signal' | 'noise';++/** Fixture timestamps are fixed, so the recency cutoff is required but not applied. */+function metadataLane(input: SoupInput): MetadataLane {+ const tree = input.initial?.filters?.emailFilter?.tree;+ const signal = tree?.and?.left?.literal?.importance;+ if (typeof signal !== 'boolean' || !tree?.and?.right?.literal?.updatedAt?.gte)+ throw new Error('Metadata backfill must bound one signal class by recency');+ return signal ? 'signal' : 'noise';+}+export type RequestRecord = {method: string;path: string;@@ -37,13 +48,17 @@ export type RequestRecord = {* No forwarding to a hosted backend, cache seeding, or filter-result injection. */export function startFixtureServer(port = 0, filterMatrix = false) {const corpus = filterMatrix ? filterCorpus(capsules) : undefined;- const metadataMail = corpus+ const metadataRows = corpus? corpus.filter((row) => row.kind === 'email' && LINKS.includes(row.linkId ?? ''))- .map((row) => row.api)- : mail;+ .map((row) => ({ signal: row.signal === true, api: row.api }))+ : mail.map((api) => ({ signal: api.isSignal, api }));+ const metadataMail: Record<MetadataLane, unknown[]> = {+ signal: metadataRows.filter((row) => row.signal).map((row) => row.api),+ noise: metadataRows.filter((row) => !row.signal).map((row) => row.api),+ };const sharedMail =corpus?.filter((row) => row.kind === 'email' && row.shared)@@ -63,26 +78,32 @@ export function startFixtureServer(port = 0, filterMatrix = false) {: [mail[1], mail[5]];const requests: RequestRecord[] = [];const pageSize = 2;- const cursors = new Map<string, number>();+ const cursors = new Map<string, { lane: MetadataLane; offset: number }>();let pagesServed = 0;- function soupPage(input: SoupInput, operation: string) {- if (operation === 'SoupMailBackfill') {- let offset = 0;- if (input.continuation) {- const saved = cursors.get(input.continuation.cursor);- if (saved === undefined) throw new Error('Unknown metadata cursor');- offset = saved;- } else if (input.initial?.emailView !== 'ALL') {- throw new Error('Metadata backfill must request ALL');- }- const nextOffset = offset + pageSize;- const nextCursor =- nextOffset < metadataMail.length ? `metadata-${nextOffset}` : null;- if (nextCursor) cursors.set(nextCursor, nextOffset);- pagesServed += 1;- return { items: metadataMail.slice(offset, nextOffset), nextCursor };+ function metadataPage(input: SoupInput) {+ let lane: MetadataLane;+ let offset = 0;+ if (input.continuation) {+ const saved = cursors.get(input.continuation.cursor);+ if (saved === undefined) throw new Error('Unknown metadata cursor');+ ({ lane, offset } = saved);+ } else if (input.initial?.emailView !== 'ALL') {+ throw new Error('Metadata backfill must request ALL');+ } else {+ lane = metadataLane(input);}+ const rows = metadataMail[lane];+ const nextOffset = offset + pageSize;+ const nextCursor =+ nextOffset < rows.length ? `metadata-${lane}-${nextOffset}` : null;+ if (nextCursor) cursors.set(nextCursor, { lane, offset: nextOffset });+ pagesServed += 1;+ return { items: rows.slice(offset, nextOffset), nextCursor };+ }++ function soupPage(input: SoupInput, operation: string) {+ if (operation === 'SoupMailBackfill') return metadataPage(input);if (operation === 'SoupSharedMailBackfill')return { items: sharedMail, nextCursor: null };if (operation === 'SoupBackfill') {@@ -229,7 +250,10 @@ export function startFixtureServer(port = 0, filterMatrix = false) {get socketCount() {return sockets.size;},- expectedMetadataPages: Math.ceil(metadataMail.length / pageSize),+ expectedMetadataPages: Object.values(metadataMail).reduce(+ (pages, rows) => pages + Math.max(1, Math.ceil(rows.length / pageSize)),+ 0+ ),initialSignalIds: signalMail.map((row) => String(row.id)),get metadataPagesServed() {return pagesServed;diff --git a/apps/web/tests/native/runner.ts b/apps/web/tests/native/runner.tsindex 19f82868f6..6f5fcdb258 100644--- a/apps/web/tests/native/runner.ts+++ b/apps/web/tests/native/runner.ts@@ -181,7 +181,7 @@ try {await browser.waitUntil(async () => {const lanes = await checkpoints(browser!);- return lanes.length === 5 && lanes.every((lane) => lane.completed);+ return lanes.length === 7 && lanes.every((lane) => lane.completed);},{ timeout: 60_000 });