Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(email-focus): show new mail in open Focus tabs within seconds (#9521)
teo21a6d138f232
diff --git a/Cargo.lock b/Cargo.lockindex ac12a8379e..c05e589109 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -7414,17 +7414,23 @@ version = "0.1.0"dependencies = ["ai_usage","chrono",+ "connection_gateway_client","doppler-config","email","jev","kafka_util",+ "macro_auth","macro_config",+ "macro_db_migrator","macro_entrypoint","macro_env","macro_env_var","macro_event_broker",+ "macro_service_urls","macro_user_id","memory",+ "models_email",+ "opentelemetry","rdkafka","rootcause","serde",@@ -7433,7 +7439,9 @@ dependencies = ["tokio","tokio-util","tracing",+ "tracing-opentelemetry","uuid",+ "wiremock","workspace-hack",]diff --git a/apps/web/src/lib/queries/email/focus.test.ts b/apps/web/src/lib/queries/email/focus.test.tsnew file mode 100644index 0000000000..93f9263433--- /dev/null+++ b/apps/web/src/lib/queries/email/focus.test.ts@@ -0,0 +1,210 @@+import { WebsocketEvent } from '@macro-inc/collaboration/websocket';+import { Telemetry } from '@macro-inc/observability';+import { createClient, type Operation } from '@urql/core';+import { createRoot, createSignal } from 'solid-js';+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';+import { map, pipe, tap } from 'wonka';+import { useEmailFocusQuery } from './focus';++const socket = vi.hoisted(() => ({+ messages: new Set<(message: { type: string; data: unknown }) => void>(),+ reconnects: new Set<() => void>(),+}));++// Exercise the real query binding without UI barrels or live connections.+vi.mock('@service-connection/websocket', async () => ({+ ...(await import('@service-connection/websocket-payload')),+ createConnectionWebsocketEffect: (+ callback: (message: { type: string; data: unknown }) => void+ ) => {+ socket.messages.add(callback);+ },+ ws: {+ addEventListener: (event: WebsocketEvent, callback: () => void) => {+ if (event === WebsocketEvent.Reconnect) socket.reconnects.add(callback);+ },+ removeEventListener: (event: WebsocketEvent, callback: () => void) => {+ if (event === WebsocketEvent.Reconnect)+ socket.reconnects.delete(callback);+ },+ },+}));+vi.mock('@entity', () => ({ isEmailEntity: () => true }));+vi.mock('@service-storage/graphql-soup', () => ({+ getGraphqlSoupClient: () => client,+ mapGraphqlSoupItem: () => undefined,+}));+vi.mock('../soup/transform-utils', () => ({+ isDisplayableSoupItem: () => false,+ mapApiSoupItemToEntity: () => undefined,+}));++const reads: Operation[] = [];+const client = createClient({+ url: '/graphql',+ exchanges: [+ () => (operations) =>+ pipe(+ operations,+ tap((operation: Operation) => {+ if (operation.kind === 'query') reads.push(operation);+ }),+ map((operation: Operation) => ({+ operation,+ data: { user: { id: 'viewer', emailFocus: [] } },+ stale: false,+ hasNext: false,+ }))+ ),+ ],+});++function push(data: unknown, type = 'refresh_email') {+ for (const message of socket.messages) {+ message({ type, data: JSON.stringify(data) });+ }+}++const FOCUS_UPDATED = { event: 'focus_updated', link_id: 'inbox' };++let visibility: DocumentVisibilityState;+let dispose: (() => void) | undefined;++function setVisibility(value: DocumentVisibilityState) {+ visibility = value;+ document.dispatchEvent(new Event('visibilitychange'));+}++/** Mount the Focus list and settle its first read. */+async function mount(enabled = true) {+ const [isEnabled, setEnabled] = createSignal(enabled);+ dispose = createRoot((disposeRoot) => {+ useEmailFocusQuery(isEnabled);+ return disposeRoot;+ });+ await vi.advanceTimersByTimeAsync(0);+ reads.length = 0;+ return { setEnabled };+}++beforeEach(() => {+ vi.useFakeTimers();+ visibility = 'visible';+ vi.spyOn(document, 'visibilityState', 'get').mockImplementation(+ () => visibility+ );+ reads.length = 0;+});++afterEach(() => {+ dispose?.();+ dispose = undefined;+ socket.messages.clear();+ socket.reconnects.clear();+ vi.useRealTimers();+ vi.restoreAllMocks();+});++describe('useEmailFocusQuery', () => {+ it('reads Focus from the server when the worker stores a classification', async () => {+ await mount();++ push(FOCUS_UPDATED);+ await vi.advanceTimersByTimeAsync(0);++ expect(reads).toHaveLength(1);+ expect(reads[0]?.context.requestPolicy).toBe('network-only');+ expect(reads[0]?.variables).toEqual({ days: 30 });+ });++ it('turns a burst of pushes into few reads', async () => {+ await mount();++ for (let index = 0; index < 20; index++) push(FOCUS_UPDATED);+ await vi.advanceTimersByTimeAsync(1_000);++ // At once, then one for everything that came while it was throttled.+ expect(reads).toHaveLength(2);+ });++ it('ignores other inbox refreshes and other messages', async () => {+ await mount();++ push({ event: 'upsert_message', link_id: 'inbox' });+ push(FOCUS_UPDATED, 'message_update');+ await vi.advanceTimersByTimeAsync(1_000);++ expect(reads).toHaveLength(0);+ });++ it('holds a hidden tab to one read when it next shows', async () => {+ await mount();+ setVisibility('hidden');++ push(FOCUS_UPDATED);+ push(FOCUS_UPDATED);+ await vi.advanceTimersByTimeAsync(10_000);+ expect(reads).toHaveLength(0);++ setVisibility('visible');+ await vi.advanceTimersByTimeAsync(1_000);+ expect(reads).toHaveLength(1);+ });++ it('reads when a hidden tab shows, in case a push was lost', async () => {+ await mount();+ setVisibility('hidden');+ await vi.advanceTimersByTimeAsync(10_000);+ expect(reads).toHaveLength(0);++ setVisibility('visible');+ await vi.advanceTimersByTimeAsync(1_000);+ expect(reads).toHaveLength(1);+ });++ it('leaves a closed Focus tab to read when it opens', async () => {+ const { setEnabled } = await mount(false);++ push(FOCUS_UPDATED);+ await vi.advanceTimersByTimeAsync(1_000);+ expect(reads).toHaveLength(0);++ setEnabled(true);+ await vi.advanceTimersByTimeAsync(1_000);+ expect(reads).toHaveLength(1);+ });++ it('reads again after a reconnect that may have missed a push', async () => {+ await mount();++ for (const reconnect of socket.reconnects) reconnect();+ await vi.advanceTimersByTimeAsync(0);++ expect(reads).toHaveLength(1);+ });++ it('stops listening when it unmounts', async () => {+ await mount();+ dispose?.();+ dispose = undefined;++ for (const reconnect of socket.reconnects) reconnect();+ await vi.advanceTimersByTimeAsync(1_000);++ expect(socket.reconnects.size).toBe(0);+ expect(reads).toHaveLength(0);+ });++ it('joins the trace the push carries', async () => {+ const remoteContext = vi.spyOn(Telemetry, 'remoteContext');+ await mount();+ const traceparent =+ '00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01';++ push({ ...FOCUS_UPDATED, traceparent });+ await vi.advanceTimersByTimeAsync(0);++ expect(remoteContext).toHaveBeenCalledWith(traceparent);+ expect(reads[0]?.context.macroTraceParent).toBeDefined();+ });+});diff --git a/apps/web/src/lib/queries/email/focus.ts b/apps/web/src/lib/queries/email/focus.tsindex 72f391c2da..74435240d1 100644--- a/apps/web/src/lib/queries/email/focus.ts+++ b/apps/web/src/lib/queries/email/focus.ts@@ -1,5 +1,14 @@+import { TRACE_PARENT_CONTEXT_KEY } from '@app/lib/urql-solid/trace-parent-exchange';import { type EmailEntity, isEmailEntity } from '@entity';import { createLiveQuery } from '@graphql-cache/solid/create-live-query';+import { WebsocketEvent } from '@macro-inc/collaboration/websocket';+import { Telemetry, type TraceContext } from '@macro-inc/observability';+import {+ createConnectionWebsocketEffect,+ parseWebsocketPayload,+ ws,+} from '@service-connection/websocket';+import type { RefreshEmailEvent } from '@service-email/generated/schemas';import {EmailFocusDocument,type EmailFocusQuery,@@ -8,11 +17,12 @@ import {getGraphqlSoupClient,mapGraphqlSoupItem,} from '@service-storage/graphql-soup';-import type { Accessor } from 'solid-js';+import { type Accessor, onCleanup } from 'solid-js';import {isDisplayableSoupItem,mapApiSoupItemToEntity,} from '../soup/transform-utils';+import { createVisibleRefresh } from '../visible-refresh';/** Days of mail the Focus list covers. */export const FOCUS_WINDOW_DAYS = 30;@@ -40,9 +50,14 @@ export function mapEmailFocus(data: EmailFocusQuery): EmailEntity[] {return threads;}-/** The viewer's Focus list, read through the shared GraphQL client. */+/**+ * The viewer's Focus list, read through the shared GraphQL client. The list is+ * one read the normalized cache can't maintain, so the worker's `focus_updated`+ * push, or a reconnect that may have missed one, reads it again. A hidden tab+ * catches up once when it next shows; a closed tab reads when it opens.+ */export function useEmailFocusQuery(enabled: Accessor<boolean>) {- return createLiveQuery(+ const query = createLiveQuery(EmailFocusDocument,() => (enabled() ? { days: FOCUS_WINDOW_DAYS } : undefined),() => ({@@ -51,4 +66,47 @@ export function useEmailFocusQuery(enabled: Accessor<boolean>) {select: mapEmailFocus,}));++ let pushed: TraceContext | undefined;+ const refresh = createVisibleRefresh(+ () => {+ if (!enabled()) return;+ const parent = pushed ?? Telemetry.captureContext();+ pushed = undefined;+ return parent.run(() =>+ Telemetry.span('email.focus.refresh', async (span) => {+ const result = await query.refetch({+ requestPolicy: 'network-only',+ // A query made in a view's flow would otherwise send in that trace.+ context: { [TRACE_PARENT_CONTEXT_KEY]: span },+ });+ span.setAttr('email.focus.threads', result.data?.length ?? 0);+ if (result.error) span.error('Focus refresh failed');+ })+ );+ },+ // The push is best effort, so one lost while hidden must not outlast the tab showing.+ { refreshOnShow: true }+ );+ onCleanup(refresh.dispose);++ createConnectionWebsocketEffect((message) => {+ if (message.type !== 'refresh_email' || !enabled()) return;+ const event = parseWebsocketPayload<RefreshEmailEvent>(+ message.type,+ message.data+ );+ if (event?.event !== 'focus_updated') return;+ if (event.traceparent) pushed = Telemetry.remoteContext(event.traceparent);+ refresh.markDirty();+ });+ const onReconnect = () => {+ if (enabled()) refresh.markDirty();+ };+ ws.addEventListener(WebsocketEvent.Reconnect, onReconnect);+ onCleanup(() =>+ ws.removeEventListener(WebsocketEvent.Reconnect, onReconnect)+ );++ return query;}diff --git a/apps/web/src/lib/queries/email/sync.ts b/apps/web/src/lib/queries/email/sync.tsindex 9bf0b625d4..cbcc01198a 100644--- a/apps/web/src/lib/queries/email/sync.ts+++ b/apps/web/src/lib/queries/email/sync.ts@@ -52,6 +52,7 @@ function asRefreshEmailEvent(payload: unknown): RefreshEmailEvent | undefined {* `SYNCING`, with a toast. The legacy `backfill` event now only signals* failure. `photo_synced` refetches the links list so the inbox's derived* `photo_url` lands once its self-contact photo finishes uploading.+ * `focus_updated` belongs to the mounted Focus list (see `useEmailFocusQuery`).*/export function handleRefreshEmail(payload: unknown): void {const event = asRefreshEmailEvent(payload);diff --git a/apps/web/src/lib/queries/subscribe-to-visible-cache-changes.ts b/apps/web/src/lib/queries/subscribe-to-visible-cache-changes.tsindex b28e27be54..0f097e3a35 100644--- a/apps/web/src/lib/queries/subscribe-to-visible-cache-changes.ts+++ b/apps/web/src/lib/queries/subscribe-to-visible-cache-changes.ts@@ -1,17 +1,10 @@-import { fireAndForget } from '@core/util/fireAndForget';import type { CacheHost } from '@graphql-cache/host/types';-import { makeEventListener } from '@solid-primitives/event-listener';-import { leadingAndTrailing, throttle } from '@solid-primitives/scheduled';import { type Accessor, untrack } from 'solid-js';--const CACHE_REFRESH_INTERVAL_MS = 250;+import { createVisibleRefresh } from './visible-refresh';/**- * Throttles cache-driven UI refreshes while visible. Hidden tabs retain one- * dirty bit instead of repeatedly reading the shared cache; becoming visible- * refreshes once against its latest state. Async refreshes finish before one- * dirty follow-up runs, so a busy cache cannot continually cancel its readers.- * This does not pause cache ingestion, mutations, or explicit/initial requests.+ * Refreshes after cache changes on `createVisibleRefresh`'s terms. This does+ * not pause cache ingestion, mutations, or explicit/initial requests.** The caller must unsubscribe on disposal.*/@@ -20,32 +13,8 @@ export function subscribeToVisibleCacheChanges(refresh: () => void | Promise<unknown>,options?: { searchBuckets: Accessor<readonly string[]> }): () => void {- let dirty = false;+ const visibleRefresh = createVisibleRefresh(refresh);let disposed = false;- let refreshing = false;- const isVisible = () => document.visibilityState === 'visible';- const runRefresh = async () => {- // Visibility can change between scheduling and the trailing callback.- if (disposed || refreshing || !dirty || !isVisible()) return;- dirty = false;- refreshing = true;- try {- const completion = refresh();- if (completion) await completion;- } catch (error) {- console.warn('Cache-driven refresh failed', error);- } finally {- refreshing = false;- // Keep changes received during a read, including the last change in a- // burst. A failed read only retries if another change made it dirty.- if (dirty && !disposed && isVisible()) scheduled();- }- };- const scheduled = leadingAndTrailing(- throttle,- () => fireAndForget(runRefresh(), 'visible-cache-refresh'),- CACHE_REFRESH_INTERVAL_MS- );const unsubscribe = host.onCacheChanged((_revision, changes) => {if (disposed) return;@@ -54,26 +23,14 @@ export function subscribeToVisibleCacheChanges(const watched = untrack(options.searchBuckets);if (!changedBuckets.some((bucket) => watched.includes(bucket))) return;}- dirty = true;- if (isVisible()) scheduled();+ visibleRefresh.markDirty();},{ includeHydration: true });- const removeVisibilityListener = makeEventListener(- document,- 'visibilitychange',- () => {- // Cancel pending background work without losing its dirty state. Reset- // the throttle so a visible tab catches up immediately, not on a timer.- scheduled.clear();- if (dirty && !disposed && isVisible()) scheduled();- }- );return () => {disposed = true;unsubscribe();- removeVisibilityListener();- scheduled.clear();+ visibleRefresh.dispose();};}diff --git a/apps/web/src/lib/queries/visible-refresh.ts b/apps/web/src/lib/queries/visible-refresh.tsnew file mode 100644index 0000000000..4c8631acbf--- /dev/null+++ b/apps/web/src/lib/queries/visible-refresh.ts@@ -0,0 +1,75 @@+import { fireAndForget } from '@core/util/fireAndForget';+import { makeEventListener } from '@solid-primitives/event-listener';+import { leadingAndTrailing, throttle } from '@solid-primitives/scheduled';++const REFRESH_INTERVAL_MS = 250;++/**+ * Throttles refreshes while visible. Hidden tabs retain one dirty bit instead+ * of repeatedly reading; becoming visible refreshes once against the latest+ * state. Async refreshes finish before one dirty follow-up runs, so a busy+ * source cannot continually cancel its readers. With `refreshOnShow`, becoming+ * visible refreshes even if nothing marked it dirty, for a source whose change+ * signal can be lost.+ *+ * The caller must dispose it.+ */+export function createVisibleRefresh(+ refresh: () => void | Promise<unknown>,+ { refreshOnShow = false }: { refreshOnShow?: boolean } = {}+): {+ markDirty: () => void;+ dispose: () => void;+} {+ let dirty = false;+ let disposed = false;+ let refreshing = false;+ const isVisible = () => document.visibilityState === 'visible';+ const runRefresh = async () => {+ // Visibility can change between scheduling and the trailing callback.+ if (disposed || refreshing || !dirty || !isVisible()) return;+ dirty = false;+ refreshing = true;+ try {+ const completion = refresh();+ if (completion) await completion;+ } catch (error) {+ console.warn('Visible refresh failed', error);+ } finally {+ refreshing = false;+ // Keep changes received during a read, including the last change in a+ // burst. A failed read only retries if another change made it dirty.+ if (dirty && !disposed && isVisible()) scheduled();+ }+ };+ const scheduled = leadingAndTrailing(+ throttle,+ () => fireAndForget(runRefresh(), 'visible-refresh'),+ REFRESH_INTERVAL_MS+ );+ const removeVisibilityListener = makeEventListener(+ document,+ 'visibilitychange',+ () => {+ // Cancel pending background work without losing its dirty state. Reset+ // the throttle so a visible tab catches up immediately, not on a timer.+ scheduled.clear();+ if (disposed || !isVisible()) return;+ if (refreshOnShow) dirty = true;+ if (dirty) scheduled();+ }+ );++ return {+ markDirty: () => {+ if (disposed) return;+ dirty = true;+ if (isVisible()) scheduled();+ },+ dispose: () => {+ disposed = true;+ removeVisibilityListener();+ scheduled.clear();+ },+ };+}diff --git a/apps/web/src/lib/service-clients/service-email/generated/schemas/index.ts b/apps/web/src/lib/service-clients/service-email/generated/schemas/index.tsindex 4123f3d80d..3eedb2c784 100644--- a/apps/web/src/lib/service-clients/service-email/generated/schemas/index.ts+++ b/apps/web/src/lib/service-clients/service-email/generated/schemas/index.ts@@ -365,6 +365,9 @@ export * from './refreshEmailEventOneOfFive';export * from './refreshEmailEventOneOfFiveEvent';export * from './refreshEmailEventOneOfNine';export * from './refreshEmailEventOneOfNineEvent';+export * from './refreshEmailEventOneOfOnefive';+export * from './refreshEmailEventOneOfOnefiveEvent';+export * from './refreshEmailEventOneOfOnefiveTraceparent';export * from './refreshEmailEventOneOfOneone';export * from './refreshEmailEventOneOfOneoneEvent';export * from './refreshEmailEventOneOfOnethree';diff --git a/apps/web/src/lib/service-clients/service-email/generated/schemas/refreshEmailEvent.ts b/apps/web/src/lib/service-clients/service-email/generated/schemas/refreshEmailEvent.tsindex dd995b274c..e5ad2fe0e9 100644--- a/apps/web/src/lib/service-clients/service-email/generated/schemas/refreshEmailEvent.ts+++ b/apps/web/src/lib/service-clients/service-email/generated/schemas/refreshEmailEvent.ts@@ -7,6 +7,7 @@import type { RefreshEmailEventOneOf } from './refreshEmailEventOneOf';import type { RefreshEmailEventOneOfFive } from './refreshEmailEventOneOfFive';import type { RefreshEmailEventOneOfNine } from './refreshEmailEventOneOfNine';+import type { RefreshEmailEventOneOfOnefive } from './refreshEmailEventOneOfOnefive';import type { RefreshEmailEventOneOfOneone } from './refreshEmailEventOneOfOneone';import type { RefreshEmailEventOneOfOnethree } from './refreshEmailEventOneOfOnethree';import type { RefreshEmailEventOneOfSeven } from './refreshEmailEventOneOfSeven';@@ -23,4 +24,5 @@ export type RefreshEmailEvent =| RefreshEmailEventOneOfSeven| RefreshEmailEventOneOfNine| RefreshEmailEventOneOfOneone- | RefreshEmailEventOneOfOnethree;+ | RefreshEmailEventOneOfOnethree+ | RefreshEmailEventOneOfOnefive;diff --git a/apps/web/src/lib/service-clients/service-email/generated/schemas/refreshEmailEventOneOfOnefive.ts b/apps/web/src/lib/service-clients/service-email/generated/schemas/refreshEmailEventOneOfOnefive.tsnew file mode 100644index 0000000000..c96f4ca85c--- /dev/null+++ b/apps/web/src/lib/service-clients/service-email/generated/schemas/refreshEmailEventOneOfOnefive.ts@@ -0,0 +1,20 @@+/**+ * Generated by orval v7.21.0 🍺+ * Do not edit manually.+ * email_service+ * OpenAPI spec version: 0.1.0+ */+import type { RefreshEmailEventOneOfOnefiveEvent } from './refreshEmailEventOneOfOnefiveEvent';+import type { RefreshEmailEventOneOfOnefiveTraceparent } from './refreshEmailEventOneOfOnefiveTraceparent';++/**+ * A Focus classification was stored for a thread in `link_id`, so its+readers read their Focus list again. `traceparent` is the W3C trace+context of the classification, for that read to join. Coalesced by the+sender; carries nothing about the mail.+ */+export type RefreshEmailEventOneOfOnefive = {+ event: RefreshEmailEventOneOfOnefiveEvent;+ link_id: string;+ traceparent?: RefreshEmailEventOneOfOnefiveTraceparent;+};diff --git a/apps/web/src/lib/service-clients/service-email/generated/schemas/refreshEmailEventOneOfOnefiveEvent.ts b/apps/web/src/lib/service-clients/service-email/generated/schemas/refreshEmailEventOneOfOnefiveEvent.tsnew file mode 100644index 0000000000..53c3403b9b--- /dev/null+++ b/apps/web/src/lib/service-clients/service-email/generated/schemas/refreshEmailEventOneOfOnefiveEvent.ts@@ -0,0 +1,14 @@+/**+ * Generated by orval v7.21.0 🍺+ * Do not edit manually.+ * email_service+ * OpenAPI spec version: 0.1.0+ */++export type RefreshEmailEventOneOfOnefiveEvent =+ (typeof RefreshEmailEventOneOfOnefiveEvent)[keyof typeof RefreshEmailEventOneOfOnefiveEvent];++// eslint-disable-next-line @typescript-eslint/no-redeclare+export const RefreshEmailEventOneOfOnefiveEvent = {+ focus_updated: 'focus_updated',+} as const;diff --git a/apps/web/src/lib/service-clients/service-email/generated/schemas/refreshEmailEventOneOfOnefiveTraceparent.ts b/apps/web/src/lib/service-clients/service-email/generated/schemas/refreshEmailEventOneOfOnefiveTraceparent.tsnew file mode 100644index 0000000000..2cfa841774--- /dev/null+++ b/apps/web/src/lib/service-clients/service-email/generated/schemas/refreshEmailEventOneOfOnefiveTraceparent.ts@@ -0,0 +1,8 @@+/**+ * Generated by orval v7.21.0 🍺+ * Do not edit manually.+ * email_service+ * OpenAPI spec version: 0.1.0+ */++export type RefreshEmailEventOneOfOnefiveTraceparent = string | null;diff --git a/apps/web/src/lib/service-clients/service-email/openapi.json b/apps/web/src/lib/service-clients/service-email/openapi.jsonindex 4671c31a92..8c7eaa66a4 100644--- a/apps/web/src/lib/service-clients/service-email/openapi.json+++ b/apps/web/src/lib/service-clients/service-email/openapi.json@@ -7295,6 +7295,24 @@"format": "int32"}}+ },+ {+ "type": "object",+ "description": "A Focus classification was stored for a thread in `link_id`, so its\nreaders read their Focus list again. `traceparent` is the W3C trace\ncontext of the classification, for that read to join. Coalesced by the\nsender; carries nothing about the mail.",+ "required": ["link_id", "event"],+ "properties": {+ "event": {+ "type": "string",+ "enum": ["focus_updated"]+ },+ "link_id": {+ "type": "string",+ "format": "uuid"+ },+ "traceparent": {+ "type": ["string", "null"]+ }+ }}],"description": "Payload for the `refresh_email` connection gateway event: identifies the\ninbox that changed and the kind of change."diff --git a/crates/connection_gateway_client/src/client/email.rs b/crates/connection_gateway_client/src/client/email.rsindex 8f024bc727..8193023201 100644--- a/crates/connection_gateway_client/src/client/email.rs+++ b/crates/connection_gateway_client/src/client/email.rs@@ -2,6 +2,8 @@ use super::ConnectionGatewayClient;use connection_gateway_models::MessageReceipt;use model_entity::EntityType;+const REFRESH_EMAIL: &str = "refresh_email";+impl ConnectionGatewayClient {/// Trigger refresh of the user's emails#[tracing::instrument(skip(self, event))]@@ -12,8 +14,27 @@ impl ConnectionGatewayClient {) -> anyhow::Result<Vec<MessageReceipt>> {self.send_message(EntityType::User.with_entity_str(user_id),- "refresh_email".to_string(),+ REFRESH_EMAIL.to_string(),+ event,+ )+ .await+ }++ /// Trigger a refresh for everyone reading an inbox. User ids hold email+ /// addresses, so they stay out of the span.+ #[tracing::instrument(skip_all, fields(users = user_ids.len()))]+ pub async fn refresh_email_for_users(+ &self,+ user_ids: &[String],+ event: serde_json::Value,+ ) -> anyhow::Result<Vec<MessageReceipt>> {+ self.batch_send_message(+ REFRESH_EMAIL.to_string(),event,+ user_ids+ .iter()+ .map(|user_id| EntityType::User.with_entity_str(user_id))+ .collect(),).await}diff --git a/crates/email/src/domain/focus.rs b/crates/email/src/domain/focus.rsindex fb89a3e8f5..f66ffa0dd0 100644--- a/crates/email/src/domain/focus.rs+++ b/crates/email/src/domain/focus.rs@@ -29,7 +29,7 @@ pub use models::{FocusRecord, FocusSignals, FocusThread, FocusVerdict, SentNote, StaleThread,StoredClassification, ThreadFocus,};-pub use ports::{FocusClassifier, FocusClassifierError, FocusStore, ProfileSource};+pub use ports::{FocusClassifier, FocusClassifierError, FocusNotifier, FocusStore, ProfileSource};pub use questions::{FOCUS_QUESTIONS, FocusQuestion};pub use rule::{FOCUS_RULE_VERSION, decide};pub use service::{ClassifyOutcome, FocusQueries, FocusService, SweepReport};diff --git a/crates/email/src/domain/focus/ports.rs b/crates/email/src/domain/focus/ports.rsindex de0ddfe5a7..57ef95e502 100644--- a/crates/email/src/domain/focus/ports.rs+++ b/crates/email/src/domain/focus/ports.rs@@ -77,6 +77,14 @@ pub trait ProfileSource: Send + Sync + 'static {) -> impl Future<Output = Result<Option<String>, Report>> + Send;}+/// Tells the people reading an inbox that its Focus changed.+pub trait FocusNotifier: Send + Sync + 'static {+ /// A classification was stored for a thread in `link_id`, an inbox `owner`+ /// holds. Returns at once: delivery is best effort and never holds up or+ /// fails a classification.+ fn focus_changed(&self, owner: &MacroUserIdStr<'static>, link_id: Uuid);+}+/// Why the classifier gave no answers.#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]pub enum FocusClassifierError {diff --git a/crates/email/src/domain/focus/service.rs b/crates/email/src/domain/focus/service.rsindex c5d5135241..0a41343418 100644--- a/crates/email/src/domain/focus/service.rs+++ b/crates/email/src/domain/focus/service.rs@@ -16,7 +16,7 @@ use super::{FocusAnswers, FocusMessage, FocusRecord, FocusThread, FocusVerdict, StaleThread,StoredClassification, ThreadFocus,},- ports::{FocusClassifier, FocusClassifierError, FocusStore, ProfileSource},+ ports::{FocusClassifier, FocusClassifierError, FocusNotifier, FocusStore, ProfileSource},questions::FOCUS_QUESTIONS,rule::{FOCUS_RULE_VERSION, decide},signals::signals,@@ -76,10 +76,11 @@ pub struct SweepReport {}/// Classifies threads for inboxes on the allowlist.-pub struct FocusService<S, P, C> {+pub struct FocusService<S, P, C, N> {store: S,profiles: P,classifier: C,+ notifier: N,enabled_domains: Vec<String>,/// Threads a sweep checked at their latest mail without storing a result/// (rejected, nothing to classify, or a result already stored), so later@@ -88,15 +89,22 @@ pub struct FocusService<S, P, C> {settled: Mutex<HashMap<Uuid, DateTime<Utc>>>,}-impl<S, P, C> FocusService<S, P, C>+impl<S, P, C, N> FocusService<S, P, C, N>whereS: FocusStore,P: ProfileSource,C: FocusClassifier,+ N: FocusNotifier,{/// Classify only inboxes whose address is at one of `enabled_domains`;/// with none, nothing is classified.- pub fn new(store: S, profiles: P, classifier: C, enabled_domains: Vec<String>) -> Self {+ pub fn new(+ store: S,+ profiles: P,+ classifier: C,+ notifier: N,+ enabled_domains: Vec<String>,+ ) -> Self {let enabled_domains = enabled_domains.into_iter().map(|domain| domain.trim().to_lowercase())@@ -106,6 +114,7 @@ wherestore,profiles,classifier,+ notifier,enabled_domains,settled: Mutex::default(),}@@ -232,7 +241,8 @@ whereanswers: FocusAnswers,model: String,) -> Result<bool, Report> {- self.store+ let stored = self+ .store.save(&FocusRecord {thread_id: thread.thread_id,link_id: thread.link_id,@@ -243,7 +253,11 @@ wheremodel,classified_at: Utc::now(),})- .await+ .await?;+ if stored {+ self.notifier.focus_changed(&thread.owner, thread.link_id);+ }+ Ok(stored)}/// Check up to `limit` stale threads from the last `window`, newest first,diff --git a/crates/email/src/domain/focus/service/test.rs b/crates/email/src/domain/focus/service/test.rsindex bcb210d078..d2ca7b7757 100644--- a/crates/email/src/domain/focus/service/test.rs+++ b/crates/email/src/domain/focus/service/test.rs@@ -122,6 +122,17 @@ impl ProfileSource for FakeProfiles {}}+#[derive(Clone, Default)]+struct FakeNotifier {+ changed: Arc<Mutex<Vec<(MacroUserIdStr<'static>, Uuid)>>>,+}++impl FocusNotifier for FakeNotifier {+ fn focus_changed(&self, owner: &MacroUserIdStr<'static>, link_id: Uuid) {+ self.changed.lock().unwrap().push((owner.clone(), link_id));+ }+}+#[derive(Clone)]struct FakeClassifier {answers: Result<Vec<f32>, FocusClassifierError>,@@ -192,11 +203,21 @@ fn service(store: &FakeStore,classifier: &FakeClassifier,domains: &[&str],-) -> FocusService<FakeStore, FakeProfiles, FakeClassifier> {+) -> FocusService<FakeStore, FakeProfiles, FakeClassifier, FakeNotifier> {+ notifying_service(store, classifier, domains, &FakeNotifier::default())+}++fn notifying_service(+ store: &FakeStore,+ classifier: &FakeClassifier,+ domains: &[&str],+ notifier: &FakeNotifier,+) -> FocusService<FakeStore, FakeProfiles, FakeClassifier, FakeNotifier> {FocusService::new(store.clone(),FakeProfiles(Some("Owner runs finance at Acme.".to_owned())),classifier.clone(),+ notifier.clone(),domains.iter().map(|domain| (*domain).to_owned()).collect(),)}@@ -530,6 +551,100 @@ async fn sweep_stops_when_the_classifier_rejects_everything() {);}+#[tokio::test]+async fn a_stored_classification_tells_the_inbox_readers() {+ let store = FakeStore::default();+ let classifier = FakeClassifier::answering(Ok(relevant()));+ let notifier = FakeNotifier::default();+ let thread = thread("[email protected]");+ let (owner, link_id) = (thread.owner.clone(), thread.link_id);+ let thread_id = insert(&store, thread);++ notifying_service(&store, &classifier, &["acme.example"], ¬ifier)+ .classify_thread(thread_id)+ .await+ .unwrap();++ assert_eq!(*notifier.changed.lock().unwrap(), vec![(owner, link_id)]);+}++#[tokio::test]+async fn nothing_stored_tells_nobody() {+ let notifier = FakeNotifier::default();++ // A later message's result is already stored.+ let refusing = FakeStore {+ refuse_saves: true,+ ..FakeStore::default()+ };+ let thread_id = insert(&refusing, thread("[email protected]"));+ let classifier = FakeClassifier::answering(Ok(relevant()));+ let outcome = notifying_service(&refusing, &classifier, &["acme.example"], ¬ifier)+ .classify_thread(thread_id)+ .await+ .unwrap();+ assert_eq!(outcome, ClassifyOutcome::UpToDate);++ // The classifier gave no answers.+ let store = FakeStore::default();+ let thread_id = insert(&store, thread("[email protected]"));+ let classifier = FakeClassifier::answering(Err(FocusClassifierError::Unavailable));+ let outcome = notifying_service(&store, &classifier, &["acme.example"], ¬ifier)+ .classify_thread(thread_id)+ .await+ .unwrap();+ assert_eq!(outcome, ClassifyOutcome::Unavailable);++ assert!(notifier.changed.lock().unwrap().is_empty());+}++#[tokio::test]+async fn deciding_again_under_a_new_rule_tells_the_inbox_readers() {+ let store = FakeStore::default();+ let classifier = FakeClassifier::answering(Err(FocusClassifierError::Unavailable));+ let notifier = FakeNotifier::default();+ let mut legacy = thread("[email protected]");+ legacy.classified = Some(stored_at(&legacy, 0));+ let (owner, link_id) = (legacy.owner.clone(), legacy.link_id);+ let id = insert(&store, legacy);++ let outcome = notifying_service(&store, &classifier, &["acme.example"], ¬ifier)+ .classify_thread(id)+ .await+ .unwrap();++ assert!(matches!(outcome, ClassifyOutcome::Redecided(_)));+ assert_eq!(*notifier.changed.lock().unwrap(), vec![(owner, link_id)]);+}++#[tokio::test]+async fn key_mail_outside_the_signal_inbox_tells_the_inbox_readers() {+ let store = FakeStore::default();+ let classifier = FakeClassifier::answering(Ok(relevant()));+ let notifier = FakeNotifier::default();+ let service = notifying_service(&store, &classifier, &["acme.example"], ¬ifier);++ let mut noise = thread("[email protected]");+ noise.is_signal = false;+ let noise = insert(&store, noise);+ assert_eq!(+ service.classify_thread(noise).await.unwrap(),+ ClassifyOutcome::NotSignal+ );+ assert!(notifier.changed.lock().unwrap().is_empty());++ let mut reschedule = thread("[email protected]");+ reschedule.is_signal = false;+ reschedule.messages[0].calendar_part = Some(CalendarPart::Change {+ until: Utc::now() + Duration::days(2),+ });+ let (owner, link_id) = (reschedule.owner.clone(), reschedule.link_id);+ let reschedule = insert(&store, reschedule);+ service.classify_thread(reschedule).await.unwrap();++ assert_eq!(*notifier.changed.lock().unwrap(), vec![(owner, link_id)]);+}+#[tokio::test]async fn an_older_rule_is_decided_again_without_the_classifier() {let store = FakeStore::default();diff --git a/crates/email/src/outbound/focus_pg/test.rs b/crates/email/src/outbound/focus_pg/test.rsindex f53a9001b0..1efa1da0ca 100644--- a/crates/email/src/outbound/focus_pg/test.rs+++ b/crates/email/src/outbound/focus_pg/test.rs@@ -6,8 +6,8 @@ use sqlx::{Pool, Postgres};use super::*;use crate::domain::{focus::{- ClassifyOutcome, FocusAnswers, FocusClassifier, FocusClassifierError, FocusQueries,- FocusQuestion, FocusService, FocusThread, FocusVerdict, ProfileSource,+ ClassifyOutcome, FocusAnswers, FocusClassifier, FocusClassifierError, FocusNotifier,+ FocusQueries, FocusQuestion, FocusService, FocusThread, FocusVerdict, ProfileSource,},models::calendar_invitation::{CalendarInvitation, InvitationDateTime, InvitationMethod},};@@ -71,14 +71,21 @@ impl FocusClassifier for FixedClassifier {}}+struct NoReaders;++impl FocusNotifier for NoReaders {+ fn focus_changed(&self, _owner: &MacroUserIdStr<'static>, _link_id: Uuid) {}+}+fn worker(repo: &FocusPgRepository,answers: Result<Vec<f32>, FocusClassifierError>,-) -> FocusService<FocusPgRepository, NoProfile, FixedClassifier> {+) -> FocusService<FocusPgRepository, NoProfile, FixedClassifier, NoReaders> {FocusService::new(repo.clone(),NoProfile,FixedClassifier { answers },+ NoReaders,vec!["acme.example".to_owned()],)}diff --git a/crates/models_email/src/email/api/refresh.rs b/crates/models_email/src/email/api/refresh.rsindex 109af1d4c9..50f3e9c454 100644--- a/crates/models_email/src/email/api/refresh.rs+++ b/crates/models_email/src/email/api/refresh.rs@@ -46,4 +46,13 @@ pub enum RefreshEmailEvent {completed_threads: i32,total_threads: i32,},+ /// A Focus classification was stored for a thread in `link_id`, so its+ /// readers read their Focus list again. `traceparent` is the W3C trace+ /// context of the classification, for that read to join. Coalesced by the+ /// sender; carries nothing about the mail.+ FocusUpdated {+ link_id: Uuid,+ #[serde(default, skip_serializing_if = "Option::is_none")]+ traceparent: Option<String>,+ },}diff --git a/docs/agent-guide/email.md b/docs/agent-guide/email.mdindex 952ceac93a..568c1e7679 100644--- a/docs/agent-guide/email.md+++ b/docs/agent-guide/email.md@@ -186,11 +186,16 @@ options) switches between **Priority**,**Importance** and **Most recent**; touch layouts have no dropdown and keep thesaved order (Priority by default). The filter menu adds a Focus group (Replyneeded, Follow up) on this tab only and has no Status or Done group there. The-list is one `user.emailFocus` GraphQL read filtered on the client; it refreshes-on open, not live. Mark Done hides the row at once and archives the thread; Undo-brings it back. The worker classifies only inboxes on its-`FOCUS_ENABLED_EMAIL_DOMAINS` allowlist, so other accounts see the empty state-`Nothing needs you`.+list is one `user.emailFocus` GraphQL read filtered on the client. New mail+reaches an open Focus tab within seconds: when the worker stores a+classification it pushes `refresh_email` (`focus_updated`) to the inbox's owner+and delegates, and the tab reads the list again without a reload. A hidden tab+reads once when it next shows, and any other tab reads when Focus opens. Time+passing stores nothing and pushes nothing, so a follow-up coming due or a+calendar change whose event ends shows at the tab's next read. Mark+Done hides the row at once and archives the thread; Undo brings it back. The+worker classifies only inboxes on its `FOCUS_ENABLED_EMAIL_DOMAINS` allowlist, so+other accounts see the empty state `Nothing needs you`.`Archived`, between All mail and Shared in the desktop Library section, listsyour own archived (Mail Done) threads: thediff --git a/docs/agent-guide/observability.md b/docs/agent-guide/observability.mdindex faf49a8591..8dc1bf5ea4 100644--- a/docs/agent-guide/observability.md+++ b/docs/agent-guide/observability.md@@ -153,6 +153,20 @@ traces. Errors carry fixed messages. Never add addresses, subjects, bodies, orfile names. The `http POST` spans carry `traceparent`, but email_service does notjoin the trace yet (see known gaps).+## Email Focus live updates++Search Tempo for `{ name="email_focus.classify" && span.thread_id="<id>" }`. One+trace covers an email event from its Kafka publish to the browser's read:++| Span | What it measures |+| --- | --- |+| `kafka.process` | One `macro.email` event in the `email-focus` group, until its classification ends. |+| `email_focus.queue` | The wait for the thread's turn: mail for a thread being classified runs after that call. `event_age_ms` is the time from publish to receipt, which includes any time the consumer stopped reading because every slot was busy. |+| `email_focus.classify` | One classification: `classify_thread`, with its mail reads, `jev.evaluate`, and `save`. `outcome` is `classified`, `redecided`, `up_to_date`, `missing`, `not_signal`, `not_enabled`, `unavailable`, `rejected`, or `failed`. `event_age_ms` is the time from publish to the end. |+| `email_focus.notify` | One push to an inbox's readers, under the newest classification it covers. `coalesced` counts the classifications it covers, and 0 marks the repeat for the replica. `delivered` counts the websockets that got it. |+| `connection_gateway.websocket_send` | The frame leaving the gateway. |+| `email.focus.refresh` | The browser's read after a push, joined through the push's `traceparent`. A read after a reconnect starts its own trace. `email.focus.threads` is the list's size. |+## Agent sessionsA session's lifetime is reconstructable from these spans. All of them carrydiff --git a/docs/agent-guide/soup-live-updates.md b/docs/agent-guide/soup-live-updates.mdindex e2686185c6..9d89ea76ae 100644--- a/docs/agent-guide/soup-live-updates.md+++ b/docs/agent-guide/soup-live-updates.md@@ -161,6 +161,12 @@ Checkpoint v14 rehydrates older projections without wiping normalized records orqueued mutations. General Soup still uses its existing server pagination; filter-onlyoffline checks use an ungrouped view with a created/updated timestamp sort.+The email Focus list is not a Soup list the cache maintains: membership comes from+the worker's classifications, so it reads `user.emailFocus` again when told to. See+the [Focus walkthrough](email.md) and `useEmailFocusQuery`. A list like it pushes+over the connection gateway and refreshes through `createVisibleRefresh`, which+holds a hidden tab to one read when it next shows.+Realtime Soup batches coalesce repeated entity IDs (including entity type), keepingthat entity's last operation in the batch. Emitted `SoupUpdated` items are non-null.If viewer-scoped hydration finds no item, the backend logs and omits that update;diff --git a/infra/stacks/email-service/focus_worker.ts b/infra/stacks/email-service/focus_worker.tsindex 76df0be0a7..94d64f7c5d 100644--- a/infra/stacks/email-service/focus_worker.ts+++ b/infra/stacks/email-service/focus_worker.ts@@ -68,7 +68,8 @@ export class EmailFocusWorker extends pulumi.ComponentResource {);this.ecr = image.ecr;- // The worker only calls out: Postgres, Kafka and the classifier API.+ // The worker only calls out: Postgres, Kafka, the classifier API and the+ // connection gateway.const serviceSg = new aws.ec2.SecurityGroup(`${BASE_NAME}-sg-${stack}`,{diff --git a/packages/observability/src/telemetry.test.ts b/packages/observability/src/telemetry.test.tsindex 841f9edff8..079d9d5e0f 100644--- a/packages/observability/src/telemetry.test.ts+++ b/packages/observability/src/telemetry.test.ts@@ -1,6 +1,7 @@// @vitest-environment jsdom-import { context, trace } from '@opentelemetry/api';+import { context, propagation, trace } from '@opentelemetry/api';import { logs } from '@opentelemetry/api-logs';+import { W3CTraceContextPropagator } from '@opentelemetry/core';import {InMemoryLogRecordExporter,LoggerProvider,@@ -17,6 +18,7 @@ import { Telemetry } from './index';import { userIdSuppressed } from './privacy';context.setGlobalContextManager(new StackContextManager().enable());+propagation.setGlobalPropagator(new W3CTraceContextPropagator());const spanExporter = new InMemorySpanExporter();trace.setGlobalTracerProvider(@@ -110,6 +112,39 @@ describe('Telemetry', () => {expect(spans.get('cache.storage')?.attributes).not.toHaveProperty('usr.id');});+ test('a remote context joins the trace another service started', async () => {+ const traceId = '0af7651916cd43dd8448eb211c80319c';+ const parentId = 'b7ad6b7169203331';+ const remote = Telemetry.remoteContext(`00-${traceId}-${parentId}-01`);+ await Promise.resolve();++ remote.run(() =>+ Telemetry.span('pushed.read', {}, async (span) => {+ span.span('pushed.read.phase').end();+ })+ );+ await Promise.resolve();++ const spans = new Map(+ spanExporter.getFinishedSpans().map((span) => [span.name, span])+ );+ const read = spans.get('pushed.read');+ expect(read?.spanContext().traceId).toBe(traceId);+ expect(read?.parentSpanContext?.spanId).toBe(parentId);+ expect(read?.parentSpanContext?.isRemote).toBe(true);+ expect(spans.get('pushed.read.phase')?.spanContext().traceId).toBe(traceId);+ });++ test('a malformed remote context starts a new trace', () => {+ Telemetry.remoteContext('not-a-traceparent').run(() =>+ Telemetry.span('pushed.read').end()+ );++ const [read] = spanExporter.getFinishedSpans();+ expect(read?.parentSpanContext).toBeUndefined();+ expect(read?.spanContext().traceId).toMatch(/^[0-9a-f]{32}$/);+ });+test('captured context survives an await and restores an unrelated caller', async () => {const root = Telemetry.span('queued.action');const parent = root.run(() => Telemetry.captureContext());diff --git a/packages/observability/src/telemetry.ts b/packages/observability/src/telemetry.tsindex 0ca346e38d..5086db8934 100644--- a/packages/observability/src/telemetry.ts+++ b/packages/observability/src/telemetry.ts@@ -88,6 +88,15 @@ export class Telemetry {return Telemetry.#tracing.captureContext();}+ /**+ * The trace another service started, for work it asked for over a transport+ * without headers. Spans started in it join that trace. A malformed value+ * starts a new trace instead.+ */+ static remoteContext(traceparent: string): TraceContext {+ return Telemetry.#tracing.remoteContext(traceparent);+ }+/** Record a point-in-time sample; timestamp is Unix time in milliseconds. */static recordSample(name: string,diff --git a/packages/observability/src/tracing.ts b/packages/observability/src/tracing.tsindex c2e4c0249d..3187a92109 100644--- a/packages/observability/src/tracing.ts+++ b/packages/observability/src/tracing.ts@@ -2,6 +2,7 @@ import {type Context,context,type Span as OtelSpan,+ propagation,ROOT_CONTEXT,SpanKind,trace,@@ -55,6 +56,11 @@ export class Tracing {return { run: (operation) => context.with(parent, operation) };}+ remoteContext(traceparent: string): TraceContext {+ const parent = propagation.extract(ROOT_CONTEXT, { traceparent });+ return { run: (operation) => context.with(parent, operation) };+ }+span(name: string, options: SpanOptions = {}): Span {return this.#startSpan(name, context.active(), options);}diff --git a/packages/sdk-docs/reference/generated/email/types.gen/type-aliases/RefreshEmailEvent.md b/packages/sdk-docs/reference/generated/email/types.gen/type-aliases/RefreshEmailEvent.mdindex 132ecd1eef..d12c551c10 100644--- a/packages/sdk-docs/reference/generated/email/types.gen/type-aliases/RefreshEmailEvent.md+++ b/packages/sdk-docs/reference/generated/email/types.gen/type-aliases/RefreshEmailEvent.md@@ -1,6 +1,6 @@# Type Alias: RefreshEmailEvent-> **RefreshEmailEvent** = \{ `event`: `"backfill"`; `link_id`: `string`; `status`: [`BackfillStatus`](BackfillStatus.md); \} \| \{ `event`: `"upsert_message"`; `link_id`: `string`; \} \| \{ `event`: `"update_labels"`; `link_id`: `string`; \} \| \{ `event`: `"delete_message"`; `link_id`: `string`; \} \| \{ `event`: `"link_removed"`; `link_id`: `string`; \} \| \{ `event`: `"photo_synced"`; `link_id`: `string`; \} \| \{ `completed_threads`: `number`; `event`: `"backfill_progress"`; `link_id`: `string`; `status`: [`BackfillStatus`](BackfillStatus.md); `total_threads`: `number`; \}+> **RefreshEmailEvent** = \{ `event`: `"backfill"`; `link_id`: `string`; `status`: [`BackfillStatus`](BackfillStatus.md); \} \| \{ `event`: `"upsert_message"`; `link_id`: `string`; \} \| \{ `event`: `"update_labels"`; `link_id`: `string`; \} \| \{ `event`: `"delete_message"`; `link_id`: `string`; \} \| \{ `event`: `"link_removed"`; `link_id`: `string`; \} \| \{ `event`: `"photo_synced"`; `link_id`: `string`; \} \| \{ `completed_threads`: `number`; `event`: `"backfill_progress"`; `link_id`: `string`; `status`: [`BackfillStatus`](BackfillStatus.md); `total_threads`: `number`; \} \| \{ `event`: `"focus_updated"`; `link_id`: `string`; `traceparent?`: `string` \| `null`; \}Payload for the `refresh_email` connection gateway event: identifies theinbox that changed and the kind of change.diff --git a/packages/sdk/generated/email/types.gen.ts b/packages/sdk/generated/email/types.gen.tsindex b0a6438564..7957735f23 100644--- a/packages/sdk/generated/email/types.gen.ts+++ b/packages/sdk/generated/email/types.gen.ts@@ -1743,6 +1743,10 @@ export type RefreshEmailEvent = {link_id: string;status: BackfillStatus;total_threads: number;+} | {+ event: 'focus_updated';+ link_id: string;+ traceparent?: string | null;};/**diff --git a/packages/sdk/specs/email.json b/packages/sdk/specs/email.jsonindex 4671c31a92..8c7eaa66a4 100644--- a/packages/sdk/specs/email.json+++ b/packages/sdk/specs/email.json@@ -7295,6 +7295,24 @@"format": "int32"}}+ },+ {+ "type": "object",+ "description": "A Focus classification was stored for a thread in `link_id`, so its\nreaders read their Focus list again. `traceparent` is the W3C trace\ncontext of the classification, for that read to join. Coalesced by the\nsender; carries nothing about the mail.",+ "required": ["link_id", "event"],+ "properties": {+ "event": {+ "type": "string",+ "enum": ["focus_updated"]+ },+ "link_id": {+ "type": "string",+ "format": "uuid"+ },+ "traceparent": {+ "type": ["string", "null"]+ }+ }}],"description": "Payload for the `refresh_email` connection gateway event: identifies the\ninbox that changed and the kind of change."diff --git a/services/email_focus_worker/Cargo.toml b/services/email_focus_worker/Cargo.tomlindex f9cbcc0be6..0c3b87d33b 100644--- a/services/email_focus_worker/Cargo.toml+++ b/services/email_focus_worker/Cargo.toml@@ -15,24 +15,34 @@ path = "src/doppler_config.rs"[dependencies]ai_usage = { path = "../../crates/ai_usage" }chrono = { workspace = true }+connection_gateway_client = { path = "../../crates/connection_gateway_client" }doppler-config = { workspace = true }email = { path = "../../crates/email", default-features = false, features = ["outbound", "ports"] }jev = { path = "../../crates/jev", features = ["outbound"] }kafka_util = { path = "../../crates/kafka_util" }+macro_auth = { path = "../../crates/macro_auth" }macro_config = { path = "../../crates/macro_config" }macro_entrypoint = { path = "../../crates/macro_entrypoint" }macro_env = { path = "../../crates/macro_env" }macro_env_var = { path = "../../crates/macro_env_var" }macro_event_broker = { path = "../../crates/macro_event_broker", features = ["outbound"] }+macro_service_urls = { path = "../../crates/macro_service_urls" }macro_user_id = { path = "../../crates/macro_user_id" }memory = { path = "../../crates/memory" }+models_email = { path = "../../crates/models_email" }+opentelemetry = { workspace = true, features = ["trace"] }rdkafka = { workspace = true }rootcause = { workspace = true }serde = { workspace = true }serde_json = { workspace = true }sqlx = { workspace = true }tokio = { workspace = true, features = ["signal", "time"] }-tokio-util = { workspace = true }+tokio-util = { workspace = true, features = ["rt"] }tracing = { workspace = true }+tracing-opentelemetry = { workspace = true }uuid = { workspace = true }workspace-hack = { version = "0.1", path = "../../crates/workspace-hack" }++[dev-dependencies]+macro_db_migrator = { path = "../../crates/macro_db_migrator" }+wiremock = { workspace = true }diff --git a/services/email_focus_worker/src/config.rs b/services/email_focus_worker/src/config.rsindex 6c574b37cb..c91b797807 100644--- a/services/email_focus_worker/src/config.rs+++ b/services/email_focus_worker/src/config.rs@@ -1,6 +1,7 @@//! Configuration for the email focus worker, loaded with `macro_config` from//! the email-service Doppler project it shares with the other email binaries.+use macro_auth::InternalApiKey;pub use macro_env::Environment;use macro_env_var::env_vars;@@ -22,6 +23,8 @@ pub struct Config {pub macro_db_url: MacroDbUrl,/// Kafka bootstrap servers; the worker reads `macro.email`.pub kafka_brokers: KafkaBrokers,+ /// Authenticates the worker's pushes to the connection gateway.+ pub internal_api_key: InternalApiKey,/// TypeSafe credential for Jev. While it is unset the worker classifies nothing.pub typesafe_api_key: jev::outbound::TypesafeApiKey,/// Comma-separated inbox domains to classify, e.g. `macro.com`. Mail fromdiff --git a/services/email_focus_worker/src/inbound/kafka_consumer.rs b/services/email_focus_worker/src/inbound/kafka_consumer.rsindex a7443dc2a4..d9e0f01224 100644--- a/services/email_focus_worker/src/inbound/kafka_consumer.rs+++ b/services/email_focus_worker/src/inbound/kafka_consumer.rs@@ -2,24 +2,37 @@//!//! Every email event on the platform passes through here, so the consumer//! hands each thread to the Focus service, which drops inboxes off the-//! allowlist after a single-row read. Offsets are committed whatever the-//! outcome: a lost or failed classification is picked up by the hourly sweep.+//! allowlist after a single-row read. Up to [`MAX_CONCURRENT_CLASSIFICATIONS`]+//! threads run side by side, so a slow classifier call delays only its own+//! thread; with every slot taken the consumer stops reading until one frees.+//!+//! An event's offset is committed when it is handed off, in the order received,+//! not when its classification ends. So an event is never replayed, and one in+//! flight when the worker dies is lost like any failed classification. The sweep+//! that runs when the worker starts picks those up, newest first.use std::{future::Future, sync::Arc};+use chrono::Utc;use email::domain::{events::{EmailMacroEvent, EmailTopicEvent},- focus::{ClassifyOutcome, FocusClassifier, FocusService, FocusStore, ProfileSource},+ focus::{+ ClassifyOutcome, FocusClassifier, FocusNotifier, FocusService, FocusStore, ProfileSource,+ },};use kafka_util::{GroupName, InitialOffset, KafkaEventConsumer};use macro_event_broker::{KafkaConsumerAdapter, MacroEvent as _, MacroEventCollection as _, MacroEventConsumerService,};-use rdkafka::consumer::CommitMode;+use rdkafka::{Message as _, consumer::CommitMode};use rootcause::Report;-use tracing::Instrument as _;+use tokio_util::sync::CancellationToken;+use tracing::{Instrument as _, field::Empty};use uuid::Uuid;+use self::classifications::Classifications;++mod classifications;#[cfg(test)]mod test;@@ -45,42 +58,120 @@ fn thread_to_classify(event: &EmailTopicEvent) -> Option<Uuid> {}}-async fn classify<S, P, C>(service: &FocusService<S, P, C>, thread_id: Uuid)-where+/// Threads classified at once. Each holds at most one thread's recent mail in+/// memory and a classifier call that its own deadline bounds.+const MAX_CONCURRENT_CLASSIFICATIONS: usize = 16;++/// How long ago the event was published, by the Kafka record's timestamp.+fn event_age_ms(published_at_ms: Option<i64>) -> Option<i64> {+ published_at_ms.map(|published_at_ms| Utc::now().timestamp_millis() - published_at_ms)+}++fn outcome_name(outcome: &ClassifyOutcome) -> &'static str {+ match outcome {+ ClassifyOutcome::Classified(_) => "classified",+ ClassifyOutcome::Redecided(_) => "redecided",+ ClassifyOutcome::UpToDate => "up_to_date",+ ClassifyOutcome::Missing => "missing",+ ClassifyOutcome::NotSignal => "not_signal",+ ClassifyOutcome::NotEnabled => "not_enabled",+ ClassifyOutcome::Unavailable => "unavailable",+ ClassifyOutcome::Rejected => "rejected",+ }+}++async fn classify<S, P, C, N>(+ service: &FocusService<S, P, C, N>,+ thread_id: Uuid,+ published_at_ms: Option<i64>,+) whereS: FocusStore,P: ProfileSource,C: FocusClassifier,+ N: FocusNotifier,{- match service.classify_thread(thread_id).await {- Ok(ClassifyOutcome::Classified(verdict)) => tracing::info!(- %thread_id,- is_focus = verdict.is_focus,- category = %verdict.category,- importance = verdict.importance,- "focus classified"- ),- Ok(outcome @ (ClassifyOutcome::Unavailable | ClassifyOutcome::Rejected)) => {- tracing::warn!(%thread_id, ?outcome, "focus classification left for the sweep");+ let span = tracing::info_span!(+ "email_focus.classify",+ %thread_id,+ outcome = Empty,+ event_age_ms = Empty,+ otel.status_code = Empty,+ otel.status_description = Empty,+ );+ let result = service+ .classify_thread(thread_id)+ .instrument(span.clone())+ .await;+ span.record("event_age_ms", event_age_ms(published_at_ms));+ let _in_span = span.enter();+ match result {+ Ok(outcome) => {+ span.record("outcome", outcome_name(&outcome));+ match outcome {+ ClassifyOutcome::Classified(verdict) | ClassifyOutcome::Redecided(verdict) => {+ tracing::info!(+ %thread_id,+ is_focus = verdict.is_focus,+ category = %verdict.category,+ importance = verdict.importance,+ "focus classified"+ );+ }+ ClassifyOutcome::Unavailable | ClassifyOutcome::Rejected => {+ tracing::warn!(%thread_id, ?outcome, "focus classification left for the sweep");+ }+ _ => tracing::debug!(%thread_id, ?outcome, "focus skipped"),+ }}- Ok(outcome) => tracing::debug!(%thread_id, ?outcome, "focus skipped"),Err(error) => {+ span.record("outcome", "failed");+ kafka_util::record_span_error(&span, "focus classification failed");tracing::warn!(%thread_id, error = ?error, "focus classification failed");}}}-/// Consume `macro.email` under the `email-focus` group until `shutdown`-/// resolves. Receive and commit failures are logged and skipped, as in the-/// activity consumer: a restart would only re-run the startup sweep.-pub async fn run_focus_consumer<S, P, C>(+/// The work for one event: waiting for a slot, then classifying, in the event's+/// trace. Nothing happens until it runs.+fn classification<S, P, C, N>(+ service: Arc<FocusService<S, P, C, N>>,+ thread_id: Uuid,+ published_at_ms: Option<i64>,+ event_span: tracing::Span,+) -> impl Future<Output = ()> + Send + 'static+where+ S: FocusStore,+ P: ProfileSource,+ C: FocusClassifier,+ N: FocusNotifier,+{+ let queued = tracing::info_span!(+ parent: &event_span,+ "email_focus.queue",+ %thread_id,+ event_age_ms = event_age_ms(published_at_ms),+ );+ async move {+ drop(queued);+ classify(&service, thread_id, published_at_ms).await;+ }+ .instrument(event_span)+}++/// Consume `macro.email` under the `email-focus` group until `shutdown`, which+/// drops the classifications in flight. Receive and commit failures are logged+/// and skipped, as in the activity consumer: a restart would only re-run the+/// startup sweep.+pub async fn run_focus_consumer<S, P, C, N>(brokers: &str,- service: Arc<FocusService<S, P, C>>,- shutdown: impl Future<Output = ()> + Send,+ service: Arc<FocusService<S, P, C, N>>,+ shutdown: CancellationToken,) -> Result<(), Report>whereS: FocusStore,P: ProfileSource,C: FocusClassifier,+ N: FocusNotifier,{let consumer = KafkaEventConsumer::<EmailFocusGroup>::from_env(brokers)?;let consumer =@@ -91,12 +182,13 @@ wheregroup = EmailFocusGroup::GROUP_NAME,"email focus consumer listening");- let mut shutdown = std::pin::pin!(shutdown);+ let mut classifications = Classifications::new(MAX_CONCURRENT_CLASSIFICATIONS);loop {let message = tokio::select! {biased;- () = &mut shutdown => return Ok(()),- result = consumer.recv() => match result {+ () = shutdown.cancelled() => break,+ () = classifications.next_finished() => continue,+ result = consumer.recv(), if classifications.has_room() => match result {Ok(message) => message,Err(error) => {tracing::error!(error = ?error, "kafka receive error");@@ -114,12 +206,11 @@ where}};if let Some(thread_id) = thread_id {- tokio::select! {- biased;- // Uncommitted: the event is redelivered to the next consumer.- () = &mut shutdown => return Ok(()),- () = classify(&service, thread_id).instrument(span.clone()) => {}- }+ let published_at_ms = message.inner().timestamp().to_millis();+ classifications.submit(+ thread_id,+ classification(Arc::clone(&service), thread_id, published_at_ms, span),+ );}if let Err(error) = consumer.inner()@@ -128,4 +219,6 @@ wheretracing::error!(error = ?error, "kafka commit error");}}+ classifications.shutdown().await;+ Ok(())}diff --git a/services/email_focus_worker/src/inbound/kafka_consumer/classifications.rs b/services/email_focus_worker/src/inbound/kafka_consumer/classifications.rsnew file mode 100644index 0000000000..7bf5f6b3c5--- /dev/null+++ b/services/email_focus_worker/src/inbound/kafka_consumer/classifications.rs@@ -0,0 +1,109 @@+//! The classifications in flight: a bounded number at once, one per thread.++use std::{+ collections::{HashMap, hash_map::Entry},+ future::Future,+};++use tokio::sync::mpsc;+use tokio_util::{sync::CancellationToken, task::TaskTracker};+use uuid::Uuid;++#[cfg(test)]+mod test;++/// Runs classifications side by side so a slow one holds up nothing but its+/// own thread. Mail for a thread being classified doesn't start a second call:+/// the newest such event runs once the current one ends, and classifying again+/// costs nothing if the first call already saw that mail.+pub struct Classifications<F> {+ capacity: usize,+ /// Threads being classified, each with what to run next if more mail came.+ running: HashMap<Uuid, Option<F>>,+ finished_tx: mpsc::UnboundedSender<Uuid>,+ finished_rx: mpsc::UnboundedReceiver<Uuid>,+ tracker: TaskTracker,+ cancel: CancellationToken,+}++/// Reports a thread's task as finished however it ends, a panic included, so+/// one bad classification can't keep its slot forever.+struct FinishedOnDrop {+ thread_id: Uuid,+ finished: mpsc::UnboundedSender<Uuid>,+}++impl Drop for FinishedOnDrop {+ fn drop(&mut self) {+ // The receiver is only gone once the worker is shutting down.+ let _ = self.finished.send(self.thread_id);+ }+}++impl<F> Classifications<F>+where+ F: Future<Output = ()> + Send + 'static,+{+ /// Allow `capacity` threads to be classified at once.+ pub fn new(capacity: usize) -> Self {+ let (finished_tx, finished_rx) = mpsc::unbounded_channel();+ Self {+ capacity,+ running: HashMap::new(),+ finished_tx,+ finished_rx,+ tracker: TaskTracker::new(),+ cancel: CancellationToken::new(),+ }+ }++ /// Whether another thread can start. Wait with [`Self::next_finished`] until+ /// it can, so the Kafka consumer stops reading instead of queueing here.+ pub fn has_room(&self) -> bool {+ self.running.len() < self.capacity+ }++ /// Run `classification` for `thread_id`, or after the one running for it.+ pub fn submit(&mut self, thread_id: Uuid, classification: F) {+ match self.running.entry(thread_id) {+ Entry::Occupied(mut running) => {+ running.insert(Some(classification));+ }+ Entry::Vacant(slot) => {+ slot.insert(None);+ self.start(thread_id, classification);+ }+ }+ }++ fn start(&self, thread_id: Uuid, classification: F) {+ let finished = FinishedOnDrop {+ thread_id,+ finished: self.finished_tx.clone(),+ };+ let cancel = self.cancel.clone();+ self.tracker.spawn(async move {+ let _finished = finished;+ cancel.run_until_cancelled(classification).await;+ });+ }++ /// Wait for a classification to end and free its slot, unless its thread+ /// has another waiting, which then takes the slot. Safe to cancel.+ pub async fn next_finished(&mut self) {+ let Some(thread_id) = self.finished_rx.recv().await else {+ return;+ };+ if let Some(next) = self.running.remove(&thread_id).flatten() {+ self.running.insert(thread_id, None);+ self.start(thread_id, next);+ }+ }++ /// Drop everything in flight and wait for its tasks to end.+ pub async fn shutdown(self) {+ self.cancel.cancel();+ self.tracker.close();+ self.tracker.wait().await;+ }+}diff --git a/services/email_focus_worker/src/inbound/kafka_consumer/classifications/test.rs b/services/email_focus_worker/src/inbound/kafka_consumer/classifications/test.rsnew file mode 100644index 0000000000..9bb4597f7c--- /dev/null+++ b/services/email_focus_worker/src/inbound/kafka_consumer/classifications/test.rs@@ -0,0 +1,148 @@+use std::{+ pin::Pin,+ sync::{+ Arc,+ atomic::{AtomicUsize, Ordering},+ },+};++use tokio::sync::oneshot;++use super::*;++type Classification = Pin<Box<dyn Future<Output = ()> + Send>>;++/// A classification that runs until released, counting its starts.+fn gated(starts: &Arc<AtomicUsize>) -> (Classification, oneshot::Sender<()>) {+ let (release, released) = oneshot::channel();+ let starts = Arc::clone(starts);+ let classification = Box::pin(async move {+ starts.fetch_add(1, Ordering::SeqCst);+ let _ = released.await;+ });+ (classification, release)+}++#[tokio::test]+async fn a_slow_thread_does_not_hold_up_another() {+ let mut classifications = Classifications::new(2);+ let starts = Arc::new(AtomicUsize::new(0));++ let (slow, _still_running) = gated(&starts);+ classifications.submit(Uuid::now_v7(), slow);++ let (classified, done) = oneshot::channel();+ classifications.submit(+ Uuid::now_v7(),+ Box::pin(async move {+ let _ = classified.send(());+ }) as Classification,+ );++ // The second finishes while the first is still waiting to be released.+ done.await.unwrap();+ classifications.next_finished().await;+ assert!(classifications.has_room());+ classifications.shutdown().await;+}++#[tokio::test]+async fn capacity_bounds_the_threads_running() {+ let mut classifications = Classifications::new(2);+ let starts = Arc::new(AtomicUsize::new(0));++ let (first, release_first) = gated(&starts);+ let (second, _still_running) = gated(&starts);+ classifications.submit(Uuid::now_v7(), first);+ assert!(classifications.has_room());+ classifications.submit(Uuid::now_v7(), second);+ assert!(!classifications.has_room());++ release_first.send(()).unwrap();+ classifications.next_finished().await;+ assert!(classifications.has_room());+ classifications.shutdown().await;+}++#[tokio::test]+async fn mail_during_a_run_classifies_once_more_afterwards() {+ let mut classifications = Classifications::new(1);+ let thread_id = Uuid::now_v7();+ let (first_started_tx, first_started) = oneshot::channel();+ let (release_first, first_released) = oneshot::channel::<()>();+ let (newest_ran_tx, newest_ran) = oneshot::channel();+ let superseded = Arc::new(AtomicUsize::new(0));++ classifications.submit(+ thread_id,+ Box::pin(async move {+ let _ = first_started_tx.send(());+ let _ = first_released.await;+ }) as Classification,+ );+ first_started.await.unwrap();++ let superseded_runs = Arc::clone(&superseded);+ classifications.submit(+ thread_id,+ Box::pin(async move {+ superseded_runs.fetch_add(1, Ordering::SeqCst);+ }),+ );+ classifications.submit(+ thread_id,+ Box::pin(async move {+ let _ = newest_ran_tx.send(());+ }),+ );++ // Two more events took no slot of their own.+ assert_eq!(classifications.running.len(), 1);+ release_first.send(()).unwrap();+ classifications.next_finished().await;+ newest_ran.await.unwrap();+ assert!(!classifications.has_room());+ classifications.next_finished().await;+ assert!(classifications.has_room());+ assert_eq!(superseded.load(Ordering::SeqCst), 0);+ classifications.shutdown().await;+}++#[tokio::test]+async fn shutdown_drops_what_is_running() {+ let mut classifications = Classifications::new(1);+ let (started_tx, started) = oneshot::channel();+ let (dropped_tx, dropped) = oneshot::channel::<()>();+ classifications.submit(+ Uuid::now_v7(),+ Box::pin(async move {+ let _dropped_with_the_future = dropped_tx;+ let _ = started_tx.send(());+ std::future::pending::<()>().await;+ }) as Classification,+ );+ started.await.unwrap();++ classifications.shutdown().await;++ // The sender went with the future when shutdown cancelled it.+ assert!(dropped.await.is_err());+}++fn bad_thread() {+ panic!("a bad thread");+}++#[tokio::test]+async fn a_panic_gives_its_slot_back() {+ let mut classifications = Classifications::new(1);+ classifications.submit(+ Uuid::now_v7(),+ Box::pin(async { bad_thread() }) as Classification,+ );++ classifications.next_finished().await;++ assert!(classifications.has_room());+ classifications.shutdown().await;+}diff --git a/services/email_focus_worker/src/inbound/sweep.rs b/services/email_focus_worker/src/inbound/sweep.rsindex f5d115b618..7fedf4a76c 100644--- a/services/email_focus_worker/src/inbound/sweep.rs+++ b/services/email_focus_worker/src/inbound/sweep.rs@@ -9,7 +9,9 @@use std::{sync::Arc, time::Duration};-use email::domain::focus::{FocusClassifier, FocusService, FocusStore, ProfileSource};+use email::domain::focus::{+ FocusClassifier, FocusNotifier, FocusService, FocusStore, ProfileSource,+};use rootcause::Report;use tokio_util::sync::CancellationToken;@@ -17,8 +19,8 @@ use tokio_util::sync::CancellationToken;const SWEEP_LIMIT: i64 = 2_000;/// Sweep every `interval` over the last `window_days` until `shutdown`.-pub async fn run_sweeps<S, P, C>(- service: Arc<FocusService<S, P, C>>,+pub async fn run_sweeps<S, P, C, N>(+ service: Arc<FocusService<S, P, C, N>>,interval: Duration,window_days: u16,shutdown: CancellationToken,@@ -27,6 +29,7 @@ whereS: FocusStore,P: ProfileSource,C: FocusClassifier,+ N: FocusNotifier,{let window = chrono::Duration::days(i64::from(window_days));let mut ticker = tokio::time::interval(interval);diff --git a/services/email_focus_worker/src/main.rs b/services/email_focus_worker/src/main.rsindex 6320041784..349dfe4d2a 100644--- a/services/email_focus_worker/src/main.rs+++ b/services/email_focus_worker/src/main.rs@@ -3,8 +3,9 @@//!//! Reads `macro.email` for new and sent messages, asks Jev about each signal//! thread and key mail in an allowlisted inbox with the owner's stored memory-//! as context, and stores the verdict in `email_thread_focus`. An hourly sweep-//! catches what the event stream missed.+//! as context, stores the verdict in `email_thread_focus`, and tells the inbox's+//! readers through the connection gateway. An hourly sweep catches what the+//! event stream missed.mod config;mod inbound {@@ -12,24 +13,31 @@ mod inbound {pub mod sweep;}mod outbound {+ pub mod gateway_notifier;pub mod jev_classifier;pub mod memory_profile;}use std::{sync::Arc, time::Duration};+use connection_gateway_client::ConnectionGatewayClient;use email::{domain::focus::FocusService, outbound::FocusPgRepository};use jev::{domain::JevClassifier, outbound::SystemOneJev};use macro_entrypoint::{MacroEntrypoint, shutdown_signal};+use macro_service_urls::ConnectionGatewayUrl;use memory::outbound::pg_memory_repo::PgMemoryRepo;use rootcause::Report;use sqlx::postgres::PgPoolOptions;-use tokio_util::sync::CancellationToken;+use tokio_util::{sync::CancellationToken, task::TaskTracker};use crate::{config::Config,inbound::{kafka_consumer::run_focus_consumer, sweep::run_sweeps},- outbound::{jev_classifier::JevFocusClassifier, memory_profile::MemoryProfiles},+ outbound::{+ gateway_notifier::{CoalescingNotifier, GatewayDelivery},+ jev_classifier::JevFocusClassifier,+ memory_profile::MemoryProfiles,+ },};#[tokio::main]@@ -59,13 +67,24 @@ async fn main() -> Result<(), Report> {.max_connections(5).connect(config.macro_db_url.as_ref()).await?;+ let shutdown = CancellationToken::new();+ let notifications = TaskTracker::new();+ let gateway = ConnectionGatewayClient::new(+ config.internal_api_key.to_string(),+ ConnectionGatewayUrl::new()?.to_string(),+ );let service = Arc::new(FocusService::new(FocusPgRepository(db.clone()),MemoryProfiles::new(PgMemoryRepo::new(db.clone())),JevFocusClassifier::new(JevClassifier::new(provider,- ai_usage::pg_recorder_with_enforcement(db, enforcement),+ ai_usage::pg_recorder_with_enforcement(db.clone(), enforcement),)),+ CoalescingNotifier::new(+ GatewayDelivery::new(gateway, db),+ notifications.clone(),+ shutdown.clone(),+ ),domains,));tracing::info!(@@ -74,7 +93,6 @@ async fn main() -> Result<(), Report> {"email focus worker started");- let shutdown = CancellationToken::new();tokio::spawn({let shutdown = shutdown.clone();async move {@@ -83,11 +101,7 @@ async fn main() -> Result<(), Report> {}});let brokers = config.kafka_brokers.to_string();- let consumer = run_focus_consumer(- &brokers,- Arc::clone(&service),- shutdown.clone().cancelled_owned(),- );+ let consumer = run_focus_consumer(&brokers, Arc::clone(&service), shutdown.clone());let sweeps = run_sweeps(service,// A zero interval would panic the timer.@@ -98,5 +112,7 @@ async fn main() -> Result<(), Report> {// Either loop ending with an error stops the worker so it restarts fresh.let result = tokio::try_join!(consumer, sweeps).map(|_| ());shutdown.cancel();+ notifications.close();+ notifications.wait().await;result}diff --git a/services/email_focus_worker/src/outbound/gateway_notifier.rs b/services/email_focus_worker/src/outbound/gateway_notifier.rsnew file mode 100644index 0000000000..4e5b4b1796--- /dev/null+++ b/services/email_focus_worker/src/outbound/gateway_notifier.rs@@ -0,0 +1,220 @@+//! Tells an inbox's readers its Focus changed, through the connection gateway.+//!+//! A burst of mail in one inbox becomes few pushes. The first goes out at once;+//! what follows collapses into one per [`NOTIFY_WINDOW`]. The last push of a+//! burst is always a repeat one window later, because readers read Focus from a+//! replica that can lag the write they were told about.++use std::{+ collections::HashMap,+ future::Future,+ sync::{Arc, Mutex, MutexGuard, PoisonError},+ time::Duration,+};++use connection_gateway_client::ConnectionGatewayClient;+use email::domain::focus::FocusNotifier;+use macro_user_id::user_id::MacroUserIdStr;+use models_email::api::refresh::RefreshEmailEvent;+use sqlx::PgPool;+use tokio_util::{sync::CancellationToken, task::TaskTracker};+use tracing::{Instrument as _, field::Empty};+use tracing_opentelemetry::OpenTelemetrySpanExt as _;+use uuid::Uuid;++#[cfg(test)]+mod test;++/// The shortest gap between two pushes about one inbox.+const NOTIFY_WINDOW: Duration = Duration::from_secs(3);++/// Sends one push about an inbox.+pub trait FocusDelivery: Send + Sync + 'static {+ /// Tell everyone reading `link_id` to read Focus again. Failures are logged.+ fn deliver(+ &self,+ owner: &MacroUserIdStr<'static>,+ link_id: Uuid,+ ) -> impl Future<Output = ()> + Send;+}++/// Classifications stored since an inbox's last push.+#[derive(Default)]+struct OpenWindow {+ /// The trace of the newest one, which the next push joins.+ newest: Option<opentelemetry::Context>,+ count: i64,+}++type OpenWindows = Arc<Mutex<HashMap<Uuid, OpenWindow>>>;++fn locked(windows: &OpenWindows) -> MutexGuard<'_, HashMap<Uuid, OpenWindow>> {+ windows.lock().unwrap_or_else(PoisonError::into_inner)+}++/// The [`FocusNotifier`] the worker runs, with one window per inbox.+pub struct CoalescingNotifier<D> {+ delivery: Arc<D>,+ windows: OpenWindows,+ tracker: TaskTracker,+ shutdown: CancellationToken,+}++impl<D: FocusDelivery> CoalescingNotifier<D> {+ /// Push through `delivery`, with each inbox's pushes on `tracker`, until `shutdown`.+ pub fn new(delivery: D, tracker: TaskTracker, shutdown: CancellationToken) -> Self {+ Self {+ delivery: Arc::new(delivery),+ windows: Arc::default(),+ tracker,+ shutdown,+ }+ }+}++impl<D: FocusDelivery> FocusNotifier for CoalescingNotifier<D> {+ fn focus_changed(&self, owner: &MacroUserIdStr<'static>, link_id: Uuid) {+ let classification = tracing::Span::current().context();+ {+ let mut windows = locked(&self.windows);+ if let Some(window) = windows.get_mut(&link_id) {+ window.newest = Some(classification);+ window.count += 1;+ return;+ }+ windows.insert(link_id, OpenWindow::default());+ }+ self.tracker.spawn(run_window(+ Arc::clone(&self.delivery),+ Arc::clone(&self.windows),+ self.shutdown.clone(),+ owner.clone(),+ link_id,+ classification,+ ));+ }+}++/// Push for one inbox until a window passes with nothing new to say.+async fn run_window<D: FocusDelivery>(+ delivery: Arc<D>,+ windows: OpenWindows,+ shutdown: CancellationToken,+ owner: MacroUserIdStr<'static>,+ link_id: Uuid,+ classification: opentelemetry::Context,+) {+ let mut parent = classification;+ let mut coalesced = 1;+ loop {+ let span = tracing::info_span!(+ "email_focus.notify",+ %link_id,+ coalesced,+ delivered = Empty,+ otel.status_code = Empty,+ otel.status_description = Empty,+ );+ // Starts a trace when there's nothing to join, not under this task's.+ let _ = span.set_parent(parent.clone());+ let push = delivery.deliver(&owner, link_id).instrument(span);+ if shutdown.run_until_cancelled(push).await.is_none() {+ return;+ }++ let window = tokio::time::sleep(NOTIFY_WINDOW);+ if shutdown.run_until_cancelled(window).await.is_none() {+ return;+ }+ let mut open = locked(&windows);+ let Some(window) = open.get_mut(&link_id) else {+ return;+ };+ if let Some(newest) = window.newest.take() {+ parent = newest;+ coalesced = std::mem::take(&mut window.count);+ continue;+ }+ // Nothing new since the last push. If that one was for a new+ // classification, repeat it for the replica; if it was the repeat, stop.+ if coalesced == 0 {+ open.remove(&link_id);+ return;+ }+ coalesced = 0;+ }+}++/// Pushes `refresh_email` to an inbox's owner and the people it's delegated to.+pub struct GatewayDelivery {+ gateway: ConnectionGatewayClient,+ db: PgPool,+}++impl GatewayDelivery {+ /// Push through `gateway`, reading delegates from `db`.+ pub fn new(gateway: ConnectionGatewayClient, db: PgPool) -> Self {+ Self { gateway, db }+ }+}++/// The current span's W3C context, for the client's read to join: the gateway+/// keeps a message's own trace context off the websocket.+fn traceparent() -> Option<String> {+ let mut carrier = HashMap::new();+ opentelemetry::global::get_text_map_propagator(|propagator| {+ propagator.inject_context(&tracing::Span::current().context(), &mut carrier);+ });+ carrier.remove("traceparent")+}++impl FocusDelivery for GatewayDelivery {+ async fn deliver(&self, owner: &MacroUserIdStr<'static>, link_id: Uuid) {+ let span = tracing::Span::current();+ let mut readers = vec![owner.as_ref().to_owned()];+ match sqlx::query_scalar!(+ "SELECT primary_macro_id FROM macro_user_links WHERE link_id = $1",+ link_id,+ )+ .fetch_all(&self.db)+ .await+ {+ Ok(delegates) => readers.extend(delegates),+ Err(error) => {+ tracing::warn!(error = ?error, %link_id, "focus notification delegates unavailable");+ }+ }+ readers.sort_unstable();+ readers.dedup();++ let event = RefreshEmailEvent::FocusUpdated {+ link_id,+ traceparent: traceparent(),+ };+ let payload = match serde_json::to_value(&event) {+ Ok(payload) => payload,+ Err(error) => {+ tracing::error!(error = ?error, %link_id, "focus notification unserializable");+ kafka_util::record_span_error(&span, "focus notification unserializable");+ return;+ }+ };+ match self+ .gateway+ .refresh_email_for_users(&readers, payload)+ .await+ {+ Ok(receipts) => {+ let delivered = receipts+ .iter()+ .map(|receipt| receipt.delivery_count)+ .sum::<u64>();+ span.record("delivered", i64::try_from(delivered).unwrap_or(i64::MAX));+ }+ Err(error) => {+ tracing::warn!(error = ?error, %link_id, "focus notification failed");+ kafka_util::record_span_error(&span, "focus notification failed");+ }+ }+ }+}diff --git a/services/email_focus_worker/src/outbound/gateway_notifier/test.rs b/services/email_focus_worker/src/outbound/gateway_notifier/test.rsnew file mode 100644index 0000000000..0fb24e7fbc--- /dev/null+++ b/services/email_focus_worker/src/outbound/gateway_notifier/test.rs@@ -0,0 +1,171 @@+use std::collections::BTreeSet;++use macro_db_migrator::MACRO_DB_MIGRATIONS;+use sqlx::{Pool, Postgres};+use tokio::{sync::mpsc, time::Instant};+use wiremock::{Mock, MockServer, ResponseTemplate, matchers};++use super::*;++/// Records each push and when it went out.+struct FakeDelivery {+ pushes: mpsc::UnboundedSender<(Uuid, Instant)>,+}++impl FocusDelivery for FakeDelivery {+ async fn deliver(&self, _owner: &MacroUserIdStr<'static>, link_id: Uuid) {+ self.pushes.send((link_id, Instant::now())).unwrap();+ }+}++fn owner() -> MacroUserIdStr<'static> {+ MacroUserIdStr::try_from_email("[email protected]").unwrap()+}++fn notifier() -> (+ CoalescingNotifier<FakeDelivery>,+ mpsc::UnboundedReceiver<(Uuid, Instant)>,+ TaskTracker,+ CancellationToken,+) {+ let (pushes, pushed) = mpsc::unbounded_channel();+ let tracker = TaskTracker::new();+ let shutdown = CancellationToken::new();+ let notifier =+ CoalescingNotifier::new(FakeDelivery { pushes }, tracker.clone(), shutdown.clone());+ (notifier, pushed, tracker, shutdown)+}++#[tokio::test(start_paused = true)]+async fn one_classification_pushes_at_once_and_repeats_for_the_replica() {+ let (notifier, mut pushed, tracker, _shutdown) = notifier();+ let link_id = Uuid::now_v7();+ let start = Instant::now();++ notifier.focus_changed(&owner(), link_id);++ assert_eq!(pushed.recv().await.unwrap(), (link_id, start));+ assert_eq!(+ pushed.recv().await.unwrap(),+ (link_id, start + NOTIFY_WINDOW)+ );++ // The window closes one quiet window later and there is nothing more to say.+ tracker.close();+ tracker.wait().await;+ assert_eq!(start.elapsed(), NOTIFY_WINDOW * 2);+ assert!(pushed.try_recv().is_err());+}++#[tokio::test(start_paused = true)]+async fn a_burst_in_one_inbox_collapses_into_few_pushes() {+ let (notifier, mut pushed, tracker, _shutdown) = notifier();+ let link_id = Uuid::now_v7();+ let start = Instant::now();++ for _ in 0..20 {+ notifier.focus_changed(&owner(), link_id);+ }++ // At once, then the rest together, then the repeat for the replica.+ assert_eq!(pushed.recv().await.unwrap().1, start);+ assert_eq!(pushed.recv().await.unwrap().1, start + NOTIFY_WINDOW);+ assert_eq!(pushed.recv().await.unwrap().1, start + NOTIFY_WINDOW * 2);+ tracker.close();+ tracker.wait().await;+ assert!(pushed.try_recv().is_err());+}++#[tokio::test(start_paused = true)]+async fn a_classification_after_the_window_closes_pushes_at_once_again() {+ let (notifier, mut pushed, _tracker, _shutdown) = notifier();+ let link_id = Uuid::now_v7();++ notifier.focus_changed(&owner(), link_id);+ pushed.recv().await.unwrap();+ pushed.recv().await.unwrap();+ tokio::time::sleep(NOTIFY_WINDOW * 2).await;++ let later = Instant::now();+ notifier.focus_changed(&owner(), link_id);+ assert_eq!(pushed.recv().await.unwrap().1, later);+}++#[tokio::test(start_paused = true)]+async fn inboxes_do_not_share_a_window() {+ let (notifier, mut pushed, _tracker, _shutdown) = notifier();+ let (first, second) = (Uuid::now_v7(), Uuid::now_v7());+ let start = Instant::now();++ notifier.focus_changed(&owner(), first);+ notifier.focus_changed(&owner(), second);++ let pushes = [pushed.recv().await.unwrap(), pushed.recv().await.unwrap()];+ assert_eq!(pushes[0].1, start);+ assert_eq!(pushes[1].1, start);+ assert_eq!(+ BTreeSet::from([pushes[0].0, pushes[1].0]),+ BTreeSet::from([first, second])+ );+}++#[tokio::test(start_paused = true)]+async fn shutdown_ends_an_open_window_without_waiting_for_it() {+ let (notifier, mut pushed, tracker, shutdown) = notifier();+ let start = Instant::now();++ notifier.focus_changed(&owner(), Uuid::now_v7());+ pushed.recv().await.unwrap();++ shutdown.cancel();+ tracker.close();+ tracker.wait().await;+ assert_eq!(start.elapsed(), Duration::ZERO);+}++/// Link f02 in the Focus fixtures: held by `[email protected]`, delegated+/// to `[email protected]`.+const DELEGATED_LINK: Uuid = uuid::uuid!("00000000-0000-0000-0000-000000000f02");++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(+ path = "../../../../../crates/email/fixtures",+ scripts("email_focus", "email_focus_linked")+ )+)]+async fn a_push_reaches_the_owner_and_delegates_as_one_refresh_email(pool: Pool<Postgres>) {+ let gateway = MockServer::start().await;+ Mock::given(matchers::method("POST"))+ .and(matchers::path("/message/batch_send"))+ .and(matchers::header("x-internal-auth-key", "test-key"))+ .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({+ "receipts": [{"user_id": "macro|[email protected]", "delivery_count": 2, "active": true}],+ })))+ .expect(1)+ .mount(&gateway)+ .await;+ let delivery = GatewayDelivery::new(+ ConnectionGatewayClient::new("test-key".to_owned(), gateway.uri()),+ pool,+ );++ let holder = MacroUserIdStr::try_from_email("[email protected]").unwrap();+ delivery.deliver(&holder, DELEGATED_LINK).await;++ let requests = gateway.received_requests().await.unwrap();+ let body: serde_json::Value = serde_json::from_slice(&requests[0].body).unwrap();+ assert_eq!(body["message_type"], "refresh_email");+ assert_eq!(body["message"]["event"], "focus_updated");+ assert_eq!(body["message"]["link_id"], DELEGATED_LINK.to_string());+ let readers = body["entities"]+ .as_array()+ .unwrap()+ .iter()+ .map(|entity| entity["entity_id"].as_str().unwrap())+ .collect::<Vec<_>>();+ assert_eq!(+ readers,+ ["macro|[email protected]", "macro|[email protected]"]+ );+}