Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(email-focus): show new mail in open Focus tabs within seconds (#9521)

teo21a6d138f232
diff --git a/Cargo.lock b/Cargo.lock
index ac12a8379e..c05e589109 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -7414,17 +7414,23 @@ version = "0.1.0"
dependencies = [
"ai_usage",
"chrono",
+ "connection_gateway_client",
"doppler-config",
"email",
"jev",
"kafka_util",
+ "macro_auth",
"macro_config",
+ "macro_db_migrator",
"macro_entrypoint",
"macro_env",
"macro_env_var",
"macro_event_broker",
+ "macro_service_urls",
"macro_user_id",
"memory",
+ "models_email",
+ "opentelemetry",
"rdkafka",
"rootcause",
"serde",
@@ -7433,7 +7439,9 @@ dependencies = [
"tokio",
"tokio-util",
"tracing",
+ "tracing-opentelemetry",
"uuid",
+ "wiremock",
"workspace-hack",
]
diff --git a/apps/web/src/lib/queries/email/focus.test.ts b/apps/web/src/lib/queries/email/focus.test.ts
new file mode 100644
index 0000000000..93f9263433
--- /dev/null
+++ b/apps/web/src/lib/queries/email/focus.test.ts
@@ -0,0 +1,210 @@
+import { WebsocketEvent } from '@macro-inc/collaboration/websocket';
+import { Telemetry } from '@macro-inc/observability';
+import { createClient, type Operation } from '@urql/core';
+import { createRoot, createSignal } from 'solid-js';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { map, pipe, tap } from 'wonka';
+import { useEmailFocusQuery } from './focus';
+
+const socket = vi.hoisted(() => ({
+ messages: new Set<(message: { type: string; data: unknown }) => void>(),
+ reconnects: new Set<() => void>(),
+}));
+
+// Exercise the real query binding without UI barrels or live connections.
+vi.mock('@service-connection/websocket', async () => ({
+ ...(await import('@service-connection/websocket-payload')),
+ createConnectionWebsocketEffect: (
+ callback: (message: { type: string; data: unknown }) => void
+ ) => {
+ socket.messages.add(callback);
+ },
+ ws: {
+ addEventListener: (event: WebsocketEvent, callback: () => void) => {
+ if (event === WebsocketEvent.Reconnect) socket.reconnects.add(callback);
+ },
+ removeEventListener: (event: WebsocketEvent, callback: () => void) => {
+ if (event === WebsocketEvent.Reconnect)
+ socket.reconnects.delete(callback);
+ },
+ },
+}));
+vi.mock('@entity', () => ({ isEmailEntity: () => true }));
+vi.mock('@service-storage/graphql-soup', () => ({
+ getGraphqlSoupClient: () => client,
+ mapGraphqlSoupItem: () => undefined,
+}));
+vi.mock('../soup/transform-utils', () => ({
+ isDisplayableSoupItem: () => false,
+ mapApiSoupItemToEntity: () => undefined,
+}));
+
+const reads: Operation[] = [];
+const client = createClient({
+ url: '/graphql',
+ exchanges: [
+ () => (operations) =>
+ pipe(
+ operations,
+ tap((operation: Operation) => {
+ if (operation.kind === 'query') reads.push(operation);
+ }),
+ map((operation: Operation) => ({
+ operation,
+ data: { user: { id: 'viewer', emailFocus: [] } },
+ stale: false,
+ hasNext: false,
+ }))
+ ),
+ ],
+});
+
+function push(data: unknown, type = 'refresh_email') {
+ for (const message of socket.messages) {
+ message({ type, data: JSON.stringify(data) });
+ }
+}
+
+const FOCUS_UPDATED = { event: 'focus_updated', link_id: 'inbox' };
+
+let visibility: DocumentVisibilityState;
+let dispose: (() => void) | undefined;
+
+function setVisibility(value: DocumentVisibilityState) {
+ visibility = value;
+ document.dispatchEvent(new Event('visibilitychange'));
+}
+
+/** Mount the Focus list and settle its first read. */
+async function mount(enabled = true) {
+ const [isEnabled, setEnabled] = createSignal(enabled);
+ dispose = createRoot((disposeRoot) => {
+ useEmailFocusQuery(isEnabled);
+ return disposeRoot;
+ });
+ await vi.advanceTimersByTimeAsync(0);
+ reads.length = 0;
+ return { setEnabled };
+}
+
+beforeEach(() => {
+ vi.useFakeTimers();
+ visibility = 'visible';
+ vi.spyOn(document, 'visibilityState', 'get').mockImplementation(
+ () => visibility
+ );
+ reads.length = 0;
+});
+
+afterEach(() => {
+ dispose?.();
+ dispose = undefined;
+ socket.messages.clear();
+ socket.reconnects.clear();
+ vi.useRealTimers();
+ vi.restoreAllMocks();
+});
+
+describe('useEmailFocusQuery', () => {
+ it('reads Focus from the server when the worker stores a classification', async () => {
+ await mount();
+
+ push(FOCUS_UPDATED);
+ await vi.advanceTimersByTimeAsync(0);
+
+ expect(reads).toHaveLength(1);
+ expect(reads[0]?.context.requestPolicy).toBe('network-only');
+ expect(reads[0]?.variables).toEqual({ days: 30 });
+ });
+
+ it('turns a burst of pushes into few reads', async () => {
+ await mount();
+
+ for (let index = 0; index < 20; index++) push(FOCUS_UPDATED);
+ await vi.advanceTimersByTimeAsync(1_000);
+
+ // At once, then one for everything that came while it was throttled.
+ expect(reads).toHaveLength(2);
+ });
+
+ it('ignores other inbox refreshes and other messages', async () => {
+ await mount();
+
+ push({ event: 'upsert_message', link_id: 'inbox' });
+ push(FOCUS_UPDATED, 'message_update');
+ await vi.advanceTimersByTimeAsync(1_000);
+
+ expect(reads).toHaveLength(0);
+ });
+
+ it('holds a hidden tab to one read when it next shows', async () => {
+ await mount();
+ setVisibility('hidden');
+
+ push(FOCUS_UPDATED);
+ push(FOCUS_UPDATED);
+ await vi.advanceTimersByTimeAsync(10_000);
+ expect(reads).toHaveLength(0);
+
+ setVisibility('visible');
+ await vi.advanceTimersByTimeAsync(1_000);
+ expect(reads).toHaveLength(1);
+ });
+
+ it('reads when a hidden tab shows, in case a push was lost', async () => {
+ await mount();
+ setVisibility('hidden');
+ await vi.advanceTimersByTimeAsync(10_000);
+ expect(reads).toHaveLength(0);
+
+ setVisibility('visible');
+ await vi.advanceTimersByTimeAsync(1_000);
+ expect(reads).toHaveLength(1);
+ });
+
+ it('leaves a closed Focus tab to read when it opens', async () => {
+ const { setEnabled } = await mount(false);
+
+ push(FOCUS_UPDATED);
+ await vi.advanceTimersByTimeAsync(1_000);
+ expect(reads).toHaveLength(0);
+
+ setEnabled(true);
+ await vi.advanceTimersByTimeAsync(1_000);
+ expect(reads).toHaveLength(1);
+ });
+
+ it('reads again after a reconnect that may have missed a push', async () => {
+ await mount();
+
+ for (const reconnect of socket.reconnects) reconnect();
+ await vi.advanceTimersByTimeAsync(0);
+
+ expect(reads).toHaveLength(1);
+ });
+
+ it('stops listening when it unmounts', async () => {
+ await mount();
+ dispose?.();
+ dispose = undefined;
+
+ for (const reconnect of socket.reconnects) reconnect();
+ await vi.advanceTimersByTimeAsync(1_000);
+
+ expect(socket.reconnects.size).toBe(0);
+ expect(reads).toHaveLength(0);
+ });
+
+ it('joins the trace the push carries', async () => {
+ const remoteContext = vi.spyOn(Telemetry, 'remoteContext');
+ await mount();
+ const traceparent =
+ '00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01';
+
+ push({ ...FOCUS_UPDATED, traceparent });
+ await vi.advanceTimersByTimeAsync(0);
+
+ expect(remoteContext).toHaveBeenCalledWith(traceparent);
+ expect(reads[0]?.context.macroTraceParent).toBeDefined();
+ });
+});
diff --git a/apps/web/src/lib/queries/email/focus.ts b/apps/web/src/lib/queries/email/focus.ts
index 72f391c2da..74435240d1 100644
--- a/apps/web/src/lib/queries/email/focus.ts
+++ b/apps/web/src/lib/queries/email/focus.ts
@@ -1,5 +1,14 @@
+import { TRACE_PARENT_CONTEXT_KEY } from '@app/lib/urql-solid/trace-parent-exchange';
import { type EmailEntity, isEmailEntity } from '@entity';
import { createLiveQuery } from '@graphql-cache/solid/create-live-query';
+import { WebsocketEvent } from '@macro-inc/collaboration/websocket';
+import { Telemetry, type TraceContext } from '@macro-inc/observability';
+import {
+ createConnectionWebsocketEffect,
+ parseWebsocketPayload,
+ ws,
+} from '@service-connection/websocket';
+import type { RefreshEmailEvent } from '@service-email/generated/schemas';
import {
EmailFocusDocument,
type EmailFocusQuery,
@@ -8,11 +17,12 @@ import {
getGraphqlSoupClient,
mapGraphqlSoupItem,
} from '@service-storage/graphql-soup';
-import type { Accessor } from 'solid-js';
+import { type Accessor, onCleanup } from 'solid-js';
import {
isDisplayableSoupItem,
mapApiSoupItemToEntity,
} from '../soup/transform-utils';
+import { createVisibleRefresh } from '../visible-refresh';
/** Days of mail the Focus list covers. */
export const FOCUS_WINDOW_DAYS = 30;
@@ -40,9 +50,14 @@ export function mapEmailFocus(data: EmailFocusQuery): EmailEntity[] {
return threads;
}
-/** The viewer's Focus list, read through the shared GraphQL client. */
+/**
+ * The viewer's Focus list, read through the shared GraphQL client. The list is
+ * one read the normalized cache can't maintain, so the worker's `focus_updated`
+ * push, or a reconnect that may have missed one, reads it again. A hidden tab
+ * catches up once when it next shows; a closed tab reads when it opens.
+ */
export function useEmailFocusQuery(enabled: Accessor<boolean>) {
- return createLiveQuery(
+ const query = createLiveQuery(
EmailFocusDocument,
() => (enabled() ? { days: FOCUS_WINDOW_DAYS } : undefined),
() => ({
@@ -51,4 +66,47 @@ export function useEmailFocusQuery(enabled: Accessor<boolean>) {
select: mapEmailFocus,
})
);
+
+ let pushed: TraceContext | undefined;
+ const refresh = createVisibleRefresh(
+ () => {
+ if (!enabled()) return;
+ const parent = pushed ?? Telemetry.captureContext();
+ pushed = undefined;
+ return parent.run(() =>
+ Telemetry.span('email.focus.refresh', async (span) => {
+ const result = await query.refetch({
+ requestPolicy: 'network-only',
+ // A query made in a view's flow would otherwise send in that trace.
+ context: { [TRACE_PARENT_CONTEXT_KEY]: span },
+ });
+ span.setAttr('email.focus.threads', result.data?.length ?? 0);
+ if (result.error) span.error('Focus refresh failed');
+ })
+ );
+ },
+ // The push is best effort, so one lost while hidden must not outlast the tab showing.
+ { refreshOnShow: true }
+ );
+ onCleanup(refresh.dispose);
+
+ createConnectionWebsocketEffect((message) => {
+ if (message.type !== 'refresh_email' || !enabled()) return;
+ const event = parseWebsocketPayload<RefreshEmailEvent>(
+ message.type,
+ message.data
+ );
+ if (event?.event !== 'focus_updated') return;
+ if (event.traceparent) pushed = Telemetry.remoteContext(event.traceparent);
+ refresh.markDirty();
+ });
+ const onReconnect = () => {
+ if (enabled()) refresh.markDirty();
+ };
+ ws.addEventListener(WebsocketEvent.Reconnect, onReconnect);
+ onCleanup(() =>
+ ws.removeEventListener(WebsocketEvent.Reconnect, onReconnect)
+ );
+
+ return query;
}
diff --git a/apps/web/src/lib/queries/email/sync.ts b/apps/web/src/lib/queries/email/sync.ts
index 9bf0b625d4..cbcc01198a 100644
--- a/apps/web/src/lib/queries/email/sync.ts
+++ b/apps/web/src/lib/queries/email/sync.ts
@@ -52,6 +52,7 @@ function asRefreshEmailEvent(payload: unknown): RefreshEmailEvent | undefined {
* `SYNCING`, with a toast. The legacy `backfill` event now only signals
* failure. `photo_synced` refetches the links list so the inbox's derived
* `photo_url` lands once its self-contact photo finishes uploading.
+ * `focus_updated` belongs to the mounted Focus list (see `useEmailFocusQuery`).
*/
export function handleRefreshEmail(payload: unknown): void {
const event = asRefreshEmailEvent(payload);
diff --git a/apps/web/src/lib/queries/subscribe-to-visible-cache-changes.ts b/apps/web/src/lib/queries/subscribe-to-visible-cache-changes.ts
index b28e27be54..0f097e3a35 100644
--- a/apps/web/src/lib/queries/subscribe-to-visible-cache-changes.ts
+++ b/apps/web/src/lib/queries/subscribe-to-visible-cache-changes.ts
@@ -1,17 +1,10 @@
-import { fireAndForget } from '@core/util/fireAndForget';
import type { CacheHost } from '@graphql-cache/host/types';
-import { makeEventListener } from '@solid-primitives/event-listener';
-import { leadingAndTrailing, throttle } from '@solid-primitives/scheduled';
import { type Accessor, untrack } from 'solid-js';
-
-const CACHE_REFRESH_INTERVAL_MS = 250;
+import { createVisibleRefresh } from './visible-refresh';
/**
- * Throttles cache-driven UI refreshes while visible. Hidden tabs retain one
- * dirty bit instead of repeatedly reading the shared cache; becoming visible
- * refreshes once against its latest state. Async refreshes finish before one
- * dirty follow-up runs, so a busy cache cannot continually cancel its readers.
- * This does not pause cache ingestion, mutations, or explicit/initial requests.
+ * Refreshes after cache changes on `createVisibleRefresh`'s terms. This does
+ * not pause cache ingestion, mutations, or explicit/initial requests.
*
* The caller must unsubscribe on disposal.
*/
@@ -20,32 +13,8 @@ export function subscribeToVisibleCacheChanges(
refresh: () => void | Promise<unknown>,
options?: { searchBuckets: Accessor<readonly string[]> }
): () => void {
- let dirty = false;
+ const visibleRefresh = createVisibleRefresh(refresh);
let disposed = false;
- let refreshing = false;
- const isVisible = () => document.visibilityState === 'visible';
- const runRefresh = async () => {
- // Visibility can change between scheduling and the trailing callback.
- if (disposed || refreshing || !dirty || !isVisible()) return;
- dirty = false;
- refreshing = true;
- try {
- const completion = refresh();
- if (completion) await completion;
- } catch (error) {
- console.warn('Cache-driven refresh failed', error);
- } finally {
- refreshing = false;
- // Keep changes received during a read, including the last change in a
- // burst. A failed read only retries if another change made it dirty.
- if (dirty && !disposed && isVisible()) scheduled();
- }
- };
- const scheduled = leadingAndTrailing(
- throttle,
- () => fireAndForget(runRefresh(), 'visible-cache-refresh'),
- CACHE_REFRESH_INTERVAL_MS
- );
const unsubscribe = host.onCacheChanged(
(_revision, changes) => {
if (disposed) return;
@@ -54,26 +23,14 @@ export function subscribeToVisibleCacheChanges(
const watched = untrack(options.searchBuckets);
if (!changedBuckets.some((bucket) => watched.includes(bucket))) return;
}
- dirty = true;
- if (isVisible()) scheduled();
+ visibleRefresh.markDirty();
},
{ includeHydration: true }
);
- const removeVisibilityListener = makeEventListener(
- document,
- 'visibilitychange',
- () => {
- // Cancel pending background work without losing its dirty state. Reset
- // the throttle so a visible tab catches up immediately, not on a timer.
- scheduled.clear();
- if (dirty && !disposed && isVisible()) scheduled();
- }
- );
return () => {
disposed = true;
unsubscribe();
- removeVisibilityListener();
- scheduled.clear();
+ visibleRefresh.dispose();
};
}
diff --git a/apps/web/src/lib/queries/visible-refresh.ts b/apps/web/src/lib/queries/visible-refresh.ts
new file mode 100644
index 0000000000..4c8631acbf
--- /dev/null
+++ b/apps/web/src/lib/queries/visible-refresh.ts
@@ -0,0 +1,75 @@
+import { fireAndForget } from '@core/util/fireAndForget';
+import { makeEventListener } from '@solid-primitives/event-listener';
+import { leadingAndTrailing, throttle } from '@solid-primitives/scheduled';
+
+const REFRESH_INTERVAL_MS = 250;
+
+/**
+ * Throttles refreshes while visible. Hidden tabs retain one dirty bit instead
+ * of repeatedly reading; becoming visible refreshes once against the latest
+ * state. Async refreshes finish before one dirty follow-up runs, so a busy
+ * source cannot continually cancel its readers. With `refreshOnShow`, becoming
+ * visible refreshes even if nothing marked it dirty, for a source whose change
+ * signal can be lost.
+ *
+ * The caller must dispose it.
+ */
+export function createVisibleRefresh(
+ refresh: () => void | Promise<unknown>,
+ { refreshOnShow = false }: { refreshOnShow?: boolean } = {}
+): {
+ markDirty: () => void;
+ dispose: () => void;
+} {
+ let dirty = false;
+ let disposed = false;
+ let refreshing = false;
+ const isVisible = () => document.visibilityState === 'visible';
+ const runRefresh = async () => {
+ // Visibility can change between scheduling and the trailing callback.
+ if (disposed || refreshing || !dirty || !isVisible()) return;
+ dirty = false;
+ refreshing = true;
+ try {
+ const completion = refresh();
+ if (completion) await completion;
+ } catch (error) {
+ console.warn('Visible refresh failed', error);
+ } finally {
+ refreshing = false;
+ // Keep changes received during a read, including the last change in a
+ // burst. A failed read only retries if another change made it dirty.
+ if (dirty && !disposed && isVisible()) scheduled();
+ }
+ };
+ const scheduled = leadingAndTrailing(
+ throttle,
+ () => fireAndForget(runRefresh(), 'visible-refresh'),
+ REFRESH_INTERVAL_MS
+ );
+ const removeVisibilityListener = makeEventListener(
+ document,
+ 'visibilitychange',
+ () => {
+ // Cancel pending background work without losing its dirty state. Reset
+ // the throttle so a visible tab catches up immediately, not on a timer.
+ scheduled.clear();
+ if (disposed || !isVisible()) return;
+ if (refreshOnShow) dirty = true;
+ if (dirty) scheduled();
+ }
+ );
+
+ return {
+ markDirty: () => {
+ if (disposed) return;
+ dirty = true;
+ if (isVisible()) scheduled();
+ },
+ dispose: () => {
+ disposed = true;
+ removeVisibilityListener();
+ scheduled.clear();
+ },
+ };
+}
diff --git a/apps/web/src/lib/service-clients/service-email/generated/schemas/index.ts b/apps/web/src/lib/service-clients/service-email/generated/schemas/index.ts
index 4123f3d80d..3eedb2c784 100644
--- a/apps/web/src/lib/service-clients/service-email/generated/schemas/index.ts
+++ b/apps/web/src/lib/service-clients/service-email/generated/schemas/index.ts
@@ -365,6 +365,9 @@ export * from './refreshEmailEventOneOfFive';
export * from './refreshEmailEventOneOfFiveEvent';
export * from './refreshEmailEventOneOfNine';
export * from './refreshEmailEventOneOfNineEvent';
+export * from './refreshEmailEventOneOfOnefive';
+export * from './refreshEmailEventOneOfOnefiveEvent';
+export * from './refreshEmailEventOneOfOnefiveTraceparent';
export * from './refreshEmailEventOneOfOneone';
export * from './refreshEmailEventOneOfOneoneEvent';
export * from './refreshEmailEventOneOfOnethree';
diff --git a/apps/web/src/lib/service-clients/service-email/generated/schemas/refreshEmailEvent.ts b/apps/web/src/lib/service-clients/service-email/generated/schemas/refreshEmailEvent.ts
index dd995b274c..e5ad2fe0e9 100644
--- a/apps/web/src/lib/service-clients/service-email/generated/schemas/refreshEmailEvent.ts
+++ b/apps/web/src/lib/service-clients/service-email/generated/schemas/refreshEmailEvent.ts
@@ -7,6 +7,7 @@
import type { RefreshEmailEventOneOf } from './refreshEmailEventOneOf';
import type { RefreshEmailEventOneOfFive } from './refreshEmailEventOneOfFive';
import type { RefreshEmailEventOneOfNine } from './refreshEmailEventOneOfNine';
+import type { RefreshEmailEventOneOfOnefive } from './refreshEmailEventOneOfOnefive';
import type { RefreshEmailEventOneOfOneone } from './refreshEmailEventOneOfOneone';
import type { RefreshEmailEventOneOfOnethree } from './refreshEmailEventOneOfOnethree';
import type { RefreshEmailEventOneOfSeven } from './refreshEmailEventOneOfSeven';
@@ -23,4 +24,5 @@ export type RefreshEmailEvent =
| RefreshEmailEventOneOfSeven
| RefreshEmailEventOneOfNine
| RefreshEmailEventOneOfOneone
- | RefreshEmailEventOneOfOnethree;
+ | RefreshEmailEventOneOfOnethree
+ | RefreshEmailEventOneOfOnefive;
diff --git a/apps/web/src/lib/service-clients/service-email/generated/schemas/refreshEmailEventOneOfOnefive.ts b/apps/web/src/lib/service-clients/service-email/generated/schemas/refreshEmailEventOneOfOnefive.ts
new file mode 100644
index 0000000000..c96f4ca85c
--- /dev/null
+++ b/apps/web/src/lib/service-clients/service-email/generated/schemas/refreshEmailEventOneOfOnefive.ts
@@ -0,0 +1,20 @@
+/**
+ * Generated by orval v7.21.0 🍺
+ * Do not edit manually.
+ * email_service
+ * OpenAPI spec version: 0.1.0
+ */
+import type { RefreshEmailEventOneOfOnefiveEvent } from './refreshEmailEventOneOfOnefiveEvent';
+import type { RefreshEmailEventOneOfOnefiveTraceparent } from './refreshEmailEventOneOfOnefiveTraceparent';
+
+/**
+ * A Focus classification was stored for a thread in `link_id`, so its
+readers read their Focus list again. `traceparent` is the W3C trace
+context of the classification, for that read to join. Coalesced by the
+sender; carries nothing about the mail.
+ */
+export type RefreshEmailEventOneOfOnefive = {
+ event: RefreshEmailEventOneOfOnefiveEvent;
+ link_id: string;
+ traceparent?: RefreshEmailEventOneOfOnefiveTraceparent;
+};
diff --git a/apps/web/src/lib/service-clients/service-email/generated/schemas/refreshEmailEventOneOfOnefiveEvent.ts b/apps/web/src/lib/service-clients/service-email/generated/schemas/refreshEmailEventOneOfOnefiveEvent.ts
new file mode 100644
index 0000000000..53c3403b9b
--- /dev/null
+++ b/apps/web/src/lib/service-clients/service-email/generated/schemas/refreshEmailEventOneOfOnefiveEvent.ts
@@ -0,0 +1,14 @@
+/**
+ * Generated by orval v7.21.0 🍺
+ * Do not edit manually.
+ * email_service
+ * OpenAPI spec version: 0.1.0
+ */
+
+export type RefreshEmailEventOneOfOnefiveEvent =
+ (typeof RefreshEmailEventOneOfOnefiveEvent)[keyof typeof RefreshEmailEventOneOfOnefiveEvent];
+
+// eslint-disable-next-line @typescript-eslint/no-redeclare
+export const RefreshEmailEventOneOfOnefiveEvent = {
+ focus_updated: 'focus_updated',
+} as const;
diff --git a/apps/web/src/lib/service-clients/service-email/generated/schemas/refreshEmailEventOneOfOnefiveTraceparent.ts b/apps/web/src/lib/service-clients/service-email/generated/schemas/refreshEmailEventOneOfOnefiveTraceparent.ts
new file mode 100644
index 0000000000..2cfa841774
--- /dev/null
+++ b/apps/web/src/lib/service-clients/service-email/generated/schemas/refreshEmailEventOneOfOnefiveTraceparent.ts
@@ -0,0 +1,8 @@
+/**
+ * Generated by orval v7.21.0 🍺
+ * Do not edit manually.
+ * email_service
+ * OpenAPI spec version: 0.1.0
+ */
+
+export type RefreshEmailEventOneOfOnefiveTraceparent = string | null;
diff --git a/apps/web/src/lib/service-clients/service-email/openapi.json b/apps/web/src/lib/service-clients/service-email/openapi.json
index 4671c31a92..8c7eaa66a4 100644
--- a/apps/web/src/lib/service-clients/service-email/openapi.json
+++ b/apps/web/src/lib/service-clients/service-email/openapi.json
@@ -7295,6 +7295,24 @@
"format": "int32"
}
}
+ },
+ {
+ "type": "object",
+ "description": "A Focus classification was stored for a thread in `link_id`, so its\nreaders read their Focus list again. `traceparent` is the W3C trace\ncontext of the classification, for that read to join. Coalesced by the\nsender; carries nothing about the mail.",
+ "required": ["link_id", "event"],
+ "properties": {
+ "event": {
+ "type": "string",
+ "enum": ["focus_updated"]
+ },
+ "link_id": {
+ "type": "string",
+ "format": "uuid"
+ },
+ "traceparent": {
+ "type": ["string", "null"]
+ }
+ }
}
],
"description": "Payload for the `refresh_email` connection gateway event: identifies the\ninbox that changed and the kind of change."
diff --git a/crates/connection_gateway_client/src/client/email.rs b/crates/connection_gateway_client/src/client/email.rs
index 8f024bc727..8193023201 100644
--- a/crates/connection_gateway_client/src/client/email.rs
+++ b/crates/connection_gateway_client/src/client/email.rs
@@ -2,6 +2,8 @@ use super::ConnectionGatewayClient;
use connection_gateway_models::MessageReceipt;
use model_entity::EntityType;
+const REFRESH_EMAIL: &str = "refresh_email";
+
impl ConnectionGatewayClient {
/// Trigger refresh of the user's emails
#[tracing::instrument(skip(self, event))]
@@ -12,8 +14,27 @@ impl ConnectionGatewayClient {
) -> anyhow::Result<Vec<MessageReceipt>> {
self.send_message(
EntityType::User.with_entity_str(user_id),
- "refresh_email".to_string(),
+ REFRESH_EMAIL.to_string(),
+ event,
+ )
+ .await
+ }
+
+ /// Trigger a refresh for everyone reading an inbox. User ids hold email
+ /// addresses, so they stay out of the span.
+ #[tracing::instrument(skip_all, fields(users = user_ids.len()))]
+ pub async fn refresh_email_for_users(
+ &self,
+ user_ids: &[String],
+ event: serde_json::Value,
+ ) -> anyhow::Result<Vec<MessageReceipt>> {
+ self.batch_send_message(
+ REFRESH_EMAIL.to_string(),
event,
+ user_ids
+ .iter()
+ .map(|user_id| EntityType::User.with_entity_str(user_id))
+ .collect(),
)
.await
}
diff --git a/crates/email/src/domain/focus.rs b/crates/email/src/domain/focus.rs
index fb89a3e8f5..f66ffa0dd0 100644
--- a/crates/email/src/domain/focus.rs
+++ b/crates/email/src/domain/focus.rs
@@ -29,7 +29,7 @@ pub use models::{
FocusRecord, FocusSignals, FocusThread, FocusVerdict, SentNote, StaleThread,
StoredClassification, ThreadFocus,
};
-pub use ports::{FocusClassifier, FocusClassifierError, FocusStore, ProfileSource};
+pub use ports::{FocusClassifier, FocusClassifierError, FocusNotifier, FocusStore, ProfileSource};
pub use questions::{FOCUS_QUESTIONS, FocusQuestion};
pub use rule::{FOCUS_RULE_VERSION, decide};
pub use service::{ClassifyOutcome, FocusQueries, FocusService, SweepReport};
diff --git a/crates/email/src/domain/focus/ports.rs b/crates/email/src/domain/focus/ports.rs
index de0ddfe5a7..57ef95e502 100644
--- a/crates/email/src/domain/focus/ports.rs
+++ b/crates/email/src/domain/focus/ports.rs
@@ -77,6 +77,14 @@ pub trait ProfileSource: Send + Sync + 'static {
) -> impl Future<Output = Result<Option<String>, Report>> + Send;
}
+/// Tells the people reading an inbox that its Focus changed.
+pub trait FocusNotifier: Send + Sync + 'static {
+ /// A classification was stored for a thread in `link_id`, an inbox `owner`
+ /// holds. Returns at once: delivery is best effort and never holds up or
+ /// fails a classification.
+ fn focus_changed(&self, owner: &MacroUserIdStr<'static>, link_id: Uuid);
+}
+
/// Why the classifier gave no answers.
#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
pub enum FocusClassifierError {
diff --git a/crates/email/src/domain/focus/service.rs b/crates/email/src/domain/focus/service.rs
index c5d5135241..0a41343418 100644
--- a/crates/email/src/domain/focus/service.rs
+++ b/crates/email/src/domain/focus/service.rs
@@ -16,7 +16,7 @@ use super::{
FocusAnswers, FocusMessage, FocusRecord, FocusThread, FocusVerdict, StaleThread,
StoredClassification, ThreadFocus,
},
- ports::{FocusClassifier, FocusClassifierError, FocusStore, ProfileSource},
+ ports::{FocusClassifier, FocusClassifierError, FocusNotifier, FocusStore, ProfileSource},
questions::FOCUS_QUESTIONS,
rule::{FOCUS_RULE_VERSION, decide},
signals::signals,
@@ -76,10 +76,11 @@ pub struct SweepReport {
}
/// Classifies threads for inboxes on the allowlist.
-pub struct FocusService<S, P, C> {
+pub struct FocusService<S, P, C, N> {
store: S,
profiles: P,
classifier: C,
+ notifier: N,
enabled_domains: Vec<String>,
/// Threads a sweep checked at their latest mail without storing a result
/// (rejected, nothing to classify, or a result already stored), so later
@@ -88,15 +89,22 @@ pub struct FocusService<S, P, C> {
settled: Mutex<HashMap<Uuid, DateTime<Utc>>>,
}
-impl<S, P, C> FocusService<S, P, C>
+impl<S, P, C, N> FocusService<S, P, C, N>
where
S: FocusStore,
P: ProfileSource,
C: FocusClassifier,
+ N: FocusNotifier,
{
/// Classify only inboxes whose address is at one of `enabled_domains`;
/// with none, nothing is classified.
- pub fn new(store: S, profiles: P, classifier: C, enabled_domains: Vec<String>) -> Self {
+ pub fn new(
+ store: S,
+ profiles: P,
+ classifier: C,
+ notifier: N,
+ enabled_domains: Vec<String>,
+ ) -> Self {
let enabled_domains = enabled_domains
.into_iter()
.map(|domain| domain.trim().to_lowercase())
@@ -106,6 +114,7 @@ where
store,
profiles,
classifier,
+ notifier,
enabled_domains,
settled: Mutex::default(),
}
@@ -232,7 +241,8 @@ where
answers: FocusAnswers,
model: String,
) -> Result<bool, Report> {
- self.store
+ let stored = self
+ .store
.save(&FocusRecord {
thread_id: thread.thread_id,
link_id: thread.link_id,
@@ -243,7 +253,11 @@ where
model,
classified_at: Utc::now(),
})
- .await
+ .await?;
+ if stored {
+ self.notifier.focus_changed(&thread.owner, thread.link_id);
+ }
+ Ok(stored)
}
/// Check up to `limit` stale threads from the last `window`, newest first,
diff --git a/crates/email/src/domain/focus/service/test.rs b/crates/email/src/domain/focus/service/test.rs
index bcb210d078..d2ca7b7757 100644
--- a/crates/email/src/domain/focus/service/test.rs
+++ b/crates/email/src/domain/focus/service/test.rs
@@ -122,6 +122,17 @@ impl ProfileSource for FakeProfiles {
}
}
+#[derive(Clone, Default)]
+struct FakeNotifier {
+ changed: Arc<Mutex<Vec<(MacroUserIdStr<'static>, Uuid)>>>,
+}
+
+impl FocusNotifier for FakeNotifier {
+ fn focus_changed(&self, owner: &MacroUserIdStr<'static>, link_id: Uuid) {
+ self.changed.lock().unwrap().push((owner.clone(), link_id));
+ }
+}
+
#[derive(Clone)]
struct FakeClassifier {
answers: Result<Vec<f32>, FocusClassifierError>,
@@ -192,11 +203,21 @@ fn service(
store: &FakeStore,
classifier: &FakeClassifier,
domains: &[&str],
-) -> FocusService<FakeStore, FakeProfiles, FakeClassifier> {
+) -> FocusService<FakeStore, FakeProfiles, FakeClassifier, FakeNotifier> {
+ notifying_service(store, classifier, domains, &FakeNotifier::default())
+}
+
+fn notifying_service(
+ store: &FakeStore,
+ classifier: &FakeClassifier,
+ domains: &[&str],
+ notifier: &FakeNotifier,
+) -> FocusService<FakeStore, FakeProfiles, FakeClassifier, FakeNotifier> {
FocusService::new(
store.clone(),
FakeProfiles(Some("Owner runs finance at Acme.".to_owned())),
classifier.clone(),
+ notifier.clone(),
domains.iter().map(|domain| (*domain).to_owned()).collect(),
)
}
@@ -530,6 +551,100 @@ async fn sweep_stops_when_the_classifier_rejects_everything() {
);
}
+#[tokio::test]
+async fn a_stored_classification_tells_the_inbox_readers() {
+ let store = FakeStore::default();
+ let classifier = FakeClassifier::answering(Ok(relevant()));
+ let notifier = FakeNotifier::default();
+ let thread = thread("[email protected]");
+ let (owner, link_id) = (thread.owner.clone(), thread.link_id);
+ let thread_id = insert(&store, thread);
+
+ notifying_service(&store, &classifier, &["acme.example"], &notifier)
+ .classify_thread(thread_id)
+ .await
+ .unwrap();
+
+ assert_eq!(*notifier.changed.lock().unwrap(), vec![(owner, link_id)]);
+}
+
+#[tokio::test]
+async fn nothing_stored_tells_nobody() {
+ let notifier = FakeNotifier::default();
+
+ // A later message's result is already stored.
+ let refusing = FakeStore {
+ refuse_saves: true,
+ ..FakeStore::default()
+ };
+ let thread_id = insert(&refusing, thread("[email protected]"));
+ let classifier = FakeClassifier::answering(Ok(relevant()));
+ let outcome = notifying_service(&refusing, &classifier, &["acme.example"], &notifier)
+ .classify_thread(thread_id)
+ .await
+ .unwrap();
+ assert_eq!(outcome, ClassifyOutcome::UpToDate);
+
+ // The classifier gave no answers.
+ let store = FakeStore::default();
+ let thread_id = insert(&store, thread("[email protected]"));
+ let classifier = FakeClassifier::answering(Err(FocusClassifierError::Unavailable));
+ let outcome = notifying_service(&store, &classifier, &["acme.example"], &notifier)
+ .classify_thread(thread_id)
+ .await
+ .unwrap();
+ assert_eq!(outcome, ClassifyOutcome::Unavailable);
+
+ assert!(notifier.changed.lock().unwrap().is_empty());
+}
+
+#[tokio::test]
+async fn deciding_again_under_a_new_rule_tells_the_inbox_readers() {
+ let store = FakeStore::default();
+ let classifier = FakeClassifier::answering(Err(FocusClassifierError::Unavailable));
+ let notifier = FakeNotifier::default();
+ let mut legacy = thread("[email protected]");
+ legacy.classified = Some(stored_at(&legacy, 0));
+ let (owner, link_id) = (legacy.owner.clone(), legacy.link_id);
+ let id = insert(&store, legacy);
+
+ let outcome = notifying_service(&store, &classifier, &["acme.example"], &notifier)
+ .classify_thread(id)
+ .await
+ .unwrap();
+
+ assert!(matches!(outcome, ClassifyOutcome::Redecided(_)));
+ assert_eq!(*notifier.changed.lock().unwrap(), vec![(owner, link_id)]);
+}
+
+#[tokio::test]
+async fn key_mail_outside_the_signal_inbox_tells_the_inbox_readers() {
+ let store = FakeStore::default();
+ let classifier = FakeClassifier::answering(Ok(relevant()));
+ let notifier = FakeNotifier::default();
+ let service = notifying_service(&store, &classifier, &["acme.example"], &notifier);
+
+ let mut noise = thread("[email protected]");
+ noise.is_signal = false;
+ let noise = insert(&store, noise);
+ assert_eq!(
+ service.classify_thread(noise).await.unwrap(),
+ ClassifyOutcome::NotSignal
+ );
+ assert!(notifier.changed.lock().unwrap().is_empty());
+
+ let mut reschedule = thread("[email protected]");
+ reschedule.is_signal = false;
+ reschedule.messages[0].calendar_part = Some(CalendarPart::Change {
+ until: Utc::now() + Duration::days(2),
+ });
+ let (owner, link_id) = (reschedule.owner.clone(), reschedule.link_id);
+ let reschedule = insert(&store, reschedule);
+ service.classify_thread(reschedule).await.unwrap();
+
+ assert_eq!(*notifier.changed.lock().unwrap(), vec![(owner, link_id)]);
+}
+
#[tokio::test]
async fn an_older_rule_is_decided_again_without_the_classifier() {
let store = FakeStore::default();
diff --git a/crates/email/src/outbound/focus_pg/test.rs b/crates/email/src/outbound/focus_pg/test.rs
index f53a9001b0..1efa1da0ca 100644
--- a/crates/email/src/outbound/focus_pg/test.rs
+++ b/crates/email/src/outbound/focus_pg/test.rs
@@ -6,8 +6,8 @@ use sqlx::{Pool, Postgres};
use super::*;
use crate::domain::{
focus::{
- ClassifyOutcome, FocusAnswers, FocusClassifier, FocusClassifierError, FocusQueries,
- FocusQuestion, FocusService, FocusThread, FocusVerdict, ProfileSource,
+ ClassifyOutcome, FocusAnswers, FocusClassifier, FocusClassifierError, FocusNotifier,
+ FocusQueries, FocusQuestion, FocusService, FocusThread, FocusVerdict, ProfileSource,
},
models::calendar_invitation::{CalendarInvitation, InvitationDateTime, InvitationMethod},
};
@@ -71,14 +71,21 @@ impl FocusClassifier for FixedClassifier {
}
}
+struct NoReaders;
+
+impl FocusNotifier for NoReaders {
+ fn focus_changed(&self, _owner: &MacroUserIdStr<'static>, _link_id: Uuid) {}
+}
+
fn worker(
repo: &FocusPgRepository,
answers: Result<Vec<f32>, FocusClassifierError>,
-) -> FocusService<FocusPgRepository, NoProfile, FixedClassifier> {
+) -> FocusService<FocusPgRepository, NoProfile, FixedClassifier, NoReaders> {
FocusService::new(
repo.clone(),
NoProfile,
FixedClassifier { answers },
+ NoReaders,
vec!["acme.example".to_owned()],
)
}
diff --git a/crates/models_email/src/email/api/refresh.rs b/crates/models_email/src/email/api/refresh.rs
index 109af1d4c9..50f3e9c454 100644
--- a/crates/models_email/src/email/api/refresh.rs
+++ b/crates/models_email/src/email/api/refresh.rs
@@ -46,4 +46,13 @@ pub enum RefreshEmailEvent {
completed_threads: i32,
total_threads: i32,
},
+ /// A Focus classification was stored for a thread in `link_id`, so its
+ /// readers read their Focus list again. `traceparent` is the W3C trace
+ /// context of the classification, for that read to join. Coalesced by the
+ /// sender; carries nothing about the mail.
+ FocusUpdated {
+ link_id: Uuid,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ traceparent: Option<String>,
+ },
}
diff --git a/docs/agent-guide/email.md b/docs/agent-guide/email.md
index 952ceac93a..568c1e7679 100644
--- a/docs/agent-guide/email.md
+++ b/docs/agent-guide/email.md
@@ -186,11 +186,16 @@ options) switches between **Priority**,
**Importance** and **Most recent**; touch layouts have no dropdown and keep the
saved order (Priority by default). The filter menu adds a Focus group (Reply
needed, Follow up) on this tab only and has no Status or Done group there. The
-list is one `user.emailFocus` GraphQL read filtered on the client; it refreshes
-on open, not live. Mark Done hides the row at once and archives the thread; Undo
-brings it back. The worker classifies only inboxes on its
-`FOCUS_ENABLED_EMAIL_DOMAINS` allowlist, so other accounts see the empty state
-`Nothing needs you`.
+list is one `user.emailFocus` GraphQL read filtered on the client. New mail
+reaches an open Focus tab within seconds: when the worker stores a
+classification it pushes `refresh_email` (`focus_updated`) to the inbox's owner
+and delegates, and the tab reads the list again without a reload. A hidden tab
+reads once when it next shows, and any other tab reads when Focus opens. Time
+passing stores nothing and pushes nothing, so a follow-up coming due or a
+calendar change whose event ends shows at the tab's next read. Mark
+Done hides the row at once and archives the thread; Undo brings it back. The
+worker classifies only inboxes on its `FOCUS_ENABLED_EMAIL_DOMAINS` allowlist, so
+other accounts see the empty state `Nothing needs you`.
`Archived`, between All mail and Shared in the desktop Library section, lists
your own archived (Mail Done) threads: the
diff --git a/docs/agent-guide/observability.md b/docs/agent-guide/observability.md
index faf49a8591..8dc1bf5ea4 100644
--- a/docs/agent-guide/observability.md
+++ b/docs/agent-guide/observability.md
@@ -153,6 +153,20 @@ traces. Errors carry fixed messages. Never add addresses, subjects, bodies, or
file names. The `http POST` spans carry `traceparent`, but email_service does not
join the trace yet (see known gaps).
+## Email Focus live updates
+
+Search Tempo for `{ name="email_focus.classify" && span.thread_id="<id>" }`. One
+trace covers an email event from its Kafka publish to the browser's read:
+
+| Span | What it measures |
+| --- | --- |
+| `kafka.process` | One `macro.email` event in the `email-focus` group, until its classification ends. |
+| `email_focus.queue` | The wait for the thread's turn: mail for a thread being classified runs after that call. `event_age_ms` is the time from publish to receipt, which includes any time the consumer stopped reading because every slot was busy. |
+| `email_focus.classify` | One classification: `classify_thread`, with its mail reads, `jev.evaluate`, and `save`. `outcome` is `classified`, `redecided`, `up_to_date`, `missing`, `not_signal`, `not_enabled`, `unavailable`, `rejected`, or `failed`. `event_age_ms` is the time from publish to the end. |
+| `email_focus.notify` | One push to an inbox's readers, under the newest classification it covers. `coalesced` counts the classifications it covers, and 0 marks the repeat for the replica. `delivered` counts the websockets that got it. |
+| `connection_gateway.websocket_send` | The frame leaving the gateway. |
+| `email.focus.refresh` | The browser's read after a push, joined through the push's `traceparent`. A read after a reconnect starts its own trace. `email.focus.threads` is the list's size. |
+
## Agent sessions
A session's lifetime is reconstructable from these spans. All of them carry
diff --git a/docs/agent-guide/soup-live-updates.md b/docs/agent-guide/soup-live-updates.md
index e2686185c6..9d89ea76ae 100644
--- a/docs/agent-guide/soup-live-updates.md
+++ b/docs/agent-guide/soup-live-updates.md
@@ -161,6 +161,12 @@ Checkpoint v14 rehydrates older projections without wiping normalized records or
queued mutations. General Soup still uses its existing server pagination; filter-only
offline checks use an ungrouped view with a created/updated timestamp sort.
+The email Focus list is not a Soup list the cache maintains: membership comes from
+the worker's classifications, so it reads `user.emailFocus` again when told to. See
+the [Focus walkthrough](email.md) and `useEmailFocusQuery`. A list like it pushes
+over the connection gateway and refreshes through `createVisibleRefresh`, which
+holds a hidden tab to one read when it next shows.
+
Realtime Soup batches coalesce repeated entity IDs (including entity type), keeping
that entity's last operation in the batch. Emitted `SoupUpdated` items are non-null.
If viewer-scoped hydration finds no item, the backend logs and omits that update;
diff --git a/infra/stacks/email-service/focus_worker.ts b/infra/stacks/email-service/focus_worker.ts
index 76df0be0a7..94d64f7c5d 100644
--- a/infra/stacks/email-service/focus_worker.ts
+++ b/infra/stacks/email-service/focus_worker.ts
@@ -68,7 +68,8 @@ export class EmailFocusWorker extends pulumi.ComponentResource {
);
this.ecr = image.ecr;
- // The worker only calls out: Postgres, Kafka and the classifier API.
+ // The worker only calls out: Postgres, Kafka, the classifier API and the
+ // connection gateway.
const serviceSg = new aws.ec2.SecurityGroup(
`${BASE_NAME}-sg-${stack}`,
{
diff --git a/packages/observability/src/telemetry.test.ts b/packages/observability/src/telemetry.test.ts
index 841f9edff8..079d9d5e0f 100644
--- a/packages/observability/src/telemetry.test.ts
+++ b/packages/observability/src/telemetry.test.ts
@@ -1,6 +1,7 @@
// @vitest-environment jsdom
-import { context, trace } from '@opentelemetry/api';
+import { context, propagation, trace } from '@opentelemetry/api';
import { logs } from '@opentelemetry/api-logs';
+import { W3CTraceContextPropagator } from '@opentelemetry/core';
import {
InMemoryLogRecordExporter,
LoggerProvider,
@@ -17,6 +18,7 @@ import { Telemetry } from './index';
import { userIdSuppressed } from './privacy';
context.setGlobalContextManager(new StackContextManager().enable());
+propagation.setGlobalPropagator(new W3CTraceContextPropagator());
const spanExporter = new InMemorySpanExporter();
trace.setGlobalTracerProvider(
@@ -110,6 +112,39 @@ describe('Telemetry', () => {
expect(spans.get('cache.storage')?.attributes).not.toHaveProperty('usr.id');
});
+ test('a remote context joins the trace another service started', async () => {
+ const traceId = '0af7651916cd43dd8448eb211c80319c';
+ const parentId = 'b7ad6b7169203331';
+ const remote = Telemetry.remoteContext(`00-${traceId}-${parentId}-01`);
+ await Promise.resolve();
+
+ remote.run(() =>
+ Telemetry.span('pushed.read', {}, async (span) => {
+ span.span('pushed.read.phase').end();
+ })
+ );
+ await Promise.resolve();
+
+ const spans = new Map(
+ spanExporter.getFinishedSpans().map((span) => [span.name, span])
+ );
+ const read = spans.get('pushed.read');
+ expect(read?.spanContext().traceId).toBe(traceId);
+ expect(read?.parentSpanContext?.spanId).toBe(parentId);
+ expect(read?.parentSpanContext?.isRemote).toBe(true);
+ expect(spans.get('pushed.read.phase')?.spanContext().traceId).toBe(traceId);
+ });
+
+ test('a malformed remote context starts a new trace', () => {
+ Telemetry.remoteContext('not-a-traceparent').run(() =>
+ Telemetry.span('pushed.read').end()
+ );
+
+ const [read] = spanExporter.getFinishedSpans();
+ expect(read?.parentSpanContext).toBeUndefined();
+ expect(read?.spanContext().traceId).toMatch(/^[0-9a-f]{32}$/);
+ });
+
test('captured context survives an await and restores an unrelated caller', async () => {
const root = Telemetry.span('queued.action');
const parent = root.run(() => Telemetry.captureContext());
diff --git a/packages/observability/src/telemetry.ts b/packages/observability/src/telemetry.ts
index 0ca346e38d..5086db8934 100644
--- a/packages/observability/src/telemetry.ts
+++ b/packages/observability/src/telemetry.ts
@@ -88,6 +88,15 @@ export class Telemetry {
return Telemetry.#tracing.captureContext();
}
+ /**
+ * The trace another service started, for work it asked for over a transport
+ * without headers. Spans started in it join that trace. A malformed value
+ * starts a new trace instead.
+ */
+ static remoteContext(traceparent: string): TraceContext {
+ return Telemetry.#tracing.remoteContext(traceparent);
+ }
+
/** Record a point-in-time sample; timestamp is Unix time in milliseconds. */
static recordSample(
name: string,
diff --git a/packages/observability/src/tracing.ts b/packages/observability/src/tracing.ts
index c2e4c0249d..3187a92109 100644
--- a/packages/observability/src/tracing.ts
+++ b/packages/observability/src/tracing.ts
@@ -2,6 +2,7 @@ import {
type Context,
context,
type Span as OtelSpan,
+ propagation,
ROOT_CONTEXT,
SpanKind,
trace,
@@ -55,6 +56,11 @@ export class Tracing {
return { run: (operation) => context.with(parent, operation) };
}
+ remoteContext(traceparent: string): TraceContext {
+ const parent = propagation.extract(ROOT_CONTEXT, { traceparent });
+ return { run: (operation) => context.with(parent, operation) };
+ }
+
span(name: string, options: SpanOptions = {}): Span {
return this.#startSpan(name, context.active(), options);
}
diff --git a/packages/sdk-docs/reference/generated/email/types.gen/type-aliases/RefreshEmailEvent.md b/packages/sdk-docs/reference/generated/email/types.gen/type-aliases/RefreshEmailEvent.md
index 132ecd1eef..d12c551c10 100644
--- a/packages/sdk-docs/reference/generated/email/types.gen/type-aliases/RefreshEmailEvent.md
+++ b/packages/sdk-docs/reference/generated/email/types.gen/type-aliases/RefreshEmailEvent.md
@@ -1,6 +1,6 @@
# Type Alias: RefreshEmailEvent
-> **RefreshEmailEvent** = \{ `event`: `"backfill"`; `link_id`: `string`; `status`: [`BackfillStatus`](BackfillStatus.md); \} \| \{ `event`: `"upsert_message"`; `link_id`: `string`; \} \| \{ `event`: `"update_labels"`; `link_id`: `string`; \} \| \{ `event`: `"delete_message"`; `link_id`: `string`; \} \| \{ `event`: `"link_removed"`; `link_id`: `string`; \} \| \{ `event`: `"photo_synced"`; `link_id`: `string`; \} \| \{ `completed_threads`: `number`; `event`: `"backfill_progress"`; `link_id`: `string`; `status`: [`BackfillStatus`](BackfillStatus.md); `total_threads`: `number`; \}
+> **RefreshEmailEvent** = \{ `event`: `"backfill"`; `link_id`: `string`; `status`: [`BackfillStatus`](BackfillStatus.md); \} \| \{ `event`: `"upsert_message"`; `link_id`: `string`; \} \| \{ `event`: `"update_labels"`; `link_id`: `string`; \} \| \{ `event`: `"delete_message"`; `link_id`: `string`; \} \| \{ `event`: `"link_removed"`; `link_id`: `string`; \} \| \{ `event`: `"photo_synced"`; `link_id`: `string`; \} \| \{ `completed_threads`: `number`; `event`: `"backfill_progress"`; `link_id`: `string`; `status`: [`BackfillStatus`](BackfillStatus.md); `total_threads`: `number`; \} \| \{ `event`: `"focus_updated"`; `link_id`: `string`; `traceparent?`: `string` \| `null`; \}
Payload for the `refresh_email` connection gateway event: identifies the
inbox that changed and the kind of change.
diff --git a/packages/sdk/generated/email/types.gen.ts b/packages/sdk/generated/email/types.gen.ts
index b0a6438564..7957735f23 100644
--- a/packages/sdk/generated/email/types.gen.ts
+++ b/packages/sdk/generated/email/types.gen.ts
@@ -1743,6 +1743,10 @@ export type RefreshEmailEvent = {
link_id: string;
status: BackfillStatus;
total_threads: number;
+} | {
+ event: 'focus_updated';
+ link_id: string;
+ traceparent?: string | null;
};
/**
diff --git a/packages/sdk/specs/email.json b/packages/sdk/specs/email.json
index 4671c31a92..8c7eaa66a4 100644
--- a/packages/sdk/specs/email.json
+++ b/packages/sdk/specs/email.json
@@ -7295,6 +7295,24 @@
"format": "int32"
}
}
+ },
+ {
+ "type": "object",
+ "description": "A Focus classification was stored for a thread in `link_id`, so its\nreaders read their Focus list again. `traceparent` is the W3C trace\ncontext of the classification, for that read to join. Coalesced by the\nsender; carries nothing about the mail.",
+ "required": ["link_id", "event"],
+ "properties": {
+ "event": {
+ "type": "string",
+ "enum": ["focus_updated"]
+ },
+ "link_id": {
+ "type": "string",
+ "format": "uuid"
+ },
+ "traceparent": {
+ "type": ["string", "null"]
+ }
+ }
}
],
"description": "Payload for the `refresh_email` connection gateway event: identifies the\ninbox that changed and the kind of change."
diff --git a/services/email_focus_worker/Cargo.toml b/services/email_focus_worker/Cargo.toml
index f9cbcc0be6..0c3b87d33b 100644
--- a/services/email_focus_worker/Cargo.toml
+++ b/services/email_focus_worker/Cargo.toml
@@ -15,24 +15,34 @@ path = "src/doppler_config.rs"
[dependencies]
ai_usage = { path = "../../crates/ai_usage" }
chrono = { workspace = true }
+connection_gateway_client = { path = "../../crates/connection_gateway_client" }
doppler-config = { workspace = true }
email = { path = "../../crates/email", default-features = false, features = ["outbound", "ports"] }
jev = { path = "../../crates/jev", features = ["outbound"] }
kafka_util = { path = "../../crates/kafka_util" }
+macro_auth = { path = "../../crates/macro_auth" }
macro_config = { path = "../../crates/macro_config" }
macro_entrypoint = { path = "../../crates/macro_entrypoint" }
macro_env = { path = "../../crates/macro_env" }
macro_env_var = { path = "../../crates/macro_env_var" }
macro_event_broker = { path = "../../crates/macro_event_broker", features = ["outbound"] }
+macro_service_urls = { path = "../../crates/macro_service_urls" }
macro_user_id = { path = "../../crates/macro_user_id" }
memory = { path = "../../crates/memory" }
+models_email = { path = "../../crates/models_email" }
+opentelemetry = { workspace = true, features = ["trace"] }
rdkafka = { workspace = true }
rootcause = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
sqlx = { workspace = true }
tokio = { workspace = true, features = ["signal", "time"] }
-tokio-util = { workspace = true }
+tokio-util = { workspace = true, features = ["rt"] }
tracing = { workspace = true }
+tracing-opentelemetry = { workspace = true }
uuid = { workspace = true }
workspace-hack = { version = "0.1", path = "../../crates/workspace-hack" }
+
+[dev-dependencies]
+macro_db_migrator = { path = "../../crates/macro_db_migrator" }
+wiremock = { workspace = true }
diff --git a/services/email_focus_worker/src/config.rs b/services/email_focus_worker/src/config.rs
index 6c574b37cb..c91b797807 100644
--- a/services/email_focus_worker/src/config.rs
+++ b/services/email_focus_worker/src/config.rs
@@ -1,6 +1,7 @@
//! Configuration for the email focus worker, loaded with `macro_config` from
//! the email-service Doppler project it shares with the other email binaries.
+use macro_auth::InternalApiKey;
pub use macro_env::Environment;
use macro_env_var::env_vars;
@@ -22,6 +23,8 @@ pub struct Config {
pub macro_db_url: MacroDbUrl,
/// Kafka bootstrap servers; the worker reads `macro.email`.
pub kafka_brokers: KafkaBrokers,
+ /// Authenticates the worker's pushes to the connection gateway.
+ pub internal_api_key: InternalApiKey,
/// TypeSafe credential for Jev. While it is unset the worker classifies nothing.
pub typesafe_api_key: jev::outbound::TypesafeApiKey,
/// Comma-separated inbox domains to classify, e.g. `macro.com`. Mail from
diff --git a/services/email_focus_worker/src/inbound/kafka_consumer.rs b/services/email_focus_worker/src/inbound/kafka_consumer.rs
index a7443dc2a4..d9e0f01224 100644
--- a/services/email_focus_worker/src/inbound/kafka_consumer.rs
+++ b/services/email_focus_worker/src/inbound/kafka_consumer.rs
@@ -2,24 +2,37 @@
//!
//! Every email event on the platform passes through here, so the consumer
//! hands each thread to the Focus service, which drops inboxes off the
-//! allowlist after a single-row read. Offsets are committed whatever the
-//! outcome: a lost or failed classification is picked up by the hourly sweep.
+//! allowlist after a single-row read. Up to [`MAX_CONCURRENT_CLASSIFICATIONS`]
+//! threads run side by side, so a slow classifier call delays only its own
+//! thread; with every slot taken the consumer stops reading until one frees.
+//!
+//! An event's offset is committed when it is handed off, in the order received,
+//! not when its classification ends. So an event is never replayed, and one in
+//! flight when the worker dies is lost like any failed classification. The sweep
+//! that runs when the worker starts picks those up, newest first.
use std::{future::Future, sync::Arc};
+use chrono::Utc;
use email::domain::{
events::{EmailMacroEvent, EmailTopicEvent},
- focus::{ClassifyOutcome, FocusClassifier, FocusService, FocusStore, ProfileSource},
+ focus::{
+ ClassifyOutcome, FocusClassifier, FocusNotifier, FocusService, FocusStore, ProfileSource,
+ },
};
use kafka_util::{GroupName, InitialOffset, KafkaEventConsumer};
use macro_event_broker::{
KafkaConsumerAdapter, MacroEvent as _, MacroEventCollection as _, MacroEventConsumerService,
};
-use rdkafka::consumer::CommitMode;
+use rdkafka::{Message as _, consumer::CommitMode};
use rootcause::Report;
-use tracing::Instrument as _;
+use tokio_util::sync::CancellationToken;
+use tracing::{Instrument as _, field::Empty};
use uuid::Uuid;
+use self::classifications::Classifications;
+
+mod classifications;
#[cfg(test)]
mod test;
@@ -45,42 +58,120 @@ fn thread_to_classify(event: &EmailTopicEvent) -> Option<Uuid> {
}
}
-async fn classify<S, P, C>(service: &FocusService<S, P, C>, thread_id: Uuid)
-where
+/// Threads classified at once. Each holds at most one thread's recent mail in
+/// memory and a classifier call that its own deadline bounds.
+const MAX_CONCURRENT_CLASSIFICATIONS: usize = 16;
+
+/// How long ago the event was published, by the Kafka record's timestamp.
+fn event_age_ms(published_at_ms: Option<i64>) -> Option<i64> {
+ published_at_ms.map(|published_at_ms| Utc::now().timestamp_millis() - published_at_ms)
+}
+
+fn outcome_name(outcome: &ClassifyOutcome) -> &'static str {
+ match outcome {
+ ClassifyOutcome::Classified(_) => "classified",
+ ClassifyOutcome::Redecided(_) => "redecided",
+ ClassifyOutcome::UpToDate => "up_to_date",
+ ClassifyOutcome::Missing => "missing",
+ ClassifyOutcome::NotSignal => "not_signal",
+ ClassifyOutcome::NotEnabled => "not_enabled",
+ ClassifyOutcome::Unavailable => "unavailable",
+ ClassifyOutcome::Rejected => "rejected",
+ }
+}
+
+async fn classify<S, P, C, N>(
+ service: &FocusService<S, P, C, N>,
+ thread_id: Uuid,
+ published_at_ms: Option<i64>,
+) where
S: FocusStore,
P: ProfileSource,
C: FocusClassifier,
+ N: FocusNotifier,
{
- match service.classify_thread(thread_id).await {
- Ok(ClassifyOutcome::Classified(verdict)) => tracing::info!(
- %thread_id,
- is_focus = verdict.is_focus,
- category = %verdict.category,
- importance = verdict.importance,
- "focus classified"
- ),
- Ok(outcome @ (ClassifyOutcome::Unavailable | ClassifyOutcome::Rejected)) => {
- tracing::warn!(%thread_id, ?outcome, "focus classification left for the sweep");
+ let span = tracing::info_span!(
+ "email_focus.classify",
+ %thread_id,
+ outcome = Empty,
+ event_age_ms = Empty,
+ otel.status_code = Empty,
+ otel.status_description = Empty,
+ );
+ let result = service
+ .classify_thread(thread_id)
+ .instrument(span.clone())
+ .await;
+ span.record("event_age_ms", event_age_ms(published_at_ms));
+ let _in_span = span.enter();
+ match result {
+ Ok(outcome) => {
+ span.record("outcome", outcome_name(&outcome));
+ match outcome {
+ ClassifyOutcome::Classified(verdict) | ClassifyOutcome::Redecided(verdict) => {
+ tracing::info!(
+ %thread_id,
+ is_focus = verdict.is_focus,
+ category = %verdict.category,
+ importance = verdict.importance,
+ "focus classified"
+ );
+ }
+ ClassifyOutcome::Unavailable | ClassifyOutcome::Rejected => {
+ tracing::warn!(%thread_id, ?outcome, "focus classification left for the sweep");
+ }
+ _ => tracing::debug!(%thread_id, ?outcome, "focus skipped"),
+ }
}
- Ok(outcome) => tracing::debug!(%thread_id, ?outcome, "focus skipped"),
Err(error) => {
+ span.record("outcome", "failed");
+ kafka_util::record_span_error(&span, "focus classification failed");
tracing::warn!(%thread_id, error = ?error, "focus classification failed");
}
}
}
-/// Consume `macro.email` under the `email-focus` group until `shutdown`
-/// resolves. Receive and commit failures are logged and skipped, as in the
-/// activity consumer: a restart would only re-run the startup sweep.
-pub async fn run_focus_consumer<S, P, C>(
+/// The work for one event: waiting for a slot, then classifying, in the event's
+/// trace. Nothing happens until it runs.
+fn classification<S, P, C, N>(
+ service: Arc<FocusService<S, P, C, N>>,
+ thread_id: Uuid,
+ published_at_ms: Option<i64>,
+ event_span: tracing::Span,
+) -> impl Future<Output = ()> + Send + 'static
+where
+ S: FocusStore,
+ P: ProfileSource,
+ C: FocusClassifier,
+ N: FocusNotifier,
+{
+ let queued = tracing::info_span!(
+ parent: &event_span,
+ "email_focus.queue",
+ %thread_id,
+ event_age_ms = event_age_ms(published_at_ms),
+ );
+ async move {
+ drop(queued);
+ classify(&service, thread_id, published_at_ms).await;
+ }
+ .instrument(event_span)
+}
+
+/// Consume `macro.email` under the `email-focus` group until `shutdown`, which
+/// drops the classifications in flight. Receive and commit failures are logged
+/// and skipped, as in the activity consumer: a restart would only re-run the
+/// startup sweep.
+pub async fn run_focus_consumer<S, P, C, N>(
brokers: &str,
- service: Arc<FocusService<S, P, C>>,
- shutdown: impl Future<Output = ()> + Send,
+ service: Arc<FocusService<S, P, C, N>>,
+ shutdown: CancellationToken,
) -> Result<(), Report>
where
S: FocusStore,
P: ProfileSource,
C: FocusClassifier,
+ N: FocusNotifier,
{
let consumer = KafkaEventConsumer::<EmailFocusGroup>::from_env(brokers)?;
let consumer =
@@ -91,12 +182,13 @@ where
group = EmailFocusGroup::GROUP_NAME,
"email focus consumer listening"
);
- let mut shutdown = std::pin::pin!(shutdown);
+ let mut classifications = Classifications::new(MAX_CONCURRENT_CLASSIFICATIONS);
loop {
let message = tokio::select! {
biased;
- () = &mut shutdown => return Ok(()),
- result = consumer.recv() => match result {
+ () = shutdown.cancelled() => break,
+ () = classifications.next_finished() => continue,
+ result = consumer.recv(), if classifications.has_room() => match result {
Ok(message) => message,
Err(error) => {
tracing::error!(error = ?error, "kafka receive error");
@@ -114,12 +206,11 @@ where
}
};
if let Some(thread_id) = thread_id {
- tokio::select! {
- biased;
- // Uncommitted: the event is redelivered to the next consumer.
- () = &mut shutdown => return Ok(()),
- () = classify(&service, thread_id).instrument(span.clone()) => {}
- }
+ let published_at_ms = message.inner().timestamp().to_millis();
+ classifications.submit(
+ thread_id,
+ classification(Arc::clone(&service), thread_id, published_at_ms, span),
+ );
}
if let Err(error) = consumer
.inner()
@@ -128,4 +219,6 @@ where
tracing::error!(error = ?error, "kafka commit error");
}
}
+ classifications.shutdown().await;
+ Ok(())
}
diff --git a/services/email_focus_worker/src/inbound/kafka_consumer/classifications.rs b/services/email_focus_worker/src/inbound/kafka_consumer/classifications.rs
new file mode 100644
index 0000000000..7bf5f6b3c5
--- /dev/null
+++ b/services/email_focus_worker/src/inbound/kafka_consumer/classifications.rs
@@ -0,0 +1,109 @@
+//! The classifications in flight: a bounded number at once, one per thread.
+
+use std::{
+ collections::{HashMap, hash_map::Entry},
+ future::Future,
+};
+
+use tokio::sync::mpsc;
+use tokio_util::{sync::CancellationToken, task::TaskTracker};
+use uuid::Uuid;
+
+#[cfg(test)]
+mod test;
+
+/// Runs classifications side by side so a slow one holds up nothing but its
+/// own thread. Mail for a thread being classified doesn't start a second call:
+/// the newest such event runs once the current one ends, and classifying again
+/// costs nothing if the first call already saw that mail.
+pub struct Classifications<F> {
+ capacity: usize,
+ /// Threads being classified, each with what to run next if more mail came.
+ running: HashMap<Uuid, Option<F>>,
+ finished_tx: mpsc::UnboundedSender<Uuid>,
+ finished_rx: mpsc::UnboundedReceiver<Uuid>,
+ tracker: TaskTracker,
+ cancel: CancellationToken,
+}
+
+/// Reports a thread's task as finished however it ends, a panic included, so
+/// one bad classification can't keep its slot forever.
+struct FinishedOnDrop {
+ thread_id: Uuid,
+ finished: mpsc::UnboundedSender<Uuid>,
+}
+
+impl Drop for FinishedOnDrop {
+ fn drop(&mut self) {
+ // The receiver is only gone once the worker is shutting down.
+ let _ = self.finished.send(self.thread_id);
+ }
+}
+
+impl<F> Classifications<F>
+where
+ F: Future<Output = ()> + Send + 'static,
+{
+ /// Allow `capacity` threads to be classified at once.
+ pub fn new(capacity: usize) -> Self {
+ let (finished_tx, finished_rx) = mpsc::unbounded_channel();
+ Self {
+ capacity,
+ running: HashMap::new(),
+ finished_tx,
+ finished_rx,
+ tracker: TaskTracker::new(),
+ cancel: CancellationToken::new(),
+ }
+ }
+
+ /// Whether another thread can start. Wait with [`Self::next_finished`] until
+ /// it can, so the Kafka consumer stops reading instead of queueing here.
+ pub fn has_room(&self) -> bool {
+ self.running.len() < self.capacity
+ }
+
+ /// Run `classification` for `thread_id`, or after the one running for it.
+ pub fn submit(&mut self, thread_id: Uuid, classification: F) {
+ match self.running.entry(thread_id) {
+ Entry::Occupied(mut running) => {
+ running.insert(Some(classification));
+ }
+ Entry::Vacant(slot) => {
+ slot.insert(None);
+ self.start(thread_id, classification);
+ }
+ }
+ }
+
+ fn start(&self, thread_id: Uuid, classification: F) {
+ let finished = FinishedOnDrop {
+ thread_id,
+ finished: self.finished_tx.clone(),
+ };
+ let cancel = self.cancel.clone();
+ self.tracker.spawn(async move {
+ let _finished = finished;
+ cancel.run_until_cancelled(classification).await;
+ });
+ }
+
+ /// Wait for a classification to end and free its slot, unless its thread
+ /// has another waiting, which then takes the slot. Safe to cancel.
+ pub async fn next_finished(&mut self) {
+ let Some(thread_id) = self.finished_rx.recv().await else {
+ return;
+ };
+ if let Some(next) = self.running.remove(&thread_id).flatten() {
+ self.running.insert(thread_id, None);
+ self.start(thread_id, next);
+ }
+ }
+
+ /// Drop everything in flight and wait for its tasks to end.
+ pub async fn shutdown(self) {
+ self.cancel.cancel();
+ self.tracker.close();
+ self.tracker.wait().await;
+ }
+}
diff --git a/services/email_focus_worker/src/inbound/kafka_consumer/classifications/test.rs b/services/email_focus_worker/src/inbound/kafka_consumer/classifications/test.rs
new file mode 100644
index 0000000000..9bb4597f7c
--- /dev/null
+++ b/services/email_focus_worker/src/inbound/kafka_consumer/classifications/test.rs
@@ -0,0 +1,148 @@
+use std::{
+ pin::Pin,
+ sync::{
+ Arc,
+ atomic::{AtomicUsize, Ordering},
+ },
+};
+
+use tokio::sync::oneshot;
+
+use super::*;
+
+type Classification = Pin<Box<dyn Future<Output = ()> + Send>>;
+
+/// A classification that runs until released, counting its starts.
+fn gated(starts: &Arc<AtomicUsize>) -> (Classification, oneshot::Sender<()>) {
+ let (release, released) = oneshot::channel();
+ let starts = Arc::clone(starts);
+ let classification = Box::pin(async move {
+ starts.fetch_add(1, Ordering::SeqCst);
+ let _ = released.await;
+ });
+ (classification, release)
+}
+
+#[tokio::test]
+async fn a_slow_thread_does_not_hold_up_another() {
+ let mut classifications = Classifications::new(2);
+ let starts = Arc::new(AtomicUsize::new(0));
+
+ let (slow, _still_running) = gated(&starts);
+ classifications.submit(Uuid::now_v7(), slow);
+
+ let (classified, done) = oneshot::channel();
+ classifications.submit(
+ Uuid::now_v7(),
+ Box::pin(async move {
+ let _ = classified.send(());
+ }) as Classification,
+ );
+
+ // The second finishes while the first is still waiting to be released.
+ done.await.unwrap();
+ classifications.next_finished().await;
+ assert!(classifications.has_room());
+ classifications.shutdown().await;
+}
+
+#[tokio::test]
+async fn capacity_bounds_the_threads_running() {
+ let mut classifications = Classifications::new(2);
+ let starts = Arc::new(AtomicUsize::new(0));
+
+ let (first, release_first) = gated(&starts);
+ let (second, _still_running) = gated(&starts);
+ classifications.submit(Uuid::now_v7(), first);
+ assert!(classifications.has_room());
+ classifications.submit(Uuid::now_v7(), second);
+ assert!(!classifications.has_room());
+
+ release_first.send(()).unwrap();
+ classifications.next_finished().await;
+ assert!(classifications.has_room());
+ classifications.shutdown().await;
+}
+
+#[tokio::test]
+async fn mail_during_a_run_classifies_once_more_afterwards() {
+ let mut classifications = Classifications::new(1);
+ let thread_id = Uuid::now_v7();
+ let (first_started_tx, first_started) = oneshot::channel();
+ let (release_first, first_released) = oneshot::channel::<()>();
+ let (newest_ran_tx, newest_ran) = oneshot::channel();
+ let superseded = Arc::new(AtomicUsize::new(0));
+
+ classifications.submit(
+ thread_id,
+ Box::pin(async move {
+ let _ = first_started_tx.send(());
+ let _ = first_released.await;
+ }) as Classification,
+ );
+ first_started.await.unwrap();
+
+ let superseded_runs = Arc::clone(&superseded);
+ classifications.submit(
+ thread_id,
+ Box::pin(async move {
+ superseded_runs.fetch_add(1, Ordering::SeqCst);
+ }),
+ );
+ classifications.submit(
+ thread_id,
+ Box::pin(async move {
+ let _ = newest_ran_tx.send(());
+ }),
+ );
+
+ // Two more events took no slot of their own.
+ assert_eq!(classifications.running.len(), 1);
+ release_first.send(()).unwrap();
+ classifications.next_finished().await;
+ newest_ran.await.unwrap();
+ assert!(!classifications.has_room());
+ classifications.next_finished().await;
+ assert!(classifications.has_room());
+ assert_eq!(superseded.load(Ordering::SeqCst), 0);
+ classifications.shutdown().await;
+}
+
+#[tokio::test]
+async fn shutdown_drops_what_is_running() {
+ let mut classifications = Classifications::new(1);
+ let (started_tx, started) = oneshot::channel();
+ let (dropped_tx, dropped) = oneshot::channel::<()>();
+ classifications.submit(
+ Uuid::now_v7(),
+ Box::pin(async move {
+ let _dropped_with_the_future = dropped_tx;
+ let _ = started_tx.send(());
+ std::future::pending::<()>().await;
+ }) as Classification,
+ );
+ started.await.unwrap();
+
+ classifications.shutdown().await;
+
+ // The sender went with the future when shutdown cancelled it.
+ assert!(dropped.await.is_err());
+}
+
+fn bad_thread() {
+ panic!("a bad thread");
+}
+
+#[tokio::test]
+async fn a_panic_gives_its_slot_back() {
+ let mut classifications = Classifications::new(1);
+ classifications.submit(
+ Uuid::now_v7(),
+ Box::pin(async { bad_thread() }) as Classification,
+ );
+
+ classifications.next_finished().await;
+
+ assert!(classifications.has_room());
+ classifications.shutdown().await;
+}
diff --git a/services/email_focus_worker/src/inbound/sweep.rs b/services/email_focus_worker/src/inbound/sweep.rs
index f5d115b618..7fedf4a76c 100644
--- a/services/email_focus_worker/src/inbound/sweep.rs
+++ b/services/email_focus_worker/src/inbound/sweep.rs
@@ -9,7 +9,9 @@
use std::{sync::Arc, time::Duration};
-use email::domain::focus::{FocusClassifier, FocusService, FocusStore, ProfileSource};
+use email::domain::focus::{
+ FocusClassifier, FocusNotifier, FocusService, FocusStore, ProfileSource,
+};
use rootcause::Report;
use tokio_util::sync::CancellationToken;
@@ -17,8 +19,8 @@ use tokio_util::sync::CancellationToken;
const SWEEP_LIMIT: i64 = 2_000;
/// Sweep every `interval` over the last `window_days` until `shutdown`.
-pub async fn run_sweeps<S, P, C>(
- service: Arc<FocusService<S, P, C>>,
+pub async fn run_sweeps<S, P, C, N>(
+ service: Arc<FocusService<S, P, C, N>>,
interval: Duration,
window_days: u16,
shutdown: CancellationToken,
@@ -27,6 +29,7 @@ where
S: FocusStore,
P: ProfileSource,
C: FocusClassifier,
+ N: FocusNotifier,
{
let window = chrono::Duration::days(i64::from(window_days));
let mut ticker = tokio::time::interval(interval);
diff --git a/services/email_focus_worker/src/main.rs b/services/email_focus_worker/src/main.rs
index 6320041784..349dfe4d2a 100644
--- a/services/email_focus_worker/src/main.rs
+++ b/services/email_focus_worker/src/main.rs
@@ -3,8 +3,9 @@
//!
//! Reads `macro.email` for new and sent messages, asks Jev about each signal
//! thread and key mail in an allowlisted inbox with the owner's stored memory
-//! as context, and stores the verdict in `email_thread_focus`. An hourly sweep
-//! catches what the event stream missed.
+//! as context, stores the verdict in `email_thread_focus`, and tells the inbox's
+//! readers through the connection gateway. An hourly sweep catches what the
+//! event stream missed.
mod config;
mod inbound {
@@ -12,24 +13,31 @@ mod inbound {
pub mod sweep;
}
mod outbound {
+ pub mod gateway_notifier;
pub mod jev_classifier;
pub mod memory_profile;
}
use std::{sync::Arc, time::Duration};
+use connection_gateway_client::ConnectionGatewayClient;
use email::{domain::focus::FocusService, outbound::FocusPgRepository};
use jev::{domain::JevClassifier, outbound::SystemOneJev};
use macro_entrypoint::{MacroEntrypoint, shutdown_signal};
+use macro_service_urls::ConnectionGatewayUrl;
use memory::outbound::pg_memory_repo::PgMemoryRepo;
use rootcause::Report;
use sqlx::postgres::PgPoolOptions;
-use tokio_util::sync::CancellationToken;
+use tokio_util::{sync::CancellationToken, task::TaskTracker};
use crate::{
config::Config,
inbound::{kafka_consumer::run_focus_consumer, sweep::run_sweeps},
- outbound::{jev_classifier::JevFocusClassifier, memory_profile::MemoryProfiles},
+ outbound::{
+ gateway_notifier::{CoalescingNotifier, GatewayDelivery},
+ jev_classifier::JevFocusClassifier,
+ memory_profile::MemoryProfiles,
+ },
};
#[tokio::main]
@@ -59,13 +67,24 @@ async fn main() -> Result<(), Report> {
.max_connections(5)
.connect(config.macro_db_url.as_ref())
.await?;
+ let shutdown = CancellationToken::new();
+ let notifications = TaskTracker::new();
+ let gateway = ConnectionGatewayClient::new(
+ config.internal_api_key.to_string(),
+ ConnectionGatewayUrl::new()?.to_string(),
+ );
let service = Arc::new(FocusService::new(
FocusPgRepository(db.clone()),
MemoryProfiles::new(PgMemoryRepo::new(db.clone())),
JevFocusClassifier::new(JevClassifier::new(
provider,
- ai_usage::pg_recorder_with_enforcement(db, enforcement),
+ ai_usage::pg_recorder_with_enforcement(db.clone(), enforcement),
)),
+ CoalescingNotifier::new(
+ GatewayDelivery::new(gateway, db),
+ notifications.clone(),
+ shutdown.clone(),
+ ),
domains,
));
tracing::info!(
@@ -74,7 +93,6 @@ async fn main() -> Result<(), Report> {
"email focus worker started"
);
- let shutdown = CancellationToken::new();
tokio::spawn({
let shutdown = shutdown.clone();
async move {
@@ -83,11 +101,7 @@ async fn main() -> Result<(), Report> {
}
});
let brokers = config.kafka_brokers.to_string();
- let consumer = run_focus_consumer(
- &brokers,
- Arc::clone(&service),
- shutdown.clone().cancelled_owned(),
- );
+ let consumer = run_focus_consumer(&brokers, Arc::clone(&service), shutdown.clone());
let sweeps = run_sweeps(
service,
// A zero interval would panic the timer.
@@ -98,5 +112,7 @@ async fn main() -> Result<(), Report> {
// Either loop ending with an error stops the worker so it restarts fresh.
let result = tokio::try_join!(consumer, sweeps).map(|_| ());
shutdown.cancel();
+ notifications.close();
+ notifications.wait().await;
result
}
diff --git a/services/email_focus_worker/src/outbound/gateway_notifier.rs b/services/email_focus_worker/src/outbound/gateway_notifier.rs
new file mode 100644
index 0000000000..4e5b4b1796
--- /dev/null
+++ b/services/email_focus_worker/src/outbound/gateway_notifier.rs
@@ -0,0 +1,220 @@
+//! Tells an inbox's readers its Focus changed, through the connection gateway.
+//!
+//! A burst of mail in one inbox becomes few pushes. The first goes out at once;
+//! what follows collapses into one per [`NOTIFY_WINDOW`]. The last push of a
+//! burst is always a repeat one window later, because readers read Focus from a
+//! replica that can lag the write they were told about.
+
+use std::{
+ collections::HashMap,
+ future::Future,
+ sync::{Arc, Mutex, MutexGuard, PoisonError},
+ time::Duration,
+};
+
+use connection_gateway_client::ConnectionGatewayClient;
+use email::domain::focus::FocusNotifier;
+use macro_user_id::user_id::MacroUserIdStr;
+use models_email::api::refresh::RefreshEmailEvent;
+use sqlx::PgPool;
+use tokio_util::{sync::CancellationToken, task::TaskTracker};
+use tracing::{Instrument as _, field::Empty};
+use tracing_opentelemetry::OpenTelemetrySpanExt as _;
+use uuid::Uuid;
+
+#[cfg(test)]
+mod test;
+
+/// The shortest gap between two pushes about one inbox.
+const NOTIFY_WINDOW: Duration = Duration::from_secs(3);
+
+/// Sends one push about an inbox.
+pub trait FocusDelivery: Send + Sync + 'static {
+ /// Tell everyone reading `link_id` to read Focus again. Failures are logged.
+ fn deliver(
+ &self,
+ owner: &MacroUserIdStr<'static>,
+ link_id: Uuid,
+ ) -> impl Future<Output = ()> + Send;
+}
+
+/// Classifications stored since an inbox's last push.
+#[derive(Default)]
+struct OpenWindow {
+ /// The trace of the newest one, which the next push joins.
+ newest: Option<opentelemetry::Context>,
+ count: i64,
+}
+
+type OpenWindows = Arc<Mutex<HashMap<Uuid, OpenWindow>>>;
+
+fn locked(windows: &OpenWindows) -> MutexGuard<'_, HashMap<Uuid, OpenWindow>> {
+ windows.lock().unwrap_or_else(PoisonError::into_inner)
+}
+
+/// The [`FocusNotifier`] the worker runs, with one window per inbox.
+pub struct CoalescingNotifier<D> {
+ delivery: Arc<D>,
+ windows: OpenWindows,
+ tracker: TaskTracker,
+ shutdown: CancellationToken,
+}
+
+impl<D: FocusDelivery> CoalescingNotifier<D> {
+ /// Push through `delivery`, with each inbox's pushes on `tracker`, until `shutdown`.
+ pub fn new(delivery: D, tracker: TaskTracker, shutdown: CancellationToken) -> Self {
+ Self {
+ delivery: Arc::new(delivery),
+ windows: Arc::default(),
+ tracker,
+ shutdown,
+ }
+ }
+}
+
+impl<D: FocusDelivery> FocusNotifier for CoalescingNotifier<D> {
+ fn focus_changed(&self, owner: &MacroUserIdStr<'static>, link_id: Uuid) {
+ let classification = tracing::Span::current().context();
+ {
+ let mut windows = locked(&self.windows);
+ if let Some(window) = windows.get_mut(&link_id) {
+ window.newest = Some(classification);
+ window.count += 1;
+ return;
+ }
+ windows.insert(link_id, OpenWindow::default());
+ }
+ self.tracker.spawn(run_window(
+ Arc::clone(&self.delivery),
+ Arc::clone(&self.windows),
+ self.shutdown.clone(),
+ owner.clone(),
+ link_id,
+ classification,
+ ));
+ }
+}
+
+/// Push for one inbox until a window passes with nothing new to say.
+async fn run_window<D: FocusDelivery>(
+ delivery: Arc<D>,
+ windows: OpenWindows,
+ shutdown: CancellationToken,
+ owner: MacroUserIdStr<'static>,
+ link_id: Uuid,
+ classification: opentelemetry::Context,
+) {
+ let mut parent = classification;
+ let mut coalesced = 1;
+ loop {
+ let span = tracing::info_span!(
+ "email_focus.notify",
+ %link_id,
+ coalesced,
+ delivered = Empty,
+ otel.status_code = Empty,
+ otel.status_description = Empty,
+ );
+ // Starts a trace when there's nothing to join, not under this task's.
+ let _ = span.set_parent(parent.clone());
+ let push = delivery.deliver(&owner, link_id).instrument(span);
+ if shutdown.run_until_cancelled(push).await.is_none() {
+ return;
+ }
+
+ let window = tokio::time::sleep(NOTIFY_WINDOW);
+ if shutdown.run_until_cancelled(window).await.is_none() {
+ return;
+ }
+ let mut open = locked(&windows);
+ let Some(window) = open.get_mut(&link_id) else {
+ return;
+ };
+ if let Some(newest) = window.newest.take() {
+ parent = newest;
+ coalesced = std::mem::take(&mut window.count);
+ continue;
+ }
+ // Nothing new since the last push. If that one was for a new
+ // classification, repeat it for the replica; if it was the repeat, stop.
+ if coalesced == 0 {
+ open.remove(&link_id);
+ return;
+ }
+ coalesced = 0;
+ }
+}
+
+/// Pushes `refresh_email` to an inbox's owner and the people it's delegated to.
+pub struct GatewayDelivery {
+ gateway: ConnectionGatewayClient,
+ db: PgPool,
+}
+
+impl GatewayDelivery {
+ /// Push through `gateway`, reading delegates from `db`.
+ pub fn new(gateway: ConnectionGatewayClient, db: PgPool) -> Self {
+ Self { gateway, db }
+ }
+}
+
+/// The current span's W3C context, for the client's read to join: the gateway
+/// keeps a message's own trace context off the websocket.
+fn traceparent() -> Option<String> {
+ let mut carrier = HashMap::new();
+ opentelemetry::global::get_text_map_propagator(|propagator| {
+ propagator.inject_context(&tracing::Span::current().context(), &mut carrier);
+ });
+ carrier.remove("traceparent")
+}
+
+impl FocusDelivery for GatewayDelivery {
+ async fn deliver(&self, owner: &MacroUserIdStr<'static>, link_id: Uuid) {
+ let span = tracing::Span::current();
+ let mut readers = vec![owner.as_ref().to_owned()];
+ match sqlx::query_scalar!(
+ "SELECT primary_macro_id FROM macro_user_links WHERE link_id = $1",
+ link_id,
+ )
+ .fetch_all(&self.db)
+ .await
+ {
+ Ok(delegates) => readers.extend(delegates),
+ Err(error) => {
+ tracing::warn!(error = ?error, %link_id, "focus notification delegates unavailable");
+ }
+ }
+ readers.sort_unstable();
+ readers.dedup();
+
+ let event = RefreshEmailEvent::FocusUpdated {
+ link_id,
+ traceparent: traceparent(),
+ };
+ let payload = match serde_json::to_value(&event) {
+ Ok(payload) => payload,
+ Err(error) => {
+ tracing::error!(error = ?error, %link_id, "focus notification unserializable");
+ kafka_util::record_span_error(&span, "focus notification unserializable");
+ return;
+ }
+ };
+ match self
+ .gateway
+ .refresh_email_for_users(&readers, payload)
+ .await
+ {
+ Ok(receipts) => {
+ let delivered = receipts
+ .iter()
+ .map(|receipt| receipt.delivery_count)
+ .sum::<u64>();
+ span.record("delivered", i64::try_from(delivered).unwrap_or(i64::MAX));
+ }
+ Err(error) => {
+ tracing::warn!(error = ?error, %link_id, "focus notification failed");
+ kafka_util::record_span_error(&span, "focus notification failed");
+ }
+ }
+ }
+}
diff --git a/services/email_focus_worker/src/outbound/gateway_notifier/test.rs b/services/email_focus_worker/src/outbound/gateway_notifier/test.rs
new file mode 100644
index 0000000000..0fb24e7fbc
--- /dev/null
+++ b/services/email_focus_worker/src/outbound/gateway_notifier/test.rs
@@ -0,0 +1,171 @@
+use std::collections::BTreeSet;
+
+use macro_db_migrator::MACRO_DB_MIGRATIONS;
+use sqlx::{Pool, Postgres};
+use tokio::{sync::mpsc, time::Instant};
+use wiremock::{Mock, MockServer, ResponseTemplate, matchers};
+
+use super::*;
+
+/// Records each push and when it went out.
+struct FakeDelivery {
+ pushes: mpsc::UnboundedSender<(Uuid, Instant)>,
+}
+
+impl FocusDelivery for FakeDelivery {
+ async fn deliver(&self, _owner: &MacroUserIdStr<'static>, link_id: Uuid) {
+ self.pushes.send((link_id, Instant::now())).unwrap();
+ }
+}
+
+fn owner() -> MacroUserIdStr<'static> {
+ MacroUserIdStr::try_from_email("[email protected]").unwrap()
+}
+
+fn notifier() -> (
+ CoalescingNotifier<FakeDelivery>,
+ mpsc::UnboundedReceiver<(Uuid, Instant)>,
+ TaskTracker,
+ CancellationToken,
+) {
+ let (pushes, pushed) = mpsc::unbounded_channel();
+ let tracker = TaskTracker::new();
+ let shutdown = CancellationToken::new();
+ let notifier =
+ CoalescingNotifier::new(FakeDelivery { pushes }, tracker.clone(), shutdown.clone());
+ (notifier, pushed, tracker, shutdown)
+}
+
+#[tokio::test(start_paused = true)]
+async fn one_classification_pushes_at_once_and_repeats_for_the_replica() {
+ let (notifier, mut pushed, tracker, _shutdown) = notifier();
+ let link_id = Uuid::now_v7();
+ let start = Instant::now();
+
+ notifier.focus_changed(&owner(), link_id);
+
+ assert_eq!(pushed.recv().await.unwrap(), (link_id, start));
+ assert_eq!(
+ pushed.recv().await.unwrap(),
+ (link_id, start + NOTIFY_WINDOW)
+ );
+
+ // The window closes one quiet window later and there is nothing more to say.
+ tracker.close();
+ tracker.wait().await;
+ assert_eq!(start.elapsed(), NOTIFY_WINDOW * 2);
+ assert!(pushed.try_recv().is_err());
+}
+
+#[tokio::test(start_paused = true)]
+async fn a_burst_in_one_inbox_collapses_into_few_pushes() {
+ let (notifier, mut pushed, tracker, _shutdown) = notifier();
+ let link_id = Uuid::now_v7();
+ let start = Instant::now();
+
+ for _ in 0..20 {
+ notifier.focus_changed(&owner(), link_id);
+ }
+
+ // At once, then the rest together, then the repeat for the replica.
+ assert_eq!(pushed.recv().await.unwrap().1, start);
+ assert_eq!(pushed.recv().await.unwrap().1, start + NOTIFY_WINDOW);
+ assert_eq!(pushed.recv().await.unwrap().1, start + NOTIFY_WINDOW * 2);
+ tracker.close();
+ tracker.wait().await;
+ assert!(pushed.try_recv().is_err());
+}
+
+#[tokio::test(start_paused = true)]
+async fn a_classification_after_the_window_closes_pushes_at_once_again() {
+ let (notifier, mut pushed, _tracker, _shutdown) = notifier();
+ let link_id = Uuid::now_v7();
+
+ notifier.focus_changed(&owner(), link_id);
+ pushed.recv().await.unwrap();
+ pushed.recv().await.unwrap();
+ tokio::time::sleep(NOTIFY_WINDOW * 2).await;
+
+ let later = Instant::now();
+ notifier.focus_changed(&owner(), link_id);
+ assert_eq!(pushed.recv().await.unwrap().1, later);
+}
+
+#[tokio::test(start_paused = true)]
+async fn inboxes_do_not_share_a_window() {
+ let (notifier, mut pushed, _tracker, _shutdown) = notifier();
+ let (first, second) = (Uuid::now_v7(), Uuid::now_v7());
+ let start = Instant::now();
+
+ notifier.focus_changed(&owner(), first);
+ notifier.focus_changed(&owner(), second);
+
+ let pushes = [pushed.recv().await.unwrap(), pushed.recv().await.unwrap()];
+ assert_eq!(pushes[0].1, start);
+ assert_eq!(pushes[1].1, start);
+ assert_eq!(
+ BTreeSet::from([pushes[0].0, pushes[1].0]),
+ BTreeSet::from([first, second])
+ );
+}
+
+#[tokio::test(start_paused = true)]
+async fn shutdown_ends_an_open_window_without_waiting_for_it() {
+ let (notifier, mut pushed, tracker, shutdown) = notifier();
+ let start = Instant::now();
+
+ notifier.focus_changed(&owner(), Uuid::now_v7());
+ pushed.recv().await.unwrap();
+
+ shutdown.cancel();
+ tracker.close();
+ tracker.wait().await;
+ assert_eq!(start.elapsed(), Duration::ZERO);
+}
+
+/// Link f02 in the Focus fixtures: held by `[email protected]`, delegated
+const DELEGATED_LINK: Uuid = uuid::uuid!("00000000-0000-0000-0000-000000000f02");
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(
+ path = "../../../../../crates/email/fixtures",
+ scripts("email_focus", "email_focus_linked")
+ )
+)]
+async fn a_push_reaches_the_owner_and_delegates_as_one_refresh_email(pool: Pool<Postgres>) {
+ let gateway = MockServer::start().await;
+ Mock::given(matchers::method("POST"))
+ .and(matchers::path("/message/batch_send"))
+ .and(matchers::header("x-internal-auth-key", "test-key"))
+ .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
+ "receipts": [{"user_id": "macro|[email protected]", "delivery_count": 2, "active": true}],
+ })))
+ .expect(1)
+ .mount(&gateway)
+ .await;
+ let delivery = GatewayDelivery::new(
+ ConnectionGatewayClient::new("test-key".to_owned(), gateway.uri()),
+ pool,
+ );
+
+ let holder = MacroUserIdStr::try_from_email("[email protected]").unwrap();
+ delivery.deliver(&holder, DELEGATED_LINK).await;
+
+ let requests = gateway.received_requests().await.unwrap();
+ let body: serde_json::Value = serde_json::from_slice(&requests[0].body).unwrap();
+ assert_eq!(body["message_type"], "refresh_email");
+ assert_eq!(body["message"]["event"], "focus_updated");
+ assert_eq!(body["message"]["link_id"], DELEGATED_LINK.to_string());
+ let readers = body["entities"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .map(|entity| entity["entity_id"].as_str().unwrap())
+ .collect::<Vec<_>>();
+ assert_eq!(
+ readers,
+ );
+}