Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(email): trace Outlook sync, Graph calls and import progress (#9480)
teo352bd7169a84
diff --git a/.agents/skills/instrumentation/SKILL.md b/.agents/skills/instrumentation/SKILL.mdindex bfaa961ff6..2b2f8fd48f 100644--- a/.agents/skills/instrumentation/SKILL.md+++ b/.agents/skills/instrumentation/SKILL.md@@ -207,6 +207,15 @@ telemetry with its own volume budget.- Agent/ACP messages: `genai_telemetry::propagation::inject` / `set_parent`carry context in `_meta`; GenAI spans follow CS-55.- Spawned tasks take the span with them (`.instrument(span)`) or link to it.+- Datadog files a client span that carries an HTTP method as operation+ `http.client.request` with resource `<METHOD> <url.template>`, whatever the+ span is called; set `operation.name` on it to keep it findable by name+ (`outlook.graph_request` does). Internal spans keep their name as+ `resource_name`.+- Record numeric span fields as `i64`: `tracing-opentelemetry` has no+ unsigned type and exports `u16`/`u32`/`u64`/`usize` values as strings.+- Background loops start one root span per claimed unit of work, not per poll,+ so idle polling exports nothing.## Worked examplesdiff --git a/.sqlx/query-094d0bb92a52fd11b65ff460f08314c2f56f8fbfa34b54b879610cd7d95b0f4b.json b/.sqlx/query-094d0bb92a52fd11b65ff460f08314c2f56f8fbfa34b54b879610cd7d95b0f4b.jsonnew file mode 100644index 0000000000..51278d417f--- /dev/null+++ b/.sqlx/query-094d0bb92a52fd11b65ff460f08314c2f56f8fbfa34b54b879610cd7d95b0f4b.json@@ -0,0 +1,86 @@+{+ "db_name": "PostgreSQL",+ "query": "\n SELECT l.id AS \"link_id!\",\n l.sync_generation AS \"sync_generation!\",\n l.is_sync_active AS \"sync_active!\",\n l.needs_reauth AS \"needs_reauth!\",\n q.queued AS \"queued!\",\n q.parked AS \"parked!\",\n q.leased AS \"leased!\",\n COALESCE(floor(EXTRACT(EPOCH FROM now() - q.oldest_due)), 0)::bigint AS \"oldest_waiting_seconds!\",\n COALESCE(ceil(EXTRACT(EPOCH FROM q.parked_until - now())), 0)::bigint AS \"parked_until_max_seconds!\",\n s.incomplete AS \"streams_incomplete!\",\n COALESCE(recent.imported_5m, 0)::bigint AS \"imported_5m!\",\n floor(EXTRACT(EPOCH FROM now() - latest.last_imported_at))::bigint AS seconds_since_last_import\n FROM email_links l\n CROSS JOIN LATERAL (\n SELECT count(*) AS queued,\n count(*) FILTER (WHERE available_at > now()) AS parked,\n count(*) FILTER (WHERE lease_until > now()) AS leased,\n min(available_at) FILTER (\n WHERE available_at <= now() AND (lease_until IS NULL OR lease_until <= now())\n ) AS oldest_due,\n max(available_at) FILTER (WHERE available_at > now()) AS parked_until\n FROM email_message_reconciliation\n WHERE link_id = l.id AND generation = l.sync_generation\n ) q\n CROSS JOIN LATERAL (\n SELECT count(*) AS incomplete\n FROM email_sync_streams\n WHERE link_id = l.id AND generation = l.sync_generation\n AND kind IN ('folder_catalog', 'mail_folder') AND NOT initial_complete\n ) s\n CROSS JOIN LATERAL (\n SELECT sum(imported) AS imported_5m\n FROM email_mailbox_import_minutes\n WHERE link_id = l.id AND minute > now() - interval '5 minutes'\n ) recent\n -- A flush carries imports from at most the minute before it, so\n -- the latest import is in one of the two newest minutes.\n CROSS JOIN LATERAL (\n SELECT max(last_imported_at) AS last_imported_at\n FROM (\n SELECT last_imported_at FROM email_mailbox_import_minutes\n WHERE link_id = l.id ORDER BY minute DESC LIMIT 2\n ) newest\n ) latest\n WHERE l.provider = 'OUTLOOK' AND (l.is_sync_active OR q.queued > 0)\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "link_id!",+ "type_info": "Uuid"+ },+ {+ "ordinal": 1,+ "name": "sync_generation!",+ "type_info": "Int8"+ },+ {+ "ordinal": 2,+ "name": "sync_active!",+ "type_info": "Bool"+ },+ {+ "ordinal": 3,+ "name": "needs_reauth!",+ "type_info": "Bool"+ },+ {+ "ordinal": 4,+ "name": "queued!",+ "type_info": "Int8"+ },+ {+ "ordinal": 5,+ "name": "parked!",+ "type_info": "Int8"+ },+ {+ "ordinal": 6,+ "name": "leased!",+ "type_info": "Int8"+ },+ {+ "ordinal": 7,+ "name": "oldest_waiting_seconds!",+ "type_info": "Int8"+ },+ {+ "ordinal": 8,+ "name": "parked_until_max_seconds!",+ "type_info": "Int8"+ },+ {+ "ordinal": 9,+ "name": "streams_incomplete!",+ "type_info": "Int8"+ },+ {+ "ordinal": 10,+ "name": "imported_5m!",+ "type_info": "Int8"+ },+ {+ "ordinal": 11,+ "name": "seconds_since_last_import",+ "type_info": "Int8"+ }+ ],+ "parameters": {+ "Left": []+ },+ "nullable": [+ false,+ false,+ false,+ false,+ null,+ null,+ null,+ null,+ null,+ null,+ null,+ null+ ]+ },+ "hash": "094d0bb92a52fd11b65ff460f08314c2f56f8fbfa34b54b879610cd7d95b0f4b"+}diff --git a/.sqlx/query-0b7d6e01be005b7456de49f2917a5449a93525fc3ebd0adf6bfe5e3be5b2526d.json b/.sqlx/query-0b7d6e01be005b7456de49f2917a5449a93525fc3ebd0adf6bfe5e3be5b2526d.jsonnew file mode 100644index 0000000000..2cf59f1478--- /dev/null+++ b/.sqlx/query-0b7d6e01be005b7456de49f2917a5449a93525fc3ebd0adf6bfe5e3be5b2526d.json@@ -0,0 +1,15 @@+{+ "db_name": "PostgreSQL",+ "query": "\n INSERT INTO email_projection_outbox(id,link_id,generation,payload)\n SELECT gen_random_uuid(),m.link_id,$2,jsonb_build_object('kind','attachment_recheck','message_id',m.id)\n FROM email_messages m WHERE m.link_id=$1 AND m.provider_id IS NOT NULL AND NOT m.is_draft\n AND EXISTS(SELECT 1 FROM email_attachments a WHERE a.message_id=m.id AND a.reference_url IS NULL)\n ",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "Uuid",+ "Int8"+ ]+ },+ "nullable": []+ },+ "hash": "0b7d6e01be005b7456de49f2917a5449a93525fc3ebd0adf6bfe5e3be5b2526d"+}diff --git a/.sqlx/query-13e4afddda3bdbd69f444a40db87de3fca8a4e6858203bf44e488c936abf4d6a.json b/.sqlx/query-13e4afddda3bdbd69f444a40db87de3fca8a4e6858203bf44e488c936abf4d6a.jsondeleted file mode 100644index cfa335ee40..0000000000--- a/.sqlx/query-13e4afddda3bdbd69f444a40db87de3fca8a4e6858203bf44e488c936abf4d6a.json+++ /dev/null@@ -1,23 +0,0 @@-{- "db_name": "PostgreSQL",- "query": "SELECT (\n EXISTS(SELECT 1 FROM email_sync_streams WHERE link_id=$1 AND generation=$2 AND kind='folder_catalog' AND initial_complete)\n AND EXISTS(SELECT 1 FROM email_sync_streams WHERE link_id=$1 AND generation=$2 AND kind='mail_folder' AND NOT attachments_rechecked)\n AND NOT EXISTS(SELECT 1 FROM email_sync_streams WHERE link_id=$1 AND generation=$2 AND kind IN ('mail_folder','folder_catalog') AND NOT initial_complete)\n AND NOT EXISTS(SELECT 1 FROM email_message_reconciliation WHERE link_id=$1 AND generation=$2)\n ) AS \"ready!\"",- "describe": {- "columns": [- {- "ordinal": 0,- "name": "ready!",- "type_info": "Bool"- }- ],- "parameters": {- "Left": [- "Uuid",- "Int8"- ]- },- "nullable": [- null- ]- },- "hash": "13e4afddda3bdbd69f444a40db87de3fca8a4e6858203bf44e488c936abf4d6a"-}diff --git a/.sqlx/query-1fa5550ddc5382947e0c08629e95d27b72e7c0de6173fdcbb7f25a2dffdf180c.json b/.sqlx/query-1fa5550ddc5382947e0c08629e95d27b72e7c0de6173fdcbb7f25a2dffdf180c.jsondeleted file mode 100644index 53f37d6e11..0000000000--- a/.sqlx/query-1fa5550ddc5382947e0c08629e95d27b72e7c0de6173fdcbb7f25a2dffdf180c.json+++ /dev/null@@ -1,15 +0,0 @@-{- "db_name": "PostgreSQL",- "query": "UPDATE email_projection_outbox SET lease_id = NULL,lease_until = NULL,\n available_at = now() + make_interval(secs => LEAST(600,30 * attempts))\n WHERE id = $1 AND lease_id = $2",- "describe": {- "columns": [],- "parameters": {- "Left": [- "Uuid",- "Uuid"- ]- },- "nullable": []- },- "hash": "1fa5550ddc5382947e0c08629e95d27b72e7c0de6173fdcbb7f25a2dffdf180c"-}diff --git a/.sqlx/query-4f501e81e5d26c98eed8aaec428c90f364eb75ecedc23dae6e55f5283714e78f.json b/.sqlx/query-3c12e585b3ce343cf3a56fc7706c14d3e4348cc45742093eb66bb5b11f50c45f.jsonsimilarity index 81%rename from .sqlx/query-4f501e81e5d26c98eed8aaec428c90f364eb75ecedc23dae6e55f5283714e78f.jsonrename to .sqlx/query-3c12e585b3ce343cf3a56fc7706c14d3e4348cc45742093eb66bb5b11f50c45f.jsonindex cea675c703..d9469ff428 100644--- a/.sqlx/query-4f501e81e5d26c98eed8aaec428c90f364eb75ecedc23dae6e55f5283714e78f.json+++ b/.sqlx/query-3c12e585b3ce343cf3a56fc7706c14d3e4348cc45742093eb66bb5b11f50c45f.json@@ -1,6 +1,6 @@{"db_name": "PostgreSQL",- "query": "\n WITH candidate AS (\n SELECT o.id FROM email_projection_outbox o JOIN email_links l ON l.id = o.link_id\n WHERE l.is_sync_active AND o.available_at <= now() AND (o.lease_until IS NULL OR o.lease_until < now())\n ORDER BY o.available_at,o.id LIMIT 1 FOR UPDATE OF o SKIP LOCKED\n )\n UPDATE email_projection_outbox o SET lease_id = $1,lease_until = now() + interval '3 minutes',attempts = attempts + 1\n FROM candidate c,email_links l WHERE o.id = c.id AND l.id = o.link_id\n RETURNING o.id,o.link_id,o.generation,o.payload,l.sync_generation,l.grant_generation\n ",+ "query": "\n WITH candidate AS (\n SELECT o.id FROM email_projection_outbox o JOIN email_links l ON l.id = o.link_id\n WHERE l.is_sync_active AND o.available_at <= now() AND (o.lease_until IS NULL OR o.lease_until < now())\n ORDER BY o.available_at,o.id LIMIT 1 FOR UPDATE OF o SKIP LOCKED\n )\n UPDATE email_projection_outbox o SET lease_id = $1,lease_until = now() + interval '3 minutes',attempts = attempts + 1\n FROM candidate c,email_links l WHERE o.id = c.id AND l.id = o.link_id\n RETURNING o.id,o.link_id,o.generation,o.payload,o.attempts,l.sync_generation,l.grant_generation\n ","describe": {"columns": [{@@ -25,11 +25,16 @@},{"ordinal": 4,+ "name": "attempts",+ "type_info": "Int4"+ },+ {+ "ordinal": 5,"name": "sync_generation","type_info": "Int8"},{- "ordinal": 5,+ "ordinal": 6,"name": "grant_generation","type_info": "Int8"}@@ -45,8 +50,9 @@false,false,false,+ false,false]},- "hash": "4f501e81e5d26c98eed8aaec428c90f364eb75ecedc23dae6e55f5283714e78f"+ "hash": "3c12e585b3ce343cf3a56fc7706c14d3e4348cc45742093eb66bb5b11f50c45f"}diff --git a/.sqlx/query-3c7c452162235d1ddd691ea33df2aed5b0b12b2bb725e9f47cd5f2dda93ea0d2.json b/.sqlx/query-3c7c452162235d1ddd691ea33df2aed5b0b12b2bb725e9f47cd5f2dda93ea0d2.jsonnew file mode 100644index 0000000000..f7ec7f647d--- /dev/null+++ b/.sqlx/query-3c7c452162235d1ddd691ea33df2aed5b0b12b2bb725e9f47cd5f2dda93ea0d2.json@@ -0,0 +1,29 @@+{+ "db_name": "PostgreSQL",+ "query": "SELECT\n (SELECT count(*) FROM email_messages WHERE link_id=$1 AND provider_id IS NOT NULL) AS \"messages!\",\n (SELECT count(*) FROM email_sync_streams WHERE link_id=$1 AND generation=$2 AND kind='mail_folder') AS \"folders!\"\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "messages!",+ "type_info": "Int8"+ },+ {+ "ordinal": 1,+ "name": "folders!",+ "type_info": "Int8"+ }+ ],+ "parameters": {+ "Left": [+ "Uuid",+ "Int8"+ ]+ },+ "nullable": [+ null,+ null+ ]+ },+ "hash": "3c7c452162235d1ddd691ea33df2aed5b0b12b2bb725e9f47cd5f2dda93ea0d2"+}diff --git a/.sqlx/query-5312773217eb0dadeae9c4a02f07631cde543a6a3685bb6c8e7451c2d7edfc6c.json b/.sqlx/query-5312773217eb0dadeae9c4a02f07631cde543a6a3685bb6c8e7451c2d7edfc6c.jsonnew file mode 100644index 0000000000..23492e8aad--- /dev/null+++ b/.sqlx/query-5312773217eb0dadeae9c4a02f07631cde543a6a3685bb6c8e7451c2d7edfc6c.json@@ -0,0 +1,16 @@+{+ "db_name": "PostgreSQL",+ "query": "INSERT INTO email_sync_streams(id,link_id,generation,kind,scope_id,position,initial_complete,scan_id,import_started_at,import_completed_at)\n SELECT gen_random_uuid(),link_id,$3,kind,scope_id,position,initial_complete,scan_id,import_started_at,import_completed_at FROM email_sync_streams WHERE link_id = $1 AND generation = $2\n ON CONFLICT DO NOTHING",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "Uuid",+ "Int8",+ "Int8"+ ]+ },+ "nullable": []+ },+ "hash": "5312773217eb0dadeae9c4a02f07631cde543a6a3685bb6c8e7451c2d7edfc6c"+}diff --git a/.sqlx/query-8886c3d33e3babda4a5f53deb9c377eed4e67151789a60405ac76a89af9bcdc9.json b/.sqlx/query-8886c3d33e3babda4a5f53deb9c377eed4e67151789a60405ac76a89af9bcdc9.jsonnew file mode 100644index 0000000000..529c4a4647--- /dev/null+++ b/.sqlx/query-8886c3d33e3babda4a5f53deb9c377eed4e67151789a60405ac76a89af9bcdc9.json@@ -0,0 +1,29 @@+{+ "db_name": "PostgreSQL",+ "query": "\n UPDATE email_sync_streams SET import_completed_at = now()\n WHERE link_id=$1 AND generation=$2 AND kind='folder_catalog' AND import_completed_at IS NULL\n RETURNING import_started_at, import_completed_at AS \"completed_at!\"\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "import_started_at",+ "type_info": "Timestamptz"+ },+ {+ "ordinal": 1,+ "name": "completed_at!",+ "type_info": "Timestamptz"+ }+ ],+ "parameters": {+ "Left": [+ "Uuid",+ "Int8"+ ]+ },+ "nullable": [+ true,+ true+ ]+ },+ "hash": "8886c3d33e3babda4a5f53deb9c377eed4e67151789a60405ac76a89af9bcdc9"+}diff --git a/.sqlx/query-88e47b1fc3e5e19bfd0f4083e078d02ba8d5b4a2eafe1cf6c1d8fbf1b4d841a3.json b/.sqlx/query-88e47b1fc3e5e19bfd0f4083e078d02ba8d5b4a2eafe1cf6c1d8fbf1b4d841a3.jsonnew file mode 100644index 0000000000..a6ce05605b--- /dev/null+++ b/.sqlx/query-88e47b1fc3e5e19bfd0f4083e078d02ba8d5b4a2eafe1cf6c1d8fbf1b4d841a3.json@@ -0,0 +1,16 @@+{+ "db_name": "PostgreSQL",+ "query": "\n INSERT INTO email_mailbox_import_minutes (link_id, minute, imported, last_imported_at)\n SELECT link_id, date_trunc('minute', now()), imported, last_imported_at\n FROM unnest($1::uuid[], $2::int[], $3::timestamptz[]) AS flushed(link_id, imported, last_imported_at)\n ON CONFLICT (link_id, minute) DO UPDATE SET\n imported = email_mailbox_import_minutes.imported + EXCLUDED.imported,\n last_imported_at = GREATEST(email_mailbox_import_minutes.last_imported_at, EXCLUDED.last_imported_at)\n ",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "UuidArray",+ "Int4Array",+ "TimestamptzArray"+ ]+ },+ "nullable": []+ },+ "hash": "88e47b1fc3e5e19bfd0f4083e078d02ba8d5b4a2eafe1cf6c1d8fbf1b4d841a3"+}diff --git a/.sqlx/query-8bddb3c4f6f73fee00f642d74491428e6f6bd3e3ce41066048b81cb682465be5.json b/.sqlx/query-8bddb3c4f6f73fee00f642d74491428e6f6bd3e3ce41066048b81cb682465be5.jsonnew file mode 100644index 0000000000..4cee539bd5--- /dev/null+++ b/.sqlx/query-8bddb3c4f6f73fee00f642d74491428e6f6bd3e3ce41066048b81cb682465be5.json@@ -0,0 +1,12 @@+{+ "db_name": "PostgreSQL",+ "query": "DELETE FROM email_mailbox_import_minutes WHERE minute < now() - interval '1 day'",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": []+ },+ "nullable": []+ },+ "hash": "8bddb3c4f6f73fee00f642d74491428e6f6bd3e3ce41066048b81cb682465be5"+}diff --git a/.sqlx/query-91d7badfc9cd004327cba280d60690632c8f3c4c802f2b8b385c72413b55e210.json b/.sqlx/query-91d7badfc9cd004327cba280d60690632c8f3c4c802f2b8b385c72413b55e210.jsondeleted file mode 100644index c970f897f4..0000000000--- a/.sqlx/query-91d7badfc9cd004327cba280d60690632c8f3c4c802f2b8b385c72413b55e210.json+++ /dev/null@@ -1,15 +0,0 @@-{- "db_name": "PostgreSQL",- "query": "\n INSERT INTO email_projection_outbox(id,link_id,generation,payload)\n SELECT gen_random_uuid(),m.link_id,$2,jsonb_build_object('kind','attachment_recheck','message_id',m.id)\n FROM email_messages m WHERE m.link_id=$1 AND m.provider_id IS NOT NULL AND NOT m.is_draft\n AND EXISTS(SELECT 1 FROM email_attachments a WHERE a.message_id=m.id AND a.reference_url IS NULL)\n ",- "describe": {- "columns": [],- "parameters": {- "Left": [- "Uuid",- "Int8"- ]- },- "nullable": []- },- "hash": "91d7badfc9cd004327cba280d60690632c8f3c4c802f2b8b385c72413b55e210"-}diff --git a/.sqlx/query-a3945ff5ad3ec5248b966f794a2434339c0d4725cfe0b2f5fcb7fae5ef66d559.json b/.sqlx/query-a3945ff5ad3ec5248b966f794a2434339c0d4725cfe0b2f5fcb7fae5ef66d559.jsondeleted file mode 100644index 167ada83b3..0000000000--- a/.sqlx/query-a3945ff5ad3ec5248b966f794a2434339c0d4725cfe0b2f5fcb7fae5ef66d559.json+++ /dev/null@@ -1,16 +0,0 @@-{- "db_name": "PostgreSQL",- "query": "INSERT INTO email_sync_streams(id,link_id,generation,kind,scope_id,position,initial_complete,scan_id)\n SELECT gen_random_uuid(),link_id,$3,kind,scope_id,position,initial_complete,scan_id FROM email_sync_streams WHERE link_id = $1 AND generation = $2\n ON CONFLICT DO NOTHING",- "describe": {- "columns": [],- "parameters": {- "Left": [- "Uuid",- "Int8",- "Int8"- ]- },- "nullable": []- },- "hash": "a3945ff5ad3ec5248b966f794a2434339c0d4725cfe0b2f5fcb7fae5ef66d559"-}diff --git a/.sqlx/query-2983e995fc823e3cc3c6fa643728283e373c5efcf8127f05d7a1257e3bf08aac.json b/.sqlx/query-b4e8e156590c9dfd3d3ed2ce0f9648e124b75544b3842ab7f98a932e4ceca191.jsonsimilarity index 51%rename from .sqlx/query-2983e995fc823e3cc3c6fa643728283e373c5efcf8127f05d7a1257e3bf08aac.jsonrename to .sqlx/query-b4e8e156590c9dfd3d3ed2ce0f9648e124b75544b3842ab7f98a932e4ceca191.jsonindex 5d469bd7fe..43384ff81f 100644--- a/.sqlx/query-2983e995fc823e3cc3c6fa643728283e373c5efcf8127f05d7a1257e3bf08aac.json+++ b/.sqlx/query-b4e8e156590c9dfd3d3ed2ce0f9648e124b75544b3842ab7f98a932e4ceca191.json@@ -1,6 +1,6 @@{"db_name": "PostgreSQL",- "query": "UPDATE email_sync_streams SET position = NULL,initial_complete = false, attachments_rechecked = false,scan_id = gen_random_uuid(),next_run_at = now(),fence = fence + 1,lease_id = NULL,lease_until = NULL\n WHERE link_id = $1 AND generation = $2 AND kind IN ('folder_catalog','mail_folder','contacts_catalog','contacts_profile','contacts')",+ "query": "UPDATE email_sync_streams SET position = NULL,initial_complete = false, attachments_rechecked = false,scan_id = gen_random_uuid(),next_run_at = now(),fence = fence + 1,lease_id = NULL,lease_until = NULL,\n import_started_at = now(),import_completed_at = NULL\n WHERE link_id = $1 AND generation = $2 AND kind IN ('folder_catalog','mail_folder','contacts_catalog','contacts_profile','contacts')","describe": {"columns": [],"parameters": {@@ -11,5 +11,5 @@},"nullable": []},- "hash": "2983e995fc823e3cc3c6fa643728283e373c5efcf8127f05d7a1257e3bf08aac"+ "hash": "b4e8e156590c9dfd3d3ed2ce0f9648e124b75544b3842ab7f98a932e4ceca191"}diff --git a/.sqlx/query-d4436ee46b2938f27dc995c4034c0b5b05dc90062a333dd41c303894287191d0.json b/.sqlx/query-d4436ee46b2938f27dc995c4034c0b5b05dc90062a333dd41c303894287191d0.jsonnew file mode 100644index 0000000000..5371a982ae--- /dev/null+++ b/.sqlx/query-d4436ee46b2938f27dc995c4034c0b5b05dc90062a333dd41c303894287191d0.json@@ -0,0 +1,16 @@+{+ "db_name": "PostgreSQL",+ "query": "UPDATE email_projection_outbox SET lease_id = NULL,lease_until = NULL,\n available_at = now() + make_interval(secs => $3)\n WHERE id = $1 AND lease_id = $2",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "Uuid",+ "Uuid",+ "Float8"+ ]+ },+ "nullable": []+ },+ "hash": "d4436ee46b2938f27dc995c4034c0b5b05dc90062a333dd41c303894287191d0"+}diff --git a/.sqlx/query-fcdaf7e42334b418b27dadf22945e16383d1f93a62e43a3aecf70e2077e11a73.json b/.sqlx/query-fcdaf7e42334b418b27dadf22945e16383d1f93a62e43a3aecf70e2077e11a73.jsonnew file mode 100644index 0000000000..d30a222f79--- /dev/null+++ b/.sqlx/query-fcdaf7e42334b418b27dadf22945e16383d1f93a62e43a3aecf70e2077e11a73.json@@ -0,0 +1,53 @@+{+ "db_name": "PostgreSQL",+ "query": "SELECT\n EXISTS(SELECT 1 FROM email_sync_streams WHERE link_id=$1 AND generation=$2 AND kind='folder_catalog' AND initial_complete) AS \"catalog_complete!\",\n EXISTS(SELECT 1 FROM email_sync_streams WHERE link_id=$1 AND generation=$2 AND kind='folder_catalog' AND import_completed_at IS NULL) AS \"import_open!\",\n EXISTS(SELECT 1 FROM email_sync_streams WHERE link_id=$1 AND generation=$2 AND kind='mail_folder' AND NOT attachments_rechecked) AS \"recheck_pending!\",\n EXISTS(SELECT 1 FROM email_sync_streams WHERE link_id=$1 AND generation=$2 AND kind IN ('mail_folder','folder_catalog') AND NOT initial_complete) AS \"streams_incomplete!\",\n EXISTS(SELECT 1 FROM email_message_reconciliation WHERE link_id=$1 AND generation=$2) AS \"queued!\",\n EXISTS(SELECT 1 FROM email_message_reconciliation WHERE link_id=$1 AND generation=$2 AND is_import) AS \"imports_queued!\"\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "catalog_complete!",+ "type_info": "Bool"+ },+ {+ "ordinal": 1,+ "name": "import_open!",+ "type_info": "Bool"+ },+ {+ "ordinal": 2,+ "name": "recheck_pending!",+ "type_info": "Bool"+ },+ {+ "ordinal": 3,+ "name": "streams_incomplete!",+ "type_info": "Bool"+ },+ {+ "ordinal": 4,+ "name": "queued!",+ "type_info": "Bool"+ },+ {+ "ordinal": 5,+ "name": "imports_queued!",+ "type_info": "Bool"+ }+ ],+ "parameters": {+ "Left": [+ "Uuid",+ "Int8"+ ]+ },+ "nullable": [+ null,+ null,+ null,+ null,+ null,+ null+ ]+ },+ "hash": "fcdaf7e42334b418b27dadf22945e16383d1f93a62e43a3aecf70e2077e11a73"+}diff --git a/Cargo.lock b/Cargo.lockindex 2d703aebb5..2dd92dfbef 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -7364,6 +7364,7 @@ dependencies = ["thiserror 2.0.18","tokio","tracing",+ "tracing-subscriber","url","uuid","wiremock",diff --git a/crates/authentication_service_client/src/microsoft_access_token.rs b/crates/authentication_service_client/src/microsoft_access_token.rsindex b6fb817911..24bc7019f6 100644--- a/crates/authentication_service_client/src/microsoft_access_token.rs+++ b/crates/authentication_service_client/src/microsoft_access_token.rs@@ -8,6 +8,10 @@ use crate::{pub struct MicrosoftAccessToken {pub access_token: String,pub scopes: Vec<String>,+ /// Whether authentication served a cached token; `None` from a version+ /// that does not report it.+ #[serde(default)]+ pub cached: Option<bool>,}/// Non-secret, owner-bound result of a completed Microsoft OAuth attempt.@@ -130,6 +134,11 @@ impl AuthServiceClient {.send().await.map_err(|_| unavailable())?;+ // Lands on the caller's token span when it declares the field.+ tracing::Span::current().record(+ "http.response.status_code",+ i64::from(response.status().as_u16()),+ );match response.status() {reqwest::StatusCode::OK => {let token: MicrosoftAccessToken =diff --git a/crates/email/AGENTS.md b/crates/email/AGENTS.mdindex b0ccdd5e15..204e2f0219 100644--- a/crates/email/AGENTS.md+++ b/crates/email/AGENTS.md@@ -111,6 +111,29 @@ tools — hexagonal, heavily feature-gated.delivery on a tracking failure, and count opens only for sent messages.Tokens are stored per message, so a retried send or Outlook draft revisionrebuilds the same URL; keep it that way or the draft fingerprints drift.+- Outlook sync telemetry (`domain/mailbox/telemetry.rs`, `progress.rs`) is a+ contract the Outlook sync monitors in `infra/stacks/monitoring` read: each+ claimed unit of `MailboxSyncService` and `MailboxProjectionService` opens+ one root `outlook.sync.*` span (idle polls open none) and records+ `outlook.outcome`, `outlook.defer_reason` and `outlook.defer_seconds`+ through `record_done`/`record_deferred`/`record_failed`. The mailbox loops+ log their own deferrals with `link_id` (gate refusals at DEBUG), so the+ worker must not log them again. Add a new defer reason in `defer_reason`.+- Map database errors in `mailbox_pg` with `db_error` (or `wrapped_db_error`+ for `email_db_client` calls) so the SQLSTATE reaches the log; keep+ `MailboxError::Persistence` for stored data that cannot be decoded.+ `MailboxError::Effect` (`defer_reason=effect`) is for a projection effect+ that fails outside the database; email-service's `MailboxEffects` maps its+ database steps with `database_failed` and everything else with `failed`.+- "outlook import completed" fires from `settle_import` (`mailbox_pg.rs`)+ under the link lock, once per import run: the folder-catalog stream's+ `import_completed_at` is stamped when every folder is enumerated and no+ import row is left. A resync must reset `import_started_at` and+ `import_completed_at` and log "outlook import started", as+ `outbound/mailbox_pg/lifecycle.rs` does.+- Import counts for the health snapshot stay in memory (`ImportTally`) and are+ flushed once a minute to `email_mailbox_import_minutes`; never write per+ imported message for telemetry.- Outlook organization commands only run in their link's current`sync_generation`. Anything that bumps it must rebind unfinished commands as`pending`, since claims skip `confirming` ones while those still block laterdiff --git a/crates/email/src/domain/mailbox.rs b/crates/email/src/domain/mailbox.rsindex 476521d40a..a649521614 100644--- a/crates/email/src/domain/mailbox.rs+++ b/crates/email/src/domain/mailbox.rs@@ -1,9 +1,14 @@//! Durable mailbox enumeration independent from provider-specific notifications.use email_api_client::domain::models::{- EmailApiError, MailFolder, MailboxChangePage, MailboxMessage, ProviderId, StreamToken,+ EmailApiError, MailFolder, MailboxChangePage, MailboxMessage, ProviderId, StreamPosition,+ StreamToken,};+use progress::{ImportCompleted, ImportTally};use std::future::Future;+use std::time::Instant;+use telemetry::{claim_failed, elapsed_ms, record_deferred, record_done, record_failed, span_int};+use tracing::{Instrument, Span};use uuid::Uuid;pub mod catalog;@@ -15,8 +20,10 @@ pub mod gmail_history;pub mod health;pub mod initialization;pub mod lifecycle;+pub mod progress;pub mod projection;pub mod settings;+pub mod telemetry;#[cfg(test)]mod test;pub mod watches;@@ -29,8 +36,47 @@ pub enum MailboxError {Provider(#[from] EmailApiError),#[error("mailbox persistence is unavailable")]Persistence,+ /// A database call failed; the cause carries no row data.+ #[error("mailbox persistence is unavailable: {0}")]+ Database(DatabaseFailure),+ /// A projection's downstream effect (event publishing, a queue, another+ /// service) failed or is not ready; the effect logs its own cause.+ #[error("a mailbox projection effect is unavailable")]+ Effect,#[error("mailbox generation or worker lease changed")]Stale,+ /// The provider no longer reports the message where it was; a later+ /// attempt either finds it or settles that it is gone.+ #[error("mailbox message location has not settled")]+ LocationUnsettled,+}++/// What failed underneath a database call. The database's own message is+/// left out because it can quote row data.+#[derive(Debug, Clone, PartialEq, Eq)]+pub enum DatabaseFailure {+ /// The database rejected the statement with this SQLSTATE.+ Sqlstate(String),+ /// No pooled connection became free in time.+ PoolTimedOut,+ /// The connection failed or closed.+ Connection,+ /// A returned value could not be decoded.+ Decode,+ /// Any other driver failure.+ Other,+}++impl std::fmt::Display for DatabaseFailure {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ match self {+ Self::Sqlstate(code) => write!(f, "sqlstate {code}"),+ Self::PoolTimedOut => f.write_str("connection pool timed out"),+ Self::Connection => f.write_str("connection failed"),+ Self::Decode => f.write_str("decode failed"),+ Self::Other => f.write_str("driver error"),+ }+ }}/// Both generations are resolved from storage, never selected by the client.@@ -70,6 +116,15 @@ pub struct CatalogLease {pub fence: i64,}+/// What a committed page changed.+#[derive(Debug, Default)]+pub struct CommittedPage {+ /// Reconciliation rows the page queued or requeued.+ pub queued: u64,+ /// Set when this page finished the mailbox's import run.+ pub import_completed: Option<ImportCompleted>,+}+pub enum SyncLease<'a> {Catalog(&'a CatalogLease),Stream(&'a StreamLease),@@ -89,7 +144,7 @@ pub trait MailboxSyncRepository: Send + Sync + 'static {&self,lease: &CatalogLease,folders: &[MailFolder],- ) -> impl Future<Output = Result<(), MailboxError>> + Send;+ ) -> impl Future<Output = Result<Option<ImportCompleted>, MailboxError>> + Send;fn release_catalog(&self,lease: &CatalogLease,@@ -105,7 +160,7 @@ pub trait MailboxSyncRepository: Send + Sync + 'static {&self,lease: &StreamLease,page: &MailboxChangePage,- ) -> impl Future<Output = Result<(), MailboxError>> + Send;+ ) -> impl Future<Output = Result<CommittedPage, MailboxError>> + Send;/// Reset only this expired stream; this operation never removes messages.fn reset_stream(&self,@@ -125,10 +180,11 @@ pub trait MailboxSyncRepository: Send + Sync + 'static {mailbox: MailboxKey,) -> impl Future<Output = Result<Vec<MailFolder>, MailboxError>> + Send;/// Finishes only the leased revision. A newer wakeup must remain pending.+ /// Reports the import run this message finished, once per run.fn complete_message(&self,lease: &MessageLease,- ) -> impl Future<Output = Result<(), MailboxError>> + Send;+ ) -> impl Future<Output = Result<Option<ImportCompleted>, MailboxError>> + Send;fn release_message(&self,lease: &MessageLease,@@ -153,6 +209,8 @@ pub trait MailboxGateway: Send + Sync + 'static {message: &ProviderId,folders: &[MailFolder],) -> impl Future<Output = Result<Option<MailboxMessage>, EmailApiError>> + Send;+ /// Provider requests admitted for the mailbox's sync generation, when known.+ fn admitted_requests(&self, mailbox: MailboxKey) -> impl Future<Output = Option<u64>> + Send;}/// The shared ingestion path writes content, normalized state and a projection@@ -173,6 +231,7 @@ pub struct MailboxSyncService<R, P, I> {repo: R,provider: P,ingest: I,+ imports: ImportTally,}impl<R, P, I> MailboxSyncService<R, P, I> {@@ -181,8 +240,14 @@ impl<R, P, I> MailboxSyncService<R, P, I> {repo,provider,ingest,+ imports: ImportTally::default(),}}++ /// The in-memory import counts the health loop flushes.+ pub fn import_tally(&self) -> ImportTally {+ self.imports.clone()+ }}impl<R: MailboxSyncRepository, P: MailboxGateway, I: MailboxIngest> MailboxSyncService<R, P, I> {@@ -190,37 +255,97 @@ impl<R: MailboxSyncRepository, P: MailboxGateway, I: MailboxIngest> MailboxSyncSlet Some(lease) = self.repo.claim_catalog(macro_uuid::generate_uuid_v7())- .await?+ .await+ .map_err(|error| claim_failed("discover", error))?else {return Ok(false);};+ let span = tracing::info_span!(+ "outlook.sync.discover",+ mailbox.link_id = %lease.mailbox.link_id,+ mailbox.sync_generation = lease.mailbox.sync_generation,+ outlook.stream_kind = "folder_catalog",+ outlook.folders = tracing::field::Empty,+ outlook.outcome = tracing::field::Empty,+ outlook.defer_reason = tracing::field::Empty,+ outlook.defer_seconds = tracing::field::Empty,+ otel.status_description = tracing::field::Empty,+ );+ self.discover(&lease).instrument(span).await+ }++ async fn discover(&self, lease: &CatalogLease) -> Result<bool, MailboxError> {+ let span = Span::current();let outcome = maintain_lease(async { Ok(self.provider.folders(lease.mailbox).await?) },- || self.repo.renew(SyncLease::Catalog(&lease)),+ || self.repo.renew(SyncLease::Catalog(lease)),).await;- match outcome {- Ok(folders) => self.repo.commit_catalog(&lease, &folders).await?,+ let result = match outcome {+ Ok(folders) => {+ span.record("outlook.folders", span_int(folders.len()));+ self.repo.commit_catalog(lease, &folders).await+ }Err(MailboxError::Provider(error)) => {+ let delay = retry_seconds(&error);+ let error = MailboxError::from(error);self.repo- .release_catalog(&lease, retry_seconds(&error))- .await?;- return Err(error.into());+ .release_catalog(lease, delay)+ .await+ .inspect_err(|release| {+ record_failed(&span, lease.mailbox.link_id, release);+ })?;+ record_deferred(&span, lease.mailbox.link_id, &error, delay);+ return Err(error);+ }+ Err(error) => Err(error),+ };+ match result {+ Ok(import_completed) => {+ record_done(&span, "imported");+ if let Some(completed) = import_completed {+ self.report_import_completed(&completed).await;+ }+ Ok(true)+ }+ Err(error) => {+ record_failed(&span, lease.mailbox.link_id, &error);+ Err(error)}- Err(error) => return Err(error),}- Ok(true)}+/// Process one bounded provider page. Notifications only make streams due;/// periodic polling recovers missed, duplicate and out-of-order notifications.pub async fn enumerate_once(&self) -> Result<bool, MailboxError> {let Some(lease) = self.repo.claim_stream(macro_uuid::generate_uuid_v7())- .await?+ .await+ .map_err(|error| claim_failed("enumerate", error))?else {return Ok(false);};+ let span = tracing::info_span!(+ "outlook.sync.enumerate",+ mailbox.link_id = %lease.mailbox.link_id,+ mailbox.sync_generation = lease.mailbox.sync_generation,+ outlook.stream_kind = "mail_folder",+ outlook.page_items = tracing::field::Empty,+ outlook.rows_queued = tracing::field::Empty,+ outlook.delta_advanced = false,+ outlook.folder_initial_complete = lease.initial_complete,+ outlook.cursor_reset = false,+ outlook.outcome = tracing::field::Empty,+ outlook.defer_reason = tracing::field::Empty,+ outlook.defer_seconds = tracing::field::Empty,+ otel.status_description = tracing::field::Empty,+ );+ self.enumerate(&lease).instrument(span).await+ }++ async fn enumerate(&self, lease: &StreamLease) -> Result<bool, MailboxError> {+ let span = Span::current();let outcome = maintain_lease(async {Ok(self@@ -228,61 +353,151 @@ impl<R: MailboxSyncRepository, P: MailboxGateway, I: MailboxIngest> MailboxSyncS.changes(lease.mailbox, &lease.folder, lease.position.as_ref()).await?)},- || self.repo.renew(SyncLease::Stream(&lease)),+ || self.repo.renew(SyncLease::Stream(lease)),).await;- match outcome {- Ok(page) => self.repo.commit_page(&lease, &page).await?,+ let result = match outcome {+ Ok(page) => {+ span.record(+ "outlook.page_items",+ span_int(page.changed.len() + page.removed.len()),+ );+ let checkpoint = matches!(page.position, StreamPosition::Checkpoint(_));+ self.repo.commit_page(lease, &page).await.map(|committed| {+ span.record("outlook.rows_queued", span_int(committed.queued));+ span.record("outlook.delta_advanced", true);+ span.record(+ "outlook.folder_initial_complete",+ lease.initial_complete || checkpoint,+ );+ committed.import_completed+ })+ }Err(MailboxError::Provider(EmailApiError::OutdatedCursor)) => {- self.repo.reset_stream(&lease).await?+ span.record("outlook.cursor_reset", true);+ self.repo.reset_stream(lease).await.map(|()| None)}Err(MailboxError::Provider(error)) => {+ let delay = retry_seconds(&error);+ let error = MailboxError::from(error);self.repo- .release_stream(&lease, retry_seconds(&error))- .await?;- return Err(error.into());+ .release_stream(lease, delay)+ .await+ .inspect_err(|release| {+ record_failed(&span, lease.mailbox.link_id, release);+ })?;+ record_deferred(&span, lease.mailbox.link_id, &error, delay);+ return Err(error);+ }+ Err(error) => Err(error),+ };+ match result {+ Ok(import_completed) => {+ record_done(&span, "imported");+ if let Some(completed) = import_completed {+ self.report_import_completed(&completed).await;+ }+ Ok(true)+ }+ Err(error) => {+ record_failed(&span, lease.mailbox.link_id, &error);+ Err(error)}- Err(error) => return Err(error),}- Ok(true)}pub async fn reconcile_once(&self) -> Result<bool, MailboxError> {let Some(lease) = self.repo.claim_message(macro_uuid::generate_uuid_v7())- .await?+ .await+ .map_err(|error| claim_failed("reconcile", error))?else {return Ok(false);};+ let span = tracing::info_span!(+ "outlook.sync.reconcile",+ mailbox.link_id = %lease.mailbox.link_id,+ mailbox.sync_generation = lease.mailbox.sync_generation,+ outlook.is_import = lease.is_import,+ outlook.attempts = lease.attempts,+ outlook.graph_calls = tracing::field::Empty,+ outlook.attachments = tracing::field::Empty,+ outlook.db_write_ms = tracing::field::Empty,+ outlook.outcome = tracing::field::Empty,+ outlook.defer_reason = tracing::field::Empty,+ outlook.defer_seconds = tracing::field::Empty,+ otel.status_description = tracing::field::Empty,+ );+ self.reconcile(&lease).instrument(span).await+ }++ async fn reconcile(&self, lease: &MessageLease) -> Result<bool, MailboxError> {+ let span = Span::current();let operation = async {let folders = self.repo.folders(lease.mailbox).await?;- match self- .provider- .message(lease.mailbox, &lease.provider_id, &folders)- .await?- {- Some(snapshot) => self.ingest.ingest(&lease, snapshot).await?,- None if lease.attempts < 3 => {- return Err(EmailApiError::Transient {- message: "mailbox message location has not settled".into(),- }- .into());+ let (snapshot, graph_calls) = email_api_client::domain::request_count::counted(+ self.provider+ .message(lease.mailbox, &lease.provider_id, &folders),+ )+ .await;+ span.record("outlook.graph_calls", i64::from(graph_calls));+ let started = Instant::now();+ let outcome = match snapshot? {+ Some(snapshot) => {+ span.record(+ "outlook.attachments",+ span_int(snapshot.content.message.attachments.len()),+ );+ self.ingest.ingest(lease, snapshot).await?;+ "imported"}- None => self.ingest.absent(&lease).await?,+ None if lease.attempts < 3 => return Err(MailboxError::LocationUnsettled),+ None => {+ self.ingest.absent(lease).await?;+ "absent"+ }+ };+ let import_completed = self.repo.complete_message(lease).await?;+ span.record("outlook.db_write_ms", elapsed_ms(started));+ Ok((outcome, import_completed))+ };+ let result = maintain_lease(operation, || self.repo.renew(SyncLease::Message(lease))).await;+ let error = match result {+ Ok((outcome, import_completed)) => {+ record_done(&span, outcome);+ if outcome == "imported" {+ self.imports+ .record(lease.mailbox.link_id, chrono::Utc::now());+ }+ if let Some(completed) = import_completed {+ self.report_import_completed(&completed).await;+ }+ return Ok(true);}- self.repo.complete_message(&lease).await+ Err(error) => error,};- let result =- maintain_lease(operation, || self.repo.renew(SyncLease::Message(&lease))).await;- if let Err(error) = &result {- let delay = match error {- MailboxError::Provider(error) => retry_seconds(error),- _ => 30,- };- self.repo.release_message(&lease, delay).await?;+ let delay = match &error {+ MailboxError::Provider(error) => retry_seconds(error),+ _ => 30,+ };+ self.repo+ .release_message(lease, delay)+ .await+ .inspect_err(|release| {+ record_failed(&span, lease.mailbox.link_id, release);+ })?;+ if matches!(error, MailboxError::Stale) {+ record_failed(&span, lease.mailbox.link_id, &error);+ } else {+ record_deferred(&span, lease.mailbox.link_id, &error, delay);}- result.map(|()| true)+ Err(error)+ }++ async fn report_import_completed(&self, completed: &ImportCompleted) {+ let graph_calls = self.provider.admitted_requests(completed.mailbox).await;+ progress::log_import_completed(completed, graph_calls);}}diff --git a/crates/email/src/domain/mailbox/commands.rs b/crates/email/src/domain/mailbox/commands.rsindex abb189503a..1f2c5f7d01 100644--- a/crates/email/src/domain/mailbox/commands.rs+++ b/crates/email/src/domain/mailbox/commands.rs@@ -122,24 +122,31 @@ impl<R: MailboxCommandRepository, G: MailboxCommandGateway> MailboxCommandServiclet Some(lease) = self.repo.claim_command(macro_uuid::generate_uuid_v7())- .await?+ .await+ .map_err(|error| super::telemetry::claim_failed("commands", error))?else {return Ok(false);};- let result =- super::maintain_lease(self.execute(&lease), || self.repo.renew_command(&lease)).await;- if let Err(error) = &result {- self.repo- .release_command(- &lease,- match error {- MailboxError::Provider(e) => super::retry_seconds(e),- _ => 30,- },- )- .await?;+ let link_id = lease.mailbox.link_id;+ async {+ let result =+ super::maintain_lease(self.execute(&lease), || self.repo.renew_command(&lease))+ .await;+ if let Err(error) = &result {+ self.repo+ .release_command(+ &lease,+ match error {+ MailboxError::Provider(e) => super::retry_seconds(e),+ _ => 30,+ },+ )+ .await?;+ }+ result.map(|()| true)}- result.map(|()| true)+ .await+ .inspect_err(|error| super::telemetry::log_unit_error(link_id, error))}async fn execute(&self, lease: &CommandLease) -> Result<(), MailboxError> {diff --git a/crates/email/src/domain/mailbox/contacts.rs b/crates/email/src/domain/mailbox/contacts.rsindex cb5b0cae1b..d74c1b5a8f 100644--- a/crates/email/src/domain/mailbox/contacts.rs+++ b/crates/email/src/domain/mailbox/contacts.rs@@ -141,33 +141,39 @@ impl<R: AddressBookRepository, G: AddressBookGateway, S: ContactPhotoStorage>let Some(work) = self.repository.claim(macro_uuid::generate_uuid_v7())- .await?+ .await+ .map_err(|error| super::telemetry::claim_failed("contacts", error))?else {return Ok(false);};- let result = maintain_lease(self.sync(&work), || self.repository.renew(&work)).await;- match result {- Err(MailboxError::Provider(EmailApiError::OutdatedCursor))- if work.kind == AddressBookWorkKind::Folder =>- {- self.repository.reset(&work).await?+ let link_id = work.stream.mailbox.link_id;+ async {+ let result = maintain_lease(self.sync(&work), || self.repository.renew(&work)).await;+ match result {+ Err(MailboxError::Provider(EmailApiError::OutdatedCursor))+ if work.kind == AddressBookWorkKind::Folder =>+ {+ self.repository.reset(&work).await?+ }+ Err(MailboxError::Provider(EmailApiError::NotFound))+ if work.kind == AddressBookWorkKind::Folder =>+ {+ self.repository.retire_folder(&work).await?+ }+ Err(MailboxError::Provider(error)) => {+ // Consent/policy failures are confined to this capability.+ self.repository+ .release(&work, retry_seconds(&error).max(60))+ .await?;+ return Err(error.into());+ }+ Err(error) => return Err(error),+ Ok(()) => {}}- Err(MailboxError::Provider(EmailApiError::NotFound))- if work.kind == AddressBookWorkKind::Folder =>- {- self.repository.retire_folder(&work).await?- }- Err(MailboxError::Provider(error)) => {- // Consent/policy failures are confined to this capability.- self.repository- .release(&work, retry_seconds(&error).max(60))- .await?;- return Err(error.into());- }- Err(error) => return Err(error),- Ok(()) => {}+ Ok(true)}- Ok(true)+ .await+ .inspect_err(|error| super::telemetry::log_unit_error(link_id, error))}async fn sync(&self, work: &AddressBookWork) -> Result<(), MailboxError> {match work.kind {@@ -202,41 +208,47 @@ impl<R: AddressBookRepository, G: AddressBookGateway, S: ContactPhotoStorage>let Some(lease) = self.repository.claim_photo(macro_uuid::generate_uuid_v7())- .await?+ .await+ .map_err(|error| super::telemetry::claim_failed("contact_photos", error))?else {return Ok(false);};- let operation = async {- let photo = self- .gateway- .photo(- lease.mailbox,- lease.folder.as_ref().map(|folder| (folder, &lease.contact)),- )- .await?;- let Some(photo) = photo else {- return self.repository.commit_photo(&lease, None, None).await;- };- let hash = format!("{:x}", Sha256::digest(&photo.bytes));- let url = if lease.photo_hash.as_deref() == Some(&hash)- && let Some(url) = &lease.photo_url- {- url.clone()- } else {- self.storage.store(lease.mailbox, &hash, photo).await?- };- self.repository- .commit_photo(&lease, Some(&hash), Some(&url))- .await- };- let result = maintain_lease(operation, || self.repository.renew_photo(&lease)).await;- if let Err(error) = &result {- let delay = match error {- MailboxError::Provider(error) => retry_seconds(error).max(60),- _ => 60,+ let link_id = lease.mailbox.link_id;+ async {+ let operation = async {+ let photo = self+ .gateway+ .photo(+ lease.mailbox,+ lease.folder.as_ref().map(|folder| (folder, &lease.contact)),+ )+ .await?;+ let Some(photo) = photo else {+ return self.repository.commit_photo(&lease, None, None).await;+ };+ let hash = format!("{:x}", Sha256::digest(&photo.bytes));+ let url = if lease.photo_hash.as_deref() == Some(&hash)+ && let Some(url) = &lease.photo_url+ {+ url.clone()+ } else {+ self.storage.store(lease.mailbox, &hash, photo).await?+ };+ self.repository+ .commit_photo(&lease, Some(&hash), Some(&url))+ .await};- self.repository.release_photo(&lease, delay).await?;+ let result = maintain_lease(operation, || self.repository.renew_photo(&lease)).await;+ if let Err(error) = &result {+ let delay = match error {+ MailboxError::Provider(error) => retry_seconds(error).max(60),+ _ => 60,+ };+ self.repository.release_photo(&lease, delay).await?;+ }+ result.map(|()| true)}- result.map(|()| true)+ .await+ .inspect_err(|error| super::telemetry::log_unit_error(link_id, error))}}diff --git a/crates/email/src/domain/mailbox/drafts.rs b/crates/email/src/domain/mailbox/drafts.rsindex 02702931a8..d60165d682 100644--- a/crates/email/src/domain/mailbox/drafts.rs+++ b/crates/email/src/domain/mailbox/drafts.rs@@ -50,41 +50,47 @@ impl<R: DraftRepository, G: DraftGateway, M: DraftMaterializer> MailboxDraftServlet Some(lease) = self.repo.claim_draft(macro_uuid::generate_uuid_v7(), self.claim_mode)- .await?+ .await+ .map_err(|error| super::telemetry::claim_failed("drafts", error))?else {return Ok(false);};- let result =- super::maintain_lease(self.execute(&lease), || self.repo.renew_draft(&lease)).await;- match result {- Ok(()) => Ok(true),- Err(MailboxError::Provider(EmailApiError::Conflict)) => {- self.repo.fail_draft(&lease, DraftFailure::Conflict).await?;- Ok(true)- }- Err(MailboxError::Provider(EmailApiError::Forbidden)) => {- self.repo.fail_draft(&lease, DraftFailure::Denied).await?;- Ok(true)- }- Err(MailboxError::Provider(EmailApiError::Permanent { .. })) => {- self.repo.fail_draft(&lease, DraftFailure::Invalid).await?;- Ok(true)- }- Err(error) => {- if !matches!(error, MailboxError::Stale) {- self.repo- .release_draft(- &lease,- match &error {- MailboxError::Provider(e) => super::retry_seconds(e),- _ => 30,- },- )- .await?;+ let link_id = lease.mailbox.link_id;+ async {+ let result =+ super::maintain_lease(self.execute(&lease), || self.repo.renew_draft(&lease)).await;+ match result {+ Ok(()) => Ok(true),+ Err(MailboxError::Provider(EmailApiError::Conflict)) => {+ self.repo.fail_draft(&lease, DraftFailure::Conflict).await?;+ Ok(true)+ }+ Err(MailboxError::Provider(EmailApiError::Forbidden)) => {+ self.repo.fail_draft(&lease, DraftFailure::Denied).await?;+ Ok(true)+ }+ Err(MailboxError::Provider(EmailApiError::Permanent { .. })) => {+ self.repo.fail_draft(&lease, DraftFailure::Invalid).await?;+ Ok(true)+ }+ Err(error) => {+ if !matches!(error, MailboxError::Stale) {+ self.repo+ .release_draft(+ &lease,+ match &error {+ MailboxError::Provider(e) => super::retry_seconds(e),+ _ => 30,+ },+ )+ .await?;+ }+ Err(error)}- Err(error)}}+ .await+ .inspect_err(|error| super::telemetry::log_unit_error(link_id, error))}async fn execute(&self, lease: &DraftLease) -> Result<(), MailboxError> {diff --git a/crates/email/src/domain/mailbox/progress.rs b/crates/email/src/domain/mailbox/progress.rsnew file mode 100644index 0000000000..b4126b4e40--- /dev/null+++ b/crates/email/src/domain/mailbox/progress.rs@@ -0,0 +1,182 @@+//! Import progress: the per-mailbox health snapshot and import lifecycle+//! events. Telemetry only; nothing here changes what sync does.++use super::{MailboxError, MailboxKey};+use chrono::{DateTime, Utc};+use std::collections::HashMap;+use std::future::Future;+use std::sync::{Arc, Mutex};+use uuid::Uuid;++#[cfg(test)]+mod test;++/// One finished import run of a mailbox's sync generation.+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct ImportCompleted {+ pub mailbox: MailboxKey,+ pub started_at: Option<DateTime<Utc>>,+ pub completed_at: DateTime<Utc>,+ /// Provider messages stored for the mailbox when the run finished.+ pub messages: i64,+ /// Mail folders the run enumerated.+ pub folders: i64,+}++/// Messages one mailbox imported since the last flush.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub struct MailboxImports {+ pub imported: u32,+ pub last_imported_at: DateTime<Utc>,+}++/// Imports this worker counted, kept in memory so the import path writes+/// nothing extra. The health loop flushes it once a minute.+#[derive(Clone, Default)]+pub struct ImportTally(Arc<Mutex<HashMap<Uuid, MailboxImports>>>);++impl ImportTally {+ pub fn record(&self, link_id: Uuid, at: DateTime<Utc>) {+ let Ok(mut imports) = self.0.lock() else {+ return;+ };+ imports+ .entry(link_id)+ .and_modify(|entry| {+ entry.imported = entry.imported.saturating_add(1);+ entry.last_imported_at = entry.last_imported_at.max(at);+ })+ .or_insert(MailboxImports {+ imported: 1,+ last_imported_at: at,+ });+ }++ fn take(&self) -> HashMap<Uuid, MailboxImports> {+ self.0+ .lock()+ .map(|mut imports| std::mem::take(&mut *imports))+ .unwrap_or_default()+ }++ /// Put back counts whose flush failed, so the next flush carries them.+ fn restore(&self, unflushed: HashMap<Uuid, MailboxImports>) {+ for (link_id, entry) in unflushed {+ let Ok(mut imports) = self.0.lock() else {+ return;+ };+ imports+ .entry(link_id)+ .and_modify(|current| {+ current.imported = current.imported.saturating_add(entry.imported);+ current.last_imported_at = current.last_imported_at.max(entry.last_imported_at);+ })+ .or_insert(entry);+ }+ }+}++/// One mailbox's sync backlog, as one health snapshot reports it.+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct MailboxBacklog {+ pub link_id: Uuid,+ pub sync_generation: i64,+ /// Messages waiting for reconciliation in the current generation.+ pub queued: i64,+ /// Queued messages backed off until a later time.+ pub parked: i64,+ /// Queued messages a worker holds right now.+ pub leased: i64,+ /// How long the longest-waiting claimable message has been due; 0 if none.+ pub oldest_waiting_seconds: i64,+ /// How far away the latest parked message's retry is; 0 if none.+ pub parked_until_max_seconds: i64,+ /// Folder and catalog streams whose initial enumeration is unfinished.+ pub streams_incomplete: i64,+ pub imported_5m: i64,+ /// `None` when nothing was imported in the last day.+ pub seconds_since_last_import: Option<i64>,+ pub sync_active: bool,+ pub needs_reauth: bool,+}++pub trait MailboxProgressRepository: Send + Sync + 'static {+ /// Add imports counted since the last flush to the current minute.+ fn record_imports(+ &self,+ imports: &HashMap<Uuid, MailboxImports>,+ ) -> impl Future<Output = Result<(), MailboxError>> + Send;+ /// One aggregate read: every sync-active Outlook mailbox, and inactive+ /// ones that still have queued work. Also drops import counts older than+ /// a day.+ fn backlogs(&self) -> impl Future<Output = Result<Vec<MailboxBacklog>, MailboxError>> + Send;+}++pub struct MailboxProgressService<R> {+ repo: R,+ tally: ImportTally,+}++impl<R: MailboxProgressRepository> MailboxProgressService<R> {+ /// `tally` is the one the worker's sync service counts imports into.+ pub fn new(repo: R, tally: ImportTally) -> Self {+ Self { repo, tally }+ }++ /// Persist this worker's import counts. Every worker calls this each minute.+ pub async fn flush(&self) -> Result<(), MailboxError> {+ let imports = self.tally.take();+ if imports.is_empty() {+ return Ok(());+ }+ let result = self.repo.record_imports(&imports).await;+ if result.is_err() {+ self.tally.restore(imports);+ }+ result+ }++ /// Log one health snapshot per mailbox. One worker calls this per minute.+ pub async fn report(&self) -> Result<usize, MailboxError> {+ let backlogs = self.repo.backlogs().await?;+ for backlog in &backlogs {+ tracing::info!(+ link_id = %backlog.link_id,+ sync_generation = backlog.sync_generation,+ queued = backlog.queued,+ parked = backlog.parked,+ leased = backlog.leased,+ oldest_waiting_seconds = backlog.oldest_waiting_seconds,+ parked_until_max_seconds = backlog.parked_until_max_seconds,+ streams_incomplete = backlog.streams_incomplete,+ imported_5m = backlog.imported_5m,+ seconds_since_last_import = backlog.seconds_since_last_import,+ sync_active = backlog.sync_active,+ needs_reauth = backlog.needs_reauth,+ "outlook mailbox health"+ );+ }+ Ok(backlogs.len())+ }+}++/// Logged once an import run's last folder and last imported message settle.+pub(super) fn log_import_completed(completed: &ImportCompleted, graph_calls: Option<u64>) {+ let duration_seconds = completed+ .started_at+ .map(|started| (completed.completed_at - started).num_seconds().max(0));+ tracing::info!(+ link_id = %completed.mailbox.link_id,+ sync_generation = completed.mailbox.sync_generation,+ duration_seconds,+ messages = completed.messages,+ folders = completed.folders,+ graph_calls = graph_calls.and_then(|calls| i64::try_from(calls).ok()),+ "outlook import completed"+ );+}++/// Logged when a mailbox starts an import run in a sync generation.+pub fn log_import_started(link_id: Uuid, sync_generation: i64) {+ tracing::info!(%link_id, sync_generation, "outlook import started");+}diff --git a/crates/email/src/domain/mailbox/progress/test.rs b/crates/email/src/domain/mailbox/progress/test.rsnew file mode 100644index 0000000000..ea14d1ff48--- /dev/null+++ b/crates/email/src/domain/mailbox/progress/test.rs@@ -0,0 +1,52 @@+use super::*;+use std::sync::atomic::{AtomicBool, Ordering};++/// Fails the first flush, then keeps what later flushes write.+#[derive(Default)]+struct FlakyStore {+ failed_once: AtomicBool,+ written: Mutex<Vec<HashMap<Uuid, MailboxImports>>>,+}++impl MailboxProgressRepository for FlakyStore {+ async fn record_imports(+ &self,+ imports: &HashMap<Uuid, MailboxImports>,+ ) -> Result<(), MailboxError> {+ if !self.failed_once.swap(true, Ordering::SeqCst) {+ return Err(MailboxError::Persistence);+ }+ self.written.lock().unwrap().push(imports.clone());+ Ok(())+ }+ async fn backlogs(&self) -> Result<Vec<MailboxBacklog>, MailboxError> {+ Ok(Vec::new())+ }+}++#[tokio::test]+async fn imports_counted_before_a_failed_flush_reach_the_next_one() {+ let link = Uuid::now_v7();+ let early = Utc::now() - chrono::Duration::seconds(30);+ let late = Utc::now();+ let tally = ImportTally::default();+ let service = MailboxProgressService::new(FlakyStore::default(), tally.clone());+ tally.record(link, early);+ tally.record(link, early);++ assert!(service.flush().await.is_err());+ tally.record(link, late);+ service.flush().await.unwrap();++ let written = service.repo.written.lock().unwrap();+ assert_eq!(+ written.as_slice(),+ [HashMap::from([(+ link,+ MailboxImports {+ imported: 3,+ last_imported_at: late,+ }+ )])]+ );+}diff --git a/crates/email/src/domain/mailbox/projection.rs b/crates/email/src/domain/mailbox/projection.rsindex 9b0b1622d5..88010affe6 100644--- a/crates/email/src/domain/mailbox/projection.rs+++ b/crates/email/src/domain/mailbox/projection.rs@@ -1,5 +1,8 @@//! Durable downstream projection and live-mail notification policy.+use super::telemetry::{+ claim_failed, elapsed_ms, record_deferred, record_done, record_failed, span_int,+};use super::{MailboxError, MailboxKey};use crate::domain::events::*;use crate::domain::models::calendar_invitation::CalendarInvitation;@@ -7,7 +10,8 @@ use email_api_client::domain::models::CalendarPart;use macro_user_id::user_id::MacroUserIdStr;use models_email::service::{link::Link, message::Message};use serde::{Deserialize, Serialize};-use std::{collections::HashSet, future::Future};+use std::{collections::HashSet, future::Future, time::Instant};+use tracing::{Instrument, Span};use uuid::Uuid;#[cfg(test)]@@ -65,6 +69,8 @@ pub struct ProjectionLease {pub mailbox: MailboxKey,pub event: ProjectionEvent,pub from_current_generation: bool,+ /// Claims of this event so far, including this one.+ pub attempts: i32,}pub struct ProjectedMessage {@@ -124,10 +130,12 @@ pub trait MailboxProjectionRepository: Send + Sync + 'static {&self,lease: &ProjectionLease,) -> impl Future<Output = Result<Option<ProjectionContext>, MailboxError>> + Send;+ /// Deletes a delivered event, or with `retry_after_seconds` makes it due+ /// again after that delay.fn finish_projection(&self,lease: &ProjectionLease,- success: bool,+ retry_after_seconds: Option<u32>,) -> impl Future<Output = Result<(), MailboxError>> + Send;}@@ -202,18 +210,68 @@ impl<R: MailboxProjectionRepository, E: MailboxProjectionEffects> MailboxProjectlet Some(lease) = self.repo.claim_projection(macro_uuid::generate_uuid_v7())- .await?+ .await+ .map_err(|error| claim_failed("project", error))?else {return Ok(false);};- let result =- super::maintain_lease(self.project(&lease), || self.repo.renew_projection(&lease))- .await;- self.repo.finish_projection(&lease, result.is_ok()).await?;- result.map(|()| true)+ let span = tracing::info_span!(+ "outlook.sync.project",+ mailbox.link_id = %lease.mailbox.link_id,+ mailbox.sync_generation = lease.mailbox.sync_generation,+ outlook.event = lease.event.kind(),+ outlook.attempts = lease.attempts,+ outlook.crm_jobs = tracing::field::Empty,+ outlook.refreshes = tracing::field::Empty,+ outlook.reindex_events = tracing::field::Empty,+ outlook.attachment_upload_ms = tracing::field::Empty,+ outlook.outcome = tracing::field::Empty,+ outlook.defer_reason = tracing::field::Empty,+ outlook.defer_seconds = tracing::field::Empty,+ otel.status_description = tracing::field::Empty,+ );+ self.project_leased(&lease).instrument(span).await}- async fn project(&self, lease: &ProjectionLease) -> Result<(), MailboxError> {+ async fn project_leased(&self, lease: &ProjectionLease) -> Result<bool, MailboxError> {+ let span = Span::current();+ let mut effects = EffectCounts::default();+ let result = super::maintain_lease(self.project(lease, &mut effects), || {+ self.repo.renew_projection(lease)+ })+ .await;+ span.record("outlook.crm_jobs", span_int(effects.crm_jobs));+ span.record("outlook.refreshes", i64::from(effects.refreshes));+ span.record("outlook.reindex_events", i64::from(effects.reindex_events));+ span.record("outlook.attachment_upload_ms", effects.attachment_upload_ms);+ let retry_after = result+ .is_err()+ .then(|| projection_retry_seconds(lease.attempts));+ self.repo+ .finish_projection(lease, retry_after)+ .await+ .inspect_err(|finish| record_failed(&span, lease.mailbox.link_id, finish))?;+ match (result, retry_after) {+ (Ok(()), _) => {+ record_done(&span, "imported");+ Ok(true)+ }+ (Err(error), Some(delay)) if !matches!(error, MailboxError::Stale) => {+ record_deferred(&span, lease.mailbox.link_id, &error, delay);+ Err(error)+ }+ (Err(error), _) => {+ record_failed(&span, lease.mailbox.link_id, &error);+ Err(error)+ }+ }+ }++ async fn project(+ &self,+ lease: &ProjectionLease,+ effects: &mut EffectCounts,+ ) -> Result<(), MailboxError> {let Some(context) = self.repo.projection_context(lease).await? else {return Ok(());};@@ -257,11 +315,13 @@ impl<R: MailboxProjectionRepository, E: MailboxProjectionEffects> MailboxProject.with_event_id(lease.id),).await?;+ effects.reindex_events += 1;}if *self_photo {self.effects.photo_refresh(context.link.id, &context.viewers).await?;+ effects.refreshes += 1;}}ProjectionEvent::LinkConnected { is_new, .. } => {@@ -288,14 +348,16 @@ impl<R: MailboxProjectionRepository, E: MailboxProjectionEffects> MailboxProject.as_ref().is_some_and(|message| message.is_present){+ let started = Instant::now();self.effects.attachments(lease.mailbox, &context, true).await?;+ effects.attachment_upload_ms += elapsed_ms(started);}return Ok(());}ProjectionEvent::Absent { thread_id, .. } => {- self.reindex(&context.link, *thread_id).await?;+ self.reindex(&context.link, *thread_id, effects).await?;}ProjectionEvent::Organization {thread_id,@@ -310,7 +372,7 @@ impl<R: MailboxProjectionRepository, E: MailboxProjectionEffects> MailboxProject).await?;}- self.reindex(&context.link, *thread_id).await?;+ self.reindex(&context.link, *thread_id, effects).await?;}ProjectionEvent::Message {actor,@@ -336,7 +398,10 @@ impl<R: MailboxProjectionRepository, E: MailboxProjectionEffects> MailboxProject.invitations(*message_id, &invitation_snapshots(calendar_parts)).await?;}- self.effects.contacts(correspondence(&context)).await?;+ let correspondence = correspondence(&context);+ let crm_jobs = correspondence.contacts.len();+ self.effects.contacts(correspondence).await?;+ effects.crm_jobs += crm_jobs;}if !import&& let Some(event) =@@ -350,24 +415,34 @@ impl<R: MailboxProjectionRepository, E: MailboxProjectionEffects> MailboxProject}}}- self.reindex(&context.link, *thread_id).await?;+ self.reindex(&context.link, *thread_id, effects).await?;self.effects.refresh(context.link.id, &context.viewers).await?;+ effects.refreshes += 1;if current.is_present {+ let started = Instant::now();self.effects.attachments(lease.mailbox, &context, import).await?;+ effects.attachment_upload_ms += elapsed_ms(started);}return Ok(());}}self.effects.refresh(context.link.id, &context.viewers)- .await+ .await?;+ effects.refreshes += 1;+ Ok(())}- async fn reindex(&self, link: &Link, thread: Uuid) -> Result<(), MailboxError> {+ async fn reindex(+ &self,+ link: &Link,+ thread: Uuid,+ effects: &mut EffectCounts,+ ) -> Result<(), MailboxError> {self.effects.publish(EmailMacroEvent::threads_reindex_requested(ThreadsReindexRequestedMetadata {@@ -377,7 +452,42 @@ impl<R: MailboxProjectionRepository, E: MailboxProjectionEffects> MailboxProjectreason: ThreadsReindexReason::ProviderChanged,},))- .await+ .await?;+ effects.reindex_events += 1;+ Ok(())+ }+}++/// Downstream effects one projection produced, for its span.+#[derive(Default)]+struct EffectCounts {+ crm_jobs: usize,+ refreshes: u32,+ reindex_events: u32,+ attachment_upload_ms: i64,+}++/// A failed projection backs off 30 s per attempt, up to ten minutes.+fn projection_retry_seconds(attempts: i32) -> u32 {+ u32::try_from(attempts)+ .unwrap_or(1)+ .saturating_mul(30)+ .clamp(30, 600)+}++impl ProjectionEvent {+ /// The event's kind, as its payload's `kind` tag names it.+ fn kind(&self) -> &'static str {+ match self {+ Self::LinkChanged => "link_changed",+ Self::AttachmentRecheck { .. } => "attachment_recheck",+ Self::ReauthorizationRequired => "reauthorization_required",+ Self::ContactsChanged { .. } => "contacts_changed",+ Self::Message { .. } => "message",+ Self::Absent { .. } => "absent",+ Self::Organization { .. } => "organization",+ Self::LinkConnected { .. } => "link_connected",+ }}}diff --git a/crates/email/src/domain/mailbox/projection/test.rs b/crates/email/src/domain/mailbox/projection/test.rsindex c9f9456d41..d17dabf9f0 100644--- a/crates/email/src/domain/mailbox/projection/test.rs+++ b/crates/email/src/domain/mailbox/projection/test.rs@@ -90,6 +90,7 @@ fn notification_retry_preserves_ids_and_staff_push_policy() {is_new: true,},from_current_generation: true,+ attempts: 1,};let first = notifications(&lease, &context);let retry = notifications(&lease, &context);diff --git a/crates/email/src/domain/mailbox/settings.rs b/crates/email/src/domain/mailbox/settings.rsindex 06c96b1916..e9b0c518b5 100644--- a/crates/email/src/domain/mailbox/settings.rs+++ b/crates/email/src/domain/mailbox/settings.rs@@ -298,24 +298,30 @@ impl<R: MailboxSettingsRepository, G: MailboxSettingsGateway> MailboxSettingsSerself.outlook_sync,self.outlook_writes,)- .await?+ .await+ .map_err(|error| super::telemetry::claim_failed("settings", error))?else {return Ok(false);};- let result = super::maintain_lease(self.execute(&lease), || {- self.repository.renew_settings(&lease)- })- .await;- if let Err(error) = &result {- let delay = match error {- MailboxError::Provider(e) => super::retry_seconds(e),- _ => 30,- };- self.repository- .finish_settings(&lease, false, delay, Some("provider_settings_unavailable"))- .await?;+ let link_id = lease.mailbox.key.link_id;+ async {+ let result = super::maintain_lease(self.execute(&lease), || {+ self.repository.renew_settings(&lease)+ })+ .await;+ if let Err(error) = &result {+ let delay = match error {+ MailboxError::Provider(e) => super::retry_seconds(e),+ _ => 30,+ };+ self.repository+ .finish_settings(&lease, false, delay, Some("provider_settings_unavailable"))+ .await?;+ }+ result.map(|()| true)}- result.map(|()| true)+ .await+ .inspect_err(|error| super::telemetry::log_unit_error(link_id, error))}async fn execute(&self, lease: &SettingsLease) -> Result<(), MailboxError> {match lease.kind.as_str() {diff --git a/crates/email/src/domain/mailbox/telemetry.rs b/crates/email/src/domain/mailbox/telemetry.rsnew file mode 100644index 0000000000..2a7a12b76a--- /dev/null+++ b/crates/email/src/domain/mailbox/telemetry.rs@@ -0,0 +1,105 @@+//! Outcomes, deferral reasons and logs shared by the Outlook sync units+//! (`outlook.sync.*` spans). Each claimed unit of work gets one root span;+//! idle polls get none.++use super::MailboxError;+use email_api_client::domain::models::{EmailApiError, GateLimit, RateLimitOrigin};+use tracing::Span;+use uuid::Uuid;++/// Why a unit of sync work was deferred or failed (`outlook.defer_reason`).+pub fn defer_reason(error: &MailboxError) -> &'static str {+ match error {+ MailboxError::Provider(EmailApiError::RateLimited { origin, .. }) => match origin {+ RateLimitOrigin::Gate(GateLimit::Window) => "gate_window",+ RateLimitOrigin::Gate(GateLimit::Concurrency) => "gate_concurrency",+ RateLimitOrigin::Gate(GateLimit::Cooldown) => "gate_cooldown",+ RateLimitOrigin::Provider { .. } => "graph_throttled",+ RateLimitOrigin::Local => "other",+ },+ MailboxError::Provider(EmailApiError::CredentialsUnavailable { .. }) => "token_unavailable",+ MailboxError::Provider(EmailApiError::AuthRequired | EmailApiError::Forbidden) => {+ "auth_required"+ }+ MailboxError::Persistence | MailboxError::Database(_) => "persistence",+ MailboxError::Effect => "effect",+ MailboxError::LocationUnsettled => "location_unsettled",+ _ => "other",+ }+}++/// A refusal by the shared request gate is routine back-pressure, so it is+/// logged at DEBUG; the span still records it.+fn is_gate_refusal(error: &MailboxError) -> bool {+ matches!(+ error,+ MailboxError::Provider(EmailApiError::RateLimited {+ origin: RateLimitOrigin::Gate(_),+ ..+ })+ )+}++/// The unit committed its work: `imported` or, for a vanished message, `absent`.+pub(super) fn record_done(span: &Span, outcome: &'static str) {+ span.record("outlook.outcome", outcome);+}++/// The unit released its work to run again after `delay_seconds`.+pub(super) fn record_deferred(+ span: &Span,+ link_id: Uuid,+ error: &MailboxError,+ delay_seconds: u32,+) {+ let reason = defer_reason(error);+ span.record("outlook.outcome", "deferred");+ span.record("outlook.defer_reason", reason);+ span.record("outlook.defer_seconds", i64::from(delay_seconds));+ if is_gate_refusal(error) {+ tracing::debug!(parent: span, %link_id, defer_reason = reason, defer_seconds = i64::from(delay_seconds), %error, "mailbox sync work deferred");+ } else {+ tracing::warn!(parent: span, %link_id, defer_reason = reason, defer_seconds = i64::from(delay_seconds), %error, "mailbox sync work deferred");+ }+}++/// The unit stopped without releasing its work, which another worker retries+/// once the lease expires, or lost its lease to a newer owner (`Stale`).+pub(super) fn record_failed(span: &Span, link_id: Uuid, error: &MailboxError) {+ let reason = defer_reason(error);+ span.record("outlook.outcome", "failed");+ span.record("outlook.defer_reason", reason);+ span.record("otel.status_description", tracing::field::display(error));+ if matches!(error, MailboxError::Stale) {+ tracing::debug!(parent: span, %link_id, defer_reason = reason, %error, "mailbox sync work failed");+ } else {+ tracing::warn!(parent: span, %link_id, defer_reason = reason, %error, "mailbox sync work failed");+ }+}++/// Logs why a unit of the other mailbox loops (contacts, drafts, commands,+/// settings, watches), which have no span, stopped or was deferred.+pub(super) fn log_unit_error(link_id: Uuid, error: &MailboxError) {+ let reason = defer_reason(error);+ if is_gate_refusal(error) || matches!(error, MailboxError::Stale) {+ tracing::debug!(%link_id, defer_reason = reason, %error, "mailbox sync work deferred");+ } else {+ tracing::warn!(%link_id, defer_reason = reason, %error, "mailbox sync work deferred");+ }+}++/// A claim query failed, so no unit (and no mailbox) was involved yet.+pub(super) fn claim_failed(unit: &'static str, error: MailboxError) -> MailboxError {+ tracing::warn!(unit, defer_reason = defer_reason(&error), %error, "mailbox sync claim failed");+ error+}++pub(super) fn elapsed_ms(started: std::time::Instant) -> i64 {+ span_int(started.elapsed().as_millis())+}++/// OpenTelemetry receives a span field as an integer only when it is+/// recorded as `i64`; unsigned values arrive as strings.+pub(super) fn span_int(value: impl TryInto<i64>) -> i64 {+ value.try_into().unwrap_or(i64::MAX)+}diff --git a/crates/email/src/domain/mailbox/watches.rs b/crates/email/src/domain/mailbox/watches.rsindex 1a6aedfc28..99915a059b 100644--- a/crates/email/src/domain/mailbox/watches.rs+++ b/crates/email/src/domain/mailbox/watches.rs@@ -158,20 +158,30 @@ impl<R: MailboxWatchRepository, G: MailboxWatchGateway> MailboxWatchService<R, Gif !self.callback_base.starts_with("https://") {return Ok(false);}- let Some(lease) = self.repository.claim_watch(Uuid::new_v4()).await? else {+ let Some(lease) = self+ .repository+ .claim_watch(Uuid::new_v4())+ .await+ .map_err(|error| super::telemetry::claim_failed("watches", error))?+ else {return Ok(false);};- let result = super::maintain_lease(self.execute(&lease), || {- self.repository.renew_watch_lease(&lease)- })- .await;- let delay = match &result {- Ok(()) => 12 * 60 * 60,- Err(MailboxError::Provider(e)) => super::retry_seconds(e),- _ => 30,- };- self.repository.release_watch(&lease, delay).await?;- result.map(|()| true)+ let link_id = lease.mailbox.link_id;+ async {+ let result = super::maintain_lease(self.execute(&lease), || {+ self.repository.renew_watch_lease(&lease)+ })+ .await;+ let delay = match &result {+ Ok(()) => 12 * 60 * 60,+ Err(MailboxError::Provider(e)) => super::retry_seconds(e),+ _ => 30,+ };+ self.repository.release_watch(&lease, delay).await?;+ result.map(|()| true)+ }+ .await+ .inspect_err(|error| super::telemetry::log_unit_error(link_id, error))}async fn execute(&self, lease: &WatchLease) -> Result<(), MailboxError> {let remote = self.gateway.watches(lease.mailbox).await?;diff --git a/crates/email/src/lib.rs b/crates/email/src/lib.rsindex f92ebe48cd..1aa29a43ef 100644--- a/crates/email/src/lib.rs+++ b/crates/email/src/lib.rs@@ -3,3 +3,5 @@ pub mod domain;pub mod inbound;#[cfg(any(feature = "outbound", feature = "http_client", feature = "s3"))]pub mod outbound;+#[cfg(test)]+pub(crate) mod telemetry_capture;diff --git a/crates/email/src/outbound/mailbox_pg.rs b/crates/email/src/outbound/mailbox_pg.rsindex ed57f9f8bd..5810140b1d 100644--- a/crates/email/src/outbound/mailbox_pg.rs+++ b/crates/email/src/outbound/mailbox_pg.rs@@ -1,5 +1,6 @@//! Postgres implementation of durable mailbox stream and reconciliation leases.+use crate::domain::mailbox::progress::ImportCompleted;use crate::domain::mailbox::*;use email_api_client::domain::models::{MailFolder, MailboxChangePage, ProviderId, StreamPosition, StreamToken,@@ -15,6 +16,7 @@ mod drafts;mod gmail_history;mod ingest;mod lifecycle;+mod progress;mod projection;mod settings;#[cfg(test)]@@ -44,8 +46,40 @@ impl PgMailboxSync {}}-fn db_error(_: sqlx::Error) -> MailboxError {- MailboxError::Persistence+fn db_error(error: sqlx::Error) -> MailboxError {+ MailboxError::Database(DatabaseFailure::from(&error))+}++/// For `email_db_client` calls, which wrap their `sqlx` errors in `anyhow`.+fn wrapped_db_error(error: anyhow::Error) -> MailboxError {+ error+ .chain()+ .find_map(|cause| cause.downcast_ref::<sqlx::Error>())+ .map_or(MailboxError::Persistence, |error| {+ MailboxError::Database(DatabaseFailure::from(error))+ })+}++/// Lets any adapter keep a database error's SQLSTATE without its message.+impl From<&sqlx::Error> for DatabaseFailure {+ fn from(error: &sqlx::Error) -> Self {+ match error {+ sqlx::Error::Database(error) => error.code().map_or(DatabaseFailure::Other, |code| {+ DatabaseFailure::Sqlstate(code.into_owned())+ }),+ sqlx::Error::PoolTimedOut => DatabaseFailure::PoolTimedOut,+ sqlx::Error::Io(_)+ | sqlx::Error::Tls(_)+ | sqlx::Error::Protocol(_)+ | sqlx::Error::PoolClosed+ | sqlx::Error::WorkerCrashed => DatabaseFailure::Connection,+ sqlx::Error::ColumnDecode { .. }+ | sqlx::Error::Decode(_)+ | sqlx::Error::ColumnNotFound(_)+ | sqlx::Error::TypeNotFound { .. } => DatabaseFailure::Decode,+ _ => DatabaseFailure::Other,+ }+ }}impl MailboxSyncRepository for PgMailboxSync {@@ -75,7 +109,7 @@ impl MailboxSyncRepository for PgMailboxSync {&self,lease: &CatalogLease,folders: &[MailFolder],- ) -> Result<(), MailboxError> {+ ) -> Result<Option<ImportCompleted>, MailboxError> {self.commit_catalog_lease(lease, folders).await}async fn release_catalog(&self, lease: &CatalogLease, delay: u32) -> Result<(), MailboxError> {@@ -117,7 +151,7 @@ impl MailboxSyncRepository for PgMailboxSync {&self,lease: &StreamLease,page: &MailboxChangePage,- ) -> Result<(), MailboxError> {+ ) -> Result<CommittedPage, MailboxError> {let (position, complete) = match &page.position {StreamPosition::Continue(position) => (position, false),StreamPosition::Checkpoint(position) => (position, true),@@ -157,7 +191,7 @@ impl MailboxSyncRepository for PgMailboxSync {if advanced.rows_affected() != 1 {return Err(MailboxError::Stale);}- sqlx::query!(+ let queued = sqlx::query!(r#"INSERT INTO email_message_reconciliation (link_id,generation,provider_id,is_import)SELECT $1,$2,provider_id,$4 FROM unnest($3::text[]) provider_id@@ -173,9 +207,14 @@ impl MailboxSyncRepository for PgMailboxSync {).execute(&mut *tx).await- .map_err(db_error)?;- Self::queue_attachment_recheck(&mut tx, lease.mailbox).await?;- tx.commit().await.map_err(db_error)+ .map_err(db_error)?+ .rows_affected();+ let import_completed = Self::settle_import(&mut tx, lease.mailbox).await?;+ tx.commit().await.map_err(db_error)?;+ Ok(CommittedPage {+ queued,+ import_completed,+ })}async fn reset_stream(&self, lease: &StreamLease) -> Result<(), MailboxError> {@@ -264,7 +303,10 @@ impl MailboxSyncRepository for PgMailboxSync {.collect()}- async fn complete_message(&self, lease: &MessageLease) -> Result<(), MailboxError> {+ async fn complete_message(+ &self,+ lease: &MessageLease,+ ) -> Result<Option<ImportCompleted>, MailboxError> {let mut tx = self.db.begin().await.map_err(db_error)?;Self::lock_attachment_completion(&mut tx, lease.mailbox).await?;sqlx::query!(r#"@@ -278,8 +320,9 @@ impl MailboxSyncRepository for PgMailboxSync {WHERE link_id = $1 AND generation = $2 AND provider_id = $3 AND lease_id = $4"#,lease.mailbox.link_id,lease.mailbox.sync_generation,lease.provider_id.as_str(),lease.lease_id).execute(&mut *tx).await.map_err(db_error)?;- Self::queue_attachment_recheck(&mut tx, lease.mailbox).await?;- tx.commit().await.map_err(db_error)+ let import_completed = Self::settle_import(&mut tx, lease.mailbox).await?;+ tx.commit().await.map_err(db_error)?;+ Ok(import_completed)}async fn release_message(&self, lease: &MessageLease, delay: u32) -> Result<(), MailboxError> {@@ -332,29 +375,54 @@ impl PgMailboxSync {Ok(())}- // Called under the link lock after the final page or reconciliation. All- // correspondence is now ingested, regardless of folder/page delivery order.- async fn queue_attachment_recheck(+ // Called under the link lock after each page or reconciliation, so one+ // caller at a time sees the import finish.+ async fn settle_import(tx: &mut sqlx::PgConnection,mailbox: MailboxKey,- ) -> Result<(), MailboxError> {- let ready=sqlx::query_scalar!(r#"SELECT (- EXISTS(SELECT 1 FROM email_sync_streams WHERE link_id=$1 AND generation=$2 AND kind='folder_catalog' AND initial_complete)- AND EXISTS(SELECT 1 FROM email_sync_streams WHERE link_id=$1 AND generation=$2 AND kind='mail_folder' AND NOT attachments_rechecked)- AND NOT EXISTS(SELECT 1 FROM email_sync_streams WHERE link_id=$1 AND generation=$2 AND kind IN ('mail_folder','folder_catalog') AND NOT initial_complete)- AND NOT EXISTS(SELECT 1 FROM email_message_reconciliation WHERE link_id=$1 AND generation=$2)- ) AS "ready!""#,mailbox.link_id,mailbox.sync_generation).fetch_one(&mut *tx).await.map_err(db_error)?;- if !ready {- return Ok(());+ ) -> Result<Option<ImportCompleted>, MailboxError> {+ let state = sqlx::query!(r#"SELECT+ EXISTS(SELECT 1 FROM email_sync_streams WHERE link_id=$1 AND generation=$2 AND kind='folder_catalog' AND initial_complete) AS "catalog_complete!",+ EXISTS(SELECT 1 FROM email_sync_streams WHERE link_id=$1 AND generation=$2 AND kind='folder_catalog' AND import_completed_at IS NULL) AS "import_open!",+ EXISTS(SELECT 1 FROM email_sync_streams WHERE link_id=$1 AND generation=$2 AND kind='mail_folder' AND NOT attachments_rechecked) AS "recheck_pending!",+ EXISTS(SELECT 1 FROM email_sync_streams WHERE link_id=$1 AND generation=$2 AND kind IN ('mail_folder','folder_catalog') AND NOT initial_complete) AS "streams_incomplete!",+ EXISTS(SELECT 1 FROM email_message_reconciliation WHERE link_id=$1 AND generation=$2) AS "queued!",+ EXISTS(SELECT 1 FROM email_message_reconciliation WHERE link_id=$1 AND generation=$2 AND is_import) AS "imports_queued!"+ "#,mailbox.link_id,mailbox.sync_generation).fetch_one(&mut *tx).await.map_err(db_error)?;+ let enumerated = state.catalog_complete && !state.streams_incomplete;+ // All correspondence is now ingested, regardless of folder/page delivery order.+ if enumerated && state.recheck_pending && !state.queued {+ sqlx::query!(r#"+ INSERT INTO email_projection_outbox(id,link_id,generation,payload)+ SELECT gen_random_uuid(),m.link_id,$2,jsonb_build_object('kind','attachment_recheck','message_id',m.id)+ FROM email_messages m WHERE m.link_id=$1 AND m.provider_id IS NOT NULL AND NOT m.is_draft+ AND EXISTS(SELECT 1 FROM email_attachments a WHERE a.message_id=m.id AND a.reference_url IS NULL)+ "#,mailbox.link_id,mailbox.sync_generation).execute(&mut *tx).await.map_err(db_error)?;+ sqlx::query!("UPDATE email_sync_streams SET attachments_rechecked=true WHERE link_id=$1 AND generation=$2 AND kind='mail_folder'",mailbox.link_id,mailbox.sync_generation)+ .execute(&mut *tx).await.map_err(db_error)?;}- sqlx::query!(r#"- INSERT INTO email_projection_outbox(id,link_id,generation,payload)- SELECT gen_random_uuid(),m.link_id,$2,jsonb_build_object('kind','attachment_recheck','message_id',m.id)- FROM email_messages m WHERE m.link_id=$1 AND m.provider_id IS NOT NULL AND NOT m.is_draft- AND EXISTS(SELECT 1 FROM email_attachments a WHERE a.message_id=m.id AND a.reference_url IS NULL)- "#,mailbox.link_id,mailbox.sync_generation).execute(&mut *tx).await.map_err(db_error)?;- sqlx::query!("UPDATE email_sync_streams SET attachments_rechecked=true WHERE link_id=$1 AND generation=$2 AND kind='mail_folder'",mailbox.link_id,mailbox.sync_generation)- .execute(&mut *tx).await.map_err(db_error)?;- Ok(())+ // The import run is done once every folder is enumerated and no import+ // row is left; live changes may still be queued.+ if !(enumerated && state.import_open && !state.imports_queued) {+ return Ok(None);+ }+ let Some(run) = sqlx::query!(r#"+ UPDATE email_sync_streams SET import_completed_at = now()+ WHERE link_id=$1 AND generation=$2 AND kind='folder_catalog' AND import_completed_at IS NULL+ RETURNING import_started_at, import_completed_at AS "completed_at!"+ "#,mailbox.link_id,mailbox.sync_generation).fetch_optional(&mut *tx).await.map_err(db_error)? else {+ return Ok(None);+ };+ let size = sqlx::query!(r#"SELECT+ (SELECT count(*) FROM email_messages WHERE link_id=$1 AND provider_id IS NOT NULL) AS "messages!",+ (SELECT count(*) FROM email_sync_streams WHERE link_id=$1 AND generation=$2 AND kind='mail_folder') AS "folders!"+ "#,mailbox.link_id,mailbox.sync_generation).fetch_one(&mut *tx).await.map_err(db_error)?;+ Ok(Some(ImportCompleted {+ mailbox,+ started_at: run.import_started_at,+ completed_at: run.completed_at,+ messages: size.messages,+ folders: size.folders,+ }))}}diff --git a/crates/email/src/outbound/mailbox_pg/catalog.rs b/crates/email/src/outbound/mailbox_pg/catalog.rsindex 14f462f086..dbf0fb9c40 100644--- a/crates/email/src/outbound/mailbox_pg/catalog.rs+++ b/crates/email/src/outbound/mailbox_pg/catalog.rs@@ -34,7 +34,7 @@ impl PgMailboxSync {&self,lease: &CatalogLease,folders: &[MailFolder],- ) -> Result<(), MailboxError> {+ ) -> Result<Option<ImportCompleted>, MailboxError> {let mut tx = self.db.begin().await.map_err(db_error)?;// Serialize against disconnect/reconnect before touching the shared catalog.let current = sqlx::query!(@@ -148,8 +148,9 @@ impl PgMailboxSync {WHERE s.link_id = $1 AND s.generation = $2 AND s.kind = 'mail_folder'AND f.link_id = s.link_id AND f.provider_id = s.scope_id AND f.deleted_at IS NOT NULL"#,lease.mailbox.link_id,lease.mailbox.sync_generation).execute(&mut *tx).await.map_err(db_error)?;- Self::queue_attachment_recheck(&mut tx, lease.mailbox).await?;- tx.commit().await.map_err(db_error)+ let import_completed = Self::settle_import(&mut tx, lease.mailbox).await?;+ tx.commit().await.map_err(db_error)?;+ Ok(import_completed)}pub(super) async fn release_catalog_lease(diff --git a/crates/email/src/outbound/mailbox_pg/commands/test.rs b/crates/email/src/outbound/mailbox_pg/commands/test.rsindex d7cd37cbc6..d885734240 100644--- a/crates/email/src/outbound/mailbox_pg/commands/test.rs+++ b/crates/email/src/outbound/mailbox_pg/commands/test.rs@@ -241,6 +241,9 @@ impl MailboxGateway for UncertainProvider {> {unreachable!("commands must not fetch bodies or attachments")}+ async fn admitted_requests(&self, _: MailboxKey) -> Option<u64> {+ None+ }}impl MailboxCommandGateway for UncertainProvider {async fn organization(diff --git a/crates/email/src/outbound/mailbox_pg/credentials.rs b/crates/email/src/outbound/mailbox_pg/credentials.rsindex 0fa2ebe708..4df2aef733 100644--- a/crates/email/src/outbound/mailbox_pg/credentials.rs+++ b/crates/email/src/outbound/mailbox_pg/credentials.rs@@ -11,8 +11,11 @@ impl MailboxCredentialHealthRepository for PgMailboxSync {mailbox: MailboxAccess,health: CredentialHealth,) -> Result<(), TokenError> {- let failed = |_| TokenError::Transient {- message: "could not persist mailbox health".into(),+ let failed = |error: sqlx::Error| TokenError::Transient {+ message: format!(+ "could not persist mailbox health: {}",+ DatabaseFailure::from(&error)+ ),};let mut tx = self.db.begin().await.map_err(failed)?;let current = sqlx::query!("SELECT sync_generation,needs_reauth FROM email_links WHERE id = $1 AND grant_generation = $2 AND is_sync_active AND provider = 'OUTLOOK' AND sync_generation = $3 FOR UPDATE",mailbox.link_id,mailbox.grant_generation,mailbox.sync_generation)diff --git a/crates/email/src/outbound/mailbox_pg/ingest.rs b/crates/email/src/outbound/mailbox_pg/ingest.rsindex c155793645..c75f070638 100644--- a/crates/email/src/outbound/mailbox_pg/ingest.rs+++ b/crates/email/src/outbound/mailbox_pg/ingest.rs@@ -34,7 +34,7 @@ impl MailboxIngest for PgMailboxSync {addresses,).await- .map_err(|_| MailboxError::Persistence)?;+ .map_err(wrapped_db_error)?;let mut tx = self.db.begin().await.map_err(db_error)?;let active = sqlx::query!(r#"SELECT id FROM email_links WHERE id = $1 AND sync_generation = $2 AND grant_generation = $3@@ -205,7 +205,7 @@ impl MailboxIngest for PgMailboxSync {None,).await- .map_err(|_| MailboxError::Persistence)?;+ .map_err(wrapped_db_error)?;}let mut state_json = serde_json::to_value(&state).map_err(|_| MailboxError::Persistence)?;if !message.is_sent && shadow.as_ref().is_some_and(|s| s.delete_requested) {@@ -247,14 +247,14 @@ impl MailboxIngest for PgMailboxSync {.map_err(db_error)?;email_db_client::threads::update::sync_thread_calendar_flag(&mut tx, thread_id).await- .map_err(|_| MailboxError::Persistence)?;+ .map_err(wrapped_db_error)?;email_db_client::messages::replying_to_id::update_thread_messages_replying_to(&mut tx,thread_id,lease.mailbox.link_id,).await- .map_err(|_| MailboxError::Persistence)?;+ .map_err(wrapped_db_error)?;let payload = serde_json::json!({"kind":"message", "message_id":message_id,"thread_id":thread_id,"provider_id":lease.provider_id.as_str(),"is_import":lease.is_import,diff --git a/crates/email/src/outbound/mailbox_pg/lifecycle.rs b/crates/email/src/outbound/mailbox_pg/lifecycle.rsindex 46b0dbdc43..c4b8211ffe 100644--- a/crates/email/src/outbound/mailbox_pg/lifecycle.rs+++ b/crates/email/src/outbound/mailbox_pg/lifecycle.rs@@ -292,7 +292,8 @@ impl InboxLifecycleRepository for PgMailboxSync {.map_or_else(macro_uuid::generate_uuid_v7, |stream| stream.id);if !in_progress || stream.is_none() {sqlx::query!("INSERT INTO email_sync_streams(id,link_id,generation,kind,scope_id) VALUES ($1,$2,$3,'folder_catalog','mail') ON CONFLICT DO NOTHING",run_id,link,current.sync_generation).execute(&mut *tx).await.map_err(failed)?;- sqlx::query!(r#"UPDATE email_sync_streams SET position = NULL,initial_complete = false, attachments_rechecked = false,scan_id = gen_random_uuid(),next_run_at = now(),fence = fence + 1,lease_id = NULL,lease_until = NULL+ sqlx::query!(r#"UPDATE email_sync_streams SET position = NULL,initial_complete = false, attachments_rechecked = false,scan_id = gen_random_uuid(),next_run_at = now(),fence = fence + 1,lease_id = NULL,lease_until = NULL,+ import_started_at = now(),import_completed_at = NULLWHERE link_id = $1 AND generation = $2 AND kind IN ('folder_catalog','mail_folder','contacts_catalog','contacts_profile','contacts')"#,link,current.sync_generation).execute(&mut *tx).await.map_err(failed)?;sqlx::query!(r#"INSERT INTO email_message_reconciliation(link_id,generation,provider_id,is_import)SELECT link_id,$2,provider_id,true FROM email_messages WHERE link_id = $1 AND provider_id IS NOT NULL@@ -305,6 +306,9 @@ impl InboxLifecycleRepository for PgMailboxSync {}};tx.commit().await.map_err(failed)?;+ if current.provider == UserProvider::Outlook && !result.already_in_progress {+ crate::domain::mailbox::progress::log_import_started(link, current.sync_generation);+ }Ok(result)}@@ -384,8 +388,9 @@ async fn rebind_work() -> Result<(), InboxLifecycleError> {sqlx::query!("UPDATE email_mailbox_watch_work SET next_run_at=now(),revision=revision+1 WHERE link_id=$1",link).execute(&mut **tx).await.map_err(failed)?;- sqlx::query!(r#"INSERT INTO email_sync_streams(id,link_id,generation,kind,scope_id,position,initial_complete,scan_id)- SELECT gen_random_uuid(),link_id,$3,kind,scope_id,position,initial_complete,scan_id FROM email_sync_streams WHERE link_id = $1 AND generation = $2+ // The import run continues in the new generation, so it keeps its stamps.+ sqlx::query!(r#"INSERT INTO email_sync_streams(id,link_id,generation,kind,scope_id,position,initial_complete,scan_id,import_started_at,import_completed_at)+ SELECT gen_random_uuid(),link_id,$3,kind,scope_id,position,initial_complete,scan_id,import_started_at,import_completed_at FROM email_sync_streams WHERE link_id = $1 AND generation = $2ON CONFLICT DO NOTHING"#,link,old,new).execute(&mut **tx).await.map_err(failed)?;sqlx::query!(r#"INSERT INTO email_message_reconciliation(link_id,generation,provider_id,is_import)SELECT link_id,$3,provider_id,is_import FROM email_message_reconciliation WHERE link_id = $1 AND generation = $2 ON CONFLICT DO NOTHING"#,link,old,new).execute(&mut **tx).await.map_err(failed)?;diff --git a/crates/email/src/outbound/mailbox_pg/lifecycle/test.rs b/crates/email/src/outbound/mailbox_pg/lifecycle/test.rsindex c3257af318..77dd576f6c 100644--- a/crates/email/src/outbound/mailbox_pg/lifecycle/test.rs+++ b/crates/email/src/outbound/mailbox_pg/lifecycle/test.rs@@ -66,6 +66,13 @@ async fn leaving_shared_custodian_rebinds_work_and_rejects_old_health_even_with_db: PgPool,) {let (repo, link, replacement) = shared(&db).await;+ sqlx::query!(+ "UPDATE email_sync_streams SET import_started_at = now() - interval '1 hour', import_completed_at = now() - interval '5 minutes' WHERE link_id = $1",+ link+ )+ .execute(&db)+ .await+ .unwrap();let command = macro_uuid::generate_uuid_v7();sqlx::query!(r#"INSERT INTO email_mailbox_commands(id,link_id,generation,actor_id,intent,status)@@ -110,15 +117,18 @@ async fn leaving_shared_custodian_rebinds_work_and_rejects_old_health_even_with_.await,Err(TokenError::ReauthRequired)));- assert_eq!(- sqlx::query_scalar!(- "SELECT count(*) FROM email_sync_streams WHERE link_id = $1 AND generation = 2",- link- )- .fetch_one(&db)- .await- .unwrap(),- Some(1)+ let rebound_stream = sqlx::query!(+ r#"SELECT import_started_at < now() - interval '30 minutes' AS "started_before!", import_completed_at IS NOT NULL AS "completed!"+ FROM email_sync_streams WHERE link_id = $1 AND generation = 2"#,+ link+ )+ .fetch_all(&db)+ .await+ .unwrap();+ assert_eq!(rebound_stream.len(), 1);+ assert!(+ rebound_stream[0].started_before && rebound_stream[0].completed,+ "the import run continues in the new generation, so it must not report completion again");assert_eq!(sqlx::query_scalar!(@@ -275,6 +285,52 @@ async fn outlook_resync_fences_page_workers_and_never_creates_a_gmail_backfill(d);}+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn outlook_resync_starts_a_new_import_run_once(db: PgPool) {+ let telemetry = crate::telemetry_capture::Telemetry::default();+ let _guard = telemetry.install();+ let (repo, link, _) = shared(&db).await;+ sqlx::query!(+ r#"INSERT INTO email_sync_streams (id,link_id,generation,kind,scope_id,initial_complete,import_started_at,import_completed_at)+ VALUES ($1,$2,1,'folder_catalog','mail',true,now() - interval '1 day',now() - interval '23 hours')"#,+ macro_uuid::generate_uuid_v7(),+ link+ )+ .execute(&db)+ .await+ .unwrap();+ sqlx::query!(+ "UPDATE email_sync_streams SET initial_complete = true WHERE link_id = $1",+ link+ )+ .execute(&db)+ .await+ .unwrap();+ let service = InboxLifecycleService::new(repo.clone(), Effects);++ service.resync(&actor("remaining"), link).await.unwrap();+ let again = service.resync(&actor("remaining"), link).await.unwrap();++ assert!(again.already_in_progress);+ let started = telemetry.events("outlook import started");+ let [(level, fields)] = started.as_slice() else {+ panic!("expected one start, got {started:?}");+ };+ assert_eq!(*level, tracing::Level::INFO);+ assert_eq!(fields["link_id"], link.to_string());+ assert_eq!(fields["sync_generation"], "1");+ let run = sqlx::query!(+ r#"SELECT import_started_at > now() - interval '1 minute' AS "restarted!", import_completed_at+ FROM email_sync_streams WHERE link_id = $1 AND kind = 'folder_catalog'"#,+ link+ )+ .fetch_one(&db)+ .await+ .unwrap();+ assert!(run.restarted);+ assert!(run.import_completed_at.is_none());+}+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]async fn gmail_resync_commits_the_backfill_with_durable_publication(db: PgPool) {let (repo, link, _) = shared(&db).await;diff --git a/crates/email/src/outbound/mailbox_pg/progress.rs b/crates/email/src/outbound/mailbox_pg/progress.rsnew file mode 100644index 0000000000..ef74ddc486--- /dev/null+++ b/crates/email/src/outbound/mailbox_pg/progress.rs@@ -0,0 +1,121 @@+use super::*;+use crate::domain::mailbox::progress::*;+use std::collections::HashMap;++#[cfg(test)]+mod test;++impl MailboxProgressRepository for PgMailboxSync {+ async fn record_imports(+ &self,+ imports: &HashMap<Uuid, MailboxImports>,+ ) -> Result<(), MailboxError> {+ let mut links = Vec::with_capacity(imports.len());+ let mut imported = Vec::with_capacity(imports.len());+ let mut last_imported = Vec::with_capacity(imports.len());+ for (link_id, entry) in imports {+ links.push(*link_id);+ imported.push(i32::try_from(entry.imported).unwrap_or(i32::MAX));+ last_imported.push(entry.last_imported_at);+ }+ sqlx::query!(+ r#"+ INSERT INTO email_mailbox_import_minutes (link_id, minute, imported, last_imported_at)+ SELECT link_id, date_trunc('minute', now()), imported, last_imported_at+ FROM unnest($1::uuid[], $2::int[], $3::timestamptz[]) AS flushed(link_id, imported, last_imported_at)+ ON CONFLICT (link_id, minute) DO UPDATE SET+ imported = email_mailbox_import_minutes.imported + EXCLUDED.imported,+ last_imported_at = GREATEST(email_mailbox_import_minutes.last_imported_at, EXCLUDED.last_imported_at)+ "#,+ &links,+ &imported,+ &last_imported+ )+ .execute(&self.db)+ .await+ .map_err(db_error)?;+ Ok(())+ }++ async fn backlogs(&self) -> Result<Vec<MailboxBacklog>, MailboxError> {+ sqlx::query!(+ "DELETE FROM email_mailbox_import_minutes WHERE minute < now() - interval '1 day'"+ )+ .execute(&self.db)+ .await+ .map_err(db_error)?;+ // One aggregate per Outlook link, each read through the+ // (link_id, generation, ...) key prefixes of the current generation's+ // queue and streams and the link's newest minute counts. Old+ // generations and the message table are never read.+ let rows = sqlx::query!(+ r#"+ SELECT l.id AS "link_id!",+ l.sync_generation AS "sync_generation!",+ l.is_sync_active AS "sync_active!",+ l.needs_reauth AS "needs_reauth!",+ q.queued AS "queued!",+ q.parked AS "parked!",+ q.leased AS "leased!",+ COALESCE(floor(EXTRACT(EPOCH FROM now() - q.oldest_due)), 0)::bigint AS "oldest_waiting_seconds!",+ COALESCE(ceil(EXTRACT(EPOCH FROM q.parked_until - now())), 0)::bigint AS "parked_until_max_seconds!",+ s.incomplete AS "streams_incomplete!",+ COALESCE(recent.imported_5m, 0)::bigint AS "imported_5m!",+ floor(EXTRACT(EPOCH FROM now() - latest.last_imported_at))::bigint AS seconds_since_last_import+ FROM email_links l+ CROSS JOIN LATERAL (+ SELECT count(*) AS queued,+ count(*) FILTER (WHERE available_at > now()) AS parked,+ count(*) FILTER (WHERE lease_until > now()) AS leased,+ min(available_at) FILTER (+ WHERE available_at <= now() AND (lease_until IS NULL OR lease_until <= now())+ ) AS oldest_due,+ max(available_at) FILTER (WHERE available_at > now()) AS parked_until+ FROM email_message_reconciliation+ WHERE link_id = l.id AND generation = l.sync_generation+ ) q+ CROSS JOIN LATERAL (+ SELECT count(*) AS incomplete+ FROM email_sync_streams+ WHERE link_id = l.id AND generation = l.sync_generation+ AND kind IN ('folder_catalog', 'mail_folder') AND NOT initial_complete+ ) s+ CROSS JOIN LATERAL (+ SELECT sum(imported) AS imported_5m+ FROM email_mailbox_import_minutes+ WHERE link_id = l.id AND minute > now() - interval '5 minutes'+ ) recent+ -- A flush carries imports from at most the minute before it, so+ -- the latest import is in one of the two newest minutes.+ CROSS JOIN LATERAL (+ SELECT max(last_imported_at) AS last_imported_at+ FROM (+ SELECT last_imported_at FROM email_mailbox_import_minutes+ WHERE link_id = l.id ORDER BY minute DESC LIMIT 2+ ) newest+ ) latest+ WHERE l.provider = 'OUTLOOK' AND (l.is_sync_active OR q.queued > 0)+ "#+ )+ .fetch_all(&self.db)+ .await+ .map_err(db_error)?;+ Ok(rows+ .into_iter()+ .map(|row| MailboxBacklog {+ link_id: row.link_id,+ sync_generation: row.sync_generation,+ queued: row.queued,+ parked: row.parked,+ leased: row.leased,+ oldest_waiting_seconds: row.oldest_waiting_seconds,+ parked_until_max_seconds: row.parked_until_max_seconds,+ streams_incomplete: row.streams_incomplete,+ imported_5m: row.imported_5m,+ seconds_since_last_import: row.seconds_since_last_import,+ sync_active: row.sync_active,+ needs_reauth: row.needs_reauth,+ })+ .collect())+ }+}diff --git a/crates/email/src/outbound/mailbox_pg/progress/test.rs b/crates/email/src/outbound/mailbox_pg/progress/test.rsnew file mode 100644index 0000000000..a5a923ebfc--- /dev/null+++ b/crates/email/src/outbound/mailbox_pg/progress/test.rs@@ -0,0 +1,446 @@+use super::*;+use crate::domain::mailbox::progress::MailboxProgressService;+use crate::telemetry_capture::Telemetry;+use email_api_client::domain::models::{+ EmailApiError, GateLimit, MailboxMessage, RateLimitOrigin, StreamPosition,+};+use email_api_client::domain::request_count;+use macro_db_migrator::MACRO_DB_MIGRATIONS;+use std::collections::VecDeque;+use std::sync::Mutex;+use std::time::Duration;++/// Provider double: answers message reads in order, sending `requests` Graph+/// requests for each, and reports a fixed admitted-request count.+#[derive(Default)]+struct Provider {+ messages: Mutex<VecDeque<Result<Option<MailboxMessage>, EmailApiError>>>,+ pages: Mutex<VecDeque<Result<MailboxChangePage, EmailApiError>>>,+ requests: u32,+}++impl Provider {+ fn answering(messages: Vec<Result<Option<MailboxMessage>, EmailApiError>>) -> Self {+ Self {+ messages: Mutex::new(messages.into()),+ requests: 2,+ ..Self::default()+ }+ }+}++impl MailboxGateway for Provider {+ async fn folders(&self, _: MailboxKey) -> Result<Vec<MailFolder>, EmailApiError> {+ Ok(Vec::new())+ }+ async fn changes(+ &self,+ _: MailboxKey,+ _: &ProviderId,+ _: Option<&StreamToken>,+ ) -> Result<MailboxChangePage, EmailApiError> {+ self.pages.lock().unwrap().pop_front().unwrap()+ }+ async fn message(+ &self,+ _: MailboxKey,+ _: &ProviderId,+ _: &[MailFolder],+ ) -> Result<Option<MailboxMessage>, EmailApiError> {+ for _ in 0..self.requests {+ request_count::record_sent();+ }+ self.messages.lock().unwrap().pop_front().unwrap()+ }+ async fn admitted_requests(&self, _: MailboxKey) -> Option<u64> {+ Some(42)+ }+}++fn sync(+ repo: &PgMailboxSync,+ provider: Provider,+) -> MailboxSyncService<PgMailboxSync, Provider, PgMailboxSync> {+ MailboxSyncService::new(repo.clone(), provider, repo.clone())+}++/// `attempts` is what the row had before its next claim adds one.+async fn queue(db: &PgPool, link: Uuid, provider_id: &str, is_import: bool, attempts: i32) {+ sqlx::query!(+ "INSERT INTO email_message_reconciliation (link_id,generation,provider_id,is_import,attempts) VALUES ($1,1,$2,$3,$4)",+ link,+ provider_id,+ is_import,+ attempts+ )+ .execute(db)+ .await+ .unwrap();+}++/// A mailbox whose only folder and catalog finished enumerating.+async fn enumerated_mailbox(db: &PgPool) -> (PgMailboxSync, Uuid) {+ let (repo, link) = super::super::test::fixture(db).await;+ sqlx::query!(+ "UPDATE email_sync_streams SET initial_complete = true, attachments_rechecked = true WHERE link_id = $1",+ link+ )+ .execute(db)+ .await+ .unwrap();+ sqlx::query!(+ "INSERT INTO email_sync_streams (id,link_id,generation,kind,scope_id,initial_complete) VALUES ($1,$2,1,'folder_catalog','mail',true)",+ macro_uuid::generate_uuid_v7(),+ link+ )+ .execute(db)+ .await+ .unwrap();+ (repo, link)+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn import_completes_once_when_its_last_import_row_settles(db: PgPool) {+ let telemetry = Telemetry::default();+ let _guard = telemetry.install();+ let (repo, link) = enumerated_mailbox(&db).await;+ queue(&db, link, "first", true, 2).await;+ queue(&db, link, "second", true, 2).await;+ let service = sync(+ &repo,+ Provider::answering(vec![Ok(None), Ok(None), Ok(None)]),+ );++ assert!(service.reconcile_once().await.unwrap());+ assert!(telemetry.events("outlook import completed").is_empty());+ assert!(service.reconcile_once().await.unwrap());+ // A live change after the import must not report it again.+ queue(&db, link, "live", false, 2).await;+ assert!(service.reconcile_once().await.unwrap());++ let completed = telemetry.events("outlook import completed");+ let [(level, fields)] = completed.as_slice() else {+ panic!("expected one completion, got {completed:?}");+ };+ assert_eq!(*level, tracing::Level::INFO);+ assert_eq!(fields["link_id"], link.to_string());+ assert_eq!(fields["sync_generation"], "1");+ assert_eq!(fields["folders"], "1");+ assert_eq!(fields["messages"], "0");+ assert_eq!(fields["graph_calls"], "42");+ assert!(fields.contains_key("duration_seconds"));+ let reconciled = telemetry.spans("outlook.sync.reconcile");+ assert_eq!(reconciled.len(), 3);+ assert!(+ reconciled+ .iter()+ .all(|span| span["outlook.outcome"] == "absent")+ );+ assert_eq!(reconciled[0]["outlook.is_import"], "true");+ assert_eq!(reconciled[0]["outlook.attempts"], "3");+ assert_eq!(reconciled[0]["outlook.graph_calls"], "2");+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn workers_finishing_the_last_rows_together_report_one_completion(db: PgPool) {+ let telemetry = Telemetry::default();+ let _guard = telemetry.install();+ let (repo, link) = enumerated_mailbox(&db).await;+ queue(&db, link, "first", true, 2).await;+ queue(&db, link, "second", true, 2).await;+ let first = sync(&repo, Provider::answering(vec![Ok(None)]));+ let second = sync(&repo, Provider::answering(vec![Ok(None)]));++ let (a, b) = tokio::join!(first.reconcile_once(), second.reconcile_once());++ assert!(a.unwrap() && b.unwrap());+ assert_eq!(telemetry.events("outlook import completed").len(), 1);+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn deferrals_name_their_reason_and_gate_refusals_stay_quiet(db: PgPool) {+ let telemetry = Telemetry::default();+ let _guard = telemetry.install();+ let (repo, link) = super::super::test::fixture(&db).await;+ for (id, attempts) in [("window", 0), ("graph", 0), ("token", 0), ("moving", 0)] {+ queue(&db, link, id, true, attempts).await;+ }+ let service = sync(+ &repo,+ Provider::answering(vec![+ Err(EmailApiError::RateLimited {+ retry_after: Some(Duration::from_secs(12)),+ origin: RateLimitOrigin::Gate(GateLimit::Window),+ }),+ Err(EmailApiError::RateLimited {+ retry_after: Some(Duration::from_secs(7)),+ origin: RateLimitOrigin::Provider { concurrency: false },+ }),+ Err(EmailApiError::CredentialsUnavailable {+ message: "authentication unavailable".into(),+ }),+ Ok(None),+ ]),+ );++ for _ in 0..4 {+ assert!(service.reconcile_once().await.is_err());+ }++ let spans = telemetry.spans("outlook.sync.reconcile");+ let reasons: Vec<_> = spans+ .iter()+ .map(|span| {+ (+ span["outlook.outcome"].as_str(),+ span["outlook.defer_reason"].as_str(),+ span["outlook.defer_seconds"].as_str(),+ )+ })+ .collect();+ assert_eq!(+ reasons,+ [+ ("deferred", "gate_window", "12"),+ ("deferred", "graph_throttled", "7"),+ ("deferred", "token_unavailable", "30"),+ ("deferred", "location_unsettled", "30"),+ ]+ );+ assert!(+ spans+ .iter()+ .all(|span| !span.contains_key("otel.status_description"))+ );+ let deferrals = telemetry.events("mailbox sync work deferred");+ let levels: Vec<_> = deferrals+ .iter()+ .map(|(level, fields)| {+ (+ *level,+ fields["defer_reason"].as_str(),+ fields["link_id"].clone(),+ )+ })+ .collect();+ assert_eq!(+ levels,+ [+ (tracing::Level::DEBUG, "gate_window", link.to_string()),+ (tracing::Level::WARN, "graph_throttled", link.to_string()),+ (tracing::Level::WARN, "token_unavailable", link.to_string()),+ (tracing::Level::WARN, "location_unsettled", link.to_string()),+ ]+ );+ let parked = sqlx::query_scalar!(+ r#"SELECT count(*) AS "parked!" FROM email_message_reconciliation WHERE link_id = $1 AND available_at > now() + interval '10 seconds'"#,+ link+ )+ .fetch_one(&db)+ .await+ .unwrap();+ assert_eq!(+ parked, 3,+ "the window refusal and both 30-second deferrals are parked"+ );+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn enumeration_reports_the_page_it_committed(db: PgPool) {+ let telemetry = Telemetry::default();+ let _guard = telemetry.install();+ let (repo, link) = super::super::test::fixture(&db).await;+ let provider = Provider::default();+ provider+ .pages+ .lock()+ .unwrap()+ .push_back(Ok(super::super::test::page()));+ let service = sync(&repo, provider);++ assert!(service.enumerate_once().await.unwrap());++ let spans = telemetry.spans("outlook.sync.enumerate");+ let [span] = spans.as_slice() else {+ panic!("expected one enumerate span, got {spans:?}");+ };+ assert_eq!(span["mailbox.link_id"], link.to_string());+ assert_eq!(span["outlook.stream_kind"], "mail_folder");+ assert_eq!(span["outlook.outcome"], "imported");+ assert_eq!(span["outlook.page_items"], "4");+ assert_eq!(span["outlook.rows_queued"], "3");+ assert_eq!(span["outlook.delta_advanced"], "true");+ assert_eq!(span["outlook.folder_initial_complete"], "true");+ assert!(matches!(+ super::super::test::page().position,+ StreamPosition::Checkpoint(_)+ ));+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn persistence_errors_name_their_sqlstate(db: PgPool) {+ let failure = sqlx::query_scalar!(r#"SELECT 1 / $1::int AS "quotient""#, 0)+ .fetch_one(&db)+ .await+ .unwrap_err();++ let error = super::super::db_error(failure);++ assert_eq!(+ error.to_string(),+ "mailbox persistence is unavailable: sqlstate 22012"+ );+ assert_eq!(+ crate::domain::mailbox::telemetry::defer_reason(&error),+ "persistence"+ );+ // Projection effects that fail outside the database are reported apart.+ assert_eq!(+ crate::domain::mailbox::telemetry::defer_reason(&MailboxError::Effect),+ "effect"+ );+}++async fn outlook_link(db: &PgPool, email: &str, active: bool, needs_reauth: bool) -> Uuid {+ let link = macro_uuid::generate_uuid_v7();+ sqlx::query!(+ r#"INSERT INTO email_links (id,macro_id,fusionauth_user_id,email_address,provider,grant_generation,is_sync_active,needs_reauth)+ VALUES ($1,$2,$3,$4,'OUTLOOK',1,$5,$6)"#,+ link,+ format!("macro|{email}"),+ format!("owner-{email}"),+ email,+ active,+ needs_reauth+ )+ .execute(db)+ .await+ .unwrap();+ link+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn health_snapshot_counts_the_current_queue_streams_and_recent_imports(db: PgPool) {+ let (repo, active) = super::super::test::fixture(&db).await;+ sqlx::query!(+ r#"INSERT INTO email_message_reconciliation (link_id,generation,provider_id,is_import,available_at,lease_id,lease_until) VALUES+ ($1,1,'due',true,now() - interval '90 seconds',NULL,NULL),+ ($1,1,'parked',true,now() + interval '600 seconds',NULL,NULL),+ ($1,1,'leased',true,now() - interval '10 seconds',$2,now() + interval '60 seconds'),+ ($1,0,'old generation',true,now() - interval '1 hour',NULL,NULL)"#,+ active,+ macro_uuid::generate_uuid_v7()+ )+ .execute(&db)+ .await+ .unwrap();+ sqlx::query!(+ r#"INSERT INTO email_mailbox_import_minutes (link_id,minute,imported,last_imported_at) VALUES+ ($1,date_trunc('minute',now() - interval '2 minutes'),5,now() - interval '100 seconds'),+ ($1,date_trunc('minute',now() - interval '10 minutes'),7,now() - interval '10 minutes'),+ ($1,date_trunc('minute',now() - interval '2 days'),9,now() - interval '2 days')"#,+ active+ )+ .execute(&db)+ .await+ .unwrap();+ let paused = outlook_link(&db, "[email protected]", false, true).await;+ queue(&db, paused, "waiting", true, 0).await;+ outlook_link(&db, "[email protected]", false, false).await;+ sqlx::query!(+ "INSERT INTO email_links (id,macro_id,fusionauth_user_id,email_address,provider) VALUES ($1,'macro|[email protected]','gmail-owner','[email protected]','GMAIL')",+ macro_uuid::generate_uuid_v7()+ )+ .execute(&db)+ .await+ .unwrap();++ let backlogs = repo.backlogs().await.unwrap();++ assert_eq!(backlogs.len(), 2, "{backlogs:?}");+ let active_backlog = backlogs.iter().find(|b| b.link_id == active).unwrap();+ assert_eq!(+ (+ active_backlog.queued,+ active_backlog.parked,+ active_backlog.leased,+ active_backlog.streams_incomplete,+ active_backlog.imported_5m,+ active_backlog.sync_active,+ active_backlog.needs_reauth,+ ),+ (3, 1, 1, 1, 5, true, false)+ );+ assert!((90..=92).contains(&active_backlog.oldest_waiting_seconds));+ assert!((598..=600).contains(&active_backlog.parked_until_max_seconds));+ assert!(matches!(+ active_backlog.seconds_since_last_import,+ Some(100..=102)+ ));+ let paused_backlog = backlogs.iter().find(|b| b.link_id == paused).unwrap();+ assert_eq!(+ (+ paused_backlog.queued,+ paused_backlog.parked,+ paused_backlog.oldest_waiting_seconds,+ paused_backlog.parked_until_max_seconds,+ paused_backlog.imported_5m,+ paused_backlog.seconds_since_last_import,+ paused_backlog.sync_active,+ paused_backlog.needs_reauth,+ ),+ (1, 0, 0, 0, 0, None, false, true)+ );+ let retained = sqlx::query_scalar!(+ r#"SELECT count(*) AS "rows!" FROM email_mailbox_import_minutes WHERE link_id = $1"#,+ active+ )+ .fetch_one(&db)+ .await+ .unwrap();+ assert_eq!(retained, 2, "counts older than a day are dropped");+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn import_counts_from_several_workers_add_up_in_the_snapshot(db: PgPool) {+ let telemetry = Telemetry::default();+ let _guard = telemetry.install();+ let (repo, link) = super::super::test::fixture(&db).await;+ let first = ImportTally::default();+ let second = ImportTally::default();+ for _ in 0..3 {+ first.record(link, chrono::Utc::now());+ }+ second.record(link, chrono::Utc::now());++ MailboxProgressService::new(repo.clone(), first)+ .flush()+ .await+ .unwrap();+ MailboxProgressService::new(repo.clone(), second.clone())+ .flush()+ .await+ .unwrap();+ // A flush with nothing new writes nothing.+ MailboxProgressService::new(repo.clone(), second)+ .flush()+ .await+ .unwrap();+ let reported = MailboxProgressService::new(repo.clone(), ImportTally::default())+ .report()+ .await+ .unwrap();++ assert_eq!(reported, 1);+ let snapshots = telemetry.events("outlook mailbox health");+ let [(level, fields)] = snapshots.as_slice() else {+ panic!("expected one snapshot, got {snapshots:?}");+ };+ assert_eq!(*level, tracing::Level::INFO);+ assert_eq!(fields["link_id"], link.to_string());+ assert_eq!(fields["imported_5m"], "4");+ assert_eq!(fields["queued"], "0");+ assert_eq!(fields["oldest_waiting_seconds"], "0");+ assert_eq!(fields["sync_active"], "true");+ assert!(fields.contains_key("seconds_since_last_import"));+}diff --git a/crates/email/src/outbound/mailbox_pg/projection.rs b/crates/email/src/outbound/mailbox_pg/projection.rsindex 842f14ced5..3f040a674b 100644--- a/crates/email/src/outbound/mailbox_pg/projection.rs+++ b/crates/email/src/outbound/mailbox_pg/projection.rs@@ -28,7 +28,7 @@ impl MailboxProjectionRepository for PgMailboxSync {)UPDATE email_projection_outbox o SET lease_id = $1,lease_until = now() + interval '3 minutes',attempts = attempts + 1FROM candidate c,email_links l WHERE o.id = c.id AND l.id = o.link_id- RETURNING o.id,o.link_id,o.generation,o.payload,l.sync_generation,l.grant_generation+ RETURNING o.id,o.link_id,o.generation,o.payload,o.attempts,l.sync_generation,l.grant_generation"#,lease_id).fetch_optional(&self.db).await.map_err(db_error)?;row.map(|row| {Ok(ProjectionLease {@@ -42,6 +42,7 @@ impl MailboxProjectionRepository for PgMailboxSync {event: serde_json::from_value(row.payload).map_err(|_| MailboxError::Persistence)?,from_current_generation: row.generation == row.sync_generation,+ attempts: row.attempts,})}).transpose()@@ -53,7 +54,7 @@ impl MailboxProjectionRepository for PgMailboxSync {) -> Result<Option<ProjectionContext>, MailboxError> {let link = email_db_client::links::get::fetch_link_by_id(&self.db, lease.mailbox.link_id).await- .map_err(|_| MailboxError::Persistence)?;+ .map_err(wrapped_db_error)?;let Some(link) = link.filter(|link| link.is_sync_active) else {return Ok(None);};@@ -89,22 +90,26 @@ impl MailboxProjectionRepository for PgMailboxSync {async fn finish_projection(&self,lease: &ProjectionLease,- success: bool,+ retry_after_seconds: Option<u32>,) -> Result<(), MailboxError> {- if success {- sqlx::query!("DELETE FROM email_projection_outbox WHERE id = $1 AND lease_id = $2 AND lease_until > now()",lease.id,lease.lease_id)- .execute(&self.db).await.map_err(db_error)?;- } else {- sqlx::query!(- r#"UPDATE email_projection_outbox SET lease_id = NULL,lease_until = NULL,- available_at = now() + make_interval(secs => LEAST(600,30 * attempts))- WHERE id = $1 AND lease_id = $2"#,- lease.id,- lease.lease_id- )- .execute(&self.db)- .await- .map_err(db_error)?;+ match retry_after_seconds {+ None => {+ sqlx::query!("DELETE FROM email_projection_outbox WHERE id = $1 AND lease_id = $2 AND lease_until > now()",lease.id,lease.lease_id)+ .execute(&self.db).await.map_err(db_error)?;+ }+ Some(delay) => {+ sqlx::query!(+ r#"UPDATE email_projection_outbox SET lease_id = NULL,lease_until = NULL,+ available_at = now() + make_interval(secs => $3)+ WHERE id = $1 AND lease_id = $2"#,+ lease.id,+ lease.lease_id,+ f64::from(delay)+ )+ .execute(&self.db)+ .await+ .map_err(db_error)?;+ }}Ok(())}@@ -141,7 +146,7 @@ impl PgMailboxSync {vec![db_message],).await- .map_err(|_| MailboxError::Persistence)?+ .map_err(wrapped_db_error)?.pop()else {return Ok(None);diff --git a/crates/email/src/outbound/mailbox_pg/test.rs b/crates/email/src/outbound/mailbox_pg/test.rsindex 2dd3383d74..a94742ab32 100644--- a/crates/email/src/outbound/mailbox_pg/test.rs+++ b/crates/email/src/outbound/mailbox_pg/test.rs@@ -385,7 +385,7 @@ async fn projection_ack_keeps_version_watermark_and_exposes_normalized_state(db:);assert!(message.is_present);repo.renew_projection(&event).await.unwrap();- repo.finish_projection(&event, true).await.unwrap();+ repo.finish_projection(&event, None).await.unwrap();repo.ingest(&work,snapshot(link, work.provider_id.as_str(), state, "v1"),@@ -429,7 +429,7 @@ async fn projection_crash_reclaims_work_and_old_worker_cannot_ack_successor(db:repo.renew_projection(&old).await,Err(MailboxError::Stale)));- repo.finish_projection(&old, true).await.unwrap();+ repo.finish_projection(&old, None).await.unwrap();assert_eq!(sqlx::query_scalar!("SELECT count(*) FROM email_projection_outbox WHERE id = $1",@@ -440,7 +440,7 @@ async fn projection_crash_reclaims_work_and_old_worker_cannot_ack_successor(db:.unwrap(),Some(1));- repo.finish_projection(&successor, true).await.unwrap();+ repo.finish_projection(&successor, None).await.unwrap();}#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]diff --git a/crates/email/src/telemetry_capture.rs b/crates/email/src/telemetry_capture.rsnew file mode 100644index 0000000000..8ac4377ba6--- /dev/null+++ b/crates/email/src/telemetry_capture.rs@@ -0,0 +1,97 @@+//! Records closed spans and events so tests can assert on telemetry fields.++use std::collections::HashMap;+use std::sync::{Arc, Mutex};+use tracing::field::{Field, Visit};+use tracing::span::{Attributes, Id, Record};+use tracing_subscriber::layer::{Context, SubscriberExt};+use tracing_subscriber::registry::LookupSpan;++pub(crate) type Fields = HashMap<&'static str, String>;++#[derive(Default)]+struct Values(Fields);++impl Visit for Values {+ fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) {+ self.0.insert(field.name(), format!("{value:?}"));+ }+ fn record_str(&mut self, field: &Field, value: &str) {+ self.0.insert(field.name(), value.to_owned());+ }+}++#[derive(Default)]+struct Captured {+ spans: Vec<(&'static str, Fields)>,+ events: Vec<(tracing::Level, Fields)>,+}++/// Install with [`Telemetry::install`] for the current thread.+#[derive(Clone, Default)]+pub(crate) struct Telemetry(Arc<Mutex<Captured>>);++impl Telemetry {+ pub(crate) fn install(&self) -> tracing::subscriber::DefaultGuard {+ let guard =+ tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()));+ // A callsite another test thread registered while this subscriber was+ // being installed can keep a cached "never" interest; recompute it.+ tracing::callsite::rebuild_interest_cache();+ guard+ }++ /// Closed spans with this name, oldest first.+ pub(crate) fn spans(&self, name: &str) -> Vec<Fields> {+ let captured = self.0.lock().unwrap();+ captured+ .spans+ .iter()+ .filter(|(span, _)| *span == name)+ .map(|(_, fields)| fields.clone())+ .collect()+ }++ /// Events whose message is `message`, with their level.+ pub(crate) fn events(&self, message: &str) -> Vec<(tracing::Level, Fields)> {+ let captured = self.0.lock().unwrap();+ captured+ .events+ .iter()+ .filter(|(_, fields)| fields.get("message").is_some_and(|text| text == message))+ .cloned()+ .collect()+ }+}++impl<S: tracing::Subscriber + for<'a> LookupSpan<'a>> tracing_subscriber::Layer<S> for Telemetry {+ fn on_new_span(&self, attrs: &Attributes<'_>, id: &Id, ctx: Context<'_, S>) {+ let mut values = Values::default();+ attrs.record(&mut values);+ if let Some(span) = ctx.span(id) {+ span.extensions_mut().insert(values);+ }+ }+ fn on_record(&self, id: &Id, record: &Record<'_>, ctx: Context<'_, S>) {+ if let Some(span) = ctx.span(id)+ && let Some(values) = span.extensions_mut().get_mut::<Values>()+ {+ record.record(values);+ }+ }+ fn on_event(&self, event: &tracing::Event<'_>, _ctx: Context<'_, S>) {+ let mut values = Values::default();+ event.record(&mut values);+ self.0+ .lock()+ .unwrap()+ .events+ .push((*event.metadata().level(), values.0));+ }+ fn on_close(&self, id: Id, ctx: Context<'_, S>) {+ if let Some(span) = ctx.span(&id) {+ let values = span.extensions_mut().remove::<Values>().unwrap_or_default();+ self.0.lock().unwrap().spans.push((span.name(), values.0));+ }+ }+}diff --git a/crates/email_api_client/AGENTS.md b/crates/email_api_client/AGENTS.mdindex ee8899c494..c50a4aae75 100644--- a/crates/email_api_client/AGENTS.md+++ b/crates/email_api_client/AGENTS.md@@ -22,6 +22,9 @@ Outlook (Microsoft Graph) outbound adapters.which owns request headers, provider error mapping, write-failure logs,and one `outlook.graph_request` span per attempt (with an`outlook.graph_admission` child for the per-mailbox gate wait).+- `src/outbound/microsoft_gate.rs`: the Redis admission every Graph request+ passes (window, concurrency, cooldown), which also counts admitted requests+ per sync generation.## Scope@@ -90,6 +93,18 @@ Outlook (Microsoft Graph) outbound adapters.through `request`/`conditional_request` rather than the raw client. Use`graph_request` only when a caller must branch on the Graph status, errorcode, or request id; the others drop them.+- `outlook.graph_request` is the Graph telemetry contract the Outlook sync+ monitors in `infra/stacks/monitoring` read. Every attempt records+ `outlook.throttle_origin` (`gate`, `graph` or `none`); a gate refusal+ (`RateLimitOrigin::Gate` with its `GateLimit`) sent nothing, so it is not+ a span error and logs at DEBUG. The span sets `operation.name` because+ Datadog files a client span with an HTTP method as `http.client.request`,+ and its `url.template` comes from `outlook/route.rs`, which keeps only+ known Graph path words so ids and user-chosen names never reach it. Record+ numeric fields as `i64`; unsigned values reach the collector as strings.+- `domain::request_count::counted` counts the Graph requests one unit of work+ sends (the transport calls `record_sent`); Outlook token failures surface+ as `EmailApiError::CredentialsUnavailable`, apart from Graph failures.- Same-mailbox Outlook replies use `createReply`. When Graph refuses it with400 `ErrorInvalidOperation`, `create_draft` creates a standalone draft thatcarries the same correlation and RFC reply headers.diff --git a/crates/email_api_client/Cargo.toml b/crates/email_api_client/Cargo.tomlindex cdc54b0dc7..73c2eff57a 100644--- a/crates/email_api_client/Cargo.toml+++ b/crates/email_api_client/Cargo.toml@@ -6,7 +6,7 @@ version = "0.1.0"[features]default = []-ports = []+ports = ["dep:tokio"]outbound-gmail = ["ports","dep:base64",@@ -44,6 +44,7 @@ serde_json = { workspace = true, optional = true }reqwest = { workspace = true, optional = true }url = { workspace = true, optional = true }thiserror = { workspace = true }+tokio = { workspace = true, optional = true }tracing = { workspace = true }uuid = { workspace = true }workspace-hack = { version = "0.1", path = "../workspace-hack" }@@ -54,4 +55,5 @@ macro_db_migrator = { path = "../macro_db_migrator" }macro_event_broker = { path = "../macro_event_broker", default-features = false }sqlx = { workspace = true }tokio = { workspace = true }+tracing-subscriber = { workspace = true }wiremock = { workspace = true }diff --git a/crates/email_api_client/src/domain/mod.rs b/crates/email_api_client/src/domain/mod.rsindex 1ff795a668..ae2440313d 100644--- a/crates/email_api_client/src/domain/mod.rs+++ b/crates/email_api_client/src/domain/mod.rs@@ -10,3 +10,7 @@ pub mod ports;/// Provider-neutral email API orchestration.#[cfg(feature = "ports")]pub mod service;++/// Counts provider requests made inside one unit of work.+#[cfg(feature = "ports")]+pub mod request_count;diff --git a/crates/email_api_client/src/domain/models/error.rs b/crates/email_api_client/src/domain/models/error.rsindex c0722f13c8..baa234eb52 100644--- a/crates/email_api_client/src/domain/models/error.rs+++ b/crates/email_api_client/src/domain/models/error.rs@@ -48,6 +48,13 @@ pub enum EmailApiError {/// A sanitized diagnostic suitable for logs.message: String,},+ /// No access token could be obtained right now, so no provider request was+ /// made. Retrying may succeed.+ #[error("email provider credentials are temporarily unavailable: {message}")]+ CredentialsUnavailable {+ /// A sanitized diagnostic suitable for logs.+ message: String,+ },/// A deterministic failure that should not be retried unchanged.#[error("permanent email provider failure: {message}")]Permanent {@@ -66,7 +73,10 @@ pub enum EmailApiError {impl EmailApiError {/// Returns whether retrying the same operation may succeed.pub fn is_transient(&self) -> bool {- matches!(self, Self::RateLimited { .. } | Self::Transient { .. })+ matches!(+ self,+ Self::RateLimited { .. } | Self::Transient { .. } | Self::CredentialsUnavailable { .. }+ )}}@@ -79,6 +89,9 @@ impl EmailApiError {pub enum RateLimitOrigin {/// Refused by local quota policy before any provider call was made.Local,+ /// Refused by the shared per-mailbox Microsoft request gate before any+ /// provider call was made, naming the limit that refused it.+ Gate(GateLimit),/// The provider itself throttled the request.Provider {/// The provider refused because the mailbox had too many requests in@@ -87,6 +100,17 @@ pub enum RateLimitOrigin {},}+/// The per-mailbox gate limit that refused a request.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum GateLimit {+ /// The mailbox spent its request allowance for the current window.+ Window,+ /// The mailbox already has its maximum number of requests in flight.+ Concurrency,+ /// The mailbox is cooling down after the provider throttled it.+ Cooldown,+}+impl From<RateLimitRefusal> for EmailApiError {fn from(refusal: RateLimitRefusal) -> Self {Self::RateLimited {diff --git a/crates/email_api_client/src/domain/models/mod.rs b/crates/email_api_client/src/domain/models/mod.rsindex 0ec5668b95..12f6abcee7 100644--- a/crates/email_api_client/src/domain/models/mod.rs+++ b/crates/email_api_client/src/domain/models/mod.rs@@ -16,7 +16,7 @@ pub use address_book::*;pub use auth::{AccessToken, TokenError, TokenFreshness};pub use changes::{ChangeBatch, ChangePage, InboxChanges, SyncCursor};pub use draft::*;-pub use error::{EmailApiError, RateLimitOrigin, RateLimitRefusal};+pub use error::{EmailApiError, GateLimit, RateLimitOrigin, RateLimitRefusal};pub use mailbox::{CalendarPart, ContactPage, MessageWithCalendarParts, ProviderSubscription, ThreadListPage,};diff --git a/crates/email_api_client/src/domain/ports/mailbox.rs b/crates/email_api_client/src/domain/ports/mailbox.rsindex c9ed8aa5af..996f422a55 100644--- a/crates/email_api_client/src/domain/ports/mailbox.rs+++ b/crates/email_api_client/src/domain/ports/mailbox.rs@@ -233,6 +233,12 @@ pub trait MailboxDraftClient: Send + Sync + 'static {bytes: &[u8],) -> impl Future<Output = Result<crate::domain::models::UploadProgress, EmailApiError>> + Send;}+/// Reads how many requests a mailbox's request gate admitted.+pub trait MailboxRequestCount: Send + Sync + 'static {+ /// Requests admitted for the mailbox's sync generation, when known.+ fn admitted_requests(&self, mailbox: MailboxAccess)+ -> impl Future<Output = Option<u64>> + Send;+}/// Binds request accounting to the same mailbox as credential acquisition.pub trait ScopedMailboxRepository: Clone + Send + Sync + 'static {/// Return a request-scoped clone; credentials remain supplied separately.@@ -254,6 +260,12 @@ pub trait MailboxRequestGate: Send + Sync + 'static {permit: uuid::Uuid,retry_after: Option<std::time::Duration>,) -> std::pin::Pin<Box<dyn Future<Output = ()> + Send + '_>>;+ /// Requests admitted for this mailbox's sync generation, or `None` when+ /// the count is unavailable.+ fn admitted(+ &self,+ mailbox: MailboxAccess,+ ) -> std::pin::Pin<Box<dyn Future<Output = Option<u64>> + Send + '_>>;}/// Settings capabilities which require explicit rule ownership and bounded cleanup.diff --git a/crates/email_api_client/src/domain/request_count.rs b/crates/email_api_client/src/domain/request_count.rsnew file mode 100644index 0000000000..9777db6647--- /dev/null+++ b/crates/email_api_client/src/domain/request_count.rs@@ -0,0 +1,29 @@+//! Counts the provider requests one unit of work sends, such as the Graph+//! calls that reconcile one message, without threading a counter through+//! every port.++use std::cell::Cell;+use std::future::Future;++#[cfg(test)]+mod test;++tokio::task_local! {+ static SENT: Cell<u32>;+}++/// Run `operation`, returning its output and how many provider requests it+/// sent. Requests from tasks it spawns are not counted.+pub async fn counted<F: Future>(operation: F) -> (F::Output, u32) {+ SENT.scope(Cell::new(0), async {+ let output = operation.await;+ (output, SENT.with(Cell::get))+ })+ .await+}++/// Count one request actually sent to the provider. Outside [`counted`] this+/// does nothing.+pub fn record_sent() {+ let _ = SENT.try_with(|sent| sent.set(sent.get().saturating_add(1)));+}diff --git a/crates/email_api_client/src/domain/request_count/test.rs b/crates/email_api_client/src/domain/request_count/test.rsnew file mode 100644index 0000000000..9fea8696ed--- /dev/null+++ b/crates/email_api_client/src/domain/request_count/test.rs@@ -0,0 +1,18 @@+use super::{counted, record_sent};++#[tokio::test]+async fn counts_only_requests_inside_the_scope() {+ record_sent();+ let ((), outer) = counted(async {+ record_sent();+ let ((), inner) = counted(async {+ record_sent();+ record_sent();+ })+ .await;+ assert_eq!(inner, 2);+ record_sent();+ })+ .await;+ assert_eq!(outer, 2);+}diff --git a/crates/email_api_client/src/domain/service.rs b/crates/email_api_client/src/domain/service.rsindex b304de7001..0df1a1b8ed 100644--- a/crates/email_api_client/src/domain/service.rs+++ b/crates/email_api_client/src/domain/service.rs@@ -208,7 +208,7 @@ wherefn log_operation_error(error: &EmailApiError) {match error {EmailApiError::RateLimited {- origin: RateLimitOrigin::Local,+ origin: RateLimitOrigin::Local | RateLimitOrigin::Gate(_),..} => tracing::debug!(error = %error),EmailApiError::RateLimited { origin, .. } => {diff --git a/crates/email_api_client/src/domain/service/mailbox.rs b/crates/email_api_client/src/domain/service/mailbox.rsindex 64f7cba865..44f003a570 100644--- a/crates/email_api_client/src/domain/service/mailbox.rs+++ b/crates/email_api_client/src/domain/service/mailbox.rs@@ -41,7 +41,21 @@ impl<R, T: MailboxTokenSource, L: ProviderRateLimiter> MailboxApiService<R, T, Lself.tokens.access_token(mailbox, freshness).await- .map_err(super::map_token_error)+ .map_err(|error| match error {+ // No provider request was made, which callers report apart+ // from provider failures.+ TokenError::Transient { message } => {+ EmailApiError::CredentialsUnavailable { message }+ }+ error => super::map_token_error(error),+ })+ }+}++impl<R: MailboxRequestCount, T, L> MailboxApiService<R, T, L> {+ /// Requests the mailbox's gate admitted in its current sync generation.+ pub async fn admitted_requests(&self, mailbox: MailboxAccess) -> Option<u64> {+ self.repository.admitted_requests(mailbox).await}}diff --git a/crates/email_api_client/src/outbound/microsoft_gate.rs b/crates/email_api_client/src/outbound/microsoft_gate.rsindex 7f146e120b..7ede00499a 100644--- a/crates/email_api_client/src/outbound/microsoft_gate.rs+++ b/crates/email_api_client/src/outbound/microsoft_gate.rs@@ -1,7 +1,7 @@//! Shared admission for every Graph request, including nested pagination.use crate::domain::{- models::{EmailApiError, MailboxAccess, RateLimitOrigin},+ models::{EmailApiError, GateLimit, MailboxAccess, RateLimitOrigin},ports::MailboxRequestGate,};use redis::aio::MultiplexedConnection;@@ -19,27 +19,34 @@ const CONCURRENT_REQUESTS: u32 = 3;const WINDOW_MS: u32 = 600_000;// Greater than the Graph transport's 60-second total request timeout.const PERMIT_MS: u32 = 90_000;+// Admitted requests are counted per sync generation for import statistics.+// Long enough to outlast any import.+const REQUEST_COUNT_MS: u64 = 30 * 24 * 3_600_000;+// Returns `{limit, delay_ms}`: limit 0 admits the request, otherwise it names+// the refusing limit (1 cooldown, 2 concurrency, 3 window).const ACQUIRE: &str = r#"local clock = redis.call('TIME')local now = tonumber(clock[1]) * 1000 + math.floor(tonumber(clock[2]) / 1000)local cooldown = redis.call('PTTL', KEYS[3])-if cooldown > 0 then return cooldown end+if cooldown > 0 then return {1, cooldown} endredis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', now)redis.call('ZREMRANGEBYSCORE', KEYS[2], '-inf', now - tonumber(ARGV[4]))if redis.call('ZCARD', KEYS[1]) >= tonumber(ARGV[2]) thenlocal first = redis.call('ZRANGE', KEYS[1], 0, 0, 'WITHSCORES')- return math.max(1, tonumber(first[2]) - now)+ return {2, math.max(1, tonumber(first[2]) - now)}endif redis.call('ZCARD', KEYS[2]) >= tonumber(ARGV[3]) thenlocal first = redis.call('ZRANGE', KEYS[2], 0, 0, 'WITHSCORES')- return math.max(1, tonumber(first[2]) + tonumber(ARGV[4]) - now)+ return {3, math.max(1, tonumber(first[2]) + tonumber(ARGV[4]) - now)}endredis.call('ZADD', KEYS[1], now + tonumber(ARGV[5]), ARGV[1])redis.call('PEXPIRE', KEYS[1], ARGV[5])redis.call('ZADD', KEYS[2], now, ARGV[1])redis.call('PEXPIRE', KEYS[2], ARGV[4])-return 0+redis.call('HINCRBY', KEYS[4], ARGV[6], 1)+redis.call('PEXPIRE', KEYS[4], ARGV[7])+return {0, 0}"#;const FINISH: &str = r#"@@ -55,7 +62,7 @@ return 1#[derive(Clone)]pub struct RedisMicrosoftRequestGate(pub MultiplexedConnection);-fn keys(mailbox: MailboxAccess) -> [String; 3] {+fn keys(mailbox: MailboxAccess) -> [String; 4] {// Hash tags colocate the Lua keys in Redis Cluster. Generations share the// quota: refreshing or reconnecting must not reset Microsoft's allowance.let scope = format!("outlook:{{{}}}", mailbox.link_id);@@ -63,6 +70,7 @@ fn keys(mailbox: MailboxAccess) -> [String; 3] {format!("{scope}:inflight"),format!("{scope}:window"),format!("{scope}:cooldown"),+ format!("{scope}:requests"),]}@@ -73,28 +81,34 @@ impl MailboxRequestGate for RedisMicrosoftRequestGate {) -> Pin<Box<dyn Future<Output = Result<Uuid, EmailApiError>> + Send + '_>> {Box::pin(async move {let permit = Uuid::now_v7();- let [inflight, window, cooldown] = keys(mailbox);- let delay: u64 = redis::Script::new(ACQUIRE)+ let [inflight, window, cooldown, requests] = keys(mailbox);+ let (limit, delay): (u8, u64) = redis::Script::new(ACQUIRE).key(inflight).key(window).key(cooldown)+ .key(requests).arg(permit.to_string()).arg(CONCURRENT_REQUESTS).arg(REQUESTS_PER_WINDOW).arg(WINDOW_MS).arg(PERMIT_MS)+ .arg(mailbox.sync_generation)+ .arg(REQUEST_COUNT_MS).invoke_async(&mut self.0.clone()).await.map_err(|_| EmailApiError::Transient {message: "Microsoft request admission is unavailable".into(),})?;- if delay > 0 {- return Err(EmailApiError::RateLimited {- retry_after: Some(Duration::from_millis(delay)),- origin: RateLimitOrigin::Local,- });- }- Ok(permit)+ let limit = match limit {+ 0 => return Ok(permit),+ 1 => GateLimit::Cooldown,+ 2 => GateLimit::Concurrency,+ _ => GateLimit::Window,+ };+ Err(EmailApiError::RateLimited {+ retry_after: Some(Duration::from_millis(delay)),+ origin: RateLimitOrigin::Gate(limit),+ })})}fn finish(@@ -104,7 +118,7 @@ impl MailboxRequestGate for RedisMicrosoftRequestGate {retry_after: Option<Duration>,) -> Pin<Box<dyn Future<Output = ()> + Send + '_>> {Box::pin(async move {- let [inflight, _, cooldown] = keys(mailbox);+ let [inflight, _, cooldown, _] = keys(mailbox);let delay = retry_after.map(|delay| delay.as_millis().clamp(1, 86_400_000) as u64).unwrap_or(0);@@ -122,4 +136,23 @@ impl MailboxRequestGate for RedisMicrosoftRequestGate {}})}+ fn admitted(+ &self,+ mailbox: MailboxAccess,+ ) -> Pin<Box<dyn Future<Output = Option<u64>> + Send + '_>> {+ Box::pin(async move {+ let [.., requests] = keys(mailbox);+ let admitted: redis::RedisResult<Option<u64>> = redis::cmd("HGET")+ .arg(requests)+ .arg(mailbox.sync_generation)+ .query_async(&mut self.0.clone())+ .await;+ admitted+ .inspect_err(|error| {+ tracing::warn!(link_id = %mailbox.link_id, %error, "failed to read admitted Microsoft requests");+ })+ .ok()+ .flatten()+ })+ }}diff --git a/crates/email_api_client/src/outbound/microsoft_gate/test.rs b/crates/email_api_client/src/outbound/microsoft_gate/test.rsindex 628d141f7a..36d0fbccaa 100644--- a/crates/email_api_client/src/outbound/microsoft_gate/test.rs+++ b/crates/email_api_client/src/outbound/microsoft_gate/test.rs@@ -39,7 +39,7 @@ async fn concurrent_requests_share_limits_and_reconnect_does_not_reset_them() {assert!(matches!(gate.acquire(reconnected).await,Err(EmailApiError::RateLimited {- origin: RateLimitOrigin::Local,+ origin: RateLimitOrigin::Gate(GateLimit::Concurrency),..})));@@ -53,10 +53,20 @@ async fn concurrent_requests_share_limits_and_reconnect_does_not_reset_them() {match gate.acquire(mailbox).await {Err(EmailApiError::RateLimited {retry_after: Some(delay),- ..+ origin: RateLimitOrigin::Gate(GateLimit::Cooldown),}) => assert!(delay > Duration::from_secs(80)),_ => panic!("a shorter cooldown must not erase an existing provider Retry-After"),}+ // Four requests were admitted, all under sync generation 1.+ assert_eq!(gate.admitted(mailbox).await, Some(4));+ assert_eq!(+ gate.admitted(MailboxAccess {+ sync_generation: 2,+ ..mailbox+ })+ .await,+ None+ );cleanup(&gate, mailbox).await;}@@ -79,7 +89,7 @@ async fn request_window_survives_permit_release_and_reconnect() {.unwrap();assert!(matches!(gate.acquire(MailboxAccess { grant_generation: 2,..mailbox }).await,- Err(EmailApiError::RateLimited { retry_after: Some(delay),.. }) if delay > Duration::from_secs(590))+ Err(EmailApiError::RateLimited { retry_after: Some(delay), origin: RateLimitOrigin::Gate(GateLimit::Window) }) if delay > Duration::from_secs(590)));cleanup(&gate, mailbox).await;}diff --git a/crates/email_api_client/src/outbound/outlook.rs b/crates/email_api_client/src/outbound/outlook.rsindex 831c90d93d..e01a7548b4 100644--- a/crates/email_api_client/src/outbound/outlook.rs+++ b/crates/email_api_client/src/outbound/outlook.rs@@ -6,6 +6,7 @@ mod contacts;mod drafts;mod invitations;mod messages;+mod route;mod rules;mod sync;mod transport;@@ -22,7 +23,10 @@ use url::Url;use crate::domain::models::EmailApiError;use crate::domain::{models::MailboxAccess,- ports::{MailboxRejectedTokenRefresh, MailboxRequestGate, ScopedMailboxRepository},+ ports::{+ MailboxRejectedTokenRefresh, MailboxRequestCount, MailboxRequestGate,+ ScopedMailboxRepository,+ },};use std::sync::Arc;@@ -87,6 +91,12 @@ impl OutlookApiClientRepository {}}+impl MailboxRequestCount for OutlookApiClientRepository {+ async fn admitted_requests(&self, mailbox: MailboxAccess) -> Option<u64> {+ self.gate.as_ref()?.admitted(mailbox).await+ }+}+impl ScopedMailboxRepository for OutlookApiClientRepository {fn for_mailbox(&self, mailbox: MailboxAccess) -> Self {Self {diff --git a/crates/email_api_client/src/outbound/outlook/route.rs b/crates/email_api_client/src/outbound/outlook/route.rsnew file mode 100644index 0000000000..d779eaa72f--- /dev/null+++ b/crates/email_api_client/src/outbound/outlook/route.rs@@ -0,0 +1,81 @@+//! Low-cardinality route templates for Graph request telemetry.++#[cfg(test)]+mod test;++/// Graph path words kept in a template. Every other segment is an id or a+/// user-chosen name, such as a category, and becomes `{id}`.+const GRAPH_PATH_WORDS: &[&str] = &[+ "$value",+ "accept",+ "archive",+ "attachments",+ "calendarView",+ "calendars",+ "cancel",+ "childFolders",+ "contactFolders",+ "contacts",+ "copy",+ "createForward",+ "createReply",+ "createReplyAll",+ "createUploadSession",+ "decline",+ "deleteditems",+ "delta",+ "drafts",+ "events",+ "forward",+ "inbox",+ "instances",+ "junkemail",+ "mailFolders",+ "mailboxSettings",+ "masterCategories",+ "me",+ "messageRules",+ "messages",+ "move",+ "msgfolderroot",+ "outbox",+ "outlook",+ "photo",+ "send",+ "sentitems",+ "subscriptions",+ "tentativelyAccept",+ "users",+];++/// The request path relative to the API root, with ids replaced by `{id}`.+/// OData key segments (`mailFolders('…')`, used by delta links) become the+/// same `mailFolders/{id}` as the path form.+pub(super) fn url_template(api_root: &str, path: &str) -> String {+ let relative = path+ .strip_prefix(api_root.trim_end_matches('/'))+ .unwrap_or(path);+ let mut template = String::new();+ for segment in relative.split('/').filter(|segment| !segment.is_empty()) {+ match segment.split_once('(') {+ Some((name, _key)) => {+ push_segment(&mut template, name);+ template.push_str("/{id}");+ }+ None => push_segment(&mut template, segment),+ }+ }+ if template.is_empty() {+ template.push('/');+ }+ template+}++fn push_segment(template: &mut String, segment: &str) {+ template.push('/');+ if GRAPH_PATH_WORDS.contains(&segment) {+ template.push_str(segment);+ } else {+ template.push_str("{id}");+ }+}diff --git a/crates/email_api_client/src/outbound/outlook/route/test.rs b/crates/email_api_client/src/outbound/outlook/route/test.rsnew file mode 100644index 0000000000..f7a8936de5--- /dev/null+++ b/crates/email_api_client/src/outbound/outlook/route/test.rs@@ -0,0 +1,42 @@+use super::url_template;++#[test]+fn ids_and_odata_keys_collapse_to_one_template_per_route() {+ assert_eq!(+ url_template(+ "/v1.0/",+ "/v1.0/me/mailFolders/AQMkADAwATM3ZmYAZS1iNjYAYi05/messages/delta"+ ),+ "/me/mailFolders/{id}/messages/delta"+ );+ assert_eq!(+ url_template(+ "/v1.0/",+ "/v1.0/me/mailFolders('AQMkADAwATM3ZmYAZS1iNjYAYi05NDc0LTAwAi0wMAoALgAAA73oe0jpHIRF')/messages/delta"+ ),+ "/me/mailFolders/{id}/messages/delta"+ );+ assert_eq!(+ url_template("/v1.0/", "/v1.0/me/messages/AAMkAGI2TG93AAA=/attachments"),+ "/me/messages/{id}/attachments"+ );+ assert_eq!(+ url_template("/v1.0/", "/v1.0/me/messages/AAMk%3D/$value"),+ "/me/messages/{id}/$value"+ );+}++#[test]+fn user_chosen_names_never_reach_the_template() {+ assert_eq!(+ url_template(+ "/v1.0/",+ "/v1.0/me/outlook/masterCategories/Quarterly%20review"+ ),+ "/me/outlook/masterCategories/{id}"+ );+ assert_eq!(+ url_template("/v1.0/", "/v1.0/users/[email protected]/messages"),+ "/users/{id}/messages"+ );+}diff --git a/crates/email_api_client/src/outbound/outlook/test.rs b/crates/email_api_client/src/outbound/outlook/test.rsindex 1707ed54ad..535572d131 100644--- a/crates/email_api_client/src/outbound/outlook/test.rs+++ b/crates/email_api_client/src/outbound/outlook/test.rs@@ -403,6 +403,12 @@ async fn request_gate_covers_nested_pages_and_receives_provider_retry_after() {self.finished.lock().unwrap().push(retry_after);Box::pin(async {})}+ fn admitted(+ &self,+ _: MailboxAccess,+ ) -> Pin<Box<dyn Future<Output = Option<u64>> + Send + '_>> {+ Box::pin(async { None })+ }}let (server, mut api, token) = setup().await;let mailbox = MailboxAccess {diff --git a/crates/email_api_client/src/outbound/outlook/transport.rs b/crates/email_api_client/src/outbound/outlook/transport.rsindex d2165e79d3..07be1c7b42 100644--- a/crates/email_api_client/src/outbound/outlook/transport.rs+++ b/crates/email_api_client/src/outbound/outlook/transport.rs@@ -1,4 +1,4 @@-use std::time::{Duration, SystemTime};+use std::time::{Duration, Instant, SystemTime};use reqwest::{Method, RequestBuilder, Response, StatusCode};use serde::de::DeserializeOwned;@@ -6,9 +6,14 @@ use serde_json::Value;use tracing::Instrument;use url::Url;-use super::{IMMUTABLE_ID_PREFERENCE, OutlookApiClientRepository};+use super::{IMMUTABLE_ID_PREFERENCE, OutlookApiClientRepository, route};use crate::domain::models::{AccessToken, EmailApiError, RateLimitOrigin, StreamToken};+#[cfg(test)]+mod test;++const GRAPH_REQUEST: &str = "outlook.graph_request";+/// A failed Graph request with the response details a caller may branch on.pub(super) struct GraphFailure {pub(super) error: EmailApiError,@@ -157,19 +162,6 @@ impl OutlookApiClientRepository {/// One Graph round trip, including its wait for the mailbox gate. A replay/// with a refreshed token is its own attempt, so each keeps its status.- #[tracing::instrument(- name = "outlook.graph_request",- skip_all,- fields(- otel.kind = "client",- http.request.method = %method,- url.path = path,- link_id = self.mailbox.map(|mailbox| tracing::field::display(mailbox.link_id)),- http.response.status_code = tracing::field::Empty,- reauthorized = reauthorized,- ),- err(level = "warn")- )]async fn graph_attempt(&self,request: RequestBuilder,@@ -177,7 +169,30 @@ impl OutlookApiClientRepository {path: &str,reauthorized: bool,) -> Result<Response, GraphFailure> {- self.gated_graph_request(request).await+ let span = tracing::info_span!(+ "outlook.graph_request",+ otel.kind = "client",+ // Datadog names a client span that carries an HTTP method+ // `http.client.request`; this keeps the span findable by name.+ operation.name = GRAPH_REQUEST,+ http.request.method = %method,+ url.template = %route::url_template(self.root.path(), path),+ mailbox.link_id = self.mailbox.map(|mailbox| tracing::field::display(mailbox.link_id)),+ mailbox.sync_generation = self.mailbox.map(|mailbox| mailbox.sync_generation),+ reauthorized,+ http.response.status_code = tracing::field::Empty,+ graph.request_id = tracing::field::Empty,+ outlook.gate_wait_ms = tracing::field::Empty,+ outlook.throttle_origin = tracing::field::Empty,+ outlook.retry_after_seconds = tracing::field::Empty,+ otel.status_description = tracing::field::Empty,+ );+ let result = self+ .gated_graph_request(request)+ .instrument(span.clone())+ .await;+ record_attempt(&span, &result);+ result}/// Includes anonymous, preauthorized attachment uploads in mailbox accounting.@@ -193,17 +208,23 @@ impl OutlookApiClientRepository {async fn gated_graph_request(&self, request: RequestBuilder) -> Result<Response, GraphFailure> {let permit = if let Some(gate) = &self.gate {let mailbox = self.mailbox.ok_or_else(invalid_response)?;+ let started = Instant::now();let permit = gate.acquire(mailbox).instrument(tracing::info_span!("outlook.graph_admission",- link_id = %mailbox.link_id+ mailbox.link_id = %mailbox.link_id))- .await?;- Some((mailbox, permit))+ .await;+ tracing::Span::current().record(+ "outlook.gate_wait_ms",+ i64::try_from(started.elapsed().as_millis()).unwrap_or(i64::MAX),+ );+ Some((mailbox, permit?))} else {None};+ crate::domain::request_count::record_sent();let result = Self::send_request(request).await;if let (Some(gate), Some((mailbox, permit))) = (&self.gate, permit) {let retry_after = match &result {@@ -227,16 +248,18 @@ impl OutlookApiClientRepository {let method = request.method().clone();let response = client.execute(request).await.map_err(transport_failed)?;let status = response.status();- tracing::Span::current().record("http.response.status_code", status.as_u16());- if status.is_success() {- return Ok(response);- }- let path = response.url().path().to_owned();let request_id = response.headers().get("request-id").and_then(|v| v.to_str().ok()).map(str::to_owned);+ let span = tracing::Span::current();+ span.record("http.response.status_code", i64::from(status.as_u16()));+ span.record("graph.request_id", request_id.as_deref());+ if status.is_success() {+ return Ok(response);+ }+ let path = response.url().path().to_owned();let retry_after = response.headers().get("Retry-After")@@ -248,6 +271,9 @@ impl OutlookApiClientRepository {.map(|date| date.duration_since(SystemTime::now()).unwrap_or_default())})});+ if let Some(delay) = retry_after {+ span.record("outlook.retry_after_seconds", whole_seconds(delay));+ }let body = response.text().await.unwrap_or_default();let error: Value = serde_json::from_str(&body).unwrap_or(Value::Null);let code = error.pointer("/error/code").and_then(Value::as_str);@@ -314,6 +340,47 @@ impl OutlookApiClientRepository {}}+/// Numbers are recorded as `i64`: OpenTelemetry receives unsigned span fields+/// as strings.+///+/// Record where a failed attempt was throttled and mark real failures as span+/// errors. A gate refusal sent nothing, so it is neither an error nor a WARN;+/// the sync unit that owns the request reports the deferral.+fn record_attempt(span: &tracing::Span, result: &Result<Response, GraphFailure>) {+ let Err(failure) = result else {+ span.record("outlook.throttle_origin", "none");+ return;+ };+ // A response's Retry-After was recorded when it arrived; a gate refusal+ // has no response, so its delay is recorded here.+ let origin = match &failure.error {+ EmailApiError::RateLimited {+ origin: RateLimitOrigin::Provider { .. },+ ..+ } => "graph",+ EmailApiError::RateLimited { retry_after, .. } => {+ if let Some(delay) = retry_after {+ span.record("outlook.retry_after_seconds", whole_seconds(*delay));+ }+ "gate"+ }+ _ => "none",+ };+ span.record("outlook.throttle_origin", origin);+ if origin == "gate" {+ tracing::debug!(parent: span, error = %failure.error, "Microsoft request gate deferred a Graph request");+ } else {+ span.record(+ "otel.status_description",+ tracing::field::display(&failure.error),+ );+ }+}++fn whole_seconds(delay: Duration) -> i64 {+ i64::try_from(delay.as_millis().div_ceil(1000)).unwrap_or(i64::MAX)+}+pub(super) fn invalid_response() -> EmailApiError {EmailApiError::Permanent {message: "invalid Microsoft Graph response".into(),diff --git a/crates/email_api_client/src/outbound/outlook/transport/test.rs b/crates/email_api_client/src/outbound/outlook/transport/test.rsnew file mode 100644index 0000000000..d01edc6e4c--- /dev/null+++ b/crates/email_api_client/src/outbound/outlook/transport/test.rs@@ -0,0 +1,210 @@+use super::*;+use crate::domain::models::{GateLimit, MailboxAccess};+use crate::domain::ports::{MailboxRequestGate, ScopedMailboxRepository};+use std::collections::HashMap;+use std::future::Future;+use std::pin::Pin;+use std::sync::{Arc, Mutex};+use tracing::field::{Field, Visit};+use tracing::span::{Attributes, Id, Record};+use tracing_subscriber::layer::{Context, SubscriberExt};+use tracing_subscriber::registry::LookupSpan;+use uuid::Uuid;+use wiremock::matchers::{method, path};+use wiremock::{Mock, ResponseTemplate};++#[derive(Default)]+struct Fields(HashMap<&'static str, String>);++impl Visit for Fields {+ fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) {+ self.0.insert(field.name(), format!("{value:?}"));+ }+ fn record_str(&mut self, field: &Field, value: &str) {+ self.0.insert(field.name(), value.to_owned());+ }+}++/// Every closed span's name and the last value recorded for each field.+#[derive(Clone, Default)]+struct Spans(Arc<Mutex<Vec<(&'static str, HashMap<&'static str, String>)>>>);++impl Spans {+ fn named(&self, name: &str) -> Vec<HashMap<&'static str, String>> {+ let spans = self.0.lock().unwrap();+ spans+ .iter()+ .filter(|(span, _)| *span == name)+ .map(|(_, fields)| fields.clone())+ .collect()+ }+}++impl<S: tracing::Subscriber + for<'a> LookupSpan<'a>> tracing_subscriber::Layer<S> for Spans {+ fn on_new_span(&self, attrs: &Attributes<'_>, id: &Id, ctx: Context<'_, S>) {+ let mut fields = Fields::default();+ attrs.record(&mut fields);+ if let Some(span) = ctx.span(id) {+ span.extensions_mut().insert(fields);+ }+ }+ fn on_record(&self, id: &Id, values: &Record<'_>, ctx: Context<'_, S>) {+ if let Some(span) = ctx.span(id)+ && let Some(fields) = span.extensions_mut().get_mut::<Fields>()+ {+ values.record(fields);+ }+ }+ fn on_close(&self, id: Id, ctx: Context<'_, S>) {+ if let Some(span) = ctx.span(&id) {+ let fields = span.extensions_mut().remove::<Fields>().unwrap_or_default();+ self.0.lock().unwrap().push((span.name(), fields.0));+ }+ }+}++struct RefusingGate;++impl MailboxRequestGate for RefusingGate {+ fn acquire(+ &self,+ _: MailboxAccess,+ ) -> Pin<Box<dyn Future<Output = Result<Uuid, EmailApiError>> + Send + '_>> {+ Box::pin(async {+ Err(EmailApiError::RateLimited {+ retry_after: Some(Duration::from_millis(2_500)),+ origin: RateLimitOrigin::Gate(GateLimit::Window),+ })+ })+ }+ fn finish(+ &self,+ _: MailboxAccess,+ _: Uuid,+ _: Option<Duration>,+ ) -> Pin<Box<dyn Future<Output = ()> + Send + '_>> {+ Box::pin(async {})+ }+ fn admitted(&self, _: MailboxAccess) -> Pin<Box<dyn Future<Output = Option<u64>> + Send + '_>> {+ Box::pin(async { None })+ }+}++const MAILBOX: MailboxAccess = MailboxAccess {+ link_id: Uuid::from_u128(0x0a),+ sync_generation: 3,+ grant_generation: 1,+};++#[tokio::test]+async fn graph_retry_after_and_request_ids_are_recorded_on_every_response() {+ let spans = Spans::default();+ let _subscriber =+ tracing::subscriber::set_default(tracing_subscriber::registry().with(spans.clone()));+ // A callsite another test thread registered while this subscriber was+ // being installed can keep a cached "never" interest; recompute it.+ tracing::callsite::rebuild_interest_cache();+ let (server, api, token) = super::super::test::setup().await;+ let api = api.for_mailbox(MAILBOX);+ Mock::given(method("GET"))+ .and(path("/v1.0/me/messages/AAMkAGI2TG93AAA="))+ .respond_with(+ ResponseTemplate::new(429)+ .insert_header("Retry-After", "7")+ .insert_header("request-id", "graph-request-1"),+ )+ .mount(&server)+ .await;+ Mock::given(method("GET"))+ .and(path("/v1.0/me/messages/AAMkBusy"))+ .respond_with(ResponseTemplate::new(503).insert_header("Retry-After", "11"))+ .mount(&server)+ .await;+ Mock::given(method("GET"))+ .and(path("/v1.0/me/messages/AAMkOK"))+ .respond_with(+ ResponseTemplate::new(200)+ .insert_header("request-id", "graph-request-2")+ .set_body_json(serde_json::json!({})),+ )+ .mount(&server)+ .await;++ let throttled = api+ .get::<Value>(+ &token,+ api.endpoint(&["me", "messages", "AAMkAGI2TG93AAA="])+ .unwrap(),+ )+ .await;+ let unavailable = api+ .get::<Value>(+ &token,+ api.endpoint(&["me", "messages", "AAMkBusy"]).unwrap(),+ )+ .await;+ let succeeded = api+ .get::<Value>(&token, api.endpoint(&["me", "messages", "AAMkOK"]).unwrap())+ .await;++ assert!(matches!(throttled, Err(EmailApiError::RateLimited { .. })));+ assert!(succeeded.is_ok());+ let requests = spans.named("outlook.graph_request");+ assert!(matches!(unavailable, Err(EmailApiError::Transient { .. })));+ let [throttled, unavailable, succeeded] = requests.as_slice() else {+ panic!("expected three Graph request spans, got {requests:?}");+ };+ assert_eq!(unavailable["http.response.status_code"], "503");+ assert_eq!(unavailable["outlook.throttle_origin"], "none");+ assert_eq!(unavailable["outlook.retry_after_seconds"], "11");+ assert_eq!(throttled["operation.name"], "outlook.graph_request");+ assert_eq!(throttled["url.template"], "/me/messages/{id}");+ assert_eq!(throttled["mailbox.link_id"], MAILBOX.link_id.to_string());+ assert_eq!(throttled["http.response.status_code"], "429");+ assert_eq!(throttled["outlook.throttle_origin"], "graph");+ assert_eq!(throttled["outlook.retry_after_seconds"], "7");+ assert_eq!(throttled["graph.request_id"], "graph-request-1");+ assert!(throttled.contains_key("otel.status_description"));++ assert_eq!(succeeded["http.response.status_code"], "200");+ assert_eq!(succeeded["outlook.throttle_origin"], "none");+ assert_eq!(succeeded["graph.request_id"], "graph-request-2");+ assert!(!succeeded.contains_key("otel.status_description"));+}++#[tokio::test]+async fn a_gate_refusal_is_a_deferral_not_a_failed_request() {+ let spans = Spans::default();+ let _subscriber =+ tracing::subscriber::set_default(tracing_subscriber::registry().with(spans.clone()));+ // A callsite another test thread registered while this subscriber was+ // being installed can keep a cached "never" interest; recompute it.+ tracing::callsite::rebuild_interest_cache();+ let (_server, mut api, token) = super::super::test::setup().await;+ api.gate = Some(Arc::new(RefusingGate));+ let api = api.for_mailbox(MAILBOX);++ let refused = api+ .get::<Value>(+ &token,+ api.endpoint(&["me", "mailFolders", "inbox"]).unwrap(),+ )+ .await;++ assert!(matches!(+ refused,+ Err(EmailApiError::RateLimited {+ origin: RateLimitOrigin::Gate(GateLimit::Window),+ ..+ })+ ));+ let requests = spans.named("outlook.graph_request");+ let [refused] = requests.as_slice() else {+ panic!("expected one Graph request span, got {requests:?}");+ };+ assert_eq!(refused["outlook.throttle_origin"], "gate");+ assert_eq!(refused["outlook.retry_after_seconds"], "3");+ assert!(refused.contains_key("outlook.gate_wait_ms"));+ assert!(!refused.contains_key("http.response.status_code"));+ assert!(!refused.contains_key("otel.status_description"));+}diff --git a/crates/macro_db_client/migrations/20261103201221_outlook_import_progress.sql b/crates/macro_db_client/migrations/20261103201221_outlook_import_progress.sqlnew file mode 100644index 0000000000..979d915698--- /dev/null+++ b/crates/macro_db_client/migrations/20261103201221_outlook_import_progress.sql@@ -0,0 +1,33 @@+-- Additive: Outlook import progress for telemetry. Both tables are small.+SET LOCAL lock_timeout = '1s';++-- Read on the folder_catalog stream: when its import run started, and when it+-- finished. The completion stamp lets "import completed" fire once per run.+-- now() is stable, so the default needs no table rewrite.+ALTER TABLE email_sync_streams+ ADD COLUMN import_started_at timestamptz DEFAULT now(),+ ADD COLUMN import_completed_at timestamptz;++-- Imports that finished before this column existed must not report again.+UPDATE email_sync_streams catalog SET import_completed_at = now()+WHERE catalog.kind = 'folder_catalog' AND catalog.initial_complete+ AND NOT EXISTS (+ SELECT 1 FROM email_sync_streams folder+ WHERE folder.link_id = catalog.link_id AND folder.generation = catalog.generation+ AND folder.kind = 'mail_folder' AND NOT folder.initial_complete+ )+ AND NOT EXISTS (+ SELECT 1 FROM email_message_reconciliation queued+ WHERE queued.link_id = catalog.link_id AND queued.generation = catalog.generation+ AND queued.is_import+ );++-- Messages each worker imported, flushed from memory once a minute so the+-- import path itself writes nothing extra. Rows older than a day are deleted.+CREATE TABLE email_mailbox_import_minutes (+ link_id uuid NOT NULL,+ minute timestamptz NOT NULL,+ imported integer NOT NULL,+ last_imported_at timestamptz NOT NULL,+ PRIMARY KEY (link_id, minute)+);diff --git a/packages/sdk/generated/storage/sdk.gen.ts b/packages/sdk/generated/storage/sdk.gen.tsindex 801912539a..acd1d8b8c3 100644--- a/packages/sdk/generated/storage/sdk.gen.ts+++ b/packages/sdk/generated/storage/sdk.gen.ts@@ -974,7 +974,7 @@ export class Sdk extends HeyApiClient {public listCrmBlockedDomains<ThrowOnError extends boolean = false>(options?: Options<ListCrmBlockedDomainsData, ThrowOnError>): RequestResult<ListCrmBlockedDomainsResponses, ListCrmBlockedDomainsErrors, ThrowOnError> {return (options?.client ?? this.client).get<ListCrmBlockedDomainsResponses, ListCrmBlockedDomainsErrors, ThrowOnError>({ url: '/crm/blocked-domains', ...options });}-+/*** Unblock a domain: email sync resumes for it, and what blocking hid is* shown again. Idempotent. Requires an admin/owner team role (403).@@ -982,7 +982,7 @@ export class Sdk extends HeyApiClient {public unblockCrmDomain<ThrowOnError extends boolean = false>(options: Options<UnblockCrmDomainData, ThrowOnError>): RequestResult<UnblockCrmDomainResponses, UnblockCrmDomainErrors, ThrowOnError> {return (options.client ?? this.client).delete<UnblockCrmDomainResponses, UnblockCrmDomainErrors, ThrowOnError>({ url: '/crm/blocked-domains/{domain}', ...options });}-+/*** Block a domain: email sync stops creating or updating CRM records for* it, and the company and contacts the CRM holds for it are hidden.@@ -991,7 +991,7 @@ export class Sdk extends HeyApiClient {public blockCrmDomain<ThrowOnError extends boolean = false>(options: Options<BlockCrmDomainData, ThrowOnError>): RequestResult<BlockCrmDomainResponses, BlockCrmDomainErrors, ThrowOnError> {return (options.client ?? this.client).put<BlockCrmDomainResponses, BlockCrmDomainErrors, ThrowOnError>({ url: '/crm/blocked-domains/{domain}', ...options });}-+/*** Manually create a CRM company for the caller's team. Any team member* may create one. A domain, when given, must not already be tracked by the team@@ -1130,7 +1130,7 @@ export class Sdk extends HeyApiClient {}});}-+/*** Look up a CRM contact by email in the caller's team. Returns a null* `contact` when no visible contact exists. Any team member may resolve adiff --git a/services/authentication_service/src/api/internal/microsoft_access_token.rs b/services/authentication_service/src/api/internal/microsoft_access_token.rsindex 6dded0e6b4..9279a05ecc 100644--- a/services/authentication_service/src/api/internal/microsoft_access_token.rs+++ b/services/authentication_service/src/api/internal/microsoft_access_token.rs@@ -37,7 +37,7 @@ pub async fn handler({Ok(token) => ([(header::CACHE_CONTROL, "no-store")],- Json(serde_json::json!({"access_token":token.as_str(),"scopes":token.scopes})),+ Json(serde_json::json!({"access_token":token.as_str(),"scopes":token.scopes,"cached":token.cached})),).into_response(),Err(diff --git a/services/authentication_service/src/domain/microsoft.rs b/services/authentication_service/src/domain/microsoft.rsindex 3c319b26e4..37a53948a6 100644--- a/services/authentication_service/src/domain/microsoft.rs+++ b/services/authentication_service/src/domain/microsoft.rs@@ -125,6 +125,8 @@ pub struct RefreshedGrant {pub struct MicrosoftAccessGrant {token: Zeroizing<String>,pub scopes: Vec<String>,+ /// Served from this process's token cache rather than a refresh.+ pub cached: bool,}impl MicrosoftAccessGrant {pub fn as_str(&self) -> &str {@@ -595,6 +597,7 @@ impl<R: MicrosoftGrantRepository, P: MicrosoftIdentityProvider, F> MicrosoftAuthreturn Ok(MicrosoftAccessGrant {token: cached.value.clone(),scopes: cached.scopes.clone(),+ cached: true,});}let lease = macro_uuid::generate_uuid_v7();@@ -651,6 +654,7 @@ impl<R: MicrosoftGrantRepository, P: MicrosoftIdentityProvider, F> MicrosoftAuthOk(MicrosoftAccessGrant {token: refreshed.access_token,scopes: refreshed.scopes,+ cached: false,})}.await;diff --git a/services/authentication_service/src/domain/microsoft/test.rs b/services/authentication_service/src/domain/microsoft/test.rsindex c8b4300956..b3d481e292 100644--- a/services/authentication_service/src/domain/microsoft/test.rs+++ b/services/authentication_service/src/domain/microsoft/test.rs@@ -473,3 +473,51 @@ async fn reconnecting_an_inbox_never_depends_on_the_flag_or_the_switch() {);}}++#[tokio::test]+async fn tokens_report_whether_the_cache_served_them() {+ let grant_id = macro_uuid::generate_uuid_v7();+ let mut repo = MockMicrosoftGrantRepository::new();+ repo.expect_active_grant()+ .times(2)+ .returning(move |_, _, _| {+ Ok(StoredGrant {+ scopes: scopes(),+ id: grant_id,+ generation: 1,+ revision: 0,+ owner: "owner".into(),+ email: "[email protected]".into(),+ envelope: envelope(),+ })+ });+ repo.expect_acquire_refresh()+ .once()+ .returning(|_, _| Ok(true));+ repo.expect_finish_refresh()+ .once()+ .returning(|_, _, _, _| Ok(true));+ let mut provider = MockMicrosoftIdentityProvider::new();+ provider.expect_refresh().once().returning(|_| {+ Ok(RefreshedGrant {+ access_token: Zeroizing::new("fresh".into()),+ refresh_token: None,+ expires_in: 3600,+ scopes: scopes(),+ })+ });+ let mut cipher = MockMicrosoftTokenCipher::new();+ cipher+ .expect_decrypt()+ .once()+ .returning(|_, _, _| Ok(MicrosoftRefreshToken::new("refresh".into())));+ let service = Service::new(repo, provider, Arc::new(cipher));+ let link = macro_uuid::generate_uuid_v7();++ let refreshed = service.access_token(link, 1, 1, false).await.unwrap();+ let cached = service.access_token(link, 1, 1, false).await.unwrap();++ assert!(!refreshed.cached);+ assert!(cached.cached);+ assert_eq!(cached.as_str(), "fresh");+}diff --git a/services/email_service/AGENTS.md b/services/email_service/AGENTS.mdindex b6abda9bfb..481e9fa1bf 100644--- a/services/email_service/AGENTS.md+++ b/services/email_service/AGENTS.md@@ -19,6 +19,9 @@ the gateway `/email` prefix.`src/backfill_completion_service.rs`: Gmail history backfill orchestration.- `src/crm_enrichment_worker.rs`: periodic pass that resolves pending CRMcompany directory entries (names/logos) left by binaries without a resolver.+- `src/mailbox_sync_worker.rs`: the Outlook mailbox loops (sync, projections,+ contacts, drafts, commands, settings, watches) and `run_health`, the+ per-minute Outlook health snapshot.- `src/bin/`: standalone binaries — `pubsub_workers` (the queue consumerprocess), plus one-off backfill/cleanup scripts.@@ -158,6 +161,14 @@ the gateway `/email` prefix.visibility while processing. Don't raise its receive batch: messages queuedbehind a long mailbox purge would be redelivered and purged twice.`PurgePacing` (`email_db_client::links::delete`) sets the purge's write rate.+- `run_health` runs in every `pubsub_workers` task: each flushes its in-memory+ Outlook import counts every minute, and the task that wins the Redis key+ `outlook:health_snapshot` logs one "outlook mailbox health" line per Outlook+ mailbox. Its fields, the `outlook.sync.*`, `outlook.graph_request` and+ `outlook.token.acquire` spans and the import lifecycle logs are read by the+ Outlook sync monitors in `infra/stacks/monitoring`; rename them only+ together. Loops whose domain service logs its own deferrals use `reported`,+ the others `record`, which logs gate refusals at DEBUG.- `GET /opens/{token}` is public and must return the same uncacheable GIF forevery token. Gmail sends add the pixel only to the provider copy in`outbound/scheduled_delivery.rs`; Outlook adds it through the materializerdiff --git a/services/email_service/src/api/email/provider_error.rs b/services/email_service/src/api/email/provider_error.rsindex 8c756b6d39..e5c0e79d8d 100644--- a/services/email_service/src/api/email/provider_error.rs+++ b/services/email_service/src/api/email/provider_error.rs@@ -29,8 +29,8 @@ pub(crate) fn provider_error_status(error: &EmailApiError) -> StatusCode {EmailApiError::NotFound => StatusCode::NOT_FOUND,EmailApiError::Conflict | EmailApiError::OutdatedCursor => StatusCode::CONFLICT,EmailApiError::SendRejected { .. } => StatusCode::BAD_REQUEST,- EmailApiError::Transient { .. } | EmailApiError::Permanent { .. } => {- StatusCode::INTERNAL_SERVER_ERROR- }+ EmailApiError::Transient { .. }+ | EmailApiError::CredentialsUnavailable { .. }+ | EmailApiError::Permanent { .. } => StatusCode::INTERNAL_SERVER_ERROR,}}diff --git a/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs b/services/email_service/src/bin/pubsub_workers/pubsub_workers.rsindex fc4629b6dc..5aeb62eb77 100644--- a/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs+++ b/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs@@ -311,12 +311,21 @@ async fn main() -> anyhow::Result<()> {);let mailbox_repository = email::outbound::mailbox_pg::PgMailboxSync::new(db.clone());if config.outlook_sync_enabled {- worker_tracker.spawn(email_service::mailbox_sync_worker::run(- email::domain::mailbox::MailboxSyncService::new(- mailbox_repository.clone(),- email_service::outbound::email_api::ProviderMailboxGateway(outlook_api.clone()),+ let mailbox_sync = email::domain::mailbox::MailboxSyncService::new(+ mailbox_repository.clone(),+ email_service::outbound::email_api::ProviderMailboxGateway(outlook_api.clone()),+ mailbox_repository.clone(),+ );+ worker_tracker.spawn(email_service::mailbox_sync_worker::run_health(+ email::domain::mailbox::progress::MailboxProgressService::new(mailbox_repository.clone(),+ mailbox_sync.import_tally(),),+ redis_client.clone(),+ worker_cancellation_token.clone(),+ ));+ worker_tracker.spawn(email_service::mailbox_sync_worker::run(+ mailbox_sync,worker_cancellation_token.clone(),));}diff --git a/services/email_service/src/mailbox_sync_worker.rs b/services/email_service/src/mailbox_sync_worker.rsindex 758dc431b2..90ee200031 100644--- a/services/email_service/src/mailbox_sync_worker.rs+++ b/services/email_service/src/mailbox_sync_worker.rs@@ -1,11 +1,19 @@//! Poll durable mailbox work. Notifications only change when that work is due.+use crate::util::redis::RedisClient;+use email::domain::mailbox::progress::{MailboxProgressRepository, MailboxProgressService};+use email::domain::mailbox::telemetry::defer_reason;use email::domain::mailbox::{MailboxError, MailboxGateway, MailboxIngest, MailboxSyncRepository, MailboxSyncService,};+use email_api_client::domain::models::{EmailApiError, RateLimitOrigin};use std::time::Duration;use tokio_util::sync::CancellationToken;+/// How often each worker flushes its import counts and one worker logs the+/// Outlook health snapshot.+const HEALTH_INTERVAL: Duration = Duration::from_secs(60);+/// Domain services own sync and retry decisions; this loop only schedules work/// and stops accepting work during shutdown. Interrupted work retains its lease.pub async fn run<R: MailboxSyncRepository, P: MailboxGateway, I: MailboxIngest>(@@ -14,10 +22,10 @@ pub async fn run<R: MailboxSyncRepository, P: MailboxGateway, I: MailboxIngest>() {loop {let cycle = async {- let mut worked = record(service.discover_once().await);- worked |= record(service.enumerate_once().await);+ let mut worked = reported(service.discover_once().await);+ worked |= reported(service.enumerate_once().await);for _ in 0..32 {- if !record(service.reconcile_once().await) {+ if !reported(service.reconcile_once().await) {break;}worked = true;@@ -39,17 +47,72 @@ pub async fn run<R: MailboxSyncRepository, P: MailboxGateway, I: MailboxIngest>(}}+/// For units that log their own deferrals and failures with their mailbox.+fn reported(result: Result<bool, MailboxError>) -> bool {+ result.unwrap_or(false)+}+fn record(result: Result<bool, MailboxError>) -> bool {match result {Ok(worked) => worked,Err(MailboxError::Stale) => false,+ // The shared request gate pacing a mailbox is routine.+ Err(+ error @ MailboxError::Provider(EmailApiError::RateLimited {+ origin: RateLimitOrigin::Gate(_),+ ..+ }),+ ) => {+ tracing::debug!(defer_reason = defer_reason(&error), %error, "mailbox sync work deferred");+ false+ }Err(error) => {- tracing::warn!(%error,"mailbox sync work deferred");+ tracing::warn!(defer_reason = defer_reason(&error), %error, "mailbox sync work deferred");false}}}+/// Every worker flushes the imports it counted each minute; whichever worker+/// claims the minute also logs one health snapshot per Outlook mailbox.+pub async fn run_health<R: MailboxProgressRepository>(+ service: MailboxProgressService<R>,+ redis: RedisClient,+ cancellation: CancellationToken,+) {+ let mut interval = tokio::time::interval(HEALTH_INTERVAL);+ interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);+ loop {+ tokio::select! {+ biased;+ _ = cancellation.cancelled() => break,+ _ = interval.tick() => {},+ }+ let report = async {+ if let Err(error) = service.flush().await {+ tracing::warn!(%error, "failed to flush Outlook import counts");+ }+ // Slightly shorter than the interval so the next minute is free.+ if redis+ .claim_outlook_health_snapshot(HEALTH_INTERVAL - Duration::from_secs(5))+ .await+ && let Err(error) = service.report().await+ {+ tracing::warn!(%error, "failed to read the Outlook health snapshot");+ }+ };+ tokio::select! {+ biased;+ _ = cancellation.cancelled() => break,+ () = report => {},+ }+ }+ // Keep the counts of the last minute.+ if let Err(error) = service.flush().await {+ tracing::warn!(%error, "failed to flush Outlook import counts");+ }+}+pub async fn run_lifecycle<R, E>(service: email::domain::mailbox::lifecycle::InboxLifecycleService<R, E>,cancellation: CancellationToken,@@ -87,9 +150,9 @@ pub async fn run_contacts<R, G, S>({loop {let cycle = async {- let mut worked = record(service.sync_once().await);+ let mut worked = reported(service.sync_once().await);for _ in 0..8 {- if !record(service.photo_once().await) {+ if !reported(service.photo_once().await) {break;}worked = true;@@ -113,7 +176,7 @@ pub async fn run_drafts<R, G, M>(M: email::domain::mailbox::drafts::ports::DraftMaterializer,{loop {- let worked = tokio::select! { biased; _ = cancellation.cancelled() => return, result = service.execute_once() => record(result) };+ let worked = tokio::select! { biased; _ = cancellation.cancelled() => return, result = service.execute_once() => reported(result) };if !worked {tokio::select! { biased; _ = cancellation.cancelled() => return, _ = tokio::time::sleep(Duration::from_secs(1)) => {} }}@@ -132,7 +195,7 @@ pub async fn run_projections<R, E>(let worked = tokio::select! {biased;_ = cancellation.cancelled() => return,- result = service.project_once() => record(result),+ result = service.project_once() => reported(result),};if !worked {tokio::select! {@@ -153,7 +216,7 @@ pub async fn run_commands<R, G>(G: email::domain::mailbox::commands::MailboxCommandGateway,{loop {- let worked = tokio::select! { biased; _ = cancellation.cancelled() => return, result = service.execute_once() => record(result) };+ let worked = tokio::select! { biased; _ = cancellation.cancelled() => return, result = service.execute_once() => reported(result) };if !worked {tokio::select! { biased; _ = cancellation.cancelled() => return, _ = tokio::time::sleep(Duration::from_secs(1)) => {} }}@@ -191,7 +254,7 @@ pub async fn run_settings<R, G>(G: email::domain::mailbox::settings::MailboxSettingsGateway,{loop {- let worked = tokio::select! {biased;_ = cancellation.cancelled()=>return,result=service.execute_once()=>record(result)};+ let worked = tokio::select! {biased;_ = cancellation.cancelled()=>return,result=service.execute_once()=>reported(result)};if !worked {tokio::select! {biased;_ = cancellation.cancelled()=>return,_ = tokio::time::sleep(Duration::from_secs(1))=>{}}}@@ -206,7 +269,7 @@ pub async fn run_watches<R, G>(G: email::domain::mailbox::watches::MailboxWatchGateway,{loop {- let worked = tokio::select! {biased;_ = cancellation.cancelled()=>return,result=service.execute_once()=>record(result)};+ let worked = tokio::select! {biased;_ = cancellation.cancelled()=>return,result=service.execute_once()=>reported(result)};if !worked {tokio::select! {biased;_ = cancellation.cancelled()=>return,_ = tokio::time::sleep(Duration::from_secs(5))=>{}}}diff --git a/services/email_service/src/outbound/email_api/microsoft.rs b/services/email_service/src/outbound/email_api/microsoft.rsindex cdf0606352..fea8f2d9b9 100644--- a/services/email_service/src/outbound/email_api/microsoft.rs+++ b/services/email_service/src/outbound/email_api/microsoft.rs@@ -3,6 +3,7 @@use authentication_service_client::{AuthServiceClient, error::AuthServiceClientError};use email::domain::mailbox::{MailboxGateway, MailboxKey};use email_api_client::domain::{models::*, ports::*, service::mailbox::MailboxApiService};+use tracing::Instrument;mod contacts;mod drafts;@@ -17,14 +18,29 @@ impl email::domain::mailbox::credentials::MailboxCredentialGateway for Microsoftmailbox: MailboxAccess,freshness: TokenFreshness,) -> Result<email::domain::mailbox::credentials::MailboxCredentialToken, TokenError> {- self.0+ let fresh = matches!(freshness, TokenFreshness::Fresh);+ let span = tracing::info_span!(+ "outlook.token.acquire",+ mailbox.link_id = %mailbox.link_id,+ mailbox.sync_generation = mailbox.sync_generation,+ outlook.token_refresh_requested = fresh,+ outlook.token_cache = tracing::field::Empty,+ outlook.outcome = tracing::field::Empty,+ http.response.status_code = tracing::field::Empty,+ otel.status_description = tracing::field::Empty,+ );+ let result = self+ .0.get_microsoft_access_token(mailbox.link_id,mailbox.grant_generation,mailbox.sync_generation,- matches!(freshness, TokenFreshness::Fresh),+ fresh,)- .await+ .instrument(span.clone())+ .await;+ record_token(&span, &result);+ result.map(|response| email::domain::mailbox::credentials::MailboxCredentialToken {token: AccessToken::new(response.access_token),@@ -42,6 +58,30 @@ impl email::domain::mailbox::credentials::MailboxCredentialGateway for Microsoft}}+/// `hit` only when authentication served a cached token; a refresh, a+/// failure or an older authentication service that does not say is a `miss`.+fn record_token(+ span: &tracing::Span,+ result: &Result<+ authentication_service_client::microsoft_access_token::MicrosoftAccessToken,+ AuthServiceClientError,+ >,+) {+ let cached = matches!(result, Ok(token) if token.cached == Some(true));+ span.record("outlook.token_cache", if cached { "hit" } else { "miss" });+ let outcome = match result {+ Ok(_) => "ok",+ Err(AuthServiceClientError::Unauthorized | AuthServiceClientError::Forbidden) => {+ "auth_required"+ }+ Err(_) => "unavailable",+ };+ span.record("outlook.outcome", outcome);+ if let Err(error) = result {+ span.record("otel.status_description", tracing::field::display(error));+ }+}+/// The adapter depends only on the foreign domain service and its capability ports./// Concrete Graph and storage adapters are supplied by the composition root.#[derive(Clone)]@@ -81,7 +121,7 @@ whereimpl<R, T, L> MailboxGateway for ProviderMailboxGateway<R, T, L>where- R: MailboxContentReader + FolderChangeReader + ScopedMailboxRepository,+ R: MailboxContentReader + FolderChangeReader + MailboxRequestCount + ScopedMailboxRepository,T: MailboxTokenSource,L: ProviderRateLimiter,{@@ -104,12 +144,19 @@ where) -> Result<Option<MailboxMessage>, EmailApiError> {self.0.message(access(mailbox), id, folders).await}+ async fn admitted_requests(&self, mailbox: MailboxKey) -> Option<u64> {+ self.0.admitted_requests(access(mailbox)).await+ }}impl<R, T, L> email::domain::mailbox::commands::MailboxCommandGatewayfor ProviderMailboxGateway<R, T, L>where- R: MailboxContentReader + FolderChangeReader + MailboxActionWriter + ScopedMailboxRepository,+ R: MailboxContentReader+ + FolderChangeReader+ + MailboxActionWriter+ + MailboxRequestCount+ + ScopedMailboxRepository,T: MailboxTokenSource,L: ProviderRateLimiter,{diff --git a/services/email_service/src/outbound/mailbox_init.rs b/services/email_service/src/outbound/mailbox_init.rsindex c941bf3874..8dafcf0375 100644--- a/services/email_service/src/outbound/mailbox_init.rs+++ b/services/email_service/src/outbound/mailbox_init.rs@@ -370,6 +370,9 @@ impl MailboxInitializationRepository for PgMailboxInitialization {sqlx::query!("INSERT INTO email_projection_outbox (id,link_id,generation,payload) VALUES ($1,$2,$3,$4)",macro_uuid::generate_uuid_v7(),link_id,link.sync_generation,event).execute(&mut *tx).await.map_err(unavailable)?;tx.commit().await.map_err(unavailable)?;+ if restart {+ email::domain::mailbox::progress::log_import_started(link_id, link.sync_generation);+ }Ok(link_id)}}diff --git a/services/email_service/src/outbound/mailbox_projection.rs b/services/email_service/src/outbound/mailbox_projection.rsindex 87de7f906b..2f9dce2855 100644--- a/services/email_service/src/outbound/mailbox_projection.rs+++ b/services/email_service/src/outbound/mailbox_projection.rs@@ -4,7 +4,7 @@ use contacts::domain::{models::messages::ContactConnection, ports::ContactsIngreuse email::domain::{events::EmailMacroEvent,invitation_extraction::InvitationExtractionRepository,- mailbox::{MailboxError, MailboxKey, projection::*},+ mailbox::{DatabaseFailure, MailboxError, MailboxKey, projection::*},models::calendar_invitation::CalendarInvitation,};use macro_event_broker::MacroEventBroker;@@ -40,9 +40,19 @@ pub struct MailboxEffects<A, I, B, N, C> {pub notifications_enabled: bool,}+/// A downstream effect (event broker, queue, another service) failed.fn failed<E: std::fmt::Debug>(error: E) -> MailboxError {tracing::warn!(?error, "mailbox projection effect deferred");- MailboxError::Persistence+ MailboxError::Effect+}++/// A database step of an effect keeps its SQLSTATE, so it reports as+/// `persistence` rather than as an effect failure.+fn database_failed(error: &dyn std::fmt::Debug, cause: Option<&sqlx::Error>) -> MailboxError {+ tracing::warn!(?error, "mailbox projection effect deferred");+ cause.map_or(MailboxError::Persistence, |cause| {+ MailboxError::Database(DatabaseFailure::from(cause))+ })}impl<A, I, B, N, C> MailboxProjectionEffects for MailboxEffects<A, I, B, N, C>@@ -191,7 +201,7 @@ whereself.invitations.save(message_id, invitations).await- .map_err(failed)+ .map_err(|report| database_failed(&report, report.downcast_current_context()))}async fn attachments(&self,@@ -223,21 +233,25 @@ where).execute(&self.db).await- .map_err(failed)?;+ .map_err(|error| database_failed(&error, Some(&error)))?;let documents = email_db_client::attachments::provider::upload::new_email_document_atts(&self.db,mailbox.link_id,provider_id,).await- .map_err(failed)?;+ .map_err(|error| {+ database_failed(&error, error.chain().find_map(|cause| cause.downcast_ref()))+ })?;let media = email_db_client::attachments::provider::upload::new_email_media_atts(&self.db,mailbox.link_id,provider_id,).await- .map_err(failed)?;+ .map_err(|error| {+ database_failed(&error, error.chain().find_map(|cause| cause.downcast_ref()))+ })?;for (attachment, destination) in documents.into_iter().map(|a| (a, AttachmentUploadDestination::Dss))@@ -280,7 +294,13 @@ wheredata,).await- .map_err(failed)?;+ .map_err(|error| match error {+ // Its message is already a string, so no SQLSTATE survives.+ crate::util::upload_attachment::UploadAttachmentError::DatabaseError(_) => {+ database_failed(&error, None)+ }+ error => failed(error),+ })?;Ok::<_, MailboxError>(())}.await;@@ -291,7 +311,7 @@ where).execute(&self.db).await- .map_err(failed)?;+ .map_err(|error| database_failed(&error, Some(&error)))?;return result;}}@@ -300,9 +320,9 @@ whereAND a.upload_claimed_at IS NOT NULLAND NOT EXISTS(SELECT 1 FROM document_email d WHERE d.email_attachment_id = a.id)AND NOT EXISTS(SELECT 1 FROM email_attachments_sfs s WHERE s.attachment_id = a.id)) AS "pending!"- "#,current.message.db_id).fetch_one(&self.db).await.map_err(failed)?;+ "#,current.message.db_id).fetch_one(&self.db).await.map_err(|error| database_failed(&error, Some(&error)))?;if pending {- return Err(MailboxError::Persistence);+ return Err(MailboxError::Effect);}Ok(())}diff --git a/services/email_service/src/pubsub/backfill/email_api_error.rs b/services/email_service/src/pubsub/backfill/email_api_error.rsindex 4f9cddc12e..347c96ac2e 100644--- a/services/email_service/src/pubsub/backfill/email_api_error.rs+++ b/services/email_service/src/pubsub/backfill/email_api_error.rs@@ -15,6 +15,7 @@ pub(crate) fn map_email_api_error(error: EmailApiError, context: &'static str) -| EmailApiError::NotFound| EmailApiError::Conflict| EmailApiError::Transient { .. }+ | EmailApiError::CredentialsUnavailable { .. }| EmailApiError::SendRejected { .. }| EmailApiError::Permanent { .. } => FailureReason::GmailApiFailed,};diff --git a/services/email_service/src/pubsub/gmail_ops/email_api_error.rs b/services/email_service/src/pubsub/gmail_ops/email_api_error.rsindex b3681084b1..df28eb76bb 100644--- a/services/email_service/src/pubsub/gmail_ops/email_api_error.rs+++ b/services/email_service/src/pubsub/gmail_ops/email_api_error.rs@@ -25,7 +25,9 @@ fn error_policy(error: &EmailApiError) -> ErrorPolicy {// here rather than silently dropping the operation as permanent.match error {EmailApiError::RateLimited { .. } => ErrorPolicy::RateLimited,- EmailApiError::Transient { .. } => ErrorPolicy::Retry,+ EmailApiError::Transient { .. } | EmailApiError::CredentialsUnavailable { .. } => {+ ErrorPolicy::Retry+ }EmailApiError::AuthRequired| EmailApiError::Forbidden| EmailApiError::NotFound@@ -93,7 +95,9 @@ fn rate_limit_route(retry_worker: bool) -> RateLimitRoute {fn processing_error(error: EmailApiError, retryable: bool) -> ProcessingError {let reason = match error {EmailApiError::RateLimited { .. } => FailureReason::GmailApiRateLimited,- EmailApiError::Transient { .. } => FailureReason::GmailApiUnavailable,+ EmailApiError::Transient { .. } | EmailApiError::CredentialsUnavailable { .. } => {+ FailureReason::GmailApiUnavailable+ }EmailApiError::AuthRequired => FailureReason::AccessTokenFetchFailed,EmailApiError::OutdatedCursor => FailureReason::OutdatedHistoryId,EmailApiError::Forbiddendiff --git a/services/email_service/src/pubsub/inbox_sync/email_api_error.rs b/services/email_service/src/pubsub/inbox_sync/email_api_error.rsindex dec1f7076d..e2ea606587 100644--- a/services/email_service/src/pubsub/inbox_sync/email_api_error.rs+++ b/services/email_service/src/pubsub/inbox_sync/email_api_error.rs@@ -73,6 +73,7 @@ fn detail(error: EmailApiError) -> DetailedError {| EmailApiError::NotFound| EmailApiError::Conflict| EmailApiError::Transient { .. }+ | EmailApiError::CredentialsUnavailable { .. }| EmailApiError::SendRejected { .. }| EmailApiError::Permanent { .. } => FailureReason::GmailApiFailed,};diff --git a/services/email_service/src/util/redis.rs b/services/email_service/src/util/redis.rsindex 4df98494c5..5164c88432 100644--- a/services/email_service/src/util/redis.rs+++ b/services/email_service/src/util/redis.rs@@ -3,6 +3,7 @@ pub mod backfill;pub mod backfill_write_slots;pub mod health_probe;pub mod lock;+pub mod outlook_health;pub mod rate_limit;#[derive(Clone)]diff --git a/services/email_service/src/util/redis/outlook_health.rs b/services/email_service/src/util/redis/outlook_health.rsnew file mode 100644index 0000000000..043be7f7df--- /dev/null+++ b/services/email_service/src/util/redis/outlook_health.rs@@ -0,0 +1,34 @@+use crate::util::redis::RedisClient;+use std::time::Duration;++const SNAPSHOT_KEY: &str = "outlook:health_snapshot";++impl RedisClient {+ /// Claims the next Outlook health snapshot with `SET … NX PX`, so one+ /// worker logs it per window however many run. Redis errors fail open:+ /// a duplicate snapshot is harmless, a missing one hides a stall.+ pub async fn claim_outlook_health_snapshot(&self, window: Duration) -> bool {+ let mut connection = match self.inner.get_multiplexed_async_connection().await {+ Ok(connection) => connection,+ Err(error) => {+ tracing::warn!(%error, "Failed to get Redis connection for the Outlook health snapshot");+ return true;+ }+ };+ let claimed: redis::RedisResult<Option<String>> = redis::cmd("SET")+ .arg(SNAPSHOT_KEY)+ .arg("1")+ .arg("NX")+ .arg("PX")+ .arg(u64::try_from(window.as_millis()).unwrap_or(u64::MAX))+ .query_async(&mut connection)+ .await;+ match claimed {+ Ok(claimed) => claimed.is_some(),+ Err(error) => {+ tracing::warn!(%error, "Failed to claim the Outlook health snapshot");+ true+ }+ }+ }+}