Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(email): trace Outlook sync, Graph calls and import progress (#9480)

teo352bd7169a84
diff --git a/.agents/skills/instrumentation/SKILL.md b/.agents/skills/instrumentation/SKILL.md
index bfaa961ff6..2b2f8fd48f 100644
--- a/.agents/skills/instrumentation/SKILL.md
+++ b/.agents/skills/instrumentation/SKILL.md
@@ -207,6 +207,15 @@ telemetry with its own volume budget.
- Agent/ACP messages: `genai_telemetry::propagation::inject` / `set_parent`
carry context in `_meta`; GenAI spans follow CS-55.
- Spawned tasks take the span with them (`.instrument(span)`) or link to it.
+- Datadog files a client span that carries an HTTP method as operation
+ `http.client.request` with resource `<METHOD> <url.template>`, whatever the
+ span is called; set `operation.name` on it to keep it findable by name
+ (`outlook.graph_request` does). Internal spans keep their name as
+ `resource_name`.
+- Record numeric span fields as `i64`: `tracing-opentelemetry` has no
+ unsigned type and exports `u16`/`u32`/`u64`/`usize` values as strings.
+- Background loops start one root span per claimed unit of work, not per poll,
+ so idle polling exports nothing.
## Worked examples
diff --git a/.sqlx/query-094d0bb92a52fd11b65ff460f08314c2f56f8fbfa34b54b879610cd7d95b0f4b.json b/.sqlx/query-094d0bb92a52fd11b65ff460f08314c2f56f8fbfa34b54b879610cd7d95b0f4b.json
new file mode 100644
index 0000000000..51278d417f
--- /dev/null
+++ b/.sqlx/query-094d0bb92a52fd11b65ff460f08314c2f56f8fbfa34b54b879610cd7d95b0f4b.json
@@ -0,0 +1,86 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n SELECT l.id AS \"link_id!\",\n l.sync_generation AS \"sync_generation!\",\n l.is_sync_active AS \"sync_active!\",\n l.needs_reauth AS \"needs_reauth!\",\n q.queued AS \"queued!\",\n q.parked AS \"parked!\",\n q.leased AS \"leased!\",\n COALESCE(floor(EXTRACT(EPOCH FROM now() - q.oldest_due)), 0)::bigint AS \"oldest_waiting_seconds!\",\n COALESCE(ceil(EXTRACT(EPOCH FROM q.parked_until - now())), 0)::bigint AS \"parked_until_max_seconds!\",\n s.incomplete AS \"streams_incomplete!\",\n COALESCE(recent.imported_5m, 0)::bigint AS \"imported_5m!\",\n floor(EXTRACT(EPOCH FROM now() - latest.last_imported_at))::bigint AS seconds_since_last_import\n FROM email_links l\n CROSS JOIN LATERAL (\n SELECT count(*) AS queued,\n count(*) FILTER (WHERE available_at > now()) AS parked,\n count(*) FILTER (WHERE lease_until > now()) AS leased,\n min(available_at) FILTER (\n WHERE available_at <= now() AND (lease_until IS NULL OR lease_until <= now())\n ) AS oldest_due,\n max(available_at) FILTER (WHERE available_at > now()) AS parked_until\n FROM email_message_reconciliation\n WHERE link_id = l.id AND generation = l.sync_generation\n ) q\n CROSS JOIN LATERAL (\n SELECT count(*) AS incomplete\n FROM email_sync_streams\n WHERE link_id = l.id AND generation = l.sync_generation\n AND kind IN ('folder_catalog', 'mail_folder') AND NOT initial_complete\n ) s\n CROSS JOIN LATERAL (\n SELECT sum(imported) AS imported_5m\n FROM email_mailbox_import_minutes\n WHERE link_id = l.id AND minute > now() - interval '5 minutes'\n ) recent\n -- A flush carries imports from at most the minute before it, so\n -- the latest import is in one of the two newest minutes.\n CROSS JOIN LATERAL (\n SELECT max(last_imported_at) AS last_imported_at\n FROM (\n SELECT last_imported_at FROM email_mailbox_import_minutes\n WHERE link_id = l.id ORDER BY minute DESC LIMIT 2\n ) newest\n ) latest\n WHERE l.provider = 'OUTLOOK' AND (l.is_sync_active OR q.queued > 0)\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "link_id!",
+ "type_info": "Uuid"
+ },
+ {
+ "ordinal": 1,
+ "name": "sync_generation!",
+ "type_info": "Int8"
+ },
+ {
+ "ordinal": 2,
+ "name": "sync_active!",
+ "type_info": "Bool"
+ },
+ {
+ "ordinal": 3,
+ "name": "needs_reauth!",
+ "type_info": "Bool"
+ },
+ {
+ "ordinal": 4,
+ "name": "queued!",
+ "type_info": "Int8"
+ },
+ {
+ "ordinal": 5,
+ "name": "parked!",
+ "type_info": "Int8"
+ },
+ {
+ "ordinal": 6,
+ "name": "leased!",
+ "type_info": "Int8"
+ },
+ {
+ "ordinal": 7,
+ "name": "oldest_waiting_seconds!",
+ "type_info": "Int8"
+ },
+ {
+ "ordinal": 8,
+ "name": "parked_until_max_seconds!",
+ "type_info": "Int8"
+ },
+ {
+ "ordinal": 9,
+ "name": "streams_incomplete!",
+ "type_info": "Int8"
+ },
+ {
+ "ordinal": 10,
+ "name": "imported_5m!",
+ "type_info": "Int8"
+ },
+ {
+ "ordinal": 11,
+ "name": "seconds_since_last_import",
+ "type_info": "Int8"
+ }
+ ],
+ "parameters": {
+ "Left": []
+ },
+ "nullable": [
+ false,
+ false,
+ false,
+ false,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null
+ ]
+ },
+ "hash": "094d0bb92a52fd11b65ff460f08314c2f56f8fbfa34b54b879610cd7d95b0f4b"
+}
diff --git a/.sqlx/query-0b7d6e01be005b7456de49f2917a5449a93525fc3ebd0adf6bfe5e3be5b2526d.json b/.sqlx/query-0b7d6e01be005b7456de49f2917a5449a93525fc3ebd0adf6bfe5e3be5b2526d.json
new file mode 100644
index 0000000000..2cf59f1478
--- /dev/null
+++ b/.sqlx/query-0b7d6e01be005b7456de49f2917a5449a93525fc3ebd0adf6bfe5e3be5b2526d.json
@@ -0,0 +1,15 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n INSERT INTO email_projection_outbox(id,link_id,generation,payload)\n SELECT gen_random_uuid(),m.link_id,$2,jsonb_build_object('kind','attachment_recheck','message_id',m.id)\n FROM email_messages m WHERE m.link_id=$1 AND m.provider_id IS NOT NULL AND NOT m.is_draft\n AND EXISTS(SELECT 1 FROM email_attachments a WHERE a.message_id=m.id AND a.reference_url IS NULL)\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Uuid",
+ "Int8"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "0b7d6e01be005b7456de49f2917a5449a93525fc3ebd0adf6bfe5e3be5b2526d"
+}
diff --git a/.sqlx/query-13e4afddda3bdbd69f444a40db87de3fca8a4e6858203bf44e488c936abf4d6a.json b/.sqlx/query-13e4afddda3bdbd69f444a40db87de3fca8a4e6858203bf44e488c936abf4d6a.json
deleted file mode 100644
index cfa335ee40..0000000000
--- a/.sqlx/query-13e4afddda3bdbd69f444a40db87de3fca8a4e6858203bf44e488c936abf4d6a.json
+++ /dev/null
@@ -1,23 +0,0 @@
-{
- "db_name": "PostgreSQL",
- "query": "SELECT (\n EXISTS(SELECT 1 FROM email_sync_streams WHERE link_id=$1 AND generation=$2 AND kind='folder_catalog' AND initial_complete)\n AND EXISTS(SELECT 1 FROM email_sync_streams WHERE link_id=$1 AND generation=$2 AND kind='mail_folder' AND NOT attachments_rechecked)\n AND NOT EXISTS(SELECT 1 FROM email_sync_streams WHERE link_id=$1 AND generation=$2 AND kind IN ('mail_folder','folder_catalog') AND NOT initial_complete)\n AND NOT EXISTS(SELECT 1 FROM email_message_reconciliation WHERE link_id=$1 AND generation=$2)\n ) AS \"ready!\"",
- "describe": {
- "columns": [
- {
- "ordinal": 0,
- "name": "ready!",
- "type_info": "Bool"
- }
- ],
- "parameters": {
- "Left": [
- "Uuid",
- "Int8"
- ]
- },
- "nullable": [
- null
- ]
- },
- "hash": "13e4afddda3bdbd69f444a40db87de3fca8a4e6858203bf44e488c936abf4d6a"
-}
diff --git a/.sqlx/query-1fa5550ddc5382947e0c08629e95d27b72e7c0de6173fdcbb7f25a2dffdf180c.json b/.sqlx/query-1fa5550ddc5382947e0c08629e95d27b72e7c0de6173fdcbb7f25a2dffdf180c.json
deleted file mode 100644
index 53f37d6e11..0000000000
--- a/.sqlx/query-1fa5550ddc5382947e0c08629e95d27b72e7c0de6173fdcbb7f25a2dffdf180c.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "db_name": "PostgreSQL",
- "query": "UPDATE email_projection_outbox SET lease_id = NULL,lease_until = NULL,\n available_at = now() + make_interval(secs => LEAST(600,30 * attempts))\n WHERE id = $1 AND lease_id = $2",
- "describe": {
- "columns": [],
- "parameters": {
- "Left": [
- "Uuid",
- "Uuid"
- ]
- },
- "nullable": []
- },
- "hash": "1fa5550ddc5382947e0c08629e95d27b72e7c0de6173fdcbb7f25a2dffdf180c"
-}
diff --git a/.sqlx/query-4f501e81e5d26c98eed8aaec428c90f364eb75ecedc23dae6e55f5283714e78f.json b/.sqlx/query-3c12e585b3ce343cf3a56fc7706c14d3e4348cc45742093eb66bb5b11f50c45f.json
similarity index 81%
rename from .sqlx/query-4f501e81e5d26c98eed8aaec428c90f364eb75ecedc23dae6e55f5283714e78f.json
rename to .sqlx/query-3c12e585b3ce343cf3a56fc7706c14d3e4348cc45742093eb66bb5b11f50c45f.json
index cea675c703..d9469ff428 100644
--- a/.sqlx/query-4f501e81e5d26c98eed8aaec428c90f364eb75ecedc23dae6e55f5283714e78f.json
+++ b/.sqlx/query-3c12e585b3ce343cf3a56fc7706c14d3e4348cc45742093eb66bb5b11f50c45f.json
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
- "query": "\n WITH candidate AS (\n SELECT o.id FROM email_projection_outbox o JOIN email_links l ON l.id = o.link_id\n WHERE l.is_sync_active AND o.available_at <= now() AND (o.lease_until IS NULL OR o.lease_until < now())\n ORDER BY o.available_at,o.id LIMIT 1 FOR UPDATE OF o SKIP LOCKED\n )\n UPDATE email_projection_outbox o SET lease_id = $1,lease_until = now() + interval '3 minutes',attempts = attempts + 1\n FROM candidate c,email_links l WHERE o.id = c.id AND l.id = o.link_id\n RETURNING o.id,o.link_id,o.generation,o.payload,l.sync_generation,l.grant_generation\n ",
+ "query": "\n WITH candidate AS (\n SELECT o.id FROM email_projection_outbox o JOIN email_links l ON l.id = o.link_id\n WHERE l.is_sync_active AND o.available_at <= now() AND (o.lease_until IS NULL OR o.lease_until < now())\n ORDER BY o.available_at,o.id LIMIT 1 FOR UPDATE OF o SKIP LOCKED\n )\n UPDATE email_projection_outbox o SET lease_id = $1,lease_until = now() + interval '3 minutes',attempts = attempts + 1\n FROM candidate c,email_links l WHERE o.id = c.id AND l.id = o.link_id\n RETURNING o.id,o.link_id,o.generation,o.payload,o.attempts,l.sync_generation,l.grant_generation\n ",
"describe": {
"columns": [
{
@@ -25,11 +25,16 @@
},
{
"ordinal": 4,
+ "name": "attempts",
+ "type_info": "Int4"
+ },
+ {
+ "ordinal": 5,
"name": "sync_generation",
"type_info": "Int8"
},
{
- "ordinal": 5,
+ "ordinal": 6,
"name": "grant_generation",
"type_info": "Int8"
}
@@ -45,8 +50,9 @@
false,
false,
false,
+ false,
false
]
},
- "hash": "4f501e81e5d26c98eed8aaec428c90f364eb75ecedc23dae6e55f5283714e78f"
+ "hash": "3c12e585b3ce343cf3a56fc7706c14d3e4348cc45742093eb66bb5b11f50c45f"
}
diff --git a/.sqlx/query-3c7c452162235d1ddd691ea33df2aed5b0b12b2bb725e9f47cd5f2dda93ea0d2.json b/.sqlx/query-3c7c452162235d1ddd691ea33df2aed5b0b12b2bb725e9f47cd5f2dda93ea0d2.json
new file mode 100644
index 0000000000..f7ec7f647d
--- /dev/null
+++ b/.sqlx/query-3c7c452162235d1ddd691ea33df2aed5b0b12b2bb725e9f47cd5f2dda93ea0d2.json
@@ -0,0 +1,29 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT\n (SELECT count(*) FROM email_messages WHERE link_id=$1 AND provider_id IS NOT NULL) AS \"messages!\",\n (SELECT count(*) FROM email_sync_streams WHERE link_id=$1 AND generation=$2 AND kind='mail_folder') AS \"folders!\"\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "messages!",
+ "type_info": "Int8"
+ },
+ {
+ "ordinal": 1,
+ "name": "folders!",
+ "type_info": "Int8"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Uuid",
+ "Int8"
+ ]
+ },
+ "nullable": [
+ null,
+ null
+ ]
+ },
+ "hash": "3c7c452162235d1ddd691ea33df2aed5b0b12b2bb725e9f47cd5f2dda93ea0d2"
+}
diff --git a/.sqlx/query-5312773217eb0dadeae9c4a02f07631cde543a6a3685bb6c8e7451c2d7edfc6c.json b/.sqlx/query-5312773217eb0dadeae9c4a02f07631cde543a6a3685bb6c8e7451c2d7edfc6c.json
new file mode 100644
index 0000000000..23492e8aad
--- /dev/null
+++ b/.sqlx/query-5312773217eb0dadeae9c4a02f07631cde543a6a3685bb6c8e7451c2d7edfc6c.json
@@ -0,0 +1,16 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "INSERT INTO email_sync_streams(id,link_id,generation,kind,scope_id,position,initial_complete,scan_id,import_started_at,import_completed_at)\n SELECT gen_random_uuid(),link_id,$3,kind,scope_id,position,initial_complete,scan_id,import_started_at,import_completed_at FROM email_sync_streams WHERE link_id = $1 AND generation = $2\n ON CONFLICT DO NOTHING",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Uuid",
+ "Int8",
+ "Int8"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "5312773217eb0dadeae9c4a02f07631cde543a6a3685bb6c8e7451c2d7edfc6c"
+}
diff --git a/.sqlx/query-8886c3d33e3babda4a5f53deb9c377eed4e67151789a60405ac76a89af9bcdc9.json b/.sqlx/query-8886c3d33e3babda4a5f53deb9c377eed4e67151789a60405ac76a89af9bcdc9.json
new file mode 100644
index 0000000000..529c4a4647
--- /dev/null
+++ b/.sqlx/query-8886c3d33e3babda4a5f53deb9c377eed4e67151789a60405ac76a89af9bcdc9.json
@@ -0,0 +1,29 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n UPDATE email_sync_streams SET import_completed_at = now()\n WHERE link_id=$1 AND generation=$2 AND kind='folder_catalog' AND import_completed_at IS NULL\n RETURNING import_started_at, import_completed_at AS \"completed_at!\"\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "import_started_at",
+ "type_info": "Timestamptz"
+ },
+ {
+ "ordinal": 1,
+ "name": "completed_at!",
+ "type_info": "Timestamptz"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Uuid",
+ "Int8"
+ ]
+ },
+ "nullable": [
+ true,
+ true
+ ]
+ },
+ "hash": "8886c3d33e3babda4a5f53deb9c377eed4e67151789a60405ac76a89af9bcdc9"
+}
diff --git a/.sqlx/query-88e47b1fc3e5e19bfd0f4083e078d02ba8d5b4a2eafe1cf6c1d8fbf1b4d841a3.json b/.sqlx/query-88e47b1fc3e5e19bfd0f4083e078d02ba8d5b4a2eafe1cf6c1d8fbf1b4d841a3.json
new file mode 100644
index 0000000000..a6ce05605b
--- /dev/null
+++ b/.sqlx/query-88e47b1fc3e5e19bfd0f4083e078d02ba8d5b4a2eafe1cf6c1d8fbf1b4d841a3.json
@@ -0,0 +1,16 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n INSERT INTO email_mailbox_import_minutes (link_id, minute, imported, last_imported_at)\n SELECT link_id, date_trunc('minute', now()), imported, last_imported_at\n FROM unnest($1::uuid[], $2::int[], $3::timestamptz[]) AS flushed(link_id, imported, last_imported_at)\n ON CONFLICT (link_id, minute) DO UPDATE SET\n imported = email_mailbox_import_minutes.imported + EXCLUDED.imported,\n last_imported_at = GREATEST(email_mailbox_import_minutes.last_imported_at, EXCLUDED.last_imported_at)\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "UuidArray",
+ "Int4Array",
+ "TimestamptzArray"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "88e47b1fc3e5e19bfd0f4083e078d02ba8d5b4a2eafe1cf6c1d8fbf1b4d841a3"
+}
diff --git a/.sqlx/query-8bddb3c4f6f73fee00f642d74491428e6f6bd3e3ce41066048b81cb682465be5.json b/.sqlx/query-8bddb3c4f6f73fee00f642d74491428e6f6bd3e3ce41066048b81cb682465be5.json
new file mode 100644
index 0000000000..4cee539bd5
--- /dev/null
+++ b/.sqlx/query-8bddb3c4f6f73fee00f642d74491428e6f6bd3e3ce41066048b81cb682465be5.json
@@ -0,0 +1,12 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "DELETE FROM email_mailbox_import_minutes WHERE minute < now() - interval '1 day'",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": []
+ },
+ "nullable": []
+ },
+ "hash": "8bddb3c4f6f73fee00f642d74491428e6f6bd3e3ce41066048b81cb682465be5"
+}
diff --git a/.sqlx/query-91d7badfc9cd004327cba280d60690632c8f3c4c802f2b8b385c72413b55e210.json b/.sqlx/query-91d7badfc9cd004327cba280d60690632c8f3c4c802f2b8b385c72413b55e210.json
deleted file mode 100644
index c970f897f4..0000000000
--- a/.sqlx/query-91d7badfc9cd004327cba280d60690632c8f3c4c802f2b8b385c72413b55e210.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "db_name": "PostgreSQL",
- "query": "\n INSERT INTO email_projection_outbox(id,link_id,generation,payload)\n SELECT gen_random_uuid(),m.link_id,$2,jsonb_build_object('kind','attachment_recheck','message_id',m.id)\n FROM email_messages m WHERE m.link_id=$1 AND m.provider_id IS NOT NULL AND NOT m.is_draft\n AND EXISTS(SELECT 1 FROM email_attachments a WHERE a.message_id=m.id AND a.reference_url IS NULL)\n ",
- "describe": {
- "columns": [],
- "parameters": {
- "Left": [
- "Uuid",
- "Int8"
- ]
- },
- "nullable": []
- },
- "hash": "91d7badfc9cd004327cba280d60690632c8f3c4c802f2b8b385c72413b55e210"
-}
diff --git a/.sqlx/query-a3945ff5ad3ec5248b966f794a2434339c0d4725cfe0b2f5fcb7fae5ef66d559.json b/.sqlx/query-a3945ff5ad3ec5248b966f794a2434339c0d4725cfe0b2f5fcb7fae5ef66d559.json
deleted file mode 100644
index 167ada83b3..0000000000
--- a/.sqlx/query-a3945ff5ad3ec5248b966f794a2434339c0d4725cfe0b2f5fcb7fae5ef66d559.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "db_name": "PostgreSQL",
- "query": "INSERT INTO email_sync_streams(id,link_id,generation,kind,scope_id,position,initial_complete,scan_id)\n SELECT gen_random_uuid(),link_id,$3,kind,scope_id,position,initial_complete,scan_id FROM email_sync_streams WHERE link_id = $1 AND generation = $2\n ON CONFLICT DO NOTHING",
- "describe": {
- "columns": [],
- "parameters": {
- "Left": [
- "Uuid",
- "Int8",
- "Int8"
- ]
- },
- "nullable": []
- },
- "hash": "a3945ff5ad3ec5248b966f794a2434339c0d4725cfe0b2f5fcb7fae5ef66d559"
-}
diff --git a/.sqlx/query-2983e995fc823e3cc3c6fa643728283e373c5efcf8127f05d7a1257e3bf08aac.json b/.sqlx/query-b4e8e156590c9dfd3d3ed2ce0f9648e124b75544b3842ab7f98a932e4ceca191.json
similarity index 51%
rename from .sqlx/query-2983e995fc823e3cc3c6fa643728283e373c5efcf8127f05d7a1257e3bf08aac.json
rename to .sqlx/query-b4e8e156590c9dfd3d3ed2ce0f9648e124b75544b3842ab7f98a932e4ceca191.json
index 5d469bd7fe..43384ff81f 100644
--- a/.sqlx/query-2983e995fc823e3cc3c6fa643728283e373c5efcf8127f05d7a1257e3bf08aac.json
+++ b/.sqlx/query-b4e8e156590c9dfd3d3ed2ce0f9648e124b75544b3842ab7f98a932e4ceca191.json
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
- "query": "UPDATE email_sync_streams SET position = NULL,initial_complete = false, attachments_rechecked = false,scan_id = gen_random_uuid(),next_run_at = now(),fence = fence + 1,lease_id = NULL,lease_until = NULL\n WHERE link_id = $1 AND generation = $2 AND kind IN ('folder_catalog','mail_folder','contacts_catalog','contacts_profile','contacts')",
+ "query": "UPDATE email_sync_streams SET position = NULL,initial_complete = false, attachments_rechecked = false,scan_id = gen_random_uuid(),next_run_at = now(),fence = fence + 1,lease_id = NULL,lease_until = NULL,\n import_started_at = now(),import_completed_at = NULL\n WHERE link_id = $1 AND generation = $2 AND kind IN ('folder_catalog','mail_folder','contacts_catalog','contacts_profile','contacts')",
"describe": {
"columns": [],
"parameters": {
@@ -11,5 +11,5 @@
},
"nullable": []
},
- "hash": "2983e995fc823e3cc3c6fa643728283e373c5efcf8127f05d7a1257e3bf08aac"
+ "hash": "b4e8e156590c9dfd3d3ed2ce0f9648e124b75544b3842ab7f98a932e4ceca191"
}
diff --git a/.sqlx/query-d4436ee46b2938f27dc995c4034c0b5b05dc90062a333dd41c303894287191d0.json b/.sqlx/query-d4436ee46b2938f27dc995c4034c0b5b05dc90062a333dd41c303894287191d0.json
new file mode 100644
index 0000000000..5371a982ae
--- /dev/null
+++ b/.sqlx/query-d4436ee46b2938f27dc995c4034c0b5b05dc90062a333dd41c303894287191d0.json
@@ -0,0 +1,16 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "UPDATE email_projection_outbox SET lease_id = NULL,lease_until = NULL,\n available_at = now() + make_interval(secs => $3)\n WHERE id = $1 AND lease_id = $2",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Uuid",
+ "Uuid",
+ "Float8"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "d4436ee46b2938f27dc995c4034c0b5b05dc90062a333dd41c303894287191d0"
+}
diff --git a/.sqlx/query-fcdaf7e42334b418b27dadf22945e16383d1f93a62e43a3aecf70e2077e11a73.json b/.sqlx/query-fcdaf7e42334b418b27dadf22945e16383d1f93a62e43a3aecf70e2077e11a73.json
new file mode 100644
index 0000000000..d30a222f79
--- /dev/null
+++ b/.sqlx/query-fcdaf7e42334b418b27dadf22945e16383d1f93a62e43a3aecf70e2077e11a73.json
@@ -0,0 +1,53 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT\n EXISTS(SELECT 1 FROM email_sync_streams WHERE link_id=$1 AND generation=$2 AND kind='folder_catalog' AND initial_complete) AS \"catalog_complete!\",\n EXISTS(SELECT 1 FROM email_sync_streams WHERE link_id=$1 AND generation=$2 AND kind='folder_catalog' AND import_completed_at IS NULL) AS \"import_open!\",\n EXISTS(SELECT 1 FROM email_sync_streams WHERE link_id=$1 AND generation=$2 AND kind='mail_folder' AND NOT attachments_rechecked) AS \"recheck_pending!\",\n EXISTS(SELECT 1 FROM email_sync_streams WHERE link_id=$1 AND generation=$2 AND kind IN ('mail_folder','folder_catalog') AND NOT initial_complete) AS \"streams_incomplete!\",\n EXISTS(SELECT 1 FROM email_message_reconciliation WHERE link_id=$1 AND generation=$2) AS \"queued!\",\n EXISTS(SELECT 1 FROM email_message_reconciliation WHERE link_id=$1 AND generation=$2 AND is_import) AS \"imports_queued!\"\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "catalog_complete!",
+ "type_info": "Bool"
+ },
+ {
+ "ordinal": 1,
+ "name": "import_open!",
+ "type_info": "Bool"
+ },
+ {
+ "ordinal": 2,
+ "name": "recheck_pending!",
+ "type_info": "Bool"
+ },
+ {
+ "ordinal": 3,
+ "name": "streams_incomplete!",
+ "type_info": "Bool"
+ },
+ {
+ "ordinal": 4,
+ "name": "queued!",
+ "type_info": "Bool"
+ },
+ {
+ "ordinal": 5,
+ "name": "imports_queued!",
+ "type_info": "Bool"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Uuid",
+ "Int8"
+ ]
+ },
+ "nullable": [
+ null,
+ null,
+ null,
+ null,
+ null,
+ null
+ ]
+ },
+ "hash": "fcdaf7e42334b418b27dadf22945e16383d1f93a62e43a3aecf70e2077e11a73"
+}
diff --git a/Cargo.lock b/Cargo.lock
index 2d703aebb5..2dd92dfbef 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -7364,6 +7364,7 @@ dependencies = [
"thiserror 2.0.18",
"tokio",
"tracing",
+ "tracing-subscriber",
"url",
"uuid",
"wiremock",
diff --git a/crates/authentication_service_client/src/microsoft_access_token.rs b/crates/authentication_service_client/src/microsoft_access_token.rs
index b6fb817911..24bc7019f6 100644
--- a/crates/authentication_service_client/src/microsoft_access_token.rs
+++ b/crates/authentication_service_client/src/microsoft_access_token.rs
@@ -8,6 +8,10 @@ use crate::{
pub struct MicrosoftAccessToken {
pub access_token: String,
pub scopes: Vec<String>,
+ /// Whether authentication served a cached token; `None` from a version
+ /// that does not report it.
+ #[serde(default)]
+ pub cached: Option<bool>,
}
/// Non-secret, owner-bound result of a completed Microsoft OAuth attempt.
@@ -130,6 +134,11 @@ impl AuthServiceClient {
.send()
.await
.map_err(|_| unavailable())?;
+ // Lands on the caller's token span when it declares the field.
+ tracing::Span::current().record(
+ "http.response.status_code",
+ i64::from(response.status().as_u16()),
+ );
match response.status() {
reqwest::StatusCode::OK => {
let token: MicrosoftAccessToken =
diff --git a/crates/email/AGENTS.md b/crates/email/AGENTS.md
index b0ccdd5e15..204e2f0219 100644
--- a/crates/email/AGENTS.md
+++ b/crates/email/AGENTS.md
@@ -111,6 +111,29 @@ tools — hexagonal, heavily feature-gated.
delivery on a tracking failure, and count opens only for sent messages.
Tokens are stored per message, so a retried send or Outlook draft revision
rebuilds the same URL; keep it that way or the draft fingerprints drift.
+- Outlook sync telemetry (`domain/mailbox/telemetry.rs`, `progress.rs`) is a
+ contract the Outlook sync monitors in `infra/stacks/monitoring` read: each
+ claimed unit of `MailboxSyncService` and `MailboxProjectionService` opens
+ one root `outlook.sync.*` span (idle polls open none) and records
+ `outlook.outcome`, `outlook.defer_reason` and `outlook.defer_seconds`
+ through `record_done`/`record_deferred`/`record_failed`. The mailbox loops
+ log their own deferrals with `link_id` (gate refusals at DEBUG), so the
+ worker must not log them again. Add a new defer reason in `defer_reason`.
+- Map database errors in `mailbox_pg` with `db_error` (or `wrapped_db_error`
+ for `email_db_client` calls) so the SQLSTATE reaches the log; keep
+ `MailboxError::Persistence` for stored data that cannot be decoded.
+ `MailboxError::Effect` (`defer_reason=effect`) is for a projection effect
+ that fails outside the database; email-service's `MailboxEffects` maps its
+ database steps with `database_failed` and everything else with `failed`.
+- "outlook import completed" fires from `settle_import` (`mailbox_pg.rs`)
+ under the link lock, once per import run: the folder-catalog stream's
+ `import_completed_at` is stamped when every folder is enumerated and no
+ import row is left. A resync must reset `import_started_at` and
+ `import_completed_at` and log "outlook import started", as
+ `outbound/mailbox_pg/lifecycle.rs` does.
+- Import counts for the health snapshot stay in memory (`ImportTally`) and are
+ flushed once a minute to `email_mailbox_import_minutes`; never write per
+ imported message for telemetry.
- Outlook organization commands only run in their link's current
`sync_generation`. Anything that bumps it must rebind unfinished commands as
`pending`, since claims skip `confirming` ones while those still block later
diff --git a/crates/email/src/domain/mailbox.rs b/crates/email/src/domain/mailbox.rs
index 476521d40a..a649521614 100644
--- a/crates/email/src/domain/mailbox.rs
+++ b/crates/email/src/domain/mailbox.rs
@@ -1,9 +1,14 @@
//! Durable mailbox enumeration independent from provider-specific notifications.
use email_api_client::domain::models::{
- EmailApiError, MailFolder, MailboxChangePage, MailboxMessage, ProviderId, StreamToken,
+ EmailApiError, MailFolder, MailboxChangePage, MailboxMessage, ProviderId, StreamPosition,
+ StreamToken,
};
+use progress::{ImportCompleted, ImportTally};
use std::future::Future;
+use std::time::Instant;
+use telemetry::{claim_failed, elapsed_ms, record_deferred, record_done, record_failed, span_int};
+use tracing::{Instrument, Span};
use uuid::Uuid;
pub mod catalog;
@@ -15,8 +20,10 @@ pub mod gmail_history;
pub mod health;
pub mod initialization;
pub mod lifecycle;
+pub mod progress;
pub mod projection;
pub mod settings;
+pub mod telemetry;
#[cfg(test)]
mod test;
pub mod watches;
@@ -29,8 +36,47 @@ pub enum MailboxError {
Provider(#[from] EmailApiError),
#[error("mailbox persistence is unavailable")]
Persistence,
+ /// A database call failed; the cause carries no row data.
+ #[error("mailbox persistence is unavailable: {0}")]
+ Database(DatabaseFailure),
+ /// A projection's downstream effect (event publishing, a queue, another
+ /// service) failed or is not ready; the effect logs its own cause.
+ #[error("a mailbox projection effect is unavailable")]
+ Effect,
#[error("mailbox generation or worker lease changed")]
Stale,
+ /// The provider no longer reports the message where it was; a later
+ /// attempt either finds it or settles that it is gone.
+ #[error("mailbox message location has not settled")]
+ LocationUnsettled,
+}
+
+/// What failed underneath a database call. The database's own message is
+/// left out because it can quote row data.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum DatabaseFailure {
+ /// The database rejected the statement with this SQLSTATE.
+ Sqlstate(String),
+ /// No pooled connection became free in time.
+ PoolTimedOut,
+ /// The connection failed or closed.
+ Connection,
+ /// A returned value could not be decoded.
+ Decode,
+ /// Any other driver failure.
+ Other,
+}
+
+impl std::fmt::Display for DatabaseFailure {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ match self {
+ Self::Sqlstate(code) => write!(f, "sqlstate {code}"),
+ Self::PoolTimedOut => f.write_str("connection pool timed out"),
+ Self::Connection => f.write_str("connection failed"),
+ Self::Decode => f.write_str("decode failed"),
+ Self::Other => f.write_str("driver error"),
+ }
+ }
}
/// Both generations are resolved from storage, never selected by the client.
@@ -70,6 +116,15 @@ pub struct CatalogLease {
pub fence: i64,
}
+/// What a committed page changed.
+#[derive(Debug, Default)]
+pub struct CommittedPage {
+ /// Reconciliation rows the page queued or requeued.
+ pub queued: u64,
+ /// Set when this page finished the mailbox's import run.
+ pub import_completed: Option<ImportCompleted>,
+}
+
pub enum SyncLease<'a> {
Catalog(&'a CatalogLease),
Stream(&'a StreamLease),
@@ -89,7 +144,7 @@ pub trait MailboxSyncRepository: Send + Sync + 'static {
&self,
lease: &CatalogLease,
folders: &[MailFolder],
- ) -> impl Future<Output = Result<(), MailboxError>> + Send;
+ ) -> impl Future<Output = Result<Option<ImportCompleted>, MailboxError>> + Send;
fn release_catalog(
&self,
lease: &CatalogLease,
@@ -105,7 +160,7 @@ pub trait MailboxSyncRepository: Send + Sync + 'static {
&self,
lease: &StreamLease,
page: &MailboxChangePage,
- ) -> impl Future<Output = Result<(), MailboxError>> + Send;
+ ) -> impl Future<Output = Result<CommittedPage, MailboxError>> + Send;
/// Reset only this expired stream; this operation never removes messages.
fn reset_stream(
&self,
@@ -125,10 +180,11 @@ pub trait MailboxSyncRepository: Send + Sync + 'static {
mailbox: MailboxKey,
) -> impl Future<Output = Result<Vec<MailFolder>, MailboxError>> + Send;
/// Finishes only the leased revision. A newer wakeup must remain pending.
+ /// Reports the import run this message finished, once per run.
fn complete_message(
&self,
lease: &MessageLease,
- ) -> impl Future<Output = Result<(), MailboxError>> + Send;
+ ) -> impl Future<Output = Result<Option<ImportCompleted>, MailboxError>> + Send;
fn release_message(
&self,
lease: &MessageLease,
@@ -153,6 +209,8 @@ pub trait MailboxGateway: Send + Sync + 'static {
message: &ProviderId,
folders: &[MailFolder],
) -> impl Future<Output = Result<Option<MailboxMessage>, EmailApiError>> + Send;
+ /// Provider requests admitted for the mailbox's sync generation, when known.
+ fn admitted_requests(&self, mailbox: MailboxKey) -> impl Future<Output = Option<u64>> + Send;
}
/// The shared ingestion path writes content, normalized state and a projection
@@ -173,6 +231,7 @@ pub struct MailboxSyncService<R, P, I> {
repo: R,
provider: P,
ingest: I,
+ imports: ImportTally,
}
impl<R, P, I> MailboxSyncService<R, P, I> {
@@ -181,8 +240,14 @@ impl<R, P, I> MailboxSyncService<R, P, I> {
repo,
provider,
ingest,
+ imports: ImportTally::default(),
}
}
+
+ /// The in-memory import counts the health loop flushes.
+ pub fn import_tally(&self) -> ImportTally {
+ self.imports.clone()
+ }
}
impl<R: MailboxSyncRepository, P: MailboxGateway, I: MailboxIngest> MailboxSyncService<R, P, I> {
@@ -190,37 +255,97 @@ impl<R: MailboxSyncRepository, P: MailboxGateway, I: MailboxIngest> MailboxSyncS
let Some(lease) = self
.repo
.claim_catalog(macro_uuid::generate_uuid_v7())
- .await?
+ .await
+ .map_err(|error| claim_failed("discover", error))?
else {
return Ok(false);
};
+ let span = tracing::info_span!(
+ "outlook.sync.discover",
+ mailbox.link_id = %lease.mailbox.link_id,
+ mailbox.sync_generation = lease.mailbox.sync_generation,
+ outlook.stream_kind = "folder_catalog",
+ outlook.folders = tracing::field::Empty,
+ outlook.outcome = tracing::field::Empty,
+ outlook.defer_reason = tracing::field::Empty,
+ outlook.defer_seconds = tracing::field::Empty,
+ otel.status_description = tracing::field::Empty,
+ );
+ self.discover(&lease).instrument(span).await
+ }
+
+ async fn discover(&self, lease: &CatalogLease) -> Result<bool, MailboxError> {
+ let span = Span::current();
let outcome = maintain_lease(
async { Ok(self.provider.folders(lease.mailbox).await?) },
- || self.repo.renew(SyncLease::Catalog(&lease)),
+ || self.repo.renew(SyncLease::Catalog(lease)),
)
.await;
- match outcome {
- Ok(folders) => self.repo.commit_catalog(&lease, &folders).await?,
+ let result = match outcome {
+ Ok(folders) => {
+ span.record("outlook.folders", span_int(folders.len()));
+ self.repo.commit_catalog(lease, &folders).await
+ }
Err(MailboxError::Provider(error)) => {
+ let delay = retry_seconds(&error);
+ let error = MailboxError::from(error);
self.repo
- .release_catalog(&lease, retry_seconds(&error))
- .await?;
- return Err(error.into());
+ .release_catalog(lease, delay)
+ .await
+ .inspect_err(|release| {
+ record_failed(&span, lease.mailbox.link_id, release);
+ })?;
+ record_deferred(&span, lease.mailbox.link_id, &error, delay);
+ return Err(error);
+ }
+ Err(error) => Err(error),
+ };
+ match result {
+ Ok(import_completed) => {
+ record_done(&span, "imported");
+ if let Some(completed) = import_completed {
+ self.report_import_completed(&completed).await;
+ }
+ Ok(true)
+ }
+ Err(error) => {
+ record_failed(&span, lease.mailbox.link_id, &error);
+ Err(error)
}
- Err(error) => return Err(error),
}
- Ok(true)
}
+
/// Process one bounded provider page. Notifications only make streams due;
/// periodic polling recovers missed, duplicate and out-of-order notifications.
pub async fn enumerate_once(&self) -> Result<bool, MailboxError> {
let Some(lease) = self
.repo
.claim_stream(macro_uuid::generate_uuid_v7())
- .await?
+ .await
+ .map_err(|error| claim_failed("enumerate", error))?
else {
return Ok(false);
};
+ let span = tracing::info_span!(
+ "outlook.sync.enumerate",
+ mailbox.link_id = %lease.mailbox.link_id,
+ mailbox.sync_generation = lease.mailbox.sync_generation,
+ outlook.stream_kind = "mail_folder",
+ outlook.page_items = tracing::field::Empty,
+ outlook.rows_queued = tracing::field::Empty,
+ outlook.delta_advanced = false,
+ outlook.folder_initial_complete = lease.initial_complete,
+ outlook.cursor_reset = false,
+ outlook.outcome = tracing::field::Empty,
+ outlook.defer_reason = tracing::field::Empty,
+ outlook.defer_seconds = tracing::field::Empty,
+ otel.status_description = tracing::field::Empty,
+ );
+ self.enumerate(&lease).instrument(span).await
+ }
+
+ async fn enumerate(&self, lease: &StreamLease) -> Result<bool, MailboxError> {
+ let span = Span::current();
let outcome = maintain_lease(
async {
Ok(self
@@ -228,61 +353,151 @@ impl<R: MailboxSyncRepository, P: MailboxGateway, I: MailboxIngest> MailboxSyncS
.changes(lease.mailbox, &lease.folder, lease.position.as_ref())
.await?)
},
- || self.repo.renew(SyncLease::Stream(&lease)),
+ || self.repo.renew(SyncLease::Stream(lease)),
)
.await;
- match outcome {
- Ok(page) => self.repo.commit_page(&lease, &page).await?,
+ let result = match outcome {
+ Ok(page) => {
+ span.record(
+ "outlook.page_items",
+ span_int(page.changed.len() + page.removed.len()),
+ );
+ let checkpoint = matches!(page.position, StreamPosition::Checkpoint(_));
+ self.repo.commit_page(lease, &page).await.map(|committed| {
+ span.record("outlook.rows_queued", span_int(committed.queued));
+ span.record("outlook.delta_advanced", true);
+ span.record(
+ "outlook.folder_initial_complete",
+ lease.initial_complete || checkpoint,
+ );
+ committed.import_completed
+ })
+ }
Err(MailboxError::Provider(EmailApiError::OutdatedCursor)) => {
- self.repo.reset_stream(&lease).await?
+ span.record("outlook.cursor_reset", true);
+ self.repo.reset_stream(lease).await.map(|()| None)
}
Err(MailboxError::Provider(error)) => {
+ let delay = retry_seconds(&error);
+ let error = MailboxError::from(error);
self.repo
- .release_stream(&lease, retry_seconds(&error))
- .await?;
- return Err(error.into());
+ .release_stream(lease, delay)
+ .await
+ .inspect_err(|release| {
+ record_failed(&span, lease.mailbox.link_id, release);
+ })?;
+ record_deferred(&span, lease.mailbox.link_id, &error, delay);
+ return Err(error);
+ }
+ Err(error) => Err(error),
+ };
+ match result {
+ Ok(import_completed) => {
+ record_done(&span, "imported");
+ if let Some(completed) = import_completed {
+ self.report_import_completed(&completed).await;
+ }
+ Ok(true)
+ }
+ Err(error) => {
+ record_failed(&span, lease.mailbox.link_id, &error);
+ Err(error)
}
- Err(error) => return Err(error),
}
- Ok(true)
}
pub async fn reconcile_once(&self) -> Result<bool, MailboxError> {
let Some(lease) = self
.repo
.claim_message(macro_uuid::generate_uuid_v7())
- .await?
+ .await
+ .map_err(|error| claim_failed("reconcile", error))?
else {
return Ok(false);
};
+ let span = tracing::info_span!(
+ "outlook.sync.reconcile",
+ mailbox.link_id = %lease.mailbox.link_id,
+ mailbox.sync_generation = lease.mailbox.sync_generation,
+ outlook.is_import = lease.is_import,
+ outlook.attempts = lease.attempts,
+ outlook.graph_calls = tracing::field::Empty,
+ outlook.attachments = tracing::field::Empty,
+ outlook.db_write_ms = tracing::field::Empty,
+ outlook.outcome = tracing::field::Empty,
+ outlook.defer_reason = tracing::field::Empty,
+ outlook.defer_seconds = tracing::field::Empty,
+ otel.status_description = tracing::field::Empty,
+ );
+ self.reconcile(&lease).instrument(span).await
+ }
+
+ async fn reconcile(&self, lease: &MessageLease) -> Result<bool, MailboxError> {
+ let span = Span::current();
let operation = async {
let folders = self.repo.folders(lease.mailbox).await?;
- match self
- .provider
- .message(lease.mailbox, &lease.provider_id, &folders)
- .await?
- {
- Some(snapshot) => self.ingest.ingest(&lease, snapshot).await?,
- None if lease.attempts < 3 => {
- return Err(EmailApiError::Transient {
- message: "mailbox message location has not settled".into(),
- }
- .into());
+ let (snapshot, graph_calls) = email_api_client::domain::request_count::counted(
+ self.provider
+ .message(lease.mailbox, &lease.provider_id, &folders),
+ )
+ .await;
+ span.record("outlook.graph_calls", i64::from(graph_calls));
+ let started = Instant::now();
+ let outcome = match snapshot? {
+ Some(snapshot) => {
+ span.record(
+ "outlook.attachments",
+ span_int(snapshot.content.message.attachments.len()),
+ );
+ self.ingest.ingest(lease, snapshot).await?;
+ "imported"
}
- None => self.ingest.absent(&lease).await?,
+ None if lease.attempts < 3 => return Err(MailboxError::LocationUnsettled),
+ None => {
+ self.ingest.absent(lease).await?;
+ "absent"
+ }
+ };
+ let import_completed = self.repo.complete_message(lease).await?;
+ span.record("outlook.db_write_ms", elapsed_ms(started));
+ Ok((outcome, import_completed))
+ };
+ let result = maintain_lease(operation, || self.repo.renew(SyncLease::Message(lease))).await;
+ let error = match result {
+ Ok((outcome, import_completed)) => {
+ record_done(&span, outcome);
+ if outcome == "imported" {
+ self.imports
+ .record(lease.mailbox.link_id, chrono::Utc::now());
+ }
+ if let Some(completed) = import_completed {
+ self.report_import_completed(&completed).await;
+ }
+ return Ok(true);
}
- self.repo.complete_message(&lease).await
+ Err(error) => error,
};
- let result =
- maintain_lease(operation, || self.repo.renew(SyncLease::Message(&lease))).await;
- if let Err(error) = &result {
- let delay = match error {
- MailboxError::Provider(error) => retry_seconds(error),
- _ => 30,
- };
- self.repo.release_message(&lease, delay).await?;
+ let delay = match &error {
+ MailboxError::Provider(error) => retry_seconds(error),
+ _ => 30,
+ };
+ self.repo
+ .release_message(lease, delay)
+ .await
+ .inspect_err(|release| {
+ record_failed(&span, lease.mailbox.link_id, release);
+ })?;
+ if matches!(error, MailboxError::Stale) {
+ record_failed(&span, lease.mailbox.link_id, &error);
+ } else {
+ record_deferred(&span, lease.mailbox.link_id, &error, delay);
}
- result.map(|()| true)
+ Err(error)
+ }
+
+ async fn report_import_completed(&self, completed: &ImportCompleted) {
+ let graph_calls = self.provider.admitted_requests(completed.mailbox).await;
+ progress::log_import_completed(completed, graph_calls);
}
}
diff --git a/crates/email/src/domain/mailbox/commands.rs b/crates/email/src/domain/mailbox/commands.rs
index abb189503a..1f2c5f7d01 100644
--- a/crates/email/src/domain/mailbox/commands.rs
+++ b/crates/email/src/domain/mailbox/commands.rs
@@ -122,24 +122,31 @@ impl<R: MailboxCommandRepository, G: MailboxCommandGateway> MailboxCommandServic
let Some(lease) = self
.repo
.claim_command(macro_uuid::generate_uuid_v7())
- .await?
+ .await
+ .map_err(|error| super::telemetry::claim_failed("commands", error))?
else {
return Ok(false);
};
- let result =
- super::maintain_lease(self.execute(&lease), || self.repo.renew_command(&lease)).await;
- if let Err(error) = &result {
- self.repo
- .release_command(
- &lease,
- match error {
- MailboxError::Provider(e) => super::retry_seconds(e),
- _ => 30,
- },
- )
- .await?;
+ let link_id = lease.mailbox.link_id;
+ async {
+ let result =
+ super::maintain_lease(self.execute(&lease), || self.repo.renew_command(&lease))
+ .await;
+ if let Err(error) = &result {
+ self.repo
+ .release_command(
+ &lease,
+ match error {
+ MailboxError::Provider(e) => super::retry_seconds(e),
+ _ => 30,
+ },
+ )
+ .await?;
+ }
+ result.map(|()| true)
}
- result.map(|()| true)
+ .await
+ .inspect_err(|error| super::telemetry::log_unit_error(link_id, error))
}
async fn execute(&self, lease: &CommandLease) -> Result<(), MailboxError> {
diff --git a/crates/email/src/domain/mailbox/contacts.rs b/crates/email/src/domain/mailbox/contacts.rs
index cb5b0cae1b..d74c1b5a8f 100644
--- a/crates/email/src/domain/mailbox/contacts.rs
+++ b/crates/email/src/domain/mailbox/contacts.rs
@@ -141,33 +141,39 @@ impl<R: AddressBookRepository, G: AddressBookGateway, S: ContactPhotoStorage>
let Some(work) = self
.repository
.claim(macro_uuid::generate_uuid_v7())
- .await?
+ .await
+ .map_err(|error| super::telemetry::claim_failed("contacts", error))?
else {
return Ok(false);
};
- let result = maintain_lease(self.sync(&work), || self.repository.renew(&work)).await;
- match result {
- Err(MailboxError::Provider(EmailApiError::OutdatedCursor))
- if work.kind == AddressBookWorkKind::Folder =>
- {
- self.repository.reset(&work).await?
+ let link_id = work.stream.mailbox.link_id;
+ async {
+ let result = maintain_lease(self.sync(&work), || self.repository.renew(&work)).await;
+ match result {
+ Err(MailboxError::Provider(EmailApiError::OutdatedCursor))
+ if work.kind == AddressBookWorkKind::Folder =>
+ {
+ self.repository.reset(&work).await?
+ }
+ Err(MailboxError::Provider(EmailApiError::NotFound))
+ if work.kind == AddressBookWorkKind::Folder =>
+ {
+ self.repository.retire_folder(&work).await?
+ }
+ Err(MailboxError::Provider(error)) => {
+ // Consent/policy failures are confined to this capability.
+ self.repository
+ .release(&work, retry_seconds(&error).max(60))
+ .await?;
+ return Err(error.into());
+ }
+ Err(error) => return Err(error),
+ Ok(()) => {}
}
- Err(MailboxError::Provider(EmailApiError::NotFound))
- if work.kind == AddressBookWorkKind::Folder =>
- {
- self.repository.retire_folder(&work).await?
- }
- Err(MailboxError::Provider(error)) => {
- // Consent/policy failures are confined to this capability.
- self.repository
- .release(&work, retry_seconds(&error).max(60))
- .await?;
- return Err(error.into());
- }
- Err(error) => return Err(error),
- Ok(()) => {}
+ Ok(true)
}
- Ok(true)
+ .await
+ .inspect_err(|error| super::telemetry::log_unit_error(link_id, error))
}
async fn sync(&self, work: &AddressBookWork) -> Result<(), MailboxError> {
match work.kind {
@@ -202,41 +208,47 @@ impl<R: AddressBookRepository, G: AddressBookGateway, S: ContactPhotoStorage>
let Some(lease) = self
.repository
.claim_photo(macro_uuid::generate_uuid_v7())
- .await?
+ .await
+ .map_err(|error| super::telemetry::claim_failed("contact_photos", error))?
else {
return Ok(false);
};
- let operation = async {
- let photo = self
- .gateway
- .photo(
- lease.mailbox,
- lease.folder.as_ref().map(|folder| (folder, &lease.contact)),
- )
- .await?;
- let Some(photo) = photo else {
- return self.repository.commit_photo(&lease, None, None).await;
- };
- let hash = format!("{:x}", Sha256::digest(&photo.bytes));
- let url = if lease.photo_hash.as_deref() == Some(&hash)
- && let Some(url) = &lease.photo_url
- {
- url.clone()
- } else {
- self.storage.store(lease.mailbox, &hash, photo).await?
- };
- self.repository
- .commit_photo(&lease, Some(&hash), Some(&url))
- .await
- };
- let result = maintain_lease(operation, || self.repository.renew_photo(&lease)).await;
- if let Err(error) = &result {
- let delay = match error {
- MailboxError::Provider(error) => retry_seconds(error).max(60),
- _ => 60,
+ let link_id = lease.mailbox.link_id;
+ async {
+ let operation = async {
+ let photo = self
+ .gateway
+ .photo(
+ lease.mailbox,
+ lease.folder.as_ref().map(|folder| (folder, &lease.contact)),
+ )
+ .await?;
+ let Some(photo) = photo else {
+ return self.repository.commit_photo(&lease, None, None).await;
+ };
+ let hash = format!("{:x}", Sha256::digest(&photo.bytes));
+ let url = if lease.photo_hash.as_deref() == Some(&hash)
+ && let Some(url) = &lease.photo_url
+ {
+ url.clone()
+ } else {
+ self.storage.store(lease.mailbox, &hash, photo).await?
+ };
+ self.repository
+ .commit_photo(&lease, Some(&hash), Some(&url))
+ .await
};
- self.repository.release_photo(&lease, delay).await?;
+ let result = maintain_lease(operation, || self.repository.renew_photo(&lease)).await;
+ if let Err(error) = &result {
+ let delay = match error {
+ MailboxError::Provider(error) => retry_seconds(error).max(60),
+ _ => 60,
+ };
+ self.repository.release_photo(&lease, delay).await?;
+ }
+ result.map(|()| true)
}
- result.map(|()| true)
+ .await
+ .inspect_err(|error| super::telemetry::log_unit_error(link_id, error))
}
}
diff --git a/crates/email/src/domain/mailbox/drafts.rs b/crates/email/src/domain/mailbox/drafts.rs
index 02702931a8..d60165d682 100644
--- a/crates/email/src/domain/mailbox/drafts.rs
+++ b/crates/email/src/domain/mailbox/drafts.rs
@@ -50,41 +50,47 @@ impl<R: DraftRepository, G: DraftGateway, M: DraftMaterializer> MailboxDraftServ
let Some(lease) = self
.repo
.claim_draft(macro_uuid::generate_uuid_v7(), self.claim_mode)
- .await?
+ .await
+ .map_err(|error| super::telemetry::claim_failed("drafts", error))?
else {
return Ok(false);
};
- let result =
- super::maintain_lease(self.execute(&lease), || self.repo.renew_draft(&lease)).await;
- match result {
- Ok(()) => Ok(true),
- Err(MailboxError::Provider(EmailApiError::Conflict)) => {
- self.repo.fail_draft(&lease, DraftFailure::Conflict).await?;
- Ok(true)
- }
- Err(MailboxError::Provider(EmailApiError::Forbidden)) => {
- self.repo.fail_draft(&lease, DraftFailure::Denied).await?;
- Ok(true)
- }
- Err(MailboxError::Provider(EmailApiError::Permanent { .. })) => {
- self.repo.fail_draft(&lease, DraftFailure::Invalid).await?;
- Ok(true)
- }
- Err(error) => {
- if !matches!(error, MailboxError::Stale) {
- self.repo
- .release_draft(
- &lease,
- match &error {
- MailboxError::Provider(e) => super::retry_seconds(e),
- _ => 30,
- },
- )
- .await?;
+ let link_id = lease.mailbox.link_id;
+ async {
+ let result =
+ super::maintain_lease(self.execute(&lease), || self.repo.renew_draft(&lease)).await;
+ match result {
+ Ok(()) => Ok(true),
+ Err(MailboxError::Provider(EmailApiError::Conflict)) => {
+ self.repo.fail_draft(&lease, DraftFailure::Conflict).await?;
+ Ok(true)
+ }
+ Err(MailboxError::Provider(EmailApiError::Forbidden)) => {
+ self.repo.fail_draft(&lease, DraftFailure::Denied).await?;
+ Ok(true)
+ }
+ Err(MailboxError::Provider(EmailApiError::Permanent { .. })) => {
+ self.repo.fail_draft(&lease, DraftFailure::Invalid).await?;
+ Ok(true)
+ }
+ Err(error) => {
+ if !matches!(error, MailboxError::Stale) {
+ self.repo
+ .release_draft(
+ &lease,
+ match &error {
+ MailboxError::Provider(e) => super::retry_seconds(e),
+ _ => 30,
+ },
+ )
+ .await?;
+ }
+ Err(error)
}
- Err(error)
}
}
+ .await
+ .inspect_err(|error| super::telemetry::log_unit_error(link_id, error))
}
async fn execute(&self, lease: &DraftLease) -> Result<(), MailboxError> {
diff --git a/crates/email/src/domain/mailbox/progress.rs b/crates/email/src/domain/mailbox/progress.rs
new file mode 100644
index 0000000000..b4126b4e40
--- /dev/null
+++ b/crates/email/src/domain/mailbox/progress.rs
@@ -0,0 +1,182 @@
+//! Import progress: the per-mailbox health snapshot and import lifecycle
+//! events. Telemetry only; nothing here changes what sync does.
+
+use super::{MailboxError, MailboxKey};
+use chrono::{DateTime, Utc};
+use std::collections::HashMap;
+use std::future::Future;
+use std::sync::{Arc, Mutex};
+use uuid::Uuid;
+
+#[cfg(test)]
+mod test;
+
+/// One finished import run of a mailbox's sync generation.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ImportCompleted {
+ pub mailbox: MailboxKey,
+ pub started_at: Option<DateTime<Utc>>,
+ pub completed_at: DateTime<Utc>,
+ /// Provider messages stored for the mailbox when the run finished.
+ pub messages: i64,
+ /// Mail folders the run enumerated.
+ pub folders: i64,
+}
+
+/// Messages one mailbox imported since the last flush.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct MailboxImports {
+ pub imported: u32,
+ pub last_imported_at: DateTime<Utc>,
+}
+
+/// Imports this worker counted, kept in memory so the import path writes
+/// nothing extra. The health loop flushes it once a minute.
+#[derive(Clone, Default)]
+pub struct ImportTally(Arc<Mutex<HashMap<Uuid, MailboxImports>>>);
+
+impl ImportTally {
+ pub fn record(&self, link_id: Uuid, at: DateTime<Utc>) {
+ let Ok(mut imports) = self.0.lock() else {
+ return;
+ };
+ imports
+ .entry(link_id)
+ .and_modify(|entry| {
+ entry.imported = entry.imported.saturating_add(1);
+ entry.last_imported_at = entry.last_imported_at.max(at);
+ })
+ .or_insert(MailboxImports {
+ imported: 1,
+ last_imported_at: at,
+ });
+ }
+
+ fn take(&self) -> HashMap<Uuid, MailboxImports> {
+ self.0
+ .lock()
+ .map(|mut imports| std::mem::take(&mut *imports))
+ .unwrap_or_default()
+ }
+
+ /// Put back counts whose flush failed, so the next flush carries them.
+ fn restore(&self, unflushed: HashMap<Uuid, MailboxImports>) {
+ for (link_id, entry) in unflushed {
+ let Ok(mut imports) = self.0.lock() else {
+ return;
+ };
+ imports
+ .entry(link_id)
+ .and_modify(|current| {
+ current.imported = current.imported.saturating_add(entry.imported);
+ current.last_imported_at = current.last_imported_at.max(entry.last_imported_at);
+ })
+ .or_insert(entry);
+ }
+ }
+}
+
+/// One mailbox's sync backlog, as one health snapshot reports it.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct MailboxBacklog {
+ pub link_id: Uuid,
+ pub sync_generation: i64,
+ /// Messages waiting for reconciliation in the current generation.
+ pub queued: i64,
+ /// Queued messages backed off until a later time.
+ pub parked: i64,
+ /// Queued messages a worker holds right now.
+ pub leased: i64,
+ /// How long the longest-waiting claimable message has been due; 0 if none.
+ pub oldest_waiting_seconds: i64,
+ /// How far away the latest parked message's retry is; 0 if none.
+ pub parked_until_max_seconds: i64,
+ /// Folder and catalog streams whose initial enumeration is unfinished.
+ pub streams_incomplete: i64,
+ pub imported_5m: i64,
+ /// `None` when nothing was imported in the last day.
+ pub seconds_since_last_import: Option<i64>,
+ pub sync_active: bool,
+ pub needs_reauth: bool,
+}
+
+pub trait MailboxProgressRepository: Send + Sync + 'static {
+ /// Add imports counted since the last flush to the current minute.
+ fn record_imports(
+ &self,
+ imports: &HashMap<Uuid, MailboxImports>,
+ ) -> impl Future<Output = Result<(), MailboxError>> + Send;
+ /// One aggregate read: every sync-active Outlook mailbox, and inactive
+ /// ones that still have queued work. Also drops import counts older than
+ /// a day.
+ fn backlogs(&self) -> impl Future<Output = Result<Vec<MailboxBacklog>, MailboxError>> + Send;
+}
+
+pub struct MailboxProgressService<R> {
+ repo: R,
+ tally: ImportTally,
+}
+
+impl<R: MailboxProgressRepository> MailboxProgressService<R> {
+ /// `tally` is the one the worker's sync service counts imports into.
+ pub fn new(repo: R, tally: ImportTally) -> Self {
+ Self { repo, tally }
+ }
+
+ /// Persist this worker's import counts. Every worker calls this each minute.
+ pub async fn flush(&self) -> Result<(), MailboxError> {
+ let imports = self.tally.take();
+ if imports.is_empty() {
+ return Ok(());
+ }
+ let result = self.repo.record_imports(&imports).await;
+ if result.is_err() {
+ self.tally.restore(imports);
+ }
+ result
+ }
+
+ /// Log one health snapshot per mailbox. One worker calls this per minute.
+ pub async fn report(&self) -> Result<usize, MailboxError> {
+ let backlogs = self.repo.backlogs().await?;
+ for backlog in &backlogs {
+ tracing::info!(
+ link_id = %backlog.link_id,
+ sync_generation = backlog.sync_generation,
+ queued = backlog.queued,
+ parked = backlog.parked,
+ leased = backlog.leased,
+ oldest_waiting_seconds = backlog.oldest_waiting_seconds,
+ parked_until_max_seconds = backlog.parked_until_max_seconds,
+ streams_incomplete = backlog.streams_incomplete,
+ imported_5m = backlog.imported_5m,
+ seconds_since_last_import = backlog.seconds_since_last_import,
+ sync_active = backlog.sync_active,
+ needs_reauth = backlog.needs_reauth,
+ "outlook mailbox health"
+ );
+ }
+ Ok(backlogs.len())
+ }
+}
+
+/// Logged once an import run's last folder and last imported message settle.
+pub(super) fn log_import_completed(completed: &ImportCompleted, graph_calls: Option<u64>) {
+ let duration_seconds = completed
+ .started_at
+ .map(|started| (completed.completed_at - started).num_seconds().max(0));
+ tracing::info!(
+ link_id = %completed.mailbox.link_id,
+ sync_generation = completed.mailbox.sync_generation,
+ duration_seconds,
+ messages = completed.messages,
+ folders = completed.folders,
+ graph_calls = graph_calls.and_then(|calls| i64::try_from(calls).ok()),
+ "outlook import completed"
+ );
+}
+
+/// Logged when a mailbox starts an import run in a sync generation.
+pub fn log_import_started(link_id: Uuid, sync_generation: i64) {
+ tracing::info!(%link_id, sync_generation, "outlook import started");
+}
diff --git a/crates/email/src/domain/mailbox/progress/test.rs b/crates/email/src/domain/mailbox/progress/test.rs
new file mode 100644
index 0000000000..ea14d1ff48
--- /dev/null
+++ b/crates/email/src/domain/mailbox/progress/test.rs
@@ -0,0 +1,52 @@
+use super::*;
+use std::sync::atomic::{AtomicBool, Ordering};
+
+/// Fails the first flush, then keeps what later flushes write.
+#[derive(Default)]
+struct FlakyStore {
+ failed_once: AtomicBool,
+ written: Mutex<Vec<HashMap<Uuid, MailboxImports>>>,
+}
+
+impl MailboxProgressRepository for FlakyStore {
+ async fn record_imports(
+ &self,
+ imports: &HashMap<Uuid, MailboxImports>,
+ ) -> Result<(), MailboxError> {
+ if !self.failed_once.swap(true, Ordering::SeqCst) {
+ return Err(MailboxError::Persistence);
+ }
+ self.written.lock().unwrap().push(imports.clone());
+ Ok(())
+ }
+ async fn backlogs(&self) -> Result<Vec<MailboxBacklog>, MailboxError> {
+ Ok(Vec::new())
+ }
+}
+
+#[tokio::test]
+async fn imports_counted_before_a_failed_flush_reach_the_next_one() {
+ let link = Uuid::now_v7();
+ let early = Utc::now() - chrono::Duration::seconds(30);
+ let late = Utc::now();
+ let tally = ImportTally::default();
+ let service = MailboxProgressService::new(FlakyStore::default(), tally.clone());
+ tally.record(link, early);
+ tally.record(link, early);
+
+ assert!(service.flush().await.is_err());
+ tally.record(link, late);
+ service.flush().await.unwrap();
+
+ let written = service.repo.written.lock().unwrap();
+ assert_eq!(
+ written.as_slice(),
+ [HashMap::from([(
+ link,
+ MailboxImports {
+ imported: 3,
+ last_imported_at: late,
+ }
+ )])]
+ );
+}
diff --git a/crates/email/src/domain/mailbox/projection.rs b/crates/email/src/domain/mailbox/projection.rs
index 9b0b1622d5..88010affe6 100644
--- a/crates/email/src/domain/mailbox/projection.rs
+++ b/crates/email/src/domain/mailbox/projection.rs
@@ -1,5 +1,8 @@
//! Durable downstream projection and live-mail notification policy.
+use super::telemetry::{
+ claim_failed, elapsed_ms, record_deferred, record_done, record_failed, span_int,
+};
use super::{MailboxError, MailboxKey};
use crate::domain::events::*;
use crate::domain::models::calendar_invitation::CalendarInvitation;
@@ -7,7 +10,8 @@ use email_api_client::domain::models::CalendarPart;
use macro_user_id::user_id::MacroUserIdStr;
use models_email::service::{link::Link, message::Message};
use serde::{Deserialize, Serialize};
-use std::{collections::HashSet, future::Future};
+use std::{collections::HashSet, future::Future, time::Instant};
+use tracing::{Instrument, Span};
use uuid::Uuid;
#[cfg(test)]
@@ -65,6 +69,8 @@ pub struct ProjectionLease {
pub mailbox: MailboxKey,
pub event: ProjectionEvent,
pub from_current_generation: bool,
+ /// Claims of this event so far, including this one.
+ pub attempts: i32,
}
pub struct ProjectedMessage {
@@ -124,10 +130,12 @@ pub trait MailboxProjectionRepository: Send + Sync + 'static {
&self,
lease: &ProjectionLease,
) -> impl Future<Output = Result<Option<ProjectionContext>, MailboxError>> + Send;
+ /// Deletes a delivered event, or with `retry_after_seconds` makes it due
+ /// again after that delay.
fn finish_projection(
&self,
lease: &ProjectionLease,
- success: bool,
+ retry_after_seconds: Option<u32>,
) -> impl Future<Output = Result<(), MailboxError>> + Send;
}
@@ -202,18 +210,68 @@ impl<R: MailboxProjectionRepository, E: MailboxProjectionEffects> MailboxProject
let Some(lease) = self
.repo
.claim_projection(macro_uuid::generate_uuid_v7())
- .await?
+ .await
+ .map_err(|error| claim_failed("project", error))?
else {
return Ok(false);
};
- let result =
- super::maintain_lease(self.project(&lease), || self.repo.renew_projection(&lease))
- .await;
- self.repo.finish_projection(&lease, result.is_ok()).await?;
- result.map(|()| true)
+ let span = tracing::info_span!(
+ "outlook.sync.project",
+ mailbox.link_id = %lease.mailbox.link_id,
+ mailbox.sync_generation = lease.mailbox.sync_generation,
+ outlook.event = lease.event.kind(),
+ outlook.attempts = lease.attempts,
+ outlook.crm_jobs = tracing::field::Empty,
+ outlook.refreshes = tracing::field::Empty,
+ outlook.reindex_events = tracing::field::Empty,
+ outlook.attachment_upload_ms = tracing::field::Empty,
+ outlook.outcome = tracing::field::Empty,
+ outlook.defer_reason = tracing::field::Empty,
+ outlook.defer_seconds = tracing::field::Empty,
+ otel.status_description = tracing::field::Empty,
+ );
+ self.project_leased(&lease).instrument(span).await
}
- async fn project(&self, lease: &ProjectionLease) -> Result<(), MailboxError> {
+ async fn project_leased(&self, lease: &ProjectionLease) -> Result<bool, MailboxError> {
+ let span = Span::current();
+ let mut effects = EffectCounts::default();
+ let result = super::maintain_lease(self.project(lease, &mut effects), || {
+ self.repo.renew_projection(lease)
+ })
+ .await;
+ span.record("outlook.crm_jobs", span_int(effects.crm_jobs));
+ span.record("outlook.refreshes", i64::from(effects.refreshes));
+ span.record("outlook.reindex_events", i64::from(effects.reindex_events));
+ span.record("outlook.attachment_upload_ms", effects.attachment_upload_ms);
+ let retry_after = result
+ .is_err()
+ .then(|| projection_retry_seconds(lease.attempts));
+ self.repo
+ .finish_projection(lease, retry_after)
+ .await
+ .inspect_err(|finish| record_failed(&span, lease.mailbox.link_id, finish))?;
+ match (result, retry_after) {
+ (Ok(()), _) => {
+ record_done(&span, "imported");
+ Ok(true)
+ }
+ (Err(error), Some(delay)) if !matches!(error, MailboxError::Stale) => {
+ record_deferred(&span, lease.mailbox.link_id, &error, delay);
+ Err(error)
+ }
+ (Err(error), _) => {
+ record_failed(&span, lease.mailbox.link_id, &error);
+ Err(error)
+ }
+ }
+ }
+
+ async fn project(
+ &self,
+ lease: &ProjectionLease,
+ effects: &mut EffectCounts,
+ ) -> Result<(), MailboxError> {
let Some(context) = self.repo.projection_context(lease).await? else {
return Ok(());
};
@@ -257,11 +315,13 @@ impl<R: MailboxProjectionRepository, E: MailboxProjectionEffects> MailboxProject
.with_event_id(lease.id),
)
.await?;
+ effects.reindex_events += 1;
}
if *self_photo {
self.effects
.photo_refresh(context.link.id, &context.viewers)
.await?;
+ effects.refreshes += 1;
}
}
ProjectionEvent::LinkConnected { is_new, .. } => {
@@ -288,14 +348,16 @@ impl<R: MailboxProjectionRepository, E: MailboxProjectionEffects> MailboxProject
.as_ref()
.is_some_and(|message| message.is_present)
{
+ let started = Instant::now();
self.effects
.attachments(lease.mailbox, &context, true)
.await?;
+ effects.attachment_upload_ms += elapsed_ms(started);
}
return Ok(());
}
ProjectionEvent::Absent { thread_id, .. } => {
- self.reindex(&context.link, *thread_id).await?;
+ self.reindex(&context.link, *thread_id, effects).await?;
}
ProjectionEvent::Organization {
thread_id,
@@ -310,7 +372,7 @@ impl<R: MailboxProjectionRepository, E: MailboxProjectionEffects> MailboxProject
)
.await?;
}
- self.reindex(&context.link, *thread_id).await?;
+ self.reindex(&context.link, *thread_id, effects).await?;
}
ProjectionEvent::Message {
actor,
@@ -336,7 +398,10 @@ impl<R: MailboxProjectionRepository, E: MailboxProjectionEffects> MailboxProject
.invitations(*message_id, &invitation_snapshots(calendar_parts))
.await?;
}
- self.effects.contacts(correspondence(&context)).await?;
+ let correspondence = correspondence(&context);
+ let crm_jobs = correspondence.contacts.len();
+ self.effects.contacts(correspondence).await?;
+ effects.crm_jobs += crm_jobs;
}
if !import
&& let Some(event) =
@@ -350,24 +415,34 @@ impl<R: MailboxProjectionRepository, E: MailboxProjectionEffects> MailboxProject
}
}
}
- self.reindex(&context.link, *thread_id).await?;
+ self.reindex(&context.link, *thread_id, effects).await?;
self.effects
.refresh(context.link.id, &context.viewers)
.await?;
+ effects.refreshes += 1;
if current.is_present {
+ let started = Instant::now();
self.effects
.attachments(lease.mailbox, &context, import)
.await?;
+ effects.attachment_upload_ms += elapsed_ms(started);
}
return Ok(());
}
}
self.effects
.refresh(context.link.id, &context.viewers)
- .await
+ .await?;
+ effects.refreshes += 1;
+ Ok(())
}
- async fn reindex(&self, link: &Link, thread: Uuid) -> Result<(), MailboxError> {
+ async fn reindex(
+ &self,
+ link: &Link,
+ thread: Uuid,
+ effects: &mut EffectCounts,
+ ) -> Result<(), MailboxError> {
self.effects
.publish(EmailMacroEvent::threads_reindex_requested(
ThreadsReindexRequestedMetadata {
@@ -377,7 +452,42 @@ impl<R: MailboxProjectionRepository, E: MailboxProjectionEffects> MailboxProject
reason: ThreadsReindexReason::ProviderChanged,
},
))
- .await
+ .await?;
+ effects.reindex_events += 1;
+ Ok(())
+ }
+}
+
+/// Downstream effects one projection produced, for its span.
+#[derive(Default)]
+struct EffectCounts {
+ crm_jobs: usize,
+ refreshes: u32,
+ reindex_events: u32,
+ attachment_upload_ms: i64,
+}
+
+/// A failed projection backs off 30 s per attempt, up to ten minutes.
+fn projection_retry_seconds(attempts: i32) -> u32 {
+ u32::try_from(attempts)
+ .unwrap_or(1)
+ .saturating_mul(30)
+ .clamp(30, 600)
+}
+
+impl ProjectionEvent {
+ /// The event's kind, as its payload's `kind` tag names it.
+ fn kind(&self) -> &'static str {
+ match self {
+ Self::LinkChanged => "link_changed",
+ Self::AttachmentRecheck { .. } => "attachment_recheck",
+ Self::ReauthorizationRequired => "reauthorization_required",
+ Self::ContactsChanged { .. } => "contacts_changed",
+ Self::Message { .. } => "message",
+ Self::Absent { .. } => "absent",
+ Self::Organization { .. } => "organization",
+ Self::LinkConnected { .. } => "link_connected",
+ }
}
}
diff --git a/crates/email/src/domain/mailbox/projection/test.rs b/crates/email/src/domain/mailbox/projection/test.rs
index c9f9456d41..d17dabf9f0 100644
--- a/crates/email/src/domain/mailbox/projection/test.rs
+++ b/crates/email/src/domain/mailbox/projection/test.rs
@@ -90,6 +90,7 @@ fn notification_retry_preserves_ids_and_staff_push_policy() {
is_new: true,
},
from_current_generation: true,
+ attempts: 1,
};
let first = notifications(&lease, &context);
let retry = notifications(&lease, &context);
diff --git a/crates/email/src/domain/mailbox/settings.rs b/crates/email/src/domain/mailbox/settings.rs
index 06c96b1916..e9b0c518b5 100644
--- a/crates/email/src/domain/mailbox/settings.rs
+++ b/crates/email/src/domain/mailbox/settings.rs
@@ -298,24 +298,30 @@ impl<R: MailboxSettingsRepository, G: MailboxSettingsGateway> MailboxSettingsSer
self.outlook_sync,
self.outlook_writes,
)
- .await?
+ .await
+ .map_err(|error| super::telemetry::claim_failed("settings", error))?
else {
return Ok(false);
};
- let result = super::maintain_lease(self.execute(&lease), || {
- self.repository.renew_settings(&lease)
- })
- .await;
- if let Err(error) = &result {
- let delay = match error {
- MailboxError::Provider(e) => super::retry_seconds(e),
- _ => 30,
- };
- self.repository
- .finish_settings(&lease, false, delay, Some("provider_settings_unavailable"))
- .await?;
+ let link_id = lease.mailbox.key.link_id;
+ async {
+ let result = super::maintain_lease(self.execute(&lease), || {
+ self.repository.renew_settings(&lease)
+ })
+ .await;
+ if let Err(error) = &result {
+ let delay = match error {
+ MailboxError::Provider(e) => super::retry_seconds(e),
+ _ => 30,
+ };
+ self.repository
+ .finish_settings(&lease, false, delay, Some("provider_settings_unavailable"))
+ .await?;
+ }
+ result.map(|()| true)
}
- result.map(|()| true)
+ .await
+ .inspect_err(|error| super::telemetry::log_unit_error(link_id, error))
}
async fn execute(&self, lease: &SettingsLease) -> Result<(), MailboxError> {
match lease.kind.as_str() {
diff --git a/crates/email/src/domain/mailbox/telemetry.rs b/crates/email/src/domain/mailbox/telemetry.rs
new file mode 100644
index 0000000000..2a7a12b76a
--- /dev/null
+++ b/crates/email/src/domain/mailbox/telemetry.rs
@@ -0,0 +1,105 @@
+//! Outcomes, deferral reasons and logs shared by the Outlook sync units
+//! (`outlook.sync.*` spans). Each claimed unit of work gets one root span;
+//! idle polls get none.
+
+use super::MailboxError;
+use email_api_client::domain::models::{EmailApiError, GateLimit, RateLimitOrigin};
+use tracing::Span;
+use uuid::Uuid;
+
+/// Why a unit of sync work was deferred or failed (`outlook.defer_reason`).
+pub fn defer_reason(error: &MailboxError) -> &'static str {
+ match error {
+ MailboxError::Provider(EmailApiError::RateLimited { origin, .. }) => match origin {
+ RateLimitOrigin::Gate(GateLimit::Window) => "gate_window",
+ RateLimitOrigin::Gate(GateLimit::Concurrency) => "gate_concurrency",
+ RateLimitOrigin::Gate(GateLimit::Cooldown) => "gate_cooldown",
+ RateLimitOrigin::Provider { .. } => "graph_throttled",
+ RateLimitOrigin::Local => "other",
+ },
+ MailboxError::Provider(EmailApiError::CredentialsUnavailable { .. }) => "token_unavailable",
+ MailboxError::Provider(EmailApiError::AuthRequired | EmailApiError::Forbidden) => {
+ "auth_required"
+ }
+ MailboxError::Persistence | MailboxError::Database(_) => "persistence",
+ MailboxError::Effect => "effect",
+ MailboxError::LocationUnsettled => "location_unsettled",
+ _ => "other",
+ }
+}
+
+/// A refusal by the shared request gate is routine back-pressure, so it is
+/// logged at DEBUG; the span still records it.
+fn is_gate_refusal(error: &MailboxError) -> bool {
+ matches!(
+ error,
+ MailboxError::Provider(EmailApiError::RateLimited {
+ origin: RateLimitOrigin::Gate(_),
+ ..
+ })
+ )
+}
+
+/// The unit committed its work: `imported` or, for a vanished message, `absent`.
+pub(super) fn record_done(span: &Span, outcome: &'static str) {
+ span.record("outlook.outcome", outcome);
+}
+
+/// The unit released its work to run again after `delay_seconds`.
+pub(super) fn record_deferred(
+ span: &Span,
+ link_id: Uuid,
+ error: &MailboxError,
+ delay_seconds: u32,
+) {
+ let reason = defer_reason(error);
+ span.record("outlook.outcome", "deferred");
+ span.record("outlook.defer_reason", reason);
+ span.record("outlook.defer_seconds", i64::from(delay_seconds));
+ if is_gate_refusal(error) {
+ tracing::debug!(parent: span, %link_id, defer_reason = reason, defer_seconds = i64::from(delay_seconds), %error, "mailbox sync work deferred");
+ } else {
+ tracing::warn!(parent: span, %link_id, defer_reason = reason, defer_seconds = i64::from(delay_seconds), %error, "mailbox sync work deferred");
+ }
+}
+
+/// The unit stopped without releasing its work, which another worker retries
+/// once the lease expires, or lost its lease to a newer owner (`Stale`).
+pub(super) fn record_failed(span: &Span, link_id: Uuid, error: &MailboxError) {
+ let reason = defer_reason(error);
+ span.record("outlook.outcome", "failed");
+ span.record("outlook.defer_reason", reason);
+ span.record("otel.status_description", tracing::field::display(error));
+ if matches!(error, MailboxError::Stale) {
+ tracing::debug!(parent: span, %link_id, defer_reason = reason, %error, "mailbox sync work failed");
+ } else {
+ tracing::warn!(parent: span, %link_id, defer_reason = reason, %error, "mailbox sync work failed");
+ }
+}
+
+/// Logs why a unit of the other mailbox loops (contacts, drafts, commands,
+/// settings, watches), which have no span, stopped or was deferred.
+pub(super) fn log_unit_error(link_id: Uuid, error: &MailboxError) {
+ let reason = defer_reason(error);
+ if is_gate_refusal(error) || matches!(error, MailboxError::Stale) {
+ tracing::debug!(%link_id, defer_reason = reason, %error, "mailbox sync work deferred");
+ } else {
+ tracing::warn!(%link_id, defer_reason = reason, %error, "mailbox sync work deferred");
+ }
+}
+
+/// A claim query failed, so no unit (and no mailbox) was involved yet.
+pub(super) fn claim_failed(unit: &'static str, error: MailboxError) -> MailboxError {
+ tracing::warn!(unit, defer_reason = defer_reason(&error), %error, "mailbox sync claim failed");
+ error
+}
+
+pub(super) fn elapsed_ms(started: std::time::Instant) -> i64 {
+ span_int(started.elapsed().as_millis())
+}
+
+/// OpenTelemetry receives a span field as an integer only when it is
+/// recorded as `i64`; unsigned values arrive as strings.
+pub(super) fn span_int(value: impl TryInto<i64>) -> i64 {
+ value.try_into().unwrap_or(i64::MAX)
+}
diff --git a/crates/email/src/domain/mailbox/watches.rs b/crates/email/src/domain/mailbox/watches.rs
index 1a6aedfc28..99915a059b 100644
--- a/crates/email/src/domain/mailbox/watches.rs
+++ b/crates/email/src/domain/mailbox/watches.rs
@@ -158,20 +158,30 @@ impl<R: MailboxWatchRepository, G: MailboxWatchGateway> MailboxWatchService<R, G
if !self.callback_base.starts_with("https://") {
return Ok(false);
}
- let Some(lease) = self.repository.claim_watch(Uuid::new_v4()).await? else {
+ let Some(lease) = self
+ .repository
+ .claim_watch(Uuid::new_v4())
+ .await
+ .map_err(|error| super::telemetry::claim_failed("watches", error))?
+ else {
return Ok(false);
};
- let result = super::maintain_lease(self.execute(&lease), || {
- self.repository.renew_watch_lease(&lease)
- })
- .await;
- let delay = match &result {
- Ok(()) => 12 * 60 * 60,
- Err(MailboxError::Provider(e)) => super::retry_seconds(e),
- _ => 30,
- };
- self.repository.release_watch(&lease, delay).await?;
- result.map(|()| true)
+ let link_id = lease.mailbox.link_id;
+ async {
+ let result = super::maintain_lease(self.execute(&lease), || {
+ self.repository.renew_watch_lease(&lease)
+ })
+ .await;
+ let delay = match &result {
+ Ok(()) => 12 * 60 * 60,
+ Err(MailboxError::Provider(e)) => super::retry_seconds(e),
+ _ => 30,
+ };
+ self.repository.release_watch(&lease, delay).await?;
+ result.map(|()| true)
+ }
+ .await
+ .inspect_err(|error| super::telemetry::log_unit_error(link_id, error))
}
async fn execute(&self, lease: &WatchLease) -> Result<(), MailboxError> {
let remote = self.gateway.watches(lease.mailbox).await?;
diff --git a/crates/email/src/lib.rs b/crates/email/src/lib.rs
index f92ebe48cd..1aa29a43ef 100644
--- a/crates/email/src/lib.rs
+++ b/crates/email/src/lib.rs
@@ -3,3 +3,5 @@ pub mod domain;
pub mod inbound;
#[cfg(any(feature = "outbound", feature = "http_client", feature = "s3"))]
pub mod outbound;
+#[cfg(test)]
+pub(crate) mod telemetry_capture;
diff --git a/crates/email/src/outbound/mailbox_pg.rs b/crates/email/src/outbound/mailbox_pg.rs
index ed57f9f8bd..5810140b1d 100644
--- a/crates/email/src/outbound/mailbox_pg.rs
+++ b/crates/email/src/outbound/mailbox_pg.rs
@@ -1,5 +1,6 @@
//! Postgres implementation of durable mailbox stream and reconciliation leases.
+use crate::domain::mailbox::progress::ImportCompleted;
use crate::domain::mailbox::*;
use email_api_client::domain::models::{
MailFolder, MailboxChangePage, ProviderId, StreamPosition, StreamToken,
@@ -15,6 +16,7 @@ mod drafts;
mod gmail_history;
mod ingest;
mod lifecycle;
+mod progress;
mod projection;
mod settings;
#[cfg(test)]
@@ -44,8 +46,40 @@ impl PgMailboxSync {
}
}
-fn db_error(_: sqlx::Error) -> MailboxError {
- MailboxError::Persistence
+fn db_error(error: sqlx::Error) -> MailboxError {
+ MailboxError::Database(DatabaseFailure::from(&error))
+}
+
+/// For `email_db_client` calls, which wrap their `sqlx` errors in `anyhow`.
+fn wrapped_db_error(error: anyhow::Error) -> MailboxError {
+ error
+ .chain()
+ .find_map(|cause| cause.downcast_ref::<sqlx::Error>())
+ .map_or(MailboxError::Persistence, |error| {
+ MailboxError::Database(DatabaseFailure::from(error))
+ })
+}
+
+/// Lets any adapter keep a database error's SQLSTATE without its message.
+impl From<&sqlx::Error> for DatabaseFailure {
+ fn from(error: &sqlx::Error) -> Self {
+ match error {
+ sqlx::Error::Database(error) => error.code().map_or(DatabaseFailure::Other, |code| {
+ DatabaseFailure::Sqlstate(code.into_owned())
+ }),
+ sqlx::Error::PoolTimedOut => DatabaseFailure::PoolTimedOut,
+ sqlx::Error::Io(_)
+ | sqlx::Error::Tls(_)
+ | sqlx::Error::Protocol(_)
+ | sqlx::Error::PoolClosed
+ | sqlx::Error::WorkerCrashed => DatabaseFailure::Connection,
+ sqlx::Error::ColumnDecode { .. }
+ | sqlx::Error::Decode(_)
+ | sqlx::Error::ColumnNotFound(_)
+ | sqlx::Error::TypeNotFound { .. } => DatabaseFailure::Decode,
+ _ => DatabaseFailure::Other,
+ }
+ }
}
impl MailboxSyncRepository for PgMailboxSync {
@@ -75,7 +109,7 @@ impl MailboxSyncRepository for PgMailboxSync {
&self,
lease: &CatalogLease,
folders: &[MailFolder],
- ) -> Result<(), MailboxError> {
+ ) -> Result<Option<ImportCompleted>, MailboxError> {
self.commit_catalog_lease(lease, folders).await
}
async fn release_catalog(&self, lease: &CatalogLease, delay: u32) -> Result<(), MailboxError> {
@@ -117,7 +151,7 @@ impl MailboxSyncRepository for PgMailboxSync {
&self,
lease: &StreamLease,
page: &MailboxChangePage,
- ) -> Result<(), MailboxError> {
+ ) -> Result<CommittedPage, MailboxError> {
let (position, complete) = match &page.position {
StreamPosition::Continue(position) => (position, false),
StreamPosition::Checkpoint(position) => (position, true),
@@ -157,7 +191,7 @@ impl MailboxSyncRepository for PgMailboxSync {
if advanced.rows_affected() != 1 {
return Err(MailboxError::Stale);
}
- sqlx::query!(
+ let queued = sqlx::query!(
r#"
INSERT INTO email_message_reconciliation (link_id,generation,provider_id,is_import)
SELECT $1,$2,provider_id,$4 FROM unnest($3::text[]) provider_id
@@ -173,9 +207,14 @@ impl MailboxSyncRepository for PgMailboxSync {
)
.execute(&mut *tx)
.await
- .map_err(db_error)?;
- Self::queue_attachment_recheck(&mut tx, lease.mailbox).await?;
- tx.commit().await.map_err(db_error)
+ .map_err(db_error)?
+ .rows_affected();
+ let import_completed = Self::settle_import(&mut tx, lease.mailbox).await?;
+ tx.commit().await.map_err(db_error)?;
+ Ok(CommittedPage {
+ queued,
+ import_completed,
+ })
}
async fn reset_stream(&self, lease: &StreamLease) -> Result<(), MailboxError> {
@@ -264,7 +303,10 @@ impl MailboxSyncRepository for PgMailboxSync {
.collect()
}
- async fn complete_message(&self, lease: &MessageLease) -> Result<(), MailboxError> {
+ async fn complete_message(
+ &self,
+ lease: &MessageLease,
+ ) -> Result<Option<ImportCompleted>, MailboxError> {
let mut tx = self.db.begin().await.map_err(db_error)?;
Self::lock_attachment_completion(&mut tx, lease.mailbox).await?;
sqlx::query!(r#"
@@ -278,8 +320,9 @@ impl MailboxSyncRepository for PgMailboxSync {
WHERE link_id = $1 AND generation = $2 AND provider_id = $3 AND lease_id = $4
"#,lease.mailbox.link_id,lease.mailbox.sync_generation,lease.provider_id.as_str(),lease.lease_id)
.execute(&mut *tx).await.map_err(db_error)?;
- Self::queue_attachment_recheck(&mut tx, lease.mailbox).await?;
- tx.commit().await.map_err(db_error)
+ let import_completed = Self::settle_import(&mut tx, lease.mailbox).await?;
+ tx.commit().await.map_err(db_error)?;
+ Ok(import_completed)
}
async fn release_message(&self, lease: &MessageLease, delay: u32) -> Result<(), MailboxError> {
@@ -332,29 +375,54 @@ impl PgMailboxSync {
Ok(())
}
- // Called under the link lock after the final page or reconciliation. All
- // correspondence is now ingested, regardless of folder/page delivery order.
- async fn queue_attachment_recheck(
+ // Called under the link lock after each page or reconciliation, so one
+ // caller at a time sees the import finish.
+ async fn settle_import(
tx: &mut sqlx::PgConnection,
mailbox: MailboxKey,
- ) -> Result<(), MailboxError> {
- let ready=sqlx::query_scalar!(r#"SELECT (
- EXISTS(SELECT 1 FROM email_sync_streams WHERE link_id=$1 AND generation=$2 AND kind='folder_catalog' AND initial_complete)
- AND EXISTS(SELECT 1 FROM email_sync_streams WHERE link_id=$1 AND generation=$2 AND kind='mail_folder' AND NOT attachments_rechecked)
- AND NOT EXISTS(SELECT 1 FROM email_sync_streams WHERE link_id=$1 AND generation=$2 AND kind IN ('mail_folder','folder_catalog') AND NOT initial_complete)
- AND NOT EXISTS(SELECT 1 FROM email_message_reconciliation WHERE link_id=$1 AND generation=$2)
- ) AS "ready!""#,mailbox.link_id,mailbox.sync_generation).fetch_one(&mut *tx).await.map_err(db_error)?;
- if !ready {
- return Ok(());
+ ) -> Result<Option<ImportCompleted>, MailboxError> {
+ let state = sqlx::query!(r#"SELECT
+ EXISTS(SELECT 1 FROM email_sync_streams WHERE link_id=$1 AND generation=$2 AND kind='folder_catalog' AND initial_complete) AS "catalog_complete!",
+ EXISTS(SELECT 1 FROM email_sync_streams WHERE link_id=$1 AND generation=$2 AND kind='folder_catalog' AND import_completed_at IS NULL) AS "import_open!",
+ EXISTS(SELECT 1 FROM email_sync_streams WHERE link_id=$1 AND generation=$2 AND kind='mail_folder' AND NOT attachments_rechecked) AS "recheck_pending!",
+ EXISTS(SELECT 1 FROM email_sync_streams WHERE link_id=$1 AND generation=$2 AND kind IN ('mail_folder','folder_catalog') AND NOT initial_complete) AS "streams_incomplete!",
+ EXISTS(SELECT 1 FROM email_message_reconciliation WHERE link_id=$1 AND generation=$2) AS "queued!",
+ EXISTS(SELECT 1 FROM email_message_reconciliation WHERE link_id=$1 AND generation=$2 AND is_import) AS "imports_queued!"
+ "#,mailbox.link_id,mailbox.sync_generation).fetch_one(&mut *tx).await.map_err(db_error)?;
+ let enumerated = state.catalog_complete && !state.streams_incomplete;
+ // All correspondence is now ingested, regardless of folder/page delivery order.
+ if enumerated && state.recheck_pending && !state.queued {
+ sqlx::query!(r#"
+ INSERT INTO email_projection_outbox(id,link_id,generation,payload)
+ SELECT gen_random_uuid(),m.link_id,$2,jsonb_build_object('kind','attachment_recheck','message_id',m.id)
+ FROM email_messages m WHERE m.link_id=$1 AND m.provider_id IS NOT NULL AND NOT m.is_draft
+ AND EXISTS(SELECT 1 FROM email_attachments a WHERE a.message_id=m.id AND a.reference_url IS NULL)
+ "#,mailbox.link_id,mailbox.sync_generation).execute(&mut *tx).await.map_err(db_error)?;
+ sqlx::query!("UPDATE email_sync_streams SET attachments_rechecked=true WHERE link_id=$1 AND generation=$2 AND kind='mail_folder'",mailbox.link_id,mailbox.sync_generation)
+ .execute(&mut *tx).await.map_err(db_error)?;
}
- sqlx::query!(r#"
- INSERT INTO email_projection_outbox(id,link_id,generation,payload)
- SELECT gen_random_uuid(),m.link_id,$2,jsonb_build_object('kind','attachment_recheck','message_id',m.id)
- FROM email_messages m WHERE m.link_id=$1 AND m.provider_id IS NOT NULL AND NOT m.is_draft
- AND EXISTS(SELECT 1 FROM email_attachments a WHERE a.message_id=m.id AND a.reference_url IS NULL)
- "#,mailbox.link_id,mailbox.sync_generation).execute(&mut *tx).await.map_err(db_error)?;
- sqlx::query!("UPDATE email_sync_streams SET attachments_rechecked=true WHERE link_id=$1 AND generation=$2 AND kind='mail_folder'",mailbox.link_id,mailbox.sync_generation)
- .execute(&mut *tx).await.map_err(db_error)?;
- Ok(())
+ // The import run is done once every folder is enumerated and no import
+ // row is left; live changes may still be queued.
+ if !(enumerated && state.import_open && !state.imports_queued) {
+ return Ok(None);
+ }
+ let Some(run) = sqlx::query!(r#"
+ UPDATE email_sync_streams SET import_completed_at = now()
+ WHERE link_id=$1 AND generation=$2 AND kind='folder_catalog' AND import_completed_at IS NULL
+ RETURNING import_started_at, import_completed_at AS "completed_at!"
+ "#,mailbox.link_id,mailbox.sync_generation).fetch_optional(&mut *tx).await.map_err(db_error)? else {
+ return Ok(None);
+ };
+ let size = sqlx::query!(r#"SELECT
+ (SELECT count(*) FROM email_messages WHERE link_id=$1 AND provider_id IS NOT NULL) AS "messages!",
+ (SELECT count(*) FROM email_sync_streams WHERE link_id=$1 AND generation=$2 AND kind='mail_folder') AS "folders!"
+ "#,mailbox.link_id,mailbox.sync_generation).fetch_one(&mut *tx).await.map_err(db_error)?;
+ Ok(Some(ImportCompleted {
+ mailbox,
+ started_at: run.import_started_at,
+ completed_at: run.completed_at,
+ messages: size.messages,
+ folders: size.folders,
+ }))
}
}
diff --git a/crates/email/src/outbound/mailbox_pg/catalog.rs b/crates/email/src/outbound/mailbox_pg/catalog.rs
index 14f462f086..dbf0fb9c40 100644
--- a/crates/email/src/outbound/mailbox_pg/catalog.rs
+++ b/crates/email/src/outbound/mailbox_pg/catalog.rs
@@ -34,7 +34,7 @@ impl PgMailboxSync {
&self,
lease: &CatalogLease,
folders: &[MailFolder],
- ) -> Result<(), MailboxError> {
+ ) -> Result<Option<ImportCompleted>, MailboxError> {
let mut tx = self.db.begin().await.map_err(db_error)?;
// Serialize against disconnect/reconnect before touching the shared catalog.
let current = sqlx::query!(
@@ -148,8 +148,9 @@ impl PgMailboxSync {
WHERE s.link_id = $1 AND s.generation = $2 AND s.kind = 'mail_folder'
AND f.link_id = s.link_id AND f.provider_id = s.scope_id AND f.deleted_at IS NOT NULL
"#,lease.mailbox.link_id,lease.mailbox.sync_generation).execute(&mut *tx).await.map_err(db_error)?;
- Self::queue_attachment_recheck(&mut tx, lease.mailbox).await?;
- tx.commit().await.map_err(db_error)
+ let import_completed = Self::settle_import(&mut tx, lease.mailbox).await?;
+ tx.commit().await.map_err(db_error)?;
+ Ok(import_completed)
}
pub(super) async fn release_catalog_lease(
diff --git a/crates/email/src/outbound/mailbox_pg/commands/test.rs b/crates/email/src/outbound/mailbox_pg/commands/test.rs
index d7cd37cbc6..d885734240 100644
--- a/crates/email/src/outbound/mailbox_pg/commands/test.rs
+++ b/crates/email/src/outbound/mailbox_pg/commands/test.rs
@@ -241,6 +241,9 @@ impl MailboxGateway for UncertainProvider {
> {
unreachable!("commands must not fetch bodies or attachments")
}
+ async fn admitted_requests(&self, _: MailboxKey) -> Option<u64> {
+ None
+ }
}
impl MailboxCommandGateway for UncertainProvider {
async fn organization(
diff --git a/crates/email/src/outbound/mailbox_pg/credentials.rs b/crates/email/src/outbound/mailbox_pg/credentials.rs
index 0fa2ebe708..4df2aef733 100644
--- a/crates/email/src/outbound/mailbox_pg/credentials.rs
+++ b/crates/email/src/outbound/mailbox_pg/credentials.rs
@@ -11,8 +11,11 @@ impl MailboxCredentialHealthRepository for PgMailboxSync {
mailbox: MailboxAccess,
health: CredentialHealth,
) -> Result<(), TokenError> {
- let failed = |_| TokenError::Transient {
- message: "could not persist mailbox health".into(),
+ let failed = |error: sqlx::Error| TokenError::Transient {
+ message: format!(
+ "could not persist mailbox health: {}",
+ DatabaseFailure::from(&error)
+ ),
};
let mut tx = self.db.begin().await.map_err(failed)?;
let current = sqlx::query!("SELECT sync_generation,needs_reauth FROM email_links WHERE id = $1 AND grant_generation = $2 AND is_sync_active AND provider = 'OUTLOOK' AND sync_generation = $3 FOR UPDATE",mailbox.link_id,mailbox.grant_generation,mailbox.sync_generation)
diff --git a/crates/email/src/outbound/mailbox_pg/ingest.rs b/crates/email/src/outbound/mailbox_pg/ingest.rs
index c155793645..c75f070638 100644
--- a/crates/email/src/outbound/mailbox_pg/ingest.rs
+++ b/crates/email/src/outbound/mailbox_pg/ingest.rs
@@ -34,7 +34,7 @@ impl MailboxIngest for PgMailboxSync {
addresses,
)
.await
- .map_err(|_| MailboxError::Persistence)?;
+ .map_err(wrapped_db_error)?;
let mut tx = self.db.begin().await.map_err(db_error)?;
let active = sqlx::query!(r#"
SELECT id FROM email_links WHERE id = $1 AND sync_generation = $2 AND grant_generation = $3
@@ -205,7 +205,7 @@ impl MailboxIngest for PgMailboxSync {
None,
)
.await
- .map_err(|_| MailboxError::Persistence)?;
+ .map_err(wrapped_db_error)?;
}
let mut state_json = serde_json::to_value(&state).map_err(|_| MailboxError::Persistence)?;
if !message.is_sent && shadow.as_ref().is_some_and(|s| s.delete_requested) {
@@ -247,14 +247,14 @@ impl MailboxIngest for PgMailboxSync {
.map_err(db_error)?;
email_db_client::threads::update::sync_thread_calendar_flag(&mut tx, thread_id)
.await
- .map_err(|_| MailboxError::Persistence)?;
+ .map_err(wrapped_db_error)?;
email_db_client::messages::replying_to_id::update_thread_messages_replying_to(
&mut tx,
thread_id,
lease.mailbox.link_id,
)
.await
- .map_err(|_| MailboxError::Persistence)?;
+ .map_err(wrapped_db_error)?;
let payload = serde_json::json!({
"kind":"message", "message_id":message_id,"thread_id":thread_id,
"provider_id":lease.provider_id.as_str(),"is_import":lease.is_import,
diff --git a/crates/email/src/outbound/mailbox_pg/lifecycle.rs b/crates/email/src/outbound/mailbox_pg/lifecycle.rs
index 46b0dbdc43..c4b8211ffe 100644
--- a/crates/email/src/outbound/mailbox_pg/lifecycle.rs
+++ b/crates/email/src/outbound/mailbox_pg/lifecycle.rs
@@ -292,7 +292,8 @@ impl InboxLifecycleRepository for PgMailboxSync {
.map_or_else(macro_uuid::generate_uuid_v7, |stream| stream.id);
if !in_progress || stream.is_none() {
sqlx::query!("INSERT INTO email_sync_streams(id,link_id,generation,kind,scope_id) VALUES ($1,$2,$3,'folder_catalog','mail') ON CONFLICT DO NOTHING",run_id,link,current.sync_generation).execute(&mut *tx).await.map_err(failed)?;
- sqlx::query!(r#"UPDATE email_sync_streams SET position = NULL,initial_complete = false, attachments_rechecked = false,scan_id = gen_random_uuid(),next_run_at = now(),fence = fence + 1,lease_id = NULL,lease_until = NULL
+ sqlx::query!(r#"UPDATE email_sync_streams SET position = NULL,initial_complete = false, attachments_rechecked = false,scan_id = gen_random_uuid(),next_run_at = now(),fence = fence + 1,lease_id = NULL,lease_until = NULL,
+ import_started_at = now(),import_completed_at = NULL
WHERE link_id = $1 AND generation = $2 AND kind IN ('folder_catalog','mail_folder','contacts_catalog','contacts_profile','contacts')"#,link,current.sync_generation).execute(&mut *tx).await.map_err(failed)?;
sqlx::query!(r#"INSERT INTO email_message_reconciliation(link_id,generation,provider_id,is_import)
SELECT link_id,$2,provider_id,true FROM email_messages WHERE link_id = $1 AND provider_id IS NOT NULL
@@ -305,6 +306,9 @@ impl InboxLifecycleRepository for PgMailboxSync {
}
};
tx.commit().await.map_err(failed)?;
+ if current.provider == UserProvider::Outlook && !result.already_in_progress {
+ crate::domain::mailbox::progress::log_import_started(link, current.sync_generation);
+ }
Ok(result)
}
@@ -384,8 +388,9 @@ async fn rebind_work(
) -> Result<(), InboxLifecycleError> {
sqlx::query!("UPDATE email_mailbox_watch_work SET next_run_at=now(),revision=revision+1 WHERE link_id=$1",link)
.execute(&mut **tx).await.map_err(failed)?;
- sqlx::query!(r#"INSERT INTO email_sync_streams(id,link_id,generation,kind,scope_id,position,initial_complete,scan_id)
- SELECT gen_random_uuid(),link_id,$3,kind,scope_id,position,initial_complete,scan_id FROM email_sync_streams WHERE link_id = $1 AND generation = $2
+ // The import run continues in the new generation, so it keeps its stamps.
+ sqlx::query!(r#"INSERT INTO email_sync_streams(id,link_id,generation,kind,scope_id,position,initial_complete,scan_id,import_started_at,import_completed_at)
+ SELECT gen_random_uuid(),link_id,$3,kind,scope_id,position,initial_complete,scan_id,import_started_at,import_completed_at FROM email_sync_streams WHERE link_id = $1 AND generation = $2
ON CONFLICT DO NOTHING"#,link,old,new).execute(&mut **tx).await.map_err(failed)?;
sqlx::query!(r#"INSERT INTO email_message_reconciliation(link_id,generation,provider_id,is_import)
SELECT link_id,$3,provider_id,is_import FROM email_message_reconciliation WHERE link_id = $1 AND generation = $2 ON CONFLICT DO NOTHING"#,link,old,new).execute(&mut **tx).await.map_err(failed)?;
diff --git a/crates/email/src/outbound/mailbox_pg/lifecycle/test.rs b/crates/email/src/outbound/mailbox_pg/lifecycle/test.rs
index c3257af318..77dd576f6c 100644
--- a/crates/email/src/outbound/mailbox_pg/lifecycle/test.rs
+++ b/crates/email/src/outbound/mailbox_pg/lifecycle/test.rs
@@ -66,6 +66,13 @@ async fn leaving_shared_custodian_rebinds_work_and_rejects_old_health_even_with_
db: PgPool,
) {
let (repo, link, replacement) = shared(&db).await;
+ sqlx::query!(
+ "UPDATE email_sync_streams SET import_started_at = now() - interval '1 hour', import_completed_at = now() - interval '5 minutes' WHERE link_id = $1",
+ link
+ )
+ .execute(&db)
+ .await
+ .unwrap();
let command = macro_uuid::generate_uuid_v7();
sqlx::query!(
r#"INSERT INTO email_mailbox_commands(id,link_id,generation,actor_id,intent,status)
@@ -110,15 +117,18 @@ async fn leaving_shared_custodian_rebinds_work_and_rejects_old_health_even_with_
.await,
Err(TokenError::ReauthRequired)
));
- assert_eq!(
- sqlx::query_scalar!(
- "SELECT count(*) FROM email_sync_streams WHERE link_id = $1 AND generation = 2",
- link
- )
- .fetch_one(&db)
- .await
- .unwrap(),
- Some(1)
+ let rebound_stream = sqlx::query!(
+ r#"SELECT import_started_at < now() - interval '30 minutes' AS "started_before!", import_completed_at IS NOT NULL AS "completed!"
+ FROM email_sync_streams WHERE link_id = $1 AND generation = 2"#,
+ link
+ )
+ .fetch_all(&db)
+ .await
+ .unwrap();
+ assert_eq!(rebound_stream.len(), 1);
+ assert!(
+ rebound_stream[0].started_before && rebound_stream[0].completed,
+ "the import run continues in the new generation, so it must not report completion again"
);
assert_eq!(
sqlx::query_scalar!(
@@ -275,6 +285,52 @@ async fn outlook_resync_fences_page_workers_and_never_creates_a_gmail_backfill(d
);
}
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn outlook_resync_starts_a_new_import_run_once(db: PgPool) {
+ let telemetry = crate::telemetry_capture::Telemetry::default();
+ let _guard = telemetry.install();
+ let (repo, link, _) = shared(&db).await;
+ sqlx::query!(
+ r#"INSERT INTO email_sync_streams (id,link_id,generation,kind,scope_id,initial_complete,import_started_at,import_completed_at)
+ VALUES ($1,$2,1,'folder_catalog','mail',true,now() - interval '1 day',now() - interval '23 hours')"#,
+ macro_uuid::generate_uuid_v7(),
+ link
+ )
+ .execute(&db)
+ .await
+ .unwrap();
+ sqlx::query!(
+ "UPDATE email_sync_streams SET initial_complete = true WHERE link_id = $1",
+ link
+ )
+ .execute(&db)
+ .await
+ .unwrap();
+ let service = InboxLifecycleService::new(repo.clone(), Effects);
+
+ service.resync(&actor("remaining"), link).await.unwrap();
+ let again = service.resync(&actor("remaining"), link).await.unwrap();
+
+ assert!(again.already_in_progress);
+ let started = telemetry.events("outlook import started");
+ let [(level, fields)] = started.as_slice() else {
+ panic!("expected one start, got {started:?}");
+ };
+ assert_eq!(*level, tracing::Level::INFO);
+ assert_eq!(fields["link_id"], link.to_string());
+ assert_eq!(fields["sync_generation"], "1");
+ let run = sqlx::query!(
+ r#"SELECT import_started_at > now() - interval '1 minute' AS "restarted!", import_completed_at
+ FROM email_sync_streams WHERE link_id = $1 AND kind = 'folder_catalog'"#,
+ link
+ )
+ .fetch_one(&db)
+ .await
+ .unwrap();
+ assert!(run.restarted);
+ assert!(run.import_completed_at.is_none());
+}
+
#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
async fn gmail_resync_commits_the_backfill_with_durable_publication(db: PgPool) {
let (repo, link, _) = shared(&db).await;
diff --git a/crates/email/src/outbound/mailbox_pg/progress.rs b/crates/email/src/outbound/mailbox_pg/progress.rs
new file mode 100644
index 0000000000..ef74ddc486
--- /dev/null
+++ b/crates/email/src/outbound/mailbox_pg/progress.rs
@@ -0,0 +1,121 @@
+use super::*;
+use crate::domain::mailbox::progress::*;
+use std::collections::HashMap;
+
+#[cfg(test)]
+mod test;
+
+impl MailboxProgressRepository for PgMailboxSync {
+ async fn record_imports(
+ &self,
+ imports: &HashMap<Uuid, MailboxImports>,
+ ) -> Result<(), MailboxError> {
+ let mut links = Vec::with_capacity(imports.len());
+ let mut imported = Vec::with_capacity(imports.len());
+ let mut last_imported = Vec::with_capacity(imports.len());
+ for (link_id, entry) in imports {
+ links.push(*link_id);
+ imported.push(i32::try_from(entry.imported).unwrap_or(i32::MAX));
+ last_imported.push(entry.last_imported_at);
+ }
+ sqlx::query!(
+ r#"
+ INSERT INTO email_mailbox_import_minutes (link_id, minute, imported, last_imported_at)
+ SELECT link_id, date_trunc('minute', now()), imported, last_imported_at
+ FROM unnest($1::uuid[], $2::int[], $3::timestamptz[]) AS flushed(link_id, imported, last_imported_at)
+ ON CONFLICT (link_id, minute) DO UPDATE SET
+ imported = email_mailbox_import_minutes.imported + EXCLUDED.imported,
+ last_imported_at = GREATEST(email_mailbox_import_minutes.last_imported_at, EXCLUDED.last_imported_at)
+ "#,
+ &links,
+ &imported,
+ &last_imported
+ )
+ .execute(&self.db)
+ .await
+ .map_err(db_error)?;
+ Ok(())
+ }
+
+ async fn backlogs(&self) -> Result<Vec<MailboxBacklog>, MailboxError> {
+ sqlx::query!(
+ "DELETE FROM email_mailbox_import_minutes WHERE minute < now() - interval '1 day'"
+ )
+ .execute(&self.db)
+ .await
+ .map_err(db_error)?;
+ // One aggregate per Outlook link, each read through the
+ // (link_id, generation, ...) key prefixes of the current generation's
+ // queue and streams and the link's newest minute counts. Old
+ // generations and the message table are never read.
+ let rows = sqlx::query!(
+ r#"
+ SELECT l.id AS "link_id!",
+ l.sync_generation AS "sync_generation!",
+ l.is_sync_active AS "sync_active!",
+ l.needs_reauth AS "needs_reauth!",
+ q.queued AS "queued!",
+ q.parked AS "parked!",
+ q.leased AS "leased!",
+ COALESCE(floor(EXTRACT(EPOCH FROM now() - q.oldest_due)), 0)::bigint AS "oldest_waiting_seconds!",
+ COALESCE(ceil(EXTRACT(EPOCH FROM q.parked_until - now())), 0)::bigint AS "parked_until_max_seconds!",
+ s.incomplete AS "streams_incomplete!",
+ COALESCE(recent.imported_5m, 0)::bigint AS "imported_5m!",
+ floor(EXTRACT(EPOCH FROM now() - latest.last_imported_at))::bigint AS seconds_since_last_import
+ FROM email_links l
+ CROSS JOIN LATERAL (
+ SELECT count(*) AS queued,
+ count(*) FILTER (WHERE available_at > now()) AS parked,
+ count(*) FILTER (WHERE lease_until > now()) AS leased,
+ min(available_at) FILTER (
+ WHERE available_at <= now() AND (lease_until IS NULL OR lease_until <= now())
+ ) AS oldest_due,
+ max(available_at) FILTER (WHERE available_at > now()) AS parked_until
+ FROM email_message_reconciliation
+ WHERE link_id = l.id AND generation = l.sync_generation
+ ) q
+ CROSS JOIN LATERAL (
+ SELECT count(*) AS incomplete
+ FROM email_sync_streams
+ WHERE link_id = l.id AND generation = l.sync_generation
+ AND kind IN ('folder_catalog', 'mail_folder') AND NOT initial_complete
+ ) s
+ CROSS JOIN LATERAL (
+ SELECT sum(imported) AS imported_5m
+ FROM email_mailbox_import_minutes
+ WHERE link_id = l.id AND minute > now() - interval '5 minutes'
+ ) recent
+ -- A flush carries imports from at most the minute before it, so
+ -- the latest import is in one of the two newest minutes.
+ CROSS JOIN LATERAL (
+ SELECT max(last_imported_at) AS last_imported_at
+ FROM (
+ SELECT last_imported_at FROM email_mailbox_import_minutes
+ WHERE link_id = l.id ORDER BY minute DESC LIMIT 2
+ ) newest
+ ) latest
+ WHERE l.provider = 'OUTLOOK' AND (l.is_sync_active OR q.queued > 0)
+ "#
+ )
+ .fetch_all(&self.db)
+ .await
+ .map_err(db_error)?;
+ Ok(rows
+ .into_iter()
+ .map(|row| MailboxBacklog {
+ link_id: row.link_id,
+ sync_generation: row.sync_generation,
+ queued: row.queued,
+ parked: row.parked,
+ leased: row.leased,
+ oldest_waiting_seconds: row.oldest_waiting_seconds,
+ parked_until_max_seconds: row.parked_until_max_seconds,
+ streams_incomplete: row.streams_incomplete,
+ imported_5m: row.imported_5m,
+ seconds_since_last_import: row.seconds_since_last_import,
+ sync_active: row.sync_active,
+ needs_reauth: row.needs_reauth,
+ })
+ .collect())
+ }
+}
diff --git a/crates/email/src/outbound/mailbox_pg/progress/test.rs b/crates/email/src/outbound/mailbox_pg/progress/test.rs
new file mode 100644
index 0000000000..a5a923ebfc
--- /dev/null
+++ b/crates/email/src/outbound/mailbox_pg/progress/test.rs
@@ -0,0 +1,446 @@
+use super::*;
+use crate::domain::mailbox::progress::MailboxProgressService;
+use crate::telemetry_capture::Telemetry;
+use email_api_client::domain::models::{
+ EmailApiError, GateLimit, MailboxMessage, RateLimitOrigin, StreamPosition,
+};
+use email_api_client::domain::request_count;
+use macro_db_migrator::MACRO_DB_MIGRATIONS;
+use std::collections::VecDeque;
+use std::sync::Mutex;
+use std::time::Duration;
+
+/// Provider double: answers message reads in order, sending `requests` Graph
+/// requests for each, and reports a fixed admitted-request count.
+#[derive(Default)]
+struct Provider {
+ messages: Mutex<VecDeque<Result<Option<MailboxMessage>, EmailApiError>>>,
+ pages: Mutex<VecDeque<Result<MailboxChangePage, EmailApiError>>>,
+ requests: u32,
+}
+
+impl Provider {
+ fn answering(messages: Vec<Result<Option<MailboxMessage>, EmailApiError>>) -> Self {
+ Self {
+ messages: Mutex::new(messages.into()),
+ requests: 2,
+ ..Self::default()
+ }
+ }
+}
+
+impl MailboxGateway for Provider {
+ async fn folders(&self, _: MailboxKey) -> Result<Vec<MailFolder>, EmailApiError> {
+ Ok(Vec::new())
+ }
+ async fn changes(
+ &self,
+ _: MailboxKey,
+ _: &ProviderId,
+ _: Option<&StreamToken>,
+ ) -> Result<MailboxChangePage, EmailApiError> {
+ self.pages.lock().unwrap().pop_front().unwrap()
+ }
+ async fn message(
+ &self,
+ _: MailboxKey,
+ _: &ProviderId,
+ _: &[MailFolder],
+ ) -> Result<Option<MailboxMessage>, EmailApiError> {
+ for _ in 0..self.requests {
+ request_count::record_sent();
+ }
+ self.messages.lock().unwrap().pop_front().unwrap()
+ }
+ async fn admitted_requests(&self, _: MailboxKey) -> Option<u64> {
+ Some(42)
+ }
+}
+
+fn sync(
+ repo: &PgMailboxSync,
+ provider: Provider,
+) -> MailboxSyncService<PgMailboxSync, Provider, PgMailboxSync> {
+ MailboxSyncService::new(repo.clone(), provider, repo.clone())
+}
+
+/// `attempts` is what the row had before its next claim adds one.
+async fn queue(db: &PgPool, link: Uuid, provider_id: &str, is_import: bool, attempts: i32) {
+ sqlx::query!(
+ "INSERT INTO email_message_reconciliation (link_id,generation,provider_id,is_import,attempts) VALUES ($1,1,$2,$3,$4)",
+ link,
+ provider_id,
+ is_import,
+ attempts
+ )
+ .execute(db)
+ .await
+ .unwrap();
+}
+
+/// A mailbox whose only folder and catalog finished enumerating.
+async fn enumerated_mailbox(db: &PgPool) -> (PgMailboxSync, Uuid) {
+ let (repo, link) = super::super::test::fixture(db).await;
+ sqlx::query!(
+ "UPDATE email_sync_streams SET initial_complete = true, attachments_rechecked = true WHERE link_id = $1",
+ link
+ )
+ .execute(db)
+ .await
+ .unwrap();
+ sqlx::query!(
+ "INSERT INTO email_sync_streams (id,link_id,generation,kind,scope_id,initial_complete) VALUES ($1,$2,1,'folder_catalog','mail',true)",
+ macro_uuid::generate_uuid_v7(),
+ link
+ )
+ .execute(db)
+ .await
+ .unwrap();
+ (repo, link)
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn import_completes_once_when_its_last_import_row_settles(db: PgPool) {
+ let telemetry = Telemetry::default();
+ let _guard = telemetry.install();
+ let (repo, link) = enumerated_mailbox(&db).await;
+ queue(&db, link, "first", true, 2).await;
+ queue(&db, link, "second", true, 2).await;
+ let service = sync(
+ &repo,
+ Provider::answering(vec![Ok(None), Ok(None), Ok(None)]),
+ );
+
+ assert!(service.reconcile_once().await.unwrap());
+ assert!(telemetry.events("outlook import completed").is_empty());
+ assert!(service.reconcile_once().await.unwrap());
+ // A live change after the import must not report it again.
+ queue(&db, link, "live", false, 2).await;
+ assert!(service.reconcile_once().await.unwrap());
+
+ let completed = telemetry.events("outlook import completed");
+ let [(level, fields)] = completed.as_slice() else {
+ panic!("expected one completion, got {completed:?}");
+ };
+ assert_eq!(*level, tracing::Level::INFO);
+ assert_eq!(fields["link_id"], link.to_string());
+ assert_eq!(fields["sync_generation"], "1");
+ assert_eq!(fields["folders"], "1");
+ assert_eq!(fields["messages"], "0");
+ assert_eq!(fields["graph_calls"], "42");
+ assert!(fields.contains_key("duration_seconds"));
+ let reconciled = telemetry.spans("outlook.sync.reconcile");
+ assert_eq!(reconciled.len(), 3);
+ assert!(
+ reconciled
+ .iter()
+ .all(|span| span["outlook.outcome"] == "absent")
+ );
+ assert_eq!(reconciled[0]["outlook.is_import"], "true");
+ assert_eq!(reconciled[0]["outlook.attempts"], "3");
+ assert_eq!(reconciled[0]["outlook.graph_calls"], "2");
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn workers_finishing_the_last_rows_together_report_one_completion(db: PgPool) {
+ let telemetry = Telemetry::default();
+ let _guard = telemetry.install();
+ let (repo, link) = enumerated_mailbox(&db).await;
+ queue(&db, link, "first", true, 2).await;
+ queue(&db, link, "second", true, 2).await;
+ let first = sync(&repo, Provider::answering(vec![Ok(None)]));
+ let second = sync(&repo, Provider::answering(vec![Ok(None)]));
+
+ let (a, b) = tokio::join!(first.reconcile_once(), second.reconcile_once());
+
+ assert!(a.unwrap() && b.unwrap());
+ assert_eq!(telemetry.events("outlook import completed").len(), 1);
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn deferrals_name_their_reason_and_gate_refusals_stay_quiet(db: PgPool) {
+ let telemetry = Telemetry::default();
+ let _guard = telemetry.install();
+ let (repo, link) = super::super::test::fixture(&db).await;
+ for (id, attempts) in [("window", 0), ("graph", 0), ("token", 0), ("moving", 0)] {
+ queue(&db, link, id, true, attempts).await;
+ }
+ let service = sync(
+ &repo,
+ Provider::answering(vec![
+ Err(EmailApiError::RateLimited {
+ retry_after: Some(Duration::from_secs(12)),
+ origin: RateLimitOrigin::Gate(GateLimit::Window),
+ }),
+ Err(EmailApiError::RateLimited {
+ retry_after: Some(Duration::from_secs(7)),
+ origin: RateLimitOrigin::Provider { concurrency: false },
+ }),
+ Err(EmailApiError::CredentialsUnavailable {
+ message: "authentication unavailable".into(),
+ }),
+ Ok(None),
+ ]),
+ );
+
+ for _ in 0..4 {
+ assert!(service.reconcile_once().await.is_err());
+ }
+
+ let spans = telemetry.spans("outlook.sync.reconcile");
+ let reasons: Vec<_> = spans
+ .iter()
+ .map(|span| {
+ (
+ span["outlook.outcome"].as_str(),
+ span["outlook.defer_reason"].as_str(),
+ span["outlook.defer_seconds"].as_str(),
+ )
+ })
+ .collect();
+ assert_eq!(
+ reasons,
+ [
+ ("deferred", "gate_window", "12"),
+ ("deferred", "graph_throttled", "7"),
+ ("deferred", "token_unavailable", "30"),
+ ("deferred", "location_unsettled", "30"),
+ ]
+ );
+ assert!(
+ spans
+ .iter()
+ .all(|span| !span.contains_key("otel.status_description"))
+ );
+ let deferrals = telemetry.events("mailbox sync work deferred");
+ let levels: Vec<_> = deferrals
+ .iter()
+ .map(|(level, fields)| {
+ (
+ *level,
+ fields["defer_reason"].as_str(),
+ fields["link_id"].clone(),
+ )
+ })
+ .collect();
+ assert_eq!(
+ levels,
+ [
+ (tracing::Level::DEBUG, "gate_window", link.to_string()),
+ (tracing::Level::WARN, "graph_throttled", link.to_string()),
+ (tracing::Level::WARN, "token_unavailable", link.to_string()),
+ (tracing::Level::WARN, "location_unsettled", link.to_string()),
+ ]
+ );
+ let parked = sqlx::query_scalar!(
+ r#"SELECT count(*) AS "parked!" FROM email_message_reconciliation WHERE link_id = $1 AND available_at > now() + interval '10 seconds'"#,
+ link
+ )
+ .fetch_one(&db)
+ .await
+ .unwrap();
+ assert_eq!(
+ parked, 3,
+ "the window refusal and both 30-second deferrals are parked"
+ );
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn enumeration_reports_the_page_it_committed(db: PgPool) {
+ let telemetry = Telemetry::default();
+ let _guard = telemetry.install();
+ let (repo, link) = super::super::test::fixture(&db).await;
+ let provider = Provider::default();
+ provider
+ .pages
+ .lock()
+ .unwrap()
+ .push_back(Ok(super::super::test::page()));
+ let service = sync(&repo, provider);
+
+ assert!(service.enumerate_once().await.unwrap());
+
+ let spans = telemetry.spans("outlook.sync.enumerate");
+ let [span] = spans.as_slice() else {
+ panic!("expected one enumerate span, got {spans:?}");
+ };
+ assert_eq!(span["mailbox.link_id"], link.to_string());
+ assert_eq!(span["outlook.stream_kind"], "mail_folder");
+ assert_eq!(span["outlook.outcome"], "imported");
+ assert_eq!(span["outlook.page_items"], "4");
+ assert_eq!(span["outlook.rows_queued"], "3");
+ assert_eq!(span["outlook.delta_advanced"], "true");
+ assert_eq!(span["outlook.folder_initial_complete"], "true");
+ assert!(matches!(
+ super::super::test::page().position,
+ StreamPosition::Checkpoint(_)
+ ));
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn persistence_errors_name_their_sqlstate(db: PgPool) {
+ let failure = sqlx::query_scalar!(r#"SELECT 1 / $1::int AS "quotient""#, 0)
+ .fetch_one(&db)
+ .await
+ .unwrap_err();
+
+ let error = super::super::db_error(failure);
+
+ assert_eq!(
+ error.to_string(),
+ "mailbox persistence is unavailable: sqlstate 22012"
+ );
+ assert_eq!(
+ crate::domain::mailbox::telemetry::defer_reason(&error),
+ "persistence"
+ );
+ // Projection effects that fail outside the database are reported apart.
+ assert_eq!(
+ crate::domain::mailbox::telemetry::defer_reason(&MailboxError::Effect),
+ "effect"
+ );
+}
+
+async fn outlook_link(db: &PgPool, email: &str, active: bool, needs_reauth: bool) -> Uuid {
+ let link = macro_uuid::generate_uuid_v7();
+ sqlx::query!(
+ r#"INSERT INTO email_links (id,macro_id,fusionauth_user_id,email_address,provider,grant_generation,is_sync_active,needs_reauth)
+ VALUES ($1,$2,$3,$4,'OUTLOOK',1,$5,$6)"#,
+ link,
+ format!("macro|{email}"),
+ format!("owner-{email}"),
+ email,
+ active,
+ needs_reauth
+ )
+ .execute(db)
+ .await
+ .unwrap();
+ link
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn health_snapshot_counts_the_current_queue_streams_and_recent_imports(db: PgPool) {
+ let (repo, active) = super::super::test::fixture(&db).await;
+ sqlx::query!(
+ r#"INSERT INTO email_message_reconciliation (link_id,generation,provider_id,is_import,available_at,lease_id,lease_until) VALUES
+ ($1,1,'due',true,now() - interval '90 seconds',NULL,NULL),
+ ($1,1,'parked',true,now() + interval '600 seconds',NULL,NULL),
+ ($1,1,'leased',true,now() - interval '10 seconds',$2,now() + interval '60 seconds'),
+ ($1,0,'old generation',true,now() - interval '1 hour',NULL,NULL)"#,
+ active,
+ macro_uuid::generate_uuid_v7()
+ )
+ .execute(&db)
+ .await
+ .unwrap();
+ sqlx::query!(
+ r#"INSERT INTO email_mailbox_import_minutes (link_id,minute,imported,last_imported_at) VALUES
+ ($1,date_trunc('minute',now() - interval '2 minutes'),5,now() - interval '100 seconds'),
+ ($1,date_trunc('minute',now() - interval '10 minutes'),7,now() - interval '10 minutes'),
+ ($1,date_trunc('minute',now() - interval '2 days'),9,now() - interval '2 days')"#,
+ active
+ )
+ .execute(&db)
+ .await
+ .unwrap();
+ let paused = outlook_link(&db, "[email protected]", false, true).await;
+ queue(&db, paused, "waiting", true, 0).await;
+ outlook_link(&db, "[email protected]", false, false).await;
+ sqlx::query!(
+ "INSERT INTO email_links (id,macro_id,fusionauth_user_id,email_address,provider) VALUES ($1,'macro|[email protected]','gmail-owner','[email protected]','GMAIL')",
+ macro_uuid::generate_uuid_v7()
+ )
+ .execute(&db)
+ .await
+ .unwrap();
+
+ let backlogs = repo.backlogs().await.unwrap();
+
+ assert_eq!(backlogs.len(), 2, "{backlogs:?}");
+ let active_backlog = backlogs.iter().find(|b| b.link_id == active).unwrap();
+ assert_eq!(
+ (
+ active_backlog.queued,
+ active_backlog.parked,
+ active_backlog.leased,
+ active_backlog.streams_incomplete,
+ active_backlog.imported_5m,
+ active_backlog.sync_active,
+ active_backlog.needs_reauth,
+ ),
+ (3, 1, 1, 1, 5, true, false)
+ );
+ assert!((90..=92).contains(&active_backlog.oldest_waiting_seconds));
+ assert!((598..=600).contains(&active_backlog.parked_until_max_seconds));
+ assert!(matches!(
+ active_backlog.seconds_since_last_import,
+ Some(100..=102)
+ ));
+ let paused_backlog = backlogs.iter().find(|b| b.link_id == paused).unwrap();
+ assert_eq!(
+ (
+ paused_backlog.queued,
+ paused_backlog.parked,
+ paused_backlog.oldest_waiting_seconds,
+ paused_backlog.parked_until_max_seconds,
+ paused_backlog.imported_5m,
+ paused_backlog.seconds_since_last_import,
+ paused_backlog.sync_active,
+ paused_backlog.needs_reauth,
+ ),
+ (1, 0, 0, 0, 0, None, false, true)
+ );
+ let retained = sqlx::query_scalar!(
+ r#"SELECT count(*) AS "rows!" FROM email_mailbox_import_minutes WHERE link_id = $1"#,
+ active
+ )
+ .fetch_one(&db)
+ .await
+ .unwrap();
+ assert_eq!(retained, 2, "counts older than a day are dropped");
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn import_counts_from_several_workers_add_up_in_the_snapshot(db: PgPool) {
+ let telemetry = Telemetry::default();
+ let _guard = telemetry.install();
+ let (repo, link) = super::super::test::fixture(&db).await;
+ let first = ImportTally::default();
+ let second = ImportTally::default();
+ for _ in 0..3 {
+ first.record(link, chrono::Utc::now());
+ }
+ second.record(link, chrono::Utc::now());
+
+ MailboxProgressService::new(repo.clone(), first)
+ .flush()
+ .await
+ .unwrap();
+ MailboxProgressService::new(repo.clone(), second.clone())
+ .flush()
+ .await
+ .unwrap();
+ // A flush with nothing new writes nothing.
+ MailboxProgressService::new(repo.clone(), second)
+ .flush()
+ .await
+ .unwrap();
+ let reported = MailboxProgressService::new(repo.clone(), ImportTally::default())
+ .report()
+ .await
+ .unwrap();
+
+ assert_eq!(reported, 1);
+ let snapshots = telemetry.events("outlook mailbox health");
+ let [(level, fields)] = snapshots.as_slice() else {
+ panic!("expected one snapshot, got {snapshots:?}");
+ };
+ assert_eq!(*level, tracing::Level::INFO);
+ assert_eq!(fields["link_id"], link.to_string());
+ assert_eq!(fields["imported_5m"], "4");
+ assert_eq!(fields["queued"], "0");
+ assert_eq!(fields["oldest_waiting_seconds"], "0");
+ assert_eq!(fields["sync_active"], "true");
+ assert!(fields.contains_key("seconds_since_last_import"));
+}
diff --git a/crates/email/src/outbound/mailbox_pg/projection.rs b/crates/email/src/outbound/mailbox_pg/projection.rs
index 842f14ced5..3f040a674b 100644
--- a/crates/email/src/outbound/mailbox_pg/projection.rs
+++ b/crates/email/src/outbound/mailbox_pg/projection.rs
@@ -28,7 +28,7 @@ impl MailboxProjectionRepository for PgMailboxSync {
)
UPDATE email_projection_outbox o SET lease_id = $1,lease_until = now() + interval '3 minutes',attempts = attempts + 1
FROM candidate c,email_links l WHERE o.id = c.id AND l.id = o.link_id
- RETURNING o.id,o.link_id,o.generation,o.payload,l.sync_generation,l.grant_generation
+ RETURNING o.id,o.link_id,o.generation,o.payload,o.attempts,l.sync_generation,l.grant_generation
"#,lease_id).fetch_optional(&self.db).await.map_err(db_error)?;
row.map(|row| {
Ok(ProjectionLease {
@@ -42,6 +42,7 @@ impl MailboxProjectionRepository for PgMailboxSync {
event: serde_json::from_value(row.payload)
.map_err(|_| MailboxError::Persistence)?,
from_current_generation: row.generation == row.sync_generation,
+ attempts: row.attempts,
})
})
.transpose()
@@ -53,7 +54,7 @@ impl MailboxProjectionRepository for PgMailboxSync {
) -> Result<Option<ProjectionContext>, MailboxError> {
let link = email_db_client::links::get::fetch_link_by_id(&self.db, lease.mailbox.link_id)
.await
- .map_err(|_| MailboxError::Persistence)?;
+ .map_err(wrapped_db_error)?;
let Some(link) = link.filter(|link| link.is_sync_active) else {
return Ok(None);
};
@@ -89,22 +90,26 @@ impl MailboxProjectionRepository for PgMailboxSync {
async fn finish_projection(
&self,
lease: &ProjectionLease,
- success: bool,
+ retry_after_seconds: Option<u32>,
) -> Result<(), MailboxError> {
- if success {
- sqlx::query!("DELETE FROM email_projection_outbox WHERE id = $1 AND lease_id = $2 AND lease_until > now()",lease.id,lease.lease_id)
- .execute(&self.db).await.map_err(db_error)?;
- } else {
- sqlx::query!(
- r#"UPDATE email_projection_outbox SET lease_id = NULL,lease_until = NULL,
- available_at = now() + make_interval(secs => LEAST(600,30 * attempts))
- WHERE id = $1 AND lease_id = $2"#,
- lease.id,
- lease.lease_id
- )
- .execute(&self.db)
- .await
- .map_err(db_error)?;
+ match retry_after_seconds {
+ None => {
+ sqlx::query!("DELETE FROM email_projection_outbox WHERE id = $1 AND lease_id = $2 AND lease_until > now()",lease.id,lease.lease_id)
+ .execute(&self.db).await.map_err(db_error)?;
+ }
+ Some(delay) => {
+ sqlx::query!(
+ r#"UPDATE email_projection_outbox SET lease_id = NULL,lease_until = NULL,
+ available_at = now() + make_interval(secs => $3)
+ WHERE id = $1 AND lease_id = $2"#,
+ lease.id,
+ lease.lease_id,
+ f64::from(delay)
+ )
+ .execute(&self.db)
+ .await
+ .map_err(db_error)?;
+ }
}
Ok(())
}
@@ -141,7 +146,7 @@ impl PgMailboxSync {
vec![db_message],
)
.await
- .map_err(|_| MailboxError::Persistence)?
+ .map_err(wrapped_db_error)?
.pop()
else {
return Ok(None);
diff --git a/crates/email/src/outbound/mailbox_pg/test.rs b/crates/email/src/outbound/mailbox_pg/test.rs
index 2dd3383d74..a94742ab32 100644
--- a/crates/email/src/outbound/mailbox_pg/test.rs
+++ b/crates/email/src/outbound/mailbox_pg/test.rs
@@ -385,7 +385,7 @@ async fn projection_ack_keeps_version_watermark_and_exposes_normalized_state(db:
);
assert!(message.is_present);
repo.renew_projection(&event).await.unwrap();
- repo.finish_projection(&event, true).await.unwrap();
+ repo.finish_projection(&event, None).await.unwrap();
repo.ingest(
&work,
snapshot(link, work.provider_id.as_str(), state, "v1"),
@@ -429,7 +429,7 @@ async fn projection_crash_reclaims_work_and_old_worker_cannot_ack_successor(db:
repo.renew_projection(&old).await,
Err(MailboxError::Stale)
));
- repo.finish_projection(&old, true).await.unwrap();
+ repo.finish_projection(&old, None).await.unwrap();
assert_eq!(
sqlx::query_scalar!(
"SELECT count(*) FROM email_projection_outbox WHERE id = $1",
@@ -440,7 +440,7 @@ async fn projection_crash_reclaims_work_and_old_worker_cannot_ack_successor(db:
.unwrap(),
Some(1)
);
- repo.finish_projection(&successor, true).await.unwrap();
+ repo.finish_projection(&successor, None).await.unwrap();
}
#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
diff --git a/crates/email/src/telemetry_capture.rs b/crates/email/src/telemetry_capture.rs
new file mode 100644
index 0000000000..8ac4377ba6
--- /dev/null
+++ b/crates/email/src/telemetry_capture.rs
@@ -0,0 +1,97 @@
+//! Records closed spans and events so tests can assert on telemetry fields.
+
+use std::collections::HashMap;
+use std::sync::{Arc, Mutex};
+use tracing::field::{Field, Visit};
+use tracing::span::{Attributes, Id, Record};
+use tracing_subscriber::layer::{Context, SubscriberExt};
+use tracing_subscriber::registry::LookupSpan;
+
+pub(crate) type Fields = HashMap<&'static str, String>;
+
+#[derive(Default)]
+struct Values(Fields);
+
+impl Visit for Values {
+ fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) {
+ self.0.insert(field.name(), format!("{value:?}"));
+ }
+ fn record_str(&mut self, field: &Field, value: &str) {
+ self.0.insert(field.name(), value.to_owned());
+ }
+}
+
+#[derive(Default)]
+struct Captured {
+ spans: Vec<(&'static str, Fields)>,
+ events: Vec<(tracing::Level, Fields)>,
+}
+
+/// Install with [`Telemetry::install`] for the current thread.
+#[derive(Clone, Default)]
+pub(crate) struct Telemetry(Arc<Mutex<Captured>>);
+
+impl Telemetry {
+ pub(crate) fn install(&self) -> tracing::subscriber::DefaultGuard {
+ let guard =
+ tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()));
+ // A callsite another test thread registered while this subscriber was
+ // being installed can keep a cached "never" interest; recompute it.
+ tracing::callsite::rebuild_interest_cache();
+ guard
+ }
+
+ /// Closed spans with this name, oldest first.
+ pub(crate) fn spans(&self, name: &str) -> Vec<Fields> {
+ let captured = self.0.lock().unwrap();
+ captured
+ .spans
+ .iter()
+ .filter(|(span, _)| *span == name)
+ .map(|(_, fields)| fields.clone())
+ .collect()
+ }
+
+ /// Events whose message is `message`, with their level.
+ pub(crate) fn events(&self, message: &str) -> Vec<(tracing::Level, Fields)> {
+ let captured = self.0.lock().unwrap();
+ captured
+ .events
+ .iter()
+ .filter(|(_, fields)| fields.get("message").is_some_and(|text| text == message))
+ .cloned()
+ .collect()
+ }
+}
+
+impl<S: tracing::Subscriber + for<'a> LookupSpan<'a>> tracing_subscriber::Layer<S> for Telemetry {
+ fn on_new_span(&self, attrs: &Attributes<'_>, id: &Id, ctx: Context<'_, S>) {
+ let mut values = Values::default();
+ attrs.record(&mut values);
+ if let Some(span) = ctx.span(id) {
+ span.extensions_mut().insert(values);
+ }
+ }
+ fn on_record(&self, id: &Id, record: &Record<'_>, ctx: Context<'_, S>) {
+ if let Some(span) = ctx.span(id)
+ && let Some(values) = span.extensions_mut().get_mut::<Values>()
+ {
+ record.record(values);
+ }
+ }
+ fn on_event(&self, event: &tracing::Event<'_>, _ctx: Context<'_, S>) {
+ let mut values = Values::default();
+ event.record(&mut values);
+ self.0
+ .lock()
+ .unwrap()
+ .events
+ .push((*event.metadata().level(), values.0));
+ }
+ fn on_close(&self, id: Id, ctx: Context<'_, S>) {
+ if let Some(span) = ctx.span(&id) {
+ let values = span.extensions_mut().remove::<Values>().unwrap_or_default();
+ self.0.lock().unwrap().spans.push((span.name(), values.0));
+ }
+ }
+}
diff --git a/crates/email_api_client/AGENTS.md b/crates/email_api_client/AGENTS.md
index ee8899c494..c50a4aae75 100644
--- a/crates/email_api_client/AGENTS.md
+++ b/crates/email_api_client/AGENTS.md
@@ -22,6 +22,9 @@ Outlook (Microsoft Graph) outbound adapters.
which owns request headers, provider error mapping, write-failure logs,
and one `outlook.graph_request` span per attempt (with an
`outlook.graph_admission` child for the per-mailbox gate wait).
+- `src/outbound/microsoft_gate.rs`: the Redis admission every Graph request
+ passes (window, concurrency, cooldown), which also counts admitted requests
+ per sync generation.
## Scope
@@ -90,6 +93,18 @@ Outlook (Microsoft Graph) outbound adapters.
through `request`/`conditional_request` rather than the raw client. Use
`graph_request` only when a caller must branch on the Graph status, error
code, or request id; the others drop them.
+- `outlook.graph_request` is the Graph telemetry contract the Outlook sync
+ monitors in `infra/stacks/monitoring` read. Every attempt records
+ `outlook.throttle_origin` (`gate`, `graph` or `none`); a gate refusal
+ (`RateLimitOrigin::Gate` with its `GateLimit`) sent nothing, so it is not
+ a span error and logs at DEBUG. The span sets `operation.name` because
+ Datadog files a client span with an HTTP method as `http.client.request`,
+ and its `url.template` comes from `outlook/route.rs`, which keeps only
+ known Graph path words so ids and user-chosen names never reach it. Record
+ numeric fields as `i64`; unsigned values reach the collector as strings.
+- `domain::request_count::counted` counts the Graph requests one unit of work
+ sends (the transport calls `record_sent`); Outlook token failures surface
+ as `EmailApiError::CredentialsUnavailable`, apart from Graph failures.
- Same-mailbox Outlook replies use `createReply`. When Graph refuses it with
400 `ErrorInvalidOperation`, `create_draft` creates a standalone draft that
carries the same correlation and RFC reply headers.
diff --git a/crates/email_api_client/Cargo.toml b/crates/email_api_client/Cargo.toml
index cdc54b0dc7..73c2eff57a 100644
--- a/crates/email_api_client/Cargo.toml
+++ b/crates/email_api_client/Cargo.toml
@@ -6,7 +6,7 @@ version = "0.1.0"
[features]
default = []
-ports = []
+ports = ["dep:tokio"]
outbound-gmail = [
"ports",
"dep:base64",
@@ -44,6 +44,7 @@ serde_json = { workspace = true, optional = true }
reqwest = { workspace = true, optional = true }
url = { workspace = true, optional = true }
thiserror = { workspace = true }
+tokio = { workspace = true, optional = true }
tracing = { workspace = true }
uuid = { workspace = true }
workspace-hack = { version = "0.1", path = "../workspace-hack" }
@@ -54,4 +55,5 @@ macro_db_migrator = { path = "../macro_db_migrator" }
macro_event_broker = { path = "../macro_event_broker", default-features = false }
sqlx = { workspace = true }
tokio = { workspace = true }
+tracing-subscriber = { workspace = true }
wiremock = { workspace = true }
diff --git a/crates/email_api_client/src/domain/mod.rs b/crates/email_api_client/src/domain/mod.rs
index 1ff795a668..ae2440313d 100644
--- a/crates/email_api_client/src/domain/mod.rs
+++ b/crates/email_api_client/src/domain/mod.rs
@@ -10,3 +10,7 @@ pub mod ports;
/// Provider-neutral email API orchestration.
#[cfg(feature = "ports")]
pub mod service;
+
+/// Counts provider requests made inside one unit of work.
+#[cfg(feature = "ports")]
+pub mod request_count;
diff --git a/crates/email_api_client/src/domain/models/error.rs b/crates/email_api_client/src/domain/models/error.rs
index c0722f13c8..baa234eb52 100644
--- a/crates/email_api_client/src/domain/models/error.rs
+++ b/crates/email_api_client/src/domain/models/error.rs
@@ -48,6 +48,13 @@ pub enum EmailApiError {
/// A sanitized diagnostic suitable for logs.
message: String,
},
+ /// No access token could be obtained right now, so no provider request was
+ /// made. Retrying may succeed.
+ #[error("email provider credentials are temporarily unavailable: {message}")]
+ CredentialsUnavailable {
+ /// A sanitized diagnostic suitable for logs.
+ message: String,
+ },
/// A deterministic failure that should not be retried unchanged.
#[error("permanent email provider failure: {message}")]
Permanent {
@@ -66,7 +73,10 @@ pub enum EmailApiError {
impl EmailApiError {
/// Returns whether retrying the same operation may succeed.
pub fn is_transient(&self) -> bool {
- matches!(self, Self::RateLimited { .. } | Self::Transient { .. })
+ matches!(
+ self,
+ Self::RateLimited { .. } | Self::Transient { .. } | Self::CredentialsUnavailable { .. }
+ )
}
}
@@ -79,6 +89,9 @@ impl EmailApiError {
pub enum RateLimitOrigin {
/// Refused by local quota policy before any provider call was made.
Local,
+ /// Refused by the shared per-mailbox Microsoft request gate before any
+ /// provider call was made, naming the limit that refused it.
+ Gate(GateLimit),
/// The provider itself throttled the request.
Provider {
/// The provider refused because the mailbox had too many requests in
@@ -87,6 +100,17 @@ pub enum RateLimitOrigin {
},
}
+/// The per-mailbox gate limit that refused a request.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum GateLimit {
+ /// The mailbox spent its request allowance for the current window.
+ Window,
+ /// The mailbox already has its maximum number of requests in flight.
+ Concurrency,
+ /// The mailbox is cooling down after the provider throttled it.
+ Cooldown,
+}
+
impl From<RateLimitRefusal> for EmailApiError {
fn from(refusal: RateLimitRefusal) -> Self {
Self::RateLimited {
diff --git a/crates/email_api_client/src/domain/models/mod.rs b/crates/email_api_client/src/domain/models/mod.rs
index 0ec5668b95..12f6abcee7 100644
--- a/crates/email_api_client/src/domain/models/mod.rs
+++ b/crates/email_api_client/src/domain/models/mod.rs
@@ -16,7 +16,7 @@ pub use address_book::*;
pub use auth::{AccessToken, TokenError, TokenFreshness};
pub use changes::{ChangeBatch, ChangePage, InboxChanges, SyncCursor};
pub use draft::*;
-pub use error::{EmailApiError, RateLimitOrigin, RateLimitRefusal};
+pub use error::{EmailApiError, GateLimit, RateLimitOrigin, RateLimitRefusal};
pub use mailbox::{
CalendarPart, ContactPage, MessageWithCalendarParts, ProviderSubscription, ThreadListPage,
};
diff --git a/crates/email_api_client/src/domain/ports/mailbox.rs b/crates/email_api_client/src/domain/ports/mailbox.rs
index c9ed8aa5af..996f422a55 100644
--- a/crates/email_api_client/src/domain/ports/mailbox.rs
+++ b/crates/email_api_client/src/domain/ports/mailbox.rs
@@ -233,6 +233,12 @@ pub trait MailboxDraftClient: Send + Sync + 'static {
bytes: &[u8],
) -> impl Future<Output = Result<crate::domain::models::UploadProgress, EmailApiError>> + Send;
}
+/// Reads how many requests a mailbox's request gate admitted.
+pub trait MailboxRequestCount: Send + Sync + 'static {
+ /// Requests admitted for the mailbox's sync generation, when known.
+ fn admitted_requests(&self, mailbox: MailboxAccess)
+ -> impl Future<Output = Option<u64>> + Send;
+}
/// Binds request accounting to the same mailbox as credential acquisition.
pub trait ScopedMailboxRepository: Clone + Send + Sync + 'static {
/// Return a request-scoped clone; credentials remain supplied separately.
@@ -254,6 +260,12 @@ pub trait MailboxRequestGate: Send + Sync + 'static {
permit: uuid::Uuid,
retry_after: Option<std::time::Duration>,
) -> std::pin::Pin<Box<dyn Future<Output = ()> + Send + '_>>;
+ /// Requests admitted for this mailbox's sync generation, or `None` when
+ /// the count is unavailable.
+ fn admitted(
+ &self,
+ mailbox: MailboxAccess,
+ ) -> std::pin::Pin<Box<dyn Future<Output = Option<u64>> + Send + '_>>;
}
/// Settings capabilities which require explicit rule ownership and bounded cleanup.
diff --git a/crates/email_api_client/src/domain/request_count.rs b/crates/email_api_client/src/domain/request_count.rs
new file mode 100644
index 0000000000..9777db6647
--- /dev/null
+++ b/crates/email_api_client/src/domain/request_count.rs
@@ -0,0 +1,29 @@
+//! Counts the provider requests one unit of work sends, such as the Graph
+//! calls that reconcile one message, without threading a counter through
+//! every port.
+
+use std::cell::Cell;
+use std::future::Future;
+
+#[cfg(test)]
+mod test;
+
+tokio::task_local! {
+ static SENT: Cell<u32>;
+}
+
+/// Run `operation`, returning its output and how many provider requests it
+/// sent. Requests from tasks it spawns are not counted.
+pub async fn counted<F: Future>(operation: F) -> (F::Output, u32) {
+ SENT.scope(Cell::new(0), async {
+ let output = operation.await;
+ (output, SENT.with(Cell::get))
+ })
+ .await
+}
+
+/// Count one request actually sent to the provider. Outside [`counted`] this
+/// does nothing.
+pub fn record_sent() {
+ let _ = SENT.try_with(|sent| sent.set(sent.get().saturating_add(1)));
+}
diff --git a/crates/email_api_client/src/domain/request_count/test.rs b/crates/email_api_client/src/domain/request_count/test.rs
new file mode 100644
index 0000000000..9fea8696ed
--- /dev/null
+++ b/crates/email_api_client/src/domain/request_count/test.rs
@@ -0,0 +1,18 @@
+use super::{counted, record_sent};
+
+#[tokio::test]
+async fn counts_only_requests_inside_the_scope() {
+ record_sent();
+ let ((), outer) = counted(async {
+ record_sent();
+ let ((), inner) = counted(async {
+ record_sent();
+ record_sent();
+ })
+ .await;
+ assert_eq!(inner, 2);
+ record_sent();
+ })
+ .await;
+ assert_eq!(outer, 2);
+}
diff --git a/crates/email_api_client/src/domain/service.rs b/crates/email_api_client/src/domain/service.rs
index b304de7001..0df1a1b8ed 100644
--- a/crates/email_api_client/src/domain/service.rs
+++ b/crates/email_api_client/src/domain/service.rs
@@ -208,7 +208,7 @@ where
fn log_operation_error(error: &EmailApiError) {
match error {
EmailApiError::RateLimited {
- origin: RateLimitOrigin::Local,
+ origin: RateLimitOrigin::Local | RateLimitOrigin::Gate(_),
..
} => tracing::debug!(error = %error),
EmailApiError::RateLimited { origin, .. } => {
diff --git a/crates/email_api_client/src/domain/service/mailbox.rs b/crates/email_api_client/src/domain/service/mailbox.rs
index 64f7cba865..44f003a570 100644
--- a/crates/email_api_client/src/domain/service/mailbox.rs
+++ b/crates/email_api_client/src/domain/service/mailbox.rs
@@ -41,7 +41,21 @@ impl<R, T: MailboxTokenSource, L: ProviderRateLimiter> MailboxApiService<R, T, L
self.tokens
.access_token(mailbox, freshness)
.await
- .map_err(super::map_token_error)
+ .map_err(|error| match error {
+ // No provider request was made, which callers report apart
+ // from provider failures.
+ TokenError::Transient { message } => {
+ EmailApiError::CredentialsUnavailable { message }
+ }
+ error => super::map_token_error(error),
+ })
+ }
+}
+
+impl<R: MailboxRequestCount, T, L> MailboxApiService<R, T, L> {
+ /// Requests the mailbox's gate admitted in its current sync generation.
+ pub async fn admitted_requests(&self, mailbox: MailboxAccess) -> Option<u64> {
+ self.repository.admitted_requests(mailbox).await
}
}
diff --git a/crates/email_api_client/src/outbound/microsoft_gate.rs b/crates/email_api_client/src/outbound/microsoft_gate.rs
index 7f146e120b..7ede00499a 100644
--- a/crates/email_api_client/src/outbound/microsoft_gate.rs
+++ b/crates/email_api_client/src/outbound/microsoft_gate.rs
@@ -1,7 +1,7 @@
//! Shared admission for every Graph request, including nested pagination.
use crate::domain::{
- models::{EmailApiError, MailboxAccess, RateLimitOrigin},
+ models::{EmailApiError, GateLimit, MailboxAccess, RateLimitOrigin},
ports::MailboxRequestGate,
};
use redis::aio::MultiplexedConnection;
@@ -19,27 +19,34 @@ const CONCURRENT_REQUESTS: u32 = 3;
const WINDOW_MS: u32 = 600_000;
// Greater than the Graph transport's 60-second total request timeout.
const PERMIT_MS: u32 = 90_000;
+// Admitted requests are counted per sync generation for import statistics.
+// Long enough to outlast any import.
+const REQUEST_COUNT_MS: u64 = 30 * 24 * 3_600_000;
+// Returns `{limit, delay_ms}`: limit 0 admits the request, otherwise it names
+// the refusing limit (1 cooldown, 2 concurrency, 3 window).
const ACQUIRE: &str = r#"
local clock = redis.call('TIME')
local now = tonumber(clock[1]) * 1000 + math.floor(tonumber(clock[2]) / 1000)
local cooldown = redis.call('PTTL', KEYS[3])
-if cooldown > 0 then return cooldown end
+if cooldown > 0 then return {1, cooldown} end
redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', now)
redis.call('ZREMRANGEBYSCORE', KEYS[2], '-inf', now - tonumber(ARGV[4]))
if redis.call('ZCARD', KEYS[1]) >= tonumber(ARGV[2]) then
local first = redis.call('ZRANGE', KEYS[1], 0, 0, 'WITHSCORES')
- return math.max(1, tonumber(first[2]) - now)
+ return {2, math.max(1, tonumber(first[2]) - now)}
end
if redis.call('ZCARD', KEYS[2]) >= tonumber(ARGV[3]) then
local first = redis.call('ZRANGE', KEYS[2], 0, 0, 'WITHSCORES')
- return math.max(1, tonumber(first[2]) + tonumber(ARGV[4]) - now)
+ return {3, math.max(1, tonumber(first[2]) + tonumber(ARGV[4]) - now)}
end
redis.call('ZADD', KEYS[1], now + tonumber(ARGV[5]), ARGV[1])
redis.call('PEXPIRE', KEYS[1], ARGV[5])
redis.call('ZADD', KEYS[2], now, ARGV[1])
redis.call('PEXPIRE', KEYS[2], ARGV[4])
-return 0
+redis.call('HINCRBY', KEYS[4], ARGV[6], 1)
+redis.call('PEXPIRE', KEYS[4], ARGV[7])
+return {0, 0}
"#;
const FINISH: &str = r#"
@@ -55,7 +62,7 @@ return 1
#[derive(Clone)]
pub struct RedisMicrosoftRequestGate(pub MultiplexedConnection);
-fn keys(mailbox: MailboxAccess) -> [String; 3] {
+fn keys(mailbox: MailboxAccess) -> [String; 4] {
// Hash tags colocate the Lua keys in Redis Cluster. Generations share the
// quota: refreshing or reconnecting must not reset Microsoft's allowance.
let scope = format!("outlook:{{{}}}", mailbox.link_id);
@@ -63,6 +70,7 @@ fn keys(mailbox: MailboxAccess) -> [String; 3] {
format!("{scope}:inflight"),
format!("{scope}:window"),
format!("{scope}:cooldown"),
+ format!("{scope}:requests"),
]
}
@@ -73,28 +81,34 @@ impl MailboxRequestGate for RedisMicrosoftRequestGate {
) -> Pin<Box<dyn Future<Output = Result<Uuid, EmailApiError>> + Send + '_>> {
Box::pin(async move {
let permit = Uuid::now_v7();
- let [inflight, window, cooldown] = keys(mailbox);
- let delay: u64 = redis::Script::new(ACQUIRE)
+ let [inflight, window, cooldown, requests] = keys(mailbox);
+ let (limit, delay): (u8, u64) = redis::Script::new(ACQUIRE)
.key(inflight)
.key(window)
.key(cooldown)
+ .key(requests)
.arg(permit.to_string())
.arg(CONCURRENT_REQUESTS)
.arg(REQUESTS_PER_WINDOW)
.arg(WINDOW_MS)
.arg(PERMIT_MS)
+ .arg(mailbox.sync_generation)
+ .arg(REQUEST_COUNT_MS)
.invoke_async(&mut self.0.clone())
.await
.map_err(|_| EmailApiError::Transient {
message: "Microsoft request admission is unavailable".into(),
})?;
- if delay > 0 {
- return Err(EmailApiError::RateLimited {
- retry_after: Some(Duration::from_millis(delay)),
- origin: RateLimitOrigin::Local,
- });
- }
- Ok(permit)
+ let limit = match limit {
+ 0 => return Ok(permit),
+ 1 => GateLimit::Cooldown,
+ 2 => GateLimit::Concurrency,
+ _ => GateLimit::Window,
+ };
+ Err(EmailApiError::RateLimited {
+ retry_after: Some(Duration::from_millis(delay)),
+ origin: RateLimitOrigin::Gate(limit),
+ })
})
}
fn finish(
@@ -104,7 +118,7 @@ impl MailboxRequestGate for RedisMicrosoftRequestGate {
retry_after: Option<Duration>,
) -> Pin<Box<dyn Future<Output = ()> + Send + '_>> {
Box::pin(async move {
- let [inflight, _, cooldown] = keys(mailbox);
+ let [inflight, _, cooldown, _] = keys(mailbox);
let delay = retry_after
.map(|delay| delay.as_millis().clamp(1, 86_400_000) as u64)
.unwrap_or(0);
@@ -122,4 +136,23 @@ impl MailboxRequestGate for RedisMicrosoftRequestGate {
}
})
}
+ fn admitted(
+ &self,
+ mailbox: MailboxAccess,
+ ) -> Pin<Box<dyn Future<Output = Option<u64>> + Send + '_>> {
+ Box::pin(async move {
+ let [.., requests] = keys(mailbox);
+ let admitted: redis::RedisResult<Option<u64>> = redis::cmd("HGET")
+ .arg(requests)
+ .arg(mailbox.sync_generation)
+ .query_async(&mut self.0.clone())
+ .await;
+ admitted
+ .inspect_err(|error| {
+ tracing::warn!(link_id = %mailbox.link_id, %error, "failed to read admitted Microsoft requests");
+ })
+ .ok()
+ .flatten()
+ })
+ }
}
diff --git a/crates/email_api_client/src/outbound/microsoft_gate/test.rs b/crates/email_api_client/src/outbound/microsoft_gate/test.rs
index 628d141f7a..36d0fbccaa 100644
--- a/crates/email_api_client/src/outbound/microsoft_gate/test.rs
+++ b/crates/email_api_client/src/outbound/microsoft_gate/test.rs
@@ -39,7 +39,7 @@ async fn concurrent_requests_share_limits_and_reconnect_does_not_reset_them() {
assert!(matches!(
gate.acquire(reconnected).await,
Err(EmailApiError::RateLimited {
- origin: RateLimitOrigin::Local,
+ origin: RateLimitOrigin::Gate(GateLimit::Concurrency),
..
})
));
@@ -53,10 +53,20 @@ async fn concurrent_requests_share_limits_and_reconnect_does_not_reset_them() {
match gate.acquire(mailbox).await {
Err(EmailApiError::RateLimited {
retry_after: Some(delay),
- ..
+ origin: RateLimitOrigin::Gate(GateLimit::Cooldown),
}) => assert!(delay > Duration::from_secs(80)),
_ => panic!("a shorter cooldown must not erase an existing provider Retry-After"),
}
+ // Four requests were admitted, all under sync generation 1.
+ assert_eq!(gate.admitted(mailbox).await, Some(4));
+ assert_eq!(
+ gate.admitted(MailboxAccess {
+ sync_generation: 2,
+ ..mailbox
+ })
+ .await,
+ None
+ );
cleanup(&gate, mailbox).await;
}
@@ -79,7 +89,7 @@ async fn request_window_survives_permit_release_and_reconnect() {
.unwrap();
assert!(
matches!(gate.acquire(MailboxAccess { grant_generation: 2,..mailbox }).await,
- Err(EmailApiError::RateLimited { retry_after: Some(delay),.. }) if delay > Duration::from_secs(590))
+ Err(EmailApiError::RateLimited { retry_after: Some(delay), origin: RateLimitOrigin::Gate(GateLimit::Window) }) if delay > Duration::from_secs(590))
);
cleanup(&gate, mailbox).await;
}
diff --git a/crates/email_api_client/src/outbound/outlook.rs b/crates/email_api_client/src/outbound/outlook.rs
index 831c90d93d..e01a7548b4 100644
--- a/crates/email_api_client/src/outbound/outlook.rs
+++ b/crates/email_api_client/src/outbound/outlook.rs
@@ -6,6 +6,7 @@ mod contacts;
mod drafts;
mod invitations;
mod messages;
+mod route;
mod rules;
mod sync;
mod transport;
@@ -22,7 +23,10 @@ use url::Url;
use crate::domain::models::EmailApiError;
use crate::domain::{
models::MailboxAccess,
- ports::{MailboxRejectedTokenRefresh, MailboxRequestGate, ScopedMailboxRepository},
+ ports::{
+ MailboxRejectedTokenRefresh, MailboxRequestCount, MailboxRequestGate,
+ ScopedMailboxRepository,
+ },
};
use std::sync::Arc;
@@ -87,6 +91,12 @@ impl OutlookApiClientRepository {
}
}
+impl MailboxRequestCount for OutlookApiClientRepository {
+ async fn admitted_requests(&self, mailbox: MailboxAccess) -> Option<u64> {
+ self.gate.as_ref()?.admitted(mailbox).await
+ }
+}
+
impl ScopedMailboxRepository for OutlookApiClientRepository {
fn for_mailbox(&self, mailbox: MailboxAccess) -> Self {
Self {
diff --git a/crates/email_api_client/src/outbound/outlook/route.rs b/crates/email_api_client/src/outbound/outlook/route.rs
new file mode 100644
index 0000000000..d779eaa72f
--- /dev/null
+++ b/crates/email_api_client/src/outbound/outlook/route.rs
@@ -0,0 +1,81 @@
+//! Low-cardinality route templates for Graph request telemetry.
+
+#[cfg(test)]
+mod test;
+
+/// Graph path words kept in a template. Every other segment is an id or a
+/// user-chosen name, such as a category, and becomes `{id}`.
+const GRAPH_PATH_WORDS: &[&str] = &[
+ "$value",
+ "accept",
+ "archive",
+ "attachments",
+ "calendarView",
+ "calendars",
+ "cancel",
+ "childFolders",
+ "contactFolders",
+ "contacts",
+ "copy",
+ "createForward",
+ "createReply",
+ "createReplyAll",
+ "createUploadSession",
+ "decline",
+ "deleteditems",
+ "delta",
+ "drafts",
+ "events",
+ "forward",
+ "inbox",
+ "instances",
+ "junkemail",
+ "mailFolders",
+ "mailboxSettings",
+ "masterCategories",
+ "me",
+ "messageRules",
+ "messages",
+ "move",
+ "msgfolderroot",
+ "outbox",
+ "outlook",
+ "photo",
+ "send",
+ "sentitems",
+ "subscriptions",
+ "tentativelyAccept",
+ "users",
+];
+
+/// The request path relative to the API root, with ids replaced by `{id}`.
+/// OData key segments (`mailFolders('…')`, used by delta links) become the
+/// same `mailFolders/{id}` as the path form.
+pub(super) fn url_template(api_root: &str, path: &str) -> String {
+ let relative = path
+ .strip_prefix(api_root.trim_end_matches('/'))
+ .unwrap_or(path);
+ let mut template = String::new();
+ for segment in relative.split('/').filter(|segment| !segment.is_empty()) {
+ match segment.split_once('(') {
+ Some((name, _key)) => {
+ push_segment(&mut template, name);
+ template.push_str("/{id}");
+ }
+ None => push_segment(&mut template, segment),
+ }
+ }
+ if template.is_empty() {
+ template.push('/');
+ }
+ template
+}
+
+fn push_segment(template: &mut String, segment: &str) {
+ template.push('/');
+ if GRAPH_PATH_WORDS.contains(&segment) {
+ template.push_str(segment);
+ } else {
+ template.push_str("{id}");
+ }
+}
diff --git a/crates/email_api_client/src/outbound/outlook/route/test.rs b/crates/email_api_client/src/outbound/outlook/route/test.rs
new file mode 100644
index 0000000000..f7a8936de5
--- /dev/null
+++ b/crates/email_api_client/src/outbound/outlook/route/test.rs
@@ -0,0 +1,42 @@
+use super::url_template;
+
+#[test]
+fn ids_and_odata_keys_collapse_to_one_template_per_route() {
+ assert_eq!(
+ url_template(
+ "/v1.0/",
+ "/v1.0/me/mailFolders/AQMkADAwATM3ZmYAZS1iNjYAYi05/messages/delta"
+ ),
+ "/me/mailFolders/{id}/messages/delta"
+ );
+ assert_eq!(
+ url_template(
+ "/v1.0/",
+ "/v1.0/me/mailFolders('AQMkADAwATM3ZmYAZS1iNjYAYi05NDc0LTAwAi0wMAoALgAAA73oe0jpHIRF')/messages/delta"
+ ),
+ "/me/mailFolders/{id}/messages/delta"
+ );
+ assert_eq!(
+ url_template("/v1.0/", "/v1.0/me/messages/AAMkAGI2TG93AAA=/attachments"),
+ "/me/messages/{id}/attachments"
+ );
+ assert_eq!(
+ url_template("/v1.0/", "/v1.0/me/messages/AAMk%3D/$value"),
+ "/me/messages/{id}/$value"
+ );
+}
+
+#[test]
+fn user_chosen_names_never_reach_the_template() {
+ assert_eq!(
+ url_template(
+ "/v1.0/",
+ "/v1.0/me/outlook/masterCategories/Quarterly%20review"
+ ),
+ "/me/outlook/masterCategories/{id}"
+ );
+ assert_eq!(
+ url_template("/v1.0/", "/v1.0/users/[email protected]/messages"),
+ "/users/{id}/messages"
+ );
+}
diff --git a/crates/email_api_client/src/outbound/outlook/test.rs b/crates/email_api_client/src/outbound/outlook/test.rs
index 1707ed54ad..535572d131 100644
--- a/crates/email_api_client/src/outbound/outlook/test.rs
+++ b/crates/email_api_client/src/outbound/outlook/test.rs
@@ -403,6 +403,12 @@ async fn request_gate_covers_nested_pages_and_receives_provider_retry_after() {
self.finished.lock().unwrap().push(retry_after);
Box::pin(async {})
}
+ fn admitted(
+ &self,
+ _: MailboxAccess,
+ ) -> Pin<Box<dyn Future<Output = Option<u64>> + Send + '_>> {
+ Box::pin(async { None })
+ }
}
let (server, mut api, token) = setup().await;
let mailbox = MailboxAccess {
diff --git a/crates/email_api_client/src/outbound/outlook/transport.rs b/crates/email_api_client/src/outbound/outlook/transport.rs
index d2165e79d3..07be1c7b42 100644
--- a/crates/email_api_client/src/outbound/outlook/transport.rs
+++ b/crates/email_api_client/src/outbound/outlook/transport.rs
@@ -1,4 +1,4 @@
-use std::time::{Duration, SystemTime};
+use std::time::{Duration, Instant, SystemTime};
use reqwest::{Method, RequestBuilder, Response, StatusCode};
use serde::de::DeserializeOwned;
@@ -6,9 +6,14 @@ use serde_json::Value;
use tracing::Instrument;
use url::Url;
-use super::{IMMUTABLE_ID_PREFERENCE, OutlookApiClientRepository};
+use super::{IMMUTABLE_ID_PREFERENCE, OutlookApiClientRepository, route};
use crate::domain::models::{AccessToken, EmailApiError, RateLimitOrigin, StreamToken};
+#[cfg(test)]
+mod test;
+
+const GRAPH_REQUEST: &str = "outlook.graph_request";
+
/// A failed Graph request with the response details a caller may branch on.
pub(super) struct GraphFailure {
pub(super) error: EmailApiError,
@@ -157,19 +162,6 @@ impl OutlookApiClientRepository {
/// One Graph round trip, including its wait for the mailbox gate. A replay
/// with a refreshed token is its own attempt, so each keeps its status.
- #[tracing::instrument(
- name = "outlook.graph_request",
- skip_all,
- fields(
- otel.kind = "client",
- http.request.method = %method,
- url.path = path,
- link_id = self.mailbox.map(|mailbox| tracing::field::display(mailbox.link_id)),
- http.response.status_code = tracing::field::Empty,
- reauthorized = reauthorized,
- ),
- err(level = "warn")
- )]
async fn graph_attempt(
&self,
request: RequestBuilder,
@@ -177,7 +169,30 @@ impl OutlookApiClientRepository {
path: &str,
reauthorized: bool,
) -> Result<Response, GraphFailure> {
- self.gated_graph_request(request).await
+ let span = tracing::info_span!(
+ "outlook.graph_request",
+ otel.kind = "client",
+ // Datadog names a client span that carries an HTTP method
+ // `http.client.request`; this keeps the span findable by name.
+ operation.name = GRAPH_REQUEST,
+ http.request.method = %method,
+ url.template = %route::url_template(self.root.path(), path),
+ mailbox.link_id = self.mailbox.map(|mailbox| tracing::field::display(mailbox.link_id)),
+ mailbox.sync_generation = self.mailbox.map(|mailbox| mailbox.sync_generation),
+ reauthorized,
+ http.response.status_code = tracing::field::Empty,
+ graph.request_id = tracing::field::Empty,
+ outlook.gate_wait_ms = tracing::field::Empty,
+ outlook.throttle_origin = tracing::field::Empty,
+ outlook.retry_after_seconds = tracing::field::Empty,
+ otel.status_description = tracing::field::Empty,
+ );
+ let result = self
+ .gated_graph_request(request)
+ .instrument(span.clone())
+ .await;
+ record_attempt(&span, &result);
+ result
}
/// Includes anonymous, preauthorized attachment uploads in mailbox accounting.
@@ -193,17 +208,23 @@ impl OutlookApiClientRepository {
async fn gated_graph_request(&self, request: RequestBuilder) -> Result<Response, GraphFailure> {
let permit = if let Some(gate) = &self.gate {
let mailbox = self.mailbox.ok_or_else(invalid_response)?;
+ let started = Instant::now();
let permit = gate
.acquire(mailbox)
.instrument(tracing::info_span!(
"outlook.graph_admission",
- link_id = %mailbox.link_id
+ mailbox.link_id = %mailbox.link_id
))
- .await?;
- Some((mailbox, permit))
+ .await;
+ tracing::Span::current().record(
+ "outlook.gate_wait_ms",
+ i64::try_from(started.elapsed().as_millis()).unwrap_or(i64::MAX),
+ );
+ Some((mailbox, permit?))
} else {
None
};
+ crate::domain::request_count::record_sent();
let result = Self::send_request(request).await;
if let (Some(gate), Some((mailbox, permit))) = (&self.gate, permit) {
let retry_after = match &result {
@@ -227,16 +248,18 @@ impl OutlookApiClientRepository {
let method = request.method().clone();
let response = client.execute(request).await.map_err(transport_failed)?;
let status = response.status();
- tracing::Span::current().record("http.response.status_code", status.as_u16());
- if status.is_success() {
- return Ok(response);
- }
- let path = response.url().path().to_owned();
let request_id = response
.headers()
.get("request-id")
.and_then(|v| v.to_str().ok())
.map(str::to_owned);
+ let span = tracing::Span::current();
+ span.record("http.response.status_code", i64::from(status.as_u16()));
+ span.record("graph.request_id", request_id.as_deref());
+ if status.is_success() {
+ return Ok(response);
+ }
+ let path = response.url().path().to_owned();
let retry_after = response
.headers()
.get("Retry-After")
@@ -248,6 +271,9 @@ impl OutlookApiClientRepository {
.map(|date| date.duration_since(SystemTime::now()).unwrap_or_default())
})
});
+ if let Some(delay) = retry_after {
+ span.record("outlook.retry_after_seconds", whole_seconds(delay));
+ }
let body = response.text().await.unwrap_or_default();
let error: Value = serde_json::from_str(&body).unwrap_or(Value::Null);
let code = error.pointer("/error/code").and_then(Value::as_str);
@@ -314,6 +340,47 @@ impl OutlookApiClientRepository {
}
}
+/// Numbers are recorded as `i64`: OpenTelemetry receives unsigned span fields
+/// as strings.
+///
+/// Record where a failed attempt was throttled and mark real failures as span
+/// errors. A gate refusal sent nothing, so it is neither an error nor a WARN;
+/// the sync unit that owns the request reports the deferral.
+fn record_attempt(span: &tracing::Span, result: &Result<Response, GraphFailure>) {
+ let Err(failure) = result else {
+ span.record("outlook.throttle_origin", "none");
+ return;
+ };
+ // A response's Retry-After was recorded when it arrived; a gate refusal
+ // has no response, so its delay is recorded here.
+ let origin = match &failure.error {
+ EmailApiError::RateLimited {
+ origin: RateLimitOrigin::Provider { .. },
+ ..
+ } => "graph",
+ EmailApiError::RateLimited { retry_after, .. } => {
+ if let Some(delay) = retry_after {
+ span.record("outlook.retry_after_seconds", whole_seconds(*delay));
+ }
+ "gate"
+ }
+ _ => "none",
+ };
+ span.record("outlook.throttle_origin", origin);
+ if origin == "gate" {
+ tracing::debug!(parent: span, error = %failure.error, "Microsoft request gate deferred a Graph request");
+ } else {
+ span.record(
+ "otel.status_description",
+ tracing::field::display(&failure.error),
+ );
+ }
+}
+
+fn whole_seconds(delay: Duration) -> i64 {
+ i64::try_from(delay.as_millis().div_ceil(1000)).unwrap_or(i64::MAX)
+}
+
pub(super) fn invalid_response() -> EmailApiError {
EmailApiError::Permanent {
message: "invalid Microsoft Graph response".into(),
diff --git a/crates/email_api_client/src/outbound/outlook/transport/test.rs b/crates/email_api_client/src/outbound/outlook/transport/test.rs
new file mode 100644
index 0000000000..d01edc6e4c
--- /dev/null
+++ b/crates/email_api_client/src/outbound/outlook/transport/test.rs
@@ -0,0 +1,210 @@
+use super::*;
+use crate::domain::models::{GateLimit, MailboxAccess};
+use crate::domain::ports::{MailboxRequestGate, ScopedMailboxRepository};
+use std::collections::HashMap;
+use std::future::Future;
+use std::pin::Pin;
+use std::sync::{Arc, Mutex};
+use tracing::field::{Field, Visit};
+use tracing::span::{Attributes, Id, Record};
+use tracing_subscriber::layer::{Context, SubscriberExt};
+use tracing_subscriber::registry::LookupSpan;
+use uuid::Uuid;
+use wiremock::matchers::{method, path};
+use wiremock::{Mock, ResponseTemplate};
+
+#[derive(Default)]
+struct Fields(HashMap<&'static str, String>);
+
+impl Visit for Fields {
+ fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) {
+ self.0.insert(field.name(), format!("{value:?}"));
+ }
+ fn record_str(&mut self, field: &Field, value: &str) {
+ self.0.insert(field.name(), value.to_owned());
+ }
+}
+
+/// Every closed span's name and the last value recorded for each field.
+#[derive(Clone, Default)]
+struct Spans(Arc<Mutex<Vec<(&'static str, HashMap<&'static str, String>)>>>);
+
+impl Spans {
+ fn named(&self, name: &str) -> Vec<HashMap<&'static str, String>> {
+ let spans = self.0.lock().unwrap();
+ spans
+ .iter()
+ .filter(|(span, _)| *span == name)
+ .map(|(_, fields)| fields.clone())
+ .collect()
+ }
+}
+
+impl<S: tracing::Subscriber + for<'a> LookupSpan<'a>> tracing_subscriber::Layer<S> for Spans {
+ fn on_new_span(&self, attrs: &Attributes<'_>, id: &Id, ctx: Context<'_, S>) {
+ let mut fields = Fields::default();
+ attrs.record(&mut fields);
+ if let Some(span) = ctx.span(id) {
+ span.extensions_mut().insert(fields);
+ }
+ }
+ fn on_record(&self, id: &Id, values: &Record<'_>, ctx: Context<'_, S>) {
+ if let Some(span) = ctx.span(id)
+ && let Some(fields) = span.extensions_mut().get_mut::<Fields>()
+ {
+ values.record(fields);
+ }
+ }
+ fn on_close(&self, id: Id, ctx: Context<'_, S>) {
+ if let Some(span) = ctx.span(&id) {
+ let fields = span.extensions_mut().remove::<Fields>().unwrap_or_default();
+ self.0.lock().unwrap().push((span.name(), fields.0));
+ }
+ }
+}
+
+struct RefusingGate;
+
+impl MailboxRequestGate for RefusingGate {
+ fn acquire(
+ &self,
+ _: MailboxAccess,
+ ) -> Pin<Box<dyn Future<Output = Result<Uuid, EmailApiError>> + Send + '_>> {
+ Box::pin(async {
+ Err(EmailApiError::RateLimited {
+ retry_after: Some(Duration::from_millis(2_500)),
+ origin: RateLimitOrigin::Gate(GateLimit::Window),
+ })
+ })
+ }
+ fn finish(
+ &self,
+ _: MailboxAccess,
+ _: Uuid,
+ _: Option<Duration>,
+ ) -> Pin<Box<dyn Future<Output = ()> + Send + '_>> {
+ Box::pin(async {})
+ }
+ fn admitted(&self, _: MailboxAccess) -> Pin<Box<dyn Future<Output = Option<u64>> + Send + '_>> {
+ Box::pin(async { None })
+ }
+}
+
+const MAILBOX: MailboxAccess = MailboxAccess {
+ link_id: Uuid::from_u128(0x0a),
+ sync_generation: 3,
+ grant_generation: 1,
+};
+
+#[tokio::test]
+async fn graph_retry_after_and_request_ids_are_recorded_on_every_response() {
+ let spans = Spans::default();
+ let _subscriber =
+ tracing::subscriber::set_default(tracing_subscriber::registry().with(spans.clone()));
+ // A callsite another test thread registered while this subscriber was
+ // being installed can keep a cached "never" interest; recompute it.
+ tracing::callsite::rebuild_interest_cache();
+ let (server, api, token) = super::super::test::setup().await;
+ let api = api.for_mailbox(MAILBOX);
+ Mock::given(method("GET"))
+ .and(path("/v1.0/me/messages/AAMkAGI2TG93AAA="))
+ .respond_with(
+ ResponseTemplate::new(429)
+ .insert_header("Retry-After", "7")
+ .insert_header("request-id", "graph-request-1"),
+ )
+ .mount(&server)
+ .await;
+ Mock::given(method("GET"))
+ .and(path("/v1.0/me/messages/AAMkBusy"))
+ .respond_with(ResponseTemplate::new(503).insert_header("Retry-After", "11"))
+ .mount(&server)
+ .await;
+ Mock::given(method("GET"))
+ .and(path("/v1.0/me/messages/AAMkOK"))
+ .respond_with(
+ ResponseTemplate::new(200)
+ .insert_header("request-id", "graph-request-2")
+ .set_body_json(serde_json::json!({})),
+ )
+ .mount(&server)
+ .await;
+
+ let throttled = api
+ .get::<Value>(
+ &token,
+ api.endpoint(&["me", "messages", "AAMkAGI2TG93AAA="])
+ .unwrap(),
+ )
+ .await;
+ let unavailable = api
+ .get::<Value>(
+ &token,
+ api.endpoint(&["me", "messages", "AAMkBusy"]).unwrap(),
+ )
+ .await;
+ let succeeded = api
+ .get::<Value>(&token, api.endpoint(&["me", "messages", "AAMkOK"]).unwrap())
+ .await;
+
+ assert!(matches!(throttled, Err(EmailApiError::RateLimited { .. })));
+ assert!(succeeded.is_ok());
+ let requests = spans.named("outlook.graph_request");
+ assert!(matches!(unavailable, Err(EmailApiError::Transient { .. })));
+ let [throttled, unavailable, succeeded] = requests.as_slice() else {
+ panic!("expected three Graph request spans, got {requests:?}");
+ };
+ assert_eq!(unavailable["http.response.status_code"], "503");
+ assert_eq!(unavailable["outlook.throttle_origin"], "none");
+ assert_eq!(unavailable["outlook.retry_after_seconds"], "11");
+ assert_eq!(throttled["operation.name"], "outlook.graph_request");
+ assert_eq!(throttled["url.template"], "/me/messages/{id}");
+ assert_eq!(throttled["mailbox.link_id"], MAILBOX.link_id.to_string());
+ assert_eq!(throttled["http.response.status_code"], "429");
+ assert_eq!(throttled["outlook.throttle_origin"], "graph");
+ assert_eq!(throttled["outlook.retry_after_seconds"], "7");
+ assert_eq!(throttled["graph.request_id"], "graph-request-1");
+ assert!(throttled.contains_key("otel.status_description"));
+
+ assert_eq!(succeeded["http.response.status_code"], "200");
+ assert_eq!(succeeded["outlook.throttle_origin"], "none");
+ assert_eq!(succeeded["graph.request_id"], "graph-request-2");
+ assert!(!succeeded.contains_key("otel.status_description"));
+}
+
+#[tokio::test]
+async fn a_gate_refusal_is_a_deferral_not_a_failed_request() {
+ let spans = Spans::default();
+ let _subscriber =
+ tracing::subscriber::set_default(tracing_subscriber::registry().with(spans.clone()));
+ // A callsite another test thread registered while this subscriber was
+ // being installed can keep a cached "never" interest; recompute it.
+ tracing::callsite::rebuild_interest_cache();
+ let (_server, mut api, token) = super::super::test::setup().await;
+ api.gate = Some(Arc::new(RefusingGate));
+ let api = api.for_mailbox(MAILBOX);
+
+ let refused = api
+ .get::<Value>(
+ &token,
+ api.endpoint(&["me", "mailFolders", "inbox"]).unwrap(),
+ )
+ .await;
+
+ assert!(matches!(
+ refused,
+ Err(EmailApiError::RateLimited {
+ origin: RateLimitOrigin::Gate(GateLimit::Window),
+ ..
+ })
+ ));
+ let requests = spans.named("outlook.graph_request");
+ let [refused] = requests.as_slice() else {
+ panic!("expected one Graph request span, got {requests:?}");
+ };
+ assert_eq!(refused["outlook.throttle_origin"], "gate");
+ assert_eq!(refused["outlook.retry_after_seconds"], "3");
+ assert!(refused.contains_key("outlook.gate_wait_ms"));
+ assert!(!refused.contains_key("http.response.status_code"));
+ assert!(!refused.contains_key("otel.status_description"));
+}
diff --git a/crates/macro_db_client/migrations/20261103201221_outlook_import_progress.sql b/crates/macro_db_client/migrations/20261103201221_outlook_import_progress.sql
new file mode 100644
index 0000000000..979d915698
--- /dev/null
+++ b/crates/macro_db_client/migrations/20261103201221_outlook_import_progress.sql
@@ -0,0 +1,33 @@
+-- Additive: Outlook import progress for telemetry. Both tables are small.
+SET LOCAL lock_timeout = '1s';
+
+-- Read on the folder_catalog stream: when its import run started, and when it
+-- finished. The completion stamp lets "import completed" fire once per run.
+-- now() is stable, so the default needs no table rewrite.
+ALTER TABLE email_sync_streams
+ ADD COLUMN import_started_at timestamptz DEFAULT now(),
+ ADD COLUMN import_completed_at timestamptz;
+
+-- Imports that finished before this column existed must not report again.
+UPDATE email_sync_streams catalog SET import_completed_at = now()
+WHERE catalog.kind = 'folder_catalog' AND catalog.initial_complete
+ AND NOT EXISTS (
+ SELECT 1 FROM email_sync_streams folder
+ WHERE folder.link_id = catalog.link_id AND folder.generation = catalog.generation
+ AND folder.kind = 'mail_folder' AND NOT folder.initial_complete
+ )
+ AND NOT EXISTS (
+ SELECT 1 FROM email_message_reconciliation queued
+ WHERE queued.link_id = catalog.link_id AND queued.generation = catalog.generation
+ AND queued.is_import
+ );
+
+-- Messages each worker imported, flushed from memory once a minute so the
+-- import path itself writes nothing extra. Rows older than a day are deleted.
+CREATE TABLE email_mailbox_import_minutes (
+ link_id uuid NOT NULL,
+ minute timestamptz NOT NULL,
+ imported integer NOT NULL,
+ last_imported_at timestamptz NOT NULL,
+ PRIMARY KEY (link_id, minute)
+);
diff --git a/packages/sdk/generated/storage/sdk.gen.ts b/packages/sdk/generated/storage/sdk.gen.ts
index 801912539a..acd1d8b8c3 100644
--- a/packages/sdk/generated/storage/sdk.gen.ts
+++ b/packages/sdk/generated/storage/sdk.gen.ts
@@ -974,7 +974,7 @@ export class Sdk extends HeyApiClient {
public listCrmBlockedDomains<ThrowOnError extends boolean = false>(options?: Options<ListCrmBlockedDomainsData, ThrowOnError>): RequestResult<ListCrmBlockedDomainsResponses, ListCrmBlockedDomainsErrors, ThrowOnError> {
return (options?.client ?? this.client).get<ListCrmBlockedDomainsResponses, ListCrmBlockedDomainsErrors, ThrowOnError>({ url: '/crm/blocked-domains', ...options });
}
-
+
/**
* Unblock a domain: email sync resumes for it, and what blocking hid is
* shown again. Idempotent. Requires an admin/owner team role (403).
@@ -982,7 +982,7 @@ export class Sdk extends HeyApiClient {
public unblockCrmDomain<ThrowOnError extends boolean = false>(options: Options<UnblockCrmDomainData, ThrowOnError>): RequestResult<UnblockCrmDomainResponses, UnblockCrmDomainErrors, ThrowOnError> {
return (options.client ?? this.client).delete<UnblockCrmDomainResponses, UnblockCrmDomainErrors, ThrowOnError>({ url: '/crm/blocked-domains/{domain}', ...options });
}
-
+
/**
* Block a domain: email sync stops creating or updating CRM records for
* it, and the company and contacts the CRM holds for it are hidden.
@@ -991,7 +991,7 @@ export class Sdk extends HeyApiClient {
public blockCrmDomain<ThrowOnError extends boolean = false>(options: Options<BlockCrmDomainData, ThrowOnError>): RequestResult<BlockCrmDomainResponses, BlockCrmDomainErrors, ThrowOnError> {
return (options.client ?? this.client).put<BlockCrmDomainResponses, BlockCrmDomainErrors, ThrowOnError>({ url: '/crm/blocked-domains/{domain}', ...options });
}
-
+
/**
* Manually create a CRM company for the caller's team. Any team member
* may create one. A domain, when given, must not already be tracked by the team
@@ -1130,7 +1130,7 @@ export class Sdk extends HeyApiClient {
}
});
}
-
+
/**
* Look up a CRM contact by email in the caller's team. Returns a null
* `contact` when no visible contact exists. Any team member may resolve a
diff --git a/services/authentication_service/src/api/internal/microsoft_access_token.rs b/services/authentication_service/src/api/internal/microsoft_access_token.rs
index 6dded0e6b4..9279a05ecc 100644
--- a/services/authentication_service/src/api/internal/microsoft_access_token.rs
+++ b/services/authentication_service/src/api/internal/microsoft_access_token.rs
@@ -37,7 +37,7 @@ pub async fn handler(
{
Ok(token) => (
[(header::CACHE_CONTROL, "no-store")],
- Json(serde_json::json!({"access_token":token.as_str(),"scopes":token.scopes})),
+ Json(serde_json::json!({"access_token":token.as_str(),"scopes":token.scopes,"cached":token.cached})),
)
.into_response(),
Err(
diff --git a/services/authentication_service/src/domain/microsoft.rs b/services/authentication_service/src/domain/microsoft.rs
index 3c319b26e4..37a53948a6 100644
--- a/services/authentication_service/src/domain/microsoft.rs
+++ b/services/authentication_service/src/domain/microsoft.rs
@@ -125,6 +125,8 @@ pub struct RefreshedGrant {
pub struct MicrosoftAccessGrant {
token: Zeroizing<String>,
pub scopes: Vec<String>,
+ /// Served from this process's token cache rather than a refresh.
+ pub cached: bool,
}
impl MicrosoftAccessGrant {
pub fn as_str(&self) -> &str {
@@ -595,6 +597,7 @@ impl<R: MicrosoftGrantRepository, P: MicrosoftIdentityProvider, F> MicrosoftAuth
return Ok(MicrosoftAccessGrant {
token: cached.value.clone(),
scopes: cached.scopes.clone(),
+ cached: true,
});
}
let lease = macro_uuid::generate_uuid_v7();
@@ -651,6 +654,7 @@ impl<R: MicrosoftGrantRepository, P: MicrosoftIdentityProvider, F> MicrosoftAuth
Ok(MicrosoftAccessGrant {
token: refreshed.access_token,
scopes: refreshed.scopes,
+ cached: false,
})
}
.await;
diff --git a/services/authentication_service/src/domain/microsoft/test.rs b/services/authentication_service/src/domain/microsoft/test.rs
index c8b4300956..b3d481e292 100644
--- a/services/authentication_service/src/domain/microsoft/test.rs
+++ b/services/authentication_service/src/domain/microsoft/test.rs
@@ -473,3 +473,51 @@ async fn reconnecting_an_inbox_never_depends_on_the_flag_or_the_switch() {
);
}
}
+
+#[tokio::test]
+async fn tokens_report_whether_the_cache_served_them() {
+ let grant_id = macro_uuid::generate_uuid_v7();
+ let mut repo = MockMicrosoftGrantRepository::new();
+ repo.expect_active_grant()
+ .times(2)
+ .returning(move |_, _, _| {
+ Ok(StoredGrant {
+ scopes: scopes(),
+ id: grant_id,
+ generation: 1,
+ revision: 0,
+ owner: "owner".into(),
+ email: "[email protected]".into(),
+ envelope: envelope(),
+ })
+ });
+ repo.expect_acquire_refresh()
+ .once()
+ .returning(|_, _| Ok(true));
+ repo.expect_finish_refresh()
+ .once()
+ .returning(|_, _, _, _| Ok(true));
+ let mut provider = MockMicrosoftIdentityProvider::new();
+ provider.expect_refresh().once().returning(|_| {
+ Ok(RefreshedGrant {
+ access_token: Zeroizing::new("fresh".into()),
+ refresh_token: None,
+ expires_in: 3600,
+ scopes: scopes(),
+ })
+ });
+ let mut cipher = MockMicrosoftTokenCipher::new();
+ cipher
+ .expect_decrypt()
+ .once()
+ .returning(|_, _, _| Ok(MicrosoftRefreshToken::new("refresh".into())));
+ let service = Service::new(repo, provider, Arc::new(cipher));
+ let link = macro_uuid::generate_uuid_v7();
+
+ let refreshed = service.access_token(link, 1, 1, false).await.unwrap();
+ let cached = service.access_token(link, 1, 1, false).await.unwrap();
+
+ assert!(!refreshed.cached);
+ assert!(cached.cached);
+ assert_eq!(cached.as_str(), "fresh");
+}
diff --git a/services/email_service/AGENTS.md b/services/email_service/AGENTS.md
index b6abda9bfb..481e9fa1bf 100644
--- a/services/email_service/AGENTS.md
+++ b/services/email_service/AGENTS.md
@@ -19,6 +19,9 @@ the gateway `/email` prefix.
`src/backfill_completion_service.rs`: Gmail history backfill orchestration.
- `src/crm_enrichment_worker.rs`: periodic pass that resolves pending CRM
company directory entries (names/logos) left by binaries without a resolver.
+- `src/mailbox_sync_worker.rs`: the Outlook mailbox loops (sync, projections,
+ contacts, drafts, commands, settings, watches) and `run_health`, the
+ per-minute Outlook health snapshot.
- `src/bin/`: standalone binaries — `pubsub_workers` (the queue consumer
process), plus one-off backfill/cleanup scripts.
@@ -158,6 +161,14 @@ the gateway `/email` prefix.
visibility while processing. Don't raise its receive batch: messages queued
behind a long mailbox purge would be redelivered and purged twice.
`PurgePacing` (`email_db_client::links::delete`) sets the purge's write rate.
+- `run_health` runs in every `pubsub_workers` task: each flushes its in-memory
+ Outlook import counts every minute, and the task that wins the Redis key
+ `outlook:health_snapshot` logs one "outlook mailbox health" line per Outlook
+ mailbox. Its fields, the `outlook.sync.*`, `outlook.graph_request` and
+ `outlook.token.acquire` spans and the import lifecycle logs are read by the
+ Outlook sync monitors in `infra/stacks/monitoring`; rename them only
+ together. Loops whose domain service logs its own deferrals use `reported`,
+ the others `record`, which logs gate refusals at DEBUG.
- `GET /opens/{token}` is public and must return the same uncacheable GIF for
every token. Gmail sends add the pixel only to the provider copy in
`outbound/scheduled_delivery.rs`; Outlook adds it through the materializer
diff --git a/services/email_service/src/api/email/provider_error.rs b/services/email_service/src/api/email/provider_error.rs
index 8c756b6d39..e5c0e79d8d 100644
--- a/services/email_service/src/api/email/provider_error.rs
+++ b/services/email_service/src/api/email/provider_error.rs
@@ -29,8 +29,8 @@ pub(crate) fn provider_error_status(error: &EmailApiError) -> StatusCode {
EmailApiError::NotFound => StatusCode::NOT_FOUND,
EmailApiError::Conflict | EmailApiError::OutdatedCursor => StatusCode::CONFLICT,
EmailApiError::SendRejected { .. } => StatusCode::BAD_REQUEST,
- EmailApiError::Transient { .. } | EmailApiError::Permanent { .. } => {
- StatusCode::INTERNAL_SERVER_ERROR
- }
+ EmailApiError::Transient { .. }
+ | EmailApiError::CredentialsUnavailable { .. }
+ | EmailApiError::Permanent { .. } => StatusCode::INTERNAL_SERVER_ERROR,
}
}
diff --git a/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs b/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs
index fc4629b6dc..5aeb62eb77 100644
--- a/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs
+++ b/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs
@@ -311,12 +311,21 @@ async fn main() -> anyhow::Result<()> {
);
let mailbox_repository = email::outbound::mailbox_pg::PgMailboxSync::new(db.clone());
if config.outlook_sync_enabled {
- worker_tracker.spawn(email_service::mailbox_sync_worker::run(
- email::domain::mailbox::MailboxSyncService::new(
- mailbox_repository.clone(),
- email_service::outbound::email_api::ProviderMailboxGateway(outlook_api.clone()),
+ let mailbox_sync = email::domain::mailbox::MailboxSyncService::new(
+ mailbox_repository.clone(),
+ email_service::outbound::email_api::ProviderMailboxGateway(outlook_api.clone()),
+ mailbox_repository.clone(),
+ );
+ worker_tracker.spawn(email_service::mailbox_sync_worker::run_health(
+ email::domain::mailbox::progress::MailboxProgressService::new(
mailbox_repository.clone(),
+ mailbox_sync.import_tally(),
),
+ redis_client.clone(),
+ worker_cancellation_token.clone(),
+ ));
+ worker_tracker.spawn(email_service::mailbox_sync_worker::run(
+ mailbox_sync,
worker_cancellation_token.clone(),
));
}
diff --git a/services/email_service/src/mailbox_sync_worker.rs b/services/email_service/src/mailbox_sync_worker.rs
index 758dc431b2..90ee200031 100644
--- a/services/email_service/src/mailbox_sync_worker.rs
+++ b/services/email_service/src/mailbox_sync_worker.rs
@@ -1,11 +1,19 @@
//! Poll durable mailbox work. Notifications only change when that work is due.
+use crate::util::redis::RedisClient;
+use email::domain::mailbox::progress::{MailboxProgressRepository, MailboxProgressService};
+use email::domain::mailbox::telemetry::defer_reason;
use email::domain::mailbox::{
MailboxError, MailboxGateway, MailboxIngest, MailboxSyncRepository, MailboxSyncService,
};
+use email_api_client::domain::models::{EmailApiError, RateLimitOrigin};
use std::time::Duration;
use tokio_util::sync::CancellationToken;
+/// How often each worker flushes its import counts and one worker logs the
+/// Outlook health snapshot.
+const HEALTH_INTERVAL: Duration = Duration::from_secs(60);
+
/// Domain services own sync and retry decisions; this loop only schedules work
/// and stops accepting work during shutdown. Interrupted work retains its lease.
pub async fn run<R: MailboxSyncRepository, P: MailboxGateway, I: MailboxIngest>(
@@ -14,10 +22,10 @@ pub async fn run<R: MailboxSyncRepository, P: MailboxGateway, I: MailboxIngest>(
) {
loop {
let cycle = async {
- let mut worked = record(service.discover_once().await);
- worked |= record(service.enumerate_once().await);
+ let mut worked = reported(service.discover_once().await);
+ worked |= reported(service.enumerate_once().await);
for _ in 0..32 {
- if !record(service.reconcile_once().await) {
+ if !reported(service.reconcile_once().await) {
break;
}
worked = true;
@@ -39,17 +47,72 @@ pub async fn run<R: MailboxSyncRepository, P: MailboxGateway, I: MailboxIngest>(
}
}
+/// For units that log their own deferrals and failures with their mailbox.
+fn reported(result: Result<bool, MailboxError>) -> bool {
+ result.unwrap_or(false)
+}
+
fn record(result: Result<bool, MailboxError>) -> bool {
match result {
Ok(worked) => worked,
Err(MailboxError::Stale) => false,
+ // The shared request gate pacing a mailbox is routine.
+ Err(
+ error @ MailboxError::Provider(EmailApiError::RateLimited {
+ origin: RateLimitOrigin::Gate(_),
+ ..
+ }),
+ ) => {
+ tracing::debug!(defer_reason = defer_reason(&error), %error, "mailbox sync work deferred");
+ false
+ }
Err(error) => {
- tracing::warn!(%error,"mailbox sync work deferred");
+ tracing::warn!(defer_reason = defer_reason(&error), %error, "mailbox sync work deferred");
false
}
}
}
+/// Every worker flushes the imports it counted each minute; whichever worker
+/// claims the minute also logs one health snapshot per Outlook mailbox.
+pub async fn run_health<R: MailboxProgressRepository>(
+ service: MailboxProgressService<R>,
+ redis: RedisClient,
+ cancellation: CancellationToken,
+) {
+ let mut interval = tokio::time::interval(HEALTH_INTERVAL);
+ interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
+ loop {
+ tokio::select! {
+ biased;
+ _ = cancellation.cancelled() => break,
+ _ = interval.tick() => {},
+ }
+ let report = async {
+ if let Err(error) = service.flush().await {
+ tracing::warn!(%error, "failed to flush Outlook import counts");
+ }
+ // Slightly shorter than the interval so the next minute is free.
+ if redis
+ .claim_outlook_health_snapshot(HEALTH_INTERVAL - Duration::from_secs(5))
+ .await
+ && let Err(error) = service.report().await
+ {
+ tracing::warn!(%error, "failed to read the Outlook health snapshot");
+ }
+ };
+ tokio::select! {
+ biased;
+ _ = cancellation.cancelled() => break,
+ () = report => {},
+ }
+ }
+ // Keep the counts of the last minute.
+ if let Err(error) = service.flush().await {
+ tracing::warn!(%error, "failed to flush Outlook import counts");
+ }
+}
+
pub async fn run_lifecycle<R, E>(
service: email::domain::mailbox::lifecycle::InboxLifecycleService<R, E>,
cancellation: CancellationToken,
@@ -87,9 +150,9 @@ pub async fn run_contacts<R, G, S>(
{
loop {
let cycle = async {
- let mut worked = record(service.sync_once().await);
+ let mut worked = reported(service.sync_once().await);
for _ in 0..8 {
- if !record(service.photo_once().await) {
+ if !reported(service.photo_once().await) {
break;
}
worked = true;
@@ -113,7 +176,7 @@ pub async fn run_drafts<R, G, M>(
M: email::domain::mailbox::drafts::ports::DraftMaterializer,
{
loop {
- let worked = tokio::select! { biased; _ = cancellation.cancelled() => return, result = service.execute_once() => record(result) };
+ let worked = tokio::select! { biased; _ = cancellation.cancelled() => return, result = service.execute_once() => reported(result) };
if !worked {
tokio::select! { biased; _ = cancellation.cancelled() => return, _ = tokio::time::sleep(Duration::from_secs(1)) => {} }
}
@@ -132,7 +195,7 @@ pub async fn run_projections<R, E>(
let worked = tokio::select! {
biased;
_ = cancellation.cancelled() => return,
- result = service.project_once() => record(result),
+ result = service.project_once() => reported(result),
};
if !worked {
tokio::select! {
@@ -153,7 +216,7 @@ pub async fn run_commands<R, G>(
G: email::domain::mailbox::commands::MailboxCommandGateway,
{
loop {
- let worked = tokio::select! { biased; _ = cancellation.cancelled() => return, result = service.execute_once() => record(result) };
+ let worked = tokio::select! { biased; _ = cancellation.cancelled() => return, result = service.execute_once() => reported(result) };
if !worked {
tokio::select! { biased; _ = cancellation.cancelled() => return, _ = tokio::time::sleep(Duration::from_secs(1)) => {} }
}
@@ -191,7 +254,7 @@ pub async fn run_settings<R, G>(
G: email::domain::mailbox::settings::MailboxSettingsGateway,
{
loop {
- let worked = tokio::select! {biased;_ = cancellation.cancelled()=>return,result=service.execute_once()=>record(result)};
+ let worked = tokio::select! {biased;_ = cancellation.cancelled()=>return,result=service.execute_once()=>reported(result)};
if !worked {
tokio::select! {biased;_ = cancellation.cancelled()=>return,_ = tokio::time::sleep(Duration::from_secs(1))=>{}}
}
@@ -206,7 +269,7 @@ pub async fn run_watches<R, G>(
G: email::domain::mailbox::watches::MailboxWatchGateway,
{
loop {
- let worked = tokio::select! {biased;_ = cancellation.cancelled()=>return,result=service.execute_once()=>record(result)};
+ let worked = tokio::select! {biased;_ = cancellation.cancelled()=>return,result=service.execute_once()=>reported(result)};
if !worked {
tokio::select! {biased;_ = cancellation.cancelled()=>return,_ = tokio::time::sleep(Duration::from_secs(5))=>{}}
}
diff --git a/services/email_service/src/outbound/email_api/microsoft.rs b/services/email_service/src/outbound/email_api/microsoft.rs
index cdf0606352..fea8f2d9b9 100644
--- a/services/email_service/src/outbound/email_api/microsoft.rs
+++ b/services/email_service/src/outbound/email_api/microsoft.rs
@@ -3,6 +3,7 @@
use authentication_service_client::{AuthServiceClient, error::AuthServiceClientError};
use email::domain::mailbox::{MailboxGateway, MailboxKey};
use email_api_client::domain::{models::*, ports::*, service::mailbox::MailboxApiService};
+use tracing::Instrument;
mod contacts;
mod drafts;
@@ -17,14 +18,29 @@ impl email::domain::mailbox::credentials::MailboxCredentialGateway for Microsoft
mailbox: MailboxAccess,
freshness: TokenFreshness,
) -> Result<email::domain::mailbox::credentials::MailboxCredentialToken, TokenError> {
- self.0
+ let fresh = matches!(freshness, TokenFreshness::Fresh);
+ let span = tracing::info_span!(
+ "outlook.token.acquire",
+ mailbox.link_id = %mailbox.link_id,
+ mailbox.sync_generation = mailbox.sync_generation,
+ outlook.token_refresh_requested = fresh,
+ outlook.token_cache = tracing::field::Empty,
+ outlook.outcome = tracing::field::Empty,
+ http.response.status_code = tracing::field::Empty,
+ otel.status_description = tracing::field::Empty,
+ );
+ let result = self
+ .0
.get_microsoft_access_token(
mailbox.link_id,
mailbox.grant_generation,
mailbox.sync_generation,
- matches!(freshness, TokenFreshness::Fresh),
+ fresh,
)
- .await
+ .instrument(span.clone())
+ .await;
+ record_token(&span, &result);
+ result
.map(
|response| email::domain::mailbox::credentials::MailboxCredentialToken {
token: AccessToken::new(response.access_token),
@@ -42,6 +58,30 @@ impl email::domain::mailbox::credentials::MailboxCredentialGateway for Microsoft
}
}
+/// `hit` only when authentication served a cached token; a refresh, a
+/// failure or an older authentication service that does not say is a `miss`.
+fn record_token(
+ span: &tracing::Span,
+ result: &Result<
+ authentication_service_client::microsoft_access_token::MicrosoftAccessToken,
+ AuthServiceClientError,
+ >,
+) {
+ let cached = matches!(result, Ok(token) if token.cached == Some(true));
+ span.record("outlook.token_cache", if cached { "hit" } else { "miss" });
+ let outcome = match result {
+ Ok(_) => "ok",
+ Err(AuthServiceClientError::Unauthorized | AuthServiceClientError::Forbidden) => {
+ "auth_required"
+ }
+ Err(_) => "unavailable",
+ };
+ span.record("outlook.outcome", outcome);
+ if let Err(error) = result {
+ span.record("otel.status_description", tracing::field::display(error));
+ }
+}
+
/// The adapter depends only on the foreign domain service and its capability ports.
/// Concrete Graph and storage adapters are supplied by the composition root.
#[derive(Clone)]
@@ -81,7 +121,7 @@ where
impl<R, T, L> MailboxGateway for ProviderMailboxGateway<R, T, L>
where
- R: MailboxContentReader + FolderChangeReader + ScopedMailboxRepository,
+ R: MailboxContentReader + FolderChangeReader + MailboxRequestCount + ScopedMailboxRepository,
T: MailboxTokenSource,
L: ProviderRateLimiter,
{
@@ -104,12 +144,19 @@ where
) -> Result<Option<MailboxMessage>, EmailApiError> {
self.0.message(access(mailbox), id, folders).await
}
+ async fn admitted_requests(&self, mailbox: MailboxKey) -> Option<u64> {
+ self.0.admitted_requests(access(mailbox)).await
+ }
}
impl<R, T, L> email::domain::mailbox::commands::MailboxCommandGateway
for ProviderMailboxGateway<R, T, L>
where
- R: MailboxContentReader + FolderChangeReader + MailboxActionWriter + ScopedMailboxRepository,
+ R: MailboxContentReader
+ + FolderChangeReader
+ + MailboxActionWriter
+ + MailboxRequestCount
+ + ScopedMailboxRepository,
T: MailboxTokenSource,
L: ProviderRateLimiter,
{
diff --git a/services/email_service/src/outbound/mailbox_init.rs b/services/email_service/src/outbound/mailbox_init.rs
index c941bf3874..8dafcf0375 100644
--- a/services/email_service/src/outbound/mailbox_init.rs
+++ b/services/email_service/src/outbound/mailbox_init.rs
@@ -370,6 +370,9 @@ impl MailboxInitializationRepository for PgMailboxInitialization {
sqlx::query!("INSERT INTO email_projection_outbox (id,link_id,generation,payload) VALUES ($1,$2,$3,$4)",macro_uuid::generate_uuid_v7(),link_id,link.sync_generation,event)
.execute(&mut *tx).await.map_err(unavailable)?;
tx.commit().await.map_err(unavailable)?;
+ if restart {
+ email::domain::mailbox::progress::log_import_started(link_id, link.sync_generation);
+ }
Ok(link_id)
}
}
diff --git a/services/email_service/src/outbound/mailbox_projection.rs b/services/email_service/src/outbound/mailbox_projection.rs
index 87de7f906b..2f9dce2855 100644
--- a/services/email_service/src/outbound/mailbox_projection.rs
+++ b/services/email_service/src/outbound/mailbox_projection.rs
@@ -4,7 +4,7 @@ use contacts::domain::{models::messages::ContactConnection, ports::ContactsIngre
use email::domain::{
events::EmailMacroEvent,
invitation_extraction::InvitationExtractionRepository,
- mailbox::{MailboxError, MailboxKey, projection::*},
+ mailbox::{DatabaseFailure, MailboxError, MailboxKey, projection::*},
models::calendar_invitation::CalendarInvitation,
};
use macro_event_broker::MacroEventBroker;
@@ -40,9 +40,19 @@ pub struct MailboxEffects<A, I, B, N, C> {
pub notifications_enabled: bool,
}
+/// A downstream effect (event broker, queue, another service) failed.
fn failed<E: std::fmt::Debug>(error: E) -> MailboxError {
tracing::warn!(?error, "mailbox projection effect deferred");
- MailboxError::Persistence
+ MailboxError::Effect
+}
+
+/// A database step of an effect keeps its SQLSTATE, so it reports as
+/// `persistence` rather than as an effect failure.
+fn database_failed(error: &dyn std::fmt::Debug, cause: Option<&sqlx::Error>) -> MailboxError {
+ tracing::warn!(?error, "mailbox projection effect deferred");
+ cause.map_or(MailboxError::Persistence, |cause| {
+ MailboxError::Database(DatabaseFailure::from(cause))
+ })
}
impl<A, I, B, N, C> MailboxProjectionEffects for MailboxEffects<A, I, B, N, C>
@@ -191,7 +201,7 @@ where
self.invitations
.save(message_id, invitations)
.await
- .map_err(failed)
+ .map_err(|report| database_failed(&report, report.downcast_current_context()))
}
async fn attachments(
&self,
@@ -223,21 +233,25 @@ where
)
.execute(&self.db)
.await
- .map_err(failed)?;
+ .map_err(|error| database_failed(&error, Some(&error)))?;
let documents = email_db_client::attachments::provider::upload::new_email_document_atts(
&self.db,
mailbox.link_id,
provider_id,
)
.await
- .map_err(failed)?;
+ .map_err(|error| {
+ database_failed(&error, error.chain().find_map(|cause| cause.downcast_ref()))
+ })?;
let media = email_db_client::attachments::provider::upload::new_email_media_atts(
&self.db,
mailbox.link_id,
provider_id,
)
.await
- .map_err(failed)?;
+ .map_err(|error| {
+ database_failed(&error, error.chain().find_map(|cause| cause.downcast_ref()))
+ })?;
for (attachment, destination) in documents
.into_iter()
.map(|a| (a, AttachmentUploadDestination::Dss))
@@ -280,7 +294,13 @@ where
data,
)
.await
- .map_err(failed)?;
+ .map_err(|error| match error {
+ // Its message is already a string, so no SQLSTATE survives.
+ crate::util::upload_attachment::UploadAttachmentError::DatabaseError(_) => {
+ database_failed(&error, None)
+ }
+ error => failed(error),
+ })?;
Ok::<_, MailboxError>(())
}
.await;
@@ -291,7 +311,7 @@ where
)
.execute(&self.db)
.await
- .map_err(failed)?;
+ .map_err(|error| database_failed(&error, Some(&error)))?;
return result;
}
}
@@ -300,9 +320,9 @@ where
AND a.upload_claimed_at IS NOT NULL
AND NOT EXISTS(SELECT 1 FROM document_email d WHERE d.email_attachment_id = a.id)
AND NOT EXISTS(SELECT 1 FROM email_attachments_sfs s WHERE s.attachment_id = a.id)) AS "pending!"
- "#,current.message.db_id).fetch_one(&self.db).await.map_err(failed)?;
+ "#,current.message.db_id).fetch_one(&self.db).await.map_err(|error| database_failed(&error, Some(&error)))?;
if pending {
- return Err(MailboxError::Persistence);
+ return Err(MailboxError::Effect);
}
Ok(())
}
diff --git a/services/email_service/src/pubsub/backfill/email_api_error.rs b/services/email_service/src/pubsub/backfill/email_api_error.rs
index 4f9cddc12e..347c96ac2e 100644
--- a/services/email_service/src/pubsub/backfill/email_api_error.rs
+++ b/services/email_service/src/pubsub/backfill/email_api_error.rs
@@ -15,6 +15,7 @@ pub(crate) fn map_email_api_error(error: EmailApiError, context: &'static str) -
| EmailApiError::NotFound
| EmailApiError::Conflict
| EmailApiError::Transient { .. }
+ | EmailApiError::CredentialsUnavailable { .. }
| EmailApiError::SendRejected { .. }
| EmailApiError::Permanent { .. } => FailureReason::GmailApiFailed,
};
diff --git a/services/email_service/src/pubsub/gmail_ops/email_api_error.rs b/services/email_service/src/pubsub/gmail_ops/email_api_error.rs
index b3681084b1..df28eb76bb 100644
--- a/services/email_service/src/pubsub/gmail_ops/email_api_error.rs
+++ b/services/email_service/src/pubsub/gmail_ops/email_api_error.rs
@@ -25,7 +25,9 @@ fn error_policy(error: &EmailApiError) -> ErrorPolicy {
// here rather than silently dropping the operation as permanent.
match error {
EmailApiError::RateLimited { .. } => ErrorPolicy::RateLimited,
- EmailApiError::Transient { .. } => ErrorPolicy::Retry,
+ EmailApiError::Transient { .. } | EmailApiError::CredentialsUnavailable { .. } => {
+ ErrorPolicy::Retry
+ }
EmailApiError::AuthRequired
| EmailApiError::Forbidden
| EmailApiError::NotFound
@@ -93,7 +95,9 @@ fn rate_limit_route(retry_worker: bool) -> RateLimitRoute {
fn processing_error(error: EmailApiError, retryable: bool) -> ProcessingError {
let reason = match error {
EmailApiError::RateLimited { .. } => FailureReason::GmailApiRateLimited,
- EmailApiError::Transient { .. } => FailureReason::GmailApiUnavailable,
+ EmailApiError::Transient { .. } | EmailApiError::CredentialsUnavailable { .. } => {
+ FailureReason::GmailApiUnavailable
+ }
EmailApiError::AuthRequired => FailureReason::AccessTokenFetchFailed,
EmailApiError::OutdatedCursor => FailureReason::OutdatedHistoryId,
EmailApiError::Forbidden
diff --git a/services/email_service/src/pubsub/inbox_sync/email_api_error.rs b/services/email_service/src/pubsub/inbox_sync/email_api_error.rs
index dec1f7076d..e2ea606587 100644
--- a/services/email_service/src/pubsub/inbox_sync/email_api_error.rs
+++ b/services/email_service/src/pubsub/inbox_sync/email_api_error.rs
@@ -73,6 +73,7 @@ fn detail(error: EmailApiError) -> DetailedError {
| EmailApiError::NotFound
| EmailApiError::Conflict
| EmailApiError::Transient { .. }
+ | EmailApiError::CredentialsUnavailable { .. }
| EmailApiError::SendRejected { .. }
| EmailApiError::Permanent { .. } => FailureReason::GmailApiFailed,
};
diff --git a/services/email_service/src/util/redis.rs b/services/email_service/src/util/redis.rs
index 4df98494c5..5164c88432 100644
--- a/services/email_service/src/util/redis.rs
+++ b/services/email_service/src/util/redis.rs
@@ -3,6 +3,7 @@ pub mod backfill;
pub mod backfill_write_slots;
pub mod health_probe;
pub mod lock;
+pub mod outlook_health;
pub mod rate_limit;
#[derive(Clone)]
diff --git a/services/email_service/src/util/redis/outlook_health.rs b/services/email_service/src/util/redis/outlook_health.rs
new file mode 100644
index 0000000000..043be7f7df
--- /dev/null
+++ b/services/email_service/src/util/redis/outlook_health.rs
@@ -0,0 +1,34 @@
+use crate::util::redis::RedisClient;
+use std::time::Duration;
+
+const SNAPSHOT_KEY: &str = "outlook:health_snapshot";
+
+impl RedisClient {
+ /// Claims the next Outlook health snapshot with `SET … NX PX`, so one
+ /// worker logs it per window however many run. Redis errors fail open:
+ /// a duplicate snapshot is harmless, a missing one hides a stall.
+ pub async fn claim_outlook_health_snapshot(&self, window: Duration) -> bool {
+ let mut connection = match self.inner.get_multiplexed_async_connection().await {
+ Ok(connection) => connection,
+ Err(error) => {
+ tracing::warn!(%error, "Failed to get Redis connection for the Outlook health snapshot");
+ return true;
+ }
+ };
+ let claimed: redis::RedisResult<Option<String>> = redis::cmd("SET")
+ .arg(SNAPSHOT_KEY)
+ .arg("1")
+ .arg("NX")
+ .arg("PX")
+ .arg(u64::try_from(window.as_millis()).unwrap_or(u64::MAX))
+ .query_async(&mut connection)
+ .await;
+ match claimed {
+ Ok(claimed) => claimed.is_some(),
+ Err(error) => {
+ tracing::warn!(%error, "Failed to claim the Outlook health snapshot");
+ true
+ }
+ }
+ }
+}