Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(email): keep unconfirmed Gmail label writes through inbox sync (#8778)
teoff284d905afa
diff --git a/.sqlx/query-19fa9694e53b16f5b7496271333b14edf2ed03fec3d2e31c396ad389769d4bad.json b/.sqlx/query-19fa9694e53b16f5b7496271333b14edf2ed03fec3d2e31c396ad389769d4bad.jsonnew file mode 100644index 0000000000..18326e6a6a--- /dev/null+++ b/.sqlx/query-19fa9694e53b16f5b7496271333b14edf2ed03fec3d2e31c396ad389769d4bad.json@@ -0,0 +1,35 @@+{+ "db_name": "PostgreSQL",+ "query": "\n WITH mine AS (\n SELECT c.message_id, c.provider_label_id, c.seq, c.label_present_before,\n (SELECT min(newer.seq) FROM email_gmail_pending_label_writes newer\n WHERE newer.message_id = c.message_id\n AND newer.provider_label_id = c.provider_label_id\n AND newer.seq > c.seq) AS newer_seq,\n (SELECT older.label_present FROM email_gmail_pending_label_writes older\n WHERE older.message_id = c.message_id\n AND older.provider_label_id = c.provider_label_id\n AND older.seq < c.seq AND older.expires_at > now()\n ORDER BY older.seq DESC\n LIMIT 1) AS older_present\n FROM email_gmail_pending_label_writes c\n WHERE c.message_id = ANY($1) AND c.write_id = $2\n ), inherited AS (\n UPDATE email_gmail_pending_label_writes c\n SET label_present_before = mine.label_present_before\n FROM mine\n WHERE c.seq = mine.newer_seq\n ), removed AS (\n DELETE FROM email_gmail_pending_label_writes c\n USING mine\n WHERE c.seq = mine.seq\n )\n SELECT\n message_id AS \"message_id!\",\n provider_label_id AS \"provider_label_id!\",\n COALESCE(older_present, label_present_before) AS \"label_present!\"\n FROM mine\n WHERE newer_seq IS NULL\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "message_id!",+ "type_info": "Uuid"+ },+ {+ "ordinal": 1,+ "name": "provider_label_id!",+ "type_info": "Text"+ },+ {+ "ordinal": 2,+ "name": "label_present!",+ "type_info": "Bool"+ }+ ],+ "parameters": {+ "Left": [+ "UuidArray",+ "Uuid"+ ]+ },+ "nullable": [+ false,+ false,+ null+ ]+ },+ "hash": "19fa9694e53b16f5b7496271333b14edf2ed03fec3d2e31c396ad389769d4bad"+}diff --git a/.sqlx/query-36a51fc6aa5d7be8b96ab56176a5ca5dea5bc2cbde7a9b742c05b7bb3355ffb9.json b/.sqlx/query-36a51fc6aa5d7be8b96ab56176a5ca5dea5bc2cbde7a9b742c05b7bb3355ffb9.jsonnew file mode 100644index 0000000000..315190c0e2--- /dev/null+++ b/.sqlx/query-36a51fc6aa5d7be8b96ab56176a5ca5dea5bc2cbde7a9b742c05b7bb3355ffb9.json@@ -0,0 +1,16 @@+{+ "db_name": "PostgreSQL",+ "query": "\n UPDATE email_messages m\n SET\n is_read = NOT EXISTS (\n SELECT 1 FROM email_message_labels ml JOIN email_labels l ON l.id = ml.label_id\n WHERE ml.message_id = m.id AND l.provider_label_id = $2\n ),\n is_starred = EXISTS (\n SELECT 1 FROM email_message_labels ml JOIN email_labels l ON l.id = ml.label_id\n WHERE ml.message_id = m.id AND l.provider_label_id = $3\n ),\n updated_at = now()\n WHERE m.id = ANY($1)\n ",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "UuidArray",+ "Text",+ "Text"+ ]+ },+ "nullable": []+ },+ "hash": "36a51fc6aa5d7be8b96ab56176a5ca5dea5bc2cbde7a9b742c05b7bb3355ffb9"+}diff --git a/.sqlx/query-3d2c746b08e2d6a6cba0ff05ebf5f947a5593c747c0aa4b586b1ca75e09622d2.json b/.sqlx/query-3d2c746b08e2d6a6cba0ff05ebf5f947a5593c747c0aa4b586b1ca75e09622d2.jsonnew file mode 100644index 0000000000..3e14b6b895--- /dev/null+++ b/.sqlx/query-3d2c746b08e2d6a6cba0ff05ebf5f947a5593c747c0aa4b586b1ca75e09622d2.json@@ -0,0 +1,18 @@+{+ "db_name": "PostgreSQL",+ "query": "\n INSERT INTO email_gmail_pending_label_writes\n (message_id, provider_label_id, write_id, label_present, label_present_before, expires_at)\n SELECT m.id, $3, $4, $5, NOT $5, now() + interval '12 hours'\n FROM email_messages m\n WHERE m.id = ANY($1) AND m.link_id = $2\n ORDER BY m.id\n ON CONFLICT (message_id, provider_label_id, write_id) DO NOTHING\n ",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "UuidArray",+ "Uuid",+ "Text",+ "Uuid",+ "Bool"+ ]+ },+ "nullable": []+ },+ "hash": "3d2c746b08e2d6a6cba0ff05ebf5f947a5593c747c0aa4b586b1ca75e09622d2"+}diff --git a/.sqlx/query-4d86a34e867a2f184c852a7a3040e4ea0cca6ff307640325618afdf5ba47fae9.json b/.sqlx/query-4d86a34e867a2f184c852a7a3040e4ea0cca6ff307640325618afdf5ba47fae9.jsondeleted file mode 100644index 459f85a008..0000000000--- a/.sqlx/query-4d86a34e867a2f184c852a7a3040e4ea0cca6ff307640325618afdf5ba47fae9.json+++ /dev/null@@ -1,16 +0,0 @@-{- "db_name": "PostgreSQL",- "query": "\n WITH removed AS (\n DELETE FROM email_message_labels\n WHERE $3::bool\n AND message_id = $1::uuid\n AND label_id NOT IN (SELECT UNNEST($2::uuid[]))\n )\n INSERT INTO email_message_labels (message_id, label_id)\n SELECT $1, label_id FROM UNNEST($2::uuid[]) AS t(label_id)\n ON CONFLICT (message_id, label_id) DO NOTHING\n ",- "describe": {- "columns": [],- "parameters": {- "Left": [- "Uuid",- "UuidArray",- "Bool"- ]- },- "nullable": []- },- "hash": "4d86a34e867a2f184c852a7a3040e4ea0cca6ff307640325618afdf5ba47fae9"-}diff --git a/.sqlx/query-4f031ebc497c13752284dcbf040112afdb169ce5f597da25ef205162d4759336.json b/.sqlx/query-4f031ebc497c13752284dcbf040112afdb169ce5f597da25ef205162d4759336.jsonnew file mode 100644index 0000000000..e0e9b6b942--- /dev/null+++ b/.sqlx/query-4f031ebc497c13752284dcbf040112afdb169ce5f597da25ef205162d4759336.json@@ -0,0 +1,18 @@+{+ "db_name": "PostgreSQL",+ "query": "\n WITH flags AS (\n SELECT\n CASE WHEN $2 THEN NOT EXISTS (\n SELECT 1 FROM email_message_labels ml JOIN email_labels l ON l.id = ml.label_id\n WHERE ml.message_id = m.id AND l.provider_label_id = $4\n ) ELSE m.is_read END AS is_read,\n CASE WHEN $3 THEN EXISTS (\n SELECT 1 FROM email_message_labels ml JOIN email_labels l ON l.id = ml.label_id\n WHERE ml.message_id = m.id AND l.provider_label_id = $5\n ) ELSE m.is_starred END AS is_starred\n FROM email_messages m\n WHERE m.id = $1\n )\n UPDATE email_messages m\n SET is_read = flags.is_read, is_starred = flags.is_starred\n FROM flags\n WHERE m.id = $1\n AND (m.is_read, m.is_starred) IS DISTINCT FROM (flags.is_read, flags.is_starred)\n ",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "Uuid",+ "Bool",+ "Bool",+ "Text",+ "Text"+ ]+ },+ "nullable": []+ },+ "hash": "4f031ebc497c13752284dcbf040112afdb169ce5f597da25ef205162d4759336"+}diff --git a/.sqlx/query-53895d46375114408f87059ccd3ce40338a6786d47cb52dcd557a801ec6164ea.json b/.sqlx/query-53895d46375114408f87059ccd3ce40338a6786d47cb52dcd557a801ec6164ea.jsonnew file mode 100644index 0000000000..b34bca5155--- /dev/null+++ b/.sqlx/query-53895d46375114408f87059ccd3ce40338a6786d47cb52dcd557a801ec6164ea.json@@ -0,0 +1,22 @@+{+ "db_name": "PostgreSQL",+ "query": "\n SELECT DISTINCT provider_label_id AS \"provider_label_id!\"\n FROM email_gmail_pending_label_writes\n WHERE message_id = $1 AND expires_at > now()\n ORDER BY provider_label_id\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "provider_label_id!",+ "type_info": "Text"+ }+ ],+ "parameters": {+ "Left": [+ "Uuid"+ ]+ },+ "nullable": [+ false+ ]+ },+ "hash": "53895d46375114408f87059ccd3ce40338a6786d47cb52dcd557a801ec6164ea"+}diff --git a/.sqlx/query-61d951608521ac57ffbe432fd208c4b24ebb0989ea86513c80e470f9c0c580b2.json b/.sqlx/query-61d951608521ac57ffbe432fd208c4b24ebb0989ea86513c80e470f9c0c580b2.jsonnew file mode 100644index 0000000000..f84d7fbd85--- /dev/null+++ b/.sqlx/query-61d951608521ac57ffbe432fd208c4b24ebb0989ea86513c80e470f9c0c580b2.json@@ -0,0 +1,15 @@+{+ "db_name": "PostgreSQL",+ "query": "\n WITH mine AS (\n SELECT message_id, provider_label_id, seq, label_present\n FROM email_gmail_pending_label_writes\n WHERE message_id = ANY($1) AND write_id = $2\n ), next AS (\n SELECT DISTINCT ON (c.message_id, c.provider_label_id) c.seq, mine.label_present\n FROM email_gmail_pending_label_writes c\n JOIN mine ON c.message_id = mine.message_id\n AND c.provider_label_id = mine.provider_label_id AND c.seq > mine.seq\n ORDER BY c.message_id, c.provider_label_id, c.seq\n ), falls_back AS (\n UPDATE email_gmail_pending_label_writes c\n SET label_present_before = next.label_present\n FROM next\n WHERE c.seq = next.seq\n )\n DELETE FROM email_gmail_pending_label_writes c\n USING mine\n WHERE c.message_id = mine.message_id\n AND c.provider_label_id = mine.provider_label_id\n AND c.seq <= mine.seq\n ",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "UuidArray",+ "Uuid"+ ]+ },+ "nullable": []+ },+ "hash": "61d951608521ac57ffbe432fd208c4b24ebb0989ea86513c80e470f9c0c580b2"+}diff --git a/.sqlx/query-6ba7d71306b4979b2b7fa8b4ecf525f4549e9f9facdfa9c7a59f90ca3facc4f0.json b/.sqlx/query-6ba7d71306b4979b2b7fa8b4ecf525f4549e9f9facdfa9c7a59f90ca3facc4f0.jsonnew file mode 100644index 0000000000..026940b66f--- /dev/null+++ b/.sqlx/query-6ba7d71306b4979b2b7fa8b4ecf525f4549e9f9facdfa9c7a59f90ca3facc4f0.json@@ -0,0 +1,23 @@+{+ "db_name": "PostgreSQL",+ "query": "\n SELECT DISTINCT thread_id\n FROM email_messages\n WHERE id = ANY($1) AND link_id = $2\n ORDER BY thread_id\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "thread_id",+ "type_info": "Uuid"+ }+ ],+ "parameters": {+ "Left": [+ "UuidArray",+ "Uuid"+ ]+ },+ "nullable": [+ false+ ]+ },+ "hash": "6ba7d71306b4979b2b7fa8b4ecf525f4549e9f9facdfa9c7a59f90ca3facc4f0"+}diff --git a/.sqlx/query-92676e78cf8fde3c016d40a70ac3ad92ac7d1ece7bd94e4a1dc9f39833324a2c.json b/.sqlx/query-92676e78cf8fde3c016d40a70ac3ad92ac7d1ece7bd94e4a1dc9f39833324a2c.jsonnew file mode 100644index 0000000000..f01a66887d--- /dev/null+++ b/.sqlx/query-92676e78cf8fde3c016d40a70ac3ad92ac7d1ece7bd94e4a1dc9f39833324a2c.json@@ -0,0 +1,42 @@+{+ "db_name": "PostgreSQL",+ "query": "\n SELECT\n m.id AS \"message_id!\",\n mine.seq IS NOT NULL AS \"claimed!\",\n EXISTS (\n SELECT 1 FROM email_gmail_pending_label_writes newer\n WHERE newer.message_id = m.id AND newer.provider_label_id = l.label\n AND newer.seq > mine.seq\n ) AS \"has_newer!\",\n EXISTS (\n SELECT 1 FROM email_gmail_pending_label_writes newer\n WHERE newer.message_id = m.id AND newer.provider_label_id = l.label\n AND newer.seq > mine.seq AND newer.expires_at > now()\n ) AS \"waits!\"\n FROM unnest($1::uuid[]) AS m(id)\n CROSS JOIN unnest($3::text[]) AS l(label)\n LEFT JOIN email_gmail_pending_label_writes mine\n ON mine.message_id = m.id AND mine.provider_label_id = l.label AND mine.write_id = $2\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "message_id!",+ "type_info": "Uuid"+ },+ {+ "ordinal": 1,+ "name": "claimed!",+ "type_info": "Bool"+ },+ {+ "ordinal": 2,+ "name": "has_newer!",+ "type_info": "Bool"+ },+ {+ "ordinal": 3,+ "name": "waits!",+ "type_info": "Bool"+ }+ ],+ "parameters": {+ "Left": [+ "UuidArray",+ "Uuid",+ "TextArray"+ ]+ },+ "nullable": [+ null,+ null,+ null,+ null+ ]+ },+ "hash": "92676e78cf8fde3c016d40a70ac3ad92ac7d1ece7bd94e4a1dc9f39833324a2c"+}diff --git a/.sqlx/query-a7a7bc06f4b34f14d7b506d307e19bcc80603467af6af7fafa964f386b2b992c.json b/.sqlx/query-a7a7bc06f4b34f14d7b506d307e19bcc80603467af6af7fafa964f386b2b992c.jsonnew file mode 100644index 0000000000..b1742305c0--- /dev/null+++ b/.sqlx/query-a7a7bc06f4b34f14d7b506d307e19bcc80603467af6af7fafa964f386b2b992c.json@@ -0,0 +1,16 @@+{+ "db_name": "PostgreSQL",+ "query": "\n DELETE FROM email_message_labels ml\n USING unnest($1::uuid[], $2::text[]) AS r(message_id, provider_label_id), email_labels l\n WHERE ml.message_id = r.message_id AND ml.label_id = l.id\n AND l.link_id = $3 AND l.provider_label_id = r.provider_label_id\n ",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "UuidArray",+ "TextArray",+ "Uuid"+ ]+ },+ "nullable": []+ },+ "hash": "a7a7bc06f4b34f14d7b506d307e19bcc80603467af6af7fafa964f386b2b992c"+}diff --git a/.sqlx/query-d3b1094fe292191892bed6feb715c0553384fad9e08285afc74375108201b129.json b/.sqlx/query-d3b1094fe292191892bed6feb715c0553384fad9e08285afc74375108201b129.jsonnew file mode 100644index 0000000000..282ab44970--- /dev/null+++ b/.sqlx/query-d3b1094fe292191892bed6feb715c0553384fad9e08285afc74375108201b129.json@@ -0,0 +1,25 @@+{+ "db_name": "PostgreSQL",+ "query": "\n SELECT m.id AS \"id!\"\n FROM unnest($1::uuid[]) AS m(id)\n WHERE EXISTS (\n SELECT 1 FROM email_effective_message_labels ml\n JOIN email_labels l ON l.id = ml.label_id\n WHERE ml.message_id = m.id AND l.link_id = $2 AND l.provider_label_id = $3\n ) <> $4\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "id!",+ "type_info": "Uuid"+ }+ ],+ "parameters": {+ "Left": [+ "UuidArray",+ "Uuid",+ "Text",+ "Bool"+ ]+ },+ "nullable": [+ null+ ]+ },+ "hash": "d3b1094fe292191892bed6feb715c0553384fad9e08285afc74375108201b129"+}diff --git a/.sqlx/query-daa957d34f4e5cb39e457ec5f4ec30f70ffddc6f134322a1422355a793b8707b.json b/.sqlx/query-daa957d34f4e5cb39e457ec5f4ec30f70ffddc6f134322a1422355a793b8707b.jsonnew file mode 100644index 0000000000..32d1e5e031--- /dev/null+++ b/.sqlx/query-daa957d34f4e5cb39e457ec5f4ec30f70ffddc6f134322a1422355a793b8707b.json@@ -0,0 +1,16 @@+{+ "db_name": "PostgreSQL",+ "query": "\n INSERT INTO email_message_labels (message_id, label_id)\n SELECT r.message_id, l.id\n FROM unnest($1::uuid[], $2::text[]) AS r(message_id, provider_label_id)\n JOIN email_labels l ON l.link_id = $3 AND l.provider_label_id = r.provider_label_id\n ON CONFLICT (message_id, label_id) DO NOTHING\n ",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "UuidArray",+ "TextArray",+ "Uuid"+ ]+ },+ "nullable": []+ },+ "hash": "daa957d34f4e5cb39e457ec5f4ec30f70ffddc6f134322a1422355a793b8707b"+}diff --git a/.sqlx/query-e6be65db250c2f9465f1b6fc8cdd26821e9ee462836eb33b45966a2f5903258f.json b/.sqlx/query-e6be65db250c2f9465f1b6fc8cdd26821e9ee462836eb33b45966a2f5903258f.jsonnew file mode 100644index 0000000000..c811865ee8--- /dev/null+++ b/.sqlx/query-e6be65db250c2f9465f1b6fc8cdd26821e9ee462836eb33b45966a2f5903258f.json@@ -0,0 +1,22 @@+{+ "db_name": "PostgreSQL",+ "query": "\n SELECT l.provider_label_id\n FROM email_message_labels ml\n JOIN email_labels l ON l.id = ml.label_id\n WHERE ml.message_id = $1\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "provider_label_id",+ "type_info": "Text"+ }+ ],+ "parameters": {+ "Left": [+ "Uuid"+ ]+ },+ "nullable": [+ false+ ]+ },+ "hash": "e6be65db250c2f9465f1b6fc8cdd26821e9ee462836eb33b45966a2f5903258f"+}diff --git a/.sqlx/query-f0a3a9ea4075ae17bd8ed306d6b062336927c9d12a39bc9baa970aee2f3b20ea.json b/.sqlx/query-f0a3a9ea4075ae17bd8ed306d6b062336927c9d12a39bc9baa970aee2f3b20ea.jsonnew file mode 100644index 0000000000..a6bb392bd7--- /dev/null+++ b/.sqlx/query-f0a3a9ea4075ae17bd8ed306d6b062336927c9d12a39bc9baa970aee2f3b20ea.json@@ -0,0 +1,18 @@+{+ "db_name": "PostgreSQL",+ "query": "\n WITH removed AS (\n DELETE FROM email_message_labels\n WHERE $3::bool\n AND message_id = $1::uuid\n AND label_id NOT IN (SELECT UNNEST($2::uuid[]))\n AND label_id NOT IN (\n SELECT id FROM email_labels\n WHERE link_id = $4 AND provider_label_id = ANY($5::text[])\n )\n )\n INSERT INTO email_message_labels (message_id, label_id)\n SELECT $1, label_id FROM UNNEST($2::uuid[]) AS t(label_id)\n ON CONFLICT (message_id, label_id) DO NOTHING\n ",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "Uuid",+ "UuidArray",+ "Bool",+ "Uuid",+ "TextArray"+ ]+ },+ "nullable": []+ },+ "hash": "f0a3a9ea4075ae17bd8ed306d6b062336927c9d12a39bc9baa970aee2f3b20ea"+}diff --git a/crates/email/AGENTS.md b/crates/email/AGENTS.mdindex 4208260304..1556967206 100644--- a/crates/email/AGENTS.md+++ b/crates/email/AGENTS.md@@ -77,6 +77,13 @@ tools — hexagonal, heavily feature-gated.dependency belongs under before wiring it in.- `domain/ports.rs` defines the traits adapters implement — call through thedomain service, never an outbound adapter directly from another crate.+- `EmailRepo::stage_thread_label_change` reads which messages change, applies+ the change and marks the changed provider messages (exactly what+ `commit_thread_label_write` enqueues) in one transaction holding the thread's+ label-write lock. The enqueue carries the write's id; a group that can't be+ queued goes back through `EmailRepo::revert_staged_label_change`, which+ restores the state an older write still on its way left (or the original),+ never a newer write's change.- Sender discovery authorizes the inbox first. Gmail aliases use the senderprovider; Outlook exposes its verified primary address without a Gmail lookup.- Outlook links have no system label rows. Move a thread in or out of the inboxdiff --git a/crates/email/fixtures/email_message_star_failure.sql b/crates/email/fixtures/email_message_star_failure.sqlnew file mode 100644index 0000000000..6894528484--- /dev/null+++ b/crates/email/fixtures/email_message_star_failure.sql@@ -0,0 +1,10 @@+-- Fail starred-state writes inside this test database, after the label write.+CREATE FUNCTION reject_message_star_state() RETURNS trigger AS $$+BEGIN+ RAISE EXCEPTION 'starred-state test failure';+END;+$$ LANGUAGE plpgsql;++CREATE TRIGGER reject_message_star_state+BEFORE UPDATE OF is_starred ON email_messages+FOR EACH ROW EXECUTE FUNCTION reject_message_star_state();diff --git a/crates/email/src/domain/models/label.rs b/crates/email/src/domain/models/label.rsindex 78ab696c2f..45cd62f32e 100644--- a/crates/email/src/domain/models/label.rs+++ b/crates/email/src/domain/models/label.rs@@ -1,3 +1,4 @@+use super::SimpleMessage;use chrono::{DateTime, Utc};use uuid::Uuid;@@ -75,6 +76,16 @@ pub struct UpdateThreadLabelsResult {pub failed_ids: Vec<Uuid>,}+/// A label change staged on a thread for Gmail, read in the same transaction+/// that applied it.+#[derive(Debug, Clone)]+pub struct StagedLabelChange {+ /// Every message of the thread.+ pub messages: Vec<SimpleMessage>,+ /// The messages that gained or lost the label.+ pub changed_ids: Vec<Uuid>,+}+/// The most threads one bulk archive request may change.pub const MAX_BULK_ARCHIVE_THREADS: usize = 100;diff --git a/crates/email/src/domain/models/mod.rs b/crates/email/src/domain/models/mod.rsindex db12921643..b290a9579d 100644--- a/crates/email/src/domain/models/mod.rs+++ b/crates/email/src/domain/models/mod.rs@@ -33,7 +33,7 @@ pub use email_filter::{EmailFilter, UpsertEmailFilterInput};pub use error::EmailErr;pub use label::{Label, LabelListVisibility, LabelType, LinkLabel, MAX_BULK_ARCHIVE_THREADS, MessageLabel,- MessageListVisibility, SetThreadsArchivedResult, UpdateThreadLabelsResult,+ MessageListVisibility, SetThreadsArchivedResult, StagedLabelChange, UpdateThreadLabelsResult,};pub use link::{EmailBackfillStatus, EmailInboxDetails, EmailSyncStatus, Link, ResolvedSendingInbox,diff --git a/crates/email/src/domain/ports.rs b/crates/email/src/domain/ports.rsindex a3f514fb27..d9db3d9815 100644--- a/crates/email/src/domain/ports.rs+++ b/crates/email/src/domain/ports.rs@@ -6,8 +6,9 @@ use crate::domain::models::{MessageTimestamps, ParsedAddresses, ParsedMessage, ParsedThread, PreviewCursorQuery,RecipientType, ResolvedDraftInput, ResolvedSendingInbox, SavedUserDraft, SenderPolicy,SendingInboxRequest, SetThreadsArchivedResult, SettledDraftIds, SimpleMessage,- SimpleMessageInfo, SourcedAttachment, Thread, ThreadRow, UpdateThreadLabelsResult,- UpsertEmailFilterInput, UpsertedContacts, UserEmailLink, UserProvider,+ SimpleMessageInfo, SourcedAttachment, StagedLabelChange, Thread, ThreadRow,+ UpdateThreadLabelsResult, UpsertEmailFilterInput, UpsertedContacts, UserEmailLink,+ UserProvider,};use entity_access::domain::models::{EditAccessLevel, EntityAccessReceipt, ViewAccessLevel};use macro_user_id::user_id::MacroUserIdStr;@@ -38,12 +39,14 @@ pub trait EmailMessageEnqueuer: Send + Sync + 'static {) -> impl Future<Output = Result<(), Self::Err>> + Send;/// Enqueue a batch of Gmail label modification operations to the gmail_ops worker queue.+ /// Each operation clears its message's `pending_write_id` marker when it finishes.fn enqueue_gmail_ops_modify_labels_batch(&self,link_id: Uuid,messages: Vec<(Uuid, String)>,labels_to_add: Vec<String>,labels_to_remove: Vec<String>,+ pending_write_id: Uuid,) -> impl Future<Output = Result<(), Self::Err>> + Send;/// Enqueue a Gmail-ops BlockSender operation for the link.@@ -420,28 +423,39 @@ pub trait EmailRepo: Send + Sync + 'static {link_id: Uuid,) -> impl Future<Output = Result<(), Self::Err>> + Send;- /// Atomically update the UNREAD label assignments and the message/thread read flags.- /// `message_ids` must be the messages of the already-authorized thread and inbox.- /// An error leaves all three representations unchanged.- fn set_thread_read_state(+ /// Atomically stage a label change on an already-authorized thread for+ /// Gmail: read which messages gain or lose the label, apply the change to+ /// every message with the read, starred and inbox flags it drives, and+ /// mark the changed messages that have a provider id — exactly those a+ /// Gmail write sends — as waiting on `write_id`, so inbox sync keeps their+ /// local state until Gmail confirms it. Label writes on one thread take+ /// turns, so each reads the state the one before it left. `None`, with+ /// nothing changed, when the thread has no messages.+ fn stage_thread_label_change(&self,thread_id: Uuid,link_id: Uuid,- message_ids: &[Uuid],- is_read: bool,- ) -> impl Future<Output = Result<(), Self::Err>> + Send;-- /// Atomically change INBOX assignments and the thread's visibility.- /// The message IDs belong to the authorized thread/inbox. Visibility is- /// separate from `add` so an enqueue failure can restore the prior flag- /// while reverting only the assignments changed by that operation.- fn set_thread_inbox_state(+ provider_label_id: &str,+ add: bool,+ write_id: Uuid,+ ) -> impl Future<Output = Result<Option<StagedLabelChange>, Self::Err>> + Send;++ /// Atomically undo a staged label change Gmail will never receive.+ /// `provider_message_ids` go back to the state Gmail will end up with:+ /// that of the newest older write still on its way, or the one before+ /// this write; a newer write keeps its own. `local_message_ids`+ /// (Macro-only drafts) lose the change. The thread's state is derived+ /// again, all while holding the thread's label-write lock.+ #[expect(clippy::too_many_arguments, reason = "the parts of one staged change")]+ fn revert_staged_label_change(&self,thread_id: Uuid,link_id: Uuid,- message_ids: &[Uuid],+ provider_label_id: &str,add: bool,- inbox_visible: bool,+ write_id: Uuid,+ provider_message_ids: &[Uuid],+ local_message_ids: &[Uuid],) -> impl Future<Output = Result<(), Self::Err>> + Send;/// Update the read status for a batch of messages, verified by link_id.@@ -1217,6 +1231,7 @@ impl EmailMessageEnqueuer for NoOpEnqueuer {_messages: Vec<(Uuid, String)>,_labels_to_add: Vec<String>,_labels_to_remove: Vec<String>,+ _pending_write_id: Uuid,) -> Result<(), Self::Err> {Ok(())}diff --git a/crates/email/src/domain/service/thread_labels.rs b/crates/email/src/domain/service/thread_labels.rsindex ed919024bd..9f55816712 100644--- a/crates/email/src/domain/service/thread_labels.rs+++ b/crates/email/src/domain/service/thread_labels.rs@@ -24,8 +24,6 @@ struct StagedThreadLabels {all_ids: Vec<Uuid>,/// Messages that gain or lose the label; only these go to Gmail or revert.changed_ids: Vec<Uuid>,- /// The thread's inbox visibility before an INBOX change.- inbox_visible: Option<bool>,}impl StagedThreadLabels {@@ -41,6 +39,16 @@ impl StagedThreadLabels {.map(|pid| (msg.db_id, pid.clone()))})}++ /// Changed Macro-only drafts, which never reach Gmail.+ fn changed_local_messages(&self) -> Vec<Uuid> {+ self.messages+ .iter()+ .filter(|msg| self.changed_ids.contains(&msg.db_id))+ .filter(|msg| msg.provider_id.as_ref().is_none_or(|pid| pid.is_empty()))+ .map(|msg| msg.db_id)+ .collect()+ }}/// Staged changes of one label on threads of one inbox, sent to Gmail as one@@ -49,6 +57,9 @@ struct ThreadLabelWrite {link: Link,label: LinkLabel,add: bool,+ /// Marks each thread's changed messages as waiting on this write when it+ /// is staged; every group of the write carries it.+ write_id: Uuid,threads: Vec<StagedThreadLabels>,}@@ -437,8 +448,9 @@ where.await;}+ let write_id = macro_uuid::generate_uuid_v7();let thread = self- .stage_thread_labels(link, thread_id, &label.provider_label_id, add)+ .stage_thread_labels(link, thread_id, &label.provider_label_id, add, write_id).await?;let successful_ids = thread.all_ids.clone();self.commit_thread_label_write(@@ -446,6 +458,7 @@ wherelink: link.clone(),label,add,+ write_id,threads: vec![thread],},actor,@@ -518,6 +531,7 @@ wherelink,label,add: !archived,+ write_id: macro_uuid::generate_uuid_v7(),threads: Vec::new(),});writes.len() - 1@@ -529,6 +543,7 @@ wherethread_id,system_labels::INBOX,!archived,+ writes[index].write_id,).await?;Ok(Some((index, thread)))@@ -574,81 +589,34 @@ where}/// Commit one thread's label change to the DB, capturing what sending it- /// to Gmail or reverting it needs.+ /// to Gmail or reverting it needs. The messages that change are read in+ /// the same transaction that changes them and marks them pending for+ /// `write_id`: only those go to Gmail, and an enqueue-failure revert+ /// restores only what this call changed.async fn stage_thread_labels(&self,link: &Link,thread_id: Uuid,provider_label_id: &str,add: bool,+ write_id: Uuid,) -> Result<StagedThreadLabels, EmailErr> {- let messages = self- .email_repo- .get_thread_label_messages(thread_id, link.id)- .await- .map_err(|e| EmailErr::RepoErr(anyhow::Error::from(e)))?;-- if messages.is_empty() {- return Err(EmailErr::ThreadEmpty);- }-if provider_label_id.is_empty() {return Err(EmailErr::EmptyProviderLabelId);}- let all_ids: Vec<Uuid> = messages.iter().map(|m| m.db_id).collect();-- // Snapshot which messages actually gain/lose the label. Only those go- // to Gmail, and an enqueue-failure revert restores only what this call- // changed — the batch writes are idempotent, and blindly inverting- // every message would strip pre-existing state. Best effort: if the- // snapshot read fails, fall back to treating all messages as changed.- let changed_ids: Vec<Uuid> = match self.email_repo.labels_by_message_ids(&all_ids).await {- Ok(labels_by_id) => all_ids- .iter()- .filter(|id| {- let has_label = labels_by_id.get(id).is_some_and(|labels| {- labels- .iter()- .any(|l| l.provider_label_id == provider_label_id)- });- has_label != add- })- .copied()- .collect(),- Err(e) => {- let err = anyhow::Error::from(e);- if provider_label_id == system_labels::INBOX {- return Err(EmailErr::RepoErr(err));- }- tracing::warn!(error=?err, "failed to snapshot message labels; revert would cover all messages");- all_ids.clone()- }- };-- let inbox_visible = if provider_label_id == system_labels::INBOX {- Some(- self.email_repo- .thread_by_id(thread_id)- .await- .map_err(|e| EmailErr::RepoErr(anyhow::Error::from(e)))?- .ok_or(EmailErr::ThreadNotFound)?- .inbox_visible,- )- } else {- None- };-- // Optimistic DB update: update all messages first- self.apply_label_db_changes(link, thread_id, &all_ids, provider_label_id, add)- .await?;+ let staged = self+ .email_repo+ .stage_thread_label_change(thread_id, link.id, provider_label_id, add, write_id)+ .await+ .map_err(|e| EmailErr::RepoErr(anyhow::Error::from(e)))?+ .ok_or(EmailErr::ThreadEmpty)?;Ok(StagedThreadLabels {thread_id,- messages,- all_ids,- changed_ids,- inbox_visible,+ all_ids: staged.messages.iter().map(|m| m.db_id).collect(),+ messages: staged.messages,+ changed_ids: staged.changed_ids,})}@@ -667,6 +635,7 @@ wherelink,label,add,+ write_id,mut threads,} = write;let provider_label_id = label.provider_label_id.as_str();@@ -692,6 +661,7 @@ whereprovider_messages,labels_to_add.clone(),labels_to_remove.clone(),+ write_id,).await{@@ -699,7 +669,7 @@ wheretracing::error!(error=?err, "failed to enqueue gmail ops label sync, reverting database changes");let unsent_threads = threads.split_off(group.start);for thread in &unsent_threads {- self.revert_label_db_changes(&link, thread, provider_label_id, add)+ self.revert_label_db_changes(&link, thread, provider_label_id, add, write_id).await;}unsent = Some(UnsentThreads {@@ -758,145 +728,38 @@ where/// Best-effort revert of the optimistic label write after the Gmail sync/// could not be enqueued. Only `changed_ids` — the messages that actually- /// gained/lost the label — are inverted, so messages that already held- /// the requested state keep it; the denormalized thread read flag is- /// restored to its derived pre-op value. Every step logs and continues on- /// failure: the caller surfaces the enqueue error regardless.+ /// gained/lost the label — go back, and a message a newer write changed+ /// since keeps that write's state. Logs on failure: the caller surfaces+ /// the enqueue error regardless, and a pending marker left behind only+ /// keeps inbox sync off that label until it expires.async fn revert_label_db_changes(&self,link: &Link,thread: &StagedThreadLabels,provider_label_id: &str,add: bool,+ write_id: Uuid,) {- let StagedThreadLabels {- thread_id,- all_ids,- changed_ids,- inbox_visible,- ..- } = thread;- let thread_id = *thread_id;- if let Some(inbox_visible) = *inbox_visible {- if let Err(e) = self- .email_repo- .set_thread_inbox_state(thread_id, link.id, changed_ids, !add, inbox_visible)- .await- {- let err = anyhow::Error::from(e);- tracing::error!(error=?err, "failed to revert thread inbox state after enqueue failure");- }- return;- }- if !changed_ids.is_empty() {- let db_result = if add {- self.email_repo- .delete_message_labels_batch(changed_ids, provider_label_id, link.id)- .await- } else {- self.email_repo- .insert_message_labels_batch(changed_ids, provider_label_id, link.id)- .await- };- if let Err(e) = db_result {- let err = anyhow::Error::from(e);- tracing::error!(error=?err, "failed to revert message labels after enqueue failure");- }- }-- if provider_label_id == system_labels::UNREAD {- // Restore is_read on the flipped messages: they were read before- // an UNREAD add (`add` == true) and unread before a removal.- if !changed_ids.is_empty()- && let Err(e) = self- .email_repo- .update_message_read_status_batch(changed_ids, link.id, add)- .await- {- let err = anyhow::Error::from(e);- tracing::error!(error=?err, "failed to revert message read status after enqueue failure");- }- // Pre-op unread set: everything outside `changed_ids` for an add,- // `changed_ids` itself for a removal — the thread was read iff- // that set was empty.- let original_is_read = if add {- changed_ids.len() == all_ids.len()- } else {- changed_ids.is_empty()- };- if let Err(e) = self- .email_repo- .update_thread_read_status(thread_id, link.id, original_is_read)- .await- {- let err = anyhow::Error::from(e);- tracing::error!(error=?err, "failed to revert thread read status after enqueue failure");- }- } else if provider_label_id == system_labels::STARRED- && !changed_ids.is_empty()- && let Err(e) = self- .email_repo- .update_message_starred_status_batch(changed_ids, link.id, !add)- .await- {- let err = anyhow::Error::from(e);- tracing::error!(error=?err, "failed to revert message starred status after enqueue failure");- }- }-- /// Apply a thread label change to the DB. UNREAD/read flags and INBOX/- /// visibility commit atomically; a failure propagates before provider sync- /// or events. Other labels retain their best-effort starred side effect.- async fn apply_label_db_changes(- &self,- link: &Link,- thread_id: Uuid,- message_ids: &[Uuid],- provider_label_id: &str,- add: bool,- ) -> Result<(), EmailErr> {- if provider_label_id == system_labels::INBOX {- return self- .email_repo- .set_thread_inbox_state(thread_id, link.id, message_ids, add, add)- .await- .map_err(|e| EmailErr::RepoErr(anyhow::Error::from(e)));- }- if provider_label_id == system_labels::UNREAD {- return self- .email_repo- .set_thread_read_state(thread_id, link.id, message_ids, !add)- .await- .map_err(|e| EmailErr::RepoErr(anyhow::Error::from(e)));- }-- let db_result = if add {- self.email_repo- .insert_message_labels_batch(message_ids, provider_label_id, link.id)- .await- } else {- self.email_repo- .delete_message_labels_batch(message_ids, provider_label_id, link.id)- .await- };-- if let Err(e) = db_result {- let err = anyhow::Error::from(e);- tracing::error!(error=?err, "failed to update message labels in database");- return Err(EmailErr::RepoErr(err));- }-- if provider_label_id == system_labels::STARRED- && let Err(e) = self- .email_repo- .update_message_starred_status_batch(message_ids, link.id, add)- .await+ let provider_ids: Vec<Uuid> = thread+ .changed_provider_messages()+ .map(|(id, _)| id)+ .collect();+ if let Err(e) = self+ .email_repo+ .revert_staged_label_change(+ thread.thread_id,+ link.id,+ provider_label_id,+ add,+ write_id,+ &provider_ids,+ &thread.changed_local_messages(),+ )+ .await{let err = anyhow::Error::from(e);- tracing::error!(error=?err, "failed to update message starred status");+ tracing::error!(error=?err, "failed to revert label change after enqueue failure");}-- Ok(())}/// Map a user-initiated thread label change onto the semanticdiff --git a/crates/email/src/outbound/email_pg_repo.rs b/crates/email/src/outbound/email_pg_repo.rsindex ea0b05a257..81f5d98507 100644--- a/crates/email/src/outbound/email_pg_repo.rs+++ b/crates/email/src/outbound/email_pg_repo.rs@@ -4,8 +4,8 @@ use crate::domain::{EmailErr, EmailFilter, EmailInboxDetails, EmailThreadMailProjection, EmailThreadMetadata,EmailThreadPreview, Label, Link, LinkLabel, MessageAttachment, MessageLabel, MessageRow,MessageTimestamps, ParsedAddresses, PreviewCursorQuery, ResolvedDraftInput,- SettledDraftIds, SimpleMessage, SimpleMessageInfo, ThreadRow, UpsertEmailFilterInput,- UpsertedContacts, UserProvider,+ SettledDraftIds, SimpleMessage, SimpleMessageInfo, StagedLabelChange, ThreadRow,+ UpsertEmailFilterInput, UpsertedContacts, UserProvider,},ports::{DraftAttachmentRepo, EmailRepo, EmailUserRepo, LinkEmailSettings, RecipientsByMessageId,@@ -437,31 +437,44 @@ impl EmailRepo for EmailPgRepo {.await}- async fn set_thread_read_state(+ async fn stage_thread_label_change(&self,thread_id: Uuid,link_id: Uuid,- message_ids: &[Uuid],- is_read: bool,- ) -> Result<(), Self::Err> {- label::set_thread_read_state(&self.pool, thread_id, link_id, message_ids, is_read).await+ provider_label_id: &str,+ add: bool,+ write_id: Uuid,+ ) -> Result<Option<StagedLabelChange>, Self::Err> {+ label::stage_thread_label_change(+ &self.pool,+ thread_id,+ link_id,+ provider_label_id,+ add,+ write_id,+ )+ .await}- async fn set_thread_inbox_state(+ async fn revert_staged_label_change(&self,thread_id: Uuid,link_id: Uuid,- message_ids: &[Uuid],+ provider_label_id: &str,add: bool,- inbox_visible: bool,+ write_id: Uuid,+ provider_message_ids: &[Uuid],+ local_message_ids: &[Uuid],) -> Result<(), Self::Err> {- label::set_thread_inbox_state(+ label::revert_staged_label_change(&self.pool,thread_id,link_id,- message_ids,+ provider_label_id,add,- inbox_visible,+ write_id,+ provider_message_ids,+ local_message_ids,).await}diff --git a/crates/email/src/outbound/email_pg_repo/label.rs b/crates/email/src/outbound/email_pg_repo/label.rsindex ec3ae6b934..37bcab409e 100644--- a/crates/email/src/outbound/email_pg_repo/label.rs+++ b/crates/email/src/outbound/email_pg_repo/label.rs@@ -1,6 +1,6 @@use crate::domain::models::{LabelListVisibility, LabelType, LinkLabel, MessageListVisibility, SimpleMessage,- label::system_labels,+ StagedLabelChange, label::system_labels,};use crate::outbound::email_pg_repo::db_types::{LabelListVisibilityDbRow, LabelTypeDbRow, MessageListVisibilityDbRow,@@ -163,9 +163,9 @@ pub(crate) async fn get_label_by_id(Ok(row.map(Into::into))}-#[tracing::instrument(skip(pool), err)]+#[tracing::instrument(skip(executor), err)]pub(crate) async fn get_thread_label_messages(- pool: &PgPool,+ executor: impl sqlx::PgExecutor<'_>,thread_id: Uuid,link_id: Uuid,) -> Result<Vec<SimpleMessage>, sqlx::Error> {@@ -202,48 +202,136 @@ pub(crate) async fn get_thread_label_messages(thread_id,link_id)- .fetch_all(pool)+ .fetch_all(executor).await?;Ok(rows.into_iter().map(Into::into).collect())}-/// Commit every local representation of a thread's read state together.-#[tracing::instrument(skip(pool, message_ids), err)]-pub(super) async fn set_thread_read_state(+/// Stage a label change on a thread for Gmail in one transaction: read which+/// messages gain or lose the label, apply the change to every message with the+/// flags it drives, and mark the changed messages Gmail knows about as waiting+/// on `write_id`. `None` when the thread has no messages.+#[tracing::instrument(skip(pool), err)]+pub(super) async fn stage_thread_label_change(pool: &PgPool,thread_id: Uuid,link_id: Uuid,- message_ids: &[Uuid],- is_read: bool,-) -> Result<(), sqlx::Error> {+ provider_label_id: &str,+ add: bool,+ write_id: Uuid,+) -> Result<Option<StagedLabelChange>, sqlx::Error> {let mut tx = pool.begin().await?;- if is_read {- delete_message_labels_batch(&mut *tx, message_ids, system_labels::UNREAD, link_id).await?;+ email_db_client::labels::pending_writes::lock_thread_label_writes(&mut tx, &[thread_id])+ .await?;++ let messages = get_thread_label_messages(&mut *tx, thread_id, link_id).await?;+ if messages.is_empty() {+ return Ok(None);+ }+ let message_ids: Vec<Uuid> = messages.iter().map(|message| message.db_id).collect();+ let changed_ids = email_db_client::labels::get::messages_changing_label(+ &mut *tx,+ link_id,+ &message_ids,+ provider_label_id,+ add,+ )+ .await?;++ if add {+ insert_message_labels_batch(&mut *tx, &message_ids, provider_label_id, link_id).await?;} else {- insert_message_labels_batch(&mut *tx, message_ids, system_labels::UNREAD, link_id).await?;+ delete_message_labels_batch(&mut *tx, &message_ids, provider_label_id, link_id).await?;}- update_message_read_status_batch(&mut *tx, message_ids, link_id, is_read).await?;- super::thread::update_thread_read_status(&mut *tx, thread_id, link_id, is_read).await?;- tx.commit().await+ if provider_label_id == system_labels::INBOX {+ set_thread_inbox_visible(&mut tx, thread_id, link_id, add).await?;+ } else if provider_label_id == system_labels::UNREAD {+ update_message_read_status_batch(&mut *tx, &message_ids, link_id, !add).await?;+ super::thread::update_thread_read_status(&mut *tx, thread_id, link_id, !add).await?;+ } else if provider_label_id == system_labels::STARRED {+ update_message_starred_status_batch(&mut *tx, &message_ids, link_id, add).await?;+ }++ let pending_ids: Vec<Uuid> = messages+ .iter()+ .filter(|message| {+ changed_ids.contains(&message.db_id)+ && message+ .provider_id+ .as_ref()+ .is_some_and(|id| !id.is_empty())+ })+ .map(|message| message.db_id)+ .collect();+ email_db_client::labels::pending_writes::mark_pending_label_writes(+ &mut *tx,+ link_id,+ &pending_ids,+ provider_label_id,+ add,+ write_id,+ )+ .await?;+ tx.commit().await?;++ Ok(Some(StagedLabelChange {+ messages,+ changed_ids,+ }))}-/// Commit INBOX assignments and the canonical thread flag in one transaction.-#[tracing::instrument(skip(pool, message_ids), err)]-pub(super) async fn set_thread_inbox_state(+/// Undo a staged label change Gmail will never receive, in one transaction+/// holding the thread's label-write lock: labels of `provider_message_ids`+/// go back to the state Gmail will end up with (see+/// `abandon_pending_label_writes`), `local_message_ids` (Macro-only drafts)+/// lose the change, and the thread's state is derived again.+#[tracing::instrument(skip(pool, provider_message_ids, local_message_ids), err)]+#[expect(clippy::too_many_arguments, reason = "the parts of one staged change")]+pub(super) async fn revert_staged_label_change(pool: &PgPool,thread_id: Uuid,link_id: Uuid,- message_ids: &[Uuid],+ provider_label_id: &str,add: bool,- inbox_visible: bool,+ write_id: Uuid,+ provider_message_ids: &[Uuid],+ local_message_ids: &[Uuid],) -> Result<(), sqlx::Error> {let mut tx = pool.begin().await?;+ email_db_client::labels::pending_writes::lock_thread_label_writes(&mut tx, &[thread_id])+ .await?;+ email_db_client::labels::pending_writes::abandon_pending_label_writes(+ &mut tx,+ link_id,+ provider_message_ids,+ write_id,+ )+ .await?;+if add {- insert_message_labels_batch(&mut *tx, message_ids, system_labels::INBOX, link_id).await?;+ delete_message_labels_batch(&mut *tx, local_message_ids, provider_label_id, link_id)+ .await?;} else {- delete_message_labels_batch(&mut *tx, message_ids, system_labels::INBOX, link_id).await?;+ insert_message_labels_batch(&mut *tx, local_message_ids, provider_label_id, link_id)+ .await?;+ }+ if provider_label_id == system_labels::UNREAD {+ update_message_read_status_batch(&mut *tx, local_message_ids, link_id, add).await?;+ } else if provider_label_id == system_labels::STARRED {+ update_message_starred_status_batch(&mut *tx, local_message_ids, link_id, !add).await?;}++ super::thread::update_thread_metadata(&mut tx, thread_id, link_id).await?;+ tx.commit().await+}++async fn set_thread_inbox_visible(+ tx: &mut sqlx::PgConnection,+ thread_id: Uuid,+ link_id: Uuid,+ inbox_visible: bool,+) -> Result<(), sqlx::Error> {sqlx::query!(r#"UPDATE email_threads@@ -254,9 +342,9 @@ pub(super) async fn set_thread_inbox_state(thread_id,link_id,)- .execute(&mut *tx)+ .execute(tx).await?;- tx.commit().await+ Ok(())}#[tracing::instrument(skip(executor), err)]@@ -356,9 +444,9 @@ pub(crate) async fn update_message_read_status_batch(Ok(())}-#[tracing::instrument(skip(pool), err)]+#[tracing::instrument(skip(executor), err)]pub(crate) async fn update_message_starred_status_batch(- pool: &PgPool,+ executor: impl sqlx::PgExecutor<'_>,message_ids: &[Uuid],link_id: Uuid,is_starred: bool,@@ -381,7 +469,7 @@ pub(crate) async fn update_message_starred_status_batch(message_ids,link_id)- .execute(pool)+ .execute(executor).await?;Ok(())diff --git a/crates/email/src/outbound/email_pg_repo/test/send_attempt/provider_draft_trash.rs b/crates/email/src/outbound/email_pg_repo/test/send_attempt/provider_draft_trash.rsindex 58755ff83b..c71ad24fe8 100644--- a/crates/email/src/outbound/email_pg_repo/test/send_attempt/provider_draft_trash.rs+++ b/crates/email/src/outbound/email_pg_repo/test/send_attempt/provider_draft_trash.rs@@ -36,7 +36,7 @@ async fn echo_then_trash(.is_draft);// The same production path used by inbox synchronization, not a direct flag UPDATE.- insert_message(pool, thread, &mut echo, link(), true).await?;+ insert_message(pool, thread, &mut echo, link(), true, None).await?;assert!(repo.get_simple_message(message(), &[link()]).await?diff --git a/crates/email/src/outbound/email_pg_repo/test/thread_archive.rs b/crates/email/src/outbound/email_pg_repo/test/thread_archive.rsindex 2478c677aa..654469b442 100644--- a/crates/email/src/outbound/email_pg_repo/test/thread_archive.rs+++ b/crates/email/src/outbound/email_pg_repo/test/thread_archive.rs@@ -68,8 +68,12 @@ fn service_with_enqueuer<E: EmailMessageEnqueuer>(type LabelWrite = (Uuid, Vec<(Uuid, String)>, Vec<String>, Vec<String>);+/// Label writes and, in the same order, the pending write id each carried.#[derive(Clone, Default)]-struct RecordingEnqueuer(Arc<std::sync::Mutex<Vec<LabelWrite>>>);+struct RecordingEnqueuer(+ Arc<std::sync::Mutex<Vec<LabelWrite>>>,+ Arc<std::sync::Mutex<Vec<Uuid>>>,+);impl EmailMessageEnqueuer for RecordingEnqueuer {type Err = anyhow::Error;const MAX_LABEL_WRITE_MESSAGES: usize = 1000;@@ -87,11 +91,13 @@ impl EmailMessageEnqueuer for RecordingEnqueuer {messages: Vec<(Uuid, String)>,labels_to_add: Vec<String>,labels_to_remove: Vec<String>,+ pending_write_id: Uuid,) -> anyhow::Result<()> {self.0.lock().unwrap().push((link_id, messages, labels_to_add, labels_to_remove));+ self.1.lock().unwrap().push(pending_write_id);Ok(())}async fn enqueue_gmail_ops_block_sender(&self, _: Uuid, _: String) -> anyhow::Result<()> {@@ -113,7 +119,7 @@ async fn archive_sends_gmail_only_the_messages_still_in_the_inbox(pool: Pool<Postgres>,) -> anyhow::Result<()> {let enqueuer = RecordingEnqueuer::default();- let service = service_with_enqueuer(pool, enqueuer.clone());+ let service = service_with_enqueuer(pool.clone(), enqueuer.clone());let owner = MacroUserIdStr::try_from_email("[email protected]")?;let thread = Uuid::parse_str("11111111-1111-1111-1111-111111111111")?;let link = Uuid::parse_str("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")?;@@ -141,10 +147,26 @@ async fn archive_sends_gmail_only_the_messages_still_in_the_inbox(vec!["INBOX".to_string()])]);+ let write_id = enqueuer.1.lock().unwrap()[0];+ assert_eq!(+ pending_markers(&pool).await?,+ vec![(first, "INBOX".to_string(), write_id)]+ );assert_inbox_state(&service.email_repo, thread, false).await?;Ok(())}+async fn pending_markers(pool: &Pool<Postgres>) -> anyhow::Result<Vec<(Uuid, String, Uuid)>> {+ Ok(sqlx::query!(+ "SELECT message_id, provider_label_id, write_id FROM email_gmail_pending_label_writes ORDER BY message_id, seq"+ )+ .fetch_all(pool)+ .await?+ .into_iter()+ .map(|marker| (marker.message_id, marker.provider_label_id, marker.write_id))+ .collect())+}+struct FailingEnqueuer;impl EmailMessageEnqueuer for FailingEnqueuer {type Err = anyhow::Error;@@ -163,6 +185,7 @@ impl EmailMessageEnqueuer for FailingEnqueuer {_: Vec<(Uuid, String)>,_: Vec<String>,_: Vec<String>,+ _: Uuid,) -> anyhow::Result<()> {anyhow::bail!("test queue unavailable")}@@ -184,7 +207,7 @@ impl EmailMessageEnqueuer for FailingEnqueuer {async fn archive_enqueue_failure_restores_only_changed_labels_and_the_original_flag(pool: Pool<Postgres>,) -> anyhow::Result<()> {- let service = service_with_enqueuer(pool, FailingEnqueuer);+ let service = service_with_enqueuer(pool.clone(), FailingEnqueuer);let thread = Uuid::parse_str("11111111-1111-1111-1111-111111111111")?;let link = Uuid::parse_str("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")?;let first = Uuid::parse_str("11111111-aaaa-aaaa-aaaa-111111111111")?;@@ -225,6 +248,49 @@ async fn archive_enqueue_failure_restores_only_changed_labels_and_the_original_fexpected);}+ assert!(pending_markers(&pool).await?.is_empty());+ Ok(())+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(+ path = "../../../../fixtures",+ scripts("email_thread_labels", "email_thread_unread", "email_thread_archive")+ )+)]+async fn failed_enqueue_hands_markers_back_to_the_write_still_on_its_way(+ pool: Pool<Postgres>,+) -> anyhow::Result<()> {+ let enqueuer = RecordingEnqueuer::default();+ let queued = service_with_enqueuer(pool.clone(), enqueuer.clone());+ let unqueued = service_with_enqueuer(pool.clone(), FailingEnqueuer);+ let owner = MacroUserIdStr::try_from_email("[email protected]")?;+ let thread = Uuid::parse_str("11111111-1111-1111-1111-111111111111")?;+ let first = Uuid::parse_str("11111111-aaaa-aaaa-aaaa-111111111111")?;+ let second = Uuid::parse_str("11111111-bbbb-bbbb-bbbb-111111111111")?;++ // The archive is queued but not yet applied by Gmail.+ queued+ .set_thread_archived_impl(owner.clone(), thread, true)+ .await?;+ let archive = enqueuer.1.lock().unwrap()[0];+ // The unarchive takes the markers over, then cannot be queued.+ assert!(+ unqueued+ .set_thread_archived_impl(owner, thread, false)+ .await+ .is_err()+ );++ assert_inbox_state(&queued.email_repo, thread, false).await?;+ assert_eq!(+ pending_markers(&pool).await?,+ vec![+ (first, "INBOX".to_string(), archive),+ (second, "INBOX".to_string(), archive),+ ]+ );Ok(())}@@ -349,7 +415,7 @@ async fn archive_rejects_inaccessible_and_empty_threads())]async fn failed_archive_rolls_back_inbox_label_removal(pool: Pool<Postgres>) -> anyhow::Result<()> {- let service = service(pool);+ let service = service(pool.clone());let thread = Uuid::parse_str("33333333-3333-3333-3333-333333333333")?;assert!(matches!(service@@ -362,6 +428,146 @@ async fn failed_archive_rolls_back_inbox_label_removal(pool: Pool<Postgres>) ->Err(EmailErr::RepoErr(_))));assert_inbox_state(&service.email_repo, thread, true).await?;+ assert!(pending_markers(&pool).await?.is_empty());+ Ok(())+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(+ path = "../../../../fixtures",+ scripts("email_thread_labels", "email_thread_unread", "email_thread_archive")+ )+)]+async fn staged_inbox_change_marks_the_messages_it_changed_in_the_same_transaction(+ pool: Pool<Postgres>,+) -> anyhow::Result<()> {+ let repo = EmailPgRepo::new(pool.clone());+ let thread = Uuid::parse_str("11111111-1111-1111-1111-111111111111")?;+ let link = Uuid::parse_str("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")?;+ let first = Uuid::parse_str("11111111-aaaa-aaaa-aaaa-111111111111")?;+ let second = Uuid::parse_str("11111111-bbbb-bbbb-bbbb-111111111111")?;+ let write_id = Uuid::from_u128(0xf1);+ repo.delete_message_labels_batch(&[second], "INBOX", link)+ .await?;++ let staged = repo+ .stage_thread_label_change(thread, link, "INBOX", false, write_id)+ .await?+ .expect("the thread has messages");++ assert_eq!(staged.changed_ids, vec![first]);+ assert_inbox_state(&repo, thread, false).await?;+ assert_eq!(+ pending_markers(&pool).await?,+ vec![(first, "INBOX".to_string(), write_id)]+ );+ Ok(())+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(+ path = "../../../../fixtures",+ scripts("email_thread_labels", "email_thread_unread", "email_thread_archive")+ )+)]+async fn a_label_write_waits_for_the_one_already_staging_on_its_thread(+ pool: Pool<Postgres>,+) -> anyhow::Result<()> {+ let repo = EmailPgRepo::new(pool.clone());+ let thread = Uuid::parse_str("11111111-1111-1111-1111-111111111111")?;+ let link = Uuid::parse_str("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")?;+ let first = Uuid::parse_str("11111111-aaaa-aaaa-aaaa-111111111111")?;++ // Another write holds the thread's label-write lock while it archives `first`.+ let mut staging = pool.begin().await?;+ email_db_client::labels::pending_writes::lock_thread_label_writes(&mut staging, &[thread])+ .await?;+ sqlx::query!(+ "DELETE FROM email_message_labels WHERE message_id = $1",+ first+ )+ .execute(&mut *staging)+ .await?;++ let blocked = tokio::time::timeout(+ std::time::Duration::from_millis(500),+ repo.stage_thread_label_change(thread, link, "INBOX", true, Uuid::from_u128(0xf2)),+ )+ .await;+ assert!(blocked.is_err(), "a second write must wait for the first");++ staging.commit().await?;+ let staged = repo+ .stage_thread_label_change(thread, link, "INBOX", true, Uuid::from_u128(0xf2))+ .await?+ .expect("the thread has messages");+ assert_eq!(staged.changed_ids, vec![first]);+ Ok(())+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(+ path = "../../../../fixtures",+ scripts(+ "email_thread_labels",+ "email_thread_unread",+ "email_thread_archive",+ "email_thread_archive_failure"+ )+ )+)]+async fn failed_staged_inbox_change_leaves_no_pending_marker(+ pool: Pool<Postgres>,+) -> anyhow::Result<()> {+ let repo = EmailPgRepo::new(pool.clone());+ let thread = Uuid::parse_str("33333333-3333-3333-3333-333333333333")?;+ let link = Uuid::parse_str("aaaaaaaa-aaaa-aaaa-aaaa-bbbbbbbbbbbb")?;++ assert!(+ repo.stage_thread_label_change(thread, link, "INBOX", false, Uuid::from_u128(0xf1))+ .await+ .is_err()+ );++ assert_inbox_state(&repo, thread, true).await?;+ assert!(pending_markers(&pool).await?.is_empty());+ Ok(())+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(+ path = "../../../../fixtures",+ scripts("email_thread_labels", "email_message_star_failure")+ )+)]+async fn failed_star_flag_rolls_back_the_label_and_its_marker(+ pool: Pool<Postgres>,+) -> anyhow::Result<()> {+ let repo = EmailPgRepo::new(pool.clone());+ let thread = Uuid::parse_str("11111111-1111-1111-1111-111111111111")?;+ let link = Uuid::parse_str("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")?;++ assert!(+ repo.stage_thread_label_change(thread, link, "STARRED", true, Uuid::from_u128(0xf1))+ .await+ .is_err()+ );++ let messages = repo.get_thread_label_messages(thread, link).await?;+ assert!(messages.iter().all(|message| !message.is_starred));+ let ids: Vec<_> = messages.iter().map(|message| message.db_id).collect();+ let labels = repo.labels_by_message_ids(&ids).await?;+ assert!(+ !labels+ .values()+ .flatten()+ .any(|label| label.provider_label_id == "STARRED")+ );+ assert!(pending_markers(&pool).await?.is_empty());Ok(())}@@ -469,7 +675,7 @@ async fn bulk_archive_sends_one_gmail_write_per_inbox_and_reports_threads_it_skipool: Pool<Postgres>,) -> anyhow::Result<()> {let enqueuer = RecordingEnqueuer::default();- let service = service_with_enqueuer(pool, enqueuer.clone());+ let service = service_with_enqueuer(pool.clone(), enqueuer.clone());let owner = MacroUserIdStr::try_from_email("[email protected]")?;let primary_thread = Uuid::parse_str("11111111-1111-1111-1111-111111111111")?;let empty_thread = Uuid::parse_str("22222222-2222-2222-2222-222222222222")?;@@ -526,6 +732,28 @@ async fn bulk_archive_sends_one_gmail_write_per_inbox_and_reports_threads_it_ski),]);+ let write_ids = enqueuer.1.lock().unwrap().clone();+ assert_ne!(write_ids[0], write_ids[1]);+ assert_eq!(+ pending_markers(&pool).await?,+ vec![+ (+ Uuid::parse_str("11111111-aaaa-aaaa-aaaa-111111111111")?,+ "INBOX".to_string(),+ write_ids[0]+ ),+ (+ Uuid::parse_str("11111111-bbbb-bbbb-bbbb-111111111111")?,+ "INBOX".to_string(),+ write_ids[0]+ ),+ (+ Uuid::parse_str("33333333-aaaa-aaaa-aaaa-333333333333")?,+ "INBOX".to_string(),+ write_ids[1]+ ),+ ]+ );assert_inbox_state(&service.email_repo, primary_thread, false).await?;assert_inbox_state(&service.email_repo, secondary_thread, false).await?;Ok(())@@ -541,7 +769,7 @@ async fn bulk_archive_sends_one_gmail_write_per_inbox_and_reports_threads_it_skiasync fn bulk_archive_reports_an_enqueue_outage_instead_of_failing_every_thread(pool: Pool<Postgres>,) -> anyhow::Result<()> {- let service = service_with_enqueuer(pool, FailingEnqueuer);+ let service = service_with_enqueuer(pool.clone(), FailingEnqueuer);let primary_thread = Uuid::parse_str("11111111-1111-1111-1111-111111111111")?;let secondary_thread = Uuid::parse_str("33333333-3333-3333-3333-333333333333")?;@@ -556,12 +784,13 @@ async fn bulk_archive_reports_an_enqueue_outage_instead_of_failing_every_thread(assert!(matches!(result, Err(EmailErr::EnqueueErr(_))));assert_inbox_state(&service.email_repo, primary_thread, true).await?;assert_inbox_state(&service.email_repo, secondary_thread, true).await?;+ assert!(pending_markers(&pool).await?.is_empty());Ok(())}/// Accepts one label write, then fails, with room for two messages per write.#[derive(Clone, Default)]-struct FailsAfterFirstWriteEnqueuer(Arc<std::sync::Mutex<Vec<LabelWrite>>>);+struct FailsAfterFirstWriteEnqueuer(Arc<std::sync::Mutex<Vec<(LabelWrite, Uuid)>>>);impl EmailMessageEnqueuer for FailsAfterFirstWriteEnqueuer {type Err = anyhow::Error;const MAX_LABEL_WRITE_MESSAGES: usize = 2;@@ -579,12 +808,16 @@ impl EmailMessageEnqueuer for FailsAfterFirstWriteEnqueuer {messages: Vec<(Uuid, String)>,labels_to_add: Vec<String>,labels_to_remove: Vec<String>,+ pending_write_id: Uuid,) -> anyhow::Result<()> {let mut writes = self.0.lock().unwrap();if !writes.is_empty() {anyhow::bail!("test queue unavailable");}- writes.push((link_id, messages, labels_to_add, labels_to_remove));+ writes.push((+ (link_id, messages, labels_to_add, labels_to_remove),+ pending_write_id,+ ));Ok(())}async fn enqueue_gmail_ops_block_sender(&self, _: Uuid, _: String) -> anyhow::Result<()> {@@ -611,7 +844,7 @@ async fn bulk_archive_reverts_only_the_threads_whose_write_could_not_be_queued(pool: Pool<Postgres>,) -> anyhow::Result<()> {let enqueuer = FailsAfterFirstWriteEnqueuer::default();- let service = service_with_enqueuer(pool, enqueuer.clone());+ let service = service_with_enqueuer(pool.clone(), enqueuer.clone());let queued_thread = Uuid::parse_str("11111111-1111-1111-1111-111111111111")?;let unqueued_thread = Uuid::parse_str("77777777-7777-7777-7777-777777777777")?;@@ -631,16 +864,31 @@ async fn bulk_archive_reverts_only_the_threads_whose_write_could_not_be_queued(});// The first thread's two messages fill the first write on their own.+ let writes = enqueuer.0.lock().unwrap().clone();assert_eq!(- enqueuer- .0- .lock()- .unwrap()+ writes.iter()- .map(|(_, messages, _, _)| messages.len())+ .map(|((_, messages, _, _), _)| messages.len()).collect::<Vec<_>>(),vec![2]);+ let write_id = writes[0].1;+ // Only the queued group keeps its markers.+ assert_eq!(+ pending_markers(&pool).await?,+ vec![+ (+ Uuid::parse_str("11111111-aaaa-aaaa-aaaa-111111111111")?,+ "INBOX".to_string(),+ write_id+ ),+ (+ Uuid::parse_str("11111111-bbbb-bbbb-bbbb-111111111111")?,+ "INBOX".to_string(),+ write_id+ ),+ ]+ );assert_inbox_state(&service.email_repo, queued_thread, false).await?;assert_inbox_state(&service.email_repo, unqueued_thread, true).await?;Ok(())diff --git a/crates/email/src/outbound/mailbox_pg/ingest.rs b/crates/email/src/outbound/mailbox_pg/ingest.rsindex f4283bace3..c155793645 100644--- a/crates/email/src/outbound/mailbox_pg/ingest.rs+++ b/crates/email/src/outbound/mailbox_pg/ingest.rs@@ -202,6 +202,7 @@ impl MailboxIngest for PgMailboxSync {lease.mailbox.link_id,recipients,false,+ None,).await.map_err(|_| MailboxError::Persistence)?;diff --git a/crates/email_api_client/src/domain/service.rs b/crates/email_api_client/src/domain/service.rsindex 5076b841f7..eddf0cd49e 100644--- a/crates/email_api_client/src/domain/service.rs+++ b/crates/email_api_client/src/domain/service.rs@@ -15,6 +15,7 @@ mod labels;/// Credential and quota orchestration for generation-bound mailbox operations.pub mod mailbox;mod messages;+pub use messages::SEPARATE_LABEL_WRITES_BELOW;mod send;mod subscription;mod sync;diff --git a/crates/email_api_client/src/domain/service/messages.rs b/crates/email_api_client/src/domain/service/messages.rsindex f75ee04b6e..4d632b86d9 100644--- a/crates/email_api_client/src/domain/service/messages.rs+++ b/crates/email_api_client/src/domain/service/messages.rs@@ -172,7 +172,8 @@ where/// Writes one message's labels. With `skip_missing`, a message the mailbox/// no longer has counts as written, as it does in a batch request.- async fn modify_one_message_labels(+ #[tracing::instrument(skip(self, labels_to_add, labels_to_remove), fields(gate_in_flight = tracing::field::Empty, gate_cap = tracing::field::Empty))]+ pub async fn modify_one_message_labels(&self,link_id: Uuid,provider_message_id: &str,@@ -211,7 +212,7 @@ where/// Gmail charges 5 quota units per `messages.modify` and 50 per/// `messages.batchModify` of any size, so below ten messages separate writes/// cost less.-const SEPARATE_LABEL_WRITES_BELOW: usize = 10;+pub const SEPARATE_LABEL_WRITES_BELOW: usize = 10;impl<R, T, L> EmailApiClientServiceImpl<R, T, L>wherediff --git a/crates/email_db_client/AGENTS.md b/crates/email_db_client/AGENTS.mdindex e651b3883b..523231314f 100644--- a/crates/email_db_client/AGENTS.md+++ b/crates/email_db_client/AGENTS.md@@ -53,6 +53,19 @@ Parsed message responses expose the draft generation captured in the same SQL sn- Forwarded-attachment delivery reads preserve `source_link_id` from the originalmessage; the destination draft inbox is only the query access scope.+- `insert_message_with_tx` leaves labels with an unexpired+ `email_gmail_pending_label_writes` marker, and the read and starred flags+ they drive, in their local state when it re-syncs a stored message. Given a+ `ResyncSnapshot` from before the Gmail fetch, it changes only labels where+ Gmail disagrees with that snapshot, so a write made during the fetch stays.+- `email_gmail_pending_label_writes` keeps one row per write on a message's+ label, newest by `seq` (staging order, which write ids need not follow).+ Only the newest row's write sends; older ones wait while a newer row hasn't+ expired, so abandoning a write can put the label back for the newest older+ write whose row hasn't expired (`label_present`) or the state before it+ (`label_present_before`, passed on by older writes given up). Code that+ stages, confirms or abandons a Gmail label write takes+ `lock_thread_label_writes` for its threads first.- Quote and alias camelCase columns to Rust field names; check`crates/macro_db_client/migrations/` for real column spelling, don't guess.diff --git a/crates/email_db_client/src/labels/get.rs b/crates/email_db_client/src/labels/get.rsindex 7d40386a0c..a828857827 100644--- a/crates/email_db_client/src/labels/get.rs+++ b/crates/email_db_client/src/labels/get.rs@@ -246,3 +246,35 @@ pub async fn fetch_label_by_id(Ok(service_label)}++/// Messages among `message_ids` that gain `provider_label_id` when `add` is+/// true, or lose it when false.+#[tracing::instrument(skip(executor, message_ids), err)]+pub async fn messages_changing_label<'e, E>(+ executor: E,+ link_id: Uuid,+ message_ids: &[Uuid],+ provider_label_id: &str,+ add: bool,+) -> Result<Vec<Uuid>, sqlx::Error>+where+ E: sqlx::Executor<'e, Database = sqlx::Postgres>,+{+ sqlx::query_scalar!(+ r#"+ SELECT m.id AS "id!"+ FROM unnest($1::uuid[]) AS m(id)+ WHERE EXISTS (+ SELECT 1 FROM email_effective_message_labels ml+ JOIN email_labels l ON l.id = ml.label_id+ WHERE ml.message_id = m.id AND l.link_id = $2 AND l.provider_label_id = $3+ ) <> $4+ "#,+ message_ids,+ link_id,+ provider_label_id,+ add,+ )+ .fetch_all(executor)+ .await+}diff --git a/crates/email_db_client/src/labels/get/test.rs b/crates/email_db_client/src/labels/get/test.rsindex e773d7935e..ca3f97e5c9 100644--- a/crates/email_db_client/src/labels/get/test.rs+++ b/crates/email_db_client/src/labels/get/test.rs@@ -1,4 +1,4 @@-use crate::labels::get::fetch_message_labels_in_bulk;+use crate::labels::get::{fetch_message_labels_in_bulk, messages_changing_label};use anyhow::Result;use macro_db_migrator::MACRO_DB_MIGRATIONS;use models_email::db::label::{LabelListVisibility, LabelType, MessageListVisibility};@@ -138,3 +138,27 @@ async fn fetch_message_labels_in_bulk_returns_empty_for_nonexistent_messages(Ok(())}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../fixtures", scripts("fetch_message_labels_in_bulk"))+)]+async fn messages_changing_label_skips_messages_already_in_the_requested_state(+ pool: Pool<Postgres>,+) -> Result<()> {+ let link_id = Uuid::parse_str("00000000-0000-0000-0000-000000000801")?;+ let in_inbox = Uuid::parse_str("00000000-0000-0000-0000-000000008501")?;+ let important = Uuid::parse_str("00000000-0000-0000-0000-000000008502")?;+ let unlabeled = Uuid::parse_str("00000000-0000-0000-0000-000000008503")?;+ let messages = [in_inbox, important, unlabeled];++ let mut gaining = messages_changing_label(&pool, link_id, &messages, "INBOX", true).await?;+ gaining.sort();+ assert_eq!(gaining, vec![important, unlabeled]);+ assert_eq!(+ messages_changing_label(&pool, link_id, &messages, "INBOX", false).await?,+ vec![in_inbox]+ );++ Ok(())+}diff --git a/crates/email_db_client/src/labels/insert.rs b/crates/email_db_client/src/labels/insert.rsindex 38c70c551f..851a7087c7 100644--- a/crates/email_db_client/src/labels/insert.rs+++ b/crates/email_db_client/src/labels/insert.rs@@ -258,15 +258,17 @@ pub async fn insert_label(Ok(populated_service_label)}-/// inserts the labels of an email into the database in a batch+/// inserts the labels of an email into the database in a batch. Labels in+/// `pending_provider_label_ids` keep their current state on the message.pub async fn insert_message_labels(tx: &mut sqlx::PgConnection,link_id: Uuid,message_id: Uuid,provider_label_ids: &[String],delete_old: bool,+ pending_provider_label_ids: &[String],) -> anyhow::Result<()> {- if provider_label_ids.is_empty() {+ if provider_label_ids.is_empty() && !delete_old {return Ok(());}@@ -308,7 +310,11 @@ pub async fn insert_message_labels();}- let label_db_ids: Vec<Uuid> = label_mappings.iter().map(|m| m.id).collect();+ let label_db_ids: Vec<Uuid> = label_mappings+ .iter()+ .filter(|m| !pending_provider_label_ids.contains(&m.provider_label_id))+ .map(|m| m.id)+ .collect();// with delete_old, records that don't match the ones to insert are removed in case we are// doing an upsert and some of the old ones got removed. An unchanged set deletes and inserts@@ -320,6 +326,10 @@ pub async fn insert_message_labels(WHERE $3::boolAND message_id = $1::uuidAND label_id NOT IN (SELECT UNNEST($2::uuid[]))+ AND label_id NOT IN (+ SELECT id FROM email_labels+ WHERE link_id = $4 AND provider_label_id = ANY($5::text[])+ ))INSERT INTO email_message_labels (message_id, label_id)SELECT $1, label_id FROM UNNEST($2::uuid[]) AS t(label_id)@@ -327,7 +337,9 @@ pub async fn insert_message_labels("#,message_id,&label_db_ids,- delete_old+ delete_old,+ link_id,+ pending_provider_label_ids).execute(&mut *tx).await?;diff --git a/crates/email_db_client/src/labels/mod.rs b/crates/email_db_client/src/labels/mod.rsindex e1dee670dd..88dae2d862 100644--- a/crates/email_db_client/src/labels/mod.rs+++ b/crates/email_db_client/src/labels/mod.rs@@ -1,3 +1,4 @@pub mod delete;pub mod get;pub mod insert;+pub mod pending_writes;diff --git a/crates/email_db_client/src/labels/pending_writes.rs b/crates/email_db_client/src/labels/pending_writes.rsnew file mode 100644index 0000000000..45eec752e8--- /dev/null+++ b/crates/email_db_client/src/labels/pending_writes.rs@@ -0,0 +1,353 @@+#[cfg(test)]+mod test;++use models_email::email::service::label::system_labels;+use sqlx::types::Uuid;+use sqlx::{Executor, PgConnection, Postgres};+use std::collections::HashMap;++/// Makes label writes that stage or undo a change on the same threads take+/// turns until the transaction ends, so each reads the state the one before+/// it left. Locks are taken in id order, so writers on several threads cannot+/// deadlock each other.+#[tracing::instrument(skip(tx, thread_ids), err)]+pub async fn lock_thread_label_writes(+ tx: &mut PgConnection,+ thread_ids: &[Uuid],+) -> Result<(), sqlx::Error> {+ let mut thread_ids = thread_ids.to_vec();+ thread_ids.sort_unstable();+ thread_ids.dedup();+ for thread_id in thread_ids {+ sqlx::query!(+ "SELECT pg_advisory_xact_lock(hashtextextended($1, 0))",+ format!("email_thread_label_write:{thread_id}"),+ )+ .execute(&mut *tx)+ .await?;+ }+ Ok(())+}++/// Records that the Gmail operations sharing `write_id` will set+/// `provider_label_id` on `message_ids` of the link to `label_present`, each a+/// change from the opposite state, so inbox sync keeps the local label state+/// until Gmail confirms it. Rows stop applying after 12 hours, longer than an+/// operation is redelivered through the gmail_ops queues, so only operations+/// that are dead-lettered or lost fall back to Gmail's labels.+#[tracing::instrument(skip(executor, message_ids), err)]+pub async fn mark_pending_label_writes<'e, E>(+ executor: E,+ link_id: Uuid,+ message_ids: &[Uuid],+ provider_label_id: &str,+ label_present: bool,+ write_id: Uuid,+) -> Result<(), sqlx::Error>+where+ E: Executor<'e, Database = Postgres>,+{+ if message_ids.is_empty() {+ return Ok(());+ }++ sqlx::query!(+ r#"+ INSERT INTO email_gmail_pending_label_writes+ (message_id, provider_label_id, write_id, label_present, label_present_before, expires_at)+ SELECT m.id, $3, $4, $5, NOT $5, now() + interval '12 hours'+ FROM email_messages m+ WHERE m.id = ANY($1) AND m.link_id = $2+ ORDER BY m.id+ ON CONFLICT (message_id, provider_label_id, write_id) DO NOTHING+ "#,+ message_ids,+ link_id,+ provider_label_id,+ write_id,+ label_present,+ )+ .execute(executor)+ .await?;++ Ok(())+}++/// What a write's operation does with one of its messages now.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum LabelWriteTurn {+ /// The write is the newest on every label: send it to Gmail.+ Send,+ /// A newer write on the message is still on its way. Its outcome decides+ /// whether this one still has to send, so try again later.+ Wait,+ /// A newer write was confirmed, or gave up on waiting for this one:+ /// nothing left to send.+ Superseded,+}++/// The turn of `write_id` on each of `message_ids` for `provider_label_ids`.+/// A write waits for every newer write whose row hasn't expired, whatever its+/// id: requests can stage in another order than they drew their ids. Bulk+/// writes can each be the newer one on a different message, so an operation+/// sends the messages it is newest on before it waits for the rest.+#[tracing::instrument(skip(executor, message_ids), err)]+pub async fn label_write_turns<'e, E>(+ executor: E,+ message_ids: &[Uuid],+ provider_label_ids: &[String],+ write_id: Uuid,+) -> Result<HashMap<Uuid, LabelWriteTurn>, sqlx::Error>+where+ E: Executor<'e, Database = Postgres>,+{+ let rows = sqlx::query!(+ r#"+ SELECT+ m.id AS "message_id!",+ mine.seq IS NOT NULL AS "claimed!",+ EXISTS (+ SELECT 1 FROM email_gmail_pending_label_writes newer+ WHERE newer.message_id = m.id AND newer.provider_label_id = l.label+ AND newer.seq > mine.seq+ ) AS "has_newer!",+ EXISTS (+ SELECT 1 FROM email_gmail_pending_label_writes newer+ WHERE newer.message_id = m.id AND newer.provider_label_id = l.label+ AND newer.seq > mine.seq AND newer.expires_at > now()+ ) AS "waits!"+ FROM unnest($1::uuid[]) AS m(id)+ CROSS JOIN unnest($3::text[]) AS l(label)+ LEFT JOIN email_gmail_pending_label_writes mine+ ON mine.message_id = m.id AND mine.provider_label_id = l.label AND mine.write_id = $2+ "#,+ message_ids,+ write_id,+ provider_label_ids,+ )+ .fetch_all(executor)+ .await?;++ let mut turns = HashMap::new();+ for row in rows {+ let turn = if !row.claimed {+ LabelWriteTurn::Superseded+ } else if row.waits {+ LabelWriteTurn::Wait+ } else if row.has_newer {+ LabelWriteTurn::Superseded+ } else {+ LabelWriteTurn::Send+ };+ turns+ .entry(row.message_id)+ .and_modify(|current: &mut LabelWriteTurn| {+ *current = match (*current, turn) {+ (LabelWriteTurn::Wait, _) | (_, LabelWriteTurn::Wait) => LabelWriteTurn::Wait,+ (LabelWriteTurn::Send, LabelWriteTurn::Send) => LabelWriteTurn::Send,+ _ => LabelWriteTurn::Superseded,+ }+ })+ .or_insert(turn);+ }+ Ok(turns)+}++/// Gmail applied `write_id` on `message_ids`: removes its rows and those of+/// every older write on the same labels, which Gmail no longer needs. A+/// newer write's row now falls back to this write's state. Callers hold+/// `lock_thread_label_writes` for the messages' threads.+#[tracing::instrument(skip(tx, message_ids), err)]+pub async fn confirm_pending_label_writes(+ tx: &mut PgConnection,+ message_ids: &[Uuid],+ write_id: Uuid,+) -> Result<(), sqlx::Error> {+ if message_ids.is_empty() {+ return Ok(());+ }++ sqlx::query!(+ r#"+ WITH mine AS (+ SELECT message_id, provider_label_id, seq, label_present+ FROM email_gmail_pending_label_writes+ WHERE message_id = ANY($1) AND write_id = $2+ ), next AS (+ SELECT DISTINCT ON (c.message_id, c.provider_label_id) c.seq, mine.label_present+ FROM email_gmail_pending_label_writes c+ JOIN mine ON c.message_id = mine.message_id+ AND c.provider_label_id = mine.provider_label_id AND c.seq > mine.seq+ ORDER BY c.message_id, c.provider_label_id, c.seq+ ), falls_back AS (+ UPDATE email_gmail_pending_label_writes c+ SET label_present_before = next.label_present+ FROM next+ WHERE c.seq = next.seq+ )+ DELETE FROM email_gmail_pending_label_writes c+ USING mine+ WHERE c.message_id = mine.message_id+ AND c.provider_label_id = mine.provider_label_id+ AND c.seq <= mine.seq+ "#,+ message_ids,+ write_id,+ )+ .execute(&mut *tx)+ .await?;++ Ok(())+}++/// Gives up `write_id`'s rows on `message_ids` because Gmail will never apply+/// them, and puts each label the write was the newest on back in the state+/// Gmail will end up with: that of the newest older write still on its way,+/// or the one before this write, which older writes given up pass on. An+/// older write whose row expired is presumed lost and no longer counts.+/// Labels a newer write changed keep that write's state. Returns the messages+/// whose labels were put back. Callers hold `lock_thread_label_writes` for+/// the messages' threads.+#[tracing::instrument(skip(tx, message_ids), err)]+pub async fn abandon_pending_label_writes(+ tx: &mut PgConnection,+ link_id: Uuid,+ message_ids: &[Uuid],+ write_id: Uuid,+) -> Result<Vec<Uuid>, sqlx::Error> {+ if message_ids.is_empty() {+ return Ok(Vec::new());+ }++ let restored = sqlx::query!(+ r#"+ WITH mine AS (+ SELECT c.message_id, c.provider_label_id, c.seq, c.label_present_before,+ (SELECT min(newer.seq) FROM email_gmail_pending_label_writes newer+ WHERE newer.message_id = c.message_id+ AND newer.provider_label_id = c.provider_label_id+ AND newer.seq > c.seq) AS newer_seq,+ (SELECT older.label_present FROM email_gmail_pending_label_writes older+ WHERE older.message_id = c.message_id+ AND older.provider_label_id = c.provider_label_id+ AND older.seq < c.seq AND older.expires_at > now()+ ORDER BY older.seq DESC+ LIMIT 1) AS older_present+ FROM email_gmail_pending_label_writes c+ WHERE c.message_id = ANY($1) AND c.write_id = $2+ ), inherited AS (+ UPDATE email_gmail_pending_label_writes c+ SET label_present_before = mine.label_present_before+ FROM mine+ WHERE c.seq = mine.newer_seq+ ), removed AS (+ DELETE FROM email_gmail_pending_label_writes c+ USING mine+ WHERE c.seq = mine.seq+ )+ SELECT+ message_id AS "message_id!",+ provider_label_id AS "provider_label_id!",+ COALESCE(older_present, label_present_before) AS "label_present!"+ FROM mine+ WHERE newer_seq IS NULL+ "#,+ message_ids,+ write_id,+ )+ .fetch_all(&mut *tx)+ .await?;+ if restored.is_empty() {+ return Ok(Vec::new());+ }++ let (present, absent): (Vec<_>, Vec<_>) =+ restored.iter().partition(|restore| restore.label_present);+ let present_ids: Vec<Uuid> = present.iter().map(|restore| restore.message_id).collect();+ let present_labels: Vec<String> = present+ .iter()+ .map(|restore| restore.provider_label_id.clone())+ .collect();+ let absent_ids: Vec<Uuid> = absent.iter().map(|restore| restore.message_id).collect();+ let absent_labels: Vec<String> = absent+ .iter()+ .map(|restore| restore.provider_label_id.clone())+ .collect();+ sqlx::query!(+ r#"+ INSERT INTO email_message_labels (message_id, label_id)+ SELECT r.message_id, l.id+ FROM unnest($1::uuid[], $2::text[]) AS r(message_id, provider_label_id)+ JOIN email_labels l ON l.link_id = $3 AND l.provider_label_id = r.provider_label_id+ ON CONFLICT (message_id, label_id) DO NOTHING+ "#,+ &present_ids,+ &present_labels,+ link_id,+ )+ .execute(&mut *tx)+ .await?;+ sqlx::query!(+ r#"+ DELETE FROM email_message_labels ml+ USING unnest($1::uuid[], $2::text[]) AS r(message_id, provider_label_id), email_labels l+ WHERE ml.message_id = r.message_id AND ml.label_id = l.id+ AND l.link_id = $3 AND l.provider_label_id = r.provider_label_id+ "#,+ &absent_ids,+ &absent_labels,+ link_id,+ )+ .execute(&mut *tx)+ .await?;++ let mut restored_ids: Vec<Uuid> = restored.iter().map(|restore| restore.message_id).collect();+ restored_ids.sort_unstable();+ restored_ids.dedup();+ sqlx::query!(+ r#"+ UPDATE email_messages m+ SET+ is_read = NOT EXISTS (+ SELECT 1 FROM email_message_labels ml JOIN email_labels l ON l.id = ml.label_id+ WHERE ml.message_id = m.id AND l.provider_label_id = $2+ ),+ is_starred = EXISTS (+ SELECT 1 FROM email_message_labels ml JOIN email_labels l ON l.id = ml.label_id+ WHERE ml.message_id = m.id AND l.provider_label_id = $3+ ),+ updated_at = now()+ WHERE m.id = ANY($1)+ "#,+ &restored_ids,+ system_labels::UNREAD,+ system_labels::STARRED,+ )+ .execute(&mut *tx)+ .await?;++ Ok(restored_ids)+}++/// Provider label ids on the message with a local change Gmail has not+/// confirmed yet.+#[tracing::instrument(skip(executor), err)]+pub async fn pending_provider_label_ids<'e, E>(+ executor: E,+ message_id: Uuid,+) -> Result<Vec<String>, sqlx::Error>+where+ E: Executor<'e, Database = Postgres>,+{+ sqlx::query_scalar!(+ r#"+ SELECT DISTINCT provider_label_id AS "provider_label_id!"+ FROM email_gmail_pending_label_writes+ WHERE message_id = $1 AND expires_at > now()+ ORDER BY provider_label_id+ "#,+ message_id,+ )+ .fetch_all(executor)+ .await+}diff --git a/crates/email_db_client/src/labels/pending_writes/test.rs b/crates/email_db_client/src/labels/pending_writes/test.rsnew file mode 100644index 0000000000..f4e3291f44--- /dev/null+++ b/crates/email_db_client/src/labels/pending_writes/test.rs@@ -0,0 +1,366 @@+use super::*;+use anyhow::Result;+use macro_db_migrator::MACRO_DB_MIGRATIONS;+use sqlx::{Pool, Postgres};++const LINK: Uuid = Uuid::from_u128(0x801);+const MESSAGE: Uuid = Uuid::from_u128(0x8501);+const INBOX_LABEL: Uuid = Uuid::from_u128(0x18001);++/// Stages a local INBOX change on `MESSAGE` the way a thread write does: the+/// label and the write's row change together.+async fn stage_inbox(pool: &Pool<Postgres>, present: bool, write_id: Uuid) -> Result<()> {+ let mut tx = pool.begin().await?;+ if present {+ sqlx::query!(+ "INSERT INTO email_message_labels (message_id, label_id) VALUES ($1, $2)",+ MESSAGE,+ INBOX_LABEL+ )+ .execute(&mut *tx)+ .await?;+ } else {+ sqlx::query!(+ "DELETE FROM email_message_labels WHERE message_id = $1 AND label_id = $2",+ MESSAGE,+ INBOX_LABEL+ )+ .execute(&mut *tx)+ .await?;+ }+ mark_pending_label_writes(&mut *tx, LINK, &[MESSAGE], "INBOX", present, write_id).await?;+ tx.commit().await?;+ Ok(())+}++async fn in_inbox(pool: &Pool<Postgres>) -> Result<bool> {+ Ok(sqlx::query_scalar!(+ r#"SELECT EXISTS (SELECT 1 FROM email_message_labels WHERE message_id = $1 AND label_id = $2) AS "exists!""#,+ MESSAGE,+ INBOX_LABEL+ )+ .fetch_one(pool)+ .await?)+}++async fn turn(pool: &Pool<Postgres>, write_id: Uuid) -> Result<LabelWriteTurn> {+ Ok(label_write_turns(pool, &[MESSAGE], &["INBOX".to_string()], write_id).await?[&MESSAGE])+}++async fn abandon(pool: &Pool<Postgres>, write_id: Uuid) -> Result<Vec<Uuid>> {+ let mut tx = pool.begin().await?;+ let restored = abandon_pending_label_writes(&mut tx, LINK, &[MESSAGE], write_id).await?;+ tx.commit().await?;+ Ok(restored)+}++async fn confirm(pool: &Pool<Postgres>, write_id: Uuid) -> Result<()> {+ let mut tx = pool.begin().await?;+ confirm_pending_label_writes(&mut tx, &[MESSAGE], write_id).await?;+ tx.commit().await?;+ Ok(())+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../fixtures", scripts("fetch_message_labels_in_bulk"))+)]+async fn marks_only_messages_of_the_link(pool: Pool<Postgres>) -> Result<()> {+ let other_link = Uuid::from_u128(0x802);+ let second = Uuid::from_u128(0x8502);+ let write_id = Uuid::from_u128(0xaa01);++ mark_pending_label_writes(&pool, LINK, &[MESSAGE], "INBOX", false, write_id).await?;+ mark_pending_label_writes(&pool, other_link, &[second], "INBOX", false, write_id).await?;++ assert_eq!(pending_provider_label_ids(&pool, MESSAGE).await?, ["INBOX"]);+ assert!(pending_provider_label_ids(&pool, second).await?.is_empty());+ Ok(())+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../fixtures", scripts("fetch_message_labels_in_bulk"))+)]+async fn an_older_write_waits_until_a_newer_one_is_confirmed(pool: Pool<Postgres>) -> Result<()> {+ let archive = Uuid::from_u128(0xaa01);+ let unarchive = Uuid::from_u128(0xaa02);++ stage_inbox(&pool, false, archive).await?;+ stage_inbox(&pool, true, unarchive).await?;+ assert_eq!(turn(&pool, archive).await?, LabelWriteTurn::Wait);+ assert_eq!(turn(&pool, unarchive).await?, LabelWriteTurn::Send);++ confirm(&pool, unarchive).await?;+ assert_eq!(turn(&pool, archive).await?, LabelWriteTurn::Superseded);+ assert!(pending_provider_label_ids(&pool, MESSAGE).await?.is_empty());+ assert!(in_inbox(&pool).await?);+ Ok(())+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../fixtures", scripts("fetch_message_labels_in_bulk"))+)]+async fn an_abandoned_write_puts_the_label_back_for_the_older_write_still_waiting(+ pool: Pool<Postgres>,+) -> Result<()> {+ let archive = Uuid::from_u128(0xaa01);+ let unarchive = Uuid::from_u128(0xaa02);++ stage_inbox(&pool, false, archive).await?;+ stage_inbox(&pool, true, unarchive).await?;+ assert_eq!(abandon(&pool, unarchive).await?, [MESSAGE]);++ assert!(!in_inbox(&pool).await?);+ assert_eq!(turn(&pool, archive).await?, LabelWriteTurn::Send);+ assert_eq!(pending_provider_label_ids(&pool, MESSAGE).await?, ["INBOX"]);+ Ok(())+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../fixtures", scripts("fetch_message_labels_in_bulk"))+)]+async fn abandoned_writes_keep_every_older_write_they_took_over(+ pool: Pool<Postgres>,+) -> Result<()> {+ let archive = Uuid::from_u128(0xaa01);+ let unarchive = Uuid::from_u128(0xaa02);+ let archive_again = Uuid::from_u128(0xaa03);++ stage_inbox(&pool, false, archive).await?;+ stage_inbox(&pool, true, unarchive).await?;+ stage_inbox(&pool, false, archive_again).await?;++ // The second archive can't be queued, then Gmail rejects the unarchive.+ abandon(&pool, archive_again).await?;+ assert!(in_inbox(&pool).await?);+ abandon(&pool, unarchive).await?;++ assert!(!in_inbox(&pool).await?);+ assert_eq!(turn(&pool, archive).await?, LabelWriteTurn::Send);+ Ok(())+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../fixtures", scripts("fetch_message_labels_in_bulk"))+)]+async fn when_every_write_is_abandoned_the_label_returns_to_its_state_before_the_first(+ pool: Pool<Postgres>,+) -> Result<()> {+ let archive = Uuid::from_u128(0xaa01);+ let unarchive = Uuid::from_u128(0xaa02);++ stage_inbox(&pool, false, archive).await?;+ stage_inbox(&pool, true, unarchive).await?;++ // Gmail rejects the archive while the unarchive waits: nothing changes+ // locally, but the unarchive now starts from the original state.+ assert!(abandon(&pool, archive).await?.is_empty());+ assert!(in_inbox(&pool).await?);+ assert_eq!(turn(&pool, unarchive).await?, LabelWriteTurn::Send);++ let archive_again = Uuid::from_u128(0xaa03);+ stage_inbox(&pool, false, archive_again).await?;+ abandon(&pool, archive_again).await?;+ abandon(&pool, unarchive).await?;+ assert!(in_inbox(&pool).await?);+ assert!(pending_provider_label_ids(&pool, MESSAGE).await?.is_empty());+ Ok(())+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../fixtures", scripts("fetch_message_labels_in_bulk"))+)]+async fn a_confirmed_write_is_what_a_newer_abandoned_write_falls_back_to(+ pool: Pool<Postgres>,+) -> Result<()> {+ let archive = Uuid::from_u128(0xaa01);+ let unarchive = Uuid::from_u128(0xaa02);+ let archive_again = Uuid::from_u128(0xaa03);++ stage_inbox(&pool, false, archive).await?;+ stage_inbox(&pool, true, unarchive).await?;+ stage_inbox(&pool, false, archive_again).await?;+ // The unarchive reached Gmail before the second archive was staged.+ confirm(&pool, unarchive).await?;+ assert_eq!(turn(&pool, archive).await?, LabelWriteTurn::Superseded);++ abandon(&pool, archive_again).await?;+ assert!(in_inbox(&pool).await?);+ assert!(pending_provider_label_ids(&pool, MESSAGE).await?.is_empty());+ Ok(())+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../fixtures", scripts("fetch_message_labels_in_bulk"))+)]+async fn a_write_waits_for_live_newer_writes_whatever_their_ids(+ pool: Pool<Postgres>,+) -> Result<()> {+ let unarchive = Uuid::from_u128(0xaa01);+ let archive = Uuid::from_u128(0xaa02);++ // Staged in the opposite order to their ids, as when two requests race.+ stage_inbox(&pool, false, archive).await?;+ stage_inbox(&pool, true, unarchive).await?;+ assert_eq!(turn(&pool, archive).await?, LabelWriteTurn::Wait);+ assert_eq!(turn(&pool, unarchive).await?, LabelWriteTurn::Send);++ // Gmail rejects the unarchive, or it can't be queued: the archive is the+ // user's last change again, and it sends.+ assert_eq!(abandon(&pool, unarchive).await?, [MESSAGE]);+ assert!(!in_inbox(&pool).await?);+ assert_eq!(turn(&pool, archive).await?, LabelWriteTurn::Send);++ let unarchive_again = Uuid::from_u128(0xaa03);+ stage_inbox(&pool, true, unarchive_again).await?;+ assert_eq!(turn(&pool, archive).await?, LabelWriteTurn::Wait);+ sqlx::query!(+ "UPDATE email_gmail_pending_label_writes SET expires_at = now() - interval '1 second' WHERE write_id = $1",+ unarchive_again+ )+ .execute(&pool)+ .await?;+ assert_eq!(turn(&pool, archive).await?, LabelWriteTurn::Superseded);+ Ok(())+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../fixtures", scripts("fetch_message_labels_in_bulk"))+)]+async fn writes_newer_than_each_other_on_two_messages_each_have_one_to_send(+ pool: Pool<Postgres>,+) -> Result<()> {+ let second = Uuid::from_u128(0x8502);+ let first_write = Uuid::from_u128(0xaa01);+ let second_write = Uuid::from_u128(0xaa02);+ let labels = ["INBOX".to_string()];++ // Two bulk writes reach the messages' threads in opposite orders.+ mark_pending_label_writes(&pool, LINK, &[MESSAGE], "INBOX", false, first_write).await?;+ mark_pending_label_writes(&pool, LINK, &[MESSAGE], "INBOX", true, second_write).await?;+ mark_pending_label_writes(&pool, LINK, &[second], "INBOX", true, second_write).await?;+ mark_pending_label_writes(&pool, LINK, &[second], "INBOX", false, first_write).await?;+ let first_turns = label_write_turns(&pool, &[MESSAGE, second], &labels, first_write).await?;+ assert_eq!(first_turns[&MESSAGE], LabelWriteTurn::Wait);+ assert_eq!(first_turns[&second], LabelWriteTurn::Send);+ let second_turns = label_write_turns(&pool, &[MESSAGE, second], &labels, second_write).await?;+ assert_eq!(second_turns[&MESSAGE], LabelWriteTurn::Send);+ assert_eq!(second_turns[&second], LabelWriteTurn::Wait);++ // Each write sends the message it is newest on, which settles the other's+ // wait.+ let mut tx = pool.begin().await?;+ confirm_pending_label_writes(&mut tx, &[second], first_write).await?;+ confirm_pending_label_writes(&mut tx, &[MESSAGE], second_write).await?;+ tx.commit().await?;+ for write_id in [first_write, second_write] {+ let turns = label_write_turns(&pool, &[MESSAGE, second], &labels, write_id).await?;+ assert!(+ turns+ .values()+ .all(|turn| *turn == LabelWriteTurn::Superseded)+ );+ }+ assert!(pending_provider_label_ids(&pool, MESSAGE).await?.is_empty());+ assert!(pending_provider_label_ids(&pool, second).await?.is_empty());+ Ok(())+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../fixtures", scripts("fetch_message_labels_in_bulk"))+)]+async fn an_abandoned_write_does_not_bring_back_an_expired_older_write(+ pool: Pool<Postgres>,+) -> Result<()> {+ let archive = Uuid::from_u128(0xaa01);+ let archive_again = Uuid::from_u128(0xaa02);++ stage_inbox(&pool, false, archive).await?;+ sqlx::query!(+ "UPDATE email_gmail_pending_label_writes SET expires_at = now() - interval '1 second' WHERE write_id = $1",+ archive+ )+ .execute(&pool)+ .await?;+ // With the marker expired, inbox sync puts Gmail's INBOX back.+ sqlx::query!(+ "INSERT INTO email_message_labels (message_id, label_id) VALUES ($1, $2)",+ MESSAGE,+ INBOX_LABEL+ )+ .execute(&pool)+ .await?;++ stage_inbox(&pool, false, archive_again).await?;+ assert_eq!(abandon(&pool, archive_again).await?, [MESSAGE]);+ assert!(in_inbox(&pool).await?);+ Ok(())+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../fixtures", scripts("fetch_message_labels_in_bulk"))+)]+async fn writes_abandoned_after_an_older_write_expired_return_to_the_state_before_them(+ pool: Pool<Postgres>,+) -> Result<()> {+ let archive = Uuid::from_u128(0xaa01);+ let archive_again = Uuid::from_u128(0xaa02);+ let unarchive = Uuid::from_u128(0xaa03);++ stage_inbox(&pool, false, archive).await?;+ sqlx::query!(+ "UPDATE email_gmail_pending_label_writes SET expires_at = now() - interval '1 second' WHERE write_id = $1",+ archive+ )+ .execute(&pool)+ .await?;+ sqlx::query!(+ "INSERT INTO email_message_labels (message_id, label_id) VALUES ($1, $2)",+ MESSAGE,+ INBOX_LABEL+ )+ .execute(&pool)+ .await?;+ stage_inbox(&pool, false, archive_again).await?;+ stage_inbox(&pool, true, unarchive).await?;++ // The second archive gives up first and passes the state before it on to+ // the unarchive, which keeps the label meanwhile.+ assert!(abandon(&pool, archive_again).await?.is_empty());+ assert!(in_inbox(&pool).await?);+ assert_eq!(abandon(&pool, unarchive).await?, [MESSAGE]);+ assert!(in_inbox(&pool).await?);+ assert!(pending_provider_label_ids(&pool, MESSAGE).await?.is_empty());+ Ok(())+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../fixtures", scripts("fetch_message_labels_in_bulk"))+)]+async fn expired_changes_no_longer_apply(pool: Pool<Postgres>) -> Result<()> {+ let write_id = Uuid::from_u128(0xaa01);++ mark_pending_label_writes(&pool, LINK, &[MESSAGE], "TRASH", true, write_id).await?;+ mark_pending_label_writes(&pool, LINK, &[MESSAGE], "UNREAD", true, write_id).await?;+ sqlx::query!(+ "UPDATE email_gmail_pending_label_writes SET expires_at = now() - interval '1 second' WHERE provider_label_id = 'TRASH'"+ )+ .execute(&pool)+ .await?;++ assert_eq!(+ pending_provider_label_ids(&pool, MESSAGE).await?,+ ["UNREAD"]+ );+ Ok(())+}diff --git a/crates/email_db_client/src/messages/get.rs b/crates/email_db_client/src/messages/get.rsindex 9dc226983f..f91aabb66c 100644--- a/crates/email_db_client/src/messages/get.rs+++ b/crates/email_db_client/src/messages/get.rs@@ -14,6 +14,30 @@ use models_email::service::message::{MessageSenderInfo, MessageToSend};use sqlx::PgPool;use sqlx::types::Uuid;+/// The threads of the link's messages among `message_ids`, each once.+#[tracing::instrument(skip(executor, message_ids), err)]+pub async fn thread_ids_of_messages<'e, E>(+ executor: E,+ link_id: Uuid,+ message_ids: &[Uuid],+) -> Result<Vec<Uuid>, sqlx::Error>+where+ E: sqlx::Executor<'e, Database = sqlx::Postgres>,+{+ sqlx::query_scalar!(+ r#"+ SELECT DISTINCT thread_id+ FROM email_messages+ WHERE id = ANY($1) AND link_id = $2+ ORDER BY thread_id+ "#,+ message_ids,+ link_id,+ )+ .fetch_all(executor)+ .await+}+/// Returns a map of message IDs to sender info for all found messages#[tracing::instrument(skip(pool), err)]pub async fn get_message_sender_and_pretty_sender(diff --git a/crates/email_db_client/src/messages/insert.rs b/crates/email_db_client/src/messages/insert.rsindex af1c67cf39..f80b875063 100644--- a/crates/email_db_client/src/messages/insert.rs+++ b/crates/email_db_client/src/messages/insert.rs@@ -4,13 +4,23 @@ pub(crate) mod test;use crate::attachments::provider;use crate::messages::replying_to_id;use crate::parse::service_to_db::addresses_from_message;-use crate::{contacts, labels, parse, threads};+use crate::{contacts, labels, messages, parse, threads};use models_email::email::db::address::UpsertedRecipients;+use models_email::email::service::label::system_labels;use models_email::email::service::message;use sqlx::PgPool;use sqlx::types::Uuid;+/// What a re-sync of a stored message read before fetching it from Gmail.+#[derive(Debug, Clone, Default)]+pub struct ResyncSnapshot {+ /// The message's provider labels before the fetch.+ pub stored_labels: Vec<String>,+ /// Labels with a pending Macro write just before or just after the fetch.+ pub pending_labels: Vec<String>,+}+/// inserts a message (and all its related parts) into the database using the passed transaction.pub async fn insert_message_with_tx(tx: &mut sqlx::PgConnection,@@ -24,10 +34,18 @@ pub async fn insert_message_with_tx(// 1. inserting thread simultaneously (thread already has latest values)// 2. backfilling messages (metadata gets updated once after all messages complete)update_thread_metadata: bool,+ // Set when re-syncing a stored message: Gmail's answer may predate a local+ // write made while it was fetched, so labels change only where Gmail+ // disagrees with what was stored before the fetch.+ resync: Option<&ResyncSnapshot>,) -> anyhow::Result<()> {if let Some(provider_thread_id) = message.provider_thread_id.as_deref() {threads::provider_identity::lock_provider_thread(tx, link_id, provider_thread_id).await?;}+ if resync.is_some() {+ // Before the row upsert, in the same order as label writes take it.+ labels::pending_writes::lock_thread_label_writes(tx, &[thread_db_id]).await?;+ }let message_db_id = insert_db_message(tx,message,@@ -39,20 +57,57 @@ pub async fn insert_message_with_tx(contacts::upsert_message::upsert_message_recipients(tx, message_db_id, &recipents).await?;- if !message.labels.is_empty() {- let provider_label_ids: Vec<String> = message- .labels- .iter()- .map(|label| label.provider_label_id.clone())- .collect();- labels::insert::insert_message_labels(- tx,- link_id,+ // Also when Gmail reports no labels, so a re-sync removes the stored ones.+ let provider_label_ids: Vec<String> = message+ .labels+ .iter()+ .map(|label| label.provider_label_id.clone())+ .collect();++ // A re-sync must not undo a Macro label write Gmail has not confirmed yet,+ // including the read and starred flags those labels drive.+ let mut kept_label_ids =+ labels::pending_writes::pending_provider_label_ids(&mut *tx, message_db_id).await?;+ if let Some(snapshot) = resync {+ kept_label_ids.extend(snapshot.pending_labels.iter().cloned());+ let stored_now = sqlx::query_scalar!(+ r#"+ SELECT l.provider_label_id+ FROM email_message_labels ml+ JOIN email_labels l ON l.id = ml.label_id+ WHERE ml.message_id = $1+ "#,message_db_id,- &provider_label_ids,- true,)+ .fetch_all(&mut *tx).await?;+ kept_label_ids.extend(+ stored_now+ .into_iter()+ .chain(provider_label_ids.iter().cloned())+ .chain(snapshot.stored_labels.iter().cloned())+ .filter(|label| {+ provider_label_ids.contains(label) == snapshot.stored_labels.contains(label)+ }),+ );+ }+ kept_label_ids.sort_unstable();+ kept_label_ids.dedup();++ labels::insert::insert_message_labels(+ tx,+ link_id,+ message_db_id,+ &provider_label_ids,+ true,+ &kept_label_ids,+ )+ .await?;++ let kept_read = kept_label_ids.iter().any(|id| id == system_labels::UNREAD);+ let kept_starred = kept_label_ids.iter().any(|id| id == system_labels::STARRED);+ if kept_read || kept_starred {+ messages::update::set_flags_from_labels(tx, message_db_id, kept_read, kept_starred).await?;}provider::insert_attachments(tx, message_db_id, &mut message.attachments).await?;@@ -265,6 +320,7 @@ impl<U: Fn(&anyhow::Error) -> bool> BackfillMessageWriter<'_, U> {self.link_id,recipients,false,+ None,).await{@@ -314,6 +370,7 @@ pub async fn insert_message(message: &mut message::Message,link_id: Uuid,update_thread_metadata: bool,+ resync: Option<&ResyncSnapshot>,) -> anyhow::Result<()> {// we have to insert addresses before inserting the message. these values are shared// across messages, so inserting them in the txn can cause deadlocks.@@ -332,6 +389,7 @@ pub async fn insert_message(link_id,recipients,update_thread_metadata,+ resync,).await{diff --git a/crates/email_db_client/src/messages/insert/test.rs b/crates/email_db_client/src/messages/insert/test.rsindex 0303183d6c..92f22f2ee4 100644--- a/crates/email_db_client/src/messages/insert/test.rs+++ b/crates/email_db_client/src/messages/insert/test.rs@@ -96,7 +96,7 @@ async fn oversized_provider_values_do_not_block_the_message_insert(let mut message = message_with_oversized_fields(link_id, thread_db_id);let message_db_id = message.db_id;- insert_message(&pool, thread_db_id, &mut message, link_id, false).await?;+ insert_message(&pool, thread_db_id, &mut message, link_id, false, None).await?;let inserted = sqlx::query!(r#"SELECT from_name FROM email_messages WHERE id = $1"#,@@ -383,7 +383,7 @@ async fn resyncing_an_unchanged_message_writes_nothing(pool: Pool<Postgres>) ->body: "parent body",}.build(link_id, thread_db_id);- insert_message(&pool, thread_db_id, &mut parent, link_id, true).await?;+ insert_message(&pool, thread_db_id, &mut parent, link_id, true, None).await?;let body = large_body();let reply = SyncedMessage {@@ -397,7 +397,7 @@ async fn resyncing_an_unchanged_message_writes_nothing(pool: Pool<Postgres>) ->};let mut first_sync = reply.build(link_id, thread_db_id);let reply_id = first_sync.db_id;- insert_message(&pool, thread_db_id, &mut first_sync, link_id, true).await?;+ insert_message(&pool, thread_db_id, &mut first_sync, link_id, true, None).await?;let replying_to = sqlx::query_scalar!(r#"SELECT replying_to_id FROM email_messages WHERE id = $1"#,@@ -413,7 +413,7 @@ async fn resyncing_an_unchanged_message_writes_nothing(pool: Pool<Postgres>) ->// The full ingest path: message upsert, recipients, labels and the replying_to update.let mut resync = reply.build(link_id, thread_db_id);- insert_message(&pool, thread_db_id, &mut resync, link_id, true).await?;+ insert_message(&pool, thread_db_id, &mut resync, link_id, true, None).await?;assert_eq!(row_versions(&pool, reply_id).await?,@@ -469,7 +469,7 @@ async fn changed_flag_updates_the_row_without_rewriting_body_toast(};let mut first_sync = synced.build(link_id, thread_db_id);let message_id = first_sync.db_id;- insert_message(&pool, thread_db_id, &mut first_sync, link_id, false).await?;+ insert_message(&pool, thread_db_id, &mut first_sync, link_id, false, None).await?;let from_contact_id = from_contact_id(&pool, message_id).await?;let mut tx = pool.begin().await?;@@ -538,7 +538,7 @@ async fn resync_with_changed_labels_and_recipients_replaces_only_the_changes(};let mut first_sync = synced.build(link_id, thread_db_id);let message_id = first_sync.db_id;- insert_message(&pool, thread_db_id, &mut first_sync, link_id, false).await?;+ insert_message(&pool, thread_db_id, &mut first_sync, link_id, false, None).await?;let before = row_versions(&pool, message_id).await?;let mut resync = SyncedMessage {@@ -547,7 +547,7 @@ async fn resync_with_changed_labels_and_recipients_replaces_only_the_changes(..synced}.build(link_id, thread_db_id);- insert_message(&pool, thread_db_id, &mut resync, link_id, false).await?;+ insert_message(&pool, thread_db_id, &mut resync, link_id, false, None).await?;let after = row_versions(&pool, message_id).await?;assert_eq!(after.message, before.message);@@ -572,3 +572,240 @@ async fn resync_with_changed_labels_and_recipients_replaces_only_the_changes(Ok(())}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../fixtures", scripts("links", "threads"))+)]+async fn resync_without_gmail_labels_removes_stored_labels_except_pending_ones(+ pool: Pool<Postgres>,+) -> anyhow::Result<()> {+ let link_id = Uuid::parse_str(LINK_ID)?;+ let thread_db_id = Uuid::parse_str(THREAD_ID)?;+ seed_labels(&pool, link_id).await?;+ let synced = SyncedMessage {+ provider_id: "archived-in-gmail",+ global_id: "<[email protected]>",+ in_reply_to: None,+ to: &["[email protected]"],+ cc: &[],+ labels: &["INBOX"],+ body: "body",+ };+ let mut first_sync = synced.build(link_id, thread_db_id);+ first_sync.is_read = true;+ let message_id = first_sync.db_id;+ insert_message(&pool, thread_db_id, &mut first_sync, link_id, true, None).await?;++ // Marked unread in Macro, not yet in Gmail.+ let mut tx = pool.begin().await?;+ labels::insert::insert_message_labels_batch(&mut *tx, &vec![message_id], "UNREAD", link_id)+ .await?;+ messages::update::update_message_read_status(&mut tx, message_id, link_id, false).await?;+ tx.commit().await?;+ labels::pending_writes::mark_pending_label_writes(+ &pool,+ link_id,+ &[message_id],+ "UNREAD",+ true,+ Uuid::now_v7(),+ )+ .await?;++ // Archived and read in Gmail: it reports no labels at all.+ let mut resync = SyncedMessage {+ labels: &[],+ ..synced+ }+ .build(link_id, thread_db_id);+ resync.is_read = true;+ insert_message(&pool, thread_db_id, &mut resync, link_id, true, None).await?;++ let labels: Vec<String> = row_versions(&pool, message_id)+ .await?+ .labels+ .into_iter()+ .map(|(label, _)| label)+ .collect();+ assert_eq!(labels, ["UNREAD"]);+ let is_read = sqlx::query_scalar!(+ "SELECT is_read FROM email_messages WHERE id = $1",+ message_id+ )+ .fetch_one(&pool)+ .await?;+ assert!(!is_read);+ Ok(())+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../fixtures", scripts("links", "threads"))+)]+async fn resync_keeps_labels_with_a_pending_gmail_write(+ pool: Pool<Postgres>,+) -> anyhow::Result<()> {+ let link_id = Uuid::parse_str(LINK_ID)?;+ let thread_db_id = Uuid::parse_str(THREAD_ID)?;+ seed_labels(&pool, link_id).await?;+ let synced = SyncedMessage {+ provider_id: "archived",+ global_id: "<[email protected]>",+ in_reply_to: None,+ to: &["[email protected]"],+ cc: &[],+ labels: &["INBOX"],+ body: "body",+ };+ let mut first_sync = synced.build(link_id, thread_db_id);+ first_sync.is_read = true;+ let message_id = first_sync.db_id;+ insert_message(&pool, thread_db_id, &mut first_sync, link_id, true, None).await?;++ // Archived and marked unread in Macro; Gmail has not applied either yet.+ let write_id = Uuid::now_v7();+ let mut tx = pool.begin().await?;+ labels::delete::delete_message_labels_batch(&mut *tx, &vec![message_id], "INBOX", link_id)+ .await?;+ labels::insert::insert_message_labels_batch(&mut *tx, &vec![message_id], "UNREAD", link_id)+ .await?;+ messages::update::update_message_read_status(&mut tx, message_id, link_id, false).await?;+ tx.commit().await?;+ for (label, present) in [("INBOX", false), ("UNREAD", true)] {+ labels::pending_writes::mark_pending_label_writes(+ &pool,+ link_id,+ &[message_id],+ label,+ present,+ write_id,+ )+ .await?;+ }++ let mut stale_resync = synced.build(link_id, thread_db_id);+ stale_resync.is_read = true;+ insert_message(&pool, thread_db_id, &mut stale_resync, link_id, true, None).await?;++ let labels_after = |versions: RowVersions| -> Vec<String> {+ versions+ .labels+ .into_iter()+ .map(|(label, _)| label)+ .collect()+ };+ assert_eq!(+ labels_after(row_versions(&pool, message_id).await?),+ ["UNREAD"]+ );+ let stored = sqlx::query!(+ r#"+ SELECT m.is_read, t.inbox_visible+ FROM email_messages m JOIN email_threads t ON t.id = m.thread_id+ WHERE m.id = $1+ "#,+ message_id+ )+ .fetch_one(&pool)+ .await?;+ assert!(!stored.is_read);+ assert!(!stored.inbox_visible);++ // Once the writes finish, a re-sync follows Gmail again.+ let mut tx = pool.begin().await?;+ labels::pending_writes::confirm_pending_label_writes(&mut tx, &[message_id], write_id).await?;+ tx.commit().await?;+ let mut resync = synced.build(link_id, thread_db_id);+ resync.is_read = true;+ insert_message(&pool, thread_db_id, &mut resync, link_id, true, None).await?;++ assert_eq!(+ labels_after(row_versions(&pool, message_id).await?),+ ["INBOX"]+ );+ let stored = sqlx::query!(+ r#"+ SELECT m.is_read, t.inbox_visible+ FROM email_messages m JOIN email_threads t ON t.id = m.thread_id+ WHERE m.id = $1+ "#,+ message_id+ )+ .fetch_one(&pool)+ .await?;+ assert!(stored.is_read);+ assert!(stored.inbox_visible);++ Ok(())+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../fixtures", scripts("links", "threads"))+)]+async fn resync_keeps_a_write_made_and_confirmed_while_gmail_was_fetched(+ pool: Pool<Postgres>,+) -> anyhow::Result<()> {+ let link_id = Uuid::parse_str(LINK_ID)?;+ let thread_db_id = Uuid::parse_str(THREAD_ID)?;+ seed_labels(&pool, link_id).await?;+ let synced = SyncedMessage {+ provider_id: "fetched-before-the-write",+ global_id: "<[email protected]>",+ in_reply_to: None,+ to: &["[email protected]"],+ cc: &[],+ labels: &["INBOX"],+ body: "body",+ };+ let mut first_sync = synced.build(link_id, thread_db_id);+ first_sync.is_read = true;+ let message_id = first_sync.db_id;+ insert_message(&pool, thread_db_id, &mut first_sync, link_id, true, None).await?;+ let snapshot = ResyncSnapshot {+ stored_labels: vec!["INBOX".to_string()],+ pending_labels: vec![],+ };++ // While the re-sync's fetch is in flight, the user marks the message+ // unread and Gmail confirms it, so no marker is left.+ let mut tx = pool.begin().await?;+ labels::insert::insert_message_labels_batch(&mut *tx, &vec![message_id], "UNREAD", link_id)+ .await?;+ messages::update::update_message_read_status(&mut tx, message_id, link_id, false).await?;+ tx.commit().await?;++ // Gmail answered before the write, after the user archived it in Gmail.+ let mut resync = SyncedMessage {+ labels: &[],+ ..synced+ }+ .build(link_id, thread_db_id);+ resync.is_read = true;+ insert_message(+ &pool,+ thread_db_id,+ &mut resync,+ link_id,+ true,+ Some(&snapshot),+ )+ .await?;++ let labels: Vec<String> = row_versions(&pool, message_id)+ .await?+ .labels+ .into_iter()+ .map(|(label, _)| label)+ .collect();+ assert_eq!(labels, ["UNREAD"]);+ let is_read = sqlx::query_scalar!(+ "SELECT is_read FROM email_messages WHERE id = $1",+ message_id+ )+ .fetch_one(&pool)+ .await?;+ assert!(!is_read);+ Ok(())+}diff --git a/crates/email_db_client/src/messages/update.rs b/crates/email_db_client/src/messages/update.rsindex a2ff5d41b3..68c8e0bb93 100644--- a/crates/email_db_client/src/messages/update.rs+++ b/crates/email_db_client/src/messages/update.rs@@ -1,8 +1,52 @@+use models_email::email::service::label::system_labels;use sqlx::types::Uuid;#[cfg(test)]mod test;+/// Sets a message's read flag from its UNREAD label when `read` is set, and its+/// starred flag from its STARRED label when `starred` is set. A message whose+/// flags already match is not written, as a re-sync usually changes nothing.+#[tracing::instrument(skip(tx), err)]+pub async fn set_flags_from_labels(+ tx: &mut sqlx::PgConnection,+ message_id: Uuid,+ read: bool,+ starred: bool,+) -> anyhow::Result<()> {+ sqlx::query!(+ r#"+ WITH flags AS (+ SELECT+ CASE WHEN $2 THEN NOT EXISTS (+ SELECT 1 FROM email_message_labels ml JOIN email_labels l ON l.id = ml.label_id+ WHERE ml.message_id = m.id AND l.provider_label_id = $4+ ) ELSE m.is_read END AS is_read,+ CASE WHEN $3 THEN EXISTS (+ SELECT 1 FROM email_message_labels ml JOIN email_labels l ON l.id = ml.label_id+ WHERE ml.message_id = m.id AND l.provider_label_id = $5+ ) ELSE m.is_starred END AS is_starred+ FROM email_messages m+ WHERE m.id = $1+ )+ UPDATE email_messages m+ SET is_read = flags.is_read, is_starred = flags.is_starred+ FROM flags+ WHERE m.id = $1+ AND (m.is_read, m.is_starred) IS DISTINCT FROM (flags.is_read, flags.is_starred)+ "#,+ message_id,+ read,+ starred,+ system_labels::UNREAD,+ system_labels::STARRED,+ )+ .execute(tx)+ .await?;++ Ok(())+}+/// Update the read status of a single message, scoped to the inbox that owns it.#[tracing::instrument(skip(tx), err)]pub async fn update_message_read_status(diff --git a/crates/email_db_client/src/messages/update/test.rs b/crates/email_db_client/src/messages/update/test.rsindex 33e4951f29..b75a6dc2aa 100644--- a/crates/email_db_client/src/messages/update/test.rs+++ b/crates/email_db_client/src/messages/update/test.rs@@ -1,4 +1,4 @@-use super::mark_message_as_sent;+use super::{mark_message_as_sent, set_flags_from_labels};use crate::messages::get::fetch_messages_metadata;use crate::threads::get::get_thread_by_id_and_link_id;use crate::threads::update::update_thread_metadata;@@ -134,3 +134,49 @@ async fn failed_finalization_rolls_back_sent_recency(pool: PgPool) -> anyhow::Reassert!(thread.latest_outbound_message_ts.is_none());Ok(())}++async fn row_version(pool: &PgPool) -> anyhow::Result<String> {+ Ok(sqlx::query_scalar!(+ r#"SELECT xmin::text AS "xmin!" FROM email_messages WHERE id = $1"#,+ MESSAGE+ )+ .fetch_one(pool)+ .await?)+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../fixtures", scripts("mark_message_as_sent"))+)]+async fn flags_that_already_match_their_labels_are_not_rewritten(+ pool: PgPool,+) -> anyhow::Result<()> {+ // The message has neither UNREAD nor STARRED: it is read and not starred.+ sqlx::query!(+ "UPDATE email_messages SET is_read = true, is_starred = false WHERE id = $1",+ MESSAGE+ )+ .execute(&pool)+ .await?;+ let before = row_version(&pool).await?;++ let mut tx = pool.begin().await?;+ set_flags_from_labels(tx.as_mut(), MESSAGE, true, true).await?;+ tx.commit().await?;+ assert_eq!(row_version(&pool).await?, before);++ sqlx::query!(+ "UPDATE email_messages SET is_read = false WHERE id = $1",+ MESSAGE+ )+ .execute(&pool)+ .await?;+ let mut tx = pool.begin().await?;+ set_flags_from_labels(tx.as_mut(), MESSAGE, true, false).await?;+ tx.commit().await?;+ let is_read = sqlx::query_scalar!("SELECT is_read FROM email_messages WHERE id = $1", MESSAGE)+ .fetch_one(&pool)+ .await?;+ assert!(is_read);+ Ok(())+}diff --git a/crates/email_db_client/src/threads/insert.rs b/crates/email_db_client/src/threads/insert.rsindex 632d8ebbe2..5a2e9f910e 100644--- a/crates/email_db_client/src/threads/insert.rs+++ b/crates/email_db_client/src/threads/insert.rs@@ -58,6 +58,7 @@ pub async fn insert_thread_and_messages(link_id,recipient_map.remove(provider_id).unwrap(),false,+ None,).await?;}@@ -69,10 +70,11 @@ pub async fn insert_thread_and_messages(update_thread_messages_replying_to(&mut tx, thread_id, link_id).await?;}- // Messages were inserted with per-message metadata updates disabled and- // insert_thread precomputes the other thread columns, so is_signal- // must be synced here or new threads would stay noise.- super::update::sync_thread_signal_flag(&mut tx, thread_id).await?;+ // Messages were inserted with per-message metadata updates disabled, and+ // insert_thread precomputes the thread columns from Gmail's labels only+ // for a new thread. Recompute them, and is_signal, from the stored+ // labels, which keep pending Macro writes and newly synced messages.+ super::update::update_thread_metadata(&mut tx, thread_id, link_id).await?;Ok(thread_id)}diff --git a/crates/email_db_client/src/threads/insert/test.rs b/crates/email_db_client/src/threads/insert/test.rsindex ea7fc45e44..0e8dce71de 100644--- a/crates/email_db_client/src/threads/insert/test.rs+++ b/crates/email_db_client/src/threads/insert/test.rs@@ -514,3 +514,104 @@ async fn a_message_that_cannot_be_stored_is_skipped_and_the_rest_of_its_thread_iassert_eq!(thread_message_count(&pool, thread_id).await?, 1);Ok(())}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../fixtures", scripts("links"))+)]+async fn whole_thread_resync_recomputes_thread_state_from_stored_labels(+ pool: Pool<Postgres>,+) -> anyhow::Result<()> {+ use crate::messages::insert::test::SyncedMessage;++ let link_id = Uuid::parse_str("00000000-0000-0000-0000-000000000001")?;+ sqlx::query!(+ "INSERT INTO email_labels (id, link_id, provider_label_id, name) VALUES ($1, $2, 'INBOX', 'INBOX')",+ Uuid::now_v7(),+ link_id+ )+ .execute(&pool)+ .await?;+ let archived = SyncedMessage {+ provider_id: "pending-archive",+ global_id: "<[email protected]>",+ in_reply_to: None,+ to: &["[email protected]"],+ cc: &[],+ labels: &["INBOX"],+ body: "archived body",+ };+ let reply = SyncedMessage {+ provider_id: "new-reply",+ global_id: "<[email protected]>",+ in_reply_to: None,+ to: &["[email protected]"],+ cc: &[],+ labels: &["INBOX"],+ body: "reply body",+ };+ let gmail_thread = |messages: &[&SyncedMessage]| {+ let thread_db_id = Uuid::now_v7();+ let ts =+ chrono::DateTime::<Utc>::from_timestamp(1_790_000_000, 0).expect("valid timestamp");+ thread::Thread {+ db_id: thread_db_id,+ provider_id: Some("pending-archive-thread".to_string()),+ link_id,+ inbox_visible: true,+ is_read: false,+ latest_inbound_message_ts: Some(ts),+ latest_outbound_message_ts: None,+ latest_non_spam_message_ts: Some(ts),+ created_at: ts,+ updated_at: ts,+ messages: messages+ .iter()+ .map(|message| message.build(link_id, thread_db_id))+ .collect(),+ }+ };+ let first_sync = gmail_thread(&[&archived]);+ let message_id = first_sync.messages[0].db_id;+ let thread_id = insert_thread_and_messages(&pool, first_sync, link_id).await?;++ // Archived in Macro; Gmail has not applied it yet.+ let mut tx = pool.begin().await?;+ crate::labels::delete::delete_message_labels_batch(+ &mut *tx,+ &vec![message_id],+ "INBOX",+ link_id,+ )+ .await?;+ sqlx::query!(+ "UPDATE email_threads SET inbox_visible = false WHERE id = $1",+ thread_id+ )+ .execute(&mut *tx)+ .await?;+ tx.commit().await?;+ crate::labels::pending_writes::mark_pending_label_writes(+ &pool,+ link_id,+ &[message_id],+ "INBOX",+ false,+ Uuid::now_v7(),+ )+ .await?;+ let inbox_visible = || async {+ get_thread_by_id_and_link_id(&pool, thread_id, link_id)+ .await+ .map(|thread| thread.expect("thread should exist").inbox_visible)+ };++ insert_thread_and_messages(&pool, gmail_thread(&[&archived]), link_id).await?;+ assert!(!inbox_visible().await?);++ // A new reply lands in the inbox, so the thread shows again.+ insert_thread_and_messages(&pool, gmail_thread(&[&archived, &reply]), link_id).await?;+ assert!(inbox_visible().await?);++ Ok(())+}diff --git a/crates/macro_db_client/migrations/20261024154458_gmail_pending_label_writes.sql b/crates/macro_db_client/migrations/20261024154458_gmail_pending_label_writes.sqlnew file mode 100644index 0000000000..9d89066dd5--- /dev/null+++ b/crates/macro_db_client/migrations/20261024154458_gmail_pending_label_writes.sql@@ -0,0 +1,25 @@+SET LOCAL lock_timeout = '1s';++-- Gmail label changes Macro applied locally but Gmail has not confirmed yet,+-- one row per write on a message's label, newest by seq. Inbox sync keeps the+-- local label state while an unexpired row exists, so a rate-limited write+-- waiting in the gmail_ops retry queue cannot be undone by sync.+-- Only the newest row's write sends that label to Gmail; an older write waits+-- until the newer one is confirmed (its rows go) or abandoned (its row goes,+-- and the label returns to the state the next older row or the first write+-- left it in). A confirmed write removes its row and every older one. Rows of+-- operations that never finish stop applying at expires_at, and go with the+-- message.+CREATE TABLE IF NOT EXISTS email_gmail_pending_label_writes (+ seq bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,+ message_id uuid NOT NULL REFERENCES email_messages(id) ON DELETE CASCADE,+ provider_label_id text NOT NULL,+ write_id uuid NOT NULL,+ -- Whether the write leaves the label on the message.+ label_present boolean NOT NULL,+ -- The label's state before the write, as Gmail will have it if this write+ -- and every newer one are abandoned.+ label_present_before boolean NOT NULL,+ expires_at timestamptz NOT NULL,+ UNIQUE (message_id, provider_label_id, write_id)+);diff --git a/crates/models_email/src/email/service/pubsub.rs b/crates/models_email/src/email/service/pubsub.rsindex 56f55cf803..aa74199ac7 100644--- a/crates/models_email/src/email/service/pubsub.rs+++ b/crates/models_email/src/email/service/pubsub.rs@@ -43,6 +43,10 @@ pub enum FailureReason {CalendarBackfillBusy,GmailApiFailed,GmailApiRateLimited,+ /// Gmail failed with a server or network error.+ GmailApiUnavailable,+ /// A newer label write on the same messages has not finished yet.+ NewerLabelWritePending,/// Every fleet-wide backfill write slot stayed taken; the operation is deferred.BackfillWritesBusy,OutdatedHistoryId,diff --git a/crates/models_email/src/gmail/gmail_ops.rs b/crates/models_email/src/gmail/gmail_ops.rsindex 30f1cced40..08a1b050f4 100644--- a/crates/models_email/src/gmail/gmail_ops.rs+++ b/crates/models_email/src/gmail/gmail_ops.rs@@ -41,6 +41,10 @@ pub struct ModifyMessageLabelsPayload {pub provider_message_id: String,pub labels_to_add: Vec<String>,pub labels_to_remove: Vec<String>,+ /// Pending label write marker this operation clears once Gmail accepts or+ /// rejects it. Absent on operations enqueued before markers existed.+ #[serde(default)]+ pub pending_write_id: Option<Uuid>,}#[derive(Debug, Deserialize, Serialize, Clone)]@@ -51,6 +55,10 @@ pub struct ModifyMessageLabelsBatchPayload {pub messages: Vec<GmailMessageRef>,pub labels_to_add: Vec<String>,pub labels_to_remove: Vec<String>,+ /// Pending label write marker this batch clears on its messages once Gmail+ /// accepts or rejects it. Absent on batches enqueued before markers existed.+ #[serde(default)]+ pub pending_write_id: Option<Uuid>,}/// The database and Gmail ids of one message.@@ -70,6 +78,7 @@ impl From<ModifyMessageLabelsPayload> for ModifyMessageLabelsBatchPayload {}],labels_to_add: payload.labels_to_add,labels_to_remove: payload.labels_to_remove,+ pending_write_id: payload.pending_write_id,}}}@@ -80,6 +89,7 @@ impl ModifyMessageLabelsBatchPayload {messages: Vec<GmailMessageRef>,labels_to_add: Vec<String>,labels_to_remove: Vec<String>,+ pending_write_id: Uuid,) -> Vec<Self> {messages.chunks(MAX_BATCH_MODIFY_MESSAGES)@@ -87,6 +97,7 @@ impl ModifyMessageLabelsBatchPayload {messages: chunk.to_vec(),labels_to_add: labels_to_add.clone(),labels_to_remove: labels_to_remove.clone(),+ pending_write_id: Some(pending_write_id),}).collect()}@@ -95,18 +106,25 @@ impl ModifyMessageLabelsBatchPayload {/// The gmail_ops operations that carry one label write. Workers released/// before `ModifyMessageLabelsBatch` existed delete it as undecodable, so/// `batches` may be set only once every worker decodes it; otherwise each-/// message goes out as its own `ModifyMessageLabels`.+/// message goes out as its own `ModifyMessageLabels`. Every operation carries+/// `pending_write_id`, so each clears the write's markers on its own messages.pub fn label_write_operations(messages: Vec<GmailMessageRef>,labels_to_add: Vec<String>,labels_to_remove: Vec<String>,+ pending_write_id: Uuid,batches: bool,) -> Vec<GmailOpsOperation> {if batches {- return ModifyMessageLabelsBatchPayload::chunked(messages, labels_to_add, labels_to_remove)- .into_iter()- .map(GmailOpsOperation::ModifyMessageLabelsBatch)- .collect();+ return ModifyMessageLabelsBatchPayload::chunked(+ messages,+ labels_to_add,+ labels_to_remove,+ pending_write_id,+ )+ .into_iter()+ .map(GmailOpsOperation::ModifyMessageLabelsBatch)+ .collect();}messages.into_iter()@@ -116,6 +134,7 @@ pub fn label_write_operations(provider_message_id: message.provider_message_id,labels_to_add: labels_to_add.clone(),labels_to_remove: labels_to_remove.clone(),+ pending_write_id: Some(pending_write_id),})}).collect()diff --git a/crates/models_email/src/gmail/gmail_ops/test.rs b/crates/models_email/src/gmail/gmail_ops/test.rsindex f9b5eab1a8..032e6c187c 100644--- a/crates/models_email/src/gmail/gmail_ops/test.rs+++ b/crates/models_email/src/gmail/gmail_ops/test.rs@@ -9,8 +9,13 @@ fn chunked_label_writes_fit_one_gmail_request_each() {}).collect();- let payloads =- ModifyMessageLabelsBatchPayload::chunked(messages.clone(), vec![], vec!["INBOX".into()]);+ let write_id = Uuid::from_u128(7);+ let payloads = ModifyMessageLabelsBatchPayload::chunked(+ messages.clone(),+ vec![],+ vec!["INBOX".into()],+ write_id,+ );assert_eq!(payloads@@ -23,7 +28,8 @@ fn chunked_label_writes_fit_one_gmail_request_each() {payloads.iter().all(|payload| payload.labels_to_add.is_empty()- && payload.labels_to_remove == vec!["INBOX".to_string()])+ && payload.labels_to_remove == vec!["INBOX".to_string()]+ && payload.pending_write_id == Some(write_id)));assert_eq!(payloads@@ -37,6 +43,63 @@ fn chunked_label_writes_fit_one_gmail_request_each() {#[test]fn chunked_label_writes_without_messages_enqueue_nothing() {assert!(- ModifyMessageLabelsBatchPayload::chunked(vec![], vec!["TRASH".into()], vec![]).is_empty()+ ModifyMessageLabelsBatchPayload::chunked(+ vec![],+ vec!["TRASH".into()],+ vec![],+ Uuid::from_u128(7)+ )+ .is_empty()+ );+}++#[test]+fn single_message_operations_keep_their_pending_write_as_a_batch() {+ let queued: GmailOpsPubsubMessage = serde_json::from_str(+ r#"{+ "linkId": "00000000-0000-0000-0000-000000000001",+ "operation": {"modify_message_labels": {+ "dbMessageId": "00000000-0000-0000-0000-000000000002",+ "providerMessageId": "message-2",+ "labelsToAdd": [],+ "labelsToRemove": ["INBOX"],+ "pendingWriteId": "00000000-0000-0000-0000-000000000007"+ }}+ }"#,+ )+ .unwrap();+ let GmailOpsOperation::ModifyMessageLabels(payload) = queued.operation else {+ panic!("expected a single-message label write");+ };++ let batch = ModifyMessageLabelsBatchPayload::from(payload);++ assert_eq!(batch.pending_write_id, Some(Uuid::from_u128(7)));+ assert_eq!(+ batch.messages,+ vec![GmailMessageRef {+ db_message_id: Uuid::from_u128(2),+ provider_message_id: "message-2".to_string(),+ }]);}++#[test]+fn label_writes_queued_before_markers_still_decode() {+ let queued: GmailOpsPubsubMessage = serde_json::from_str(+ r#"{+ "linkId": "00000000-0000-0000-0000-000000000001",+ "operation": {"modify_message_labels_batch": {+ "messages": [{"dbMessageId": "00000000-0000-0000-0000-000000000002", "providerMessageId": "message-2"}],+ "labelsToAdd": ["TRASH"],+ "labelsToRemove": []+ }}+ }"#,+ )+ .unwrap();+ let GmailOpsOperation::ModifyMessageLabelsBatch(batch) = queued.operation else {+ panic!("expected a batched label write");+ };++ assert_eq!(batch.pending_write_id, None);+}diff --git a/crates/sqs_client/src/email.rs b/crates/sqs_client/src/email.rsindex 29d406d188..69afa38e27 100644--- a/crates/sqs_client/src/email.rs+++ b/crates/sqs_client/src/email.rs@@ -44,6 +44,7 @@ impl SQS {messages: Vec<(Uuid, String)>,labels_to_add: Vec<String>,labels_to_remove: Vec<String>,+ pending_write_id: Uuid,) -> Vec<models_email::gmail::gmail_ops::GmailOpsPubsubMessage> {use models_email::gmail::gmail_ops::{GmailMessageRef, GmailOpsPubsubMessage, label_write_operations,@@ -60,6 +61,7 @@ impl SQS {messages,labels_to_add,labels_to_remove,+ pending_write_id,self.gmail_label_write_batches,).into_iter()@@ -232,9 +234,15 @@ impl EmailMessageEnqueuer for SQS {messages: Vec<(Uuid, String)>,labels_to_add: Vec<String>,labels_to_remove: Vec<String>,+ pending_write_id: Uuid,) -> Result<(), Self::Err> {- let messages =- self.gmail_label_write_messages(link_id, messages, labels_to_add, labels_to_remove);+ let messages = self.gmail_label_write_messages(+ link_id,+ messages,+ labels_to_add,+ labels_to_remove,+ pending_write_id,+ );if !self.gmail_label_write_batches {return self.enqueue_gmail_ops_notifications_batch(messages).await;}diff --git a/crates/sqs_client/src/email/test.rs b/crates/sqs_client/src/email/test.rsindex 62bbd4ec62..b8d75a653a 100644--- a/crates/sqs_client/src/email/test.rs+++ b/crates/sqs_client/src/email/test.rs@@ -62,6 +62,8 @@ fn decode_with_released_worker(serde_json::from_str(&serde_json::to_string(message).unwrap())}+const WRITE: Uuid = Uuid::from_u128(0xb1);+fn label_write(count: u128) -> Vec<(Uuid, String)> {(0..count).map(|index| (Uuid::from_u128(index), format!("message-{index}")))@@ -76,6 +78,7 @@ fn released_workers_decode_everything_a_default_producer_sends() {label_write(1001),vec![],vec!["INBOX".to_string()],+ WRITE,);messages.extend([GmailOpsPubsubMessage {@@ -143,6 +146,7 @@ fn label_write_batches_are_sent_only_when_enabled() {label_write(1001),vec!["TRASH".to_string()],vec![],+ WRITE,);assert_eq!(@@ -163,3 +167,29 @@ fn label_write_batches_are_sent_only_when_enabled() {.all(|message| decode_with_released_worker(message).is_err()));}++#[test]+fn every_label_write_operation_carries_the_write_id_with_or_without_batches() {+ for batches in [false, true] {+ let messages = client()+ .gmail_label_write_batches(batches)+ .gmail_label_write_messages(+ Uuid::from_u128(0xa1),+ label_write(1001),+ vec![],+ vec!["INBOX".to_string()],+ WRITE,+ );++ let write_ids: Vec<_> = messages+ .iter()+ .map(|message| match &message.operation {+ GmailOpsOperation::ModifyMessageLabels(payload) => payload.pending_write_id,+ GmailOpsOperation::ModifyMessageLabelsBatch(payload) => payload.pending_write_id,+ other => panic!("expected a label write, got {other}"),+ })+ .collect();+ assert_eq!(write_ids.len(), if batches { 2 } else { 1001 });+ assert!(write_ids.iter().all(|write_id| *write_id == Some(WRITE)));+ }+}diff --git a/services/email_service/AGENTS.md b/services/email_service/AGENTS.mdindex f4caa9b05d..53171ac9f0 100644--- a/services/email_service/AGENTS.md+++ b/services/email_service/AGENTS.md@@ -42,7 +42,8 @@ the gateway `/email` prefix.## Checks-- `cargo test -p email_service` (`SQLX_OFFLINE` unset).+- `cargo test -p email_service` (`SQLX_OFFLINE` unset; the `label_write_sim`+ tests also need Redis, at `REDIS_URI` or `redis://127.0.0.1:6379`).- `just check` (from this directory) runs `cargo check` with `SQLX_OFFLINE=true`.- Local SQS: `just create-localstack` / `create-queues` before running`pubsub_workers` locally; queue names are in the `justfile`.@@ -107,18 +108,48 @@ the gateway `/email` prefix.so a stalled write can't lose its slot to another writer. Live inboxsync writes take no slot. Keep new bulk backfill writes behind the same slots.- `gmail_ops` workers decode both label write forms: single-message- `ModifyMessageLabels` and `ModifyMessageLabelsBatch` (up to 1,000 messages;- one `batchModify` from ten messages up, separate `messages.modify` calls- below that, at 5 units each against a flat 50; a permanent failure reverts- the whole batch). Producers send the batch form only with- `GMAIL_LABEL_WRITE_BATCHES_ENABLED` (here and in `document_storage_service`,- default off), because workers released before it delete batches as- undecodable. Rollout: deploy with the switch off; once every- `pubsub_workers` task runs that release, set it to `true` in the+ `ModifyMessageLabels` and `ModifyMessageLabelsBatch` (up to 1,000 messages).+ From ten messages up a batch is one `batchModify`, which a permanent failure+ reverts as a whole. Below that, gmail_ops sends one `messages.modify` per+ message (5 units each against a flat 50), checks the message's turn just+ before each, and settles each message as Gmail answers: a permanent+ rejection reverts only the messages Gmail hasn't accepted, and a resumed+ older write skips messages a newer write took over. Producers send the+ batch form only with `GMAIL_LABEL_WRITE_BATCHES_ENABLED` (here and in+ `document_storage_service`, default off), because workers released before+ it delete batches as undecodable. Rollout: deploy with the switch off; once+ every `pubsub_workers` task runs that release, set it to `true` in the`email-service` and `document-storage` Doppler projects (dev, then prd) andredeploy both services. Before rolling workers back past that release, turnit off and let the gmail-ops queues drain. Retire the single-message formonly after every producer sends batches.+- Gmail label writes leave a marker in `email_gmail_pending_label_writes`.+ Inbox sync keeps the local state of a label marked before or after it reads+ Gmail (`inbox_sync::operations::pending_labels`); a re-sync or recovery+ backfill of a stored message also keeps labels Gmail's answer agrees with+ the pre-fetch snapshot on. gmail_ops sends only messages its write is the+ newest on, and sends them before an operation with a newer write still on+ its way (whatever its id) waits in the retry queue+ (`NewerLabelWritePending`), so bulk writes that are each newer on one of the+ other's messages don't wait on each other. A waiting message sends if the+ newer write is rejected. A permanent failure gives up the write's unaccepted+ and waiting messages and puts back what the newest older write still on its+ way, or the state before the write, says; an expired marker no longer+ counts. Both label write forms carry the write's `pendingWriteId`; with+ batches off, each message's operation settles only that message's markers.+ Markers expire after 12 hours, which must outlast an operation's redelivery+ through the gmail_ops queues.+- `pubsub::label_write_sim` wires one mailbox to the production label-write,+ gmail_ops and inbox sync code with fake Gmail and SQS. Add a test there for a+ new race between writes and syncs; its ignored `randomized_shard_*` tests run+ the seeded simulation (`SIM_SEEDS`, `SIM_OVERLAP`, `SIM_BATCHES`, see+ `sim/test.rs`). Scenarios run in both label write forms (`scenario!`).+- Local stacks run debug workers on tokio's 2 MiB worker stacks, where every+ awaited future takes its own stack slot. Pass a Gmail call through a helper+ as a `BoxFuture`, and box an operation the dispatch awaits in several arms;+ a `label_write_sim` test runs the label paths on 1.5 MiB worker stacks.+- gmail_ops moves transient Gmail failures to the retry queue like rate limits;+ the primary queue dead-letters after three receives.- Header addresses that Gmail accepts can fail `MacroUserIdStr::try_from_email`.Skip the bad address with a WARN that omits it (the parse error echoes it)instead of failing the step; see `pubsub::util::build_contact_connections`.diff --git a/services/email_service/fixtures/gmail_ops_label_revert.sql b/services/email_service/fixtures/gmail_ops_label_revert.sqlnew file mode 100644index 0000000000..3775daf342--- /dev/null+++ b/services/email_service/fixtures/gmail_ops_label_revert.sql@@ -0,0 +1,22 @@+-- One Gmail inbox whose three messages were archived in Macro by write+-- ...00f1, which Gmail has not confirmed yet.+INSERT INTO email_links (id, macro_id, fusionauth_user_id, email_address, provider, is_sync_active, created_at, updated_at)+VALUES ('00000000-0000-0000-0000-0000000000a1', 'macro|[email protected]', 'revert-user', '[email protected]', 'GMAIL', true, NOW(), NOW());++INSERT INTO email_threads (id, provider_id, link_id, inbox_visible, is_read, created_at, updated_at)+VALUES ('00000000-0000-0000-0000-0000000000b1', 'thread-1', '00000000-0000-0000-0000-0000000000a1', false, true, NOW(), NOW());++INSERT INTO email_labels (id, link_id, provider_label_id, name, message_list_visibility, label_list_visibility, type, created_at)+VALUES ('00000000-0000-0000-0000-0000000000c1', '00000000-0000-0000-0000-0000000000a1', 'INBOX', 'INBOX', 'Show', 'LabelShow', 'System', NOW());++INSERT INTO email_messages (id, thread_id, link_id, provider_id, is_read, is_starred, is_sent, is_draft, has_attachments, internal_date_ts, created_at, updated_at)+VALUES+ ('00000000-0000-0000-0000-00000000000a', '00000000-0000-0000-0000-0000000000b1', '00000000-0000-0000-0000-0000000000a1', 'msg-a', true, false, false, false, false, '2026-10-01T10:00:00Z', NOW(), NOW()),+ ('00000000-0000-0000-0000-00000000000b', '00000000-0000-0000-0000-0000000000b1', '00000000-0000-0000-0000-0000000000a1', 'msg-b', true, false, false, false, false, '2026-10-01T11:00:00Z', NOW(), NOW()),+ ('00000000-0000-0000-0000-00000000000c', '00000000-0000-0000-0000-0000000000b1', '00000000-0000-0000-0000-0000000000a1', 'msg-c', true, false, false, false, false, '2026-10-01T12:00:00Z', NOW(), NOW());++INSERT INTO email_gmail_pending_label_writes (message_id, provider_label_id, write_id, label_present, label_present_before, expires_at)+VALUES+ ('00000000-0000-0000-0000-00000000000a', 'INBOX', '00000000-0000-0000-0000-0000000000f1', false, true, NOW() + interval '12 hours'),+ ('00000000-0000-0000-0000-00000000000b', 'INBOX', '00000000-0000-0000-0000-0000000000f1', false, true, NOW() + interval '12 hours'),+ ('00000000-0000-0000-0000-00000000000c', 'INBOX', '00000000-0000-0000-0000-0000000000f1', false, true, NOW() + interval '12 hours');diff --git a/services/email_service/src/api/email/messages/labels.rs b/services/email_service/src/api/email/messages/labels.rsindex c26108a71a..3d2080a516 100644--- a/services/email_service/src/api/email/messages/labels.rs+++ b/services/email_service/src/api/email/messages/labels.rs@@ -14,6 +14,9 @@ use models_email::service;use sqlx::types::Uuid;use utoipa::ToSchema;+#[cfg(test)]+mod test;+#[derive(serde::Serialize, serde::Deserialize, Debug, ToSchema)]pub struct UpdateLabelBatchRequest {pub message_ids: Vec<Uuid>,@@ -146,6 +149,10 @@ pub async fn handler(}let message_db_ids: Vec<Uuid> = db_messages.iter().map(|m| m.db_id).collect();+ let mut thread_ids: Vec<Uuid> = db_messages.iter().map(|m| m.thread_db_id).collect();+ thread_ids.sort_unstable();+ thread_ids.dedup();+ let pending_write_id = macro_uuid::generate_uuid_v7();let provider_label_id = label.provider_label_id.clone();let is_adding = body.value;@@ -161,55 +168,97 @@ pub async fn handler(.into_response()})?;- let transaction_result = async {- if !message_db_ids.is_empty() {- if is_adding {- email_db_client::labels::insert::insert_message_labels_batch(- &mut *tx,- &message_db_ids,- &provider_label_id,- link.id,- )- .await- .context("Failed to add label to messages in database")?;- } else {- email_db_client::labels::delete::delete_message_labels_batch(- &mut *tx,- &message_db_ids,- &provider_label_id,- link.id,- )- .await- .context("Failed to remove label from messages in database")?;- }+ // The changed messages Gmail knows about; drafts have no provider_id.+ let transaction_result: anyhow::Result<Vec<(Uuid, String)>> = async {+ if message_db_ids.is_empty() {+ return Ok(Vec::new());+ }+ email_db_client::labels::pending_writes::lock_thread_label_writes(&mut tx, &thread_ids)+ .await+ .context("Failed to lock the messages' threads")?;+ let changed_ids = email_db_client::labels::get::messages_changing_label(+ &mut *tx,+ link.id,+ &message_db_ids,+ &provider_label_id,+ is_adding,+ )+ .await+ .context("Failed to read which messages change")?;+ let provider_messages: Vec<(Uuid, String)> = db_messages+ .iter()+ .filter(|msg| changed_ids.contains(&msg.db_id))+ .filter_map(|msg| {+ msg.provider_id+ .as_ref()+ .filter(|pid| !pid.is_empty())+ .map(|pid| (msg.db_id, pid.clone()))+ })+ .collect();+ let provider_message_ids: Vec<Uuid> = provider_messages.iter().map(|(id, _)| *id).collect();+ email_db_client::labels::pending_writes::mark_pending_label_writes(+ &mut *tx,+ link.id,+ &provider_message_ids,+ &provider_label_id,+ is_adding,+ pending_write_id,+ )+ .await+ .context("Failed to mark pending label writes")?;- if provider_label_id.as_str() == service::label::system_labels::UNREAD {- email_db_client::messages::update::update_message_read_status_batch(- &mut *tx,- message_db_ids.clone(),- link.id,- !is_adding,- )- .await- .context("Failed to update message read status")?;- } else if provider_label_id.as_str() == service::label::system_labels::STARRED {- email_db_client::messages::update::update_message_starred_status_batch(- &mut *tx,- message_db_ids.clone(),- link.id,- is_adding,- )+ if is_adding {+ email_db_client::labels::insert::insert_message_labels_batch(+ &mut *tx,+ &message_db_ids,+ &provider_label_id,+ link.id,+ )+ .await+ .context("Failed to add label to messages in database")?;+ } else {+ email_db_client::labels::delete::delete_message_labels_batch(+ &mut *tx,+ &message_db_ids,+ &provider_label_id,+ link.id,+ )+ .await+ .context("Failed to remove label from messages in database")?;+ }++ if provider_label_id.as_str() == service::label::system_labels::UNREAD {+ email_db_client::messages::update::update_message_read_status_batch(+ &mut *tx,+ message_db_ids.clone(),+ link.id,+ !is_adding,+ )+ .await+ .context("Failed to update message read status")?;+ } else if provider_label_id.as_str() == service::label::system_labels::STARRED {+ email_db_client::messages::update::update_message_starred_status_batch(+ &mut *tx,+ message_db_ids.clone(),+ link.id,+ is_adding,+ )+ .await+ .context("Failed to update message starred status")?;+ }++ for thread_id in &thread_ids {+ email_db_client::threads::update::update_thread_metadata(&mut tx, *thread_id, link.id).await- .context("Failed to update message starred status")?;- }+ .context("Failed to update thread state")?;}- anyhow::Ok(())+ Ok(provider_messages)}.await;- match transaction_result {- Ok(_) => {+ let provider_messages = match transaction_result {+ Ok(provider_messages) => {tx.commit().await.map_err(|e| {tracing::error!(error=?e, "unable to commit transaction");(@@ -220,6 +269,7 @@ pub async fn handler().into_response()})?;+ provider_messages}Err(e) => {tracing::error!(error=?e, "Transaction failed, rolling back");@@ -234,20 +284,17 @@ pub async fn handler().into_response());}- }+ };// Publish semantic macro.email events, rolled up to one per affected// thread (the batch may span several threads in the same inbox).let actor = Some(link.macro_id.clone());- let mut event_thread_ids: Vec<Uuid> = db_messages.iter().map(|m| m.thread_db_id).collect();- event_thread_ids.sort_unstable();- event_thread_ids.dedup();- for thread_id in event_thread_ids {+ for thread_id in &thread_ids {let event = EmailMacroEvent::thread_label_change(link.id,link.macro_id.clone(),actor.clone(),- thread_id,+ *thread_id,LabelRef {label_id: label.id,provider_label_id: provider_label_id.clone(),@@ -261,43 +308,104 @@ pub async fn handler(}}- // One label write for the whole batch (drafts have no provider_id, skip them)+ // One label write for the whole batchlet (labels_to_add, labels_to_remove) = if is_adding {(vec![provider_label_id.clone()], Vec::new())} else {(Vec::new(), vec![provider_label_id.clone()])};+ let provider_message_ids: Vec<Uuid> = provider_messages.iter().map(|(id, _)| *id).collect();- let provider_messages: Vec<(Uuid, String)> = db_messages- .iter()- .filter_map(|msg| {- msg.provider_id- .as_ref()- .filter(|pid| !pid.is_empty())- .map(|pid| (msg.db_id, pid.clone()))- })- .collect();-- ctx.sqs_client- .enqueue_gmail_ops_modify_labels_batch(+ let mut failed_ids = Vec::new();+ if !provider_messages.is_empty()+ && ctx+ .sqs_client+ .enqueue_gmail_ops_modify_labels_batch(+ link.id,+ provider_messages,+ labels_to_add,+ labels_to_remove,+ pending_write_id,+ )+ .await+ .inspect_err(|e| {+ tracing::error!(error=?e, "Failed to enqueue gmail ops label write");+ })+ .is_err()+ {+ // Gmail never learns about this change.+ failed_ids = undo_unsent_write(+ &ctx.db,link.id,- provider_messages,- labels_to_add,- labels_to_remove,+ &thread_ids,+ &provider_message_ids,+ pending_write_id,)- .await- .inspect_err(|e| {- tracing::error!(error=?e, "Failed to enqueue gmail ops label write");- })- .ok();+ .await?;+ }Ok((StatusCode::OK,Json(UpdateLabelBatchResponse {- successful_ids: message_db_ids,- failed_ids: vec![],+ successful_ids: message_db_ids+ .into_iter()+ .filter(|id| !failed_ids.contains(id))+ .collect(),+ failed_ids,missing_ids,}),).into_response())}++/// Undoes a write that could not be queued and returns the messages put back,+/// which the response lists as failed. If that fails too, the change stays in+/// the database without ever reaching Gmail, so the request fails rather than+/// reporting success.+async fn undo_unsent_write(+ db: &sqlx::PgPool,+ link_id: Uuid,+ thread_ids: &[Uuid],+ message_ids: &[Uuid],+ write_id: Uuid,+) -> Result<Vec<Uuid>, Response> {+ revert_unsent_write(db, link_id, thread_ids, message_ids, write_id)+ .await+ .map_err(|e| {+ tracing::error!(error=?e, "Failed to revert label write after enqueue failure");+ (+ StatusCode::INTERNAL_SERVER_ERROR,+ Json(ErrorResponse {+ message: "unable to queue the label change".into(),+ }),+ )+ .into_response()+ })+}++/// Puts the labels of a write Gmail will never receive back in the state+/// Gmail will end up with, and derives the threads' state again, holding the+/// threads' label-write lock. Returns the messages put back.+async fn revert_unsent_write(+ db: &sqlx::PgPool,+ link_id: Uuid,+ thread_ids: &[Uuid],+ message_ids: &[Uuid],+ write_id: Uuid,+) -> anyhow::Result<Vec<Uuid>> {+ let mut tx = db.begin().await?;+ email_db_client::labels::pending_writes::lock_thread_label_writes(&mut tx, thread_ids).await?;+ let restored = email_db_client::labels::pending_writes::abandon_pending_label_writes(+ &mut tx,+ link_id,+ message_ids,+ write_id,+ )+ .await?;+ for thread_id in thread_ids {+ email_db_client::threads::update::update_thread_metadata(&mut tx, *thread_id, link_id)+ .await?;+ }+ tx.commit().await?;+ Ok(restored)+}diff --git a/services/email_service/src/api/email/messages/labels/test.rs b/services/email_service/src/api/email/messages/labels/test.rsnew file mode 100644index 0000000000..67dee6fc62--- /dev/null+++ b/services/email_service/src/api/email/messages/labels/test.rs@@ -0,0 +1,20 @@+use super::*;+use macro_db_migrator::MACRO_DB_MIGRATIONS;+use sqlx::PgPool;++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn a_write_that_can_be_neither_queued_nor_undone_fails_the_request(pool: PgPool) {+ pool.close().await;++ let response = undo_unsent_write(+ &pool,+ Uuid::now_v7(),+ &[Uuid::now_v7()],+ &[Uuid::now_v7()],+ Uuid::now_v7(),+ )+ .await+ .unwrap_err();++ assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);+}diff --git a/services/email_service/src/pubsub/backfill/backfill_message.rs b/services/email_service/src/pubsub/backfill/backfill_message.rsindex 33c9047898..1caaa5a874 100644--- a/services/email_service/src/pubsub/backfill/backfill_message.rs+++ b/services/email_service/src/pubsub/backfill/backfill_message.rs@@ -2,6 +2,7 @@ use crate::pubsub::backfill::db_error::map_db_error;use crate::pubsub::backfill::email_api_error::map_email_api_error;use crate::pubsub::backfill::{increment_counters, write_slot};use crate::pubsub::context::PubSubContext;+use crate::pubsub::inbox_sync::operations::pending_labels;use crate::pubsub::util::{CrmContactRecipient, enqueue_populate_crm_contacts};use crate::util::process_pre_insert::process_message_pre_insert;use email_api_client::domain::models::CalendarPart;@@ -23,10 +24,27 @@ pub async fn backfill_message(link: &link::Link,) -> Result<(), ProcessingError> {let p = &scope.payload;- let fetched = ctx- .email_api- .get_message(link.id, &p.message_provider_id)+ // Sync may have stored the message since the backfill listed it; a+ // re-sync must not undo a label write made meanwhile.+ let stored =+ email_db_client::messages::get_simple_messages::get_simple_message_by_provider_and_link(+ &ctx.db,+ &p.message_provider_id,+ &link.id,+ ).await+ .map_err(|e| map_db_error(e, "Failed to look up the stored message"))?;+ let fetch = Box::pin(ctx.email_api.get_message(link.id, &p.message_provider_id));+ let (fetched, resync) = match &stored {+ Some(stored) => {+ let (fetched, snapshot) = pending_labels::resync_around(&ctx.db, stored.db_id, fetch)+ .await+ .map_err(|e| map_db_error(e, "Failed to read the stored message"))?;+ (fetched, Some(snapshot))+ }+ None => (fetch.await, None),+ };+ let fetched = fetched.map_err(|error| map_email_api_error(error, "Failed to get provider message"))?.ok_or_else(|| {ProcessingError::NonRetryable(DetailedError {@@ -48,6 +66,7 @@ pub async fn backfill_message(link.id,// we update the thread metadata once all messages in the thread have been backfilledfalse,+ resync.as_ref(),)).await;inserted.map_err(|e| map_db_error(e, "Failed to insert final message into database"))?;diff --git a/services/email_service/src/pubsub/backfill/mod.rs b/services/email_service/src/pubsub/backfill/mod.rsindex 764154760a..6dc9d5d425 100644--- a/services/email_service/src/pubsub/backfill/mod.rs+++ b/services/email_service/src/pubsub/backfill/mod.rs@@ -1,5 +1,5 @@mod backfill_attachment;-mod backfill_message;+pub(in crate::pubsub) mod backfill_message;mod backfill_thread;pub(crate) mod db_error;mod deferral;diff --git a/services/email_service/src/pubsub/gmail_ops/email_api_error.rs b/services/email_service/src/pubsub/gmail_ops/email_api_error.rsindex 2140963c84..b3681084b1 100644--- a/services/email_service/src/pubsub/gmail_ops/email_api_error.rs+++ b/services/email_service/src/pubsub/gmail_ops/email_api_error.rs@@ -42,8 +42,9 @@ pub(crate) async fn handle_email_api_error(error: EmailApiError,) -> ProcessingError {match error_policy(&error) {- ErrorPolicy::RateLimited => handle_rate_limit(ctx, data, error).await,- ErrorPolicy::Retry => processing_error(error, true),+ // A Gmail outage can outlast the primary queue's few receives, so it+ // waits in the retry queue like a rate limit.+ ErrorPolicy::RateLimited | ErrorPolicy::Retry => handle_rate_limit(ctx, data, error).await,ErrorPolicy::Permanent => processing_error(error, false),}}@@ -56,14 +57,16 @@ async fn handle_rate_limit(if rate_limit_route(ctx.retry_worker) == RateLimitRoute::RetryAfterVisibilityTimeout {tracing::info!(link_id = %data.link_id,- "Gmail API rate limited in retry worker, message will be retried after visibility timeout"+ error = %error,+ "Gmail API rate limited or unavailable in retry worker, message will be retried after visibility timeout");return processing_error(error, true);}tracing::info!(link_id = %data.link_id,- "Gmail API rate limited, moving message from primary queue to retry queue"+ error = %error,+ "Gmail API rate limited or unavailable, moving message from primary queue to retry queue");if let Err(enqueue_error) = ctx.sqs_client@@ -90,12 +93,12 @@ fn rate_limit_route(retry_worker: bool) -> RateLimitRoute {fn processing_error(error: EmailApiError, retryable: bool) -> ProcessingError {let reason = match error {EmailApiError::RateLimited { .. } => FailureReason::GmailApiRateLimited,+ EmailApiError::Transient { .. } => FailureReason::GmailApiUnavailable,EmailApiError::AuthRequired => FailureReason::AccessTokenFetchFailed,EmailApiError::OutdatedCursor => FailureReason::OutdatedHistoryId,EmailApiError::Forbidden| EmailApiError::NotFound| EmailApiError::Conflict- | EmailApiError::Transient { .. }| EmailApiError::SendRejected { .. }| EmailApiError::Permanent { .. } => FailureReason::GmailApiFailed,};diff --git a/services/email_service/src/pubsub/gmail_ops/email_api_error/test.rs b/services/email_service/src/pubsub/gmail_ops/email_api_error/test.rsindex f03d0b004b..2db12b22dd 100644--- a/services/email_service/src/pubsub/gmail_ops/email_api_error/test.rs+++ b/services/email_service/src/pubsub/gmail_ops/email_api_error/test.rs@@ -32,16 +32,24 @@ fn rate_limits_use_queue_routing_policy() {}#[test]-fn transient_failures_are_retried() {+fn transient_failures_are_deferred_like_rate_limits() {let error = EmailApiError::Transient {message: "provider unavailable".to_string(),};assert_eq!(error_policy(&error), ErrorPolicy::Retry);+ // Moved to the retry queue from the primary worker, redelivered there.+ assert!(matches!(+ processing_error(error.clone(), false),+ ProcessingError::NonRetryable(DetailedError {+ reason: FailureReason::GmailApiUnavailable,+ ..+ })+ ));assert!(matches!(processing_error(error, true),ProcessingError::Retryable(DetailedError {- reason: FailureReason::GmailApiFailed,+ reason: FailureReason::GmailApiUnavailable,..})));diff --git a/services/email_service/src/pubsub/gmail_ops/error_handlers.rs b/services/email_service/src/pubsub/gmail_ops/error_handlers.rsindex a89c56cc98..cd49ec156b 100644--- a/services/email_service/src/pubsub/gmail_ops/error_handlers.rs+++ b/services/email_service/src/pubsub/gmail_ops/error_handlers.rs@@ -11,10 +11,15 @@ pub async fn handle_non_retryable_error(data: &GmailOpsPubsubMessage,e: &DetailedError,) -> anyhow::Result<()> {- // A rate-limited operation is only non-retryable here once it has been+ // A deferred operation is only non-retryable here once it has been// re-enqueued to the retry queue, so deleting it loses nothing.- if e.reason == FailureReason::GmailApiRateLimited {- tracing::debug!(error = ?e, payload = ?data.operation, "Rate-limited gmail ops message deferred. The message will be deleted.");+ if matches!(+ e.reason,+ FailureReason::GmailApiRateLimited+ | FailureReason::GmailApiUnavailable+ | FailureReason::NewerLabelWritePending+ ) {+ tracing::debug!(error = ?e, payload = ?data.operation, "Deferred gmail ops message moved to the retry queue. The message will be deleted.");} else {tracing::error!(error = ?e, payload = ?data.operation, "Non-retryable error processing gmail ops message. The message will be deleted.");}diff --git a/services/email_service/src/pubsub/gmail_ops/operations/error.rs b/services/email_service/src/pubsub/gmail_ops/operations/error.rsnew file mode 100644index 0000000000..665d16531e--- /dev/null+++ b/services/email_service/src/pubsub/gmail_ops/operations/error.rs@@ -0,0 +1,19 @@+use email_api_client::domain::models::EmailApiError;++/// Why a gmail_ops operation did not finish.+#[derive(Debug)]+pub enum OperationError {+ Gmail(EmailApiError),+ /// Recording the operation's outcome failed. Gmail writes are idempotent,+ /// so the operation runs again.+ Database(anyhow::Error),+ /// A newer label write on one of the operation's messages is still on its+ /// way. If Gmail rejects it, this one has to send after all.+ WaitingForNewerWrite,+}++impl From<EmailApiError> for OperationError {+ fn from(error: EmailApiError) -> Self {+ Self::Gmail(error)+ }+}diff --git a/services/email_service/src/pubsub/gmail_ops/operations/mod.rs b/services/email_service/src/pubsub/gmail_ops/operations/mod.rsindex fad70c0ff4..9b66f2a8ae 100644--- a/services/email_service/src/pubsub/gmail_ops/operations/mod.rs+++ b/services/email_service/src/pubsub/gmail_ops/operations/mod.rs@@ -1,4 +1,7 @@pub mod block_sender;pub mod delete_label;+mod error;pub mod modify_message_labels;pub mod unblock_sender;++pub use error::OperationError;diff --git a/services/email_service/src/pubsub/gmail_ops/operations/modify_message_labels.rs b/services/email_service/src/pubsub/gmail_ops/operations/modify_message_labels.rsindex 9f0b8aa203..587866ba6f 100644--- a/services/email_service/src/pubsub/gmail_ops/operations/modify_message_labels.rs+++ b/services/email_service/src/pubsub/gmail_ops/operations/modify_message_labels.rs@@ -1,17 +1,42 @@use crate::pubsub::gmail_ops::email_api_error::is_permanent_mutation_error;+use crate::pubsub::gmail_ops::operations::OperationError;use crate::pubsub::gmail_ops::worker::GmailOpsContext;use anyhow::Context;-use email_api_client::domain::models::EmailApiError;-use models_email::gmail::gmail_ops::ModifyMessageLabelsBatchPayload;+use email_api_client::domain::service::SEPARATE_LABEL_WRITES_BELOW;+use email_db_client::labels::pending_writes::{self, LabelWriteTurn};+use futures::future::BoxFuture;+use models_email::gmail::gmail_ops::{GmailMessageRef, ModifyMessageLabelsBatchPayload};use models_email::service;use models_email::service::link::Link;+use sqlx::{PgPool, Postgres, Transaction};use uuid::Uuid;-/// Modifies labels for a batch of messages: one Gmail batch request from ten-/// messages up, one request per message below that. A permanent failure-/// reverts every message in the batch, and a message Gmail no longer has is-/// skipped; transient errors (5xx, network) and rate limits keep the-/// optimistic DB changes for the retry, which resends the whole batch.+#[cfg(test)]+mod test;++/// Modifies labels for a batch of messages. A message whose newer write was+/// confirmed is left out. The messages the write is the newest on are sent+/// first; one whose newer write is still on its way then makes the operation+/// wait for it. Sending first means two writes that are each newer on one of+/// the other's messages never wait for each other. From ten messages up the+/// messages sent are one Gmail request, which a permanent failure reverts as+/// a whole, with the waiting messages. Below ten, each message is its own+/// request and is settled as Gmail answers it (see [`send_one_at_a_time`]).+/// A message Gmail no longer has is skipped; transient errors (5xx, network)+/// and rate limits keep the optimistic DB changes for the retry.+///+/// Boxed so this large future lives on the heap rather than in the frames+/// that await it: unoptimized builds give every awaited future its own stack+/// slot, and the gmail_ops dispatch awaits it in two places, deep enough to+/// overflow a worker thread's stack in a local build.+pub fn modify_message_labels<'a>(+ ctx: &'a GmailOpsContext,+ link: &'a Link,+ payload: &'a ModifyMessageLabelsBatchPayload,+) -> BoxFuture<'a, Result<(), OperationError>> {+ Box::pin(modify_message_labels_unboxed(ctx, link, payload))+}+#[tracing::instrument(skip(ctx, link, payload),fields(@@ -19,23 +44,234 @@ use uuid::Uuid;labels_to_add = ?payload.labels_to_add,labels_to_remove = ?payload.labels_to_remove,),- err+ err(Debug))]-pub async fn modify_message_labels(+async fn modify_message_labels_unboxed(ctx: &GmailOpsContext,link: &Link,payload: &ModifyMessageLabelsBatchPayload,-) -> Result<(), EmailApiError> {- let provider_message_ids: Vec<String> = payload+) -> Result<(), OperationError> {+ let Some(write_id) = payload.pending_write_id else {+ // Queued before markers existed: nothing to coordinate with.+ let messages: Vec<&GmailMessageRef> = payload.messages.iter().collect();+ return send(ctx, link, payload, &messages, &[]).await;+ };++ let message_ids: Vec<Uuid> = payload.messages.iter()- .map(|message| message.provider_message_id.clone())+ .map(|message| message.db_message_id).collect();- let db_message_ids: Vec<Uuid> = payload+ let turns = label_write_turns(&ctx.db, payload, &message_ids, write_id).await?;+ let ids_with = |turn: LabelWriteTurn| -> Vec<Uuid> {+ payload+ .messages+ .iter()+ .map(|message| message.db_message_id)+ .filter(|id| turns.get(id).copied().unwrap_or(LabelWriteTurn::Superseded) == turn)+ .collect()+ };+ abandon_superseded(ctx, link, &ids_with(LabelWriteTurn::Superseded), write_id).await?;++ let mut waiting = ids_with(LabelWriteTurn::Wait);+ let messages: Vec<&GmailMessageRef> = payload.messages+ .iter()+ .filter(|message| turns.get(&message.db_message_id) == Some(&LabelWriteTurn::Send))+ .collect();+ if messages.len() >= SEPARATE_LABEL_WRITES_BELOW {+ send(ctx, link, payload, &messages, &waiting).await?;+ } else if !messages.is_empty() {+ send_one_at_a_time(ctx, link, payload, write_id, &messages, &mut waiting).await?;+ }++ if !waiting.is_empty() {+ tracing::info!(+ db_message_ids = ?waiting,+ "Newer label writes on these messages are still on their way, waiting for them"+ );+ return Err(OperationError::WaitingForNewerWrite);+ }+ if messages.is_empty() {+ tracing::info!("Newer label writes took over every message, skipping Gmail");+ }+ Ok(())+}++async fn label_write_turns(+ db: &PgPool,+ payload: &ModifyMessageLabelsBatchPayload,+ message_ids: &[Uuid],+ write_id: Uuid,+) -> Result<std::collections::HashMap<Uuid, LabelWriteTurn>, OperationError> {+ let provider_label_ids: Vec<String> = payload+ .labels_to_add+ .iter()+ .chain(&payload.labels_to_remove)+ .cloned()+ .collect();+ pending_writes::label_write_turns(db, message_ids, &provider_label_ids, write_id)+ .await+ .context("Failed to read the label write's turn")+ .map_err(OperationError::Database)+}++/// Drops the write's rows on messages a newer write has taken over.+async fn abandon_superseded(+ ctx: &GmailOpsContext,+ link: &Link,+ message_ids: &[Uuid],+ write_id: Uuid,+) -> Result<(), OperationError> {+ if message_ids.is_empty() {+ return Ok(());+ }+ let mut tx = begin_locked(&ctx.db, link.id, message_ids).await?;+ pending_writes::abandon_pending_label_writes(&mut tx, link.id, message_ids, write_id)+ .await+ .context("Failed to drop superseded pending label writes")+ .map_err(OperationError::Database)?;+ commit(tx).await+}++/// Clears the write's markers on messages Gmail accepted it for. Inbox sync+/// applies Gmail's labels to them again; failing here retries the operation+/// rather than leaving sync off those labels until the markers expire.+async fn confirm(+ ctx: &GmailOpsContext,+ link: &Link,+ message_ids: &[Uuid],+ write_id: Uuid,+) -> Result<(), OperationError> {+ let mut tx = begin_locked(&ctx.db, link.id, message_ids).await?;+ pending_writes::confirm_pending_label_writes(&mut tx, message_ids, write_id)+ .await+ .context("Failed to confirm pending label writes")+ .map_err(OperationError::Database)?;+ commit(tx).await+}++/// Sends `messages` one Gmail request each, settling each message as Gmail+/// answers it. Before each request after the first, the message's turn is+/// read again: a message a newer write took over meanwhile is skipped, and+/// one with a newer write still on its way joins `waiting`. A permanent+/// rejection reverts the rejected message, those not sent yet and the waiting+/// ones; messages Gmail accepted keep their change. A retry finds the+/// accepted messages settled and doesn't send them again. Boxed for the+/// reason given on [`modify_message_labels`].+fn send_one_at_a_time<'a>(+ ctx: &'a GmailOpsContext,+ link: &'a Link,+ payload: &'a ModifyMessageLabelsBatchPayload,+ write_id: Uuid,+ messages: &'a [&'a GmailMessageRef],+ waiting: &'a mut Vec<Uuid>,+) -> BoxFuture<'a, Result<(), OperationError>> {+ Box::pin(send_one_at_a_time_unboxed(+ ctx, link, payload, write_id, messages, waiting,+ ))+}++async fn send_one_at_a_time_unboxed(+ ctx: &GmailOpsContext,+ link: &Link,+ payload: &ModifyMessageLabelsBatchPayload,+ write_id: Uuid,+ messages: &[&GmailMessageRef],+ waiting: &mut Vec<Uuid>,+) -> Result<(), OperationError> {+ // As in a batch request, a message Gmail no longer has counts as written+ // when the write has others.+ let skip_missing = payload.messages.len() > 1;+ for (index, message) in messages.iter().enumerate() {+ let message_id = message.db_message_id;+ if index > 0 {+ let turns = label_write_turns(&ctx.db, payload, &[message_id], write_id).await?;+ match turns+ .get(&message_id)+ .copied()+ .unwrap_or(LabelWriteTurn::Superseded)+ {+ LabelWriteTurn::Send => {}+ LabelWriteTurn::Superseded => {+ abandon_superseded(ctx, link, &[message_id], write_id).await?;+ continue;+ }+ LabelWriteTurn::Wait => {+ waiting.push(message_id);+ continue;+ }+ }+ }+ let result = ctx+ .email_api+ .modify_one_message_labels(+ link.id,+ &message.provider_message_id,+ &payload.labels_to_add,+ &payload.labels_to_remove,+ skip_missing,+ )+ .await;+ match result {+ Ok(()) => confirm(ctx, link, &[message_id], write_id).await?,+ Err(error) if is_permanent_mutation_error(&error) => {+ let unaccepted: Vec<Uuid> = messages[index..]+ .iter()+ .map(|message| message.db_message_id)+ .chain(waiting.iter().copied())+ .collect();+ tracing::error!(+ error = ?error,+ db_message_ids = ?unaccepted,+ "Permanent Gmail error modifying labels, reverting the messages Gmail hasn't accepted"+ );+ revert_db_changes(&ctx.db, link, payload, unaccepted)+ .await+ .map_err(OperationError::Database)?;+ return Err(OperationError::Gmail(error));+ }+ Err(error) => {+ tracing::warn!(+ error = ?error,+ db_message_id = %message_id,+ "Retryable Gmail error modifying labels, preserving optimistic database changes"+ );+ return Err(OperationError::Gmail(error));+ }+ }+ }+ Ok(())+}++/// Sends `messages` to Gmail in one call and records the outcome. A permanent+/// failure reverts them and the `waiting` messages. Boxed for the reason+/// given on [`modify_message_labels`]: the operation awaits it in two places.+fn send<'a>(+ ctx: &'a GmailOpsContext,+ link: &'a Link,+ payload: &'a ModifyMessageLabelsBatchPayload,+ messages: &'a [&'a GmailMessageRef],+ waiting: &'a [Uuid],+) -> BoxFuture<'a, Result<(), OperationError>> {+ Box::pin(send_unboxed(ctx, link, payload, messages, waiting))+}++async fn send_unboxed(+ ctx: &GmailOpsContext,+ link: &Link,+ payload: &ModifyMessageLabelsBatchPayload,+ messages: &[&GmailMessageRef],+ waiting: &[Uuid],+) -> Result<(), OperationError> {+ let db_message_ids: Vec<Uuid> = messages.iter().map(|message| message.db_message_id).collect();+ let provider_message_ids: Vec<String> = messages+ .iter()+ .map(|message| message.provider_message_id.clone())+ .collect();let result = ctx.email_api.modify_message_labels(@@ -46,114 +282,169 @@ pub async fn modify_message_labels().await;- if let Err(error) = &result {- if is_permanent_mutation_error(error) {+ match result {+ Ok(()) => {+ if let Some(write_id) = payload.pending_write_id {+ confirm(ctx, link, &db_message_ids, write_id).await?;+ }+ Ok(())+ }+ Err(error) if is_permanent_mutation_error(&error) => {+ let unaccepted: Vec<Uuid> = db_message_ids.iter().chain(waiting).copied().collect();tracing::error!(error = ?error,- ?db_message_ids,+ db_message_ids = ?unaccepted,"Permanent Gmail error modifying labels, reverting database changes");- revert_db_changes(ctx, link, payload, db_message_ids).await;- } else {+ revert_db_changes(&ctx.db, link, payload, unaccepted)+ .await+ .map_err(OperationError::Database)?;+ Err(OperationError::Gmail(error))+ }+ Err(error) => {tracing::warn!(error = ?error,?db_message_ids,"Retryable Gmail error modifying labels, preserving optimistic database changes");+ Err(OperationError::Gmail(error))}}+}++/// Starts a transaction holding the label-write lock of the messages' threads.+async fn begin_locked<'c>(+ db: &'c PgPool,+ link_id: Uuid,+ message_ids: &[Uuid],+) -> Result<Transaction<'c, Postgres>, OperationError> {+ async {+ let mut tx = db.begin().await?;+ let thread_ids =+ email_db_client::messages::get::thread_ids_of_messages(&mut *tx, link_id, message_ids)+ .await?;+ pending_writes::lock_thread_label_writes(&mut tx, &thread_ids).await?;+ Ok(tx)+ }+ .await+ .map_err(|e: sqlx::Error| {+ OperationError::Database(anyhow::Error::from(e).context("Failed to lock label writes"))+ })+}- result+async fn commit(tx: Transaction<'_, Postgres>) -> Result<(), OperationError> {+ tx.commit()+ .await+ .context("Failed to commit label write changes")+ .map_err(OperationError::Database)}-/// Reverts the optimistic DB changes for the messages of a batch that failed-/// in Gmail.+/// Reverts the optimistic DB changes Gmail rejected. With a write id, only+/// labels the write was the newest on go back, to the state Gmail will end up+/// with; a newer write keeps its own. Each affected thread's state is derived+/// again from its messages.async fn revert_db_changes(- ctx: &GmailOpsContext,+ db: &PgPool,link: &Link,payload: &ModifyMessageLabelsBatchPayload,- failed_ids: Vec<Uuid>,-) {- let mut tx = match ctx.db.begin().await {- Ok(tx) => tx,- Err(e) => {- tracing::error!(error=?e, "Failed to begin transaction for reversion");- return;+ sent_ids: Vec<Uuid>,+) -> anyhow::Result<()> {+ let mut tx = db+ .begin()+ .await+ .context("Failed to begin transaction for reversion")?;+ let thread_ids =+ email_db_client::messages::get::thread_ids_of_messages(&mut *tx, link.id, &sent_ids)+ .await+ .context("Failed to read the threads of the reverted messages")?;+ pending_writes::lock_thread_label_writes(&mut tx, &thread_ids)+ .await+ .context("Failed to lock the threads of the reverted messages")?;+ let reverted_ids = match payload.pending_write_id {+ Some(write_id) => {+ pending_writes::abandon_pending_label_writes(&mut tx, link.id, &sent_ids, write_id)+ .await+ .context("Failed to abandon pending label writes")?+ }+ None => {+ invert_label_change(&mut tx, link, payload, &sent_ids).await?;+ sent_ids}};- let is_adding = !payload.labels_to_add.is_empty();+ for thread_id in thread_ids {+ email_db_client::threads::update::update_thread_metadata(&mut tx, thread_id, link.id)+ .await+ .context("Failed to recompute thread state after revert")?;+ }+ tx.commit()+ .await+ .context("Unable to commit transaction for revert")?;+ tracing::info!(+ db_message_ids = ?reverted_ids,+ "Successfully reverted database changes after Gmail API failure"+ );+ Ok(())+}++/// Undoes the first label change of an operation queued before markers+/// existed on every message it sent.+async fn invert_label_change(+ tx: &mut sqlx::PgConnection,+ link: &Link,+ payload: &ModifyMessageLabelsBatchPayload,+ message_ids: &[Uuid],+) -> anyhow::Result<()> {+ let is_adding = !payload.labels_to_add.is_empty();let Some(provider_label_id) = (if is_adding {payload.labels_to_add.first()} else {payload.labels_to_remove.first()}) else {tracing::error!("No label IDs in payload, cannot revert");- return;+ return Ok(());};+ let message_ids = message_ids.to_vec();- let revert_result = async {- if is_adding {- email_db_client::labels::delete::delete_message_labels_batch(- &mut *tx,- &failed_ids,- provider_label_id,- link.id,- )- .await- .context("Failed to revert adding labels")?;- } else {- email_db_client::labels::insert::insert_message_labels_batch(- &mut *tx,- &failed_ids,- provider_label_id,- link.id,- )- .await- .context("Failed to revert removing labels")?;- }-- if *provider_label_id == service::label::system_labels::UNREAD {- email_db_client::messages::update::update_message_read_status_batch(- &mut *tx,- failed_ids.clone(),- link.id,- is_adding,- )- .await- .context("Failed to revert message read status")?;- } else if *provider_label_id == service::label::system_labels::STARRED {- email_db_client::messages::update::update_message_starred_status_batch(- &mut *tx,- failed_ids.clone(),- link.id,- !is_adding,- )- .await- .context("Failed to revert message starred status")?;- }-- anyhow::Ok(())+ if is_adding {+ email_db_client::labels::delete::delete_message_labels_batch(+ &mut *tx,+ &message_ids,+ provider_label_id,+ link.id,+ )+ .await+ .context("Failed to revert adding labels")?;+ } else {+ email_db_client::labels::insert::insert_message_labels_batch(+ &mut *tx,+ &message_ids,+ provider_label_id,+ link.id,+ )+ .await+ .context("Failed to revert removing labels")?;}- .await;-- match revert_result {- Ok(_) => {- if let Err(e) = tx.commit().await {- tracing::error!(error=?e, "Unable to commit transaction for revert");- } else {- tracing::info!(- db_message_ids = ?failed_ids,- "Successfully reverted database changes after Gmail API failure"- );- }- }- Err(e) => {- tracing::error!(error=?e, "Revert failed, rolling back");- if let Err(rollback_err) = tx.rollback().await {- tracing::error!(error=?rollback_err, "Failed to rollback revert transaction");- }- }++ if *provider_label_id == service::label::system_labels::UNREAD {+ email_db_client::messages::update::update_message_read_status_batch(+ &mut *tx,+ message_ids,+ link.id,+ is_adding,+ )+ .await+ .context("Failed to revert message read status")?;+ } else if *provider_label_id == service::label::system_labels::STARRED {+ email_db_client::messages::update::update_message_starred_status_batch(+ &mut *tx,+ message_ids,+ link.id,+ !is_adding,+ )+ .await+ .context("Failed to revert message starred status")?;}+ Ok(())}diff --git a/services/email_service/src/pubsub/gmail_ops/operations/modify_message_labels/test.rs b/services/email_service/src/pubsub/gmail_ops/operations/modify_message_labels/test.rsnew file mode 100644index 0000000000..430d344c82--- /dev/null+++ b/services/email_service/src/pubsub/gmail_ops/operations/modify_message_labels/test.rs@@ -0,0 +1,284 @@+use super::*;+use macro_db_migrator::MACRO_DB_MIGRATIONS;+use models_email::gmail::gmail_ops::ModifyMessageLabelsPayload;++const THREAD: Uuid = Uuid::from_u128(0xb1);+const ARCHIVE: Uuid = Uuid::from_u128(0xf1);+const UNARCHIVE: Uuid = Uuid::from_u128(0xf2);++async fn messages_with_label(pool: &PgPool, provider_label_id: &str) -> Vec<Uuid> {+ sqlx::query_scalar!(+ r#"+ SELECT ml.message_id FROM email_message_labels ml+ JOIN email_labels l ON l.id = ml.label_id+ WHERE l.provider_label_id = $1+ ORDER BY ml.message_id+ "#,+ provider_label_id+ )+ .fetch_all(pool)+ .await+ .unwrap()+}++async fn pending_markers(pool: &PgPool) -> Vec<(Uuid, Uuid)> {+ sqlx::query!(+ "SELECT message_id, write_id FROM email_gmail_pending_label_writes ORDER BY message_id, seq"+ )+ .fetch_all(pool)+ .await+ .unwrap()+ .into_iter()+ .map(|marker| (marker.message_id, marker.write_id))+ .collect()+}++async fn thread_state(pool: &PgPool) -> (bool, bool) {+ let thread = sqlx::query!(+ "SELECT inbox_visible, is_read FROM email_threads WHERE id = $1",+ THREAD+ )+ .fetch_one(pool)+ .await+ .unwrap();+ (thread.inbox_visible, thread.is_read)+}++async fn link(pool: &PgPool) -> Link {+ email_db_client::links::get::fetch_link_by_id(pool, Uuid::from_u128(0xa1))+ .await+ .unwrap()+ .unwrap()+}++fn message(id: u128) -> GmailMessageRef {+ GmailMessageRef {+ db_message_id: Uuid::from_u128(id),+ provider_message_id: format!("msg-{}", char::from_digit(id as u32, 16).unwrap()),+ }+}++fn inbox_write(+ write_id: Uuid,+ present: bool,+ messages: Vec<GmailMessageRef>,+) -> ModifyMessageLabelsBatchPayload {+ let label = vec!["INBOX".to_string()];+ ModifyMessageLabelsBatchPayload {+ messages,+ labels_to_add: if present { label.clone() } else { vec![] },+ labels_to_remove: if present { vec![] } else { label },+ pending_write_id: Some(write_id),+ }+}++/// A newer write unarchives message B locally, as staging does.+async fn unarchive_b(pool: &PgPool) {+ let mut tx = pool.begin().await.unwrap();+ sqlx::query!(+ "INSERT INTO email_message_labels (message_id, label_id) VALUES ($1, $2)",+ Uuid::from_u128(0xb),+ Uuid::from_u128(0xc1),+ )+ .execute(&mut *tx)+ .await+ .unwrap();+ pending_writes::mark_pending_label_writes(+ &mut *tx,+ Uuid::from_u128(0xa1),+ &[Uuid::from_u128(0xb)],+ "INBOX",+ true,+ UNARCHIVE,+ )+ .await+ .unwrap();+ tx.commit().await.unwrap();+}++fn message_ids(payload: &ModifyMessageLabelsBatchPayload) -> Vec<Uuid> {+ payload+ .messages+ .iter()+ .map(|message| message.db_message_id)+ .collect()+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../../../fixtures", scripts("gmail_ops_label_revert"))+)]+async fn an_older_write_waits_for_a_newer_one_and_sends_once_gmail_rejects_it(pool: PgPool) {+ unarchive_b(&pool).await;+ let archive = inbox_write(+ ARCHIVE,+ false,+ vec![message(0xa), message(0xb), message(0xc)],+ );+ let turns = label_write_turns(&pool, &archive, &message_ids(&archive), ARCHIVE)+ .await+ .unwrap();+ assert_eq!(turns[&Uuid::from_u128(0xa)], LabelWriteTurn::Send);+ assert_eq!(turns[&Uuid::from_u128(0xb)], LabelWriteTurn::Wait);++ revert_db_changes(+ &pool,+ &link(&pool).await,+ &inbox_write(UNARCHIVE, true, vec![message(0xb)]),+ vec![Uuid::from_u128(0xb)],+ )+ .await+ .unwrap();++ // B is archived again, and the archive now sends it with the others.+ assert!(messages_with_label(&pool, "INBOX").await.is_empty());+ let turns = label_write_turns(&pool, &archive, &message_ids(&archive), ARCHIVE)+ .await+ .unwrap();+ assert!(turns.values().all(|turn| *turn == LabelWriteTurn::Send));+ assert_eq!(thread_state(&pool).await, (false, true));+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../../../fixtures", scripts("gmail_ops_label_revert"))+)]+async fn a_confirmed_newer_write_supersedes_the_older_one(pool: PgPool) {+ unarchive_b(&pool).await;+ let mut tx = pool.begin().await.unwrap();+ pending_writes::confirm_pending_label_writes(&mut tx, &[Uuid::from_u128(0xb)], UNARCHIVE)+ .await+ .unwrap();+ tx.commit().await.unwrap();++ let archive = inbox_write(ARCHIVE, false, vec![message(0xa), message(0xb)]);+ let turns = label_write_turns(&pool, &archive, &message_ids(&archive), ARCHIVE)+ .await+ .unwrap();+ assert_eq!(turns[&Uuid::from_u128(0xa)], LabelWriteTurn::Send);+ assert_eq!(turns[&Uuid::from_u128(0xb)], LabelWriteTurn::Superseded);+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../../../fixtures", scripts("gmail_ops_label_revert"))+)]+async fn rejected_batch_reverts_only_the_messages_its_write_was_newest_on(pool: PgPool) {+ unarchive_b(&pool).await;++ revert_db_changes(+ &pool,+ &link(&pool).await,+ &inbox_write(ARCHIVE, false, vec![message(0xa), message(0xb)]),+ vec![Uuid::from_u128(0xa), Uuid::from_u128(0xb)],+ )+ .await+ .unwrap();++ // B keeps the newer unarchive; C was not in this batch.+ assert_eq!(+ messages_with_label(&pool, "INBOX").await,+ vec![Uuid::from_u128(0xa), Uuid::from_u128(0xb)]+ );+ assert_eq!(+ pending_markers(&pool).await,+ vec![+ (Uuid::from_u128(0xb), UNARCHIVE),+ (Uuid::from_u128(0xc), ARCHIVE),+ ]+ );+ assert_eq!(thread_state(&pool).await, (true, true));+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../../../fixtures", scripts("gmail_ops_label_revert"))+)]+async fn rejected_mark_read_restores_the_unread_message_and_thread(pool: PgPool) {+ let mark_read = Uuid::from_u128(0xf3);+ sqlx::query!(+ r#"+ INSERT INTO email_labels (id, link_id, provider_label_id, name, message_list_visibility, label_list_visibility, type, created_at)+ VALUES ($1, $2, 'UNREAD', 'UNREAD', 'Show', 'LabelShow', 'System', NOW())+ "#,+ Uuid::from_u128(0xc2),+ Uuid::from_u128(0xa1),+ )+ .execute(&pool)+ .await+ .unwrap();+ pending_writes::mark_pending_label_writes(+ &pool,+ Uuid::from_u128(0xa1),+ &[Uuid::from_u128(0xa)],+ "UNREAD",+ false,+ mark_read,+ )+ .await+ .unwrap();+ let batch = ModifyMessageLabelsBatchPayload {+ messages: vec![message(0xa)],+ labels_to_add: vec![],+ labels_to_remove: vec!["UNREAD".to_string()],+ pending_write_id: Some(mark_read),+ };++ revert_db_changes(+ &pool,+ &link(&pool).await,+ &batch,+ vec![Uuid::from_u128(0xa)],+ )+ .await+ .unwrap();++ assert_eq!(+ messages_with_label(&pool, "UNREAD").await,+ vec![Uuid::from_u128(0xa)]+ );+ let is_read = sqlx::query_scalar!(+ "SELECT is_read FROM email_messages WHERE id = $1",+ Uuid::from_u128(0xa)+ )+ .fetch_one(&pool)+ .await+ .unwrap();+ assert!(!is_read);+ assert_eq!(thread_state(&pool).await, (false, false));+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../../../fixtures", scripts("gmail_ops_label_revert"))+)]+async fn rejected_single_message_operation_reverts_its_message(pool: PgPool) {+ let batch = ModifyMessageLabelsBatchPayload::from(ModifyMessageLabelsPayload {+ db_message_id: Uuid::from_u128(0xc),+ provider_message_id: "msg-c".to_string(),+ labels_to_add: vec![],+ labels_to_remove: vec!["INBOX".to_string()],+ pending_write_id: Some(ARCHIVE),+ });++ revert_db_changes(+ &pool,+ &link(&pool).await,+ &batch,+ vec![Uuid::from_u128(0xc)],+ )+ .await+ .unwrap();++ assert_eq!(+ messages_with_label(&pool, "INBOX").await,+ vec![Uuid::from_u128(0xc)]+ );+ assert_eq!(+ pending_markers(&pool).await,+ vec![+ (Uuid::from_u128(0xa), ARCHIVE),+ (Uuid::from_u128(0xb), ARCHIVE),+ ]+ );+}diff --git a/services/email_service/src/pubsub/gmail_ops/process.rs b/services/email_service/src/pubsub/gmail_ops/process.rsindex f4502de192..c565395d05 100644--- a/services/email_service/src/pubsub/gmail_ops/process.rs+++ b/services/email_service/src/pubsub/gmail_ops/process.rs@@ -1,5 +1,6 @@use crate::pubsub::gmail_ops::email_api_error::handle_email_api_error;use crate::pubsub::gmail_ops::error_handlers::prefix_error_source;+use crate::pubsub::gmail_ops::operations::OperationError;use crate::pubsub::gmail_ops::operations::block_sender::block_sender;use crate::pubsub::gmail_ops::operations::delete_label::delete_label;use crate::pubsub::gmail_ops::operations::modify_message_labels::modify_message_labels;@@ -74,19 +75,31 @@ async fn inner_process_message("modify_message_labels",modify_message_labels(ctx, &link, payload).await,),- GmailOpsOperation::DeleteLabel(payload) => {- ("delete_label", delete_label(ctx, &link, payload).await)- }- GmailOpsOperation::BlockSender(payload) => {- ("block_sender", block_sender(ctx, &link, payload).await)- }- GmailOpsOperation::UnblockSender(payload) => {- ("unblock_sender", unblock_sender(ctx, &link, payload).await)- }+ GmailOpsOperation::DeleteLabel(payload) => (+ "delete_label",+ delete_label(ctx, &link, payload).await.map_err(Into::into),+ ),+ GmailOpsOperation::BlockSender(payload) => (+ "block_sender",+ block_sender(ctx, &link, payload).await.map_err(Into::into),+ ),+ GmailOpsOperation::UnblockSender(payload) => (+ "unblock_sender",+ unblock_sender(ctx, &link, payload)+ .await+ .map_err(Into::into),+ ),};if let Err(error) = operation_result {- let processing_error = handle_email_api_error(ctx, data, error).await;+ let processing_error = match error {+ OperationError::Gmail(error) => handle_email_api_error(ctx, data, error).await,+ OperationError::Database(source) => ProcessingError::Retryable(DetailedError {+ reason: FailureReason::DatabaseQueryFailed,+ source,+ }),+ OperationError::WaitingForNewerWrite => wait_in_retry_queue(ctx, data).await,+ };return Err(prefix_error_source(processing_error, operation_name));}@@ -94,6 +107,32 @@ async fn inner_process_message(Ok(())}+/// Moves an operation waiting for a newer label write to the retry queue,+/// whose receive limit outlasts the newer write's own retries.+async fn wait_in_retry_queue(+ ctx: &GmailOpsContext,+ data: &GmailOpsPubsubMessage,+) -> ProcessingError {+ let waiting = || DetailedError {+ reason: FailureReason::NewerLabelWritePending,+ source: anyhow!("a newer label write on its messages is still on its way"),+ };+ if ctx.retry_worker {+ return ProcessingError::Retryable(waiting());+ }+ if let Err(enqueue_error) = ctx+ .sqs_client+ .enqueue_gmail_ops_retry_notification(data.clone())+ .await+ {+ return ProcessingError::Retryable(DetailedError {+ reason: FailureReason::SqsEnqueueFailed,+ source: enqueue_error.context("Failed to enqueue gmail ops retry message"),+ });+ }+ ProcessingError::NonRetryable(waiting())+}+/// Extracts the message from the SQS message body.#[tracing::instrument(skip(message))]fn extract_gmail_ops_message(diff --git a/services/email_service/src/pubsub/inbox_sync/mod.rs b/services/email_service/src/pubsub/inbox_sync/mod.rsindex 8b472c0358..32a744966d 100644--- a/services/email_service/src/pubsub/inbox_sync/mod.rs+++ b/services/email_service/src/pubsub/inbox_sync/mod.rs@@ -1,5 +1,5 @@mod email_api_error;mod error_handlers;-mod operations;+pub(in crate::pubsub) mod operations;pub(crate) mod process;pub mod worker;diff --git a/services/email_service/src/pubsub/inbox_sync/operations/mod.rs b/services/email_service/src/pubsub/inbox_sync/operations/mod.rsindex c50c3c3df8..c06188d9b0 100644--- a/services/email_service/src/pubsub/inbox_sync/operations/mod.rs+++ b/services/email_service/src/pubsub/inbox_sync/operations/mod.rs@@ -1,4 +1,5 @@pub(in crate::pubsub) mod delete_message;pub(in crate::pubsub) mod gmail_message;+pub(in crate::pubsub) mod pending_labels;pub(in crate::pubsub) mod update_labels;pub(in crate::pubsub) mod upsert_message;diff --git a/services/email_service/src/pubsub/inbox_sync/operations/pending_labels.rs b/services/email_service/src/pubsub/inbox_sync/operations/pending_labels.rsnew file mode 100644index 0000000000..9874129f7f--- /dev/null+++ b/services/email_service/src/pubsub/inbox_sync/operations/pending_labels.rs@@ -0,0 +1,77 @@+use email_db_client::labels::pending_writes::pending_provider_label_ids;+use email_db_client::messages::insert::ResyncSnapshot;+use futures::future::BoxFuture;+use sqlx::PgPool;+use uuid::Uuid;++#[cfg(test)]+mod test;++/// A read of Gmail's labels for a message, with the labels that had a+/// pending Macro write just before and just after it.+pub(in crate::pubsub) struct PendingAroundRead<T> {+ pub value: T,+ before: Vec<String>,+ after: Vec<String>,+}++impl<T> PendingAroundRead<T> {+ /// Labels whose local state the read may not reflect: Gmail may have+ /// answered before applying a write pending at either point.+ pub fn untrusted_labels(&self) -> Vec<String> {+ let mut labels = self.before.clone();+ labels.extend(+ self.after+ .iter()+ .filter(|label| !self.before.contains(label))+ .cloned(),+ );+ labels+ }+}++/// Runs `read` between two reads of the message's pending label writes.+///+/// `read` is boxed: a Gmail read is a large future, and an unoptimized build+/// gives a future passed by value a stack slot in every frame it passes+/// through and another where it is awaited, enough to overflow a worker+/// thread's stack on the inbox sync path in a local build.+pub(in crate::pubsub) async fn read_around<T>(+ db: &PgPool,+ message_id: Uuid,+ read: BoxFuture<'_, T>,+) -> Result<PendingAroundRead<T>, sqlx::Error> {+ let before = pending_provider_label_ids(db, message_id).await?;+ let value = read.await;+ let after = pending_provider_label_ids(db, message_id).await?;+ Ok(PendingAroundRead {+ value,+ before,+ after,+ })+}++/// Runs `read`, a Gmail fetch of a stored message that is about to be+/// re-synced, after reading its stored labels and between two reads of its+/// pending label writes. `read` is boxed for the reason given on+/// [`read_around`].+pub(in crate::pubsub) async fn resync_around<T>(+ db: &PgPool,+ message_id: Uuid,+ read: BoxFuture<'_, T>,+) -> anyhow::Result<(T, ResyncSnapshot)> {+ let stored_labels = email_db_client::labels::get::fetch_message_labels(db, message_id)+ .await?+ .into_iter()+ .map(|label| label.provider_label_id)+ .collect();+ let read = read_around(db, message_id, read).await?;+ let pending_labels = read.untrusted_labels();+ Ok((+ read.value,+ ResyncSnapshot {+ stored_labels,+ pending_labels,+ },+ ))+}diff --git a/services/email_service/src/pubsub/inbox_sync/operations/pending_labels/test.rs b/services/email_service/src/pubsub/inbox_sync/operations/pending_labels/test.rsnew file mode 100644index 0000000000..0dd362d607--- /dev/null+++ b/services/email_service/src/pubsub/inbox_sync/operations/pending_labels/test.rs@@ -0,0 +1,87 @@+use super::*;+use macro_db_migrator::MACRO_DB_MIGRATIONS;++const MESSAGE: Uuid = Uuid::from_u128(0xa);++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../../../fixtures", scripts("gmail_ops_label_revert"))+)]+async fn a_write_gmail_confirms_during_the_read_is_not_trusted_from_it(pool: PgPool) {+ let read = read_around(+ &pool,+ MESSAGE,+ Box::pin(async {+ // gmail_ops applies the archive and clears its marker while Gmail's+ // answer, still showing INBOX, is on its way.+ sqlx::query!(+ "DELETE FROM email_gmail_pending_label_writes WHERE message_id = $1",+ MESSAGE+ )+ .execute(&pool)+ .await+ .unwrap();+ vec!["INBOX".to_string()]+ }),+ )+ .await+ .unwrap();++ assert_eq!(read.untrusted_labels(), vec!["INBOX".to_string()]);+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../../../fixtures", scripts("gmail_ops_label_revert"))+)]+async fn a_write_staged_during_the_read_is_not_trusted_from_it(pool: PgPool) {+ let read = read_around(&pool, MESSAGE, Box::pin(async {+ sqlx::query!(+ r#"+ INSERT INTO email_gmail_pending_label_writes+ (message_id, provider_label_id, write_id, label_present, label_present_before, expires_at)+ VALUES ($1, 'STARRED', $2, true, false, now() + interval '12 hours')+ "#,+ MESSAGE,+ Uuid::from_u128(0xf9),+ )+ .execute(&pool)+ .await+ .unwrap();+ }))+ .await+ .unwrap();++ assert_eq!(+ read.untrusted_labels(),+ vec!["INBOX".to_string(), "STARRED".to_string()]+ );+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../../../fixtures", scripts("gmail_ops_label_revert"))+)]+async fn a_resync_snapshot_has_the_labels_stored_before_the_fetch(pool: PgPool) {+ let (fetched, snapshot) = resync_around(+ &pool,+ MESSAGE,+ Box::pin(async {+ sqlx::query!(+ "INSERT INTO email_message_labels (message_id, label_id) VALUES ($1, $2)",+ MESSAGE,+ Uuid::from_u128(0xc1),+ )+ .execute(&pool)+ .await+ .unwrap();+ "fetched"+ }),+ )+ .await+ .unwrap();++ assert_eq!(fetched, "fetched");+ assert!(snapshot.stored_labels.is_empty());+ assert_eq!(snapshot.pending_labels, vec!["INBOX".to_string()]);+}diff --git a/services/email_service/src/pubsub/inbox_sync/operations/update_labels.rs b/services/email_service/src/pubsub/inbox_sync/operations/update_labels.rsindex eb968876cc..daf444fc97 100644--- a/services/email_service/src/pubsub/inbox_sync/operations/update_labels.rs+++ b/services/email_service/src/pubsub/inbox_sync/operations/update_labels.rs@@ -1,3 +1,4 @@+use super::pending_labels;use crate::pubsub::context::PubSubContext;use crate::pubsub::inbox_sync::email_api_error::handle_operation_error;use crate::pubsub::util::{@@ -92,11 +93,25 @@ pub async fn update_labels(})})?;- let provider_labels = ctx- .email_api- .get_message_label_ids(link.id, &payload.provider_message_id)- .await;- let gmail_message_labels = match provider_labels {+ // After the database labels: a write accepted in between has its marker+ // in place with its label change.+ let provider_labels = pending_labels::read_around(+ &ctx.db,+ db_message.db_id,+ Box::pin(+ ctx.email_api+ .get_message_label_ids(link.id, &payload.provider_message_id),+ ),+ )+ .await+ .map_err(|e| {+ ProcessingError::Retryable(DetailedError {+ reason: FailureReason::DatabaseQueryFailed,+ source: anyhow::Error::from(e).context("Failed to get pending label writes from db"),+ })+ })?;+ let pending_label_ids = provider_labels.untrusted_labels();+ let gmail_message_labels = match provider_labels.value {Ok(Some(labels)) => labels,Err(error) => {return Err(handle_operation_error(@@ -114,21 +129,24 @@ pub async fn update_labels(}};- let labels_to_add: Vec<String> = gmail_message_labels- .iter()- .filter(|gmail_label| {- !db_message_labels- .iter()- .any(|db_label| &db_label.provider_label_id == *gmail_label)- })- .cloned()- .collect();-- let labels_to_delete: Vec<String> = db_message_labels+ let db_label_ids: Vec<String> = db_message_labels.iter()- .filter(|db_label| !gmail_message_labels.contains(&db_label.provider_label_id)).map(|label| label.provider_label_id.clone()).collect();+ let LabelDiff {+ labels_to_add,+ labels_to_delete,+ kept_pending,+ } = label_diff(&gmail_message_labels, &db_label_ids, &pending_label_ids);++ if !kept_pending.is_empty() {+ tracing::info!(+ link_id = %link.id,+ db_message_id = %db_message.db_id,+ kept_labels = ?kept_pending,+ "Kept local labels with a pending Gmail write over the provider's labels"+ );+ }let has_label_changes = !labels_to_add.is_empty() || !labels_to_delete.is_empty();@@ -177,6 +195,39 @@ pub async fn update_labels(Ok(())}+/// How one message's database labels change to match Gmail's.+#[derive(Debug, PartialEq, Eq)]+struct LabelDiff {+ labels_to_add: Vec<String>,+ labels_to_delete: Vec<String>,+ /// Labels Gmail disagrees on that keep their local state, because Macro's+ /// own write to them has not reached Gmail yet.+ kept_pending: Vec<String>,+}++fn label_diff(+ gmail_labels: &[String],+ db_labels: &[String],+ pending_labels: &[String],+) -> LabelDiff {+ let (kept_additions, labels_to_add): (Vec<String>, Vec<String>) = gmail_labels+ .iter()+ .filter(|label| !db_labels.contains(label))+ .cloned()+ .partition(|label| pending_labels.contains(label));+ let (kept_deletions, labels_to_delete): (Vec<String>, Vec<String>) = db_labels+ .iter()+ .filter(|label| !gmail_labels.contains(label))+ .cloned()+ .partition(|label| pending_labels.contains(label));++ LabelDiff {+ labels_to_add,+ labels_to_delete,+ kept_pending: [kept_additions, kept_deletions].concat(),+ }+}+/// System labels that never appear in a `thread_labels_updated` event:/// INBOX/TRASH/UNREAD/STARRED/SPAM map to dedicated thread-state events, and/// IMPORTANT/SENT/DRAFT changes are not published.@@ -385,14 +436,22 @@ pub async fn add_message_labels(})?;let result = async {- insert::insert_message_labels(&mut tx, link.id, message_db_id, provider_label_ids, false)- .await- .map_err(|e| {- ProcessingError::Retryable(DetailedError {- reason: FailureReason::DatabaseQueryFailed,- source: e.context("Failed to insert message labels".to_string()),- })- })?;+ // `label_diff` already left pending labels out of `provider_label_ids`.+ insert::insert_message_labels(+ &mut tx,+ link.id,+ message_db_id,+ provider_label_ids,+ false,+ &[],+ )+ .await+ .map_err(|e| {+ ProcessingError::Retryable(DetailedError {+ reason: FailureReason::DatabaseQueryFailed,+ source: e.context("Failed to insert message labels".to_string()),+ })+ })?;// if we are adding unread label, mark message as not readif provider_label_idsdiff --git a/services/email_service/src/pubsub/inbox_sync/operations/update_labels/test.rs b/services/email_service/src/pubsub/inbox_sync/operations/update_labels/test.rsindex 24cba89e10..d2420382a5 100644--- a/services/email_service/src/pubsub/inbox_sync/operations/update_labels/test.rs+++ b/services/email_service/src/pubsub/inbox_sync/operations/update_labels/test.rs@@ -70,3 +70,54 @@ fn ignores_diffs_without_spam_changes() {fn spam_remains_excluded_from_user_label_events() {assert!(!is_user_label(service::label::system_labels::SPAM));}++#[test]+fn pending_archive_keeps_inbox_removed_while_gmail_still_has_it() {+ let gmail = vec!["INBOX".to_string(), "UNREAD".to_string()];+ let db = vec!["UNREAD".to_string()];+ let pending = vec!["INBOX".to_string()];++ assert_eq!(+ label_diff(&gmail, &db, &pending),+ LabelDiff {+ labels_to_add: vec![],+ labels_to_delete: vec![],+ kept_pending: vec!["INBOX".to_string()],+ }+ );+}++#[test]+fn pending_label_add_is_kept_while_other_provider_changes_apply() {+ let gmail = vec!["INBOX".to_string(), "STARRED".to_string()];+ let db = vec![+ "INBOX".to_string(),+ "UNREAD".to_string(),+ "TRASH".to_string(),+ ];+ let pending = vec!["TRASH".to_string()];++ assert_eq!(+ label_diff(&gmail, &db, &pending),+ LabelDiff {+ labels_to_add: vec!["STARRED".to_string()],+ labels_to_delete: vec!["UNREAD".to_string()],+ kept_pending: vec!["TRASH".to_string()],+ }+ );+}++#[test]+fn without_pending_writes_the_database_follows_gmail() {+ let gmail = vec!["INBOX".to_string()];+ let db = vec!["TRASH".to_string()];++ assert_eq!(+ label_diff(&gmail, &db, &[]),+ LabelDiff {+ labels_to_add: vec!["INBOX".to_string()],+ labels_to_delete: vec!["TRASH".to_string()],+ kept_pending: vec![],+ }+ );+}diff --git a/services/email_service/src/pubsub/inbox_sync/operations/upsert_message.rs b/services/email_service/src/pubsub/inbox_sync/operations/upsert_message.rsindex a403d2584a..790f6d9e1e 100644--- a/services/email_service/src/pubsub/inbox_sync/operations/upsert_message.rs+++ b/services/email_service/src/pubsub/inbox_sync/operations/upsert_message.rs@@ -1,3 +1,4 @@+use super::pending_labels;use crate::pubsub::context::PubSubContext;use crate::pubsub::inbox_sync::email_api_error::handle_operation_error;use crate::pubsub::util::{@@ -17,10 +18,11 @@ use email::domain::events::{use email::domain::models::{PreviewCursorQuery, PreviewView, PreviewViewStandardLabel};use email::domain::ports::EmailRepo;use email::outbound::EmailPgRepo;-use email_api_client::domain::models::EmailApiError;+use email_api_client::domain::models::{EmailApiError, MessageWithCalendarParts};use email_db_client::attachments::provider::upload_filters::{attachment_is_document, attachment_is_media,};+use email_db_client::messages::insert::ResyncSnapshot;use email_db_client::threads;use email_utils::{dedupe_emails, is_macro_notification_sender};use filter_ast::Expr;@@ -81,6 +83,28 @@ fn select_message_sync_event(}}+/// Fetches the message from Gmail, or `None` when Gmail no longer has it.+async fn fetch_provider_message(+ ctx: &PubSubContext,+ link: &link::Link,+ payload: &UpsertMessagePayload,+) -> result::Result<Option<MessageWithCalendarParts>, ProcessingError> {+ match ctx+ .email_api+ .get_message(link.id, &payload.provider_message_id)+ .await+ {+ Ok(fetched) => Ok(fetched),+ Err(error) => Err(handle_operation_error(+ ctx,+ link.id,+ InboxSyncOperation::UpsertMessage(payload.clone()),+ error,+ )+ .await),+ }+}+// upsert a message into the db. could be a new message or an existing one that had changes#[tracing::instrument(skip(ctx))]pub async fn upsert_message(@@ -88,26 +112,47 @@ pub async fn upsert_message(link: &link::Link,payload: &UpsertMessagePayload,) -> result::Result<(), ProcessingError> {- let provider_message = ctx- .email_api- .get_message(link.id, &payload.provider_message_id)- .await;- let fetched = match provider_message {- Ok(Some(fetched)) => fetched,- Ok(None) => {- tracing::debug!(provider_message_id = %payload.provider_message_id, link_id = %link.id,- "Message not found in gmail when attempting to upsert");- return Ok(());- }- Err(error) => {- return Err(handle_operation_error(- ctx,- link.id,- InboxSyncOperation::UpsertMessage(payload.clone()),- error,+ // Snapshot the previous draft state before the upsert. Provider drafts are mutable,+ // and this state distinguishes a draft-to-sent transition from an unchanged message.+ let existing_message =+ email_db_client::messages::get_simple_messages::get_simple_message_by_provider_and_link(+ &ctx.db,+ &payload.provider_message_id,+ &link.id,+ )+ .await+ .map_err(|e| {+ ProcessingError::NonRetryable(DetailedError {+ reason: FailureReason::DatabaseQueryFailed,+ source: e+ .context("Failed to fetch existing message before provider upsert".to_string()),+ })+ })?;+ let existing_message_was_draft = existing_message.as_ref().map(|message| message.is_draft);+ let message_already_exists = existing_message.is_some();++ let (fetched, resync) = match &existing_message {+ Some(existing) => {+ let (fetched, snapshot) = pending_labels::resync_around(+ &ctx.db,+ existing.db_id,+ Box::pin(fetch_provider_message(ctx, link, payload)),)- .await);+ .await+ .map_err(|e| {+ ProcessingError::Retryable(DetailedError {+ reason: FailureReason::DatabaseQueryFailed,+ source: e.context("Failed to read the stored message before its re-sync"),+ })+ })?;+ (fetched?, Some(snapshot))}+ None => (fetch_provider_message(ctx, link, payload).await?, None),+ };+ let Some(fetched) = fetched else {+ tracing::debug!(provider_message_id = %payload.provider_message_id, link_id = %link.id,+ "Message not found in gmail when attempting to upsert");+ return Ok(());};let mut message = fetched.message;@@ -194,25 +239,6 @@ pub async fn upsert_message(})})?;- // Snapshot the previous draft state before the upsert. Provider drafts are mutable,- // and this state distinguishes a draft-to-sent transition from an unchanged message.- let existing_message =- email_db_client::messages::get_simple_messages::get_simple_message_by_provider_and_link(- &ctx.db,- &payload.provider_message_id,- &link.id,- )- .await- .map_err(|e| {- ProcessingError::NonRetryable(DetailedError {- reason: FailureReason::DatabaseQueryFailed,- source: e- .context("Failed to fetch existing message before provider upsert".to_string()),- })- })?;- let existing_message_was_draft = existing_message.as_ref().map(|message| message.is_draft);- let message_already_exists = existing_message.is_some();-let is_new_thread = !thread_provider_to_db_map.contains_key(&provider_thread_id);// if the message's thread doesn't exist in the database, we need to fetch and insert the whole thread.@@ -220,7 +246,14 @@ pub async fn upsert_message(if let Some(thread_db_id) = thread_provider_to_db_map.get(&provider_thread_id) {// Retryable: a concurrent delete can drop the thread after the lookup// above, and the redelivery then takes the whole-thread branch below.- process_and_insert_message(&ctx.db, link.id, *thread_db_id, &mut message).await?;+ process_and_insert_message(+ &ctx.db,+ link.id,+ *thread_db_id,+ &mut message,+ resync.as_ref(),+ )+ .await?;} else {// Propagated verbatim: fetch_and_insert_thread already routed provider// errors through handle_operation_error (retry queue vs redelivery),@@ -689,17 +722,25 @@ async fn process_and_insert_message(link_id: Uuid,thread_db_id: Uuid,message: &mut Message,+ resync: Option<&ResyncSnapshot>,) -> result::Result<(), ProcessingError> {process_message_pre_insert(message).await;- email_db_client::messages::insert::insert_message(db, thread_db_id, message, link_id, true)- .await- .map_err(|e| {- ProcessingError::Retryable(DetailedError {- reason: FailureReason::DatabaseQueryFailed,- source: e.context("Failed to insert message".to_string()),- })+ email_db_client::messages::insert::insert_message(+ db,+ thread_db_id,+ message,+ link_id,+ true,+ resync,+ )+ .await+ .map_err(|e| {+ ProcessingError::Retryable(DetailedError {+ reason: FailureReason::DatabaseQueryFailed,+ source: e.context("Failed to insert message".to_string()),})+ })}/// Notify downstream services about new message in a user's inboxdiff --git a/services/email_service/src/pubsub/inbox_sync/operations/upsert_message/test.rs b/services/email_service/src/pubsub/inbox_sync/operations/upsert_message/test.rsindex 9fdeb1e77d..ae935bea6d 100644--- a/services/email_service/src/pubsub/inbox_sync/operations/upsert_message/test.rs+++ b/services/email_service/src/pubsub/inbox_sync/operations/upsert_message/test.rs@@ -391,7 +391,7 @@ async fn inserting_into_a_concurrently_deleted_thread_is_retryable(pool: PgPool).unwrap();let mut message = received_message(link_id, thread_db_id);- let result = process_and_insert_message(&pool, link_id, thread_db_id, &mut message).await;+ let result = process_and_insert_message(&pool, link_id, thread_db_id, &mut message, None).await;match result {Err(ProcessingError::Retryable(e)) => assert!(diff --git a/services/email_service/src/pubsub/label_write_sim/fake_gmail.rs b/services/email_service/src/pubsub/label_write_sim/fake_gmail.rsnew file mode 100644index 0000000000..f365fd1b96--- /dev/null+++ b/services/email_service/src/pubsub/label_write_sim/fake_gmail.rs@@ -0,0 +1,472 @@+//! An in-process Gmail REST server holding one mailbox's labels and history.+//!+//! The real `GmailClient` talks to it over HTTP, so request encoding, status+//! mapping (429 → provider rate limit, 5xx → transient, 400 → permanent) and+//! response decoding are the production code paths.++use axum::Router;+use axum::body::Bytes;+use axum::extract::State;+use axum::http::{Method, StatusCode, Uri};+use axum::response::{IntoResponse, Response};+use base64::Engine;+use serde_json::{Value, json};+use std::collections::{BTreeMap, BTreeSet, VecDeque};+use std::sync::{Arc, Mutex};+use tokio::sync::oneshot;++/// How the fake answers one request.+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, PartialOrd, Ord)]+pub enum Reply {+ #[default]+ Ok,+ /// 429 "Too many concurrent requests for user".+ RateLimited,+ /// 503, mapped to `EmailApiError::Transient`.+ Unavailable,+ /// 400, mapped to `EmailApiError::Permanent`.+ Rejected,+}++#[derive(Clone, Copy, Debug, PartialEq, Eq)]+pub enum Kind {+ Write,+ ReadLabels,+ ReadFull,+ Other,+}++#[derive(Clone, Debug)]+pub struct Logged {+ pub kind: Kind,+ pub ids: Vec<String>,+ pub add: Vec<String>,+ pub remove: Vec<String>,+ pub reply: Reply,+ /// Labels Gmail returned for a read.+ pub returned: Vec<String>,+}++#[derive(Clone, Debug)]+pub struct GmailMessage {+ pub thread_id: String,+ pub labels: BTreeSet<String>,+ pub internal_date_ms: i64,+}++#[derive(Clone, Debug)]+pub struct HistoryEntry {+ pub id: u64,+ pub message_id: String,+ pub thread_id: String,+ pub added: Vec<String>,+ pub removed: Vec<String>,+ pub message_added: bool,+}++/// Where a paused request waits.+#[derive(Clone, Copy, Debug, PartialEq, Eq)]+pub enum PauseAt {+ /// The request has not reached Gmail yet: it is applied on release.+ BeforeApply,+ /// Gmail answered (and applied a write) on arrival; the answer is+ /// delivered on release.+ AfterApply,+}++struct Pause {+ kind: Kind,+ message: Option<String>,+ at: PauseAt,+ arrived: oneshot::Sender<()>,+ release: oneshot::Receiver<()>,+}++pub struct PauseHandle {+ pub arrived: oneshot::Receiver<()>,+ pub release: oneshot::Sender<()>,+}++#[derive(Default)]+pub struct GmailState {+ pub messages: BTreeMap<String, GmailMessage>,+ pub labels: Vec<(String, String, String)>,+ pub history_id: u64,+ pub history: Vec<HistoryEntry>,+ pub next_writes: VecDeque<Reply>,+ pub default_write: Reply,+ pub next_reads: VecDeque<Reply>,+ pub default_read: Reply,+ pub log: Vec<Logged>,+ pause: Option<Pause>,+}++#[derive(Clone)]+pub struct FakeGmail {+ pub state: Arc<Mutex<GmailState>>,+ pub url: String,+}++enum Request {+ Modify {+ ids: Vec<String>,+ add: Vec<String>,+ remove: Vec<String>,+ },+ GetMessage {+ id: String,+ minimal: bool,+ },+ GetThread {+ id: String,+ },+ ListLabels,+ History {+ start: u64,+ },+ Other,+}++impl Request {+ fn kind(&self) -> Kind {+ match self {+ Request::Modify { .. } => Kind::Write,+ Request::GetMessage { minimal: true, .. } => Kind::ReadLabels,+ Request::GetMessage { minimal: false, .. } => Kind::ReadFull,+ _ => Kind::Other,+ }+ }++ fn message_ids(&self) -> Vec<String> {+ match self {+ Request::Modify { ids, .. } => ids.clone(),+ Request::GetMessage { id, .. } => vec![id.clone()],+ _ => Vec::new(),+ }+ }+}++impl FakeGmail {+ pub async fn start(state: GmailState) -> FakeGmail {+ let state = Arc::new(Mutex::new(state));+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0")+ .await+ .expect("bind fake gmail");+ let url = format!("http://{}/gmail/v1", listener.local_addr().unwrap());+ let fake = FakeGmail { state, url };+ let app = Router::new().fallback(handle).with_state(fake.clone());+ tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });+ fake+ }++ pub fn lock(&self) -> std::sync::MutexGuard<'_, GmailState> {+ self.state.lock().unwrap()+ }++ /// Holds the next request of `kind` (for `message`, when given) until the+ /// returned handle releases it.+ pub fn pause_next(&self, kind: Kind, message: Option<&str>, at: PauseAt) -> PauseHandle {+ let (arrived_tx, arrived_rx) = oneshot::channel();+ let (release_tx, release_rx) = oneshot::channel();+ self.lock().pause = Some(Pause {+ kind,+ message: message.map(str::to_string),+ at,+ arrived: arrived_tx,+ release: release_rx,+ });+ PauseHandle {+ arrived: arrived_rx,+ release: release_tx,+ }+ }++ pub fn clear_pause(&self) {+ self.lock().pause = None;+ }++ /// A change made in Gmail itself (web UI, phone), recorded in history.+ pub fn external_change(&self, message: &str, add: &[&str], remove: &[&str]) {+ let mut state = self.lock();+ let add: Vec<String> = add.iter().map(|label| label.to_string()).collect();+ let remove: Vec<String> = remove.iter().map(|label| label.to_string()).collect();+ state.apply(&[message.to_string()], &add, &remove);+ }++ fn choose_reply(&self, request: &Request) -> Reply {+ let mut state = self.lock();+ match request.kind() {+ Kind::Write => state.next_writes.pop_front().unwrap_or(state.default_write),+ Kind::ReadLabels | Kind::ReadFull => {+ state.next_reads.pop_front().unwrap_or(state.default_read)+ }+ Kind::Other => Reply::Ok,+ }+ }++ fn respond(&self, request: &Request, reply: Reply) -> Response {+ let mut state = self.lock();+ let kind = request.kind();+ let (add, remove) = match request {+ Request::Modify { add, remove, .. } => (add.clone(), remove.clone()),+ _ => (Vec::new(), Vec::new()),+ };+ let mut entry = Logged {+ kind,+ ids: request.message_ids(),+ add,+ remove,+ reply,+ returned: Vec::new(),+ };+ let response = match reply {+ Reply::RateLimited => error(429, "Too many concurrent requests for user."),+ Reply::Unavailable => error(503, "The service is currently unavailable."),+ Reply::Rejected => error(400, "Invalid label"),+ Reply::Ok => match request {+ Request::Modify { ids, add, remove } => {+ state.apply(ids, add, remove);+ StatusCode::NO_CONTENT.into_response()+ }+ Request::GetMessage { id, minimal } => match state.messages.get(id) {+ None => error(404, "Requested entity was not found."),+ Some(message) => {+ entry.returned = message.labels.iter().cloned().collect();+ let body = if *minimal {+ json!({+ "id": id,+ "threadId": message.thread_id,+ "labelIds": message.labels,+ })+ } else {+ full_message(id, message, state.history_id)+ };+ (StatusCode::OK, axum::Json(body)).into_response()+ }+ },+ Request::GetThread { id } => {+ let messages: Vec<Value> = state+ .messages+ .iter()+ .filter(|(_, message)| message.thread_id == *id)+ .map(|(message_id, message)| {+ full_message(message_id, message, state.history_id)+ })+ .collect();+ (+ StatusCode::OK,+ axum::Json(json!({ "id": id, "messages": messages })),+ )+ .into_response()+ }+ Request::ListLabels => {+ let labels: Vec<Value> = state+ .labels+ .iter()+ .map(|(id, name, label_type)| {+ json!({+ "id": id,+ "name": name,+ "type": label_type,+ "messageListVisibility": "show",+ "labelListVisibility": "labelShow",+ })+ })+ .collect();+ (StatusCode::OK, axum::Json(json!({ "labels": labels }))).into_response()+ }+ Request::History { start } => {+ let history: Vec<Value> = state+ .history+ .iter()+ .filter(|entry| entry.id > *start)+ .map(history_record)+ .collect();+ (+ StatusCode::OK,+ axum::Json(json!({+ "history": history,+ "historyId": state.history_id.to_string(),+ })),+ )+ .into_response()+ }+ Request::Other => (StatusCode::OK, axum::Json(json!({}))).into_response(),+ },+ };+ state.log.push(entry);+ response+ }+}++impl GmailState {+ fn apply(&mut self, ids: &[String], add: &[String], remove: &[String]) {+ for id in ids {+ // batchModify ignores ids that name no message.+ let Some(message) = self.messages.get_mut(id) else {+ continue;+ };+ let mut added = Vec::new();+ let mut removed = Vec::new();+ for label in add {+ if message.labels.insert(label.clone()) {+ added.push(label.clone());+ }+ }+ for label in remove {+ if message.labels.remove(label) {+ removed.push(label.clone());+ }+ }+ if added.is_empty() && removed.is_empty() {+ continue;+ }+ let thread_id = message.thread_id.clone();+ self.history_id += 1;+ let entry = HistoryEntry {+ id: self.history_id,+ message_id: id.clone(),+ thread_id,+ added,+ removed,+ message_added: false,+ };+ self.history.push(entry);+ }+ }++ fn take_pause_for(&mut self, request: &Request) -> Option<Pause> {+ let pause = self.pause.as_ref()?;+ if pause.kind != request.kind() {+ return None;+ }+ if let Some(message) = &pause.message+ && !request.message_ids().contains(message)+ {+ return None;+ }+ self.pause.take()+ }+}++async fn handle(State(gmail): State<FakeGmail>, method: Method, uri: Uri, body: Bytes) -> Response {+ let request = parse(&method, &uri, &body);+ // The answer is decided on arrival, even when it is applied later.+ let reply = gmail.choose_reply(&request);+ let pause = gmail.lock().take_pause_for(&request);+ match pause {+ None => gmail.respond(&request, reply),+ Some(pause) => match pause.at {+ PauseAt::BeforeApply => {+ let _ = pause.arrived.send(());+ let _ = pause.release.await;+ gmail.respond(&request, reply)+ }+ PauseAt::AfterApply => {+ let response = gmail.respond(&request, reply);+ let _ = pause.arrived.send(());+ let _ = pause.release.await;+ response+ }+ },+ }+}++fn parse(method: &Method, uri: &Uri, body: &[u8]) -> Request {+ let path = uri+ .path()+ .trim_start_matches("/gmail/v1/users/me/")+ .to_string();+ let query = uri.query().unwrap_or("");+ let segments: Vec<&str> = path.split('/').collect();+ let json: Value = serde_json::from_slice(body).unwrap_or(Value::Null);+ let strings = |field: &str| -> Vec<String> {+ json.get(field)+ .and_then(Value::as_array)+ .map(|values| {+ values+ .iter()+ .filter_map(|value| value.as_str().map(str::to_string))+ .collect()+ })+ .unwrap_or_default()+ };+ match (method.as_str(), segments.as_slice()) {+ ("POST", ["messages", "batchModify"]) => Request::Modify {+ ids: strings("ids"),+ add: strings("addLabelIds"),+ remove: strings("removeLabelIds"),+ },+ ("POST", ["messages", id, "modify"]) => Request::Modify {+ ids: vec![id.to_string()],+ add: strings("addLabelIds"),+ remove: strings("removeLabelIds"),+ },+ ("GET", ["messages", id]) => Request::GetMessage {+ id: id.to_string(),+ minimal: query.contains("format=minimal"),+ },+ ("GET", ["threads", id]) => Request::GetThread { id: id.to_string() },+ ("GET", ["labels"]) => Request::ListLabels,+ ("GET", ["history"]) => Request::History {+ start: query+ .split('&')+ .find_map(|pair| pair.strip_prefix("startHistoryId="))+ .and_then(|value| value.parse().ok())+ .unwrap_or(0),+ },+ _ => Request::Other,+ }+}++fn error(code: u16, message: &str) -> Response {+ (+ StatusCode::from_u16(code).unwrap(),+ axum::Json(json!({ "error": { "code": code, "message": message } })),+ )+ .into_response()+}++fn full_message(id: &str, message: &GmailMessage, history_id: u64) -> Value {+ let body = base64::engine::general_purpose::URL_SAFE.encode(format!("Body of {id}"));+ let date = chrono::DateTime::from_timestamp_millis(message.internal_date_ms)+ .unwrap()+ .to_rfc2822();+ json!({+ "id": id,+ "threadId": message.thread_id,+ "labelIds": message.labels,+ "snippet": format!("Body of {id}"),+ "sizeEstimate": 100,+ "historyId": history_id.to_string(),+ "internalDate": message.internal_date_ms.to_string(),+ "payload": {+ "partId": "",+ "mimeType": "text/plain",+ "filename": "",+ "headers": [+ { "name": "From", "value": "Alice Sender <[email protected]>" },+ { "name": "To", "value": "Sim Owner <[email protected]>" },+ { "name": "Subject", "value": format!("Thread {}", message.thread_id) },+ { "name": "Date", "value": date },+ { "name": "Message-ID", "value": format!("<{id}@example.org>") },+ ],+ "body": { "size": 12, "data": body },+ },+ })+}++fn history_record(entry: &HistoryEntry) -> Value {+ let message = json!({ "id": entry.message_id, "threadId": entry.thread_id });+ let mut record = json!({ "id": entry.id.to_string(), "messages": [message] });+ if !entry.added.is_empty() {+ record["labelsAdded"] = json!([{ "message": message, "labelIds": entry.added }]);+ }+ if !entry.removed.is_empty() {+ record["labelsRemoved"] = json!([{ "message": message, "labelIds": entry.removed }]);+ }+ if entry.message_added {+ record["messagesAdded"] = json!([{ "message": message }]);+ }+ record+}diff --git a/services/email_service/src/pubsub/label_write_sim/fake_sqs.rs b/services/email_service/src/pubsub/label_write_sim/fake_sqs.rsnew file mode 100644index 0000000000..20f585376f--- /dev/null+++ b/services/email_service/src/pubsub/label_write_sim/fake_sqs.rs@@ -0,0 +1,218 @@+//! An in-process SQS endpoint speaking the AWS JSON 1.0 protocol.+//!+//! The production `sqs_client::SQS` and `sqs_worker::SQSWorker` send real SDK+//! requests here. Sends, deletes and visibility changes are recorded for the+//! simulation, which owns delivery and the virtual clock.++use axum::Router;+use axum::body::Bytes;+use axum::extract::State;+use axum::http::{HeaderMap, StatusCode};+use axum::response::{IntoResponse, Response};+use serde_json::{Value, json};+use std::sync::{Arc, Mutex};+use tokio::sync::oneshot;++#[derive(Clone, Debug)]+pub struct Sent {+ pub queue_url: String,+ pub body: String,+ pub delay_seconds: u64,+ pub message_id: String,+}++#[derive(Clone, Debug)]+pub enum Ack {+ Deleted { receipt: String },+ Visibility { receipt: String, seconds: u64 },+}++#[derive(Default)]+pub struct SqsState {+ pub sent: Vec<Sent>,+ pub acks: Vec<Ack>,+ /// While set, every SendMessage fails, as if the SDK had used up its+ /// retries against an SQS outage.+ pub outage: bool,+ pub failed_sends: Vec<Sent>,+ next_id: u64,+ /// Holds the next SendMessage until released.+ paused_send: Option<(oneshot::Sender<()>, oneshot::Receiver<()>)>,+}++#[derive(Clone)]+pub struct FakeSqs {+ pub state: Arc<Mutex<SqsState>>,+ pub url: String,+}++impl FakeSqs {+ pub async fn start() -> FakeSqs {+ let state = Arc::new(Mutex::new(SqsState::default()));+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0")+ .await+ .expect("bind fake sqs");+ let url = format!("http://{}", listener.local_addr().unwrap());+ let fake = FakeSqs { state, url };+ let app = Router::new().fallback(handle).with_state(fake.clone());+ tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });+ fake+ }++ pub fn queue_url(&self, name: &str) -> String {+ format!("{}/000000000000/{name}", self.url)+ }++ pub fn lock(&self) -> std::sync::MutexGuard<'_, SqsState> {+ self.state.lock().unwrap()+ }++ pub fn take(&self) -> (Vec<Sent>, Vec<Ack>) {+ let mut state = self.lock();+ (+ std::mem::take(&mut state.sent),+ std::mem::take(&mut state.acks),+ )+ }++ pub fn set_outage(&self, outage: bool) {+ self.lock().outage = outage;+ }++ /// Holds the next SendMessage. Returns (arrived, release).+ pub fn pause_next_send(&self) -> (oneshot::Receiver<()>, oneshot::Sender<()>) {+ let (arrived_tx, arrived_rx) = oneshot::channel();+ let (release_tx, release_rx) = oneshot::channel();+ self.lock().paused_send = Some((arrived_tx, release_rx));+ (arrived_rx, release_tx)+ }+}++/// An SDK client for `endpoint` without SDK retries: an injected outage fails+/// the call at once instead of after the default backoff.+pub fn client(endpoint: &str, retries: bool) -> aws_sdk_sqs::Client {+ let mut config = aws_sdk_sqs::Config::builder()+ .behavior_version(aws_sdk_sqs::config::BehaviorVersion::latest())+ .region(aws_sdk_sqs::config::Region::new("us-east-1"))+ .credentials_provider(aws_sdk_sqs::config::Credentials::new(+ "test", "test", None, None, "harness",+ ))+ .endpoint_url(endpoint);+ if !retries {+ config = config.retry_config(aws_sdk_sqs::config::retry::RetryConfig::disabled());+ }+ aws_sdk_sqs::Client::from_conf(config.build())+}++async fn handle(State(sqs): State<FakeSqs>, headers: HeaderMap, body: Bytes) -> Response {+ let target = headers+ .get("x-amz-target")+ .and_then(|value| value.to_str().ok())+ .unwrap_or("")+ .trim_start_matches("AmazonSQS.")+ .to_string();+ let request: Value = serde_json::from_slice(&body).unwrap_or(Value::Null);+ let text = |field: &str| {+ request+ .get(field)+ .and_then(Value::as_str)+ .unwrap_or("")+ .to_string()+ };+ if target == "SendMessage" || target == "SendMessageBatch" {+ let paused = sqs.lock().paused_send.take();+ if let Some((arrived, release)) = paused {+ let _ = arrived.send(());+ let _ = release.await;+ }+ }+ let mut state = sqs.lock();+ match target.as_str() {+ "SendMessage" => {+ state.next_id += 1;+ let sent = Sent {+ queue_url: text("QueueUrl"),+ body: text("MessageBody"),+ delay_seconds: request+ .get("DelaySeconds")+ .and_then(Value::as_u64)+ .unwrap_or(0),+ message_id: format!("msg-{}", state.next_id),+ };+ if state.outage {+ state.failed_sends.push(sent);+ return (+ StatusCode::INTERNAL_SERVER_ERROR,+ [("content-type", "application/x-amz-json-1.0")],+ json!({ "__type": "com.amazonaws.sqs#InternalError", "message": "injected outage" })+ .to_string(),+ )+ .into_response();+ }+ let message_id = sent.message_id.clone();+ state.sent.push(sent);+ json_response(json!({ "MessageId": message_id }))+ }+ "SendMessageBatch" => {+ if state.outage {+ return (+ StatusCode::INTERNAL_SERVER_ERROR,+ [("content-type", "application/x-amz-json-1.0")],+ json!({ "__type": "com.amazonaws.sqs#InternalError", "message": "injected outage" })+ .to_string(),+ )+ .into_response();+ }+ let queue_url = text("QueueUrl");+ let mut successful = Vec::new();+ for entry in request+ .get("Entries")+ .and_then(Value::as_array)+ .cloned()+ .unwrap_or_default()+ {+ state.next_id += 1;+ let message_id = format!("msg-{}", state.next_id);+ state.sent.push(Sent {+ queue_url: queue_url.clone(),+ body: entry["MessageBody"].as_str().unwrap_or("").to_string(),+ delay_seconds: entry["DelaySeconds"].as_u64().unwrap_or(0),+ message_id: message_id.clone(),+ });+ successful.push(json!({+ "Id": entry["Id"],+ "MessageId": message_id,+ "MD5OfMessageBody": "",+ }));+ }+ json_response(json!({ "Successful": successful, "Failed": [] }))+ }+ "DeleteMessage" => {+ state.acks.push(Ack::Deleted {+ receipt: text("ReceiptHandle"),+ });+ json_response(json!({}))+ }+ "ChangeMessageVisibility" => {+ state.acks.push(Ack::Visibility {+ receipt: text("ReceiptHandle"),+ seconds: request+ .get("VisibilityTimeout")+ .and_then(Value::as_u64)+ .unwrap_or(0),+ });+ json_response(json!({}))+ }+ "ReceiveMessage" => json_response(json!({ "Messages": [] })),+ other => panic!("fake SQS does not implement {other}"),+ }+}++fn json_response(body: Value) -> Response {+ (+ StatusCode::OK,+ [("content-type", "application/x-amz-json-1.0")],+ body.to_string(),+ )+ .into_response()+}diff --git a/services/email_service/src/pubsub/label_write_sim/mod.rs b/services/email_service/src/pubsub/label_write_sim/mod.rsnew file mode 100644index 0000000000..e116327e42--- /dev/null+++ b/services/email_service/src/pubsub/label_write_sim/mod.rs@@ -0,0 +1,12 @@+//! One Gmail mailbox wired to the production label-write, gmail_ops and inbox+//! sync code, with fake Gmail and SQS endpoints. Its tests replay known races+//! between Macro's label writes, Gmail's answers and syncs, and a seeded+//! simulation checks that sync never undoes an unconfirmed write and that+//! Gmail and the database end on the user's last effective action.++mod fake_gmail;+mod fake_sqs;+mod sim;+mod test;+mod wiring;+mod world;diff --git a/services/email_service/src/pubsub/label_write_sim/sim.rs b/services/email_service/src/pubsub/label_write_sim/sim.rsnew file mode 100644index 0000000000..1bbfea6e48--- /dev/null+++ b/services/email_service/src/pubsub/label_write_sim/sim.rs@@ -0,0 +1,1056 @@+//! Seeded simulation of user label actions, Gmail answers, queue redelivery+//! and inbox syncs, run against the real worker and domain code.+//!+//! SQS is modelled on a virtual clock: the fake SQS endpoint records what the+//! production code sends, deletes and delays, and the simulation decides when+//! each message is delivered to `gmail_ops::process::process_message` or+//! `inbox_sync::process::process_message`.++mod generator;+mod model;+mod test;++use super::fake_gmail::{Kind, Logged, PauseAt, Reply};+use super::fake_sqs::Ack;+use super::wiring::{self, LabelOp};+use super::world::{self, MESSAGES, THREADS, TRACKED, WORK, World};+use aws_sdk_sqs::types::{Message, MessageSystemAttributeName};+use email::domain::models::mailbox_action::MailboxAction;+use email::domain::ports::EmailService;+use models_email::gmail::inbox_sync::{+ InboxSyncOperation, InboxSyncPubsubMessage, UpdateLabelsPayload, UpsertMessagePayload,+};+use rand::rngs::StdRng;+use rand::{Rng, SeedableRng};+use std::collections::{BTreeMap, BTreeSet, HashMap};+use std::fmt::Write as _;+use std::time::Duration;+use uuid::Uuid;++#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]+pub enum Queue {+ Ops,+ OpsRetry,+ Sync,+ SyncRetry,+}++impl Queue {+ fn max_receives(self) -> u32 {+ match self {+ Queue::Ops | Queue::Sync => 3,+ Queue::OpsRetry | Queue::SyncRetry => 100,+ }+ }+}++#[derive(Clone, Debug)]+pub struct Queued {+ pub id: String,+ pub queue: Queue,+ pub body: String,+ pub visible_at: u64,+ pub receives: u32,+ /// The label write this message carries, if it is a gmail_ops write.+ pub lineage: Option<u64>,+}++#[derive(Clone, Copy, Debug, PartialEq, Eq)]+enum EntryState {+ Live,+ Rejected,+ DeadLettered,+}++#[derive(Clone, Debug)]+struct Entry {+ seq: u64,+ lineage: u64,+ value: bool,+ state: EntryState,+}++#[derive(Clone, Copy, Debug, PartialEq, Eq)]+enum OpStatus {+ Queued,+ Applied,+ Rejected,+ Skipped,+ DeadLettered,+}++#[derive(Clone, Debug)]+struct OpTrack {+ seq: u64,+ op: LabelOp,+ label: String,+ value: bool,+ status: OpStatus,+ /// Messages Gmail accepted this write for, over all its deliveries.+ accepted: BTreeSet<String>,+}++#[derive(Clone, Debug)]+pub enum Action {+ Archive(usize, bool),+ Read(usize, bool),+ Star(usize, bool),+ Label(usize, bool),+ Trash(usize, bool),+ BulkArchive(Vec<usize>, bool),+}++#[derive(Clone, Debug)]+pub enum Step {+ Action {+ action: Action,+ enqueue_fails: bool,+ },+ Deliver {+ reply: Reply,+ gate_refuses: bool,+ enqueue_fails: bool,+ },+ Sync {+ message: usize,+ reply: Reply,+ },+ Resync {+ message: usize,+ reply: Reply,+ },+ Wait {+ seconds: u64,+ },+ /// Twelve hours pass: every queued message is worked off, then markers age+ /// past their expiry.+ Expire,+ /// `first` reaches Gmail, `seconds` run to completion, then `first`+ /// finishes.+ Overlap {+ first: Box<Step>,+ seconds: Vec<Step>,+ at: PauseAt,+ },+}++#[derive(Clone, Debug)]+pub struct Violation {+ pub invariant: &'static str,+ pub step: usize,+ pub detail: String,+}++impl Violation {+ /// The kind of step after which the violation showed, e.g. `Sync`.+ pub fn step_kind(&self) -> String {+ let text = self+ .detail+ .trim_start_matches("after ")+ .trim_start_matches("drain ");+ text.split(|c: char| !c.is_alphanumeric())+ .next()+ .unwrap_or("")+ .to_string()+ }+}++#[derive(Clone, Copy, Debug)]+pub struct Config {+ pub steps: usize,+ pub overlap: bool,+ /// Producers send label write batches rather than one operation per+ /// message.+ pub label_write_batches: bool,+}++pub struct Sim<'w> {+ pub world: &'w mut World,+ rng: StdRng,+ pub now: u64,+ pub queued: Vec<Queued>,+ pub dead_letters: Vec<Queued>,+ cells: BTreeMap<(String, String), Vec<Entry>>,+ ops: BTreeMap<u64, OpTrack>,+ lineage_by_body: HashMap<String, u64>,+ /// The lineage carrying each write to each provider message: with label+ /// write batches off, one write is an operation per message.+ lineage_by_write: HashMap<(Uuid, String), u64>,+ next_lineage: u64,+ action_seq: u64,+ next_message: u64,+ pub trace: Vec<String>,+ pub violations: Vec<Violation>,+ /// Cells whose latest write dead-lettered.+ pub lost_cells: BTreeSet<(String, String)>,+ pub stats: Stats,+ step_index: usize,+}++#[derive(Clone, Debug, Default)]+pub struct Stats {+ pub actions: u64,+ pub failed_actions: u64,+ pub deliveries: u64,+ pub syncs: u64,+ pub resyncs: u64,+ pub gmail_writes: u64,+ pub gmail_429: u64,+ pub gmail_5xx: u64,+ pub gmail_400: u64,+ pub transient_stale_applies: u64,+ pub skipped_superseded: u64,+ pub dead_letters: u64,+ pub overlaps: u64,+ pub expiries: u64,+}++pub enum Source {+ Sync(usize),+ Resync(usize),+ /// The earliest message already on the queue.+ Queued(Queue),+}++pub struct Paused {+ delivery: Delivery,+ task: tokio::task::JoinHandle<()>,+ release: tokio::sync::oneshot::Sender<()>,+ window_from: u64,+ at: PauseAt,+ action_seq: u64,+}++struct Delivery {+ queued: Queued,+ message: Message,+ log_from: u64,+ /// Gmail requests made by other steps while this one was paused.+ excluded: std::ops::Range<u64>,+ /// When Gmail applied this delivery's write before later actions ran:+ /// the last action it could have seen.+ applied_before: Option<u64>,+}++impl<'w> Sim<'w> {+ pub fn new(world: &'w mut World, seed: u64) -> Sim<'w> {+ let cells = MESSAGES+ .iter()+ .flat_map(|(id, _, _)| {+ TRACKED+ .iter()+ .map(move |label| ((id.to_string(), label.to_string()), Vec::new()))+ })+ .collect();+ Sim {+ world,+ rng: StdRng::seed_from_u64(seed),+ now: 0,+ queued: Vec::new(),+ dead_letters: Vec::new(),+ cells,+ ops: BTreeMap::new(),+ lineage_by_body: HashMap::new(),+ lineage_by_write: HashMap::new(),+ next_lineage: 0,+ action_seq: 0,+ next_message: 0,+ trace: Vec::new(),+ violations: Vec::new(),+ lost_cells: BTreeSet::new(),+ stats: Stats::default(),+ step_index: 0,+ }+ }++ // --------------------------------------------------------------- queues++ fn queue_of(&self, url: &str) -> Option<Queue> {+ let queues = &self.world.queues;+ if url == queues.ops {+ Some(Queue::Ops)+ } else if url == queues.ops_retry {+ Some(Queue::OpsRetry)+ } else if url == queues.sync {+ Some(Queue::Sync)+ } else if url == queues.sync_retry {+ Some(Queue::SyncRetry)+ } else {+ None+ }+ }++ /// Takes what the code sent to SQS since the last call into the model.+ /// Writes first seen here belong to the user action `seq`.+ fn absorb_sends(&mut self, seq: Option<u64>) {+ let (sent, acks) = self.world.sqs.take();+ for sent in sent {+ let Some(queue) = self.queue_of(&sent.queue_url) else {+ continue;+ };+ let lineage = if matches!(queue, Queue::Ops | Queue::OpsRetry) {+ self.lineage_for(&sent.body, seq)+ } else {+ None+ };+ self.queued.push(Queued {+ id: sent.message_id,+ queue,+ body: sent.body,+ visible_at: self.now + sent.delay_seconds,+ receives: 0,+ lineage,+ });+ }+ assert!(acks.is_empty(), "acks outside a delivery: {acks:?}");+ }++ /// A send made by a user action is a new write; any other send of a label+ /// write is a re-enqueue (primary → retry) of the write with that body.+ /// On `main` two actions can produce the same body; the newest wins.+ fn lineage_for(&mut self, body: &str, seq: Option<u64>) -> Option<u64> {+ if seq.is_none()+ && let Some(lineage) = self.lineage_by_body.get(body)+ {+ return Some(*lineage);+ }+ let op = wiring::decode_label_op(body)?;+ let lineage = self.next_lineage;+ self.next_lineage += 1;+ let seq = seq.unwrap_or_else(|| panic!("label write sent outside a user action: {body}"));+ let (label, value) = match (op.add.first(), op.remove.first()) {+ (Some(label), None) => (label.clone(), true),+ (None, Some(label)) => (label.clone(), false),+ _ => panic!("label write changes more than one label: {op:?}"),+ };+ for (_, provider_id) in &op.messages {+ self.cells+ .get_mut(&(provider_id.clone(), label.clone()))+ .unwrap_or_else(|| panic!("untracked cell {provider_id}/{label}"))+ .push(Entry {+ seq,+ lineage,+ value,+ state: EntryState::Live,+ });+ }+ if let Some(write_id) = op.write_id {+ for (_, provider_id) in &op.messages {+ self.lineage_by_write+ .insert((write_id, provider_id.clone()), lineage);+ }+ }+ self.lineage_by_body.insert(body.to_string(), lineage);+ self.ops.insert(+ lineage,+ OpTrack {+ seq,+ op,+ label,+ value,+ status: OpStatus::Queued,+ accepted: BTreeSet::new(),+ },+ );+ Some(lineage)+ }++ fn set_entries(&mut self, lineage: u64, messages: &[String], state: EntryState) {+ let label = self.ops[&lineage].label.clone();+ for message in messages {+ for entry in self+ .cells+ .get_mut(&(message.clone(), label.clone()))+ .unwrap()+ .iter_mut()+ .filter(|entry| entry.lineage == lineage)+ {+ entry.state = state;+ }+ }+ }++ /// Picks the next message to deliver: due ones first, otherwise the clock+ /// moves to the earliest.+ fn next_due(&mut self) -> Option<usize> {+ if self.queued.is_empty() {+ return None;+ }+ let earliest = self.queued.iter().map(|queued| queued.visible_at).min()?;+ if earliest > self.now {+ self.now = earliest;+ }+ let due: Vec<usize> = (0..self.queued.len())+ .filter(|index| self.queued[*index].visible_at <= self.now)+ .collect();+ Some(due[self.rng.random_range(0..due.len())])+ }++ /// Takes a message off its queue for delivery, or dead-letters it once it+ /// has used up its receives.+ fn begin_delivery(&mut self, index: usize) -> Option<Delivery> {+ let mut queued = self.queued.remove(index);+ if queued.receives >= queued.queue.max_receives() {+ self.stats.dead_letters += 1;+ if let Some(lineage) = queued.lineage {+ let messages: Vec<String> = self.ops[&lineage]+ .op+ .messages+ .iter()+ .map(|(_, provider_id)| provider_id.clone())+ .collect();+ self.ops.get_mut(&lineage).unwrap().status = OpStatus::DeadLettered;+ self.set_entries(lineage, &messages, EntryState::DeadLettered);+ }+ self.trace.push(format!(+ "t={} dead-letter {:?} {}",+ self.now,+ queued.queue,+ short(&queued.body)+ ));+ self.dead_letters.push(queued);+ return None;+ }+ queued.receives += 1;+ let message = Message::builder()+ .message_id(&queued.id)+ .receipt_handle(format!("{}#{}", queued.id, queued.receives))+ .body(&queued.body)+ .attributes(+ MessageSystemAttributeName::ApproximateReceiveCount,+ queued.receives.to_string(),+ )+ .build();+ let log_from = self.world.gmail.lock().log.len() as u64;+ Some(Delivery {+ queued,+ message,+ log_from,+ excluded: 0..0,+ applied_before: None,+ })+ }++ /// Hands the message to the worker entry point for its queue, as the+ /// worker loop does after a receive.+ fn spawn_delivery(world: &World, delivery: &Delivery) -> tokio::task::JoinHandle<()> {+ let message = delivery.message.clone();+ match delivery.queued.queue {+ Queue::Ops | Queue::OpsRetry => {+ let ctx = if delivery.queued.queue == Queue::Ops {+ world.ops.clone()+ } else {+ world.ops_retry.clone()+ };+ tokio::spawn(async move {+ let _ = crate::pubsub::gmail_ops::process::process_message(ctx, &message).await;+ })+ }+ Queue::Sync | Queue::SyncRetry => {+ let ctx = if delivery.queued.queue == Queue::Sync {+ world.sync.clone()+ } else {+ world.sync_retry.clone()+ };+ tokio::spawn(async move {+ let _ =+ crate::pubsub::inbox_sync::process::process_message(ctx, &message).await;+ })+ }+ }+ }++ async fn run_delivery(world: &World, delivery: &Delivery) {+ Self::spawn_delivery(world, delivery)+ .await+ .expect("worker task panicked");+ }++ /// Applies a finished delivery's acknowledgements and checks that a label+ /// write left its queue only for an accountable reason.+ fn finish_delivery(&mut self, delivery: Delivery) {+ self.stats.deliveries += 1;+ let (sent, acks) = self.world.sqs.take();+ let receipt = delivery.message.receipt_handle().unwrap().to_string();+ let mut queued = delivery.queued;+ let mut deleted = false;+ let mut visibility = None;+ let mut foreign_acks = Vec::new();+ for ack in acks {+ match ack {+ Ack::Deleted { receipt: r } if r == receipt => deleted = true,+ Ack::Visibility {+ receipt: r,+ seconds,+ } if r == receipt => visibility = Some(seconds),+ other => foreign_acks.push(other),+ }+ }+ assert!(foreign_acks.is_empty(), "unexpected acks {foreign_acks:?}");+ let resent_to_retry = sent.iter().any(|sent| {+ sent.body == queued.body && self.queue_of(&sent.queue_url) == Some(Queue::OpsRetry)+ });+ // The hand-off to the retry queue carries this delivery's write.+ if resent_to_retry && let Some(lineage) = queued.lineage {+ self.lineage_by_body.insert(queued.body.clone(), lineage);+ }+ // Put the sends back for absorb_sends.+ self.world.sqs.lock().sent.extend(sent);+ let log: Vec<Logged> = self.world.gmail.lock().log[delivery.log_from as usize..]+ .iter()+ .enumerate()+ .filter(|(offset, _)| {+ !delivery+ .excluded+ .contains(&(delivery.log_from + *offset as u64))+ })+ .map(|(_, entry)| entry.clone())+ .collect();+ for entry in &log {+ match entry.reply {+ Reply::RateLimited => self.stats.gmail_429 += 1,+ Reply::Unavailable => self.stats.gmail_5xx += 1,+ Reply::Rejected => self.stats.gmail_400 += 1,+ Reply::Ok => {}+ }+ }+ self.trace.push(format!(+ "t={} deliver {:?}#{} {} -> deleted={} visibility={:?} gmail={:?}",+ self.now,+ queued.queue,+ queued.receives,+ short(&queued.body),+ deleted,+ visibility,+ log.iter()+ .map(|entry| format!("{:?}{:?}{:?}", entry.kind, entry.ids, entry.reply))+ .collect::<Vec<_>>()+ ));+ if let Some(lineage) = queued.lineage {+ self.account_write(+ lineage,+ &log,+ deleted,+ resent_to_retry,+ delivery.applied_before,+ );+ }+ if !deleted {+ queued.visible_at = self.now + visibility.unwrap_or(60);+ self.queued.push(queued);+ }+ }++ // ---------------------------------------------------------------- steps++ async fn prepare(&mut self, step: &Step) {+ wiring::reset_admission(&mut self.world.redis, self.world.link_id).await;+ {+ let mut gmail = self.world.gmail.lock();+ gmail.default_write = Reply::Ok;+ gmail.default_read = Reply::Ok;+ gmail.next_writes.clear();+ gmail.next_reads.clear();+ }+ self.world.sqs.set_outage(false);+ match step {+ Step::Deliver {+ reply,+ gate_refuses,+ enqueue_fails,+ } => {+ {+ let mut gmail = self.world.gmail.lock();+ gmail.next_writes.push_back(*reply);+ gmail.next_reads.push_back(*reply);+ }+ if *gate_refuses {+ wiring::refuse_admission(&mut self.world.redis, self.world.link_id).await;+ }+ self.world.sqs.set_outage(*enqueue_fails);+ }+ Step::Sync { reply, .. } | Step::Resync { reply, .. } => {+ self.world.gmail.lock().next_reads.push_back(*reply);+ }+ Step::Action { enqueue_fails, .. } => self.world.sqs.set_outage(*enqueue_fails),+ _ => {}+ }+ }++ fn sync_message(&mut self, message: usize, resync: bool) -> Queued {+ let provider_message_id = MESSAGES[message].0.to_string();+ let operation = if resync {+ InboxSyncOperation::UpsertMessage(UpsertMessagePayload {+ provider_message_id,+ })+ } else {+ InboxSyncOperation::UpdateLabels(UpdateLabelsPayload {+ provider_message_id,+ })+ };+ self.next_message += 1;+ Queued {+ id: format!("sync-{}", self.next_message),+ queue: Queue::Sync,+ body: serde_json::to_string(&InboxSyncPubsubMessage {+ link_id: self.world.link_id,+ operation,+ })+ .unwrap(),+ visible_at: self.now,+ receives: 0,+ lineage: None,+ }+ }++ pub async fn run_step(&mut self, step: Step) {+ self.prepare(&step).await;+ match step {+ Step::Action {+ action,+ enqueue_fails,+ } => {+ let _ = self.act(action, enqueue_fails).await;+ }+ Step::Deliver { .. } => {+ if let Some(index) = self.next_due()+ && let Some(delivery) = self.begin_delivery(index)+ {+ Self::run_delivery(self.world, &delivery).await;+ self.finish_delivery(delivery);+ }+ }+ Step::Sync { message, .. } | Step::Resync { message, .. } => {+ let resync = matches!(step, Step::Resync { .. });+ if resync {+ self.stats.resyncs += 1;+ } else {+ self.stats.syncs += 1;+ }+ let queued = self.sync_message(message, resync);+ self.queued.push(queued);+ let index = self.queued.len() - 1;+ let delivery = self.begin_delivery(index).unwrap();+ Self::run_delivery(self.world, &delivery).await;+ self.finish_delivery(delivery);+ }+ Step::Wait { seconds } => {+ self.now += seconds;+ self.trace.push(format!("t={} wait {seconds}s", self.now));+ }+ Step::Expire => Box::pin(self.expire()).await,+ Step::Overlap { first, seconds, at } => {+ Box::pin(self.overlap(*first, seconds, at)).await+ }+ }+ self.absorb_sends(None);+ self.world.sqs.set_outage(false);+ }++ pub async fn act(&mut self, action: Action, enqueue_fails: bool) -> Result<(), String> {+ self.world.sqs.set_outage(enqueue_fails);+ self.stats.actions += 1;+ self.action_seq += 1;+ let seq = self.action_seq;+ let owner = world::owner();+ let thread = |index: usize| self.world.thread_ids[THREADS[index]];+ let result: Result<(), String> = match &action {+ Action::Archive(t, value) => self+ .world+ .set_thread_archived(owner, thread(*t), *value)+ .await+ .map_err(|error| format!("{error:?}")),+ Action::Read(t, value) => self+ .world+ .change_thread_mailbox_state(owner, thread(*t), MailboxAction::Read(*value))+ .await+ .map_err(|error| format!("{error:?}")),+ Action::Star(t, value) => self+ .world+ .change_thread_mailbox_state(owner, thread(*t), MailboxAction::Flagged(*value))+ .await+ .map_err(|error| format!("{error:?}")),+ Action::Trash(t, value) => self+ .world+ .change_thread_mailbox_state(owner, thread(*t), MailboxAction::Trashed(*value))+ .await+ .map_err(|error| format!("{error:?}")),+ Action::Label(t, value) => self+ .world+ .update_thread_labels_for_user(+ owner,+ thread(*t),+ self.world.label_ids[WORK],+ *value,+ )+ .await+ .map(|_| ())+ .map_err(|error| format!("{error:?}")),+ Action::BulkArchive(threads, value) => {+ let ids = threads.iter().map(|t| thread(*t)).collect();+ wiring::archive_many(&self.world.email, owner, ids, *value)+ .await+ .and_then(|failed| {+ if failed.is_empty() {+ Ok(())+ } else {+ Err(format!("failed threads {failed:?}"))+ }+ })+ }+ };+ if result.is_err() {+ self.stats.failed_actions += 1;+ }+ self.trace.push(format!(+ "t={} action#{seq} {action:?} enqueue_fails={enqueue_fails} -> {result:?}",+ self.now+ ));+ self.world.sqs.set_outage(false);+ self.absorb_sends(Some(seq));+ result+ }++ /// Twelve hours: work off the queues with the same mix of Gmail answers,+ /// then age every marker past its expiry.+ async fn expire(&mut self) {+ self.stats.expiries += 1;+ self.trace+ .push(format!("t={} expire: drain then age markers", self.now));+ self.drain(true).await;+ self.now += 12 * 3600;+ let ids: Vec<Uuid> = self.world.message_ids.values().copied().collect();+ wiring::age_markers(&self.world.db, &ids).await;+ }++ /// Delivers until every queue is empty. `faulty` keeps Gmail failing at+ /// random; otherwise Gmail accepts everything.+ pub async fn drain(&mut self, faulty: bool) {+ for _ in 0..5_000 {+ if self.queued.is_empty() {+ return;+ }+ let step = if faulty {+ Step::Deliver {+ reply: self.random_reply(0.6),+ gate_refuses: false,+ enqueue_fails: false,+ }+ } else {+ Step::Deliver {+ reply: Reply::Ok,+ gate_refuses: false,+ enqueue_fails: false,+ }+ };+ let description = format!("drain {step:?}");+ Box::pin(self.run_step(step)).await;+ Box::pin(self.check_step(&description)).await;+ }+ panic!("queues did not drain: {:?}", self.queued);+ }++ async fn overlap(&mut self, first: Step, seconds: Vec<Step>, at: PauseAt) {+ self.stats.overlaps += 1;+ if let Step::Deliver { reply, .. } | Step::Sync { reply, .. } | Step::Resync { reply, .. } =+ &first+ {+ let mut gmail = self.world.gmail.lock();+ gmail.next_writes.push_back(*reply);+ gmail.next_reads.push_back(*reply);+ }+ let started = match &first {+ Step::Deliver { .. } => self.next_due().and_then(|index| self.begin_delivery(index)),+ Step::Sync { message, .. } | Step::Resync { message, .. } => {+ let resync = matches!(first, Step::Resync { .. });+ let queued = self.sync_message(*message, resync);+ self.queued.push(queued);+ let index = self.queued.len() - 1;+ self.begin_delivery(index)+ }+ _ => None,+ };+ let Some(mut delivery) = started else {+ self.clear_replies();+ for step in seconds {+ self.run_concurrently(step).await;+ }+ return;+ };+ let kind = match (&first, delivery.queued.queue) {+ (_, Queue::Ops | Queue::OpsRetry) => Kind::Write,+ (Step::Resync { .. }, _) => Kind::ReadFull,+ _ => Kind::ReadLabels,+ };+ self.trace.push(format!(+ "t={} overlap: {} paused {at:?} at its Gmail {kind:?}",+ self.now,+ short(&delivery.queued.body)+ ));+ let pause = self.world.gmail.pause_next(kind, None, at);+ let mut task = Self::spawn_delivery(self.world, &delivery);+ let arrived = tokio::select! {+ _ = &mut task => false,+ arrived = pause.arrived => arrived.is_ok(),+ };+ self.world.gmail.clear_pause();+ self.clear_replies();+ if !arrived {+ self.finish_delivery(delivery);+ // Its sends (a hand-off to the retry queue) belong to it, not to+ // the next user action.+ self.absorb_sends(None);+ for step in seconds {+ self.run_concurrently(step).await;+ }+ return;+ }+ // `first` is held at Gmail while the other steps run to completion.+ // Their deliveries are accounted as they finish: until the release,+ // every SQS ack and Gmail request is theirs.+ let window_from = self.world.gmail.lock().log.len() as u64;+ if at == PauseAt::AfterApply {+ delivery.applied_before = Some(self.action_seq);+ }+ for step in seconds {+ self.trace+ .push(format!("t={} while paused: {step:?}", self.now));+ self.run_concurrently(step).await;+ }+ delivery.excluded = window_from..self.world.gmail.lock().log.len() as u64;+ let _ = pause.release.send(());+ tokio::time::timeout(Duration::from_secs(60), task)+ .await+ .expect("paused delivery finished")+ .expect("worker task panicked");+ self.finish_delivery(delivery);+ }++ fn clear_replies(&mut self) {+ let mut gmail = self.world.gmail.lock();+ gmail.next_writes.clear();+ gmail.next_reads.clear();+ }++ /// Runs `step` to completion while another delivery is paused at Gmail.+ async fn run_concurrently(&mut self, step: Step) {+ match step {+ Step::Action {+ action,+ enqueue_fails,+ } => {+ let _ = self.act(action, enqueue_fails).await;+ }+ Step::Deliver { reply, .. } => {+ self.world.gmail.lock().next_writes.push_back(reply);+ self.world.gmail.lock().next_reads.push_back(reply);+ if let Some(index) = self.next_due()+ && let Some(delivery) = self.begin_delivery(index)+ {+ Self::run_delivery(self.world, &delivery).await;+ self.finish_delivery(delivery);+ }+ self.clear_replies();+ }+ Step::Sync { message, reply } | Step::Resync { message, reply } => {+ let resync = matches!(step, Step::Resync { .. });+ self.world.gmail.lock().next_reads.push_back(reply);+ let queued = self.sync_message(message, resync);+ self.queued.push(queued);+ let index = self.queued.len() - 1;+ if let Some(delivery) = self.begin_delivery(index) {+ Self::run_delivery(self.world, &delivery).await;+ self.finish_delivery(delivery);+ }+ self.clear_replies();+ }+ _ => {}+ }+ self.absorb_sends(None);+ }++ // ------------------------------------------------------ scenario steps++ /// Delivers the earliest message on `queue` (moving the clock to it)+ /// with Gmail answering `reply`. Returns false when the queue is empty.+ pub async fn deliver_from(&mut self, queue: Queue, reply: Reply, enqueue_fails: bool) -> bool {+ self.deliver_answering(queue, &[reply], enqueue_fails).await+ }++ /// Like [`Self::deliver_from`], with Gmail answering the delivery's+ /// writes with `replies` in order and accepting any after them.+ pub async fn deliver_answering(+ &mut self,+ queue: Queue,+ replies: &[Reply],+ enqueue_fails: bool,+ ) -> bool {+ let Some(index) = (0..self.queued.len())+ .filter(|index| self.queued[*index].queue == queue)+ .min_by_key(|index| (self.queued[*index].visible_at, *index))+ else {+ return false;+ };+ self.now = self.now.max(self.queued[index].visible_at);+ self.prepare(&Step::Deliver {+ reply: replies[0],+ gate_refuses: false,+ enqueue_fails,+ })+ .await;+ self.world+ .gmail+ .lock()+ .next_writes+ .extend(replies[1..].iter().copied());+ if let Some(delivery) = self.begin_delivery(index) {+ Self::run_delivery(self.world, &delivery).await;+ self.finish_delivery(delivery);+ }+ self.absorb_sends(None);+ self.world.sqs.set_outage(false);+ self.clear_replies();+ true+ }++ /// Delivers every message now on `queue` once, oldest first.+ pub async fn deliver_all(&mut self, queue: Queue, reply: Reply) -> usize {+ let count = self+ .queued+ .iter()+ .filter(|queued| queued.queue == queue)+ .count();+ for _ in 0..count {+ self.deliver_from(queue, reply, false).await;+ }+ count+ }++ pub fn queue_len(&self, queue: Queue) -> usize {+ self.queued+ .iter()+ .filter(|queued| queued.queue == queue)+ .count()+ }++ /// Starts a delivery and holds its first Gmail request of `kind`.+ pub async fn start_paused(+ &mut self,+ source: Source,+ kind: Kind,+ at: PauseAt,+ ) -> Option<Paused> {+ self.prepare(&Step::Wait { seconds: 0 }).await;+ let index = match source {+ Source::Sync(message) | Source::Resync(message) => {+ let queued = self.sync_message(message, matches!(source, Source::Resync(_)));+ self.queued.push(queued);+ self.queued.len() - 1+ }+ Source::Queued(queue) => (0..self.queued.len())+ .filter(|index| self.queued[*index].queue == queue)+ .min_by_key(|index| (self.queued[*index].visible_at, *index))?,+ };+ self.now = self.now.max(self.queued[index].visible_at);+ let delivery = self.begin_delivery(index)?;+ let pause = self.world.gmail.pause_next(kind, None, at);+ let mut task = Self::spawn_delivery(self.world, &delivery);+ let arrived = tokio::select! {+ _ = &mut task => false,+ arrived = pause.arrived => arrived.is_ok(),+ };+ assert!(arrived, "delivery finished without reaching Gmail");+ Some(Paused {+ delivery,+ task,+ release: pause.release,+ window_from: self.world.gmail.lock().log.len() as u64,+ at,+ action_seq: self.action_seq,+ })+ }++ pub async fn finish_paused(&mut self, paused: Paused) {+ let Paused {+ mut delivery,+ task,+ release,+ window_from,+ at,+ action_seq,+ } = paused;+ delivery.excluded = window_from..self.world.gmail.lock().log.len() as u64;+ if at == PauseAt::AfterApply {+ delivery.applied_before = Some(action_seq);+ }+ let _ = release.send(());+ task.await.expect("worker task panicked");+ self.finish_delivery(delivery);+ self.absorb_sends(None);+ }++ pub async fn sync(&mut self, message: usize) {+ self.run_step(Step::Sync {+ message,+ reply: Reply::Ok,+ })+ .await;+ }++ pub async fn resync(&mut self, message: usize) {+ self.run_step(Step::Resync {+ message,+ reply: Reply::Ok,+ })+ .await;+ }++ /// Runs one seed: random steps with checks after each, then quiescence.+ pub async fn run(&mut self, config: Config) {+ for index in 0..config.steps {+ self.step_index = index;+ let step = self.random_step(config);+ let description = format!("{step:?}");+ self.run_step(step).await;+ self.check_step(&description).await;+ if !self.violations.is_empty() {+ return;+ }+ }+ self.step_index = config.steps;+ self.quiesce().await;+ }++ pub fn report(&self) -> String {+ let mut out = String::new();+ for violation in &self.violations {+ let _ = writeln!(+ out,+ " [{}] step {}: {}",+ violation.invariant, violation.step, violation.detail+ );+ }+ let _ = writeln!(out, " trace:");+ for line in &self.trace {+ let _ = writeln!(out, " {line}");+ }+ out+ }+}++fn short(body: &str) -> String {+ let value: serde_json::Value = serde_json::from_str(body).unwrap_or_default();+ let operation = &value["operation"];+ let text = operation.to_string();+ let text = text.replace(&"0".repeat(8), "");+ if text.len() > 160 {+ format!("{}…", &text[..160])+ } else {+ text+ }+}diff --git a/services/email_service/src/pubsub/label_write_sim/sim/generator.rs b/services/email_service/src/pubsub/label_write_sim/sim/generator.rsnew file mode 100644index 0000000000..918e48dea9--- /dev/null+++ b/services/email_service/src/pubsub/label_write_sim/sim/generator.rs@@ -0,0 +1,119 @@+//! Random steps: user actions, Gmail answers, deliveries, syncs and waits.++use super::super::fake_gmail::{PauseAt, Reply};+use super::super::world::{MESSAGES, THREADS};+use super::{Action, Config, Sim, Step};+use rand::Rng;++impl Sim<'_> {+ pub(super) fn random_reply(&mut self, ok: f64) -> Reply {+ let roll: f64 = self.rng.random();+ if roll < ok {+ Reply::Ok+ } else if roll < ok + (1.0 - ok) * 0.65 {+ Reply::RateLimited+ } else if roll < ok + (1.0 - ok) * 0.9 || std::env::var("SIM_NO_REJECT").is_ok() {+ Reply::Unavailable+ } else {+ Reply::Rejected+ }+ }++ pub(super) fn random_action(&mut self) -> Action {+ let thread = self.rng.random_range(0..THREADS.len());+ let value = self.rng.random_bool(0.5);+ match self.rng.random_range(0..100) {+ 0..35 => Action::Archive(thread, value),+ 35..55 => Action::Read(thread, value),+ 55..65 => Action::Star(thread, value),+ 65..75 => Action::Label(thread, value),+ 75..85 => Action::Trash(thread, value),+ _ => {+ let threads: Vec<usize> = (0..THREADS.len())+ .filter(|_| self.rng.random_bool(0.6))+ .collect();+ if threads.is_empty() {+ Action::Archive(thread, value)+ } else {+ Action::BulkArchive(threads, value)+ }+ }+ }+ }++ pub(super) fn random_simple_step(&mut self) -> Step {+ match self.rng.random_range(0..100) {+ 0..32 => Step::Action {+ action: self.random_action(),+ enqueue_fails: self.rng.random_bool(0.08),+ },+ 32..70 => Step::Deliver {+ reply: self.random_reply(0.45),+ gate_refuses: self.rng.random_bool(0.1),+ enqueue_fails: self.rng.random_bool(0.05),+ },+ 70..86 => Step::Sync {+ message: self.rng.random_range(0..MESSAGES.len()),+ reply: if self.rng.random_bool(0.85) {+ Reply::Ok+ } else {+ Reply::RateLimited+ },+ },+ 86..92 => Step::Resync {+ message: self.rng.random_range(0..MESSAGES.len()),+ reply: if self.rng.random_bool(0.9) {+ Reply::Ok+ } else {+ Reply::RateLimited+ },+ },+ _ => Step::Wait {+ seconds: self.rng.random_range(1..400),+ },+ }+ }++ pub fn random_step(&mut self, config: Config) -> Step {+ let roll = self.rng.random_range(0..1000);+ if roll < 8 {+ return Step::Expire;+ }+ if config.overlap && roll < 90 {+ let first = match self.rng.random_range(0..3) {+ 0 => Step::Sync {+ message: self.rng.random_range(0..MESSAGES.len()),+ reply: Reply::Ok,+ },+ 1 => Step::Resync {+ message: self.rng.random_range(0..MESSAGES.len()),+ reply: Reply::Ok,+ },+ _ => Step::Deliver {+ reply: self.random_reply(0.6),+ gate_refuses: false,+ enqueue_fails: false,+ },+ };+ let count = self.rng.random_range(1..=3);+ let mut seconds = Vec::new();+ while seconds.len() < count {+ let step = self.random_simple_step();+ if !matches!(step, Step::Wait { .. }) {+ seconds.push(step);+ }+ }+ let at = if self.rng.random_bool(0.5) {+ PauseAt::BeforeApply+ } else {+ PauseAt::AfterApply+ };+ return Step::Overlap {+ first: Box::new(first),+ seconds,+ at,+ };+ }+ self.random_simple_step()+ }+}diff --git a/services/email_service/src/pubsub/label_write_sim/sim/model.rs b/services/email_service/src/pubsub/label_write_sim/sim/model.rsnew file mode 100644index 0000000000..59ffa9fd7d--- /dev/null+++ b/services/email_service/src/pubsub/label_write_sim/sim/model.rs@@ -0,0 +1,324 @@+//! The user's intent per message label, how each queued label write ended,+//! and the invariants checked after every step.++use super::super::fake_gmail::{Kind, Logged, Reply};+use super::super::wiring;+use super::super::world::{MESSAGES, TRACKED};+use super::{EntryState, OpStatus, Sim, Step, Violation};+use std::collections::{BTreeSet, HashMap};+use uuid::Uuid;++impl Sim<'_> {+ pub(super) fn initial(message: &str, label: &str) -> bool {+ MESSAGES+ .iter()+ .find(|(id, _, _)| *id == message)+ .is_some_and(|(_, _, labels)| labels.contains(&label))+ }++ /// The user's last effective intent for a cell, or `None` when its latest+ /// write dead-lettered.+ pub(super) fn intent(&self, message: &str, label: &str) -> Option<bool> {+ let entries = &self.cells[&(message.to_string(), label.to_string())];+ let latest = entries+ .iter()+ .filter(|entry| entry.state != EntryState::Rejected)+ .max_by_key(|entry| entry.seq);+ match latest {+ None => Some(Self::initial(message, label)),+ Some(entry) if entry.state == EntryState::DeadLettered => None,+ Some(entry) => Some(entry.value),+ }+ }++ /// The newest non-rejected write on a cell.+ pub(super) fn latest_seq(&self, message: &str, label: &str) -> Option<u64> {+ self.cells[&(message.to_string(), label.to_string())]+ .iter()+ .filter(|entry| entry.state != EntryState::Rejected)+ .map(|entry| entry.seq)+ .max()+ }++ pub(super) fn violate(&mut self, invariant: &'static str, detail: String) {+ self.violations.push(Violation {+ invariant,+ step: self.step_index,+ detail,+ });+ }++ pub(super) fn account_write(+ &mut self,+ lineage: u64,+ log: &[Logged],+ deleted: bool,+ resent: bool,+ applied_before: Option<u64>,+ ) {+ let track = self.ops[&lineage].clone();+ let in_op: Vec<String> = track+ .op+ .messages+ .iter()+ .map(|(_, provider_id)| provider_id.clone())+ .collect();+ let writes: Vec<&Logged> = log+ .iter()+ .filter(|entry| {+ entry.kind == Kind::Write+ && entry.ids.iter().all(|id| in_op.contains(id))+ && (if track.value {+ entry.add.contains(&track.label)+ } else {+ entry.remove.contains(&track.label)+ })+ })+ .collect();+ self.stats.gmail_writes += writes.len() as u64;+ for (index, write) in writes.iter().enumerate() {+ if write.reply != Reply::Ok {+ continue;+ }+ // Only the first write can be the one Gmail applied while later+ // steps ran; any per-message write after it was sent later.+ let cutoff = match index {+ 0 => applied_before.unwrap_or(u64::MAX),+ _ => u64::MAX,+ };+ for id in &write.ids {+ let newer = self.cells[&(id.clone(), track.label.clone())]+ .iter()+ .filter(|entry| entry.state != EntryState::Rejected && entry.seq <= cutoff)+ .max_by_key(|entry| entry.seq)+ .map(|entry| (entry.seq, entry.value, entry.lineage));+ if let Some((latest, value, newer_lineage)) = newer+ && latest > track.seq+ && value != track.value+ {+ // The newer write is still queued and will set Gmail+ // again; sync keeps its local state meanwhile.+ if self.ops[&newer_lineage].status == OpStatus::Queued {+ self.stats.transient_stale_applies += 1;+ continue;+ }+ self.violate(+ "gmail-applied-stale-write",+ format!(+ "write from action {} set {id}/{} = {} at Gmail after newer action {latest}",+ track.seq, track.label, track.value+ ),+ );+ }+ }+ }+ let accepted_now = writes+ .iter()+ .filter(|write| write.reply == Reply::Ok)+ .flat_map(|write| write.ids.iter().cloned());+ self.ops+ .get_mut(&lineage)+ .unwrap()+ .accepted+ .extend(accepted_now);+ if !deleted || resent {+ return;+ }+ let accepted = self.ops[&lineage].accepted.clone();+ let sent_ids: BTreeSet<String> = writes+ .iter()+ .flat_map(|write| write.ids.iter().cloned())+ .collect();+ let outcome = if writes.iter().any(|write| write.reply == Reply::Rejected) {+ OpStatus::Rejected+ } else if writes.iter().any(|write| write.reply == Reply::Ok) {+ OpStatus::Applied+ } else if writes.is_empty() {+ OpStatus::Skipped+ } else {+ self.violate(+ "write-dropped",+ format!(+ "write from action {} deleted after only retryable Gmail errors",+ track.seq+ ),+ );+ OpStatus::Skipped+ };+ let superseded = |sim: &Self, id: &String| {+ sim.latest_seq(id, &track.label)+ .is_some_and(|latest| latest > track.seq)+ };+ if outcome == OpStatus::Rejected {+ // A rejection gives the write up on every message Gmail hasn't+ // accepted: the rejected one, those not sent yet and those+ // waiting for a newer write.+ let rejected: Vec<String> = in_op+ .iter()+ .filter(|id| !accepted.contains(*id))+ .cloned()+ .collect();+ self.set_entries(lineage, &rejected, EntryState::Rejected);+ }+ if outcome == OpStatus::Skipped {+ self.stats.skipped_superseded += 1;+ }+ for id in &in_op {+ if sent_ids.contains(id) || accepted.contains(id) || outcome == OpStatus::Rejected {+ continue;+ }+ if !superseded(self, id) {+ self.violate(+ "write-dropped",+ format!(+ "write from action {} finished without sending {id}/{}, and no newer action owns it",+ track.seq, track.label+ ),+ );+ }+ }+ self.ops.get_mut(&lineage).unwrap().status = outcome;+ }++ pub async fn check_step(&mut self, step: &str) {+ let db = self.world.db_labels().await;+ let gmail = self.world.gmail_labels();+ let mut lost = BTreeSet::new();+ for (message, _, _) in MESSAGES {+ for label in TRACKED {+ let Some(intent) = self.intent(message, label) else {+ lost.insert((message.to_string(), label.to_string()));+ continue;+ };+ let in_db = db[message].contains(label);+ if in_db != intent {+ let pending = gmail[message].contains(label) != intent;+ let invariant = if pending {+ "pending-write-reverted"+ } else {+ "db-diverged-from-intent"+ };+ self.violate(+ invariant,+ format!(+ "after {step}: {message}/{label} db={in_db} intent={intent} gmail={}",+ gmail[message].contains(label)+ ),+ );+ }+ }+ }+ self.lost_cells.extend(lost);+ for problem in self.world.flag_mismatches().await {+ self.violate("thread-flags-mismatch", format!("after {step}: {problem}"));+ }+ self.check_markers(step).await;+ }++ /// Every live marker belongs to a write still on its way to Gmail.+ pub(super) async fn check_markers(&mut self, step: &str) {+ let ids: Vec<Uuid> = self.world.message_ids.values().copied().collect();+ let provider_by_db: HashMap<Uuid, String> = self+ .world+ .message_ids+ .iter()+ .map(|(provider, db)| (*db, provider.clone()))+ .collect();+ let queued_lineages: BTreeSet<u64> = self+ .queued+ .iter()+ .filter_map(|queued| queued.lineage)+ .collect();+ for marker in wiring::markers(&self.world.db, &ids).await {+ if !marker.live {+ continue;+ }+ let provider_id = provider_by_db[&marker.message_id].clone();+ let owner = self+ .lineage_by_write+ .get(&(marker.write_id, provider_id))+ .copied();+ let in_flight = owner.is_some_and(|lineage| queued_lineages.contains(&lineage));+ if !in_flight {+ let status = owner.map(|lineage| self.ops[&lineage].status);+ if status == Some(OpStatus::DeadLettered) {+ continue;+ }+ self.violate(+ "stale-marker",+ format!(+ "after {step}: marker {}/{} owned by write {:?} whose op is {status:?}",+ provider_by_db[&marker.message_id], marker.label, owner+ ),+ );+ }+ }+ }++ /// Gmail accepts everything, every queue drains, every message gets the+ /// echo sync its Gmail change would trigger, and then Gmail, the database+ /// and the user's last intent must agree.+ pub async fn quiesce(&mut self) {+ self.trace.push(format!("t={} quiesce", self.now));+ self.drain(false).await;+ for message in 0..MESSAGES.len() {+ self.run_step(Step::Sync {+ message,+ reply: Reply::Ok,+ })+ .await;+ }+ self.drain(false).await;+ self.check_step("quiesce").await;+ let db = self.world.db_labels().await;+ let gmail = self.world.gmail_labels();+ for (message, _, _) in MESSAGES {+ for label in TRACKED {+ let Some(intent) = self.intent(message, label) else {+ continue;+ };+ let at_gmail = gmail[message].contains(label);+ if at_gmail != intent || db[message].contains(label) != at_gmail {+ self.violate(+ "no-convergence",+ format!(+ "{message}/{label}: intent={intent} gmail={at_gmail} db={}",+ db[message].contains(label)+ ),+ );+ }+ }+ }+ let ids: Vec<Uuid> = self.world.message_ids.values().copied().collect();+ let provider_by_db: HashMap<Uuid, String> = self+ .world+ .message_ids+ .iter()+ .map(|(provider, db)| (*db, provider.clone()))+ .collect();+ for marker in wiring::markers(&self.world.db, &ids).await {+ let provider_id = provider_by_db[&marker.message_id].clone();+ let owner = self+ .lineage_by_write+ .get(&(marker.write_id, provider_id))+ .copied();+ let dead =+ owner.is_some_and(|lineage| self.ops[&lineage].status == OpStatus::DeadLettered);+ if marker.live && !dead {+ self.violate(+ "marker-left-after-quiesce",+ format!("{marker:?} owned by {owner:?}"),+ );+ }+ }+ for (lineage, track) in &self.ops {+ if track.status == OpStatus::Queued {+ self.violations.push(Violation {+ invariant: "write-never-finished",+ step: self.step_index,+ detail: format!("lineage {lineage} from action {}", track.seq),+ });+ }+ }+ }+}diff --git a/services/email_service/src/pubsub/label_write_sim/sim/test.rs b/services/email_service/src/pubsub/label_write_sim/sim/test.rsnew file mode 100644index 0000000000..f7b114a1ba--- /dev/null+++ b/services/email_service/src/pubsub/label_write_sim/sim/test.rs@@ -0,0 +1,210 @@+//! The seeded simulation. The smoke test runs a fixed set of seeds; the+//! ignored shards run more, configured by environment variables:+//!+//! ```text+//! SIM_SEEDS=4000 SIM_STEPS=30 SIM_OVERLAP=0 SIM_OUT=/tmp/sim-1.txt \+//! cargo test -p email_service --lib label_write_sim::sim::test::randomized_shard_1 \+//! -- --exact --ignored --nocapture+//! ```+//!+//! `SIM_OVERLAP=1` holds one operation at its Gmail call while 1–3 other+//! steps run. `SIM_BATCHES=1` has producers send label write batches instead+//! of one operation per message. `SIM_NO_REJECT=1` turns permanent Gmail rejections into 503s.+//! `SIM_START` picks the first seed, `SIM_TRACE=1` prints every seed's trace+//! and `SIM_VERBOSE=1` the traces of violating seeds.++use super::super::world::World;+use super::{Config, Sim, Stats};+use sqlx::PgPool;+use std::collections::BTreeMap;+use std::fmt::Write as _;++fn env_number(name: &str, default: u64) -> u64 {+ std::env::var(name)+ .ok()+ .and_then(|value| value.parse().ok())+ .unwrap_or(default)+}++#[derive(Default)]+struct Summary {+ seeds: u64,+ clean: u64,+ by_invariant: BTreeMap<String, u64>,+ by_step: BTreeMap<String, u64>,+ examples: BTreeMap<String, Vec<(u64, String)>>,+ violating: Vec<String>,+ stats: Stats,+ lost_cell_seeds: u64,+}++impl Summary {+ fn add(&mut self, seed: u64, sim: &Sim<'_>) {+ self.seeds += 1;+ let stats = &sim.stats;+ let total = &mut self.stats;+ total.actions += stats.actions;+ total.failed_actions += stats.failed_actions;+ total.deliveries += stats.deliveries;+ total.syncs += stats.syncs;+ total.resyncs += stats.resyncs;+ total.gmail_writes += stats.gmail_writes;+ total.gmail_429 += stats.gmail_429;+ total.gmail_5xx += stats.gmail_5xx;+ total.gmail_400 += stats.gmail_400;+ total.skipped_superseded += stats.skipped_superseded;+ total.transient_stale_applies += stats.transient_stale_applies;+ total.dead_letters += stats.dead_letters;+ total.overlaps += stats.overlaps;+ total.expiries += stats.expiries;+ if !sim.lost_cells.is_empty() {+ self.lost_cell_seeds += 1;+ }+ let Some(first) = sim.violations.first() else {+ self.clean += 1;+ return;+ };+ *self+ .by_invariant+ .entry(first.invariant.to_string())+ .or_default() += 1;+ *self+ .by_step+ .entry(format!("{} after {}", first.invariant, first.step_kind()))+ .or_default() += 1;+ let detail: String = first.detail.chars().take(240).collect();+ self.violating.push(format!(+ "seed {seed} [{}] step {}: {detail}",+ first.invariant, first.step+ ));+ let examples = self+ .examples+ .entry(first.invariant.to_string())+ .or_default();+ if examples.len() < 3 {+ examples.push((seed, sim.report()));+ }+ }++ fn render(&self, config: Config, start: u64) -> String {+ let mut out = String::new();+ let _ = writeln!(+ out,+ "seeds={}..{} steps={} overlap={} batches={} clean={} violating={}",+ start,+ start + self.seeds,+ config.steps,+ config.overlap,+ config.label_write_batches,+ self.clean,+ self.seeds - self.clean+ );+ let _ = writeln!(out, "stats: {:?}", self.stats);+ let _ = writeln!(+ out,+ "seeds with a dead-lettered write: {}",+ self.lost_cell_seeds+ );+ for (invariant, count) in &self.by_invariant {+ let _ = writeln!(out, "first violation {invariant}: {count} seeds");+ }+ for (key, count) in &self.by_step {+ let _ = writeln!(out, " {key}: {count}");+ }+ let _ = writeln!(out, "all violating seeds:");+ for line in &self.violating {+ let _ = writeln!(out, " {line}");+ }+ for (invariant, examples) in &self.examples {+ for (seed, report) in examples {+ let _ = writeln!(out, "--- example {invariant} seed {seed}\n{report}");+ }+ }+ out+ }+}++async fn run_seeds(pool: PgPool, shard: u32, start: u64, seeds: u64, config: Config) -> Summary {+ let mut world = World::new(pool, shard, config.label_write_batches).await;+ let mut summary = Summary::default();+ for seed in start..start + seeds {+ world.reset().await;+ let mut sim = Sim::new(&mut world, seed);+ sim.run(config).await;+ summary.add(seed, &sim);+ if std::env::var("SIM_TRACE").is_ok()+ || (std::env::var("SIM_VERBOSE").is_ok() && !sim.violations.is_empty())+ {+ println!("seed {seed}:\n{}", sim.report());+ }+ }+ summary+}++async fn fifty_random_seeds(pool: PgPool, shard: u32, label_write_batches: bool) {+ let config = Config {+ steps: 30,+ overlap: false,+ label_write_batches,+ };+ let summary = run_seeds(pool, shard, 1_000_000, 50, config).await;+ assert_eq!(+ summary.clean,+ summary.seeds,+ "{}",+ summary.render(config, 1_000_000)+ );+}++#[sqlx::test(migrator = "macro_db_migrator::MACRO_DB_MIGRATIONS")]+async fn fifty_random_seeds_keep_gmail_the_database_and_the_user_in_agreement(pool: PgPool) {+ fifty_random_seeds(pool, 50, false).await;+}++#[sqlx::test(migrator = "macro_db_migrator::MACRO_DB_MIGRATIONS")]+async fn fifty_random_seeds_agree_with_label_write_batches(pool: PgPool) {+ fifty_random_seeds(pool, 51, true).await;+}++async fn run_shard(pool: PgPool, shard: u32) {+ let seeds = env_number("SIM_SEEDS", 50);+ let start = env_number("SIM_START", u64::from(shard) * 1_000_000);+ let config = Config {+ steps: env_number("SIM_STEPS", 30) as usize,+ overlap: env_number("SIM_OVERLAP", 0) == 1,+ label_write_batches: env_number("SIM_BATCHES", 0) == 1,+ };+ let started = std::time::Instant::now();+ let summary = run_seeds(pool, shard, start, seeds, config).await;+ let rendered = summary.render(config, start);+ println!("{rendered}elapsed={:?}", started.elapsed());+ if let Ok(path) = std::env::var("SIM_OUT") {+ std::fs::write(path, &rendered).unwrap();+ }+}++// One test per shard: sqlx names each test's database after the test, so+// parallel processes need different tests.+#[sqlx::test(migrator = "macro_db_migrator::MACRO_DB_MIGRATIONS")]+#[ignore = "seeded simulation, configured by SIM_* environment variables"]+async fn randomized_shard_1(pool: PgPool) {+ run_shard(pool, 1).await;+}++#[sqlx::test(migrator = "macro_db_migrator::MACRO_DB_MIGRATIONS")]+#[ignore = "seeded simulation, configured by SIM_* environment variables"]+async fn randomized_shard_2(pool: PgPool) {+ run_shard(pool, 2).await;+}++#[sqlx::test(migrator = "macro_db_migrator::MACRO_DB_MIGRATIONS")]+#[ignore = "seeded simulation, configured by SIM_* environment variables"]+async fn randomized_shard_3(pool: PgPool) {+ run_shard(pool, 3).await;+}++#[sqlx::test(migrator = "macro_db_migrator::MACRO_DB_MIGRATIONS")]+#[ignore = "seeded simulation, configured by SIM_* environment variables"]+async fn randomized_shard_4(pool: PgPool) {+ run_shard(pool, 4).await;+}diff --git a/services/email_service/src/pubsub/label_write_sim/test.rs b/services/email_service/src/pubsub/label_write_sim/test.rsnew file mode 100644index 0000000000..e100b21b64--- /dev/null+++ b/services/email_service/src/pubsub/label_write_sim/test.rs@@ -0,0 +1,756 @@+//! Deterministic races between Macro's Gmail label writes, Gmail's answers+//! and inbox syncs, through the production worker and domain code.++use super::fake_gmail::{Kind, PauseAt, Reply};+use super::sim::{Action, Queue, Sim, Source};+use super::wiring;+use super::world::{+ INBOX, MESSAGES, STARRED, THREADS, UNREAD, WORK, World, on_worker_stacks, owner,+};+use sqlx::PgPool;++const T1: usize = 0;+const T2: usize = 1;+const T3: usize = 2;+const M1A: usize = 0;+const M1B: usize = 1;+const M1C: usize = 2;+const M2A: usize = 3;+const M2B: usize = 4;+const M3A: usize = 5;++/// Defines `$name` as a module with the scenario run once for each label+/// write format: one operation per message, which producers send by default,+/// and batches. The two runs use different mailboxes, so they don't share the+/// request gate's state.+macro_rules! scenario {+ ($name:ident, $shard:expr, |$world:ident| $body:block) => {+ mod $name {+ use super::*;++ #[sqlx::test(migrator = "macro_db_migrator::MACRO_DB_MIGRATIONS")]+ async fn single_message_writes(pool: PgPool) {+ run(World::new(pool, $shard, false).await).await;+ }++ #[sqlx::test(migrator = "macro_db_migrator::MACRO_DB_MIGRATIONS")]+ async fn batched_writes(pool: PgPool) {+ run(World::new(pool, $shard + 100, true).await).await;+ }++ async fn run(mut $world: World) $body+ }+ };+}++async fn in_inbox(sim: &Sim<'_>, thread: usize) -> bool {+ sim.world.thread_inbox_visible(THREADS[thread]).await+}++async fn db_has(sim: &Sim<'_>, message: usize, label: &str) -> bool {+ sim.world.db_labels().await[MESSAGES[message].0].contains(label)+}++fn gmail_has(sim: &Sim<'_>, message: usize, label: &str) -> bool {+ sim.world.gmail_labels()[MESSAGES[message].0].contains(label)+}++async fn live_markers(sim: &Sim<'_>) -> usize {+ let ids: Vec<_> = sim.world.message_ids.values().copied().collect();+ wiring::markers(&sim.world.db, &ids)+ .await+ .into_iter()+ .filter(|marker| marker.live)+ .count()+}++async fn sync_thread(sim: &mut Sim<'_>, thread: usize) {+ for (index, (_, thread_id, _)) in MESSAGES.iter().enumerate() {+ if *thread_id == THREADS[thread] {+ sim.sync(index).await;+ }+ }+}++scenario!(+ a_rate_limited_archive_survives_sync_and_resync_until_gmail_accepts_it,+ 901,+ |world| {+ let sim = &mut Sim::new(&mut world, 1);+ sim.act(Action::Archive(T1, true), false).await.unwrap();+ sim.deliver_all(Queue::Ops, Reply::RateLimited).await;+ let operations = if sim.world.label_write_batches { 1 } else { 3 };+ assert_eq!(sim.queue_len(Queue::OpsRetry), operations);++ sim.sync(M1A).await;+ sim.resync(M1B).await;+ assert!(gmail_has(sim, M1A, INBOX));+ assert!(!in_inbox(sim, T1).await);+ assert!(!db_has(sim, M1A, INBOX).await);+ assert!(!db_has(sim, M1B, INBOX).await);++ while sim.deliver_from(Queue::OpsRetry, Reply::Ok, false).await {}+ sync_thread(sim, T1).await;+ assert!([M1A, M1B, M1C].iter().all(|m| !gmail_has(sim, *m, INBOX)));+ assert!(!in_inbox(sim, T1).await);+ assert_eq!(live_markers(sim).await, 0);+ assert!(sim.world.flag_mismatches().await.is_empty());+ }+);++scenario!(+ an_old_archive_retried_after_a_newer_unarchive_sends_nothing,+ 902,+ |world| {+ let sim = &mut Sim::new(&mut world, 1);+ sim.act(Action::Archive(T1, true), false).await.unwrap();+ sim.deliver_all(Queue::Ops, Reply::RateLimited).await;+ sim.act(Action::Archive(T1, false), false).await.unwrap();+ sim.deliver_all(Queue::Ops, Reply::Ok).await;++ let requests_before = sim.world.gmail.lock().log.len();+ while sim.deliver_from(Queue::OpsRetry, Reply::Ok, false).await {}+ assert_eq!(sim.world.gmail.lock().log.len(), requests_before);+ sync_thread(sim, T1).await;+ assert!(gmail_has(sim, M1A, INBOX));+ assert!(in_inbox(sim, T1).await);+ }+);++scenario!(+ an_older_star_rejected_after_unstar_and_star_keeps_the_last_star,+ 903,+ |world| {+ let sim = &mut Sim::new(&mut world, 1);+ sim.act(Action::Star(T3, true), false).await.unwrap();+ sim.deliver_all(Queue::Ops, Reply::RateLimited).await;+ sim.act(Action::Star(T3, false), false).await.unwrap();+ sim.act(Action::Star(T3, true), false).await.unwrap();+ sim.deliver_all(Queue::Ops, Reply::Ok).await;+ while sim+ .deliver_from(Queue::OpsRetry, Reply::Rejected, false)+ .await+ {}++ sim.sync(M3A).await;+ assert!(db_has(sim, M3A, STARRED).await);+ assert!(gmail_has(sim, M3A, STARRED));+ assert!(sim.world.flag_mismatches().await.is_empty());+ }+);++scenario!(+ an_unarchive_that_cannot_be_queued_leaves_the_queued_archive_in_charge,+ 904,+ |world| {+ let sim = &mut Sim::new(&mut world, 1);+ sim.act(Action::Archive(T1, true), false).await.unwrap();+ sim.deliver_all(Queue::Ops, Reply::RateLimited).await;+ assert!(sim.act(Action::Archive(T1, false), true).await.is_err());+ assert!(!in_inbox(sim, T1).await);+ assert_eq!(live_markers(sim).await, 3);++ sim.sync(M1A).await;+ sim.resync(M1B).await;+ assert!(!in_inbox(sim, T1).await);+ while sim.deliver_from(Queue::OpsRetry, Reply::Ok, false).await {}+ sync_thread(sim, T1).await;+ assert!(!gmail_has(sim, M1A, INBOX));+ assert!(!in_inbox(sim, T1).await);+ }+);++/// Gmail answers a read before the archive lands, the archive lands and+/// clears its marker, the echo sync runs, and only then does the stale+/// answer arrive.+async fn stale_fetch_after_the_write_lands(sim: &mut Sim<'_>, resync: bool) {+ sim.act(Action::Archive(T1, true), false).await.unwrap();+ sim.deliver_all(Queue::Ops, Reply::RateLimited).await;+ let (source, kind) = if resync {+ (Source::Resync(M1A), Kind::ReadFull)+ } else {+ (Source::Sync(M1A), Kind::ReadLabels)+ };+ let paused = sim+ .start_paused(source, kind, PauseAt::AfterApply)+ .await+ .unwrap();+ while sim.deliver_from(Queue::OpsRetry, Reply::Ok, false).await {}+ sim.sync(M1A).await;+ assert!(!db_has(sim, M1A, INBOX).await);+ sim.finish_paused(paused).await;++ assert!(!db_has(sim, M1A, INBOX).await);+ assert!(!in_inbox(sim, T1).await);+ assert!(!gmail_has(sim, M1A, INBOX));+}++scenario!(+ a_stale_label_fetch_does_not_bring_back_an_archive_that_landed_meanwhile,+ 905,+ |world| {+ stale_fetch_after_the_write_lands(&mut Sim::new(&mut world, 1), false).await;+ }+);++scenario!(+ a_stale_resync_does_not_bring_back_an_archive_that_landed_meanwhile,+ 906,+ |world| {+ stale_fetch_after_the_write_lands(&mut Sim::new(&mut world, 1), true).await;+ }+);++scenario!(+ a_resync_with_no_gmail_labels_removes_all_but_the_pending_ones,+ 907,+ |world| {+ let sim = &mut Sim::new(&mut world, 1);+ sim.act(Action::Read(T3, true), false).await.unwrap();+ sim.deliver_all(Queue::Ops, Reply::RateLimited).await;+ // In Gmail's own UI the user archives, unlabels and reads it.+ sim.world+ .gmail+ .external_change("m3a", &[], &[INBOX, WORK, UNREAD]);++ sim.resync(M3A).await;+ assert!(!db_has(sim, M3A, INBOX).await);+ assert!(!db_has(sim, M3A, WORK).await);+ assert!(!db_has(sim, M3A, UNREAD).await);+ assert!(!in_inbox(sim, T3).await);+ assert!(sim.world.flag_mismatches().await.is_empty());+ }+);++scenario!(+ a_rejected_write_restores_the_labels_and_the_thread_state,+ 908,+ |world| {+ let sim = &mut Sim::new(&mut world, 1);+ sim.act(Action::Archive(T2, true), false).await.unwrap();+ sim.deliver_all(Queue::Ops, Reply::Rejected).await;+ assert!(db_has(sim, M2A, INBOX).await);+ assert!(in_inbox(sim, T2).await);++ sim.act(Action::Read(T2, true), false).await.unwrap();+ sim.deliver_all(Queue::Ops, Reply::Rejected).await;+ assert!(db_has(sim, M2A, UNREAD).await);+ assert!(sim.world.flag_mismatches().await.is_empty());+ }+);++scenario!(+ a_rejected_unarchive_lets_the_older_archive_waiting_behind_it_reach_gmail,+ 909,+ |world| {+ let sim = &mut Sim::new(&mut world, 1);+ sim.act(Action::Archive(T3, true), false).await.unwrap();+ sim.deliver_all(Queue::Ops, Reply::RateLimited).await;+ sim.act(Action::Archive(T3, false), false).await.unwrap();+ // The archive's retry runs while the unarchive is still queued: it waits.+ let requests_before = sim.world.gmail.lock().log.len();+ sim.deliver_from(Queue::OpsRetry, Reply::Ok, false).await;+ assert_eq!(sim.world.gmail.lock().log.len(), requests_before);+ assert_eq!(sim.queue_len(Queue::OpsRetry), 1);++ sim.deliver_all(Queue::Ops, Reply::Rejected).await;+ assert!(!in_inbox(sim, T3).await);+ while sim.deliver_from(Queue::OpsRetry, Reply::Ok, false).await {}+ assert!(!gmail_has(sim, M3A, INBOX));+ assert_eq!(live_markers(sim).await, 0);+ sim.sync(M3A).await;+ assert!(!in_inbox(sim, T3).await);+ }+);++scenario!(+ a_failed_enqueue_and_a_rejection_keep_the_first_archive_they_took_over,+ 910,+ |world| {+ let sim = &mut Sim::new(&mut world, 1);+ sim.act(Action::Archive(T3, true), false).await.unwrap();+ sim.deliver_all(Queue::Ops, Reply::RateLimited).await;+ sim.act(Action::Archive(T3, false), false).await.unwrap();+ assert!(sim.act(Action::Archive(T3, true), true).await.is_err());+ sim.deliver_all(Queue::Ops, Reply::Rejected).await;+ assert!(!in_inbox(sim, T3).await);++ let requests_before = sim.world.gmail.lock().log.len();+ while sim.deliver_from(Queue::OpsRetry, Reply::Ok, false).await {}+ assert!(sim.world.gmail.lock().log.len() > requests_before);+ assert!(!gmail_has(sim, M3A, INBOX));+ sim.sync(M3A).await;+ assert!(!in_inbox(sim, T3).await);+ }+);++/// A star staged, sent and confirmed entirely while a sync's Gmail fetch is+/// in flight.+async fn star_inside_the_fetch(sim: &mut Sim<'_>, resync: bool) {+ let (source, kind) = if resync {+ (Source::Resync(M1A), Kind::ReadFull)+ } else {+ (Source::Sync(M1A), Kind::ReadLabels)+ };+ let paused = sim+ .start_paused(source, kind, PauseAt::AfterApply)+ .await+ .unwrap();+ sim.act(Action::Star(T1, true), false).await.unwrap();+ sim.deliver_all(Queue::Ops, Reply::Ok).await;+ sim.sync(M1A).await;+ assert!(db_has(sim, M1A, STARRED).await);+ sim.finish_paused(paused).await;++ assert!(db_has(sim, M1A, STARRED).await);+ assert!(gmail_has(sim, M1A, STARRED));+ assert!(sim.world.flag_mismatches().await.is_empty());+}++scenario!(+ a_resync_keeps_a_write_made_and_confirmed_during_its_fetch,+ 911,+ |world| {+ star_inside_the_fetch(&mut Sim::new(&mut world, 1), true).await;+ }+);++scenario!(+ a_label_sync_keeps_a_write_made_and_confirmed_during_its_fetch,+ 912,+ |world| {+ star_inside_the_fetch(&mut Sim::new(&mut world, 1), false).await;+ }+);++scenario!(+ a_recovery_backfill_does_not_bring_back_an_archive_that_landed_during_its_fetch,+ 913,+ |world| {+ use models_email::email::service::backfill::{BackfillMessagePayload, JobScopedPayload};+ let sim = &mut Sim::new(&mut world, 1);+ sim.act(Action::Archive(T1, true), false).await.unwrap();+ sim.deliver_all(Queue::Ops, Reply::RateLimited).await;+ let ctx = sim.world.sync.clone();+ let link = email_db_client::links::get::fetch_link_by_id(&ctx.db, sim.world.link_id)+ .await+ .unwrap()+ .unwrap();+ let scope = JobScopedPayload {+ link_id: link.id,+ job_id: uuid::Uuid::from_u128(0xb0),+ payload: BackfillMessagePayload {+ thread_provider_id: THREADS[T1].to_string(),+ thread_db_id: sim.world.thread_ids[THREADS[T1]],+ message_provider_id: MESSAGES[M1A].0.to_string(),+ },+ };+ wiring::reset_admission(&mut sim.world.redis, sim.world.link_id).await;+ let pause = sim+ .world+ .gmail+ .pause_next(Kind::ReadFull, Some("m1a"), PauseAt::AfterApply);+ let mut backfill = tokio::spawn(async move {+ let _ =+ crate::pubsub::backfill::backfill_message::backfill_message(&ctx, &scope, &link)+ .await;+ });+ let arrived = tokio::select! {+ _ = &mut backfill => false,+ arrived = pause.arrived => arrived.is_ok(),+ };+ assert!(arrived, "the backfill's fetch reaches Gmail");+ while sim.deliver_from(Queue::OpsRetry, Reply::Ok, false).await {}+ sim.sync(M1A).await;+ assert!(!db_has(sim, M1A, INBOX).await);+ let _ = pause.release.send(());+ backfill.await.unwrap();++ assert!(!db_has(sim, M1A, INBOX).await);+ assert!(!gmail_has(sim, M1A, INBOX));+ assert!(sim.world.flag_mismatches().await.is_empty());+ }+);++scenario!(+ an_unqueued_mark_unread_reverts_without_undoing_a_newer_mark_read,+ 914,+ |world| {+ use email::domain::models::mailbox_action::MailboxAction;+ use email::domain::ports::EmailService;+ let sim = &mut Sim::new(&mut world, 1);+ let (arrived, release) = sim.world.sqs.pause_next_send();+ let owner = owner();+ let thread = sim.world.thread_ids[THREADS[T1]];+ let email = sim.world.email.clone();+ // Mark t1 unread; its enqueue is held, then fails.+ let first = {+ let email = email.clone();+ let owner = owner.clone();+ tokio::spawn(async move {+ .change_thread_mailbox_state(owner, thread, MailboxAction::Read(false))+ .await+ })+ };+ arrived.await.unwrap();+ // Meanwhile the user marks t1 read; this write is queued normally.+ let second = email+ .change_thread_mailbox_state(owner, thread, MailboxAction::Read(true))+ .await;+ sim.world.sqs.set_outage(true);+ let _ = release.send(());+ let first = first.await.unwrap();+ sim.world.sqs.set_outage(false);++ assert!(first.is_err());+ assert!(second.is_ok());+ for message in [M1A, M1B, M1C] {+ assert!(!db_has(sim, message, UNREAD).await);+ }+ assert!(sim.world.flag_mismatches().await.is_empty());+ }+);++/// With label write batches off, archiving a three-message thread queues one+/// operation per message, all for the same write. Each settles only its own+/// message: an accepted one clears its marker, a rejected one puts its+/// message back, and one still queued keeps its message archived through a+/// sync.+#[sqlx::test(migrator = "macro_db_migrator::MACRO_DB_MIGRATIONS")]+async fn single_message_writes_each_settle_their_own_message(pool: PgPool) {+ let mut world = World::new(pool, 916, false).await;+ let sim = &mut Sim::new(&mut world, 1);+ sim.act(Action::Archive(T1, true), false).await.unwrap();+ assert_eq!(sim.queue_len(Queue::Ops), 3);+ assert_eq!(live_markers(sim).await, 3);++ sim.deliver_from(Queue::Ops, Reply::Ok, false).await;+ let accepted = only(+ T1_MESSAGES+ .into_iter()+ .filter(|m| !gmail_has(sim, *m, INBOX)),+ );+ assert_eq!(live_markers(sim).await, 2);++ sim.deliver_from(Queue::Ops, Reply::Rejected, false).await;+ let mut back_in_inbox = Vec::new();+ for message in T1_MESSAGES {+ if db_has(sim, message, INBOX).await {+ back_in_inbox.push(message);+ }+ }+ let rejected = only(back_in_inbox.into_iter());+ assert!(in_inbox(sim, T1).await);+ assert_eq!(live_markers(sim).await, 1);++ let queued = only(+ T1_MESSAGES+ .into_iter()+ .filter(|m| *m != accepted && *m != rejected),+ );+ sim.sync(queued).await;+ assert!(gmail_has(sim, queued, INBOX));+ assert!(!db_has(sim, queued, INBOX).await);++ sim.deliver_from(Queue::Ops, Reply::Ok, false).await;+ sync_thread(sim, T1).await;+ assert_eq!(live_markers(sim).await, 0);+ for message in T1_MESSAGES {+ assert_eq!(db_has(sim, message, INBOX).await, message == rejected);+ assert_eq!(gmail_has(sim, message, INBOX), message == rejected);+ }+ assert!(sim.world.flag_mismatches().await.is_empty());+}++/// With batches on, a write to fewer than ten messages goes to Gmail one+/// request per message. When Gmail rejects one after accepting another, only+/// the messages it hasn't accepted go back, so the database already agrees+/// with Gmail and no marker is left.+#[sqlx::test(migrator = "macro_db_migrator::MACRO_DB_MIGRATIONS")]+async fn a_rejection_partway_through_a_batch_keeps_what_gmail_accepted(pool: PgPool) {+ let mut world = World::new(pool, 917, true).await;+ let sim = &mut Sim::new(&mut world, 1);+ sim.act(Action::Archive(T1, true), false).await.unwrap();+ assert_eq!(sim.queue_len(Queue::Ops), 1);++ sim.deliver_answering(Queue::Ops, &[Reply::Ok, Reply::Rejected], false)+ .await;+ let accepted = only(+ T1_MESSAGES+ .into_iter()+ .filter(|m| !gmail_has(sim, *m, INBOX)),+ );+ assert_eq!(live_markers(sim).await, 0);+ assert_t1_inbox_is_all_but(sim, accepted).await;++ sync_thread(sim, T1).await;+ assert_t1_inbox_is_all_but(sim, accepted).await;+ assert!(in_inbox(sim, T1).await);+ assert!(sim.world.flag_mismatches().await.is_empty());+}++/// Seeds 4000735 and 8000641: with batches on, Gmail accepts a write's first+/// message and the next one is rate limited. The retry doesn't send the+/// accepted message again, and when Gmail rejects it, only the messages Gmail+/// never accepted go back.+#[sqlx::test(migrator = "macro_db_migrator::MACRO_DB_MIGRATIONS")]+async fn a_rejected_retry_keeps_the_messages_gmail_accepted_before_it(pool: PgPool) {+ let mut world = World::new(pool, 918, true).await;+ let sim = &mut Sim::new(&mut world, 1);+ sim.act(Action::Archive(T1, true), false).await.unwrap();+ sim.deliver_answering(Queue::Ops, &[Reply::Ok, Reply::RateLimited], false)+ .await;+ let accepted = only(+ T1_MESSAGES+ .into_iter()+ .filter(|m| !gmail_has(sim, *m, INBOX)),+ );+ assert_eq!(live_markers(sim).await, 2);+ assert_eq!(sim.queue_len(Queue::OpsRetry), 1);++ let requests_before = sim.world.gmail.lock().log.len();+ sim.deliver_answering(Queue::OpsRetry, &[Reply::Rejected], false)+ .await;+ let resent: Vec<String> = sim.world.gmail.lock().log[requests_before..]+ .iter()+ .flat_map(|request| request.ids.clone())+ .collect();+ assert!(!resent.contains(&MESSAGES[accepted].0.to_string()));+ assert_eq!(live_markers(sim).await, 0);+ assert_t1_inbox_is_all_but(sim, accepted).await;++ sync_thread(sim, T1).await;+ assert_t1_inbox_is_all_but(sim, accepted).await;+ assert!(in_inbox(sim, T1).await);+ assert!(sim.world.flag_mismatches().await.is_empty());+}++/// Seed 7000286: with batches on, an archive of a three-message thread is held+/// after Gmail applies its first message. The user unarchives the thread and+/// that write reaches Gmail first. When the archive resumes, it leaves the+/// other two messages to the unarchive.+#[sqlx::test(migrator = "macro_db_migrator::MACRO_DB_MIGRATIONS")]+async fn a_resumed_older_write_skips_messages_a_newer_write_took_over(pool: PgPool) {+ let mut world = World::new(pool, 919, true).await;+ let sim = &mut Sim::new(&mut world, 1);+ sim.act(Action::Archive(T1, true), false).await.unwrap();+ let paused = sim+ .start_paused(Source::Queued(Queue::Ops), Kind::Write, PauseAt::AfterApply)+ .await+ .unwrap();+ sim.act(Action::Archive(T1, false), false).await.unwrap();+ sim.deliver_all(Queue::Ops, Reply::Ok).await;+ let requests_before = sim.world.gmail.lock().log.len();+ sim.finish_paused(paused).await;++ assert_eq!(sim.world.gmail.lock().log.len(), requests_before);+ for message in T1_MESSAGES {+ assert!(gmail_has(sim, message, INBOX));+ assert!(db_has(sim, message, INBOX).await);+ }+ assert_eq!(live_markers(sim).await, 0);+ sync_thread(sim, T1).await;+ for message in T1_MESSAGES {+ assert!(db_has(sim, message, INBOX).await);+ }+ assert!(in_inbox(sim, T1).await);+ assert!(sim.world.flag_mismatches().await.is_empty());+}++/// Two bulk writes reach t1 and t2 in opposite orders, so each is the newer+/// one on a thread: the archive on t2, the unarchive on t1. On t2 the+/// archive drew the earlier id, as when requests race. The unarchive sends t1+/// before it waits for the archive on t2, and when Gmail rejects the archive,+/// t2 goes back to the unarchive, which then sends it.+#[sqlx::test(migrator = "macro_db_migrator::MACRO_DB_MIGRATIONS")]+async fn crossing_bulk_writes_each_send_where_they_are_newest_and_fall_back_on_rejection(+ pool: PgPool,+) {+ use crate::pubsub::gmail_ops::operations::OperationError;+ use crate::pubsub::gmail_ops::operations::modify_message_labels::modify_message_labels;+ use email::domain::ports::EmailRepo;+ use models_email::gmail::gmail_ops::{GmailMessageRef, ModifyMessageLabelsBatchPayload};+ use uuid::Uuid;++ let mut world = World::new(pool, 920, true).await;+ let sim = &mut Sim::new(&mut world, 1);+ sim.act(Action::Archive(T2, true), false).await.unwrap();+ sim.deliver_all(Queue::Ops, Reply::Ok).await;++ let archive = Uuid::from_u128(0xc1);+ let unarchive = Uuid::from_u128(0xc2);+ let repo = email::outbound::EmailPgRepo::new(sim.world.db.clone());+ for (thread, add, write_id) in [+ (T1, false, archive),+ (T2, true, unarchive),+ (T1, true, unarchive),+ (T2, false, archive),+ ] {+ repo.stage_thread_label_change(+ sim.world.thread_ids[THREADS[thread]],+ sim.world.link_id,+ INBOX,+ add,+ write_id,+ )+ .await+ .unwrap()+ .unwrap();+ }+ let write = |add: bool, write_id: Uuid, messages: [usize; 5]| {+ let inbox = vec![INBOX.to_string()];+ ModifyMessageLabelsBatchPayload {+ messages: messages+ .iter()+ .map(|message| GmailMessageRef {+ db_message_id: sim.world.message_ids[MESSAGES[*message].0],+ provider_message_id: MESSAGES[*message].0.to_string(),+ })+ .collect(),+ labels_to_add: if add { inbox.clone() } else { vec![] },+ labels_to_remove: if add { vec![] } else { inbox },+ pending_write_id: Some(write_id),+ }+ };+ let archive = write(false, archive, [M1A, M1B, M1C, M2A, M2B]);+ let unarchive = write(true, unarchive, [M2A, M2B, M1A, M1B, M1C]);+ let link = email_db_client::links::get::fetch_link_by_id(&sim.world.db, sim.world.link_id)+ .await+ .unwrap()+ .unwrap();++ wiring::reset_admission(&mut sim.world.redis, sim.world.link_id).await;+ let requests_before = sim.world.gmail.lock().log.len();+ let result = modify_message_labels(&sim.world.ops, &link, &unarchive).await;+ assert!(matches!(result, Err(OperationError::WaitingForNewerWrite)));+ let sent: Vec<String> = sim.world.gmail.lock().log[requests_before..]+ .iter()+ .filter(|request| request.kind == Kind::Write)+ .flat_map(|request| request.ids.clone())+ .collect();+ assert_eq!(sent, ["m1a", "m1b", "m1c"]);++ wiring::reset_admission(&mut sim.world.redis, sim.world.link_id).await;+ sim.world+ .gmail+ .lock()+ .next_writes+ .push_back(Reply::Rejected);+ let result = modify_message_labels(&sim.world.ops, &link, &archive).await;+ assert!(matches!(result, Err(OperationError::Gmail(_))));+ assert!(in_inbox(sim, T2).await);++ wiring::reset_admission(&mut sim.world.redis, sim.world.link_id).await;+ modify_message_labels(&sim.world.ops, &link, &unarchive)+ .await+ .unwrap();+ for message in [M1A, M1B, M1C, M2A, M2B] {+ assert!(db_has(sim, message, INBOX).await);+ assert!(gmail_has(sim, message, INBOX));+ }+ assert_eq!(live_markers(sim).await, 0);+ assert!(sim.world.flag_mismatches().await.is_empty());+}++/// Every message of t1 but `archived` is in the inbox, in the database and at+/// Gmail.+async fn assert_t1_inbox_is_all_but(sim: &Sim<'_>, archived: usize) {+ for message in T1_MESSAGES {+ assert_eq!(db_has(sim, message, INBOX).await, message != archived);+ assert_eq!(gmail_has(sim, message, INBOX), message != archived);+ }+}++const T1_MESSAGES: [usize; 3] = [M1A, M1B, M1C];++fn only(mut messages: impl Iterator<Item = usize>) -> usize {+ let message = messages.next().expect("one message");+ assert!(messages.next().is_none(), "exactly one message");+ message+}++/// Local stacks run the email workers unoptimized on tokio's 2 MiB worker+/// stacks. A label write and a label sync of the same mailbox, run at the same+/// time, must fit well within that: a Gmail read passed by value once made the+/// sync path overflow it.+fn label_writes_and_a_label_sync_fit_on_small_worker_stacks(+ pool: PgPool,+ shard: u32,+ label_write_batches: bool,+) {+ use aws_sdk_sqs::types::{Message, MessageSystemAttributeName};+ use email::domain::ports::EmailService;+ use models_email::gmail::inbox_sync::{+ InboxSyncOperation, InboxSyncPubsubMessage, UpdateLabelsPayload,+ };+ let message = |id: String, body: String| {+ Message::builder()+ .receipt_handle(format!("{id}#1"))+ .message_id(id)+ .body(body)+ .attributes(MessageSystemAttributeName::ApproximateReceiveCount, "1")+ .build()+ };+ on_worker_stacks(&pool, 1536 << 10, move |pool| async move {+ let mut world = World::new(pool, shard, label_write_batches).await;+ for round in 0..3 {+ world.reset().await;+ world+ .set_thread_archived(owner(), world.thread_ids[THREADS[T1]], true)+ .await+ .unwrap();+ let (sent, _) = world.sqs.take();+ let writes: Vec<_> = sent+ .into_iter()+ .filter(|sent| sent.queue_url == world.queues.ops)+ .enumerate()+ .map(|(index, sent)| message(format!("write-{round}-{index}"), sent.body))+ .collect();+ let label_sync = serde_json::to_string(&InboxSyncPubsubMessage {+ link_id: world.link_id,+ operation: InboxSyncOperation::UpdateLabels(UpdateLabelsPayload {+ provider_message_id: MESSAGES[M1A].0.to_string(),+ }),+ })+ .unwrap();+ let label_sync = message(format!("label-sync-{round}"), label_sync);+ let sync = world.sync.clone();+ let label_sync = tokio::spawn(async move {+ crate::pubsub::inbox_sync::process::process_message(sync, &label_sync).await+ });+ let writes: Vec<_> = writes+ .into_iter()+ .map(|write| {+ let ops = world.ops.clone();+ tokio::spawn(async move {+ crate::pubsub::gmail_ops::process::process_message(ops, &write).await+ })+ })+ .collect();+ for write in writes {+ write.await.unwrap().unwrap();+ }+ label_sync.await.unwrap().unwrap();+ }+ assert!(!gmail_has_inbox_on_t1(&world));+ });+}++#[sqlx::test(migrator = "macro_db_migrator::MACRO_DB_MIGRATIONS")]+async fn single_message_writes_and_a_label_sync_fit_on_small_worker_stacks(pool: PgPool) {+ label_writes_and_a_label_sync_fit_on_small_worker_stacks(pool, 915, false);+}++#[sqlx::test(migrator = "macro_db_migrator::MACRO_DB_MIGRATIONS")]+async fn a_label_write_batch_and_a_label_sync_fit_on_small_worker_stacks(pool: PgPool) {+ label_writes_and_a_label_sync_fit_on_small_worker_stacks(pool, 1015, true);+}++fn gmail_has_inbox_on_t1(world: &World) -> bool {+ let gmail = world.gmail_labels();+ [M1A, M1B, M1C]+ .iter()+ .any(|message| gmail[MESSAGES[*message].0].contains(INBOX))+}diff --git a/services/email_service/src/pubsub/label_write_sim/wiring.rs b/services/email_service/src/pubsub/label_write_sim/wiring.rsnew file mode 100644index 0000000000..26eae5171a--- /dev/null+++ b/services/email_service/src/pubsub/label_write_sim/wiring.rs@@ -0,0 +1,183 @@+//! The production Gmail client and request gate wired to the fake Gmail, and+//! direct access to the gate's Redis state, the pending label write markers+//! and queued label writes.++use crate::outbound::email_api::{+ EmailServiceTokenSource, GmailApi, RateBudget, RedisGmailRequestGate,+};+use email::domain::ports::EmailService;+use email_api_client::GmailApiClientRepository;+use macro_user_id::user_id::MacroUserIdStr;+use models_email::gmail::gmail_ops::{GmailOpsOperation, GmailOpsPubsubMessage};+use sqlx::PgPool;+use uuid::Uuid;++pub fn redis_client(client: redis::Client) -> crate::util::redis::RedisClient {+ crate::util::redis::RedisClient::new(client, 60)+}++pub async fn gmail_api(+ db: PgPool,+ redis: &redis::Client,+ gmail_url: &str,+ services_url: &str,+ sqs_client: sqs_client::SQS,+) -> GmailApi {+ let client = gmail_client::GmailClient::new_with_urls(+ Some("projects/sim/topics/gmail".to_string()),+ gmail_url.to_string(),+ format!("{services_url}/people"),+ format!("{services_url}/certs"),+ "sim".to_string(),+ );+ GmailApi::new(+ GmailApiClientRepository::new(client),+ EmailServiceTokenSource::new(+ db,+ redis.get_multiplexed_async_connection().await.unwrap(),+ authentication_service_client::AuthServiceClient::new(+ "secret".to_string(),+ services_url.to_string(),+ ),+ sqs_client,+ ),+ RedisGmailRequestGate::new(+ redis.get_connection_manager().await.unwrap(),+ RateBudget::Live,+ ),+ )+}++fn gate_keys(link_id: Uuid) -> [String; 6] {+ let scope = format!("gmail:{{{link_id}}}");+ [+ format!("{scope}:inflight"),+ format!("{scope}:window"),+ format!("{scope}:cooldown"),+ format!("{scope}:waiting"),+ format!("{scope}:backfill_cooldown"),+ format!("{scope}:limit"),+ ]+}++/// Clears the request gate's state, including the concurrency limit it learns+/// from a Gmail 429, so each simulated step starts admitted regardless of+/// wall-clock time.+pub async fn reset_admission(redis: &mut redis::aio::MultiplexedConnection, link_id: Uuid) {+ let _: () = redis::cmd("DEL")+ .arg(&gate_keys(link_id))+ .query_async(redis)+ .await+ .unwrap();+}++/// Makes the real gate refuse the next request locally, as after a Gmail 429+/// on another worker.+pub async fn refuse_admission(redis: &mut redis::aio::MultiplexedConnection, link_id: Uuid) {+ let [_, _, cooldown, ..] = gate_keys(link_id);+ let _: () = redis::cmd("SET")+ .arg(cooldown)+ .arg("1")+ .arg("PX")+ .arg(5_000)+ .query_async(redis)+ .await+ .unwrap();+}++pub async fn clear_markers(db: &PgPool, message_ids: &[Uuid]) {+ sqlx::query("DELETE FROM email_gmail_pending_label_writes WHERE message_id = ANY($1)")+ .bind(message_ids)+ .execute(db)+ .await+ .unwrap();+}++#[derive(Clone, Debug)]+pub struct Marker {+ pub message_id: Uuid,+ pub label: String,+ pub write_id: Uuid,+ pub live: bool,+}++pub async fn markers(db: &PgPool, message_ids: &[Uuid]) -> Vec<Marker> {+ let rows: Vec<(Uuid, String, Uuid, bool)> = sqlx::query_as(+ "SELECT message_id, provider_label_id, write_id, expires_at > now()+ FROM email_gmail_pending_label_writes WHERE message_id = ANY($1)+ ORDER BY message_id, provider_label_id",+ )+ .bind(message_ids)+ .fetch_all(db)+ .await+ .unwrap();+ rows.into_iter()+ .map(|(message_id, label, write_id, live)| Marker {+ message_id,+ label,+ write_id,+ live,+ })+ .collect()+}++/// Moves every marker 12 hours closer to its expiry.+pub async fn age_markers(db: &PgPool, message_ids: &[Uuid]) {+ sqlx::query(+ "UPDATE email_gmail_pending_label_writes+ SET expires_at = expires_at - interval '12 hours'+ WHERE message_id = ANY($1)",+ )+ .bind(message_ids)+ .execute(db)+ .await+ .unwrap();+}++/// A queued label write, decoded from its SQS body.+#[derive(Clone, Debug, PartialEq, Eq)]+pub struct LabelOp {+ /// (database id, provider id) of each message the op sends.+ pub messages: Vec<(Uuid, String)>,+ pub add: Vec<String>,+ pub remove: Vec<String>,+ pub write_id: Option<Uuid>,+}++pub fn decode_label_op(body: &str) -> Option<LabelOp> {+ let message: GmailOpsPubsubMessage = serde_json::from_str(body).ok()?;+ match message.operation {+ GmailOpsOperation::ModifyMessageLabels(payload) => Some(LabelOp {+ messages: vec![(payload.db_message_id, payload.provider_message_id)],+ add: payload.labels_to_add,+ remove: payload.labels_to_remove,+ write_id: payload.pending_write_id,+ }),+ GmailOpsOperation::ModifyMessageLabelsBatch(payload) => Some(LabelOp {+ messages: payload+ .messages+ .into_iter()+ .map(|message| (message.db_message_id, message.provider_message_id))+ .collect(),+ add: payload.labels_to_add,+ remove: payload.labels_to_remove,+ write_id: payload.pending_write_id,+ }),+ _ => None,+ }+}++/// Bulk archive through the bulk route's service call. Returns the threads+/// the service reported as not changed.+pub async fn archive_many<S: EmailService>(+ service: &S,+ owner: MacroUserIdStr<'static>,+ threads: Vec<Uuid>,+ archived: bool,+) -> Result<Vec<Uuid>, String> {+ service+ .set_threads_archived(owner, threads, archived)+ .await+ .map(|result| result.failed_thread_ids)+ .map_err(|error| format!("{error:?}"))+}diff --git a/services/email_service/src/pubsub/label_write_sim/world.rs b/services/email_service/src/pubsub/label_write_sim/world.rsnew file mode 100644index 0000000000..7809eaea02--- /dev/null+++ b/services/email_service/src/pubsub/label_write_sim/world.rs@@ -0,0 +1,604 @@+//! One Gmail mailbox wired to the production worker and domain code.+//!+//! Real: Postgres (migrated schema), Redis (request gate or limiter, token+//! cache), the `GmailApi` service with its `GmailClient`, the gmail_ops and+//! inbox_sync `process_message` entry points, and the `EmailServiceImpl` the+//! HTTP routes use for archive, read, star, label and trash actions.+//! Faked: Gmail's HTTP API, SQS's HTTP API, and catch-all 200 responses for+//! the auth service, SFS, DSS and connection gateway.++use super::fake_gmail::{FakeGmail, GmailMessage, GmailState};+use super::fake_sqs::{self, FakeSqs};+use super::wiring;+use crate::pubsub::context::{CrmMetadataResolver, PubSubContext};+use crate::pubsub::gmail_ops::worker::GmailOpsContext;+use axum::Router;+use axum::response::IntoResponse;+use email::domain::service::EmailServiceImpl;+use email::outbound::EmailPgRepo;+use frecency::domain::services::FrecencyQueryServiceImpl;+use frecency::outbound::postgres::FrecencyPgStorage;+use macro_event_broker::{KafkaEventPublisher, MacroEventBrokerService};+use macro_user_id::user_id::MacroUserIdStr;+use models_email::email::service::cache::TokenCacheKey;+use sqlx::PgPool;+use std::collections::{BTreeMap, BTreeSet};+use std::sync::Arc;+use tokio_util::task::TaskTracker;+use uuid::Uuid;++pub const INBOX: &str = "INBOX";+pub const UNREAD: &str = "UNREAD";+pub const STARRED: &str = "STARRED";+pub const TRASH: &str = "TRASH";+pub const WORK: &str = "Label_work";++pub const THREADS: [&str; 3] = ["t1", "t2", "t3"];+/// Provider message ids, their thread, and their starting labels.+pub const MESSAGES: [(&str, &str, &[&str]); 6] = [+ ("m1a", "t1", &[INBOX, UNREAD]),+ ("m1b", "t1", &[INBOX, STARRED]),+ ("m1c", "t1", &[INBOX]),+ ("m2a", "t2", &[INBOX, UNREAD]),+ ("m2b", "t2", &[INBOX]),+ ("m3a", "t3", &[INBOX, UNREAD, WORK]),+];+/// Labels the simulation tracks per message.+pub const TRACKED: [&str; 5] = [INBOX, UNREAD, STARRED, TRASH, WORK];+const LABELS: [(&str, &str, &str); 7] = [+ ("INBOX", "INBOX", "system"),+ ("UNREAD", "UNREAD", "system"),+ ("STARRED", "STARRED", "system"),+ ("TRASH", "TRASH", "system"),+ ("SPAM", "SPAM", "system"),+ ("SENT", "SENT", "system"),+ ("Label_work", "Work", "user"),+];++pub type EmailSvc = EmailServiceImpl<+ EmailPgRepo,+ FrecencyQueryServiceImpl<FrecencyPgStorage>,+ sqs_client::SQS,+ crm::domain::service::CrmServiceImpl<+ crm::outbound::companies_repo::CompaniesRepositoryImpl,+ crm::outbound::no_op_resolver::NoOpCompanyMetadataResolver,+ >,+ entity_access_management::domain::service::EntityAccessManagementServiceImpl<+ entity_access_management::outbound::PgRepository,+ >,+ MacroEventBrokerService<KafkaEventPublisher, TaskTracker>,+>;++pub struct Queues {+ pub ops: String,+ pub ops_retry: String,+ pub sync: String,+ pub sync_retry: String,+}++pub struct World {+ pub db: PgPool,+ pub gmail: FakeGmail,+ pub sqs: FakeSqs,+ pub redis: redis::aio::MultiplexedConnection,+ pub link_id: Uuid,+ pub queues: Queues,+ pub ops: GmailOpsContext,+ pub ops_retry: GmailOpsContext,+ pub sync: PubSubContext,+ pub sync_retry: PubSubContext,+ pub email: EmailSvc,+ /// Provider message id → database id.+ pub message_ids: BTreeMap<String, Uuid>,+ /// Provider thread id → database id.+ pub thread_ids: BTreeMap<String, Uuid>,+ pub label_ids: BTreeMap<String, Uuid>,+ /// Whether producers send label writes as batches or, as they do by+ /// default, one single-message operation per changed message.+ pub label_write_batches: bool,+}++/// Runs `test` on a runtime of its own, connected to `pool`'s database, with+/// two worker threads of `worker_stack` bytes of stack each, as the email+/// workers have. Tasks `test` spawns run on those threads.+pub fn on_worker_stacks<F, Fut>(pool: &PgPool, worker_stack: usize, test: F)+where+ F: FnOnce(PgPool) -> Fut + Send + 'static,+ Fut: std::future::Future<Output = ()>,+{+ let options = pool.connect_options().as_ref().clone();+ let outcome = std::thread::spawn(move || {+ tokio::runtime::Builder::new_multi_thread()+ .worker_threads(2)+ .thread_stack_size(worker_stack)+ .enable_all()+ .build()+ .expect("test runtime")+ .block_on(async move {+ let pool = sqlx::postgres::PgPoolOptions::new()+ .connect_with(options)+ .await+ .expect("test database");+ test(pool).await;+ });+ })+ .join();+ if let Err(panic) = outcome {+ std::panic::resume_unwind(panic);+ }+}++pub fn owner() -> MacroUserIdStr<'static> {+ MacroUserIdStr::try_from_email("[email protected]").unwrap()+}++pub fn link_uuid(shard: u32) -> Uuid {+ Uuid::from_u128(0x5100_0000_0000_0000_0000_0000_0000_0000 + u128::from(shard))+}++fn thread_uuid(shard: u32, index: usize) -> Uuid {+ Uuid::from_u128(+ 0x5200_0000_0000_0000_0000_0000_0000_0000 + (u128::from(shard) << 16) + index as u128,+ )+}++fn message_uuid(shard: u32, index: usize) -> Uuid {+ Uuid::from_u128(+ 0x5300_0000_0000_0000_0000_0000_0000_0000 + (u128::from(shard) << 16) + index as u128,+ )+}++fn label_uuid(shard: u32, index: usize) -> Uuid {+ Uuid::from_u128(+ 0x5400_0000_0000_0000_0000_0000_0000_0000 + (u128::from(shard) << 16) + index as u128,+ )+}++pub fn initial_gmail() -> GmailState {+ let mut state = GmailState::default();+ state.history_id = 100;+ state.labels = LABELS+ .iter()+ .map(|(id, name, kind)| (id.to_string(), name.to_string(), kind.to_string()))+ .collect();+ for (index, (id, thread, labels)) in MESSAGES.iter().enumerate() {+ state.messages.insert(+ id.to_string(),+ GmailMessage {+ thread_id: thread.to_string(),+ labels: labels.iter().map(|label| label.to_string()).collect(),+ internal_date_ms: 1_790_000_000_000 + index as i64 * 60_000,+ },+ );+ }+ state+}++impl World {+ pub async fn new(db: PgPool, shard: u32, label_write_batches: bool) -> World {+ let gmail = FakeGmail::start(initial_gmail()).await;+ let sqs = FakeSqs::start().await;+ let services_url = start_catch_all().await;+ let link_id = link_uuid(shard);+ let queues = Queues {+ ops: sqs.queue_url("gmail-ops"),+ ops_retry: sqs.queue_url("gmail-ops-retry"),+ sync: sqs.queue_url("gmail-webhook"),+ sync_retry: sqs.queue_url("gmail-webhook-retry"),+ };+ let mut world = World::compose(+ db,+ gmail,+ sqs.clone(),+ fake_sqs::client(&sqs.url, false),+ &services_url,+ link_id,+ queues,+ label_write_batches,+ )+ .await;+ world.seed(shard).await;+ world+ }++ /// Wires the production contexts to `gmail` and an SQS endpoint.+ async fn compose(+ db: PgPool,+ gmail: FakeGmail,+ sqs: FakeSqs,+ sqs_sdk: aws_sdk_sqs::Client,+ services_url: &str,+ link_id: Uuid,+ queues: Queues,+ label_write_batches: bool,+ ) -> World {+ let redis_url =+ std::env::var("REDIS_URI").unwrap_or_else(|_| "redis://127.0.0.1:6379".to_string());+ let redis_client = redis::Client::open(redis_url).expect("redis url");+ let redis = redis_client+ .get_multiplexed_async_connection()+ .await+ .expect("redis connection");+ let sqs_client = sqs_client::SQS::new(sqs_sdk.clone())+ .gmail_inbox_sync_queue(&queues.sync)+ .gmail_inbox_sync_retry_queue(&queues.sync_retry)+ .gmail_ops_queue(&queues.ops)+ .gmail_ops_retry_queue(&queues.ops_retry)+ .gmail_label_write_batches(label_write_batches)+ .email_backfill_queue(&format!("{services_url}/queue/backfill"))+ .email_link_manager_queue(&format!("{services_url}/queue/link-manager"))+ .sfs_uploader_queue(&format!("{services_url}/queue/sfs"));+ let email_api = wiring::gmail_api(+ db.clone(),+ &redis_client,+ &gmail.url,+ services_url,+ sqs_client.clone(),+ )+ .await;+ let worker =+ |queue: &str| sqs_worker::SQSWorker::new(sqs_sdk.clone(), queue.to_string(), 1, 1);+ let ops_context = |retry_worker: bool, queue: &str| GmailOpsContext {+ db: db.clone(),+ sqs_worker: worker(queue),+ sqs_client: sqs_client.clone(),+ email_api: email_api.clone(),+ retry_worker,+ };+ let broker = MacroEventBrokerService::new(+ KafkaEventPublisher::new("127.0.0.1:9092").expect("kafka publisher"),+ TaskTracker::new(),+ );+ let crm_service = crm::domain::service::CrmServiceImpl::new(+ crm::outbound::companies_repo::CompaniesRepositoryImpl::new(db.clone()),+ CrmMetadataResolver::Unfurl(+ crm::outbound::unfurl_resolver::UnfurlCompanyMetadataResolver::new(Arc::new(+ unfurl::domain::service::UnfurlServiceImpl::new(+ unfurl::outbound::ReqwestUnfurlFetcher::new().expect("unfurl fetcher"),+ ),+ )),+ ),+ );+ let sync_context = |retry_worker: bool, queue: &str| PubSubContext {+ gmail_history: email::domain::mailbox::gmail_history::GmailHistoryService(+ email::outbound::mailbox_pg::PgMailboxSync::new(db.clone()),+ ),+ invitation_extractor: crate::pubsub::invitation_extraction::compose(+ db.clone(),+ email_api.clone(),+ ),+ db: db.clone(),+ sqs_worker: worker(queue),+ sqs_client: sqs_client.clone(),+ contacts_ingress: Arc::new(contacts::domain::service::SqsContactsIngress {+ queue: contacts::outbound::ingress::SqsContactsQueue::new(+ sqs_sdk.clone(),+ format!("{services_url}/queue/contacts"),+ ),+ }),+ email_api: email_api.clone(),+ redis_client: wiring::redis_client(redis_client.clone()),+ notification_ingress_service: Arc::new(+ notification::domain::service::SqsNotificationIngress {+ queue: notification::outbound::queue::SqsQueue::new(+ sqs_sdk.clone(),+ format!("{services_url}/queue/notifications"),+ ),+ },+ ),+ sfs_client: static_file_service_client::StaticFileServiceClient::new(+ "key".to_string(),+ services_url.to_string(),+ ),+ connection_gateway_client:+ connection_gateway_client::client::ConnectionGatewayClient::new(+ "key".to_string(),+ services_url.to_string(),+ ),+ dss_client: document_storage_service_client::DocumentStorageServiceClient::new(+ "key".to_string(),+ services_url.to_string(),+ ),+ system_properties_service: Arc::new(+ system_properties::SystemPropertiesServiceImpl::new(+ system_properties::PgSystemPropertiesRepository::new(db.clone()),+ ),+ ),+ crm_service: crm_service.clone(),+ macro_event_broker: broker.clone(),+ notifications_enabled: false,+ retry_worker,+ };+ let email = EmailServiceImpl::new(+ EmailPgRepo::new(db.clone()),+ FrecencyQueryServiceImpl::new(FrecencyPgStorage::new(db.clone())),+ sqs_client.clone(),+ crm::domain::service::CrmServiceImpl::new(+ crm::outbound::companies_repo::CompaniesRepositoryImpl::new(db.clone()),+ crm::outbound::no_op_resolver::NoOpCompanyMetadataResolver,+ ),+ entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(+ entity_access_management::outbound::PgRepository::new(db.clone()),+ ),+ 0,+ )+ .with_macro_event_broker(broker.clone());++ World {+ ops: ops_context(false, &queues.ops),+ ops_retry: ops_context(true, &queues.ops_retry),+ sync: sync_context(false, &queues.sync),+ sync_retry: sync_context(true, &queues.sync_retry),+ db,+ gmail,+ sqs,+ redis,+ link_id,+ queues,+ email,+ message_ids: BTreeMap::new(),+ thread_ids: BTreeMap::new(),+ label_ids: BTreeMap::new(),+ label_write_batches,+ }+ }++ async fn seed(&mut self, shard: u32) {+ let link = self.link_id;+ sqlx::query(+ "INSERT INTO email_links (id, macro_id, fusionauth_user_id, email_address, provider, is_sync_active, created_at, updated_at)+ VALUES ($1, 'macro|[email protected]', $2, $3, 'GMAIL', true, NOW(), NOW())",+ )+ .bind(link)+ .bind(format!("sim-user-{shard}"))+ .bind(format!("sim{shard}@example.com"))+ .execute(&self.db)+ .await+ .unwrap();+ sqlx::query("INSERT INTO email_gmail_histories (link_id, history_id) VALUES ($1, '100')")+ .bind(link)+ .execute(&self.db)+ .await+ .unwrap();+ for (index, (id, name, kind)) in LABELS.iter().enumerate() {+ let label_id = label_uuid(shard, index);+ let label_type = if *kind == "system" { "System" } else { "User" };+ sqlx::query(&format!(+ "INSERT INTO email_labels (id, link_id, provider_label_id, name, message_list_visibility, label_list_visibility, type, created_at)+ VALUES ($1, $2, $3, $4, 'Show', 'LabelShow', '{label_type}', NOW())"+ ))+ .bind(label_id)+ .bind(link)+ .bind(id)+ .bind(name)+ .execute(&self.db)+ .await+ .unwrap();+ self.label_ids.insert(id.to_string(), label_id);+ }+ for (index, thread) in THREADS.iter().enumerate() {+ let thread_id = thread_uuid(shard, index);+ sqlx::query(+ "INSERT INTO email_threads (id, provider_id, link_id, inbox_visible, is_read, created_at, updated_at)+ VALUES ($1, $2, $3, true, false, NOW(), NOW())",+ )+ .bind(thread_id)+ .bind(thread)+ .bind(link)+ .execute(&self.db)+ .await+ .unwrap();+ self.thread_ids.insert(thread.to_string(), thread_id);+ }+ for (index, (id, thread, labels)) in MESSAGES.iter().enumerate() {+ let message_id = message_uuid(shard, index);+ sqlx::query(+ "INSERT INTO email_messages (id, thread_id, link_id, provider_id, provider_thread_id, is_read, is_starred, is_sent, is_draft, has_attachments, internal_date_ts, created_at, updated_at)+ VALUES ($1, $2, $3, $4, $5, $6, $7, false, false, false, to_timestamp($8::double precision / 1000), NOW(), NOW())",+ )+ .bind(message_id)+ .bind(self.thread_ids[*thread])+ .bind(link)+ .bind(id)+ .bind(thread)+ .bind(!labels.contains(&UNREAD))+ .bind(labels.contains(&STARRED))+ .bind((1_790_000_000_000 + index as i64 * 60_000) as f64)+ .execute(&self.db)+ .await+ .unwrap();+ self.message_ids.insert(id.to_string(), message_id);+ }+ self.reset().await;+ }++ /// Puts the mailbox, Gmail, queues and admission state back to the start.+ pub async fn reset(&mut self) {+ let message_ids: Vec<Uuid> = self.message_ids.values().copied().collect();+ sqlx::query("DELETE FROM email_message_labels WHERE message_id = ANY($1)")+ .bind(&message_ids)+ .execute(&self.db)+ .await+ .unwrap();+ wiring::clear_markers(&self.db, &message_ids).await;+ for (id, _, labels) in MESSAGES {+ let message_id = self.message_ids[id];+ for label in labels {+ sqlx::query(+ "INSERT INTO email_message_labels (message_id, label_id) VALUES ($1, $2)",+ )+ .bind(message_id)+ .bind(self.label_ids[*label])+ .execute(&self.db)+ .await+ .unwrap();+ }+ sqlx::query("UPDATE email_messages SET is_read = $2, is_starred = $3 WHERE id = $1")+ .bind(message_id)+ .bind(!labels.contains(&UNREAD))+ .bind(labels.contains(&STARRED))+ .execute(&self.db)+ .await+ .unwrap();+ }+ for thread_id in self.thread_ids.values() {+ let mut connection = self.db.acquire().await.unwrap();+ email_db_client::threads::update::update_thread_metadata(+ &mut connection,+ *thread_id,+ self.link_id,+ )+ .await+ .unwrap();+ }+ sqlx::query("UPDATE email_gmail_histories SET history_id = '100' WHERE link_id = $1")+ .bind(self.link_id)+ .execute(&self.db)+ .await+ .unwrap();+ *self.gmail.lock() = initial_gmail();+ {+ let mut sqs = self.sqs.lock();+ sqs.sent.clear();+ sqs.acks.clear();+ sqs.failed_sends.clear();+ sqs.outage = false;+ }+ self.seed_token().await;+ wiring::reset_admission(&mut self.redis, self.link_id).await;+ }++ pub async fn seed_token(&mut self) {+ let link = email_db_client::links::get::fetch_link_by_id(&self.db, self.link_id)+ .await+ .unwrap()+ .unwrap();+ let key = TokenCacheKey::new(+ &link.fusionauth_user_id,+ link.email_address.0.as_ref(),+ link.provider.as_str(),+ )+ .to_redis_key();+ let _: () = redis::cmd("SET")+ .arg(key)+ .arg("fake-access-token")+ .arg("EX")+ .arg(86_400)+ .query_async(&mut self.redis)+ .await+ .unwrap();+ }++ /// Labels per provider message id in the database.+ pub async fn db_labels(&self) -> BTreeMap<String, BTreeSet<String>> {+ let rows: Vec<(String, String)> = sqlx::query_as(+ "SELECT m.provider_id, l.provider_label_id+ FROM email_messages m+ JOIN email_message_labels ml ON ml.message_id = m.id+ JOIN email_labels l ON l.id = ml.label_id+ WHERE m.link_id = $1",+ )+ .bind(self.link_id)+ .fetch_all(&self.db)+ .await+ .unwrap();+ let mut labels: BTreeMap<String, BTreeSet<String>> = MESSAGES+ .iter()+ .map(|(id, _, _)| (id.to_string(), BTreeSet::new()))+ .collect();+ for (message, label) in rows {+ labels.entry(message).or_default().insert(label);+ }+ labels+ }++ pub fn gmail_labels(&self) -> BTreeMap<String, BTreeSet<String>> {+ self.gmail+ .lock()+ .messages+ .iter()+ .map(|(id, message)| (id.clone(), message.labels.clone()))+ .collect()+ }++ /// Threads and messages whose stored flags disagree with their labels.+ pub async fn flag_mismatches(&self) -> Vec<String> {+ let mut problems = Vec::new();+ let messages: Vec<(String, bool, bool, bool, bool)> = sqlx::query_as(+ "SELECT m.provider_id, m.is_read, m.is_starred,+ EXISTS (SELECT 1 FROM email_message_labels ml JOIN email_labels l ON l.id = ml.label_id+ WHERE ml.message_id = m.id AND l.provider_label_id = 'UNREAD'),+ EXISTS (SELECT 1 FROM email_message_labels ml JOIN email_labels l ON l.id = ml.label_id+ WHERE ml.message_id = m.id AND l.provider_label_id = 'STARRED')+ FROM email_messages m WHERE m.link_id = $1",+ )+ .bind(self.link_id)+ .fetch_all(&self.db)+ .await+ .unwrap();+ for (id, is_read, is_starred, unread, starred) in messages {+ if is_read == unread {+ problems.push(format!(+ "message {id}: is_read={is_read} but UNREAD={unread}"+ ));+ }+ if is_starred != starred {+ problems.push(format!(+ "message {id}: is_starred={is_starred} but STARRED={starred}"+ ));+ }+ }+ let threads: Vec<(String, bool, bool, bool, bool)> = sqlx::query_as(+ "SELECT t.provider_id, t.inbox_visible, t.is_read,+ COALESCE(bool_or((f.in_inbox AND NOT f.in_sent)), false),+ COALESCE(bool_and(m.is_read), true)+ FROM email_threads t+ JOIN email_messages m ON m.thread_id = t.id+ JOIN email_message_mailbox_facts f ON f.id = m.id+ WHERE t.link_id = $1 AND f.is_present+ GROUP BY t.id",+ )+ .bind(self.link_id)+ .fetch_all(&self.db)+ .await+ .unwrap();+ for (id, inbox_visible, is_read, expected_visible, expected_read) in threads {+ if inbox_visible != expected_visible {+ problems.push(format!(+ "thread {id}: inbox_visible={inbox_visible}, messages say {expected_visible}"+ ));+ }+ if is_read != expected_read {+ problems.push(format!(+ "thread {id}: is_read={is_read}, messages say {expected_read}"+ ));+ }+ }+ problems+ }++ pub async fn thread_inbox_visible(&self, thread: &str) -> bool {+ sqlx::query_scalar("SELECT inbox_visible FROM email_threads WHERE id = $1")+ .bind(self.thread_ids[thread])+ .fetch_one(&self.db)+ .await+ .unwrap()+ }+}++/// Answers 200 `{}` to everything: auth, SFS, DSS, connection gateway, and+/// the SQS queues the simulation does not model.+async fn start_catch_all() -> String {+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();+ let url = format!("http://{}", listener.local_addr().unwrap());+ let app = Router::new().fallback(|| async {+ (+ axum::http::StatusCode::OK,+ [("content-type", "application/json")],+ "{}",+ )+ .into_response()+ });+ tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });+ url+}diff --git a/services/email_service/src/pubsub/mod.rs b/services/email_service/src/pubsub/mod.rsindex 638ba412ab..7eaa5a3d9c 100644--- a/services/email_service/src/pubsub/mod.rs+++ b/services/email_service/src/pubsub/mod.rs@@ -7,6 +7,8 @@ pub use context::CrmMetadataResolver;pub mod crm_cleanup;pub mod gmail_ops;pub mod inbox_sync;+#[cfg(test)]+mod label_write_sim;pub mod link_manager;pub mod scheduled;#[cfg(feature = "sfs_delete")]