Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(email): keep unconfirmed Gmail label writes through inbox sync (#8778)

teoff284d905afa
diff --git a/.sqlx/query-19fa9694e53b16f5b7496271333b14edf2ed03fec3d2e31c396ad389769d4bad.json b/.sqlx/query-19fa9694e53b16f5b7496271333b14edf2ed03fec3d2e31c396ad389769d4bad.json
new file mode 100644
index 0000000000..18326e6a6a
--- /dev/null
+++ b/.sqlx/query-19fa9694e53b16f5b7496271333b14edf2ed03fec3d2e31c396ad389769d4bad.json
@@ -0,0 +1,35 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n WITH mine AS (\n SELECT c.message_id, c.provider_label_id, c.seq, c.label_present_before,\n (SELECT min(newer.seq) FROM email_gmail_pending_label_writes newer\n WHERE newer.message_id = c.message_id\n AND newer.provider_label_id = c.provider_label_id\n AND newer.seq > c.seq) AS newer_seq,\n (SELECT older.label_present FROM email_gmail_pending_label_writes older\n WHERE older.message_id = c.message_id\n AND older.provider_label_id = c.provider_label_id\n AND older.seq < c.seq AND older.expires_at > now()\n ORDER BY older.seq DESC\n LIMIT 1) AS older_present\n FROM email_gmail_pending_label_writes c\n WHERE c.message_id = ANY($1) AND c.write_id = $2\n ), inherited AS (\n UPDATE email_gmail_pending_label_writes c\n SET label_present_before = mine.label_present_before\n FROM mine\n WHERE c.seq = mine.newer_seq\n ), removed AS (\n DELETE FROM email_gmail_pending_label_writes c\n USING mine\n WHERE c.seq = mine.seq\n )\n SELECT\n message_id AS \"message_id!\",\n provider_label_id AS \"provider_label_id!\",\n COALESCE(older_present, label_present_before) AS \"label_present!\"\n FROM mine\n WHERE newer_seq IS NULL\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "message_id!",
+ "type_info": "Uuid"
+ },
+ {
+ "ordinal": 1,
+ "name": "provider_label_id!",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 2,
+ "name": "label_present!",
+ "type_info": "Bool"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "UuidArray",
+ "Uuid"
+ ]
+ },
+ "nullable": [
+ false,
+ false,
+ null
+ ]
+ },
+ "hash": "19fa9694e53b16f5b7496271333b14edf2ed03fec3d2e31c396ad389769d4bad"
+}
diff --git a/.sqlx/query-36a51fc6aa5d7be8b96ab56176a5ca5dea5bc2cbde7a9b742c05b7bb3355ffb9.json b/.sqlx/query-36a51fc6aa5d7be8b96ab56176a5ca5dea5bc2cbde7a9b742c05b7bb3355ffb9.json
new file mode 100644
index 0000000000..315190c0e2
--- /dev/null
+++ b/.sqlx/query-36a51fc6aa5d7be8b96ab56176a5ca5dea5bc2cbde7a9b742c05b7bb3355ffb9.json
@@ -0,0 +1,16 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n UPDATE email_messages m\n SET\n is_read = NOT EXISTS (\n SELECT 1 FROM email_message_labels ml JOIN email_labels l ON l.id = ml.label_id\n WHERE ml.message_id = m.id AND l.provider_label_id = $2\n ),\n is_starred = EXISTS (\n SELECT 1 FROM email_message_labels ml JOIN email_labels l ON l.id = ml.label_id\n WHERE ml.message_id = m.id AND l.provider_label_id = $3\n ),\n updated_at = now()\n WHERE m.id = ANY($1)\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "UuidArray",
+ "Text",
+ "Text"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "36a51fc6aa5d7be8b96ab56176a5ca5dea5bc2cbde7a9b742c05b7bb3355ffb9"
+}
diff --git a/.sqlx/query-3d2c746b08e2d6a6cba0ff05ebf5f947a5593c747c0aa4b586b1ca75e09622d2.json b/.sqlx/query-3d2c746b08e2d6a6cba0ff05ebf5f947a5593c747c0aa4b586b1ca75e09622d2.json
new file mode 100644
index 0000000000..3e14b6b895
--- /dev/null
+++ b/.sqlx/query-3d2c746b08e2d6a6cba0ff05ebf5f947a5593c747c0aa4b586b1ca75e09622d2.json
@@ -0,0 +1,18 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n INSERT INTO email_gmail_pending_label_writes\n (message_id, provider_label_id, write_id, label_present, label_present_before, expires_at)\n SELECT m.id, $3, $4, $5, NOT $5, now() + interval '12 hours'\n FROM email_messages m\n WHERE m.id = ANY($1) AND m.link_id = $2\n ORDER BY m.id\n ON CONFLICT (message_id, provider_label_id, write_id) DO NOTHING\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "UuidArray",
+ "Uuid",
+ "Text",
+ "Uuid",
+ "Bool"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "3d2c746b08e2d6a6cba0ff05ebf5f947a5593c747c0aa4b586b1ca75e09622d2"
+}
diff --git a/.sqlx/query-4d86a34e867a2f184c852a7a3040e4ea0cca6ff307640325618afdf5ba47fae9.json b/.sqlx/query-4d86a34e867a2f184c852a7a3040e4ea0cca6ff307640325618afdf5ba47fae9.json
deleted file mode 100644
index 459f85a008..0000000000
--- a/.sqlx/query-4d86a34e867a2f184c852a7a3040e4ea0cca6ff307640325618afdf5ba47fae9.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "db_name": "PostgreSQL",
- "query": "\n WITH removed AS (\n DELETE FROM email_message_labels\n WHERE $3::bool\n AND message_id = $1::uuid\n AND label_id NOT IN (SELECT UNNEST($2::uuid[]))\n )\n INSERT INTO email_message_labels (message_id, label_id)\n SELECT $1, label_id FROM UNNEST($2::uuid[]) AS t(label_id)\n ON CONFLICT (message_id, label_id) DO NOTHING\n ",
- "describe": {
- "columns": [],
- "parameters": {
- "Left": [
- "Uuid",
- "UuidArray",
- "Bool"
- ]
- },
- "nullable": []
- },
- "hash": "4d86a34e867a2f184c852a7a3040e4ea0cca6ff307640325618afdf5ba47fae9"
-}
diff --git a/.sqlx/query-4f031ebc497c13752284dcbf040112afdb169ce5f597da25ef205162d4759336.json b/.sqlx/query-4f031ebc497c13752284dcbf040112afdb169ce5f597da25ef205162d4759336.json
new file mode 100644
index 0000000000..e0e9b6b942
--- /dev/null
+++ b/.sqlx/query-4f031ebc497c13752284dcbf040112afdb169ce5f597da25ef205162d4759336.json
@@ -0,0 +1,18 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n WITH flags AS (\n SELECT\n CASE WHEN $2 THEN NOT EXISTS (\n SELECT 1 FROM email_message_labels ml JOIN email_labels l ON l.id = ml.label_id\n WHERE ml.message_id = m.id AND l.provider_label_id = $4\n ) ELSE m.is_read END AS is_read,\n CASE WHEN $3 THEN EXISTS (\n SELECT 1 FROM email_message_labels ml JOIN email_labels l ON l.id = ml.label_id\n WHERE ml.message_id = m.id AND l.provider_label_id = $5\n ) ELSE m.is_starred END AS is_starred\n FROM email_messages m\n WHERE m.id = $1\n )\n UPDATE email_messages m\n SET is_read = flags.is_read, is_starred = flags.is_starred\n FROM flags\n WHERE m.id = $1\n AND (m.is_read, m.is_starred) IS DISTINCT FROM (flags.is_read, flags.is_starred)\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Uuid",
+ "Bool",
+ "Bool",
+ "Text",
+ "Text"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "4f031ebc497c13752284dcbf040112afdb169ce5f597da25ef205162d4759336"
+}
diff --git a/.sqlx/query-53895d46375114408f87059ccd3ce40338a6786d47cb52dcd557a801ec6164ea.json b/.sqlx/query-53895d46375114408f87059ccd3ce40338a6786d47cb52dcd557a801ec6164ea.json
new file mode 100644
index 0000000000..b34bca5155
--- /dev/null
+++ b/.sqlx/query-53895d46375114408f87059ccd3ce40338a6786d47cb52dcd557a801ec6164ea.json
@@ -0,0 +1,22 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n SELECT DISTINCT provider_label_id AS \"provider_label_id!\"\n FROM email_gmail_pending_label_writes\n WHERE message_id = $1 AND expires_at > now()\n ORDER BY provider_label_id\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "provider_label_id!",
+ "type_info": "Text"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Uuid"
+ ]
+ },
+ "nullable": [
+ false
+ ]
+ },
+ "hash": "53895d46375114408f87059ccd3ce40338a6786d47cb52dcd557a801ec6164ea"
+}
diff --git a/.sqlx/query-61d951608521ac57ffbe432fd208c4b24ebb0989ea86513c80e470f9c0c580b2.json b/.sqlx/query-61d951608521ac57ffbe432fd208c4b24ebb0989ea86513c80e470f9c0c580b2.json
new file mode 100644
index 0000000000..f84d7fbd85
--- /dev/null
+++ b/.sqlx/query-61d951608521ac57ffbe432fd208c4b24ebb0989ea86513c80e470f9c0c580b2.json
@@ -0,0 +1,15 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n WITH mine AS (\n SELECT message_id, provider_label_id, seq, label_present\n FROM email_gmail_pending_label_writes\n WHERE message_id = ANY($1) AND write_id = $2\n ), next AS (\n SELECT DISTINCT ON (c.message_id, c.provider_label_id) c.seq, mine.label_present\n FROM email_gmail_pending_label_writes c\n JOIN mine ON c.message_id = mine.message_id\n AND c.provider_label_id = mine.provider_label_id AND c.seq > mine.seq\n ORDER BY c.message_id, c.provider_label_id, c.seq\n ), falls_back AS (\n UPDATE email_gmail_pending_label_writes c\n SET label_present_before = next.label_present\n FROM next\n WHERE c.seq = next.seq\n )\n DELETE FROM email_gmail_pending_label_writes c\n USING mine\n WHERE c.message_id = mine.message_id\n AND c.provider_label_id = mine.provider_label_id\n AND c.seq <= mine.seq\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "UuidArray",
+ "Uuid"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "61d951608521ac57ffbe432fd208c4b24ebb0989ea86513c80e470f9c0c580b2"
+}
diff --git a/.sqlx/query-6ba7d71306b4979b2b7fa8b4ecf525f4549e9f9facdfa9c7a59f90ca3facc4f0.json b/.sqlx/query-6ba7d71306b4979b2b7fa8b4ecf525f4549e9f9facdfa9c7a59f90ca3facc4f0.json
new file mode 100644
index 0000000000..026940b66f
--- /dev/null
+++ b/.sqlx/query-6ba7d71306b4979b2b7fa8b4ecf525f4549e9f9facdfa9c7a59f90ca3facc4f0.json
@@ -0,0 +1,23 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n SELECT DISTINCT thread_id\n FROM email_messages\n WHERE id = ANY($1) AND link_id = $2\n ORDER BY thread_id\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "thread_id",
+ "type_info": "Uuid"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "UuidArray",
+ "Uuid"
+ ]
+ },
+ "nullable": [
+ false
+ ]
+ },
+ "hash": "6ba7d71306b4979b2b7fa8b4ecf525f4549e9f9facdfa9c7a59f90ca3facc4f0"
+}
diff --git a/.sqlx/query-92676e78cf8fde3c016d40a70ac3ad92ac7d1ece7bd94e4a1dc9f39833324a2c.json b/.sqlx/query-92676e78cf8fde3c016d40a70ac3ad92ac7d1ece7bd94e4a1dc9f39833324a2c.json
new file mode 100644
index 0000000000..f01a66887d
--- /dev/null
+++ b/.sqlx/query-92676e78cf8fde3c016d40a70ac3ad92ac7d1ece7bd94e4a1dc9f39833324a2c.json
@@ -0,0 +1,42 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n SELECT\n m.id AS \"message_id!\",\n mine.seq IS NOT NULL AS \"claimed!\",\n EXISTS (\n SELECT 1 FROM email_gmail_pending_label_writes newer\n WHERE newer.message_id = m.id AND newer.provider_label_id = l.label\n AND newer.seq > mine.seq\n ) AS \"has_newer!\",\n EXISTS (\n SELECT 1 FROM email_gmail_pending_label_writes newer\n WHERE newer.message_id = m.id AND newer.provider_label_id = l.label\n AND newer.seq > mine.seq AND newer.expires_at > now()\n ) AS \"waits!\"\n FROM unnest($1::uuid[]) AS m(id)\n CROSS JOIN unnest($3::text[]) AS l(label)\n LEFT JOIN email_gmail_pending_label_writes mine\n ON mine.message_id = m.id AND mine.provider_label_id = l.label AND mine.write_id = $2\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "message_id!",
+ "type_info": "Uuid"
+ },
+ {
+ "ordinal": 1,
+ "name": "claimed!",
+ "type_info": "Bool"
+ },
+ {
+ "ordinal": 2,
+ "name": "has_newer!",
+ "type_info": "Bool"
+ },
+ {
+ "ordinal": 3,
+ "name": "waits!",
+ "type_info": "Bool"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "UuidArray",
+ "Uuid",
+ "TextArray"
+ ]
+ },
+ "nullable": [
+ null,
+ null,
+ null,
+ null
+ ]
+ },
+ "hash": "92676e78cf8fde3c016d40a70ac3ad92ac7d1ece7bd94e4a1dc9f39833324a2c"
+}
diff --git a/.sqlx/query-a7a7bc06f4b34f14d7b506d307e19bcc80603467af6af7fafa964f386b2b992c.json b/.sqlx/query-a7a7bc06f4b34f14d7b506d307e19bcc80603467af6af7fafa964f386b2b992c.json
new file mode 100644
index 0000000000..b1742305c0
--- /dev/null
+++ b/.sqlx/query-a7a7bc06f4b34f14d7b506d307e19bcc80603467af6af7fafa964f386b2b992c.json
@@ -0,0 +1,16 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n DELETE FROM email_message_labels ml\n USING unnest($1::uuid[], $2::text[]) AS r(message_id, provider_label_id), email_labels l\n WHERE ml.message_id = r.message_id AND ml.label_id = l.id\n AND l.link_id = $3 AND l.provider_label_id = r.provider_label_id\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "UuidArray",
+ "TextArray",
+ "Uuid"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "a7a7bc06f4b34f14d7b506d307e19bcc80603467af6af7fafa964f386b2b992c"
+}
diff --git a/.sqlx/query-d3b1094fe292191892bed6feb715c0553384fad9e08285afc74375108201b129.json b/.sqlx/query-d3b1094fe292191892bed6feb715c0553384fad9e08285afc74375108201b129.json
new file mode 100644
index 0000000000..282ab44970
--- /dev/null
+++ b/.sqlx/query-d3b1094fe292191892bed6feb715c0553384fad9e08285afc74375108201b129.json
@@ -0,0 +1,25 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n SELECT m.id AS \"id!\"\n FROM unnest($1::uuid[]) AS m(id)\n WHERE EXISTS (\n SELECT 1 FROM email_effective_message_labels ml\n JOIN email_labels l ON l.id = ml.label_id\n WHERE ml.message_id = m.id AND l.link_id = $2 AND l.provider_label_id = $3\n ) <> $4\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "id!",
+ "type_info": "Uuid"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "UuidArray",
+ "Uuid",
+ "Text",
+ "Bool"
+ ]
+ },
+ "nullable": [
+ null
+ ]
+ },
+ "hash": "d3b1094fe292191892bed6feb715c0553384fad9e08285afc74375108201b129"
+}
diff --git a/.sqlx/query-daa957d34f4e5cb39e457ec5f4ec30f70ffddc6f134322a1422355a793b8707b.json b/.sqlx/query-daa957d34f4e5cb39e457ec5f4ec30f70ffddc6f134322a1422355a793b8707b.json
new file mode 100644
index 0000000000..32d1e5e031
--- /dev/null
+++ b/.sqlx/query-daa957d34f4e5cb39e457ec5f4ec30f70ffddc6f134322a1422355a793b8707b.json
@@ -0,0 +1,16 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n INSERT INTO email_message_labels (message_id, label_id)\n SELECT r.message_id, l.id\n FROM unnest($1::uuid[], $2::text[]) AS r(message_id, provider_label_id)\n JOIN email_labels l ON l.link_id = $3 AND l.provider_label_id = r.provider_label_id\n ON CONFLICT (message_id, label_id) DO NOTHING\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "UuidArray",
+ "TextArray",
+ "Uuid"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "daa957d34f4e5cb39e457ec5f4ec30f70ffddc6f134322a1422355a793b8707b"
+}
diff --git a/.sqlx/query-e6be65db250c2f9465f1b6fc8cdd26821e9ee462836eb33b45966a2f5903258f.json b/.sqlx/query-e6be65db250c2f9465f1b6fc8cdd26821e9ee462836eb33b45966a2f5903258f.json
new file mode 100644
index 0000000000..c811865ee8
--- /dev/null
+++ b/.sqlx/query-e6be65db250c2f9465f1b6fc8cdd26821e9ee462836eb33b45966a2f5903258f.json
@@ -0,0 +1,22 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n SELECT l.provider_label_id\n FROM email_message_labels ml\n JOIN email_labels l ON l.id = ml.label_id\n WHERE ml.message_id = $1\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "provider_label_id",
+ "type_info": "Text"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Uuid"
+ ]
+ },
+ "nullable": [
+ false
+ ]
+ },
+ "hash": "e6be65db250c2f9465f1b6fc8cdd26821e9ee462836eb33b45966a2f5903258f"
+}
diff --git a/.sqlx/query-f0a3a9ea4075ae17bd8ed306d6b062336927c9d12a39bc9baa970aee2f3b20ea.json b/.sqlx/query-f0a3a9ea4075ae17bd8ed306d6b062336927c9d12a39bc9baa970aee2f3b20ea.json
new file mode 100644
index 0000000000..a6bb392bd7
--- /dev/null
+++ b/.sqlx/query-f0a3a9ea4075ae17bd8ed306d6b062336927c9d12a39bc9baa970aee2f3b20ea.json
@@ -0,0 +1,18 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n WITH removed AS (\n DELETE FROM email_message_labels\n WHERE $3::bool\n AND message_id = $1::uuid\n AND label_id NOT IN (SELECT UNNEST($2::uuid[]))\n AND label_id NOT IN (\n SELECT id FROM email_labels\n WHERE link_id = $4 AND provider_label_id = ANY($5::text[])\n )\n )\n INSERT INTO email_message_labels (message_id, label_id)\n SELECT $1, label_id FROM UNNEST($2::uuid[]) AS t(label_id)\n ON CONFLICT (message_id, label_id) DO NOTHING\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Uuid",
+ "UuidArray",
+ "Bool",
+ "Uuid",
+ "TextArray"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "f0a3a9ea4075ae17bd8ed306d6b062336927c9d12a39bc9baa970aee2f3b20ea"
+}
diff --git a/crates/email/AGENTS.md b/crates/email/AGENTS.md
index 4208260304..1556967206 100644
--- a/crates/email/AGENTS.md
+++ b/crates/email/AGENTS.md
@@ -77,6 +77,13 @@ tools — hexagonal, heavily feature-gated.
dependency belongs under before wiring it in.
- `domain/ports.rs` defines the traits adapters implement — call through the
domain service, never an outbound adapter directly from another crate.
+- `EmailRepo::stage_thread_label_change` reads which messages change, applies
+ the change and marks the changed provider messages (exactly what
+ `commit_thread_label_write` enqueues) in one transaction holding the thread's
+ label-write lock. The enqueue carries the write's id; a group that can't be
+ queued goes back through `EmailRepo::revert_staged_label_change`, which
+ restores the state an older write still on its way left (or the original),
+ never a newer write's change.
- Sender discovery authorizes the inbox first. Gmail aliases use the sender
provider; Outlook exposes its verified primary address without a Gmail lookup.
- Outlook links have no system label rows. Move a thread in or out of the inbox
diff --git a/crates/email/fixtures/email_message_star_failure.sql b/crates/email/fixtures/email_message_star_failure.sql
new file mode 100644
index 0000000000..6894528484
--- /dev/null
+++ b/crates/email/fixtures/email_message_star_failure.sql
@@ -0,0 +1,10 @@
+-- Fail starred-state writes inside this test database, after the label write.
+CREATE FUNCTION reject_message_star_state() RETURNS trigger AS $$
+BEGIN
+ RAISE EXCEPTION 'starred-state test failure';
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE TRIGGER reject_message_star_state
+BEFORE UPDATE OF is_starred ON email_messages
+FOR EACH ROW EXECUTE FUNCTION reject_message_star_state();
diff --git a/crates/email/src/domain/models/label.rs b/crates/email/src/domain/models/label.rs
index 78ab696c2f..45cd62f32e 100644
--- a/crates/email/src/domain/models/label.rs
+++ b/crates/email/src/domain/models/label.rs
@@ -1,3 +1,4 @@
+use super::SimpleMessage;
use chrono::{DateTime, Utc};
use uuid::Uuid;
@@ -75,6 +76,16 @@ pub struct UpdateThreadLabelsResult {
pub failed_ids: Vec<Uuid>,
}
+/// A label change staged on a thread for Gmail, read in the same transaction
+/// that applied it.
+#[derive(Debug, Clone)]
+pub struct StagedLabelChange {
+ /// Every message of the thread.
+ pub messages: Vec<SimpleMessage>,
+ /// The messages that gained or lost the label.
+ pub changed_ids: Vec<Uuid>,
+}
+
/// The most threads one bulk archive request may change.
pub const MAX_BULK_ARCHIVE_THREADS: usize = 100;
diff --git a/crates/email/src/domain/models/mod.rs b/crates/email/src/domain/models/mod.rs
index db12921643..b290a9579d 100644
--- a/crates/email/src/domain/models/mod.rs
+++ b/crates/email/src/domain/models/mod.rs
@@ -33,7 +33,7 @@ pub use email_filter::{EmailFilter, UpsertEmailFilterInput};
pub use error::EmailErr;
pub use label::{
Label, LabelListVisibility, LabelType, LinkLabel, MAX_BULK_ARCHIVE_THREADS, MessageLabel,
- MessageListVisibility, SetThreadsArchivedResult, UpdateThreadLabelsResult,
+ MessageListVisibility, SetThreadsArchivedResult, StagedLabelChange, UpdateThreadLabelsResult,
};
pub use link::{
EmailBackfillStatus, EmailInboxDetails, EmailSyncStatus, Link, ResolvedSendingInbox,
diff --git a/crates/email/src/domain/ports.rs b/crates/email/src/domain/ports.rs
index a3f514fb27..d9db3d9815 100644
--- a/crates/email/src/domain/ports.rs
+++ b/crates/email/src/domain/ports.rs
@@ -6,8 +6,9 @@ use crate::domain::models::{
MessageTimestamps, ParsedAddresses, ParsedMessage, ParsedThread, PreviewCursorQuery,
RecipientType, ResolvedDraftInput, ResolvedSendingInbox, SavedUserDraft, SenderPolicy,
SendingInboxRequest, SetThreadsArchivedResult, SettledDraftIds, SimpleMessage,
- SimpleMessageInfo, SourcedAttachment, Thread, ThreadRow, UpdateThreadLabelsResult,
- UpsertEmailFilterInput, UpsertedContacts, UserEmailLink, UserProvider,
+ SimpleMessageInfo, SourcedAttachment, StagedLabelChange, Thread, ThreadRow,
+ UpdateThreadLabelsResult, UpsertEmailFilterInput, UpsertedContacts, UserEmailLink,
+ UserProvider,
};
use entity_access::domain::models::{EditAccessLevel, EntityAccessReceipt, ViewAccessLevel};
use macro_user_id::user_id::MacroUserIdStr;
@@ -38,12 +39,14 @@ pub trait EmailMessageEnqueuer: Send + Sync + 'static {
) -> impl Future<Output = Result<(), Self::Err>> + Send;
/// Enqueue a batch of Gmail label modification operations to the gmail_ops worker queue.
+ /// Each operation clears its message's `pending_write_id` marker when it finishes.
fn enqueue_gmail_ops_modify_labels_batch(
&self,
link_id: Uuid,
messages: Vec<(Uuid, String)>,
labels_to_add: Vec<String>,
labels_to_remove: Vec<String>,
+ pending_write_id: Uuid,
) -> impl Future<Output = Result<(), Self::Err>> + Send;
/// Enqueue a Gmail-ops BlockSender operation for the link.
@@ -420,28 +423,39 @@ pub trait EmailRepo: Send + Sync + 'static {
link_id: Uuid,
) -> impl Future<Output = Result<(), Self::Err>> + Send;
- /// Atomically update the UNREAD label assignments and the message/thread read flags.
- /// `message_ids` must be the messages of the already-authorized thread and inbox.
- /// An error leaves all three representations unchanged.
- fn set_thread_read_state(
+ /// Atomically stage a label change on an already-authorized thread for
+ /// Gmail: read which messages gain or lose the label, apply the change to
+ /// every message with the read, starred and inbox flags it drives, and
+ /// mark the changed messages that have a provider id — exactly those a
+ /// Gmail write sends — as waiting on `write_id`, so inbox sync keeps their
+ /// local state until Gmail confirms it. Label writes on one thread take
+ /// turns, so each reads the state the one before it left. `None`, with
+ /// nothing changed, when the thread has no messages.
+ fn stage_thread_label_change(
&self,
thread_id: Uuid,
link_id: Uuid,
- message_ids: &[Uuid],
- is_read: bool,
- ) -> impl Future<Output = Result<(), Self::Err>> + Send;
-
- /// Atomically change INBOX assignments and the thread's visibility.
- /// The message IDs belong to the authorized thread/inbox. Visibility is
- /// separate from `add` so an enqueue failure can restore the prior flag
- /// while reverting only the assignments changed by that operation.
- fn set_thread_inbox_state(
+ provider_label_id: &str,
+ add: bool,
+ write_id: Uuid,
+ ) -> impl Future<Output = Result<Option<StagedLabelChange>, Self::Err>> + Send;
+
+ /// Atomically undo a staged label change Gmail will never receive.
+ /// `provider_message_ids` go back to the state Gmail will end up with:
+ /// that of the newest older write still on its way, or the one before
+ /// this write; a newer write keeps its own. `local_message_ids`
+ /// (Macro-only drafts) lose the change. The thread's state is derived
+ /// again, all while holding the thread's label-write lock.
+ #[expect(clippy::too_many_arguments, reason = "the parts of one staged change")]
+ fn revert_staged_label_change(
&self,
thread_id: Uuid,
link_id: Uuid,
- message_ids: &[Uuid],
+ provider_label_id: &str,
add: bool,
- inbox_visible: bool,
+ write_id: Uuid,
+ provider_message_ids: &[Uuid],
+ local_message_ids: &[Uuid],
) -> impl Future<Output = Result<(), Self::Err>> + Send;
/// Update the read status for a batch of messages, verified by link_id.
@@ -1217,6 +1231,7 @@ impl EmailMessageEnqueuer for NoOpEnqueuer {
_messages: Vec<(Uuid, String)>,
_labels_to_add: Vec<String>,
_labels_to_remove: Vec<String>,
+ _pending_write_id: Uuid,
) -> Result<(), Self::Err> {
Ok(())
}
diff --git a/crates/email/src/domain/service/thread_labels.rs b/crates/email/src/domain/service/thread_labels.rs
index ed919024bd..9f55816712 100644
--- a/crates/email/src/domain/service/thread_labels.rs
+++ b/crates/email/src/domain/service/thread_labels.rs
@@ -24,8 +24,6 @@ struct StagedThreadLabels {
all_ids: Vec<Uuid>,
/// Messages that gain or lose the label; only these go to Gmail or revert.
changed_ids: Vec<Uuid>,
- /// The thread's inbox visibility before an INBOX change.
- inbox_visible: Option<bool>,
}
impl StagedThreadLabels {
@@ -41,6 +39,16 @@ impl StagedThreadLabels {
.map(|pid| (msg.db_id, pid.clone()))
})
}
+
+ /// Changed Macro-only drafts, which never reach Gmail.
+ fn changed_local_messages(&self) -> Vec<Uuid> {
+ self.messages
+ .iter()
+ .filter(|msg| self.changed_ids.contains(&msg.db_id))
+ .filter(|msg| msg.provider_id.as_ref().is_none_or(|pid| pid.is_empty()))
+ .map(|msg| msg.db_id)
+ .collect()
+ }
}
/// Staged changes of one label on threads of one inbox, sent to Gmail as one
@@ -49,6 +57,9 @@ struct ThreadLabelWrite {
link: Link,
label: LinkLabel,
add: bool,
+ /// Marks each thread's changed messages as waiting on this write when it
+ /// is staged; every group of the write carries it.
+ write_id: Uuid,
threads: Vec<StagedThreadLabels>,
}
@@ -437,8 +448,9 @@ where
.await;
}
+ let write_id = macro_uuid::generate_uuid_v7();
let thread = self
- .stage_thread_labels(link, thread_id, &label.provider_label_id, add)
+ .stage_thread_labels(link, thread_id, &label.provider_label_id, add, write_id)
.await?;
let successful_ids = thread.all_ids.clone();
self.commit_thread_label_write(
@@ -446,6 +458,7 @@ where
link: link.clone(),
label,
add,
+ write_id,
threads: vec![thread],
},
actor,
@@ -518,6 +531,7 @@ where
link,
label,
add: !archived,
+ write_id: macro_uuid::generate_uuid_v7(),
threads: Vec::new(),
});
writes.len() - 1
@@ -529,6 +543,7 @@ where
thread_id,
system_labels::INBOX,
!archived,
+ writes[index].write_id,
)
.await?;
Ok(Some((index, thread)))
@@ -574,81 +589,34 @@ where
}
/// Commit one thread's label change to the DB, capturing what sending it
- /// to Gmail or reverting it needs.
+ /// to Gmail or reverting it needs. The messages that change are read in
+ /// the same transaction that changes them and marks them pending for
+ /// `write_id`: only those go to Gmail, and an enqueue-failure revert
+ /// restores only what this call changed.
async fn stage_thread_labels(
&self,
link: &Link,
thread_id: Uuid,
provider_label_id: &str,
add: bool,
+ write_id: Uuid,
) -> Result<StagedThreadLabels, EmailErr> {
- let messages = self
- .email_repo
- .get_thread_label_messages(thread_id, link.id)
- .await
- .map_err(|e| EmailErr::RepoErr(anyhow::Error::from(e)))?;
-
- if messages.is_empty() {
- return Err(EmailErr::ThreadEmpty);
- }
-
if provider_label_id.is_empty() {
return Err(EmailErr::EmptyProviderLabelId);
}
- let all_ids: Vec<Uuid> = messages.iter().map(|m| m.db_id).collect();
-
- // Snapshot which messages actually gain/lose the label. Only those go
- // to Gmail, and an enqueue-failure revert restores only what this call
- // changed — the batch writes are idempotent, and blindly inverting
- // every message would strip pre-existing state. Best effort: if the
- // snapshot read fails, fall back to treating all messages as changed.
- let changed_ids: Vec<Uuid> = match self.email_repo.labels_by_message_ids(&all_ids).await {
- Ok(labels_by_id) => all_ids
- .iter()
- .filter(|id| {
- let has_label = labels_by_id.get(id).is_some_and(|labels| {
- labels
- .iter()
- .any(|l| l.provider_label_id == provider_label_id)
- });
- has_label != add
- })
- .copied()
- .collect(),
- Err(e) => {
- let err = anyhow::Error::from(e);
- if provider_label_id == system_labels::INBOX {
- return Err(EmailErr::RepoErr(err));
- }
- tracing::warn!(error=?err, "failed to snapshot message labels; revert would cover all messages");
- all_ids.clone()
- }
- };
-
- let inbox_visible = if provider_label_id == system_labels::INBOX {
- Some(
- self.email_repo
- .thread_by_id(thread_id)
- .await
- .map_err(|e| EmailErr::RepoErr(anyhow::Error::from(e)))?
- .ok_or(EmailErr::ThreadNotFound)?
- .inbox_visible,
- )
- } else {
- None
- };
-
- // Optimistic DB update: update all messages first
- self.apply_label_db_changes(link, thread_id, &all_ids, provider_label_id, add)
- .await?;
+ let staged = self
+ .email_repo
+ .stage_thread_label_change(thread_id, link.id, provider_label_id, add, write_id)
+ .await
+ .map_err(|e| EmailErr::RepoErr(anyhow::Error::from(e)))?
+ .ok_or(EmailErr::ThreadEmpty)?;
Ok(StagedThreadLabels {
thread_id,
- messages,
- all_ids,
- changed_ids,
- inbox_visible,
+ all_ids: staged.messages.iter().map(|m| m.db_id).collect(),
+ messages: staged.messages,
+ changed_ids: staged.changed_ids,
})
}
@@ -667,6 +635,7 @@ where
link,
label,
add,
+ write_id,
mut threads,
} = write;
let provider_label_id = label.provider_label_id.as_str();
@@ -692,6 +661,7 @@ where
provider_messages,
labels_to_add.clone(),
labels_to_remove.clone(),
+ write_id,
)
.await
{
@@ -699,7 +669,7 @@ where
tracing::error!(error=?err, "failed to enqueue gmail ops label sync, reverting database changes");
let unsent_threads = threads.split_off(group.start);
for thread in &unsent_threads {
- self.revert_label_db_changes(&link, thread, provider_label_id, add)
+ self.revert_label_db_changes(&link, thread, provider_label_id, add, write_id)
.await;
}
unsent = Some(UnsentThreads {
@@ -758,145 +728,38 @@ where
/// Best-effort revert of the optimistic label write after the Gmail sync
/// could not be enqueued. Only `changed_ids` — the messages that actually
- /// gained/lost the label — are inverted, so messages that already held
- /// the requested state keep it; the denormalized thread read flag is
- /// restored to its derived pre-op value. Every step logs and continues on
- /// failure: the caller surfaces the enqueue error regardless.
+ /// gained/lost the label — go back, and a message a newer write changed
+ /// since keeps that write's state. Logs on failure: the caller surfaces
+ /// the enqueue error regardless, and a pending marker left behind only
+ /// keeps inbox sync off that label until it expires.
async fn revert_label_db_changes(
&self,
link: &Link,
thread: &StagedThreadLabels,
provider_label_id: &str,
add: bool,
+ write_id: Uuid,
) {
- let StagedThreadLabels {
- thread_id,
- all_ids,
- changed_ids,
- inbox_visible,
- ..
- } = thread;
- let thread_id = *thread_id;
- if let Some(inbox_visible) = *inbox_visible {
- if let Err(e) = self
- .email_repo
- .set_thread_inbox_state(thread_id, link.id, changed_ids, !add, inbox_visible)
- .await
- {
- let err = anyhow::Error::from(e);
- tracing::error!(error=?err, "failed to revert thread inbox state after enqueue failure");
- }
- return;
- }
- if !changed_ids.is_empty() {
- let db_result = if add {
- self.email_repo
- .delete_message_labels_batch(changed_ids, provider_label_id, link.id)
- .await
- } else {
- self.email_repo
- .insert_message_labels_batch(changed_ids, provider_label_id, link.id)
- .await
- };
- if let Err(e) = db_result {
- let err = anyhow::Error::from(e);
- tracing::error!(error=?err, "failed to revert message labels after enqueue failure");
- }
- }
-
- if provider_label_id == system_labels::UNREAD {
- // Restore is_read on the flipped messages: they were read before
- // an UNREAD add (`add` == true) and unread before a removal.
- if !changed_ids.is_empty()
- && let Err(e) = self
- .email_repo
- .update_message_read_status_batch(changed_ids, link.id, add)
- .await
- {
- let err = anyhow::Error::from(e);
- tracing::error!(error=?err, "failed to revert message read status after enqueue failure");
- }
- // Pre-op unread set: everything outside `changed_ids` for an add,
- // `changed_ids` itself for a removal — the thread was read iff
- // that set was empty.
- let original_is_read = if add {
- changed_ids.len() == all_ids.len()
- } else {
- changed_ids.is_empty()
- };
- if let Err(e) = self
- .email_repo
- .update_thread_read_status(thread_id, link.id, original_is_read)
- .await
- {
- let err = anyhow::Error::from(e);
- tracing::error!(error=?err, "failed to revert thread read status after enqueue failure");
- }
- } else if provider_label_id == system_labels::STARRED
- && !changed_ids.is_empty()
- && let Err(e) = self
- .email_repo
- .update_message_starred_status_batch(changed_ids, link.id, !add)
- .await
- {
- let err = anyhow::Error::from(e);
- tracing::error!(error=?err, "failed to revert message starred status after enqueue failure");
- }
- }
-
- /// Apply a thread label change to the DB. UNREAD/read flags and INBOX/
- /// visibility commit atomically; a failure propagates before provider sync
- /// or events. Other labels retain their best-effort starred side effect.
- async fn apply_label_db_changes(
- &self,
- link: &Link,
- thread_id: Uuid,
- message_ids: &[Uuid],
- provider_label_id: &str,
- add: bool,
- ) -> Result<(), EmailErr> {
- if provider_label_id == system_labels::INBOX {
- return self
- .email_repo
- .set_thread_inbox_state(thread_id, link.id, message_ids, add, add)
- .await
- .map_err(|e| EmailErr::RepoErr(anyhow::Error::from(e)));
- }
- if provider_label_id == system_labels::UNREAD {
- return self
- .email_repo
- .set_thread_read_state(thread_id, link.id, message_ids, !add)
- .await
- .map_err(|e| EmailErr::RepoErr(anyhow::Error::from(e)));
- }
-
- let db_result = if add {
- self.email_repo
- .insert_message_labels_batch(message_ids, provider_label_id, link.id)
- .await
- } else {
- self.email_repo
- .delete_message_labels_batch(message_ids, provider_label_id, link.id)
- .await
- };
-
- if let Err(e) = db_result {
- let err = anyhow::Error::from(e);
- tracing::error!(error=?err, "failed to update message labels in database");
- return Err(EmailErr::RepoErr(err));
- }
-
- if provider_label_id == system_labels::STARRED
- && let Err(e) = self
- .email_repo
- .update_message_starred_status_batch(message_ids, link.id, add)
- .await
+ let provider_ids: Vec<Uuid> = thread
+ .changed_provider_messages()
+ .map(|(id, _)| id)
+ .collect();
+ if let Err(e) = self
+ .email_repo
+ .revert_staged_label_change(
+ thread.thread_id,
+ link.id,
+ provider_label_id,
+ add,
+ write_id,
+ &provider_ids,
+ &thread.changed_local_messages(),
+ )
+ .await
{
let err = anyhow::Error::from(e);
- tracing::error!(error=?err, "failed to update message starred status");
+ tracing::error!(error=?err, "failed to revert label change after enqueue failure");
}
-
- Ok(())
}
/// Map a user-initiated thread label change onto the semantic
diff --git a/crates/email/src/outbound/email_pg_repo.rs b/crates/email/src/outbound/email_pg_repo.rs
index ea0b05a257..81f5d98507 100644
--- a/crates/email/src/outbound/email_pg_repo.rs
+++ b/crates/email/src/outbound/email_pg_repo.rs
@@ -4,8 +4,8 @@ use crate::domain::{
EmailErr, EmailFilter, EmailInboxDetails, EmailThreadMailProjection, EmailThreadMetadata,
EmailThreadPreview, Label, Link, LinkLabel, MessageAttachment, MessageLabel, MessageRow,
MessageTimestamps, ParsedAddresses, PreviewCursorQuery, ResolvedDraftInput,
- SettledDraftIds, SimpleMessage, SimpleMessageInfo, ThreadRow, UpsertEmailFilterInput,
- UpsertedContacts, UserProvider,
+ SettledDraftIds, SimpleMessage, SimpleMessageInfo, StagedLabelChange, ThreadRow,
+ UpsertEmailFilterInput, UpsertedContacts, UserProvider,
},
ports::{
DraftAttachmentRepo, EmailRepo, EmailUserRepo, LinkEmailSettings, RecipientsByMessageId,
@@ -437,31 +437,44 @@ impl EmailRepo for EmailPgRepo {
.await
}
- async fn set_thread_read_state(
+ async fn stage_thread_label_change(
&self,
thread_id: Uuid,
link_id: Uuid,
- message_ids: &[Uuid],
- is_read: bool,
- ) -> Result<(), Self::Err> {
- label::set_thread_read_state(&self.pool, thread_id, link_id, message_ids, is_read).await
+ provider_label_id: &str,
+ add: bool,
+ write_id: Uuid,
+ ) -> Result<Option<StagedLabelChange>, Self::Err> {
+ label::stage_thread_label_change(
+ &self.pool,
+ thread_id,
+ link_id,
+ provider_label_id,
+ add,
+ write_id,
+ )
+ .await
}
- async fn set_thread_inbox_state(
+ async fn revert_staged_label_change(
&self,
thread_id: Uuid,
link_id: Uuid,
- message_ids: &[Uuid],
+ provider_label_id: &str,
add: bool,
- inbox_visible: bool,
+ write_id: Uuid,
+ provider_message_ids: &[Uuid],
+ local_message_ids: &[Uuid],
) -> Result<(), Self::Err> {
- label::set_thread_inbox_state(
+ label::revert_staged_label_change(
&self.pool,
thread_id,
link_id,
- message_ids,
+ provider_label_id,
add,
- inbox_visible,
+ write_id,
+ provider_message_ids,
+ local_message_ids,
)
.await
}
diff --git a/crates/email/src/outbound/email_pg_repo/label.rs b/crates/email/src/outbound/email_pg_repo/label.rs
index ec3ae6b934..37bcab409e 100644
--- a/crates/email/src/outbound/email_pg_repo/label.rs
+++ b/crates/email/src/outbound/email_pg_repo/label.rs
@@ -1,6 +1,6 @@
use crate::domain::models::{
LabelListVisibility, LabelType, LinkLabel, MessageListVisibility, SimpleMessage,
- label::system_labels,
+ StagedLabelChange, label::system_labels,
};
use crate::outbound::email_pg_repo::db_types::{
LabelListVisibilityDbRow, LabelTypeDbRow, MessageListVisibilityDbRow,
@@ -163,9 +163,9 @@ pub(crate) async fn get_label_by_id(
Ok(row.map(Into::into))
}
-#[tracing::instrument(skip(pool), err)]
+#[tracing::instrument(skip(executor), err)]
pub(crate) async fn get_thread_label_messages(
- pool: &PgPool,
+ executor: impl sqlx::PgExecutor<'_>,
thread_id: Uuid,
link_id: Uuid,
) -> Result<Vec<SimpleMessage>, sqlx::Error> {
@@ -202,48 +202,136 @@ pub(crate) async fn get_thread_label_messages(
thread_id,
link_id
)
- .fetch_all(pool)
+ .fetch_all(executor)
.await?;
Ok(rows.into_iter().map(Into::into).collect())
}
-/// Commit every local representation of a thread's read state together.
-#[tracing::instrument(skip(pool, message_ids), err)]
-pub(super) async fn set_thread_read_state(
+/// Stage a label change on a thread for Gmail in one transaction: read which
+/// messages gain or lose the label, apply the change to every message with the
+/// flags it drives, and mark the changed messages Gmail knows about as waiting
+/// on `write_id`. `None` when the thread has no messages.
+#[tracing::instrument(skip(pool), err)]
+pub(super) async fn stage_thread_label_change(
pool: &PgPool,
thread_id: Uuid,
link_id: Uuid,
- message_ids: &[Uuid],
- is_read: bool,
-) -> Result<(), sqlx::Error> {
+ provider_label_id: &str,
+ add: bool,
+ write_id: Uuid,
+) -> Result<Option<StagedLabelChange>, sqlx::Error> {
let mut tx = pool.begin().await?;
- if is_read {
- delete_message_labels_batch(&mut *tx, message_ids, system_labels::UNREAD, link_id).await?;
+ email_db_client::labels::pending_writes::lock_thread_label_writes(&mut tx, &[thread_id])
+ .await?;
+
+ let messages = get_thread_label_messages(&mut *tx, thread_id, link_id).await?;
+ if messages.is_empty() {
+ return Ok(None);
+ }
+ let message_ids: Vec<Uuid> = messages.iter().map(|message| message.db_id).collect();
+ let changed_ids = email_db_client::labels::get::messages_changing_label(
+ &mut *tx,
+ link_id,
+ &message_ids,
+ provider_label_id,
+ add,
+ )
+ .await?;
+
+ if add {
+ insert_message_labels_batch(&mut *tx, &message_ids, provider_label_id, link_id).await?;
} else {
- insert_message_labels_batch(&mut *tx, message_ids, system_labels::UNREAD, link_id).await?;
+ delete_message_labels_batch(&mut *tx, &message_ids, provider_label_id, link_id).await?;
}
- update_message_read_status_batch(&mut *tx, message_ids, link_id, is_read).await?;
- super::thread::update_thread_read_status(&mut *tx, thread_id, link_id, is_read).await?;
- tx.commit().await
+ if provider_label_id == system_labels::INBOX {
+ set_thread_inbox_visible(&mut tx, thread_id, link_id, add).await?;
+ } else if provider_label_id == system_labels::UNREAD {
+ update_message_read_status_batch(&mut *tx, &message_ids, link_id, !add).await?;
+ super::thread::update_thread_read_status(&mut *tx, thread_id, link_id, !add).await?;
+ } else if provider_label_id == system_labels::STARRED {
+ update_message_starred_status_batch(&mut *tx, &message_ids, link_id, add).await?;
+ }
+
+ let pending_ids: Vec<Uuid> = messages
+ .iter()
+ .filter(|message| {
+ changed_ids.contains(&message.db_id)
+ && message
+ .provider_id
+ .as_ref()
+ .is_some_and(|id| !id.is_empty())
+ })
+ .map(|message| message.db_id)
+ .collect();
+ email_db_client::labels::pending_writes::mark_pending_label_writes(
+ &mut *tx,
+ link_id,
+ &pending_ids,
+ provider_label_id,
+ add,
+ write_id,
+ )
+ .await?;
+ tx.commit().await?;
+
+ Ok(Some(StagedLabelChange {
+ messages,
+ changed_ids,
+ }))
}
-/// Commit INBOX assignments and the canonical thread flag in one transaction.
-#[tracing::instrument(skip(pool, message_ids), err)]
-pub(super) async fn set_thread_inbox_state(
+/// Undo a staged label change Gmail will never receive, in one transaction
+/// holding the thread's label-write lock: labels of `provider_message_ids`
+/// go back to the state Gmail will end up with (see
+/// `abandon_pending_label_writes`), `local_message_ids` (Macro-only drafts)
+/// lose the change, and the thread's state is derived again.
+#[tracing::instrument(skip(pool, provider_message_ids, local_message_ids), err)]
+#[expect(clippy::too_many_arguments, reason = "the parts of one staged change")]
+pub(super) async fn revert_staged_label_change(
pool: &PgPool,
thread_id: Uuid,
link_id: Uuid,
- message_ids: &[Uuid],
+ provider_label_id: &str,
add: bool,
- inbox_visible: bool,
+ write_id: Uuid,
+ provider_message_ids: &[Uuid],
+ local_message_ids: &[Uuid],
) -> Result<(), sqlx::Error> {
let mut tx = pool.begin().await?;
+ email_db_client::labels::pending_writes::lock_thread_label_writes(&mut tx, &[thread_id])
+ .await?;
+ email_db_client::labels::pending_writes::abandon_pending_label_writes(
+ &mut tx,
+ link_id,
+ provider_message_ids,
+ write_id,
+ )
+ .await?;
+
if add {
- insert_message_labels_batch(&mut *tx, message_ids, system_labels::INBOX, link_id).await?;
+ delete_message_labels_batch(&mut *tx, local_message_ids, provider_label_id, link_id)
+ .await?;
} else {
- delete_message_labels_batch(&mut *tx, message_ids, system_labels::INBOX, link_id).await?;
+ insert_message_labels_batch(&mut *tx, local_message_ids, provider_label_id, link_id)
+ .await?;
+ }
+ if provider_label_id == system_labels::UNREAD {
+ update_message_read_status_batch(&mut *tx, local_message_ids, link_id, add).await?;
+ } else if provider_label_id == system_labels::STARRED {
+ update_message_starred_status_batch(&mut *tx, local_message_ids, link_id, !add).await?;
}
+
+ super::thread::update_thread_metadata(&mut tx, thread_id, link_id).await?;
+ tx.commit().await
+}
+
+async fn set_thread_inbox_visible(
+ tx: &mut sqlx::PgConnection,
+ thread_id: Uuid,
+ link_id: Uuid,
+ inbox_visible: bool,
+) -> Result<(), sqlx::Error> {
sqlx::query!(
r#"
UPDATE email_threads
@@ -254,9 +342,9 @@ pub(super) async fn set_thread_inbox_state(
thread_id,
link_id,
)
- .execute(&mut *tx)
+ .execute(tx)
.await?;
- tx.commit().await
+ Ok(())
}
#[tracing::instrument(skip(executor), err)]
@@ -356,9 +444,9 @@ pub(crate) async fn update_message_read_status_batch(
Ok(())
}
-#[tracing::instrument(skip(pool), err)]
+#[tracing::instrument(skip(executor), err)]
pub(crate) async fn update_message_starred_status_batch(
- pool: &PgPool,
+ executor: impl sqlx::PgExecutor<'_>,
message_ids: &[Uuid],
link_id: Uuid,
is_starred: bool,
@@ -381,7 +469,7 @@ pub(crate) async fn update_message_starred_status_batch(
message_ids,
link_id
)
- .execute(pool)
+ .execute(executor)
.await?;
Ok(())
diff --git a/crates/email/src/outbound/email_pg_repo/test/send_attempt/provider_draft_trash.rs b/crates/email/src/outbound/email_pg_repo/test/send_attempt/provider_draft_trash.rs
index 58755ff83b..c71ad24fe8 100644
--- a/crates/email/src/outbound/email_pg_repo/test/send_attempt/provider_draft_trash.rs
+++ b/crates/email/src/outbound/email_pg_repo/test/send_attempt/provider_draft_trash.rs
@@ -36,7 +36,7 @@ async fn echo_then_trash(
.is_draft
);
// The same production path used by inbox synchronization, not a direct flag UPDATE.
- insert_message(pool, thread, &mut echo, link(), true).await?;
+ insert_message(pool, thread, &mut echo, link(), true, None).await?;
assert!(
repo.get_simple_message(message(), &[link()])
.await?
diff --git a/crates/email/src/outbound/email_pg_repo/test/thread_archive.rs b/crates/email/src/outbound/email_pg_repo/test/thread_archive.rs
index 2478c677aa..654469b442 100644
--- a/crates/email/src/outbound/email_pg_repo/test/thread_archive.rs
+++ b/crates/email/src/outbound/email_pg_repo/test/thread_archive.rs
@@ -68,8 +68,12 @@ fn service_with_enqueuer<E: EmailMessageEnqueuer>(
type LabelWrite = (Uuid, Vec<(Uuid, String)>, Vec<String>, Vec<String>);
+/// Label writes and, in the same order, the pending write id each carried.
#[derive(Clone, Default)]
-struct RecordingEnqueuer(Arc<std::sync::Mutex<Vec<LabelWrite>>>);
+struct RecordingEnqueuer(
+ Arc<std::sync::Mutex<Vec<LabelWrite>>>,
+ Arc<std::sync::Mutex<Vec<Uuid>>>,
+);
impl EmailMessageEnqueuer for RecordingEnqueuer {
type Err = anyhow::Error;
const MAX_LABEL_WRITE_MESSAGES: usize = 1000;
@@ -87,11 +91,13 @@ impl EmailMessageEnqueuer for RecordingEnqueuer {
messages: Vec<(Uuid, String)>,
labels_to_add: Vec<String>,
labels_to_remove: Vec<String>,
+ pending_write_id: Uuid,
) -> anyhow::Result<()> {
self.0
.lock()
.unwrap()
.push((link_id, messages, labels_to_add, labels_to_remove));
+ self.1.lock().unwrap().push(pending_write_id);
Ok(())
}
async fn enqueue_gmail_ops_block_sender(&self, _: Uuid, _: String) -> anyhow::Result<()> {
@@ -113,7 +119,7 @@ async fn archive_sends_gmail_only_the_messages_still_in_the_inbox(
pool: Pool<Postgres>,
) -> anyhow::Result<()> {
let enqueuer = RecordingEnqueuer::default();
- let service = service_with_enqueuer(pool, enqueuer.clone());
+ let service = service_with_enqueuer(pool.clone(), enqueuer.clone());
let owner = MacroUserIdStr::try_from_email("[email protected]")?;
let thread = Uuid::parse_str("11111111-1111-1111-1111-111111111111")?;
let link = Uuid::parse_str("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")?;
@@ -141,10 +147,26 @@ async fn archive_sends_gmail_only_the_messages_still_in_the_inbox(
vec!["INBOX".to_string()]
)]
);
+ let write_id = enqueuer.1.lock().unwrap()[0];
+ assert_eq!(
+ pending_markers(&pool).await?,
+ vec![(first, "INBOX".to_string(), write_id)]
+ );
assert_inbox_state(&service.email_repo, thread, false).await?;
Ok(())
}
+async fn pending_markers(pool: &Pool<Postgres>) -> anyhow::Result<Vec<(Uuid, String, Uuid)>> {
+ Ok(sqlx::query!(
+ "SELECT message_id, provider_label_id, write_id FROM email_gmail_pending_label_writes ORDER BY message_id, seq"
+ )
+ .fetch_all(pool)
+ .await?
+ .into_iter()
+ .map(|marker| (marker.message_id, marker.provider_label_id, marker.write_id))
+ .collect())
+}
+
struct FailingEnqueuer;
impl EmailMessageEnqueuer for FailingEnqueuer {
type Err = anyhow::Error;
@@ -163,6 +185,7 @@ impl EmailMessageEnqueuer for FailingEnqueuer {
_: Vec<(Uuid, String)>,
_: Vec<String>,
_: Vec<String>,
+ _: Uuid,
) -> anyhow::Result<()> {
anyhow::bail!("test queue unavailable")
}
@@ -184,7 +207,7 @@ impl EmailMessageEnqueuer for FailingEnqueuer {
async fn archive_enqueue_failure_restores_only_changed_labels_and_the_original_flag(
pool: Pool<Postgres>,
) -> anyhow::Result<()> {
- let service = service_with_enqueuer(pool, FailingEnqueuer);
+ let service = service_with_enqueuer(pool.clone(), FailingEnqueuer);
let thread = Uuid::parse_str("11111111-1111-1111-1111-111111111111")?;
let link = Uuid::parse_str("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")?;
let first = Uuid::parse_str("11111111-aaaa-aaaa-aaaa-111111111111")?;
@@ -225,6 +248,49 @@ async fn archive_enqueue_failure_restores_only_changed_labels_and_the_original_f
expected
);
}
+ assert!(pending_markers(&pool).await?.is_empty());
+ Ok(())
+}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(
+ path = "../../../../fixtures",
+ scripts("email_thread_labels", "email_thread_unread", "email_thread_archive")
+ )
+)]
+async fn failed_enqueue_hands_markers_back_to_the_write_still_on_its_way(
+ pool: Pool<Postgres>,
+) -> anyhow::Result<()> {
+ let enqueuer = RecordingEnqueuer::default();
+ let queued = service_with_enqueuer(pool.clone(), enqueuer.clone());
+ let unqueued = service_with_enqueuer(pool.clone(), FailingEnqueuer);
+ let owner = MacroUserIdStr::try_from_email("[email protected]")?;
+ let thread = Uuid::parse_str("11111111-1111-1111-1111-111111111111")?;
+ let first = Uuid::parse_str("11111111-aaaa-aaaa-aaaa-111111111111")?;
+ let second = Uuid::parse_str("11111111-bbbb-bbbb-bbbb-111111111111")?;
+
+ // The archive is queued but not yet applied by Gmail.
+ queued
+ .set_thread_archived_impl(owner.clone(), thread, true)
+ .await?;
+ let archive = enqueuer.1.lock().unwrap()[0];
+ // The unarchive takes the markers over, then cannot be queued.
+ assert!(
+ unqueued
+ .set_thread_archived_impl(owner, thread, false)
+ .await
+ .is_err()
+ );
+
+ assert_inbox_state(&queued.email_repo, thread, false).await?;
+ assert_eq!(
+ pending_markers(&pool).await?,
+ vec![
+ (first, "INBOX".to_string(), archive),
+ (second, "INBOX".to_string(), archive),
+ ]
+ );
Ok(())
}
@@ -349,7 +415,7 @@ async fn archive_rejects_inaccessible_and_empty_threads(
)
)]
async fn failed_archive_rolls_back_inbox_label_removal(pool: Pool<Postgres>) -> anyhow::Result<()> {
- let service = service(pool);
+ let service = service(pool.clone());
let thread = Uuid::parse_str("33333333-3333-3333-3333-333333333333")?;
assert!(matches!(
service
@@ -362,6 +428,146 @@ async fn failed_archive_rolls_back_inbox_label_removal(pool: Pool<Postgres>) ->
Err(EmailErr::RepoErr(_))
));
assert_inbox_state(&service.email_repo, thread, true).await?;
+ assert!(pending_markers(&pool).await?.is_empty());
+ Ok(())
+}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(
+ path = "../../../../fixtures",
+ scripts("email_thread_labels", "email_thread_unread", "email_thread_archive")
+ )
+)]
+async fn staged_inbox_change_marks_the_messages_it_changed_in_the_same_transaction(
+ pool: Pool<Postgres>,
+) -> anyhow::Result<()> {
+ let repo = EmailPgRepo::new(pool.clone());
+ let thread = Uuid::parse_str("11111111-1111-1111-1111-111111111111")?;
+ let link = Uuid::parse_str("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")?;
+ let first = Uuid::parse_str("11111111-aaaa-aaaa-aaaa-111111111111")?;
+ let second = Uuid::parse_str("11111111-bbbb-bbbb-bbbb-111111111111")?;
+ let write_id = Uuid::from_u128(0xf1);
+ repo.delete_message_labels_batch(&[second], "INBOX", link)
+ .await?;
+
+ let staged = repo
+ .stage_thread_label_change(thread, link, "INBOX", false, write_id)
+ .await?
+ .expect("the thread has messages");
+
+ assert_eq!(staged.changed_ids, vec![first]);
+ assert_inbox_state(&repo, thread, false).await?;
+ assert_eq!(
+ pending_markers(&pool).await?,
+ vec![(first, "INBOX".to_string(), write_id)]
+ );
+ Ok(())
+}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(
+ path = "../../../../fixtures",
+ scripts("email_thread_labels", "email_thread_unread", "email_thread_archive")
+ )
+)]
+async fn a_label_write_waits_for_the_one_already_staging_on_its_thread(
+ pool: Pool<Postgres>,
+) -> anyhow::Result<()> {
+ let repo = EmailPgRepo::new(pool.clone());
+ let thread = Uuid::parse_str("11111111-1111-1111-1111-111111111111")?;
+ let link = Uuid::parse_str("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")?;
+ let first = Uuid::parse_str("11111111-aaaa-aaaa-aaaa-111111111111")?;
+
+ // Another write holds the thread's label-write lock while it archives `first`.
+ let mut staging = pool.begin().await?;
+ email_db_client::labels::pending_writes::lock_thread_label_writes(&mut staging, &[thread])
+ .await?;
+ sqlx::query!(
+ "DELETE FROM email_message_labels WHERE message_id = $1",
+ first
+ )
+ .execute(&mut *staging)
+ .await?;
+
+ let blocked = tokio::time::timeout(
+ std::time::Duration::from_millis(500),
+ repo.stage_thread_label_change(thread, link, "INBOX", true, Uuid::from_u128(0xf2)),
+ )
+ .await;
+ assert!(blocked.is_err(), "a second write must wait for the first");
+
+ staging.commit().await?;
+ let staged = repo
+ .stage_thread_label_change(thread, link, "INBOX", true, Uuid::from_u128(0xf2))
+ .await?
+ .expect("the thread has messages");
+ assert_eq!(staged.changed_ids, vec![first]);
+ Ok(())
+}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(
+ path = "../../../../fixtures",
+ scripts(
+ "email_thread_labels",
+ "email_thread_unread",
+ "email_thread_archive",
+ "email_thread_archive_failure"
+ )
+ )
+)]
+async fn failed_staged_inbox_change_leaves_no_pending_marker(
+ pool: Pool<Postgres>,
+) -> anyhow::Result<()> {
+ let repo = EmailPgRepo::new(pool.clone());
+ let thread = Uuid::parse_str("33333333-3333-3333-3333-333333333333")?;
+ let link = Uuid::parse_str("aaaaaaaa-aaaa-aaaa-aaaa-bbbbbbbbbbbb")?;
+
+ assert!(
+ repo.stage_thread_label_change(thread, link, "INBOX", false, Uuid::from_u128(0xf1))
+ .await
+ .is_err()
+ );
+
+ assert_inbox_state(&repo, thread, true).await?;
+ assert!(pending_markers(&pool).await?.is_empty());
+ Ok(())
+}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(
+ path = "../../../../fixtures",
+ scripts("email_thread_labels", "email_message_star_failure")
+ )
+)]
+async fn failed_star_flag_rolls_back_the_label_and_its_marker(
+ pool: Pool<Postgres>,
+) -> anyhow::Result<()> {
+ let repo = EmailPgRepo::new(pool.clone());
+ let thread = Uuid::parse_str("11111111-1111-1111-1111-111111111111")?;
+ let link = Uuid::parse_str("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")?;
+
+ assert!(
+ repo.stage_thread_label_change(thread, link, "STARRED", true, Uuid::from_u128(0xf1))
+ .await
+ .is_err()
+ );
+
+ let messages = repo.get_thread_label_messages(thread, link).await?;
+ assert!(messages.iter().all(|message| !message.is_starred));
+ let ids: Vec<_> = messages.iter().map(|message| message.db_id).collect();
+ let labels = repo.labels_by_message_ids(&ids).await?;
+ assert!(
+ !labels
+ .values()
+ .flatten()
+ .any(|label| label.provider_label_id == "STARRED")
+ );
+ assert!(pending_markers(&pool).await?.is_empty());
Ok(())
}
@@ -469,7 +675,7 @@ async fn bulk_archive_sends_one_gmail_write_per_inbox_and_reports_threads_it_ski
pool: Pool<Postgres>,
) -> anyhow::Result<()> {
let enqueuer = RecordingEnqueuer::default();
- let service = service_with_enqueuer(pool, enqueuer.clone());
+ let service = service_with_enqueuer(pool.clone(), enqueuer.clone());
let owner = MacroUserIdStr::try_from_email("[email protected]")?;
let primary_thread = Uuid::parse_str("11111111-1111-1111-1111-111111111111")?;
let empty_thread = Uuid::parse_str("22222222-2222-2222-2222-222222222222")?;
@@ -526,6 +732,28 @@ async fn bulk_archive_sends_one_gmail_write_per_inbox_and_reports_threads_it_ski
),
]
);
+ let write_ids = enqueuer.1.lock().unwrap().clone();
+ assert_ne!(write_ids[0], write_ids[1]);
+ assert_eq!(
+ pending_markers(&pool).await?,
+ vec![
+ (
+ Uuid::parse_str("11111111-aaaa-aaaa-aaaa-111111111111")?,
+ "INBOX".to_string(),
+ write_ids[0]
+ ),
+ (
+ Uuid::parse_str("11111111-bbbb-bbbb-bbbb-111111111111")?,
+ "INBOX".to_string(),
+ write_ids[0]
+ ),
+ (
+ Uuid::parse_str("33333333-aaaa-aaaa-aaaa-333333333333")?,
+ "INBOX".to_string(),
+ write_ids[1]
+ ),
+ ]
+ );
assert_inbox_state(&service.email_repo, primary_thread, false).await?;
assert_inbox_state(&service.email_repo, secondary_thread, false).await?;
Ok(())
@@ -541,7 +769,7 @@ async fn bulk_archive_sends_one_gmail_write_per_inbox_and_reports_threads_it_ski
async fn bulk_archive_reports_an_enqueue_outage_instead_of_failing_every_thread(
pool: Pool<Postgres>,
) -> anyhow::Result<()> {
- let service = service_with_enqueuer(pool, FailingEnqueuer);
+ let service = service_with_enqueuer(pool.clone(), FailingEnqueuer);
let primary_thread = Uuid::parse_str("11111111-1111-1111-1111-111111111111")?;
let secondary_thread = Uuid::parse_str("33333333-3333-3333-3333-333333333333")?;
@@ -556,12 +784,13 @@ async fn bulk_archive_reports_an_enqueue_outage_instead_of_failing_every_thread(
assert!(matches!(result, Err(EmailErr::EnqueueErr(_))));
assert_inbox_state(&service.email_repo, primary_thread, true).await?;
assert_inbox_state(&service.email_repo, secondary_thread, true).await?;
+ assert!(pending_markers(&pool).await?.is_empty());
Ok(())
}
/// Accepts one label write, then fails, with room for two messages per write.
#[derive(Clone, Default)]
-struct FailsAfterFirstWriteEnqueuer(Arc<std::sync::Mutex<Vec<LabelWrite>>>);
+struct FailsAfterFirstWriteEnqueuer(Arc<std::sync::Mutex<Vec<(LabelWrite, Uuid)>>>);
impl EmailMessageEnqueuer for FailsAfterFirstWriteEnqueuer {
type Err = anyhow::Error;
const MAX_LABEL_WRITE_MESSAGES: usize = 2;
@@ -579,12 +808,16 @@ impl EmailMessageEnqueuer for FailsAfterFirstWriteEnqueuer {
messages: Vec<(Uuid, String)>,
labels_to_add: Vec<String>,
labels_to_remove: Vec<String>,
+ pending_write_id: Uuid,
) -> anyhow::Result<()> {
let mut writes = self.0.lock().unwrap();
if !writes.is_empty() {
anyhow::bail!("test queue unavailable");
}
- writes.push((link_id, messages, labels_to_add, labels_to_remove));
+ writes.push((
+ (link_id, messages, labels_to_add, labels_to_remove),
+ pending_write_id,
+ ));
Ok(())
}
async fn enqueue_gmail_ops_block_sender(&self, _: Uuid, _: String) -> anyhow::Result<()> {
@@ -611,7 +844,7 @@ async fn bulk_archive_reverts_only_the_threads_whose_write_could_not_be_queued(
pool: Pool<Postgres>,
) -> anyhow::Result<()> {
let enqueuer = FailsAfterFirstWriteEnqueuer::default();
- let service = service_with_enqueuer(pool, enqueuer.clone());
+ let service = service_with_enqueuer(pool.clone(), enqueuer.clone());
let queued_thread = Uuid::parse_str("11111111-1111-1111-1111-111111111111")?;
let unqueued_thread = Uuid::parse_str("77777777-7777-7777-7777-777777777777")?;
@@ -631,16 +864,31 @@ async fn bulk_archive_reverts_only_the_threads_whose_write_could_not_be_queued(
}
);
// The first thread's two messages fill the first write on their own.
+ let writes = enqueuer.0.lock().unwrap().clone();
assert_eq!(
- enqueuer
- .0
- .lock()
- .unwrap()
+ writes
.iter()
- .map(|(_, messages, _, _)| messages.len())
+ .map(|((_, messages, _, _), _)| messages.len())
.collect::<Vec<_>>(),
vec![2]
);
+ let write_id = writes[0].1;
+ // Only the queued group keeps its markers.
+ assert_eq!(
+ pending_markers(&pool).await?,
+ vec![
+ (
+ Uuid::parse_str("11111111-aaaa-aaaa-aaaa-111111111111")?,
+ "INBOX".to_string(),
+ write_id
+ ),
+ (
+ Uuid::parse_str("11111111-bbbb-bbbb-bbbb-111111111111")?,
+ "INBOX".to_string(),
+ write_id
+ ),
+ ]
+ );
assert_inbox_state(&service.email_repo, queued_thread, false).await?;
assert_inbox_state(&service.email_repo, unqueued_thread, true).await?;
Ok(())
diff --git a/crates/email/src/outbound/mailbox_pg/ingest.rs b/crates/email/src/outbound/mailbox_pg/ingest.rs
index f4283bace3..c155793645 100644
--- a/crates/email/src/outbound/mailbox_pg/ingest.rs
+++ b/crates/email/src/outbound/mailbox_pg/ingest.rs
@@ -202,6 +202,7 @@ impl MailboxIngest for PgMailboxSync {
lease.mailbox.link_id,
recipients,
false,
+ None,
)
.await
.map_err(|_| MailboxError::Persistence)?;
diff --git a/crates/email_api_client/src/domain/service.rs b/crates/email_api_client/src/domain/service.rs
index 5076b841f7..eddf0cd49e 100644
--- a/crates/email_api_client/src/domain/service.rs
+++ b/crates/email_api_client/src/domain/service.rs
@@ -15,6 +15,7 @@ mod labels;
/// Credential and quota orchestration for generation-bound mailbox operations.
pub mod mailbox;
mod messages;
+pub use messages::SEPARATE_LABEL_WRITES_BELOW;
mod send;
mod subscription;
mod sync;
diff --git a/crates/email_api_client/src/domain/service/messages.rs b/crates/email_api_client/src/domain/service/messages.rs
index f75ee04b6e..4d632b86d9 100644
--- a/crates/email_api_client/src/domain/service/messages.rs
+++ b/crates/email_api_client/src/domain/service/messages.rs
@@ -172,7 +172,8 @@ where
/// Writes one message's labels. With `skip_missing`, a message the mailbox
/// no longer has counts as written, as it does in a batch request.
- async fn modify_one_message_labels(
+ #[tracing::instrument(skip(self, labels_to_add, labels_to_remove), fields(gate_in_flight = tracing::field::Empty, gate_cap = tracing::field::Empty))]
+ pub async fn modify_one_message_labels(
&self,
link_id: Uuid,
provider_message_id: &str,
@@ -211,7 +212,7 @@ where
/// Gmail charges 5 quota units per `messages.modify` and 50 per
/// `messages.batchModify` of any size, so below ten messages separate writes
/// cost less.
-const SEPARATE_LABEL_WRITES_BELOW: usize = 10;
+pub const SEPARATE_LABEL_WRITES_BELOW: usize = 10;
impl<R, T, L> EmailApiClientServiceImpl<R, T, L>
where
diff --git a/crates/email_db_client/AGENTS.md b/crates/email_db_client/AGENTS.md
index e651b3883b..523231314f 100644
--- a/crates/email_db_client/AGENTS.md
+++ b/crates/email_db_client/AGENTS.md
@@ -53,6 +53,19 @@ Parsed message responses expose the draft generation captured in the same SQL sn
- Forwarded-attachment delivery reads preserve `source_link_id` from the original
message; the destination draft inbox is only the query access scope.
+- `insert_message_with_tx` leaves labels with an unexpired
+ `email_gmail_pending_label_writes` marker, and the read and starred flags
+ they drive, in their local state when it re-syncs a stored message. Given a
+ `ResyncSnapshot` from before the Gmail fetch, it changes only labels where
+ Gmail disagrees with that snapshot, so a write made during the fetch stays.
+- `email_gmail_pending_label_writes` keeps one row per write on a message's
+ label, newest by `seq` (staging order, which write ids need not follow).
+ Only the newest row's write sends; older ones wait while a newer row hasn't
+ expired, so abandoning a write can put the label back for the newest older
+ write whose row hasn't expired (`label_present`) or the state before it
+ (`label_present_before`, passed on by older writes given up). Code that
+ stages, confirms or abandons a Gmail label write takes
+ `lock_thread_label_writes` for its threads first.
- Quote and alias camelCase columns to Rust field names; check
`crates/macro_db_client/migrations/` for real column spelling, don't guess.
diff --git a/crates/email_db_client/src/labels/get.rs b/crates/email_db_client/src/labels/get.rs
index 7d40386a0c..a828857827 100644
--- a/crates/email_db_client/src/labels/get.rs
+++ b/crates/email_db_client/src/labels/get.rs
@@ -246,3 +246,35 @@ pub async fn fetch_label_by_id(
Ok(service_label)
}
+
+/// Messages among `message_ids` that gain `provider_label_id` when `add` is
+/// true, or lose it when false.
+#[tracing::instrument(skip(executor, message_ids), err)]
+pub async fn messages_changing_label<'e, E>(
+ executor: E,
+ link_id: Uuid,
+ message_ids: &[Uuid],
+ provider_label_id: &str,
+ add: bool,
+) -> Result<Vec<Uuid>, sqlx::Error>
+where
+ E: sqlx::Executor<'e, Database = sqlx::Postgres>,
+{
+ sqlx::query_scalar!(
+ r#"
+ SELECT m.id AS "id!"
+ FROM unnest($1::uuid[]) AS m(id)
+ WHERE EXISTS (
+ SELECT 1 FROM email_effective_message_labels ml
+ JOIN email_labels l ON l.id = ml.label_id
+ WHERE ml.message_id = m.id AND l.link_id = $2 AND l.provider_label_id = $3
+ ) <> $4
+ "#,
+ message_ids,
+ link_id,
+ provider_label_id,
+ add,
+ )
+ .fetch_all(executor)
+ .await
+}
diff --git a/crates/email_db_client/src/labels/get/test.rs b/crates/email_db_client/src/labels/get/test.rs
index e773d7935e..ca3f97e5c9 100644
--- a/crates/email_db_client/src/labels/get/test.rs
+++ b/crates/email_db_client/src/labels/get/test.rs
@@ -1,4 +1,4 @@
-use crate::labels::get::fetch_message_labels_in_bulk;
+use crate::labels::get::{fetch_message_labels_in_bulk, messages_changing_label};
use anyhow::Result;
use macro_db_migrator::MACRO_DB_MIGRATIONS;
use models_email::db::label::{LabelListVisibility, LabelType, MessageListVisibility};
@@ -138,3 +138,27 @@ async fn fetch_message_labels_in_bulk_returns_empty_for_nonexistent_messages(
Ok(())
}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../fixtures", scripts("fetch_message_labels_in_bulk"))
+)]
+async fn messages_changing_label_skips_messages_already_in_the_requested_state(
+ pool: Pool<Postgres>,
+) -> Result<()> {
+ let link_id = Uuid::parse_str("00000000-0000-0000-0000-000000000801")?;
+ let in_inbox = Uuid::parse_str("00000000-0000-0000-0000-000000008501")?;
+ let important = Uuid::parse_str("00000000-0000-0000-0000-000000008502")?;
+ let unlabeled = Uuid::parse_str("00000000-0000-0000-0000-000000008503")?;
+ let messages = [in_inbox, important, unlabeled];
+
+ let mut gaining = messages_changing_label(&pool, link_id, &messages, "INBOX", true).await?;
+ gaining.sort();
+ assert_eq!(gaining, vec![important, unlabeled]);
+ assert_eq!(
+ messages_changing_label(&pool, link_id, &messages, "INBOX", false).await?,
+ vec![in_inbox]
+ );
+
+ Ok(())
+}
diff --git a/crates/email_db_client/src/labels/insert.rs b/crates/email_db_client/src/labels/insert.rs
index 38c70c551f..851a7087c7 100644
--- a/crates/email_db_client/src/labels/insert.rs
+++ b/crates/email_db_client/src/labels/insert.rs
@@ -258,15 +258,17 @@ pub async fn insert_label(
Ok(populated_service_label)
}
-/// inserts the labels of an email into the database in a batch
+/// inserts the labels of an email into the database in a batch. Labels in
+/// `pending_provider_label_ids` keep their current state on the message.
pub async fn insert_message_labels(
tx: &mut sqlx::PgConnection,
link_id: Uuid,
message_id: Uuid,
provider_label_ids: &[String],
delete_old: bool,
+ pending_provider_label_ids: &[String],
) -> anyhow::Result<()> {
- if provider_label_ids.is_empty() {
+ if provider_label_ids.is_empty() && !delete_old {
return Ok(());
}
@@ -308,7 +310,11 @@ pub async fn insert_message_labels(
);
}
- let label_db_ids: Vec<Uuid> = label_mappings.iter().map(|m| m.id).collect();
+ let label_db_ids: Vec<Uuid> = label_mappings
+ .iter()
+ .filter(|m| !pending_provider_label_ids.contains(&m.provider_label_id))
+ .map(|m| m.id)
+ .collect();
// with delete_old, records that don't match the ones to insert are removed in case we are
// doing an upsert and some of the old ones got removed. An unchanged set deletes and inserts
@@ -320,6 +326,10 @@ pub async fn insert_message_labels(
WHERE $3::bool
AND message_id = $1::uuid
AND label_id NOT IN (SELECT UNNEST($2::uuid[]))
+ AND label_id NOT IN (
+ SELECT id FROM email_labels
+ WHERE link_id = $4 AND provider_label_id = ANY($5::text[])
+ )
)
INSERT INTO email_message_labels (message_id, label_id)
SELECT $1, label_id FROM UNNEST($2::uuid[]) AS t(label_id)
@@ -327,7 +337,9 @@ pub async fn insert_message_labels(
"#,
message_id,
&label_db_ids,
- delete_old
+ delete_old,
+ link_id,
+ pending_provider_label_ids
)
.execute(&mut *tx)
.await?;
diff --git a/crates/email_db_client/src/labels/mod.rs b/crates/email_db_client/src/labels/mod.rs
index e1dee670dd..88dae2d862 100644
--- a/crates/email_db_client/src/labels/mod.rs
+++ b/crates/email_db_client/src/labels/mod.rs
@@ -1,3 +1,4 @@
pub mod delete;
pub mod get;
pub mod insert;
+pub mod pending_writes;
diff --git a/crates/email_db_client/src/labels/pending_writes.rs b/crates/email_db_client/src/labels/pending_writes.rs
new file mode 100644
index 0000000000..45eec752e8
--- /dev/null
+++ b/crates/email_db_client/src/labels/pending_writes.rs
@@ -0,0 +1,353 @@
+#[cfg(test)]
+mod test;
+
+use models_email::email::service::label::system_labels;
+use sqlx::types::Uuid;
+use sqlx::{Executor, PgConnection, Postgres};
+use std::collections::HashMap;
+
+/// Makes label writes that stage or undo a change on the same threads take
+/// turns until the transaction ends, so each reads the state the one before
+/// it left. Locks are taken in id order, so writers on several threads cannot
+/// deadlock each other.
+#[tracing::instrument(skip(tx, thread_ids), err)]
+pub async fn lock_thread_label_writes(
+ tx: &mut PgConnection,
+ thread_ids: &[Uuid],
+) -> Result<(), sqlx::Error> {
+ let mut thread_ids = thread_ids.to_vec();
+ thread_ids.sort_unstable();
+ thread_ids.dedup();
+ for thread_id in thread_ids {
+ sqlx::query!(
+ "SELECT pg_advisory_xact_lock(hashtextextended($1, 0))",
+ format!("email_thread_label_write:{thread_id}"),
+ )
+ .execute(&mut *tx)
+ .await?;
+ }
+ Ok(())
+}
+
+/// Records that the Gmail operations sharing `write_id` will set
+/// `provider_label_id` on `message_ids` of the link to `label_present`, each a
+/// change from the opposite state, so inbox sync keeps the local label state
+/// until Gmail confirms it. Rows stop applying after 12 hours, longer than an
+/// operation is redelivered through the gmail_ops queues, so only operations
+/// that are dead-lettered or lost fall back to Gmail's labels.
+#[tracing::instrument(skip(executor, message_ids), err)]
+pub async fn mark_pending_label_writes<'e, E>(
+ executor: E,
+ link_id: Uuid,
+ message_ids: &[Uuid],
+ provider_label_id: &str,
+ label_present: bool,
+ write_id: Uuid,
+) -> Result<(), sqlx::Error>
+where
+ E: Executor<'e, Database = Postgres>,
+{
+ if message_ids.is_empty() {
+ return Ok(());
+ }
+
+ sqlx::query!(
+ r#"
+ INSERT INTO email_gmail_pending_label_writes
+ (message_id, provider_label_id, write_id, label_present, label_present_before, expires_at)
+ SELECT m.id, $3, $4, $5, NOT $5, now() + interval '12 hours'
+ FROM email_messages m
+ WHERE m.id = ANY($1) AND m.link_id = $2
+ ORDER BY m.id
+ ON CONFLICT (message_id, provider_label_id, write_id) DO NOTHING
+ "#,
+ message_ids,
+ link_id,
+ provider_label_id,
+ write_id,
+ label_present,
+ )
+ .execute(executor)
+ .await?;
+
+ Ok(())
+}
+
+/// What a write's operation does with one of its messages now.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum LabelWriteTurn {
+ /// The write is the newest on every label: send it to Gmail.
+ Send,
+ /// A newer write on the message is still on its way. Its outcome decides
+ /// whether this one still has to send, so try again later.
+ Wait,
+ /// A newer write was confirmed, or gave up on waiting for this one:
+ /// nothing left to send.
+ Superseded,
+}
+
+/// The turn of `write_id` on each of `message_ids` for `provider_label_ids`.
+/// A write waits for every newer write whose row hasn't expired, whatever its
+/// id: requests can stage in another order than they drew their ids. Bulk
+/// writes can each be the newer one on a different message, so an operation
+/// sends the messages it is newest on before it waits for the rest.
+#[tracing::instrument(skip(executor, message_ids), err)]
+pub async fn label_write_turns<'e, E>(
+ executor: E,
+ message_ids: &[Uuid],
+ provider_label_ids: &[String],
+ write_id: Uuid,
+) -> Result<HashMap<Uuid, LabelWriteTurn>, sqlx::Error>
+where
+ E: Executor<'e, Database = Postgres>,
+{
+ let rows = sqlx::query!(
+ r#"
+ SELECT
+ m.id AS "message_id!",
+ mine.seq IS NOT NULL AS "claimed!",
+ EXISTS (
+ SELECT 1 FROM email_gmail_pending_label_writes newer
+ WHERE newer.message_id = m.id AND newer.provider_label_id = l.label
+ AND newer.seq > mine.seq
+ ) AS "has_newer!",
+ EXISTS (
+ SELECT 1 FROM email_gmail_pending_label_writes newer
+ WHERE newer.message_id = m.id AND newer.provider_label_id = l.label
+ AND newer.seq > mine.seq AND newer.expires_at > now()
+ ) AS "waits!"
+ FROM unnest($1::uuid[]) AS m(id)
+ CROSS JOIN unnest($3::text[]) AS l(label)
+ LEFT JOIN email_gmail_pending_label_writes mine
+ ON mine.message_id = m.id AND mine.provider_label_id = l.label AND mine.write_id = $2
+ "#,
+ message_ids,
+ write_id,
+ provider_label_ids,
+ )
+ .fetch_all(executor)
+ .await?;
+
+ let mut turns = HashMap::new();
+ for row in rows {
+ let turn = if !row.claimed {
+ LabelWriteTurn::Superseded
+ } else if row.waits {
+ LabelWriteTurn::Wait
+ } else if row.has_newer {
+ LabelWriteTurn::Superseded
+ } else {
+ LabelWriteTurn::Send
+ };
+ turns
+ .entry(row.message_id)
+ .and_modify(|current: &mut LabelWriteTurn| {
+ *current = match (*current, turn) {
+ (LabelWriteTurn::Wait, _) | (_, LabelWriteTurn::Wait) => LabelWriteTurn::Wait,
+ (LabelWriteTurn::Send, LabelWriteTurn::Send) => LabelWriteTurn::Send,
+ _ => LabelWriteTurn::Superseded,
+ }
+ })
+ .or_insert(turn);
+ }
+ Ok(turns)
+}
+
+/// Gmail applied `write_id` on `message_ids`: removes its rows and those of
+/// every older write on the same labels, which Gmail no longer needs. A
+/// newer write's row now falls back to this write's state. Callers hold
+/// `lock_thread_label_writes` for the messages' threads.
+#[tracing::instrument(skip(tx, message_ids), err)]
+pub async fn confirm_pending_label_writes(
+ tx: &mut PgConnection,
+ message_ids: &[Uuid],
+ write_id: Uuid,
+) -> Result<(), sqlx::Error> {
+ if message_ids.is_empty() {
+ return Ok(());
+ }
+
+ sqlx::query!(
+ r#"
+ WITH mine AS (
+ SELECT message_id, provider_label_id, seq, label_present
+ FROM email_gmail_pending_label_writes
+ WHERE message_id = ANY($1) AND write_id = $2
+ ), next AS (
+ SELECT DISTINCT ON (c.message_id, c.provider_label_id) c.seq, mine.label_present
+ FROM email_gmail_pending_label_writes c
+ JOIN mine ON c.message_id = mine.message_id
+ AND c.provider_label_id = mine.provider_label_id AND c.seq > mine.seq
+ ORDER BY c.message_id, c.provider_label_id, c.seq
+ ), falls_back AS (
+ UPDATE email_gmail_pending_label_writes c
+ SET label_present_before = next.label_present
+ FROM next
+ WHERE c.seq = next.seq
+ )
+ DELETE FROM email_gmail_pending_label_writes c
+ USING mine
+ WHERE c.message_id = mine.message_id
+ AND c.provider_label_id = mine.provider_label_id
+ AND c.seq <= mine.seq
+ "#,
+ message_ids,
+ write_id,
+ )
+ .execute(&mut *tx)
+ .await?;
+
+ Ok(())
+}
+
+/// Gives up `write_id`'s rows on `message_ids` because Gmail will never apply
+/// them, and puts each label the write was the newest on back in the state
+/// Gmail will end up with: that of the newest older write still on its way,
+/// or the one before this write, which older writes given up pass on. An
+/// older write whose row expired is presumed lost and no longer counts.
+/// Labels a newer write changed keep that write's state. Returns the messages
+/// whose labels were put back. Callers hold `lock_thread_label_writes` for
+/// the messages' threads.
+#[tracing::instrument(skip(tx, message_ids), err)]
+pub async fn abandon_pending_label_writes(
+ tx: &mut PgConnection,
+ link_id: Uuid,
+ message_ids: &[Uuid],
+ write_id: Uuid,
+) -> Result<Vec<Uuid>, sqlx::Error> {
+ if message_ids.is_empty() {
+ return Ok(Vec::new());
+ }
+
+ let restored = sqlx::query!(
+ r#"
+ WITH mine AS (
+ SELECT c.message_id, c.provider_label_id, c.seq, c.label_present_before,
+ (SELECT min(newer.seq) FROM email_gmail_pending_label_writes newer
+ WHERE newer.message_id = c.message_id
+ AND newer.provider_label_id = c.provider_label_id
+ AND newer.seq > c.seq) AS newer_seq,
+ (SELECT older.label_present FROM email_gmail_pending_label_writes older
+ WHERE older.message_id = c.message_id
+ AND older.provider_label_id = c.provider_label_id
+ AND older.seq < c.seq AND older.expires_at > now()
+ ORDER BY older.seq DESC
+ LIMIT 1) AS older_present
+ FROM email_gmail_pending_label_writes c
+ WHERE c.message_id = ANY($1) AND c.write_id = $2
+ ), inherited AS (
+ UPDATE email_gmail_pending_label_writes c
+ SET label_present_before = mine.label_present_before
+ FROM mine
+ WHERE c.seq = mine.newer_seq
+ ), removed AS (
+ DELETE FROM email_gmail_pending_label_writes c
+ USING mine
+ WHERE c.seq = mine.seq
+ )
+ SELECT
+ message_id AS "message_id!",
+ provider_label_id AS "provider_label_id!",
+ COALESCE(older_present, label_present_before) AS "label_present!"
+ FROM mine
+ WHERE newer_seq IS NULL
+ "#,
+ message_ids,
+ write_id,
+ )
+ .fetch_all(&mut *tx)
+ .await?;
+ if restored.is_empty() {
+ return Ok(Vec::new());
+ }
+
+ let (present, absent): (Vec<_>, Vec<_>) =
+ restored.iter().partition(|restore| restore.label_present);
+ let present_ids: Vec<Uuid> = present.iter().map(|restore| restore.message_id).collect();
+ let present_labels: Vec<String> = present
+ .iter()
+ .map(|restore| restore.provider_label_id.clone())
+ .collect();
+ let absent_ids: Vec<Uuid> = absent.iter().map(|restore| restore.message_id).collect();
+ let absent_labels: Vec<String> = absent
+ .iter()
+ .map(|restore| restore.provider_label_id.clone())
+ .collect();
+ sqlx::query!(
+ r#"
+ INSERT INTO email_message_labels (message_id, label_id)
+ SELECT r.message_id, l.id
+ FROM unnest($1::uuid[], $2::text[]) AS r(message_id, provider_label_id)
+ JOIN email_labels l ON l.link_id = $3 AND l.provider_label_id = r.provider_label_id
+ ON CONFLICT (message_id, label_id) DO NOTHING
+ "#,
+ &present_ids,
+ &present_labels,
+ link_id,
+ )
+ .execute(&mut *tx)
+ .await?;
+ sqlx::query!(
+ r#"
+ DELETE FROM email_message_labels ml
+ USING unnest($1::uuid[], $2::text[]) AS r(message_id, provider_label_id), email_labels l
+ WHERE ml.message_id = r.message_id AND ml.label_id = l.id
+ AND l.link_id = $3 AND l.provider_label_id = r.provider_label_id
+ "#,
+ &absent_ids,
+ &absent_labels,
+ link_id,
+ )
+ .execute(&mut *tx)
+ .await?;
+
+ let mut restored_ids: Vec<Uuid> = restored.iter().map(|restore| restore.message_id).collect();
+ restored_ids.sort_unstable();
+ restored_ids.dedup();
+ sqlx::query!(
+ r#"
+ UPDATE email_messages m
+ SET
+ is_read = NOT EXISTS (
+ SELECT 1 FROM email_message_labels ml JOIN email_labels l ON l.id = ml.label_id
+ WHERE ml.message_id = m.id AND l.provider_label_id = $2
+ ),
+ is_starred = EXISTS (
+ SELECT 1 FROM email_message_labels ml JOIN email_labels l ON l.id = ml.label_id
+ WHERE ml.message_id = m.id AND l.provider_label_id = $3
+ ),
+ updated_at = now()
+ WHERE m.id = ANY($1)
+ "#,
+ &restored_ids,
+ system_labels::UNREAD,
+ system_labels::STARRED,
+ )
+ .execute(&mut *tx)
+ .await?;
+
+ Ok(restored_ids)
+}
+
+/// Provider label ids on the message with a local change Gmail has not
+/// confirmed yet.
+#[tracing::instrument(skip(executor), err)]
+pub async fn pending_provider_label_ids<'e, E>(
+ executor: E,
+ message_id: Uuid,
+) -> Result<Vec<String>, sqlx::Error>
+where
+ E: Executor<'e, Database = Postgres>,
+{
+ sqlx::query_scalar!(
+ r#"
+ SELECT DISTINCT provider_label_id AS "provider_label_id!"
+ FROM email_gmail_pending_label_writes
+ WHERE message_id = $1 AND expires_at > now()
+ ORDER BY provider_label_id
+ "#,
+ message_id,
+ )
+ .fetch_all(executor)
+ .await
+}
diff --git a/crates/email_db_client/src/labels/pending_writes/test.rs b/crates/email_db_client/src/labels/pending_writes/test.rs
new file mode 100644
index 0000000000..f4e3291f44
--- /dev/null
+++ b/crates/email_db_client/src/labels/pending_writes/test.rs
@@ -0,0 +1,366 @@
+use super::*;
+use anyhow::Result;
+use macro_db_migrator::MACRO_DB_MIGRATIONS;
+use sqlx::{Pool, Postgres};
+
+const LINK: Uuid = Uuid::from_u128(0x801);
+const MESSAGE: Uuid = Uuid::from_u128(0x8501);
+const INBOX_LABEL: Uuid = Uuid::from_u128(0x18001);
+
+/// Stages a local INBOX change on `MESSAGE` the way a thread write does: the
+/// label and the write's row change together.
+async fn stage_inbox(pool: &Pool<Postgres>, present: bool, write_id: Uuid) -> Result<()> {
+ let mut tx = pool.begin().await?;
+ if present {
+ sqlx::query!(
+ "INSERT INTO email_message_labels (message_id, label_id) VALUES ($1, $2)",
+ MESSAGE,
+ INBOX_LABEL
+ )
+ .execute(&mut *tx)
+ .await?;
+ } else {
+ sqlx::query!(
+ "DELETE FROM email_message_labels WHERE message_id = $1 AND label_id = $2",
+ MESSAGE,
+ INBOX_LABEL
+ )
+ .execute(&mut *tx)
+ .await?;
+ }
+ mark_pending_label_writes(&mut *tx, LINK, &[MESSAGE], "INBOX", present, write_id).await?;
+ tx.commit().await?;
+ Ok(())
+}
+
+async fn in_inbox(pool: &Pool<Postgres>) -> Result<bool> {
+ Ok(sqlx::query_scalar!(
+ r#"SELECT EXISTS (SELECT 1 FROM email_message_labels WHERE message_id = $1 AND label_id = $2) AS "exists!""#,
+ MESSAGE,
+ INBOX_LABEL
+ )
+ .fetch_one(pool)
+ .await?)
+}
+
+async fn turn(pool: &Pool<Postgres>, write_id: Uuid) -> Result<LabelWriteTurn> {
+ Ok(label_write_turns(pool, &[MESSAGE], &["INBOX".to_string()], write_id).await?[&MESSAGE])
+}
+
+async fn abandon(pool: &Pool<Postgres>, write_id: Uuid) -> Result<Vec<Uuid>> {
+ let mut tx = pool.begin().await?;
+ let restored = abandon_pending_label_writes(&mut tx, LINK, &[MESSAGE], write_id).await?;
+ tx.commit().await?;
+ Ok(restored)
+}
+
+async fn confirm(pool: &Pool<Postgres>, write_id: Uuid) -> Result<()> {
+ let mut tx = pool.begin().await?;
+ confirm_pending_label_writes(&mut tx, &[MESSAGE], write_id).await?;
+ tx.commit().await?;
+ Ok(())
+}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../fixtures", scripts("fetch_message_labels_in_bulk"))
+)]
+async fn marks_only_messages_of_the_link(pool: Pool<Postgres>) -> Result<()> {
+ let other_link = Uuid::from_u128(0x802);
+ let second = Uuid::from_u128(0x8502);
+ let write_id = Uuid::from_u128(0xaa01);
+
+ mark_pending_label_writes(&pool, LINK, &[MESSAGE], "INBOX", false, write_id).await?;
+ mark_pending_label_writes(&pool, other_link, &[second], "INBOX", false, write_id).await?;
+
+ assert_eq!(pending_provider_label_ids(&pool, MESSAGE).await?, ["INBOX"]);
+ assert!(pending_provider_label_ids(&pool, second).await?.is_empty());
+ Ok(())
+}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../fixtures", scripts("fetch_message_labels_in_bulk"))
+)]
+async fn an_older_write_waits_until_a_newer_one_is_confirmed(pool: Pool<Postgres>) -> Result<()> {
+ let archive = Uuid::from_u128(0xaa01);
+ let unarchive = Uuid::from_u128(0xaa02);
+
+ stage_inbox(&pool, false, archive).await?;
+ stage_inbox(&pool, true, unarchive).await?;
+ assert_eq!(turn(&pool, archive).await?, LabelWriteTurn::Wait);
+ assert_eq!(turn(&pool, unarchive).await?, LabelWriteTurn::Send);
+
+ confirm(&pool, unarchive).await?;
+ assert_eq!(turn(&pool, archive).await?, LabelWriteTurn::Superseded);
+ assert!(pending_provider_label_ids(&pool, MESSAGE).await?.is_empty());
+ assert!(in_inbox(&pool).await?);
+ Ok(())
+}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../fixtures", scripts("fetch_message_labels_in_bulk"))
+)]
+async fn an_abandoned_write_puts_the_label_back_for_the_older_write_still_waiting(
+ pool: Pool<Postgres>,
+) -> Result<()> {
+ let archive = Uuid::from_u128(0xaa01);
+ let unarchive = Uuid::from_u128(0xaa02);
+
+ stage_inbox(&pool, false, archive).await?;
+ stage_inbox(&pool, true, unarchive).await?;
+ assert_eq!(abandon(&pool, unarchive).await?, [MESSAGE]);
+
+ assert!(!in_inbox(&pool).await?);
+ assert_eq!(turn(&pool, archive).await?, LabelWriteTurn::Send);
+ assert_eq!(pending_provider_label_ids(&pool, MESSAGE).await?, ["INBOX"]);
+ Ok(())
+}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../fixtures", scripts("fetch_message_labels_in_bulk"))
+)]
+async fn abandoned_writes_keep_every_older_write_they_took_over(
+ pool: Pool<Postgres>,
+) -> Result<()> {
+ let archive = Uuid::from_u128(0xaa01);
+ let unarchive = Uuid::from_u128(0xaa02);
+ let archive_again = Uuid::from_u128(0xaa03);
+
+ stage_inbox(&pool, false, archive).await?;
+ stage_inbox(&pool, true, unarchive).await?;
+ stage_inbox(&pool, false, archive_again).await?;
+
+ // The second archive can't be queued, then Gmail rejects the unarchive.
+ abandon(&pool, archive_again).await?;
+ assert!(in_inbox(&pool).await?);
+ abandon(&pool, unarchive).await?;
+
+ assert!(!in_inbox(&pool).await?);
+ assert_eq!(turn(&pool, archive).await?, LabelWriteTurn::Send);
+ Ok(())
+}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../fixtures", scripts("fetch_message_labels_in_bulk"))
+)]
+async fn when_every_write_is_abandoned_the_label_returns_to_its_state_before_the_first(
+ pool: Pool<Postgres>,
+) -> Result<()> {
+ let archive = Uuid::from_u128(0xaa01);
+ let unarchive = Uuid::from_u128(0xaa02);
+
+ stage_inbox(&pool, false, archive).await?;
+ stage_inbox(&pool, true, unarchive).await?;
+
+ // Gmail rejects the archive while the unarchive waits: nothing changes
+ // locally, but the unarchive now starts from the original state.
+ assert!(abandon(&pool, archive).await?.is_empty());
+ assert!(in_inbox(&pool).await?);
+ assert_eq!(turn(&pool, unarchive).await?, LabelWriteTurn::Send);
+
+ let archive_again = Uuid::from_u128(0xaa03);
+ stage_inbox(&pool, false, archive_again).await?;
+ abandon(&pool, archive_again).await?;
+ abandon(&pool, unarchive).await?;
+ assert!(in_inbox(&pool).await?);
+ assert!(pending_provider_label_ids(&pool, MESSAGE).await?.is_empty());
+ Ok(())
+}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../fixtures", scripts("fetch_message_labels_in_bulk"))
+)]
+async fn a_confirmed_write_is_what_a_newer_abandoned_write_falls_back_to(
+ pool: Pool<Postgres>,
+) -> Result<()> {
+ let archive = Uuid::from_u128(0xaa01);
+ let unarchive = Uuid::from_u128(0xaa02);
+ let archive_again = Uuid::from_u128(0xaa03);
+
+ stage_inbox(&pool, false, archive).await?;
+ stage_inbox(&pool, true, unarchive).await?;
+ stage_inbox(&pool, false, archive_again).await?;
+ // The unarchive reached Gmail before the second archive was staged.
+ confirm(&pool, unarchive).await?;
+ assert_eq!(turn(&pool, archive).await?, LabelWriteTurn::Superseded);
+
+ abandon(&pool, archive_again).await?;
+ assert!(in_inbox(&pool).await?);
+ assert!(pending_provider_label_ids(&pool, MESSAGE).await?.is_empty());
+ Ok(())
+}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../fixtures", scripts("fetch_message_labels_in_bulk"))
+)]
+async fn a_write_waits_for_live_newer_writes_whatever_their_ids(
+ pool: Pool<Postgres>,
+) -> Result<()> {
+ let unarchive = Uuid::from_u128(0xaa01);
+ let archive = Uuid::from_u128(0xaa02);
+
+ // Staged in the opposite order to their ids, as when two requests race.
+ stage_inbox(&pool, false, archive).await?;
+ stage_inbox(&pool, true, unarchive).await?;
+ assert_eq!(turn(&pool, archive).await?, LabelWriteTurn::Wait);
+ assert_eq!(turn(&pool, unarchive).await?, LabelWriteTurn::Send);
+
+ // Gmail rejects the unarchive, or it can't be queued: the archive is the
+ // user's last change again, and it sends.
+ assert_eq!(abandon(&pool, unarchive).await?, [MESSAGE]);
+ assert!(!in_inbox(&pool).await?);
+ assert_eq!(turn(&pool, archive).await?, LabelWriteTurn::Send);
+
+ let unarchive_again = Uuid::from_u128(0xaa03);
+ stage_inbox(&pool, true, unarchive_again).await?;
+ assert_eq!(turn(&pool, archive).await?, LabelWriteTurn::Wait);
+ sqlx::query!(
+ "UPDATE email_gmail_pending_label_writes SET expires_at = now() - interval '1 second' WHERE write_id = $1",
+ unarchive_again
+ )
+ .execute(&pool)
+ .await?;
+ assert_eq!(turn(&pool, archive).await?, LabelWriteTurn::Superseded);
+ Ok(())
+}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../fixtures", scripts("fetch_message_labels_in_bulk"))
+)]
+async fn writes_newer_than_each_other_on_two_messages_each_have_one_to_send(
+ pool: Pool<Postgres>,
+) -> Result<()> {
+ let second = Uuid::from_u128(0x8502);
+ let first_write = Uuid::from_u128(0xaa01);
+ let second_write = Uuid::from_u128(0xaa02);
+ let labels = ["INBOX".to_string()];
+
+ // Two bulk writes reach the messages' threads in opposite orders.
+ mark_pending_label_writes(&pool, LINK, &[MESSAGE], "INBOX", false, first_write).await?;
+ mark_pending_label_writes(&pool, LINK, &[MESSAGE], "INBOX", true, second_write).await?;
+ mark_pending_label_writes(&pool, LINK, &[second], "INBOX", true, second_write).await?;
+ mark_pending_label_writes(&pool, LINK, &[second], "INBOX", false, first_write).await?;
+ let first_turns = label_write_turns(&pool, &[MESSAGE, second], &labels, first_write).await?;
+ assert_eq!(first_turns[&MESSAGE], LabelWriteTurn::Wait);
+ assert_eq!(first_turns[&second], LabelWriteTurn::Send);
+ let second_turns = label_write_turns(&pool, &[MESSAGE, second], &labels, second_write).await?;
+ assert_eq!(second_turns[&MESSAGE], LabelWriteTurn::Send);
+ assert_eq!(second_turns[&second], LabelWriteTurn::Wait);
+
+ // Each write sends the message it is newest on, which settles the other's
+ // wait.
+ let mut tx = pool.begin().await?;
+ confirm_pending_label_writes(&mut tx, &[second], first_write).await?;
+ confirm_pending_label_writes(&mut tx, &[MESSAGE], second_write).await?;
+ tx.commit().await?;
+ for write_id in [first_write, second_write] {
+ let turns = label_write_turns(&pool, &[MESSAGE, second], &labels, write_id).await?;
+ assert!(
+ turns
+ .values()
+ .all(|turn| *turn == LabelWriteTurn::Superseded)
+ );
+ }
+ assert!(pending_provider_label_ids(&pool, MESSAGE).await?.is_empty());
+ assert!(pending_provider_label_ids(&pool, second).await?.is_empty());
+ Ok(())
+}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../fixtures", scripts("fetch_message_labels_in_bulk"))
+)]
+async fn an_abandoned_write_does_not_bring_back_an_expired_older_write(
+ pool: Pool<Postgres>,
+) -> Result<()> {
+ let archive = Uuid::from_u128(0xaa01);
+ let archive_again = Uuid::from_u128(0xaa02);
+
+ stage_inbox(&pool, false, archive).await?;
+ sqlx::query!(
+ "UPDATE email_gmail_pending_label_writes SET expires_at = now() - interval '1 second' WHERE write_id = $1",
+ archive
+ )
+ .execute(&pool)
+ .await?;
+ // With the marker expired, inbox sync puts Gmail's INBOX back.
+ sqlx::query!(
+ "INSERT INTO email_message_labels (message_id, label_id) VALUES ($1, $2)",
+ MESSAGE,
+ INBOX_LABEL
+ )
+ .execute(&pool)
+ .await?;
+
+ stage_inbox(&pool, false, archive_again).await?;
+ assert_eq!(abandon(&pool, archive_again).await?, [MESSAGE]);
+ assert!(in_inbox(&pool).await?);
+ Ok(())
+}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../fixtures", scripts("fetch_message_labels_in_bulk"))
+)]
+async fn writes_abandoned_after_an_older_write_expired_return_to_the_state_before_them(
+ pool: Pool<Postgres>,
+) -> Result<()> {
+ let archive = Uuid::from_u128(0xaa01);
+ let archive_again = Uuid::from_u128(0xaa02);
+ let unarchive = Uuid::from_u128(0xaa03);
+
+ stage_inbox(&pool, false, archive).await?;
+ sqlx::query!(
+ "UPDATE email_gmail_pending_label_writes SET expires_at = now() - interval '1 second' WHERE write_id = $1",
+ archive
+ )
+ .execute(&pool)
+ .await?;
+ sqlx::query!(
+ "INSERT INTO email_message_labels (message_id, label_id) VALUES ($1, $2)",
+ MESSAGE,
+ INBOX_LABEL
+ )
+ .execute(&pool)
+ .await?;
+ stage_inbox(&pool, false, archive_again).await?;
+ stage_inbox(&pool, true, unarchive).await?;
+
+ // The second archive gives up first and passes the state before it on to
+ // the unarchive, which keeps the label meanwhile.
+ assert!(abandon(&pool, archive_again).await?.is_empty());
+ assert!(in_inbox(&pool).await?);
+ assert_eq!(abandon(&pool, unarchive).await?, [MESSAGE]);
+ assert!(in_inbox(&pool).await?);
+ assert!(pending_provider_label_ids(&pool, MESSAGE).await?.is_empty());
+ Ok(())
+}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../fixtures", scripts("fetch_message_labels_in_bulk"))
+)]
+async fn expired_changes_no_longer_apply(pool: Pool<Postgres>) -> Result<()> {
+ let write_id = Uuid::from_u128(0xaa01);
+
+ mark_pending_label_writes(&pool, LINK, &[MESSAGE], "TRASH", true, write_id).await?;
+ mark_pending_label_writes(&pool, LINK, &[MESSAGE], "UNREAD", true, write_id).await?;
+ sqlx::query!(
+ "UPDATE email_gmail_pending_label_writes SET expires_at = now() - interval '1 second' WHERE provider_label_id = 'TRASH'"
+ )
+ .execute(&pool)
+ .await?;
+
+ assert_eq!(
+ pending_provider_label_ids(&pool, MESSAGE).await?,
+ ["UNREAD"]
+ );
+ Ok(())
+}
diff --git a/crates/email_db_client/src/messages/get.rs b/crates/email_db_client/src/messages/get.rs
index 9dc226983f..f91aabb66c 100644
--- a/crates/email_db_client/src/messages/get.rs
+++ b/crates/email_db_client/src/messages/get.rs
@@ -14,6 +14,30 @@ use models_email::service::message::{MessageSenderInfo, MessageToSend};
use sqlx::PgPool;
use sqlx::types::Uuid;
+/// The threads of the link's messages among `message_ids`, each once.
+#[tracing::instrument(skip(executor, message_ids), err)]
+pub async fn thread_ids_of_messages<'e, E>(
+ executor: E,
+ link_id: Uuid,
+ message_ids: &[Uuid],
+) -> Result<Vec<Uuid>, sqlx::Error>
+where
+ E: sqlx::Executor<'e, Database = sqlx::Postgres>,
+{
+ sqlx::query_scalar!(
+ r#"
+ SELECT DISTINCT thread_id
+ FROM email_messages
+ WHERE id = ANY($1) AND link_id = $2
+ ORDER BY thread_id
+ "#,
+ message_ids,
+ link_id,
+ )
+ .fetch_all(executor)
+ .await
+}
+
/// Returns a map of message IDs to sender info for all found messages
#[tracing::instrument(skip(pool), err)]
pub async fn get_message_sender_and_pretty_sender(
diff --git a/crates/email_db_client/src/messages/insert.rs b/crates/email_db_client/src/messages/insert.rs
index af1c67cf39..f80b875063 100644
--- a/crates/email_db_client/src/messages/insert.rs
+++ b/crates/email_db_client/src/messages/insert.rs
@@ -4,13 +4,23 @@ pub(crate) mod test;
use crate::attachments::provider;
use crate::messages::replying_to_id;
use crate::parse::service_to_db::addresses_from_message;
-use crate::{contacts, labels, parse, threads};
+use crate::{contacts, labels, messages, parse, threads};
use models_email::email::db::address::UpsertedRecipients;
+use models_email::email::service::label::system_labels;
use models_email::email::service::message;
use sqlx::PgPool;
use sqlx::types::Uuid;
+/// What a re-sync of a stored message read before fetching it from Gmail.
+#[derive(Debug, Clone, Default)]
+pub struct ResyncSnapshot {
+ /// The message's provider labels before the fetch.
+ pub stored_labels: Vec<String>,
+ /// Labels with a pending Macro write just before or just after the fetch.
+ pub pending_labels: Vec<String>,
+}
+
/// inserts a message (and all its related parts) into the database using the passed transaction.
pub async fn insert_message_with_tx(
tx: &mut sqlx::PgConnection,
@@ -24,10 +34,18 @@ pub async fn insert_message_with_tx(
// 1. inserting thread simultaneously (thread already has latest values)
// 2. backfilling messages (metadata gets updated once after all messages complete)
update_thread_metadata: bool,
+ // Set when re-syncing a stored message: Gmail's answer may predate a local
+ // write made while it was fetched, so labels change only where Gmail
+ // disagrees with what was stored before the fetch.
+ resync: Option<&ResyncSnapshot>,
) -> anyhow::Result<()> {
if let Some(provider_thread_id) = message.provider_thread_id.as_deref() {
threads::provider_identity::lock_provider_thread(tx, link_id, provider_thread_id).await?;
}
+ if resync.is_some() {
+ // Before the row upsert, in the same order as label writes take it.
+ labels::pending_writes::lock_thread_label_writes(tx, &[thread_db_id]).await?;
+ }
let message_db_id = insert_db_message(
tx,
message,
@@ -39,20 +57,57 @@ pub async fn insert_message_with_tx(
contacts::upsert_message::upsert_message_recipients(tx, message_db_id, &recipents).await?;
- if !message.labels.is_empty() {
- let provider_label_ids: Vec<String> = message
- .labels
- .iter()
- .map(|label| label.provider_label_id.clone())
- .collect();
- labels::insert::insert_message_labels(
- tx,
- link_id,
+ // Also when Gmail reports no labels, so a re-sync removes the stored ones.
+ let provider_label_ids: Vec<String> = message
+ .labels
+ .iter()
+ .map(|label| label.provider_label_id.clone())
+ .collect();
+
+ // A re-sync must not undo a Macro label write Gmail has not confirmed yet,
+ // including the read and starred flags those labels drive.
+ let mut kept_label_ids =
+ labels::pending_writes::pending_provider_label_ids(&mut *tx, message_db_id).await?;
+ if let Some(snapshot) = resync {
+ kept_label_ids.extend(snapshot.pending_labels.iter().cloned());
+ let stored_now = sqlx::query_scalar!(
+ r#"
+ SELECT l.provider_label_id
+ FROM email_message_labels ml
+ JOIN email_labels l ON l.id = ml.label_id
+ WHERE ml.message_id = $1
+ "#,
message_db_id,
- &provider_label_ids,
- true,
)
+ .fetch_all(&mut *tx)
.await?;
+ kept_label_ids.extend(
+ stored_now
+ .into_iter()
+ .chain(provider_label_ids.iter().cloned())
+ .chain(snapshot.stored_labels.iter().cloned())
+ .filter(|label| {
+ provider_label_ids.contains(label) == snapshot.stored_labels.contains(label)
+ }),
+ );
+ }
+ kept_label_ids.sort_unstable();
+ kept_label_ids.dedup();
+
+ labels::insert::insert_message_labels(
+ tx,
+ link_id,
+ message_db_id,
+ &provider_label_ids,
+ true,
+ &kept_label_ids,
+ )
+ .await?;
+
+ let kept_read = kept_label_ids.iter().any(|id| id == system_labels::UNREAD);
+ let kept_starred = kept_label_ids.iter().any(|id| id == system_labels::STARRED);
+ if kept_read || kept_starred {
+ messages::update::set_flags_from_labels(tx, message_db_id, kept_read, kept_starred).await?;
}
provider::insert_attachments(tx, message_db_id, &mut message.attachments).await?;
@@ -265,6 +320,7 @@ impl<U: Fn(&anyhow::Error) -> bool> BackfillMessageWriter<'_, U> {
self.link_id,
recipients,
false,
+ None,
)
.await
{
@@ -314,6 +370,7 @@ pub async fn insert_message(
message: &mut message::Message,
link_id: Uuid,
update_thread_metadata: bool,
+ resync: Option<&ResyncSnapshot>,
) -> anyhow::Result<()> {
// we have to insert addresses before inserting the message. these values are shared
// across messages, so inserting them in the txn can cause deadlocks.
@@ -332,6 +389,7 @@ pub async fn insert_message(
link_id,
recipients,
update_thread_metadata,
+ resync,
)
.await
{
diff --git a/crates/email_db_client/src/messages/insert/test.rs b/crates/email_db_client/src/messages/insert/test.rs
index 0303183d6c..92f22f2ee4 100644
--- a/crates/email_db_client/src/messages/insert/test.rs
+++ b/crates/email_db_client/src/messages/insert/test.rs
@@ -96,7 +96,7 @@ async fn oversized_provider_values_do_not_block_the_message_insert(
let mut message = message_with_oversized_fields(link_id, thread_db_id);
let message_db_id = message.db_id;
- insert_message(&pool, thread_db_id, &mut message, link_id, false).await?;
+ insert_message(&pool, thread_db_id, &mut message, link_id, false, None).await?;
let inserted = sqlx::query!(
r#"SELECT from_name FROM email_messages WHERE id = $1"#,
@@ -383,7 +383,7 @@ async fn resyncing_an_unchanged_message_writes_nothing(pool: Pool<Postgres>) ->
body: "parent body",
}
.build(link_id, thread_db_id);
- insert_message(&pool, thread_db_id, &mut parent, link_id, true).await?;
+ insert_message(&pool, thread_db_id, &mut parent, link_id, true, None).await?;
let body = large_body();
let reply = SyncedMessage {
@@ -397,7 +397,7 @@ async fn resyncing_an_unchanged_message_writes_nothing(pool: Pool<Postgres>) ->
};
let mut first_sync = reply.build(link_id, thread_db_id);
let reply_id = first_sync.db_id;
- insert_message(&pool, thread_db_id, &mut first_sync, link_id, true).await?;
+ insert_message(&pool, thread_db_id, &mut first_sync, link_id, true, None).await?;
let replying_to = sqlx::query_scalar!(
r#"SELECT replying_to_id FROM email_messages WHERE id = $1"#,
@@ -413,7 +413,7 @@ async fn resyncing_an_unchanged_message_writes_nothing(pool: Pool<Postgres>) ->
// The full ingest path: message upsert, recipients, labels and the replying_to update.
let mut resync = reply.build(link_id, thread_db_id);
- insert_message(&pool, thread_db_id, &mut resync, link_id, true).await?;
+ insert_message(&pool, thread_db_id, &mut resync, link_id, true, None).await?;
assert_eq!(
row_versions(&pool, reply_id).await?,
@@ -469,7 +469,7 @@ async fn changed_flag_updates_the_row_without_rewriting_body_toast(
};
let mut first_sync = synced.build(link_id, thread_db_id);
let message_id = first_sync.db_id;
- insert_message(&pool, thread_db_id, &mut first_sync, link_id, false).await?;
+ insert_message(&pool, thread_db_id, &mut first_sync, link_id, false, None).await?;
let from_contact_id = from_contact_id(&pool, message_id).await?;
let mut tx = pool.begin().await?;
@@ -538,7 +538,7 @@ async fn resync_with_changed_labels_and_recipients_replaces_only_the_changes(
};
let mut first_sync = synced.build(link_id, thread_db_id);
let message_id = first_sync.db_id;
- insert_message(&pool, thread_db_id, &mut first_sync, link_id, false).await?;
+ insert_message(&pool, thread_db_id, &mut first_sync, link_id, false, None).await?;
let before = row_versions(&pool, message_id).await?;
let mut resync = SyncedMessage {
@@ -547,7 +547,7 @@ async fn resync_with_changed_labels_and_recipients_replaces_only_the_changes(
..synced
}
.build(link_id, thread_db_id);
- insert_message(&pool, thread_db_id, &mut resync, link_id, false).await?;
+ insert_message(&pool, thread_db_id, &mut resync, link_id, false, None).await?;
let after = row_versions(&pool, message_id).await?;
assert_eq!(after.message, before.message);
@@ -572,3 +572,240 @@ async fn resync_with_changed_labels_and_recipients_replaces_only_the_changes(
Ok(())
}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../fixtures", scripts("links", "threads"))
+)]
+async fn resync_without_gmail_labels_removes_stored_labels_except_pending_ones(
+ pool: Pool<Postgres>,
+) -> anyhow::Result<()> {
+ let link_id = Uuid::parse_str(LINK_ID)?;
+ let thread_db_id = Uuid::parse_str(THREAD_ID)?;
+ seed_labels(&pool, link_id).await?;
+ let synced = SyncedMessage {
+ provider_id: "archived-in-gmail",
+ global_id: "<[email protected]>",
+ in_reply_to: None,
+ to: &["[email protected]"],
+ cc: &[],
+ labels: &["INBOX"],
+ body: "body",
+ };
+ let mut first_sync = synced.build(link_id, thread_db_id);
+ first_sync.is_read = true;
+ let message_id = first_sync.db_id;
+ insert_message(&pool, thread_db_id, &mut first_sync, link_id, true, None).await?;
+
+ // Marked unread in Macro, not yet in Gmail.
+ let mut tx = pool.begin().await?;
+ labels::insert::insert_message_labels_batch(&mut *tx, &vec![message_id], "UNREAD", link_id)
+ .await?;
+ messages::update::update_message_read_status(&mut tx, message_id, link_id, false).await?;
+ tx.commit().await?;
+ labels::pending_writes::mark_pending_label_writes(
+ &pool,
+ link_id,
+ &[message_id],
+ "UNREAD",
+ true,
+ Uuid::now_v7(),
+ )
+ .await?;
+
+ // Archived and read in Gmail: it reports no labels at all.
+ let mut resync = SyncedMessage {
+ labels: &[],
+ ..synced
+ }
+ .build(link_id, thread_db_id);
+ resync.is_read = true;
+ insert_message(&pool, thread_db_id, &mut resync, link_id, true, None).await?;
+
+ let labels: Vec<String> = row_versions(&pool, message_id)
+ .await?
+ .labels
+ .into_iter()
+ .map(|(label, _)| label)
+ .collect();
+ assert_eq!(labels, ["UNREAD"]);
+ let is_read = sqlx::query_scalar!(
+ "SELECT is_read FROM email_messages WHERE id = $1",
+ message_id
+ )
+ .fetch_one(&pool)
+ .await?;
+ assert!(!is_read);
+ Ok(())
+}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../fixtures", scripts("links", "threads"))
+)]
+async fn resync_keeps_labels_with_a_pending_gmail_write(
+ pool: Pool<Postgres>,
+) -> anyhow::Result<()> {
+ let link_id = Uuid::parse_str(LINK_ID)?;
+ let thread_db_id = Uuid::parse_str(THREAD_ID)?;
+ seed_labels(&pool, link_id).await?;
+ let synced = SyncedMessage {
+ provider_id: "archived",
+ global_id: "<[email protected]>",
+ in_reply_to: None,
+ to: &["[email protected]"],
+ cc: &[],
+ labels: &["INBOX"],
+ body: "body",
+ };
+ let mut first_sync = synced.build(link_id, thread_db_id);
+ first_sync.is_read = true;
+ let message_id = first_sync.db_id;
+ insert_message(&pool, thread_db_id, &mut first_sync, link_id, true, None).await?;
+
+ // Archived and marked unread in Macro; Gmail has not applied either yet.
+ let write_id = Uuid::now_v7();
+ let mut tx = pool.begin().await?;
+ labels::delete::delete_message_labels_batch(&mut *tx, &vec![message_id], "INBOX", link_id)
+ .await?;
+ labels::insert::insert_message_labels_batch(&mut *tx, &vec![message_id], "UNREAD", link_id)
+ .await?;
+ messages::update::update_message_read_status(&mut tx, message_id, link_id, false).await?;
+ tx.commit().await?;
+ for (label, present) in [("INBOX", false), ("UNREAD", true)] {
+ labels::pending_writes::mark_pending_label_writes(
+ &pool,
+ link_id,
+ &[message_id],
+ label,
+ present,
+ write_id,
+ )
+ .await?;
+ }
+
+ let mut stale_resync = synced.build(link_id, thread_db_id);
+ stale_resync.is_read = true;
+ insert_message(&pool, thread_db_id, &mut stale_resync, link_id, true, None).await?;
+
+ let labels_after = |versions: RowVersions| -> Vec<String> {
+ versions
+ .labels
+ .into_iter()
+ .map(|(label, _)| label)
+ .collect()
+ };
+ assert_eq!(
+ labels_after(row_versions(&pool, message_id).await?),
+ ["UNREAD"]
+ );
+ let stored = sqlx::query!(
+ r#"
+ SELECT m.is_read, t.inbox_visible
+ FROM email_messages m JOIN email_threads t ON t.id = m.thread_id
+ WHERE m.id = $1
+ "#,
+ message_id
+ )
+ .fetch_one(&pool)
+ .await?;
+ assert!(!stored.is_read);
+ assert!(!stored.inbox_visible);
+
+ // Once the writes finish, a re-sync follows Gmail again.
+ let mut tx = pool.begin().await?;
+ labels::pending_writes::confirm_pending_label_writes(&mut tx, &[message_id], write_id).await?;
+ tx.commit().await?;
+ let mut resync = synced.build(link_id, thread_db_id);
+ resync.is_read = true;
+ insert_message(&pool, thread_db_id, &mut resync, link_id, true, None).await?;
+
+ assert_eq!(
+ labels_after(row_versions(&pool, message_id).await?),
+ ["INBOX"]
+ );
+ let stored = sqlx::query!(
+ r#"
+ SELECT m.is_read, t.inbox_visible
+ FROM email_messages m JOIN email_threads t ON t.id = m.thread_id
+ WHERE m.id = $1
+ "#,
+ message_id
+ )
+ .fetch_one(&pool)
+ .await?;
+ assert!(stored.is_read);
+ assert!(stored.inbox_visible);
+
+ Ok(())
+}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../fixtures", scripts("links", "threads"))
+)]
+async fn resync_keeps_a_write_made_and_confirmed_while_gmail_was_fetched(
+ pool: Pool<Postgres>,
+) -> anyhow::Result<()> {
+ let link_id = Uuid::parse_str(LINK_ID)?;
+ let thread_db_id = Uuid::parse_str(THREAD_ID)?;
+ seed_labels(&pool, link_id).await?;
+ let synced = SyncedMessage {
+ provider_id: "fetched-before-the-write",
+ global_id: "<[email protected]>",
+ in_reply_to: None,
+ to: &["[email protected]"],
+ cc: &[],
+ labels: &["INBOX"],
+ body: "body",
+ };
+ let mut first_sync = synced.build(link_id, thread_db_id);
+ first_sync.is_read = true;
+ let message_id = first_sync.db_id;
+ insert_message(&pool, thread_db_id, &mut first_sync, link_id, true, None).await?;
+ let snapshot = ResyncSnapshot {
+ stored_labels: vec!["INBOX".to_string()],
+ pending_labels: vec![],
+ };
+
+ // While the re-sync's fetch is in flight, the user marks the message
+ // unread and Gmail confirms it, so no marker is left.
+ let mut tx = pool.begin().await?;
+ labels::insert::insert_message_labels_batch(&mut *tx, &vec![message_id], "UNREAD", link_id)
+ .await?;
+ messages::update::update_message_read_status(&mut tx, message_id, link_id, false).await?;
+ tx.commit().await?;
+
+ // Gmail answered before the write, after the user archived it in Gmail.
+ let mut resync = SyncedMessage {
+ labels: &[],
+ ..synced
+ }
+ .build(link_id, thread_db_id);
+ resync.is_read = true;
+ insert_message(
+ &pool,
+ thread_db_id,
+ &mut resync,
+ link_id,
+ true,
+ Some(&snapshot),
+ )
+ .await?;
+
+ let labels: Vec<String> = row_versions(&pool, message_id)
+ .await?
+ .labels
+ .into_iter()
+ .map(|(label, _)| label)
+ .collect();
+ assert_eq!(labels, ["UNREAD"]);
+ let is_read = sqlx::query_scalar!(
+ "SELECT is_read FROM email_messages WHERE id = $1",
+ message_id
+ )
+ .fetch_one(&pool)
+ .await?;
+ assert!(!is_read);
+ Ok(())
+}
diff --git a/crates/email_db_client/src/messages/update.rs b/crates/email_db_client/src/messages/update.rs
index a2ff5d41b3..68c8e0bb93 100644
--- a/crates/email_db_client/src/messages/update.rs
+++ b/crates/email_db_client/src/messages/update.rs
@@ -1,8 +1,52 @@
+use models_email::email::service::label::system_labels;
use sqlx::types::Uuid;
#[cfg(test)]
mod test;
+/// Sets a message's read flag from its UNREAD label when `read` is set, and its
+/// starred flag from its STARRED label when `starred` is set. A message whose
+/// flags already match is not written, as a re-sync usually changes nothing.
+#[tracing::instrument(skip(tx), err)]
+pub async fn set_flags_from_labels(
+ tx: &mut sqlx::PgConnection,
+ message_id: Uuid,
+ read: bool,
+ starred: bool,
+) -> anyhow::Result<()> {
+ sqlx::query!(
+ r#"
+ WITH flags AS (
+ SELECT
+ CASE WHEN $2 THEN NOT EXISTS (
+ SELECT 1 FROM email_message_labels ml JOIN email_labels l ON l.id = ml.label_id
+ WHERE ml.message_id = m.id AND l.provider_label_id = $4
+ ) ELSE m.is_read END AS is_read,
+ CASE WHEN $3 THEN EXISTS (
+ SELECT 1 FROM email_message_labels ml JOIN email_labels l ON l.id = ml.label_id
+ WHERE ml.message_id = m.id AND l.provider_label_id = $5
+ ) ELSE m.is_starred END AS is_starred
+ FROM email_messages m
+ WHERE m.id = $1
+ )
+ UPDATE email_messages m
+ SET is_read = flags.is_read, is_starred = flags.is_starred
+ FROM flags
+ WHERE m.id = $1
+ AND (m.is_read, m.is_starred) IS DISTINCT FROM (flags.is_read, flags.is_starred)
+ "#,
+ message_id,
+ read,
+ starred,
+ system_labels::UNREAD,
+ system_labels::STARRED,
+ )
+ .execute(tx)
+ .await?;
+
+ Ok(())
+}
+
/// Update the read status of a single message, scoped to the inbox that owns it.
#[tracing::instrument(skip(tx), err)]
pub async fn update_message_read_status(
diff --git a/crates/email_db_client/src/messages/update/test.rs b/crates/email_db_client/src/messages/update/test.rs
index 33e4951f29..b75a6dc2aa 100644
--- a/crates/email_db_client/src/messages/update/test.rs
+++ b/crates/email_db_client/src/messages/update/test.rs
@@ -1,4 +1,4 @@
-use super::mark_message_as_sent;
+use super::{mark_message_as_sent, set_flags_from_labels};
use crate::messages::get::fetch_messages_metadata;
use crate::threads::get::get_thread_by_id_and_link_id;
use crate::threads::update::update_thread_metadata;
@@ -134,3 +134,49 @@ async fn failed_finalization_rolls_back_sent_recency(pool: PgPool) -> anyhow::Re
assert!(thread.latest_outbound_message_ts.is_none());
Ok(())
}
+
+async fn row_version(pool: &PgPool) -> anyhow::Result<String> {
+ Ok(sqlx::query_scalar!(
+ r#"SELECT xmin::text AS "xmin!" FROM email_messages WHERE id = $1"#,
+ MESSAGE
+ )
+ .fetch_one(pool)
+ .await?)
+}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../fixtures", scripts("mark_message_as_sent"))
+)]
+async fn flags_that_already_match_their_labels_are_not_rewritten(
+ pool: PgPool,
+) -> anyhow::Result<()> {
+ // The message has neither UNREAD nor STARRED: it is read and not starred.
+ sqlx::query!(
+ "UPDATE email_messages SET is_read = true, is_starred = false WHERE id = $1",
+ MESSAGE
+ )
+ .execute(&pool)
+ .await?;
+ let before = row_version(&pool).await?;
+
+ let mut tx = pool.begin().await?;
+ set_flags_from_labels(tx.as_mut(), MESSAGE, true, true).await?;
+ tx.commit().await?;
+ assert_eq!(row_version(&pool).await?, before);
+
+ sqlx::query!(
+ "UPDATE email_messages SET is_read = false WHERE id = $1",
+ MESSAGE
+ )
+ .execute(&pool)
+ .await?;
+ let mut tx = pool.begin().await?;
+ set_flags_from_labels(tx.as_mut(), MESSAGE, true, false).await?;
+ tx.commit().await?;
+ let is_read = sqlx::query_scalar!("SELECT is_read FROM email_messages WHERE id = $1", MESSAGE)
+ .fetch_one(&pool)
+ .await?;
+ assert!(is_read);
+ Ok(())
+}
diff --git a/crates/email_db_client/src/threads/insert.rs b/crates/email_db_client/src/threads/insert.rs
index 632d8ebbe2..5a2e9f910e 100644
--- a/crates/email_db_client/src/threads/insert.rs
+++ b/crates/email_db_client/src/threads/insert.rs
@@ -58,6 +58,7 @@ pub async fn insert_thread_and_messages(
link_id,
recipient_map.remove(provider_id).unwrap(),
false,
+ None,
)
.await?;
}
@@ -69,10 +70,11 @@ pub async fn insert_thread_and_messages(
update_thread_messages_replying_to(&mut tx, thread_id, link_id).await?;
}
- // Messages were inserted with per-message metadata updates disabled and
- // insert_thread precomputes the other thread columns, so is_signal
- // must be synced here or new threads would stay noise.
- super::update::sync_thread_signal_flag(&mut tx, thread_id).await?;
+ // Messages were inserted with per-message metadata updates disabled, and
+ // insert_thread precomputes the thread columns from Gmail's labels only
+ // for a new thread. Recompute them, and is_signal, from the stored
+ // labels, which keep pending Macro writes and newly synced messages.
+ super::update::update_thread_metadata(&mut tx, thread_id, link_id).await?;
Ok(thread_id)
}
diff --git a/crates/email_db_client/src/threads/insert/test.rs b/crates/email_db_client/src/threads/insert/test.rs
index ea7fc45e44..0e8dce71de 100644
--- a/crates/email_db_client/src/threads/insert/test.rs
+++ b/crates/email_db_client/src/threads/insert/test.rs
@@ -514,3 +514,104 @@ async fn a_message_that_cannot_be_stored_is_skipped_and_the_rest_of_its_thread_i
assert_eq!(thread_message_count(&pool, thread_id).await?, 1);
Ok(())
}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../fixtures", scripts("links"))
+)]
+async fn whole_thread_resync_recomputes_thread_state_from_stored_labels(
+ pool: Pool<Postgres>,
+) -> anyhow::Result<()> {
+ use crate::messages::insert::test::SyncedMessage;
+
+ let link_id = Uuid::parse_str("00000000-0000-0000-0000-000000000001")?;
+ sqlx::query!(
+ "INSERT INTO email_labels (id, link_id, provider_label_id, name) VALUES ($1, $2, 'INBOX', 'INBOX')",
+ Uuid::now_v7(),
+ link_id
+ )
+ .execute(&pool)
+ .await?;
+ let archived = SyncedMessage {
+ provider_id: "pending-archive",
+ global_id: "<[email protected]>",
+ in_reply_to: None,
+ to: &["[email protected]"],
+ cc: &[],
+ labels: &["INBOX"],
+ body: "archived body",
+ };
+ let reply = SyncedMessage {
+ provider_id: "new-reply",
+ global_id: "<[email protected]>",
+ in_reply_to: None,
+ to: &["[email protected]"],
+ cc: &[],
+ labels: &["INBOX"],
+ body: "reply body",
+ };
+ let gmail_thread = |messages: &[&SyncedMessage]| {
+ let thread_db_id = Uuid::now_v7();
+ let ts =
+ chrono::DateTime::<Utc>::from_timestamp(1_790_000_000, 0).expect("valid timestamp");
+ thread::Thread {
+ db_id: thread_db_id,
+ provider_id: Some("pending-archive-thread".to_string()),
+ link_id,
+ inbox_visible: true,
+ is_read: false,
+ latest_inbound_message_ts: Some(ts),
+ latest_outbound_message_ts: None,
+ latest_non_spam_message_ts: Some(ts),
+ created_at: ts,
+ updated_at: ts,
+ messages: messages
+ .iter()
+ .map(|message| message.build(link_id, thread_db_id))
+ .collect(),
+ }
+ };
+ let first_sync = gmail_thread(&[&archived]);
+ let message_id = first_sync.messages[0].db_id;
+ let thread_id = insert_thread_and_messages(&pool, first_sync, link_id).await?;
+
+ // Archived in Macro; Gmail has not applied it yet.
+ let mut tx = pool.begin().await?;
+ crate::labels::delete::delete_message_labels_batch(
+ &mut *tx,
+ &vec![message_id],
+ "INBOX",
+ link_id,
+ )
+ .await?;
+ sqlx::query!(
+ "UPDATE email_threads SET inbox_visible = false WHERE id = $1",
+ thread_id
+ )
+ .execute(&mut *tx)
+ .await?;
+ tx.commit().await?;
+ crate::labels::pending_writes::mark_pending_label_writes(
+ &pool,
+ link_id,
+ &[message_id],
+ "INBOX",
+ false,
+ Uuid::now_v7(),
+ )
+ .await?;
+ let inbox_visible = || async {
+ get_thread_by_id_and_link_id(&pool, thread_id, link_id)
+ .await
+ .map(|thread| thread.expect("thread should exist").inbox_visible)
+ };
+
+ insert_thread_and_messages(&pool, gmail_thread(&[&archived]), link_id).await?;
+ assert!(!inbox_visible().await?);
+
+ // A new reply lands in the inbox, so the thread shows again.
+ insert_thread_and_messages(&pool, gmail_thread(&[&archived, &reply]), link_id).await?;
+ assert!(inbox_visible().await?);
+
+ Ok(())
+}
diff --git a/crates/macro_db_client/migrations/20261024154458_gmail_pending_label_writes.sql b/crates/macro_db_client/migrations/20261024154458_gmail_pending_label_writes.sql
new file mode 100644
index 0000000000..9d89066dd5
--- /dev/null
+++ b/crates/macro_db_client/migrations/20261024154458_gmail_pending_label_writes.sql
@@ -0,0 +1,25 @@
+SET LOCAL lock_timeout = '1s';
+
+-- Gmail label changes Macro applied locally but Gmail has not confirmed yet,
+-- one row per write on a message's label, newest by seq. Inbox sync keeps the
+-- local label state while an unexpired row exists, so a rate-limited write
+-- waiting in the gmail_ops retry queue cannot be undone by sync.
+-- Only the newest row's write sends that label to Gmail; an older write waits
+-- until the newer one is confirmed (its rows go) or abandoned (its row goes,
+-- and the label returns to the state the next older row or the first write
+-- left it in). A confirmed write removes its row and every older one. Rows of
+-- operations that never finish stop applying at expires_at, and go with the
+-- message.
+CREATE TABLE IF NOT EXISTS email_gmail_pending_label_writes (
+ seq bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+ message_id uuid NOT NULL REFERENCES email_messages(id) ON DELETE CASCADE,
+ provider_label_id text NOT NULL,
+ write_id uuid NOT NULL,
+ -- Whether the write leaves the label on the message.
+ label_present boolean NOT NULL,
+ -- The label's state before the write, as Gmail will have it if this write
+ -- and every newer one are abandoned.
+ label_present_before boolean NOT NULL,
+ expires_at timestamptz NOT NULL,
+ UNIQUE (message_id, provider_label_id, write_id)
+);
diff --git a/crates/models_email/src/email/service/pubsub.rs b/crates/models_email/src/email/service/pubsub.rs
index 56f55cf803..aa74199ac7 100644
--- a/crates/models_email/src/email/service/pubsub.rs
+++ b/crates/models_email/src/email/service/pubsub.rs
@@ -43,6 +43,10 @@ pub enum FailureReason {
CalendarBackfillBusy,
GmailApiFailed,
GmailApiRateLimited,
+ /// Gmail failed with a server or network error.
+ GmailApiUnavailable,
+ /// A newer label write on the same messages has not finished yet.
+ NewerLabelWritePending,
/// Every fleet-wide backfill write slot stayed taken; the operation is deferred.
BackfillWritesBusy,
OutdatedHistoryId,
diff --git a/crates/models_email/src/gmail/gmail_ops.rs b/crates/models_email/src/gmail/gmail_ops.rs
index 30f1cced40..08a1b050f4 100644
--- a/crates/models_email/src/gmail/gmail_ops.rs
+++ b/crates/models_email/src/gmail/gmail_ops.rs
@@ -41,6 +41,10 @@ pub struct ModifyMessageLabelsPayload {
pub provider_message_id: String,
pub labels_to_add: Vec<String>,
pub labels_to_remove: Vec<String>,
+ /// Pending label write marker this operation clears once Gmail accepts or
+ /// rejects it. Absent on operations enqueued before markers existed.
+ #[serde(default)]
+ pub pending_write_id: Option<Uuid>,
}
#[derive(Debug, Deserialize, Serialize, Clone)]
@@ -51,6 +55,10 @@ pub struct ModifyMessageLabelsBatchPayload {
pub messages: Vec<GmailMessageRef>,
pub labels_to_add: Vec<String>,
pub labels_to_remove: Vec<String>,
+ /// Pending label write marker this batch clears on its messages once Gmail
+ /// accepts or rejects it. Absent on batches enqueued before markers existed.
+ #[serde(default)]
+ pub pending_write_id: Option<Uuid>,
}
/// The database and Gmail ids of one message.
@@ -70,6 +78,7 @@ impl From<ModifyMessageLabelsPayload> for ModifyMessageLabelsBatchPayload {
}],
labels_to_add: payload.labels_to_add,
labels_to_remove: payload.labels_to_remove,
+ pending_write_id: payload.pending_write_id,
}
}
}
@@ -80,6 +89,7 @@ impl ModifyMessageLabelsBatchPayload {
messages: Vec<GmailMessageRef>,
labels_to_add: Vec<String>,
labels_to_remove: Vec<String>,
+ pending_write_id: Uuid,
) -> Vec<Self> {
messages
.chunks(MAX_BATCH_MODIFY_MESSAGES)
@@ -87,6 +97,7 @@ impl ModifyMessageLabelsBatchPayload {
messages: chunk.to_vec(),
labels_to_add: labels_to_add.clone(),
labels_to_remove: labels_to_remove.clone(),
+ pending_write_id: Some(pending_write_id),
})
.collect()
}
@@ -95,18 +106,25 @@ impl ModifyMessageLabelsBatchPayload {
/// The gmail_ops operations that carry one label write. Workers released
/// before `ModifyMessageLabelsBatch` existed delete it as undecodable, so
/// `batches` may be set only once every worker decodes it; otherwise each
-/// message goes out as its own `ModifyMessageLabels`.
+/// message goes out as its own `ModifyMessageLabels`. Every operation carries
+/// `pending_write_id`, so each clears the write's markers on its own messages.
pub fn label_write_operations(
messages: Vec<GmailMessageRef>,
labels_to_add: Vec<String>,
labels_to_remove: Vec<String>,
+ pending_write_id: Uuid,
batches: bool,
) -> Vec<GmailOpsOperation> {
if batches {
- return ModifyMessageLabelsBatchPayload::chunked(messages, labels_to_add, labels_to_remove)
- .into_iter()
- .map(GmailOpsOperation::ModifyMessageLabelsBatch)
- .collect();
+ return ModifyMessageLabelsBatchPayload::chunked(
+ messages,
+ labels_to_add,
+ labels_to_remove,
+ pending_write_id,
+ )
+ .into_iter()
+ .map(GmailOpsOperation::ModifyMessageLabelsBatch)
+ .collect();
}
messages
.into_iter()
@@ -116,6 +134,7 @@ pub fn label_write_operations(
provider_message_id: message.provider_message_id,
labels_to_add: labels_to_add.clone(),
labels_to_remove: labels_to_remove.clone(),
+ pending_write_id: Some(pending_write_id),
})
})
.collect()
diff --git a/crates/models_email/src/gmail/gmail_ops/test.rs b/crates/models_email/src/gmail/gmail_ops/test.rs
index f9b5eab1a8..032e6c187c 100644
--- a/crates/models_email/src/gmail/gmail_ops/test.rs
+++ b/crates/models_email/src/gmail/gmail_ops/test.rs
@@ -9,8 +9,13 @@ fn chunked_label_writes_fit_one_gmail_request_each() {
})
.collect();
- let payloads =
- ModifyMessageLabelsBatchPayload::chunked(messages.clone(), vec![], vec!["INBOX".into()]);
+ let write_id = Uuid::from_u128(7);
+ let payloads = ModifyMessageLabelsBatchPayload::chunked(
+ messages.clone(),
+ vec![],
+ vec!["INBOX".into()],
+ write_id,
+ );
assert_eq!(
payloads
@@ -23,7 +28,8 @@ fn chunked_label_writes_fit_one_gmail_request_each() {
payloads
.iter()
.all(|payload| payload.labels_to_add.is_empty()
- && payload.labels_to_remove == vec!["INBOX".to_string()])
+ && payload.labels_to_remove == vec!["INBOX".to_string()]
+ && payload.pending_write_id == Some(write_id))
);
assert_eq!(
payloads
@@ -37,6 +43,63 @@ fn chunked_label_writes_fit_one_gmail_request_each() {
#[test]
fn chunked_label_writes_without_messages_enqueue_nothing() {
assert!(
- ModifyMessageLabelsBatchPayload::chunked(vec![], vec!["TRASH".into()], vec![]).is_empty()
+ ModifyMessageLabelsBatchPayload::chunked(
+ vec![],
+ vec!["TRASH".into()],
+ vec![],
+ Uuid::from_u128(7)
+ )
+ .is_empty()
+ );
+}
+
+#[test]
+fn single_message_operations_keep_their_pending_write_as_a_batch() {
+ let queued: GmailOpsPubsubMessage = serde_json::from_str(
+ r#"{
+ "linkId": "00000000-0000-0000-0000-000000000001",
+ "operation": {"modify_message_labels": {
+ "dbMessageId": "00000000-0000-0000-0000-000000000002",
+ "providerMessageId": "message-2",
+ "labelsToAdd": [],
+ "labelsToRemove": ["INBOX"],
+ "pendingWriteId": "00000000-0000-0000-0000-000000000007"
+ }}
+ }"#,
+ )
+ .unwrap();
+ let GmailOpsOperation::ModifyMessageLabels(payload) = queued.operation else {
+ panic!("expected a single-message label write");
+ };
+
+ let batch = ModifyMessageLabelsBatchPayload::from(payload);
+
+ assert_eq!(batch.pending_write_id, Some(Uuid::from_u128(7)));
+ assert_eq!(
+ batch.messages,
+ vec![GmailMessageRef {
+ db_message_id: Uuid::from_u128(2),
+ provider_message_id: "message-2".to_string(),
+ }]
);
}
+
+#[test]
+fn label_writes_queued_before_markers_still_decode() {
+ let queued: GmailOpsPubsubMessage = serde_json::from_str(
+ r#"{
+ "linkId": "00000000-0000-0000-0000-000000000001",
+ "operation": {"modify_message_labels_batch": {
+ "messages": [{"dbMessageId": "00000000-0000-0000-0000-000000000002", "providerMessageId": "message-2"}],
+ "labelsToAdd": ["TRASH"],
+ "labelsToRemove": []
+ }}
+ }"#,
+ )
+ .unwrap();
+ let GmailOpsOperation::ModifyMessageLabelsBatch(batch) = queued.operation else {
+ panic!("expected a batched label write");
+ };
+
+ assert_eq!(batch.pending_write_id, None);
+}
diff --git a/crates/sqs_client/src/email.rs b/crates/sqs_client/src/email.rs
index 29d406d188..69afa38e27 100644
--- a/crates/sqs_client/src/email.rs
+++ b/crates/sqs_client/src/email.rs
@@ -44,6 +44,7 @@ impl SQS {
messages: Vec<(Uuid, String)>,
labels_to_add: Vec<String>,
labels_to_remove: Vec<String>,
+ pending_write_id: Uuid,
) -> Vec<models_email::gmail::gmail_ops::GmailOpsPubsubMessage> {
use models_email::gmail::gmail_ops::{
GmailMessageRef, GmailOpsPubsubMessage, label_write_operations,
@@ -60,6 +61,7 @@ impl SQS {
messages,
labels_to_add,
labels_to_remove,
+ pending_write_id,
self.gmail_label_write_batches,
)
.into_iter()
@@ -232,9 +234,15 @@ impl EmailMessageEnqueuer for SQS {
messages: Vec<(Uuid, String)>,
labels_to_add: Vec<String>,
labels_to_remove: Vec<String>,
+ pending_write_id: Uuid,
) -> Result<(), Self::Err> {
- let messages =
- self.gmail_label_write_messages(link_id, messages, labels_to_add, labels_to_remove);
+ let messages = self.gmail_label_write_messages(
+ link_id,
+ messages,
+ labels_to_add,
+ labels_to_remove,
+ pending_write_id,
+ );
if !self.gmail_label_write_batches {
return self.enqueue_gmail_ops_notifications_batch(messages).await;
}
diff --git a/crates/sqs_client/src/email/test.rs b/crates/sqs_client/src/email/test.rs
index 62bbd4ec62..b8d75a653a 100644
--- a/crates/sqs_client/src/email/test.rs
+++ b/crates/sqs_client/src/email/test.rs
@@ -62,6 +62,8 @@ fn decode_with_released_worker(
serde_json::from_str(&serde_json::to_string(message).unwrap())
}
+const WRITE: Uuid = Uuid::from_u128(0xb1);
+
fn label_write(count: u128) -> Vec<(Uuid, String)> {
(0..count)
.map(|index| (Uuid::from_u128(index), format!("message-{index}")))
@@ -76,6 +78,7 @@ fn released_workers_decode_everything_a_default_producer_sends() {
label_write(1001),
vec![],
vec!["INBOX".to_string()],
+ WRITE,
);
messages.extend([
GmailOpsPubsubMessage {
@@ -143,6 +146,7 @@ fn label_write_batches_are_sent_only_when_enabled() {
label_write(1001),
vec!["TRASH".to_string()],
vec![],
+ WRITE,
);
assert_eq!(
@@ -163,3 +167,29 @@ fn label_write_batches_are_sent_only_when_enabled() {
.all(|message| decode_with_released_worker(message).is_err())
);
}
+
+#[test]
+fn every_label_write_operation_carries_the_write_id_with_or_without_batches() {
+ for batches in [false, true] {
+ let messages = client()
+ .gmail_label_write_batches(batches)
+ .gmail_label_write_messages(
+ Uuid::from_u128(0xa1),
+ label_write(1001),
+ vec![],
+ vec!["INBOX".to_string()],
+ WRITE,
+ );
+
+ let write_ids: Vec<_> = messages
+ .iter()
+ .map(|message| match &message.operation {
+ GmailOpsOperation::ModifyMessageLabels(payload) => payload.pending_write_id,
+ GmailOpsOperation::ModifyMessageLabelsBatch(payload) => payload.pending_write_id,
+ other => panic!("expected a label write, got {other}"),
+ })
+ .collect();
+ assert_eq!(write_ids.len(), if batches { 2 } else { 1001 });
+ assert!(write_ids.iter().all(|write_id| *write_id == Some(WRITE)));
+ }
+}
diff --git a/services/email_service/AGENTS.md b/services/email_service/AGENTS.md
index f4caa9b05d..53171ac9f0 100644
--- a/services/email_service/AGENTS.md
+++ b/services/email_service/AGENTS.md
@@ -42,7 +42,8 @@ the gateway `/email` prefix.
## Checks
-- `cargo test -p email_service` (`SQLX_OFFLINE` unset).
+- `cargo test -p email_service` (`SQLX_OFFLINE` unset; the `label_write_sim`
+ tests also need Redis, at `REDIS_URI` or `redis://127.0.0.1:6379`).
- `just check` (from this directory) runs `cargo check` with `SQLX_OFFLINE=true`.
- Local SQS: `just create-localstack` / `create-queues` before running
`pubsub_workers` locally; queue names are in the `justfile`.
@@ -107,18 +108,48 @@ the gateway `/email` prefix.
so a stalled write can't lose its slot to another writer. Live inbox
sync writes take no slot. Keep new bulk backfill writes behind the same slots.
- `gmail_ops` workers decode both label write forms: single-message
- `ModifyMessageLabels` and `ModifyMessageLabelsBatch` (up to 1,000 messages;
- one `batchModify` from ten messages up, separate `messages.modify` calls
- below that, at 5 units each against a flat 50; a permanent failure reverts
- the whole batch). Producers send the batch form only with
- `GMAIL_LABEL_WRITE_BATCHES_ENABLED` (here and in `document_storage_service`,
- default off), because workers released before it delete batches as
- undecodable. Rollout: deploy with the switch off; once every
- `pubsub_workers` task runs that release, set it to `true` in the
+ `ModifyMessageLabels` and `ModifyMessageLabelsBatch` (up to 1,000 messages).
+ From ten messages up a batch is one `batchModify`, which a permanent failure
+ reverts as a whole. Below that, gmail_ops sends one `messages.modify` per
+ message (5 units each against a flat 50), checks the message's turn just
+ before each, and settles each message as Gmail answers: a permanent
+ rejection reverts only the messages Gmail hasn't accepted, and a resumed
+ older write skips messages a newer write took over. Producers send the
+ batch form only with `GMAIL_LABEL_WRITE_BATCHES_ENABLED` (here and in
+ `document_storage_service`, default off), because workers released before
+ it delete batches as undecodable. Rollout: deploy with the switch off; once
+ every `pubsub_workers` task runs that release, set it to `true` in the
`email-service` and `document-storage` Doppler projects (dev, then prd) and
redeploy both services. Before rolling workers back past that release, turn
it off and let the gmail-ops queues drain. Retire the single-message form
only after every producer sends batches.
+- Gmail label writes leave a marker in `email_gmail_pending_label_writes`.
+ Inbox sync keeps the local state of a label marked before or after it reads
+ Gmail (`inbox_sync::operations::pending_labels`); a re-sync or recovery
+ backfill of a stored message also keeps labels Gmail's answer agrees with
+ the pre-fetch snapshot on. gmail_ops sends only messages its write is the
+ newest on, and sends them before an operation with a newer write still on
+ its way (whatever its id) waits in the retry queue
+ (`NewerLabelWritePending`), so bulk writes that are each newer on one of the
+ other's messages don't wait on each other. A waiting message sends if the
+ newer write is rejected. A permanent failure gives up the write's unaccepted
+ and waiting messages and puts back what the newest older write still on its
+ way, or the state before the write, says; an expired marker no longer
+ counts. Both label write forms carry the write's `pendingWriteId`; with
+ batches off, each message's operation settles only that message's markers.
+ Markers expire after 12 hours, which must outlast an operation's redelivery
+ through the gmail_ops queues.
+- `pubsub::label_write_sim` wires one mailbox to the production label-write,
+ gmail_ops and inbox sync code with fake Gmail and SQS. Add a test there for a
+ new race between writes and syncs; its ignored `randomized_shard_*` tests run
+ the seeded simulation (`SIM_SEEDS`, `SIM_OVERLAP`, `SIM_BATCHES`, see
+ `sim/test.rs`). Scenarios run in both label write forms (`scenario!`).
+- Local stacks run debug workers on tokio's 2 MiB worker stacks, where every
+ awaited future takes its own stack slot. Pass a Gmail call through a helper
+ as a `BoxFuture`, and box an operation the dispatch awaits in several arms;
+ a `label_write_sim` test runs the label paths on 1.5 MiB worker stacks.
+- gmail_ops moves transient Gmail failures to the retry queue like rate limits;
+ the primary queue dead-letters after three receives.
- Header addresses that Gmail accepts can fail `MacroUserIdStr::try_from_email`.
Skip the bad address with a WARN that omits it (the parse error echoes it)
instead of failing the step; see `pubsub::util::build_contact_connections`.
diff --git a/services/email_service/fixtures/gmail_ops_label_revert.sql b/services/email_service/fixtures/gmail_ops_label_revert.sql
new file mode 100644
index 0000000000..3775daf342
--- /dev/null
+++ b/services/email_service/fixtures/gmail_ops_label_revert.sql
@@ -0,0 +1,22 @@
+-- One Gmail inbox whose three messages were archived in Macro by write
+-- ...00f1, which Gmail has not confirmed yet.
+INSERT INTO email_links (id, macro_id, fusionauth_user_id, email_address, provider, is_sync_active, created_at, updated_at)
+VALUES ('00000000-0000-0000-0000-0000000000a1', 'macro|[email protected]', 'revert-user', '[email protected]', 'GMAIL', true, NOW(), NOW());
+
+INSERT INTO email_threads (id, provider_id, link_id, inbox_visible, is_read, created_at, updated_at)
+VALUES ('00000000-0000-0000-0000-0000000000b1', 'thread-1', '00000000-0000-0000-0000-0000000000a1', false, true, NOW(), NOW());
+
+INSERT INTO email_labels (id, link_id, provider_label_id, name, message_list_visibility, label_list_visibility, type, created_at)
+VALUES ('00000000-0000-0000-0000-0000000000c1', '00000000-0000-0000-0000-0000000000a1', 'INBOX', 'INBOX', 'Show', 'LabelShow', 'System', NOW());
+
+INSERT INTO email_messages (id, thread_id, link_id, provider_id, is_read, is_starred, is_sent, is_draft, has_attachments, internal_date_ts, created_at, updated_at)
+VALUES
+ ('00000000-0000-0000-0000-00000000000a', '00000000-0000-0000-0000-0000000000b1', '00000000-0000-0000-0000-0000000000a1', 'msg-a', true, false, false, false, false, '2026-10-01T10:00:00Z', NOW(), NOW()),
+ ('00000000-0000-0000-0000-00000000000b', '00000000-0000-0000-0000-0000000000b1', '00000000-0000-0000-0000-0000000000a1', 'msg-b', true, false, false, false, false, '2026-10-01T11:00:00Z', NOW(), NOW()),
+ ('00000000-0000-0000-0000-00000000000c', '00000000-0000-0000-0000-0000000000b1', '00000000-0000-0000-0000-0000000000a1', 'msg-c', true, false, false, false, false, '2026-10-01T12:00:00Z', NOW(), NOW());
+
+INSERT INTO email_gmail_pending_label_writes (message_id, provider_label_id, write_id, label_present, label_present_before, expires_at)
+VALUES
+ ('00000000-0000-0000-0000-00000000000a', 'INBOX', '00000000-0000-0000-0000-0000000000f1', false, true, NOW() + interval '12 hours'),
+ ('00000000-0000-0000-0000-00000000000b', 'INBOX', '00000000-0000-0000-0000-0000000000f1', false, true, NOW() + interval '12 hours'),
+ ('00000000-0000-0000-0000-00000000000c', 'INBOX', '00000000-0000-0000-0000-0000000000f1', false, true, NOW() + interval '12 hours');
diff --git a/services/email_service/src/api/email/messages/labels.rs b/services/email_service/src/api/email/messages/labels.rs
index c26108a71a..3d2080a516 100644
--- a/services/email_service/src/api/email/messages/labels.rs
+++ b/services/email_service/src/api/email/messages/labels.rs
@@ -14,6 +14,9 @@ use models_email::service;
use sqlx::types::Uuid;
use utoipa::ToSchema;
+#[cfg(test)]
+mod test;
+
#[derive(serde::Serialize, serde::Deserialize, Debug, ToSchema)]
pub struct UpdateLabelBatchRequest {
pub message_ids: Vec<Uuid>,
@@ -146,6 +149,10 @@ pub async fn handler(
}
let message_db_ids: Vec<Uuid> = db_messages.iter().map(|m| m.db_id).collect();
+ let mut thread_ids: Vec<Uuid> = db_messages.iter().map(|m| m.thread_db_id).collect();
+ thread_ids.sort_unstable();
+ thread_ids.dedup();
+ let pending_write_id = macro_uuid::generate_uuid_v7();
let provider_label_id = label.provider_label_id.clone();
let is_adding = body.value;
@@ -161,55 +168,97 @@ pub async fn handler(
.into_response()
})?;
- let transaction_result = async {
- if !message_db_ids.is_empty() {
- if is_adding {
- email_db_client::labels::insert::insert_message_labels_batch(
- &mut *tx,
- &message_db_ids,
- &provider_label_id,
- link.id,
- )
- .await
- .context("Failed to add label to messages in database")?;
- } else {
- email_db_client::labels::delete::delete_message_labels_batch(
- &mut *tx,
- &message_db_ids,
- &provider_label_id,
- link.id,
- )
- .await
- .context("Failed to remove label from messages in database")?;
- }
+ // The changed messages Gmail knows about; drafts have no provider_id.
+ let transaction_result: anyhow::Result<Vec<(Uuid, String)>> = async {
+ if message_db_ids.is_empty() {
+ return Ok(Vec::new());
+ }
+ email_db_client::labels::pending_writes::lock_thread_label_writes(&mut tx, &thread_ids)
+ .await
+ .context("Failed to lock the messages' threads")?;
+ let changed_ids = email_db_client::labels::get::messages_changing_label(
+ &mut *tx,
+ link.id,
+ &message_db_ids,
+ &provider_label_id,
+ is_adding,
+ )
+ .await
+ .context("Failed to read which messages change")?;
+ let provider_messages: Vec<(Uuid, String)> = db_messages
+ .iter()
+ .filter(|msg| changed_ids.contains(&msg.db_id))
+ .filter_map(|msg| {
+ msg.provider_id
+ .as_ref()
+ .filter(|pid| !pid.is_empty())
+ .map(|pid| (msg.db_id, pid.clone()))
+ })
+ .collect();
+ let provider_message_ids: Vec<Uuid> = provider_messages.iter().map(|(id, _)| *id).collect();
+ email_db_client::labels::pending_writes::mark_pending_label_writes(
+ &mut *tx,
+ link.id,
+ &provider_message_ids,
+ &provider_label_id,
+ is_adding,
+ pending_write_id,
+ )
+ .await
+ .context("Failed to mark pending label writes")?;
- if provider_label_id.as_str() == service::label::system_labels::UNREAD {
- email_db_client::messages::update::update_message_read_status_batch(
- &mut *tx,
- message_db_ids.clone(),
- link.id,
- !is_adding,
- )
- .await
- .context("Failed to update message read status")?;
- } else if provider_label_id.as_str() == service::label::system_labels::STARRED {
- email_db_client::messages::update::update_message_starred_status_batch(
- &mut *tx,
- message_db_ids.clone(),
- link.id,
- is_adding,
- )
+ if is_adding {
+ email_db_client::labels::insert::insert_message_labels_batch(
+ &mut *tx,
+ &message_db_ids,
+ &provider_label_id,
+ link.id,
+ )
+ .await
+ .context("Failed to add label to messages in database")?;
+ } else {
+ email_db_client::labels::delete::delete_message_labels_batch(
+ &mut *tx,
+ &message_db_ids,
+ &provider_label_id,
+ link.id,
+ )
+ .await
+ .context("Failed to remove label from messages in database")?;
+ }
+
+ if provider_label_id.as_str() == service::label::system_labels::UNREAD {
+ email_db_client::messages::update::update_message_read_status_batch(
+ &mut *tx,
+ message_db_ids.clone(),
+ link.id,
+ !is_adding,
+ )
+ .await
+ .context("Failed to update message read status")?;
+ } else if provider_label_id.as_str() == service::label::system_labels::STARRED {
+ email_db_client::messages::update::update_message_starred_status_batch(
+ &mut *tx,
+ message_db_ids.clone(),
+ link.id,
+ is_adding,
+ )
+ .await
+ .context("Failed to update message starred status")?;
+ }
+
+ for thread_id in &thread_ids {
+ email_db_client::threads::update::update_thread_metadata(&mut tx, *thread_id, link.id)
.await
- .context("Failed to update message starred status")?;
- }
+ .context("Failed to update thread state")?;
}
- anyhow::Ok(())
+ Ok(provider_messages)
}
.await;
- match transaction_result {
- Ok(_) => {
+ let provider_messages = match transaction_result {
+ Ok(provider_messages) => {
tx.commit().await.map_err(|e| {
tracing::error!(error=?e, "unable to commit transaction");
(
@@ -220,6 +269,7 @@ pub async fn handler(
)
.into_response()
})?;
+ provider_messages
}
Err(e) => {
tracing::error!(error=?e, "Transaction failed, rolling back");
@@ -234,20 +284,17 @@ pub async fn handler(
)
.into_response());
}
- }
+ };
// Publish semantic macro.email events, rolled up to one per affected
// thread (the batch may span several threads in the same inbox).
let actor = Some(link.macro_id.clone());
- let mut event_thread_ids: Vec<Uuid> = db_messages.iter().map(|m| m.thread_db_id).collect();
- event_thread_ids.sort_unstable();
- event_thread_ids.dedup();
- for thread_id in event_thread_ids {
+ for thread_id in &thread_ids {
let event = EmailMacroEvent::thread_label_change(
link.id,
link.macro_id.clone(),
actor.clone(),
- thread_id,
+ *thread_id,
LabelRef {
label_id: label.id,
provider_label_id: provider_label_id.clone(),
@@ -261,43 +308,104 @@ pub async fn handler(
}
}
- // One label write for the whole batch (drafts have no provider_id, skip them)
+ // One label write for the whole batch
let (labels_to_add, labels_to_remove) = if is_adding {
(vec![provider_label_id.clone()], Vec::new())
} else {
(Vec::new(), vec![provider_label_id.clone()])
};
+ let provider_message_ids: Vec<Uuid> = provider_messages.iter().map(|(id, _)| *id).collect();
- let provider_messages: Vec<(Uuid, String)> = db_messages
- .iter()
- .filter_map(|msg| {
- msg.provider_id
- .as_ref()
- .filter(|pid| !pid.is_empty())
- .map(|pid| (msg.db_id, pid.clone()))
- })
- .collect();
-
- ctx.sqs_client
- .enqueue_gmail_ops_modify_labels_batch(
+ let mut failed_ids = Vec::new();
+ if !provider_messages.is_empty()
+ && ctx
+ .sqs_client
+ .enqueue_gmail_ops_modify_labels_batch(
+ link.id,
+ provider_messages,
+ labels_to_add,
+ labels_to_remove,
+ pending_write_id,
+ )
+ .await
+ .inspect_err(|e| {
+ tracing::error!(error=?e, "Failed to enqueue gmail ops label write");
+ })
+ .is_err()
+ {
+ // Gmail never learns about this change.
+ failed_ids = undo_unsent_write(
+ &ctx.db,
link.id,
- provider_messages,
- labels_to_add,
- labels_to_remove,
+ &thread_ids,
+ &provider_message_ids,
+ pending_write_id,
)
- .await
- .inspect_err(|e| {
- tracing::error!(error=?e, "Failed to enqueue gmail ops label write");
- })
- .ok();
+ .await?;
+ }
Ok((
StatusCode::OK,
Json(UpdateLabelBatchResponse {
- successful_ids: message_db_ids,
- failed_ids: vec![],
+ successful_ids: message_db_ids
+ .into_iter()
+ .filter(|id| !failed_ids.contains(id))
+ .collect(),
+ failed_ids,
missing_ids,
}),
)
.into_response())
}
+
+/// Undoes a write that could not be queued and returns the messages put back,
+/// which the response lists as failed. If that fails too, the change stays in
+/// the database without ever reaching Gmail, so the request fails rather than
+/// reporting success.
+async fn undo_unsent_write(
+ db: &sqlx::PgPool,
+ link_id: Uuid,
+ thread_ids: &[Uuid],
+ message_ids: &[Uuid],
+ write_id: Uuid,
+) -> Result<Vec<Uuid>, Response> {
+ revert_unsent_write(db, link_id, thread_ids, message_ids, write_id)
+ .await
+ .map_err(|e| {
+ tracing::error!(error=?e, "Failed to revert label write after enqueue failure");
+ (
+ StatusCode::INTERNAL_SERVER_ERROR,
+ Json(ErrorResponse {
+ message: "unable to queue the label change".into(),
+ }),
+ )
+ .into_response()
+ })
+}
+
+/// Puts the labels of a write Gmail will never receive back in the state
+/// Gmail will end up with, and derives the threads' state again, holding the
+/// threads' label-write lock. Returns the messages put back.
+async fn revert_unsent_write(
+ db: &sqlx::PgPool,
+ link_id: Uuid,
+ thread_ids: &[Uuid],
+ message_ids: &[Uuid],
+ write_id: Uuid,
+) -> anyhow::Result<Vec<Uuid>> {
+ let mut tx = db.begin().await?;
+ email_db_client::labels::pending_writes::lock_thread_label_writes(&mut tx, thread_ids).await?;
+ let restored = email_db_client::labels::pending_writes::abandon_pending_label_writes(
+ &mut tx,
+ link_id,
+ message_ids,
+ write_id,
+ )
+ .await?;
+ for thread_id in thread_ids {
+ email_db_client::threads::update::update_thread_metadata(&mut tx, *thread_id, link_id)
+ .await?;
+ }
+ tx.commit().await?;
+ Ok(restored)
+}
diff --git a/services/email_service/src/api/email/messages/labels/test.rs b/services/email_service/src/api/email/messages/labels/test.rs
new file mode 100644
index 0000000000..67dee6fc62
--- /dev/null
+++ b/services/email_service/src/api/email/messages/labels/test.rs
@@ -0,0 +1,20 @@
+use super::*;
+use macro_db_migrator::MACRO_DB_MIGRATIONS;
+use sqlx::PgPool;
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn a_write_that_can_be_neither_queued_nor_undone_fails_the_request(pool: PgPool) {
+ pool.close().await;
+
+ let response = undo_unsent_write(
+ &pool,
+ Uuid::now_v7(),
+ &[Uuid::now_v7()],
+ &[Uuid::now_v7()],
+ Uuid::now_v7(),
+ )
+ .await
+ .unwrap_err();
+
+ assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
+}
diff --git a/services/email_service/src/pubsub/backfill/backfill_message.rs b/services/email_service/src/pubsub/backfill/backfill_message.rs
index 33c9047898..1caaa5a874 100644
--- a/services/email_service/src/pubsub/backfill/backfill_message.rs
+++ b/services/email_service/src/pubsub/backfill/backfill_message.rs
@@ -2,6 +2,7 @@ use crate::pubsub::backfill::db_error::map_db_error;
use crate::pubsub::backfill::email_api_error::map_email_api_error;
use crate::pubsub::backfill::{increment_counters, write_slot};
use crate::pubsub::context::PubSubContext;
+use crate::pubsub::inbox_sync::operations::pending_labels;
use crate::pubsub::util::{CrmContactRecipient, enqueue_populate_crm_contacts};
use crate::util::process_pre_insert::process_message_pre_insert;
use email_api_client::domain::models::CalendarPart;
@@ -23,10 +24,27 @@ pub async fn backfill_message(
link: &link::Link,
) -> Result<(), ProcessingError> {
let p = &scope.payload;
- let fetched = ctx
- .email_api
- .get_message(link.id, &p.message_provider_id)
+ // Sync may have stored the message since the backfill listed it; a
+ // re-sync must not undo a label write made meanwhile.
+ let stored =
+ email_db_client::messages::get_simple_messages::get_simple_message_by_provider_and_link(
+ &ctx.db,
+ &p.message_provider_id,
+ &link.id,
+ )
.await
+ .map_err(|e| map_db_error(e, "Failed to look up the stored message"))?;
+ let fetch = Box::pin(ctx.email_api.get_message(link.id, &p.message_provider_id));
+ let (fetched, resync) = match &stored {
+ Some(stored) => {
+ let (fetched, snapshot) = pending_labels::resync_around(&ctx.db, stored.db_id, fetch)
+ .await
+ .map_err(|e| map_db_error(e, "Failed to read the stored message"))?;
+ (fetched, Some(snapshot))
+ }
+ None => (fetch.await, None),
+ };
+ let fetched = fetched
.map_err(|error| map_email_api_error(error, "Failed to get provider message"))?
.ok_or_else(|| {
ProcessingError::NonRetryable(DetailedError {
@@ -48,6 +66,7 @@ pub async fn backfill_message(
link.id,
// we update the thread metadata once all messages in the thread have been backfilled
false,
+ resync.as_ref(),
))
.await;
inserted.map_err(|e| map_db_error(e, "Failed to insert final message into database"))?;
diff --git a/services/email_service/src/pubsub/backfill/mod.rs b/services/email_service/src/pubsub/backfill/mod.rs
index 764154760a..6dc9d5d425 100644
--- a/services/email_service/src/pubsub/backfill/mod.rs
+++ b/services/email_service/src/pubsub/backfill/mod.rs
@@ -1,5 +1,5 @@
mod backfill_attachment;
-mod backfill_message;
+pub(in crate::pubsub) mod backfill_message;
mod backfill_thread;
pub(crate) mod db_error;
mod deferral;
diff --git a/services/email_service/src/pubsub/gmail_ops/email_api_error.rs b/services/email_service/src/pubsub/gmail_ops/email_api_error.rs
index 2140963c84..b3681084b1 100644
--- a/services/email_service/src/pubsub/gmail_ops/email_api_error.rs
+++ b/services/email_service/src/pubsub/gmail_ops/email_api_error.rs
@@ -42,8 +42,9 @@ pub(crate) async fn handle_email_api_error(
error: EmailApiError,
) -> ProcessingError {
match error_policy(&error) {
- ErrorPolicy::RateLimited => handle_rate_limit(ctx, data, error).await,
- ErrorPolicy::Retry => processing_error(error, true),
+ // A Gmail outage can outlast the primary queue's few receives, so it
+ // waits in the retry queue like a rate limit.
+ ErrorPolicy::RateLimited | ErrorPolicy::Retry => handle_rate_limit(ctx, data, error).await,
ErrorPolicy::Permanent => processing_error(error, false),
}
}
@@ -56,14 +57,16 @@ async fn handle_rate_limit(
if rate_limit_route(ctx.retry_worker) == RateLimitRoute::RetryAfterVisibilityTimeout {
tracing::info!(
link_id = %data.link_id,
- "Gmail API rate limited in retry worker, message will be retried after visibility timeout"
+ error = %error,
+ "Gmail API rate limited or unavailable in retry worker, message will be retried after visibility timeout"
);
return processing_error(error, true);
}
tracing::info!(
link_id = %data.link_id,
- "Gmail API rate limited, moving message from primary queue to retry queue"
+ error = %error,
+ "Gmail API rate limited or unavailable, moving message from primary queue to retry queue"
);
if let Err(enqueue_error) = ctx
.sqs_client
@@ -90,12 +93,12 @@ fn rate_limit_route(retry_worker: bool) -> RateLimitRoute {
fn processing_error(error: EmailApiError, retryable: bool) -> ProcessingError {
let reason = match error {
EmailApiError::RateLimited { .. } => FailureReason::GmailApiRateLimited,
+ EmailApiError::Transient { .. } => FailureReason::GmailApiUnavailable,
EmailApiError::AuthRequired => FailureReason::AccessTokenFetchFailed,
EmailApiError::OutdatedCursor => FailureReason::OutdatedHistoryId,
EmailApiError::Forbidden
| EmailApiError::NotFound
| EmailApiError::Conflict
- | EmailApiError::Transient { .. }
| EmailApiError::SendRejected { .. }
| EmailApiError::Permanent { .. } => FailureReason::GmailApiFailed,
};
diff --git a/services/email_service/src/pubsub/gmail_ops/email_api_error/test.rs b/services/email_service/src/pubsub/gmail_ops/email_api_error/test.rs
index f03d0b004b..2db12b22dd 100644
--- a/services/email_service/src/pubsub/gmail_ops/email_api_error/test.rs
+++ b/services/email_service/src/pubsub/gmail_ops/email_api_error/test.rs
@@ -32,16 +32,24 @@ fn rate_limits_use_queue_routing_policy() {
}
#[test]
-fn transient_failures_are_retried() {
+fn transient_failures_are_deferred_like_rate_limits() {
let error = EmailApiError::Transient {
message: "provider unavailable".to_string(),
};
assert_eq!(error_policy(&error), ErrorPolicy::Retry);
+ // Moved to the retry queue from the primary worker, redelivered there.
+ assert!(matches!(
+ processing_error(error.clone(), false),
+ ProcessingError::NonRetryable(DetailedError {
+ reason: FailureReason::GmailApiUnavailable,
+ ..
+ })
+ ));
assert!(matches!(
processing_error(error, true),
ProcessingError::Retryable(DetailedError {
- reason: FailureReason::GmailApiFailed,
+ reason: FailureReason::GmailApiUnavailable,
..
})
));
diff --git a/services/email_service/src/pubsub/gmail_ops/error_handlers.rs b/services/email_service/src/pubsub/gmail_ops/error_handlers.rs
index a89c56cc98..cd49ec156b 100644
--- a/services/email_service/src/pubsub/gmail_ops/error_handlers.rs
+++ b/services/email_service/src/pubsub/gmail_ops/error_handlers.rs
@@ -11,10 +11,15 @@ pub async fn handle_non_retryable_error(
data: &GmailOpsPubsubMessage,
e: &DetailedError,
) -> anyhow::Result<()> {
- // A rate-limited operation is only non-retryable here once it has been
+ // A deferred operation is only non-retryable here once it has been
// re-enqueued to the retry queue, so deleting it loses nothing.
- if e.reason == FailureReason::GmailApiRateLimited {
- tracing::debug!(error = ?e, payload = ?data.operation, "Rate-limited gmail ops message deferred. The message will be deleted.");
+ if matches!(
+ e.reason,
+ FailureReason::GmailApiRateLimited
+ | FailureReason::GmailApiUnavailable
+ | FailureReason::NewerLabelWritePending
+ ) {
+ tracing::debug!(error = ?e, payload = ?data.operation, "Deferred gmail ops message moved to the retry queue. The message will be deleted.");
} else {
tracing::error!(error = ?e, payload = ?data.operation, "Non-retryable error processing gmail ops message. The message will be deleted.");
}
diff --git a/services/email_service/src/pubsub/gmail_ops/operations/error.rs b/services/email_service/src/pubsub/gmail_ops/operations/error.rs
new file mode 100644
index 0000000000..665d16531e
--- /dev/null
+++ b/services/email_service/src/pubsub/gmail_ops/operations/error.rs
@@ -0,0 +1,19 @@
+use email_api_client::domain::models::EmailApiError;
+
+/// Why a gmail_ops operation did not finish.
+#[derive(Debug)]
+pub enum OperationError {
+ Gmail(EmailApiError),
+ /// Recording the operation's outcome failed. Gmail writes are idempotent,
+ /// so the operation runs again.
+ Database(anyhow::Error),
+ /// A newer label write on one of the operation's messages is still on its
+ /// way. If Gmail rejects it, this one has to send after all.
+ WaitingForNewerWrite,
+}
+
+impl From<EmailApiError> for OperationError {
+ fn from(error: EmailApiError) -> Self {
+ Self::Gmail(error)
+ }
+}
diff --git a/services/email_service/src/pubsub/gmail_ops/operations/mod.rs b/services/email_service/src/pubsub/gmail_ops/operations/mod.rs
index fad70c0ff4..9b66f2a8ae 100644
--- a/services/email_service/src/pubsub/gmail_ops/operations/mod.rs
+++ b/services/email_service/src/pubsub/gmail_ops/operations/mod.rs
@@ -1,4 +1,7 @@
pub mod block_sender;
pub mod delete_label;
+mod error;
pub mod modify_message_labels;
pub mod unblock_sender;
+
+pub use error::OperationError;
diff --git a/services/email_service/src/pubsub/gmail_ops/operations/modify_message_labels.rs b/services/email_service/src/pubsub/gmail_ops/operations/modify_message_labels.rs
index 9f0b8aa203..587866ba6f 100644
--- a/services/email_service/src/pubsub/gmail_ops/operations/modify_message_labels.rs
+++ b/services/email_service/src/pubsub/gmail_ops/operations/modify_message_labels.rs
@@ -1,17 +1,42 @@
use crate::pubsub::gmail_ops::email_api_error::is_permanent_mutation_error;
+use crate::pubsub::gmail_ops::operations::OperationError;
use crate::pubsub::gmail_ops::worker::GmailOpsContext;
use anyhow::Context;
-use email_api_client::domain::models::EmailApiError;
-use models_email::gmail::gmail_ops::ModifyMessageLabelsBatchPayload;
+use email_api_client::domain::service::SEPARATE_LABEL_WRITES_BELOW;
+use email_db_client::labels::pending_writes::{self, LabelWriteTurn};
+use futures::future::BoxFuture;
+use models_email::gmail::gmail_ops::{GmailMessageRef, ModifyMessageLabelsBatchPayload};
use models_email::service;
use models_email::service::link::Link;
+use sqlx::{PgPool, Postgres, Transaction};
use uuid::Uuid;
-/// Modifies labels for a batch of messages: one Gmail batch request from ten
-/// messages up, one request per message below that. A permanent failure
-/// reverts every message in the batch, and a message Gmail no longer has is
-/// skipped; transient errors (5xx, network) and rate limits keep the
-/// optimistic DB changes for the retry, which resends the whole batch.
+#[cfg(test)]
+mod test;
+
+/// Modifies labels for a batch of messages. A message whose newer write was
+/// confirmed is left out. The messages the write is the newest on are sent
+/// first; one whose newer write is still on its way then makes the operation
+/// wait for it. Sending first means two writes that are each newer on one of
+/// the other's messages never wait for each other. From ten messages up the
+/// messages sent are one Gmail request, which a permanent failure reverts as
+/// a whole, with the waiting messages. Below ten, each message is its own
+/// request and is settled as Gmail answers it (see [`send_one_at_a_time`]).
+/// A message Gmail no longer has is skipped; transient errors (5xx, network)
+/// and rate limits keep the optimistic DB changes for the retry.
+///
+/// Boxed so this large future lives on the heap rather than in the frames
+/// that await it: unoptimized builds give every awaited future its own stack
+/// slot, and the gmail_ops dispatch awaits it in two places, deep enough to
+/// overflow a worker thread's stack in a local build.
+pub fn modify_message_labels<'a>(
+ ctx: &'a GmailOpsContext,
+ link: &'a Link,
+ payload: &'a ModifyMessageLabelsBatchPayload,
+) -> BoxFuture<'a, Result<(), OperationError>> {
+ Box::pin(modify_message_labels_unboxed(ctx, link, payload))
+}
+
#[tracing::instrument(
skip(ctx, link, payload),
fields(
@@ -19,23 +44,234 @@ use uuid::Uuid;
labels_to_add = ?payload.labels_to_add,
labels_to_remove = ?payload.labels_to_remove,
),
- err
+ err(Debug)
)]
-pub async fn modify_message_labels(
+async fn modify_message_labels_unboxed(
ctx: &GmailOpsContext,
link: &Link,
payload: &ModifyMessageLabelsBatchPayload,
-) -> Result<(), EmailApiError> {
- let provider_message_ids: Vec<String> = payload
+) -> Result<(), OperationError> {
+ let Some(write_id) = payload.pending_write_id else {
+ // Queued before markers existed: nothing to coordinate with.
+ let messages: Vec<&GmailMessageRef> = payload.messages.iter().collect();
+ return send(ctx, link, payload, &messages, &[]).await;
+ };
+
+ let message_ids: Vec<Uuid> = payload
.messages
.iter()
- .map(|message| message.provider_message_id.clone())
+ .map(|message| message.db_message_id)
.collect();
- let db_message_ids: Vec<Uuid> = payload
+ let turns = label_write_turns(&ctx.db, payload, &message_ids, write_id).await?;
+ let ids_with = |turn: LabelWriteTurn| -> Vec<Uuid> {
+ payload
+ .messages
+ .iter()
+ .map(|message| message.db_message_id)
+ .filter(|id| turns.get(id).copied().unwrap_or(LabelWriteTurn::Superseded) == turn)
+ .collect()
+ };
+ abandon_superseded(ctx, link, &ids_with(LabelWriteTurn::Superseded), write_id).await?;
+
+ let mut waiting = ids_with(LabelWriteTurn::Wait);
+ let messages: Vec<&GmailMessageRef> = payload
.messages
+ .iter()
+ .filter(|message| turns.get(&message.db_message_id) == Some(&LabelWriteTurn::Send))
+ .collect();
+ if messages.len() >= SEPARATE_LABEL_WRITES_BELOW {
+ send(ctx, link, payload, &messages, &waiting).await?;
+ } else if !messages.is_empty() {
+ send_one_at_a_time(ctx, link, payload, write_id, &messages, &mut waiting).await?;
+ }
+
+ if !waiting.is_empty() {
+ tracing::info!(
+ db_message_ids = ?waiting,
+ "Newer label writes on these messages are still on their way, waiting for them"
+ );
+ return Err(OperationError::WaitingForNewerWrite);
+ }
+ if messages.is_empty() {
+ tracing::info!("Newer label writes took over every message, skipping Gmail");
+ }
+ Ok(())
+}
+
+async fn label_write_turns(
+ db: &PgPool,
+ payload: &ModifyMessageLabelsBatchPayload,
+ message_ids: &[Uuid],
+ write_id: Uuid,
+) -> Result<std::collections::HashMap<Uuid, LabelWriteTurn>, OperationError> {
+ let provider_label_ids: Vec<String> = payload
+ .labels_to_add
+ .iter()
+ .chain(&payload.labels_to_remove)
+ .cloned()
+ .collect();
+ pending_writes::label_write_turns(db, message_ids, &provider_label_ids, write_id)
+ .await
+ .context("Failed to read the label write's turn")
+ .map_err(OperationError::Database)
+}
+
+/// Drops the write's rows on messages a newer write has taken over.
+async fn abandon_superseded(
+ ctx: &GmailOpsContext,
+ link: &Link,
+ message_ids: &[Uuid],
+ write_id: Uuid,
+) -> Result<(), OperationError> {
+ if message_ids.is_empty() {
+ return Ok(());
+ }
+ let mut tx = begin_locked(&ctx.db, link.id, message_ids).await?;
+ pending_writes::abandon_pending_label_writes(&mut tx, link.id, message_ids, write_id)
+ .await
+ .context("Failed to drop superseded pending label writes")
+ .map_err(OperationError::Database)?;
+ commit(tx).await
+}
+
+/// Clears the write's markers on messages Gmail accepted it for. Inbox sync
+/// applies Gmail's labels to them again; failing here retries the operation
+/// rather than leaving sync off those labels until the markers expire.
+async fn confirm(
+ ctx: &GmailOpsContext,
+ link: &Link,
+ message_ids: &[Uuid],
+ write_id: Uuid,
+) -> Result<(), OperationError> {
+ let mut tx = begin_locked(&ctx.db, link.id, message_ids).await?;
+ pending_writes::confirm_pending_label_writes(&mut tx, message_ids, write_id)
+ .await
+ .context("Failed to confirm pending label writes")
+ .map_err(OperationError::Database)?;
+ commit(tx).await
+}
+
+/// Sends `messages` one Gmail request each, settling each message as Gmail
+/// answers it. Before each request after the first, the message's turn is
+/// read again: a message a newer write took over meanwhile is skipped, and
+/// one with a newer write still on its way joins `waiting`. A permanent
+/// rejection reverts the rejected message, those not sent yet and the waiting
+/// ones; messages Gmail accepted keep their change. A retry finds the
+/// accepted messages settled and doesn't send them again. Boxed for the
+/// reason given on [`modify_message_labels`].
+fn send_one_at_a_time<'a>(
+ ctx: &'a GmailOpsContext,
+ link: &'a Link,
+ payload: &'a ModifyMessageLabelsBatchPayload,
+ write_id: Uuid,
+ messages: &'a [&'a GmailMessageRef],
+ waiting: &'a mut Vec<Uuid>,
+) -> BoxFuture<'a, Result<(), OperationError>> {
+ Box::pin(send_one_at_a_time_unboxed(
+ ctx, link, payload, write_id, messages, waiting,
+ ))
+}
+
+async fn send_one_at_a_time_unboxed(
+ ctx: &GmailOpsContext,
+ link: &Link,
+ payload: &ModifyMessageLabelsBatchPayload,
+ write_id: Uuid,
+ messages: &[&GmailMessageRef],
+ waiting: &mut Vec<Uuid>,
+) -> Result<(), OperationError> {
+ // As in a batch request, a message Gmail no longer has counts as written
+ // when the write has others.
+ let skip_missing = payload.messages.len() > 1;
+ for (index, message) in messages.iter().enumerate() {
+ let message_id = message.db_message_id;
+ if index > 0 {
+ let turns = label_write_turns(&ctx.db, payload, &[message_id], write_id).await?;
+ match turns
+ .get(&message_id)
+ .copied()
+ .unwrap_or(LabelWriteTurn::Superseded)
+ {
+ LabelWriteTurn::Send => {}
+ LabelWriteTurn::Superseded => {
+ abandon_superseded(ctx, link, &[message_id], write_id).await?;
+ continue;
+ }
+ LabelWriteTurn::Wait => {
+ waiting.push(message_id);
+ continue;
+ }
+ }
+ }
+ let result = ctx
+ .email_api
+ .modify_one_message_labels(
+ link.id,
+ &message.provider_message_id,
+ &payload.labels_to_add,
+ &payload.labels_to_remove,
+ skip_missing,
+ )
+ .await;
+ match result {
+ Ok(()) => confirm(ctx, link, &[message_id], write_id).await?,
+ Err(error) if is_permanent_mutation_error(&error) => {
+ let unaccepted: Vec<Uuid> = messages[index..]
+ .iter()
+ .map(|message| message.db_message_id)
+ .chain(waiting.iter().copied())
+ .collect();
+ tracing::error!(
+ error = ?error,
+ db_message_ids = ?unaccepted,
+ "Permanent Gmail error modifying labels, reverting the messages Gmail hasn't accepted"
+ );
+ revert_db_changes(&ctx.db, link, payload, unaccepted)
+ .await
+ .map_err(OperationError::Database)?;
+ return Err(OperationError::Gmail(error));
+ }
+ Err(error) => {
+ tracing::warn!(
+ error = ?error,
+ db_message_id = %message_id,
+ "Retryable Gmail error modifying labels, preserving optimistic database changes"
+ );
+ return Err(OperationError::Gmail(error));
+ }
+ }
+ }
+ Ok(())
+}
+
+/// Sends `messages` to Gmail in one call and records the outcome. A permanent
+/// failure reverts them and the `waiting` messages. Boxed for the reason
+/// given on [`modify_message_labels`]: the operation awaits it in two places.
+fn send<'a>(
+ ctx: &'a GmailOpsContext,
+ link: &'a Link,
+ payload: &'a ModifyMessageLabelsBatchPayload,
+ messages: &'a [&'a GmailMessageRef],
+ waiting: &'a [Uuid],
+) -> BoxFuture<'a, Result<(), OperationError>> {
+ Box::pin(send_unboxed(ctx, link, payload, messages, waiting))
+}
+
+async fn send_unboxed(
+ ctx: &GmailOpsContext,
+ link: &Link,
+ payload: &ModifyMessageLabelsBatchPayload,
+ messages: &[&GmailMessageRef],
+ waiting: &[Uuid],
+) -> Result<(), OperationError> {
+ let db_message_ids: Vec<Uuid> = messages
.iter()
.map(|message| message.db_message_id)
.collect();
+ let provider_message_ids: Vec<String> = messages
+ .iter()
+ .map(|message| message.provider_message_id.clone())
+ .collect();
let result = ctx
.email_api
.modify_message_labels(
@@ -46,114 +282,169 @@ pub async fn modify_message_labels(
)
.await;
- if let Err(error) = &result {
- if is_permanent_mutation_error(error) {
+ match result {
+ Ok(()) => {
+ if let Some(write_id) = payload.pending_write_id {
+ confirm(ctx, link, &db_message_ids, write_id).await?;
+ }
+ Ok(())
+ }
+ Err(error) if is_permanent_mutation_error(&error) => {
+ let unaccepted: Vec<Uuid> = db_message_ids.iter().chain(waiting).copied().collect();
tracing::error!(
error = ?error,
- ?db_message_ids,
+ db_message_ids = ?unaccepted,
"Permanent Gmail error modifying labels, reverting database changes"
);
- revert_db_changes(ctx, link, payload, db_message_ids).await;
- } else {
+ revert_db_changes(&ctx.db, link, payload, unaccepted)
+ .await
+ .map_err(OperationError::Database)?;
+ Err(OperationError::Gmail(error))
+ }
+ Err(error) => {
tracing::warn!(
error = ?error,
?db_message_ids,
"Retryable Gmail error modifying labels, preserving optimistic database changes"
);
+ Err(OperationError::Gmail(error))
}
}
+}
+
+/// Starts a transaction holding the label-write lock of the messages' threads.
+async fn begin_locked<'c>(
+ db: &'c PgPool,
+ link_id: Uuid,
+ message_ids: &[Uuid],
+) -> Result<Transaction<'c, Postgres>, OperationError> {
+ async {
+ let mut tx = db.begin().await?;
+ let thread_ids =
+ email_db_client::messages::get::thread_ids_of_messages(&mut *tx, link_id, message_ids)
+ .await?;
+ pending_writes::lock_thread_label_writes(&mut tx, &thread_ids).await?;
+ Ok(tx)
+ }
+ .await
+ .map_err(|e: sqlx::Error| {
+ OperationError::Database(anyhow::Error::from(e).context("Failed to lock label writes"))
+ })
+}
- result
+async fn commit(tx: Transaction<'_, Postgres>) -> Result<(), OperationError> {
+ tx.commit()
+ .await
+ .context("Failed to commit label write changes")
+ .map_err(OperationError::Database)
}
-/// Reverts the optimistic DB changes for the messages of a batch that failed
-/// in Gmail.
+/// Reverts the optimistic DB changes Gmail rejected. With a write id, only
+/// labels the write was the newest on go back, to the state Gmail will end up
+/// with; a newer write keeps its own. Each affected thread's state is derived
+/// again from its messages.
async fn revert_db_changes(
- ctx: &GmailOpsContext,
+ db: &PgPool,
link: &Link,
payload: &ModifyMessageLabelsBatchPayload,
- failed_ids: Vec<Uuid>,
-) {
- let mut tx = match ctx.db.begin().await {
- Ok(tx) => tx,
- Err(e) => {
- tracing::error!(error=?e, "Failed to begin transaction for reversion");
- return;
+ sent_ids: Vec<Uuid>,
+) -> anyhow::Result<()> {
+ let mut tx = db
+ .begin()
+ .await
+ .context("Failed to begin transaction for reversion")?;
+ let thread_ids =
+ email_db_client::messages::get::thread_ids_of_messages(&mut *tx, link.id, &sent_ids)
+ .await
+ .context("Failed to read the threads of the reverted messages")?;
+ pending_writes::lock_thread_label_writes(&mut tx, &thread_ids)
+ .await
+ .context("Failed to lock the threads of the reverted messages")?;
+ let reverted_ids = match payload.pending_write_id {
+ Some(write_id) => {
+ pending_writes::abandon_pending_label_writes(&mut tx, link.id, &sent_ids, write_id)
+ .await
+ .context("Failed to abandon pending label writes")?
+ }
+ None => {
+ invert_label_change(&mut tx, link, payload, &sent_ids).await?;
+ sent_ids
}
};
- let is_adding = !payload.labels_to_add.is_empty();
+ for thread_id in thread_ids {
+ email_db_client::threads::update::update_thread_metadata(&mut tx, thread_id, link.id)
+ .await
+ .context("Failed to recompute thread state after revert")?;
+ }
+ tx.commit()
+ .await
+ .context("Unable to commit transaction for revert")?;
+ tracing::info!(
+ db_message_ids = ?reverted_ids,
+ "Successfully reverted database changes after Gmail API failure"
+ );
+ Ok(())
+}
+
+/// Undoes the first label change of an operation queued before markers
+/// existed on every message it sent.
+async fn invert_label_change(
+ tx: &mut sqlx::PgConnection,
+ link: &Link,
+ payload: &ModifyMessageLabelsBatchPayload,
+ message_ids: &[Uuid],
+) -> anyhow::Result<()> {
+ let is_adding = !payload.labels_to_add.is_empty();
let Some(provider_label_id) = (if is_adding {
payload.labels_to_add.first()
} else {
payload.labels_to_remove.first()
}) else {
tracing::error!("No label IDs in payload, cannot revert");
- return;
+ return Ok(());
};
+ let message_ids = message_ids.to_vec();
- let revert_result = async {
- if is_adding {
- email_db_client::labels::delete::delete_message_labels_batch(
- &mut *tx,
- &failed_ids,
- provider_label_id,
- link.id,
- )
- .await
- .context("Failed to revert adding labels")?;
- } else {
- email_db_client::labels::insert::insert_message_labels_batch(
- &mut *tx,
- &failed_ids,
- provider_label_id,
- link.id,
- )
- .await
- .context("Failed to revert removing labels")?;
- }
-
- if *provider_label_id == service::label::system_labels::UNREAD {
- email_db_client::messages::update::update_message_read_status_batch(
- &mut *tx,
- failed_ids.clone(),
- link.id,
- is_adding,
- )
- .await
- .context("Failed to revert message read status")?;
- } else if *provider_label_id == service::label::system_labels::STARRED {
- email_db_client::messages::update::update_message_starred_status_batch(
- &mut *tx,
- failed_ids.clone(),
- link.id,
- !is_adding,
- )
- .await
- .context("Failed to revert message starred status")?;
- }
-
- anyhow::Ok(())
+ if is_adding {
+ email_db_client::labels::delete::delete_message_labels_batch(
+ &mut *tx,
+ &message_ids,
+ provider_label_id,
+ link.id,
+ )
+ .await
+ .context("Failed to revert adding labels")?;
+ } else {
+ email_db_client::labels::insert::insert_message_labels_batch(
+ &mut *tx,
+ &message_ids,
+ provider_label_id,
+ link.id,
+ )
+ .await
+ .context("Failed to revert removing labels")?;
}
- .await;
-
- match revert_result {
- Ok(_) => {
- if let Err(e) = tx.commit().await {
- tracing::error!(error=?e, "Unable to commit transaction for revert");
- } else {
- tracing::info!(
- db_message_ids = ?failed_ids,
- "Successfully reverted database changes after Gmail API failure"
- );
- }
- }
- Err(e) => {
- tracing::error!(error=?e, "Revert failed, rolling back");
- if let Err(rollback_err) = tx.rollback().await {
- tracing::error!(error=?rollback_err, "Failed to rollback revert transaction");
- }
- }
+
+ if *provider_label_id == service::label::system_labels::UNREAD {
+ email_db_client::messages::update::update_message_read_status_batch(
+ &mut *tx,
+ message_ids,
+ link.id,
+ is_adding,
+ )
+ .await
+ .context("Failed to revert message read status")?;
+ } else if *provider_label_id == service::label::system_labels::STARRED {
+ email_db_client::messages::update::update_message_starred_status_batch(
+ &mut *tx,
+ message_ids,
+ link.id,
+ !is_adding,
+ )
+ .await
+ .context("Failed to revert message starred status")?;
}
+ Ok(())
}
diff --git a/services/email_service/src/pubsub/gmail_ops/operations/modify_message_labels/test.rs b/services/email_service/src/pubsub/gmail_ops/operations/modify_message_labels/test.rs
new file mode 100644
index 0000000000..430d344c82
--- /dev/null
+++ b/services/email_service/src/pubsub/gmail_ops/operations/modify_message_labels/test.rs
@@ -0,0 +1,284 @@
+use super::*;
+use macro_db_migrator::MACRO_DB_MIGRATIONS;
+use models_email::gmail::gmail_ops::ModifyMessageLabelsPayload;
+
+const THREAD: Uuid = Uuid::from_u128(0xb1);
+const ARCHIVE: Uuid = Uuid::from_u128(0xf1);
+const UNARCHIVE: Uuid = Uuid::from_u128(0xf2);
+
+async fn messages_with_label(pool: &PgPool, provider_label_id: &str) -> Vec<Uuid> {
+ sqlx::query_scalar!(
+ r#"
+ SELECT ml.message_id FROM email_message_labels ml
+ JOIN email_labels l ON l.id = ml.label_id
+ WHERE l.provider_label_id = $1
+ ORDER BY ml.message_id
+ "#,
+ provider_label_id
+ )
+ .fetch_all(pool)
+ .await
+ .unwrap()
+}
+
+async fn pending_markers(pool: &PgPool) -> Vec<(Uuid, Uuid)> {
+ sqlx::query!(
+ "SELECT message_id, write_id FROM email_gmail_pending_label_writes ORDER BY message_id, seq"
+ )
+ .fetch_all(pool)
+ .await
+ .unwrap()
+ .into_iter()
+ .map(|marker| (marker.message_id, marker.write_id))
+ .collect()
+}
+
+async fn thread_state(pool: &PgPool) -> (bool, bool) {
+ let thread = sqlx::query!(
+ "SELECT inbox_visible, is_read FROM email_threads WHERE id = $1",
+ THREAD
+ )
+ .fetch_one(pool)
+ .await
+ .unwrap();
+ (thread.inbox_visible, thread.is_read)
+}
+
+async fn link(pool: &PgPool) -> Link {
+ email_db_client::links::get::fetch_link_by_id(pool, Uuid::from_u128(0xa1))
+ .await
+ .unwrap()
+ .unwrap()
+}
+
+fn message(id: u128) -> GmailMessageRef {
+ GmailMessageRef {
+ db_message_id: Uuid::from_u128(id),
+ provider_message_id: format!("msg-{}", char::from_digit(id as u32, 16).unwrap()),
+ }
+}
+
+fn inbox_write(
+ write_id: Uuid,
+ present: bool,
+ messages: Vec<GmailMessageRef>,
+) -> ModifyMessageLabelsBatchPayload {
+ let label = vec!["INBOX".to_string()];
+ ModifyMessageLabelsBatchPayload {
+ messages,
+ labels_to_add: if present { label.clone() } else { vec![] },
+ labels_to_remove: if present { vec![] } else { label },
+ pending_write_id: Some(write_id),
+ }
+}
+
+/// A newer write unarchives message B locally, as staging does.
+async fn unarchive_b(pool: &PgPool) {
+ let mut tx = pool.begin().await.unwrap();
+ sqlx::query!(
+ "INSERT INTO email_message_labels (message_id, label_id) VALUES ($1, $2)",
+ Uuid::from_u128(0xb),
+ Uuid::from_u128(0xc1),
+ )
+ .execute(&mut *tx)
+ .await
+ .unwrap();
+ pending_writes::mark_pending_label_writes(
+ &mut *tx,
+ Uuid::from_u128(0xa1),
+ &[Uuid::from_u128(0xb)],
+ "INBOX",
+ true,
+ UNARCHIVE,
+ )
+ .await
+ .unwrap();
+ tx.commit().await.unwrap();
+}
+
+fn message_ids(payload: &ModifyMessageLabelsBatchPayload) -> Vec<Uuid> {
+ payload
+ .messages
+ .iter()
+ .map(|message| message.db_message_id)
+ .collect()
+}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../../../fixtures", scripts("gmail_ops_label_revert"))
+)]
+async fn an_older_write_waits_for_a_newer_one_and_sends_once_gmail_rejects_it(pool: PgPool) {
+ unarchive_b(&pool).await;
+ let archive = inbox_write(
+ ARCHIVE,
+ false,
+ vec![message(0xa), message(0xb), message(0xc)],
+ );
+ let turns = label_write_turns(&pool, &archive, &message_ids(&archive), ARCHIVE)
+ .await
+ .unwrap();
+ assert_eq!(turns[&Uuid::from_u128(0xa)], LabelWriteTurn::Send);
+ assert_eq!(turns[&Uuid::from_u128(0xb)], LabelWriteTurn::Wait);
+
+ revert_db_changes(
+ &pool,
+ &link(&pool).await,
+ &inbox_write(UNARCHIVE, true, vec![message(0xb)]),
+ vec![Uuid::from_u128(0xb)],
+ )
+ .await
+ .unwrap();
+
+ // B is archived again, and the archive now sends it with the others.
+ assert!(messages_with_label(&pool, "INBOX").await.is_empty());
+ let turns = label_write_turns(&pool, &archive, &message_ids(&archive), ARCHIVE)
+ .await
+ .unwrap();
+ assert!(turns.values().all(|turn| *turn == LabelWriteTurn::Send));
+ assert_eq!(thread_state(&pool).await, (false, true));
+}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../../../fixtures", scripts("gmail_ops_label_revert"))
+)]
+async fn a_confirmed_newer_write_supersedes_the_older_one(pool: PgPool) {
+ unarchive_b(&pool).await;
+ let mut tx = pool.begin().await.unwrap();
+ pending_writes::confirm_pending_label_writes(&mut tx, &[Uuid::from_u128(0xb)], UNARCHIVE)
+ .await
+ .unwrap();
+ tx.commit().await.unwrap();
+
+ let archive = inbox_write(ARCHIVE, false, vec![message(0xa), message(0xb)]);
+ let turns = label_write_turns(&pool, &archive, &message_ids(&archive), ARCHIVE)
+ .await
+ .unwrap();
+ assert_eq!(turns[&Uuid::from_u128(0xa)], LabelWriteTurn::Send);
+ assert_eq!(turns[&Uuid::from_u128(0xb)], LabelWriteTurn::Superseded);
+}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../../../fixtures", scripts("gmail_ops_label_revert"))
+)]
+async fn rejected_batch_reverts_only_the_messages_its_write_was_newest_on(pool: PgPool) {
+ unarchive_b(&pool).await;
+
+ revert_db_changes(
+ &pool,
+ &link(&pool).await,
+ &inbox_write(ARCHIVE, false, vec![message(0xa), message(0xb)]),
+ vec![Uuid::from_u128(0xa), Uuid::from_u128(0xb)],
+ )
+ .await
+ .unwrap();
+
+ // B keeps the newer unarchive; C was not in this batch.
+ assert_eq!(
+ messages_with_label(&pool, "INBOX").await,
+ vec![Uuid::from_u128(0xa), Uuid::from_u128(0xb)]
+ );
+ assert_eq!(
+ pending_markers(&pool).await,
+ vec![
+ (Uuid::from_u128(0xb), UNARCHIVE),
+ (Uuid::from_u128(0xc), ARCHIVE),
+ ]
+ );
+ assert_eq!(thread_state(&pool).await, (true, true));
+}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../../../fixtures", scripts("gmail_ops_label_revert"))
+)]
+async fn rejected_mark_read_restores_the_unread_message_and_thread(pool: PgPool) {
+ let mark_read = Uuid::from_u128(0xf3);
+ sqlx::query!(
+ r#"
+ INSERT INTO email_labels (id, link_id, provider_label_id, name, message_list_visibility, label_list_visibility, type, created_at)
+ VALUES ($1, $2, 'UNREAD', 'UNREAD', 'Show', 'LabelShow', 'System', NOW())
+ "#,
+ Uuid::from_u128(0xc2),
+ Uuid::from_u128(0xa1),
+ )
+ .execute(&pool)
+ .await
+ .unwrap();
+ pending_writes::mark_pending_label_writes(
+ &pool,
+ Uuid::from_u128(0xa1),
+ &[Uuid::from_u128(0xa)],
+ "UNREAD",
+ false,
+ mark_read,
+ )
+ .await
+ .unwrap();
+ let batch = ModifyMessageLabelsBatchPayload {
+ messages: vec![message(0xa)],
+ labels_to_add: vec![],
+ labels_to_remove: vec!["UNREAD".to_string()],
+ pending_write_id: Some(mark_read),
+ };
+
+ revert_db_changes(
+ &pool,
+ &link(&pool).await,
+ &batch,
+ vec![Uuid::from_u128(0xa)],
+ )
+ .await
+ .unwrap();
+
+ assert_eq!(
+ messages_with_label(&pool, "UNREAD").await,
+ vec![Uuid::from_u128(0xa)]
+ );
+ let is_read = sqlx::query_scalar!(
+ "SELECT is_read FROM email_messages WHERE id = $1",
+ Uuid::from_u128(0xa)
+ )
+ .fetch_one(&pool)
+ .await
+ .unwrap();
+ assert!(!is_read);
+ assert_eq!(thread_state(&pool).await, (false, false));
+}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../../../fixtures", scripts("gmail_ops_label_revert"))
+)]
+async fn rejected_single_message_operation_reverts_its_message(pool: PgPool) {
+ let batch = ModifyMessageLabelsBatchPayload::from(ModifyMessageLabelsPayload {
+ db_message_id: Uuid::from_u128(0xc),
+ provider_message_id: "msg-c".to_string(),
+ labels_to_add: vec![],
+ labels_to_remove: vec!["INBOX".to_string()],
+ pending_write_id: Some(ARCHIVE),
+ });
+
+ revert_db_changes(
+ &pool,
+ &link(&pool).await,
+ &batch,
+ vec![Uuid::from_u128(0xc)],
+ )
+ .await
+ .unwrap();
+
+ assert_eq!(
+ messages_with_label(&pool, "INBOX").await,
+ vec![Uuid::from_u128(0xc)]
+ );
+ assert_eq!(
+ pending_markers(&pool).await,
+ vec![
+ (Uuid::from_u128(0xa), ARCHIVE),
+ (Uuid::from_u128(0xb), ARCHIVE),
+ ]
+ );
+}
diff --git a/services/email_service/src/pubsub/gmail_ops/process.rs b/services/email_service/src/pubsub/gmail_ops/process.rs
index f4502de192..c565395d05 100644
--- a/services/email_service/src/pubsub/gmail_ops/process.rs
+++ b/services/email_service/src/pubsub/gmail_ops/process.rs
@@ -1,5 +1,6 @@
use crate::pubsub::gmail_ops::email_api_error::handle_email_api_error;
use crate::pubsub::gmail_ops::error_handlers::prefix_error_source;
+use crate::pubsub::gmail_ops::operations::OperationError;
use crate::pubsub::gmail_ops::operations::block_sender::block_sender;
use crate::pubsub::gmail_ops::operations::delete_label::delete_label;
use crate::pubsub::gmail_ops::operations::modify_message_labels::modify_message_labels;
@@ -74,19 +75,31 @@ async fn inner_process_message(
"modify_message_labels",
modify_message_labels(ctx, &link, payload).await,
),
- GmailOpsOperation::DeleteLabel(payload) => {
- ("delete_label", delete_label(ctx, &link, payload).await)
- }
- GmailOpsOperation::BlockSender(payload) => {
- ("block_sender", block_sender(ctx, &link, payload).await)
- }
- GmailOpsOperation::UnblockSender(payload) => {
- ("unblock_sender", unblock_sender(ctx, &link, payload).await)
- }
+ GmailOpsOperation::DeleteLabel(payload) => (
+ "delete_label",
+ delete_label(ctx, &link, payload).await.map_err(Into::into),
+ ),
+ GmailOpsOperation::BlockSender(payload) => (
+ "block_sender",
+ block_sender(ctx, &link, payload).await.map_err(Into::into),
+ ),
+ GmailOpsOperation::UnblockSender(payload) => (
+ "unblock_sender",
+ unblock_sender(ctx, &link, payload)
+ .await
+ .map_err(Into::into),
+ ),
};
if let Err(error) = operation_result {
- let processing_error = handle_email_api_error(ctx, data, error).await;
+ let processing_error = match error {
+ OperationError::Gmail(error) => handle_email_api_error(ctx, data, error).await,
+ OperationError::Database(source) => ProcessingError::Retryable(DetailedError {
+ reason: FailureReason::DatabaseQueryFailed,
+ source,
+ }),
+ OperationError::WaitingForNewerWrite => wait_in_retry_queue(ctx, data).await,
+ };
return Err(prefix_error_source(processing_error, operation_name));
}
@@ -94,6 +107,32 @@ async fn inner_process_message(
Ok(())
}
+/// Moves an operation waiting for a newer label write to the retry queue,
+/// whose receive limit outlasts the newer write's own retries.
+async fn wait_in_retry_queue(
+ ctx: &GmailOpsContext,
+ data: &GmailOpsPubsubMessage,
+) -> ProcessingError {
+ let waiting = || DetailedError {
+ reason: FailureReason::NewerLabelWritePending,
+ source: anyhow!("a newer label write on its messages is still on its way"),
+ };
+ if ctx.retry_worker {
+ return ProcessingError::Retryable(waiting());
+ }
+ if let Err(enqueue_error) = ctx
+ .sqs_client
+ .enqueue_gmail_ops_retry_notification(data.clone())
+ .await
+ {
+ return ProcessingError::Retryable(DetailedError {
+ reason: FailureReason::SqsEnqueueFailed,
+ source: enqueue_error.context("Failed to enqueue gmail ops retry message"),
+ });
+ }
+ ProcessingError::NonRetryable(waiting())
+}
+
/// Extracts the message from the SQS message body.
#[tracing::instrument(skip(message))]
fn extract_gmail_ops_message(
diff --git a/services/email_service/src/pubsub/inbox_sync/mod.rs b/services/email_service/src/pubsub/inbox_sync/mod.rs
index 8b472c0358..32a744966d 100644
--- a/services/email_service/src/pubsub/inbox_sync/mod.rs
+++ b/services/email_service/src/pubsub/inbox_sync/mod.rs
@@ -1,5 +1,5 @@
mod email_api_error;
mod error_handlers;
-mod operations;
+pub(in crate::pubsub) mod operations;
pub(crate) mod process;
pub mod worker;
diff --git a/services/email_service/src/pubsub/inbox_sync/operations/mod.rs b/services/email_service/src/pubsub/inbox_sync/operations/mod.rs
index c50c3c3df8..c06188d9b0 100644
--- a/services/email_service/src/pubsub/inbox_sync/operations/mod.rs
+++ b/services/email_service/src/pubsub/inbox_sync/operations/mod.rs
@@ -1,4 +1,5 @@
pub(in crate::pubsub) mod delete_message;
pub(in crate::pubsub) mod gmail_message;
+pub(in crate::pubsub) mod pending_labels;
pub(in crate::pubsub) mod update_labels;
pub(in crate::pubsub) mod upsert_message;
diff --git a/services/email_service/src/pubsub/inbox_sync/operations/pending_labels.rs b/services/email_service/src/pubsub/inbox_sync/operations/pending_labels.rs
new file mode 100644
index 0000000000..9874129f7f
--- /dev/null
+++ b/services/email_service/src/pubsub/inbox_sync/operations/pending_labels.rs
@@ -0,0 +1,77 @@
+use email_db_client::labels::pending_writes::pending_provider_label_ids;
+use email_db_client::messages::insert::ResyncSnapshot;
+use futures::future::BoxFuture;
+use sqlx::PgPool;
+use uuid::Uuid;
+
+#[cfg(test)]
+mod test;
+
+/// A read of Gmail's labels for a message, with the labels that had a
+/// pending Macro write just before and just after it.
+pub(in crate::pubsub) struct PendingAroundRead<T> {
+ pub value: T,
+ before: Vec<String>,
+ after: Vec<String>,
+}
+
+impl<T> PendingAroundRead<T> {
+ /// Labels whose local state the read may not reflect: Gmail may have
+ /// answered before applying a write pending at either point.
+ pub fn untrusted_labels(&self) -> Vec<String> {
+ let mut labels = self.before.clone();
+ labels.extend(
+ self.after
+ .iter()
+ .filter(|label| !self.before.contains(label))
+ .cloned(),
+ );
+ labels
+ }
+}
+
+/// Runs `read` between two reads of the message's pending label writes.
+///
+/// `read` is boxed: a Gmail read is a large future, and an unoptimized build
+/// gives a future passed by value a stack slot in every frame it passes
+/// through and another where it is awaited, enough to overflow a worker
+/// thread's stack on the inbox sync path in a local build.
+pub(in crate::pubsub) async fn read_around<T>(
+ db: &PgPool,
+ message_id: Uuid,
+ read: BoxFuture<'_, T>,
+) -> Result<PendingAroundRead<T>, sqlx::Error> {
+ let before = pending_provider_label_ids(db, message_id).await?;
+ let value = read.await;
+ let after = pending_provider_label_ids(db, message_id).await?;
+ Ok(PendingAroundRead {
+ value,
+ before,
+ after,
+ })
+}
+
+/// Runs `read`, a Gmail fetch of a stored message that is about to be
+/// re-synced, after reading its stored labels and between two reads of its
+/// pending label writes. `read` is boxed for the reason given on
+/// [`read_around`].
+pub(in crate::pubsub) async fn resync_around<T>(
+ db: &PgPool,
+ message_id: Uuid,
+ read: BoxFuture<'_, T>,
+) -> anyhow::Result<(T, ResyncSnapshot)> {
+ let stored_labels = email_db_client::labels::get::fetch_message_labels(db, message_id)
+ .await?
+ .into_iter()
+ .map(|label| label.provider_label_id)
+ .collect();
+ let read = read_around(db, message_id, read).await?;
+ let pending_labels = read.untrusted_labels();
+ Ok((
+ read.value,
+ ResyncSnapshot {
+ stored_labels,
+ pending_labels,
+ },
+ ))
+}
diff --git a/services/email_service/src/pubsub/inbox_sync/operations/pending_labels/test.rs b/services/email_service/src/pubsub/inbox_sync/operations/pending_labels/test.rs
new file mode 100644
index 0000000000..0dd362d607
--- /dev/null
+++ b/services/email_service/src/pubsub/inbox_sync/operations/pending_labels/test.rs
@@ -0,0 +1,87 @@
+use super::*;
+use macro_db_migrator::MACRO_DB_MIGRATIONS;
+
+const MESSAGE: Uuid = Uuid::from_u128(0xa);
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../../../fixtures", scripts("gmail_ops_label_revert"))
+)]
+async fn a_write_gmail_confirms_during_the_read_is_not_trusted_from_it(pool: PgPool) {
+ let read = read_around(
+ &pool,
+ MESSAGE,
+ Box::pin(async {
+ // gmail_ops applies the archive and clears its marker while Gmail's
+ // answer, still showing INBOX, is on its way.
+ sqlx::query!(
+ "DELETE FROM email_gmail_pending_label_writes WHERE message_id = $1",
+ MESSAGE
+ )
+ .execute(&pool)
+ .await
+ .unwrap();
+ vec!["INBOX".to_string()]
+ }),
+ )
+ .await
+ .unwrap();
+
+ assert_eq!(read.untrusted_labels(), vec!["INBOX".to_string()]);
+}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../../../fixtures", scripts("gmail_ops_label_revert"))
+)]
+async fn a_write_staged_during_the_read_is_not_trusted_from_it(pool: PgPool) {
+ let read = read_around(&pool, MESSAGE, Box::pin(async {
+ sqlx::query!(
+ r#"
+ INSERT INTO email_gmail_pending_label_writes
+ (message_id, provider_label_id, write_id, label_present, label_present_before, expires_at)
+ VALUES ($1, 'STARRED', $2, true, false, now() + interval '12 hours')
+ "#,
+ MESSAGE,
+ Uuid::from_u128(0xf9),
+ )
+ .execute(&pool)
+ .await
+ .unwrap();
+ }))
+ .await
+ .unwrap();
+
+ assert_eq!(
+ read.untrusted_labels(),
+ vec!["INBOX".to_string(), "STARRED".to_string()]
+ );
+}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../../../fixtures", scripts("gmail_ops_label_revert"))
+)]
+async fn a_resync_snapshot_has_the_labels_stored_before_the_fetch(pool: PgPool) {
+ let (fetched, snapshot) = resync_around(
+ &pool,
+ MESSAGE,
+ Box::pin(async {
+ sqlx::query!(
+ "INSERT INTO email_message_labels (message_id, label_id) VALUES ($1, $2)",
+ MESSAGE,
+ Uuid::from_u128(0xc1),
+ )
+ .execute(&pool)
+ .await
+ .unwrap();
+ "fetched"
+ }),
+ )
+ .await
+ .unwrap();
+
+ assert_eq!(fetched, "fetched");
+ assert!(snapshot.stored_labels.is_empty());
+ assert_eq!(snapshot.pending_labels, vec!["INBOX".to_string()]);
+}
diff --git a/services/email_service/src/pubsub/inbox_sync/operations/update_labels.rs b/services/email_service/src/pubsub/inbox_sync/operations/update_labels.rs
index eb968876cc..daf444fc97 100644
--- a/services/email_service/src/pubsub/inbox_sync/operations/update_labels.rs
+++ b/services/email_service/src/pubsub/inbox_sync/operations/update_labels.rs
@@ -1,3 +1,4 @@
+use super::pending_labels;
use crate::pubsub::context::PubSubContext;
use crate::pubsub::inbox_sync::email_api_error::handle_operation_error;
use crate::pubsub::util::{
@@ -92,11 +93,25 @@ pub async fn update_labels(
})
})?;
- let provider_labels = ctx
- .email_api
- .get_message_label_ids(link.id, &payload.provider_message_id)
- .await;
- let gmail_message_labels = match provider_labels {
+ // After the database labels: a write accepted in between has its marker
+ // in place with its label change.
+ let provider_labels = pending_labels::read_around(
+ &ctx.db,
+ db_message.db_id,
+ Box::pin(
+ ctx.email_api
+ .get_message_label_ids(link.id, &payload.provider_message_id),
+ ),
+ )
+ .await
+ .map_err(|e| {
+ ProcessingError::Retryable(DetailedError {
+ reason: FailureReason::DatabaseQueryFailed,
+ source: anyhow::Error::from(e).context("Failed to get pending label writes from db"),
+ })
+ })?;
+ let pending_label_ids = provider_labels.untrusted_labels();
+ let gmail_message_labels = match provider_labels.value {
Ok(Some(labels)) => labels,
Err(error) => {
return Err(handle_operation_error(
@@ -114,21 +129,24 @@ pub async fn update_labels(
}
};
- let labels_to_add: Vec<String> = gmail_message_labels
- .iter()
- .filter(|gmail_label| {
- !db_message_labels
- .iter()
- .any(|db_label| &db_label.provider_label_id == *gmail_label)
- })
- .cloned()
- .collect();
-
- let labels_to_delete: Vec<String> = db_message_labels
+ let db_label_ids: Vec<String> = db_message_labels
.iter()
- .filter(|db_label| !gmail_message_labels.contains(&db_label.provider_label_id))
.map(|label| label.provider_label_id.clone())
.collect();
+ let LabelDiff {
+ labels_to_add,
+ labels_to_delete,
+ kept_pending,
+ } = label_diff(&gmail_message_labels, &db_label_ids, &pending_label_ids);
+
+ if !kept_pending.is_empty() {
+ tracing::info!(
+ link_id = %link.id,
+ db_message_id = %db_message.db_id,
+ kept_labels = ?kept_pending,
+ "Kept local labels with a pending Gmail write over the provider's labels"
+ );
+ }
let has_label_changes = !labels_to_add.is_empty() || !labels_to_delete.is_empty();
@@ -177,6 +195,39 @@ pub async fn update_labels(
Ok(())
}
+/// How one message's database labels change to match Gmail's.
+#[derive(Debug, PartialEq, Eq)]
+struct LabelDiff {
+ labels_to_add: Vec<String>,
+ labels_to_delete: Vec<String>,
+ /// Labels Gmail disagrees on that keep their local state, because Macro's
+ /// own write to them has not reached Gmail yet.
+ kept_pending: Vec<String>,
+}
+
+fn label_diff(
+ gmail_labels: &[String],
+ db_labels: &[String],
+ pending_labels: &[String],
+) -> LabelDiff {
+ let (kept_additions, labels_to_add): (Vec<String>, Vec<String>) = gmail_labels
+ .iter()
+ .filter(|label| !db_labels.contains(label))
+ .cloned()
+ .partition(|label| pending_labels.contains(label));
+ let (kept_deletions, labels_to_delete): (Vec<String>, Vec<String>) = db_labels
+ .iter()
+ .filter(|label| !gmail_labels.contains(label))
+ .cloned()
+ .partition(|label| pending_labels.contains(label));
+
+ LabelDiff {
+ labels_to_add,
+ labels_to_delete,
+ kept_pending: [kept_additions, kept_deletions].concat(),
+ }
+}
+
/// System labels that never appear in a `thread_labels_updated` event:
/// INBOX/TRASH/UNREAD/STARRED/SPAM map to dedicated thread-state events, and
/// IMPORTANT/SENT/DRAFT changes are not published.
@@ -385,14 +436,22 @@ pub async fn add_message_labels(
})?;
let result = async {
- insert::insert_message_labels(&mut tx, link.id, message_db_id, provider_label_ids, false)
- .await
- .map_err(|e| {
- ProcessingError::Retryable(DetailedError {
- reason: FailureReason::DatabaseQueryFailed,
- source: e.context("Failed to insert message labels".to_string()),
- })
- })?;
+ // `label_diff` already left pending labels out of `provider_label_ids`.
+ insert::insert_message_labels(
+ &mut tx,
+ link.id,
+ message_db_id,
+ provider_label_ids,
+ false,
+ &[],
+ )
+ .await
+ .map_err(|e| {
+ ProcessingError::Retryable(DetailedError {
+ reason: FailureReason::DatabaseQueryFailed,
+ source: e.context("Failed to insert message labels".to_string()),
+ })
+ })?;
// if we are adding unread label, mark message as not read
if provider_label_ids
diff --git a/services/email_service/src/pubsub/inbox_sync/operations/update_labels/test.rs b/services/email_service/src/pubsub/inbox_sync/operations/update_labels/test.rs
index 24cba89e10..d2420382a5 100644
--- a/services/email_service/src/pubsub/inbox_sync/operations/update_labels/test.rs
+++ b/services/email_service/src/pubsub/inbox_sync/operations/update_labels/test.rs
@@ -70,3 +70,54 @@ fn ignores_diffs_without_spam_changes() {
fn spam_remains_excluded_from_user_label_events() {
assert!(!is_user_label(service::label::system_labels::SPAM));
}
+
+#[test]
+fn pending_archive_keeps_inbox_removed_while_gmail_still_has_it() {
+ let gmail = vec!["INBOX".to_string(), "UNREAD".to_string()];
+ let db = vec!["UNREAD".to_string()];
+ let pending = vec!["INBOX".to_string()];
+
+ assert_eq!(
+ label_diff(&gmail, &db, &pending),
+ LabelDiff {
+ labels_to_add: vec![],
+ labels_to_delete: vec![],
+ kept_pending: vec!["INBOX".to_string()],
+ }
+ );
+}
+
+#[test]
+fn pending_label_add_is_kept_while_other_provider_changes_apply() {
+ let gmail = vec!["INBOX".to_string(), "STARRED".to_string()];
+ let db = vec![
+ "INBOX".to_string(),
+ "UNREAD".to_string(),
+ "TRASH".to_string(),
+ ];
+ let pending = vec!["TRASH".to_string()];
+
+ assert_eq!(
+ label_diff(&gmail, &db, &pending),
+ LabelDiff {
+ labels_to_add: vec!["STARRED".to_string()],
+ labels_to_delete: vec!["UNREAD".to_string()],
+ kept_pending: vec!["TRASH".to_string()],
+ }
+ );
+}
+
+#[test]
+fn without_pending_writes_the_database_follows_gmail() {
+ let gmail = vec!["INBOX".to_string()];
+ let db = vec!["TRASH".to_string()];
+
+ assert_eq!(
+ label_diff(&gmail, &db, &[]),
+ LabelDiff {
+ labels_to_add: vec!["INBOX".to_string()],
+ labels_to_delete: vec!["TRASH".to_string()],
+ kept_pending: vec![],
+ }
+ );
+}
diff --git a/services/email_service/src/pubsub/inbox_sync/operations/upsert_message.rs b/services/email_service/src/pubsub/inbox_sync/operations/upsert_message.rs
index a403d2584a..790f6d9e1e 100644
--- a/services/email_service/src/pubsub/inbox_sync/operations/upsert_message.rs
+++ b/services/email_service/src/pubsub/inbox_sync/operations/upsert_message.rs
@@ -1,3 +1,4 @@
+use super::pending_labels;
use crate::pubsub::context::PubSubContext;
use crate::pubsub::inbox_sync::email_api_error::handle_operation_error;
use crate::pubsub::util::{
@@ -17,10 +18,11 @@ use email::domain::events::{
use email::domain::models::{PreviewCursorQuery, PreviewView, PreviewViewStandardLabel};
use email::domain::ports::EmailRepo;
use email::outbound::EmailPgRepo;
-use email_api_client::domain::models::EmailApiError;
+use email_api_client::domain::models::{EmailApiError, MessageWithCalendarParts};
use email_db_client::attachments::provider::upload_filters::{
attachment_is_document, attachment_is_media,
};
+use email_db_client::messages::insert::ResyncSnapshot;
use email_db_client::threads;
use email_utils::{dedupe_emails, is_macro_notification_sender};
use filter_ast::Expr;
@@ -81,6 +83,28 @@ fn select_message_sync_event(
}
}
+/// Fetches the message from Gmail, or `None` when Gmail no longer has it.
+async fn fetch_provider_message(
+ ctx: &PubSubContext,
+ link: &link::Link,
+ payload: &UpsertMessagePayload,
+) -> result::Result<Option<MessageWithCalendarParts>, ProcessingError> {
+ match ctx
+ .email_api
+ .get_message(link.id, &payload.provider_message_id)
+ .await
+ {
+ Ok(fetched) => Ok(fetched),
+ Err(error) => Err(handle_operation_error(
+ ctx,
+ link.id,
+ InboxSyncOperation::UpsertMessage(payload.clone()),
+ error,
+ )
+ .await),
+ }
+}
+
// upsert a message into the db. could be a new message or an existing one that had changes
#[tracing::instrument(skip(ctx))]
pub async fn upsert_message(
@@ -88,26 +112,47 @@ pub async fn upsert_message(
link: &link::Link,
payload: &UpsertMessagePayload,
) -> result::Result<(), ProcessingError> {
- let provider_message = ctx
- .email_api
- .get_message(link.id, &payload.provider_message_id)
- .await;
- let fetched = match provider_message {
- Ok(Some(fetched)) => fetched,
- Ok(None) => {
- tracing::debug!(provider_message_id = %payload.provider_message_id, link_id = %link.id,
- "Message not found in gmail when attempting to upsert");
- return Ok(());
- }
- Err(error) => {
- return Err(handle_operation_error(
- ctx,
- link.id,
- InboxSyncOperation::UpsertMessage(payload.clone()),
- error,
+ // Snapshot the previous draft state before the upsert. Provider drafts are mutable,
+ // and this state distinguishes a draft-to-sent transition from an unchanged message.
+ let existing_message =
+ email_db_client::messages::get_simple_messages::get_simple_message_by_provider_and_link(
+ &ctx.db,
+ &payload.provider_message_id,
+ &link.id,
+ )
+ .await
+ .map_err(|e| {
+ ProcessingError::NonRetryable(DetailedError {
+ reason: FailureReason::DatabaseQueryFailed,
+ source: e
+ .context("Failed to fetch existing message before provider upsert".to_string()),
+ })
+ })?;
+ let existing_message_was_draft = existing_message.as_ref().map(|message| message.is_draft);
+ let message_already_exists = existing_message.is_some();
+
+ let (fetched, resync) = match &existing_message {
+ Some(existing) => {
+ let (fetched, snapshot) = pending_labels::resync_around(
+ &ctx.db,
+ existing.db_id,
+ Box::pin(fetch_provider_message(ctx, link, payload)),
)
- .await);
+ .await
+ .map_err(|e| {
+ ProcessingError::Retryable(DetailedError {
+ reason: FailureReason::DatabaseQueryFailed,
+ source: e.context("Failed to read the stored message before its re-sync"),
+ })
+ })?;
+ (fetched?, Some(snapshot))
}
+ None => (fetch_provider_message(ctx, link, payload).await?, None),
+ };
+ let Some(fetched) = fetched else {
+ tracing::debug!(provider_message_id = %payload.provider_message_id, link_id = %link.id,
+ "Message not found in gmail when attempting to upsert");
+ return Ok(());
};
let mut message = fetched.message;
@@ -194,25 +239,6 @@ pub async fn upsert_message(
})
})?;
- // Snapshot the previous draft state before the upsert. Provider drafts are mutable,
- // and this state distinguishes a draft-to-sent transition from an unchanged message.
- let existing_message =
- email_db_client::messages::get_simple_messages::get_simple_message_by_provider_and_link(
- &ctx.db,
- &payload.provider_message_id,
- &link.id,
- )
- .await
- .map_err(|e| {
- ProcessingError::NonRetryable(DetailedError {
- reason: FailureReason::DatabaseQueryFailed,
- source: e
- .context("Failed to fetch existing message before provider upsert".to_string()),
- })
- })?;
- let existing_message_was_draft = existing_message.as_ref().map(|message| message.is_draft);
- let message_already_exists = existing_message.is_some();
-
let is_new_thread = !thread_provider_to_db_map.contains_key(&provider_thread_id);
// if the message's thread doesn't exist in the database, we need to fetch and insert the whole thread.
@@ -220,7 +246,14 @@ pub async fn upsert_message(
if let Some(thread_db_id) = thread_provider_to_db_map.get(&provider_thread_id) {
// Retryable: a concurrent delete can drop the thread after the lookup
// above, and the redelivery then takes the whole-thread branch below.
- process_and_insert_message(&ctx.db, link.id, *thread_db_id, &mut message).await?;
+ process_and_insert_message(
+ &ctx.db,
+ link.id,
+ *thread_db_id,
+ &mut message,
+ resync.as_ref(),
+ )
+ .await?;
} else {
// Propagated verbatim: fetch_and_insert_thread already routed provider
// errors through handle_operation_error (retry queue vs redelivery),
@@ -689,17 +722,25 @@ async fn process_and_insert_message(
link_id: Uuid,
thread_db_id: Uuid,
message: &mut Message,
+ resync: Option<&ResyncSnapshot>,
) -> result::Result<(), ProcessingError> {
process_message_pre_insert(message).await;
- email_db_client::messages::insert::insert_message(db, thread_db_id, message, link_id, true)
- .await
- .map_err(|e| {
- ProcessingError::Retryable(DetailedError {
- reason: FailureReason::DatabaseQueryFailed,
- source: e.context("Failed to insert message".to_string()),
- })
+ email_db_client::messages::insert::insert_message(
+ db,
+ thread_db_id,
+ message,
+ link_id,
+ true,
+ resync,
+ )
+ .await
+ .map_err(|e| {
+ ProcessingError::Retryable(DetailedError {
+ reason: FailureReason::DatabaseQueryFailed,
+ source: e.context("Failed to insert message".to_string()),
})
+ })
}
/// Notify downstream services about new message in a user's inbox
diff --git a/services/email_service/src/pubsub/inbox_sync/operations/upsert_message/test.rs b/services/email_service/src/pubsub/inbox_sync/operations/upsert_message/test.rs
index 9fdeb1e77d..ae935bea6d 100644
--- a/services/email_service/src/pubsub/inbox_sync/operations/upsert_message/test.rs
+++ b/services/email_service/src/pubsub/inbox_sync/operations/upsert_message/test.rs
@@ -391,7 +391,7 @@ async fn inserting_into_a_concurrently_deleted_thread_is_retryable(pool: PgPool)
.unwrap();
let mut message = received_message(link_id, thread_db_id);
- let result = process_and_insert_message(&pool, link_id, thread_db_id, &mut message).await;
+ let result = process_and_insert_message(&pool, link_id, thread_db_id, &mut message, None).await;
match result {
Err(ProcessingError::Retryable(e)) => assert!(
diff --git a/services/email_service/src/pubsub/label_write_sim/fake_gmail.rs b/services/email_service/src/pubsub/label_write_sim/fake_gmail.rs
new file mode 100644
index 0000000000..f365fd1b96
--- /dev/null
+++ b/services/email_service/src/pubsub/label_write_sim/fake_gmail.rs
@@ -0,0 +1,472 @@
+//! An in-process Gmail REST server holding one mailbox's labels and history.
+//!
+//! The real `GmailClient` talks to it over HTTP, so request encoding, status
+//! mapping (429 → provider rate limit, 5xx → transient, 400 → permanent) and
+//! response decoding are the production code paths.
+
+use axum::Router;
+use axum::body::Bytes;
+use axum::extract::State;
+use axum::http::{Method, StatusCode, Uri};
+use axum::response::{IntoResponse, Response};
+use base64::Engine;
+use serde_json::{Value, json};
+use std::collections::{BTreeMap, BTreeSet, VecDeque};
+use std::sync::{Arc, Mutex};
+use tokio::sync::oneshot;
+
+/// How the fake answers one request.
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub enum Reply {
+ #[default]
+ Ok,
+ /// 429 "Too many concurrent requests for user".
+ RateLimited,
+ /// 503, mapped to `EmailApiError::Transient`.
+ Unavailable,
+ /// 400, mapped to `EmailApiError::Permanent`.
+ Rejected,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum Kind {
+ Write,
+ ReadLabels,
+ ReadFull,
+ Other,
+}
+
+#[derive(Clone, Debug)]
+pub struct Logged {
+ pub kind: Kind,
+ pub ids: Vec<String>,
+ pub add: Vec<String>,
+ pub remove: Vec<String>,
+ pub reply: Reply,
+ /// Labels Gmail returned for a read.
+ pub returned: Vec<String>,
+}
+
+#[derive(Clone, Debug)]
+pub struct GmailMessage {
+ pub thread_id: String,
+ pub labels: BTreeSet<String>,
+ pub internal_date_ms: i64,
+}
+
+#[derive(Clone, Debug)]
+pub struct HistoryEntry {
+ pub id: u64,
+ pub message_id: String,
+ pub thread_id: String,
+ pub added: Vec<String>,
+ pub removed: Vec<String>,
+ pub message_added: bool,
+}
+
+/// Where a paused request waits.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum PauseAt {
+ /// The request has not reached Gmail yet: it is applied on release.
+ BeforeApply,
+ /// Gmail answered (and applied a write) on arrival; the answer is
+ /// delivered on release.
+ AfterApply,
+}
+
+struct Pause {
+ kind: Kind,
+ message: Option<String>,
+ at: PauseAt,
+ arrived: oneshot::Sender<()>,
+ release: oneshot::Receiver<()>,
+}
+
+pub struct PauseHandle {
+ pub arrived: oneshot::Receiver<()>,
+ pub release: oneshot::Sender<()>,
+}
+
+#[derive(Default)]
+pub struct GmailState {
+ pub messages: BTreeMap<String, GmailMessage>,
+ pub labels: Vec<(String, String, String)>,
+ pub history_id: u64,
+ pub history: Vec<HistoryEntry>,
+ pub next_writes: VecDeque<Reply>,
+ pub default_write: Reply,
+ pub next_reads: VecDeque<Reply>,
+ pub default_read: Reply,
+ pub log: Vec<Logged>,
+ pause: Option<Pause>,
+}
+
+#[derive(Clone)]
+pub struct FakeGmail {
+ pub state: Arc<Mutex<GmailState>>,
+ pub url: String,
+}
+
+enum Request {
+ Modify {
+ ids: Vec<String>,
+ add: Vec<String>,
+ remove: Vec<String>,
+ },
+ GetMessage {
+ id: String,
+ minimal: bool,
+ },
+ GetThread {
+ id: String,
+ },
+ ListLabels,
+ History {
+ start: u64,
+ },
+ Other,
+}
+
+impl Request {
+ fn kind(&self) -> Kind {
+ match self {
+ Request::Modify { .. } => Kind::Write,
+ Request::GetMessage { minimal: true, .. } => Kind::ReadLabels,
+ Request::GetMessage { minimal: false, .. } => Kind::ReadFull,
+ _ => Kind::Other,
+ }
+ }
+
+ fn message_ids(&self) -> Vec<String> {
+ match self {
+ Request::Modify { ids, .. } => ids.clone(),
+ Request::GetMessage { id, .. } => vec![id.clone()],
+ _ => Vec::new(),
+ }
+ }
+}
+
+impl FakeGmail {
+ pub async fn start(state: GmailState) -> FakeGmail {
+ let state = Arc::new(Mutex::new(state));
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
+ .await
+ .expect("bind fake gmail");
+ let url = format!("http://{}/gmail/v1", listener.local_addr().unwrap());
+ let fake = FakeGmail { state, url };
+ let app = Router::new().fallback(handle).with_state(fake.clone());
+ tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
+ fake
+ }
+
+ pub fn lock(&self) -> std::sync::MutexGuard<'_, GmailState> {
+ self.state.lock().unwrap()
+ }
+
+ /// Holds the next request of `kind` (for `message`, when given) until the
+ /// returned handle releases it.
+ pub fn pause_next(&self, kind: Kind, message: Option<&str>, at: PauseAt) -> PauseHandle {
+ let (arrived_tx, arrived_rx) = oneshot::channel();
+ let (release_tx, release_rx) = oneshot::channel();
+ self.lock().pause = Some(Pause {
+ kind,
+ message: message.map(str::to_string),
+ at,
+ arrived: arrived_tx,
+ release: release_rx,
+ });
+ PauseHandle {
+ arrived: arrived_rx,
+ release: release_tx,
+ }
+ }
+
+ pub fn clear_pause(&self) {
+ self.lock().pause = None;
+ }
+
+ /// A change made in Gmail itself (web UI, phone), recorded in history.
+ pub fn external_change(&self, message: &str, add: &[&str], remove: &[&str]) {
+ let mut state = self.lock();
+ let add: Vec<String> = add.iter().map(|label| label.to_string()).collect();
+ let remove: Vec<String> = remove.iter().map(|label| label.to_string()).collect();
+ state.apply(&[message.to_string()], &add, &remove);
+ }
+
+ fn choose_reply(&self, request: &Request) -> Reply {
+ let mut state = self.lock();
+ match request.kind() {
+ Kind::Write => state.next_writes.pop_front().unwrap_or(state.default_write),
+ Kind::ReadLabels | Kind::ReadFull => {
+ state.next_reads.pop_front().unwrap_or(state.default_read)
+ }
+ Kind::Other => Reply::Ok,
+ }
+ }
+
+ fn respond(&self, request: &Request, reply: Reply) -> Response {
+ let mut state = self.lock();
+ let kind = request.kind();
+ let (add, remove) = match request {
+ Request::Modify { add, remove, .. } => (add.clone(), remove.clone()),
+ _ => (Vec::new(), Vec::new()),
+ };
+ let mut entry = Logged {
+ kind,
+ ids: request.message_ids(),
+ add,
+ remove,
+ reply,
+ returned: Vec::new(),
+ };
+ let response = match reply {
+ Reply::RateLimited => error(429, "Too many concurrent requests for user."),
+ Reply::Unavailable => error(503, "The service is currently unavailable."),
+ Reply::Rejected => error(400, "Invalid label"),
+ Reply::Ok => match request {
+ Request::Modify { ids, add, remove } => {
+ state.apply(ids, add, remove);
+ StatusCode::NO_CONTENT.into_response()
+ }
+ Request::GetMessage { id, minimal } => match state.messages.get(id) {
+ None => error(404, "Requested entity was not found."),
+ Some(message) => {
+ entry.returned = message.labels.iter().cloned().collect();
+ let body = if *minimal {
+ json!({
+ "id": id,
+ "threadId": message.thread_id,
+ "labelIds": message.labels,
+ })
+ } else {
+ full_message(id, message, state.history_id)
+ };
+ (StatusCode::OK, axum::Json(body)).into_response()
+ }
+ },
+ Request::GetThread { id } => {
+ let messages: Vec<Value> = state
+ .messages
+ .iter()
+ .filter(|(_, message)| message.thread_id == *id)
+ .map(|(message_id, message)| {
+ full_message(message_id, message, state.history_id)
+ })
+ .collect();
+ (
+ StatusCode::OK,
+ axum::Json(json!({ "id": id, "messages": messages })),
+ )
+ .into_response()
+ }
+ Request::ListLabels => {
+ let labels: Vec<Value> = state
+ .labels
+ .iter()
+ .map(|(id, name, label_type)| {
+ json!({
+ "id": id,
+ "name": name,
+ "type": label_type,
+ "messageListVisibility": "show",
+ "labelListVisibility": "labelShow",
+ })
+ })
+ .collect();
+ (StatusCode::OK, axum::Json(json!({ "labels": labels }))).into_response()
+ }
+ Request::History { start } => {
+ let history: Vec<Value> = state
+ .history
+ .iter()
+ .filter(|entry| entry.id > *start)
+ .map(history_record)
+ .collect();
+ (
+ StatusCode::OK,
+ axum::Json(json!({
+ "history": history,
+ "historyId": state.history_id.to_string(),
+ })),
+ )
+ .into_response()
+ }
+ Request::Other => (StatusCode::OK, axum::Json(json!({}))).into_response(),
+ },
+ };
+ state.log.push(entry);
+ response
+ }
+}
+
+impl GmailState {
+ fn apply(&mut self, ids: &[String], add: &[String], remove: &[String]) {
+ for id in ids {
+ // batchModify ignores ids that name no message.
+ let Some(message) = self.messages.get_mut(id) else {
+ continue;
+ };
+ let mut added = Vec::new();
+ let mut removed = Vec::new();
+ for label in add {
+ if message.labels.insert(label.clone()) {
+ added.push(label.clone());
+ }
+ }
+ for label in remove {
+ if message.labels.remove(label) {
+ removed.push(label.clone());
+ }
+ }
+ if added.is_empty() && removed.is_empty() {
+ continue;
+ }
+ let thread_id = message.thread_id.clone();
+ self.history_id += 1;
+ let entry = HistoryEntry {
+ id: self.history_id,
+ message_id: id.clone(),
+ thread_id,
+ added,
+ removed,
+ message_added: false,
+ };
+ self.history.push(entry);
+ }
+ }
+
+ fn take_pause_for(&mut self, request: &Request) -> Option<Pause> {
+ let pause = self.pause.as_ref()?;
+ if pause.kind != request.kind() {
+ return None;
+ }
+ if let Some(message) = &pause.message
+ && !request.message_ids().contains(message)
+ {
+ return None;
+ }
+ self.pause.take()
+ }
+}
+
+async fn handle(State(gmail): State<FakeGmail>, method: Method, uri: Uri, body: Bytes) -> Response {
+ let request = parse(&method, &uri, &body);
+ // The answer is decided on arrival, even when it is applied later.
+ let reply = gmail.choose_reply(&request);
+ let pause = gmail.lock().take_pause_for(&request);
+ match pause {
+ None => gmail.respond(&request, reply),
+ Some(pause) => match pause.at {
+ PauseAt::BeforeApply => {
+ let _ = pause.arrived.send(());
+ let _ = pause.release.await;
+ gmail.respond(&request, reply)
+ }
+ PauseAt::AfterApply => {
+ let response = gmail.respond(&request, reply);
+ let _ = pause.arrived.send(());
+ let _ = pause.release.await;
+ response
+ }
+ },
+ }
+}
+
+fn parse(method: &Method, uri: &Uri, body: &[u8]) -> Request {
+ let path = uri
+ .path()
+ .trim_start_matches("/gmail/v1/users/me/")
+ .to_string();
+ let query = uri.query().unwrap_or("");
+ let segments: Vec<&str> = path.split('/').collect();
+ let json: Value = serde_json::from_slice(body).unwrap_or(Value::Null);
+ let strings = |field: &str| -> Vec<String> {
+ json.get(field)
+ .and_then(Value::as_array)
+ .map(|values| {
+ values
+ .iter()
+ .filter_map(|value| value.as_str().map(str::to_string))
+ .collect()
+ })
+ .unwrap_or_default()
+ };
+ match (method.as_str(), segments.as_slice()) {
+ ("POST", ["messages", "batchModify"]) => Request::Modify {
+ ids: strings("ids"),
+ add: strings("addLabelIds"),
+ remove: strings("removeLabelIds"),
+ },
+ ("POST", ["messages", id, "modify"]) => Request::Modify {
+ ids: vec![id.to_string()],
+ add: strings("addLabelIds"),
+ remove: strings("removeLabelIds"),
+ },
+ ("GET", ["messages", id]) => Request::GetMessage {
+ id: id.to_string(),
+ minimal: query.contains("format=minimal"),
+ },
+ ("GET", ["threads", id]) => Request::GetThread { id: id.to_string() },
+ ("GET", ["labels"]) => Request::ListLabels,
+ ("GET", ["history"]) => Request::History {
+ start: query
+ .split('&')
+ .find_map(|pair| pair.strip_prefix("startHistoryId="))
+ .and_then(|value| value.parse().ok())
+ .unwrap_or(0),
+ },
+ _ => Request::Other,
+ }
+}
+
+fn error(code: u16, message: &str) -> Response {
+ (
+ StatusCode::from_u16(code).unwrap(),
+ axum::Json(json!({ "error": { "code": code, "message": message } })),
+ )
+ .into_response()
+}
+
+fn full_message(id: &str, message: &GmailMessage, history_id: u64) -> Value {
+ let body = base64::engine::general_purpose::URL_SAFE.encode(format!("Body of {id}"));
+ let date = chrono::DateTime::from_timestamp_millis(message.internal_date_ms)
+ .unwrap()
+ .to_rfc2822();
+ json!({
+ "id": id,
+ "threadId": message.thread_id,
+ "labelIds": message.labels,
+ "snippet": format!("Body of {id}"),
+ "sizeEstimate": 100,
+ "historyId": history_id.to_string(),
+ "internalDate": message.internal_date_ms.to_string(),
+ "payload": {
+ "partId": "",
+ "mimeType": "text/plain",
+ "filename": "",
+ "headers": [
+ { "name": "From", "value": "Alice Sender <[email protected]>" },
+ { "name": "To", "value": "Sim Owner <[email protected]>" },
+ { "name": "Subject", "value": format!("Thread {}", message.thread_id) },
+ { "name": "Date", "value": date },
+ { "name": "Message-ID", "value": format!("<{id}@example.org>") },
+ ],
+ "body": { "size": 12, "data": body },
+ },
+ })
+}
+
+fn history_record(entry: &HistoryEntry) -> Value {
+ let message = json!({ "id": entry.message_id, "threadId": entry.thread_id });
+ let mut record = json!({ "id": entry.id.to_string(), "messages": [message] });
+ if !entry.added.is_empty() {
+ record["labelsAdded"] = json!([{ "message": message, "labelIds": entry.added }]);
+ }
+ if !entry.removed.is_empty() {
+ record["labelsRemoved"] = json!([{ "message": message, "labelIds": entry.removed }]);
+ }
+ if entry.message_added {
+ record["messagesAdded"] = json!([{ "message": message }]);
+ }
+ record
+}
diff --git a/services/email_service/src/pubsub/label_write_sim/fake_sqs.rs b/services/email_service/src/pubsub/label_write_sim/fake_sqs.rs
new file mode 100644
index 0000000000..20f585376f
--- /dev/null
+++ b/services/email_service/src/pubsub/label_write_sim/fake_sqs.rs
@@ -0,0 +1,218 @@
+//! An in-process SQS endpoint speaking the AWS JSON 1.0 protocol.
+//!
+//! The production `sqs_client::SQS` and `sqs_worker::SQSWorker` send real SDK
+//! requests here. Sends, deletes and visibility changes are recorded for the
+//! simulation, which owns delivery and the virtual clock.
+
+use axum::Router;
+use axum::body::Bytes;
+use axum::extract::State;
+use axum::http::{HeaderMap, StatusCode};
+use axum::response::{IntoResponse, Response};
+use serde_json::{Value, json};
+use std::sync::{Arc, Mutex};
+use tokio::sync::oneshot;
+
+#[derive(Clone, Debug)]
+pub struct Sent {
+ pub queue_url: String,
+ pub body: String,
+ pub delay_seconds: u64,
+ pub message_id: String,
+}
+
+#[derive(Clone, Debug)]
+pub enum Ack {
+ Deleted { receipt: String },
+ Visibility { receipt: String, seconds: u64 },
+}
+
+#[derive(Default)]
+pub struct SqsState {
+ pub sent: Vec<Sent>,
+ pub acks: Vec<Ack>,
+ /// While set, every SendMessage fails, as if the SDK had used up its
+ /// retries against an SQS outage.
+ pub outage: bool,
+ pub failed_sends: Vec<Sent>,
+ next_id: u64,
+ /// Holds the next SendMessage until released.
+ paused_send: Option<(oneshot::Sender<()>, oneshot::Receiver<()>)>,
+}
+
+#[derive(Clone)]
+pub struct FakeSqs {
+ pub state: Arc<Mutex<SqsState>>,
+ pub url: String,
+}
+
+impl FakeSqs {
+ pub async fn start() -> FakeSqs {
+ let state = Arc::new(Mutex::new(SqsState::default()));
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
+ .await
+ .expect("bind fake sqs");
+ let url = format!("http://{}", listener.local_addr().unwrap());
+ let fake = FakeSqs { state, url };
+ let app = Router::new().fallback(handle).with_state(fake.clone());
+ tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
+ fake
+ }
+
+ pub fn queue_url(&self, name: &str) -> String {
+ format!("{}/000000000000/{name}", self.url)
+ }
+
+ pub fn lock(&self) -> std::sync::MutexGuard<'_, SqsState> {
+ self.state.lock().unwrap()
+ }
+
+ pub fn take(&self) -> (Vec<Sent>, Vec<Ack>) {
+ let mut state = self.lock();
+ (
+ std::mem::take(&mut state.sent),
+ std::mem::take(&mut state.acks),
+ )
+ }
+
+ pub fn set_outage(&self, outage: bool) {
+ self.lock().outage = outage;
+ }
+
+ /// Holds the next SendMessage. Returns (arrived, release).
+ pub fn pause_next_send(&self) -> (oneshot::Receiver<()>, oneshot::Sender<()>) {
+ let (arrived_tx, arrived_rx) = oneshot::channel();
+ let (release_tx, release_rx) = oneshot::channel();
+ self.lock().paused_send = Some((arrived_tx, release_rx));
+ (arrived_rx, release_tx)
+ }
+}
+
+/// An SDK client for `endpoint` without SDK retries: an injected outage fails
+/// the call at once instead of after the default backoff.
+pub fn client(endpoint: &str, retries: bool) -> aws_sdk_sqs::Client {
+ let mut config = aws_sdk_sqs::Config::builder()
+ .behavior_version(aws_sdk_sqs::config::BehaviorVersion::latest())
+ .region(aws_sdk_sqs::config::Region::new("us-east-1"))
+ .credentials_provider(aws_sdk_sqs::config::Credentials::new(
+ "test", "test", None, None, "harness",
+ ))
+ .endpoint_url(endpoint);
+ if !retries {
+ config = config.retry_config(aws_sdk_sqs::config::retry::RetryConfig::disabled());
+ }
+ aws_sdk_sqs::Client::from_conf(config.build())
+}
+
+async fn handle(State(sqs): State<FakeSqs>, headers: HeaderMap, body: Bytes) -> Response {
+ let target = headers
+ .get("x-amz-target")
+ .and_then(|value| value.to_str().ok())
+ .unwrap_or("")
+ .trim_start_matches("AmazonSQS.")
+ .to_string();
+ let request: Value = serde_json::from_slice(&body).unwrap_or(Value::Null);
+ let text = |field: &str| {
+ request
+ .get(field)
+ .and_then(Value::as_str)
+ .unwrap_or("")
+ .to_string()
+ };
+ if target == "SendMessage" || target == "SendMessageBatch" {
+ let paused = sqs.lock().paused_send.take();
+ if let Some((arrived, release)) = paused {
+ let _ = arrived.send(());
+ let _ = release.await;
+ }
+ }
+ let mut state = sqs.lock();
+ match target.as_str() {
+ "SendMessage" => {
+ state.next_id += 1;
+ let sent = Sent {
+ queue_url: text("QueueUrl"),
+ body: text("MessageBody"),
+ delay_seconds: request
+ .get("DelaySeconds")
+ .and_then(Value::as_u64)
+ .unwrap_or(0),
+ message_id: format!("msg-{}", state.next_id),
+ };
+ if state.outage {
+ state.failed_sends.push(sent);
+ return (
+ StatusCode::INTERNAL_SERVER_ERROR,
+ [("content-type", "application/x-amz-json-1.0")],
+ json!({ "__type": "com.amazonaws.sqs#InternalError", "message": "injected outage" })
+ .to_string(),
+ )
+ .into_response();
+ }
+ let message_id = sent.message_id.clone();
+ state.sent.push(sent);
+ json_response(json!({ "MessageId": message_id }))
+ }
+ "SendMessageBatch" => {
+ if state.outage {
+ return (
+ StatusCode::INTERNAL_SERVER_ERROR,
+ [("content-type", "application/x-amz-json-1.0")],
+ json!({ "__type": "com.amazonaws.sqs#InternalError", "message": "injected outage" })
+ .to_string(),
+ )
+ .into_response();
+ }
+ let queue_url = text("QueueUrl");
+ let mut successful = Vec::new();
+ for entry in request
+ .get("Entries")
+ .and_then(Value::as_array)
+ .cloned()
+ .unwrap_or_default()
+ {
+ state.next_id += 1;
+ let message_id = format!("msg-{}", state.next_id);
+ state.sent.push(Sent {
+ queue_url: queue_url.clone(),
+ body: entry["MessageBody"].as_str().unwrap_or("").to_string(),
+ delay_seconds: entry["DelaySeconds"].as_u64().unwrap_or(0),
+ message_id: message_id.clone(),
+ });
+ successful.push(json!({
+ "Id": entry["Id"],
+ "MessageId": message_id,
+ "MD5OfMessageBody": "",
+ }));
+ }
+ json_response(json!({ "Successful": successful, "Failed": [] }))
+ }
+ "DeleteMessage" => {
+ state.acks.push(Ack::Deleted {
+ receipt: text("ReceiptHandle"),
+ });
+ json_response(json!({}))
+ }
+ "ChangeMessageVisibility" => {
+ state.acks.push(Ack::Visibility {
+ receipt: text("ReceiptHandle"),
+ seconds: request
+ .get("VisibilityTimeout")
+ .and_then(Value::as_u64)
+ .unwrap_or(0),
+ });
+ json_response(json!({}))
+ }
+ "ReceiveMessage" => json_response(json!({ "Messages": [] })),
+ other => panic!("fake SQS does not implement {other}"),
+ }
+}
+
+fn json_response(body: Value) -> Response {
+ (
+ StatusCode::OK,
+ [("content-type", "application/x-amz-json-1.0")],
+ body.to_string(),
+ )
+ .into_response()
+}
diff --git a/services/email_service/src/pubsub/label_write_sim/mod.rs b/services/email_service/src/pubsub/label_write_sim/mod.rs
new file mode 100644
index 0000000000..e116327e42
--- /dev/null
+++ b/services/email_service/src/pubsub/label_write_sim/mod.rs
@@ -0,0 +1,12 @@
+//! One Gmail mailbox wired to the production label-write, gmail_ops and inbox
+//! sync code, with fake Gmail and SQS endpoints. Its tests replay known races
+//! between Macro's label writes, Gmail's answers and syncs, and a seeded
+//! simulation checks that sync never undoes an unconfirmed write and that
+//! Gmail and the database end on the user's last effective action.
+
+mod fake_gmail;
+mod fake_sqs;
+mod sim;
+mod test;
+mod wiring;
+mod world;
diff --git a/services/email_service/src/pubsub/label_write_sim/sim.rs b/services/email_service/src/pubsub/label_write_sim/sim.rs
new file mode 100644
index 0000000000..1bbfea6e48
--- /dev/null
+++ b/services/email_service/src/pubsub/label_write_sim/sim.rs
@@ -0,0 +1,1056 @@
+//! Seeded simulation of user label actions, Gmail answers, queue redelivery
+//! and inbox syncs, run against the real worker and domain code.
+//!
+//! SQS is modelled on a virtual clock: the fake SQS endpoint records what the
+//! production code sends, deletes and delays, and the simulation decides when
+//! each message is delivered to `gmail_ops::process::process_message` or
+//! `inbox_sync::process::process_message`.
+
+mod generator;
+mod model;
+mod test;
+
+use super::fake_gmail::{Kind, Logged, PauseAt, Reply};
+use super::fake_sqs::Ack;
+use super::wiring::{self, LabelOp};
+use super::world::{self, MESSAGES, THREADS, TRACKED, WORK, World};
+use aws_sdk_sqs::types::{Message, MessageSystemAttributeName};
+use email::domain::models::mailbox_action::MailboxAction;
+use email::domain::ports::EmailService;
+use models_email::gmail::inbox_sync::{
+ InboxSyncOperation, InboxSyncPubsubMessage, UpdateLabelsPayload, UpsertMessagePayload,
+};
+use rand::rngs::StdRng;
+use rand::{Rng, SeedableRng};
+use std::collections::{BTreeMap, BTreeSet, HashMap};
+use std::fmt::Write as _;
+use std::time::Duration;
+use uuid::Uuid;
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum Queue {
+ Ops,
+ OpsRetry,
+ Sync,
+ SyncRetry,
+}
+
+impl Queue {
+ fn max_receives(self) -> u32 {
+ match self {
+ Queue::Ops | Queue::Sync => 3,
+ Queue::OpsRetry | Queue::SyncRetry => 100,
+ }
+ }
+}
+
+#[derive(Clone, Debug)]
+pub struct Queued {
+ pub id: String,
+ pub queue: Queue,
+ pub body: String,
+ pub visible_at: u64,
+ pub receives: u32,
+ /// The label write this message carries, if it is a gmail_ops write.
+ pub lineage: Option<u64>,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+enum EntryState {
+ Live,
+ Rejected,
+ DeadLettered,
+}
+
+#[derive(Clone, Debug)]
+struct Entry {
+ seq: u64,
+ lineage: u64,
+ value: bool,
+ state: EntryState,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+enum OpStatus {
+ Queued,
+ Applied,
+ Rejected,
+ Skipped,
+ DeadLettered,
+}
+
+#[derive(Clone, Debug)]
+struct OpTrack {
+ seq: u64,
+ op: LabelOp,
+ label: String,
+ value: bool,
+ status: OpStatus,
+ /// Messages Gmail accepted this write for, over all its deliveries.
+ accepted: BTreeSet<String>,
+}
+
+#[derive(Clone, Debug)]
+pub enum Action {
+ Archive(usize, bool),
+ Read(usize, bool),
+ Star(usize, bool),
+ Label(usize, bool),
+ Trash(usize, bool),
+ BulkArchive(Vec<usize>, bool),
+}
+
+#[derive(Clone, Debug)]
+pub enum Step {
+ Action {
+ action: Action,
+ enqueue_fails: bool,
+ },
+ Deliver {
+ reply: Reply,
+ gate_refuses: bool,
+ enqueue_fails: bool,
+ },
+ Sync {
+ message: usize,
+ reply: Reply,
+ },
+ Resync {
+ message: usize,
+ reply: Reply,
+ },
+ Wait {
+ seconds: u64,
+ },
+ /// Twelve hours pass: every queued message is worked off, then markers age
+ /// past their expiry.
+ Expire,
+ /// `first` reaches Gmail, `seconds` run to completion, then `first`
+ /// finishes.
+ Overlap {
+ first: Box<Step>,
+ seconds: Vec<Step>,
+ at: PauseAt,
+ },
+}
+
+#[derive(Clone, Debug)]
+pub struct Violation {
+ pub invariant: &'static str,
+ pub step: usize,
+ pub detail: String,
+}
+
+impl Violation {
+ /// The kind of step after which the violation showed, e.g. `Sync`.
+ pub fn step_kind(&self) -> String {
+ let text = self
+ .detail
+ .trim_start_matches("after ")
+ .trim_start_matches("drain ");
+ text.split(|c: char| !c.is_alphanumeric())
+ .next()
+ .unwrap_or("")
+ .to_string()
+ }
+}
+
+#[derive(Clone, Copy, Debug)]
+pub struct Config {
+ pub steps: usize,
+ pub overlap: bool,
+ /// Producers send label write batches rather than one operation per
+ /// message.
+ pub label_write_batches: bool,
+}
+
+pub struct Sim<'w> {
+ pub world: &'w mut World,
+ rng: StdRng,
+ pub now: u64,
+ pub queued: Vec<Queued>,
+ pub dead_letters: Vec<Queued>,
+ cells: BTreeMap<(String, String), Vec<Entry>>,
+ ops: BTreeMap<u64, OpTrack>,
+ lineage_by_body: HashMap<String, u64>,
+ /// The lineage carrying each write to each provider message: with label
+ /// write batches off, one write is an operation per message.
+ lineage_by_write: HashMap<(Uuid, String), u64>,
+ next_lineage: u64,
+ action_seq: u64,
+ next_message: u64,
+ pub trace: Vec<String>,
+ pub violations: Vec<Violation>,
+ /// Cells whose latest write dead-lettered.
+ pub lost_cells: BTreeSet<(String, String)>,
+ pub stats: Stats,
+ step_index: usize,
+}
+
+#[derive(Clone, Debug, Default)]
+pub struct Stats {
+ pub actions: u64,
+ pub failed_actions: u64,
+ pub deliveries: u64,
+ pub syncs: u64,
+ pub resyncs: u64,
+ pub gmail_writes: u64,
+ pub gmail_429: u64,
+ pub gmail_5xx: u64,
+ pub gmail_400: u64,
+ pub transient_stale_applies: u64,
+ pub skipped_superseded: u64,
+ pub dead_letters: u64,
+ pub overlaps: u64,
+ pub expiries: u64,
+}
+
+pub enum Source {
+ Sync(usize),
+ Resync(usize),
+ /// The earliest message already on the queue.
+ Queued(Queue),
+}
+
+pub struct Paused {
+ delivery: Delivery,
+ task: tokio::task::JoinHandle<()>,
+ release: tokio::sync::oneshot::Sender<()>,
+ window_from: u64,
+ at: PauseAt,
+ action_seq: u64,
+}
+
+struct Delivery {
+ queued: Queued,
+ message: Message,
+ log_from: u64,
+ /// Gmail requests made by other steps while this one was paused.
+ excluded: std::ops::Range<u64>,
+ /// When Gmail applied this delivery's write before later actions ran:
+ /// the last action it could have seen.
+ applied_before: Option<u64>,
+}
+
+impl<'w> Sim<'w> {
+ pub fn new(world: &'w mut World, seed: u64) -> Sim<'w> {
+ let cells = MESSAGES
+ .iter()
+ .flat_map(|(id, _, _)| {
+ TRACKED
+ .iter()
+ .map(move |label| ((id.to_string(), label.to_string()), Vec::new()))
+ })
+ .collect();
+ Sim {
+ world,
+ rng: StdRng::seed_from_u64(seed),
+ now: 0,
+ queued: Vec::new(),
+ dead_letters: Vec::new(),
+ cells,
+ ops: BTreeMap::new(),
+ lineage_by_body: HashMap::new(),
+ lineage_by_write: HashMap::new(),
+ next_lineage: 0,
+ action_seq: 0,
+ next_message: 0,
+ trace: Vec::new(),
+ violations: Vec::new(),
+ lost_cells: BTreeSet::new(),
+ stats: Stats::default(),
+ step_index: 0,
+ }
+ }
+
+ // --------------------------------------------------------------- queues
+
+ fn queue_of(&self, url: &str) -> Option<Queue> {
+ let queues = &self.world.queues;
+ if url == queues.ops {
+ Some(Queue::Ops)
+ } else if url == queues.ops_retry {
+ Some(Queue::OpsRetry)
+ } else if url == queues.sync {
+ Some(Queue::Sync)
+ } else if url == queues.sync_retry {
+ Some(Queue::SyncRetry)
+ } else {
+ None
+ }
+ }
+
+ /// Takes what the code sent to SQS since the last call into the model.
+ /// Writes first seen here belong to the user action `seq`.
+ fn absorb_sends(&mut self, seq: Option<u64>) {
+ let (sent, acks) = self.world.sqs.take();
+ for sent in sent {
+ let Some(queue) = self.queue_of(&sent.queue_url) else {
+ continue;
+ };
+ let lineage = if matches!(queue, Queue::Ops | Queue::OpsRetry) {
+ self.lineage_for(&sent.body, seq)
+ } else {
+ None
+ };
+ self.queued.push(Queued {
+ id: sent.message_id,
+ queue,
+ body: sent.body,
+ visible_at: self.now + sent.delay_seconds,
+ receives: 0,
+ lineage,
+ });
+ }
+ assert!(acks.is_empty(), "acks outside a delivery: {acks:?}");
+ }
+
+ /// A send made by a user action is a new write; any other send of a label
+ /// write is a re-enqueue (primary → retry) of the write with that body.
+ /// On `main` two actions can produce the same body; the newest wins.
+ fn lineage_for(&mut self, body: &str, seq: Option<u64>) -> Option<u64> {
+ if seq.is_none()
+ && let Some(lineage) = self.lineage_by_body.get(body)
+ {
+ return Some(*lineage);
+ }
+ let op = wiring::decode_label_op(body)?;
+ let lineage = self.next_lineage;
+ self.next_lineage += 1;
+ let seq = seq.unwrap_or_else(|| panic!("label write sent outside a user action: {body}"));
+ let (label, value) = match (op.add.first(), op.remove.first()) {
+ (Some(label), None) => (label.clone(), true),
+ (None, Some(label)) => (label.clone(), false),
+ _ => panic!("label write changes more than one label: {op:?}"),
+ };
+ for (_, provider_id) in &op.messages {
+ self.cells
+ .get_mut(&(provider_id.clone(), label.clone()))
+ .unwrap_or_else(|| panic!("untracked cell {provider_id}/{label}"))
+ .push(Entry {
+ seq,
+ lineage,
+ value,
+ state: EntryState::Live,
+ });
+ }
+ if let Some(write_id) = op.write_id {
+ for (_, provider_id) in &op.messages {
+ self.lineage_by_write
+ .insert((write_id, provider_id.clone()), lineage);
+ }
+ }
+ self.lineage_by_body.insert(body.to_string(), lineage);
+ self.ops.insert(
+ lineage,
+ OpTrack {
+ seq,
+ op,
+ label,
+ value,
+ status: OpStatus::Queued,
+ accepted: BTreeSet::new(),
+ },
+ );
+ Some(lineage)
+ }
+
+ fn set_entries(&mut self, lineage: u64, messages: &[String], state: EntryState) {
+ let label = self.ops[&lineage].label.clone();
+ for message in messages {
+ for entry in self
+ .cells
+ .get_mut(&(message.clone(), label.clone()))
+ .unwrap()
+ .iter_mut()
+ .filter(|entry| entry.lineage == lineage)
+ {
+ entry.state = state;
+ }
+ }
+ }
+
+ /// Picks the next message to deliver: due ones first, otherwise the clock
+ /// moves to the earliest.
+ fn next_due(&mut self) -> Option<usize> {
+ if self.queued.is_empty() {
+ return None;
+ }
+ let earliest = self.queued.iter().map(|queued| queued.visible_at).min()?;
+ if earliest > self.now {
+ self.now = earliest;
+ }
+ let due: Vec<usize> = (0..self.queued.len())
+ .filter(|index| self.queued[*index].visible_at <= self.now)
+ .collect();
+ Some(due[self.rng.random_range(0..due.len())])
+ }
+
+ /// Takes a message off its queue for delivery, or dead-letters it once it
+ /// has used up its receives.
+ fn begin_delivery(&mut self, index: usize) -> Option<Delivery> {
+ let mut queued = self.queued.remove(index);
+ if queued.receives >= queued.queue.max_receives() {
+ self.stats.dead_letters += 1;
+ if let Some(lineage) = queued.lineage {
+ let messages: Vec<String> = self.ops[&lineage]
+ .op
+ .messages
+ .iter()
+ .map(|(_, provider_id)| provider_id.clone())
+ .collect();
+ self.ops.get_mut(&lineage).unwrap().status = OpStatus::DeadLettered;
+ self.set_entries(lineage, &messages, EntryState::DeadLettered);
+ }
+ self.trace.push(format!(
+ "t={} dead-letter {:?} {}",
+ self.now,
+ queued.queue,
+ short(&queued.body)
+ ));
+ self.dead_letters.push(queued);
+ return None;
+ }
+ queued.receives += 1;
+ let message = Message::builder()
+ .message_id(&queued.id)
+ .receipt_handle(format!("{}#{}", queued.id, queued.receives))
+ .body(&queued.body)
+ .attributes(
+ MessageSystemAttributeName::ApproximateReceiveCount,
+ queued.receives.to_string(),
+ )
+ .build();
+ let log_from = self.world.gmail.lock().log.len() as u64;
+ Some(Delivery {
+ queued,
+ message,
+ log_from,
+ excluded: 0..0,
+ applied_before: None,
+ })
+ }
+
+ /// Hands the message to the worker entry point for its queue, as the
+ /// worker loop does after a receive.
+ fn spawn_delivery(world: &World, delivery: &Delivery) -> tokio::task::JoinHandle<()> {
+ let message = delivery.message.clone();
+ match delivery.queued.queue {
+ Queue::Ops | Queue::OpsRetry => {
+ let ctx = if delivery.queued.queue == Queue::Ops {
+ world.ops.clone()
+ } else {
+ world.ops_retry.clone()
+ };
+ tokio::spawn(async move {
+ let _ = crate::pubsub::gmail_ops::process::process_message(ctx, &message).await;
+ })
+ }
+ Queue::Sync | Queue::SyncRetry => {
+ let ctx = if delivery.queued.queue == Queue::Sync {
+ world.sync.clone()
+ } else {
+ world.sync_retry.clone()
+ };
+ tokio::spawn(async move {
+ let _ =
+ crate::pubsub::inbox_sync::process::process_message(ctx, &message).await;
+ })
+ }
+ }
+ }
+
+ async fn run_delivery(world: &World, delivery: &Delivery) {
+ Self::spawn_delivery(world, delivery)
+ .await
+ .expect("worker task panicked");
+ }
+
+ /// Applies a finished delivery's acknowledgements and checks that a label
+ /// write left its queue only for an accountable reason.
+ fn finish_delivery(&mut self, delivery: Delivery) {
+ self.stats.deliveries += 1;
+ let (sent, acks) = self.world.sqs.take();
+ let receipt = delivery.message.receipt_handle().unwrap().to_string();
+ let mut queued = delivery.queued;
+ let mut deleted = false;
+ let mut visibility = None;
+ let mut foreign_acks = Vec::new();
+ for ack in acks {
+ match ack {
+ Ack::Deleted { receipt: r } if r == receipt => deleted = true,
+ Ack::Visibility {
+ receipt: r,
+ seconds,
+ } if r == receipt => visibility = Some(seconds),
+ other => foreign_acks.push(other),
+ }
+ }
+ assert!(foreign_acks.is_empty(), "unexpected acks {foreign_acks:?}");
+ let resent_to_retry = sent.iter().any(|sent| {
+ sent.body == queued.body && self.queue_of(&sent.queue_url) == Some(Queue::OpsRetry)
+ });
+ // The hand-off to the retry queue carries this delivery's write.
+ if resent_to_retry && let Some(lineage) = queued.lineage {
+ self.lineage_by_body.insert(queued.body.clone(), lineage);
+ }
+ // Put the sends back for absorb_sends.
+ self.world.sqs.lock().sent.extend(sent);
+ let log: Vec<Logged> = self.world.gmail.lock().log[delivery.log_from as usize..]
+ .iter()
+ .enumerate()
+ .filter(|(offset, _)| {
+ !delivery
+ .excluded
+ .contains(&(delivery.log_from + *offset as u64))
+ })
+ .map(|(_, entry)| entry.clone())
+ .collect();
+ for entry in &log {
+ match entry.reply {
+ Reply::RateLimited => self.stats.gmail_429 += 1,
+ Reply::Unavailable => self.stats.gmail_5xx += 1,
+ Reply::Rejected => self.stats.gmail_400 += 1,
+ Reply::Ok => {}
+ }
+ }
+ self.trace.push(format!(
+ "t={} deliver {:?}#{} {} -> deleted={} visibility={:?} gmail={:?}",
+ self.now,
+ queued.queue,
+ queued.receives,
+ short(&queued.body),
+ deleted,
+ visibility,
+ log.iter()
+ .map(|entry| format!("{:?}{:?}{:?}", entry.kind, entry.ids, entry.reply))
+ .collect::<Vec<_>>()
+ ));
+ if let Some(lineage) = queued.lineage {
+ self.account_write(
+ lineage,
+ &log,
+ deleted,
+ resent_to_retry,
+ delivery.applied_before,
+ );
+ }
+ if !deleted {
+ queued.visible_at = self.now + visibility.unwrap_or(60);
+ self.queued.push(queued);
+ }
+ }
+
+ // ---------------------------------------------------------------- steps
+
+ async fn prepare(&mut self, step: &Step) {
+ wiring::reset_admission(&mut self.world.redis, self.world.link_id).await;
+ {
+ let mut gmail = self.world.gmail.lock();
+ gmail.default_write = Reply::Ok;
+ gmail.default_read = Reply::Ok;
+ gmail.next_writes.clear();
+ gmail.next_reads.clear();
+ }
+ self.world.sqs.set_outage(false);
+ match step {
+ Step::Deliver {
+ reply,
+ gate_refuses,
+ enqueue_fails,
+ } => {
+ {
+ let mut gmail = self.world.gmail.lock();
+ gmail.next_writes.push_back(*reply);
+ gmail.next_reads.push_back(*reply);
+ }
+ if *gate_refuses {
+ wiring::refuse_admission(&mut self.world.redis, self.world.link_id).await;
+ }
+ self.world.sqs.set_outage(*enqueue_fails);
+ }
+ Step::Sync { reply, .. } | Step::Resync { reply, .. } => {
+ self.world.gmail.lock().next_reads.push_back(*reply);
+ }
+ Step::Action { enqueue_fails, .. } => self.world.sqs.set_outage(*enqueue_fails),
+ _ => {}
+ }
+ }
+
+ fn sync_message(&mut self, message: usize, resync: bool) -> Queued {
+ let provider_message_id = MESSAGES[message].0.to_string();
+ let operation = if resync {
+ InboxSyncOperation::UpsertMessage(UpsertMessagePayload {
+ provider_message_id,
+ })
+ } else {
+ InboxSyncOperation::UpdateLabels(UpdateLabelsPayload {
+ provider_message_id,
+ })
+ };
+ self.next_message += 1;
+ Queued {
+ id: format!("sync-{}", self.next_message),
+ queue: Queue::Sync,
+ body: serde_json::to_string(&InboxSyncPubsubMessage {
+ link_id: self.world.link_id,
+ operation,
+ })
+ .unwrap(),
+ visible_at: self.now,
+ receives: 0,
+ lineage: None,
+ }
+ }
+
+ pub async fn run_step(&mut self, step: Step) {
+ self.prepare(&step).await;
+ match step {
+ Step::Action {
+ action,
+ enqueue_fails,
+ } => {
+ let _ = self.act(action, enqueue_fails).await;
+ }
+ Step::Deliver { .. } => {
+ if let Some(index) = self.next_due()
+ && let Some(delivery) = self.begin_delivery(index)
+ {
+ Self::run_delivery(self.world, &delivery).await;
+ self.finish_delivery(delivery);
+ }
+ }
+ Step::Sync { message, .. } | Step::Resync { message, .. } => {
+ let resync = matches!(step, Step::Resync { .. });
+ if resync {
+ self.stats.resyncs += 1;
+ } else {
+ self.stats.syncs += 1;
+ }
+ let queued = self.sync_message(message, resync);
+ self.queued.push(queued);
+ let index = self.queued.len() - 1;
+ let delivery = self.begin_delivery(index).unwrap();
+ Self::run_delivery(self.world, &delivery).await;
+ self.finish_delivery(delivery);
+ }
+ Step::Wait { seconds } => {
+ self.now += seconds;
+ self.trace.push(format!("t={} wait {seconds}s", self.now));
+ }
+ Step::Expire => Box::pin(self.expire()).await,
+ Step::Overlap { first, seconds, at } => {
+ Box::pin(self.overlap(*first, seconds, at)).await
+ }
+ }
+ self.absorb_sends(None);
+ self.world.sqs.set_outage(false);
+ }
+
+ pub async fn act(&mut self, action: Action, enqueue_fails: bool) -> Result<(), String> {
+ self.world.sqs.set_outage(enqueue_fails);
+ self.stats.actions += 1;
+ self.action_seq += 1;
+ let seq = self.action_seq;
+ let owner = world::owner();
+ let thread = |index: usize| self.world.thread_ids[THREADS[index]];
+ let result: Result<(), String> = match &action {
+ Action::Archive(t, value) => self
+ .world
+ .email
+ .set_thread_archived(owner, thread(*t), *value)
+ .await
+ .map_err(|error| format!("{error:?}")),
+ Action::Read(t, value) => self
+ .world
+ .email
+ .change_thread_mailbox_state(owner, thread(*t), MailboxAction::Read(*value))
+ .await
+ .map_err(|error| format!("{error:?}")),
+ Action::Star(t, value) => self
+ .world
+ .email
+ .change_thread_mailbox_state(owner, thread(*t), MailboxAction::Flagged(*value))
+ .await
+ .map_err(|error| format!("{error:?}")),
+ Action::Trash(t, value) => self
+ .world
+ .email
+ .change_thread_mailbox_state(owner, thread(*t), MailboxAction::Trashed(*value))
+ .await
+ .map_err(|error| format!("{error:?}")),
+ Action::Label(t, value) => self
+ .world
+ .email
+ .update_thread_labels_for_user(
+ owner,
+ thread(*t),
+ self.world.label_ids[WORK],
+ *value,
+ )
+ .await
+ .map(|_| ())
+ .map_err(|error| format!("{error:?}")),
+ Action::BulkArchive(threads, value) => {
+ let ids = threads.iter().map(|t| thread(*t)).collect();
+ wiring::archive_many(&self.world.email, owner, ids, *value)
+ .await
+ .and_then(|failed| {
+ if failed.is_empty() {
+ Ok(())
+ } else {
+ Err(format!("failed threads {failed:?}"))
+ }
+ })
+ }
+ };
+ if result.is_err() {
+ self.stats.failed_actions += 1;
+ }
+ self.trace.push(format!(
+ "t={} action#{seq} {action:?} enqueue_fails={enqueue_fails} -> {result:?}",
+ self.now
+ ));
+ self.world.sqs.set_outage(false);
+ self.absorb_sends(Some(seq));
+ result
+ }
+
+ /// Twelve hours: work off the queues with the same mix of Gmail answers,
+ /// then age every marker past its expiry.
+ async fn expire(&mut self) {
+ self.stats.expiries += 1;
+ self.trace
+ .push(format!("t={} expire: drain then age markers", self.now));
+ self.drain(true).await;
+ self.now += 12 * 3600;
+ let ids: Vec<Uuid> = self.world.message_ids.values().copied().collect();
+ wiring::age_markers(&self.world.db, &ids).await;
+ }
+
+ /// Delivers until every queue is empty. `faulty` keeps Gmail failing at
+ /// random; otherwise Gmail accepts everything.
+ pub async fn drain(&mut self, faulty: bool) {
+ for _ in 0..5_000 {
+ if self.queued.is_empty() {
+ return;
+ }
+ let step = if faulty {
+ Step::Deliver {
+ reply: self.random_reply(0.6),
+ gate_refuses: false,
+ enqueue_fails: false,
+ }
+ } else {
+ Step::Deliver {
+ reply: Reply::Ok,
+ gate_refuses: false,
+ enqueue_fails: false,
+ }
+ };
+ let description = format!("drain {step:?}");
+ Box::pin(self.run_step(step)).await;
+ Box::pin(self.check_step(&description)).await;
+ }
+ panic!("queues did not drain: {:?}", self.queued);
+ }
+
+ async fn overlap(&mut self, first: Step, seconds: Vec<Step>, at: PauseAt) {
+ self.stats.overlaps += 1;
+ if let Step::Deliver { reply, .. } | Step::Sync { reply, .. } | Step::Resync { reply, .. } =
+ &first
+ {
+ let mut gmail = self.world.gmail.lock();
+ gmail.next_writes.push_back(*reply);
+ gmail.next_reads.push_back(*reply);
+ }
+ let started = match &first {
+ Step::Deliver { .. } => self.next_due().and_then(|index| self.begin_delivery(index)),
+ Step::Sync { message, .. } | Step::Resync { message, .. } => {
+ let resync = matches!(first, Step::Resync { .. });
+ let queued = self.sync_message(*message, resync);
+ self.queued.push(queued);
+ let index = self.queued.len() - 1;
+ self.begin_delivery(index)
+ }
+ _ => None,
+ };
+ let Some(mut delivery) = started else {
+ self.clear_replies();
+ for step in seconds {
+ self.run_concurrently(step).await;
+ }
+ return;
+ };
+ let kind = match (&first, delivery.queued.queue) {
+ (_, Queue::Ops | Queue::OpsRetry) => Kind::Write,
+ (Step::Resync { .. }, _) => Kind::ReadFull,
+ _ => Kind::ReadLabels,
+ };
+ self.trace.push(format!(
+ "t={} overlap: {} paused {at:?} at its Gmail {kind:?}",
+ self.now,
+ short(&delivery.queued.body)
+ ));
+ let pause = self.world.gmail.pause_next(kind, None, at);
+ let mut task = Self::spawn_delivery(self.world, &delivery);
+ let arrived = tokio::select! {
+ _ = &mut task => false,
+ arrived = pause.arrived => arrived.is_ok(),
+ };
+ self.world.gmail.clear_pause();
+ self.clear_replies();
+ if !arrived {
+ self.finish_delivery(delivery);
+ // Its sends (a hand-off to the retry queue) belong to it, not to
+ // the next user action.
+ self.absorb_sends(None);
+ for step in seconds {
+ self.run_concurrently(step).await;
+ }
+ return;
+ }
+ // `first` is held at Gmail while the other steps run to completion.
+ // Their deliveries are accounted as they finish: until the release,
+ // every SQS ack and Gmail request is theirs.
+ let window_from = self.world.gmail.lock().log.len() as u64;
+ if at == PauseAt::AfterApply {
+ delivery.applied_before = Some(self.action_seq);
+ }
+ for step in seconds {
+ self.trace
+ .push(format!("t={} while paused: {step:?}", self.now));
+ self.run_concurrently(step).await;
+ }
+ delivery.excluded = window_from..self.world.gmail.lock().log.len() as u64;
+ let _ = pause.release.send(());
+ tokio::time::timeout(Duration::from_secs(60), task)
+ .await
+ .expect("paused delivery finished")
+ .expect("worker task panicked");
+ self.finish_delivery(delivery);
+ }
+
+ fn clear_replies(&mut self) {
+ let mut gmail = self.world.gmail.lock();
+ gmail.next_writes.clear();
+ gmail.next_reads.clear();
+ }
+
+ /// Runs `step` to completion while another delivery is paused at Gmail.
+ async fn run_concurrently(&mut self, step: Step) {
+ match step {
+ Step::Action {
+ action,
+ enqueue_fails,
+ } => {
+ let _ = self.act(action, enqueue_fails).await;
+ }
+ Step::Deliver { reply, .. } => {
+ self.world.gmail.lock().next_writes.push_back(reply);
+ self.world.gmail.lock().next_reads.push_back(reply);
+ if let Some(index) = self.next_due()
+ && let Some(delivery) = self.begin_delivery(index)
+ {
+ Self::run_delivery(self.world, &delivery).await;
+ self.finish_delivery(delivery);
+ }
+ self.clear_replies();
+ }
+ Step::Sync { message, reply } | Step::Resync { message, reply } => {
+ let resync = matches!(step, Step::Resync { .. });
+ self.world.gmail.lock().next_reads.push_back(reply);
+ let queued = self.sync_message(message, resync);
+ self.queued.push(queued);
+ let index = self.queued.len() - 1;
+ if let Some(delivery) = self.begin_delivery(index) {
+ Self::run_delivery(self.world, &delivery).await;
+ self.finish_delivery(delivery);
+ }
+ self.clear_replies();
+ }
+ _ => {}
+ }
+ self.absorb_sends(None);
+ }
+
+ // ------------------------------------------------------ scenario steps
+
+ /// Delivers the earliest message on `queue` (moving the clock to it)
+ /// with Gmail answering `reply`. Returns false when the queue is empty.
+ pub async fn deliver_from(&mut self, queue: Queue, reply: Reply, enqueue_fails: bool) -> bool {
+ self.deliver_answering(queue, &[reply], enqueue_fails).await
+ }
+
+ /// Like [`Self::deliver_from`], with Gmail answering the delivery's
+ /// writes with `replies` in order and accepting any after them.
+ pub async fn deliver_answering(
+ &mut self,
+ queue: Queue,
+ replies: &[Reply],
+ enqueue_fails: bool,
+ ) -> bool {
+ let Some(index) = (0..self.queued.len())
+ .filter(|index| self.queued[*index].queue == queue)
+ .min_by_key(|index| (self.queued[*index].visible_at, *index))
+ else {
+ return false;
+ };
+ self.now = self.now.max(self.queued[index].visible_at);
+ self.prepare(&Step::Deliver {
+ reply: replies[0],
+ gate_refuses: false,
+ enqueue_fails,
+ })
+ .await;
+ self.world
+ .gmail
+ .lock()
+ .next_writes
+ .extend(replies[1..].iter().copied());
+ if let Some(delivery) = self.begin_delivery(index) {
+ Self::run_delivery(self.world, &delivery).await;
+ self.finish_delivery(delivery);
+ }
+ self.absorb_sends(None);
+ self.world.sqs.set_outage(false);
+ self.clear_replies();
+ true
+ }
+
+ /// Delivers every message now on `queue` once, oldest first.
+ pub async fn deliver_all(&mut self, queue: Queue, reply: Reply) -> usize {
+ let count = self
+ .queued
+ .iter()
+ .filter(|queued| queued.queue == queue)
+ .count();
+ for _ in 0..count {
+ self.deliver_from(queue, reply, false).await;
+ }
+ count
+ }
+
+ pub fn queue_len(&self, queue: Queue) -> usize {
+ self.queued
+ .iter()
+ .filter(|queued| queued.queue == queue)
+ .count()
+ }
+
+ /// Starts a delivery and holds its first Gmail request of `kind`.
+ pub async fn start_paused(
+ &mut self,
+ source: Source,
+ kind: Kind,
+ at: PauseAt,
+ ) -> Option<Paused> {
+ self.prepare(&Step::Wait { seconds: 0 }).await;
+ let index = match source {
+ Source::Sync(message) | Source::Resync(message) => {
+ let queued = self.sync_message(message, matches!(source, Source::Resync(_)));
+ self.queued.push(queued);
+ self.queued.len() - 1
+ }
+ Source::Queued(queue) => (0..self.queued.len())
+ .filter(|index| self.queued[*index].queue == queue)
+ .min_by_key(|index| (self.queued[*index].visible_at, *index))?,
+ };
+ self.now = self.now.max(self.queued[index].visible_at);
+ let delivery = self.begin_delivery(index)?;
+ let pause = self.world.gmail.pause_next(kind, None, at);
+ let mut task = Self::spawn_delivery(self.world, &delivery);
+ let arrived = tokio::select! {
+ _ = &mut task => false,
+ arrived = pause.arrived => arrived.is_ok(),
+ };
+ assert!(arrived, "delivery finished without reaching Gmail");
+ Some(Paused {
+ delivery,
+ task,
+ release: pause.release,
+ window_from: self.world.gmail.lock().log.len() as u64,
+ at,
+ action_seq: self.action_seq,
+ })
+ }
+
+ pub async fn finish_paused(&mut self, paused: Paused) {
+ let Paused {
+ mut delivery,
+ task,
+ release,
+ window_from,
+ at,
+ action_seq,
+ } = paused;
+ delivery.excluded = window_from..self.world.gmail.lock().log.len() as u64;
+ if at == PauseAt::AfterApply {
+ delivery.applied_before = Some(action_seq);
+ }
+ let _ = release.send(());
+ task.await.expect("worker task panicked");
+ self.finish_delivery(delivery);
+ self.absorb_sends(None);
+ }
+
+ pub async fn sync(&mut self, message: usize) {
+ self.run_step(Step::Sync {
+ message,
+ reply: Reply::Ok,
+ })
+ .await;
+ }
+
+ pub async fn resync(&mut self, message: usize) {
+ self.run_step(Step::Resync {
+ message,
+ reply: Reply::Ok,
+ })
+ .await;
+ }
+
+ /// Runs one seed: random steps with checks after each, then quiescence.
+ pub async fn run(&mut self, config: Config) {
+ for index in 0..config.steps {
+ self.step_index = index;
+ let step = self.random_step(config);
+ let description = format!("{step:?}");
+ self.run_step(step).await;
+ self.check_step(&description).await;
+ if !self.violations.is_empty() {
+ return;
+ }
+ }
+ self.step_index = config.steps;
+ self.quiesce().await;
+ }
+
+ pub fn report(&self) -> String {
+ let mut out = String::new();
+ for violation in &self.violations {
+ let _ = writeln!(
+ out,
+ " [{}] step {}: {}",
+ violation.invariant, violation.step, violation.detail
+ );
+ }
+ let _ = writeln!(out, " trace:");
+ for line in &self.trace {
+ let _ = writeln!(out, " {line}");
+ }
+ out
+ }
+}
+
+fn short(body: &str) -> String {
+ let value: serde_json::Value = serde_json::from_str(body).unwrap_or_default();
+ let operation = &value["operation"];
+ let text = operation.to_string();
+ let text = text.replace(&"0".repeat(8), "");
+ if text.len() > 160 {
+ format!("{}…", &text[..160])
+ } else {
+ text
+ }
+}
diff --git a/services/email_service/src/pubsub/label_write_sim/sim/generator.rs b/services/email_service/src/pubsub/label_write_sim/sim/generator.rs
new file mode 100644
index 0000000000..918e48dea9
--- /dev/null
+++ b/services/email_service/src/pubsub/label_write_sim/sim/generator.rs
@@ -0,0 +1,119 @@
+//! Random steps: user actions, Gmail answers, deliveries, syncs and waits.
+
+use super::super::fake_gmail::{PauseAt, Reply};
+use super::super::world::{MESSAGES, THREADS};
+use super::{Action, Config, Sim, Step};
+use rand::Rng;
+
+impl Sim<'_> {
+ pub(super) fn random_reply(&mut self, ok: f64) -> Reply {
+ let roll: f64 = self.rng.random();
+ if roll < ok {
+ Reply::Ok
+ } else if roll < ok + (1.0 - ok) * 0.65 {
+ Reply::RateLimited
+ } else if roll < ok + (1.0 - ok) * 0.9 || std::env::var("SIM_NO_REJECT").is_ok() {
+ Reply::Unavailable
+ } else {
+ Reply::Rejected
+ }
+ }
+
+ pub(super) fn random_action(&mut self) -> Action {
+ let thread = self.rng.random_range(0..THREADS.len());
+ let value = self.rng.random_bool(0.5);
+ match self.rng.random_range(0..100) {
+ 0..35 => Action::Archive(thread, value),
+ 35..55 => Action::Read(thread, value),
+ 55..65 => Action::Star(thread, value),
+ 65..75 => Action::Label(thread, value),
+ 75..85 => Action::Trash(thread, value),
+ _ => {
+ let threads: Vec<usize> = (0..THREADS.len())
+ .filter(|_| self.rng.random_bool(0.6))
+ .collect();
+ if threads.is_empty() {
+ Action::Archive(thread, value)
+ } else {
+ Action::BulkArchive(threads, value)
+ }
+ }
+ }
+ }
+
+ pub(super) fn random_simple_step(&mut self) -> Step {
+ match self.rng.random_range(0..100) {
+ 0..32 => Step::Action {
+ action: self.random_action(),
+ enqueue_fails: self.rng.random_bool(0.08),
+ },
+ 32..70 => Step::Deliver {
+ reply: self.random_reply(0.45),
+ gate_refuses: self.rng.random_bool(0.1),
+ enqueue_fails: self.rng.random_bool(0.05),
+ },
+ 70..86 => Step::Sync {
+ message: self.rng.random_range(0..MESSAGES.len()),
+ reply: if self.rng.random_bool(0.85) {
+ Reply::Ok
+ } else {
+ Reply::RateLimited
+ },
+ },
+ 86..92 => Step::Resync {
+ message: self.rng.random_range(0..MESSAGES.len()),
+ reply: if self.rng.random_bool(0.9) {
+ Reply::Ok
+ } else {
+ Reply::RateLimited
+ },
+ },
+ _ => Step::Wait {
+ seconds: self.rng.random_range(1..400),
+ },
+ }
+ }
+
+ pub fn random_step(&mut self, config: Config) -> Step {
+ let roll = self.rng.random_range(0..1000);
+ if roll < 8 {
+ return Step::Expire;
+ }
+ if config.overlap && roll < 90 {
+ let first = match self.rng.random_range(0..3) {
+ 0 => Step::Sync {
+ message: self.rng.random_range(0..MESSAGES.len()),
+ reply: Reply::Ok,
+ },
+ 1 => Step::Resync {
+ message: self.rng.random_range(0..MESSAGES.len()),
+ reply: Reply::Ok,
+ },
+ _ => Step::Deliver {
+ reply: self.random_reply(0.6),
+ gate_refuses: false,
+ enqueue_fails: false,
+ },
+ };
+ let count = self.rng.random_range(1..=3);
+ let mut seconds = Vec::new();
+ while seconds.len() < count {
+ let step = self.random_simple_step();
+ if !matches!(step, Step::Wait { .. }) {
+ seconds.push(step);
+ }
+ }
+ let at = if self.rng.random_bool(0.5) {
+ PauseAt::BeforeApply
+ } else {
+ PauseAt::AfterApply
+ };
+ return Step::Overlap {
+ first: Box::new(first),
+ seconds,
+ at,
+ };
+ }
+ self.random_simple_step()
+ }
+}
diff --git a/services/email_service/src/pubsub/label_write_sim/sim/model.rs b/services/email_service/src/pubsub/label_write_sim/sim/model.rs
new file mode 100644
index 0000000000..59ffa9fd7d
--- /dev/null
+++ b/services/email_service/src/pubsub/label_write_sim/sim/model.rs
@@ -0,0 +1,324 @@
+//! The user's intent per message label, how each queued label write ended,
+//! and the invariants checked after every step.
+
+use super::super::fake_gmail::{Kind, Logged, Reply};
+use super::super::wiring;
+use super::super::world::{MESSAGES, TRACKED};
+use super::{EntryState, OpStatus, Sim, Step, Violation};
+use std::collections::{BTreeSet, HashMap};
+use uuid::Uuid;
+
+impl Sim<'_> {
+ pub(super) fn initial(message: &str, label: &str) -> bool {
+ MESSAGES
+ .iter()
+ .find(|(id, _, _)| *id == message)
+ .is_some_and(|(_, _, labels)| labels.contains(&label))
+ }
+
+ /// The user's last effective intent for a cell, or `None` when its latest
+ /// write dead-lettered.
+ pub(super) fn intent(&self, message: &str, label: &str) -> Option<bool> {
+ let entries = &self.cells[&(message.to_string(), label.to_string())];
+ let latest = entries
+ .iter()
+ .filter(|entry| entry.state != EntryState::Rejected)
+ .max_by_key(|entry| entry.seq);
+ match latest {
+ None => Some(Self::initial(message, label)),
+ Some(entry) if entry.state == EntryState::DeadLettered => None,
+ Some(entry) => Some(entry.value),
+ }
+ }
+
+ /// The newest non-rejected write on a cell.
+ pub(super) fn latest_seq(&self, message: &str, label: &str) -> Option<u64> {
+ self.cells[&(message.to_string(), label.to_string())]
+ .iter()
+ .filter(|entry| entry.state != EntryState::Rejected)
+ .map(|entry| entry.seq)
+ .max()
+ }
+
+ pub(super) fn violate(&mut self, invariant: &'static str, detail: String) {
+ self.violations.push(Violation {
+ invariant,
+ step: self.step_index,
+ detail,
+ });
+ }
+
+ pub(super) fn account_write(
+ &mut self,
+ lineage: u64,
+ log: &[Logged],
+ deleted: bool,
+ resent: bool,
+ applied_before: Option<u64>,
+ ) {
+ let track = self.ops[&lineage].clone();
+ let in_op: Vec<String> = track
+ .op
+ .messages
+ .iter()
+ .map(|(_, provider_id)| provider_id.clone())
+ .collect();
+ let writes: Vec<&Logged> = log
+ .iter()
+ .filter(|entry| {
+ entry.kind == Kind::Write
+ && entry.ids.iter().all(|id| in_op.contains(id))
+ && (if track.value {
+ entry.add.contains(&track.label)
+ } else {
+ entry.remove.contains(&track.label)
+ })
+ })
+ .collect();
+ self.stats.gmail_writes += writes.len() as u64;
+ for (index, write) in writes.iter().enumerate() {
+ if write.reply != Reply::Ok {
+ continue;
+ }
+ // Only the first write can be the one Gmail applied while later
+ // steps ran; any per-message write after it was sent later.
+ let cutoff = match index {
+ 0 => applied_before.unwrap_or(u64::MAX),
+ _ => u64::MAX,
+ };
+ for id in &write.ids {
+ let newer = self.cells[&(id.clone(), track.label.clone())]
+ .iter()
+ .filter(|entry| entry.state != EntryState::Rejected && entry.seq <= cutoff)
+ .max_by_key(|entry| entry.seq)
+ .map(|entry| (entry.seq, entry.value, entry.lineage));
+ if let Some((latest, value, newer_lineage)) = newer
+ && latest > track.seq
+ && value != track.value
+ {
+ // The newer write is still queued and will set Gmail
+ // again; sync keeps its local state meanwhile.
+ if self.ops[&newer_lineage].status == OpStatus::Queued {
+ self.stats.transient_stale_applies += 1;
+ continue;
+ }
+ self.violate(
+ "gmail-applied-stale-write",
+ format!(
+ "write from action {} set {id}/{} = {} at Gmail after newer action {latest}",
+ track.seq, track.label, track.value
+ ),
+ );
+ }
+ }
+ }
+ let accepted_now = writes
+ .iter()
+ .filter(|write| write.reply == Reply::Ok)
+ .flat_map(|write| write.ids.iter().cloned());
+ self.ops
+ .get_mut(&lineage)
+ .unwrap()
+ .accepted
+ .extend(accepted_now);
+ if !deleted || resent {
+ return;
+ }
+ let accepted = self.ops[&lineage].accepted.clone();
+ let sent_ids: BTreeSet<String> = writes
+ .iter()
+ .flat_map(|write| write.ids.iter().cloned())
+ .collect();
+ let outcome = if writes.iter().any(|write| write.reply == Reply::Rejected) {
+ OpStatus::Rejected
+ } else if writes.iter().any(|write| write.reply == Reply::Ok) {
+ OpStatus::Applied
+ } else if writes.is_empty() {
+ OpStatus::Skipped
+ } else {
+ self.violate(
+ "write-dropped",
+ format!(
+ "write from action {} deleted after only retryable Gmail errors",
+ track.seq
+ ),
+ );
+ OpStatus::Skipped
+ };
+ let superseded = |sim: &Self, id: &String| {
+ sim.latest_seq(id, &track.label)
+ .is_some_and(|latest| latest > track.seq)
+ };
+ if outcome == OpStatus::Rejected {
+ // A rejection gives the write up on every message Gmail hasn't
+ // accepted: the rejected one, those not sent yet and those
+ // waiting for a newer write.
+ let rejected: Vec<String> = in_op
+ .iter()
+ .filter(|id| !accepted.contains(*id))
+ .cloned()
+ .collect();
+ self.set_entries(lineage, &rejected, EntryState::Rejected);
+ }
+ if outcome == OpStatus::Skipped {
+ self.stats.skipped_superseded += 1;
+ }
+ for id in &in_op {
+ if sent_ids.contains(id) || accepted.contains(id) || outcome == OpStatus::Rejected {
+ continue;
+ }
+ if !superseded(self, id) {
+ self.violate(
+ "write-dropped",
+ format!(
+ "write from action {} finished without sending {id}/{}, and no newer action owns it",
+ track.seq, track.label
+ ),
+ );
+ }
+ }
+ self.ops.get_mut(&lineage).unwrap().status = outcome;
+ }
+
+ pub async fn check_step(&mut self, step: &str) {
+ let db = self.world.db_labels().await;
+ let gmail = self.world.gmail_labels();
+ let mut lost = BTreeSet::new();
+ for (message, _, _) in MESSAGES {
+ for label in TRACKED {
+ let Some(intent) = self.intent(message, label) else {
+ lost.insert((message.to_string(), label.to_string()));
+ continue;
+ };
+ let in_db = db[message].contains(label);
+ if in_db != intent {
+ let pending = gmail[message].contains(label) != intent;
+ let invariant = if pending {
+ "pending-write-reverted"
+ } else {
+ "db-diverged-from-intent"
+ };
+ self.violate(
+ invariant,
+ format!(
+ "after {step}: {message}/{label} db={in_db} intent={intent} gmail={}",
+ gmail[message].contains(label)
+ ),
+ );
+ }
+ }
+ }
+ self.lost_cells.extend(lost);
+ for problem in self.world.flag_mismatches().await {
+ self.violate("thread-flags-mismatch", format!("after {step}: {problem}"));
+ }
+ self.check_markers(step).await;
+ }
+
+ /// Every live marker belongs to a write still on its way to Gmail.
+ pub(super) async fn check_markers(&mut self, step: &str) {
+ let ids: Vec<Uuid> = self.world.message_ids.values().copied().collect();
+ let provider_by_db: HashMap<Uuid, String> = self
+ .world
+ .message_ids
+ .iter()
+ .map(|(provider, db)| (*db, provider.clone()))
+ .collect();
+ let queued_lineages: BTreeSet<u64> = self
+ .queued
+ .iter()
+ .filter_map(|queued| queued.lineage)
+ .collect();
+ for marker in wiring::markers(&self.world.db, &ids).await {
+ if !marker.live {
+ continue;
+ }
+ let provider_id = provider_by_db[&marker.message_id].clone();
+ let owner = self
+ .lineage_by_write
+ .get(&(marker.write_id, provider_id))
+ .copied();
+ let in_flight = owner.is_some_and(|lineage| queued_lineages.contains(&lineage));
+ if !in_flight {
+ let status = owner.map(|lineage| self.ops[&lineage].status);
+ if status == Some(OpStatus::DeadLettered) {
+ continue;
+ }
+ self.violate(
+ "stale-marker",
+ format!(
+ "after {step}: marker {}/{} owned by write {:?} whose op is {status:?}",
+ provider_by_db[&marker.message_id], marker.label, owner
+ ),
+ );
+ }
+ }
+ }
+
+ /// Gmail accepts everything, every queue drains, every message gets the
+ /// echo sync its Gmail change would trigger, and then Gmail, the database
+ /// and the user's last intent must agree.
+ pub async fn quiesce(&mut self) {
+ self.trace.push(format!("t={} quiesce", self.now));
+ self.drain(false).await;
+ for message in 0..MESSAGES.len() {
+ self.run_step(Step::Sync {
+ message,
+ reply: Reply::Ok,
+ })
+ .await;
+ }
+ self.drain(false).await;
+ self.check_step("quiesce").await;
+ let db = self.world.db_labels().await;
+ let gmail = self.world.gmail_labels();
+ for (message, _, _) in MESSAGES {
+ for label in TRACKED {
+ let Some(intent) = self.intent(message, label) else {
+ continue;
+ };
+ let at_gmail = gmail[message].contains(label);
+ if at_gmail != intent || db[message].contains(label) != at_gmail {
+ self.violate(
+ "no-convergence",
+ format!(
+ "{message}/{label}: intent={intent} gmail={at_gmail} db={}",
+ db[message].contains(label)
+ ),
+ );
+ }
+ }
+ }
+ let ids: Vec<Uuid> = self.world.message_ids.values().copied().collect();
+ let provider_by_db: HashMap<Uuid, String> = self
+ .world
+ .message_ids
+ .iter()
+ .map(|(provider, db)| (*db, provider.clone()))
+ .collect();
+ for marker in wiring::markers(&self.world.db, &ids).await {
+ let provider_id = provider_by_db[&marker.message_id].clone();
+ let owner = self
+ .lineage_by_write
+ .get(&(marker.write_id, provider_id))
+ .copied();
+ let dead =
+ owner.is_some_and(|lineage| self.ops[&lineage].status == OpStatus::DeadLettered);
+ if marker.live && !dead {
+ self.violate(
+ "marker-left-after-quiesce",
+ format!("{marker:?} owned by {owner:?}"),
+ );
+ }
+ }
+ for (lineage, track) in &self.ops {
+ if track.status == OpStatus::Queued {
+ self.violations.push(Violation {
+ invariant: "write-never-finished",
+ step: self.step_index,
+ detail: format!("lineage {lineage} from action {}", track.seq),
+ });
+ }
+ }
+ }
+}
diff --git a/services/email_service/src/pubsub/label_write_sim/sim/test.rs b/services/email_service/src/pubsub/label_write_sim/sim/test.rs
new file mode 100644
index 0000000000..f7b114a1ba
--- /dev/null
+++ b/services/email_service/src/pubsub/label_write_sim/sim/test.rs
@@ -0,0 +1,210 @@
+//! The seeded simulation. The smoke test runs a fixed set of seeds; the
+//! ignored shards run more, configured by environment variables:
+//!
+//! ```text
+//! SIM_SEEDS=4000 SIM_STEPS=30 SIM_OVERLAP=0 SIM_OUT=/tmp/sim-1.txt \
+//! cargo test -p email_service --lib label_write_sim::sim::test::randomized_shard_1 \
+//! -- --exact --ignored --nocapture
+//! ```
+//!
+//! `SIM_OVERLAP=1` holds one operation at its Gmail call while 1–3 other
+//! steps run. `SIM_BATCHES=1` has producers send label write batches instead
+//! of one operation per message. `SIM_NO_REJECT=1` turns permanent Gmail rejections into 503s.
+//! `SIM_START` picks the first seed, `SIM_TRACE=1` prints every seed's trace
+//! and `SIM_VERBOSE=1` the traces of violating seeds.
+
+use super::super::world::World;
+use super::{Config, Sim, Stats};
+use sqlx::PgPool;
+use std::collections::BTreeMap;
+use std::fmt::Write as _;
+
+fn env_number(name: &str, default: u64) -> u64 {
+ std::env::var(name)
+ .ok()
+ .and_then(|value| value.parse().ok())
+ .unwrap_or(default)
+}
+
+#[derive(Default)]
+struct Summary {
+ seeds: u64,
+ clean: u64,
+ by_invariant: BTreeMap<String, u64>,
+ by_step: BTreeMap<String, u64>,
+ examples: BTreeMap<String, Vec<(u64, String)>>,
+ violating: Vec<String>,
+ stats: Stats,
+ lost_cell_seeds: u64,
+}
+
+impl Summary {
+ fn add(&mut self, seed: u64, sim: &Sim<'_>) {
+ self.seeds += 1;
+ let stats = &sim.stats;
+ let total = &mut self.stats;
+ total.actions += stats.actions;
+ total.failed_actions += stats.failed_actions;
+ total.deliveries += stats.deliveries;
+ total.syncs += stats.syncs;
+ total.resyncs += stats.resyncs;
+ total.gmail_writes += stats.gmail_writes;
+ total.gmail_429 += stats.gmail_429;
+ total.gmail_5xx += stats.gmail_5xx;
+ total.gmail_400 += stats.gmail_400;
+ total.skipped_superseded += stats.skipped_superseded;
+ total.transient_stale_applies += stats.transient_stale_applies;
+ total.dead_letters += stats.dead_letters;
+ total.overlaps += stats.overlaps;
+ total.expiries += stats.expiries;
+ if !sim.lost_cells.is_empty() {
+ self.lost_cell_seeds += 1;
+ }
+ let Some(first) = sim.violations.first() else {
+ self.clean += 1;
+ return;
+ };
+ *self
+ .by_invariant
+ .entry(first.invariant.to_string())
+ .or_default() += 1;
+ *self
+ .by_step
+ .entry(format!("{} after {}", first.invariant, first.step_kind()))
+ .or_default() += 1;
+ let detail: String = first.detail.chars().take(240).collect();
+ self.violating.push(format!(
+ "seed {seed} [{}] step {}: {detail}",
+ first.invariant, first.step
+ ));
+ let examples = self
+ .examples
+ .entry(first.invariant.to_string())
+ .or_default();
+ if examples.len() < 3 {
+ examples.push((seed, sim.report()));
+ }
+ }
+
+ fn render(&self, config: Config, start: u64) -> String {
+ let mut out = String::new();
+ let _ = writeln!(
+ out,
+ "seeds={}..{} steps={} overlap={} batches={} clean={} violating={}",
+ start,
+ start + self.seeds,
+ config.steps,
+ config.overlap,
+ config.label_write_batches,
+ self.clean,
+ self.seeds - self.clean
+ );
+ let _ = writeln!(out, "stats: {:?}", self.stats);
+ let _ = writeln!(
+ out,
+ "seeds with a dead-lettered write: {}",
+ self.lost_cell_seeds
+ );
+ for (invariant, count) in &self.by_invariant {
+ let _ = writeln!(out, "first violation {invariant}: {count} seeds");
+ }
+ for (key, count) in &self.by_step {
+ let _ = writeln!(out, " {key}: {count}");
+ }
+ let _ = writeln!(out, "all violating seeds:");
+ for line in &self.violating {
+ let _ = writeln!(out, " {line}");
+ }
+ for (invariant, examples) in &self.examples {
+ for (seed, report) in examples {
+ let _ = writeln!(out, "--- example {invariant} seed {seed}\n{report}");
+ }
+ }
+ out
+ }
+}
+
+async fn run_seeds(pool: PgPool, shard: u32, start: u64, seeds: u64, config: Config) -> Summary {
+ let mut world = World::new(pool, shard, config.label_write_batches).await;
+ let mut summary = Summary::default();
+ for seed in start..start + seeds {
+ world.reset().await;
+ let mut sim = Sim::new(&mut world, seed);
+ sim.run(config).await;
+ summary.add(seed, &sim);
+ if std::env::var("SIM_TRACE").is_ok()
+ || (std::env::var("SIM_VERBOSE").is_ok() && !sim.violations.is_empty())
+ {
+ println!("seed {seed}:\n{}", sim.report());
+ }
+ }
+ summary
+}
+
+async fn fifty_random_seeds(pool: PgPool, shard: u32, label_write_batches: bool) {
+ let config = Config {
+ steps: 30,
+ overlap: false,
+ label_write_batches,
+ };
+ let summary = run_seeds(pool, shard, 1_000_000, 50, config).await;
+ assert_eq!(
+ summary.clean,
+ summary.seeds,
+ "{}",
+ summary.render(config, 1_000_000)
+ );
+}
+
+#[sqlx::test(migrator = "macro_db_migrator::MACRO_DB_MIGRATIONS")]
+async fn fifty_random_seeds_keep_gmail_the_database_and_the_user_in_agreement(pool: PgPool) {
+ fifty_random_seeds(pool, 50, false).await;
+}
+
+#[sqlx::test(migrator = "macro_db_migrator::MACRO_DB_MIGRATIONS")]
+async fn fifty_random_seeds_agree_with_label_write_batches(pool: PgPool) {
+ fifty_random_seeds(pool, 51, true).await;
+}
+
+async fn run_shard(pool: PgPool, shard: u32) {
+ let seeds = env_number("SIM_SEEDS", 50);
+ let start = env_number("SIM_START", u64::from(shard) * 1_000_000);
+ let config = Config {
+ steps: env_number("SIM_STEPS", 30) as usize,
+ overlap: env_number("SIM_OVERLAP", 0) == 1,
+ label_write_batches: env_number("SIM_BATCHES", 0) == 1,
+ };
+ let started = std::time::Instant::now();
+ let summary = run_seeds(pool, shard, start, seeds, config).await;
+ let rendered = summary.render(config, start);
+ println!("{rendered}elapsed={:?}", started.elapsed());
+ if let Ok(path) = std::env::var("SIM_OUT") {
+ std::fs::write(path, &rendered).unwrap();
+ }
+}
+
+// One test per shard: sqlx names each test's database after the test, so
+// parallel processes need different tests.
+#[sqlx::test(migrator = "macro_db_migrator::MACRO_DB_MIGRATIONS")]
+#[ignore = "seeded simulation, configured by SIM_* environment variables"]
+async fn randomized_shard_1(pool: PgPool) {
+ run_shard(pool, 1).await;
+}
+
+#[sqlx::test(migrator = "macro_db_migrator::MACRO_DB_MIGRATIONS")]
+#[ignore = "seeded simulation, configured by SIM_* environment variables"]
+async fn randomized_shard_2(pool: PgPool) {
+ run_shard(pool, 2).await;
+}
+
+#[sqlx::test(migrator = "macro_db_migrator::MACRO_DB_MIGRATIONS")]
+#[ignore = "seeded simulation, configured by SIM_* environment variables"]
+async fn randomized_shard_3(pool: PgPool) {
+ run_shard(pool, 3).await;
+}
+
+#[sqlx::test(migrator = "macro_db_migrator::MACRO_DB_MIGRATIONS")]
+#[ignore = "seeded simulation, configured by SIM_* environment variables"]
+async fn randomized_shard_4(pool: PgPool) {
+ run_shard(pool, 4).await;
+}
diff --git a/services/email_service/src/pubsub/label_write_sim/test.rs b/services/email_service/src/pubsub/label_write_sim/test.rs
new file mode 100644
index 0000000000..e100b21b64
--- /dev/null
+++ b/services/email_service/src/pubsub/label_write_sim/test.rs
@@ -0,0 +1,756 @@
+//! Deterministic races between Macro's Gmail label writes, Gmail's answers
+//! and inbox syncs, through the production worker and domain code.
+
+use super::fake_gmail::{Kind, PauseAt, Reply};
+use super::sim::{Action, Queue, Sim, Source};
+use super::wiring;
+use super::world::{
+ INBOX, MESSAGES, STARRED, THREADS, UNREAD, WORK, World, on_worker_stacks, owner,
+};
+use sqlx::PgPool;
+
+const T1: usize = 0;
+const T2: usize = 1;
+const T3: usize = 2;
+const M1A: usize = 0;
+const M1B: usize = 1;
+const M1C: usize = 2;
+const M2A: usize = 3;
+const M2B: usize = 4;
+const M3A: usize = 5;
+
+/// Defines `$name` as a module with the scenario run once for each label
+/// write format: one operation per message, which producers send by default,
+/// and batches. The two runs use different mailboxes, so they don't share the
+/// request gate's state.
+macro_rules! scenario {
+ ($name:ident, $shard:expr, |$world:ident| $body:block) => {
+ mod $name {
+ use super::*;
+
+ #[sqlx::test(migrator = "macro_db_migrator::MACRO_DB_MIGRATIONS")]
+ async fn single_message_writes(pool: PgPool) {
+ run(World::new(pool, $shard, false).await).await;
+ }
+
+ #[sqlx::test(migrator = "macro_db_migrator::MACRO_DB_MIGRATIONS")]
+ async fn batched_writes(pool: PgPool) {
+ run(World::new(pool, $shard + 100, true).await).await;
+ }
+
+ async fn run(mut $world: World) $body
+ }
+ };
+}
+
+async fn in_inbox(sim: &Sim<'_>, thread: usize) -> bool {
+ sim.world.thread_inbox_visible(THREADS[thread]).await
+}
+
+async fn db_has(sim: &Sim<'_>, message: usize, label: &str) -> bool {
+ sim.world.db_labels().await[MESSAGES[message].0].contains(label)
+}
+
+fn gmail_has(sim: &Sim<'_>, message: usize, label: &str) -> bool {
+ sim.world.gmail_labels()[MESSAGES[message].0].contains(label)
+}
+
+async fn live_markers(sim: &Sim<'_>) -> usize {
+ let ids: Vec<_> = sim.world.message_ids.values().copied().collect();
+ wiring::markers(&sim.world.db, &ids)
+ .await
+ .into_iter()
+ .filter(|marker| marker.live)
+ .count()
+}
+
+async fn sync_thread(sim: &mut Sim<'_>, thread: usize) {
+ for (index, (_, thread_id, _)) in MESSAGES.iter().enumerate() {
+ if *thread_id == THREADS[thread] {
+ sim.sync(index).await;
+ }
+ }
+}
+
+scenario!(
+ a_rate_limited_archive_survives_sync_and_resync_until_gmail_accepts_it,
+ 901,
+ |world| {
+ let sim = &mut Sim::new(&mut world, 1);
+ sim.act(Action::Archive(T1, true), false).await.unwrap();
+ sim.deliver_all(Queue::Ops, Reply::RateLimited).await;
+ let operations = if sim.world.label_write_batches { 1 } else { 3 };
+ assert_eq!(sim.queue_len(Queue::OpsRetry), operations);
+
+ sim.sync(M1A).await;
+ sim.resync(M1B).await;
+ assert!(gmail_has(sim, M1A, INBOX));
+ assert!(!in_inbox(sim, T1).await);
+ assert!(!db_has(sim, M1A, INBOX).await);
+ assert!(!db_has(sim, M1B, INBOX).await);
+
+ while sim.deliver_from(Queue::OpsRetry, Reply::Ok, false).await {}
+ sync_thread(sim, T1).await;
+ assert!([M1A, M1B, M1C].iter().all(|m| !gmail_has(sim, *m, INBOX)));
+ assert!(!in_inbox(sim, T1).await);
+ assert_eq!(live_markers(sim).await, 0);
+ assert!(sim.world.flag_mismatches().await.is_empty());
+ }
+);
+
+scenario!(
+ an_old_archive_retried_after_a_newer_unarchive_sends_nothing,
+ 902,
+ |world| {
+ let sim = &mut Sim::new(&mut world, 1);
+ sim.act(Action::Archive(T1, true), false).await.unwrap();
+ sim.deliver_all(Queue::Ops, Reply::RateLimited).await;
+ sim.act(Action::Archive(T1, false), false).await.unwrap();
+ sim.deliver_all(Queue::Ops, Reply::Ok).await;
+
+ let requests_before = sim.world.gmail.lock().log.len();
+ while sim.deliver_from(Queue::OpsRetry, Reply::Ok, false).await {}
+ assert_eq!(sim.world.gmail.lock().log.len(), requests_before);
+ sync_thread(sim, T1).await;
+ assert!(gmail_has(sim, M1A, INBOX));
+ assert!(in_inbox(sim, T1).await);
+ }
+);
+
+scenario!(
+ an_older_star_rejected_after_unstar_and_star_keeps_the_last_star,
+ 903,
+ |world| {
+ let sim = &mut Sim::new(&mut world, 1);
+ sim.act(Action::Star(T3, true), false).await.unwrap();
+ sim.deliver_all(Queue::Ops, Reply::RateLimited).await;
+ sim.act(Action::Star(T3, false), false).await.unwrap();
+ sim.act(Action::Star(T3, true), false).await.unwrap();
+ sim.deliver_all(Queue::Ops, Reply::Ok).await;
+ while sim
+ .deliver_from(Queue::OpsRetry, Reply::Rejected, false)
+ .await
+ {}
+
+ sim.sync(M3A).await;
+ assert!(db_has(sim, M3A, STARRED).await);
+ assert!(gmail_has(sim, M3A, STARRED));
+ assert!(sim.world.flag_mismatches().await.is_empty());
+ }
+);
+
+scenario!(
+ an_unarchive_that_cannot_be_queued_leaves_the_queued_archive_in_charge,
+ 904,
+ |world| {
+ let sim = &mut Sim::new(&mut world, 1);
+ sim.act(Action::Archive(T1, true), false).await.unwrap();
+ sim.deliver_all(Queue::Ops, Reply::RateLimited).await;
+ assert!(sim.act(Action::Archive(T1, false), true).await.is_err());
+ assert!(!in_inbox(sim, T1).await);
+ assert_eq!(live_markers(sim).await, 3);
+
+ sim.sync(M1A).await;
+ sim.resync(M1B).await;
+ assert!(!in_inbox(sim, T1).await);
+ while sim.deliver_from(Queue::OpsRetry, Reply::Ok, false).await {}
+ sync_thread(sim, T1).await;
+ assert!(!gmail_has(sim, M1A, INBOX));
+ assert!(!in_inbox(sim, T1).await);
+ }
+);
+
+/// Gmail answers a read before the archive lands, the archive lands and
+/// clears its marker, the echo sync runs, and only then does the stale
+/// answer arrive.
+async fn stale_fetch_after_the_write_lands(sim: &mut Sim<'_>, resync: bool) {
+ sim.act(Action::Archive(T1, true), false).await.unwrap();
+ sim.deliver_all(Queue::Ops, Reply::RateLimited).await;
+ let (source, kind) = if resync {
+ (Source::Resync(M1A), Kind::ReadFull)
+ } else {
+ (Source::Sync(M1A), Kind::ReadLabels)
+ };
+ let paused = sim
+ .start_paused(source, kind, PauseAt::AfterApply)
+ .await
+ .unwrap();
+ while sim.deliver_from(Queue::OpsRetry, Reply::Ok, false).await {}
+ sim.sync(M1A).await;
+ assert!(!db_has(sim, M1A, INBOX).await);
+ sim.finish_paused(paused).await;
+
+ assert!(!db_has(sim, M1A, INBOX).await);
+ assert!(!in_inbox(sim, T1).await);
+ assert!(!gmail_has(sim, M1A, INBOX));
+}
+
+scenario!(
+ a_stale_label_fetch_does_not_bring_back_an_archive_that_landed_meanwhile,
+ 905,
+ |world| {
+ stale_fetch_after_the_write_lands(&mut Sim::new(&mut world, 1), false).await;
+ }
+);
+
+scenario!(
+ a_stale_resync_does_not_bring_back_an_archive_that_landed_meanwhile,
+ 906,
+ |world| {
+ stale_fetch_after_the_write_lands(&mut Sim::new(&mut world, 1), true).await;
+ }
+);
+
+scenario!(
+ a_resync_with_no_gmail_labels_removes_all_but_the_pending_ones,
+ 907,
+ |world| {
+ let sim = &mut Sim::new(&mut world, 1);
+ sim.act(Action::Read(T3, true), false).await.unwrap();
+ sim.deliver_all(Queue::Ops, Reply::RateLimited).await;
+ // In Gmail's own UI the user archives, unlabels and reads it.
+ sim.world
+ .gmail
+ .external_change("m3a", &[], &[INBOX, WORK, UNREAD]);
+
+ sim.resync(M3A).await;
+ assert!(!db_has(sim, M3A, INBOX).await);
+ assert!(!db_has(sim, M3A, WORK).await);
+ assert!(!db_has(sim, M3A, UNREAD).await);
+ assert!(!in_inbox(sim, T3).await);
+ assert!(sim.world.flag_mismatches().await.is_empty());
+ }
+);
+
+scenario!(
+ a_rejected_write_restores_the_labels_and_the_thread_state,
+ 908,
+ |world| {
+ let sim = &mut Sim::new(&mut world, 1);
+ sim.act(Action::Archive(T2, true), false).await.unwrap();
+ sim.deliver_all(Queue::Ops, Reply::Rejected).await;
+ assert!(db_has(sim, M2A, INBOX).await);
+ assert!(in_inbox(sim, T2).await);
+
+ sim.act(Action::Read(T2, true), false).await.unwrap();
+ sim.deliver_all(Queue::Ops, Reply::Rejected).await;
+ assert!(db_has(sim, M2A, UNREAD).await);
+ assert!(sim.world.flag_mismatches().await.is_empty());
+ }
+);
+
+scenario!(
+ a_rejected_unarchive_lets_the_older_archive_waiting_behind_it_reach_gmail,
+ 909,
+ |world| {
+ let sim = &mut Sim::new(&mut world, 1);
+ sim.act(Action::Archive(T3, true), false).await.unwrap();
+ sim.deliver_all(Queue::Ops, Reply::RateLimited).await;
+ sim.act(Action::Archive(T3, false), false).await.unwrap();
+ // The archive's retry runs while the unarchive is still queued: it waits.
+ let requests_before = sim.world.gmail.lock().log.len();
+ sim.deliver_from(Queue::OpsRetry, Reply::Ok, false).await;
+ assert_eq!(sim.world.gmail.lock().log.len(), requests_before);
+ assert_eq!(sim.queue_len(Queue::OpsRetry), 1);
+
+ sim.deliver_all(Queue::Ops, Reply::Rejected).await;
+ assert!(!in_inbox(sim, T3).await);
+ while sim.deliver_from(Queue::OpsRetry, Reply::Ok, false).await {}
+ assert!(!gmail_has(sim, M3A, INBOX));
+ assert_eq!(live_markers(sim).await, 0);
+ sim.sync(M3A).await;
+ assert!(!in_inbox(sim, T3).await);
+ }
+);
+
+scenario!(
+ a_failed_enqueue_and_a_rejection_keep_the_first_archive_they_took_over,
+ 910,
+ |world| {
+ let sim = &mut Sim::new(&mut world, 1);
+ sim.act(Action::Archive(T3, true), false).await.unwrap();
+ sim.deliver_all(Queue::Ops, Reply::RateLimited).await;
+ sim.act(Action::Archive(T3, false), false).await.unwrap();
+ assert!(sim.act(Action::Archive(T3, true), true).await.is_err());
+ sim.deliver_all(Queue::Ops, Reply::Rejected).await;
+ assert!(!in_inbox(sim, T3).await);
+
+ let requests_before = sim.world.gmail.lock().log.len();
+ while sim.deliver_from(Queue::OpsRetry, Reply::Ok, false).await {}
+ assert!(sim.world.gmail.lock().log.len() > requests_before);
+ assert!(!gmail_has(sim, M3A, INBOX));
+ sim.sync(M3A).await;
+ assert!(!in_inbox(sim, T3).await);
+ }
+);
+
+/// A star staged, sent and confirmed entirely while a sync's Gmail fetch is
+/// in flight.
+async fn star_inside_the_fetch(sim: &mut Sim<'_>, resync: bool) {
+ let (source, kind) = if resync {
+ (Source::Resync(M1A), Kind::ReadFull)
+ } else {
+ (Source::Sync(M1A), Kind::ReadLabels)
+ };
+ let paused = sim
+ .start_paused(source, kind, PauseAt::AfterApply)
+ .await
+ .unwrap();
+ sim.act(Action::Star(T1, true), false).await.unwrap();
+ sim.deliver_all(Queue::Ops, Reply::Ok).await;
+ sim.sync(M1A).await;
+ assert!(db_has(sim, M1A, STARRED).await);
+ sim.finish_paused(paused).await;
+
+ assert!(db_has(sim, M1A, STARRED).await);
+ assert!(gmail_has(sim, M1A, STARRED));
+ assert!(sim.world.flag_mismatches().await.is_empty());
+}
+
+scenario!(
+ a_resync_keeps_a_write_made_and_confirmed_during_its_fetch,
+ 911,
+ |world| {
+ star_inside_the_fetch(&mut Sim::new(&mut world, 1), true).await;
+ }
+);
+
+scenario!(
+ a_label_sync_keeps_a_write_made_and_confirmed_during_its_fetch,
+ 912,
+ |world| {
+ star_inside_the_fetch(&mut Sim::new(&mut world, 1), false).await;
+ }
+);
+
+scenario!(
+ a_recovery_backfill_does_not_bring_back_an_archive_that_landed_during_its_fetch,
+ 913,
+ |world| {
+ use models_email::email::service::backfill::{BackfillMessagePayload, JobScopedPayload};
+ let sim = &mut Sim::new(&mut world, 1);
+ sim.act(Action::Archive(T1, true), false).await.unwrap();
+ sim.deliver_all(Queue::Ops, Reply::RateLimited).await;
+ let ctx = sim.world.sync.clone();
+ let link = email_db_client::links::get::fetch_link_by_id(&ctx.db, sim.world.link_id)
+ .await
+ .unwrap()
+ .unwrap();
+ let scope = JobScopedPayload {
+ link_id: link.id,
+ job_id: uuid::Uuid::from_u128(0xb0),
+ payload: BackfillMessagePayload {
+ thread_provider_id: THREADS[T1].to_string(),
+ thread_db_id: sim.world.thread_ids[THREADS[T1]],
+ message_provider_id: MESSAGES[M1A].0.to_string(),
+ },
+ };
+ wiring::reset_admission(&mut sim.world.redis, sim.world.link_id).await;
+ let pause = sim
+ .world
+ .gmail
+ .pause_next(Kind::ReadFull, Some("m1a"), PauseAt::AfterApply);
+ let mut backfill = tokio::spawn(async move {
+ let _ =
+ crate::pubsub::backfill::backfill_message::backfill_message(&ctx, &scope, &link)
+ .await;
+ });
+ let arrived = tokio::select! {
+ _ = &mut backfill => false,
+ arrived = pause.arrived => arrived.is_ok(),
+ };
+ assert!(arrived, "the backfill's fetch reaches Gmail");
+ while sim.deliver_from(Queue::OpsRetry, Reply::Ok, false).await {}
+ sim.sync(M1A).await;
+ assert!(!db_has(sim, M1A, INBOX).await);
+ let _ = pause.release.send(());
+ backfill.await.unwrap();
+
+ assert!(!db_has(sim, M1A, INBOX).await);
+ assert!(!gmail_has(sim, M1A, INBOX));
+ assert!(sim.world.flag_mismatches().await.is_empty());
+ }
+);
+
+scenario!(
+ an_unqueued_mark_unread_reverts_without_undoing_a_newer_mark_read,
+ 914,
+ |world| {
+ use email::domain::models::mailbox_action::MailboxAction;
+ use email::domain::ports::EmailService;
+ let sim = &mut Sim::new(&mut world, 1);
+ let (arrived, release) = sim.world.sqs.pause_next_send();
+ let owner = owner();
+ let thread = sim.world.thread_ids[THREADS[T1]];
+ let email = sim.world.email.clone();
+ // Mark t1 unread; its enqueue is held, then fails.
+ let first = {
+ let email = email.clone();
+ let owner = owner.clone();
+ tokio::spawn(async move {
+ email
+ .change_thread_mailbox_state(owner, thread, MailboxAction::Read(false))
+ .await
+ })
+ };
+ arrived.await.unwrap();
+ // Meanwhile the user marks t1 read; this write is queued normally.
+ let second = email
+ .change_thread_mailbox_state(owner, thread, MailboxAction::Read(true))
+ .await;
+ sim.world.sqs.set_outage(true);
+ let _ = release.send(());
+ let first = first.await.unwrap();
+ sim.world.sqs.set_outage(false);
+
+ assert!(first.is_err());
+ assert!(second.is_ok());
+ for message in [M1A, M1B, M1C] {
+ assert!(!db_has(sim, message, UNREAD).await);
+ }
+ assert!(sim.world.flag_mismatches().await.is_empty());
+ }
+);
+
+/// With label write batches off, archiving a three-message thread queues one
+/// operation per message, all for the same write. Each settles only its own
+/// message: an accepted one clears its marker, a rejected one puts its
+/// message back, and one still queued keeps its message archived through a
+/// sync.
+#[sqlx::test(migrator = "macro_db_migrator::MACRO_DB_MIGRATIONS")]
+async fn single_message_writes_each_settle_their_own_message(pool: PgPool) {
+ let mut world = World::new(pool, 916, false).await;
+ let sim = &mut Sim::new(&mut world, 1);
+ sim.act(Action::Archive(T1, true), false).await.unwrap();
+ assert_eq!(sim.queue_len(Queue::Ops), 3);
+ assert_eq!(live_markers(sim).await, 3);
+
+ sim.deliver_from(Queue::Ops, Reply::Ok, false).await;
+ let accepted = only(
+ T1_MESSAGES
+ .into_iter()
+ .filter(|m| !gmail_has(sim, *m, INBOX)),
+ );
+ assert_eq!(live_markers(sim).await, 2);
+
+ sim.deliver_from(Queue::Ops, Reply::Rejected, false).await;
+ let mut back_in_inbox = Vec::new();
+ for message in T1_MESSAGES {
+ if db_has(sim, message, INBOX).await {
+ back_in_inbox.push(message);
+ }
+ }
+ let rejected = only(back_in_inbox.into_iter());
+ assert!(in_inbox(sim, T1).await);
+ assert_eq!(live_markers(sim).await, 1);
+
+ let queued = only(
+ T1_MESSAGES
+ .into_iter()
+ .filter(|m| *m != accepted && *m != rejected),
+ );
+ sim.sync(queued).await;
+ assert!(gmail_has(sim, queued, INBOX));
+ assert!(!db_has(sim, queued, INBOX).await);
+
+ sim.deliver_from(Queue::Ops, Reply::Ok, false).await;
+ sync_thread(sim, T1).await;
+ assert_eq!(live_markers(sim).await, 0);
+ for message in T1_MESSAGES {
+ assert_eq!(db_has(sim, message, INBOX).await, message == rejected);
+ assert_eq!(gmail_has(sim, message, INBOX), message == rejected);
+ }
+ assert!(sim.world.flag_mismatches().await.is_empty());
+}
+
+/// With batches on, a write to fewer than ten messages goes to Gmail one
+/// request per message. When Gmail rejects one after accepting another, only
+/// the messages it hasn't accepted go back, so the database already agrees
+/// with Gmail and no marker is left.
+#[sqlx::test(migrator = "macro_db_migrator::MACRO_DB_MIGRATIONS")]
+async fn a_rejection_partway_through_a_batch_keeps_what_gmail_accepted(pool: PgPool) {
+ let mut world = World::new(pool, 917, true).await;
+ let sim = &mut Sim::new(&mut world, 1);
+ sim.act(Action::Archive(T1, true), false).await.unwrap();
+ assert_eq!(sim.queue_len(Queue::Ops), 1);
+
+ sim.deliver_answering(Queue::Ops, &[Reply::Ok, Reply::Rejected], false)
+ .await;
+ let accepted = only(
+ T1_MESSAGES
+ .into_iter()
+ .filter(|m| !gmail_has(sim, *m, INBOX)),
+ );
+ assert_eq!(live_markers(sim).await, 0);
+ assert_t1_inbox_is_all_but(sim, accepted).await;
+
+ sync_thread(sim, T1).await;
+ assert_t1_inbox_is_all_but(sim, accepted).await;
+ assert!(in_inbox(sim, T1).await);
+ assert!(sim.world.flag_mismatches().await.is_empty());
+}
+
+/// Seeds 4000735 and 8000641: with batches on, Gmail accepts a write's first
+/// message and the next one is rate limited. The retry doesn't send the
+/// accepted message again, and when Gmail rejects it, only the messages Gmail
+/// never accepted go back.
+#[sqlx::test(migrator = "macro_db_migrator::MACRO_DB_MIGRATIONS")]
+async fn a_rejected_retry_keeps_the_messages_gmail_accepted_before_it(pool: PgPool) {
+ let mut world = World::new(pool, 918, true).await;
+ let sim = &mut Sim::new(&mut world, 1);
+ sim.act(Action::Archive(T1, true), false).await.unwrap();
+ sim.deliver_answering(Queue::Ops, &[Reply::Ok, Reply::RateLimited], false)
+ .await;
+ let accepted = only(
+ T1_MESSAGES
+ .into_iter()
+ .filter(|m| !gmail_has(sim, *m, INBOX)),
+ );
+ assert_eq!(live_markers(sim).await, 2);
+ assert_eq!(sim.queue_len(Queue::OpsRetry), 1);
+
+ let requests_before = sim.world.gmail.lock().log.len();
+ sim.deliver_answering(Queue::OpsRetry, &[Reply::Rejected], false)
+ .await;
+ let resent: Vec<String> = sim.world.gmail.lock().log[requests_before..]
+ .iter()
+ .flat_map(|request| request.ids.clone())
+ .collect();
+ assert!(!resent.contains(&MESSAGES[accepted].0.to_string()));
+ assert_eq!(live_markers(sim).await, 0);
+ assert_t1_inbox_is_all_but(sim, accepted).await;
+
+ sync_thread(sim, T1).await;
+ assert_t1_inbox_is_all_but(sim, accepted).await;
+ assert!(in_inbox(sim, T1).await);
+ assert!(sim.world.flag_mismatches().await.is_empty());
+}
+
+/// Seed 7000286: with batches on, an archive of a three-message thread is held
+/// after Gmail applies its first message. The user unarchives the thread and
+/// that write reaches Gmail first. When the archive resumes, it leaves the
+/// other two messages to the unarchive.
+#[sqlx::test(migrator = "macro_db_migrator::MACRO_DB_MIGRATIONS")]
+async fn a_resumed_older_write_skips_messages_a_newer_write_took_over(pool: PgPool) {
+ let mut world = World::new(pool, 919, true).await;
+ let sim = &mut Sim::new(&mut world, 1);
+ sim.act(Action::Archive(T1, true), false).await.unwrap();
+ let paused = sim
+ .start_paused(Source::Queued(Queue::Ops), Kind::Write, PauseAt::AfterApply)
+ .await
+ .unwrap();
+ sim.act(Action::Archive(T1, false), false).await.unwrap();
+ sim.deliver_all(Queue::Ops, Reply::Ok).await;
+ let requests_before = sim.world.gmail.lock().log.len();
+ sim.finish_paused(paused).await;
+
+ assert_eq!(sim.world.gmail.lock().log.len(), requests_before);
+ for message in T1_MESSAGES {
+ assert!(gmail_has(sim, message, INBOX));
+ assert!(db_has(sim, message, INBOX).await);
+ }
+ assert_eq!(live_markers(sim).await, 0);
+ sync_thread(sim, T1).await;
+ for message in T1_MESSAGES {
+ assert!(db_has(sim, message, INBOX).await);
+ }
+ assert!(in_inbox(sim, T1).await);
+ assert!(sim.world.flag_mismatches().await.is_empty());
+}
+
+/// Two bulk writes reach t1 and t2 in opposite orders, so each is the newer
+/// one on a thread: the archive on t2, the unarchive on t1. On t2 the
+/// archive drew the earlier id, as when requests race. The unarchive sends t1
+/// before it waits for the archive on t2, and when Gmail rejects the archive,
+/// t2 goes back to the unarchive, which then sends it.
+#[sqlx::test(migrator = "macro_db_migrator::MACRO_DB_MIGRATIONS")]
+async fn crossing_bulk_writes_each_send_where_they_are_newest_and_fall_back_on_rejection(
+ pool: PgPool,
+) {
+ use crate::pubsub::gmail_ops::operations::OperationError;
+ use crate::pubsub::gmail_ops::operations::modify_message_labels::modify_message_labels;
+ use email::domain::ports::EmailRepo;
+ use models_email::gmail::gmail_ops::{GmailMessageRef, ModifyMessageLabelsBatchPayload};
+ use uuid::Uuid;
+
+ let mut world = World::new(pool, 920, true).await;
+ let sim = &mut Sim::new(&mut world, 1);
+ sim.act(Action::Archive(T2, true), false).await.unwrap();
+ sim.deliver_all(Queue::Ops, Reply::Ok).await;
+
+ let archive = Uuid::from_u128(0xc1);
+ let unarchive = Uuid::from_u128(0xc2);
+ let repo = email::outbound::EmailPgRepo::new(sim.world.db.clone());
+ for (thread, add, write_id) in [
+ (T1, false, archive),
+ (T2, true, unarchive),
+ (T1, true, unarchive),
+ (T2, false, archive),
+ ] {
+ repo.stage_thread_label_change(
+ sim.world.thread_ids[THREADS[thread]],
+ sim.world.link_id,
+ INBOX,
+ add,
+ write_id,
+ )
+ .await
+ .unwrap()
+ .unwrap();
+ }
+ let write = |add: bool, write_id: Uuid, messages: [usize; 5]| {
+ let inbox = vec![INBOX.to_string()];
+ ModifyMessageLabelsBatchPayload {
+ messages: messages
+ .iter()
+ .map(|message| GmailMessageRef {
+ db_message_id: sim.world.message_ids[MESSAGES[*message].0],
+ provider_message_id: MESSAGES[*message].0.to_string(),
+ })
+ .collect(),
+ labels_to_add: if add { inbox.clone() } else { vec![] },
+ labels_to_remove: if add { vec![] } else { inbox },
+ pending_write_id: Some(write_id),
+ }
+ };
+ let archive = write(false, archive, [M1A, M1B, M1C, M2A, M2B]);
+ let unarchive = write(true, unarchive, [M2A, M2B, M1A, M1B, M1C]);
+ let link = email_db_client::links::get::fetch_link_by_id(&sim.world.db, sim.world.link_id)
+ .await
+ .unwrap()
+ .unwrap();
+
+ wiring::reset_admission(&mut sim.world.redis, sim.world.link_id).await;
+ let requests_before = sim.world.gmail.lock().log.len();
+ let result = modify_message_labels(&sim.world.ops, &link, &unarchive).await;
+ assert!(matches!(result, Err(OperationError::WaitingForNewerWrite)));
+ let sent: Vec<String> = sim.world.gmail.lock().log[requests_before..]
+ .iter()
+ .filter(|request| request.kind == Kind::Write)
+ .flat_map(|request| request.ids.clone())
+ .collect();
+ assert_eq!(sent, ["m1a", "m1b", "m1c"]);
+
+ wiring::reset_admission(&mut sim.world.redis, sim.world.link_id).await;
+ sim.world
+ .gmail
+ .lock()
+ .next_writes
+ .push_back(Reply::Rejected);
+ let result = modify_message_labels(&sim.world.ops, &link, &archive).await;
+ assert!(matches!(result, Err(OperationError::Gmail(_))));
+ assert!(in_inbox(sim, T2).await);
+
+ wiring::reset_admission(&mut sim.world.redis, sim.world.link_id).await;
+ modify_message_labels(&sim.world.ops, &link, &unarchive)
+ .await
+ .unwrap();
+ for message in [M1A, M1B, M1C, M2A, M2B] {
+ assert!(db_has(sim, message, INBOX).await);
+ assert!(gmail_has(sim, message, INBOX));
+ }
+ assert_eq!(live_markers(sim).await, 0);
+ assert!(sim.world.flag_mismatches().await.is_empty());
+}
+
+/// Every message of t1 but `archived` is in the inbox, in the database and at
+/// Gmail.
+async fn assert_t1_inbox_is_all_but(sim: &Sim<'_>, archived: usize) {
+ for message in T1_MESSAGES {
+ assert_eq!(db_has(sim, message, INBOX).await, message != archived);
+ assert_eq!(gmail_has(sim, message, INBOX), message != archived);
+ }
+}
+
+const T1_MESSAGES: [usize; 3] = [M1A, M1B, M1C];
+
+fn only(mut messages: impl Iterator<Item = usize>) -> usize {
+ let message = messages.next().expect("one message");
+ assert!(messages.next().is_none(), "exactly one message");
+ message
+}
+
+/// Local stacks run the email workers unoptimized on tokio's 2 MiB worker
+/// stacks. A label write and a label sync of the same mailbox, run at the same
+/// time, must fit well within that: a Gmail read passed by value once made the
+/// sync path overflow it.
+fn label_writes_and_a_label_sync_fit_on_small_worker_stacks(
+ pool: PgPool,
+ shard: u32,
+ label_write_batches: bool,
+) {
+ use aws_sdk_sqs::types::{Message, MessageSystemAttributeName};
+ use email::domain::ports::EmailService;
+ use models_email::gmail::inbox_sync::{
+ InboxSyncOperation, InboxSyncPubsubMessage, UpdateLabelsPayload,
+ };
+ let message = |id: String, body: String| {
+ Message::builder()
+ .receipt_handle(format!("{id}#1"))
+ .message_id(id)
+ .body(body)
+ .attributes(MessageSystemAttributeName::ApproximateReceiveCount, "1")
+ .build()
+ };
+ on_worker_stacks(&pool, 1536 << 10, move |pool| async move {
+ let mut world = World::new(pool, shard, label_write_batches).await;
+ for round in 0..3 {
+ world.reset().await;
+ world
+ .email
+ .set_thread_archived(owner(), world.thread_ids[THREADS[T1]], true)
+ .await
+ .unwrap();
+ let (sent, _) = world.sqs.take();
+ let writes: Vec<_> = sent
+ .into_iter()
+ .filter(|sent| sent.queue_url == world.queues.ops)
+ .enumerate()
+ .map(|(index, sent)| message(format!("write-{round}-{index}"), sent.body))
+ .collect();
+ let label_sync = serde_json::to_string(&InboxSyncPubsubMessage {
+ link_id: world.link_id,
+ operation: InboxSyncOperation::UpdateLabels(UpdateLabelsPayload {
+ provider_message_id: MESSAGES[M1A].0.to_string(),
+ }),
+ })
+ .unwrap();
+ let label_sync = message(format!("label-sync-{round}"), label_sync);
+ let sync = world.sync.clone();
+ let label_sync = tokio::spawn(async move {
+ crate::pubsub::inbox_sync::process::process_message(sync, &label_sync).await
+ });
+ let writes: Vec<_> = writes
+ .into_iter()
+ .map(|write| {
+ let ops = world.ops.clone();
+ tokio::spawn(async move {
+ crate::pubsub::gmail_ops::process::process_message(ops, &write).await
+ })
+ })
+ .collect();
+ for write in writes {
+ write.await.unwrap().unwrap();
+ }
+ label_sync.await.unwrap().unwrap();
+ }
+ assert!(!gmail_has_inbox_on_t1(&world));
+ });
+}
+
+#[sqlx::test(migrator = "macro_db_migrator::MACRO_DB_MIGRATIONS")]
+async fn single_message_writes_and_a_label_sync_fit_on_small_worker_stacks(pool: PgPool) {
+ label_writes_and_a_label_sync_fit_on_small_worker_stacks(pool, 915, false);
+}
+
+#[sqlx::test(migrator = "macro_db_migrator::MACRO_DB_MIGRATIONS")]
+async fn a_label_write_batch_and_a_label_sync_fit_on_small_worker_stacks(pool: PgPool) {
+ label_writes_and_a_label_sync_fit_on_small_worker_stacks(pool, 1015, true);
+}
+
+fn gmail_has_inbox_on_t1(world: &World) -> bool {
+ let gmail = world.gmail_labels();
+ [M1A, M1B, M1C]
+ .iter()
+ .any(|message| gmail[MESSAGES[*message].0].contains(INBOX))
+}
diff --git a/services/email_service/src/pubsub/label_write_sim/wiring.rs b/services/email_service/src/pubsub/label_write_sim/wiring.rs
new file mode 100644
index 0000000000..26eae5171a
--- /dev/null
+++ b/services/email_service/src/pubsub/label_write_sim/wiring.rs
@@ -0,0 +1,183 @@
+//! The production Gmail client and request gate wired to the fake Gmail, and
+//! direct access to the gate's Redis state, the pending label write markers
+//! and queued label writes.
+
+use crate::outbound::email_api::{
+ EmailServiceTokenSource, GmailApi, RateBudget, RedisGmailRequestGate,
+};
+use email::domain::ports::EmailService;
+use email_api_client::GmailApiClientRepository;
+use macro_user_id::user_id::MacroUserIdStr;
+use models_email::gmail::gmail_ops::{GmailOpsOperation, GmailOpsPubsubMessage};
+use sqlx::PgPool;
+use uuid::Uuid;
+
+pub fn redis_client(client: redis::Client) -> crate::util::redis::RedisClient {
+ crate::util::redis::RedisClient::new(client, 60)
+}
+
+pub async fn gmail_api(
+ db: PgPool,
+ redis: &redis::Client,
+ gmail_url: &str,
+ services_url: &str,
+ sqs_client: sqs_client::SQS,
+) -> GmailApi {
+ let client = gmail_client::GmailClient::new_with_urls(
+ Some("projects/sim/topics/gmail".to_string()),
+ gmail_url.to_string(),
+ format!("{services_url}/people"),
+ format!("{services_url}/certs"),
+ "sim".to_string(),
+ );
+ GmailApi::new(
+ GmailApiClientRepository::new(client),
+ EmailServiceTokenSource::new(
+ db,
+ redis.get_multiplexed_async_connection().await.unwrap(),
+ authentication_service_client::AuthServiceClient::new(
+ "secret".to_string(),
+ services_url.to_string(),
+ ),
+ sqs_client,
+ ),
+ RedisGmailRequestGate::new(
+ redis.get_connection_manager().await.unwrap(),
+ RateBudget::Live,
+ ),
+ )
+}
+
+fn gate_keys(link_id: Uuid) -> [String; 6] {
+ let scope = format!("gmail:{{{link_id}}}");
+ [
+ format!("{scope}:inflight"),
+ format!("{scope}:window"),
+ format!("{scope}:cooldown"),
+ format!("{scope}:waiting"),
+ format!("{scope}:backfill_cooldown"),
+ format!("{scope}:limit"),
+ ]
+}
+
+/// Clears the request gate's state, including the concurrency limit it learns
+/// from a Gmail 429, so each simulated step starts admitted regardless of
+/// wall-clock time.
+pub async fn reset_admission(redis: &mut redis::aio::MultiplexedConnection, link_id: Uuid) {
+ let _: () = redis::cmd("DEL")
+ .arg(&gate_keys(link_id))
+ .query_async(redis)
+ .await
+ .unwrap();
+}
+
+/// Makes the real gate refuse the next request locally, as after a Gmail 429
+/// on another worker.
+pub async fn refuse_admission(redis: &mut redis::aio::MultiplexedConnection, link_id: Uuid) {
+ let [_, _, cooldown, ..] = gate_keys(link_id);
+ let _: () = redis::cmd("SET")
+ .arg(cooldown)
+ .arg("1")
+ .arg("PX")
+ .arg(5_000)
+ .query_async(redis)
+ .await
+ .unwrap();
+}
+
+pub async fn clear_markers(db: &PgPool, message_ids: &[Uuid]) {
+ sqlx::query("DELETE FROM email_gmail_pending_label_writes WHERE message_id = ANY($1)")
+ .bind(message_ids)
+ .execute(db)
+ .await
+ .unwrap();
+}
+
+#[derive(Clone, Debug)]
+pub struct Marker {
+ pub message_id: Uuid,
+ pub label: String,
+ pub write_id: Uuid,
+ pub live: bool,
+}
+
+pub async fn markers(db: &PgPool, message_ids: &[Uuid]) -> Vec<Marker> {
+ let rows: Vec<(Uuid, String, Uuid, bool)> = sqlx::query_as(
+ "SELECT message_id, provider_label_id, write_id, expires_at > now()
+ FROM email_gmail_pending_label_writes WHERE message_id = ANY($1)
+ ORDER BY message_id, provider_label_id",
+ )
+ .bind(message_ids)
+ .fetch_all(db)
+ .await
+ .unwrap();
+ rows.into_iter()
+ .map(|(message_id, label, write_id, live)| Marker {
+ message_id,
+ label,
+ write_id,
+ live,
+ })
+ .collect()
+}
+
+/// Moves every marker 12 hours closer to its expiry.
+pub async fn age_markers(db: &PgPool, message_ids: &[Uuid]) {
+ sqlx::query(
+ "UPDATE email_gmail_pending_label_writes
+ SET expires_at = expires_at - interval '12 hours'
+ WHERE message_id = ANY($1)",
+ )
+ .bind(message_ids)
+ .execute(db)
+ .await
+ .unwrap();
+}
+
+/// A queued label write, decoded from its SQS body.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct LabelOp {
+ /// (database id, provider id) of each message the op sends.
+ pub messages: Vec<(Uuid, String)>,
+ pub add: Vec<String>,
+ pub remove: Vec<String>,
+ pub write_id: Option<Uuid>,
+}
+
+pub fn decode_label_op(body: &str) -> Option<LabelOp> {
+ let message: GmailOpsPubsubMessage = serde_json::from_str(body).ok()?;
+ match message.operation {
+ GmailOpsOperation::ModifyMessageLabels(payload) => Some(LabelOp {
+ messages: vec![(payload.db_message_id, payload.provider_message_id)],
+ add: payload.labels_to_add,
+ remove: payload.labels_to_remove,
+ write_id: payload.pending_write_id,
+ }),
+ GmailOpsOperation::ModifyMessageLabelsBatch(payload) => Some(LabelOp {
+ messages: payload
+ .messages
+ .into_iter()
+ .map(|message| (message.db_message_id, message.provider_message_id))
+ .collect(),
+ add: payload.labels_to_add,
+ remove: payload.labels_to_remove,
+ write_id: payload.pending_write_id,
+ }),
+ _ => None,
+ }
+}
+
+/// Bulk archive through the bulk route's service call. Returns the threads
+/// the service reported as not changed.
+pub async fn archive_many<S: EmailService>(
+ service: &S,
+ owner: MacroUserIdStr<'static>,
+ threads: Vec<Uuid>,
+ archived: bool,
+) -> Result<Vec<Uuid>, String> {
+ service
+ .set_threads_archived(owner, threads, archived)
+ .await
+ .map(|result| result.failed_thread_ids)
+ .map_err(|error| format!("{error:?}"))
+}
diff --git a/services/email_service/src/pubsub/label_write_sim/world.rs b/services/email_service/src/pubsub/label_write_sim/world.rs
new file mode 100644
index 0000000000..7809eaea02
--- /dev/null
+++ b/services/email_service/src/pubsub/label_write_sim/world.rs
@@ -0,0 +1,604 @@
+//! One Gmail mailbox wired to the production worker and domain code.
+//!
+//! Real: Postgres (migrated schema), Redis (request gate or limiter, token
+//! cache), the `GmailApi` service with its `GmailClient`, the gmail_ops and
+//! inbox_sync `process_message` entry points, and the `EmailServiceImpl` the
+//! HTTP routes use for archive, read, star, label and trash actions.
+//! Faked: Gmail's HTTP API, SQS's HTTP API, and catch-all 200 responses for
+//! the auth service, SFS, DSS and connection gateway.
+
+use super::fake_gmail::{FakeGmail, GmailMessage, GmailState};
+use super::fake_sqs::{self, FakeSqs};
+use super::wiring;
+use crate::pubsub::context::{CrmMetadataResolver, PubSubContext};
+use crate::pubsub::gmail_ops::worker::GmailOpsContext;
+use axum::Router;
+use axum::response::IntoResponse;
+use email::domain::service::EmailServiceImpl;
+use email::outbound::EmailPgRepo;
+use frecency::domain::services::FrecencyQueryServiceImpl;
+use frecency::outbound::postgres::FrecencyPgStorage;
+use macro_event_broker::{KafkaEventPublisher, MacroEventBrokerService};
+use macro_user_id::user_id::MacroUserIdStr;
+use models_email::email::service::cache::TokenCacheKey;
+use sqlx::PgPool;
+use std::collections::{BTreeMap, BTreeSet};
+use std::sync::Arc;
+use tokio_util::task::TaskTracker;
+use uuid::Uuid;
+
+pub const INBOX: &str = "INBOX";
+pub const UNREAD: &str = "UNREAD";
+pub const STARRED: &str = "STARRED";
+pub const TRASH: &str = "TRASH";
+pub const WORK: &str = "Label_work";
+
+pub const THREADS: [&str; 3] = ["t1", "t2", "t3"];
+/// Provider message ids, their thread, and their starting labels.
+pub const MESSAGES: [(&str, &str, &[&str]); 6] = [
+ ("m1a", "t1", &[INBOX, UNREAD]),
+ ("m1b", "t1", &[INBOX, STARRED]),
+ ("m1c", "t1", &[INBOX]),
+ ("m2a", "t2", &[INBOX, UNREAD]),
+ ("m2b", "t2", &[INBOX]),
+ ("m3a", "t3", &[INBOX, UNREAD, WORK]),
+];
+/// Labels the simulation tracks per message.
+pub const TRACKED: [&str; 5] = [INBOX, UNREAD, STARRED, TRASH, WORK];
+const LABELS: [(&str, &str, &str); 7] = [
+ ("INBOX", "INBOX", "system"),
+ ("UNREAD", "UNREAD", "system"),
+ ("STARRED", "STARRED", "system"),
+ ("TRASH", "TRASH", "system"),
+ ("SPAM", "SPAM", "system"),
+ ("SENT", "SENT", "system"),
+ ("Label_work", "Work", "user"),
+];
+
+pub type EmailSvc = EmailServiceImpl<
+ EmailPgRepo,
+ FrecencyQueryServiceImpl<FrecencyPgStorage>,
+ sqs_client::SQS,
+ crm::domain::service::CrmServiceImpl<
+ crm::outbound::companies_repo::CompaniesRepositoryImpl,
+ crm::outbound::no_op_resolver::NoOpCompanyMetadataResolver,
+ >,
+ entity_access_management::domain::service::EntityAccessManagementServiceImpl<
+ entity_access_management::outbound::PgRepository,
+ >,
+ MacroEventBrokerService<KafkaEventPublisher, TaskTracker>,
+>;
+
+pub struct Queues {
+ pub ops: String,
+ pub ops_retry: String,
+ pub sync: String,
+ pub sync_retry: String,
+}
+
+pub struct World {
+ pub db: PgPool,
+ pub gmail: FakeGmail,
+ pub sqs: FakeSqs,
+ pub redis: redis::aio::MultiplexedConnection,
+ pub link_id: Uuid,
+ pub queues: Queues,
+ pub ops: GmailOpsContext,
+ pub ops_retry: GmailOpsContext,
+ pub sync: PubSubContext,
+ pub sync_retry: PubSubContext,
+ pub email: EmailSvc,
+ /// Provider message id → database id.
+ pub message_ids: BTreeMap<String, Uuid>,
+ /// Provider thread id → database id.
+ pub thread_ids: BTreeMap<String, Uuid>,
+ pub label_ids: BTreeMap<String, Uuid>,
+ /// Whether producers send label writes as batches or, as they do by
+ /// default, one single-message operation per changed message.
+ pub label_write_batches: bool,
+}
+
+/// Runs `test` on a runtime of its own, connected to `pool`'s database, with
+/// two worker threads of `worker_stack` bytes of stack each, as the email
+/// workers have. Tasks `test` spawns run on those threads.
+pub fn on_worker_stacks<F, Fut>(pool: &PgPool, worker_stack: usize, test: F)
+where
+ F: FnOnce(PgPool) -> Fut + Send + 'static,
+ Fut: std::future::Future<Output = ()>,
+{
+ let options = pool.connect_options().as_ref().clone();
+ let outcome = std::thread::spawn(move || {
+ tokio::runtime::Builder::new_multi_thread()
+ .worker_threads(2)
+ .thread_stack_size(worker_stack)
+ .enable_all()
+ .build()
+ .expect("test runtime")
+ .block_on(async move {
+ let pool = sqlx::postgres::PgPoolOptions::new()
+ .connect_with(options)
+ .await
+ .expect("test database");
+ test(pool).await;
+ });
+ })
+ .join();
+ if let Err(panic) = outcome {
+ std::panic::resume_unwind(panic);
+ }
+}
+
+pub fn owner() -> MacroUserIdStr<'static> {
+ MacroUserIdStr::try_from_email("[email protected]").unwrap()
+}
+
+pub fn link_uuid(shard: u32) -> Uuid {
+ Uuid::from_u128(0x5100_0000_0000_0000_0000_0000_0000_0000 + u128::from(shard))
+}
+
+fn thread_uuid(shard: u32, index: usize) -> Uuid {
+ Uuid::from_u128(
+ 0x5200_0000_0000_0000_0000_0000_0000_0000 + (u128::from(shard) << 16) + index as u128,
+ )
+}
+
+fn message_uuid(shard: u32, index: usize) -> Uuid {
+ Uuid::from_u128(
+ 0x5300_0000_0000_0000_0000_0000_0000_0000 + (u128::from(shard) << 16) + index as u128,
+ )
+}
+
+fn label_uuid(shard: u32, index: usize) -> Uuid {
+ Uuid::from_u128(
+ 0x5400_0000_0000_0000_0000_0000_0000_0000 + (u128::from(shard) << 16) + index as u128,
+ )
+}
+
+pub fn initial_gmail() -> GmailState {
+ let mut state = GmailState::default();
+ state.history_id = 100;
+ state.labels = LABELS
+ .iter()
+ .map(|(id, name, kind)| (id.to_string(), name.to_string(), kind.to_string()))
+ .collect();
+ for (index, (id, thread, labels)) in MESSAGES.iter().enumerate() {
+ state.messages.insert(
+ id.to_string(),
+ GmailMessage {
+ thread_id: thread.to_string(),
+ labels: labels.iter().map(|label| label.to_string()).collect(),
+ internal_date_ms: 1_790_000_000_000 + index as i64 * 60_000,
+ },
+ );
+ }
+ state
+}
+
+impl World {
+ pub async fn new(db: PgPool, shard: u32, label_write_batches: bool) -> World {
+ let gmail = FakeGmail::start(initial_gmail()).await;
+ let sqs = FakeSqs::start().await;
+ let services_url = start_catch_all().await;
+ let link_id = link_uuid(shard);
+ let queues = Queues {
+ ops: sqs.queue_url("gmail-ops"),
+ ops_retry: sqs.queue_url("gmail-ops-retry"),
+ sync: sqs.queue_url("gmail-webhook"),
+ sync_retry: sqs.queue_url("gmail-webhook-retry"),
+ };
+ let mut world = World::compose(
+ db,
+ gmail,
+ sqs.clone(),
+ fake_sqs::client(&sqs.url, false),
+ &services_url,
+ link_id,
+ queues,
+ label_write_batches,
+ )
+ .await;
+ world.seed(shard).await;
+ world
+ }
+
+ /// Wires the production contexts to `gmail` and an SQS endpoint.
+ async fn compose(
+ db: PgPool,
+ gmail: FakeGmail,
+ sqs: FakeSqs,
+ sqs_sdk: aws_sdk_sqs::Client,
+ services_url: &str,
+ link_id: Uuid,
+ queues: Queues,
+ label_write_batches: bool,
+ ) -> World {
+ let redis_url =
+ std::env::var("REDIS_URI").unwrap_or_else(|_| "redis://127.0.0.1:6379".to_string());
+ let redis_client = redis::Client::open(redis_url).expect("redis url");
+ let redis = redis_client
+ .get_multiplexed_async_connection()
+ .await
+ .expect("redis connection");
+ let sqs_client = sqs_client::SQS::new(sqs_sdk.clone())
+ .gmail_inbox_sync_queue(&queues.sync)
+ .gmail_inbox_sync_retry_queue(&queues.sync_retry)
+ .gmail_ops_queue(&queues.ops)
+ .gmail_ops_retry_queue(&queues.ops_retry)
+ .gmail_label_write_batches(label_write_batches)
+ .email_backfill_queue(&format!("{services_url}/queue/backfill"))
+ .email_link_manager_queue(&format!("{services_url}/queue/link-manager"))
+ .sfs_uploader_queue(&format!("{services_url}/queue/sfs"));
+ let email_api = wiring::gmail_api(
+ db.clone(),
+ &redis_client,
+ &gmail.url,
+ services_url,
+ sqs_client.clone(),
+ )
+ .await;
+ let worker =
+ |queue: &str| sqs_worker::SQSWorker::new(sqs_sdk.clone(), queue.to_string(), 1, 1);
+ let ops_context = |retry_worker: bool, queue: &str| GmailOpsContext {
+ db: db.clone(),
+ sqs_worker: worker(queue),
+ sqs_client: sqs_client.clone(),
+ email_api: email_api.clone(),
+ retry_worker,
+ };
+ let broker = MacroEventBrokerService::new(
+ KafkaEventPublisher::new("127.0.0.1:9092").expect("kafka publisher"),
+ TaskTracker::new(),
+ );
+ let crm_service = crm::domain::service::CrmServiceImpl::new(
+ crm::outbound::companies_repo::CompaniesRepositoryImpl::new(db.clone()),
+ CrmMetadataResolver::Unfurl(
+ crm::outbound::unfurl_resolver::UnfurlCompanyMetadataResolver::new(Arc::new(
+ unfurl::domain::service::UnfurlServiceImpl::new(
+ unfurl::outbound::ReqwestUnfurlFetcher::new().expect("unfurl fetcher"),
+ ),
+ )),
+ ),
+ );
+ let sync_context = |retry_worker: bool, queue: &str| PubSubContext {
+ gmail_history: email::domain::mailbox::gmail_history::GmailHistoryService(
+ email::outbound::mailbox_pg::PgMailboxSync::new(db.clone()),
+ ),
+ invitation_extractor: crate::pubsub::invitation_extraction::compose(
+ db.clone(),
+ email_api.clone(),
+ ),
+ db: db.clone(),
+ sqs_worker: worker(queue),
+ sqs_client: sqs_client.clone(),
+ contacts_ingress: Arc::new(contacts::domain::service::SqsContactsIngress {
+ queue: contacts::outbound::ingress::SqsContactsQueue::new(
+ sqs_sdk.clone(),
+ format!("{services_url}/queue/contacts"),
+ ),
+ }),
+ email_api: email_api.clone(),
+ redis_client: wiring::redis_client(redis_client.clone()),
+ notification_ingress_service: Arc::new(
+ notification::domain::service::SqsNotificationIngress {
+ queue: notification::outbound::queue::SqsQueue::new(
+ sqs_sdk.clone(),
+ format!("{services_url}/queue/notifications"),
+ ),
+ },
+ ),
+ sfs_client: static_file_service_client::StaticFileServiceClient::new(
+ "key".to_string(),
+ services_url.to_string(),
+ ),
+ connection_gateway_client:
+ connection_gateway_client::client::ConnectionGatewayClient::new(
+ "key".to_string(),
+ services_url.to_string(),
+ ),
+ dss_client: document_storage_service_client::DocumentStorageServiceClient::new(
+ "key".to_string(),
+ services_url.to_string(),
+ ),
+ system_properties_service: Arc::new(
+ system_properties::SystemPropertiesServiceImpl::new(
+ system_properties::PgSystemPropertiesRepository::new(db.clone()),
+ ),
+ ),
+ crm_service: crm_service.clone(),
+ macro_event_broker: broker.clone(),
+ notifications_enabled: false,
+ retry_worker,
+ };
+ let email = EmailServiceImpl::new(
+ EmailPgRepo::new(db.clone()),
+ FrecencyQueryServiceImpl::new(FrecencyPgStorage::new(db.clone())),
+ sqs_client.clone(),
+ crm::domain::service::CrmServiceImpl::new(
+ crm::outbound::companies_repo::CompaniesRepositoryImpl::new(db.clone()),
+ crm::outbound::no_op_resolver::NoOpCompanyMetadataResolver,
+ ),
+ entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(
+ entity_access_management::outbound::PgRepository::new(db.clone()),
+ ),
+ 0,
+ )
+ .with_macro_event_broker(broker.clone());
+
+ World {
+ ops: ops_context(false, &queues.ops),
+ ops_retry: ops_context(true, &queues.ops_retry),
+ sync: sync_context(false, &queues.sync),
+ sync_retry: sync_context(true, &queues.sync_retry),
+ db,
+ gmail,
+ sqs,
+ redis,
+ link_id,
+ queues,
+ email,
+ message_ids: BTreeMap::new(),
+ thread_ids: BTreeMap::new(),
+ label_ids: BTreeMap::new(),
+ label_write_batches,
+ }
+ }
+
+ async fn seed(&mut self, shard: u32) {
+ let link = self.link_id;
+ sqlx::query(
+ "INSERT INTO email_links (id, macro_id, fusionauth_user_id, email_address, provider, is_sync_active, created_at, updated_at)
+ VALUES ($1, 'macro|[email protected]', $2, $3, 'GMAIL', true, NOW(), NOW())",
+ )
+ .bind(link)
+ .bind(format!("sim-user-{shard}"))
+ .bind(format!("sim{shard}@example.com"))
+ .execute(&self.db)
+ .await
+ .unwrap();
+ sqlx::query("INSERT INTO email_gmail_histories (link_id, history_id) VALUES ($1, '100')")
+ .bind(link)
+ .execute(&self.db)
+ .await
+ .unwrap();
+ for (index, (id, name, kind)) in LABELS.iter().enumerate() {
+ let label_id = label_uuid(shard, index);
+ let label_type = if *kind == "system" { "System" } else { "User" };
+ sqlx::query(&format!(
+ "INSERT INTO email_labels (id, link_id, provider_label_id, name, message_list_visibility, label_list_visibility, type, created_at)
+ VALUES ($1, $2, $3, $4, 'Show', 'LabelShow', '{label_type}', NOW())"
+ ))
+ .bind(label_id)
+ .bind(link)
+ .bind(id)
+ .bind(name)
+ .execute(&self.db)
+ .await
+ .unwrap();
+ self.label_ids.insert(id.to_string(), label_id);
+ }
+ for (index, thread) in THREADS.iter().enumerate() {
+ let thread_id = thread_uuid(shard, index);
+ sqlx::query(
+ "INSERT INTO email_threads (id, provider_id, link_id, inbox_visible, is_read, created_at, updated_at)
+ VALUES ($1, $2, $3, true, false, NOW(), NOW())",
+ )
+ .bind(thread_id)
+ .bind(thread)
+ .bind(link)
+ .execute(&self.db)
+ .await
+ .unwrap();
+ self.thread_ids.insert(thread.to_string(), thread_id);
+ }
+ for (index, (id, thread, labels)) in MESSAGES.iter().enumerate() {
+ let message_id = message_uuid(shard, index);
+ sqlx::query(
+ "INSERT INTO email_messages (id, thread_id, link_id, provider_id, provider_thread_id, is_read, is_starred, is_sent, is_draft, has_attachments, internal_date_ts, created_at, updated_at)
+ VALUES ($1, $2, $3, $4, $5, $6, $7, false, false, false, to_timestamp($8::double precision / 1000), NOW(), NOW())",
+ )
+ .bind(message_id)
+ .bind(self.thread_ids[*thread])
+ .bind(link)
+ .bind(id)
+ .bind(thread)
+ .bind(!labels.contains(&UNREAD))
+ .bind(labels.contains(&STARRED))
+ .bind((1_790_000_000_000 + index as i64 * 60_000) as f64)
+ .execute(&self.db)
+ .await
+ .unwrap();
+ self.message_ids.insert(id.to_string(), message_id);
+ }
+ self.reset().await;
+ }
+
+ /// Puts the mailbox, Gmail, queues and admission state back to the start.
+ pub async fn reset(&mut self) {
+ let message_ids: Vec<Uuid> = self.message_ids.values().copied().collect();
+ sqlx::query("DELETE FROM email_message_labels WHERE message_id = ANY($1)")
+ .bind(&message_ids)
+ .execute(&self.db)
+ .await
+ .unwrap();
+ wiring::clear_markers(&self.db, &message_ids).await;
+ for (id, _, labels) in MESSAGES {
+ let message_id = self.message_ids[id];
+ for label in labels {
+ sqlx::query(
+ "INSERT INTO email_message_labels (message_id, label_id) VALUES ($1, $2)",
+ )
+ .bind(message_id)
+ .bind(self.label_ids[*label])
+ .execute(&self.db)
+ .await
+ .unwrap();
+ }
+ sqlx::query("UPDATE email_messages SET is_read = $2, is_starred = $3 WHERE id = $1")
+ .bind(message_id)
+ .bind(!labels.contains(&UNREAD))
+ .bind(labels.contains(&STARRED))
+ .execute(&self.db)
+ .await
+ .unwrap();
+ }
+ for thread_id in self.thread_ids.values() {
+ let mut connection = self.db.acquire().await.unwrap();
+ email_db_client::threads::update::update_thread_metadata(
+ &mut connection,
+ *thread_id,
+ self.link_id,
+ )
+ .await
+ .unwrap();
+ }
+ sqlx::query("UPDATE email_gmail_histories SET history_id = '100' WHERE link_id = $1")
+ .bind(self.link_id)
+ .execute(&self.db)
+ .await
+ .unwrap();
+ *self.gmail.lock() = initial_gmail();
+ {
+ let mut sqs = self.sqs.lock();
+ sqs.sent.clear();
+ sqs.acks.clear();
+ sqs.failed_sends.clear();
+ sqs.outage = false;
+ }
+ self.seed_token().await;
+ wiring::reset_admission(&mut self.redis, self.link_id).await;
+ }
+
+ pub async fn seed_token(&mut self) {
+ let link = email_db_client::links::get::fetch_link_by_id(&self.db, self.link_id)
+ .await
+ .unwrap()
+ .unwrap();
+ let key = TokenCacheKey::new(
+ &link.fusionauth_user_id,
+ link.email_address.0.as_ref(),
+ link.provider.as_str(),
+ )
+ .to_redis_key();
+ let _: () = redis::cmd("SET")
+ .arg(key)
+ .arg("fake-access-token")
+ .arg("EX")
+ .arg(86_400)
+ .query_async(&mut self.redis)
+ .await
+ .unwrap();
+ }
+
+ /// Labels per provider message id in the database.
+ pub async fn db_labels(&self) -> BTreeMap<String, BTreeSet<String>> {
+ let rows: Vec<(String, String)> = sqlx::query_as(
+ "SELECT m.provider_id, l.provider_label_id
+ FROM email_messages m
+ JOIN email_message_labels ml ON ml.message_id = m.id
+ JOIN email_labels l ON l.id = ml.label_id
+ WHERE m.link_id = $1",
+ )
+ .bind(self.link_id)
+ .fetch_all(&self.db)
+ .await
+ .unwrap();
+ let mut labels: BTreeMap<String, BTreeSet<String>> = MESSAGES
+ .iter()
+ .map(|(id, _, _)| (id.to_string(), BTreeSet::new()))
+ .collect();
+ for (message, label) in rows {
+ labels.entry(message).or_default().insert(label);
+ }
+ labels
+ }
+
+ pub fn gmail_labels(&self) -> BTreeMap<String, BTreeSet<String>> {
+ self.gmail
+ .lock()
+ .messages
+ .iter()
+ .map(|(id, message)| (id.clone(), message.labels.clone()))
+ .collect()
+ }
+
+ /// Threads and messages whose stored flags disagree with their labels.
+ pub async fn flag_mismatches(&self) -> Vec<String> {
+ let mut problems = Vec::new();
+ let messages: Vec<(String, bool, bool, bool, bool)> = sqlx::query_as(
+ "SELECT m.provider_id, m.is_read, m.is_starred,
+ EXISTS (SELECT 1 FROM email_message_labels ml JOIN email_labels l ON l.id = ml.label_id
+ WHERE ml.message_id = m.id AND l.provider_label_id = 'UNREAD'),
+ EXISTS (SELECT 1 FROM email_message_labels ml JOIN email_labels l ON l.id = ml.label_id
+ WHERE ml.message_id = m.id AND l.provider_label_id = 'STARRED')
+ FROM email_messages m WHERE m.link_id = $1",
+ )
+ .bind(self.link_id)
+ .fetch_all(&self.db)
+ .await
+ .unwrap();
+ for (id, is_read, is_starred, unread, starred) in messages {
+ if is_read == unread {
+ problems.push(format!(
+ "message {id}: is_read={is_read} but UNREAD={unread}"
+ ));
+ }
+ if is_starred != starred {
+ problems.push(format!(
+ "message {id}: is_starred={is_starred} but STARRED={starred}"
+ ));
+ }
+ }
+ let threads: Vec<(String, bool, bool, bool, bool)> = sqlx::query_as(
+ "SELECT t.provider_id, t.inbox_visible, t.is_read,
+ COALESCE(bool_or((f.in_inbox AND NOT f.in_sent)), false),
+ COALESCE(bool_and(m.is_read), true)
+ FROM email_threads t
+ JOIN email_messages m ON m.thread_id = t.id
+ JOIN email_message_mailbox_facts f ON f.id = m.id
+ WHERE t.link_id = $1 AND f.is_present
+ GROUP BY t.id",
+ )
+ .bind(self.link_id)
+ .fetch_all(&self.db)
+ .await
+ .unwrap();
+ for (id, inbox_visible, is_read, expected_visible, expected_read) in threads {
+ if inbox_visible != expected_visible {
+ problems.push(format!(
+ "thread {id}: inbox_visible={inbox_visible}, messages say {expected_visible}"
+ ));
+ }
+ if is_read != expected_read {
+ problems.push(format!(
+ "thread {id}: is_read={is_read}, messages say {expected_read}"
+ ));
+ }
+ }
+ problems
+ }
+
+ pub async fn thread_inbox_visible(&self, thread: &str) -> bool {
+ sqlx::query_scalar("SELECT inbox_visible FROM email_threads WHERE id = $1")
+ .bind(self.thread_ids[thread])
+ .fetch_one(&self.db)
+ .await
+ .unwrap()
+ }
+}
+
+/// Answers 200 `{}` to everything: auth, SFS, DSS, connection gateway, and
+/// the SQS queues the simulation does not model.
+async fn start_catch_all() -> String {
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let url = format!("http://{}", listener.local_addr().unwrap());
+ let app = Router::new().fallback(|| async {
+ (
+ axum::http::StatusCode::OK,
+ [("content-type", "application/json")],
+ "{}",
+ )
+ .into_response()
+ });
+ tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
+ url
+}
diff --git a/services/email_service/src/pubsub/mod.rs b/services/email_service/src/pubsub/mod.rs
index 638ba412ab..7eaa5a3d9c 100644
--- a/services/email_service/src/pubsub/mod.rs
+++ b/services/email_service/src/pubsub/mod.rs
@@ -7,6 +7,8 @@ pub use context::CrmMetadataResolver;
pub mod crm_cleanup;
pub mod gmail_ops;
pub mod inbox_sync;
+#[cfg(test)]
+mod label_write_sim;
pub mod link_manager;
pub mod scheduled;
#[cfg(feature = "sfs_delete")]