Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(email): gate each page of Gmail history and contacts on its own (#8809)
teo66c5b8fe006c
diff --git a/crates/email_api_client/AGENTS.md b/crates/email_api_client/AGENTS.mdindex 67941df91a..ee8899c494 100644--- a/crates/email_api_client/AGENTS.md+++ b/crates/email_api_client/AGENTS.md@@ -69,7 +69,11 @@ Outlook (Microsoft Graph) outbound adapters.`call_with_token`; a call made with a bare token is invisible to thein-flight cap. Their spans declare `gate_in_flight` and `gate_cap` asempty fields, which admission fills with the mailbox's load so it sits next- to the call's outcome.+ to the call's outcome. Paginated reads (history, contacts) are page-level+ ports that the service loops over, one permit per page, so every page is+ charged to the mailbox's window and checks its cooldown; a refused page+ fails the read, and the span keeps the last page's load. Keep new paginated+ provider calls page-level rather than looping in the adapter.- Link-based reads go through `read_with_token`, which retries once with afresh token when the provider answers `AuthRequired`; a cached token canoutlive a revoked grant, and only the fresh fetch lets the token source markdiff --git a/crates/email_api_client/src/domain/models/changes.rs b/crates/email_api_client/src/domain/models/changes.rsindex 1d9a674654..c8e370cc43 100644--- a/crates/email_api_client/src/domain/models/changes.rs+++ b/crates/email_api_client/src/domain/models/changes.rs@@ -34,6 +34,55 @@ pub struct InboxChanges {pub labels_to_update: HashSet<String>,}+impl InboxChanges {+ /// Adds changes the provider recorded after these, with the same result+ /// as reading both in one pass: a later deletion drops an earlier upsert+ /// or label refresh, an earlier deletion blocks later ones, and an upsert+ /// supersedes a label refresh.+ pub fn append_later(&mut self, later: InboxChanges) {+ let earlier = std::mem::take(self);+ self.message_ids_to_upsert = earlier+ .message_ids_to_upsert+ .difference(&later.message_ids_to_delete)+ .chain(+ later+ .message_ids_to_upsert+ .difference(&earlier.message_ids_to_delete),+ )+ .cloned()+ .collect();+ self.labels_to_update = earlier+ .labels_to_update+ .iter()+ .filter(|id| {+ !later.message_ids_to_upsert.contains(*id)+ && !later.message_ids_to_delete.contains(*id)+ })+ .chain(later.labels_to_update.iter().filter(|id| {+ !earlier.message_ids_to_upsert.contains(*id)+ && !earlier.message_ids_to_delete.contains(*id)+ }))+ .cloned()+ .collect();+ self.message_ids_to_delete = earlier+ .message_ids_to_delete+ .union(&later.message_ids_to_delete)+ .cloned()+ .collect();+ }+}++/// One provider page of inbox changes following a cursor.+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct ChangePage {+ /// Changes recorded in this page.+ pub changes: InboxChanges,+ /// The mailbox's cursor when the page was read, if the provider reported one.+ pub cursor: Option<SyncCursor>,+ /// Opaque token for the next page, or `None` when this is the last page.+ pub next_page_token: Option<String>,+}+/// A page of inbox changes and the cursor that follows those changes.#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]pub struct ChangeBatch {diff --git a/crates/email_api_client/src/domain/models/mailbox.rs b/crates/email_api_client/src/domain/models/mailbox.rsindex 71f5b43580..2d48cfed94 100644--- a/crates/email_api_client/src/domain/models/mailbox.rs+++ b/crates/email_api_client/src/domain/models/mailbox.rs@@ -1,6 +1,7 @@use chrono::{DateTime, Utc};use models_email::email::service::message::Message;use models_email::email::service::thread::ThreadSummary;+use models_email::service::contact::Contact;use serde::{Deserialize, Serialize};use super::SyncCursor;@@ -57,3 +58,15 @@ pub struct ThreadListPage {/// Opaque token for the next page, or `None` when this is the last page.pub next_page_token: Option<String>,}++/// One provider page of contacts.+#[derive(Debug, Clone)]+pub struct ContactPage {+ /// Contacts returned in this page.+ pub contacts: Vec<Contact>,+ /// Token for the next incremental synchronization, which the provider+ /// returns with the last page.+ pub next_sync_token: Option<String>,+ /// Opaque token for the next page, or `None` when this is the last page.+ pub next_page_token: Option<String>,+}diff --git a/crates/email_api_client/src/domain/models/mod.rs b/crates/email_api_client/src/domain/models/mod.rsindex 398a57bd43..0ec5668b95 100644--- a/crates/email_api_client/src/domain/models/mod.rs+++ b/crates/email_api_client/src/domain/models/mod.rs@@ -14,10 +14,12 @@ mod send;pub use address_book::*;pub use auth::{AccessToken, TokenError, TokenFreshness};-pub use changes::{ChangeBatch, InboxChanges, SyncCursor};+pub use changes::{ChangeBatch, ChangePage, InboxChanges, SyncCursor};pub use draft::*;pub use error::{EmailApiError, RateLimitOrigin, RateLimitRefusal};-pub use mailbox::{CalendarPart, MessageWithCalendarParts, ProviderSubscription, ThreadListPage};+pub use mailbox::{+ CalendarPart, ContactPage, MessageWithCalendarParts, ProviderSubscription, ThreadListPage,+};pub use mailbox_settings::*;pub use operation::ApiOperationKind;pub use permit::{MailboxLoad, RequestPermit};diff --git a/crates/email_api_client/src/domain/ports.rs b/crates/email_api_client/src/domain/ports.rsindex 084620fa31..028f706d93 100644--- a/crates/email_api_client/src/domain/ports.rs+++ b/crates/email_api_client/src/domain/ports.rs@@ -6,12 +6,12 @@ pub use mailbox::*;use std::future::Future;use models_email::email::service::label::Label;-use models_email::service::contact::{Contact, ContactList};+use models_email::service::contact::Contact;use models_email::service::link::Link;use uuid::Uuid;use super::models::{- AccessToken, ApiOperationKind, CalendarPart, ChangeBatch, EmailApiError,+ AccessToken, ApiOperationKind, CalendarPart, ChangePage, ContactPage, EmailApiError,MessageWithCalendarParts, ProviderSubscription, RateLimitRefusal, RequestPermit, SendRequest,SentIds, SyncCursor, ThreadListPage, TokenError, TokenFreshness,};@@ -27,12 +27,14 @@ pub trait MailboxSyncClient: Send + Sync + 'static {access_token: &AccessToken,) -> impl Future<Output = Result<u64, EmailApiError>> + Send;- /// Returns mailbox changes that follow `cursor`.- fn list_changes(+ /// Returns one page of the mailbox changes that follow `cursor`. Pass the+ /// previous page's `next_page_token` to continue.+ fn list_changes_page(&self,access_token: &AccessToken,cursor: &SyncCursor,- ) -> impl Future<Output = Result<ChangeBatch, EmailApiError>> + Send;+ page_token: Option<&str>,+ ) -> impl Future<Output = Result<ChangePage, EmailApiError>> + Send;}/// Mailbox push-notification subscription capabilities.@@ -225,21 +227,24 @@ pub trait MailboxContactsClient: Send + Sync + 'static {link_id: Uuid,) -> impl Future<Output = Result<Contact, EmailApiError>> + Send;- /// Fetches and normalizes primary contacts from an optional synchronization token.- fn list_contacts(+ /// Fetches and normalizes one page of primary contacts from an optional+ /// synchronization token. Pass the previous page's `next_page_token` to continue.+ fn list_contacts_page(&self,access_token: &AccessToken,link_id: Uuid,sync_token: Option<&str>,- ) -> impl Future<Output = Result<ContactList, EmailApiError>> + Send;+ page_token: Option<&str>,+ ) -> impl Future<Output = Result<ContactPage, EmailApiError>> + Send;- /// Fetches and normalizes automatically collected contacts.- fn list_other_contacts(+ /// Fetches and normalizes one page of automatically collected contacts.+ fn list_other_contacts_page(&self,access_token: &AccessToken,link_id: Uuid,sync_token: Option<&str>,- ) -> impl Future<Output = Result<ContactList, EmailApiError>> + Send;+ page_token: Option<&str>,+ ) -> impl Future<Output = Result<ContactPage, EmailApiError>> + Send;}/// Blocked-sender management capabilities.@@ -380,11 +385,12 @@ impl MailboxSyncClient for NoOpMailboxClient {Err(unavailable())}- async fn list_changes(+ async fn list_changes_page(&self,_: &AccessToken,_: &SyncCursor,- ) -> Result<ChangeBatch, EmailApiError> {+ _: Option<&str>,+ ) -> Result<ChangePage, EmailApiError> {Err(unavailable())}}@@ -545,21 +551,23 @@ impl MailboxContactsClient for NoOpMailboxClient {Err(unavailable())}- async fn list_contacts(+ async fn list_contacts_page(&self,_: &AccessToken,_: Uuid,_: Option<&str>,- ) -> Result<ContactList, EmailApiError> {+ _: Option<&str>,+ ) -> Result<ContactPage, EmailApiError> {Err(unavailable())}- async fn list_other_contacts(+ async fn list_other_contacts_page(&self,_: &AccessToken,_: Uuid,_: Option<&str>,- ) -> Result<ContactList, EmailApiError> {+ _: Option<&str>,+ ) -> Result<ContactPage, EmailApiError> {Err(unavailable())}}diff --git a/crates/email_api_client/src/domain/service/contacts.rs b/crates/email_api_client/src/domain/service/contacts.rsindex ddd3ae3182..a5895ec16c 100644--- a/crates/email_api_client/src/domain/service/contacts.rs+++ b/crates/email_api_client/src/domain/service/contacts.rs@@ -1,7 +1,7 @@use models_email::service::contact::{Contact, ContactList};use uuid::Uuid;-use super::super::models::EmailApiError;+use super::super::models::{AccessToken, ContactPage, EmailApiError};use super::super::ports::{MailboxContactsClient, ProviderRequestGate, ProviderTokenSource};use super::{ApiOperationKind, EmailApiClientServiceImpl};@@ -26,43 +26,92 @@ where////// Failures log at WARN: an expired sync token ([`EmailApiError::OutdatedCursor`])/// or a grant without the contacts scopes is routine, and callers recover or skip.- #[tracing::instrument(skip(self, sync_token), fields(retry_count = 0, gate_in_flight = tracing::field::Empty, gate_cap = tracing::field::Empty), err(level = "warn"))]+ #[tracing::instrument(+ skip(self, sync_token),+ fields(+ retry_count = 0,+ pages = tracing::field::Empty,+ gate_in_flight = tracing::field::Empty,+ gate_cap = tracing::field::Empty+ ),+ err(level = "warn")+ )]pub async fn list_contacts(&self,link_id: Uuid,sync_token: Option<&str>,) -> Result<ContactList, EmailApiError> {- self.retry_with_fresh_token(- link_id,- ApiOperationKind::ListContacts,- |token| async move {- self.repository- .list_contacts(&token, link_id, sync_token)- .await- },- )- .await?+ self.list_contact_pages(link_id, |token, page_token| async move {+ self.repository+ .list_contacts_page(&token, link_id, sync_token, page_token.as_deref())+ .await+ })+ .await}/// Lists automatically collected contacts, optionally continuing a synchronization.////// Failures log at WARN for the same reasons as [`Self::list_contacts`].- #[tracing::instrument(skip(self, sync_token), fields(retry_count = 0, gate_in_flight = tracing::field::Empty, gate_cap = tracing::field::Empty), err(level = "warn"))]+ #[tracing::instrument(+ skip(self, sync_token),+ fields(+ retry_count = 0,+ pages = tracing::field::Empty,+ gate_in_flight = tracing::field::Empty,+ gate_cap = tracing::field::Empty+ ),+ err(level = "warn")+ )]pub async fn list_other_contacts(&self,link_id: Uuid,sync_token: Option<&str>,) -> Result<ContactList, EmailApiError> {- self.retry_with_fresh_token(- link_id,- ApiOperationKind::ListContacts,- |token| async move {- self.repository- .list_other_contacts(&token, link_id, sync_token)- .await- },- )- .await?+ self.list_contact_pages(link_id, |token, page_token| async move {+ self.repository+ .list_other_contacts_page(&token, link_id, sync_token, page_token.as_deref())+ .await+ })+ .await+ }++ /// Reads every page of a contact list. Each page is its own admission, and+ /// a refusal or failure on any page fails the whole read.+ async fn list_contact_pages<F, Fut>(+ &self,+ link_id: Uuid,+ read_page: F,+ ) -> Result<ContactList, EmailApiError>+ where+ F: Fn(AccessToken, Option<String>) -> Fut,+ Fut: Future<Output = Result<ContactPage, EmailApiError>>,+ {+ let mut contacts = Vec::new();+ let mut next_sync_token = None;+ let mut page_token: Option<String> = None;+ let mut pages = 0;+ loop {+ let page = self+ .retry_with_fresh_token(link_id, ApiOperationKind::ListContacts, |token| {+ read_page(token, page_token.clone())+ })+ .await??;+ pages += 1;+ contacts.extend(page.contacts);+ next_sync_token = page.next_sync_token.or(next_sync_token);+ page_token = page.next_page_token;+ if page_token.is_none() {+ break;+ }+ }+ tracing::Span::current().record("pages", pages);+ let next_sync_token = next_sync_token.ok_or_else(|| EmailApiError::Permanent {+ message: "the provider returned no sync token with the last contacts page".to_string(),+ })?;+ Ok(ContactList {+ contacts,+ next_sync_token,+ })}}diff --git a/crates/email_api_client/src/domain/service/contacts/test.rs b/crates/email_api_client/src/domain/service/contacts/test.rsindex 54df24189f..fb1b20aebf 100644--- a/crates/email_api_client/src/domain/service/contacts/test.rs+++ b/crates/email_api_client/src/domain/service/contacts/test.rs@@ -1,11 +1,13 @@use std::sync::{Arc, Mutex};-use models_email::service::contact::{Contact, ContactList};+use std::collections::VecDeque;++use models_email::service::contact::Contact;use uuid::Uuid;use super::super::test_support::{Call, FakeGate, FakeTokenSource, call_log};use super::*;-use crate::domain::models::{AccessToken, TokenFreshness};+use crate::domain::models::{AccessToken, ContactPage, TokenFreshness};#[derive(Debug, Clone, PartialEq, Eq)]enum ContactCall {@@ -30,12 +32,13 @@ impl MailboxContactsClient for ContactsClient {Err(expected_error())}- async fn list_contacts(+ async fn list_contacts_page(&self,access_token: &AccessToken,link_id: Uuid,sync_token: Option<&str>,- ) -> Result<ContactList, EmailApiError> {+ _: Option<&str>,+ ) -> Result<ContactPage, EmailApiError> {assert_eq!(access_token.expose_secret(), "token");*self.call.lock().unwrap() = Some(ContactCall::Contacts(link_id,@@ -44,12 +47,13 @@ impl MailboxContactsClient for ContactsClient {Err(expected_error())}- async fn list_other_contacts(+ async fn list_other_contacts_page(&self,access_token: &AccessToken,link_id: Uuid,sync_token: Option<&str>,- ) -> Result<ContactList, EmailApiError> {+ _: Option<&str>,+ ) -> Result<ContactPage, EmailApiError> {assert_eq!(access_token.expose_secret(), "token");*self.call.lock().unwrap() = Some(ContactCall::OtherContacts(link_id,@@ -106,24 +110,26 @@ impl MailboxContactsClient for RejectsCachedToken {unreachable!("not used by this test")}- async fn list_contacts(+ async fn list_contacts_page(&self,access_token: &AccessToken,_: Uuid,_: Option<&str>,- ) -> Result<ContactList, EmailApiError> {+ _: Option<&str>,+ ) -> Result<ContactPage, EmailApiError> {match access_token.expose_secret() {"cached" => Err(EmailApiError::AuthRequired),_ => Err(expected_error()),}}- async fn list_other_contacts(+ async fn list_other_contacts_page(&self,_: &AccessToken,_: Uuid,_: Option<&str>,- ) -> Result<ContactList, EmailApiError> {+ _: Option<&str>,+ ) -> Result<ContactPage, EmailApiError> {unreachable!("not used by this test")}}@@ -188,3 +194,132 @@ async fn contact_reads_use_correct_operation_kinds_and_forward_sync_tokens() {).await;}++/// Serves configured pages of other contacts and records the tokens it was given.+#[derive(Default)]+struct PagedContacts {+ pages: Mutex<VecDeque<ContactPage>>,+ requests: Mutex<Vec<(Option<String>, Option<String>)>>,+}++impl MailboxContactsClient for PagedContacts {+ async fn get_self_contact(&self, _: &AccessToken, _: Uuid) -> Result<Contact, EmailApiError> {+ unreachable!("not used by this test")+ }++ async fn list_contacts_page(+ &self,+ _: &AccessToken,+ _: Uuid,+ _: Option<&str>,+ _: Option<&str>,+ ) -> Result<ContactPage, EmailApiError> {+ unreachable!("not used by this test")+ }++ async fn list_other_contacts_page(+ &self,+ _: &AccessToken,+ _: Uuid,+ sync_token: Option<&str>,+ page_token: Option<&str>,+ ) -> Result<ContactPage, EmailApiError> {+ self.requests.lock().unwrap().push((+ sync_token.map(str::to_string),+ page_token.map(str::to_string),+ ));+ Ok(self+ .pages+ .lock()+ .unwrap()+ .pop_front()+ .expect("the test configures every page the service reads"))+ }+}++fn contact(link_id: Uuid, email: &str) -> Contact {+ Contact {+ id: Uuid::nil(),+ link_id,+ name: None,+ email_address: Some(email.to_string()),+ original_photo_url: None,+ sfs_photo_url: None,+ }+}++#[tokio::test]+async fn each_contacts_page_is_admitted_on_its_own_and_the_pages_are_combined() {+ let calls = call_log();+ let link_id = Uuid::new_v4();+ let repository = PagedContacts {+ pages: Mutex::new(VecDeque::from([+ ContactPage {+ contacts: vec![contact(link_id, "[email protected]")],+ next_sync_token: None,+ next_page_token: Some("page-2".into()),+ },+ ContactPage {+ contacts: vec![contact(link_id, "[email protected]")],+ next_sync_token: Some("new-sync".into()),+ next_page_token: None,+ },+ ])),+ ..PagedContacts::default()+ };+ let service = EmailApiClientServiceImpl::new(+ repository,+ FakeTokenSource::new(calls.clone(), Ok(AccessToken::new("token"))),+ FakeGate::new(calls.clone(), Ok(())),+ );++ let list = service+ .list_other_contacts(link_id, Some("old-sync"))+ .await+ .unwrap();++ assert_eq!(+ list.contacts+ .iter()+ .map(|contact| contact.email_address.as_deref())+ .collect::<Vec<_>>(),+ vec![Some("[email protected]"), Some("[email protected]")]+ );+ assert_eq!(list.next_sync_token, "new-sync");+ assert_eq!(+ *service.repository.requests.lock().unwrap(),+ vec![+ (Some("old-sync".into()), None),+ (Some("old-sync".into()), Some("page-2".into())),+ ]+ );+ let page = [+ Call::Acquire(link_id, ApiOperationKind::ListContacts),+ Call::Token(link_id, TokenFreshness::Cached),+ Call::Finish(link_id, None),+ ];+ assert_eq!(*calls.lock().unwrap(), [page.clone(), page].concat());+}++#[tokio::test]+async fn contacts_without_a_sync_token_on_the_last_page_fail() {+ let calls = call_log();+ let link_id = Uuid::new_v4();+ let service = EmailApiClientServiceImpl::new(+ PagedContacts {+ pages: Mutex::new(VecDeque::from([ContactPage {+ contacts: vec![],+ next_sync_token: None,+ next_page_token: None,+ }])),+ ..PagedContacts::default()+ },+ FakeTokenSource::new(calls.clone(), Ok(AccessToken::new("token"))),+ FakeGate::new(calls.clone(), Ok(())),+ );++ assert!(matches!(+ service.list_other_contacts(link_id, None).await,+ Err(EmailApiError::Permanent { .. })+ ));+}diff --git a/crates/email_api_client/src/domain/service/sync.rs b/crates/email_api_client/src/domain/service/sync.rsindex b5e5984a81..0e8dcee027 100644--- a/crates/email_api_client/src/domain/service/sync.rs+++ b/crates/email_api_client/src/domain/service/sync.rs@@ -1,6 +1,6 @@use uuid::Uuid;-use super::super::models::{ChangeBatch, EmailApiError, SyncCursor};+use super::super::models::{ChangeBatch, EmailApiError, InboxChanges, SyncCursor};use super::super::ports::{MailboxSyncClient, ProviderRequestGate, ProviderTokenSource};use super::{ApiOperationKind, EmailApiClientServiceImpl};@@ -20,16 +20,50 @@ where}/// Lists mailbox changes following `cursor`.- #[tracing::instrument(skip(self), fields(retry_count = 0, gate_in_flight = tracing::field::Empty, gate_cap = tracing::field::Empty))]+ ///+ /// Each page is its own admission, so it takes a permit, is charged to the+ /// mailbox's window and waits out a cooldown. A refusal or failure on any+ /// page fails the whole read; callers redeliver it from `cursor`.+ #[tracing::instrument(+ skip(self),+ fields(+ retry_count = 0,+ pages = tracing::field::Empty,+ gate_in_flight = tracing::field::Empty,+ gate_cap = tracing::field::Empty+ )+ )]pub async fn list_changes(&self,link_id: Uuid,cursor: &SyncCursor,) -> Result<ChangeBatch, EmailApiError> {- self.read_with_token(link_id, ApiOperationKind::ListChanges, |token| async move {- self.repository.list_changes(&token, cursor).await- })- .await+ let mut changes = InboxChanges::default();+ let mut next_cursor = None;+ let mut page_token: Option<String> = None;+ let mut pages = 0;+ loop {+ let page_token_ref = page_token.as_deref();+ let page = self+ .read_with_token(link_id, ApiOperationKind::ListChanges, |token| async move {+ self.repository+ .list_changes_page(&token, cursor, page_token_ref)+ .await+ })+ .await?;+ pages += 1;+ changes.append_later(page.changes);+ next_cursor = page.cursor.or(next_cursor);+ page_token = page.next_page_token;+ if page_token.is_none() {+ break;+ }+ }+ tracing::Span::current().record("pages", pages);+ Ok(ChangeBatch::new(+ changes,+ next_cursor.unwrap_or_else(|| cursor.clone()),+ ))}}diff --git a/crates/email_api_client/src/domain/service/sync/test.rs b/crates/email_api_client/src/domain/service/sync/test.rsindex 55523a4a5c..f517c68225 100644--- a/crates/email_api_client/src/domain/service/sync/test.rs+++ b/crates/email_api_client/src/domain/service/sync/test.rs@@ -1,14 +1,29 @@-use std::sync::{Arc, Mutex};+use std::collections::{HashSet, VecDeque};+use std::sync::Mutex;+use std::time::Duration;use uuid::Uuid;-use super::super::test_support::{Call, FakeGate, FakeTokenSource, call_log};+use super::super::test_support::{Call, CallLog, FakeGate, FakeTokenSource, call_log};use super::*;-use crate::domain::models::{AccessToken, InboxChanges, TokenFreshness};+use crate::domain::models::{+ AccessToken, ChangePage, InboxChanges, RateLimitOrigin, RateLimitRefusal, RequestPermit,+ TokenFreshness,+};-#[derive(Clone, Default)]+#[derive(Default)]struct SyncClient {- cursor: Arc<Mutex<Option<String>>>,+ pages: Mutex<VecDeque<ChangePage>>,+ requests: Mutex<Vec<(String, Option<String>)>>,+}++impl SyncClient {+ fn with_pages(pages: Vec<ChangePage>) -> Self {+ Self {+ pages: Mutex::new(pages.into()),+ requests: Mutex::default(),+ }+ }}impl MailboxSyncClient for SyncClient {@@ -17,23 +32,62 @@ impl MailboxSyncClient for SyncClient {Ok(42)}- async fn list_changes(+ async fn list_changes_page(&self,access_token: &AccessToken,cursor: &SyncCursor,- ) -> Result<ChangeBatch, EmailApiError> {+ page_token: Option<&str>,+ ) -> Result<ChangePage, EmailApiError> {assert_eq!(access_token.expose_secret(), "token");- *self.cursor.lock().unwrap() = Some(cursor.as_str().to_string());- Ok(ChangeBatch::new(- InboxChanges::default(),- SyncCursor::gmail("next"),- ))+ self.requests+ .lock()+ .unwrap()+ .push((cursor.as_str().to_string(), page_token.map(str::to_string)));+ Ok(self+ .pages+ .lock()+ .unwrap()+ .pop_front()+ .expect("the test configures every page the service reads"))+ }+}++/// Admits the first `admitted` requests and refuses the rest.+struct AdmitsFirst {+ calls: CallLog,+ admitted: Mutex<usize>,+}++impl ProviderRequestGate for AdmitsFirst {+ async fn acquire(+ &self,+ link_id: Uuid,+ operation: ApiOperationKind,+ ) -> Result<RequestPermit, RateLimitRefusal> {+ self.calls+ .lock()+ .unwrap()+ .push(Call::Acquire(link_id, operation));+ let mut admitted = self.admitted.lock().unwrap();+ if *admitted == 0 {+ return Err(RateLimitRefusal::new(Some(Duration::from_secs(1))));+ }+ *admitted -= 1;+ Ok(RequestPermit::new(link_id, Uuid::nil(), |_, _| {}))+ }++ async fn finish(&self, permit: RequestPermit, error: Option<&EmailApiError>) {+ self.calls+ .lock()+ .unwrap()+ .push(Call::Finish(permit.link_id(), error.cloned()));+ permit.disarm();}}fn service(repository: SyncClient,- calls: &super::super::test_support::CallLog,+ calls: &CallLog,) -> EmailApiClientServiceImpl<SyncClient, FakeTokenSource, FakeGate> {EmailApiClientServiceImpl::new(repository,@@ -42,6 +96,10 @@ fn service()}+fn ids(values: &[&str]) -> HashSet<String> {+ values.iter().map(|value| value.to_string()).collect()+}+#[tokio::test]async fn get_thread_count_uses_profile_quota_and_forwards_link_id() {let calls = call_log();@@ -66,17 +124,23 @@ async fn get_thread_count_uses_profile_quota_and_forwards_link_id() {#[tokio::test]async fn list_changes_uses_changes_quota_and_forwards_cursor() {let calls = call_log();- let repository = SyncClient::default();- let captured_cursor = repository.cursor.clone();+ let repository = SyncClient::with_pages(vec![ChangePage {+ changes: InboxChanges::default(),+ cursor: Some(SyncCursor::gmail("next")),+ next_page_token: None,+ }]);let link_id = Uuid::new_v4();- let cursor = SyncCursor::gmail("cursor");- let result = service(repository, &calls)- .list_changes(link_id, &cursor)+ let service = service(repository, &calls);+ let result = service+ .list_changes(link_id, &SyncCursor::gmail("cursor")).await;assert_eq!(result.unwrap().next_cursor, SyncCursor::gmail("next"));- assert_eq!(*captured_cursor.lock().unwrap(), Some("cursor".into()));+ assert_eq!(+ *service.repository.requests.lock().unwrap(),+ vec![("cursor".into(), None)]+ );assert_eq!(*calls.lock().unwrap(),vec![@@ -86,3 +150,115 @@ async fn list_changes_uses_changes_quota_and_forwards_cursor() {]);}++#[tokio::test]+async fn each_history_page_is_admitted_on_its_own_and_later_pages_win() {+ let calls = call_log();+ let repository = SyncClient::with_pages(vec![+ ChangePage {+ changes: InboxChanges {+ message_ids_to_upsert: ids(&["kept", "deleted-later"]),+ ..InboxChanges::default()+ },+ cursor: Some(SyncCursor::gmail("200")),+ next_page_token: Some("page-2".into()),+ },+ ChangePage {+ changes: InboxChanges {+ message_ids_to_delete: ids(&["deleted-later"]),+ labels_to_update: ids(&["relabelled"]),+ ..InboxChanges::default()+ },+ cursor: Some(SyncCursor::gmail("300")),+ next_page_token: Some("page-3".into()),+ },+ ChangePage {+ changes: InboxChanges::default(),+ cursor: None,+ next_page_token: None,+ },+ ]);+ let link_id = Uuid::new_v4();++ let service = service(repository, &calls);+ let batch = service+ .list_changes(link_id, &SyncCursor::gmail("100"))+ .await+ .unwrap();++ assert_eq!(+ batch,+ ChangeBatch::new(+ InboxChanges {+ message_ids_to_upsert: ids(&["kept"]),+ message_ids_to_delete: ids(&["deleted-later"]),+ labels_to_update: ids(&["relabelled"]),+ },+ SyncCursor::gmail("300"),+ )+ );+ assert_eq!(+ *service.repository.requests.lock().unwrap(),+ vec![+ ("100".into(), None),+ ("100".into(), Some("page-2".into())),+ ("100".into(), Some("page-3".into())),+ ]+ );+ let page = [+ Call::Acquire(link_id, ApiOperationKind::ListChanges),+ Call::Token(link_id, TokenFreshness::Cached),+ Call::Finish(link_id, None),+ ];+ assert_eq!(+ *calls.lock().unwrap(),+ [page.clone(), page.clone(), page].concat()+ );+}++#[tokio::test]+async fn a_refused_page_fails_the_read_so_it_is_redelivered() {+ let calls = call_log();+ let repository = SyncClient::with_pages(vec![ChangePage {+ changes: InboxChanges {+ message_ids_to_upsert: ids(&["first-page"]),+ ..InboxChanges::default()+ },+ cursor: Some(SyncCursor::gmail("200")),+ next_page_token: Some("page-2".into()),+ }]);+ let link_id = Uuid::new_v4();+ let service = EmailApiClientServiceImpl::new(+ repository,+ FakeTokenSource::new(calls.clone(), Ok(AccessToken::new("token"))),+ AdmitsFirst {+ calls: calls.clone(),+ admitted: Mutex::new(1),+ },+ );++ let result = service+ .list_changes(link_id, &SyncCursor::gmail("100"))+ .await;++ assert_eq!(+ result,+ Err(EmailApiError::RateLimited {+ retry_after: Some(Duration::from_secs(1)),+ origin: RateLimitOrigin::Local,+ })+ );+ assert_eq!(+ *service.repository.requests.lock().unwrap(),+ vec![("100".into(), None)]+ );+ assert_eq!(+ *calls.lock().unwrap(),+ vec![+ Call::Acquire(link_id, ApiOperationKind::ListChanges),+ Call::Token(link_id, TokenFreshness::Cached),+ Call::Finish(link_id, None),+ Call::Acquire(link_id, ApiOperationKind::ListChanges),+ ]+ );+}diff --git a/crates/email_api_client/src/domain/service/test_support.rs b/crates/email_api_client/src/domain/service/test_support.rsindex 34d6be531c..0c888fabbc 100644--- a/crates/email_api_client/src/domain/service/test_support.rs+++ b/crates/email_api_client/src/domain/service/test_support.rs@@ -1,11 +1,11 @@use std::sync::{Arc, Mutex};use models_email::email::service::label::Label;-use models_email::service::contact::{Contact, ContactList};+use models_email::service::contact::Contact;use uuid::Uuid;use super::super::models::{- AccessToken, ApiOperationKind, ChangeBatch, EmailApiError, ProviderSubscription,+ AccessToken, ApiOperationKind, ChangePage, ContactPage, EmailApiError, ProviderSubscription,RateLimitRefusal, RequestPermit, SendRequest, SentIds, SyncCursor, ThreadListPage, TokenError,TokenFreshness,};@@ -157,12 +157,13 @@ impl MailboxSyncClient for FakeRepository {self.unavailable("get_thread_count")}- async fn list_changes(+ async fn list_changes_page(&self,_: &AccessToken,_: &SyncCursor,- ) -> Result<ChangeBatch, EmailApiError> {- self.unavailable("list_changes")+ _: Option<&str>,+ ) -> Result<ChangePage, EmailApiError> {+ self.unavailable("list_changes_page")}}@@ -297,22 +298,24 @@ impl MailboxContactsClient for FakeRepository {self.unavailable("get_self_contact")}- async fn list_contacts(+ async fn list_contacts_page(&self,_: &AccessToken,_: Uuid,_: Option<&str>,- ) -> Result<ContactList, EmailApiError> {- self.unavailable("list_contacts")+ _: Option<&str>,+ ) -> Result<ContactPage, EmailApiError> {+ self.unavailable("list_contacts_page")}- async fn list_other_contacts(+ async fn list_other_contacts_page(&self,_: &AccessToken,_: Uuid,_: Option<&str>,- ) -> Result<ContactList, EmailApiError> {- self.unavailable("list_other_contacts")+ _: Option<&str>,+ ) -> Result<ContactPage, EmailApiError> {+ self.unavailable("list_other_contacts_page")}}diff --git a/crates/email_api_client/src/outbound/gmail/contacts.rs b/crates/email_api_client/src/outbound/gmail/contacts.rsindex 65ea815913..f416ffb8b6 100644--- a/crates/email_api_client/src/outbound/gmail/contacts.rs+++ b/crates/email_api_client/src/outbound/gmail/contacts.rs@@ -1,9 +1,9 @@//! Gmail contact capability implementation.-use models_email::service::contact::{Contact, ContactList};+use models_email::service::contact::Contact;use uuid::Uuid;-use crate::domain::models::{AccessToken, EmailApiError};+use crate::domain::models::{AccessToken, ContactPage, EmailApiError};use crate::domain::ports::MailboxContactsClient;use super::convert::map_person_to_contact;@@ -23,45 +23,59 @@ impl MailboxContactsClient for GmailApiClientRepository {Ok(map_person_to_contact(link_id, person))}- async fn list_contacts(+ async fn list_contacts_page(&self,access_token: &AccessToken,link_id: Uuid,sync_token: Option<&str>,- ) -> Result<ContactList, EmailApiError> {- let (people, next_sync_token) = self+ page_token: Option<&str>,+ ) -> Result<ContactPage, EmailApiError> {+ let page = self.client- .get_contacts(access_token.expose_secret(), sync_token)+ .get_contacts_page(access_token.expose_secret(), sync_token, page_token).await.map_err(map_contacts_error)?;- Ok(contact_list(link_id, people, next_sync_token))+ Ok(contact_page(+ link_id,+ page.connections,+ page.next_sync_token,+ page.next_page_token,+ ))}- async fn list_other_contacts(+ async fn list_other_contacts_page(&self,access_token: &AccessToken,link_id: Uuid,sync_token: Option<&str>,- ) -> Result<ContactList, EmailApiError> {- let (people, next_sync_token) = self+ page_token: Option<&str>,+ ) -> Result<ContactPage, EmailApiError> {+ let page = self.client- .get_other_contacts(access_token.expose_secret(), sync_token)+ .get_other_contacts_page(access_token.expose_secret(), sync_token, page_token).await.map_err(map_contacts_error)?;- Ok(contact_list(link_id, people, next_sync_token))+ Ok(contact_page(+ link_id,+ page.other_contacts,+ page.next_sync_token,+ page.next_page_token,+ ))}}-fn contact_list(+fn contact_page(link_id: Uuid,people: Vec<models_email::gmail::contacts::PersonResource>,- next_sync_token: String,-) -> ContactList {- ContactList {+ next_sync_token: Option<String>,+ next_page_token: Option<String>,+) -> ContactPage {+ ContactPage {contacts: people.into_iter().map(|person| map_person_to_contact(link_id, person)).collect(),next_sync_token,+ next_page_token,}}diff --git a/crates/email_api_client/src/outbound/gmail/convert/history.rs b/crates/email_api_client/src/outbound/gmail/convert/history.rsindex 7f75690751..9b0b0be880 100644--- a/crates/email_api_client/src/outbound/gmail/convert/history.rs+++ b/crates/email_api_client/src/outbound/gmail/convert/history.rs@@ -1,15 +1,19 @@use models_email::gmail::{HistoryListResponse, HistoryRecord};-use crate::domain::models::{ChangeBatch, InboxChanges, SyncCursor};+use crate::domain::models::{ChangePage, InboxChanges, SyncCursor};-pub(crate) fn map_history_list_response_to_changes(response: HistoryListResponse) -> ChangeBatch {+pub(crate) fn map_history_page(response: HistoryListResponse) -> ChangePage {let mut changes = InboxChanges::default();for record in response.history.unwrap_or_default() {apply_history_record(&mut changes, &record);}- ChangeBatch::new(changes, SyncCursor::gmail(response.history_id))+ ChangePage {+ changes,+ cursor: (!response.history_id.is_empty()).then(|| SyncCursor::gmail(response.history_id)),+ next_page_token: response.next_page_token,+ }}fn apply_history_record(changes: &mut InboxChanges, record: &HistoryRecord) {diff --git a/crates/email_api_client/src/outbound/gmail/convert/history/test.rs b/crates/email_api_client/src/outbound/gmail/convert/history/test.rsindex e74be161c4..162494fcfc 100644--- a/crates/email_api_client/src/outbound/gmail/convert/history/test.rs+++ b/crates/email_api_client/src/outbound/gmail/convert/history/test.rs@@ -2,7 +2,7 @@ use models_email::gmail::{HistoryListResponse, HistoryMessage, HistoryRecord, LabelAdded, MessageAdded, MessageDeleted,};-use super::map_history_list_response_to_changes;+use super::map_history_page;use crate::domain::models::SyncCursor;fn message(id: &str) -> HistoryMessage {@@ -12,6 +12,47 @@ fn message(id: &str) -> HistoryMessage {}}+fn page(records: &[HistoryRecord]) -> HistoryListResponse {+ HistoryListResponse {+ history: Some(records.to_vec()),+ next_page_token: None,+ history_id: "cursor".into(),+ }+}++fn record(messages: &[&str], added: &[&str], deleted: &[&str], labelled: &[&str]) -> HistoryRecord {+ HistoryRecord {+ id: "1".into(),+ messages: Some(messages.iter().map(|id| message(id)).collect()),+ messages_added: Some(+ added+ .iter()+ .map(|id| MessageAdded {+ message: message(id),+ })+ .collect(),+ ),+ messages_deleted: Some(+ deleted+ .iter()+ .map(|id| MessageDeleted {+ message: message(id),+ })+ .collect(),+ ),+ labels_added: Some(+ labelled+ .iter()+ .map(|id| LabelAdded {+ message: message(id),+ label_ids: vec!["STARRED".into()],+ })+ .collect(),+ ),+ labels_removed: None,+ }+}+#[test]fn resolves_overlapping_changes_with_deletion_precedence() {let response = HistoryListResponse {@@ -34,23 +75,64 @@ fn resolves_overlapping_changes_with_deletion_precedence() {}]),};- let batch = map_history_list_response_to_changes(response);+ let page = map_history_page(response);- assert_eq!(batch.next_cursor, SyncCursor::gmail("cursor-2"));- assert!(batch.changes.message_ids_to_upsert.contains("implicit"));- assert!(batch.changes.message_ids_to_delete.contains("deleted"));- assert!(!batch.changes.message_ids_to_upsert.contains("deleted"));- assert!(batch.changes.labels_to_update.contains("labels"));+ assert_eq!(page.cursor, Some(SyncCursor::gmail("cursor-2")));+ assert!(page.changes.message_ids_to_upsert.contains("implicit"));+ assert!(page.changes.message_ids_to_delete.contains("deleted"));+ assert!(!page.changes.message_ids_to_upsert.contains("deleted"));+ assert!(page.changes.labels_to_update.contains("labels"));}#[test]fn handles_an_empty_history_page() {- let batch = map_history_list_response_to_changes(HistoryListResponse {+ let page = map_history_page(HistoryListResponse {history: None,- next_page_token: None,+ next_page_token: Some("next".into()),history_id: "cursor".into(),});- assert!(batch.changes.message_ids_to_upsert.is_empty());- assert_eq!(batch.next_cursor.as_str(), "cursor");+ assert!(page.changes.message_ids_to_upsert.is_empty());+ assert_eq!(page.cursor, Some(SyncCursor::gmail("cursor")));+ assert_eq!(page.next_page_token.as_deref(), Some("next"));+}++#[test]+fn a_page_without_a_history_id_reports_no_cursor() {+ let page = map_history_page(HistoryListResponse {+ history: None,+ next_page_token: None,+ history_id: String::new(),+ });++ assert_eq!(page.cursor, None);+}++#[test]+fn appending_pages_matches_reading_the_history_in_one_pass() {+ // Each record interacts with earlier ones: deletions after upserts and+ // label changes, upserts and label changes after deletions, and upserts+ // after label changes.+ let records = [+ record(&[], &["a"], &[], &["b"]),+ record(&["c"], &[], &["a"], &["d"]),+ record(&[], &["a"], &[], &["c", "e"]),+ record(&["b"], &[], &["e", "f"], &[]),+ record(&[], &["g"], &[], &["f", "d"]),+ record(&["g"], &[], &["g"], &["h"]),+ ];+ let whole = map_history_page(page(&records)).changes;++ for first_split in 0..=records.len() {+ for second_split in first_split..=records.len() {+ let mut changes = map_history_page(page(&records[..first_split])).changes;+ changes+ .append_later(map_history_page(page(&records[first_split..second_split])).changes);+ changes.append_later(map_history_page(page(&records[second_split..])).changes);+ assert_eq!(+ changes, whole,+ "pages split after records {first_split} and {second_split}"+ );+ }+ }}diff --git a/crates/email_api_client/src/outbound/gmail/convert/mod.rs b/crates/email_api_client/src/outbound/gmail/convert/mod.rsindex 2d878c4613..0cafebcc49 100644--- a/crates/email_api_client/src/outbound/gmail/convert/mod.rs+++ b/crates/email_api_client/src/outbound/gmail/convert/mod.rs@@ -8,7 +8,7 @@ mod payload;mod thread;pub(crate) use contact::map_person_to_contact;-pub(crate) use history::map_history_list_response_to_changes;+pub(crate) use history::map_history_page;pub(crate) use label::{map_label_to_service, map_labels_to_service};pub(crate) use message::map_message_resource_to_service;pub(crate) use thread::map_thread_resource_to_service;diff --git a/crates/email_api_client/src/outbound/gmail/sync.rs b/crates/email_api_client/src/outbound/gmail/sync.rsindex 772a2dd789..4b0fec1461 100644--- a/crates/email_api_client/src/outbound/gmail/sync.rs+++ b/crates/email_api_client/src/outbound/gmail/sync.rs@@ -1,9 +1,9 @@//! Gmail mailbox-sync capability implementation.-use crate::domain::models::{AccessToken, ChangeBatch, EmailApiError, SyncCursor};+use crate::domain::models::{AccessToken, ChangePage, EmailApiError, SyncCursor};use crate::domain::ports::MailboxSyncClient;-use super::convert::map_history_list_response_to_changes;+use super::convert::map_history_page;use super::{GmailApiClientRepository, map_gmail_error, map_history_error};impl MailboxSyncClient for GmailApiClientRepository {@@ -21,16 +21,17 @@ impl MailboxSyncClient for GmailApiClientRepository {})}- async fn list_changes(+ async fn list_changes_page(&self,access_token: &AccessToken,cursor: &SyncCursor,- ) -> Result<ChangeBatch, EmailApiError> {+ page_token: Option<&str>,+ ) -> Result<ChangePage, EmailApiError> {let history = self.client- .get_history(access_token.expose_secret(), cursor.as_str())+ .get_history_page(access_token.expose_secret(), cursor.as_str(), page_token).await.map_err(map_history_error)?;- Ok(map_history_list_response_to_changes(history))+ Ok(map_history_page(history))}}diff --git a/crates/email_api_client/src/outbound/gmail/test/contacts.rs b/crates/email_api_client/src/outbound/gmail/test/contacts.rsindex f8a4da0582..736b7c4e06 100644--- a/crates/email_api_client/src/outbound/gmail/test/contacts.rs+++ b/crates/email_api_client/src/outbound/gmail/test/contacts.rs@@ -1,5 +1,5 @@use uuid::Uuid;-use wiremock::matchers::{method, path, query_param};+use wiremock::matchers::{method, path, query_param, query_param_is_missing};use wiremock::{Mock, ResponseTemplate};use super::repository;@@ -7,16 +7,17 @@ use crate::domain::models::AccessToken;use crate::domain::ports::MailboxContactsClient;#[tokio::test]-async fn paginates_contacts_and_returns_the_final_sync_token() {+async fn reads_contacts_one_page_at_a_time() {let (server, repository) = repository().await;Mock::given(method("GET")).and(path("/people/me/connections")).and(query_param("requestSyncToken", "true"))+ .and(query_param_is_missing("pageToken")).respond_with(ResponseTemplate::new(200).set_body_raw(include_str!("fixtures/contacts_page_1.json"),"application/json",))- .up_to_n_times(1)+ .expect(1).mount(&server).await;Mock::given(method("GET"))@@ -26,17 +27,28 @@ async fn paginates_contacts_and_returns_the_final_sync_token() {include_str!("fixtures/contacts_page_2.json"),"application/json",))+ .expect(1).mount(&server).await;+ let token = AccessToken::new("token");+ let link_id = Uuid::now_v7();++ let first = repository+ .list_contacts_page(&token, link_id, None, None)+ .await+ .unwrap();+ assert_eq!(first.contacts.len(), 1);+ assert_eq!(first.next_page_token.as_deref(), Some("contacts-next"));+ assert_eq!(first.next_sync_token, None);- let contacts = repository- .list_contacts(&AccessToken::new("token"), Uuid::now_v7(), None)+ let last = repository+ .list_contacts_page(&token, link_id, None, Some("contacts-next")).await.unwrap();- assert_eq!(contacts.contacts.len(), 2);- assert_eq!(contacts.next_sync_token, "contacts-sync-final");+ assert_eq!(last.next_page_token, None);+ assert_eq!(last.next_sync_token.as_deref(), Some("contacts-sync-final"));assert_eq!(- contacts.contacts[1].original_photo_url.as_deref(),+ last.contacts[0].original_photo_url.as_deref(),Some("https://lh3.googleusercontent.test/second=s128"));}@@ -74,9 +86,9 @@ async fn normalizes_self_and_other_contacts() {);let other = repository- .list_other_contacts(&AccessToken::new("token"), link_id, Some("previous"))+ .list_other_contacts_page(&AccessToken::new("token"), link_id, Some("previous"), None).await.unwrap();- assert_eq!(other.next_sync_token, "other-sync-final");+ assert_eq!(other.next_sync_token.as_deref(), Some("other-sync-final"));assert_eq!(other.contacts[0].link_id, link_id);}diff --git a/crates/email_api_client/src/outbound/gmail/test/sync.rs b/crates/email_api_client/src/outbound/gmail/test/sync.rsindex e767e3b6c7..505cd9449f 100644--- a/crates/email_api_client/src/outbound/gmail/test/sync.rs+++ b/crates/email_api_client/src/outbound/gmail/test/sync.rs@@ -1,4 +1,4 @@-use wiremock::matchers::{method, path, query_param};+use wiremock::matchers::{method, path, query_param, query_param_is_missing};use wiremock::{Mock, ResponseTemplate};use super::repository;@@ -27,37 +27,49 @@ async fn reads_profile_thread_count() {}#[tokio::test]-async fn paginates_and_partitions_history() {+async fn reads_and_partitions_one_history_page_at_a_time() {let (server, repository) = repository().await;Mock::given(method("GET")).and(path("/users/me/history")).and(query_param("startHistoryId", "100"))+ .and(query_param_is_missing("pageToken")).respond_with(ResponseTemplate::new(200).set_body_raw(include_str!("fixtures/history_page_1.json"),"application/json",))- .up_to_n_times(1)+ .expect(1).mount(&server).await;Mock::given(method("GET")).and(path("/users/me/history"))+ .and(query_param("startHistoryId", "100")).and(query_param("pageToken", "next-page")).respond_with(ResponseTemplate::new(200).set_body_raw(include_str!("fixtures/history_page_2.json"),"application/json",))+ .expect(1).mount(&server).await;+ let token = AccessToken::new("token");+ let cursor = SyncCursor::gmail("100");- let batch = repository- .list_changes(&AccessToken::new("token"), &SyncCursor::gmail("100"))+ let first = repository+ .list_changes_page(&token, &cursor, None).await.unwrap();+ assert_eq!(first.cursor, Some(SyncCursor::gmail("101")));+ assert_eq!(first.next_page_token.as_deref(), Some("next-page"));+ assert!(first.changes.message_ids_to_upsert.contains("added"));+ assert!(first.changes.message_ids_to_delete.contains("deleted"));- assert_eq!(batch.next_cursor.as_str(), "102");- assert!(batch.changes.message_ids_to_upsert.contains("added"));- assert!(batch.changes.message_ids_to_delete.contains("deleted"));- assert!(batch.changes.labels_to_update.contains("labels"));+ let last = repository+ .list_changes_page(&token, &cursor, Some("next-page"))+ .await+ .unwrap();+ assert_eq!(last.cursor, Some(SyncCursor::gmail("102")));+ assert_eq!(last.next_page_token, None);+ assert!(last.changes.labels_to_update.contains("labels"));}#[tokio::test]@@ -74,7 +86,11 @@ async fn only_history_not_found_is_an_outdated_cursor() {assert_eq!(repository- .list_changes(&AccessToken::new("token"), &SyncCursor::gmail("stale"))+ .list_changes_page(+ &AccessToken::new("token"),+ &SyncCursor::gmail("stale"),+ None+ ).await,Err(EmailApiError::OutdatedCursor));diff --git a/crates/gmail_client/AGENTS.md b/crates/gmail_client/AGENTS.mdindex 137f5bad3b..ad55373906 100644--- a/crates/gmail_client/AGENTS.md+++ b/crates/gmail_client/AGENTS.md@@ -45,6 +45,9 @@ contacts, watch) behind a single public facade; most modules are- Requests time out after 60 s (`REQUEST_TIMEOUT`). The email service'sGmail request gate holds a permit for each call and sizes its lease abovethis (`PERMIT_MS` in `gmail_gate.rs`); keep the two in step.+- History and contacts are fetched one page per call (`get_history_page`,+ `get_contacts_page`, `get_other_contacts_page`) so each page takes its own+ gate permit. Don't add methods that loop over pages here.## Relateddiff --git a/crates/gmail_client/src/contacts.rs b/crates/gmail_client/src/contacts.rsindex 2ba421a69c..b42d2f6c23 100644--- a/crates/gmail_client/src/contacts.rs+++ b/crates/gmail_client/src/contacts.rs@@ -26,119 +26,83 @@ pub(crate) async fn get_self_connection(decode_json_response(response).await}-/// Fetches all pages of the user's contacts and returns raw People resources.+/// Gets one page of the user's contacts as raw People resources.////// Failures log at WARN: expired sync tokens and links granted without the/// contacts scopes are routine, and the contacts sync recovers or skips them.-#[tracing::instrument(skip(client, access_token, sync_token), err(level = "warn"))]-pub(crate) async fn list_connections(+#[tracing::instrument(+ skip(client, access_token, sync_token, page_token),+ err(level = "warn")+)]+pub(crate) async fn get_connections_page(client: &GmailClient,access_token: &str,sync_token: Option<&str>,-) -> Result<(Vec<PersonResource>, String), GmailApiHttpError> {- let mut people = Vec::new();- let mut page_token: Option<String> = None;- let mut next_sync_token: Option<String> = None;-- loop {- let mut request = client- .inner- .get(format!("{}/people/me/connections", client.contacts_url))- .bearer_auth(access_token)- .query(&[- ("personFields", PERSON_FIELDS),- ("requestSyncToken", "true"),- ]);- if let Some(token) = sync_token {- request = request.query(&[("syncToken", token)]);- } else {- request = request.query(&[("pageSize", "1000")]);- }- if let Some(token) = page_token.as_deref() {- request = request.query(&[("pageToken", token)]);- }-- let response = request.send().await.map_err(GmailApiHttpError::transport)?;- if !response.status().is_success() {- return Err(unsuccessful_response(response).await);- }-- let page: ConnectionsResponse = decode_json_response(response).await?;- people.extend(page.connections);- if page.next_sync_token.is_some() {- next_sync_token = page.next_sync_token;- }- page_token = page.next_page_token;+ page_token: Option<&str>,+) -> Result<ConnectionsResponse, GmailApiHttpError> {+ let mut request = client+ .inner+ .get(format!("{}/people/me/connections", client.contacts_url))+ .bearer_auth(access_token)+ .query(&[+ ("personFields", PERSON_FIELDS),+ ("requestSyncToken", "true"),+ ]);+ if let Some(token) = sync_token {+ request = request.query(&[("syncToken", token)]);+ } else {+ request = request.query(&[("pageSize", "1000")]);+ }+ if let Some(token) = page_token {+ request = request.query(&[("pageToken", token)]);+ }- if page_token.is_none() {- break;- }+ let response = request.send().await.map_err(GmailApiHttpError::transport)?;+ if !response.status().is_success() {+ return Err(unsuccessful_response(response).await);}- let next_sync_token = next_sync_token.ok_or_else(|| {- GmailApiHttpError::InvalidResponse(- "People API did not return a nextSyncToken for connections".to_string(),- )- })?;- Ok((people, next_sync_token))+ decode_json_response(response).await}-/// Fetches all pages of "Other Contacts" and returns raw People resources.+/// Gets one page of "Other Contacts" as raw People resources.///-/// Failures log at WARN for the same reasons as [`list_connections`].-#[tracing::instrument(skip(client, access_token, sync_token), err(level = "warn"))]-pub(crate) async fn list_other_contacts(+/// Failures log at WARN for the same reasons as [`get_connections_page`].+#[tracing::instrument(+ skip(client, access_token, sync_token, page_token),+ err(level = "warn")+)]+pub(crate) async fn get_other_contacts_page(client: &GmailClient,access_token: &str,sync_token: Option<&str>,-) -> Result<(Vec<PersonResource>, String), GmailApiHttpError> {- let mut people = Vec::new();- let mut page_token: Option<String> = None;- let mut next_sync_token: Option<String> = None;-- loop {- let mut request = client- .inner- .get(format!("{}/otherContacts", client.contacts_url))- .bearer_auth(access_token)- .query(&[- ("readMask", PERSON_FIELDS),- ("sources", "READ_SOURCE_TYPE_CONTACT"),- ("sources", "READ_SOURCE_TYPE_PROFILE"),- ("requestSyncToken", "true"),- ]);- if let Some(token) = sync_token {- request = request.query(&[("syncToken", token)]);- } else {- request = request.query(&[("pageSize", "1000")]);- }- if let Some(token) = page_token.as_deref() {- request = request.query(&[("pageToken", token)]);- }-- let response = request.send().await.map_err(GmailApiHttpError::transport)?;- if !response.status().is_success() {- return Err(unsuccessful_response(response).await);- }-- let page: OtherContactsResponse = decode_json_response(response).await?;- people.extend(page.other_contacts);- if page.next_sync_token.is_some() {- next_sync_token = page.next_sync_token;- }- page_token = page.next_page_token;+ page_token: Option<&str>,+) -> Result<OtherContactsResponse, GmailApiHttpError> {+ let mut request = client+ .inner+ .get(format!("{}/otherContacts", client.contacts_url))+ .bearer_auth(access_token)+ .query(&[+ ("readMask", PERSON_FIELDS),+ ("sources", "READ_SOURCE_TYPE_CONTACT"),+ ("sources", "READ_SOURCE_TYPE_PROFILE"),+ ("requestSyncToken", "true"),+ ]);+ if let Some(token) = sync_token {+ request = request.query(&[("syncToken", token)]);+ } else {+ request = request.query(&[("pageSize", "1000")]);+ }+ if let Some(token) = page_token {+ request = request.query(&[("pageToken", token)]);+ }- if page_token.is_none() {- break;- }+ let response = request.send().await.map_err(GmailApiHttpError::transport)?;+ if !response.status().is_success() {+ return Err(unsuccessful_response(response).await);}- let next_sync_token = next_sync_token.ok_or_else(|| {- GmailApiHttpError::InvalidResponse(- "People API did not return a nextSyncToken for other contacts".to_string(),- )- })?;- Ok((people, next_sync_token))+ decode_json_response(response).await}#[cfg(test)]diff --git a/crates/gmail_client/src/contacts/test.rs b/crates/gmail_client/src/contacts/test.rsindex abed4c4b34..dbe531d4de 100644--- a/crates/gmail_client/src/contacts/test.rs+++ b/crates/gmail_client/src/contacts/test.rs@@ -14,7 +14,7 @@ fn client(server: &MockServer) -> GmailClient {}#[tokio::test]-async fn connections_paginate_and_return_the_final_sync_token() {+async fn each_connections_page_is_one_request_that_returns_the_next_page_token() {let server = MockServer::start().await;Mock::given(method("GET")).and(path("/people/me/connections"))@@ -39,16 +39,22 @@ async fn connections_paginate_and_return_the_final_sync_token() {.mount(&server).await;- let (people, sync_token) = list_connections(&client(&server), "token", None)+ let first = get_connections_page(&client(&server), "token", None, None).await- .expect("connections should decode");+ .expect("first page should decode");+ assert_eq!(first.connections.len(), 1);+ assert_eq!(first.next_page_token.as_deref(), Some("page-2"));+ assert_eq!(first.next_sync_token, None);- assert_eq!(people.len(), 2);- assert_eq!(sync_token, "sync-final");+ let last = get_connections_page(&client(&server), "token", None, Some("page-2"))+ .await+ .expect("last page should decode");+ assert_eq!(last.next_page_token, None);+ assert_eq!(last.next_sync_token.as_deref(), Some("sync-final"));}#[tokio::test]-async fn other_contacts_incremental_sync_paginates() {+async fn other_contacts_pages_send_the_sync_token_with_every_page() {let server = MockServer::start().await;Mock::given(method("GET")).and(path("/otherContacts"))@@ -74,48 +80,16 @@ async fn other_contacts_incremental_sync_paginates() {.mount(&server).await;- let (people, sync_token) = list_other_contacts(&client(&server), "token", Some("old-sync"))- .await- .expect("other contacts should decode");-- assert_eq!(people.len(), 2);- assert_eq!(sync_token, "new-sync");-}--#[tokio::test]-async fn missing_connections_sync_token_is_an_invalid_response() {- let server = MockServer::start().await;- Mock::given(method("GET"))- .and(path("/people/me/connections"))- .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({- "connections": []- })))- .mount(&server)- .await;-- let error = list_connections(&client(&server), "token", None)+ let first = get_other_contacts_page(&client(&server), "token", Some("old-sync"), None).await- .expect_err("missing sync token should fail");-- assert!(matches!(error, GmailApiHttpError::InvalidResponse(_)));-}+ .expect("first page should decode");+ assert_eq!(first.next_page_token.as_deref(), Some("page-2"));-#[tokio::test]-async fn missing_other_contacts_sync_token_is_an_invalid_response() {- let server = MockServer::start().await;- Mock::given(method("GET"))- .and(path("/otherContacts"))- .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({- "otherContacts": []- })))- .mount(&server)- .await;-- let error = list_other_contacts(&client(&server), "token", None)+ let last = get_other_contacts_page(&client(&server), "token", Some("old-sync"), Some("page-2")).await- .expect_err("missing sync token should fail");-- assert!(matches!(error, GmailApiHttpError::InvalidResponse(_)));+ .expect("last page should decode");+ assert_eq!(last.other_contacts.len(), 1);+ assert_eq!(last.next_sync_token.as_deref(), Some("new-sync"));}#[tokio::test]diff --git a/crates/gmail_client/src/history.rs b/crates/gmail_client/src/history.rsindex 141bcb5a5e..49776ef6d4 100644--- a/crates/gmail_client/src/history.rs+++ b/crates/gmail_client/src/history.rs@@ -4,53 +4,33 @@ use models_email::gmail::HistoryListResponse;const MAX_RESULTS: usize = 500;-/// Gets the changes to a user's inbox that have occurred since `start_history_id`.-/// Returns the raw Gmail response for conversion by the caller.-#[tracing::instrument(skip(client, access_token), err)]-pub(crate) async fn get_history(+/// Gets one page of the changes to a user's inbox that have occurred since+/// `start_history_id`. Returns the raw Gmail response for conversion by the caller.+#[tracing::instrument(skip(client, access_token, page_token), err)]+pub(crate) async fn get_history_page(client: &GmailClient,access_token: &str,start_history_id: &str,+ page_token: Option<&str>,) -> Result<HistoryListResponse, GmailApiHttpError> {- let mut history = Vec::new();- let mut history_id = String::new();- let mut page_token: Option<String> = None;-- loop {- let mut request = client- .inner- .get(format!("{}/users/me/history", client.base_url))- .bearer_auth(access_token)- .query(&[- ("startHistoryId", start_history_id),- ("maxResults", &MAX_RESULTS.to_string()),- ]);- if let Some(token) = page_token.as_deref() {- request = request.query(&[("pageToken", token)]);- }-- let response = request.send().await.map_err(GmailApiHttpError::transport)?;- if !response.status().is_success() {- return Err(unsuccessful_response(response).await);- }-- let page: HistoryListResponse = decode_json_response(response).await?;- if let Some(page_history) = page.history {- history.extend(page_history);- }- if !page.history_id.is_empty() {- history_id = page.history_id;- }- page_token = page.next_page_token;+ let mut request = client+ .inner+ .get(format!("{}/users/me/history", client.base_url))+ .bearer_auth(access_token)+ .query(&[+ ("startHistoryId", start_history_id),+ ("maxResults", &MAX_RESULTS.to_string()),+ ]);+ if let Some(token) = page_token {+ request = request.query(&[("pageToken", token)]);+ }- if page_token.is_none() {- return Ok(HistoryListResponse {- history: Some(history),- next_page_token: None,- history_id,- });- }+ let response = request.send().await.map_err(GmailApiHttpError::transport)?;+ if !response.status().is_success() {+ return Err(unsuccessful_response(response).await);}++ decode_json_response(response).await}#[cfg(test)]diff --git a/crates/gmail_client/src/history/test.rs b/crates/gmail_client/src/history/test.rsindex 3cf371fb50..2bd71a6b58 100644--- a/crates/gmail_client/src/history/test.rs+++ b/crates/gmail_client/src/history/test.rs@@ -15,7 +15,7 @@ fn client(server: &MockServer) -> GmailClient {}#[tokio::test]-async fn history_paginates_and_advances_to_the_latest_cursor() {+async fn each_history_page_is_one_request_that_returns_the_next_page_token() {let server = MockServer::start().await;Mock::given(method("GET")).and(path("/users/me/history"))@@ -35,6 +35,7 @@ async fn history_paginates_and_advances_to_the_latest_cursor() {.await;Mock::given(method("GET")).and(path("/users/me/history"))+ .and(query_param("startHistoryId", "100")).and(query_param("pageToken", "next")).respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({"history": [{@@ -47,13 +48,18 @@ async fn history_paginates_and_advances_to_the_latest_cursor() {.mount(&server).await;- let response = get_history(&client(&server), "token", "100")+ let first = get_history_page(&client(&server), "token", "100", None).await- .expect("history should decode");+ .expect("first page should decode");+ assert_eq!(first.history_id, "101");+ assert_eq!(first.next_page_token.as_deref(), Some("next"));- assert_eq!(response.history_id, "102");- assert_eq!(response.history.unwrap().len(), 2);- assert!(response.next_page_token.is_none());+ let last = get_history_page(&client(&server), "token", "100", Some("next"))+ .await+ .expect("last page should decode");+ assert_eq!(last.history_id, "102");+ assert_eq!(last.history.unwrap().len(), 1);+ assert!(last.next_page_token.is_none());}#[tokio::test]@@ -68,7 +74,7 @@ async fn stale_history_not_found_status_is_preserved() {.mount(&server).await;- let error = get_history(&client(&server), "token", "stale")+ let error = get_history_page(&client(&server), "token", "stale", None).await.expect_err("stale history should fail");diff --git a/crates/gmail_client/src/lib.rs b/crates/gmail_client/src/lib.rsindex b02db81072..fd4dcfd09c 100644--- a/crates/gmail_client/src/lib.rs+++ b/crates/gmail_client/src/lib.rs@@ -49,7 +49,7 @@ use crate::contacts::get_self_connection;use crate::messages::{get_message, get_message_label_ids, list_messages};use crate::threads::get_thread;pub use error::GmailApiHttpError;-use models_email::gmail::contacts::PersonResource;+use models_email::gmail::contacts::{ConnectionsResponse, OtherContactsResponse, PersonResource};pub use models_email::gmail::filters::Filter;use models_email::gmail::inbox_sync::{GoogleJwtClaims, GooglePublicKeys, JwtVerificationError, KeyMap,@@ -181,15 +181,17 @@ impl GmailClient {get_thread(self, access_token, thread_id).await}- /// Gets the changes to a user's inbox that have occurred since start_history_id.- /// Returns raw HistoryListResponse - callers should map to InboxChanges using convert module.- #[tracing::instrument(skip(self, access_token), err)]- pub async fn get_history(+ /// Gets one page of the changes to a user's inbox since `start_history_id`.+ /// Pass the previous page's `next_page_token` to continue. Returns the raw+ /// HistoryListResponse; callers map it to inbox changes.+ #[tracing::instrument(skip(self, access_token, page_token), err)]+ pub async fn get_history_page(&self,access_token: &str,start_history_id: &str,+ page_token: Option<&str>,) -> Result<HistoryListResponse, GmailApiHttpError> {- history::get_history(self, access_token, start_history_id).await+ history::get_history_page(self, access_token, start_history_id, page_token).await}/// Fetches the user's raw Gmail profile.@@ -383,31 +385,31 @@ impl GmailClient {get_self_connection(self, access_token).await}- /// Fetches all of the user's main contacts, handling pagination.- /// Returns raw Gmail PersonResource objects and a sync token for future incremental updates.- /// Callers should map PersonResource to service layer Contact.+ /// Fetches one page of the user's main contacts. Pass the previous page's+ /// `next_page_token` to continue; the last page carries the sync token for+ /// future incremental updates. Callers map the people to service contacts./// Failures log at WARN: expired sync tokens and missing contacts scopes are routine.- #[tracing::instrument(skip(self, access_token, sync_token), err(level = "warn"))]- pub async fn get_contacts(+ #[tracing::instrument(skip(self, access_token, sync_token, page_token), err(level = "warn"))]+ pub async fn get_contacts_page(&self,access_token: &str,sync_token: Option<&str>,- ) -> Result<(Vec<PersonResource>, String), GmailApiHttpError> {- contacts::list_connections(self, access_token, sync_token).await+ page_token: Option<&str>,+ ) -> Result<ConnectionsResponse, GmailApiHttpError> {+ contacts::get_connections_page(self, access_token, sync_token, page_token).await}- /// Fetches all of the user's "Other Contacts", handling pagination.- /// These are typically contacts auto-created from interactions.- /// Returns raw Gmail PersonResource objects and a sync token.- /// Callers should map PersonResource to service layer Contact.+ /// Fetches one page of the user's "Other Contacts", typically created from+ /// interactions. Paging and sync tokens work as in [`Self::get_contacts_page`]./// Failures log at WARN: expired sync tokens and missing contacts scopes are routine.- #[tracing::instrument(skip(self, access_token, sync_token), err(level = "warn"))]- pub async fn get_other_contacts(+ #[tracing::instrument(skip(self, access_token, sync_token, page_token), err(level = "warn"))]+ pub async fn get_other_contacts_page(&self,access_token: &str,sync_token: Option<&str>,- ) -> Result<(Vec<PersonResource>, String), GmailApiHttpError> {- contacts::list_other_contacts(self, access_token, sync_token).await+ page_token: Option<&str>,+ ) -> Result<OtherContactsResponse, GmailApiHttpError> {+ contacts::get_other_contacts_page(self, access_token, sync_token, page_token).await}/// Creates a new Gmail filter.