Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(email): gate each page of Gmail history and contacts on its own (#8809)

teo66c5b8fe006c
diff --git a/crates/email_api_client/AGENTS.md b/crates/email_api_client/AGENTS.md
index 67941df91a..ee8899c494 100644
--- a/crates/email_api_client/AGENTS.md
+++ b/crates/email_api_client/AGENTS.md
@@ -69,7 +69,11 @@ Outlook (Microsoft Graph) outbound adapters.
`call_with_token`; a call made with a bare token is invisible to the
in-flight cap. Their spans declare `gate_in_flight` and `gate_cap` as
empty fields, which admission fills with the mailbox's load so it sits next
- to the call's outcome.
+ to the call's outcome. Paginated reads (history, contacts) are page-level
+ ports that the service loops over, one permit per page, so every page is
+ charged to the mailbox's window and checks its cooldown; a refused page
+ fails the read, and the span keeps the last page's load. Keep new paginated
+ provider calls page-level rather than looping in the adapter.
- Link-based reads go through `read_with_token`, which retries once with a
fresh token when the provider answers `AuthRequired`; a cached token can
outlive a revoked grant, and only the fresh fetch lets the token source mark
diff --git a/crates/email_api_client/src/domain/models/changes.rs b/crates/email_api_client/src/domain/models/changes.rs
index 1d9a674654..c8e370cc43 100644
--- a/crates/email_api_client/src/domain/models/changes.rs
+++ b/crates/email_api_client/src/domain/models/changes.rs
@@ -34,6 +34,55 @@ pub struct InboxChanges {
pub labels_to_update: HashSet<String>,
}
+impl InboxChanges {
+ /// Adds changes the provider recorded after these, with the same result
+ /// as reading both in one pass: a later deletion drops an earlier upsert
+ /// or label refresh, an earlier deletion blocks later ones, and an upsert
+ /// supersedes a label refresh.
+ pub fn append_later(&mut self, later: InboxChanges) {
+ let earlier = std::mem::take(self);
+ self.message_ids_to_upsert = earlier
+ .message_ids_to_upsert
+ .difference(&later.message_ids_to_delete)
+ .chain(
+ later
+ .message_ids_to_upsert
+ .difference(&earlier.message_ids_to_delete),
+ )
+ .cloned()
+ .collect();
+ self.labels_to_update = earlier
+ .labels_to_update
+ .iter()
+ .filter(|id| {
+ !later.message_ids_to_upsert.contains(*id)
+ && !later.message_ids_to_delete.contains(*id)
+ })
+ .chain(later.labels_to_update.iter().filter(|id| {
+ !earlier.message_ids_to_upsert.contains(*id)
+ && !earlier.message_ids_to_delete.contains(*id)
+ }))
+ .cloned()
+ .collect();
+ self.message_ids_to_delete = earlier
+ .message_ids_to_delete
+ .union(&later.message_ids_to_delete)
+ .cloned()
+ .collect();
+ }
+}
+
+/// One provider page of inbox changes following a cursor.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ChangePage {
+ /// Changes recorded in this page.
+ pub changes: InboxChanges,
+ /// The mailbox's cursor when the page was read, if the provider reported one.
+ pub cursor: Option<SyncCursor>,
+ /// Opaque token for the next page, or `None` when this is the last page.
+ pub next_page_token: Option<String>,
+}
+
/// A page of inbox changes and the cursor that follows those changes.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ChangeBatch {
diff --git a/crates/email_api_client/src/domain/models/mailbox.rs b/crates/email_api_client/src/domain/models/mailbox.rs
index 71f5b43580..2d48cfed94 100644
--- a/crates/email_api_client/src/domain/models/mailbox.rs
+++ b/crates/email_api_client/src/domain/models/mailbox.rs
@@ -1,6 +1,7 @@
use chrono::{DateTime, Utc};
use models_email::email::service::message::Message;
use models_email::email::service::thread::ThreadSummary;
+use models_email::service::contact::Contact;
use serde::{Deserialize, Serialize};
use super::SyncCursor;
@@ -57,3 +58,15 @@ pub struct ThreadListPage {
/// Opaque token for the next page, or `None` when this is the last page.
pub next_page_token: Option<String>,
}
+
+/// One provider page of contacts.
+#[derive(Debug, Clone)]
+pub struct ContactPage {
+ /// Contacts returned in this page.
+ pub contacts: Vec<Contact>,
+ /// Token for the next incremental synchronization, which the provider
+ /// returns with the last page.
+ pub next_sync_token: Option<String>,
+ /// Opaque token for the next page, or `None` when this is the last page.
+ pub next_page_token: Option<String>,
+}
diff --git a/crates/email_api_client/src/domain/models/mod.rs b/crates/email_api_client/src/domain/models/mod.rs
index 398a57bd43..0ec5668b95 100644
--- a/crates/email_api_client/src/domain/models/mod.rs
+++ b/crates/email_api_client/src/domain/models/mod.rs
@@ -14,10 +14,12 @@ mod send;
pub use address_book::*;
pub use auth::{AccessToken, TokenError, TokenFreshness};
-pub use changes::{ChangeBatch, InboxChanges, SyncCursor};
+pub use changes::{ChangeBatch, ChangePage, InboxChanges, SyncCursor};
pub use draft::*;
pub use error::{EmailApiError, RateLimitOrigin, RateLimitRefusal};
-pub use mailbox::{CalendarPart, MessageWithCalendarParts, ProviderSubscription, ThreadListPage};
+pub use mailbox::{
+ CalendarPart, ContactPage, MessageWithCalendarParts, ProviderSubscription, ThreadListPage,
+};
pub use mailbox_settings::*;
pub use operation::ApiOperationKind;
pub use permit::{MailboxLoad, RequestPermit};
diff --git a/crates/email_api_client/src/domain/ports.rs b/crates/email_api_client/src/domain/ports.rs
index 084620fa31..028f706d93 100644
--- a/crates/email_api_client/src/domain/ports.rs
+++ b/crates/email_api_client/src/domain/ports.rs
@@ -6,12 +6,12 @@ pub use mailbox::*;
use std::future::Future;
use models_email::email::service::label::Label;
-use models_email::service::contact::{Contact, ContactList};
+use models_email::service::contact::Contact;
use models_email::service::link::Link;
use uuid::Uuid;
use super::models::{
- AccessToken, ApiOperationKind, CalendarPart, ChangeBatch, EmailApiError,
+ AccessToken, ApiOperationKind, CalendarPart, ChangePage, ContactPage, EmailApiError,
MessageWithCalendarParts, ProviderSubscription, RateLimitRefusal, RequestPermit, SendRequest,
SentIds, SyncCursor, ThreadListPage, TokenError, TokenFreshness,
};
@@ -27,12 +27,14 @@ pub trait MailboxSyncClient: Send + Sync + 'static {
access_token: &AccessToken,
) -> impl Future<Output = Result<u64, EmailApiError>> + Send;
- /// Returns mailbox changes that follow `cursor`.
- fn list_changes(
+ /// Returns one page of the mailbox changes that follow `cursor`. Pass the
+ /// previous page's `next_page_token` to continue.
+ fn list_changes_page(
&self,
access_token: &AccessToken,
cursor: &SyncCursor,
- ) -> impl Future<Output = Result<ChangeBatch, EmailApiError>> + Send;
+ page_token: Option<&str>,
+ ) -> impl Future<Output = Result<ChangePage, EmailApiError>> + Send;
}
/// Mailbox push-notification subscription capabilities.
@@ -225,21 +227,24 @@ pub trait MailboxContactsClient: Send + Sync + 'static {
link_id: Uuid,
) -> impl Future<Output = Result<Contact, EmailApiError>> + Send;
- /// Fetches and normalizes primary contacts from an optional synchronization token.
- fn list_contacts(
+ /// Fetches and normalizes one page of primary contacts from an optional
+ /// synchronization token. Pass the previous page's `next_page_token` to continue.
+ fn list_contacts_page(
&self,
access_token: &AccessToken,
link_id: Uuid,
sync_token: Option<&str>,
- ) -> impl Future<Output = Result<ContactList, EmailApiError>> + Send;
+ page_token: Option<&str>,
+ ) -> impl Future<Output = Result<ContactPage, EmailApiError>> + Send;
- /// Fetches and normalizes automatically collected contacts.
- fn list_other_contacts(
+ /// Fetches and normalizes one page of automatically collected contacts.
+ fn list_other_contacts_page(
&self,
access_token: &AccessToken,
link_id: Uuid,
sync_token: Option<&str>,
- ) -> impl Future<Output = Result<ContactList, EmailApiError>> + Send;
+ page_token: Option<&str>,
+ ) -> impl Future<Output = Result<ContactPage, EmailApiError>> + Send;
}
/// Blocked-sender management capabilities.
@@ -380,11 +385,12 @@ impl MailboxSyncClient for NoOpMailboxClient {
Err(unavailable())
}
- async fn list_changes(
+ async fn list_changes_page(
&self,
_: &AccessToken,
_: &SyncCursor,
- ) -> Result<ChangeBatch, EmailApiError> {
+ _: Option<&str>,
+ ) -> Result<ChangePage, EmailApiError> {
Err(unavailable())
}
}
@@ -545,21 +551,23 @@ impl MailboxContactsClient for NoOpMailboxClient {
Err(unavailable())
}
- async fn list_contacts(
+ async fn list_contacts_page(
&self,
_: &AccessToken,
_: Uuid,
_: Option<&str>,
- ) -> Result<ContactList, EmailApiError> {
+ _: Option<&str>,
+ ) -> Result<ContactPage, EmailApiError> {
Err(unavailable())
}
- async fn list_other_contacts(
+ async fn list_other_contacts_page(
&self,
_: &AccessToken,
_: Uuid,
_: Option<&str>,
- ) -> Result<ContactList, EmailApiError> {
+ _: Option<&str>,
+ ) -> Result<ContactPage, EmailApiError> {
Err(unavailable())
}
}
diff --git a/crates/email_api_client/src/domain/service/contacts.rs b/crates/email_api_client/src/domain/service/contacts.rs
index ddd3ae3182..a5895ec16c 100644
--- a/crates/email_api_client/src/domain/service/contacts.rs
+++ b/crates/email_api_client/src/domain/service/contacts.rs
@@ -1,7 +1,7 @@
use models_email::service::contact::{Contact, ContactList};
use uuid::Uuid;
-use super::super::models::EmailApiError;
+use super::super::models::{AccessToken, ContactPage, EmailApiError};
use super::super::ports::{MailboxContactsClient, ProviderRequestGate, ProviderTokenSource};
use super::{ApiOperationKind, EmailApiClientServiceImpl};
@@ -26,43 +26,92 @@ where
///
/// Failures log at WARN: an expired sync token ([`EmailApiError::OutdatedCursor`])
/// or a grant without the contacts scopes is routine, and callers recover or skip.
- #[tracing::instrument(skip(self, sync_token), fields(retry_count = 0, gate_in_flight = tracing::field::Empty, gate_cap = tracing::field::Empty), err(level = "warn"))]
+ #[tracing::instrument(
+ skip(self, sync_token),
+ fields(
+ retry_count = 0,
+ pages = tracing::field::Empty,
+ gate_in_flight = tracing::field::Empty,
+ gate_cap = tracing::field::Empty
+ ),
+ err(level = "warn")
+ )]
pub async fn list_contacts(
&self,
link_id: Uuid,
sync_token: Option<&str>,
) -> Result<ContactList, EmailApiError> {
- self.retry_with_fresh_token(
- link_id,
- ApiOperationKind::ListContacts,
- |token| async move {
- self.repository
- .list_contacts(&token, link_id, sync_token)
- .await
- },
- )
- .await?
+ self.list_contact_pages(link_id, |token, page_token| async move {
+ self.repository
+ .list_contacts_page(&token, link_id, sync_token, page_token.as_deref())
+ .await
+ })
+ .await
}
/// Lists automatically collected contacts, optionally continuing a synchronization.
///
/// Failures log at WARN for the same reasons as [`Self::list_contacts`].
- #[tracing::instrument(skip(self, sync_token), fields(retry_count = 0, gate_in_flight = tracing::field::Empty, gate_cap = tracing::field::Empty), err(level = "warn"))]
+ #[tracing::instrument(
+ skip(self, sync_token),
+ fields(
+ retry_count = 0,
+ pages = tracing::field::Empty,
+ gate_in_flight = tracing::field::Empty,
+ gate_cap = tracing::field::Empty
+ ),
+ err(level = "warn")
+ )]
pub async fn list_other_contacts(
&self,
link_id: Uuid,
sync_token: Option<&str>,
) -> Result<ContactList, EmailApiError> {
- self.retry_with_fresh_token(
- link_id,
- ApiOperationKind::ListContacts,
- |token| async move {
- self.repository
- .list_other_contacts(&token, link_id, sync_token)
- .await
- },
- )
- .await?
+ self.list_contact_pages(link_id, |token, page_token| async move {
+ self.repository
+ .list_other_contacts_page(&token, link_id, sync_token, page_token.as_deref())
+ .await
+ })
+ .await
+ }
+
+ /// Reads every page of a contact list. Each page is its own admission, and
+ /// a refusal or failure on any page fails the whole read.
+ async fn list_contact_pages<F, Fut>(
+ &self,
+ link_id: Uuid,
+ read_page: F,
+ ) -> Result<ContactList, EmailApiError>
+ where
+ F: Fn(AccessToken, Option<String>) -> Fut,
+ Fut: Future<Output = Result<ContactPage, EmailApiError>>,
+ {
+ let mut contacts = Vec::new();
+ let mut next_sync_token = None;
+ let mut page_token: Option<String> = None;
+ let mut pages = 0;
+ loop {
+ let page = self
+ .retry_with_fresh_token(link_id, ApiOperationKind::ListContacts, |token| {
+ read_page(token, page_token.clone())
+ })
+ .await??;
+ pages += 1;
+ contacts.extend(page.contacts);
+ next_sync_token = page.next_sync_token.or(next_sync_token);
+ page_token = page.next_page_token;
+ if page_token.is_none() {
+ break;
+ }
+ }
+ tracing::Span::current().record("pages", pages);
+ let next_sync_token = next_sync_token.ok_or_else(|| EmailApiError::Permanent {
+ message: "the provider returned no sync token with the last contacts page".to_string(),
+ })?;
+ Ok(ContactList {
+ contacts,
+ next_sync_token,
+ })
}
}
diff --git a/crates/email_api_client/src/domain/service/contacts/test.rs b/crates/email_api_client/src/domain/service/contacts/test.rs
index 54df24189f..fb1b20aebf 100644
--- a/crates/email_api_client/src/domain/service/contacts/test.rs
+++ b/crates/email_api_client/src/domain/service/contacts/test.rs
@@ -1,11 +1,13 @@
use std::sync::{Arc, Mutex};
-use models_email::service::contact::{Contact, ContactList};
+use std::collections::VecDeque;
+
+use models_email::service::contact::Contact;
use uuid::Uuid;
use super::super::test_support::{Call, FakeGate, FakeTokenSource, call_log};
use super::*;
-use crate::domain::models::{AccessToken, TokenFreshness};
+use crate::domain::models::{AccessToken, ContactPage, TokenFreshness};
#[derive(Debug, Clone, PartialEq, Eq)]
enum ContactCall {
@@ -30,12 +32,13 @@ impl MailboxContactsClient for ContactsClient {
Err(expected_error())
}
- async fn list_contacts(
+ async fn list_contacts_page(
&self,
access_token: &AccessToken,
link_id: Uuid,
sync_token: Option<&str>,
- ) -> Result<ContactList, EmailApiError> {
+ _: Option<&str>,
+ ) -> Result<ContactPage, EmailApiError> {
assert_eq!(access_token.expose_secret(), "token");
*self.call.lock().unwrap() = Some(ContactCall::Contacts(
link_id,
@@ -44,12 +47,13 @@ impl MailboxContactsClient for ContactsClient {
Err(expected_error())
}
- async fn list_other_contacts(
+ async fn list_other_contacts_page(
&self,
access_token: &AccessToken,
link_id: Uuid,
sync_token: Option<&str>,
- ) -> Result<ContactList, EmailApiError> {
+ _: Option<&str>,
+ ) -> Result<ContactPage, EmailApiError> {
assert_eq!(access_token.expose_secret(), "token");
*self.call.lock().unwrap() = Some(ContactCall::OtherContacts(
link_id,
@@ -106,24 +110,26 @@ impl MailboxContactsClient for RejectsCachedToken {
unreachable!("not used by this test")
}
- async fn list_contacts(
+ async fn list_contacts_page(
&self,
access_token: &AccessToken,
_: Uuid,
_: Option<&str>,
- ) -> Result<ContactList, EmailApiError> {
+ _: Option<&str>,
+ ) -> Result<ContactPage, EmailApiError> {
match access_token.expose_secret() {
"cached" => Err(EmailApiError::AuthRequired),
_ => Err(expected_error()),
}
}
- async fn list_other_contacts(
+ async fn list_other_contacts_page(
&self,
_: &AccessToken,
_: Uuid,
_: Option<&str>,
- ) -> Result<ContactList, EmailApiError> {
+ _: Option<&str>,
+ ) -> Result<ContactPage, EmailApiError> {
unreachable!("not used by this test")
}
}
@@ -188,3 +194,132 @@ async fn contact_reads_use_correct_operation_kinds_and_forward_sync_tokens() {
)
.await;
}
+
+/// Serves configured pages of other contacts and records the tokens it was given.
+#[derive(Default)]
+struct PagedContacts {
+ pages: Mutex<VecDeque<ContactPage>>,
+ requests: Mutex<Vec<(Option<String>, Option<String>)>>,
+}
+
+impl MailboxContactsClient for PagedContacts {
+ async fn get_self_contact(&self, _: &AccessToken, _: Uuid) -> Result<Contact, EmailApiError> {
+ unreachable!("not used by this test")
+ }
+
+ async fn list_contacts_page(
+ &self,
+ _: &AccessToken,
+ _: Uuid,
+ _: Option<&str>,
+ _: Option<&str>,
+ ) -> Result<ContactPage, EmailApiError> {
+ unreachable!("not used by this test")
+ }
+
+ async fn list_other_contacts_page(
+ &self,
+ _: &AccessToken,
+ _: Uuid,
+ sync_token: Option<&str>,
+ page_token: Option<&str>,
+ ) -> Result<ContactPage, EmailApiError> {
+ self.requests.lock().unwrap().push((
+ sync_token.map(str::to_string),
+ page_token.map(str::to_string),
+ ));
+ Ok(self
+ .pages
+ .lock()
+ .unwrap()
+ .pop_front()
+ .expect("the test configures every page the service reads"))
+ }
+}
+
+fn contact(link_id: Uuid, email: &str) -> Contact {
+ Contact {
+ id: Uuid::nil(),
+ link_id,
+ name: None,
+ email_address: Some(email.to_string()),
+ original_photo_url: None,
+ sfs_photo_url: None,
+ }
+}
+
+#[tokio::test]
+async fn each_contacts_page_is_admitted_on_its_own_and_the_pages_are_combined() {
+ let calls = call_log();
+ let link_id = Uuid::new_v4();
+ let repository = PagedContacts {
+ pages: Mutex::new(VecDeque::from([
+ ContactPage {
+ contacts: vec![contact(link_id, "[email protected]")],
+ next_sync_token: None,
+ next_page_token: Some("page-2".into()),
+ },
+ ContactPage {
+ contacts: vec![contact(link_id, "[email protected]")],
+ next_sync_token: Some("new-sync".into()),
+ next_page_token: None,
+ },
+ ])),
+ ..PagedContacts::default()
+ };
+ let service = EmailApiClientServiceImpl::new(
+ repository,
+ FakeTokenSource::new(calls.clone(), Ok(AccessToken::new("token"))),
+ FakeGate::new(calls.clone(), Ok(())),
+ );
+
+ let list = service
+ .list_other_contacts(link_id, Some("old-sync"))
+ .await
+ .unwrap();
+
+ assert_eq!(
+ list.contacts
+ .iter()
+ .map(|contact| contact.email_address.as_deref())
+ .collect::<Vec<_>>(),
+ vec![Some("[email protected]"), Some("[email protected]")]
+ );
+ assert_eq!(list.next_sync_token, "new-sync");
+ assert_eq!(
+ *service.repository.requests.lock().unwrap(),
+ vec![
+ (Some("old-sync".into()), None),
+ (Some("old-sync".into()), Some("page-2".into())),
+ ]
+ );
+ let page = [
+ Call::Acquire(link_id, ApiOperationKind::ListContacts),
+ Call::Token(link_id, TokenFreshness::Cached),
+ Call::Finish(link_id, None),
+ ];
+ assert_eq!(*calls.lock().unwrap(), [page.clone(), page].concat());
+}
+
+#[tokio::test]
+async fn contacts_without_a_sync_token_on_the_last_page_fail() {
+ let calls = call_log();
+ let link_id = Uuid::new_v4();
+ let service = EmailApiClientServiceImpl::new(
+ PagedContacts {
+ pages: Mutex::new(VecDeque::from([ContactPage {
+ contacts: vec![],
+ next_sync_token: None,
+ next_page_token: None,
+ }])),
+ ..PagedContacts::default()
+ },
+ FakeTokenSource::new(calls.clone(), Ok(AccessToken::new("token"))),
+ FakeGate::new(calls.clone(), Ok(())),
+ );
+
+ assert!(matches!(
+ service.list_other_contacts(link_id, None).await,
+ Err(EmailApiError::Permanent { .. })
+ ));
+}
diff --git a/crates/email_api_client/src/domain/service/sync.rs b/crates/email_api_client/src/domain/service/sync.rs
index b5e5984a81..0e8dcee027 100644
--- a/crates/email_api_client/src/domain/service/sync.rs
+++ b/crates/email_api_client/src/domain/service/sync.rs
@@ -1,6 +1,6 @@
use uuid::Uuid;
-use super::super::models::{ChangeBatch, EmailApiError, SyncCursor};
+use super::super::models::{ChangeBatch, EmailApiError, InboxChanges, SyncCursor};
use super::super::ports::{MailboxSyncClient, ProviderRequestGate, ProviderTokenSource};
use super::{ApiOperationKind, EmailApiClientServiceImpl};
@@ -20,16 +20,50 @@ where
}
/// Lists mailbox changes following `cursor`.
- #[tracing::instrument(skip(self), fields(retry_count = 0, gate_in_flight = tracing::field::Empty, gate_cap = tracing::field::Empty))]
+ ///
+ /// Each page is its own admission, so it takes a permit, is charged to the
+ /// mailbox's window and waits out a cooldown. A refusal or failure on any
+ /// page fails the whole read; callers redeliver it from `cursor`.
+ #[tracing::instrument(
+ skip(self),
+ fields(
+ retry_count = 0,
+ pages = tracing::field::Empty,
+ gate_in_flight = tracing::field::Empty,
+ gate_cap = tracing::field::Empty
+ )
+ )]
pub async fn list_changes(
&self,
link_id: Uuid,
cursor: &SyncCursor,
) -> Result<ChangeBatch, EmailApiError> {
- self.read_with_token(link_id, ApiOperationKind::ListChanges, |token| async move {
- self.repository.list_changes(&token, cursor).await
- })
- .await
+ let mut changes = InboxChanges::default();
+ let mut next_cursor = None;
+ let mut page_token: Option<String> = None;
+ let mut pages = 0;
+ loop {
+ let page_token_ref = page_token.as_deref();
+ let page = self
+ .read_with_token(link_id, ApiOperationKind::ListChanges, |token| async move {
+ self.repository
+ .list_changes_page(&token, cursor, page_token_ref)
+ .await
+ })
+ .await?;
+ pages += 1;
+ changes.append_later(page.changes);
+ next_cursor = page.cursor.or(next_cursor);
+ page_token = page.next_page_token;
+ if page_token.is_none() {
+ break;
+ }
+ }
+ tracing::Span::current().record("pages", pages);
+ Ok(ChangeBatch::new(
+ changes,
+ next_cursor.unwrap_or_else(|| cursor.clone()),
+ ))
}
}
diff --git a/crates/email_api_client/src/domain/service/sync/test.rs b/crates/email_api_client/src/domain/service/sync/test.rs
index 55523a4a5c..f517c68225 100644
--- a/crates/email_api_client/src/domain/service/sync/test.rs
+++ b/crates/email_api_client/src/domain/service/sync/test.rs
@@ -1,14 +1,29 @@
-use std::sync::{Arc, Mutex};
+use std::collections::{HashSet, VecDeque};
+use std::sync::Mutex;
+use std::time::Duration;
use uuid::Uuid;
-use super::super::test_support::{Call, FakeGate, FakeTokenSource, call_log};
+use super::super::test_support::{Call, CallLog, FakeGate, FakeTokenSource, call_log};
use super::*;
-use crate::domain::models::{AccessToken, InboxChanges, TokenFreshness};
+use crate::domain::models::{
+ AccessToken, ChangePage, InboxChanges, RateLimitOrigin, RateLimitRefusal, RequestPermit,
+ TokenFreshness,
+};
-#[derive(Clone, Default)]
+#[derive(Default)]
struct SyncClient {
- cursor: Arc<Mutex<Option<String>>>,
+ pages: Mutex<VecDeque<ChangePage>>,
+ requests: Mutex<Vec<(String, Option<String>)>>,
+}
+
+impl SyncClient {
+ fn with_pages(pages: Vec<ChangePage>) -> Self {
+ Self {
+ pages: Mutex::new(pages.into()),
+ requests: Mutex::default(),
+ }
+ }
}
impl MailboxSyncClient for SyncClient {
@@ -17,23 +32,62 @@ impl MailboxSyncClient for SyncClient {
Ok(42)
}
- async fn list_changes(
+ async fn list_changes_page(
&self,
access_token: &AccessToken,
cursor: &SyncCursor,
- ) -> Result<ChangeBatch, EmailApiError> {
+ page_token: Option<&str>,
+ ) -> Result<ChangePage, EmailApiError> {
assert_eq!(access_token.expose_secret(), "token");
- *self.cursor.lock().unwrap() = Some(cursor.as_str().to_string());
- Ok(ChangeBatch::new(
- InboxChanges::default(),
- SyncCursor::gmail("next"),
- ))
+ self.requests
+ .lock()
+ .unwrap()
+ .push((cursor.as_str().to_string(), page_token.map(str::to_string)));
+ Ok(self
+ .pages
+ .lock()
+ .unwrap()
+ .pop_front()
+ .expect("the test configures every page the service reads"))
+ }
+}
+
+/// Admits the first `admitted` requests and refuses the rest.
+struct AdmitsFirst {
+ calls: CallLog,
+ admitted: Mutex<usize>,
+}
+
+impl ProviderRequestGate for AdmitsFirst {
+ async fn acquire(
+ &self,
+ link_id: Uuid,
+ operation: ApiOperationKind,
+ ) -> Result<RequestPermit, RateLimitRefusal> {
+ self.calls
+ .lock()
+ .unwrap()
+ .push(Call::Acquire(link_id, operation));
+ let mut admitted = self.admitted.lock().unwrap();
+ if *admitted == 0 {
+ return Err(RateLimitRefusal::new(Some(Duration::from_secs(1))));
+ }
+ *admitted -= 1;
+ Ok(RequestPermit::new(link_id, Uuid::nil(), |_, _| {}))
+ }
+
+ async fn finish(&self, permit: RequestPermit, error: Option<&EmailApiError>) {
+ self.calls
+ .lock()
+ .unwrap()
+ .push(Call::Finish(permit.link_id(), error.cloned()));
+ permit.disarm();
}
}
fn service(
repository: SyncClient,
- calls: &super::super::test_support::CallLog,
+ calls: &CallLog,
) -> EmailApiClientServiceImpl<SyncClient, FakeTokenSource, FakeGate> {
EmailApiClientServiceImpl::new(
repository,
@@ -42,6 +96,10 @@ fn service(
)
}
+fn ids(values: &[&str]) -> HashSet<String> {
+ values.iter().map(|value| value.to_string()).collect()
+}
+
#[tokio::test]
async fn get_thread_count_uses_profile_quota_and_forwards_link_id() {
let calls = call_log();
@@ -66,17 +124,23 @@ async fn get_thread_count_uses_profile_quota_and_forwards_link_id() {
#[tokio::test]
async fn list_changes_uses_changes_quota_and_forwards_cursor() {
let calls = call_log();
- let repository = SyncClient::default();
- let captured_cursor = repository.cursor.clone();
+ let repository = SyncClient::with_pages(vec![ChangePage {
+ changes: InboxChanges::default(),
+ cursor: Some(SyncCursor::gmail("next")),
+ next_page_token: None,
+ }]);
let link_id = Uuid::new_v4();
- let cursor = SyncCursor::gmail("cursor");
- let result = service(repository, &calls)
- .list_changes(link_id, &cursor)
+ let service = service(repository, &calls);
+ let result = service
+ .list_changes(link_id, &SyncCursor::gmail("cursor"))
.await;
assert_eq!(result.unwrap().next_cursor, SyncCursor::gmail("next"));
- assert_eq!(*captured_cursor.lock().unwrap(), Some("cursor".into()));
+ assert_eq!(
+ *service.repository.requests.lock().unwrap(),
+ vec![("cursor".into(), None)]
+ );
assert_eq!(
*calls.lock().unwrap(),
vec![
@@ -86,3 +150,115 @@ async fn list_changes_uses_changes_quota_and_forwards_cursor() {
]
);
}
+
+#[tokio::test]
+async fn each_history_page_is_admitted_on_its_own_and_later_pages_win() {
+ let calls = call_log();
+ let repository = SyncClient::with_pages(vec![
+ ChangePage {
+ changes: InboxChanges {
+ message_ids_to_upsert: ids(&["kept", "deleted-later"]),
+ ..InboxChanges::default()
+ },
+ cursor: Some(SyncCursor::gmail("200")),
+ next_page_token: Some("page-2".into()),
+ },
+ ChangePage {
+ changes: InboxChanges {
+ message_ids_to_delete: ids(&["deleted-later"]),
+ labels_to_update: ids(&["relabelled"]),
+ ..InboxChanges::default()
+ },
+ cursor: Some(SyncCursor::gmail("300")),
+ next_page_token: Some("page-3".into()),
+ },
+ ChangePage {
+ changes: InboxChanges::default(),
+ cursor: None,
+ next_page_token: None,
+ },
+ ]);
+ let link_id = Uuid::new_v4();
+
+ let service = service(repository, &calls);
+ let batch = service
+ .list_changes(link_id, &SyncCursor::gmail("100"))
+ .await
+ .unwrap();
+
+ assert_eq!(
+ batch,
+ ChangeBatch::new(
+ InboxChanges {
+ message_ids_to_upsert: ids(&["kept"]),
+ message_ids_to_delete: ids(&["deleted-later"]),
+ labels_to_update: ids(&["relabelled"]),
+ },
+ SyncCursor::gmail("300"),
+ )
+ );
+ assert_eq!(
+ *service.repository.requests.lock().unwrap(),
+ vec![
+ ("100".into(), None),
+ ("100".into(), Some("page-2".into())),
+ ("100".into(), Some("page-3".into())),
+ ]
+ );
+ let page = [
+ Call::Acquire(link_id, ApiOperationKind::ListChanges),
+ Call::Token(link_id, TokenFreshness::Cached),
+ Call::Finish(link_id, None),
+ ];
+ assert_eq!(
+ *calls.lock().unwrap(),
+ [page.clone(), page.clone(), page].concat()
+ );
+}
+
+#[tokio::test]
+async fn a_refused_page_fails_the_read_so_it_is_redelivered() {
+ let calls = call_log();
+ let repository = SyncClient::with_pages(vec![ChangePage {
+ changes: InboxChanges {
+ message_ids_to_upsert: ids(&["first-page"]),
+ ..InboxChanges::default()
+ },
+ cursor: Some(SyncCursor::gmail("200")),
+ next_page_token: Some("page-2".into()),
+ }]);
+ let link_id = Uuid::new_v4();
+ let service = EmailApiClientServiceImpl::new(
+ repository,
+ FakeTokenSource::new(calls.clone(), Ok(AccessToken::new("token"))),
+ AdmitsFirst {
+ calls: calls.clone(),
+ admitted: Mutex::new(1),
+ },
+ );
+
+ let result = service
+ .list_changes(link_id, &SyncCursor::gmail("100"))
+ .await;
+
+ assert_eq!(
+ result,
+ Err(EmailApiError::RateLimited {
+ retry_after: Some(Duration::from_secs(1)),
+ origin: RateLimitOrigin::Local,
+ })
+ );
+ assert_eq!(
+ *service.repository.requests.lock().unwrap(),
+ vec![("100".into(), None)]
+ );
+ assert_eq!(
+ *calls.lock().unwrap(),
+ vec![
+ Call::Acquire(link_id, ApiOperationKind::ListChanges),
+ Call::Token(link_id, TokenFreshness::Cached),
+ Call::Finish(link_id, None),
+ Call::Acquire(link_id, ApiOperationKind::ListChanges),
+ ]
+ );
+}
diff --git a/crates/email_api_client/src/domain/service/test_support.rs b/crates/email_api_client/src/domain/service/test_support.rs
index 34d6be531c..0c888fabbc 100644
--- a/crates/email_api_client/src/domain/service/test_support.rs
+++ b/crates/email_api_client/src/domain/service/test_support.rs
@@ -1,11 +1,11 @@
use std::sync::{Arc, Mutex};
use models_email::email::service::label::Label;
-use models_email::service::contact::{Contact, ContactList};
+use models_email::service::contact::Contact;
use uuid::Uuid;
use super::super::models::{
- AccessToken, ApiOperationKind, ChangeBatch, EmailApiError, ProviderSubscription,
+ AccessToken, ApiOperationKind, ChangePage, ContactPage, EmailApiError, ProviderSubscription,
RateLimitRefusal, RequestPermit, SendRequest, SentIds, SyncCursor, ThreadListPage, TokenError,
TokenFreshness,
};
@@ -157,12 +157,13 @@ impl MailboxSyncClient for FakeRepository {
self.unavailable("get_thread_count")
}
- async fn list_changes(
+ async fn list_changes_page(
&self,
_: &AccessToken,
_: &SyncCursor,
- ) -> Result<ChangeBatch, EmailApiError> {
- self.unavailable("list_changes")
+ _: Option<&str>,
+ ) -> Result<ChangePage, EmailApiError> {
+ self.unavailable("list_changes_page")
}
}
@@ -297,22 +298,24 @@ impl MailboxContactsClient for FakeRepository {
self.unavailable("get_self_contact")
}
- async fn list_contacts(
+ async fn list_contacts_page(
&self,
_: &AccessToken,
_: Uuid,
_: Option<&str>,
- ) -> Result<ContactList, EmailApiError> {
- self.unavailable("list_contacts")
+ _: Option<&str>,
+ ) -> Result<ContactPage, EmailApiError> {
+ self.unavailable("list_contacts_page")
}
- async fn list_other_contacts(
+ async fn list_other_contacts_page(
&self,
_: &AccessToken,
_: Uuid,
_: Option<&str>,
- ) -> Result<ContactList, EmailApiError> {
- self.unavailable("list_other_contacts")
+ _: Option<&str>,
+ ) -> Result<ContactPage, EmailApiError> {
+ self.unavailable("list_other_contacts_page")
}
}
diff --git a/crates/email_api_client/src/outbound/gmail/contacts.rs b/crates/email_api_client/src/outbound/gmail/contacts.rs
index 65ea815913..f416ffb8b6 100644
--- a/crates/email_api_client/src/outbound/gmail/contacts.rs
+++ b/crates/email_api_client/src/outbound/gmail/contacts.rs
@@ -1,9 +1,9 @@
//! Gmail contact capability implementation.
-use models_email::service::contact::{Contact, ContactList};
+use models_email::service::contact::Contact;
use uuid::Uuid;
-use crate::domain::models::{AccessToken, EmailApiError};
+use crate::domain::models::{AccessToken, ContactPage, EmailApiError};
use crate::domain::ports::MailboxContactsClient;
use super::convert::map_person_to_contact;
@@ -23,45 +23,59 @@ impl MailboxContactsClient for GmailApiClientRepository {
Ok(map_person_to_contact(link_id, person))
}
- async fn list_contacts(
+ async fn list_contacts_page(
&self,
access_token: &AccessToken,
link_id: Uuid,
sync_token: Option<&str>,
- ) -> Result<ContactList, EmailApiError> {
- let (people, next_sync_token) = self
+ page_token: Option<&str>,
+ ) -> Result<ContactPage, EmailApiError> {
+ let page = self
.client
- .get_contacts(access_token.expose_secret(), sync_token)
+ .get_contacts_page(access_token.expose_secret(), sync_token, page_token)
.await
.map_err(map_contacts_error)?;
- Ok(contact_list(link_id, people, next_sync_token))
+ Ok(contact_page(
+ link_id,
+ page.connections,
+ page.next_sync_token,
+ page.next_page_token,
+ ))
}
- async fn list_other_contacts(
+ async fn list_other_contacts_page(
&self,
access_token: &AccessToken,
link_id: Uuid,
sync_token: Option<&str>,
- ) -> Result<ContactList, EmailApiError> {
- let (people, next_sync_token) = self
+ page_token: Option<&str>,
+ ) -> Result<ContactPage, EmailApiError> {
+ let page = self
.client
- .get_other_contacts(access_token.expose_secret(), sync_token)
+ .get_other_contacts_page(access_token.expose_secret(), sync_token, page_token)
.await
.map_err(map_contacts_error)?;
- Ok(contact_list(link_id, people, next_sync_token))
+ Ok(contact_page(
+ link_id,
+ page.other_contacts,
+ page.next_sync_token,
+ page.next_page_token,
+ ))
}
}
-fn contact_list(
+fn contact_page(
link_id: Uuid,
people: Vec<models_email::gmail::contacts::PersonResource>,
- next_sync_token: String,
-) -> ContactList {
- ContactList {
+ next_sync_token: Option<String>,
+ next_page_token: Option<String>,
+) -> ContactPage {
+ ContactPage {
contacts: people
.into_iter()
.map(|person| map_person_to_contact(link_id, person))
.collect(),
next_sync_token,
+ next_page_token,
}
}
diff --git a/crates/email_api_client/src/outbound/gmail/convert/history.rs b/crates/email_api_client/src/outbound/gmail/convert/history.rs
index 7f75690751..9b0b0be880 100644
--- a/crates/email_api_client/src/outbound/gmail/convert/history.rs
+++ b/crates/email_api_client/src/outbound/gmail/convert/history.rs
@@ -1,15 +1,19 @@
use models_email::gmail::{HistoryListResponse, HistoryRecord};
-use crate::domain::models::{ChangeBatch, InboxChanges, SyncCursor};
+use crate::domain::models::{ChangePage, InboxChanges, SyncCursor};
-pub(crate) fn map_history_list_response_to_changes(response: HistoryListResponse) -> ChangeBatch {
+pub(crate) fn map_history_page(response: HistoryListResponse) -> ChangePage {
let mut changes = InboxChanges::default();
for record in response.history.unwrap_or_default() {
apply_history_record(&mut changes, &record);
}
- ChangeBatch::new(changes, SyncCursor::gmail(response.history_id))
+ ChangePage {
+ changes,
+ cursor: (!response.history_id.is_empty()).then(|| SyncCursor::gmail(response.history_id)),
+ next_page_token: response.next_page_token,
+ }
}
fn apply_history_record(changes: &mut InboxChanges, record: &HistoryRecord) {
diff --git a/crates/email_api_client/src/outbound/gmail/convert/history/test.rs b/crates/email_api_client/src/outbound/gmail/convert/history/test.rs
index e74be161c4..162494fcfc 100644
--- a/crates/email_api_client/src/outbound/gmail/convert/history/test.rs
+++ b/crates/email_api_client/src/outbound/gmail/convert/history/test.rs
@@ -2,7 +2,7 @@ use models_email::gmail::{
HistoryListResponse, HistoryMessage, HistoryRecord, LabelAdded, MessageAdded, MessageDeleted,
};
-use super::map_history_list_response_to_changes;
+use super::map_history_page;
use crate::domain::models::SyncCursor;
fn message(id: &str) -> HistoryMessage {
@@ -12,6 +12,47 @@ fn message(id: &str) -> HistoryMessage {
}
}
+fn page(records: &[HistoryRecord]) -> HistoryListResponse {
+ HistoryListResponse {
+ history: Some(records.to_vec()),
+ next_page_token: None,
+ history_id: "cursor".into(),
+ }
+}
+
+fn record(messages: &[&str], added: &[&str], deleted: &[&str], labelled: &[&str]) -> HistoryRecord {
+ HistoryRecord {
+ id: "1".into(),
+ messages: Some(messages.iter().map(|id| message(id)).collect()),
+ messages_added: Some(
+ added
+ .iter()
+ .map(|id| MessageAdded {
+ message: message(id),
+ })
+ .collect(),
+ ),
+ messages_deleted: Some(
+ deleted
+ .iter()
+ .map(|id| MessageDeleted {
+ message: message(id),
+ })
+ .collect(),
+ ),
+ labels_added: Some(
+ labelled
+ .iter()
+ .map(|id| LabelAdded {
+ message: message(id),
+ label_ids: vec!["STARRED".into()],
+ })
+ .collect(),
+ ),
+ labels_removed: None,
+ }
+}
+
#[test]
fn resolves_overlapping_changes_with_deletion_precedence() {
let response = HistoryListResponse {
@@ -34,23 +75,64 @@ fn resolves_overlapping_changes_with_deletion_precedence() {
}]),
};
- let batch = map_history_list_response_to_changes(response);
+ let page = map_history_page(response);
- assert_eq!(batch.next_cursor, SyncCursor::gmail("cursor-2"));
- assert!(batch.changes.message_ids_to_upsert.contains("implicit"));
- assert!(batch.changes.message_ids_to_delete.contains("deleted"));
- assert!(!batch.changes.message_ids_to_upsert.contains("deleted"));
- assert!(batch.changes.labels_to_update.contains("labels"));
+ assert_eq!(page.cursor, Some(SyncCursor::gmail("cursor-2")));
+ assert!(page.changes.message_ids_to_upsert.contains("implicit"));
+ assert!(page.changes.message_ids_to_delete.contains("deleted"));
+ assert!(!page.changes.message_ids_to_upsert.contains("deleted"));
+ assert!(page.changes.labels_to_update.contains("labels"));
}
#[test]
fn handles_an_empty_history_page() {
- let batch = map_history_list_response_to_changes(HistoryListResponse {
+ let page = map_history_page(HistoryListResponse {
history: None,
- next_page_token: None,
+ next_page_token: Some("next".into()),
history_id: "cursor".into(),
});
- assert!(batch.changes.message_ids_to_upsert.is_empty());
- assert_eq!(batch.next_cursor.as_str(), "cursor");
+ assert!(page.changes.message_ids_to_upsert.is_empty());
+ assert_eq!(page.cursor, Some(SyncCursor::gmail("cursor")));
+ assert_eq!(page.next_page_token.as_deref(), Some("next"));
+}
+
+#[test]
+fn a_page_without_a_history_id_reports_no_cursor() {
+ let page = map_history_page(HistoryListResponse {
+ history: None,
+ next_page_token: None,
+ history_id: String::new(),
+ });
+
+ assert_eq!(page.cursor, None);
+}
+
+#[test]
+fn appending_pages_matches_reading_the_history_in_one_pass() {
+ // Each record interacts with earlier ones: deletions after upserts and
+ // label changes, upserts and label changes after deletions, and upserts
+ // after label changes.
+ let records = [
+ record(&[], &["a"], &[], &["b"]),
+ record(&["c"], &[], &["a"], &["d"]),
+ record(&[], &["a"], &[], &["c", "e"]),
+ record(&["b"], &[], &["e", "f"], &[]),
+ record(&[], &["g"], &[], &["f", "d"]),
+ record(&["g"], &[], &["g"], &["h"]),
+ ];
+ let whole = map_history_page(page(&records)).changes;
+
+ for first_split in 0..=records.len() {
+ for second_split in first_split..=records.len() {
+ let mut changes = map_history_page(page(&records[..first_split])).changes;
+ changes
+ .append_later(map_history_page(page(&records[first_split..second_split])).changes);
+ changes.append_later(map_history_page(page(&records[second_split..])).changes);
+ assert_eq!(
+ changes, whole,
+ "pages split after records {first_split} and {second_split}"
+ );
+ }
+ }
}
diff --git a/crates/email_api_client/src/outbound/gmail/convert/mod.rs b/crates/email_api_client/src/outbound/gmail/convert/mod.rs
index 2d878c4613..0cafebcc49 100644
--- a/crates/email_api_client/src/outbound/gmail/convert/mod.rs
+++ b/crates/email_api_client/src/outbound/gmail/convert/mod.rs
@@ -8,7 +8,7 @@ mod payload;
mod thread;
pub(crate) use contact::map_person_to_contact;
-pub(crate) use history::map_history_list_response_to_changes;
+pub(crate) use history::map_history_page;
pub(crate) use label::{map_label_to_service, map_labels_to_service};
pub(crate) use message::map_message_resource_to_service;
pub(crate) use thread::map_thread_resource_to_service;
diff --git a/crates/email_api_client/src/outbound/gmail/sync.rs b/crates/email_api_client/src/outbound/gmail/sync.rs
index 772a2dd789..4b0fec1461 100644
--- a/crates/email_api_client/src/outbound/gmail/sync.rs
+++ b/crates/email_api_client/src/outbound/gmail/sync.rs
@@ -1,9 +1,9 @@
//! Gmail mailbox-sync capability implementation.
-use crate::domain::models::{AccessToken, ChangeBatch, EmailApiError, SyncCursor};
+use crate::domain::models::{AccessToken, ChangePage, EmailApiError, SyncCursor};
use crate::domain::ports::MailboxSyncClient;
-use super::convert::map_history_list_response_to_changes;
+use super::convert::map_history_page;
use super::{GmailApiClientRepository, map_gmail_error, map_history_error};
impl MailboxSyncClient for GmailApiClientRepository {
@@ -21,16 +21,17 @@ impl MailboxSyncClient for GmailApiClientRepository {
})
}
- async fn list_changes(
+ async fn list_changes_page(
&self,
access_token: &AccessToken,
cursor: &SyncCursor,
- ) -> Result<ChangeBatch, EmailApiError> {
+ page_token: Option<&str>,
+ ) -> Result<ChangePage, EmailApiError> {
let history = self
.client
- .get_history(access_token.expose_secret(), cursor.as_str())
+ .get_history_page(access_token.expose_secret(), cursor.as_str(), page_token)
.await
.map_err(map_history_error)?;
- Ok(map_history_list_response_to_changes(history))
+ Ok(map_history_page(history))
}
}
diff --git a/crates/email_api_client/src/outbound/gmail/test/contacts.rs b/crates/email_api_client/src/outbound/gmail/test/contacts.rs
index f8a4da0582..736b7c4e06 100644
--- a/crates/email_api_client/src/outbound/gmail/test/contacts.rs
+++ b/crates/email_api_client/src/outbound/gmail/test/contacts.rs
@@ -1,5 +1,5 @@
use uuid::Uuid;
-use wiremock::matchers::{method, path, query_param};
+use wiremock::matchers::{method, path, query_param, query_param_is_missing};
use wiremock::{Mock, ResponseTemplate};
use super::repository;
@@ -7,16 +7,17 @@ use crate::domain::models::AccessToken;
use crate::domain::ports::MailboxContactsClient;
#[tokio::test]
-async fn paginates_contacts_and_returns_the_final_sync_token() {
+async fn reads_contacts_one_page_at_a_time() {
let (server, repository) = repository().await;
Mock::given(method("GET"))
.and(path("/people/me/connections"))
.and(query_param("requestSyncToken", "true"))
+ .and(query_param_is_missing("pageToken"))
.respond_with(ResponseTemplate::new(200).set_body_raw(
include_str!("fixtures/contacts_page_1.json"),
"application/json",
))
- .up_to_n_times(1)
+ .expect(1)
.mount(&server)
.await;
Mock::given(method("GET"))
@@ -26,17 +27,28 @@ async fn paginates_contacts_and_returns_the_final_sync_token() {
include_str!("fixtures/contacts_page_2.json"),
"application/json",
))
+ .expect(1)
.mount(&server)
.await;
+ let token = AccessToken::new("token");
+ let link_id = Uuid::now_v7();
+
+ let first = repository
+ .list_contacts_page(&token, link_id, None, None)
+ .await
+ .unwrap();
+ assert_eq!(first.contacts.len(), 1);
+ assert_eq!(first.next_page_token.as_deref(), Some("contacts-next"));
+ assert_eq!(first.next_sync_token, None);
- let contacts = repository
- .list_contacts(&AccessToken::new("token"), Uuid::now_v7(), None)
+ let last = repository
+ .list_contacts_page(&token, link_id, None, Some("contacts-next"))
.await
.unwrap();
- assert_eq!(contacts.contacts.len(), 2);
- assert_eq!(contacts.next_sync_token, "contacts-sync-final");
+ assert_eq!(last.next_page_token, None);
+ assert_eq!(last.next_sync_token.as_deref(), Some("contacts-sync-final"));
assert_eq!(
- contacts.contacts[1].original_photo_url.as_deref(),
+ last.contacts[0].original_photo_url.as_deref(),
Some("https://lh3.googleusercontent.test/second=s128")
);
}
@@ -74,9 +86,9 @@ async fn normalizes_self_and_other_contacts() {
);
let other = repository
- .list_other_contacts(&AccessToken::new("token"), link_id, Some("previous"))
+ .list_other_contacts_page(&AccessToken::new("token"), link_id, Some("previous"), None)
.await
.unwrap();
- assert_eq!(other.next_sync_token, "other-sync-final");
+ assert_eq!(other.next_sync_token.as_deref(), Some("other-sync-final"));
assert_eq!(other.contacts[0].link_id, link_id);
}
diff --git a/crates/email_api_client/src/outbound/gmail/test/sync.rs b/crates/email_api_client/src/outbound/gmail/test/sync.rs
index e767e3b6c7..505cd9449f 100644
--- a/crates/email_api_client/src/outbound/gmail/test/sync.rs
+++ b/crates/email_api_client/src/outbound/gmail/test/sync.rs
@@ -1,4 +1,4 @@
-use wiremock::matchers::{method, path, query_param};
+use wiremock::matchers::{method, path, query_param, query_param_is_missing};
use wiremock::{Mock, ResponseTemplate};
use super::repository;
@@ -27,37 +27,49 @@ async fn reads_profile_thread_count() {
}
#[tokio::test]
-async fn paginates_and_partitions_history() {
+async fn reads_and_partitions_one_history_page_at_a_time() {
let (server, repository) = repository().await;
Mock::given(method("GET"))
.and(path("/users/me/history"))
.and(query_param("startHistoryId", "100"))
+ .and(query_param_is_missing("pageToken"))
.respond_with(ResponseTemplate::new(200).set_body_raw(
include_str!("fixtures/history_page_1.json"),
"application/json",
))
- .up_to_n_times(1)
+ .expect(1)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path("/users/me/history"))
+ .and(query_param("startHistoryId", "100"))
.and(query_param("pageToken", "next-page"))
.respond_with(ResponseTemplate::new(200).set_body_raw(
include_str!("fixtures/history_page_2.json"),
"application/json",
))
+ .expect(1)
.mount(&server)
.await;
+ let token = AccessToken::new("token");
+ let cursor = SyncCursor::gmail("100");
- let batch = repository
- .list_changes(&AccessToken::new("token"), &SyncCursor::gmail("100"))
+ let first = repository
+ .list_changes_page(&token, &cursor, None)
.await
.unwrap();
+ assert_eq!(first.cursor, Some(SyncCursor::gmail("101")));
+ assert_eq!(first.next_page_token.as_deref(), Some("next-page"));
+ assert!(first.changes.message_ids_to_upsert.contains("added"));
+ assert!(first.changes.message_ids_to_delete.contains("deleted"));
- assert_eq!(batch.next_cursor.as_str(), "102");
- assert!(batch.changes.message_ids_to_upsert.contains("added"));
- assert!(batch.changes.message_ids_to_delete.contains("deleted"));
- assert!(batch.changes.labels_to_update.contains("labels"));
+ let last = repository
+ .list_changes_page(&token, &cursor, Some("next-page"))
+ .await
+ .unwrap();
+ assert_eq!(last.cursor, Some(SyncCursor::gmail("102")));
+ assert_eq!(last.next_page_token, None);
+ assert!(last.changes.labels_to_update.contains("labels"));
}
#[tokio::test]
@@ -74,7 +86,11 @@ async fn only_history_not_found_is_an_outdated_cursor() {
assert_eq!(
repository
- .list_changes(&AccessToken::new("token"), &SyncCursor::gmail("stale"))
+ .list_changes_page(
+ &AccessToken::new("token"),
+ &SyncCursor::gmail("stale"),
+ None
+ )
.await,
Err(EmailApiError::OutdatedCursor)
);
diff --git a/crates/gmail_client/AGENTS.md b/crates/gmail_client/AGENTS.md
index 137f5bad3b..ad55373906 100644
--- a/crates/gmail_client/AGENTS.md
+++ b/crates/gmail_client/AGENTS.md
@@ -45,6 +45,9 @@ contacts, watch) behind a single public facade; most modules are
- Requests time out after 60 s (`REQUEST_TIMEOUT`). The email service's
Gmail request gate holds a permit for each call and sizes its lease above
this (`PERMIT_MS` in `gmail_gate.rs`); keep the two in step.
+- History and contacts are fetched one page per call (`get_history_page`,
+ `get_contacts_page`, `get_other_contacts_page`) so each page takes its own
+ gate permit. Don't add methods that loop over pages here.
## Related
diff --git a/crates/gmail_client/src/contacts.rs b/crates/gmail_client/src/contacts.rs
index 2ba421a69c..b42d2f6c23 100644
--- a/crates/gmail_client/src/contacts.rs
+++ b/crates/gmail_client/src/contacts.rs
@@ -26,119 +26,83 @@ pub(crate) async fn get_self_connection(
decode_json_response(response).await
}
-/// Fetches all pages of the user's contacts and returns raw People resources.
+/// Gets one page of the user's contacts as raw People resources.
///
/// Failures log at WARN: expired sync tokens and links granted without the
/// contacts scopes are routine, and the contacts sync recovers or skips them.
-#[tracing::instrument(skip(client, access_token, sync_token), err(level = "warn"))]
-pub(crate) async fn list_connections(
+#[tracing::instrument(
+ skip(client, access_token, sync_token, page_token),
+ err(level = "warn")
+)]
+pub(crate) async fn get_connections_page(
client: &GmailClient,
access_token: &str,
sync_token: Option<&str>,
-) -> Result<(Vec<PersonResource>, String), GmailApiHttpError> {
- let mut people = Vec::new();
- let mut page_token: Option<String> = None;
- let mut next_sync_token: Option<String> = None;
-
- loop {
- let mut request = client
- .inner
- .get(format!("{}/people/me/connections", client.contacts_url))
- .bearer_auth(access_token)
- .query(&[
- ("personFields", PERSON_FIELDS),
- ("requestSyncToken", "true"),
- ]);
- if let Some(token) = sync_token {
- request = request.query(&[("syncToken", token)]);
- } else {
- request = request.query(&[("pageSize", "1000")]);
- }
- if let Some(token) = page_token.as_deref() {
- request = request.query(&[("pageToken", token)]);
- }
-
- let response = request.send().await.map_err(GmailApiHttpError::transport)?;
- if !response.status().is_success() {
- return Err(unsuccessful_response(response).await);
- }
-
- let page: ConnectionsResponse = decode_json_response(response).await?;
- people.extend(page.connections);
- if page.next_sync_token.is_some() {
- next_sync_token = page.next_sync_token;
- }
- page_token = page.next_page_token;
+ page_token: Option<&str>,
+) -> Result<ConnectionsResponse, GmailApiHttpError> {
+ let mut request = client
+ .inner
+ .get(format!("{}/people/me/connections", client.contacts_url))
+ .bearer_auth(access_token)
+ .query(&[
+ ("personFields", PERSON_FIELDS),
+ ("requestSyncToken", "true"),
+ ]);
+ if let Some(token) = sync_token {
+ request = request.query(&[("syncToken", token)]);
+ } else {
+ request = request.query(&[("pageSize", "1000")]);
+ }
+ if let Some(token) = page_token {
+ request = request.query(&[("pageToken", token)]);
+ }
- if page_token.is_none() {
- break;
- }
+ let response = request.send().await.map_err(GmailApiHttpError::transport)?;
+ if !response.status().is_success() {
+ return Err(unsuccessful_response(response).await);
}
- let next_sync_token = next_sync_token.ok_or_else(|| {
- GmailApiHttpError::InvalidResponse(
- "People API did not return a nextSyncToken for connections".to_string(),
- )
- })?;
- Ok((people, next_sync_token))
+ decode_json_response(response).await
}
-/// Fetches all pages of "Other Contacts" and returns raw People resources.
+/// Gets one page of "Other Contacts" as raw People resources.
///
-/// Failures log at WARN for the same reasons as [`list_connections`].
-#[tracing::instrument(skip(client, access_token, sync_token), err(level = "warn"))]
-pub(crate) async fn list_other_contacts(
+/// Failures log at WARN for the same reasons as [`get_connections_page`].
+#[tracing::instrument(
+ skip(client, access_token, sync_token, page_token),
+ err(level = "warn")
+)]
+pub(crate) async fn get_other_contacts_page(
client: &GmailClient,
access_token: &str,
sync_token: Option<&str>,
-) -> Result<(Vec<PersonResource>, String), GmailApiHttpError> {
- let mut people = Vec::new();
- let mut page_token: Option<String> = None;
- let mut next_sync_token: Option<String> = None;
-
- loop {
- let mut request = client
- .inner
- .get(format!("{}/otherContacts", client.contacts_url))
- .bearer_auth(access_token)
- .query(&[
- ("readMask", PERSON_FIELDS),
- ("sources", "READ_SOURCE_TYPE_CONTACT"),
- ("sources", "READ_SOURCE_TYPE_PROFILE"),
- ("requestSyncToken", "true"),
- ]);
- if let Some(token) = sync_token {
- request = request.query(&[("syncToken", token)]);
- } else {
- request = request.query(&[("pageSize", "1000")]);
- }
- if let Some(token) = page_token.as_deref() {
- request = request.query(&[("pageToken", token)]);
- }
-
- let response = request.send().await.map_err(GmailApiHttpError::transport)?;
- if !response.status().is_success() {
- return Err(unsuccessful_response(response).await);
- }
-
- let page: OtherContactsResponse = decode_json_response(response).await?;
- people.extend(page.other_contacts);
- if page.next_sync_token.is_some() {
- next_sync_token = page.next_sync_token;
- }
- page_token = page.next_page_token;
+ page_token: Option<&str>,
+) -> Result<OtherContactsResponse, GmailApiHttpError> {
+ let mut request = client
+ .inner
+ .get(format!("{}/otherContacts", client.contacts_url))
+ .bearer_auth(access_token)
+ .query(&[
+ ("readMask", PERSON_FIELDS),
+ ("sources", "READ_SOURCE_TYPE_CONTACT"),
+ ("sources", "READ_SOURCE_TYPE_PROFILE"),
+ ("requestSyncToken", "true"),
+ ]);
+ if let Some(token) = sync_token {
+ request = request.query(&[("syncToken", token)]);
+ } else {
+ request = request.query(&[("pageSize", "1000")]);
+ }
+ if let Some(token) = page_token {
+ request = request.query(&[("pageToken", token)]);
+ }
- if page_token.is_none() {
- break;
- }
+ let response = request.send().await.map_err(GmailApiHttpError::transport)?;
+ if !response.status().is_success() {
+ return Err(unsuccessful_response(response).await);
}
- let next_sync_token = next_sync_token.ok_or_else(|| {
- GmailApiHttpError::InvalidResponse(
- "People API did not return a nextSyncToken for other contacts".to_string(),
- )
- })?;
- Ok((people, next_sync_token))
+ decode_json_response(response).await
}
#[cfg(test)]
diff --git a/crates/gmail_client/src/contacts/test.rs b/crates/gmail_client/src/contacts/test.rs
index abed4c4b34..dbe531d4de 100644
--- a/crates/gmail_client/src/contacts/test.rs
+++ b/crates/gmail_client/src/contacts/test.rs
@@ -14,7 +14,7 @@ fn client(server: &MockServer) -> GmailClient {
}
#[tokio::test]
-async fn connections_paginate_and_return_the_final_sync_token() {
+async fn each_connections_page_is_one_request_that_returns_the_next_page_token() {
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/people/me/connections"))
@@ -39,16 +39,22 @@ async fn connections_paginate_and_return_the_final_sync_token() {
.mount(&server)
.await;
- let (people, sync_token) = list_connections(&client(&server), "token", None)
+ let first = get_connections_page(&client(&server), "token", None, None)
.await
- .expect("connections should decode");
+ .expect("first page should decode");
+ assert_eq!(first.connections.len(), 1);
+ assert_eq!(first.next_page_token.as_deref(), Some("page-2"));
+ assert_eq!(first.next_sync_token, None);
- assert_eq!(people.len(), 2);
- assert_eq!(sync_token, "sync-final");
+ let last = get_connections_page(&client(&server), "token", None, Some("page-2"))
+ .await
+ .expect("last page should decode");
+ assert_eq!(last.next_page_token, None);
+ assert_eq!(last.next_sync_token.as_deref(), Some("sync-final"));
}
#[tokio::test]
-async fn other_contacts_incremental_sync_paginates() {
+async fn other_contacts_pages_send_the_sync_token_with_every_page() {
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/otherContacts"))
@@ -74,48 +80,16 @@ async fn other_contacts_incremental_sync_paginates() {
.mount(&server)
.await;
- let (people, sync_token) = list_other_contacts(&client(&server), "token", Some("old-sync"))
- .await
- .expect("other contacts should decode");
-
- assert_eq!(people.len(), 2);
- assert_eq!(sync_token, "new-sync");
-}
-
-#[tokio::test]
-async fn missing_connections_sync_token_is_an_invalid_response() {
- let server = MockServer::start().await;
- Mock::given(method("GET"))
- .and(path("/people/me/connections"))
- .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
- "connections": []
- })))
- .mount(&server)
- .await;
-
- let error = list_connections(&client(&server), "token", None)
+ let first = get_other_contacts_page(&client(&server), "token", Some("old-sync"), None)
.await
- .expect_err("missing sync token should fail");
-
- assert!(matches!(error, GmailApiHttpError::InvalidResponse(_)));
-}
+ .expect("first page should decode");
+ assert_eq!(first.next_page_token.as_deref(), Some("page-2"));
-#[tokio::test]
-async fn missing_other_contacts_sync_token_is_an_invalid_response() {
- let server = MockServer::start().await;
- Mock::given(method("GET"))
- .and(path("/otherContacts"))
- .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
- "otherContacts": []
- })))
- .mount(&server)
- .await;
-
- let error = list_other_contacts(&client(&server), "token", None)
+ let last = get_other_contacts_page(&client(&server), "token", Some("old-sync"), Some("page-2"))
.await
- .expect_err("missing sync token should fail");
-
- assert!(matches!(error, GmailApiHttpError::InvalidResponse(_)));
+ .expect("last page should decode");
+ assert_eq!(last.other_contacts.len(), 1);
+ assert_eq!(last.next_sync_token.as_deref(), Some("new-sync"));
}
#[tokio::test]
diff --git a/crates/gmail_client/src/history.rs b/crates/gmail_client/src/history.rs
index 141bcb5a5e..49776ef6d4 100644
--- a/crates/gmail_client/src/history.rs
+++ b/crates/gmail_client/src/history.rs
@@ -4,53 +4,33 @@ use models_email::gmail::HistoryListResponse;
const MAX_RESULTS: usize = 500;
-/// Gets the changes to a user's inbox that have occurred since `start_history_id`.
-/// Returns the raw Gmail response for conversion by the caller.
-#[tracing::instrument(skip(client, access_token), err)]
-pub(crate) async fn get_history(
+/// Gets one page of the changes to a user's inbox that have occurred since
+/// `start_history_id`. Returns the raw Gmail response for conversion by the caller.
+#[tracing::instrument(skip(client, access_token, page_token), err)]
+pub(crate) async fn get_history_page(
client: &GmailClient,
access_token: &str,
start_history_id: &str,
+ page_token: Option<&str>,
) -> Result<HistoryListResponse, GmailApiHttpError> {
- let mut history = Vec::new();
- let mut history_id = String::new();
- let mut page_token: Option<String> = None;
-
- loop {
- let mut request = client
- .inner
- .get(format!("{}/users/me/history", client.base_url))
- .bearer_auth(access_token)
- .query(&[
- ("startHistoryId", start_history_id),
- ("maxResults", &MAX_RESULTS.to_string()),
- ]);
- if let Some(token) = page_token.as_deref() {
- request = request.query(&[("pageToken", token)]);
- }
-
- let response = request.send().await.map_err(GmailApiHttpError::transport)?;
- if !response.status().is_success() {
- return Err(unsuccessful_response(response).await);
- }
-
- let page: HistoryListResponse = decode_json_response(response).await?;
- if let Some(page_history) = page.history {
- history.extend(page_history);
- }
- if !page.history_id.is_empty() {
- history_id = page.history_id;
- }
- page_token = page.next_page_token;
+ let mut request = client
+ .inner
+ .get(format!("{}/users/me/history", client.base_url))
+ .bearer_auth(access_token)
+ .query(&[
+ ("startHistoryId", start_history_id),
+ ("maxResults", &MAX_RESULTS.to_string()),
+ ]);
+ if let Some(token) = page_token {
+ request = request.query(&[("pageToken", token)]);
+ }
- if page_token.is_none() {
- return Ok(HistoryListResponse {
- history: Some(history),
- next_page_token: None,
- history_id,
- });
- }
+ let response = request.send().await.map_err(GmailApiHttpError::transport)?;
+ if !response.status().is_success() {
+ return Err(unsuccessful_response(response).await);
}
+
+ decode_json_response(response).await
}
#[cfg(test)]
diff --git a/crates/gmail_client/src/history/test.rs b/crates/gmail_client/src/history/test.rs
index 3cf371fb50..2bd71a6b58 100644
--- a/crates/gmail_client/src/history/test.rs
+++ b/crates/gmail_client/src/history/test.rs
@@ -15,7 +15,7 @@ fn client(server: &MockServer) -> GmailClient {
}
#[tokio::test]
-async fn history_paginates_and_advances_to_the_latest_cursor() {
+async fn each_history_page_is_one_request_that_returns_the_next_page_token() {
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/users/me/history"))
@@ -35,6 +35,7 @@ async fn history_paginates_and_advances_to_the_latest_cursor() {
.await;
Mock::given(method("GET"))
.and(path("/users/me/history"))
+ .and(query_param("startHistoryId", "100"))
.and(query_param("pageToken", "next"))
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
"history": [{
@@ -47,13 +48,18 @@ async fn history_paginates_and_advances_to_the_latest_cursor() {
.mount(&server)
.await;
- let response = get_history(&client(&server), "token", "100")
+ let first = get_history_page(&client(&server), "token", "100", None)
.await
- .expect("history should decode");
+ .expect("first page should decode");
+ assert_eq!(first.history_id, "101");
+ assert_eq!(first.next_page_token.as_deref(), Some("next"));
- assert_eq!(response.history_id, "102");
- assert_eq!(response.history.unwrap().len(), 2);
- assert!(response.next_page_token.is_none());
+ let last = get_history_page(&client(&server), "token", "100", Some("next"))
+ .await
+ .expect("last page should decode");
+ assert_eq!(last.history_id, "102");
+ assert_eq!(last.history.unwrap().len(), 1);
+ assert!(last.next_page_token.is_none());
}
#[tokio::test]
@@ -68,7 +74,7 @@ async fn stale_history_not_found_status_is_preserved() {
.mount(&server)
.await;
- let error = get_history(&client(&server), "token", "stale")
+ let error = get_history_page(&client(&server), "token", "stale", None)
.await
.expect_err("stale history should fail");
diff --git a/crates/gmail_client/src/lib.rs b/crates/gmail_client/src/lib.rs
index b02db81072..fd4dcfd09c 100644
--- a/crates/gmail_client/src/lib.rs
+++ b/crates/gmail_client/src/lib.rs
@@ -49,7 +49,7 @@ use crate::contacts::get_self_connection;
use crate::messages::{get_message, get_message_label_ids, list_messages};
use crate::threads::get_thread;
pub use error::GmailApiHttpError;
-use models_email::gmail::contacts::PersonResource;
+use models_email::gmail::contacts::{ConnectionsResponse, OtherContactsResponse, PersonResource};
pub use models_email::gmail::filters::Filter;
use models_email::gmail::inbox_sync::{
GoogleJwtClaims, GooglePublicKeys, JwtVerificationError, KeyMap,
@@ -181,15 +181,17 @@ impl GmailClient {
get_thread(self, access_token, thread_id).await
}
- /// Gets the changes to a user's inbox that have occurred since start_history_id.
- /// Returns raw HistoryListResponse - callers should map to InboxChanges using convert module.
- #[tracing::instrument(skip(self, access_token), err)]
- pub async fn get_history(
+ /// Gets one page of the changes to a user's inbox since `start_history_id`.
+ /// Pass the previous page's `next_page_token` to continue. Returns the raw
+ /// HistoryListResponse; callers map it to inbox changes.
+ #[tracing::instrument(skip(self, access_token, page_token), err)]
+ pub async fn get_history_page(
&self,
access_token: &str,
start_history_id: &str,
+ page_token: Option<&str>,
) -> Result<HistoryListResponse, GmailApiHttpError> {
- history::get_history(self, access_token, start_history_id).await
+ history::get_history_page(self, access_token, start_history_id, page_token).await
}
/// Fetches the user's raw Gmail profile.
@@ -383,31 +385,31 @@ impl GmailClient {
get_self_connection(self, access_token).await
}
- /// Fetches all of the user's main contacts, handling pagination.
- /// Returns raw Gmail PersonResource objects and a sync token for future incremental updates.
- /// Callers should map PersonResource to service layer Contact.
+ /// Fetches one page of the user's main contacts. Pass the previous page's
+ /// `next_page_token` to continue; the last page carries the sync token for
+ /// future incremental updates. Callers map the people to service contacts.
/// Failures log at WARN: expired sync tokens and missing contacts scopes are routine.
- #[tracing::instrument(skip(self, access_token, sync_token), err(level = "warn"))]
- pub async fn get_contacts(
+ #[tracing::instrument(skip(self, access_token, sync_token, page_token), err(level = "warn"))]
+ pub async fn get_contacts_page(
&self,
access_token: &str,
sync_token: Option<&str>,
- ) -> Result<(Vec<PersonResource>, String), GmailApiHttpError> {
- contacts::list_connections(self, access_token, sync_token).await
+ page_token: Option<&str>,
+ ) -> Result<ConnectionsResponse, GmailApiHttpError> {
+ contacts::get_connections_page(self, access_token, sync_token, page_token).await
}
- /// Fetches all of the user's "Other Contacts", handling pagination.
- /// These are typically contacts auto-created from interactions.
- /// Returns raw Gmail PersonResource objects and a sync token.
- /// Callers should map PersonResource to service layer Contact.
+ /// Fetches one page of the user's "Other Contacts", typically created from
+ /// interactions. Paging and sync tokens work as in [`Self::get_contacts_page`].
/// Failures log at WARN: expired sync tokens and missing contacts scopes are routine.
- #[tracing::instrument(skip(self, access_token, sync_token), err(level = "warn"))]
- pub async fn get_other_contacts(
+ #[tracing::instrument(skip(self, access_token, sync_token, page_token), err(level = "warn"))]
+ pub async fn get_other_contacts_page(
&self,
access_token: &str,
sync_token: Option<&str>,
- ) -> Result<(Vec<PersonResource>, String), GmailApiHttpError> {
- contacts::list_other_contacts(self, access_token, sync_token).await
+ page_token: Option<&str>,
+ ) -> Result<OtherContactsResponse, GmailApiHttpError> {
+ contacts::get_other_contacts_page(self, access_token, sync_token, page_token).await
}
/// Creates a new Gmail filter.