Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

22.5 KB
Raw Blame
//! One Gmail mailbox wired to the production worker and domain code.
//!
//! Real: Postgres (migrated schema), Redis (request gate or limiter, token
//! cache), the `GmailApi` service with its `GmailClient`, the gmail_ops and
//! inbox_sync `process_message` entry points, and the `EmailServiceImpl` the
//! HTTP routes use for archive, read, star, label and trash actions.
//! Faked: Gmail's HTTP API, SQS's HTTP API, and catch-all 200 responses for
//! the auth service, SFS, DSS and connection gateway.

use super::fake_gmail::{FakeGmail, GmailMessage, GmailState};
use super::fake_sqs::{self, FakeSqs};
use super::wiring;
use crate::pubsub::context::{CrmMetadataResolver, PubSubContext};
use crate::pubsub::gmail_ops::worker::GmailOpsContext;
use axum::Router;
use axum::response::IntoResponse;
use email::domain::service::EmailServiceImpl;
use email::outbound::EmailPgRepo;
use frecency::domain::services::FrecencyQueryServiceImpl;
use frecency::outbound::postgres::FrecencyPgStorage;
use macro_event_broker::{KafkaEventPublisher, MacroEventBrokerService};
use macro_user_id::user_id::MacroUserIdStr;
use models_email::email::service::cache::TokenCacheKey;
use sqlx::PgPool;
use std::collections::{BTreeMap, BTreeSet};
use std::sync::Arc;
use tokio_util::task::TaskTracker;
use uuid::Uuid;

pub const INBOX: &str = "INBOX";
pub const UNREAD: &str = "UNREAD";
pub const STARRED: &str = "STARRED";
pub const TRASH: &str = "TRASH";
pub const WORK: &str = "Label_work";

pub const THREADS: [&str; 3] = ["t1", "t2", "t3"];
/// Provider message ids, their thread, and their starting labels.
pub const MESSAGES: [(&str, &str, &[&str]); 6] = [
    ("m1a", "t1", &[INBOX, UNREAD]),
    ("m1b", "t1", &[INBOX, STARRED]),
    ("m1c", "t1", &[INBOX]),
    ("m2a", "t2", &[INBOX, UNREAD]),
    ("m2b", "t2", &[INBOX]),
    ("m3a", "t3", &[INBOX, UNREAD, WORK]),
];
/// Labels the simulation tracks per message.
pub const TRACKED: [&str; 5] = [INBOX, UNREAD, STARRED, TRASH, WORK];
const LABELS: [(&str, &str, &str); 7] = [
    ("INBOX", "INBOX", "system"),
    ("UNREAD", "UNREAD", "system"),
    ("STARRED", "STARRED", "system"),
    ("TRASH", "TRASH", "system"),
    ("SPAM", "SPAM", "system"),
    ("SENT", "SENT", "system"),
    ("Label_work", "Work", "user"),
];

pub type EmailSvc = EmailServiceImpl<
    EmailPgRepo,
    FrecencyQueryServiceImpl<FrecencyPgStorage>,
    sqs_client::SQS,
    crm::domain::service::CrmServiceImpl<
        crm::outbound::companies_repo::CompaniesRepositoryImpl,
        crm::outbound::no_op_resolver::NoOpCompanyMetadataResolver,
    >,
    entity_access_management::domain::service::EntityAccessManagementServiceImpl<
        entity_access_management::outbound::PgRepository,
    >,
    MacroEventBrokerService<KafkaEventPublisher, TaskTracker>,
>;

pub struct Queues {
    pub ops: String,
    pub ops_retry: String,
    pub sync: String,
    pub sync_retry: String,
}

pub struct World {
    pub db: PgPool,
    pub gmail: FakeGmail,
    pub sqs: FakeSqs,
    pub redis: redis::aio::MultiplexedConnection,
    pub link_id: Uuid,
    pub queues: Queues,
    pub ops: GmailOpsContext,
    pub ops_retry: GmailOpsContext,
    pub sync: PubSubContext,
    pub sync_retry: PubSubContext,
    pub email: EmailSvc,
    /// Provider message id → database id.
    pub message_ids: BTreeMap<String, Uuid>,
    /// Provider thread id → database id.
    pub thread_ids: BTreeMap<String, Uuid>,
    pub label_ids: BTreeMap<String, Uuid>,
    /// Whether producers send label writes as batches or, as they do by
    /// default, one single-message operation per changed message.
    pub label_write_batches: bool,
}

/// Runs `test` on a runtime of its own, connected to `pool`'s database, with
/// two worker threads of `worker_stack` bytes of stack each, as the email
/// workers have. Tasks `test` spawns run on those threads.
pub fn on_worker_stacks<F, Fut>(pool: &PgPool, worker_stack: usize, test: F)
where
    F: FnOnce(PgPool) -> Fut + Send + 'static,
    Fut: std::future::Future<Output = ()>,
{
    let options = pool.connect_options().as_ref().clone();
    let outcome = std::thread::spawn(move || {
        tokio::runtime::Builder::new_multi_thread()
            .worker_threads(2)
            .thread_stack_size(worker_stack)
            .enable_all()
            .build()
            .expect("test runtime")
            .block_on(async move {
                let pool = sqlx::postgres::PgPoolOptions::new()
                    .connect_with(options)
                    .await
                    .expect("test database");
                test(pool).await;
            });
    })
    .join();
    if let Err(panic) = outcome {
        std::panic::resume_unwind(panic);
    }
}

pub fn owner() -> MacroUserIdStr<'static> {
    MacroUserIdStr::try_from_email("[email protected]").unwrap()
}

pub fn link_uuid(shard: u32) -> Uuid {
    Uuid::from_u128(0x5100_0000_0000_0000_0000_0000_0000_0000 + u128::from(shard))
}

fn thread_uuid(shard: u32, index: usize) -> Uuid {
    Uuid::from_u128(
        0x5200_0000_0000_0000_0000_0000_0000_0000 + (u128::from(shard) << 16) + index as u128,
    )
}

fn message_uuid(shard: u32, index: usize) -> Uuid {
    Uuid::from_u128(
        0x5300_0000_0000_0000_0000_0000_0000_0000 + (u128::from(shard) << 16) + index as u128,
    )
}

fn label_uuid(shard: u32, index: usize) -> Uuid {
    Uuid::from_u128(
        0x5400_0000_0000_0000_0000_0000_0000_0000 + (u128::from(shard) << 16) + index as u128,
    )
}

pub fn initial_gmail() -> GmailState {
    let mut state = GmailState::default();
    state.history_id = 100;
    state.labels = LABELS
        .iter()
        .map(|(id, name, kind)| (id.to_string(), name.to_string(), kind.to_string()))
        .collect();
    for (index, (id, thread, labels)) in MESSAGES.iter().enumerate() {
        state.messages.insert(
            id.to_string(),
            GmailMessage {
                thread_id: thread.to_string(),
                labels: labels.iter().map(|label| label.to_string()).collect(),
                internal_date_ms: 1_790_000_000_000 + index as i64 * 60_000,
            },
        );
    }
    state
}

impl World {
    pub async fn new(db: PgPool, shard: u32, label_write_batches: bool) -> World {
        let gmail = FakeGmail::start(initial_gmail()).await;
        let sqs = FakeSqs::start().await;
        let services_url = start_catch_all().await;
        let link_id = link_uuid(shard);
        let queues = Queues {
            ops: sqs.queue_url("gmail-ops"),
            ops_retry: sqs.queue_url("gmail-ops-retry"),
            sync: sqs.queue_url("gmail-webhook"),
            sync_retry: sqs.queue_url("gmail-webhook-retry"),
        };
        let mut world = World::compose(
            db,
            gmail,
            sqs.clone(),
            fake_sqs::client(&sqs.url, false),
            &services_url,
            link_id,
            queues,
            label_write_batches,
        )
        .await;
        world.seed(shard).await;
        world
    }

    /// Wires the production contexts to `gmail` and an SQS endpoint.
    async fn compose(
        db: PgPool,
        gmail: FakeGmail,
        sqs: FakeSqs,
        sqs_sdk: aws_sdk_sqs::Client,
        services_url: &str,
        link_id: Uuid,
        queues: Queues,
        label_write_batches: bool,
    ) -> World {
        let redis_url =
            std::env::var("REDIS_URI").unwrap_or_else(|_| "redis://127.0.0.1:6379".to_string());
        let redis_client = redis::Client::open(redis_url).expect("redis url");
        let redis = redis_client
            .get_multiplexed_async_connection()
            .await
            .expect("redis connection");
        let sqs_client = sqs_client::SQS::new(sqs_sdk.clone())
            .gmail_inbox_sync_queue(&queues.sync)
            .gmail_inbox_sync_retry_queue(&queues.sync_retry)
            .gmail_ops_queue(&queues.ops)
            .gmail_ops_retry_queue(&queues.ops_retry)
            .gmail_label_write_batches(label_write_batches)
            .email_backfill_queue(&format!("{services_url}/queue/backfill"))
            .email_link_manager_queue(&format!("{services_url}/queue/link-manager"))
            .sfs_uploader_queue(&format!("{services_url}/queue/sfs"));
        let email_api = wiring::gmail_api(
            db.clone(),
            &redis_client,
            &gmail.url,
            services_url,
            sqs_client.clone(),
        )
        .await;
        let worker =
            |queue: &str| sqs_worker::SQSWorker::new(sqs_sdk.clone(), queue.to_string(), 1, 1);
        let ops_context = |retry_worker: bool, queue: &str| GmailOpsContext {
            db: db.clone(),
            sqs_worker: worker(queue),
            sqs_client: sqs_client.clone(),
            email_api: email_api.clone(),
            retry_worker,
        };
        let broker = MacroEventBrokerService::new(
            KafkaEventPublisher::new("127.0.0.1:9092").expect("kafka publisher"),
            TaskTracker::new(),
        );
        let crm_service = crm::domain::service::CrmServiceImpl::new(
            crm::outbound::companies_repo::CompaniesRepositoryImpl::new(db.clone()),
            CrmMetadataResolver::new(Arc::new(unfurl::domain::service::UnfurlServiceImpl::new(
                unfurl::outbound::ReqwestUnfurlFetcher::new().expect("unfurl fetcher"),
            ))),
        );
        let sync_context = |retry_worker: bool, queue: &str| PubSubContext {
            gmail_history: email::domain::mailbox::gmail_history::GmailHistoryService(
                email::outbound::mailbox_pg::PgMailboxSync::new(db.clone()),
            ),
            invitation_extractor: crate::pubsub::invitation_extraction::compose(
                db.clone(),
                email_api.clone(),
            ),
            db: db.clone(),
            sqs_worker: worker(queue),
            sqs_client: sqs_client.clone(),
            contacts_ingress: Arc::new(contacts::domain::service::SqsContactsIngress {
                queue: contacts::outbound::ingress::SqsContactsQueue::new(
                    sqs_sdk.clone(),
                    format!("{services_url}/queue/contacts"),
                ),
            }),
            email_api: email_api.clone(),
            redis_client: wiring::redis_client(redis_client.clone()),
            notification_ingress_service: Arc::new(
                notification::domain::service::SqsNotificationIngress {
                    queue: notification::outbound::queue::SqsQueue::new(
                        sqs_sdk.clone(),
                        format!("{services_url}/queue/notifications"),
                    ),
                },
            ),
            sfs_client: static_file_service_client::StaticFileServiceClient::new(
                "key".to_string(),
                services_url.to_string(),
            ),
            connection_gateway_client:
                connection_gateway_client::client::ConnectionGatewayClient::new(
                    "key".to_string(),
                    services_url.to_string(),
                ),
            dss_client: document_storage_service_client::DocumentStorageServiceClient::new(
                "key".to_string(),
                services_url.to_string(),
            ),
            system_properties_service: Arc::new(
                system_properties::SystemPropertiesServiceImpl::new(
                    system_properties::PgSystemPropertiesRepository::new(db.clone()),
                ),
            ),
            crm_service: crm_service.clone(),
            macro_event_broker: broker.clone(),
            notifications_enabled: false,
            retry_worker,
        };
        let email = EmailServiceImpl::new(
            EmailPgRepo::new(db.clone()),
            FrecencyQueryServiceImpl::new(FrecencyPgStorage::new(db.clone())),
            sqs_client.clone(),
            crm::domain::service::CrmServiceImpl::new(
                crm::outbound::companies_repo::CompaniesRepositoryImpl::new(db.clone()),
                crm::outbound::no_op_resolver::NoOpCompanyMetadataResolver,
            ),
            entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(
                entity_access_management::outbound::PgRepository::new(db.clone()),
            ),
            0,
        )
        .with_macro_event_broker(broker.clone());

        World {
            ops: ops_context(false, &queues.ops),
            ops_retry: ops_context(true, &queues.ops_retry),
            sync: sync_context(false, &queues.sync),
            sync_retry: sync_context(true, &queues.sync_retry),
            db,
            gmail,
            sqs,
            redis,
            link_id,
            queues,
            email,
            message_ids: BTreeMap::new(),
            thread_ids: BTreeMap::new(),
            label_ids: BTreeMap::new(),
            label_write_batches,
        }
    }

    async fn seed(&mut self, shard: u32) {
        let link = self.link_id;
        sqlx::query(
            "INSERT INTO email_links (id, macro_id, fusionauth_user_id, email_address, provider, is_sync_active, created_at, updated_at)
             VALUES ($1, 'macro|[email protected]', $2, $3, 'GMAIL', true, NOW(), NOW())",
        )
        .bind(link)
        .bind(format!("sim-user-{shard}"))
        .bind(format!("sim{shard}@example.com"))
        .execute(&self.db)
        .await
        .unwrap();
        sqlx::query("INSERT INTO email_gmail_histories (link_id, history_id) VALUES ($1, '100')")
            .bind(link)
            .execute(&self.db)
            .await
            .unwrap();
        for (index, (id, name, kind)) in LABELS.iter().enumerate() {
            let label_id = label_uuid(shard, index);
            let label_type = if *kind == "system" { "System" } else { "User" };
            sqlx::query(&format!(
                "INSERT INTO email_labels (id, link_id, provider_label_id, name, message_list_visibility, label_list_visibility, type, created_at)
                 VALUES ($1, $2, $3, $4, 'Show', 'LabelShow', '{label_type}', NOW())"
            ))
            .bind(label_id)
            .bind(link)
            .bind(id)
            .bind(name)
            .execute(&self.db)
            .await
            .unwrap();
            self.label_ids.insert(id.to_string(), label_id);
        }
        for (index, thread) in THREADS.iter().enumerate() {
            let thread_id = thread_uuid(shard, index);
            sqlx::query(
                "INSERT INTO email_threads (id, provider_id, link_id, inbox_visible, is_read, created_at, updated_at)
                 VALUES ($1, $2, $3, true, false, NOW(), NOW())",
            )
            .bind(thread_id)
            .bind(thread)
            .bind(link)
            .execute(&self.db)
            .await
            .unwrap();
            self.thread_ids.insert(thread.to_string(), thread_id);
        }
        for (index, (id, thread, labels)) in MESSAGES.iter().enumerate() {
            let message_id = message_uuid(shard, index);
            sqlx::query(
                "INSERT INTO email_messages (id, thread_id, link_id, provider_id, provider_thread_id, is_read, is_starred, is_sent, is_draft, has_attachments, internal_date_ts, created_at, updated_at)
                 VALUES ($1, $2, $3, $4, $5, $6, $7, false, false, false, to_timestamp($8::double precision / 1000), NOW(), NOW())",
            )
            .bind(message_id)
            .bind(self.thread_ids[*thread])
            .bind(link)
            .bind(id)
            .bind(thread)
            .bind(!labels.contains(&UNREAD))
            .bind(labels.contains(&STARRED))
            .bind((1_790_000_000_000 + index as i64 * 60_000) as f64)
            .execute(&self.db)
            .await
            .unwrap();
            self.message_ids.insert(id.to_string(), message_id);
        }
        self.reset().await;
    }

    /// Puts the mailbox, Gmail, queues and admission state back to the start.
    pub async fn reset(&mut self) {
        let message_ids: Vec<Uuid> = self.message_ids.values().copied().collect();
        sqlx::query("DELETE FROM email_message_labels WHERE message_id = ANY($1)")
            .bind(&message_ids)
            .execute(&self.db)
            .await
            .unwrap();
        wiring::clear_markers(&self.db, &message_ids).await;
        for (id, _, labels) in MESSAGES {
            let message_id = self.message_ids[id];
            for label in labels {
                sqlx::query(
                    "INSERT INTO email_message_labels (message_id, label_id) VALUES ($1, $2)",
                )
                .bind(message_id)
                .bind(self.label_ids[*label])
                .execute(&self.db)
                .await
                .unwrap();
            }
            sqlx::query("UPDATE email_messages SET is_read = $2, is_starred = $3 WHERE id = $1")
                .bind(message_id)
                .bind(!labels.contains(&UNREAD))
                .bind(labels.contains(&STARRED))
                .execute(&self.db)
                .await
                .unwrap();
        }
        for thread_id in self.thread_ids.values() {
            let mut connection = self.db.acquire().await.unwrap();
            email_db_client::threads::update::update_thread_metadata(
                &mut connection,
                *thread_id,
                self.link_id,
            )
            .await
            .unwrap();
        }
        sqlx::query("UPDATE email_gmail_histories SET history_id = '100' WHERE link_id = $1")
            .bind(self.link_id)
            .execute(&self.db)
            .await
            .unwrap();
        *self.gmail.lock() = initial_gmail();
        {
            let mut sqs = self.sqs.lock();
            sqs.sent.clear();
            sqs.acks.clear();
            sqs.failed_sends.clear();
            sqs.outage = false;
        }
        self.seed_token().await;
        wiring::reset_admission(&mut self.redis, self.link_id).await;
    }

    pub async fn seed_token(&mut self) {
        let link = email_db_client::links::get::fetch_link_by_id(&self.db, self.link_id)
            .await
            .unwrap()
            .unwrap();
        let key = TokenCacheKey::new(
            &link.fusionauth_user_id,
            link.email_address.0.as_ref(),
            link.provider.as_str(),
        )
        .to_redis_key();
        let _: () = redis::cmd("SET")
            .arg(key)
            .arg("fake-access-token")
            .arg("EX")
            .arg(86_400)
            .query_async(&mut self.redis)
            .await
            .unwrap();
    }

    /// Labels per provider message id in the database.
    pub async fn db_labels(&self) -> BTreeMap<String, BTreeSet<String>> {
        let rows: Vec<(String, String)> = sqlx::query_as(
            "SELECT m.provider_id, l.provider_label_id
             FROM email_messages m
             JOIN email_message_labels ml ON ml.message_id = m.id
             JOIN email_labels l ON l.id = ml.label_id
             WHERE m.link_id = $1",
        )
        .bind(self.link_id)
        .fetch_all(&self.db)
        .await
        .unwrap();
        let mut labels: BTreeMap<String, BTreeSet<String>> = MESSAGES
            .iter()
            .map(|(id, _, _)| (id.to_string(), BTreeSet::new()))
            .collect();
        for (message, label) in rows {
            labels.entry(message).or_default().insert(label);
        }
        labels
    }

    pub fn gmail_labels(&self) -> BTreeMap<String, BTreeSet<String>> {
        self.gmail
            .lock()
            .messages
            .iter()
            .map(|(id, message)| (id.clone(), message.labels.clone()))
            .collect()
    }

    /// Threads and messages whose stored flags disagree with their labels.
    pub async fn flag_mismatches(&self) -> Vec<String> {
        let mut problems = Vec::new();
        let messages: Vec<(String, bool, bool, bool, bool)> = sqlx::query_as(
            "SELECT m.provider_id, m.is_read, m.is_starred,
                EXISTS (SELECT 1 FROM email_message_labels ml JOIN email_labels l ON l.id = ml.label_id
                        WHERE ml.message_id = m.id AND l.provider_label_id = 'UNREAD'),
                EXISTS (SELECT 1 FROM email_message_labels ml JOIN email_labels l ON l.id = ml.label_id
                        WHERE ml.message_id = m.id AND l.provider_label_id = 'STARRED')
             FROM email_messages m WHERE m.link_id = $1",
        )
        .bind(self.link_id)
        .fetch_all(&self.db)
        .await
        .unwrap();
        for (id, is_read, is_starred, unread, starred) in messages {
            if is_read == unread {
                problems.push(format!(
                    "message {id}: is_read={is_read} but UNREAD={unread}"
                ));
            }
            if is_starred != starred {
                problems.push(format!(
                    "message {id}: is_starred={is_starred} but STARRED={starred}"
                ));
            }
        }
        let threads: Vec<(String, bool, bool, bool, bool)> = sqlx::query_as(
            "SELECT t.provider_id, t.inbox_visible, t.is_read,
                COALESCE(bool_or((f.in_inbox AND NOT f.in_sent)), false),
                COALESCE(bool_and(m.is_read), true)
             FROM email_threads t
             JOIN email_messages m ON m.thread_id = t.id
             JOIN email_message_mailbox_facts f ON f.id = m.id
             WHERE t.link_id = $1 AND f.is_present
             GROUP BY t.id",
        )
        .bind(self.link_id)
        .fetch_all(&self.db)
        .await
        .unwrap();
        for (id, inbox_visible, is_read, expected_visible, expected_read) in threads {
            if inbox_visible != expected_visible {
                problems.push(format!(
                    "thread {id}: inbox_visible={inbox_visible}, messages say {expected_visible}"
                ));
            }
            if is_read != expected_read {
                problems.push(format!(
                    "thread {id}: is_read={is_read}, messages say {expected_read}"
                ));
            }
        }
        problems
    }

    pub async fn thread_inbox_visible(&self, thread: &str) -> bool {
        sqlx::query_scalar("SELECT inbox_visible FROM email_threads WHERE id = $1")
            .bind(self.thread_ids[thread])
            .fetch_one(&self.db)
            .await
            .unwrap()
    }
}

/// Answers 200 `{}` to everything: auth, SFS, DSS, connection gateway, and
/// the SQS queues the simulation does not model.
async fn start_catch_all() -> String {
    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
    let url = format!("http://{}", listener.local_addr().unwrap());
    let app = Router::new().fallback(|| async {
        (
            axum::http::StatusCode::OK,
            [("content-type", "application/json")],
            "{}",
        )
            .into_response()
    });
    tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
    url
}