Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(whatsapp-session): run linked WhatsApp accounts in a session service (#8538)
Wolf Mermelstein0c732498ad58
diff --git a/.github/services-config.json b/.github/services-config.jsonindex a371a8030f..9a4f656fac 100644--- a/.github/services-config.json+++ b/.github/services-config.json@@ -302,6 +302,22 @@"preview-gateway": {"source_paths": [],"stack_path": "infra/stacks/preview-gateway/**"+ },+ "whatsapp-session-service": {+ "deploy_enabled": false,+ "bootstrap_pending": "Needs the whatsapp-session-service Doppler project (dev/prd) with its keys, including WHATSAPP_SESSION_KMS_KEY_ID from the stack's first deploy, and an explicit go-ahead to run personal WhatsApp sessions.",+ "source_paths": [+ "services/whatsapp_session_service/**",+ "crates/whatsapp_linked/**",+ "crates/whatsapp_store_pg/**",+ "crates/whatsmeow/**",+ "crates/whatsmeow_proto/**",+ "crates/whatsmeow_signal/**"+ ],+ "stack_path": "infra/stacks/whatsapp-session-service/**",+ "deploy_binaries": [+ "whatsapp_session_service"+ ]}}}diff --git a/.github/workflows/deploy-service-generic.yml b/.github/workflows/deploy-service-generic.ymlindex ecd5fc4182..431d20ec01 100644--- a/.github/workflows/deploy-service-generic.yml+++ b/.github/workflows/deploy-service-generic.yml@@ -35,6 +35,7 @@ on:- search-upload- static-file-service- unfurl-service+ - whatsapp-session-service- opensearch- search-event-queue- search-text-extractor-queuediff --git a/.github/workspace-dep-closures.json b/.github/workspace-dep-closures.jsonindex b1ce4ad643..e59b130178 100644--- a/.github/workspace-dep-closures.json+++ b/.github/workspace-dep-closures.json@@ -14000,6 +14000,57 @@"crates/whatsapp_linked","crates/workspace-hack"],+ "whatsapp_session_service": [+ "crates/agent",+ "crates/ai_toolset",+ "crates/ai_usage",+ "crates/attachment",+ "crates/bot_id",+ "crates/bot_token",+ "crates/cowlike",+ "crates/database_env_vars",+ "crates/document_sub_type",+ "crates/entity_access",+ "crates/entity_access_db_utils",+ "crates/entity_registry/entity_registry",+ "crates/entity_registry/entity_registry_db_utils",+ "crates/entity_registry/shared_entity_registry",+ "crates/genai_telemetry",+ "crates/harness_id",+ "crates/harness_token",+ "crates/macro_auth",+ "crates/macro_authorization",+ "crates/macro_aws_config",+ "crates/macro_config",+ "crates/macro_config_derive",+ "crates/macro_entrypoint",+ "crates/macro_env",+ "crates/macro_env_var",+ "crates/macro_service_urls",+ "crates/macro_user_id",+ "crates/macro_uuid",+ "crates/model",+ "crates/model-entity",+ "crates/model-error-response",+ "crates/model_file_type",+ "crates/model_owner",+ "crates/model_user",+ "crates/models_bulk_upload",+ "crates/models_entity_access_management",+ "crates/models_pagination",+ "crates/models_permissions",+ "crates/non_empty",+ "crates/prompt",+ "crates/remote_env_var",+ "crates/share_permission_db_utils",+ "crates/whatsapp_linked",+ "crates/whatsapp_store_pg",+ "crates/whatsmeow",+ "crates/whatsmeow_proto",+ "crates/whatsmeow_signal",+ "crates/workspace-hack",+ "services/whatsapp_session_service"+ ],"whatsapp_store_pg": ["crates/whatsapp_store_pg","crates/whatsmeow",diff --git a/.sqlx/query-08926aeffe64402c98fe63e2a117ceca96b884431998ad1b59e702c39d71cb82.json b/.sqlx/query-08926aeffe64402c98fe63e2a117ceca96b884431998ad1b59e702c39d71cb82.jsonnew file mode 100644index 0000000000..d43888e559--- /dev/null+++ b/.sqlx/query-08926aeffe64402c98fe63e2a117ceca96b884431998ad1b59e702c39d71cb82.json@@ -0,0 +1,22 @@+{+ "db_name": "PostgreSQL",+ "query": "SELECT pg_try_advisory_lock(hashtextextended('whatsapp-account:' || $1::uuid::text, 0)) AS \"locked!\"",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "locked!",+ "type_info": "Bool"+ }+ ],+ "parameters": {+ "Left": [+ "Uuid"+ ]+ },+ "nullable": [+ null+ ]+ },+ "hash": "08926aeffe64402c98fe63e2a117ceca96b884431998ad1b59e702c39d71cb82"+}diff --git a/.sqlx/query-ceb8cfbdc19c2bd8b642a20295eb8b3bf30c874a6b95d4f29eef5145c412e876.json b/.sqlx/query-34fe6907fd98b764249e05ba1ea47805e75dcf650cd566a5dbab54f311d8bbee.jsonsimilarity index 52%rename from .sqlx/query-ceb8cfbdc19c2bd8b642a20295eb8b3bf30c874a6b95d4f29eef5145c412e876.jsonrename to .sqlx/query-34fe6907fd98b764249e05ba1ea47805e75dcf650cd566a5dbab54f311d8bbee.jsonindex 1b49e6e536..6cb36c6b15 100644--- a/.sqlx/query-ceb8cfbdc19c2bd8b642a20295eb8b3bf30c874a6b95d4f29eef5145c412e876.json+++ b/.sqlx/query-34fe6907fd98b764249e05ba1ea47805e75dcf650cd566a5dbab54f311d8bbee.json@@ -1,6 +1,6 @@{"db_name": "PostgreSQL",- "query": "WITH needing AS (\n SELECT EXISTS (\n SELECT 1 FROM pg_locks l\n WHERE l.locktype = 'advisory' AND l.objsubid = 1\n AND l.database = (SELECT oid FROM pg_database WHERE datname = current_database())\n AND ((l.classid::bigint << 32) | l.objid::bigint) = hashtext('whatsapp-account:' || a.id::text)::bigint) AS locked\n FROM whatsapp_linked_account a WHERE (a.state = 'connected' OR EXISTS (\n SELECT 1 FROM whatsapp_pairing p WHERE p.account_id = a.id AND p.consumed_at IS NULL AND p.expires_at > now())))\n SELECT\n (SELECT count(*) FROM needing WHERE locked) AS \"running!\",\n (SELECT count(*) FROM needing WHERE NOT locked) AS \"unowned!\",\n (SELECT COALESCE(EXTRACT(EPOCH FROM now() - min(created_at)), 0)::bigint FROM whatsapp_inbound_event) AS \"oldest!\"",+ "query": "WITH needing AS (\n SELECT EXISTS (\n SELECT 1 FROM pg_locks l\n WHERE l.locktype = 'advisory' AND l.objsubid = 1\n AND l.database = (SELECT oid FROM pg_database WHERE datname = current_database())\n AND ((l.classid::bigint << 32) | l.objid::bigint) = hashtextextended('whatsapp-account:' || a.id::text, 0)) AS locked\n FROM whatsapp_linked_account a WHERE (a.state = 'connected' OR EXISTS (\n SELECT 1 FROM whatsapp_pairing p WHERE p.account_id = a.id AND p.consumed_at IS NULL AND p.expires_at > now())))\n SELECT\n (SELECT count(*) FROM needing WHERE locked) AS \"running!\",\n (SELECT count(*) FROM needing WHERE NOT locked) AS \"unowned!\",\n (SELECT COALESCE(EXTRACT(EPOCH FROM now() - min(created_at)), 0)::bigint FROM whatsapp_inbound_event) AS \"oldest!\"","describe": {"columns": [{@@ -28,5 +28,5 @@null]},- "hash": "ceb8cfbdc19c2bd8b642a20295eb8b3bf30c874a6b95d4f29eef5145c412e876"+ "hash": "34fe6907fd98b764249e05ba1ea47805e75dcf650cd566a5dbab54f311d8bbee"}diff --git a/.sqlx/query-3d444801dac44f47578f48713f8affb7aa14a5a823eb6fe8b6f1d62d457ee785.json b/.sqlx/query-3d444801dac44f47578f48713f8affb7aa14a5a823eb6fe8b6f1d62d457ee785.jsonnew file mode 100644index 0000000000..a5e956ed91--- /dev/null+++ b/.sqlx/query-3d444801dac44f47578f48713f8affb7aa14a5a823eb6fe8b6f1d62d457ee785.json@@ -0,0 +1,20 @@+{+ "db_name": "PostgreSQL",+ "query": "SELECT pg_backend_pid() AS \"pid!\"",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "pid!",+ "type_info": "Int4"+ }+ ],+ "parameters": {+ "Left": []+ },+ "nullable": [+ null+ ]+ },+ "hash": "3d444801dac44f47578f48713f8affb7aa14a5a823eb6fe8b6f1d62d457ee785"+}diff --git a/.sqlx/query-8ecb2252025833f307649422bdf3bacb398453456b796f005d7d7d1cce527f04.json b/.sqlx/query-8ecb2252025833f307649422bdf3bacb398453456b796f005d7d7d1cce527f04.jsonnew file mode 100644index 0000000000..016509d31c--- /dev/null+++ b/.sqlx/query-8ecb2252025833f307649422bdf3bacb398453456b796f005d7d7d1cce527f04.json@@ -0,0 +1,22 @@+{+ "db_name": "PostgreSQL",+ "query": "SELECT pg_advisory_unlock(hashtextextended('whatsapp-account:' || $1::uuid::text, 0)) AS \"unlocked!\"",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "unlocked!",+ "type_info": "Bool"+ }+ ],+ "parameters": {+ "Left": [+ "Uuid"+ ]+ },+ "nullable": [+ null+ ]+ },+ "hash": "8ecb2252025833f307649422bdf3bacb398453456b796f005d7d7d1cce527f04"+}diff --git a/.sqlx/query-736363207d5a3891f62c011bb974267c3ba7e64110e00db59e3b23dd5159e743.json b/.sqlx/query-add7d9621be97026468f00045b9d9d154a9e52f08d0112664858783da832d12c.jsonsimilarity index 77%rename from .sqlx/query-736363207d5a3891f62c011bb974267c3ba7e64110e00db59e3b23dd5159e743.jsonrename to .sqlx/query-add7d9621be97026468f00045b9d9d154a9e52f08d0112664858783da832d12c.jsonindex e56d1bc66f..cc4b3cd67e 100644--- a/.sqlx/query-736363207d5a3891f62c011bb974267c3ba7e64110e00db59e3b23dd5159e743.json+++ b/.sqlx/query-add7d9621be97026468f00045b9d9d154a9e52f08d0112664858783da832d12c.json@@ -1,6 +1,6 @@{"db_name": "PostgreSQL",- "query": "SELECT a.id FROM whatsapp_linked_account a\n WHERE (a.state = 'connected' OR EXISTS (\n SELECT 1 FROM whatsapp_pairing p WHERE p.account_id = a.id AND p.consumed_at IS NULL AND p.expires_at > now()))\n AND NOT EXISTS (\n SELECT 1 FROM pg_locks l\n WHERE l.locktype = 'advisory' AND l.objsubid = 1\n AND l.database = (SELECT oid FROM pg_database WHERE datname = current_database())\n AND ((l.classid::bigint << 32) | l.objid::bigint) = hashtext('whatsapp-account:' || a.id::text)::bigint)\n ORDER BY a.updated_at, a.id LIMIT $1",+ "query": "SELECT a.id FROM whatsapp_linked_account a\n WHERE (a.state = 'connected' OR EXISTS (\n SELECT 1 FROM whatsapp_pairing p WHERE p.account_id = a.id AND p.consumed_at IS NULL AND p.expires_at > now()))\n AND NOT EXISTS (\n SELECT 1 FROM pg_locks l\n WHERE l.locktype = 'advisory' AND l.objsubid = 1\n AND l.database = (SELECT oid FROM pg_database WHERE datname = current_database())\n AND ((l.classid::bigint << 32) | l.objid::bigint) = hashtextextended('whatsapp-account:' || a.id::text, 0))\n ORDER BY a.updated_at, a.id LIMIT $1","describe": {"columns": [{@@ -18,5 +18,5 @@false]},- "hash": "736363207d5a3891f62c011bb974267c3ba7e64110e00db59e3b23dd5159e743"+ "hash": "add7d9621be97026468f00045b9d9d154a9e52f08d0112664858783da832d12c"}diff --git a/Cargo.lock b/Cargo.lockindex b1bcff0bd7..678d0959df 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -21629,6 +21629,43 @@ dependencies = ["workspace-hack",]+[[package]]+name = "whatsapp_session_service"+version = "0.1.0"+dependencies = [+ "async-trait",+ "aws-sdk-kms",+ "axum",+ "chrono",+ "database_env_vars",+ "doppler-config",+ "futures",+ "macro_aws_config",+ "macro_config",+ "macro_db_migrator",+ "macro_entrypoint",+ "macro_env",+ "macro_env_var",+ "macro_service_urls",+ "rootcause",+ "rustls",+ "serde",+ "serde_json",+ "sha2 0.10.9",+ "sqlx",+ "thiserror 2.0.18",+ "tokio",+ "tokio-util",+ "tracing",+ "uuid",+ "whatsapp_fake_server",+ "whatsapp_linked",+ "whatsapp_store_pg",+ "whatsmeow",+ "whatsmeow_signal",+ "workspace-hack",+]+[[package]]name = "whatsapp_store_pg"version = "0.1.0"diff --git a/Cargo.toml b/Cargo.tomlindex a789190ebd..f51432ea13 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -179,6 +179,7 @@ members = ["services/upload_extractor_lambda_handler","services/upload_extractor_lambda_trigger","services/user_link_cleanup_handler",+ "services/whatsapp_session_service","services/worker_trigger","tooling/native_app_server","tooling/notification_sandbox",diff --git a/crates/whatsapp_linked/src/domain/ports.rs b/crates/whatsapp_linked/src/domain/ports.rsindex 787d8d3173..b55601c794 100644--- a/crates/whatsapp_linked/src/domain/ports.rs+++ b/crates/whatsapp_linked/src/domain/ports.rs@@ -238,7 +238,7 @@ pub trait SessionRepository: Send + Sync {/// The account's lifecycle, unfenced.async fn account_state(&self, account: Uuid) -> LinkedResult<Option<AccountState>>;/// Accounts needing a replica whose advisory lock- /// (`hashtext('whatsapp-account:' || id)`) nobody holds.+ /// (`hashtextextended('whatsapp-account:' || id, 0)`) nobody holds.async fn discoverable_accounts(&self, limit: i64) -> LinkedResult<Vec<Uuid>>;/// Fleet counts for health checks.async fn fleet_health(&self) -> LinkedResult<FleetHealth>;diff --git a/crates/whatsapp_linked/src/outbound/pg/session.rs b/crates/whatsapp_linked/src/outbound/pg/session.rsindex f96b99d7d7..18dffe96c9 100644--- a/crates/whatsapp_linked/src/outbound/pg/session.rs+++ b/crates/whatsapp_linked/src/outbound/pg/session.rs@@ -75,7 +75,7 @@ impl SessionRepository for PgLinkedRepository {SELECT 1 FROM pg_locks lWHERE l.locktype = 'advisory' AND l.objsubid = 1AND l.database = (SELECT oid FROM pg_database WHERE datname = current_database())- AND ((l.classid::bigint << 32) | l.objid::bigint) = hashtext('whatsapp-account:' || a.id::text)::bigint)+ AND ((l.classid::bigint << 32) | l.objid::bigint) = hashtextextended('whatsapp-account:' || a.id::text, 0))ORDER BY a.updated_at, a.id LIMIT $1"#,limit)@@ -93,7 +93,7 @@ impl SessionRepository for PgLinkedRepository {SELECT 1 FROM pg_locks lWHERE l.locktype = 'advisory' AND l.objsubid = 1AND l.database = (SELECT oid FROM pg_database WHERE datname = current_database())- AND ((l.classid::bigint << 32) | l.objid::bigint) = hashtext('whatsapp-account:' || a.id::text)::bigint) AS locked+ AND ((l.classid::bigint << 32) | l.objid::bigint) = hashtextextended('whatsapp-account:' || a.id::text, 0)) AS lockedFROM whatsapp_linked_account a WHERE (a.state = 'connected' OR EXISTS (SELECT 1 FROM whatsapp_pairing p WHERE p.account_id = a.id AND p.consumed_at IS NULL AND p.expires_at > now())))SELECTdiff --git a/crates/whatsmeow/AGENTS.md b/crates/whatsmeow/AGENTS.mdindex 0dd13accfb..6f84fb66df 100644--- a/crates/whatsmeow/AGENTS.md+++ b/crates/whatsmeow/AGENTS.md@@ -128,7 +128,10 @@ WASA/bot parts of `msgsecret.go`, reporting tokens, `armadillomessage.go`, `pushright before a zero `NoiseSocket` panics), and `client::test::capture_request`gets the Rust side; both replace the iq `id` and usync `sid`.- Go formats JIDs (and sometimes node XML) into error messages; ported errors- leave them out, since `#[tracing::instrument(err)]` logs them.+ leave them out, since `#[tracing::instrument(err)]` logs them. `IqError`+ names only the node's tag, and `SignalError::Internal` /+ `StoreError::Internal` leave their report out of the message (it stays in+ the field and `Debug`); `errors/test.rs` formats each and asserts no JID.- Methods that spawn work which can reconnect return `BoxFuture` (see`Client::auto_reconnect`): without it, `Send` inference cycles through`connect`.diff --git a/crates/whatsmeow/src/errors.rs b/crates/whatsmeow/src/errors.rsindex 2236797eaf..dba02758fe 100644--- a/crates/whatsmeow/src/errors.rs+++ b/crates/whatsmeow/src/errors.rs@@ -419,15 +419,23 @@ impl IqError {}}+/// Deviation from Go: an unknown error or unexpected response names only the+/// node's tag, since nodes carry JIDs and the message ends up in logs and+/// spans; the nodes stay in `error_node`, `raw_node` and `Debug`.impl fmt::Display for IqError {fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {if self.code == 0 {if let Some(error_node) = &self.error_node {- write!(formatter, "info query returned unknown error: {error_node}")+ write!(+ formatter,+ "info query returned unknown error: <{}>",+ error_node.tag+ )} else if let Some(raw_node) = &self.raw_node {write!(formatter,- "info query returned unexpected response: {raw_node}"+ "info query returned unexpected response: <{}>",+ raw_node.tag)} else {formatter.write_str("unknown info query error")@@ -495,3 +503,6 @@ pub fn element_missing(tag: &str, in_: &str) -> ClientError {in_: in_.to_owned(),}}++#[cfg(test)]+mod test;diff --git a/crates/whatsmeow/src/errors/test.rs b/crates/whatsmeow/src/errors/test.rsnew file mode 100644index 0000000000..c50dc225e1--- /dev/null+++ b/crates/whatsmeow/src/errors/test.rs@@ -0,0 +1,142 @@+// This Source Code Form is subject to the terms of the Mozilla Public+// License, v. 2.0. If a copy of the MPL was not distributed with this+// file, You can obtain one at http://mozilla.org/MPL/2.0/.++//! Error messages reach logs and spans through `#[tracing::instrument(err)]`,+//! so none of them may name a JID, even when the error carries one.++use std::sync::Arc;++use super::*;+use crate::binary::{AttributeValue, Attrs, NodeContent};+use crate::prekeys::node_to_pre_key_bundle;+use crate::store::{LidStore, MemoryStore};+use crate::types::parse_jid;++const USER: &str = "1234:[email protected]";+const LID: &str = "9876:5@lid";+const GROUP: &str = "[email protected]";++fn jid(value: &str) -> AttributeValue {+ parse_jid(value).unwrap().into()+}++fn node(tag: &str, attrs: Vec<(&str, AttributeValue)>, children: Vec<Node>) -> Node {+ Node {+ tag: tag.to_owned(),+ attrs: attrs+ .into_iter()+ .map(|(key, value)| (key.to_owned(), value))+ .collect::<Attrs>(),+ content: (!children.is_empty()).then_some(NodeContent::Nodes(children)),+ }+}++/// An IQ response naming a user, a LID and a group, with `error` as its+/// only child when given.+fn response(error: Option<Node>) -> Node {+ node(+ "iq",+ vec![+ ("from", jid(GROUP)),+ ("participant", jid(USER)),+ ("type", AttributeValue::from("error")),+ ],+ error.into_iter().collect(),+ )+}++fn error_node(code: Option<&str>) -> Node {+ let mut attrs = vec![+ ("jid", jid(LID)),+ ("text", AttributeValue::from("forbidden")),+ ];+ if let Some(code) = code {+ attrs.push(("code", AttributeValue::from(code)));+ }+ node(+ "error",+ attrs,+ vec![node("item", vec![("jid", jid(USER))], vec![])],+ )+}++fn assert_no_jid(message: &str) {+ for jid_part in ["@s.whatsapp.net", "@lid", "@g.us", "1234", "9876", "120363"] {+ assert!(!message.contains(jid_part), "{message:?} names a JID");+ }+}++#[tokio::test]+async fn error_messages_never_name_jids() {+ let unknown_iq = parse_iq_error(&response(Some(error_node(None))));+ let coded_iq = parse_iq_error(&response(Some(error_node(Some("403")))));+ let unexpected_iq = IqError {+ raw_node: Some(response(None)),+ ..IqError::default()+ };+ assert_eq!(+ unknown_iq.to_string(),+ "info query returned unknown error: <error>"+ );+ assert_eq!(+ unexpected_iq.to_string(),+ "info query returned unexpected response: <iq>"+ );+ // The detail stays reachable.+ assert!(format!("{unknown_iq:?}").contains("lid"));+ assert!(unknown_iq.raw_node.is_some());++ let store = Arc::new(MemoryStore::default());+ let lid_error = store+ .put_lid_mapping(&parse_jid(USER).unwrap(), &parse_jid(LID).unwrap())+ .await+ .unwrap_err();+ let pn_error = store+ .get_pn_for_lid(&parse_jid(USER).unwrap())+ .await+ .unwrap_err();+ let prekey_error = node_to_pre_key_bundle(+ 5,+ &node(+ "user",+ vec![("jid", jid(USER))],+ vec![error_node(Some("406"))],+ ),+ )+ .unwrap_err();++ let errors: Vec<ClientError> = vec![+ unknown_iq.clone().into(),+ coded_iq.into(),+ unexpected_iq.into(),+ ClientError::from(unknown_iq.clone()).wrap("failed to update group participants"),+ wrap_iq_error(+ ClientError::GroupNotFound,+ ClientError::from(unknown_iq.clone()),+ ),+ DisconnectedError {+ action: "info query".to_owned(),+ node: Some(response(None)),+ }+ .into(),+ prekey_error,+ ClientError::Signal(SignalError::Internal(+ rootcause::report!("session with {USER} not found").into_dynamic(),+ )),+ ClientError::Store(StoreError::Internal(+ rootcause::report!("invalid value {LID}").into_dynamic(),+ )),+ ClientError::Store(lid_error).wrap("failed to store LID mapping"),+ ClientError::Store(pn_error),+ ];+ for error in &errors {+ let message = error.to_string();+ assert_no_jid(&message);+ let mut source = std::error::Error::source(error);+ while let Some(inner) = source {+ assert_no_jid(&inner.to_string());+ source = inner.source();+ }+ }+}diff --git a/crates/whatsmeow/src/prekeys.rs b/crates/whatsmeow/src/prekeys.rsindex cc45408a98..161447dabc 100644--- a/crates/whatsmeow/src/prekeys.rs+++ b/crates/whatsmeow/src/prekeys.rs@@ -251,8 +251,20 @@ fn node_bytes<'node>(node: &'node Node, tag: &str) -> Option<&'node [u8]> {/// Parses a `<user>` node of a prekey response into a bundle.pub fn node_to_pre_key_bundle(device_id: u32, node: &Node) -> Result<PreKeyBundle, ClientError> {if let Some(error_node) = node.get_optional_child_by_tag(&["error"]) {+ // Deviation from Go: only the error's code and text attributes are+ // formatted, so a JID in the node can't reach the message.+ let summary = Node {+ tag: error_node.tag.clone(),+ attrs: error_node+ .attrs+ .iter()+ .filter(|(key, _)| matches!(key.as_str(), "code" | "text"))+ .map(|(key, value)| (key.clone(), value.clone()))+ .collect(),+ content: None,+ };return Err(ClientError::Other(format!(- "got error getting prekeys: {error_node}"+ "got error getting prekeys: {summary}")));}let registration_id = node_bytes(node, "registration")diff --git a/crates/whatsmeow/src/send.rs b/crates/whatsmeow/src/send.rsindex 2457933664..cf243cffa7 100644--- a/crates/whatsmeow/src/send.rs+++ b/crates/whatsmeow/src/send.rs@@ -1023,7 +1023,8 @@ impl Client {}};if !session_exists {- return Err(ClientError::NoSession.wrap(format!("with {address}")));+ // Deviation from Go: the error doesn't name the address.+ return Err(ClientError::NoSession);}}let ciphertext = {diff --git a/crates/whatsmeow/src/signal.rs b/crates/whatsmeow/src/signal.rsindex 30aa01741c..21036d74ff 100644--- a/crates/whatsmeow/src/signal.rs+++ b/crates/whatsmeow/src/signal.rs@@ -124,8 +124,10 @@ pub enum SignalError {/// The store failed.#[error(transparent)]Store(#[from] StoreError),- /// Any other protocol failure.- #[error("signal protocol error: {0:?}")]+ /// Any other protocol failure. The message leaves the report out, since+ /// libsignal names addresses (JIDs) in its errors; read it from the field+ /// or `Debug`.+ #[error("signal protocol error")]Internal(rootcause::Report),}diff --git a/crates/whatsmeow/src/store/memorystore.rs b/crates/whatsmeow/src/store/memorystore.rsindex e37a9d90e9..8c4aa3db5a 100644--- a/crates/whatsmeow/src/store/memorystore.rs+++ b/crates/whatsmeow/src/store/memorystore.rs@@ -758,9 +758,9 @@ impl LidStore for MemoryStore {async fn put_lid_mapping(&self, lid: &Jid, jid: &Jid) -> Result<(), StoreError> {if lid.server != HIDDEN_USER_SERVER || jid.server != DEFAULT_USER_SERVER {- return Err(StoreError::InvalidArgument(format!(- "invalid PutLIDMapping call {lid}/{jid}"- )));+ return Err(StoreError::InvalidArgument(+ "invalid PutLIDMapping call: expected a LID and a phone number JID".to_owned(),+ ));}put_mapping(&mut self.state(), lid, jid);Ok(())@@ -768,9 +768,9 @@ impl LidStore for MemoryStore {async fn get_pn_for_lid(&self, lid: &Jid) -> Result<Option<Jid>, StoreError> {if lid.server != HIDDEN_USER_SERVER {- return Err(StoreError::InvalidArgument(format!(- "invalid GetPNForLID call with non-LID JID {lid}"- )));+ return Err(StoreError::InvalidArgument(+ "invalid GetPNForLID call with non-LID JID".to_owned(),+ ));}Ok(self.state().pn_by_lid.get(&lid.user).map(|user| Jid {user: user.clone(),@@ -782,9 +782,9 @@ impl LidStore for MemoryStore {async fn get_lid_for_pn(&self, pn: &Jid) -> Result<Option<Jid>, StoreError> {if pn.server != DEFAULT_USER_SERVER {- return Err(StoreError::InvalidArgument(format!(- "invalid GetLIDForPN call with non-PN JID {pn}"- )));+ return Err(StoreError::InvalidArgument(+ "invalid GetLIDForPN call with non-PN JID".to_owned(),+ ));}Ok(self.state().lid_by_pn.get(&pn.user).map(|user| Jid {user: user.clone(),diff --git a/crates/whatsmeow/src/store/store.rs b/crates/whatsmeow/src/store/store.rsindex 8bb8dcb7e2..b0b0d71b80 100644--- a/crates/whatsmeow/src/store/store.rs+++ b/crates/whatsmeow/src/store/store.rs@@ -34,8 +34,10 @@ pub enum StoreError {/// The call was invalid, like a LID lookup with a phone number JID.#[error("{0}")]InvalidArgument(String),- /// The backing storage failed.- #[error("store error: {0:?}")]+ /// The backing storage failed. The message leaves the report out, since+ /// storage errors can quote stored values (JIDs); read it from the field+ /// or `Debug`.+ #[error("store error")]Internal(rootcause::Report),}diff --git a/docker/docker-compose.yml b/docker/docker-compose.ymlindex ed181356e1..29c1774ff8 100644--- a/docker/docker-compose.yml+++ b/docker/docker-compose.yml@@ -357,6 +357,34 @@ services:# password it has no terminal to read.- agent-harness-service+ # ============================================================================+ # whatsapp_session_service+ # Personal WhatsApp linked devices: one outbound websocket per account,+ # leased through Postgres advisory locks on WHATSAPP_SESSION_LOCK_DATABASE_URL+ # (the same local Postgres; nothing pools it). Nothing calls it, so no host+ # port and no proxy route. OVERRIDE_WHATSAPP_* stay unset: those are for+ # tests against the fake server.+ # ============================================================================+ whatsapp_session_service:+ <<: [*common-env, *rust-services-image]+ command: ["/app/out/whatsapp_session_service"]+ restart: on-failure+ # Drain releases accounts in batches before exiting, as on ECS.+ stop_grace_period: 120s+ depends_on:+ - postgres+ networks:+ databases:+ services:+ aliases:+ - whatsapp-session-service+ healthcheck:+ test: ["CMD", "curl", "-f", "http://localhost:8080/health"]+ interval: 30s+ timeout: 10s+ retries: 3+ start_period: 10s+# ============================================================================static_file_service:<<: [*common-env, *rust-services-image]diff --git a/infra/bun.lock b/infra/bun.lockindex c8dac7540c..fc9099622b 100644--- a/infra/bun.lock+++ b/infra/bun.lock@@ -254,6 +254,10 @@"@pulumi/pulumi": "^3.181.0",},},+ "stacks/whatsapp-session-service": {+ "name": "whatsapp-session-service-stack",+ "version": "0.0.0",+ },},"overrides": {"ts-node": "10.8.0",@@ -911,6 +915,8 @@"walk-up-path": ["[email protected]", "", {}, "sha512-3hu+tD8YzSLGuFYtPRb48vdhKMi0KQV5sn+uWr8+7dMEq/2G/dtLrdDinkLjqq5TIbIBjYJ4Ax/n3YiaW7QM8A=="],+ "whatsapp-session-service-stack": ["whatsapp-session-service-stack@workspace:stacks/whatsapp-session-service"],+"which": ["[email protected]", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="],"wrap-ansi": ["[email protected]", "", { "dependencies": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", "strip-ansi": "^6.0.0" } }, "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q=="],diff --git a/infra/stacks/doppler-projects/index.ts b/infra/stacks/doppler-projects/index.tsindex 4dd648d24f..6cc6c87aa5 100644--- a/infra/stacks/doppler-projects/index.ts+++ b/infra/stacks/doppler-projects/index.ts@@ -20,6 +20,7 @@ const SERVICE_NAMES = ['preview-gateway','slack-import-worker','lexical-service',+ 'whatsapp-session-service',];for (const service_name of SERVICE_NAMES) {diff --git a/infra/stacks/monitoring/AGENTS.md b/infra/stacks/monitoring/AGENTS.mdindex 1c394bd6e6..bab17a84d0 100644--- a/infra/stacks/monitoring/AGENTS.md+++ b/infra/stacks/monitoring/AGENTS.md@@ -22,3 +22,6 @@ From `infra/`:- Do not use native committed-offset lag for the independent notificationconsumers. Their per-instance log measurements preserve broadcast semantics.- `notificationRetentionDashboardUrl` is a stack output, not an app route.+- `monitors/whatsapp-session.ts` reads log events by `telemetry_event`; its+ header is the field contract with `services/whatsapp_session_service`.+ Renaming a field there silently turns its monitor into missing data.diff --git a/infra/stacks/monitoring/index.ts b/infra/stacks/monitoring/index.tsindex f72d16d022..f02f2df0b0 100644--- a/infra/stacks/monitoring/index.ts+++ b/infra/stacks/monitoring/index.ts@@ -20,6 +20,7 @@ import './synthetics';// Declared here first.import './monitors/ai-editing';import './monitors/graphql-soup';+import './monitors/whatsapp-session';import { notificationRetentionDashboard } from './monitors/notification-retention';export const notificationRetentionDashboardUrl =diff --git a/infra/stacks/monitoring/monitors/whatsapp-session.ts b/infra/stacks/monitoring/monitors/whatsapp-session.tsnew file mode 100644index 0000000000..70221799c7--- /dev/null+++ b/infra/stacks/monitoring/monitors/whatsapp-session.ts@@ -0,0 +1,86 @@+// WhatsApp linked-device sessions (services/whatsapp_session_service).+//+// New monitors, not adopted. Every one reads a structured log event the service+// emits, keyed by `telemetry_event`, so the fields below are its contract:+//+// - `whatsapp_session_health`, once a minute per replica:+// `accounts_running`, `accounts_unowned` (connected or pairing accounts whose+// advisory lock no session holds), `oldest_unprocessed_inbound_seconds`.+// - `whatsapp_session_reconnect`, once per websocket reconnect: `account_id`.+// - `whatsapp_send_outcome`, once per finished send: `outcome`+// (`sent` | `failed` | `unknown`).+// - `whatsapp_session_drain`, once per SIGTERM drain: `drain_duration_ms`,+// `accounts_released`.+//+// Ids are opaque account uuids; never phone numbers, JIDs, or content.+// Thresholds are starting points: there is no baseline before launch.+import * as datadog from '@pulumi/datadog';++const CONTEXT = `Runbook: services/whatsapp_session_service/AGENTS.md. To stop new+sends, turn off WHATSAPP_LINKED_SENDING_ENABLED in the cloud-storage-service+Doppler config; pending sends stay recorded.+@webhook-macro-alert-hook`;++const HEALTH = '@telemetry_event:whatsapp_session_health';++for (const environment of ['dev', 'prod']) {+ const env = `env:${environment}`;+ const label = `[${environment.toUpperCase()}] WhatsApp sessions`;+ const common = {+ tags: [env, 'service:whatsapp-session-service'],+ priority: environment === 'prod' ? '2' : '3',+ includeTags: true,+ requireFullWindow: false,+ };++ new datadog.Monitor(`whatsapp-session-unowned-accounts-${environment}`, {+ ...common,+ name: `${label}: accounts without an owner`,+ type: 'log alert',+ query: `logs("${HEALTH} ${env}").index("*").rollup("min", "@accounts_unowned").last("10m") > 0`,+ message: `{{value}} connected accounts have had no replica holding their lease for ten minutes, so their groups receive and send nothing. Check replica count, WHATSAPP_SESSION_MAX_ACCOUNTS headroom, and the lock connection (WHATSAPP_SESSION_LOCK_DATABASE_URL must be the direct MacroDB endpoint).\n${CONTEXT}`,+ monitorThresholds: { critical: '0' },+ // Health events stop when the service is not deployed; that is not zero.+ onMissingData: 'resolve',+ });++ new datadog.Monitor(`whatsapp-session-reconnect-churn-${environment}`, {+ ...common,+ name: `${label}: {{account_id.name}} reconnect churn`,+ type: 'log alert',+ query: `logs("@telemetry_event:whatsapp_session_reconnect ${env}").index("*").rollup("count").by("@account_id").last("15m") > 10`,+ message: `Account {{account_id.name}} reconnected {{value}} times in 15 minutes. A flapping session can get the number flagged by WhatsApp; check the disconnect reasons on its spans, and whether two replicas are fighting over the lease (lease_epoch bumps).\n${CONTEXT}`,+ monitorThresholds: { critical: '10', warning: '5' },+ onMissingData: 'resolve',+ });++ new datadog.Monitor(`whatsapp-session-unknown-sends-${environment}`, {+ ...common,+ name: `${label}: sends with unknown outcome`,+ type: 'log alert',+ query: `logs("@telemetry_event:whatsapp_send_outcome @outcome:unknown ${env}").index("*").rollup("count").last("15m") > 5`,+ message: `{{value}} sends timed out after submission in 15 minutes. They are never resent automatically; the sender sees them as unknown. Check WhatsApp connectivity and drain/takeover timing around deploys.\n${CONTEXT}`,+ monitorThresholds: { critical: '5', warning: '0' },+ onMissingData: 'resolve',+ });++ new datadog.Monitor(`whatsapp-session-inbound-lag-${environment}`, {+ ...common,+ name: `${label}: oldest unprocessed inbound event`,+ type: 'log alert',+ query: `logs("${HEALTH} ${env}").index("*").rollup("max", "@oldest_unprocessed_inbound_seconds").last("10m") > 300`,+ message: `An inbound WhatsApp event has waited {{value}}s for ingestion. Messages are reaching the replica but not the channel; check the ingestion spans and MacroDB latency.\n${CONTEXT}`,+ monitorThresholds: { critical: '300', warning: '60' },+ onMissingData: 'resolve',+ });++ new datadog.Monitor(`whatsapp-session-drain-duration-${environment}`, {+ ...common,+ name: `${label}: drain close to the stop timeout`,+ type: 'log alert',+ query: `logs("@telemetry_event:whatsapp_session_drain ${env}").index("*").rollup("max", "@drain_duration_ms").last("1h") > 100000`,+ message: `A replica took {{value}}ms to drain on SIGTERM. ECS kills the task at 120s, cutting off in-flight sends (they become unknown). Raise WHATSAPP_SESSION_DRAIN_BATCH or lower WHATSAPP_SESSION_MAX_ACCOUNTS.\n${CONTEXT}`,+ monitorThresholds: { critical: '100000', warning: '80000' },+ onMissingData: 'resolve',+ });+}diff --git a/infra/stacks/whatsapp-session-service/AGENTS.md b/infra/stacks/whatsapp-session-service/AGENTS.mdnew file mode 100644index 0000000000..2c1c71abe2--- /dev/null+++ b/infra/stacks/whatsapp-session-service/AGENTS.md@@ -0,0 +1,63 @@+# whatsapp-session-service stack++## What it is++ECS service for `services/whatsapp_session_service`: personal WhatsApp linked+devices, one outbound websocket per account, leased between replicas with+Postgres advisory locks. Nothing calls it: no load balancer, no gateway route,+no service URL. Egress is 443 (WhatsApp, AWS APIs, Datadog) and 5432 (MacroDB).++## Start here++- `service.ts`: KMS key, security group, task definition, rolling deployment.+- `infra/stacks/monitoring/monitors/whatsapp-session.ts`: its Datadog monitors+ and the telemetry they expect.++## Config (Doppler project `whatsapp-session-service`, `dev` and `prd`)++| Key | Value |+| --- | --- |+| `DATABASE_URL` | Pooled MacroDB URL, as other services. |+| `WHATSAPP_SESSION_LOCK_DATABASE_URL` | MacroDB **instance** endpoint (`macro-db-<stack>`, the `macrodb` stack's `endpoint` output), never the RDS Proxy (`macrodb-proxy-<stack>`). |+| `WHATSAPP_SESSION_KMS_KEY_ID` | `alias/whatsapp-session-service-<stack>` (`sessionKmsKeyAlias` output). |+| `WHATSAPP_SESSION_MAX_ACCOUNTS` | `500` |+| `WHATSAPP_SESSION_DRAIN_BATCH` | `20` |++The stack writes `WHATSAPP_SESSION_KMS_KEY_ID` into Doppler itself+(`doppler.Secret` in `index.ts`), since `MacroConfig` reads only+`APP_SECRETS_JSON` in ECS; the others are set by hand. The Doppler provider+needs a token that can write the project: before the first `pulumi up` of a+stack, `pulumi config set --secret doppler:dopplerToken <token>` (as+`agent-harness-service` has). Local values come+from `xtask_local` (`local_env.rs`).++## Checks++From `infra/`: `bun run check`, `bunx biome check stacks/whatsapp-session-service`.++## Gotchas++- Session advisory locks and `LISTEN` live on one Postgres session. RDS Proxy+ multiplexes, pins, and closes idle client connections (`IdleClientTimeout`);+ losing that session silently drops every lock the replica holds. Keep the+ lock connection outside transactions too: MacroDB sets+ `idle_in_transaction_session_timeout` to 5 minutes.+- Budget connections: each replica holds its pool plus the direct lock+ connection; during a deploy (`maximumPercent: 200`) double the replicas.+- `stopTimeout` is Fargate's 120 s maximum and the sidecars match it; the app+ container depends on them so they stop last and the drain's logs and spans+ ship. Drain must finish inside 120 s.+- No ECS container health check: the runtime image has no HTTP client. A+ replica that loses its lock connection must exit non-zero so ECS replaces it.+- No autoscaling: replicas own accounts, so scaling in forces takeovers. Change+ `desired_count` in `Pulumi.<stack>.yaml`.+- The KMS key only answers calls whose encryption context has+ `macro:purpose = whatsapp-session` (`crates/whatsapp_store_pg/src/kms.rs`).+ Changing that string strands every stored session.+- `bootstrap_pending` in `.github/services-config.json` keeps it out of+ deploys until the Doppler project exists and someone enables it.++## Related++- Rules: `.agents/rules/infra/pulumi.md`, `.agents/rules/config-and-service-urls.md`+- Docs: `docs/development/infrastructure.md`, `docs/guides/new-service.md`diff --git a/infra/stacks/whatsapp-session-service/CLAUDE.md b/infra/stacks/whatsapp-session-service/CLAUDE.mdnew file mode 120000index 0000000000..47dc3e3d86--- /dev/null+++ b/infra/stacks/whatsapp-session-service/CLAUDE.md@@ -0,0 +1 @@+AGENTS.md\ No newline at end of filediff --git a/infra/stacks/whatsapp-session-service/Pulumi.dev.yaml b/infra/stacks/whatsapp-session-service/Pulumi.dev.yamlnew file mode 100644index 0000000000..cfb3863b2b--- /dev/null+++ b/infra/stacks/whatsapp-session-service/Pulumi.dev.yaml@@ -0,0 +1,5 @@+config:+ aws-native:region: us-east-1+ aws:region: us-east-1+ whatsapp-session-service:session_kms_deletion_window_days: 7+ whatsapp-session-service:desired_count: 2diff --git a/infra/stacks/whatsapp-session-service/Pulumi.prod.yaml b/infra/stacks/whatsapp-session-service/Pulumi.prod.yamlnew file mode 100644index 0000000000..55c395807d--- /dev/null+++ b/infra/stacks/whatsapp-session-service/Pulumi.prod.yaml@@ -0,0 +1,5 @@+config:+ aws-native:region: us-east-1+ aws:region: us-east-1+ whatsapp-session-service:session_kms_deletion_window_days: 30+ whatsapp-session-service:desired_count: 2diff --git a/infra/stacks/whatsapp-session-service/Pulumi.yaml b/infra/stacks/whatsapp-session-service/Pulumi.yamlnew file mode 100644index 0000000000..356fc11652--- /dev/null+++ b/infra/stacks/whatsapp-session-service/Pulumi.yaml@@ -0,0 +1,3 @@+name: whatsapp-session-service+runtime: nodejs+description: Macro WhatsApp linked-device session servicediff --git a/infra/stacks/whatsapp-session-service/index.ts b/infra/stacks/whatsapp-session-service/index.tsnew file mode 100644index 0000000000..d0700ad563--- /dev/null+++ b/infra/stacks/whatsapp-session-service/index.ts@@ -0,0 +1,51 @@+import * as pulumi from '@pulumi/pulumi';+import * as doppler from '@pulumiverse/doppler';+import { config, stack } from '../../packages/shared';+import { get_coparse_api_vpc } from '../../packages/vpc';+import { WhatsappSessionService } from './service';++const tags = {+ environment: stack,+ tech_lead: 'wolf',+ project: 'whatsapp-session-service',+};++// Config (DATABASE_URL, WHATSAPP_SESSION_LOCK_DATABASE_URL,+// WHATSAPP_SESSION_KMS_KEY_ID, ...) arrives through the Doppler-synced+// APP_SECRETS_JSON; see AGENTS.md for the key list. Values this stack+// creates are written into Doppler here.++const cloudStorageStack = new pulumi.StackReference('cloud-storage-stack', {+ name: `macro-inc/document-storage/${stack}`,+});++const service = new WhatsappSessionService(+ `whatsapp-session-service-${stack}`,+ {+ vpc: get_coparse_api_vpc(),+ ecsClusterArn: cloudStorageStack+ .getOutput('cloudStorageClusterArn')+ .apply((value) => value as string),+ ecsClusterName: cloudStorageStack+ .getOutput('cloudStorageClusterName')+ .apply((value) => value as string),+ desiredCount: config.requireNumber('desired_count'),+ kmsDeletionWindowInDays: config.requireNumber(+ 'session_kms_deletion_window_days'+ ),+ tags,+ }+);++// MacroConfig reads only APP_SECRETS_JSON in ECS, so the key this stack+// creates reaches the service through Doppler.+new doppler.Secret(`whatsapp-session-WHATSAPP_SESSION_KMS_KEY_ID-${stack}`, {+ project: 'whatsapp-session-service',+ config: stack === 'prod' ? 'prd' : 'dev',+ name: 'WHATSAPP_SESSION_KMS_KEY_ID',+ value: service.kmsKeyAlias,+});++export const sessionKmsKeyAlias = service.kmsKeyAlias;+export const sessionKmsKeyArn = service.kmsKeyArn;+export const logGroupName = service.logGroupName;diff --git a/infra/stacks/whatsapp-session-service/package.json b/infra/stacks/whatsapp-session-service/package.jsonnew file mode 100644index 0000000000..935fcbd1a1--- /dev/null+++ b/infra/stacks/whatsapp-session-service/package.json@@ -0,0 +1,7 @@+{+ "name": "whatsapp-session-service-stack",+ "version": "0.0.0",+ "private": true,+ "license": "MIT",+ "main": "index.ts"+}diff --git a/infra/stacks/whatsapp-session-service/service.ts b/infra/stacks/whatsapp-session-service/service.tsnew file mode 100644index 0000000000..15276f9807--- /dev/null+++ b/infra/stacks/whatsapp-session-service/service.ts@@ -0,0 +1,362 @@+import * as aws from '@pulumi/aws';+import * as awsx from '@pulumi/awsx';+import * as pulumi from '@pulumi/pulumi';+import {+ DATADOG_API_KEY,+ DEFAULT_CONTINUE_BEFORE_STEADY_STATE,+ EcsDeploymentFailureAlarm,+ withTelemetry,+} from '../../packages/resources';+import { EcrImage } from '../../packages/service';+import {+ CLOUD_TRAIL_SNS_TOPIC_ARN,+ DopplerEcsEnvironment,+ RDS_PORT,+ stack,+} from '../../packages/shared';++const BASE_NAME = pulumi.getProject();+const REPO_ROOT = '../../..';++const SESSION_KMS_ACTIONS = ['kms:GenerateDataKey', 'kms:Decrypt'];+// Matches `ENCRYPTION_PURPOSE` in crates/whatsapp_store_pg/src/kms.rs. The key+// refuses any call that does not carry it, so this key cannot be used as a+// general-purpose encryption oracle even by its own role.+const SESSION_KMS_PURPOSE = 'whatsapp-session';++// Fargate's ceiling. Drain on SIGTERM releases accounts in batches and waits+// for in-flight sends' server acks before closing each socket.+const STOP_TIMEOUT_SECONDS = 120;++type WhatsappSessionServiceArgs = {+ vpc: {+ vpcId: pulumi.Input<string>;+ privateSubnetIds: pulumi.Input<string[]>;+ };+ ecsClusterArn: pulumi.Input<string>;+ ecsClusterName: pulumi.Input<string>;+ desiredCount: number;+ kmsDeletionWindowInDays: number;+ tags: { [key: string]: string };+};++/**+ * Holds personal WhatsApp linked-device sessions: one outbound websocket per+ * account, leased through Postgres advisory locks. Nothing calls it, so it has+ * no load balancer, target group, or inbound rule.+ */+export class WhatsappSessionService extends pulumi.ComponentResource {+ readonly role: aws.iam.Role;+ readonly serviceSg: aws.ec2.SecurityGroup;+ readonly service: awsx.ecs.FargateService;+ readonly kmsKeyArn: pulumi.Output<string>;+ readonly kmsKeyAlias: pulumi.Output<string>;+ readonly logGroupName: pulumi.Output<string>;++ constructor(+ name: string,+ args: WhatsappSessionServiceArgs,+ opts?: pulumi.ComponentResourceOptions+ ) {+ super('my:components:WhatsappSessionService', name, {}, opts);+ const { tags } = args;+ const child = { parent: this };++ this.role = new aws.iam.Role(+ `${BASE_NAME}-role`,+ {+ name: `${BASE_NAME}-role-${stack}`,+ assumeRolePolicy: aws.iam.assumeRolePolicyForPrincipal({+ Service: 'ecs-tasks.amazonaws.com',+ }),+ tags,+ },+ child+ );++ // A key of its own: the Cursor, Codex, and Microsoft keys protect other+ // secrets with other readers. Only this task role may generate or unwrap+ // account data keys; DSS gets nothing.+ const accountRootArn = pulumi.interpolate`arn:aws:iam::${aws.getCallerIdentityOutput().accountId}:root`;+ const kmsKey = new aws.kms.Key(+ `${BASE_NAME}-session-key`,+ {+ description: `WhatsApp linked-device session data keys for ${stack}`,+ deletionWindowInDays: args.kmsDeletionWindowInDays,+ enableKeyRotation: true,+ policy: aws.iam.getPolicyDocumentOutput({+ statements: [+ {+ sid: 'AllowAccountKeyAdministration',+ effect: 'Allow',+ principals: [{ type: 'AWS', identifiers: [accountRootArn] }],+ actions: [+ 'kms:CancelKeyDeletion',+ 'kms:Create*',+ 'kms:Delete*',+ 'kms:Describe*',+ 'kms:Disable*',+ 'kms:Enable*',+ 'kms:Get*',+ 'kms:List*',+ 'kms:Put*',+ 'kms:Revoke*',+ 'kms:ScheduleKeyDeletion',+ 'kms:TagResource',+ 'kms:UntagResource',+ 'kms:Update*',+ ],+ resources: ['*'],+ },+ {+ sid: 'AllowSessionServiceEnvelopeEncryption',+ effect: 'Allow',+ principals: [{ type: 'AWS', identifiers: [this.role.arn] }],+ actions: SESSION_KMS_ACTIONS,+ resources: ['*'],+ conditions: [+ {+ test: 'StringEquals',+ variable: 'kms:EncryptionContext:macro:purpose',+ values: [SESSION_KMS_PURPOSE],+ },+ ],+ },+ ],+ }).json,+ tags,+ },+ { parent: this, protect: stack === 'prod' }+ );+ const kmsAlias = new aws.kms.Alias(+ `${BASE_NAME}-session-key-alias`,+ {+ name: `alias/${BASE_NAME}-${stack}`,+ targetKeyId: kmsKey.keyId,+ },+ child+ );+ new aws.iam.RolePolicy(+ `${BASE_NAME}-session-kms-policy`,+ {+ role: this.role.id,+ policy: {+ Version: '2012-10-17',+ Statement: [+ {+ Action: SESSION_KMS_ACTIONS,+ Resource: kmsKey.arn,+ Effect: 'Allow',+ Condition: {+ StringEquals: {+ 'kms:EncryptionContext:macro:purpose': SESSION_KMS_PURPOSE,+ },+ },+ },+ ],+ },+ },+ child+ );+ this.kmsKeyArn = kmsKey.arn;+ this.kmsKeyAlias = kmsAlias.name;++ this.serviceSg = new aws.ec2.SecurityGroup(+ `${BASE_NAME}-sg-${stack}`,+ {+ name: `${BASE_NAME}-sg-${stack}`,+ vpcId: args.vpc.vpcId,+ description: 'WhatsApp session service: egress only',+ tags,+ },+ child+ );+ // WhatsApp's websocket and media hosts, KMS, Secrets Manager, ECR, and the+ // Datadog intake are all HTTPS. DNS and the Fargate metadata endpoints are+ // not filtered by security groups.+ const httpsEgress = new aws.vpc.SecurityGroupEgressRule(+ `${BASE_NAME}-https-out`,+ {+ securityGroupId: this.serviceSg.id,+ description: 'HTTPS to WhatsApp, AWS APIs, and Datadog',+ cidrIpv4: '0.0.0.0/0',+ ipProtocol: 'tcp',+ fromPort: 443,+ toPort: 443,+ tags,+ },+ child+ );+ // MacroDB (direct instance endpoint and RDS Proxy) lives in this VPC.+ const vpcCidr = aws.ec2.getVpcOutput({ id: args.vpc.vpcId }).cidrBlock;+ const postgresEgress = new aws.vpc.SecurityGroupEgressRule(+ `${BASE_NAME}-postgres-out`,+ {+ securityGroupId: this.serviceSg.id,+ description: 'Postgres to MacroDB',+ cidrIpv4: vpcCidr,+ ipProtocol: 'tcp',+ fromPort: RDS_PORT,+ toPort: RDS_PORT,+ tags,+ },+ child+ );++ // The FireLens router's own diagnostics; application logs ship to Datadog.+ const logGroup = new aws.cloudwatch.LogGroup(+ `${BASE_NAME}-log-group`,+ {+ name: `/ecs/${BASE_NAME}-${stack}`,+ retentionInDays: 30,+ tags,+ },+ child+ );+ this.logGroupName = logGroup.name;++ const image = new EcrImage(+ `${BASE_NAME}-ecr-image-${stack}`,+ {+ repositoryId: `${BASE_NAME}-ecr-${stack}`,+ repositoryName: `${BASE_NAME}-${stack}`,+ imageId: `${BASE_NAME}-image-${stack}`,+ imagePath: REPO_ROOT,+ dockerfile: 'docker/Dockerfile',+ platform: { family: 'linux', architecture: 'amd64' },+ buildArgs: { SERVICE_NAME: 'whatsapp_session_service' },+ tags,+ },+ child+ );++ const dopplerEcsEnvironment = new DopplerEcsEnvironment(+ BASE_NAME,+ { tags },+ child+ );++ this.service = new awsx.ecs.FargateService(+ BASE_NAME,+ {+ cluster: args.ecsClusterArn,+ desiredCount: args.desiredCount,+ // Start the replacement before stopping the old task: the old task+ // drains and NOTIFYs `whatsapp_account_released` so a running replica+ // takes each account over at once.+ deploymentMinimumHealthyPercent: 100,+ deploymentMaximumPercent: 200,+ continueBeforeSteadyState: DEFAULT_CONTINUE_BEFORE_STEADY_STATE,+ deploymentCircuitBreaker: { enable: true, rollback: true },+ networkConfiguration: {+ subnets: args.vpc.privateSubnetIds,+ securityGroups: [this.serviceSg.id],+ assignPublicIp: false,+ },+ taskDefinitionArgs: {+ cpu: stack === 'prod' ? '1024' : '512',+ memory: stack === 'prod' ? '4096' : '2048',+ taskRole: { roleArn: this.role.arn },+ executionRole: { roleArn: dopplerEcsEnvironment.executionRole.arn },+ runtimePlatform: {+ operatingSystemFamily: 'LINUX',+ cpuArchitecture: 'X86_64',+ },+ containers: withTelemetry(+ BASE_NAME,+ {+ service: {+ name: 'service',+ image: image.image.imageUri,+ essential: true,+ stopTimeout: STOP_TIMEOUT_SECONDS,+ // Stopping runs in reverse dependency order, so the sidecars+ // outlive the drain and its logs and spans still ship.+ dependsOn: [+ { containerName: 'log_router', condition: 'START' },+ { containerName: 'datadog-agent', condition: 'START' },+ ],+ secrets: [...dopplerEcsEnvironment.containerSecrets],+ environment: [+ { name: 'ENVIRONMENT', value: stack },+ { name: 'DD_SERVICE', value: BASE_NAME },+ { name: 'DD_ENV', value: stack },+ { name: 'RUST_BACKTRACE', value: '1' },+ ],+ // `GET /health` on 8080 is for local compose; the image has+ // no HTTP client for an ECS container health check.+ portMappings: [{ containerPort: 8080, hostPort: 8080 }],+ logConfiguration: {+ logDriver: 'awsfirelens',+ options: {+ Name: 'datadog',+ Host: 'http-intake.logs.us5.datadoghq.com',+ apikey: DATADOG_API_KEY,+ dd_service: BASE_NAME,+ dd_source: 'fargate',+ dd_tags: `env:${stack}`,+ provider: 'ecs',+ },+ },+ },+ },+ {+ logRouter: {+ stopTimeout: STOP_TIMEOUT_SECONDS,+ logConfiguration: {+ logDriver: 'awslogs',+ options: {+ 'awslogs-group': logGroup.name,+ 'awslogs-region': 'us-east-1',+ 'awslogs-stream-prefix': 'log_router',+ },+ },+ },+ datadogAgent: { stopTimeout: STOP_TIMEOUT_SECONDS },+ }+ ),+ },+ tags,+ },+ { parent: this, dependsOn: [httpsEgress, postgresEgress] }+ );++ new EcsDeploymentFailureAlarm(+ `${BASE_NAME}-deployment-failure-alarm`,+ {+ serviceName: BASE_NAME,+ serviceArn: this.service.service.arn,+ tags,+ },+ child+ );++ for (const [metricName, label] of [+ ['CPUUtilization', 'CPU'],+ ['MemoryUtilization', 'Memory'],+ ]) {+ new aws.cloudwatch.MetricAlarm(+ `${BASE_NAME}-high-${label.toLowerCase()}-alarm`,+ {+ name: `${BASE_NAME}-high-${label.toLowerCase()}-alarm-${stack}`,+ metricName,+ namespace: 'AWS/ECS',+ statistic: 'Average',+ period: 180,+ evaluationPeriods: 2,+ threshold: 80,+ comparisonOperator: 'GreaterThanThreshold',+ dimensions: {+ ClusterName: args.ecsClusterName,+ ServiceName: this.service.service.name,+ },+ alarmDescription: `High ${label} usage alarm for ${BASE_NAME}.`,+ actionsEnabled: true,+ alarmActions: [CLOUD_TRAIL_SNS_TOPIC_ARN],+ tags,+ },+ child+ );+ }+ }+}diff --git a/infra/stacks/whatsapp-session-service/tsconfig.json b/infra/stacks/whatsapp-session-service/tsconfig.jsonnew file mode 100644index 0000000000..4082f16a5d--- /dev/null+++ b/infra/stacks/whatsapp-session-service/tsconfig.json@@ -0,0 +1,3 @@+{+ "extends": "../../tsconfig.json"+}diff --git a/nix/cloud-storage.nix b/nix/cloud-storage.nixindex 2bbcede5f4..1b935457eb 100644--- a/nix/cloud-storage.nix+++ b/nix/cloud-storage.nix@@ -428,6 +428,11 @@packageName = "unfurl_service";binaries = [ "unfurl_service" ];}+ {+ serviceName = "whatsapp-session-service";+ packageName = "whatsapp_session_service";+ binaries = [ "whatsapp_session_service" ];+ }];localStackBinaryDefinitions = [@@ -558,6 +563,7 @@"notification-service""static-file-service""unfurl-service"+ "whatsapp-session-service"];localStackBinaries = pkgs.buildEnv {diff --git a/services/whatsapp_session_service/AGENTS.md b/services/whatsapp_session_service/AGENTS.mdnew file mode 100644index 0000000000..f0bee95ed0--- /dev/null+++ b/services/whatsapp_session_service/AGENTS.md@@ -0,0 +1,123 @@+# whatsapp_session_service++## What it is++WhatsApp linked devices: runs one whatsmeow client per linked personal+WhatsApp account, on whichever replica holds the account's Postgres advisory+lock. It pairs devices (QR codes into `whatsapp_pairing`), keeps group rosters+fresh, ingests standalone `whatsapp_group` traffic through `whatsapp_linked`'s+`SessionService`, and dispatches the sends DSS records. No HTTP API besides+`GET /health`; DSS talks to it only through rows and `NOTIFY`.++## How it works++- `src/replica.rs`: `replica::start(ReplicaDeps, shutdown)` composes one+ replica; `main.rs` and end-to-end tests both use it (tests inject the fake+ server's dialer, `test_signal()`, a fake KMS and short `Timings`).+- `domain/supervisor.rs`: which accounts this replica runs. Takes free accounts+ (`pg_try_advisory_lock(hashtextextended('whatsapp-account:' || id, 0))`) on rescan and+ on `NOTIFY`, up to `WHATSAPP_SESSION_MAX_ACCOUNTS`; on SIGTERM releases them+ in batches of `WHATSAPP_SESSION_DRAIN_BATCH` (finish sends, disconnect,+ unlock, `NOTIFY whatsapp_account_released`), all within 100 s.+- `domain/account.rs`: one account. `begin_lease` bumps `lease_epoch`; every+ domain and store write is fenced by it, and `Fenced` stops the account.+ Pairing an account that is not connected deletes the stored device first+ (fresh keys); `whatsapp_account_changed` to `disconnected` logs the device+ out at WhatsApp (the remote unlink DSS's disconnect promises).+- `domain/inbound.rs`: durable inbound. The client's event handler (with+ `synchronous_ack`) calls the `InboundGate`: drop unadopted groups, else+ `enqueue_inbound` (one fenced insert), and only then is the node acked;+ `replay_deferred` ingests the queue from the runner. The handler never+ waits on the runner or the socket.+- `domain/media.rs`: per-account import loop (`claim_media_imports` →+ download → `complete_media_import` / `fail_media_import`). Media sends+ upload before `begin_dispatch`. Two transfers per account, 32 MiB cap.+- `outbound/lease.rs`: the lock connection (one non-pooled `PgConnection`,+ pinged every 5 s; if it dies every account stops and the process exits+ non-zero so ECS replaces it), release announcements, and discovery through+ `SessionService` (`discoverable_accounts`, `fleet_health`).+- `outbound/signals.rs`: `LISTEN` for `whatsapp_{account_released,+ pairing_requested, send_requested, groups_requested, account_changed}`.+- `outbound/whatsapp.rs` (+ `events.rs`): `whatsapp_store_pg` device store,+ whatsmeow `Client` (`whatsmeow_signal::libsignal()`), event mapping. Only+ group events become inbound events; the gate drops groups the account has+ not adopted. Adopted groups stay cached until `whatsapp_account_changed`;+ a miss always waits for a load that started after the event arrived+ (single-flight), so an adoption that committed just before is seen. Media locators (direct path, media key, hashes) are sealed with+ `PgWhatsAppStore::seal("media-locator", …)` before they leave the process.+- Passkey pairing: `PairPasskeyRequest` → `PairingUpdate::Passkey { challenge }`;+ on `whatsapp_pairing_requested`, and on every tick in case the notify was+ missed, the runner takes the owner's assertion (`take_passkey_response`)+ and forwards it; a confirmation that needs the+ handoff shows its code as the pairing's `pair_code`, then confirms.++## Config (Doppler `whatsapp-session-service`, `dev` and `prd`)++- `DATABASE_URL`: pooled MacroDB URL.+- `WHATSAPP_SESSION_LOCK_DATABASE_URL`: the RDS **instance** endpoint+ (`macro-db-<stack>`, the macrodb stack's `endpoint` output), never+ `macrodb-proxy-<stack>`: RDS Proxy multiplexes and reaps idle connections,+ which drops advisory locks and `LISTEN`. Never open a transaction on it:+ MacroDB kills sessions idle in transaction after 5 minutes.+- `WHATSAPP_SESSION_KMS_KEY_ID`: `alias/whatsapp-session-service-<stack>`+ (GenerateDataKey + Decrypt).+- `WHATSAPP_SESSION_MAX_ACCOUNTS` (local 500), `WHATSAPP_SESSION_DRAIN_BATCH`+ (local 20).+- Tests only, process env: `OVERRIDE_WHATSAPP_WEBSOCKET_URL` (fake server)+ and `OVERRIDE_WHATSAPP_MEDIA_HOST` (`scheme://host[:port]` of its media+ host).++## Telemetry++Spans: `whatsapp.account.{lease,release,rescan,connect}`,+`whatsapp.inbound.ingest`, `whatsapp.media.import`, `whatsapp.send.dispatch`,+`whatsapp.groups.refresh`, `whatsapp.drain`. Log events read by `infra/stacks/monitoring/monitors/+whatsapp-session.ts` (`telemetry_event` = `whatsapp_session_health`,+`whatsapp_session_reconnect`, `whatsapp_send_outcome`,+`whatsapp_session_drain`): keep their field names. Opaque ids only; never+JIDs, phone numbers, QR codes or message text.++## Runbook++- **Stuck lease** (an account no replica runs, `accounts_unowned` > 0): find+ the holder with `SELECT pid FROM pg_locks WHERE locktype = 'advisory' AND+ objsubid = 1 AND ((classid::bigint << 32) | objid::bigint) =+ hashtextextended('whatsapp-account:<id>', 0)`; if its replica is gone,+ `pg_terminate_backend(pid)` frees it and the next rescan (30 s) takes it.+- **All accounts offline**: check the lock URL points at the instance endpoint,+ replicas are not crash-looping on a lost lock connection, and capacity+ (`accounts_running` vs `WHATSAPP_SESSION_MAX_ACCOUNTS` × replicas).+- **KMS failure**: accounts exit `Failed` at open (`whatsapp.account.connect`+ outcome `failed`, error "KMS refused"); check the task role can use the key+ alias and the key is enabled. Nothing is lost; the next rescan retries.+- **Unknown sends**: never resent automatically; a later echo with the same+ message id settles them.++## Checks++- `cargo test -p whatsapp_session_service` (`SQLX_OFFLINE` unset; the lease+ tests need Postgres).++## Gotchas++- Release order matters: unlock before `NOTIFY`, or the next replica's+ `try_lock` loses the race.+- The event handler runs on whatsmeow's node-handler queue, not the socket+ reader, so a database insert there cannot block send acks; it must still+ never wait on the runner (session events go through an unbounded channel).+- Every path that acknowledges an event without storing it logs a debug+ event `whatsapp.inbound.skipped` with a `reason` (`not_group_content`,+ `group_not_adopted`, `invalid_event`). Once stored, the domain may still+ drop an event as not observed (consent boundary).+- whatsmeow's media futures are large: the adapter boxes them, or nesting+ them in the runner overflows a 2 MiB stack in debug builds.+- Every send sets `no_disconnect_retry`: a send cut off by a disconnect is+ recorded `unknown` instead of being resent after reconnect.+- whatsmeow emits a sender key distribution as its own message event with+ the content's id; `events.rs` drops it, or it would claim the id.+- Builds from the shared `docker/Dockerfile`; there is no per-service one.++## Related++- `crates/whatsapp_linked/AGENTS.md`, `crates/whatsapp_store_pg/AGENTS.md`,+ `crates/whatsmeow/AGENTS.md`, `infra/stacks/whatsapp-session-service/`diff --git a/services/whatsapp_session_service/CLAUDE.md b/services/whatsapp_session_service/CLAUDE.mdnew file mode 120000index 0000000000..47dc3e3d86--- /dev/null+++ b/services/whatsapp_session_service/CLAUDE.md@@ -0,0 +1 @@+AGENTS.md\ No newline at end of filediff --git a/services/whatsapp_session_service/Cargo.toml b/services/whatsapp_session_service/Cargo.tomlnew file mode 100644index 0000000000..9a37c014f8--- /dev/null+++ b/services/whatsapp_session_service/Cargo.toml@@ -0,0 +1,50 @@+[package]+default-run = "whatsapp_session_service"+edition = "2024"+name = "whatsapp_session_service"+publish = false+version = "0.1.0"++[[bin]]+name = "whatsapp_session_service"+path = "src/main.rs"++[[bin]]+name = "whatsapp_session_service_doppler_config"+path = "src/doppler_config.rs"++[dependencies]+async-trait = { workspace = true }+aws-sdk-kms = { workspace = true }+axum = { workspace = true }+chrono = { workspace = true }+futures = { workspace = true }+database_env_vars = { path = "../../crates/database_env_vars" }+doppler-config = { workspace = true }+macro_aws_config = { path = "../../crates/macro_aws_config" }+macro_config = { path = "../../crates/macro_config" }+macro_env_var = { path = "../../crates/macro_env_var" }+macro_entrypoint = { path = "../../crates/macro_entrypoint" }+macro_env = { path = "../../crates/macro_env" }+macro_service_urls = { path = "../../crates/macro_service_urls" }+rootcause = { workspace = true }+rustls = { workspace = true }+serde = { workspace = true }+serde_json = { workspace = true }+sqlx = { workspace = true }+thiserror = { workspace = true }+tokio = { workspace = true }+tokio-util = { workspace = true }+tracing = { workspace = true }+uuid = { workspace = true }+whatsapp_store_pg = { path = "../../crates/whatsapp_store_pg" }+whatsmeow = { path = "../../crates/whatsmeow" }+whatsmeow_signal = { path = "../../crates/whatsmeow_signal" }+whatsapp_linked = { path = "../../crates/whatsapp_linked", features = ["outbound"] }+workspace-hack = { version = "0.1", path = "../../crates/workspace-hack" }++[dev-dependencies]+macro_db_migrator = { path = "../../crates/macro_db_migrator" }+sha2 = { workspace = true }+whatsapp_fake_server = { path = "../../crates/whatsapp_fake_server" }+whatsapp_store_pg = { path = "../../crates/whatsapp_store_pg", features = ["fake-kms"] }diff --git a/services/whatsapp_session_service/src/config.rs b/services/whatsapp_session_service/src/config.rsnew file mode 100644index 0000000000..08a5ae8374--- /dev/null+++ b/services/whatsapp_session_service/src/config.rs@@ -0,0 +1,26 @@+//! Startup configuration. Every value comes from Doppler (`APP_SECRETS_JSON` in+//! ECS); `main.rs` reads the test-only overrides from the process environment.++use database_env_vars::DatabaseUrl;+use macro_env_var::env_vars;++env_vars! {+ /// Direct, non-pooled MacroDB URL for the advisory-lock connection and `LISTEN`.+ pub struct WhatsappSessionLockDatabaseUrl;+}++/// Settings supplied by the deployment.+#[derive(macro_config::MacroConfig)]+#[serde(rename_all = "SCREAMING_SNAKE_CASE")]+pub struct Config {+ /// Pooled MacroDB URL for every read and fenced write.+ pub database_url: DatabaseUrl,+ /// Direct, non-pooled MacroDB URL: the advisory-lock connection and `LISTEN`.+ pub whatsapp_session_lock_database_url: WhatsappSessionLockDatabaseUrl,+ /// KMS key that wraps each account's data key.+ pub whatsapp_session_kms_key_id: String,+ /// Most accounts one replica runs.+ pub whatsapp_session_max_accounts: u32,+ /// Accounts released per drain step on shutdown.+ pub whatsapp_session_drain_batch: u32,+}diff --git a/services/whatsapp_session_service/src/domain.rs b/services/whatsapp_session_service/src/domain.rsnew file mode 100644index 0000000000..2f4888c0aa--- /dev/null+++ b/services/whatsapp_session_service/src/domain.rs@@ -0,0 +1,8 @@+//! Account ownership and the per-account session, independent of Postgres and+//! of the WhatsApp protocol.++pub mod account;+pub mod inbound;+pub mod media;+pub mod ports;+pub mod supervisor;diff --git a/services/whatsapp_session_service/src/domain/account.rs b/services/whatsapp_session_service/src/domain/account.rsnew file mode 100644index 0000000000..2531a61d34--- /dev/null+++ b/services/whatsapp_session_service/src/domain/account.rs@@ -0,0 +1,712 @@+//! One leased account: take the lease epoch, open the device, pair or+//! connect, ingest adopted-group traffic, dispatch sends, and hand the account+//! back cleanly.++use std::sync::Arc;+use std::time::Duration;++use tokio::sync::{Notify, Semaphore, mpsc, watch};+use tokio_util::sync::CancellationToken;+use tracing::Instrument;+use uuid::Uuid;+use whatsapp_linked::domain::events::{+ ClaimedSend, Lease, OutgoingContent, PairingUpdate, PairingWork, SendOutcome, SessionScope,+};+use whatsapp_linked::domain::models::{AccountState, LinkedError, PairingState};+use whatsapp_linked::domain::ports::LinkedResult;+use whatsapp_linked::domain::session::SessionService;++use super::inbound::InboundGate;+use super::media::{MEDIA_LIMIT_BYTES, MediaImports, TRANSFERS_PER_ACCOUNT};+use super::ports::{+ AccountControl, AccountExit, AccountRunner, AccountSignal, SendFailure, SessionConnector,+ SessionError, SessionEvent, UploadedMedia, WhatsAppSession,+};++#[cfg(test)]+mod test;++/// Most `replay_deferred` rounds per wake-up, so a large backlog cannot+/// starve signals and sends.+const REPLAY_ROUNDS: usize = 8;++/// Timings and batch sizes for one account.+#[derive(Debug, Clone, Copy)]+pub struct AccountConfig {+ /// How long a submitted send may wait for the server's acknowledgement+ /// before its outcome is recorded as unknown.+ pub send_timeout: Duration,+ /// Sends claimed per round trip.+ pub send_batch: i64,+ /// How often to look for sends, deferred events and pairing expiry+ /// without a signal.+ pub tick_every: Duration,+}++/// Runs leased accounts against the linked-devices domain.+pub struct AccountSessions<Connector> {+ sessions: SessionService,+ connector: Connector,+ config: AccountConfig,+}++impl<Connector: SessionConnector> AccountSessions<Connector> {+ /// Wires the domain service and the WhatsApp connector.+ pub fn new(sessions: SessionService, connector: Connector, config: AccountConfig) -> Self {+ AccountSessions {+ sessions,+ connector,+ config,+ }+ }+}++/// How the event loop should continue.+enum Next {+ Continue,+ Exit(AccountExit),+}++fn is_fenced(report: &rootcause::Report<LinkedError>) -> bool {+ matches!(report.current_context(), LinkedError::Fenced)+}++/// Fenced ends the account; any other domain error is logged and survived.+fn survive<Value>(+ result: LinkedResult<Value>,+ step: &'static str,+) -> Result<Option<Value>, AccountExit> {+ match result {+ Ok(value) => Ok(Some(value)),+ Err(report) if is_fenced(&report) => Err(AccountExit::Fenced),+ Err(report) => {+ tracing::warn!(+ step,+ code = report.current_context().code(),+ error = %report,+ "WhatsApp account step failed"+ );+ Ok(None)+ }+ }+}++impl<Connector: SessionConnector> AccountRunner for AccountSessions<Connector> {+ async fn run(&self, account: Uuid, control: AccountControl) -> AccountExit {+ let lease = match self.sessions.begin_lease(account).await {+ Ok(lease) => lease,+ Err(report) if matches!(report.current_context(), LinkedError::NotFound) => {+ return AccountExit::Idle;+ }+ Err(report) => {+ tracing::warn!(+ account_id = %account,+ code = report.current_context().code(),+ error = %report,+ "Taking the WhatsApp account lease failed"+ );+ return AccountExit::Failed;+ }+ };+ let pairing = match survive(+ self.sessions.live_pairing(lease.scope).await,+ "live_pairing",+ ) {+ Ok(Some(pairing)) => pairing,+ Ok(None) => return AccountExit::Failed,+ Err(exit) => return exit,+ };+ // A `pairing` account without a live transaction has nothing to show.+ if lease.state != AccountState::Connected && pairing.is_none() {+ return AccountExit::Idle;+ }+ let connect_span = tracing::info_span!(+ "whatsapp.account.connect",+ account_id = %account,+ lease_epoch = lease.scope.lease_epoch,+ paired = tracing::field::Empty,+ pairing = pairing.is_some(),+ outcome = tracing::field::Empty,+ );+ let gate = Arc::new(InboundGate::new(self.sessions.clone(), lease.scope));+ let opened = self+ .open(&lease, pairing.as_ref(), Arc::clone(&gate))+ .instrument(connect_span.clone())+ .await;+ connect_span.record(+ "outcome",+ match &opened {+ Ok(Some(_)) => "connecting",+ Ok(None) => "idle",+ Err(AccountExit::Fenced) => "fenced",+ Err(_) => "failed",+ },+ );+ let (session, events) = match opened {+ Ok(Some(opened)) => opened,+ Ok(None) => return AccountExit::Idle,+ Err(exit) => return exit,+ };+ let session = Arc::new(session);+ let transfers = Arc::new(Semaphore::new(TRANSFERS_PER_ACCOUNT));+ let media_wake = Arc::new(Notify::new());+ let (connected, connected_receiver) = watch::channel(false);+ let media_stop = CancellationToken::new();+ let imports = tokio::spawn(+ MediaImports {+ sessions: self.sessions.clone(),+ scope: lease.scope,+ session: Arc::clone(&session),+ transfers: Arc::clone(&transfers),+ wake: Arc::clone(&media_wake),+ connected: connected_receiver,+ every: self.config.tick_every,+ fenced: gate.fenced.clone(),+ }+ .run(media_stop.clone()),+ );+ let mut running = Running {+ sessions: &self.sessions,+ config: self.config,+ scope: lease.scope,+ session,+ gate,+ transfers,+ media_wake,+ connected,+ pairing,+ reconnecting: false,+ };+ let exit = running.serve(events, control).await;+ media_stop.cancel();+ if exit == AccountExit::Stopped {+ imports.abort();+ }+ let _ = imports.await;+ exit+ }+}++impl<Connector: SessionConnector> AccountSessions<Connector> {+ /// Opens the device and starts connecting, or `None` when an unpaired+ /// device has no pairing transaction to serve. Pairing an account that+ /// is not connected starts from a fresh device: the old one was logged+ /// out or disconnected and its keys are useless.+ async fn open(+ &self,+ lease: &Lease,+ pairing: Option<&PairingWork>,+ gate: Arc<InboundGate>,+ ) -> Result<Option<(Connector::Session, mpsc::UnboundedReceiver<SessionEvent>)>, AccountExit>+ {+ let fresh_device = pairing.is_some() && lease.state != AccountState::Connected;+ let (session, events) = match self.connector.open(lease.scope, fresh_device, gate).await {+ Ok(opened) => opened,+ Err(SessionError::Fenced) => return Err(AccountExit::Fenced),+ Err(error) => {+ tracing::error!(error = %error, "Opening the WhatsApp device failed");+ return Err(AccountExit::Failed);+ }+ };+ let paired = session.identity_jid().await.is_some();+ tracing::Span::current().record("paired", paired);+ if !paired {+ if pairing.is_none() {+ return Ok(None);+ }+ if let Err(error) = session.start_pairing().await {+ tracing::error!(error = %error, "Starting WhatsApp pairing failed");+ return Err(AccountExit::Failed);+ }+ }+ if let Err(error) = session.connect().await {+ tracing::error!(error = %error, "Connecting to WhatsApp failed");+ session.disconnect().await;+ return Err(AccountExit::Failed);+ }+ Ok(Some((session, events)))+ }+}++struct Running<'service, Session> {+ sessions: &'service SessionService,+ config: AccountConfig,+ scope: SessionScope,+ session: Arc<Session>,+ /// Stores adopted-group events from the client and filters the rest.+ gate: Arc<InboundGate>,+ /// Upload and download slots shared with the import loop.+ transfers: Arc<Semaphore>,+ /// Wakes the import loop after ingestion.+ media_wake: Arc<Notify>,+ connected: watch::Sender<bool>,+ pairing: Option<PairingWork>,+ reconnecting: bool,+}++impl<Session: WhatsAppSession> Running<'_, Session> {+ fn is_connected(&self) -> bool {+ *self.connected.borrow()+ }++ async fn serve(+ &mut self,+ mut events: mpsc::UnboundedReceiver<SessionEvent>,+ mut control: AccountControl,+ ) -> AccountExit {+ let gate = Arc::clone(&self.gate);+ let mut tick = tokio::time::interval(self.config.tick_every);+ tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);+ let exit = loop {+ let next = tokio::select! {+ biased;+ () = control.stop.cancelled() => Next::Exit(AccountExit::Stopped),+ () = control.release.cancelled() => Next::Exit(self.release().await),+ () = gate.fenced.cancelled() => Next::Exit(AccountExit::Fenced),+ () = gate.queued.notified() => {+ let result = self.replay().await;+ self.exit_on(result)+ }+ event = events.recv() => match event {+ Some(event) => self.on_event(event).await,+ None => Next::Exit(AccountExit::Failed),+ },+ Some(signal) = control.signals.recv() => self.on_signal(signal).await,+ _ = tick.tick() => self.on_tick().await,+ };+ if let Next::Exit(exit) = next {+ break exit;+ }+ };+ if exit != AccountExit::Released {+ self.session.disconnect().await;+ }+ exit+ }++ /// Drain: no new sends, disconnect, and hand undispatched claims back.+ async fn release(&mut self) -> AccountExit {+ self.session.disconnect().await;+ match survive(+ self.sessions.release_lease(self.scope).await,+ "release_lease",+ ) {+ Ok(_) => AccountExit::Released,+ Err(exit) => exit,+ }+ }++ fn exit_on(&self, result: Result<(), AccountExit>) -> Next {+ match result {+ Ok(()) => Next::Continue,+ Err(exit) => Next::Exit(exit),+ }+ }++ async fn on_event(&mut self, event: SessionEvent) -> Next {+ match event {+ SessionEvent::Qr(code) => {+ let result = self.update_pairing(PairingUpdate::Qr(code)).await;+ self.exit_on(result)+ }+ SessionEvent::PairSuccess { identity_jid } => {+ let result = self+ .update_pairing(PairingUpdate::Success { identity_jid })+ .await;+ self.exit_on(result)+ }+ SessionEvent::PasskeyRequested { challenge } => {+ let result = self+ .update_pairing(PairingUpdate::Passkey { challenge })+ .await;+ self.exit_on(result)+ }+ SessionEvent::PasskeyConfirmation { code } => {+ let result = self.confirm_passkey(code).await;+ self.exit_on(result)+ }+ SessionEvent::PairFailed => {+ let result = self.update_pairing(PairingUpdate::Failed).await;+ match result {+ Ok(()) => Next::Exit(AccountExit::Idle),+ Err(exit) => Next::Exit(exit),+ }+ }+ SessionEvent::Connected => {+ if self.reconnecting {+ tracing::info!(+ telemetry_event = "whatsapp_session_reconnect",+ account_id = %self.scope.account_id,+ "WhatsApp session reconnected"+ );+ }+ self.connected.send_replace(true);+ self.reconnecting = false;+ let result = self.on_connected().await;+ self.exit_on(result)+ }+ SessionEvent::Disconnected => {+ self.connected.send_replace(false);+ self.reconnecting = true;+ Next::Continue+ }+ SessionEvent::LoggedOut => {+ match survive(self.sessions.logged_out(self.scope).await, "logged_out") {+ Ok(_) => Next::Exit(AccountExit::Idle),+ Err(exit) => Next::Exit(exit),+ }+ }+ SessionEvent::Replaced => {+ tracing::error!(+ account_id = %self.scope.account_id,+ "Another WhatsApp connection replaced this account's session"+ );+ Next::Exit(AccountExit::Failed)+ }+ }+ }++ async fn on_signal(&mut self, signal: AccountSignal) -> Next {+ let result = match signal {+ AccountSignal::Pairing(pairing) => self.check_pairing(pairing).await,+ AccountSignal::Send => self.dispatch_sends().await,+ AccountSignal::Groups => self.refresh_groups().await,+ AccountSignal::Changed => self.account_changed().await,+ AccountSignal::Resync => self.resync().await,+ };+ self.exit_on(result)+ }++ async fn on_tick(&mut self) -> Next {+ let result = async {+ if let Some(pairing) = self.pairing.as_ref().map(|pairing| pairing.id) {+ self.check_pairing(pairing).await?;+ }+ self.replay().await?;+ if self.is_connected() {+ self.dispatch_sends().await?;+ }+ Ok(())+ }+ .await;+ self.exit_on(result)+ }++ async fn resync(&mut self) -> Result<(), AccountExit> {+ if let Some(pairing) = self.pairing.as_ref().map(|pairing| pairing.id) {+ self.check_pairing(pairing).await?;+ }+ self.replay().await?;+ if self.is_connected() {+ self.gate.reload().await;+ self.refresh_groups().await?;+ self.dispatch_sends().await?;+ }+ Ok(())+ }++ /// Ingests what the gate queued (and deferred events now due), then lets+ /// the import loop look for new attachments.+ async fn replay(&mut self) -> Result<(), AccountExit> {+ for _ in 0..REPLAY_ROUNDS {+ match survive(+ self.sessions.replay_deferred(self.scope).await,+ "replay_deferred",+ )? {+ Some(applied) if applied > 0 => {}+ _ => break,+ }+ }+ self.media_wake.notify_one();+ Ok(())+ }++ async fn on_connected(&mut self) -> Result<(), AccountExit> {+ if let Some(pairing) = self.pairing.clone()+ && let Some(identity_jid) = self.session.identity_jid().await+ {+ // A reconnect of an already paired device completes its pairing.+ if pairing.state != PairingState::Success {+ self.update_pairing(PairingUpdate::Success { identity_jid })+ .await?;+ }+ }+ self.gate.reload().await;+ self.refresh_groups().await?;+ self.replay().await?;+ self.dispatch_sends().await+ }++ /// The phone shows a code: surface it on the pairing row, then confirm.+ async fn confirm_passkey(&mut self, code: String) -> Result<(), AccountExit> {+ self.update_pairing(PairingUpdate::PairCode(code)).await?;+ if let Err(error) = self.session.confirm_passkey().await {+ tracing::warn!(error = %error, "Confirming the WhatsApp passkey pairing failed");+ self.update_pairing(PairingUpdate::Failed).await?;+ }+ Ok(())+ }++ /// Reports pairing progress; a terminal transaction ends the account.+ async fn update_pairing(&mut self, update: PairingUpdate) -> Result<(), AccountExit> {+ let Some(pairing) = self.pairing.as_ref().map(|pairing| pairing.id) else {+ return Ok(());+ };+ let success = matches!(update, PairingUpdate::Success { .. });+ match self+ .sessions+ .update_pairing(self.scope, pairing, update)+ .await+ {+ Ok(work) => self.after_pairing(work).await,+ Err(report) if is_fenced(&report) => Err(AccountExit::Fenced),+ Err(report) if success => {+ // The phone paired as someone it must not be: unlink the device.+ tracing::warn!(+ code = report.current_context().code(),+ "WhatsApp pairing rejected; logging the device out"+ );+ self.session.logout().await;+ Err(AccountExit::Idle)+ }+ Err(report) => {+ tracing::warn!(+ code = report.current_context().code(),+ error = %report,+ "Recording WhatsApp pairing progress failed"+ );+ Ok(())+ }+ }+ }++ async fn check_pairing(&mut self, pairing: Uuid) -> Result<(), AccountExit> {+ match survive(self.sessions.pairing_work(pairing).await, "pairing_work")? {+ Some(Some(work)) if work.account_id == self.scope.account_id => {+ let pending_response = work.passkey_response.is_some();+ self.after_pairing(work).await?;+ if pending_response {+ self.forward_passkey(pairing).await?;+ }+ Ok(())+ }+ _ => Ok(()),+ }+ }++ /// Hands the owner's passkey assertion to WhatsApp, once.+ async fn forward_passkey(&mut self, pairing: Uuid) -> Result<(), AccountExit> {+ let Some(Some(response)) = survive(+ self.sessions+ .take_passkey_response(self.scope, pairing)+ .await,+ "take_passkey_response",+ )?+ else {+ return Ok(());+ };+ if let Err(error) = self.session.send_passkey_response(response).await {+ tracing::warn!(error = %error, "Forwarding the WhatsApp passkey assertion failed");+ self.update_pairing(PairingUpdate::Failed).await?;+ }+ Ok(())+ }++ async fn after_pairing(&mut self, work: PairingWork) -> Result<(), AccountExit> {+ match work.state {+ PairingState::Waiting | PairingState::Qr | PairingState::Passkey => {+ self.pairing = Some(work);+ Ok(())+ }+ PairingState::Success => {+ self.pairing = None;+ Ok(())+ }+ PairingState::Cancelled => {+ self.pairing = None;+ if self.session.identity_jid().await.is_some() && !self.is_connected() {+ self.session.logout().await;+ }+ if self.is_connected() {+ Ok(())+ } else {+ Err(AccountExit::Idle)+ }+ }+ PairingState::Expired | PairingState::Failed => {+ self.pairing = None;+ if self.is_connected() {+ Ok(())+ } else {+ Err(AccountExit::Idle)+ }+ }+ }+ }++ /// Consent or adopted groups changed. An owner who disconnected asked for+ /// the device to be unlinked at WhatsApp.+ async fn account_changed(&mut self) -> Result<(), AccountExit> {+ match self.sessions.account_state(self.scope.account_id).await {+ Ok(Some(AccountState::Disconnected)) => {+ self.session.logout().await;+ Err(AccountExit::Idle)+ }+ Ok(Some(AccountState::LoggedOut) | None) => Err(AccountExit::Idle),+ Ok(Some(AccountState::Pairing | AccountState::Connected)) => {+ self.gate.reload().await;+ Ok(())+ }+ Err(report) => {+ tracing::warn!(error = %report, "Reading the WhatsApp account state failed");+ self.gate.reload().await;+ Ok(())+ }+ }+ }++ async fn refresh_groups(&mut self) -> Result<(), AccountExit> {+ if !self.is_connected() {+ return Ok(());+ }+ let span = tracing::info_span!(+ "whatsapp.groups.refresh",+ account_id = %self.scope.account_id,+ count = tracing::field::Empty,+ );+ async {+ match self.session.joined_groups().await {+ Ok(groups) => {+ tracing::Span::current().record("count", groups.len());+ survive(+ self.sessions.update_groups(self.scope, &groups, true).await,+ "update_groups",+ )?;+ Ok(())+ }+ Err(SessionError::Fenced) => Err(AccountExit::Fenced),+ Err(error) => {+ tracing::warn!(error = %error, "Listing WhatsApp groups failed");+ Ok(())+ }+ }+ }+ .instrument(span)+ .await+ }++ /// Claims pending sends and dispatches them one at a time, in group order.+ async fn dispatch_sends(&mut self) -> Result<(), AccountExit> {+ if !self.is_connected() {+ return Ok(());+ }+ loop {+ let Some(claimed) = survive(+ self.sessions+ .claim_sends(self.scope, self.config.send_batch)+ .await,+ "claim_sends",+ )?+ else {+ return Ok(());+ };+ if claimed.is_empty() {+ return Ok(());+ }+ for send in &claimed {+ self.dispatch(send).await?;+ }+ }+ }++ /// Uploads a media send's attachment, before the point of no return. A+ /// failed upload reached nobody, so the send is recorded rejected.+ async fn upload(+ &mut self,+ send: &ClaimedSend,+ ) -> Result<Result<Option<UploadedMedia>, &'static str>, AccountExit> {+ if !matches!(send.content, OutgoingContent::Media { .. }) {+ return Ok(Ok(None));+ }+ let Some(media) = survive(+ self.sessions.outbound_media(self.scope, send.id).await,+ "outbound_media",+ )?+ else {+ return Ok(Err("media_unavailable"));+ };+ if media.bytes.len() as u64 > MEDIA_LIMIT_BYTES {+ return Ok(Err("media_too_large"));+ }+ let Ok(_permit) = self.transfers.acquire().await else {+ return Ok(Err("media_upload_failed"));+ };+ Ok(match self.session.upload_media(&media).await {+ Ok(uploaded) => Ok(Some(uploaded)),+ Err(SendFailure::Rejected(code)) => Err(code),+ Err(SendFailure::Unknown) => Err("media_upload_failed"),+ })+ }++ async fn dispatch(&mut self, send: &ClaimedSend) -> Result<(), AccountExit> {+ let span = tracing::info_span!(+ "whatsapp.send.dispatch",+ account_id = %self.scope.account_id,+ send_id = %send.id,+ group_sequence = send.group_sequence,+ outcome = tracing::field::Empty,+ );+ async {+ let uploaded = self.upload(send).await?;+ let message_id = self.session.new_message_id().await;+ // Recorded before submission: from here on the send is never retried.+ if survive(+ self.sessions+ .begin_dispatch(self.scope, send.id, &message_id)+ .await,+ "begin_dispatch",+ )?+ .is_none()+ {+ return Ok(());+ }+ let submitted = match &uploaded {+ Ok(media) => {+ tokio::time::timeout(+ self.config.send_timeout,+ self.session.send(send, &message_id, media.as_ref()),+ )+ .await+ }+ Err(code) => Ok(Err(SendFailure::Rejected(code))),+ };+ let (outcome, name) = match submitted {+ Ok(Ok(server_at)) => (SendOutcome::Accepted { server_at }, "sent"),+ Ok(Err(SendFailure::Rejected(code))) => (+ SendOutcome::Rejected {+ code: code.to_owned(),+ },+ "failed",+ ),+ Ok(Err(SendFailure::Unknown)) | Err(_) => (SendOutcome::Unknown, "unknown"),+ };+ tracing::Span::current().record("outcome", name);+ tracing::info!(+ telemetry_event = "whatsapp_send_outcome",+ account_id = %self.scope.account_id,+ send_id = %send.id,+ outcome = name,+ "WhatsApp send finished"+ );+ survive(+ self.sessions+ .record_send_outcome(self.scope, send.id, outcome)+ .await,+ "record_send_outcome",+ )?;+ Ok(())+ }+ .instrument(span)+ .await+ }+}diff --git a/services/whatsapp_session_service/src/domain/account/test.rs b/services/whatsapp_session_service/src/domain/account/test.rsnew file mode 100644index 0000000000..6a128fe203--- /dev/null+++ b/services/whatsapp_session_service/src/domain/account/test.rs@@ -0,0 +1,550 @@+//! The account runner over the real linked-devices repository, with an+//! in-memory WhatsApp session the test drives.++use std::sync::{Arc, Mutex};+use std::time::Duration;++use chrono::{DateTime, Utc};+use macro_db_migrator::MACRO_DB_MIGRATIONS;+use sqlx::PgPool;+use tokio::sync::mpsc;+use tokio_util::sync::CancellationToken;+use uuid::Uuid;+use whatsapp_linked::domain::events::{+ ClaimedSend, InboundEvent, InboundKind, MediaImport, Message, MessageKind, OutboundMedia,+ SessionScope,+};+use whatsapp_linked::domain::models::{GroupInfo, Identity, Participant};+use whatsapp_linked::domain::session::SessionService;+use whatsapp_linked::outbound::pg::PgLinkedRepository;++use super::{AccountConfig, AccountSessions};+use crate::domain::inbound::InboundGate;+use crate::domain::ports::{+ AccountControl, AccountExit, AccountRunner, AccountSignal, DownloadFailure, InboundSink,+ SendFailure, SessionConnector, SessionError, SessionEvent, UploadedMedia, WhatsAppSession,+};++const IDENTITY: &str = "[email protected]";+const GROUP: &str = "[email protected]";++/// Calls the runner made, in order.+#[derive(Debug, Clone, PartialEq, Eq)]+enum Call {+ /// Lease epoch and whether the device starts fresh.+ Open(i64, bool),+ StartPairing,+ Connect,+ JoinedGroups,+ Logout,+ Disconnect,+ PasskeyResponse(serde_json::Value),+ ConfirmPasskey,+}++#[derive(Clone)]+struct FakeWhatsApp {+ calls: mpsc::UnboundedSender<Call>,+ events: Arc<Mutex<Option<mpsc::UnboundedSender<SessionEvent>>>>,+ inbound: Arc<Mutex<Option<Arc<dyn InboundSink>>>>,+ identity: Arc<Mutex<Option<String>>>,+}++struct FakeSession {+ whatsapp: FakeWhatsApp,+}++impl SessionConnector for FakeWhatsApp {+ type Session = FakeSession;++ async fn open(+ &self,+ scope: SessionScope,+ fresh_device: bool,+ inbound: Arc<dyn InboundSink>,+ ) -> Result<(FakeSession, mpsc::UnboundedReceiver<SessionEvent>), SessionError> {+ let (sender, receiver) = mpsc::unbounded_channel();+ *self.events.lock().unwrap() = Some(sender);+ *self.inbound.lock().unwrap() = Some(inbound);+ self.calls+ .send(Call::Open(scope.lease_epoch, fresh_device))+ .unwrap();+ Ok((+ FakeSession {+ whatsapp: self.clone(),+ },+ receiver,+ ))+ }+}++impl WhatsAppSession for FakeSession {+ async fn identity_jid(&self) -> Option<String> {+ self.whatsapp.identity.lock().unwrap().clone()+ }++ async fn start_pairing(&self) -> Result<(), SessionError> {+ self.whatsapp.calls.send(Call::StartPairing).unwrap();+ Ok(())+ }++ async fn send_passkey_response(&self, response: serde_json::Value) -> Result<(), SessionError> {+ self.whatsapp+ .calls+ .send(Call::PasskeyResponse(response))+ .unwrap();+ Ok(())+ }++ async fn confirm_passkey(&self) -> Result<(), SessionError> {+ self.whatsapp.calls.send(Call::ConfirmPasskey).unwrap();+ Ok(())+ }++ async fn upload_media(&self, _media: &OutboundMedia) -> Result<UploadedMedia, SendFailure> {+ Err(SendFailure::Rejected("not_in_this_test"))+ }++ async fn download_media(&self, _import: &MediaImport) -> Result<Vec<u8>, DownloadFailure> {+ Err(DownloadFailure::Retryable)+ }++ async fn connect(&self) -> Result<(), SessionError> {+ self.whatsapp.calls.send(Call::Connect).unwrap();+ Ok(())+ }++ async fn new_message_id(&self) -> String {+ "3EB0FAKE".to_owned()+ }++ async fn send(+ &self,+ _send: &ClaimedSend,+ _message_id: &str,+ _media: Option<&UploadedMedia>,+ ) -> Result<DateTime<Utc>, SendFailure> {+ Err(SendFailure::Unknown)+ }++ async fn joined_groups(&self) -> Result<Vec<GroupInfo>, SessionError> {+ self.whatsapp.calls.send(Call::JoinedGroups).unwrap();+ Ok(vec![GroupInfo {+ jid: GROUP.to_owned(),+ name: "Launch".to_owned(),+ participants: vec![Participant {+ identity: Identity {+ jid: IDENTITY.to_owned(),+ alternate_jid: None,+ display_name: None,+ },+ admin: true,+ }],+ announce: false,+ disappearing: false,+ community: false,+ observed_at: Utc::now(),+ group: None,+ }])+ }++ async fn logout(&self) {+ self.whatsapp.calls.send(Call::Logout).unwrap();+ }++ async fn disconnect(&self) {+ self.whatsapp.calls.send(Call::Disconnect).unwrap();+ }+}++/// A user with an account in `pairing` and a live pairing transaction.+async fn insert_pairing_account(pool: &PgPool) -> (Uuid, Uuid) {+ let user_id = "macro|whatsapp-owner";+ sqlx::query!(+ r#"+ WITH new_macro_user AS (+ INSERT INTO macro_user (id, username, email, stripe_customer_id)+ VALUES (gen_random_uuid(), $2, $2, 'stripe_owner')+ RETURNING id+ )+ INSERT INTO "User" ("id", "email", "macro_user_id")+ SELECT $1, $2, new_macro_user.id FROM new_macro_user+ "#,+ user_id,+ "[email protected]",+ )+ .execute(pool)+ .await+ .unwrap();+ let account = Uuid::now_v7();+ sqlx::query!(+ "INSERT INTO whatsapp_linked_account (id, user_id, state) VALUES ($1, $2, 'pairing')",+ account,+ user_id,+ )+ .execute(pool)+ .await+ .unwrap();+ let pairing = Uuid::now_v7();+ sqlx::query!(+ r#"INSERT INTO whatsapp_pairing (id, account_id, user_id, generation, state, expires_at)+ VALUES ($1, $2, $3, 1, 'waiting', now() + interval '5 minutes')"#,+ pairing,+ account,+ user_id,+ )+ .execute(pool)+ .await+ .unwrap();+ (account, pairing)+}++fn runner(+ pool: &PgPool,+) -> (+ Arc<AccountSessions<FakeWhatsApp>>,+ FakeWhatsApp,+ mpsc::UnboundedReceiver<Call>,+) {+ runner_ticking(Duration::from_secs(600), pool)+}++fn runner_ticking(+ tick_every: Duration,+ pool: &PgPool,+) -> (+ Arc<AccountSessions<FakeWhatsApp>>,+ FakeWhatsApp,+ mpsc::UnboundedReceiver<Call>,+) {+ let (calls, call_receiver) = mpsc::unbounded_channel();+ let whatsapp = FakeWhatsApp {+ calls,+ events: Arc::default(),+ inbound: Arc::default(),+ identity: Arc::default(),+ };+ let runner = Arc::new(AccountSessions::new(+ SessionService::new(Arc::new(PgLinkedRepository::new(pool.clone()))),+ whatsapp.clone(),+ AccountConfig {+ send_timeout: Duration::from_secs(5),+ send_batch: 16,+ tick_every,+ },+ ));+ (runner, whatsapp, call_receiver)+}++struct Started {+ signals: mpsc::Sender<AccountSignal>,+ release: CancellationToken,+ running: tokio::task::JoinHandle<AccountExit>,+}++fn start(runner: &Arc<AccountSessions<FakeWhatsApp>>, account: Uuid) -> Started {+ let (signals, signal_receiver) = mpsc::channel(8);+ let release = CancellationToken::new();+ let control = AccountControl {+ signals: signal_receiver,+ release: release.clone(),+ stop: CancellationToken::new(),+ };+ let runner = Arc::clone(runner);+ let running = tokio::spawn(async move { runner.run(account, control).await });+ Started {+ signals,+ release,+ running,+ }+}++fn event_sender(whatsapp: &FakeWhatsApp) -> mpsc::UnboundedSender<SessionEvent> {+ whatsapp.events.lock().unwrap().clone().unwrap()+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn pairing_records_the_qr_and_identity_then_refreshes_groups(pool: PgPool) {+ let (account, pairing) = insert_pairing_account(&pool).await;+ let (runner, whatsapp, mut calls) = runner(&pool);+ let started = start(&runner, account);+ assert_eq!(calls.recv().await, Some(Call::Open(1, true)));+ assert_eq!(calls.recv().await, Some(Call::StartPairing));+ assert_eq!(calls.recv().await, Some(Call::Connect));++ let events = event_sender(&whatsapp);+ events.send(SessionEvent::Qr("qr-1".to_owned())).unwrap();+ events+ .send(SessionEvent::PairSuccess {+ identity_jid: IDENTITY.to_owned(),+ })+ .unwrap();+ *whatsapp.identity.lock().unwrap() = Some(IDENTITY.to_owned());+ events.send(SessionEvent::Connected).unwrap();+ assert_eq!(calls.recv().await, Some(Call::JoinedGroups));+ started.release.cancel();+ assert_eq!(started.running.await.unwrap(), AccountExit::Released);+ assert_eq!(calls.recv().await, Some(Call::Disconnect));++ let paired = sqlx::query!(+ "SELECT state, consumed_at IS NOT NULL AS \"consumed!\", qr_code FROM whatsapp_pairing WHERE id = $1",+ pairing+ )+ .fetch_one(&pool)+ .await+ .unwrap();+ assert_eq!(paired.state, "success");+ assert!(paired.consumed);+ assert_eq!(paired.qr_code, None, "codes are cleared once pairing ends");+ let linked = sqlx::query!(+ "SELECT state, identity_jid, lease_epoch FROM whatsapp_linked_account WHERE id = $1",+ account+ )+ .fetch_one(&pool)+ .await+ .unwrap();+ assert_eq!(linked.state, "connected");+ assert_eq!(linked.identity_jid.as_deref(), Some(IDENTITY));+ assert_eq!(linked.lease_epoch, 1);+ let roster = sqlx::query_scalar!(+ "SELECT group_jid FROM whatsapp_group_roster WHERE account_id = $1",+ account+ )+ .fetch_all(&pool)+ .await+ .unwrap();+ assert_eq!(roster, vec![GROUP.to_owned()]);+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn a_replica_whose_epoch_was_taken_over_cannot_write_and_stops(pool: PgPool) {+ let (account, pairing) = insert_pairing_account(&pool).await;+ let (runner, whatsapp, mut calls) = runner(&pool);+ let started = start(&runner, account);+ assert_eq!(calls.recv().await, Some(Call::Open(1, true)));+ assert_eq!(calls.recv().await, Some(Call::StartPairing));+ assert_eq!(calls.recv().await, Some(Call::Connect));++ let takeover = SessionService::new(Arc::new(PgLinkedRepository::new(pool.clone())))+ .begin_lease(account)+ .await+ .unwrap();+ assert_eq!(takeover.scope.lease_epoch, 2);+ event_sender(&whatsapp)+ .send(SessionEvent::Qr("stale-qr".to_owned()))+ .unwrap();++ assert_eq!(started.running.await.unwrap(), AccountExit::Fenced);+ assert_eq!(calls.recv().await, Some(Call::Disconnect));+ let qr_code = sqlx::query_scalar!(+ "SELECT qr_code FROM whatsapp_pairing WHERE id = $1",+ pairing+ )+ .fetch_one(&pool)+ .await+ .unwrap();+ assert_eq!(qr_code, None, "the fenced replica's write never landed");+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn an_owner_who_disconnects_gets_the_device_unlinked(pool: PgPool) {+ let (account, _pairing) = insert_pairing_account(&pool).await;+ sqlx::query!(+ "UPDATE whatsapp_linked_account SET state = 'connected', identity_jid = $2 WHERE id = $1",+ account,+ IDENTITY,+ )+ .execute(&pool)+ .await+ .unwrap();+ sqlx::query!(+ "UPDATE whatsapp_pairing SET state = 'success', consumed_at = now() WHERE account_id = $1",+ account+ )+ .execute(&pool)+ .await+ .unwrap();+ let (runner, whatsapp, mut calls) = runner(&pool);+ *whatsapp.identity.lock().unwrap() = Some(IDENTITY.to_owned());+ let started = start(&runner, account);+ assert_eq!(+ calls.recv().await,+ Some(Call::Open(1, false)),+ "a connected account keeps its device"+ );+ assert_eq!(calls.recv().await, Some(Call::Connect));++ sqlx::query!(+ "UPDATE whatsapp_linked_account SET state = 'disconnected' WHERE id = $1",+ account+ )+ .execute(&pool)+ .await+ .unwrap();+ started.signals.send(AccountSignal::Changed).await.unwrap();++ assert_eq!(calls.recv().await, Some(Call::Logout));+ assert_eq!(started.running.await.unwrap(), AccountExit::Idle);+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn passkey_pairing_shows_the_challenge_forwards_the_assertion_once_and_confirms(+ pool: PgPool,+) {+ let (account, pairing) = insert_pairing_account(&pool).await;+ let (runner, whatsapp, mut calls) = runner(&pool);+ let started = start(&runner, account);+ assert_eq!(calls.recv().await, Some(Call::Open(1, true)));+ assert_eq!(calls.recv().await, Some(Call::StartPairing));+ assert_eq!(calls.recv().await, Some(Call::Connect));++ let challenge = serde_json::json!({ "challenge": "c2lnbg", "rpId": "whatsapp.com" });+ event_sender(&whatsapp)+ .send(SessionEvent::PasskeyRequested {+ challenge: challenge.clone(),+ })+ .unwrap();+ // The owner answers through DSS, which stores the assertion and signals.+ let assertion = serde_json::json!({ "id": "credential", "type": "public-key" });+ // Events are handled in order: once Connected was handled, so was the request.+ event_sender(&whatsapp)+ .send(SessionEvent::Connected)+ .unwrap();+ assert_eq!(calls.recv().await, Some(Call::JoinedGroups));+ let shown = sqlx::query!(+ "SELECT state, passkey_challenge FROM whatsapp_pairing WHERE id = $1",+ pairing+ )+ .fetch_one(&pool)+ .await+ .unwrap();+ assert_eq!(shown.state, "passkey");+ assert_eq!(shown.passkey_challenge, Some(challenge));+ sqlx::query!(+ "UPDATE whatsapp_pairing SET passkey_response = $2 WHERE id = $1",+ pairing,+ assertion+ )+ .execute(&pool)+ .await+ .unwrap();+ started+ .signals+ .send(AccountSignal::Pairing(pairing))+ .await+ .unwrap();+ assert_eq!(calls.recv().await, Some(Call::PasskeyResponse(assertion)));++ event_sender(&whatsapp)+ .send(SessionEvent::PasskeyConfirmation {+ code: "ABCD-EFGH".to_owned(),+ })+ .unwrap();+ assert_eq!(calls.recv().await, Some(Call::ConfirmPasskey));+ started.release.cancel();+ assert_eq!(started.running.await.unwrap(), AccountExit::Released);+ let confirmed = sqlx::query!(+ "SELECT pair_code, passkey_response FROM whatsapp_pairing WHERE id = $1",+ pairing+ )+ .fetch_one(&pool)+ .await+ .unwrap();+ assert_eq!(confirmed.pair_code.as_deref(), Some("ABCD-EFGH"));+ assert_eq!(+ confirmed.passkey_response, None,+ "the assertion is forwarded once"+ );+}++fn group_text(event: u128) -> InboundEvent {+ InboundEvent {+ event_id: Uuid::from_u128(event),+ group_jid: GROUP.to_owned(),+ source_at: Utc::now(),+ observed_at: Utc::now(),+ kind: InboundKind::Message(Message {+ source_id: format!("3EB0{event}"),+ sender: Identity {+ jid: "[email protected]".to_owned(),+ alternate_jid: None,+ display_name: None,+ },+ kind: MessageKind::Text,+ text: "hello".to_owned(),+ history: false,+ revision: 0,+ target: None,+ media: vec![],+ }),+ }+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn the_gate_acks_unadopted_traffic_unstored_and_refuses_once_fenced(pool: PgPool) {+ let (account, _pairing) = insert_pairing_account(&pool).await;+ let sessions = SessionService::new(Arc::new(PgLinkedRepository::new(pool.clone())));+ let stale = sessions.begin_lease(account).await.unwrap().scope;+ let current = sessions.begin_lease(account).await.unwrap().scope;++ let gate = InboundGate::new(sessions.clone(), current);+ assert!(+ gate.accept(&group_text(1)).await,+ "an unadopted group is acknowledged and dropped"+ );+ let stored = sqlx::query_scalar!(+ r#"SELECT count(*) AS "count!" FROM whatsapp_inbound_event WHERE account_id = $1"#,+ account+ )+ .fetch_one(&pool)+ .await+ .unwrap();+ assert_eq!(stored, 0);++ let fenced = InboundGate::new(sessions, stale);+ assert!(+ !fenced.accept(&group_text(2)).await,+ "a fenced replica leaves the node unacknowledged"+ );+ assert!(fenced.fenced.is_cancelled(), "and stops the account");+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn a_passkey_answer_is_forwarded_on_the_tick_without_a_signal(pool: PgPool) {+ let (account, pairing) = insert_pairing_account(&pool).await;+ let (runner, whatsapp, mut calls) = runner_ticking(Duration::from_millis(50), &pool);+ let started = start(&runner, account);+ assert_eq!(calls.recv().await, Some(Call::Open(1, true)));+ assert_eq!(calls.recv().await, Some(Call::StartPairing));+ assert_eq!(calls.recv().await, Some(Call::Connect));+ event_sender(&whatsapp)+ .send(SessionEvent::PasskeyRequested {+ challenge: serde_json::json!({ "rpId": "whatsapp.com" }),+ })+ .unwrap();+ event_sender(&whatsapp)+ .send(SessionEvent::Connected)+ .unwrap();+ assert_eq!(calls.recv().await, Some(Call::JoinedGroups));++ // DSS stored the assertion but its NOTIFY never reached this replica.+ let assertion = serde_json::json!({ "id": "credential", "type": "public-key" });+ sqlx::query!(+ "UPDATE whatsapp_pairing SET passkey_response = $2 WHERE id = $1",+ pairing,+ assertion+ )+ .execute(&pool)+ .await+ .unwrap();++ let forwarded = loop {+ match calls.recv().await {+ Some(Call::JoinedGroups) => continue,+ other => break other,+ }+ };+ assert_eq!(forwarded, Some(Call::PasskeyResponse(assertion)));+ started.release.cancel();+ assert_eq!(started.running.await.unwrap(), AccountExit::Released);+}diff --git a/services/whatsapp_session_service/src/domain/inbound.rs b/services/whatsapp_session_service/src/domain/inbound.rsnew file mode 100644index 0000000000..3ee0678f3c--- /dev/null+++ b/services/whatsapp_session_service/src/domain/inbound.rs@@ -0,0 +1,209 @@+//! Durable inbound: the client's event handler stores each adopted-group+//! event (one fenced insert) before the protocol node is acknowledged, and+//! the account runner ingests the queue afterwards.++use std::collections::HashSet;+use std::future::Future;+use std::sync::atomic::{AtomicU64, Ordering};++use tokio::sync::{Mutex, Notify};+use tokio_util::sync::CancellationToken;+use tracing::Instrument;+use whatsapp_linked::domain::events::{InboundEvent, SessionScope};+use whatsapp_linked::domain::models::LinkedError;+use whatsapp_linked::domain::ports::LinkedResult;+use whatsapp_linked::domain::session::SessionService;++use super::ports::InboundSink;++#[cfg(test)]+mod test;++/// What the gate needs from the linked-devices domain.+pub trait InboundStore: Send + Sync + 'static {+ /// Groups the account observes now.+ fn observed_groups(+ &self,+ scope: SessionScope,+ ) -> impl Future<Output = LinkedResult<Vec<String>>> + Send;++ /// Stores an event durably for later ingestion.+ fn enqueue_inbound(+ &self,+ scope: SessionScope,+ event: &InboundEvent,+ ) -> impl Future<Output = LinkedResult<()>> + Send;+}++impl InboundStore for SessionService {+ async fn observed_groups(&self, scope: SessionScope) -> LinkedResult<Vec<String>> {+ SessionService::observed_groups(self, scope).await+ }++ async fn enqueue_inbound(&self, scope: SessionScope, event: &InboundEvent) -> LinkedResult<()> {+ SessionService::enqueue_inbound(self, scope, event).await+ }+}++#[derive(Default)]+struct Observed {+ groups: HashSet<String>,+ /// Ticket of the load `groups` came from (0: never loaded).+ loaded_ticket: u64,+}++/// Filters group events to adopted groups and enqueues them durably.+///+/// Adopted groups stay cached until [`InboundGate::reload`]+/// (`whatsapp_account_changed`). A miss never trusts the cache: it waits for+/// a load that started after the event arrived (joining one already in+/// flight, or starting one), so an adoption that committed just before the+/// event is always seen.+pub struct InboundGate<Store = SessionService> {+ store: Store,+ scope: SessionScope,+ /// Locked across a load, so concurrent misses share one read.+ observed: Mutex<Observed>,+ /// Tickets handed to loads in start order.+ load_tickets: AtomicU64,+ /// Woken after an event is enqueued.+ pub(crate) queued: Notify,+ /// Cancelled when a write was fenced: the account must stop.+ pub(crate) fenced: CancellationToken,+}++impl<Store: InboundStore> InboundGate<Store> {+ /// A gate for the account the lease `scope` holds.+ pub fn new(store: Store, scope: SessionScope) -> Self {+ InboundGate {+ store,+ scope,+ observed: Mutex::new(Observed::default()),+ load_tickets: AtomicU64::new(0),+ queued: Notify::new(),+ fenced: CancellationToken::new(),+ }+ }++ /// Re-reads the adopted groups.+ pub(crate) async fn reload(&self) {+ let mut observed = self.observed.lock().await;+ self.reload_locked(&mut observed).await;+ }++ /// `false` when the adopted groups could not be read.+ async fn reload_locked(&self, observed: &mut Observed) -> bool {+ let ticket = self.load_tickets.fetch_add(1, Ordering::SeqCst) + 1;+ match self.store.observed_groups(self.scope).await {+ Ok(groups) => {+ observed.groups = groups.into_iter().collect();+ observed.loaded_ticket = ticket;+ tracing::debug!(+ name: "whatsapp.observed_groups.reloaded",+ account_id = %self.scope.account_id,+ count = observed.groups.len(),+ "Reloaded the WhatsApp groups this account observes"+ );+ true+ }+ Err(report) => {+ if matches!(report.current_context(), LinkedError::Fenced) {+ self.fenced.cancel();+ } else {+ tracing::warn!(+ code = report.current_context().code(),+ error = %report,+ "Reading the adopted WhatsApp groups failed"+ );+ }+ false+ }+ }+ }++ /// Whether the group is adopted, `None` when that could not be decided.+ async fn observes(&self, group_jid: &str) -> Option<bool> {+ // Loads with a later ticket started after this event arrived.+ let arrived = self.load_tickets.load(Ordering::SeqCst);+ let mut observed = self.observed.lock().await;+ if observed.groups.contains(group_jid) {+ return Some(true);+ }+ // A load that started after arrival finished while we waited for the+ // lock: its answer is fresh enough. Otherwise load now.+ if observed.loaded_ticket <= arrived && !self.reload_locked(&mut observed).await {+ return None;+ }+ Some(observed.groups.contains(group_jid))+ }+}++// Object safe for the client's event handler, hence `async_trait`.+#[async_trait::async_trait]+impl<Store: InboundStore> InboundSink for InboundGate<Store> {+ async fn accept(&self, event: &InboundEvent) -> bool {+ if self.fenced.is_cancelled() {+ return false;+ }+ // Undecided (or fenced) leaves the node unacknowledged for redelivery.+ let Some(observed) = self.observes(&event.group_jid).await else {+ return false;+ };+ let span = tracing::info_span!(+ "whatsapp.inbound.ingest",+ account_id = %self.scope.account_id,+ event_id = %event.event_id,+ observed,+ outcome = tracing::field::Empty,+ );+ if !observed {+ span.record("outcome", "dropped");+ tracing::debug!(+ name: "whatsapp.inbound.skipped",+ parent: &span,+ account_id = %self.scope.account_id,+ event_id = %event.event_id,+ reason = "group_not_adopted",+ "Acknowledged a WhatsApp event without storing it"+ );+ return true;+ }+ match self+ .store+ .enqueue_inbound(self.scope, event)+ .instrument(span.clone())+ .await+ {+ Ok(()) => {+ span.record("outcome", "queued");+ self.queued.notify_one();+ true+ }+ Err(report) => {+ let error = report.current_context();+ span.record("outcome", error.code());+ match error {+ LinkedError::Fenced => self.fenced.cancel(),+ // Invalid events are never storable; ack so they are not+ // redelivered forever.+ LinkedError::Invalid(_) => {+ tracing::debug!(+ name: "whatsapp.inbound.skipped",+ parent: &span,+ account_id = %self.scope.account_id,+ event_id = %event.event_id,+ reason = "invalid_event",+ "Acknowledged a WhatsApp event without storing it"+ );+ return true;+ }+ _ => tracing::warn!(+ code = error.code(),+ "Queueing a WhatsApp event failed; leaving it unacknowledged"+ ),+ }+ false+ }+ }+ }+}diff --git a/services/whatsapp_session_service/src/domain/inbound/test.rs b/services/whatsapp_session_service/src/domain/inbound/test.rsnew file mode 100644index 0000000000..375d072cc1--- /dev/null+++ b/services/whatsapp_session_service/src/domain/inbound/test.rs@@ -0,0 +1,98 @@+use std::sync::Mutex;++use chrono::Utc;+use uuid::Uuid;+use whatsapp_linked::domain::events::{+ InboundEvent, InboundKind, Message, MessageKind, SessionScope,+};+use whatsapp_linked::domain::models::Identity;+use whatsapp_linked::domain::ports::LinkedResult;++use super::{InboundGate, InboundStore};+use crate::domain::ports::InboundSink;++const GROUP: &str = "[email protected]";+const SCOPE: SessionScope = SessionScope {+ account_id: Uuid::from_u128(1),+ lease_epoch: 1,+};++/// Adopted groups the test changes, and what was loaded and stored.+#[derive(Default)]+struct FakeStore {+ adopted: Mutex<Vec<String>>,+ loads: Mutex<usize>,+ stored: Mutex<Vec<Uuid>>,+}++impl InboundStore for FakeStore {+ async fn observed_groups(&self, _scope: SessionScope) -> LinkedResult<Vec<String>> {+ *self.loads.lock().unwrap() += 1;+ Ok(self.adopted.lock().unwrap().clone())+ }++ async fn enqueue_inbound(+ &self,+ _scope: SessionScope,+ event: &InboundEvent,+ ) -> LinkedResult<()> {+ self.stored.lock().unwrap().push(event.event_id);+ Ok(())+ }+}++fn text(event: u128) -> InboundEvent {+ InboundEvent {+ event_id: Uuid::from_u128(event),+ group_jid: GROUP.to_owned(),+ source_at: Utc::now(),+ observed_at: Utc::now(),+ kind: InboundKind::Message(Message {+ source_id: format!("3EB0{event}"),+ sender: Identity {+ jid: "[email protected]".to_owned(),+ alternate_jid: None,+ display_name: None,+ },+ kind: MessageKind::Text,+ text: "hello".to_owned(),+ history: false,+ revision: 0,+ target: None,+ media: vec![],+ }),+ }+}++#[tokio::test]+async fn an_adoption_committed_after_a_stale_load_is_seen_by_the_next_message() {+ let gate = InboundGate::new(FakeStore::default(), SCOPE);+ gate.reload().await;+ // The adoption commits right after that load; its NOTIFY is still in flight.+ gate.store.adopted.lock().unwrap().push(GROUP.to_owned());++ assert!(gate.accept(&text(1)).await);++ assert_eq!(*gate.store.stored.lock().unwrap(), vec![Uuid::from_u128(1)]);+}++#[tokio::test]+async fn adopted_groups_stay_cached_and_unadopted_ones_are_checked_every_time() {+ let gate = InboundGate::new(FakeStore::default(), SCOPE);+ gate.store.adopted.lock().unwrap().push(GROUP.to_owned());+ gate.reload().await;++ assert!(gate.accept(&text(1)).await);+ assert!(gate.accept(&text(2)).await);+ assert_eq!(*gate.store.loads.lock().unwrap(), 1, "hits use the cache");++ let mut other = text(3);+ other.group_jid = "[email protected]".to_owned();+ assert!(gate.accept(&other).await, "an unadopted group is acked");+ assert!(gate.accept(&other).await);+ assert_eq!(*gate.store.loads.lock().unwrap(), 3, "every miss loads");+ assert_eq!(+ *gate.store.stored.lock().unwrap(),+ vec![Uuid::from_u128(1), Uuid::from_u128(2)]+ );+}diff --git a/services/whatsapp_session_service/src/domain/media.rs b/services/whatsapp_session_service/src/domain/media.rsnew file mode 100644index 0000000000..454e067c1a--- /dev/null+++ b/services/whatsapp_session_service/src/domain/media.rs@@ -0,0 +1,150 @@+//! Attachment downloads for one account: claim observed attachments, fetch+//! and decrypt them over the account's connection, and hand the bytes to the+//! domain, at most [`TRANSFERS_PER_ACCOUNT`] transfers at a time.++use std::sync::Arc;+use std::time::Duration;++use futures::future::join_all;+use tokio::sync::{Notify, Semaphore, watch};+use tokio_util::sync::CancellationToken;+use tracing::Instrument;+use whatsapp_linked::domain::events::{MediaFailure, MediaImport, SessionScope};+use whatsapp_linked::domain::models::LinkedError;+use whatsapp_linked::domain::session::SessionService;++use super::ports::{DownloadFailure, WhatsAppSession};++/// Concurrent uploads and downloads per account, shared by sends and imports.+pub const TRANSFERS_PER_ACCOUNT: usize = 2;+/// Largest attachment moved in either direction.+pub const MEDIA_LIMIT_BYTES: u64 = 32 * 1024 * 1024;+/// Attachments claimed per round trip.+const IMPORT_BATCH: i64 = 4;++/// Everything the download loop needs.+pub(crate) struct MediaImports<Session> {+ pub(crate) sessions: SessionService,+ pub(crate) scope: SessionScope,+ pub(crate) session: Arc<Session>,+ pub(crate) transfers: Arc<Semaphore>,+ /// Woken when new events were ingested (they may reserve attachments).+ pub(crate) wake: Arc<Notify>,+ /// Downloads only run while connected.+ pub(crate) connected: watch::Receiver<bool>,+ pub(crate) every: Duration,+ /// Cancelled when a write was fenced.+ pub(crate) fenced: CancellationToken,+}++impl<Session: WhatsAppSession> MediaImports<Session> {+ /// Imports until `stop`.+ pub(crate) async fn run(mut self, stop: CancellationToken) {+ loop {+ tokio::select! {+ () = stop.cancelled() => return,+ () = self.fenced.cancelled() => return,+ () = self.wake.notified() => {}+ () = tokio::time::sleep(self.every) => {}+ }+ let connected = tokio::select! {+ () = stop.cancelled() => return,+ connected = self.connected.wait_for(|connected| *connected) => connected.is_ok(),+ };+ if !connected {+ return;+ }+ if !self.import_claimed().await {+ return;+ }+ }+ }++ /// Imports until nothing is claimable; `false` once fenced.+ async fn import_claimed(&self) -> bool {+ loop {+ let claimed = match self+ .sessions+ .claim_media_imports(self.scope, IMPORT_BATCH)+ .await+ {+ Ok(claimed) => claimed,+ Err(report) if matches!(report.current_context(), LinkedError::Fenced) => {+ self.fenced.cancel();+ return false;+ }+ Err(report) => {+ tracing::warn!(error = %report, "Claiming WhatsApp attachments failed");+ return true;+ }+ };+ if claimed.is_empty() {+ return true;+ }+ join_all(claimed.iter().map(|import| self.import(import))).await;+ if self.fenced.is_cancelled() {+ return false;+ }+ }+ }++ async fn import(&self, import: &MediaImport) {+ let Ok(_permit) = self.transfers.acquire().await else {+ return;+ };+ let span = tracing::info_span!(+ "whatsapp.media.import",+ account_id = %self.scope.account_id,+ binding_id = %import.attachment.binding_id,+ ordinal = import.attachment.ordinal,+ size = import.size,+ outcome = tracing::field::Empty,+ );+ async {+ let downloaded = if import.size > MEDIA_LIMIT_BYTES {+ Err(DownloadFailure::Permanent)+ } else {+ self.session.download_media(import).await+ };+ let result = match downloaded {+ Ok(bytes) if bytes.len() as u64 > MEDIA_LIMIT_BYTES => {+ self.fail(import, MediaFailure::Permanent).await+ }+ Ok(bytes) => {+ tracing::Span::current().record("outcome", "imported");+ self.sessions+ .complete_media_import(self.scope, &import.attachment, &bytes)+ .await+ }+ Err(DownloadFailure::Retryable) => self.fail(import, MediaFailure::Retryable).await,+ Err(DownloadFailure::Permanent) => self.fail(import, MediaFailure::Permanent).await,+ };+ if let Err(report) = result {+ if matches!(report.current_context(), LinkedError::Fenced) {+ self.fenced.cancel();+ } else {+ tracing::warn!(error = %report, "Recording a WhatsApp attachment failed");+ }+ }+ }+ .instrument(span)+ .await+ }++ async fn fail(+ &self,+ import: &MediaImport,+ failure: MediaFailure,+ ) -> whatsapp_linked::domain::ports::LinkedResult<()> {+ tracing::Span::current().record(+ "outcome",+ match failure {+ MediaFailure::Retryable => "retry",+ MediaFailure::Permanent => "failed",+ },+ );+ self.sessions+ .fail_media_import(self.scope, &import.attachment, failure)+ .await+ }+}diff --git a/services/whatsapp_session_service/src/domain/ports.rs b/services/whatsapp_session_service/src/domain/ports.rsnew file mode 100644index 0000000000..7c0c15b11d--- /dev/null+++ b/services/whatsapp_session_service/src/domain/ports.rs@@ -0,0 +1,296 @@+//! What the supervisor and the account runner need from the outside: account+//! locks, the account directory, a runner per leased account, and the+//! account's WhatsApp connection.++use std::future::Future;+use std::sync::Arc;++use chrono::{DateTime, Utc};+use tokio::sync::mpsc;+use tokio_util::sync::CancellationToken;+use uuid::Uuid;+use whatsapp_linked::domain::events::{+ ClaimedSend, InboundEvent, MediaImport, OutboundMedia, SessionScope,+};+use whatsapp_linked::domain::models::GroupInfo;++/// Errors from taking, holding or announcing account locks.+#[derive(Debug, thiserror::Error)]+pub enum LeaseError {+ /// The lock connection is gone, so every lock it held may already be free.+ #[error("the account lock connection was lost")]+ Lost,+ /// Any other storage failure.+ #[error("account lease storage failed: {0}")]+ Internal(rootcause::Report),+}++/// Session-level advisory locks, one per account, held by this replica.+pub trait AccountLocks: Send + Sync + 'static {+ /// Takes the account's lock without waiting. `false` when another replica+ /// holds it.+ fn try_lock(&self, account: Uuid) -> impl Future<Output = Result<bool, LeaseError>> + Send;++ /// Releases a lock this replica holds.+ fn unlock(&self, account: Uuid) -> impl Future<Output = Result<(), LeaseError>> + Send;++ /// Cancelled as soon as the locks may be gone. Accounts must stop at once.+ fn lost(&self) -> CancellationToken;+}++/// Finds accounts that need a replica and tells other replicas about releases.+pub trait AccountDirectory: Send + Sync + 'static {+ /// Accounts that need a session (connected, or with a live pairing+ /// transaction) and that no replica holds, oldest first.+ fn candidates(&self, limit: u32) -> impl Future<Output = Result<Vec<Uuid>, LeaseError>> + Send;++ /// The account a pairing transaction belongs to.+ fn pairing_account(+ &self,+ pairing: Uuid,+ ) -> impl Future<Output = Result<Option<Uuid>, LeaseError>> + Send;++ /// `NOTIFY whatsapp_account_released`: the account still needs a session+ /// and is free to take.+ fn announce_released(+ &self,+ account: Uuid,+ ) -> impl Future<Output = Result<(), LeaseError>> + Send;++ /// Fleet-wide facts for the periodic health report.+ fn health(&self) -> impl Future<Output = Result<FleetHealth, LeaseError>> + Send;+}++/// Fleet-wide facts every replica reports once a minute.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub struct FleetHealth {+ /// Accounts that need a session but whose lock no replica holds.+ pub accounts_unowned: u64,+ /// Age of the oldest inbound event still waiting to be applied.+ pub oldest_unprocessed_inbound_seconds: u64,+}++/// A signal for one account this replica runs.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum AccountSignal {+ /// `whatsapp_pairing_requested`: a pairing transaction started or changed.+ Pairing(Uuid),+ /// `whatsapp_send_requested`: claim pending sends.+ Send,+ /// `whatsapp_groups_requested`: refresh the group list.+ Groups,+ /// `whatsapp_account_changed`: consent or adopted groups changed.+ Changed,+ /// Notifications may have been missed; re-read everything.+ Resync,+}++/// What a runner receives for its account.+pub struct AccountControl {+ /// Signals for this account, coalesced when the runner is busy.+ pub signals: mpsc::Receiver<AccountSignal>,+ /// Graceful release (drain): stop taking sends, wait for in-flight+ /// acknowledgements, disconnect, then return [`AccountExit::Released`].+ pub release: CancellationToken,+ /// Hard stop: the lock is gone. Return without another write.+ pub stop: CancellationToken,+}++/// Why a runner returned.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum AccountExit {+ /// Nothing to run: no session and no live pairing. Not announced.+ Idle,+ /// Released on request while the account still needs a session.+ Released,+ /// Another replica took the account over.+ Fenced,+ /// Stopped by `stop`.+ Stopped,+ /// Gave up after an error. Not announced, so a failing account doesn't+ /// bounce between replicas; the next rescan retries it.+ Failed,+}++/// Runs one account for as long as this replica holds its lock.+pub trait AccountRunner: Send + Sync + 'static {+ /// Runs `account` until it is idle, released, fenced or stopped.+ fn run(+ &self,+ account: Uuid,+ control: AccountControl,+ ) -> impl Future<Output = AccountExit> + Send;+}++/// What happened on an account's WhatsApp connection, already mapped onto+/// the linked-devices domain types. Group traffic does not come this way: it+/// goes straight to the [`InboundSink`] so it is stored before it is acked.+#[derive(Debug, Clone)]+pub enum SessionEvent {+ /// A new QR code to show.+ Qr(String),+ /// Pairing finished as this phone-number identity.+ PairSuccess {+ /// Verified phone-number JID of the linked account.+ identity_jid: String,+ },+ /// Pairing failed or timed out.+ PairFailed,+ /// WhatsApp asked for a passkey assertion; `challenge` is the WebAuthn+ /// request options to show the owner.+ PasskeyRequested {+ /// WebAuthn request options.+ challenge: serde_json::Value,+ },+ /// The phone shows a code the owner must compare before the passkey+ /// pairing is confirmed.+ PasskeyConfirmation {+ /// Code to show next to the one on the phone.+ code: String,+ },+ /// Connected and authenticated.+ Connected,+ /// The websocket dropped; the client reconnects by itself.+ Disconnected,+ /// WhatsApp logged the device out or rejected its credentials.+ LoggedOut,+ /// Another connection with the same keys replaced this one.+ Replaced,+}++/// Where the client hands group events before acknowledging them.+///+/// Object safe (`Arc<dyn InboundSink>` inside the client's event handler),+/// hence `async_trait`.+#[async_trait::async_trait]+pub trait InboundSink: Send + Sync {+ /// Stores the event (or drops it when its group is not adopted). `false`+ /// leaves the protocol node unacknowledged so WhatsApp redelivers it.+ /// Must never wait on the account's WhatsApp connection.+ async fn accept(&self, event: &InboundEvent) -> bool;+}++/// Errors opening or driving a WhatsApp session.+#[derive(Debug, thiserror::Error)]+pub enum SessionError {+ /// Another replica took the account over.+ #[error("the account lease moved to another replica")]+ Fenced,+ /// KMS refused to unwrap or generate the account's data key.+ #[error("KMS refused the account's data key")]+ Kms,+ /// Any other failure.+ #[error("WhatsApp session failed: {0}")]+ Internal(rootcause::Report),+}++/// Why a send did not get an acknowledgement.+#[derive(Debug, Clone, PartialEq, Eq)]+pub enum SendFailure {+ /// WhatsApp refused it; nothing reached the group.+ Rejected(&'static str),+ /// It may have reached WhatsApp.+ Unknown,+}++/// Why an attachment could not be downloaded.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum DownloadFailure {+ /// Try again later (network, not connected, rate limited).+ Retryable,+ /// Never downloadable (expired, tampered, too large, unreadable locator).+ Permanent,+}++/// An attachment uploaded to WhatsApp's media hosts, ready to reference+/// from one message.+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct UploadedMedia {+ /// Media host URL.+ pub url: String,+ /// Path on the media host.+ pub direct_path: String,+ /// Key the file was encrypted with.+ pub media_key: Vec<u8>,+ /// SHA-256 of the encrypted file.+ pub file_enc_sha256: Vec<u8>,+ /// SHA-256 of the plaintext.+ pub file_sha256: Vec<u8>,+ /// Plaintext length.+ pub file_length: u64,+}++/// One account's WhatsApp connection.+pub trait WhatsAppSession: Send + Sync + 'static {+ /// The device's phone-number JID once paired.+ fn identity_jid(&self) -> impl Future<Output = Option<String>> + Send;++ /// Starts emitting [`SessionEvent::Qr`] codes; call before `connect` on an+ /// unpaired device.+ fn start_pairing(&self) -> impl Future<Output = Result<(), SessionError>> + Send;++ /// Forwards the owner's WebAuthn assertion for a passkey pairing.+ fn send_passkey_response(+ &self,+ response: serde_json::Value,+ ) -> impl Future<Output = Result<(), SessionError>> + Send;++ /// Confirms a passkey pairing once its code was shown.+ fn confirm_passkey(&self) -> impl Future<Output = Result<(), SessionError>> + Send;++ /// Connects in the background, reconnecting by itself until `disconnect`.+ fn connect(&self) -> impl Future<Output = Result<(), SessionError>> + Send;++ /// A fresh WhatsApp message id for the next send.+ fn new_message_id(&self) -> impl Future<Output = String> + Send;++ /// Encrypts and uploads an attachment for a later send.+ fn upload_media(+ &self,+ media: &OutboundMedia,+ ) -> impl Future<Output = Result<UploadedMedia, SendFailure>> + Send;++ /// Sends one claimed intent with `message_id` and waits for the server's+ /// acknowledgement, returning its timestamp. A media send carries the+ /// attachment it uploaded first.+ fn send(+ &self,+ send: &ClaimedSend,+ message_id: &str,+ media: Option<&UploadedMedia>,+ ) -> impl Future<Output = Result<DateTime<Utc>, SendFailure>> + Send;++ /// Downloads and decrypts an observed attachment from its sealed locator.+ fn download_media(+ &self,+ import: &MediaImport,+ ) -> impl Future<Output = Result<Vec<u8>, DownloadFailure>> + Send;++ /// Full metadata for every group the account is in.+ fn joined_groups(&self) -> impl Future<Output = Result<Vec<GroupInfo>, SessionError>> + Send;++ /// Unlinks the device at WhatsApp.+ fn logout(&self) -> impl Future<Output = ()> + Send;++ /// Closes the connection without unlinking.+ fn disconnect(&self) -> impl Future<Output = ()> + Send;+}++/// Opens an account's device store and WhatsApp client.+pub trait SessionConnector: Send + Sync + 'static {+ /// The connection type.+ type Session: WhatsAppSession;++ /// Opens the session for the holder of `scope`'s epoch. With+ /// `fresh_device`, a stored device is deleted first and new keys are+ /// generated. Group events go to `inbound`; the rest arrive on the+ /// returned receiver, which never makes the client wait.+ fn open(+ &self,+ scope: SessionScope,+ fresh_device: bool,+ inbound: Arc<dyn InboundSink>,+ ) -> impl Future<+ Output = Result<(Self::Session, mpsc::UnboundedReceiver<SessionEvent>), SessionError>,+ > + Send;+}diff --git a/services/whatsapp_session_service/src/domain/supervisor.rs b/services/whatsapp_session_service/src/domain/supervisor.rsnew file mode 100644index 0000000000..fd3f1a126c--- /dev/null+++ b/services/whatsapp_session_service/src/domain/supervisor.rs@@ -0,0 +1,449 @@+//! Account ownership on one replica: which accounts it runs, taking free ones,+//! handing them over on drain, and stopping everything when the locks are lost.++use std::collections::{HashMap, HashSet};+use std::sync::Arc;+use std::time::Duration;++use tokio::sync::mpsc;+use tokio::task::{Id, JoinError, JoinSet};+use tokio_util::sync::CancellationToken;+use tracing::Instrument;+use uuid::Uuid;++use super::ports::{+ AccountControl, AccountDirectory, AccountExit, AccountLocks, AccountRunner, AccountSignal,+ LeaseError,+};++#[cfg(test)]+mod test;++/// `NOTIFY` channel, payload `<account_id>`: the account still needs a session+/// and is free to take.+pub const ACCOUNT_RELEASED: &str = "whatsapp_account_released";++/// Signals for this replica, one per `NOTIFY` (see `outbound::signals`).+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum Signal {+ /// `whatsapp_account_released`: another replica let the account go.+ AccountReleased(Uuid),+ /// `whatsapp_pairing_requested`, carrying the pairing id.+ PairingRequested(Uuid),+ /// `whatsapp_send_requested`.+ SendRequested(Uuid),+ /// `whatsapp_groups_requested`.+ GroupsRequested(Uuid),+ /// `whatsapp_account_changed`.+ AccountChanged(Uuid),+ /// The listener reconnected, so notifications may have been missed.+ Resync,+}++/// Limits and timings.+#[derive(Debug, Clone, Copy)]+pub struct SupervisorConfig {+ /// Most accounts this replica runs at once.+ pub max_accounts: u32,+ /// Accounts released per drain step.+ pub drain_batch: u32,+ /// How often to look for free accounts without a signal.+ pub rescan_every: Duration,+ /// How long a released account may take to finish in-flight sends.+ pub release_timeout: Duration,+ /// How long a stopped account may take to return.+ pub stop_timeout: Duration,+ /// Longest the whole drain may take; what is left is stopped at once.+ pub drain_deadline: Duration,+ /// How often to report fleet health.+ pub health_every: Duration,+}++/// Why the supervisor returned early.+#[derive(Debug, thiserror::Error)]+pub enum SupervisorError {+ /// The lock connection died; every account was stopped.+ #[error("account locks were lost")]+ LocksLost,+ /// The signal listener stopped.+ #[error("the signal listener stopped")]+ SignalsClosed,+}++/// Unread signals per account before new ones are dropped. Every signal makes+/// the runner re-read its state, so dropped duplicates lose nothing.+const ACCOUNT_SIGNAL_CAPACITY: usize = 8;++struct Running {+ signals: mpsc::Sender<AccountSignal>,+ release: CancellationToken,+ stop: CancellationToken,+}++/// Runs the accounts this replica holds locks for.+pub struct Supervisor<Locks, Directory, Runner> {+ locks: Locks,+ directory: Directory,+ runner: Arc<Runner>,+ config: SupervisorConfig,+ accounts: HashMap<Uuid, Running>,+ tasks: JoinSet<AccountExit>,+ task_accounts: HashMap<Id, Uuid>,+}++impl<Locks, Directory, Runner> Supervisor<Locks, Directory, Runner>+where+ Locks: AccountLocks,+ Directory: AccountDirectory,+ Runner: AccountRunner,+{+ /// A supervisor over one lock connection.+ pub fn new(+ locks: Locks,+ directory: Directory,+ runner: Arc<Runner>,+ config: SupervisorConfig,+ ) -> Self {+ Supervisor {+ locks,+ directory,+ runner,+ config,+ accounts: HashMap::new(),+ tasks: JoinSet::new(),+ task_accounts: HashMap::new(),+ }+ }++ /// Runs until `shutdown` (then drains and returns `Ok`) or until the locks+ /// are lost (then stops every account and returns the error).+ pub async fn run(+ mut self,+ signals: &mut mpsc::Receiver<Signal>,+ shutdown: CancellationToken,+ ) -> Result<(), SupervisorError> {+ let lost = self.locks.lost();+ let mut rescan = tokio::time::interval(self.config.rescan_every);+ rescan.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);+ let mut health = tokio::time::interval(self.config.health_every);+ health.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);+ loop {+ let step = tokio::select! {+ biased;+ () = lost.cancelled() => Err(LeaseError::Lost),+ () = shutdown.cancelled() => {+ return self.drain().await;+ }+ Some(joined) = self.tasks.join_next_with_id() => {+ self.finished(joined, false).await+ }+ signal = signals.recv() => match signal {+ Some(signal) => self.handle(signal).await,+ None => {+ self.stop_all().await;+ return Err(SupervisorError::SignalsClosed);+ }+ },+ _ = rescan.tick() => self.rescan().await,+ _ = health.tick() => self.report_health().await,+ };+ match step {+ Ok(()) => {}+ Err(LeaseError::Lost) => {+ self.stop_all().await;+ return Err(SupervisorError::LocksLost);+ }+ Err(LeaseError::Internal(report)) => {+ tracing::warn!(error = %report, "WhatsApp account supervision step failed");+ }+ }+ }+ }++ async fn handle(&mut self, signal: Signal) -> Result<(), LeaseError> {+ match signal {+ Signal::AccountReleased(account) => self.take(account).await,+ Signal::PairingRequested(pairing) => {+ match self.directory.pairing_account(pairing).await? {+ Some(account) if self.accounts.contains_key(&account) => {+ self.forward(account, AccountSignal::Pairing(pairing));+ Ok(())+ }+ Some(account) => self.take(account).await,+ None => Ok(()),+ }+ }+ Signal::SendRequested(account) => {+ self.forward(account, AccountSignal::Send);+ Ok(())+ }+ Signal::GroupsRequested(account) => {+ self.forward(account, AccountSignal::Groups);+ Ok(())+ }+ Signal::AccountChanged(account) if self.accounts.contains_key(&account) => {+ self.forward(account, AccountSignal::Changed);+ Ok(())+ }+ Signal::AccountChanged(account) => self.take(account).await,+ Signal::Resync => {+ for running in self.accounts.values() {+ let _ = running.signals.try_send(AccountSignal::Resync);+ }+ self.rescan().await+ }+ }+ }++ fn forward(&self, account: Uuid, signal: AccountSignal) {+ if let Some(running) = self.accounts.get(&account) {+ // Full means the runner already has work queued; see the capacity.+ let _ = running.signals.try_send(signal);+ }+ }++ fn at_capacity(&self) -> bool {+ self.accounts.len() >= self.config.max_accounts as usize+ }++ /// Takes the account's lock if it is free and this replica has room.+ async fn take(&mut self, account: Uuid) -> Result<(), LeaseError> {+ if self.accounts.contains_key(&account) || self.at_capacity() {+ return Ok(());+ }+ let span = tracing::info_span!(+ "whatsapp.account.lease",+ account_id = %account,+ acquired = tracing::field::Empty,+ held = self.accounts.len(),+ );+ let acquired = self+ .locks+ .try_lock(account)+ .instrument(span.clone())+ .await?;+ span.record("acquired", acquired);+ if acquired {+ self.start(account);+ }+ Ok(())+ }++ fn start(&mut self, account: Uuid) {+ let (sender, receiver) = mpsc::channel(ACCOUNT_SIGNAL_CAPACITY);+ let release = CancellationToken::new();+ let stop = self.locks.lost().child_token();+ let control = AccountControl {+ signals: receiver,+ release: release.clone(),+ stop: stop.clone(),+ };+ let runner = Arc::clone(&self.runner);+ let task = self+ .tasks+ .spawn(async move { runner.run(account, control).await });+ self.task_accounts.insert(task.id(), account);+ self.accounts.insert(+ account,+ Running {+ signals: sender,+ release,+ stop,+ },+ );+ }++ /// Unlocks an account whose runner returned, and announces it when it+ /// still needs a session.+ async fn finished(+ &mut self,+ joined: Result<(Id, AccountExit), JoinError>,+ draining: bool,+ ) -> Result<(), LeaseError> {+ let (id, exit) = match joined {+ Ok((id, exit)) => (id, exit),+ Err(error) => {+ tracing::error!(error = %error, "WhatsApp account task panicked");+ (error.id(), AccountExit::Failed)+ }+ };+ let Some(account) = self.task_accounts.remove(&id) else {+ return Ok(());+ };+ self.accounts.remove(&account);+ let announce = exit == AccountExit::Released || (draining && exit == AccountExit::Stopped);+ let span = tracing::info_span!(+ "whatsapp.account.release",+ account_id = %account,+ exit = ?exit,+ announced = announce,+ );+ async {+ self.locks.unlock(account).await?;+ if announce {+ self.directory.announce_released(account).await?;+ }+ Ok(())+ }+ .instrument(span)+ .await+ }++ async fn report_health(&self) -> Result<(), LeaseError> {+ let health = self.directory.health().await?;+ tracing::info!(+ telemetry_event = "whatsapp_session_health",+ accounts_running = self.accounts.len(),+ accounts_unowned = health.accounts_unowned,+ oldest_unprocessed_inbound_seconds = health.oldest_unprocessed_inbound_seconds,+ "WhatsApp session health"+ );+ Ok(())+ }++ async fn rescan(&mut self) -> Result<(), LeaseError> {+ if self.at_capacity() {+ return Ok(());+ }+ let room = self.config.max_accounts - self.accounts.len() as u32;+ let span = tracing::info_span!(+ "whatsapp.account.rescan",+ room,+ candidates = tracing::field::Empty,+ );+ let candidates = self+ .directory+ .candidates(room)+ .instrument(span.clone())+ .await?;+ span.record("candidates", candidates.len());+ for account in candidates {+ self.take(account).await?;+ }+ Ok(())+ }++ /// Hands every account over in batches, so other replicas take them while+ /// the rest keep running. Accounts still running at `drain_deadline` are+ /// stopped together, so the whole drain fits the platform's stop timeout.+ async fn drain(mut self) -> Result<(), SupervisorError> {+ let started = tokio::time::Instant::now();+ let drain_end = started + self.config.drain_deadline;+ let span = tracing::info_span!(+ "whatsapp.drain",+ accounts = self.accounts.len(),+ batch = self.config.drain_batch,+ released = tracing::field::Empty,+ timed_out = tracing::field::Empty,+ );+ let lost = self.locks.lost();+ let mut remaining: Vec<Uuid> = self.accounts.keys().copied().collect();+ let batch_size = self.config.drain_batch.max(1) as usize;+ let mut released = 0u32;+ let mut timed_out = 0u32;+ while !remaining.is_empty() {+ let past_deadline = tokio::time::Instant::now() >= drain_end;+ let take = if past_deadline {+ remaining.len()+ } else {+ batch_size.min(remaining.len())+ };+ let batch: Vec<Uuid> = remaining.drain(..take).collect();+ let mut pending: HashSet<Uuid> = batch+ .iter()+ .copied()+ .filter(|account| self.accounts.contains_key(account))+ .collect();+ for account in &pending {+ if let Some(running) = self.accounts.get(account) {+ if past_deadline {+ running.stop.cancel();+ } else {+ running.release.cancel();+ }+ }+ }+ let deadline = tokio::time::sleep_until(+ (tokio::time::Instant::now() + self.config.release_timeout).min(drain_end),+ );+ tokio::pin!(deadline);+ let mut stopped = past_deadline;+ while !pending.is_empty() {+ let step = tokio::select! {+ biased;+ () = lost.cancelled() => Err(LeaseError::Lost),+ joined = self.tasks.join_next_with_id() => {+ let Some(joined) = joined else { break };+ let account = match &joined {+ Ok((id, _)) => self.task_accounts.get(id).copied(),+ Err(error) => self.task_accounts.get(&error.id()).copied(),+ };+ if let Some(account) = account {+ pending.remove(&account);+ }+ let result = self.finished(joined, true).await;+ if result.is_ok() {+ released += 1;+ }+ result+ }+ () = &mut deadline, if !stopped => {+ stopped = true;+ timed_out += pending.len() as u32;+ for account in &pending {+ if let Some(running) = self.accounts.get(account) {+ running.stop.cancel();+ }+ }+ Ok(())+ }+ };+ match step {+ Ok(()) => {}+ Err(LeaseError::Lost) => {+ span.record("released", released);+ span.record("timed_out", timed_out);+ self.stop_all().await;+ return Err(SupervisorError::LocksLost);+ }+ Err(LeaseError::Internal(report)) => {+ tracing::warn!(parent: &span, error = %report, "Releasing a WhatsApp account failed");+ }+ }+ }+ }+ span.record("released", released);+ span.record("timed_out", timed_out);+ tracing::info!(+ parent: &span,+ telemetry_event = "whatsapp_session_drain",+ drain_duration_ms = started.elapsed().as_millis() as u64,+ accounts_released = released,+ accounts_timed_out = timed_out,+ "WhatsApp session drain finished"+ );+ Ok(())+ }++ /// The locks are gone: stop every account without another write.+ async fn stop_all(&mut self) {+ for running in self.accounts.values() {+ running.stop.cancel();+ }+ let count = self.accounts.len();+ if tokio::time::timeout(self.config.stop_timeout, async {+ while self.tasks.join_next().await.is_some() {}+ })+ .await+ .is_err()+ {+ tracing::error!(+ count,+ "WhatsApp accounts did not stop in time; aborting them"+ );+ self.tasks.shutdown().await;+ }+ self.accounts.clear();+ self.task_accounts.clear();+ }+}diff --git a/services/whatsapp_session_service/src/domain/supervisor/test.rs b/services/whatsapp_session_service/src/domain/supervisor/test.rsnew file mode 100644index 0000000000..ac0f4fc0ba--- /dev/null+++ b/services/whatsapp_session_service/src/domain/supervisor/test.rs@@ -0,0 +1,312 @@+use std::collections::HashSet;+use std::sync::{Arc, Mutex};+use std::time::Duration;++use tokio::sync::mpsc;+use tokio_util::sync::CancellationToken;+use uuid::Uuid;++use super::{Signal, Supervisor, SupervisorConfig, SupervisorError};+use crate::domain::ports::{+ AccountControl, AccountDirectory, AccountExit, AccountLocks, AccountRunner, AccountSignal,+ FleetHealth, LeaseError,+};++/// Locks shared by every fake replica, like advisory locks on one server.+#[derive(Clone, Default)]+struct FakeLocks {+ held: Arc<Mutex<HashSet<Uuid>>>,+ lost: CancellationToken,+}++impl AccountLocks for FakeLocks {+ async fn try_lock(&self, account: Uuid) -> Result<bool, LeaseError> {+ Ok(self.held.lock().unwrap().insert(account))+ }++ async fn unlock(&self, account: Uuid) -> Result<(), LeaseError> {+ self.held.lock().unwrap().remove(&account);+ Ok(())+ }++ fn lost(&self) -> CancellationToken {+ self.lost.clone()+ }+}++struct FakeDirectory {+ candidates: Vec<Uuid>,+ announced: mpsc::UnboundedSender<Uuid>,+}++impl AccountDirectory for FakeDirectory {+ async fn candidates(&self, limit: u32) -> Result<Vec<Uuid>, LeaseError> {+ Ok(self+ .candidates+ .iter()+ .copied()+ .take(limit as usize)+ .collect())+ }++ async fn pairing_account(&self, _pairing: Uuid) -> Result<Option<Uuid>, LeaseError> {+ Ok(None)+ }++ async fn announce_released(&self, account: Uuid) -> Result<(), LeaseError> {+ self.announced.send(account).unwrap();+ Ok(())+ }++ async fn health(&self) -> Result<FleetHealth, LeaseError> {+ Ok(FleetHealth {+ accounts_unowned: 0,+ oldest_unprocessed_inbound_seconds: 0,+ })+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+enum RunnerEvent {+ Started(Uuid),+ Signal(Uuid, AccountSignal),+ Released(Uuid),+ Stopped(Uuid),+}++/// Runs until released or stopped. With `ignore_release`, only a stop ends+/// it, like an account whose in-flight send never gets an acknowledgement.+struct FakeRunner {+ events: mpsc::UnboundedSender<RunnerEvent>,+ ignore_release: bool,+}++impl AccountRunner for FakeRunner {+ async fn run(&self, account: Uuid, mut control: AccountControl) -> AccountExit {+ self.events.send(RunnerEvent::Started(account)).unwrap();+ loop {+ tokio::select! {+ () = control.stop.cancelled() => {+ self.events.send(RunnerEvent::Stopped(account)).unwrap();+ return AccountExit::Stopped;+ }+ () = control.release.cancelled(), if !self.ignore_release => {+ self.events.send(RunnerEvent::Released(account)).unwrap();+ return AccountExit::Released;+ }+ Some(signal) = control.signals.recv() => {+ self.events.send(RunnerEvent::Signal(account, signal)).unwrap();+ }+ }+ }+ }+}++const CONFIG: SupervisorConfig = SupervisorConfig {+ max_accounts: 2,+ drain_batch: 1,+ rescan_every: Duration::from_secs(30),+ release_timeout: Duration::from_secs(10),+ stop_timeout: Duration::from_secs(5),+ drain_deadline: Duration::from_secs(100),+ health_every: Duration::from_secs(60),+};++#[tokio::test(start_paused = true)]+async fn takes_free_accounts_up_to_its_cap_and_forwards_their_signals() {+ let first = Uuid::from_u128(1);+ let second = Uuid::from_u128(2);+ let third = Uuid::from_u128(3);+ let locks = FakeLocks::default();+ let (announced, _announcements) = mpsc::unbounded_channel();+ let (events, mut runner_events) = mpsc::unbounded_channel();+ let supervisor = Supervisor::new(+ locks.clone(),+ FakeDirectory {+ candidates: vec![first, second, third],+ announced,+ },+ Arc::new(FakeRunner {+ events,+ ignore_release: false,+ }),+ CONFIG,+ );+ let (signals, mut receiver) = mpsc::channel(8);+ let shutdown = CancellationToken::new();+ let stop = shutdown.clone();+ let running = tokio::spawn(async move { supervisor.run(&mut receiver, stop).await });++ assert_eq!(+ runner_events.recv().await,+ Some(RunnerEvent::Started(first))+ );+ assert_eq!(+ runner_events.recv().await,+ Some(RunnerEvent::Started(second))+ );+ signals.send(Signal::SendRequested(second)).await.unwrap();+ signals.send(Signal::SendRequested(third)).await.unwrap();+ signals.send(Signal::GroupsRequested(first)).await.unwrap();+ assert_eq!(+ runner_events.recv().await,+ Some(RunnerEvent::Signal(second, AccountSignal::Send))+ );+ assert_eq!(+ runner_events.recv().await,+ Some(RunnerEvent::Signal(first, AccountSignal::Groups))+ );+ assert_eq!(+ *locks.held.lock().unwrap(),+ HashSet::from([first, second]),+ "the third account stays free for another replica"+ );++ shutdown.cancel();+ running.await.unwrap().unwrap();+}++#[tokio::test(start_paused = true)]+async fn drain_releases_every_account_then_unlocks_and_announces_it() {+ let first = Uuid::from_u128(1);+ let second = Uuid::from_u128(2);+ let locks = FakeLocks::default();+ let (announced, mut announcements) = mpsc::unbounded_channel();+ let (events, mut runner_events) = mpsc::unbounded_channel();+ let supervisor = Supervisor::new(+ locks.clone(),+ FakeDirectory {+ candidates: vec![first, second],+ announced,+ },+ Arc::new(FakeRunner {+ events,+ ignore_release: false,+ }),+ CONFIG,+ );+ let (_signals, mut receiver) = mpsc::channel(8);+ let shutdown = CancellationToken::new();+ let stop = shutdown.clone();+ let running = tokio::spawn(async move { supervisor.run(&mut receiver, stop).await });+ assert_eq!(+ runner_events.recv().await,+ Some(RunnerEvent::Started(first))+ );+ assert_eq!(+ runner_events.recv().await,+ Some(RunnerEvent::Started(second))+ );++ shutdown.cancel();+ running.await.unwrap().unwrap();++ let mut released = vec![runner_events.recv().await, runner_events.recv().await];+ released.sort_by_key(|event| format!("{event:?}"));+ assert_eq!(+ released,+ vec![+ Some(RunnerEvent::Released(first)),+ Some(RunnerEvent::Released(second))+ ]+ );+ let mut announced = vec![announcements.recv().await, announcements.recv().await];+ announced.sort();+ assert_eq!(announced, vec![Some(first), Some(second)]);+ assert!(locks.held.lock().unwrap().is_empty());+}++#[tokio::test(start_paused = true)]+async fn drain_stops_an_account_that_does_not_release_in_time_and_still_announces_it() {+ let account = Uuid::from_u128(1);+ let locks = FakeLocks::default();+ let (announced, mut announcements) = mpsc::unbounded_channel();+ let (events, mut runner_events) = mpsc::unbounded_channel();+ let supervisor = Supervisor::new(+ locks.clone(),+ FakeDirectory {+ candidates: vec![account],+ announced,+ },+ Arc::new(FakeRunner {+ events,+ ignore_release: true,+ }),+ CONFIG,+ );+ let (_signals, mut receiver) = mpsc::channel(8);+ let shutdown = CancellationToken::new();+ let stop = shutdown.clone();+ let running = tokio::spawn(async move { supervisor.run(&mut receiver, stop).await });+ assert_eq!(+ runner_events.recv().await,+ Some(RunnerEvent::Started(account))+ );++ shutdown.cancel();+ running.await.unwrap().unwrap();++ assert_eq!(+ runner_events.recv().await,+ Some(RunnerEvent::Stopped(account))+ );+ assert_eq!(announcements.recv().await, Some(account));+ assert!(locks.held.lock().unwrap().is_empty());+}++#[tokio::test(start_paused = true)]+async fn losing_the_locks_stops_every_account_without_unlocking_or_announcing() {+ let first = Uuid::from_u128(1);+ let second = Uuid::from_u128(2);+ let locks = FakeLocks::default();+ let (announced, mut announcements) = mpsc::unbounded_channel();+ let (events, mut runner_events) = mpsc::unbounded_channel();+ let supervisor = Supervisor::new(+ locks.clone(),+ FakeDirectory {+ candidates: vec![first, second],+ announced,+ },+ Arc::new(FakeRunner {+ events,+ ignore_release: false,+ }),+ CONFIG,+ );+ let (_signals, mut receiver) = mpsc::channel(8);+ let running = tokio::spawn(async move {+ supervisor+ .run(&mut receiver, CancellationToken::new())+ .await+ });+ assert_eq!(+ runner_events.recv().await,+ Some(RunnerEvent::Started(first))+ );+ assert_eq!(+ runner_events.recv().await,+ Some(RunnerEvent::Started(second))+ );++ locks.lost.cancel();++ assert!(matches!(+ running.await.unwrap(),+ Err(SupervisorError::LocksLost)+ ));+ let mut stopped = vec![runner_events.recv().await, runner_events.recv().await];+ stopped.sort_by_key(|event| format!("{event:?}"));+ assert_eq!(+ stopped,+ vec![+ Some(RunnerEvent::Stopped(first)),+ Some(RunnerEvent::Stopped(second))+ ]+ );+ assert!(announcements.try_recv().is_err());+ assert_eq!(+ *locks.held.lock().unwrap(),+ HashSet::from([first, second]),+ "a lost connection's locks are freed by Postgres, not unlocked"+ );+}diff --git a/services/whatsapp_session_service/src/doppler_config.rs b/services/whatsapp_session_service/src/doppler_config.rsnew file mode 100644index 0000000000..5cc4c04a52--- /dev/null+++ b/services/whatsapp_session_service/src/doppler_config.rs@@ -0,0 +1,22 @@+//! Checks that Doppler `dev` and `prd` hold every key [`Config`] reads.++use macro_env::Environment;+use whatsapp_session_service::config::Config;++const DOPPLER_PROJECT: &str = "whatsapp-session-service";++#[tokio::main]+async fn main() -> Result<(), rootcause::Report> {+ for environment in [Environment::Develop, Environment::Production] {+ doppler_config::DopplerConfig::builder()+ .token_from_env("DOPPLER_TOKEN")+ .config(environment.to_doppler_slug())+ .project(DOPPLER_PROJECT)+ .build()+ .map_err(|error| rootcause::report!("invalid Doppler settings: {error:?}"))?+ .load::<Config>()+ .await+ .map_err(|error| rootcause::report!("Doppler config is incomplete: {error:?}"))?;+ }+ Ok(())+}diff --git a/services/whatsapp_session_service/src/inbound.rs b/services/whatsapp_session_service/src/inbound.rsnew file mode 100644index 0000000000..9d257f7a80--- /dev/null+++ b/services/whatsapp_session_service/src/inbound.rs@@ -0,0 +1,3 @@+//! The health endpoint.++pub mod health;diff --git a/services/whatsapp_session_service/src/inbound/health.rs b/services/whatsapp_session_service/src/inbound/health.rsnew file mode 100644index 0000000000..45f5bc8983--- /dev/null+++ b/services/whatsapp_session_service/src/inbound/health.rs@@ -0,0 +1,28 @@+//! `GET /health`: always 200 while the process runs, so a draining replica is+//! not killed early; `ready` turns false once the drain starts.++use axum::extract::State;+use axum::routing::get;+use axum::{Json, Router};+use serde::Serialize;+use tokio_util::sync::CancellationToken;++/// Health response body.+#[derive(Debug, Serialize, PartialEq, Eq)]+pub struct Health {+ /// False while draining.+ pub ready: bool,+}++async fn health(State(draining): State<CancellationToken>) -> Json<Health> {+ Json(Health {+ ready: !draining.is_cancelled(),+ })+}++/// The health router; `draining` is the shutdown token.+pub fn router(draining: CancellationToken) -> Router {+ Router::new()+ .route("/health", get(health))+ .with_state(draining)+}diff --git a/services/whatsapp_session_service/src/lib.rs b/services/whatsapp_session_service/src/lib.rsnew file mode 100644index 0000000000..69127a5631--- /dev/null+++ b/services/whatsapp_session_service/src/lib.rs@@ -0,0 +1,10 @@+// The account runner future nests the domain, store and client futures.+#![recursion_limit = "256"]+//! WhatsApp session service: runs the linked WhatsApp accounts this replica+//! holds advisory locks for. See `AGENTS.md`.++pub mod config;+pub mod domain;+pub mod inbound;+pub mod outbound;+pub mod replica;diff --git a/services/whatsapp_session_service/src/main.rs b/services/whatsapp_session_service/src/main.rsnew file mode 100644index 0000000000..25c4351e4e--- /dev/null+++ b/services/whatsapp_session_service/src/main.rs@@ -0,0 +1,97 @@+// The account runner future nests the domain, store and client futures.+#![recursion_limit = "256"]+//! Composition root of the WhatsApp session service.++use std::sync::Arc;+use std::time::Duration;++use macro_entrypoint::{MacroEntrypoint, shutdown_signal};+use sqlx::postgres::{PgConnectOptions, PgPoolOptions};+use tokio_util::sync::CancellationToken;+use whatsapp_session_service::config::Config;+use whatsapp_session_service::inbound::health;+use whatsapp_session_service::replica::{self, ReplicaDeps, Timings};+use whatsapp_store_pg::kms::AwsWhatsAppKms;+use whatsmeow::{Dialer, WebSocketDialer};++/// Test-only: point every client at a fake WhatsApp server.+const WEBSOCKET_URL_OVERRIDE: &str = "OVERRIDE_WHATSAPP_WEBSOCKET_URL";+/// Test-only: the fake server's media host, as `scheme://host[:port]`.+const MEDIA_HOST_OVERRIDE: &str = "OVERRIDE_WHATSAPP_MEDIA_HOST";+const PORT: u16 = 8080;++#[tokio::main]+async fn main() -> Result<(), rootcause::Report> {+ let entrypoint = MacroEntrypoint::default().init();+ let result = run().await;+ entrypoint.shutdown();+ result+}++async fn run() -> Result<(), rootcause::Report> {+ // whatsmeow dials wss:// with rustls directly.+ let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();+ let config = macro_config::ConfigLoader::load::<Config>()?;+ let pool = PgPoolOptions::new()+ .max_connections(20)+ .acquire_timeout(Duration::from_secs(10))+ .connect(config.database_url.as_ref())+ .await?;+ let lock_options: PgConnectOptions =+ config.whatsapp_session_lock_database_url.as_ref().parse()?;+ let dialer: Arc<dyn Dialer> =+ match macro_service_urls::read_override_env(WEBSOCKET_URL_OVERRIDE)? {+ Some(url) => Arc::new(WebSocketDialer { url }),+ None => Arc::new(WebSocketDialer::default()),+ };+ let kms = AwsWhatsAppKms::new(+ aws_sdk_kms::Client::new(¯o_aws_config::get_macro_aws_config().await),+ config.whatsapp_session_kms_key_id.clone(),+ );++ let shutdown = CancellationToken::new();+ let listener = tokio::net::TcpListener::bind(("0.0.0.0", PORT)).await?;+ let server_stop = CancellationToken::new();+ let server = tokio::spawn({+ let stop = server_stop.clone();+ let app = health::router(shutdown.clone());+ async move {+ axum::serve(listener, app)+ .with_graceful_shutdown(stop.cancelled_owned())+ .await+ }+ });+ tokio::spawn({+ let shutdown = shutdown.clone();+ async move {+ shutdown_signal().await;+ shutdown.cancel();+ }+ });++ let replica = replica::start(+ ReplicaDeps {+ pool,+ lock_options,+ kms,+ signal: whatsmeow_signal::libsignal(),+ dialer,+ media_base_url: macro_service_urls::read_override_env(MEDIA_HOST_OVERRIDE)?,+ max_accounts: config.whatsapp_session_max_accounts,+ drain_batch: config.whatsapp_session_drain_batch,+ timings: Timings::production(),+ },+ shutdown,+ )+ .await?;+ let result = replica.wait().await;+ server_stop.cancel();+ match server.await {+ Ok(Ok(())) => {}+ Ok(Err(error)) => tracing::error!(error = %error, "The health server failed"),+ Err(error) => tracing::error!(error = %error, "The health server task panicked"),+ }+ // Losing the lock connection exits non-zero so ECS replaces the task.+ result?;+ Ok(())+}diff --git a/services/whatsapp_session_service/src/outbound.rs b/services/whatsapp_session_service/src/outbound.rsnew file mode 100644index 0000000000..9f360f1cf3--- /dev/null+++ b/services/whatsapp_session_service/src/outbound.rs@@ -0,0 +1,5 @@+//! Postgres and WhatsApp adapters.++pub mod lease;+pub mod signals;+pub mod whatsapp;diff --git a/services/whatsapp_session_service/src/outbound/lease.rs b/services/whatsapp_session_service/src/outbound/lease.rsnew file mode 100644index 0000000000..7b939ced5c--- /dev/null+++ b/services/whatsapp_session_service/src/outbound/lease.rs@@ -0,0 +1,262 @@+//! Account ownership in Postgres: session-level advisory locks on one+//! dedicated connection, candidate discovery, and release announcements.+//!+//! The lock key of an account is `hashtextextended('whatsapp-account:' || id, 0)`+//! (64-bit, so keys of different accounts practically never collide). Locks+//! live as long as the lock connection: if it dies, Postgres frees them and+//! [`PgAccountLocks::lost`] fires so every account on this replica stops.+//! Fencing (`whatsapp_linked_account.lease_epoch`) is the domain's job; a lock+//! only decides who may bump the epoch.++use std::time::Duration;++use sqlx::postgres::{PgConnectOptions, PgPool};+use sqlx::{ConnectOptions, Connection, PgConnection};+use tokio::sync::{mpsc, oneshot};+use tokio_util::sync::CancellationToken;+use tokio_util::task::TaskTracker;+use uuid::Uuid;++use whatsapp_linked::domain::models::LinkedError;+use whatsapp_linked::domain::session::SessionService;++use crate::domain::ports::{AccountDirectory, AccountLocks, FleetHealth, LeaseError};+use crate::domain::supervisor::ACCOUNT_RELEASED;++#[cfg(test)]+mod test;++/// Server-side TCP keepalive settings for the lock connection, so Postgres+/// frees a dead replica's locks within about 25 seconds.+const SERVER_KEEPALIVES: [(&str, &str); 3] = [+ ("tcp_keepalives_idle", "10"),+ ("tcp_keepalives_interval", "5"),+ ("tcp_keepalives_count", "3"),+];++impl From<sqlx::Error> for LeaseError {+ fn from(error: sqlx::Error) -> Self {+ LeaseError::Internal(rootcause::Report::new(error).into_dynamic())+ }+}++enum Command {+ TryLock(Uuid, oneshot::Sender<Result<bool, sqlx::Error>>),+ Unlock(Uuid, oneshot::Sender<Result<bool, sqlx::Error>>),+}++/// Advisory locks held on one dedicated, non-pooled connection.+///+/// The connection is owned by one task; dropping every handle closes it,+/// which releases any lock still held.+#[derive(Clone)]+pub struct PgAccountLocks {+ commands: mpsc::Sender<Command>,+ lost: CancellationToken,+ backend_pid: i32,+}++impl PgAccountLocks {+ /// Opens the lock connection. Every `keepalive` it pings the server; a ping+ /// or lock call that fails or takes longer than `keepalive` counts as a+ /// lost connection.+ pub async fn connect(+ options: &PgConnectOptions,+ keepalive: Duration,+ tracker: &TaskTracker,+ ) -> Result<Self, LeaseError> {+ let mut connection = options.clone().options(SERVER_KEEPALIVES).connect().await?;+ let backend_pid: i32 = sqlx::query_scalar!(r#"SELECT pg_backend_pid() AS "pid!""#)+ .fetch_one(&mut connection)+ .await?;+ let (commands, receiver) = mpsc::channel(64);+ let lost = CancellationToken::new();+ tracker.spawn(hold(connection, receiver, keepalive, lost.clone()));+ Ok(PgAccountLocks {+ commands,+ lost,+ backend_pid,+ })+ }++ /// The server process of the lock connection, for operators and tests.+ pub fn backend_pid(&self) -> i32 {+ self.backend_pid+ }++ async fn call(+ &self,+ command: impl FnOnce(oneshot::Sender<Result<bool, sqlx::Error>>) -> Command,+ ) -> Result<bool, LeaseError> {+ let (reply, response) = oneshot::channel();+ self.commands+ .send(command(reply))+ .await+ .map_err(|_| LeaseError::Lost)?;+ match response.await {+ Ok(Ok(value)) => Ok(value),+ Ok(Err(_)) | Err(_) => Err(LeaseError::Lost),+ }+ }+}++/// Owns the lock connection until it fails or every handle is dropped.+async fn hold(+ mut connection: PgConnection,+ mut commands: mpsc::Receiver<Command>,+ keepalive: Duration,+ lost: CancellationToken,+) {+ let mut ping = tokio::time::interval(keepalive);+ ping.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);+ loop {+ let outcome = tokio::select! {+ command = commands.recv() => match command {+ Some(Command::TryLock(account, reply)) => {+ let result = bounded(keepalive, try_lock(&mut connection, account)).await;+ let failure = failed("try_lock", &result);+ let _ = reply.send(result);+ failure+ }+ Some(Command::Unlock(account, reply)) => {+ let result = bounded(keepalive, unlock(&mut connection, account)).await;+ let failure = failed("unlock", &result);+ let _ = reply.send(result);+ failure+ }+ None => {+ // Closing releases every lock this connection holds.+ if let Err(error) = connection.close().await {+ tracing::warn!(error = %error, "Closing the WhatsApp lock connection failed");+ }+ return;+ }+ },+ _ = ping.tick() => {+ let result = bounded(keepalive, connection.ping()).await;+ failed("ping", &result)+ }+ };+ if let Some((operation, error)) = outcome {+ tracing::error!(+ operation,+ error = %error,+ "WhatsApp account lock connection lost; stopping every account"+ );+ lost.cancel();+ return;+ }+ }+}++/// The failed operation and its error, rendered before the result is handed on.+fn failed<Value>(+ operation: &'static str,+ result: &Result<Value, sqlx::Error>,+) -> Option<(&'static str, String)> {+ result+ .as_ref()+ .err()+ .map(|error| (operation, error.to_string()))+}++async fn bounded<Value>(+ limit: Duration,+ operation: impl Future<Output = Result<Value, sqlx::Error>>,+) -> Result<Value, sqlx::Error> {+ tokio::time::timeout(limit, operation)+ .await+ .unwrap_or_else(|_| Err(sqlx::Error::Io(std::io::ErrorKind::TimedOut.into())))+}++async fn try_lock(connection: &mut PgConnection, account: Uuid) -> Result<bool, sqlx::Error> {+ sqlx::query_scalar!(+ r#"SELECT pg_try_advisory_lock(hashtextextended('whatsapp-account:' || $1::uuid::text, 0)) AS "locked!""#,+ account+ )+ .fetch_one(connection)+ .await+}++async fn unlock(connection: &mut PgConnection, account: Uuid) -> Result<bool, sqlx::Error> {+ sqlx::query_scalar!(+ r#"SELECT pg_advisory_unlock(hashtextextended('whatsapp-account:' || $1::uuid::text, 0)) AS "unlocked!""#,+ account+ )+ .fetch_one(connection)+ .await+}++impl AccountLocks for PgAccountLocks {+ async fn try_lock(&self, account: Uuid) -> Result<bool, LeaseError> {+ self.call(|reply| Command::TryLock(account, reply)).await+ }++ async fn unlock(&self, account: Uuid) -> Result<(), LeaseError> {+ let held = self.call(|reply| Command::Unlock(account, reply)).await?;+ if !held {+ tracing::warn!(account_id = %account, "Unlocked a WhatsApp account this replica did not hold");+ }+ Ok(())+ }++ fn lost(&self) -> CancellationToken {+ self.lost.clone()+ }+}++/// Discovery through the linked-devices domain, and release announcements+/// (the service's own `NOTIFY` channel) over the shared pool.+#[derive(Clone)]+pub struct PgAccountDirectory {+ pool: PgPool,+ sessions: SessionService,+}++impl PgAccountDirectory {+ /// Wraps the shared pool and the domain service.+ pub fn new(pool: PgPool, sessions: SessionService) -> Self {+ PgAccountDirectory { pool, sessions }+ }+}++fn domain_error(report: rootcause::Report<LinkedError>) -> LeaseError {+ LeaseError::Internal(report.into_dynamic())+}++impl AccountDirectory for PgAccountDirectory {+ async fn candidates(&self, limit: u32) -> Result<Vec<Uuid>, LeaseError> {+ self.sessions+ .discoverable_accounts(i64::from(limit))+ .await+ .map_err(domain_error)+ }++ async fn pairing_account(&self, pairing: Uuid) -> Result<Option<Uuid>, LeaseError> {+ Ok(self+ .sessions+ .pairing_work(pairing)+ .await+ .map_err(domain_error)?+ .map(|work| work.account_id))+ }++ async fn announce_released(&self, account: Uuid) -> Result<(), LeaseError> {+ sqlx::query!(+ "SELECT pg_notify($1, $2)",+ ACCOUNT_RELEASED,+ account.to_string()+ )+ .execute(&self.pool)+ .await?;+ Ok(())+ }++ async fn health(&self) -> Result<FleetHealth, LeaseError> {+ let health = self.sessions.fleet_health().await.map_err(domain_error)?;+ Ok(FleetHealth {+ accounts_unowned: health.accounts_unowned,+ oldest_unprocessed_inbound_seconds: health.oldest_unprocessed_inbound_seconds,+ })+ }+}diff --git a/services/whatsapp_session_service/src/outbound/lease/test.rs b/services/whatsapp_session_service/src/outbound/lease/test.rsnew file mode 100644index 0000000000..ed9c9b72eb--- /dev/null+++ b/services/whatsapp_session_service/src/outbound/lease/test.rs@@ -0,0 +1,232 @@+//! Account ownership against a real Postgres: contention between replicas,+//! handover on release, and a lost lock connection.++use std::sync::Arc;+use std::time::{Duration, Instant};++use macro_db_migrator::MACRO_DB_MIGRATIONS;+use sqlx::PgPool;+use sqlx::postgres::{PgConnectOptions, PgPoolOptions};+use tokio::sync::mpsc;+use tokio_util::sync::CancellationToken;+use tokio_util::task::TaskTracker;+use uuid::Uuid;+use whatsapp_linked::domain::session::SessionService;+use whatsapp_linked::outbound::pg::PgLinkedRepository;++use super::{PgAccountDirectory, PgAccountLocks};+use crate::domain::ports::{+ AccountControl, AccountDirectory, AccountExit, AccountLocks, AccountRunner,+};+use crate::domain::supervisor::{Supervisor, SupervisorConfig, SupervisorError};+use crate::outbound::signals::{SIGNAL_CAPACITY, SignalListener};++const KEEPALIVE: Duration = Duration::from_millis(100);++async fn insert_account(pool: &PgPool, user_id: &str, state: &str) -> Uuid {+ sqlx::query!(+ r#"+ WITH new_macro_user AS (+ INSERT INTO macro_user (id, username, email, stripe_customer_id)+ VALUES (gen_random_uuid(), $2, $2, $3)+ RETURNING id+ )+ INSERT INTO "User" ("id", "email", "macro_user_id")+ SELECT $1, $2, new_macro_user.id FROM new_macro_user+ "#,+ user_id,+ format!("{user_id}@example.com"),+ format!("stripe_{user_id}"),+ )+ .execute(pool)+ .await+ .unwrap();+ let account = Uuid::now_v7();+ sqlx::query!(+ "INSERT INTO whatsapp_linked_account (id, user_id, state) VALUES ($1, $2, $3)",+ account,+ user_id,+ state,+ )+ .execute(pool)+ .await+ .unwrap();+ account+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn only_one_replica_locks_an_account_and_discovery_skips_it(+ pool_options: PgPoolOptions,+ options: PgConnectOptions,+) {+ let pool = pool_options.connect_with(options.clone()).await.unwrap();+ let connected = insert_account(&pool, "macro|connected", "connected").await;+ let pairing = insert_account(&pool, "macro|pairing", "pairing").await;+ insert_account(&pool, "macro|logged-out", "logged_out").await;+ let tracker = TaskTracker::new();+ let first = PgAccountLocks::connect(&options, KEEPALIVE, &tracker)+ .await+ .unwrap();+ let second = PgAccountLocks::connect(&options, KEEPALIVE, &tracker)+ .await+ .unwrap();+ let directory = PgAccountDirectory::new(+ pool.clone(),+ SessionService::new(Arc::new(PgLinkedRepository::new(pool.clone()))),+ );++ sqlx::query!(+ r#"INSERT INTO whatsapp_pairing (id, account_id, user_id, generation, state, expires_at)+ VALUES ($1, $2, 'macro|pairing', 1, 'waiting', now() + interval '5 minutes')"#,+ Uuid::now_v7(),+ pairing,+ )+ .execute(&pool)+ .await+ .unwrap();+ insert_account(&pool, "macro|stale-pairing", "pairing").await;++ let mut candidates = directory.candidates(10).await.unwrap();+ candidates.sort();+ let mut expected = vec![connected, pairing];+ expected.sort();+ assert_eq!(+ candidates, expected,+ "logged-out accounts and pairings without a live transaction need no session"+ );++ assert!(first.try_lock(connected).await.unwrap());+ assert!(!second.try_lock(connected).await.unwrap());+ assert_eq!(directory.candidates(10).await.unwrap(), vec![pairing]);+ assert_eq!(directory.health().await.unwrap().accounts_unowned, 1);++ first.unlock(connected).await.unwrap();+ assert!(second.try_lock(connected).await.unwrap());+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn killing_the_lock_connection_frees_its_locks_and_reports_the_loss(+ pool_options: PgPoolOptions,+ options: PgConnectOptions,+) {+ let pool = pool_options.connect_with(options.clone()).await.unwrap();+ let account = insert_account(&pool, "macro|owner", "connected").await;+ let tracker = TaskTracker::new();+ let doomed = PgAccountLocks::connect(&options, KEEPALIVE, &tracker)+ .await+ .unwrap();+ let survivor = PgAccountLocks::connect(&options, KEEPALIVE, &tracker)+ .await+ .unwrap();+ assert!(doomed.try_lock(account).await.unwrap());++ let terminated = sqlx::query_scalar!(+ r#"SELECT pg_terminate_backend($1) AS "terminated!""#,+ doomed.backend_pid()+ )+ .fetch_one(&pool)+ .await+ .unwrap();+ assert!(terminated);++ tokio::time::timeout(KEEPALIVE * 10, doomed.lost().cancelled())+ .await+ .expect("the next keepalive notices the dead connection");+ assert!(matches!(+ doomed.try_lock(account).await,+ Err(crate::domain::ports::LeaseError::Lost)+ ));+ assert!(survivor.try_lock(account).await.unwrap());+}++/// Reports which account it started, then runs until released or stopped.+struct ReportingRunner {+ started: mpsc::UnboundedSender<(Uuid, Instant)>,+}++impl AccountRunner for ReportingRunner {+ async fn run(&self, account: Uuid, control: AccountControl) -> AccountExit {+ self.started.send((account, Instant::now())).unwrap();+ tokio::select! {+ () = control.release.cancelled() => AccountExit::Released,+ () = control.stop.cancelled() => AccountExit::Stopped,+ }+ }+}++struct Replica {+ shutdown: CancellationToken,+ started: mpsc::UnboundedReceiver<(Uuid, Instant)>,+ running: tokio::task::JoinHandle<Result<(), SupervisorError>>,+}++async fn start_replica(+ pool: &PgPool,+ options: &PgConnectOptions,+ tracker: &TaskTracker,+) -> Replica {+ let locks = PgAccountLocks::connect(options, KEEPALIVE, tracker)+ .await+ .unwrap();+ let listener = SignalListener::connect(options).await.unwrap();+ let (signal_sender, mut signals) = mpsc::channel(SIGNAL_CAPACITY);+ let shutdown = CancellationToken::new();+ tracker.spawn(listener.run(signal_sender, shutdown.clone()));+ let (started_sender, started) = mpsc::unbounded_channel();+ let supervisor = Supervisor::new(+ locks,+ PgAccountDirectory::new(+ pool.clone(),+ SessionService::new(Arc::new(PgLinkedRepository::new(pool.clone()))),+ ),+ Arc::new(ReportingRunner {+ started: started_sender,+ }),+ SupervisorConfig {+ max_accounts: 10,+ drain_batch: 10,+ // Long enough that only a signal can explain a fast handover.+ rescan_every: Duration::from_secs(600),+ release_timeout: Duration::from_secs(5),+ stop_timeout: Duration::from_secs(5),+ drain_deadline: Duration::from_secs(100),+ health_every: Duration::from_secs(600),+ },+ );+ let stop = shutdown.clone();+ let running = tokio::spawn(async move { supervisor.run(&mut signals, stop).await });+ Replica {+ shutdown,+ started,+ running,+ }+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn a_draining_replica_hands_its_account_over_without_waiting_for_a_rescan(+ pool_options: PgPoolOptions,+ options: PgConnectOptions,+) {+ let pool = pool_options.connect_with(options.clone()).await.unwrap();+ let account = insert_account(&pool, "macro|owner", "connected").await;+ let tracker = TaskTracker::new();+ let mut draining = start_replica(&pool, &options, &tracker).await;+ let (first_owner, _) = draining.started.recv().await.unwrap();+ assert_eq!(first_owner, account);+ let mut taking_over = start_replica(&pool, &options, &tracker).await;++ draining.shutdown.cancel();+ draining.running.await.unwrap().unwrap();++ // The next rescan is ten minutes away: only the release notification+ // can hand the account over within this bound.+ let (second_owner, _) =+ tokio::time::timeout(Duration::from_secs(30), taking_over.started.recv())+ .await+ .expect("the release notification hands the account over")+ .unwrap();+ assert_eq!(second_owner, account);++ taking_over.shutdown.cancel();+ taking_over.running.await.unwrap().unwrap();+}diff --git a/services/whatsapp_session_service/src/outbound/signals.rs b/services/whatsapp_session_service/src/outbound/signals.rsnew file mode 100644index 0000000000..22587008e0--- /dev/null+++ b/services/whatsapp_session_service/src/outbound/signals.rs@@ -0,0 +1,106 @@+//! `LISTEN` for the signals DSS and other replicas send through Postgres.+//! An outbound adapter (it holds a Postgres connection) that feeds the+//! supervisor.+//!+//! The listener runs on its own direct connection (a transaction pooler would+//! drop `LISTEN`). When it reconnects, notifications sent in between are lost,+//! so it emits [`Signal::Resync`] and the supervisor re-reads everything.++use sqlx::postgres::{PgConnectOptions, PgListener, PgPoolOptions};+use tokio::sync::mpsc;+use tokio_util::sync::CancellationToken;+use uuid::Uuid;++use crate::domain::supervisor::{ACCOUNT_RELEASED, Signal};++/// `whatsapp_pairing_requested`, `<pairing_id>`.+pub const PAIRING_REQUESTED: &str = "whatsapp_pairing_requested";+/// `whatsapp_send_requested`, `<account_id>`.+pub const SEND_REQUESTED: &str = "whatsapp_send_requested";+/// `whatsapp_groups_requested`, `<account_id>`.+pub const GROUPS_REQUESTED: &str = "whatsapp_groups_requested";+/// `whatsapp_account_changed`, `<account_id>`.+pub const ACCOUNT_CHANGED: &str = "whatsapp_account_changed";++const CHANNELS: [&str; 5] = [+ ACCOUNT_RELEASED,+ PAIRING_REQUESTED,+ SEND_REQUESTED,+ GROUPS_REQUESTED,+ ACCOUNT_CHANGED,+];++/// Signals queued for the supervisor before the listener waits.+pub const SIGNAL_CAPACITY: usize = 1024;++/// Errors starting the listener.+#[derive(Debug, thiserror::Error)]+pub enum ListenError {+ /// Connecting or subscribing failed.+ #[error("listening for WhatsApp signals failed: {0}")]+ Database(#[from] sqlx::Error),+}++/// A subscribed listener; [`SignalListener::run`] forwards its notifications.+pub struct SignalListener {+ listener: PgListener,+}++impl SignalListener {+ /// Connects and subscribes before returning, so no signal sent after this+ /// returns is missed.+ pub async fn connect(options: &PgConnectOptions) -> Result<Self, ListenError> {+ let pool = PgPoolOptions::new()+ .max_connections(1)+ .connect_lazy_with(options.clone());+ let mut listener = PgListener::connect_with(&pool).await?;+ listener.listen_all(CHANNELS).await?;+ Ok(SignalListener { listener })+ }++ /// Forwards notifications until `stop` or until the supervisor goes away.+ pub async fn run(mut self, signals: mpsc::Sender<Signal>, stop: CancellationToken) {+ loop {+ let received = tokio::select! {+ () = stop.cancelled() => return,+ received = self.listener.try_recv() => received,+ };+ let signal = match received {+ Ok(Some(notification)) => {+ match parse(notification.channel(), notification.payload()) {+ Some(signal) => signal,+ None => {+ tracing::warn!(+ channel = notification.channel(),+ "Ignoring a malformed WhatsApp signal"+ );+ continue;+ }+ }+ }+ // The connection dropped; the next call reconnects and resubscribes.+ Ok(None) => Signal::Resync,+ Err(error) => {+ tracing::warn!(error = %error, "WhatsApp signal listener failed; reconnecting");+ Signal::Resync+ }+ };+ if signals.send(signal).await.is_err() {+ return;+ }+ }+ }+}++/// Maps one notification to a signal; `None` for unknown channels or payloads.+pub fn parse(channel: &str, payload: &str) -> Option<Signal> {+ let id = Uuid::parse_str(payload).ok()?;+ match channel {+ ACCOUNT_RELEASED => Some(Signal::AccountReleased(id)),+ PAIRING_REQUESTED => Some(Signal::PairingRequested(id)),+ SEND_REQUESTED => Some(Signal::SendRequested(id)),+ GROUPS_REQUESTED => Some(Signal::GroupsRequested(id)),+ ACCOUNT_CHANGED => Some(Signal::AccountChanged(id)),+ _ => None,+ }+}diff --git a/services/whatsapp_session_service/src/outbound/whatsapp.rs b/services/whatsapp_session_service/src/outbound/whatsapp.rsnew file mode 100644index 0000000000..7cca58d031--- /dev/null+++ b/services/whatsapp_session_service/src/outbound/whatsapp.rs@@ -0,0 +1,629 @@+//! The account's WhatsApp connection: the encrypted Postgres device store,+//! a whatsmeow [`Client`], and the mapping between whatsmeow and the domain.++use std::sync::Arc;+use std::time::Duration;++use async_trait::async_trait;+use chrono::{DateTime, Utc};+use sqlx::PgPool;+use tokio::sync::mpsc;+use tokio_util::sync::CancellationToken;+use uuid::Uuid;+use whatsapp_linked::domain::events::{+ ClaimedSend, MediaImport, OutboundMedia, OutgoingContent, SessionScope, Target,+};+use whatsapp_linked::domain::models::{GroupInfo, Identity, MediaKind, Participant};+use whatsapp_store_pg::kms::WhatsAppKms;+use whatsapp_store_pg::{PgWhatsAppStore, WhatsAppStoreError};+use whatsmeow::download::{DownloadError, MediaType};+use whatsmeow::proto::wa_e2e;+use whatsmeow::send::{SendRequestExtra, build_edit};+use whatsmeow::signal::CurveSignatures;+use whatsmeow::store::Device;+use whatsmeow::types::{Jid, WebAuthnResponse};+use whatsmeow::{+ Client, ClientConfig, ClientError, Dialer, Event, EventHandler, QrChannelItem, Signal,+};++use crate::domain::ports::{+ DownloadFailure, InboundSink, SendFailure, SessionConnector, SessionError, SessionEvent,+ UploadedMedia, WhatsAppSession,+};+use events::{Mapped, MediaLocator};++pub mod events;+#[cfg(test)]+mod test;++/// Purpose of sealed media locators (see `PgWhatsAppStore::seal`).+const LOCATOR_PURPOSE: &str = "media-locator";++fn internal(error: impl std::error::Error + Send + Sync + 'static) -> SessionError {+ SessionError::Internal(rootcause::Report::new(error).into_dynamic())+}++/// Opens accounts' devices and clients.+pub struct WhatsmeowConnector<Kms> {+ pool: PgPool,+ kms: Kms,+ signal: Arc<dyn Signal>,+ dialer: Arc<dyn Dialer>,+ media_base_url: Option<String>,+ send_timeout: Duration,+}++impl<Kms: WhatsAppKms + 'static> WhatsmeowConnector<Kms> {+ /// `signal` is `whatsmeow_signal::libsignal()` in production; `dialer`+ /// and `media_base_url` point at WhatsApp, or at a fake server in tests.+ pub fn new(+ pool: PgPool,+ kms: Kms,+ signal: Arc<dyn Signal>,+ dialer: Arc<dyn Dialer>,+ media_base_url: Option<String>,+ send_timeout: Duration,+ ) -> Self {+ WhatsmeowConnector {+ pool,+ kms,+ signal,+ dialer,+ media_base_url,+ send_timeout,+ }+ }++ async fn open_store(&self, scope: SessionScope) -> Result<Arc<PgWhatsAppStore>, SessionError> {+ Ok(Arc::new(+ PgWhatsAppStore::open(+ self.pool.clone(),+ scope.account_id,+ scope.lease_epoch,+ &self.kms,+ )+ .await+ .map_err(store_error)?,+ ))+ }++ async fn load_device(&self, store: &Arc<PgWhatsAppStore>) -> Result<Device, SessionError> {+ store+ .load_device(&Signatures(self.signal.as_ref()))+ .await+ .map_err(store_error)+ }+}++fn store_error(error: WhatsAppStoreError) -> SessionError {+ match error {+ WhatsAppStoreError::Fenced => SessionError::Fenced,+ WhatsAppStoreError::Kms(_) => SessionError::Kms,+ error => internal(error),+ }+}++/// `load_device` takes a sized signer; the client's Signal is a trait object.+struct Signatures<'signal>(&'signal dyn Signal);++impl CurveSignatures for Signatures<'_> {+ fn calculate_signature(&self, private_key: &[u8; 32], message: &[u8]) -> [u8; 64] {+ self.0.calculate_signature(private_key, message)+ }++ fn verify_signature(+ &self,+ public_key: &[u8; 32],+ message: &[u8],+ signature: &[u8; 64],+ ) -> bool {+ self.0.verify_signature(public_key, message, signature)+ }+}++impl<Kms: WhatsAppKms + 'static> SessionConnector for WhatsmeowConnector<Kms> {+ type Session = WhatsmeowSession;++ async fn open(+ &self,+ scope: SessionScope,+ fresh_device: bool,+ inbound: Arc<dyn InboundSink>,+ ) -> Result<(WhatsmeowSession, mpsc::UnboundedReceiver<SessionEvent>), SessionError> {+ let store = self.open_store(scope).await?;+ let mut device = self.load_device(&store).await?;+ if fresh_device && device.id.is_some() {+ // Deleting the device row drops its data key and every stored+ // value; reopening generates both anew.+ device.delete().await.map_err(internal)?;+ let store = self.open_store(scope).await?;+ device = self.load_device(&store).await?;+ return self.start(scope, store, device, inbound);+ }+ self.start(scope, store, device, inbound)+ }+}++impl<Kms: WhatsAppKms + 'static> WhatsmeowConnector<Kms> {+ fn start(+ &self,+ scope: SessionScope,+ store: Arc<PgWhatsAppStore>,+ device: Device,+ inbound: Arc<dyn InboundSink>,+ ) -> Result<(WhatsmeowSession, mpsc::UnboundedReceiver<SessionEvent>), SessionError> {+ let client = Client::new(+ device,+ Arc::clone(&self.signal),+ ClientConfig {+ dialer: Arc::clone(&self.dialer),+ initial_auto_reconnect: true,+ // Group events are stored before their node is acknowledged.+ synchronous_ack: true,+ media_base_url: self.media_base_url.clone(),+ ..ClientConfig::default()+ },+ );+ let (sender, receiver) = mpsc::unbounded_channel();+ client.add_event_handler(Arc::new(Forwarder {+ account: scope.account_id,+ store: Arc::clone(&store),+ events: sender.clone(),+ inbound,+ }));+ Ok((+ WhatsmeowSession {+ client,+ store,+ events: sender,+ pairing: CancellationToken::new(),+ send_timeout: self.send_timeout,+ },+ receiver,+ ))+ }+}++/// Stores group events through the inbound sink before the client acks+/// them; passes everything else on without waiting. It never waits on the+/// runner, which may itself be waiting for a send's acknowledgement.+struct Forwarder {+ account: Uuid,+ store: Arc<PgWhatsAppStore>,+ events: mpsc::UnboundedSender<SessionEvent>,+ inbound: Arc<dyn InboundSink>,+}++// Object safe (`Arc<dyn EventHandler>`), hence `async_trait`.+#[async_trait]+impl EventHandler for Forwarder {+ async fn handle(&self, event: &Event) -> bool {+ let seal = |locator: &MediaLocator| {+ let plaintext = serde_json::to_vec(locator).ok()?;+ self.store+ .seal(LOCATOR_PURPOSE, &plaintext)+ .inspect_err(+ |error| tracing::warn!(error = %error, "Sealing a media locator failed"),+ )+ .ok()+ };+ match events::map_event(self.account, event, Utc::now(), &seal) {+ Some(Mapped::Inbound(event)) => self.inbound.accept(&event).await,+ Some(Mapped::Session(event)) => {+ let _ = self.events.send(event);+ true+ }+ None => {+ if matches!(event, Event::Message(_)) {+ tracing::debug!(+ name: "whatsapp.inbound.skipped",+ account_id = %self.account,+ reason = "not_group_content",+ "Acknowledged a WhatsApp message without storing it"+ );+ }+ true+ }+ }+ }+}++/// One account's whatsmeow client.+pub struct WhatsmeowSession {+ client: Arc<Client>,+ store: Arc<PgWhatsAppStore>,+ events: mpsc::UnboundedSender<SessionEvent>,+ pairing: CancellationToken,+ send_timeout: Duration,+}++impl Drop for WhatsmeowSession {+ fn drop(&mut self) {+ self.pairing.cancel();+ }+}++fn parse_jid(jid: &str) -> Result<Jid, SendFailure> {+ jid.parse()+ .map_err(|_| SendFailure::Rejected("invalid_jid"))+}++fn rejected(error: &ClientError) -> Option<&'static str> {+ // Only errors raised before the message could reach WhatsApp are safe to+ // call rejected; anything else may have been delivered.+ match error {+ ClientError::NotConnected | ClientError::NotLoggedIn => Some("not_connected"),+ ClientError::NotInGroup | ClientError::GroupNotFound => Some("not_in_group"),+ ClientError::NoPermissionInGroup => Some("no_permission"),+ ClientError::RecipientAdJid+ | ClientError::BroadcastListUnsupported+ | ClientError::UnknownServer => Some("invalid_recipient"),+ _ => None,+ }+}++fn media_type(kind: MediaKind) -> MediaType {+ match kind {+ MediaKind::Image => MediaType::Image,+ MediaKind::Audio => MediaType::Audio,+ MediaKind::Video => MediaType::Video,+ MediaKind::Document => MediaType::Document,+ }+}++/// Whether a download can ever succeed after this error.+fn download_failure(error: &ClientError) -> DownloadFailure {+ let ClientError::Download(error) = error else {+ return DownloadFailure::Retryable;+ };+ match error.as_ref() {+ DownloadError::Http {+ status_code: 403 | 404 | 410,+ ..+ } => DownloadFailure::Permanent,+ DownloadError::Http { .. } => DownloadFailure::Retryable,+ _ => DownloadFailure::Permanent,+ }+}++fn quote_context(quote: Option<&Target>) -> Option<Box<wa_e2e::ContextInfo>> {+ quote.map(+ |Target {+ source_id,+ sender_jid,+ }| {+ Box::new(wa_e2e::ContextInfo {+ stanza_id: Some(source_id.clone()),+ participant: Some(sender_jid.clone()),+ ..Default::default()+ })+ },+ )+}++/// The message part carrying an uploaded attachment.+fn media_message(+ kind: MediaKind,+ mime: &str,+ filename: &str,+ caption: Option<&str>,+ quote: Option<&Target>,+ uploaded: &UploadedMedia,+) -> wa_e2e::Message {+ let caption = caption.map(str::to_owned);+ let context_info = quote_context(quote);+ let url = Some(uploaded.url.clone());+ let direct_path = Some(uploaded.direct_path.clone());+ let media_key = Some(uploaded.media_key.clone());+ let file_enc_sha256 = Some(uploaded.file_enc_sha256.clone());+ let file_sha256 = Some(uploaded.file_sha256.clone());+ let file_length = Some(uploaded.file_length);+ let mimetype = Some(mime.to_owned());+ match kind {+ MediaKind::Image => wa_e2e::Message {+ image_message: Some(Box::new(wa_e2e::ImageMessage {+ url,+ direct_path,+ media_key,+ file_enc_sha256,+ file_sha256,+ file_length,+ mimetype,+ caption,+ context_info,+ ..Default::default()+ })),+ ..Default::default()+ },+ MediaKind::Video => wa_e2e::Message {+ video_message: Some(Box::new(wa_e2e::VideoMessage {+ url,+ direct_path,+ media_key,+ file_enc_sha256,+ file_sha256,+ file_length,+ mimetype,+ caption,+ context_info,+ ..Default::default()+ })),+ ..Default::default()+ },+ // Audio has no caption.+ MediaKind::Audio => wa_e2e::Message {+ audio_message: Some(Box::new(wa_e2e::AudioMessage {+ url,+ direct_path,+ media_key,+ file_enc_sha256,+ file_sha256,+ file_length,+ mimetype,+ context_info,+ ..Default::default()+ })),+ ..Default::default()+ },+ MediaKind::Document => wa_e2e::Message {+ document_message: Some(Box::new(wa_e2e::DocumentMessage {+ url,+ direct_path,+ media_key,+ file_enc_sha256,+ file_sha256,+ file_length,+ mimetype,+ file_name: Some(filename.to_owned()),+ title: Some(filename.to_owned()),+ caption,+ context_info,+ ..Default::default()+ })),+ ..Default::default()+ },+ }+}++impl WhatsmeowSession {+ async fn outgoing(+ &self,+ chat: &Jid,+ content: &OutgoingContent,+ media: Option<&UploadedMedia>,+ ) -> Result<wa_e2e::Message, SendFailure> {+ Ok(match content {+ OutgoingContent::Text { text, quote: None } => wa_e2e::Message {+ conversation: Some(text.clone()),+ ..Default::default()+ },+ OutgoingContent::Text {+ text,+ quote: Some(quote),+ } => wa_e2e::Message {+ extended_text_message: Some(Box::new(wa_e2e::ExtendedTextMessage {+ text: Some(text.clone()),+ context_info: quote_context(Some(quote)),+ ..Default::default()+ })),+ ..Default::default()+ },+ OutgoingContent::Media {+ kind,+ mime,+ filename,+ caption,+ quote,+ } => {+ let uploaded = media.ok_or(SendFailure::Rejected("media_not_uploaded"))?;+ media_message(+ *kind,+ mime,+ filename,+ caption.as_deref(),+ quote.as_ref(),+ uploaded,+ )+ }+ OutgoingContent::Reaction { target, emoji } => {+ let sender = parse_jid(&target.sender_jid)?;+ self.client+ .build_reaction(chat, &sender, &target.source_id, emoji)+ .await+ }+ OutgoingContent::Edit { target, text } => build_edit(+ chat,+ &target.source_id,+ wa_e2e::Message {+ conversation: Some(text.clone()),+ ..Default::default()+ },+ ),+ OutgoingContent::Delete { target } => {+ self.client+ .build_revoke(chat, &Jid::default(), &target.source_id)+ .await+ }+ })+ }+}++fn group_info(group: whatsmeow::types::GroupInfo, observed_at: DateTime<Utc>) -> GroupInfo {+ GroupInfo {+ jid: group.jid.to_string(),+ name: group.name.name,+ participants: group+ .participants+ .iter()+ .map(|participant| {+ let (jid, alternate) = if participant.phone_number.is_empty() {+ (&participant.jid, &participant.lid)+ } else {+ (&participant.phone_number, &participant.lid)+ };+ Participant {+ identity: Identity {+ jid: jid.to_non_ad().to_string(),+ alternate_jid: (!alternate.is_empty() && alternate != jid)+ .then(|| alternate.to_non_ad().to_string()),+ display_name: (!participant.display_name.is_empty())+ .then(|| participant.display_name.clone()),+ },+ admin: participant.is_admin || participant.is_super_admin,+ }+ })+ .collect(),+ announce: group.announce.is_announce,+ disappearing: group.ephemeral.is_ephemeral,+ community: group.parent.is_parent || !group.linked_parent.linked_parent_jid.is_empty(),+ observed_at,+ group: None,+ }+}++impl WhatsAppSession for WhatsmeowSession {+ async fn identity_jid(&self) -> Option<String> {+ let own = self.client.get_own_id().await;+ (!own.is_empty()).then(|| own.to_non_ad().to_string())+ }++ async fn start_pairing(&self) -> Result<(), SessionError> {+ let mut codes = self+ .client+ .get_qr_channel(self.pairing.clone())+ .await+ .map_err(internal)?;+ let events = self.events.clone();+ tokio::spawn(async move {+ while let Some(item) = codes.recv().await {+ let event = match item {+ QrChannelItem::Code { code, .. } => SessionEvent::Qr(code),+ QrChannelItem::Error(_)+ | QrChannelItem::Timeout+ | QrChannelItem::ErrUnexpectedEvent+ | QrChannelItem::ClientOutdated+ | QrChannelItem::ScannedWithoutMultidevice => SessionEvent::PairFailed,+ // Success and passkey steps arrive as client events.+ _ => continue,+ };+ if events.send(event).is_err() {+ return;+ }+ }+ });+ Ok(())+ }++ async fn send_passkey_response(&self, response: serde_json::Value) -> Result<(), SessionError> {+ let response: WebAuthnResponse = serde_json::from_value(response).map_err(internal)?;+ self.client+ .send_passkey_response(&response)+ .await+ .map_err(internal)+ }++ async fn confirm_passkey(&self) -> Result<(), SessionError> {+ self.client+ .send_passkey_confirmation()+ .await+ .map_err(internal)+ }++ async fn connect(&self) -> Result<(), SessionError> {+ self.client.connect().await.map_err(internal)+ }++ async fn new_message_id(&self) -> String {+ self.client.generate_message_id().await+ }++ async fn upload_media(&self, media: &OutboundMedia) -> Result<UploadedMedia, SendFailure> {+ // Boxed: the client's media futures are large enough to overflow a+ // 2 MiB stack when nested in the runner's future.+ let uploaded = Box::pin(self.client.upload(&media.bytes, media_type(media.kind)))+ .await+ .map_err(|error| {+ tracing::warn!(error = %error, "Uploading a WhatsApp attachment failed");+ SendFailure::Rejected("media_upload_failed")+ })?;+ Ok(UploadedMedia {+ url: uploaded.url,+ direct_path: uploaded.direct_path,+ media_key: uploaded.media_key,+ file_enc_sha256: uploaded.file_enc_sha256,+ file_sha256: uploaded.file_sha256,+ file_length: uploaded.file_length,+ })+ }++ async fn send(+ &self,+ send: &ClaimedSend,+ message_id: &str,+ media: Option<&UploadedMedia>,+ ) -> Result<DateTime<Utc>, SendFailure> {+ let chat = parse_jid(&send.group_jid)?;+ let message = self.outgoing(&chat, &send.content, media).await?;+ let response = Box::pin(self.client.send_message(+ &chat,+ message,+ SendRequestExtra {+ id: message_id.to_owned(),+ timeout: Some(self.send_timeout),+ // A send cut off by a disconnect is unknown, never resent.+ no_disconnect_retry: true,+ ..SendRequestExtra::default()+ },+ ))+ .await+ .map_err(|error| match rejected(&error) {+ Some(code) => SendFailure::Rejected(code),+ None => SendFailure::Unknown,+ })?;+ Ok(response.timestamp.unwrap_or_else(Utc::now))+ }++ async fn download_media(&self, import: &MediaImport) -> Result<Vec<u8>, DownloadFailure> {+ let plaintext = self+ .store+ .unseal(LOCATOR_PURPOSE, &import.locator)+ .map_err(|_| DownloadFailure::Permanent)?;+ let locator: MediaLocator =+ serde_json::from_slice(&plaintext).map_err(|_| DownloadFailure::Permanent)?;+ Box::pin(self.client.download_media_with_path(+ &locator.direct_path,+ Some(&locator.file_enc_sha256),+ Some(&locator.file_sha256),+ Some(&locator.media_key),+ Some(locator.media_type.media_type()),+ "",+ false,+ ))+ .await+ .map_err(|error| {+ let failure = download_failure(&error);+ tracing::warn!(error = %error, failure = ?failure, "Downloading a WhatsApp attachment failed");+ failure+ })+ }++ async fn joined_groups(&self) -> Result<Vec<GroupInfo>, SessionError> {+ let observed_at = Utc::now();+ let groups = self.client.get_joined_groups().await.map_err(internal)?;+ Ok(groups+ .into_iter()+ .map(|group| group_info(group, observed_at))+ .collect())+ }++ async fn logout(&self) {+ if let Err(error) = self.client.logout().await {+ tracing::warn!(error = %error, "Logging the WhatsApp device out failed");+ }+ }++ async fn disconnect(&self) {+ self.pairing.cancel();+ self.client.disconnect().await;+ }+}diff --git a/services/whatsapp_session_service/src/outbound/whatsapp/events.rs b/services/whatsapp_session_service/src/outbound/whatsapp/events.rsnew file mode 100644index 0000000000..60901e29c2--- /dev/null+++ b/services/whatsapp_session_service/src/outbound/whatsapp/events.rs@@ -0,0 +1,438 @@+//! Maps whatsmeow events onto the linked-devices domain's session events.+//! Only group traffic becomes [`InboundEvent`]s; the account runner then drops+//! everything outside adopted groups.++use chrono::{DateTime, Utc};+use serde::{Deserialize, Serialize};+use uuid::Uuid;+use whatsapp_linked::domain::events::{+ InboundEvent, InboundKind, Media, Message, MessageKind, Receipt, Target,+};+use whatsapp_linked::domain::models::Identity;+use whatsmeow::Event;+use whatsmeow::download::{DownloadableMessage, MediaType};+use whatsmeow::proto::wa_common::MessageKey;+use whatsmeow::proto::wa_e2e;+use whatsmeow::proto::wa_e2e::protocol_message;+use whatsmeow::types::{GROUP_SERVER, Jid};++use crate::domain::ports::SessionEvent;++#[cfg(test)]+mod test;++/// Namespace for event ids, so a redelivered protocol event keeps its id.+const EVENT_NAMESPACE: Uuid = Uuid::from_u128(0x6c1e_4f0a_93d2_4b7e_8a55_0f3c_2d19_7e41);++/// Readable stand-in for message shapes Macro does not import.+const UNSUPPORTED_TEXT: &str = "Sent something Macro can't show yet";++/// What a whatsmeow event means for the account.+#[derive(Debug)]+pub enum Mapped {+ /// Connection or pairing progress for the runner.+ Session(SessionEvent),+ /// Group traffic to store before acknowledging.+ Inbound(Box<InboundEvent>),+}++/// Where and how to fetch an observed attachment. Sealed with the account's+/// data key before it leaves the process: the media key decrypts the file.+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]+pub struct MediaLocator {+ /// Path on the media host.+ pub direct_path: String,+ /// Key the file is encrypted with.+ pub media_key: Vec<u8>,+ /// SHA-256 of the encrypted file.+ pub file_enc_sha256: Vec<u8>,+ /// SHA-256 of the plaintext.+ pub file_sha256: Vec<u8>,+ /// Which media keys to derive.+ pub media_type: LocatorMediaType,+}++/// The media key expansion of a locator.+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]+#[serde(rename_all = "snake_case")]+pub enum LocatorMediaType {+ /// Image keys.+ Image,+ /// Video keys.+ Video,+ /// Audio keys.+ Audio,+ /// Document keys.+ Document,+}++impl LocatorMediaType {+ /// whatsmeow's media type.+ pub fn media_type(self) -> MediaType {+ match self {+ LocatorMediaType::Image => MediaType::Image,+ LocatorMediaType::Video => MediaType::Video,+ LocatorMediaType::Audio => MediaType::Audio,+ LocatorMediaType::Document => MediaType::Document,+ }+ }+}++/// The meaning of a whatsmeow event, if the account cares about it. `seal`+/// turns a [`MediaLocator`] into the opaque bytes stored with the attachment;+/// `None` drops the attachment to a placeholder.+pub fn map_event(+ account: Uuid,+ event: &Event,+ now: DateTime<Utc>,+ seal: &dyn Fn(&MediaLocator) -> Option<Vec<u8>>,+) -> Option<Mapped> {+ Some(match event {+ Event::Message(message) => {+ Mapped::Inbound(Box::new(map_message(account, message, now, seal)?))+ }+ Event::Receipt(receipt) => Mapped::Inbound(Box::new(map_receipt(account, receipt, now)?)),+ event => Mapped::Session(map_session(event)?),+ })+}++fn map_session(event: &Event) -> Option<SessionEvent> {+ match event {+ Event::PairPasskeyRequest(request) => Some(SessionEvent::PasskeyRequested {+ challenge: serde_json::to_value(&request.public_key).ok()?,+ }),+ // With `skip_handoff_ux` the QR channel confirms on its own.+ Event::PairPasskeyConfirmation(confirmation) if !confirmation.skip_handoff_ux => {+ Some(SessionEvent::PasskeyConfirmation {+ code: confirmation.code.clone(),+ })+ }+ Event::PairSuccess(success) => Some(SessionEvent::PairSuccess {+ identity_jid: success.id.to_non_ad().to_string(),+ }),+ Event::PairError(_) => Some(SessionEvent::PairFailed),+ Event::Connected(_) => Some(SessionEvent::Connected),+ Event::Disconnected(_) | Event::KeepAliveTimeout(_) => Some(SessionEvent::Disconnected),+ Event::LoggedOut(_) => Some(SessionEvent::LoggedOut),+ Event::StreamReplaced(_) => Some(SessionEvent::Replaced),+ _ => None,+ }+}++fn person(jid: &Jid) -> Option<String> {+ (!jid.is_empty()).then(|| jid.to_non_ad().to_string())+}++fn person_text(jid: &str) -> Option<String> {+ jid.parse::<Jid>().ok().and_then(|jid| person(&jid))+}++fn event_id(account: Uuid, parts: &[&str]) -> Uuid {+ let mut name = account.to_string();+ for part in parts {+ name.push('\n');+ name.push_str(part);+ }+ Uuid::new_v5(&EVENT_NAMESPACE, name.as_bytes())+}++/// The author of the message a key points at: the reacting or quoting+/// person themselves when `from_me`, otherwise the key's participant.+fn target(key: &MessageKey, author: &str) -> Option<Target> {+ let sender_jid = if key.from_me.unwrap_or(false) {+ author.to_owned()+ } else {+ person_text(key.participant.as_deref()?)?+ };+ Some(Target {+ source_id: key.id.clone()?,+ sender_jid,+ })+}++/// whatsmeow delivers a sender key distribution (sent with a sender's first+/// group message and on every key rotation) as its own event with the same+/// message id as the content. It is not content; importing it would claim+/// that id and drop the real message.+fn carries_only_keys(message: &wa_e2e::Message) -> bool {+ let content = wa_e2e::Message {+ sender_key_distribution_message: None,+ message_context_info: None,+ ..message.clone()+ };+ content == wa_e2e::Message::default()+}++fn text_of(message: &wa_e2e::Message) -> Option<(String, Option<&wa_e2e::ContextInfo>)> {+ if let Some(text) = &message.conversation {+ return Some((text.clone(), None));+ }+ let extended = message.extended_text_message.as_ref()?;+ Some((+ extended.text.clone().unwrap_or_default(),+ extended.context_info.as_deref(),+ ))+}++/// One attachment part of a message, with its caption.+struct Attachment<'part> {+ part: &'part (dyn DownloadableMessage + Sync),+ media_type: LocatorMediaType,+ mime: Option<&'part str>,+ filename: Option<&'part str>,+ size: Option<u64>,+ caption: Option<&'part str>,+}++fn attachment_of(message: &wa_e2e::Message) -> Option<Attachment<'_>> {+ if let Some(image) = &message.image_message {+ return Some(Attachment {+ part: image.as_ref(),+ media_type: LocatorMediaType::Image,+ mime: image.mimetype.as_deref(),+ filename: None,+ size: image.file_length,+ caption: image.caption.as_deref(),+ });+ }+ if let Some(video) = &message.video_message {+ return Some(Attachment {+ part: video.as_ref(),+ media_type: LocatorMediaType::Video,+ mime: video.mimetype.as_deref(),+ filename: None,+ size: video.file_length,+ caption: video.caption.as_deref(),+ });+ }+ if let Some(audio) = &message.audio_message {+ return Some(Attachment {+ part: audio.as_ref(),+ media_type: LocatorMediaType::Audio,+ mime: audio.mimetype.as_deref(),+ filename: None,+ size: audio.file_length,+ caption: None,+ });+ }+ let document = message.document_message.as_ref()?;+ Some(Attachment {+ part: document.as_ref(),+ media_type: LocatorMediaType::Document,+ mime: document.mimetype.as_deref(),+ filename: document.file_name.as_deref(),+ size: document.file_length,+ caption: document.caption.as_deref(),+ })+}++/// The domain's attachment for a media part, or `None` when it lacks what a+/// download needs.+fn media_of(+ attachment: &Attachment<'_>,+ seal: &dyn Fn(&MediaLocator) -> Option<Vec<u8>>,+) -> Option<Media> {+ let part = attachment.part;+ let sha256: [u8; 32] = part.get_file_sha256()?.try_into().ok()?;+ let mime = attachment+ .mime+ .unwrap_or("application/octet-stream")+ .split(';')+ .next()+ .unwrap_or_default()+ .trim()+ .to_owned();+ let filename = match attachment.filename {+ Some(name) if !name.is_empty() => name.to_owned(),+ _ => {+ let extension = mime.rsplit('/').next().unwrap_or("bin");+ let stem = match attachment.media_type {+ LocatorMediaType::Image => "image",+ LocatorMediaType::Video => "video",+ LocatorMediaType::Audio => "audio",+ LocatorMediaType::Document => "document",+ };+ format!("{stem}.{extension}")+ }+ };+ let locator = seal(&MediaLocator {+ direct_path: part.get_direct_path().to_owned(),+ media_key: part.get_media_key()?.to_vec(),+ file_enc_sha256: part.get_file_enc_sha256()?.to_vec(),+ file_sha256: sha256.to_vec(),+ media_type: attachment.media_type,+ })?;+ Some(Media {+ mime,+ filename,+ size: attachment.size?,+ sha256,+ locator,+ })+}++fn map_message(+ account: Uuid,+ event: &whatsmeow::types::events::Message,+ now: DateTime<Utc>,+ seal: &dyn Fn(&MediaLocator) -> Option<Vec<u8>>,+) -> Option<InboundEvent> {+ let info = &event.info;+ if info.chat.server != GROUP_SERVER {+ return None;+ }+ let author = person(&info.sender)?;+ let sender = Identity {+ jid: author.clone(),+ alternate_jid: person(&info.sender_alt),+ display_name: (!info.push_name.is_empty()).then(|| info.push_name.clone()),+ };+ let source_at = info.timestamp.unwrap_or(now);+ let message = event.message.as_ref()?;+ if carries_only_keys(message) {+ return None;+ }+ let mut media = Vec::new();+ let (kind, text, target, revision) = if let Some(protocol) = &message.protocol_message {+ let key = protocol.key.as_ref()?;+ let edit_target = Target {+ source_id: key.id.clone()?,+ sender_jid: author.clone(),+ };+ match protocol+ .r#type+ .and_then(|value| protocol_message::Type::try_from(value).ok())+ {+ Some(protocol_message::Type::Revoke) => {+ (MessageKind::Delete, String::new(), Some(edit_target), 0)+ }+ Some(protocol_message::Type::MessageEdit) => {+ let (text, _) = protocol.edited_message.as_deref().and_then(text_of)?;+ let revision = protocol+ .timestamp_ms+ .unwrap_or_else(|| source_at.timestamp_millis());+ (MessageKind::Edit, text, Some(edit_target), revision)+ }+ // Key shares, history notifications and the like are not group content.+ _ => return None,+ }+ } else if let Some(reaction) = &message.reaction_message {+ let target = target(reaction.key.as_ref()?, &author)?;+ let revision = reaction.sender_timestamp_ms.unwrap_or(0);+ (+ MessageKind::Reaction,+ reaction.text.clone().unwrap_or_default(),+ Some(target),+ revision,+ )+ } else if let Some(attachment) = attachment_of(message) {+ match media_of(&attachment, seal) {+ Some(part) => {+ media.push(part);+ (+ MessageKind::Text,+ attachment.caption.unwrap_or_default().to_owned(),+ None,+ 0,+ )+ }+ None => (+ MessageKind::Unsupported,+ UNSUPPORTED_TEXT.to_owned(),+ None,+ 0,+ ),+ }+ } else if let Some((text, context)) = text_of(message) {+ let quote = context.and_then(|context| {+ Some(Target {+ source_id: context.stanza_id.clone()?,+ sender_jid: person_text(context.participant.as_deref()?)?,+ })+ });+ (MessageKind::Text, text, quote, 0)+ } else {+ (+ MessageKind::Unsupported,+ UNSUPPORTED_TEXT.to_owned(),+ None,+ 0,+ )+ };+ let kind_name = match kind {+ MessageKind::Text => "text",+ MessageKind::Reaction => "reaction",+ MessageKind::Edit => "edit",+ MessageKind::Delete => "delete",+ MessageKind::Unsupported => "unsupported",+ };+ let revision_text = revision.to_string();+ Some(InboundEvent {+ event_id: event_id(+ account,+ &[+ "message",+ &info.chat.to_string(),+ &author,+ info.id.as_str(),+ kind_name,+ &revision_text,+ ],+ ),+ group_jid: info.chat.to_non_ad().to_string(),+ source_at,+ observed_at: now,+ kind: InboundKind::Message(Message {+ source_id: info.id.to_string(),+ sender,+ kind,+ text,+ history: false,+ revision,+ target,+ media,+ }),+ })+}++fn map_receipt(+ account: Uuid,+ receipt: &whatsmeow::types::events::Receipt,+ now: DateTime<Utc>,+) -> Option<InboundEvent> {+ let source = &receipt.source;+ if source.chat.server != GROUP_SERVER || receipt.message_ids.is_empty() {+ return None;+ }+ let participant = person(&source.sender)?;+ let state = match receipt.r#type.as_str() {+ "" => "delivered",+ other => other,+ };+ let source_ids: Vec<String> = receipt+ .message_ids+ .iter()+ .map(ToString::to_string)+ .collect();+ let joined = source_ids.join(",");+ Some(InboundEvent {+ event_id: event_id(+ account,+ &[+ "receipt",+ &source.chat.to_string(),+ &participant,+ state,+ &joined,+ ],+ ),+ group_jid: source.chat.to_non_ad().to_string(),+ source_at: receipt.timestamp.unwrap_or(now),+ observed_at: now,+ kind: InboundKind::Receipt(Receipt {+ source_ids,+ participant_jid: participant,+ state: state.to_owned(),+ }),+ })+}diff --git a/services/whatsapp_session_service/src/outbound/whatsapp/events/test.rs b/services/whatsapp_session_service/src/outbound/whatsapp/events/test.rsnew file mode 100644index 0000000000..ae0b16cf6c--- /dev/null+++ b/services/whatsapp_session_service/src/outbound/whatsapp/events/test.rs@@ -0,0 +1,396 @@+use chrono::{DateTime, TimeZone, Utc};+use uuid::Uuid;+use whatsapp_linked::domain::events::{InboundKind, MessageKind, Target};+use whatsmeow::Event;+use whatsmeow::proto::wa_common::MessageKey;+use whatsmeow::proto::wa_e2e;+use whatsmeow::proto::wa_e2e::protocol_message;+use whatsmeow::types::events::{Message, Receipt};+use whatsmeow::types::{MessageInfo, MessageSource};++use super::{LocatorMediaType, Mapped, MediaLocator, map_event};+use crate::domain::ports::SessionEvent;++const ACCOUNT: Uuid = Uuid::from_u128(7);++fn now() -> DateTime<Utc> {+ Utc.with_ymd_and_hms(2026, 10, 21, 12, 0, 0).unwrap()+}++fn group_message(id: &str, message: wa_e2e::Message) -> Event {+ Event::Message(Box::new(Message {+ info: MessageInfo {+ source: MessageSource {+ chat: "[email protected]".parse().unwrap(),+ sender: "15550001111:[email protected]".parse().unwrap(),+ sender_alt: "987654321@lid".parse().unwrap(),+ is_group: true,+ ..MessageSource::default()+ },+ id: id.to_owned(),+ push_name: "Ada".to_owned(),+ timestamp: Some(Utc.with_ymd_and_hms(2026, 10, 21, 11, 59, 0).unwrap()),+ ..MessageInfo::default()+ },+ message: Some(message),+ ..Message::default()+ }))+}++/// Tests keep the locator readable; the adapter seals it with the account key.+fn plain_locator(locator: &MediaLocator) -> Option<Vec<u8>> {+ serde_json::to_vec(locator).ok()+}++fn inbound(event: Option<Mapped>) -> whatsapp_linked::domain::events::InboundEvent {+ match event {+ Some(Mapped::Inbound(event)) => *event,+ other => panic!("expected an inbound event, got {other:?}"),+ }+}++#[test]+fn a_group_text_becomes_a_message_from_its_author_with_a_stable_id() {+ let event = group_message(+ "3EB0AAAA",+ wa_e2e::Message {+ conversation: Some("hello".to_owned()),+ ..Default::default()+ },+ );++ let first = inbound(map_event(ACCOUNT, &event, now(), &plain_locator));+ let again = inbound(map_event(ACCOUNT, &event, now(), &plain_locator));++ assert_eq!(+ first.event_id, again.event_id,+ "redelivery keeps the event id"+ );+ assert_eq!(first.group_jid, "[email protected]");+ let InboundKind::Message(message) = first.kind else {+ panic!("expected a message");+ };+ assert_eq!(message.source_id, "3EB0AAAA");+ assert_eq!(message.sender.jid, "[email protected]");+ assert_eq!(+ message.sender.alternate_jid.as_deref(),+ Some("987654321@lid")+ );+ assert_eq!(message.sender.display_name.as_deref(), Some("Ada"));+ assert_eq!(message.kind, MessageKind::Text);+ assert_eq!(message.text, "hello");+ assert_eq!(message.target, None);+}++#[test]+fn a_quote_reaction_edit_and_delete_point_at_their_target() {+ let quote = inbound(map_event(+ ACCOUNT,+ &group_message(+ "3EB0B",+ wa_e2e::Message {+ extended_text_message: Some(Box::new(wa_e2e::ExtendedTextMessage {+ text: Some("agreed".to_owned()),+ context_info: Some(Box::new(wa_e2e::ContextInfo {+ stanza_id: Some("3EB0A".to_owned()),+ participant: Some("[email protected]".to_owned()),+ ..Default::default()+ })),+ ..Default::default()+ })),+ ..Default::default()+ },+ ),+ now(),+ &plain_locator,+ ));+ let reaction = inbound(map_event(+ ACCOUNT,+ &group_message(+ "3EB0C",+ wa_e2e::Message {+ reaction_message: Some(wa_e2e::ReactionMessage {+ key: Some(MessageKey {+ from_me: Some(true),+ id: Some("3EB0B".to_owned()),+ ..Default::default()+ }),+ text: Some("👍".to_owned()),+ sender_timestamp_ms: Some(42),+ ..Default::default()+ }),+ ..Default::default()+ },+ ),+ now(),+ &plain_locator,+ ));+ let edit = inbound(map_event(+ ACCOUNT,+ &group_message(+ "3EB0D",+ wa_e2e::Message {+ protocol_message: Some(Box::new(wa_e2e::ProtocolMessage {+ key: Some(MessageKey {+ id: Some("3EB0B".to_owned()),+ ..Default::default()+ }),+ r#type: Some(protocol_message::Type::MessageEdit as i32),+ edited_message: Some(Box::new(wa_e2e::Message {+ conversation: Some("agreed!".to_owned()),+ ..Default::default()+ })),+ timestamp_ms: Some(1_000),+ ..Default::default()+ })),+ ..Default::default()+ },+ ),+ now(),+ &plain_locator,+ ));+ let delete = inbound(map_event(+ ACCOUNT,+ &group_message(+ "3EB0E",+ wa_e2e::Message {+ protocol_message: Some(Box::new(wa_e2e::ProtocolMessage {+ key: Some(MessageKey {+ id: Some("3EB0B".to_owned()),+ ..Default::default()+ }),+ r#type: Some(protocol_message::Type::Revoke as i32),+ ..Default::default()+ })),+ ..Default::default()+ },+ ),+ now(),+ &plain_locator,+ ));++ let own_message = Target {+ source_id: "3EB0B".to_owned(),+ sender_jid: "[email protected]".to_owned(),+ };+ let [quote, reaction, edit, delete] =+ [quote, reaction, edit, delete].map(|event| match event.kind {+ InboundKind::Message(message) => message,+ other => panic!("expected a message, got {other:?}"),+ });+ assert_eq!(quote.kind, MessageKind::Text);+ assert_eq!(+ quote.target,+ Some(Target {+ source_id: "3EB0A".to_owned(),+ sender_jid: "[email protected]".to_owned(),+ })+ );+ assert_eq!(reaction.kind, MessageKind::Reaction);+ assert_eq!(reaction.text, "👍");+ assert_eq!(reaction.target.as_ref(), Some(&own_message));+ assert_eq!(edit.kind, MessageKind::Edit);+ assert_eq!(edit.text, "agreed!");+ assert_eq!(edit.revision, 1_000);+ assert_eq!(edit.target.as_ref(), Some(&own_message));+ assert_eq!(delete.kind, MessageKind::Delete);+ assert_eq!(delete.target.as_ref(), Some(&own_message));+}++#[test]+fn media_becomes_a_placeholder_and_direct_chats_are_ignored() {+ let media = inbound(map_event(+ ACCOUNT,+ &group_message(+ "3EB0F",+ wa_e2e::Message {+ image_message: Some(Box::default()),+ ..Default::default()+ },+ ),+ now(),+ &plain_locator,+ ));+ let InboundKind::Message(media) = media.kind else {+ panic!("expected a message");+ };+ assert_eq!(media.kind, MessageKind::Unsupported);++ let direct = Event::Message(Box::new(Message {+ info: MessageInfo {+ source: MessageSource {+ chat: "[email protected]".parse().unwrap(),+ sender: "[email protected]".parse().unwrap(),+ ..MessageSource::default()+ },+ id: "3EB0G".to_owned(),+ ..MessageInfo::default()+ },+ message: Some(wa_e2e::Message {+ conversation: Some("private".to_owned()),+ ..Default::default()+ }),+ ..Message::default()+ }));+ assert!(map_event(ACCOUNT, &direct, now(), &plain_locator).is_none());+}++#[test]+fn a_group_read_receipt_names_the_participant_and_messages() {+ let event = Event::Receipt(Receipt {+ source: MessageSource {+ chat: "[email protected]".parse().unwrap(),+ sender: "15552223333:[email protected]".parse().unwrap(),+ is_group: true,+ ..MessageSource::default()+ },+ message_ids: vec!["3EB0A".to_owned(), "3EB0B".to_owned()],+ r#type: whatsmeow::types::ReceiptType::Read,+ ..Receipt::default()+ });++ let InboundKind::Receipt(receipt) =+ inbound(map_event(ACCOUNT, &event, now(), &plain_locator)).kind+ else {+ panic!("expected a receipt");+ };+ assert_eq!(receipt.participant_jid, "[email protected]");+ assert_eq!(receipt.source_ids, vec!["3EB0A", "3EB0B"]);+ assert_eq!(receipt.state, "read");+}++#[test]+fn a_sender_key_distribution_alone_is_not_a_message() {+ let keys_only = group_message(+ "3EB0SAME",+ wa_e2e::Message {+ sender_key_distribution_message: Some(wa_e2e::SenderKeyDistributionMessage {+ group_id: Some("[email protected]".to_owned()),+ axolotl_sender_key_distribution_message: Some(vec![1, 2, 3]),+ }),+ message_context_info: Some(wa_e2e::MessageContextInfo::default()),+ ..Default::default()+ },+ );+ let content = group_message(+ "3EB0SAME",+ wa_e2e::Message {+ conversation: Some("first message".to_owned()),+ sender_key_distribution_message: Some(wa_e2e::SenderKeyDistributionMessage::default()),+ ..Default::default()+ },+ );++ assert!(map_event(ACCOUNT, &keys_only, now(), &plain_locator).is_none());+ let InboundKind::Message(message) =+ inbound(map_event(ACCOUNT, &content, now(), &plain_locator)).kind+ else {+ panic!("expected a message");+ };+ assert_eq!(message.kind, MessageKind::Text);+ assert_eq!(message.text, "first message");+}++#[test]+fn an_image_becomes_an_attachment_with_its_caption_and_locator() {+ let event = group_message(+ "3EB0IMG",+ wa_e2e::Message {+ image_message: Some(Box::new(wa_e2e::ImageMessage {+ direct_path: Some("/v/t62/abc".to_owned()),+ media_key: Some(vec![1; 32]),+ file_enc_sha256: Some(vec![2; 32]),+ file_sha256: Some(vec![3; 32]),+ file_length: Some(1234),+ mimetype: Some("image/jpeg".to_owned()),+ caption: Some("look".to_owned()),+ ..Default::default()+ })),+ ..Default::default()+ },+ );++ let InboundKind::Message(message) =+ inbound(map_event(ACCOUNT, &event, now(), &plain_locator)).kind+ else {+ panic!("expected a message");+ };+ assert_eq!(message.kind, MessageKind::Text);+ assert_eq!(message.text, "look");+ assert_eq!(message.media.len(), 1);+ let media = &message.media[0];+ assert_eq!(media.mime, "image/jpeg");+ assert_eq!(media.filename, "image.jpeg");+ assert_eq!(media.size, 1234);+ assert_eq!(media.sha256, [3; 32]);+ let locator: MediaLocator = serde_json::from_slice(&media.locator).unwrap();+ assert_eq!(+ locator,+ MediaLocator {+ direct_path: "/v/t62/abc".to_owned(),+ media_key: vec![1; 32],+ file_enc_sha256: vec![2; 32],+ file_sha256: vec![3; 32],+ media_type: LocatorMediaType::Image,+ }+ );+}++#[test]+fn an_attachment_that_cannot_be_sealed_becomes_a_placeholder() {+ let event = group_message(+ "3EB0DOC",+ wa_e2e::Message {+ document_message: Some(Box::new(wa_e2e::DocumentMessage {+ direct_path: Some("/v/t62/doc".to_owned()),+ media_key: Some(vec![1; 32]),+ file_enc_sha256: Some(vec![2; 32]),+ file_sha256: Some(vec![3; 32]),+ file_length: Some(10),+ mimetype: Some("application/pdf".to_owned()),+ file_name: Some("plan.pdf".to_owned()),+ ..Default::default()+ })),+ ..Default::default()+ },+ );++ let InboundKind::Message(message) = inbound(map_event(ACCOUNT, &event, now(), &|_| None)).kind+ else {+ panic!("expected a message");+ };+ assert_eq!(message.kind, MessageKind::Unsupported);+ assert!(message.media.is_empty());+}++#[test]+fn passkey_requests_carry_the_challenge_and_handoff_confirmations_their_code() {+ let request =+ Event::PairPasskeyRequest(Box::new(whatsmeow::types::events::PairPasskeyRequest {+ public_key: whatsmeow::types::WebAuthnPublicKey {+ relying_party_id: "whatsapp.com".to_owned(),+ ..Default::default()+ },+ }));+ let Some(Mapped::Session(SessionEvent::PasskeyRequested { challenge })) =+ map_event(ACCOUNT, &request, now(), &plain_locator)+ else {+ panic!("expected a passkey request");+ };+ assert_eq!(challenge["rpId"], "whatsapp.com");++ let shown = Event::PairPasskeyConfirmation(whatsmeow::types::events::PairPasskeyConfirmation {+ code: "123456".to_owned(),+ skip_handoff_ux: false,+ });+ assert!(matches!(+ map_event(ACCOUNT, &shown, now(), &plain_locator),+ Some(Mapped::Session(SessionEvent::PasskeyConfirmation { code })) if code == "123456"+ ));+ let automatic =+ Event::PairPasskeyConfirmation(whatsmeow::types::events::PairPasskeyConfirmation {+ code: "123456".to_owned(),+ skip_handoff_ux: true,+ });+ assert!(map_event(ACCOUNT, &automatic, now(), &plain_locator).is_none());+}diff --git a/services/whatsapp_session_service/src/outbound/whatsapp/test.rs b/services/whatsapp_session_service/src/outbound/whatsapp/test.rsnew file mode 100644index 0000000000..4063782897--- /dev/null+++ b/services/whatsapp_session_service/src/outbound/whatsapp/test.rs@@ -0,0 +1,396 @@+//! The real connector (encrypted Postgres store, whatsmeow client) paired+//! against the fake WhatsApp server by a fake phone scanning the QR code.++use std::sync::Arc;+use std::time::Duration;++use macro_db_migrator::MACRO_DB_MIGRATIONS;+use sha2::{Digest, Sha256};+use sqlx::PgPool;+use tokio::sync::mpsc;+use tokio_util::sync::CancellationToken;+use uuid::Uuid;+use whatsapp_fake_server::{FakeWhatsApp, test_signal};+use whatsapp_linked::domain::events::{+ AttachmentRef, ClaimedSend, InboundEvent, InboundKind, MediaImport, OutboundMedia,+ OutgoingContent, SessionScope,+};+use whatsapp_linked::domain::models::MediaKind;+use whatsapp_linked::domain::session::SessionService;+use whatsapp_linked::outbound::pg::PgLinkedRepository;+use whatsapp_store_pg::kms::FakeWhatsAppKms;++use super::WhatsmeowConnector;+use crate::domain::account::{AccountConfig, AccountSessions};+use crate::domain::ports::{+ AccountControl, AccountExit, AccountRunner, InboundSink, SessionConnector, SessionError,+ SessionEvent, WhatsAppSession,+};++/// Passes every session event on to the runner and copies it to the test.+struct Tee<Connector> {+ inner: Connector,+ copies: mpsc::UnboundedSender<SessionEvent>,+}++impl<Connector: SessionConnector> SessionConnector for Tee<Connector> {+ type Session = Connector::Session;++ async fn open(+ &self,+ scope: SessionScope,+ fresh_device: bool,+ inbound: Arc<dyn InboundSink>,+ ) -> Result<(Self::Session, mpsc::UnboundedReceiver<SessionEvent>), SessionError> {+ let (session, mut events) = self.inner.open(scope, fresh_device, inbound).await?;+ let (sender, receiver) = mpsc::unbounded_channel();+ let copies = self.copies.clone();+ tokio::spawn(async move {+ while let Some(event) = events.recv().await {+ let _ = copies.send(event.clone());+ if sender.send(event).is_err() {+ return;+ }+ }+ });+ Ok((session, receiver))+ }+}++async fn insert_pairing_account(pool: &PgPool) -> Uuid {+ let user_id = "macro|whatsapp-owner";+ sqlx::query!(+ r#"+ WITH new_macro_user AS (+ INSERT INTO macro_user (id, username, email, stripe_customer_id)+ VALUES (gen_random_uuid(), $2, $2, 'stripe_owner')+ RETURNING id+ )+ INSERT INTO "User" ("id", "email", "macro_user_id")+ SELECT $1, $2, new_macro_user.id FROM new_macro_user+ "#,+ user_id,+ "[email protected]",+ )+ .execute(pool)+ .await+ .unwrap();+ let account = Uuid::now_v7();+ sqlx::query!(+ "INSERT INTO whatsapp_linked_account (id, user_id, state) VALUES ($1, $2, 'pairing')",+ account,+ user_id,+ )+ .execute(pool)+ .await+ .unwrap();+ sqlx::query!(+ r#"INSERT INTO whatsapp_pairing (id, account_id, user_id, generation, state, expires_at)+ VALUES ($1, $2, $3, 1, 'waiting', now() + interval '5 minutes')"#,+ Uuid::now_v7(),+ account,+ user_id,+ )+ .execute(pool)+ .await+ .unwrap();+ account+}++async fn next_matching(+ copies: &mut mpsc::UnboundedReceiver<SessionEvent>,+ wanted: impl Fn(&SessionEvent) -> bool,+) -> SessionEvent {+ tokio::time::timeout(Duration::from_secs(20), async {+ loop {+ let event = copies.recv().await.expect("the session is running");+ if wanted(&event) {+ return event;+ }+ }+ })+ .await+ .expect("the fake server answers within the timeout")+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn a_phone_scanning_the_qr_links_the_account_through_the_encrypted_store(pool: PgPool) {+ let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();+ let whatsapp = FakeWhatsApp::start().await.unwrap();+ let phone = whatsapp.add_user("15550001111", "Ada").await;+ let account = insert_pairing_account(&pool).await;+ let (copies, mut copied) = mpsc::unbounded_channel();+ let runner = Arc::new(AccountSessions::new(+ SessionService::new(Arc::new(PgLinkedRepository::new(pool.clone()))),+ Tee {+ inner: WhatsmeowConnector::new(+ pool.clone(),+ FakeWhatsAppKms::new("arn:aws:kms:us-east-1:123456789012:key/test"),+ test_signal(),+ whatsapp.dialer(),+ Some(whatsapp.media_base_url()),+ Duration::from_secs(10),+ ),+ copies,+ },+ AccountConfig {+ send_timeout: Duration::from_secs(10),+ send_batch: 16,+ tick_every: Duration::from_secs(600),+ },+ ));+ let (_signals, signal_receiver) = mpsc::channel(8);+ let release = CancellationToken::new();+ let control = AccountControl {+ signals: signal_receiver,+ release: release.clone(),+ stop: CancellationToken::new(),+ };+ let running = tokio::spawn({+ let runner = Arc::clone(&runner);+ async move { runner.run(account, control).await }+ });++ let SessionEvent::Qr(code) =+ next_matching(&mut copied, |event| matches!(event, SessionEvent::Qr(_))).await+ else {+ unreachable!();+ };+ phone.scan_qr(&code).await.unwrap();+ let SessionEvent::PairSuccess { identity_jid } = next_matching(&mut copied, |event| {+ matches!(event, SessionEvent::PairSuccess { .. })+ })+ .await+ else {+ unreachable!();+ };+ assert_eq!(identity_jid, "[email protected]");+ next_matching(&mut copied, |event| {+ matches!(event, SessionEvent::Connected)+ })+ .await;++ release.cancel();+ assert_eq!(running.await.unwrap(), AccountExit::Released);+ let linked = sqlx::query!(+ "SELECT state, identity_jid FROM whatsapp_linked_account WHERE id = $1",+ account+ )+ .fetch_one(&pool)+ .await+ .unwrap();+ assert_eq!(linked.state, "connected");+ assert_eq!(+ linked.identity_jid.as_deref(),+ Some("[email protected]")+ );+ let stored_device = sqlx::query_scalar!(+ r#"SELECT count(*) AS "count!" FROM whatsapp_store_value WHERE account_id = $1"#,+ account+ )+ .fetch_one(&pool)+ .await+ .unwrap();+ assert!(+ stored_device > 0,+ "the paired device's keys are in Postgres"+ );++ // Pairing again after WhatsApp logged the device out starts from a fresh+ // device: a stored identity would make the QR channel refuse to start.+ sqlx::query!(+ "UPDATE whatsapp_linked_account SET state = 'logged_out' WHERE id = $1",+ account+ )+ .execute(&pool)+ .await+ .unwrap();+ sqlx::query!(+ r#"INSERT INTO whatsapp_pairing (id, account_id, user_id, generation, state, expires_at)+ VALUES ($1, $2, 'macro|whatsapp-owner', 1, 'waiting', now() + interval '5 minutes')"#,+ Uuid::now_v7(),+ account,+ )+ .execute(&pool)+ .await+ .unwrap();+ let (_signals, signal_receiver) = mpsc::channel(8);+ let release = CancellationToken::new();+ let control = AccountControl {+ signals: signal_receiver,+ release: release.clone(),+ stop: CancellationToken::new(),+ };+ let running = tokio::spawn({+ let runner = Arc::clone(&runner);+ async move { runner.run(account, control).await }+ });+ next_matching(&mut copied, |event| matches!(event, SessionEvent::Qr(_))).await;+ release.cancel();+ assert_eq!(running.await.unwrap(), AccountExit::Released);+}++/// Keeps every group event the client hands over, and acknowledges it.+struct RecordingSink(mpsc::UnboundedSender<InboundEvent>);++#[async_trait::async_trait]+impl InboundSink for RecordingSink {+ async fn accept(&self, event: &InboundEvent) -> bool {+ let _ = self.0.send(event.clone());+ true+ }+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn attachments_travel_both_ways_with_a_sealed_locator(pool: PgPool) {+ let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();+ let whatsapp = FakeWhatsApp::start().await.unwrap();+ let owner = whatsapp.add_user("15550001111", "Ada").await;+ let peer = whatsapp.add_user("15550002222", "Bob").await;+ let account = insert_pairing_account(&pool).await;+ let scope = SessionService::new(Arc::new(PgLinkedRepository::new(pool.clone())))+ .begin_lease(account)+ .await+ .unwrap()+ .scope;+ let connector = WhatsmeowConnector::new(+ pool.clone(),+ FakeWhatsAppKms::new("arn:aws:kms:us-east-1:123456789012:key/test"),+ test_signal(),+ whatsapp.dialer(),+ Some(whatsapp.media_base_url()),+ Duration::from_secs(10),+ );+ let (inbound, mut received) = mpsc::unbounded_channel();+ let (session, mut events) = connector+ .open(scope, false, Arc::new(RecordingSink(inbound)))+ .await+ .unwrap();+ session.start_pairing().await.unwrap();+ session.connect().await.unwrap();+ let SessionEvent::Qr(code) =+ next_matching(&mut events, |event| matches!(event, SessionEvent::Qr(_))).await+ else {+ unreachable!();+ };+ owner.scan_qr(&code).await.unwrap();+ next_matching(&mut events, |event| {+ matches!(event, SessionEvent::Connected)+ })+ .await;+ let group = whatsapp.create_group("Photos", &[&owner, &peer]);++ let photo = b"an encrypted photo from the group".repeat(40);+ let incoming = peer.send_image(&group, &photo, "look").await.unwrap();+ let event = tokio::time::timeout(Duration::from_secs(20), async {+ loop {+ let event = received.recv().await.unwrap();+ if let InboundKind::Message(message) = &event.kind+ && message.source_id == incoming+ {+ return message.clone();+ }+ }+ })+ .await+ .unwrap();+ assert_eq!(event.text, "look");+ let media = event.media.first().expect("the image is an attachment");+ assert_eq!(media.mime, "image/jpeg");+ assert!(+ !media+ .locator+ .windows(b"direct".len())+ .any(|window| window == b"direct"),+ "the locator is sealed, not readable JSON"+ );+ let import = MediaImport {+ attachment: AttachmentRef {+ binding_id: Uuid::now_v7(),+ person_id: Uuid::now_v7(),+ source_id: incoming.clone(),+ ordinal: 0,+ },+ mime: media.mime.clone(),+ filename: media.filename.clone(),+ size: media.size,+ sha256: media.sha256,+ locator: media.locator.clone(),+ };+ assert_eq!(session.download_media(&import).await.unwrap(), photo);++ let reply = b"a photo sent from Macro".repeat(30);+ let uploaded = session+ .upload_media(&OutboundMedia {+ kind: MediaKind::Image,+ mime: "image/jpeg".to_owned(),+ filename: "reply.jpg".to_owned(),+ bytes: reply.clone(),+ sha256: Sha256::digest(&reply).into(),+ })+ .await+ .unwrap();+ let send = ClaimedSend {+ id: Uuid::now_v7(),+ group_jid: group.to_string(),+ group_sequence: 1,+ content: OutgoingContent::Media {+ kind: MediaKind::Image,+ mime: "image/jpeg".to_owned(),+ filename: "reply.jpg".to_owned(),+ caption: Some("from macro".to_owned()),+ quote: None,+ },+ };+ let message_id = session.new_message_id().await;+ session+ .send(&send, &message_id, Some(&uploaded))+ .await+ .unwrap();+ let delivered = peer+ .wait_for_message(Duration::from_secs(20), |message| message.id == message_id)+ .await+ .expect("the peer's phone receives the image");+ assert_eq!(peer.download_image(&delivered).await.unwrap(), reply);+ session.disconnect().await;+}++/// IQ errors keep the whole response node, whose attributes carry JIDs; the+/// session error's message (what `%error` logs) must not repeat them.+#[test]+fn session_errors_never_show_the_jids_of_an_iq_response() {+ use whatsmeow::binary::{AttributeValue, Node, NodeContent};+ use whatsmeow::errors::{ClientError, IqError};++ let phone = "15551234567";+ let jid: whatsmeow::types::Jid = format!("{phone}@s.whatsapp.net").parse().unwrap();+ let response = Node {+ tag: "iq".to_owned(),+ attrs: [+ ("from".to_owned(), AttributeValue::from(&jid)),+ ("participant".to_owned(), AttributeValue::from(&jid)),+ ]+ .into(),+ content: Some(NodeContent::Nodes(vec![Node {+ tag: "item".to_owned(),+ attrs: [("jid".to_owned(), AttributeValue::from(&jid))].into(),+ content: None,+ }])),+ };+ let error = super::internal(ClientError::from(IqError {+ raw_node: Some(response.clone()),+ ..IqError::default()+ }));+ let unknown = super::internal(ClientError::from(IqError {+ error_node: Some(response),+ ..IqError::default()+ }));++ for message in [error.to_string(), unknown.to_string()] {+ assert!(+ message.starts_with("WhatsApp session failed: "),+ "{message}"+ );+ assert!(!message.contains(phone), "{message}");+ }+}diff --git a/services/whatsapp_session_service/src/replica.rs b/services/whatsapp_session_service/src/replica.rsnew file mode 100644index 0000000000..395a04eed8--- /dev/null+++ b/services/whatsapp_session_service/src/replica.rs@@ -0,0 +1,185 @@+//! One replica's composition: the lock and `LISTEN` connections, the+//! supervisor and the account runner. The binary and end-to-end tests both+//! start replicas through [`start`], injecting WhatsApp, Signal and KMS.++use std::sync::Arc;+use std::time::Duration;++use sqlx::PgPool;+use sqlx::postgres::PgConnectOptions;+use tokio::sync::mpsc;+use tokio::task::JoinHandle;+use tokio_util::sync::CancellationToken;+use tokio_util::task::TaskTracker;+use whatsapp_linked::domain::session::SessionService;+use whatsapp_linked::outbound::pg::PgLinkedRepository;+use whatsapp_store_pg::kms::WhatsAppKms;+use whatsmeow::{Dialer, Signal};++use crate::domain::account::{AccountConfig, AccountSessions};+use crate::domain::ports::LeaseError;+use crate::domain::supervisor::{Supervisor, SupervisorConfig, SupervisorError};+use crate::outbound::lease::{PgAccountDirectory, PgAccountLocks};+use crate::outbound::signals::{ListenError, SIGNAL_CAPACITY, SignalListener};+use crate::outbound::whatsapp::WhatsmeowConnector;++/// Every timing a replica uses.+#[derive(Debug, Clone, Copy)]+pub struct Timings {+ /// Ping interval of the lock connection; a dead one is noticed within it.+ pub lock_keepalive: Duration,+ /// A submitted send without an acknowledgement after this is unknown.+ pub send_timeout: Duration,+ /// How often to look for free accounts without a signal.+ pub rescan_every: Duration,+ /// How often an account looks for sends, deferred events and pairing+ /// expiry without a signal.+ pub tick_every: Duration,+ /// How long a released account may take to finish in-flight sends.+ pub release_timeout: Duration,+ /// How long a stopped account may take to return.+ pub stop_timeout: Duration,+ /// Longest the whole drain may take.+ pub drain_deadline: Duration,+ /// How often to report fleet health.+ pub health_every: Duration,+}++impl Timings {+ /// Production timings. The drain fits Fargate's 120 s stop timeout with+ /// room to flush telemetry.+ pub fn production() -> Self {+ let send_timeout = Duration::from_secs(30);+ Timings {+ lock_keepalive: Duration::from_secs(5),+ send_timeout,+ rescan_every: Duration::from_secs(30),+ tick_every: Duration::from_secs(30),+ release_timeout: send_timeout + Duration::from_secs(5),+ stop_timeout: Duration::from_secs(10),+ drain_deadline: Duration::from_secs(100),+ health_every: Duration::from_secs(60),+ }+ }+}++/// What a replica is built from.+pub struct ReplicaDeps<Kms> {+ /// Pooled MacroDB connections for reads and fenced writes.+ pub pool: PgPool,+ /// Direct (non-pooled) MacroDB connection options for the advisory locks+ /// and `LISTEN`.+ pub lock_options: PgConnectOptions,+ /// Wraps and unwraps account data keys.+ pub kms: Kms,+ /// The Signal implementation for every client.+ pub signal: Arc<dyn Signal>,+ /// Opens WhatsApp websockets.+ pub dialer: Arc<dyn Dialer>,+ /// Sends media transfers to one `scheme://host[:port]` instead of+ /// WhatsApp's media hosts (tests only).+ pub media_base_url: Option<String>,+ /// Most accounts this replica runs.+ pub max_accounts: u32,+ /// Accounts released per drain step.+ pub drain_batch: u32,+ /// Timings.+ pub timings: Timings,+}++/// Errors starting or running a replica.+#[derive(Debug, thiserror::Error)]+pub enum ReplicaError {+ /// Subscribing to signals failed.+ #[error(transparent)]+ Listen(#[from] ListenError),+ /// Opening the lock connection failed.+ #[error(transparent)]+ Lease(#[from] LeaseError),+ /// The supervisor stopped early.+ #[error(transparent)]+ Supervisor(#[from] SupervisorError),+ /// The supervisor task panicked.+ #[error("the replica task panicked")]+ Panicked,+}++/// A running replica.+pub struct Replica {+ lock_backend_pid: i32,+ running: JoinHandle<Result<(), SupervisorError>>,+}++impl Replica {+ /// The Postgres backend of the lock connection; terminating it takes+ /// every lock from this replica.+ pub fn lock_backend_pid(&self) -> i32 {+ self.lock_backend_pid+ }++ /// Waits until the replica drained after `shutdown`, or failed.+ pub async fn wait(self) -> Result<(), ReplicaError> {+ self.running.await.map_err(|_| ReplicaError::Panicked)??;+ Ok(())+ }+}++/// Connects the replica and starts supervising; drains when `shutdown` is+/// cancelled.+pub async fn start<Kms: WhatsAppKms + 'static>(+ deps: ReplicaDeps<Kms>,+ shutdown: CancellationToken,+) -> Result<Replica, ReplicaError> {+ let timings = deps.timings;+ let tracker = TaskTracker::new();+ let listener_stop = CancellationToken::new();+ let listener = SignalListener::connect(&deps.lock_options).await?;+ let (signal_sender, mut signals) = mpsc::channel(SIGNAL_CAPACITY);+ tracker.spawn(listener.run(signal_sender, listener_stop.clone()));+ let locks =+ PgAccountLocks::connect(&deps.lock_options, timings.lock_keepalive, &tracker).await?;+ let lock_backend_pid = locks.backend_pid();++ let sessions = SessionService::new(Arc::new(PgLinkedRepository::new(deps.pool.clone())));+ let runner = AccountSessions::new(+ sessions.clone(),+ WhatsmeowConnector::new(+ deps.pool.clone(),+ deps.kms,+ deps.signal,+ deps.dialer,+ deps.media_base_url,+ timings.send_timeout,+ ),+ AccountConfig {+ send_timeout: timings.send_timeout,+ send_batch: 16,+ tick_every: timings.tick_every,+ },+ );+ let supervisor = Supervisor::new(+ locks,+ PgAccountDirectory::new(deps.pool, sessions),+ Arc::new(runner),+ SupervisorConfig {+ max_accounts: deps.max_accounts,+ drain_batch: deps.drain_batch,+ rescan_every: timings.rescan_every,+ release_timeout: timings.release_timeout,+ stop_timeout: timings.stop_timeout,+ drain_deadline: timings.drain_deadline,+ health_every: timings.health_every,+ },+ );+ let running = tokio::spawn(async move {+ let result = supervisor.run(&mut signals, shutdown).await;+ listener_stop.cancel();+ tracker.close();+ tracker.wait().await;+ result+ });+ Ok(Replica {+ lock_backend_pid,+ running,+ })+}diff --git a/tooling/xtask/crates/xtask_local/src/local/inventory.rs b/tooling/xtask/crates/xtask_local/src/local/inventory.rsindex fe5d9e77f1..28ab6ca0bc 100644--- a/tooling/xtask/crates/xtask_local/src/local/inventory.rs+++ b/tooling/xtask/crates/xtask_local/src/local/inventory.rs@@ -290,6 +290,20 @@ pub const RUST_SERVICES: &[RustService] = &[opt_in: false,no_default_features: false,},+ RustService {+ compose_name: "whatsapp_session_service",+ cargo_bin: "whatsapp_session_service",+ package: "whatsapp_session_service",+ // Nothing calls it: it only dials out (WhatsApp, Postgres, KMS).+ host_port: None,+ path_prefix: None,+ is_websocket: false,+ // Local-only: under `run-dev` it would take leases on shared-dev+ // accounts from the deployed replicas.+ modes: &[Mode::Local],+ opt_in: false,+ no_default_features: false,+ },];/// The Rust services that participate in `mode` (opt-in services list no modes,diff --git a/tooling/xtask/crates/xtask_local/src/local/inventory/test.rs b/tooling/xtask/crates/xtask_local/src/local/inventory/test.rsindex 74c5704c92..e37d56208c 100644--- a/tooling/xtask/crates/xtask_local/src/local/inventory/test.rs+++ b/tooling/xtask/crates/xtask_local/src/local/inventory/test.rs@@ -3,7 +3,8 @@ use super::*;#[test]fn local_binaries_are_unique_and_complete() {let bins = local_binaries();- assert_eq!(bins.len(), 15, "{bins:?}");+ assert_eq!(bins.len(), 16, "{bins:?}");+ assert!(bins.contains(&"whatsapp_session_service"));assert!(bins.contains(&"pubsub_workers"));assert!(bins.contains(&"seed_cli"));assert!(bins.contains(&"document_upload_finalizer_local_worker"));diff --git a/tooling/xtask/crates/xtask_local/src/local/local_env.rs b/tooling/xtask/crates/xtask_local/src/local/local_env.rsindex e093735d43..bfd5bf1094 100644--- a/tooling/xtask/crates/xtask_local/src/local/local_env.rs+++ b/tooling/xtask/crates/xtask_local/src/local/local_env.rs@@ -245,6 +245,12 @@ impl InfraEnv {fn write(&self, env: &mut BTreeMap<String, String>) {env.insert("DATABASE_URL".into(), self.database_url.clone());env.insert("DATABASE_URL_READONLY".into(), self.database_url.clone());+ // The session service's advisory locks need a session that no pooler+ // multiplexes. Locally nothing pools, so it is the same database.+ env.insert(+ "WHATSAPP_SESSION_LOCK_DATABASE_URL".into(),+ self.database_url.clone(),+ );env.insert("REDIS_URI".into(), self.redis_uri.clone());env.insert("DOCUMENT_STORAGE_SERVICE_REDIS_URI".into(),@@ -384,6 +390,10 @@ impl InfraEnv {"CODEX_OAUTH_KMS_KEY_ID".into(),resources::CODEX_OAUTH_KMS_ALIAS.into(),);+ env.insert(+ "WHATSAPP_SESSION_KMS_KEY_ID".into(),+ resources::WHATSAPP_SESSION_KMS_ALIAS.into(),+ );// Dummy creds: the SDK talks to LocalStack, never real AWS.env.insert("AWS_ACCESS_KEY_ID".into(), "test".into());env.insert("AWS_SECRET_ACCESS_KEY".into(), "test".into());@@ -688,6 +698,9 @@ impl BootStubEnv {// The worker is opt-in and ships paused; an explicit env-file can enable it.env.insert("SLACK_IMPORT_ENABLED".into(), "false".into());env.insert("SLACK_IMPORT_CONCURRENCY".into(), "1".into());+ // whatsapp_session_service's per-replica account cap and drain step.+ env.insert("WHATSAPP_SESSION_MAX_ACCOUNTS".into(), "500".into());+ env.insert("WHATSAPP_SESSION_DRAIN_BATCH".into(), "20".into());// connection_gateway config reads `REDIS_HOST` (a Redis URL, not a// hostname — see `redis::Client::open`).env.insert("REDIS_HOST".into(), "redis://redis:6379".into());diff --git a/tooling/xtask/crates/xtask_local/src/local/local_env/test.rs b/tooling/xtask/crates/xtask_local/src/local/local_env/test.rsindex 42e225822e..75326796ce 100644--- a/tooling/xtask/crates/xtask_local/src/local/local_env/test.rs+++ b/tooling/xtask/crates/xtask_local/src/local/local_env/test.rs@@ -80,6 +80,10 @@ fn emits_required_keys() {"BASE_URL","DATABASE_URL","DATABASE_URL_READONLY",+ "WHATSAPP_SESSION_LOCK_DATABASE_URL",+ "WHATSAPP_SESSION_KMS_KEY_ID",+ "WHATSAPP_SESSION_MAX_ACCOUNTS",+ "WHATSAPP_SESSION_DRAIN_BATCH","REDIS_URI","OPENSEARCH_URL","LOCAL_AWS_URL",@@ -436,6 +440,22 @@ fn the_agent_harness_uses_local_containers_and_wipes_daytona() {);}+#[test]+fn the_whatsapp_session_service_locks_on_the_local_postgres_and_kms() {+ let env = local_env();+ assert_eq!(+ env.get("WHATSAPP_SESSION_LOCK_DATABASE_URL"),+ env.get("DATABASE_URL")+ );+ assert_eq!(+ env.get("WHATSAPP_SESSION_KMS_KEY_ID").map(String::as_str),+ Some(resources::WHATSAPP_SESSION_KMS_ALIAS)+ );+ // Tests point the client at a fake server; a local stack never does.+ assert!(!env.contains_key("OVERRIDE_WHATSAPP_WEBSOCKET_URL"));+ assert!(!env.contains_key("OVERRIDE_WHATSAPP_MEDIA_HOST"));+}+/// Sandboxes and the harness are both containers, so they reach each other on a/// shared Compose network and never over the host's loopback. The network name/// has to track the instance, or a named stack's sandboxes join the wrong one.diff --git a/tooling/xtask/crates/xtask_local/src/local/localstack.rs b/tooling/xtask/crates/xtask_local/src/local/localstack.rsindex b170af9c44..1b6b2c0d56 100644--- a/tooling/xtask/crates/xtask_local/src/local/localstack.rs+++ b/tooling/xtask/crates/xtask_local/src/local/localstack.rs@@ -84,6 +84,10 @@ async fn create_kms_keys(kms: &aws_sdk_kms::Client) -> Result<()> {resources::CODEX_OAUTH_KMS_ALIAS,"ChatGPT OAuth envelope encryption key (local)",),+ (+ resources::WHATSAPP_SESSION_KMS_ALIAS,+ "WhatsApp session data key encryption key (local)",+ ),] {if existing.aliases()diff --git a/tooling/xtask/crates/xtask_local/src/local/resources.rs b/tooling/xtask/crates/xtask_local/src/local/resources.rsindex 485d1a1e65..7ceecbe18d 100644--- a/tooling/xtask/crates/xtask_local/src/local/resources.rs+++ b/tooling/xtask/crates/xtask_local/src/local/resources.rs@@ -43,6 +43,9 @@ pub const CURSOR_API_KEY_KMS_ALIAS: &str = "alias/macro-local-cursor-api-key";/// Dedicated envelope-encryption key for local ChatGPT account connections.pub const CODEX_OAUTH_KMS_ALIAS: &str = "alias/macro-local-codex-oauth";+/// The WhatsApp session service's key for per-account data keys.+pub const WHATSAPP_SESSION_KMS_ALIAS: &str = "alias/macro-local-whatsapp-session";+/// The full LocalStack URL for `queue` (docker-network host — services run in/// containers and reach LocalStack by its compose alias).pub fn queue_url(queue: &str) -> String {