Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(connection_gateway): never wait on a websocket that stopped draining (#6385)
Wolf Mermelstein4ec7abf1dada
diff --git a/crates/connection_gateway_client/src/client.rs b/crates/connection_gateway_client/src/client.rsindex 39168fa475..1bc186e419 100644--- a/crates/connection_gateway_client/src/client.rs+++ b/crates/connection_gateway_client/src/client.rs@@ -10,6 +10,18 @@ use model_entity::Entity;mod calendar;mod email;+/// How long any one call to the connection gateway may take.+///+/// Delivery is best-effort for every caller - the gateway drops frames a+/// consumer cannot take - so a caller gains nothing by waiting indefinitely,+/// and some lose a great deal: an agent session's frame writer holds a+/// one-minute budget per frame and tears the session down when it runs out.+///+/// Every endpoint here pushes one realtime message and returns; the slowest+/// runs a p99 of 81ms in production. This is a backstop against the gateway+/// itself stalling, not a budget anything is expected to spend.+const REQUEST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(15);+/// HTTP client for communicating with the connection gateway service.#[derive(Clone, Debug)]pub struct ConnectionGatewayClient {@@ -36,6 +48,7 @@ impl ConnectionGatewayClient {let client = reqwest::Client::builder().default_headers(headers)+ .timeout(REQUEST_TIMEOUT).build().unwrap();diff --git a/services/connection_gateway/src/service/connection.rs b/services/connection_gateway/src/service/connection.rsindex 6cf49a7e01..904c34775b 100644--- a/services/connection_gateway/src/service/connection.rs+++ b/services/connection_gateway/src/service/connection.rs@@ -1,4 +1,3 @@-use crate::constants::SLOW_WEBSOCKET_OPERATION_THRESHOLD;use crate::model::{connection::{Connection, StoredConnectionEntity},message::{Message, OutgoingMessage},@@ -147,18 +146,23 @@ impl ConnectionManager {/// IMPORTANT: this removes all entities associated with the connection_id/// and not just a specific entity like document, chat, channel activity.pub async fn remove_connection(&self, connection_id: &str) -> Result<()> {- // if the connection exists, then we kill the task- // that forwards the messages to the websocket- if let Some(connection) = self.connections.get(connection_id) {- // Only abort the task if it has not already been aborted+ // Removing twice is the normal case, not an edge one: a refused send+ // removes the connection and aborts its forwarder, and the socket+ // handler removes it again when it sees that task end. Taking the+ // entry out is the one step that cannot happen twice, so it is what+ // the count and the abort hang off - `fetch_sub` on a count already+ // at zero wraps to `usize::MAX`.+ if let Some((_, connection)) = self.connections.remove(connection_id) {+ self.connection_count+ .fetch_sub(1, std::sync::atomic::Ordering::SeqCst);+ // Killing the task that forwards messages to the websocket.if !connection.abort_handle.is_finished() {connection.abort_handle.abort();}}- self.connections.remove(connection_id);- self.connection_count- .fetch_sub(1, std::sync::atomic::Ordering::SeqCst);+ // Unconditional: a connection this instance never held still has rows+ // to clear, and the delete is idempotent.self.persistence.remove_all_entries_for_by_connection_id(connection_id).await?;@@ -200,9 +204,19 @@ impl ConnectionManager {Ok(())}- /// Sends a message to a a connection- /// If the connection is not found, or dropped, then we remove the connection id- /// from the map, and should kill (TODO:) any remaining tasks associated with the connection+ /// Offers a message to a connection's outbound queue, and drops the+ /// connection if it will not take it.+ ///+ /// Never waits. The queue is this connection's alone but the callers are+ /// everyone, so a consumer that has stopped draining must cost them+ /// nothing: waiting for capacity here is indistinguishable from being+ /// down to whoever is publishing, and a publisher holding its own+ /// deadline fails the work behind the send rather than the one delivery.+ ///+ /// A full queue is not a slow consumer, it is a consumer already a whole+ /// queue behind, so it is treated the same as a closed one: the+ /// connection goes, which also aborts the forwarder that stopped+ /// draining, and the client reconnects.#[tracing::instrument(name = "connection_gateway.queue_send",err,@@ -211,7 +225,6 @@ impl ConnectionManager {connection.id = id,connection.queue.observed_depth = tracing::field::Empty,connection.queue.max_capacity = tracing::field::Empty,- connection.queue.wait_ms = tracing::field::Empty,))]pub async fn send_message(&self, id: &str, message: Message) -> Result<()> {@@ -227,25 +240,13 @@ impl ConnectionManager {span.record("connection.queue.observed_depth", depth as u64);span.record("connection.queue.max_capacity", max_capacity as u64);- let started = tokio::time::Instant::now();- let mut send = std::pin::pin!(sender.send(OutgoingMessage::Message(message)));- let result = tokio::select! {- result = &mut send => result,- () = tokio::time::sleep(SLOW_WEBSOCKET_OPERATION_THRESHOLD) => {- let depth = max_capacity.saturating_sub(sender.capacity());- tracing::warn!(- connection.id = id,- connection.queue.observed_depth = depth,- connection.queue.max_capacity = max_capacity,- "websocket outbound queue send is blocked"- );- send.await- }- };- let wait = started.elapsed();- span.record("connection.queue.wait_ms", wait.as_millis() as u64);-- if let Err(err) = result {+ if let Err(err) = sender.try_send(OutgoingMessage::Message(message)) {+ tracing::warn!(+ connection.id = id,+ connection.queue.observed_depth = depth,+ connection.queue.max_capacity = max_capacity,+ "websocket outbound queue would not take a message; dropping the connection"+ );self.remove_connection(id).await?;anyhow::bail!("failed to send message: {}", err);}diff --git a/services/connection_gateway/src/service/connection/test.rs b/services/connection_gateway/src/service/connection/test.rsindex 184167e8d9..66878e6b51 100644--- a/services/connection_gateway/src/service/connection/test.rs+++ b/services/connection_gateway/src/service/connection/test.rs@@ -66,9 +66,39 @@ impl ConnectionRepo for UnusedRepo {}#[tokio::test(start_paused = true)]-async fn a_saturated_connection_reports_slow_without_cancelling_the_send() {+async fn a_connection_with_capacity_takes_the_message() {let manager = ConnectionManager::new(UnusedRepo);let (sender, mut receiver) = tokio::sync::mpsc::channel(1);+ let forwarder = tokio::spawn(std::future::pending::<()>());+ manager.connections.insert(+ "connection".to_owned(),+ Connection {+ sender,+ abort_handle: forwarder.abort_handle(),+ },+ );++ manager+ .send_message(+ "connection",+ Message::new("test".to_owned(), "first".to_owned()),+ )+ .await+ .unwrap();++ assert!(receiver.recv().await.is_some(), "the message must arrive");+ assert!(+ manager.connections.get("connection").is_some(),+ "a healthy connection must be kept"+ );+}++#[tokio::test(start_paused = true)]+async fn a_saturated_connection_is_dropped_rather_than_waited_on() {+ let manager = ConnectionManager::new(UnusedRepo);+ // Held for the whole test: dropping it would close the channel, which is+ // the already-handled case. This one stays open, full, and never read.+ let (sender, _receiver) = tokio::sync::mpsc::channel(1);sender.send(OutgoingMessage::Message(Message::new("test".to_owned(),@@ -85,21 +115,58 @@ async fn a_saturated_connection_reports_slow_without_cancelling_the_send() {},);- let send = tokio::spawn({- let manager = manager.clone();- async move {- manager- .send_message(- "connection",- Message::new("test".to_owned(), "second".to_owned()),- )- .await- }- });- tokio::task::yield_now().await;- tokio::time::advance(SLOW_WEBSOCKET_OPERATION_THRESHOLD).await;- assert!(!send.is_finished(), "telemetry must not cancel the send");-- receiver.recv().await.unwrap();- send.await.unwrap().unwrap();+ // Time only moves here when something awaits it, so a zero elapsed is+ // what "the publisher never waits" looks like.+ let started = tokio::time::Instant::now();+ let result = manager+ .send_message(+ "connection",+ Message::new("test".to_owned(), "second".to_owned()),+ )+ .await;++ assert_eq!(+ started.elapsed(),+ std::time::Duration::ZERO,+ "a publisher must never wait on a consumer"+ );+ assert!(result.is_err(), "a full queue must refuse the message");+ assert!(+ manager.connections.get("connection").is_none(),+ "the saturated connection must be dropped"+ );+ assert!(+ forwarder.await.unwrap_err().is_cancelled(),+ "its forwarder must be aborted"+ );+}++#[tokio::test(start_paused = true)]+async fn removing_a_connection_twice_does_not_wrap_the_count() {+ let manager = ConnectionManager::new(UnusedRepo);+ let (sender, _receiver) = tokio::sync::mpsc::channel(1);+ let forwarder = tokio::spawn(std::future::pending::<()>());+ manager.connections.insert(+ "connection".to_owned(),+ Connection {+ sender,+ abort_handle: forwarder.abort_handle(),+ },+ );+ manager+ .connection_count+ .store(1, std::sync::atomic::Ordering::SeqCst);++ // A refused send removes it, then the socket handler removes it again+ // when it notices the forwarder has ended.+ manager.remove_connection("connection").await.unwrap();+ manager.remove_connection("connection").await.unwrap();++ assert_eq!(+ manager+ .connection_count+ .load(std::sync::atomic::Ordering::SeqCst),+ 0,+ "only the removal that took the entry may decrement the count"+ );}diff --git a/services/connection_gateway/src/service/sender.rs b/services/connection_gateway/src/service/sender.rsindex 9ec9f96573..51fcaa5ca5 100644--- a/services/connection_gateway/src/service/sender.rs+++ b/services/connection_gateway/src/service/sender.rs@@ -7,7 +7,7 @@ use crate::model::message::Message;use crate::model::sender::MessageReceipt;use crate::service::redis::MessageWithConnection;use anyhow::Result;-use futures::future::try_join_all;+use futures::future::join_all;use model_entity::Entity;use redis::aio::MultiplexedConnection;use std::collections::HashMap;@@ -146,12 +146,20 @@ async fn send_messages_to_connections(}});- let local_results = try_join_all(local_send_futures).await.unwrap_or_default();- let remote_results = try_join_all(remote_send_futures).await.unwrap_or_default();+ // Independent per connection, so they are joined rather than tried: one+ // connection refusing a message is now an ordinary outcome (a saturated+ // queue is ejected), and it must not cancel delivery to the healthy+ // connections beside it or discard the receipts they produced.+ let local_results = join_all(local_send_futures).await;+ let remote_results = join_all(remote_send_futures).await;let mut receipts: HashMap<String, MessageReceipt> = HashMap::new();- for delivery in local_results.into_iter().chain(remote_results.into_iter()) {+ for delivery in local_results+ .into_iter()+ .chain(remote_results.into_iter())+ .filter_map(std::result::Result::ok)+ {if let Some(receipt) = receipts.get_mut(&delivery.user_id) {receipt.delivery_count += 1;receipt.active = receipt.active || delivery.active;