Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(connection_gateway): never wait on a websocket that stopped draining (#6385)

Wolf Mermelstein4ec7abf1dada
diff --git a/crates/connection_gateway_client/src/client.rs b/crates/connection_gateway_client/src/client.rs
index 39168fa475..1bc186e419 100644
--- a/crates/connection_gateway_client/src/client.rs
+++ b/crates/connection_gateway_client/src/client.rs
@@ -10,6 +10,18 @@ use model_entity::Entity;
mod calendar;
mod email;
+/// How long any one call to the connection gateway may take.
+///
+/// Delivery is best-effort for every caller - the gateway drops frames a
+/// consumer cannot take - so a caller gains nothing by waiting indefinitely,
+/// and some lose a great deal: an agent session's frame writer holds a
+/// one-minute budget per frame and tears the session down when it runs out.
+///
+/// Every endpoint here pushes one realtime message and returns; the slowest
+/// runs a p99 of 81ms in production. This is a backstop against the gateway
+/// itself stalling, not a budget anything is expected to spend.
+const REQUEST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(15);
+
/// HTTP client for communicating with the connection gateway service.
#[derive(Clone, Debug)]
pub struct ConnectionGatewayClient {
@@ -36,6 +48,7 @@ impl ConnectionGatewayClient {
let client = reqwest::Client::builder()
.default_headers(headers)
+ .timeout(REQUEST_TIMEOUT)
.build()
.unwrap();
diff --git a/services/connection_gateway/src/service/connection.rs b/services/connection_gateway/src/service/connection.rs
index 6cf49a7e01..904c34775b 100644
--- a/services/connection_gateway/src/service/connection.rs
+++ b/services/connection_gateway/src/service/connection.rs
@@ -1,4 +1,3 @@
-use crate::constants::SLOW_WEBSOCKET_OPERATION_THRESHOLD;
use crate::model::{
connection::{Connection, StoredConnectionEntity},
message::{Message, OutgoingMessage},
@@ -147,18 +146,23 @@ impl ConnectionManager {
/// IMPORTANT: this removes all entities associated with the connection_id
/// and not just a specific entity like document, chat, channel activity.
pub async fn remove_connection(&self, connection_id: &str) -> Result<()> {
- // if the connection exists, then we kill the task
- // that forwards the messages to the websocket
- if let Some(connection) = self.connections.get(connection_id) {
- // Only abort the task if it has not already been aborted
+ // Removing twice is the normal case, not an edge one: a refused send
+ // removes the connection and aborts its forwarder, and the socket
+ // handler removes it again when it sees that task end. Taking the
+ // entry out is the one step that cannot happen twice, so it is what
+ // the count and the abort hang off - `fetch_sub` on a count already
+ // at zero wraps to `usize::MAX`.
+ if let Some((_, connection)) = self.connections.remove(connection_id) {
+ self.connection_count
+ .fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
+ // Killing the task that forwards messages to the websocket.
if !connection.abort_handle.is_finished() {
connection.abort_handle.abort();
}
}
- self.connections.remove(connection_id);
- self.connection_count
- .fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
+ // Unconditional: a connection this instance never held still has rows
+ // to clear, and the delete is idempotent.
self.persistence
.remove_all_entries_for_by_connection_id(connection_id)
.await?;
@@ -200,9 +204,19 @@ impl ConnectionManager {
Ok(())
}
- /// Sends a message to a a connection
- /// If the connection is not found, or dropped, then we remove the connection id
- /// from the map, and should kill (TODO:) any remaining tasks associated with the connection
+ /// Offers a message to a connection's outbound queue, and drops the
+ /// connection if it will not take it.
+ ///
+ /// Never waits. The queue is this connection's alone but the callers are
+ /// everyone, so a consumer that has stopped draining must cost them
+ /// nothing: waiting for capacity here is indistinguishable from being
+ /// down to whoever is publishing, and a publisher holding its own
+ /// deadline fails the work behind the send rather than the one delivery.
+ ///
+ /// A full queue is not a slow consumer, it is a consumer already a whole
+ /// queue behind, so it is treated the same as a closed one: the
+ /// connection goes, which also aborts the forwarder that stopped
+ /// draining, and the client reconnects.
#[tracing::instrument(
name = "connection_gateway.queue_send",
err,
@@ -211,7 +225,6 @@ impl ConnectionManager {
connection.id = id,
connection.queue.observed_depth = tracing::field::Empty,
connection.queue.max_capacity = tracing::field::Empty,
- connection.queue.wait_ms = tracing::field::Empty,
)
)]
pub async fn send_message(&self, id: &str, message: Message) -> Result<()> {
@@ -227,25 +240,13 @@ impl ConnectionManager {
span.record("connection.queue.observed_depth", depth as u64);
span.record("connection.queue.max_capacity", max_capacity as u64);
- let started = tokio::time::Instant::now();
- let mut send = std::pin::pin!(sender.send(OutgoingMessage::Message(message)));
- let result = tokio::select! {
- result = &mut send => result,
- () = tokio::time::sleep(SLOW_WEBSOCKET_OPERATION_THRESHOLD) => {
- let depth = max_capacity.saturating_sub(sender.capacity());
- tracing::warn!(
- connection.id = id,
- connection.queue.observed_depth = depth,
- connection.queue.max_capacity = max_capacity,
- "websocket outbound queue send is blocked"
- );
- send.await
- }
- };
- let wait = started.elapsed();
- span.record("connection.queue.wait_ms", wait.as_millis() as u64);
-
- if let Err(err) = result {
+ if let Err(err) = sender.try_send(OutgoingMessage::Message(message)) {
+ tracing::warn!(
+ connection.id = id,
+ connection.queue.observed_depth = depth,
+ connection.queue.max_capacity = max_capacity,
+ "websocket outbound queue would not take a message; dropping the connection"
+ );
self.remove_connection(id).await?;
anyhow::bail!("failed to send message: {}", err);
}
diff --git a/services/connection_gateway/src/service/connection/test.rs b/services/connection_gateway/src/service/connection/test.rs
index 184167e8d9..66878e6b51 100644
--- a/services/connection_gateway/src/service/connection/test.rs
+++ b/services/connection_gateway/src/service/connection/test.rs
@@ -66,9 +66,39 @@ impl ConnectionRepo for UnusedRepo {
}
#[tokio::test(start_paused = true)]
-async fn a_saturated_connection_reports_slow_without_cancelling_the_send() {
+async fn a_connection_with_capacity_takes_the_message() {
let manager = ConnectionManager::new(UnusedRepo);
let (sender, mut receiver) = tokio::sync::mpsc::channel(1);
+ let forwarder = tokio::spawn(std::future::pending::<()>());
+ manager.connections.insert(
+ "connection".to_owned(),
+ Connection {
+ sender,
+ abort_handle: forwarder.abort_handle(),
+ },
+ );
+
+ manager
+ .send_message(
+ "connection",
+ Message::new("test".to_owned(), "first".to_owned()),
+ )
+ .await
+ .unwrap();
+
+ assert!(receiver.recv().await.is_some(), "the message must arrive");
+ assert!(
+ manager.connections.get("connection").is_some(),
+ "a healthy connection must be kept"
+ );
+}
+
+#[tokio::test(start_paused = true)]
+async fn a_saturated_connection_is_dropped_rather_than_waited_on() {
+ let manager = ConnectionManager::new(UnusedRepo);
+ // Held for the whole test: dropping it would close the channel, which is
+ // the already-handled case. This one stays open, full, and never read.
+ let (sender, _receiver) = tokio::sync::mpsc::channel(1);
sender
.send(OutgoingMessage::Message(Message::new(
"test".to_owned(),
@@ -85,21 +115,58 @@ async fn a_saturated_connection_reports_slow_without_cancelling_the_send() {
},
);
- let send = tokio::spawn({
- let manager = manager.clone();
- async move {
- manager
- .send_message(
- "connection",
- Message::new("test".to_owned(), "second".to_owned()),
- )
- .await
- }
- });
- tokio::task::yield_now().await;
- tokio::time::advance(SLOW_WEBSOCKET_OPERATION_THRESHOLD).await;
- assert!(!send.is_finished(), "telemetry must not cancel the send");
-
- receiver.recv().await.unwrap();
- send.await.unwrap().unwrap();
+ // Time only moves here when something awaits it, so a zero elapsed is
+ // what "the publisher never waits" looks like.
+ let started = tokio::time::Instant::now();
+ let result = manager
+ .send_message(
+ "connection",
+ Message::new("test".to_owned(), "second".to_owned()),
+ )
+ .await;
+
+ assert_eq!(
+ started.elapsed(),
+ std::time::Duration::ZERO,
+ "a publisher must never wait on a consumer"
+ );
+ assert!(result.is_err(), "a full queue must refuse the message");
+ assert!(
+ manager.connections.get("connection").is_none(),
+ "the saturated connection must be dropped"
+ );
+ assert!(
+ forwarder.await.unwrap_err().is_cancelled(),
+ "its forwarder must be aborted"
+ );
+}
+
+#[tokio::test(start_paused = true)]
+async fn removing_a_connection_twice_does_not_wrap_the_count() {
+ let manager = ConnectionManager::new(UnusedRepo);
+ let (sender, _receiver) = tokio::sync::mpsc::channel(1);
+ let forwarder = tokio::spawn(std::future::pending::<()>());
+ manager.connections.insert(
+ "connection".to_owned(),
+ Connection {
+ sender,
+ abort_handle: forwarder.abort_handle(),
+ },
+ );
+ manager
+ .connection_count
+ .store(1, std::sync::atomic::Ordering::SeqCst);
+
+ // A refused send removes it, then the socket handler removes it again
+ // when it notices the forwarder has ended.
+ manager.remove_connection("connection").await.unwrap();
+ manager.remove_connection("connection").await.unwrap();
+
+ assert_eq!(
+ manager
+ .connection_count
+ .load(std::sync::atomic::Ordering::SeqCst),
+ 0,
+ "only the removal that took the entry may decrement the count"
+ );
}
diff --git a/services/connection_gateway/src/service/sender.rs b/services/connection_gateway/src/service/sender.rs
index 9ec9f96573..51fcaa5ca5 100644
--- a/services/connection_gateway/src/service/sender.rs
+++ b/services/connection_gateway/src/service/sender.rs
@@ -7,7 +7,7 @@ use crate::model::message::Message;
use crate::model::sender::MessageReceipt;
use crate::service::redis::MessageWithConnection;
use anyhow::Result;
-use futures::future::try_join_all;
+use futures::future::join_all;
use model_entity::Entity;
use redis::aio::MultiplexedConnection;
use std::collections::HashMap;
@@ -146,12 +146,20 @@ async fn send_messages_to_connections(
}
});
- let local_results = try_join_all(local_send_futures).await.unwrap_or_default();
- let remote_results = try_join_all(remote_send_futures).await.unwrap_or_default();
+ // Independent per connection, so they are joined rather than tried: one
+ // connection refusing a message is now an ordinary outcome (a saturated
+ // queue is ejected), and it must not cancel delivery to the healthy
+ // connections beside it or discard the receipts they produced.
+ let local_results = join_all(local_send_futures).await;
+ let remote_results = join_all(remote_send_futures).await;
let mut receipts: HashMap<String, MessageReceipt> = HashMap::new();
- for delivery in local_results.into_iter().chain(remote_results.into_iter()) {
+ for delivery in local_results
+ .into_iter()
+ .chain(remote_results.into_iter())
+ .filter_map(std::result::Result::ok)
+ {
if let Some(receipt) = receipts.get_mut(&delivery.user_id) {
receipt.delivery_count += 1;
receipt.active = receipt.active || delivery.active;