Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(connection-gateway): stop paging on stalled clients the gateway drops on purpose (#9294)

teo44b408d01aa0
diff --git a/services/connection_gateway/src/service/connection.rs b/services/connection_gateway/src/service/connection.rs
index ff106d9d53..180d978223 100644
--- a/services/connection_gateway/src/service/connection.rs
+++ b/services/connection_gateway/src/service/connection.rs
@@ -73,6 +73,15 @@ pub trait ConnectionRepo: Send + Sync {
) -> anyhow::Result<()>;
}
+/// What became of a message offered to a connection's outbound queue.
+#[derive(Debug, PartialEq, Eq)]
+pub enum SendOutcome {
+ /// The message is queued for the connection's forwarder to write.
+ Queued,
+ /// The connection would not take the message, so it was dropped.
+ Dropped,
+}
+
#[derive(Clone)]
pub struct ConnectionManager {
pub connections: Arc<DashMap<String, Connection>>,
@@ -234,7 +243,9 @@ impl ConnectionManager {
/// A full queue is not a slow consumer, it is a consumer already a whole
/// queue behind, so it is treated the same as a closed one: the
/// connection goes, which also aborts the forwarder that stopped
- /// draining, and the client reconnects.
+ /// draining, and the client reconnects. That is the designed way to shed
+ /// a client that stopped reading, not a failed send, so it is reported as
+ /// [SendOutcome::Dropped] rather than an error.
#[tracing::instrument(
name = "connection_gateway.queue_send",
err,
@@ -243,9 +254,10 @@ impl ConnectionManager {
connection.id = id,
connection.queue.observed_depth = tracing::field::Empty,
connection.queue.max_capacity = tracing::field::Empty,
+ connection.dropped = tracing::field::Empty,
)
)]
- pub async fn send_message(&self, id: &str, message: Message) -> Result<()> {
+ pub async fn send_message(&self, id: &str, message: Message) -> Result<SendOutcome> {
let sender = match self.connections.get(id) {
Some(connection) => connection.sender.clone(),
None => anyhow::bail!("connection not found"),
@@ -259,17 +271,19 @@ impl ConnectionManager {
span.record("connection.queue.max_capacity", max_capacity as u64);
if let Err(err) = sender.try_send(OutgoingMessage::Message(message)) {
+ span.record("connection.dropped", true);
tracing::warn!(
+ error = %err,
connection.id = id,
connection.queue.observed_depth = depth,
connection.queue.max_capacity = max_capacity,
"websocket outbound queue would not take a message; dropping the connection"
);
self.remove_connection(id).await?;
- anyhow::bail!("failed to send message: {}", err);
+ return Ok(SendOutcome::Dropped);
}
- Ok(())
+ Ok(SendOutcome::Queued)
}
/// Send a raw binary frame to a connection held by this instance.
diff --git a/services/connection_gateway/src/service/connection/test.rs b/services/connection_gateway/src/service/connection/test.rs
index 8cc1d71d16..cf1f34b888 100644
--- a/services/connection_gateway/src/service/connection/test.rs
+++ b/services/connection_gateway/src/service/connection/test.rs
@@ -79,7 +79,7 @@ async fn a_connection_with_capacity_takes_the_message() {
},
);
- manager
+ let outcome = manager
.send_message(
"connection",
Message::new("test".to_owned(), "first".to_owned()),
@@ -87,6 +87,7 @@ async fn a_connection_with_capacity_takes_the_message() {
.await
.unwrap();
+ assert_eq!(outcome, SendOutcome::Queued);
assert!(receiver.recv().await.is_some(), "the message must arrive");
assert!(
manager.connections.get("connection").is_some(),
@@ -131,7 +132,11 @@ async fn a_saturated_connection_is_dropped_rather_than_waited_on() {
std::time::Duration::ZERO,
"a publisher must never wait on a consumer"
);
- assert!(result.is_err(), "a full queue must refuse the message");
+ assert_eq!(
+ result.unwrap(),
+ SendOutcome::Dropped,
+ "a full queue must refuse the message without failing the send"
+ );
assert!(
manager.connections.get("connection").is_none(),
"the saturated connection must be dropped"
diff --git a/services/connection_gateway/src/service/sender.rs b/services/connection_gateway/src/service/sender.rs
index 341de7ec0c..3544b42fb4 100644
--- a/services/connection_gateway/src/service/sender.rs
+++ b/services/connection_gateway/src/service/sender.rs
@@ -5,6 +5,7 @@ use crate::context::ApiContext;
use crate::model::connection::StoredConnectionEntity;
use crate::model::message::Message;
use crate::model::sender::MessageReceipt;
+use crate::service::connection::SendOutcome;
use crate::service::redis::MessageWithConnection;
use anyhow::Result;
use futures::future::join_all;
@@ -92,12 +93,13 @@ async fn send_messages_to_connections(
async move {
let instant = Instant::now();
- let active = api_context
- .connection_manager
- .send_message(connection.connection_id.as_str(), message.clone())
- .await
- .is_ok()
- && connection.is_active_in_threshold(Some(DEFAULT_TIMEOUT_THRESHOLD));
+ let active = matches!(
+ api_context
+ .connection_manager
+ .send_message(connection.connection_id.as_str(), message.clone())
+ .await,
+ Ok(SendOutcome::Queued)
+ ) && connection.is_active_in_threshold(Some(DEFAULT_TIMEOUT_THRESHOLD));
tracing::trace!(
"sent message to connection directly in {:?}",