Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(connection-gateway): stop paging on stalled clients the gateway drops on purpose (#9294)
teo44b408d01aa0
diff --git a/services/connection_gateway/src/service/connection.rs b/services/connection_gateway/src/service/connection.rsindex ff106d9d53..180d978223 100644--- a/services/connection_gateway/src/service/connection.rs+++ b/services/connection_gateway/src/service/connection.rs@@ -73,6 +73,15 @@ pub trait ConnectionRepo: Send + Sync {) -> anyhow::Result<()>;}+/// What became of a message offered to a connection's outbound queue.+#[derive(Debug, PartialEq, Eq)]+pub enum SendOutcome {+ /// The message is queued for the connection's forwarder to write.+ Queued,+ /// The connection would not take the message, so it was dropped.+ Dropped,+}+#[derive(Clone)]pub struct ConnectionManager {pub connections: Arc<DashMap<String, Connection>>,@@ -234,7 +243,9 @@ impl ConnectionManager {/// A full queue is not a slow consumer, it is a consumer already a whole/// queue behind, so it is treated the same as a closed one: the/// connection goes, which also aborts the forwarder that stopped- /// draining, and the client reconnects.+ /// draining, and the client reconnects. That is the designed way to shed+ /// a client that stopped reading, not a failed send, so it is reported as+ /// [SendOutcome::Dropped] rather than an error.#[tracing::instrument(name = "connection_gateway.queue_send",err,@@ -243,9 +254,10 @@ impl ConnectionManager {connection.id = id,connection.queue.observed_depth = tracing::field::Empty,connection.queue.max_capacity = tracing::field::Empty,+ connection.dropped = tracing::field::Empty,))]- pub async fn send_message(&self, id: &str, message: Message) -> Result<()> {+ pub async fn send_message(&self, id: &str, message: Message) -> Result<SendOutcome> {let sender = match self.connections.get(id) {Some(connection) => connection.sender.clone(),None => anyhow::bail!("connection not found"),@@ -259,17 +271,19 @@ impl ConnectionManager {span.record("connection.queue.max_capacity", max_capacity as u64);if let Err(err) = sender.try_send(OutgoingMessage::Message(message)) {+ span.record("connection.dropped", true);tracing::warn!(+ error = %err,connection.id = id,connection.queue.observed_depth = depth,connection.queue.max_capacity = max_capacity,"websocket outbound queue would not take a message; dropping the connection");self.remove_connection(id).await?;- anyhow::bail!("failed to send message: {}", err);+ return Ok(SendOutcome::Dropped);}- Ok(())+ Ok(SendOutcome::Queued)}/// Send a raw binary frame to a connection held by this instance.diff --git a/services/connection_gateway/src/service/connection/test.rs b/services/connection_gateway/src/service/connection/test.rsindex 8cc1d71d16..cf1f34b888 100644--- a/services/connection_gateway/src/service/connection/test.rs+++ b/services/connection_gateway/src/service/connection/test.rs@@ -79,7 +79,7 @@ async fn a_connection_with_capacity_takes_the_message() {},);- manager+ let outcome = manager.send_message("connection",Message::new("test".to_owned(), "first".to_owned()),@@ -87,6 +87,7 @@ async fn a_connection_with_capacity_takes_the_message() {.await.unwrap();+ assert_eq!(outcome, SendOutcome::Queued);assert!(receiver.recv().await.is_some(), "the message must arrive");assert!(manager.connections.get("connection").is_some(),@@ -131,7 +132,11 @@ async fn a_saturated_connection_is_dropped_rather_than_waited_on() {std::time::Duration::ZERO,"a publisher must never wait on a consumer");- assert!(result.is_err(), "a full queue must refuse the message");+ assert_eq!(+ result.unwrap(),+ SendOutcome::Dropped,+ "a full queue must refuse the message without failing the send"+ );assert!(manager.connections.get("connection").is_none(),"the saturated connection must be dropped"diff --git a/services/connection_gateway/src/service/sender.rs b/services/connection_gateway/src/service/sender.rsindex 341de7ec0c..3544b42fb4 100644--- a/services/connection_gateway/src/service/sender.rs+++ b/services/connection_gateway/src/service/sender.rs@@ -5,6 +5,7 @@ use crate::context::ApiContext;use crate::model::connection::StoredConnectionEntity;use crate::model::message::Message;use crate::model::sender::MessageReceipt;+use crate::service::connection::SendOutcome;use crate::service::redis::MessageWithConnection;use anyhow::Result;use futures::future::join_all;@@ -92,12 +93,13 @@ async fn send_messages_to_connections(async move {let instant = Instant::now();- let active = api_context- .connection_manager- .send_message(connection.connection_id.as_str(), message.clone())- .await- .is_ok()- && connection.is_active_in_threshold(Some(DEFAULT_TIMEOUT_THRESHOLD));+ let active = matches!(+ api_context+ .connection_manager+ .send_message(connection.connection_id.as_str(), message.clone())+ .await,+ Ok(SendOutcome::Queued)+ ) && connection.is_active_in_threshold(Some(DEFAULT_TIMEOUT_THRESHOLD));tracing::trace!("sent message to connection directly in {:?}",