Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

fix(channels): release admitted messages in receive order

oxoxDev9ad75f6e4e1e
diff --git a/crates/openhuman-core/src/channels/runtime/README.md b/crates/openhuman-core/src/channels/runtime/README.md
index 7bef181b58..ea28f7197f 100644
--- a/crates/openhuman-core/src/channels/runtime/README.md
+++ b/crates/openhuman-core/src/channels/runtime/README.md
@@ -34,7 +34,9 @@ the host does not reconstruct provider-specific outbound behavior.
The dispatch loop admits each message first: preparation, the
`ChannelMessageReceived` event, runtime commands and approval replies run
-immediately. What remains goes through the per-conversation debouncer
+immediately. Admissions run concurrently, but their results enter the
+debouncer in receive order, so a slow admission never lets a later message
+overtake it. What remains goes through the per-conversation debouncer
([`inbound_debounce`](../../inbound_debounce/README.md)) keyed by
`history_key`. Messages from one sender within the quiet window
(`[inbound_debounce]`, 1.5 s by default, 4 s once the batch holds media,
diff --git a/crates/openhuman-core/src/channels/runtime/dispatch/processor.rs b/crates/openhuman-core/src/channels/runtime/dispatch/processor.rs
index ba5553d7f1..d4c0479ecf 100644
--- a/crates/openhuman-core/src/channels/runtime/dispatch/processor.rs
+++ b/crates/openhuman-core/src/channels/runtime/dispatch/processor.rs
@@ -20,12 +20,14 @@
mod admit;
mod approval;
mod bus_turn;
+mod in_order;
mod turn;
use super::inbound_batch::{merge_channel_batch, QueuedInbound};
use crate::channels::context::ChannelRuntimeContext;
use crate::config::InboundDebounceConfig;
use crate::inbound_debounce::{KeyedDebouncer, Push};
+use in_order::InOrder;
use std::collections::{HashMap, VecDeque};
use std::sync::Arc;
use tinychannels_bus::{ChannelInboundEnvelope, ChannelMessage};
@@ -105,6 +107,8 @@ pub(crate) async fn run_message_dispatch_loop(
let mut inbound_open = true;
let mut next_seq: u64 = 0;
let mut admitting = JoinSet::new();
+ let mut admit_seqs: HashMap<tokio::task::Id, u64> = HashMap::new();
+ let mut admitted = InOrder::new();
let mut turns = JoinSet::new();
let mut turn_keys: HashMap<tokio::task::Id, String> = HashMap::new();
let mut typing = JoinSet::new();
@@ -151,8 +155,8 @@ pub(crate) async fn run_message_dispatch_loop(
Some(message) => {
let seq = next_seq;
next_seq += 1;
- let ctx = Arc::clone(&ctx);
- admitting.spawn(async move { (seq, admit_runtime_message(ctx, message).await) });
+ let handle = admitting.spawn(admit_runtime_message(Arc::clone(&ctx), message));
+ admit_seqs.insert(handle.id(), seq);
}
None => {
tracing::debug!(
@@ -163,41 +167,19 @@ pub(crate) async fn run_message_dispatch_loop(
}
}
}
- Some(joined) = admitting.join_next(), if !admitting.is_empty() => {
- match joined {
- Ok((seq, Some(inbound))) => {
- let settings = debounce.settings_for(&inbound.msg.channel);
- let channel = inbound.msg.channel.clone();
- let reply_target = inbound.msg.reply_target.clone();
- let key = inbound.history_key.clone();
- let push = debouncer.push(
- key,
- QueuedInbound { seq, inbound },
- &settings,
- Instant::now(),
- );
- tracing::debug!(
- channel = %channel,
- seq,
- outcome = ?push,
- tracked = debouncer.tracked(),
- "[dispatch][debounce] buffered inbound message"
- );
- if push == Push::Opened {
- if let Some(target) = ctx.channels_by_name.get(&channel).cloned() {
- typing.spawn(async move {
- if let Err(error) = target.start_typing(&reply_target).await {
- tracing::debug!(
- "[dispatch][debounce] typing start failed on {}: {error}",
- target.name()
- );
- }
- });
- }
- }
+ Some(joined) = admitting.join_next_with_id(), if !admitting.is_empty() => {
+ let (id, result) = match joined {
+ Ok((id, result)) => (id, result),
+ Err(error) => {
+ let id = error.id();
+ log_join_error("admission", Some(error));
+ (id, None)
+ }
+ };
+ if let Some(seq) = admit_seqs.remove(&id) {
+ for (seq, inbound) in admitted.finish(seq, result) {
+ buffer_admitted(&ctx, &debounce, &mut debouncer, &mut typing, seq, inbound);
}
- Ok((_, None)) => {}
- Err(error) => log_join_error("admission", Some(error)),
}
}
Some(joined) = turns.join_next_with_id(), if !turns.is_empty() => {
@@ -224,6 +206,46 @@ pub(crate) async fn run_message_dispatch_loop(
}
}
+fn buffer_admitted(
+ ctx: &Arc<ChannelRuntimeContext>,
+ debounce: &InboundDebounceConfig,
+ debouncer: &mut KeyedDebouncer<String, QueuedInbound>,
+ typing: &mut JoinSet<()>,
+ seq: u64,
+ inbound: AdmittedInbound,
+) {
+ let settings = debounce.settings_for(&inbound.msg.channel);
+ let channel = inbound.msg.channel.clone();
+ let reply_target = inbound.msg.reply_target.clone();
+ let key = inbound.history_key.clone();
+ let push = debouncer.push(
+ key,
+ QueuedInbound { seq, inbound },
+ &settings,
+ Instant::now(),
+ );
+ tracing::debug!(
+ channel = %channel,
+ seq,
+ outcome = ?push,
+ tracked = debouncer.tracked(),
+ "[dispatch][debounce] buffered inbound message"
+ );
+ if push != Push::Opened {
+ return;
+ }
+ if let Some(target) = ctx.channels_by_name.get(&channel).cloned() {
+ typing.spawn(async move {
+ if let Err(error) = target.start_typing(&reply_target).await {
+ tracing::debug!(
+ "[dispatch][debounce] typing start failed on {}: {error}",
+ target.name()
+ );
+ }
+ });
+ }
+}
+
fn log_join_error(stage: &str, error: Option<tokio::task::JoinError>) {
if let Some(error) = error {
tracing::warn!("channel dispatch {stage} worker failed: {error}");
diff --git a/crates/openhuman-core/src/channels/runtime/dispatch/processor/in_order.rs b/crates/openhuman-core/src/channels/runtime/dispatch/processor/in_order.rs
new file mode 100644
index 0000000000..994756fa9a
--- /dev/null
+++ b/crates/openhuman-core/src/channels/runtime/dispatch/processor/in_order.rs
@@ -0,0 +1,44 @@
+//! Releases admission results in the order their messages were received.
+
+use std::collections::BTreeMap;
+
+pub(super) struct InOrder<T> {
+ next: u64,
+ finished: BTreeMap<u64, Option<T>>,
+}
+
+impl<T> InOrder<T> {
+ pub(super) fn new() -> Self {
+ Self {
+ next: 0,
+ finished: BTreeMap::new(),
+ }
+ }
+
+ /// Record the admission result for `seq` (`None` when it was rejected or
+ /// failed) and return every accepted item that can now be released, in
+ /// receive order.
+ pub(super) fn finish(&mut self, seq: u64, result: Option<T>) -> Vec<(u64, T)> {
+ if seq < self.next {
+ return Vec::new();
+ }
+ self.finished.insert(seq, result);
+ let mut released = Vec::new();
+ while let Some(result) = self.finished.remove(&self.next) {
+ if let Some(item) = result {
+ released.push((self.next, item));
+ }
+ self.next += 1;
+ }
+ released
+ }
+
+ /// Results held back behind an earlier admission still running.
+ pub(super) fn waiting(&self) -> usize {
+ self.finished.len()
+ }
+}
+
+#[cfg(test)]
+#[path = "in_order_tests.rs"]
+mod tests;
diff --git a/crates/openhuman-core/src/channels/runtime/dispatch/processor/in_order_tests.rs b/crates/openhuman-core/src/channels/runtime/dispatch/processor/in_order_tests.rs
new file mode 100644
index 0000000000..a137c6f211
--- /dev/null
+++ b/crates/openhuman-core/src/channels/runtime/dispatch/processor/in_order_tests.rs
@@ -0,0 +1,32 @@
+use super::InOrder;
+
+#[test]
+fn releases_in_receive_order_when_admissions_finish_out_of_order() {
+ let mut order = InOrder::new();
+ assert!(order.finish(1, Some("two")).is_empty());
+ assert!(order.finish(2, Some("three")).is_empty());
+ assert_eq!(order.waiting(), 2);
+ assert_eq!(
+ order.finish(0, Some("one")),
+ vec![(0, "one"), (1, "two"), (2, "three")]
+ );
+ assert_eq!(order.waiting(), 0);
+}
+
+#[test]
+fn rejected_admissions_do_not_leave_a_gap() {
+ let mut order = InOrder::new();
+ assert!(order.finish(1, Some("two")).is_empty());
+ assert_eq!(order.finish(0, None), vec![(1, "two")]);
+ assert_eq!(order.finish(2, Some("three")), vec![(2, "three")]);
+ assert_eq!(order.finish(3, None), Vec::<(u64, &str)>::new());
+ assert_eq!(order.finish(4, Some("five")), vec![(4, "five")]);
+}
+
+#[test]
+fn stale_sequence_numbers_are_ignored() {
+ let mut order = InOrder::new();
+ assert_eq!(order.finish(0, Some("one")), vec![(0, "one")]);
+ assert!(order.finish(0, Some("again")).is_empty());
+ assert_eq!(order.waiting(), 0);
+}