Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

refactor(orchestration): remove delivery slot and drain recovery helpers

Steven Enamakel4d930c8e4def
diff --git a/crates/openhuman-core/src/agent/orchestration/background_delivery.rs b/crates/openhuman-core/src/agent/orchestration/background_delivery.rs
index 37c8421dda..b34d0cc1bb 100644
--- a/crates/openhuman-core/src/agent/orchestration/background_delivery.rs
+++ b/crates/openhuman-core/src/agent/orchestration/background_delivery.rs
@@ -48,14 +48,14 @@ use super::background_completions;
use super::busy_guard::is_busy;
use super::completion_notice::build_undelivered_notice;
use super::completion_owners;
-use super::delivery_drain::{DrainFence, RECOVERY_BUSY_POLL};
+use super::delivery_drain::DrainFence;
#[path = "delivery_events.rs"]
mod delivery_events;
use crate::core::runtime::CoreContext;
#[cfg(test)]
-pub(super) use delivery_events::drain_schedule;
-#[cfg(test)]
pub(super) use delivery_events::Drain;
+#[cfg(test)]
+pub(super) use delivery_events::drain_schedule;
pub(super) use delivery_events::{drain_schedule_in, owning_profiles};
/// Coalesce completions landing within this window into one delivery turn.
@@ -138,8 +138,8 @@ fn schedule_delivery_inner(
.push((thread_id.clone(), delay));
#[cfg(not(test))]
let _ = delay; // The owner applies debounce/recovery timing inside its delivery API.
- // Scoped: the thread -> workspace table is keyed per profile, so the
- // delayed task must run under the profile that scheduled it.
+ // Scoped: the thread -> workspace table is keyed per profile, so the
+ // delayed task must run under the profile that scheduled it.
ctx.clone().spawn(async move {
let outcome = try_deliver(ctx.clone(), thread_id.clone(), fence.clone(), wait_idle).await;
if outcome == DeliveryOutcome::ParentBusy {
@@ -149,7 +149,13 @@ fn schedule_delivery_inner(
"[background_delivery] recovered drain still busy; waiting again \
thread_id={thread_id}"
);
- schedule_delivery_inner(ctx, thread_id, RECOVERY_BUSY_POLL, true, fence);
+ schedule_delivery_inner(
+ ctx,
+ thread_id,
+ CompletionDeliveryConfig::default().busy_poll,
+ true,
+ fence,
+ );
}
});
}
@@ -248,7 +254,7 @@ async fn try_deliver(
recovery_delay: RECOVERY_DELAY,
retry_backoff: Duration::from_secs(2),
busy_poll: RECOVERY_BUSY_POLL,
- max_busy_polls: super::delivery_drain::RECOVERY_BUSY_POLLS,
+ max_busy_polls: CompletionDeliveryConfig::default().max_busy_polls,
batch_size: usize::MAX,
},
delivery_leases().clone(),
@@ -480,4 +486,3 @@ pub(crate) fn register_background_delivery() {
#[cfg(test)]
#[path = "background_delivery_tests.rs"]
mod tests;
-
diff --git a/crates/openhuman-core/src/agent/orchestration/background_delivery/slot.rs b/crates/openhuman-core/src/agent/orchestration/background_delivery/slot.rs
deleted file mode 100644
index 70fb5ab9eb..0000000000
--- a/crates/openhuman-core/src/agent/orchestration/background_delivery/slot.rs
+++ /dev/null
@@ -1,54 +0,0 @@
-//! The per-thread (per profile) delivery slot: at most one delivery turn is in
-//! flight for a thread, and the batch it claimed is leased until it settles.
-
-use std::sync::Arc;
-
-use tinyagents_tasks::{CompletionReceipt, CompletionRouter};
-
-use crate::core::runtime::tenant;
-use crate::core::runtime::DispatchCtx;
-
-/// The per-thread (per profile) delivery slot, plus the batch lease.
-/// Dropping it frees both: `release` on an already-settled record is a no-op,
-/// so a settled batch is untouched and an abandoned one is claimable again.
-pub(super) struct DeliverySlot {
- /// [`tenant::profile_key_in`] of the thread id.
- pub(super) key: String,
- router: Arc<CompletionRouter>,
- held: Vec<CompletionReceipt>,
-}
-
-impl DeliverySlot {
- /// `None` when a delivery is already in flight for the thread.
- pub(super) fn claim(
- ctx: &DispatchCtx,
- thread_id: &str,
- router: Arc<CompletionRouter>,
- ) -> Option<Self> {
- let key = tenant::profile_key_in(&super::super::busy_guard::caller(ctx), thread_id);
- let mut d = super::delivering().lock().expect("delivering poisoned");
- if !d.insert(key.clone()) {
- return None;
- }
- Some(Self {
- key,
- router,
- held: Vec::new(),
- })
- }
-
- pub(super) fn hold(&mut self, receipts: &[CompletionReceipt]) {
- self.held = receipts.to_vec();
- }
-}
-
-impl Drop for DeliverySlot {
- fn drop(&mut self) {
- if !self.held.is_empty() {
- self.router.release(&self.held);
- }
- if let Ok(mut d) = super::delivering().lock() {
- d.remove(&self.key);
- }
- }
-}
diff --git a/crates/openhuman-core/src/agent/orchestration/background_delivery_recovery_tests.rs b/crates/openhuman-core/src/agent/orchestration/background_delivery_recovery_tests.rs
deleted file mode 100644
index 09eeeb39ff..0000000000
--- a/crates/openhuman-core/src/agent/orchestration/background_delivery_recovery_tests.rs
+++ /dev/null
@@ -1,81 +0,0 @@
-use super::*;
-use crate::core::runtime::DispatchCtx;
-
-#[tokio::test]
-async fn an_unexpired_completion_claim_stays_leased_after_router_reopen() {
- let _g = test_guard().await;
- let ws = workspace();
- let w = ws.path();
- record(
- w,
- "bd-claimed-restart",
- "sub-1",
- "claimed before restart",
- "thread-claimed-restart",
- )
- .await;
-
- let router = router_for_workspace(w);
- let claimed = router
- .claim_pending("thread-claimed-restart", usize::MAX)
- .unwrap();
- assert_eq!(claimed.len(), 1, "the first owner holds the completion");
- drop(claimed);
- drop(router);
- forget_workspace_for_test(w);
-
- let reopened = router_for_workspace(w);
- assert!(
- reopened
- .claim_pending("thread-claimed-restart", usize::MAX)
- .unwrap()
- .is_empty(),
- "an unexpired durable claim prevents duplicate delivery after reopening"
- );
- assert_eq!(
- pending_ids(w, "thread-claimed-restart"),
- ["sub-1"],
- "the leased completion remains pending for delivery after its visibility lease expires"
- );
-}
-
-#[tokio::test]
-async fn a_mixed_batch_that_partly_gives_up_still_retries_the_rest() {
- let _g = test_guard().await;
- let ws = workspace();
- let w = ws.path();
- record(w, "bd-mixed", "sub-old", "old", "thread-mixed").await;
- let router = router_for_workspace(w);
- for _ in 0..(DEFAULT_MAX_ATTEMPTS - 1) {
- try_deliver_with(
- &DispatchCtx::ambient(),
- "thread-mixed".into(),
- router.clone(),
- |_t, _n| async move { Err::<String, String>("checkout failed".to_string()) },
- |_t, _n| async move { unreachable!("below the ceiling") },
- )
- .await;
- }
- // A newer result joins the old one, which is on its last attempt.
- record(w, "bd-mixed", "sub-new", "new", "thread-mixed").await;
-
- let gave_up = Arc::new(Mutex::new(String::new()));
- let sink = Arc::clone(&gave_up);
- let retry = try_deliver_with(
- &DispatchCtx::ambient(),
- "thread-mixed".into(),
- router,
- |_t, _n| async move { Err::<String, String>("checkout failed".to_string()) },
- move |_t, notice| async move {
- *sink.lock().expect("sink") = notice;
- },
- )
- .await;
-
- assert!(gave_up.lock().expect("sink").contains("sub-old"));
- assert!(
- retry.is_some(),
- "the newer record must still be rescheduled"
- );
- assert_eq!(pending_ids(w, "thread-mixed"), ["sub-new"]);
-}
diff --git a/crates/openhuman-core/src/agent/orchestration/background_delivery_slots_tests.rs b/crates/openhuman-core/src/agent/orchestration/background_delivery_slots_tests.rs
deleted file mode 100644
index 6104f6d2a1..0000000000
--- a/crates/openhuman-core/src/agent/orchestration/background_delivery_slots_tests.rs
+++ /dev/null
@@ -1,276 +0,0 @@
-//! Busy state and delivery slots are per profile. In SaaS two profiles can use
-//! the same thread id and the same web-chat session id (`{client_id,
-//! thread_id}`); one profile's in-flight turn or delivery must neither hold
-//! back nor stand in for the other's. Desktop keys stay the bare ids.
-
-use super::*;
-use crate::agent::orchestration::background_completions::{
- TestWorkspace, pending_for_ctx, record_completion, router_for_workspace,
- router_for_workspace_ctx,
-};
-use crate::core::runtime::DispatchCtx;
-use crate::core::runtime::{ContextOverlay, DomainSet};
-use crate::tools::toolpacks::ToolGroups;
-
-fn profile(name: &str, workspace: &Path) -> Arc<CoreContext> {
- let config = crate::config::Config {
- workspace_dir: workspace.to_path_buf(),
- ..crate::config::Config::default()
- };
- CoreContext::for_test(DomainSet::full(), None).derive_with(
- ContextOverlay::new(config, DomainSet::kernel(), ToolGroups::none())
- .profile(name)
- .session_agent(name),
- )
-}
-
-fn unique(prefix: &str) -> String {
- format!("{prefix}-{}", uuid::Uuid::new_v4().simple())
-}
-
-fn web_session(thread: &str) -> String {
- format!(r#"{{"client_id":"c","thread_id":"{thread}"}}"#)
-}
-
-/// Two profiles, each with one pending result on the same thread id.
-struct Pair {
- alice: Arc<CoreContext>,
- bob: Arc<CoreContext>,
- ws_a: TestWorkspace,
- ws_b: TestWorkspace,
- thread: String,
- session: String,
-}
-
-async fn pair() -> Pair {
- let (ws_a, ws_b) = (TestWorkspace::new(), TestWorkspace::new());
- let alice = profile(&unique("u-alice"), ws_a.path());
- let bob = profile(&unique("u-bob"), ws_b.path());
- let thread = unique("t");
- let session = web_session(&thread);
- for (ctx, ws, task) in [(&alice, &ws_a, "task-alice"), (&bob, &ws_b, "task-bob")] {
- {
- let scope_ctx = crate::core::runtime::DispatchCtx::new(Arc::clone(ctx));
- record_completion(
- &scope_ctx,
- ws.path(),
- &session,
- task,
- "researcher",
- "done",
- Some(thread.clone()),
- )
- .await
- };
- }
- Pair {
- alice,
- bob,
- ws_a,
- ws_b,
- thread,
- session,
- }
-}
-
-fn in_scope<R>(ctx: &Arc<CoreContext>, f: impl FnOnce() -> R) -> R {
- CoreContext::sync_scope(Arc::clone(ctx), f)
-}
-
-#[tokio::test]
-async fn one_profiles_busy_turn_does_not_block_anothers_thread() {
- let _guard = crate::config::TEST_ENV_LOCK.lock().await;
- let p = pair().await;
- let turn = in_scope(&p.alice, || {
- TurnBusy::start(&DispatchCtx::ambient(), &p.session)
- });
-
- assert!(in_scope(&p.alice, || is_busy(
- &DispatchCtx::ambient(),
- &p.thread
- )));
- assert!(!in_scope(&p.bob, || is_busy(
- &DispatchCtx::ambient(),
- &p.thread
- )));
- let bob_ctx = DispatchCtx::new(Arc::clone(&p.bob));
- let router_b = router_for_workspace_ctx(&bob_ctx, p.ws_b.path());
- let batch = in_scope(&p.bob, || {
- claim_ready(&DispatchCtx::ambient(), &router_b, &p.thread)
- })
- .expect("bob's thread is idle while alice's turn runs");
- assert_eq!(batch[0].task_id, "task-bob");
- let receipts: Vec<_> = batch
- .iter()
- .map(|record| record.receipt().expect("claimed record has receipt"))
- .collect();
- router_b.release(&receipts);
-
- // Bob's Stop clears only Bob's turns; Alice stays busy.
- assert_eq!(
- in_scope(&p.bob, || clear_busy_for_thread(
- &DispatchCtx::ambient(),
- &p.thread
- )),
- 0
- );
- assert!(in_scope(&p.alice, || is_busy(
- &DispatchCtx::ambient(),
- &p.thread
- )));
-
- drop(turn);
- assert!(!in_scope(&p.alice, || is_busy(
- &DispatchCtx::ambient(),
- &p.thread
- )));
-}
-
-#[tokio::test]
-async fn a_delivery_in_flight_for_one_profile_does_not_take_anothers_slot() {
- let _guard = crate::config::TEST_ENV_LOCK.lock().await;
- let p = pair().await;
- let (alice_ctx, bob_ctx) = (
- DispatchCtx::new(Arc::clone(&p.alice)),
- DispatchCtx::new(Arc::clone(&p.bob)),
- );
- let (router_a, router_b) = (
- router_for_workspace_ctx(&alice_ctx, p.ws_a.path()),
- router_for_workspace_ctx(&bob_ctx, p.ws_b.path()),
- );
-
- let held = in_scope(&p.alice, || {
- DeliverySlot::claim(&DispatchCtx::ambient(), &p.thread, router_a.clone())
- })
- .expect("alice claims her slot");
- assert!(
- in_scope(&p.alice, || DeliverySlot::claim(
- &DispatchCtx::ambient(),
- &p.thread,
- router_a.clone()
- ))
- .is_none(),
- "one delivery per thread within a profile"
- );
- let bob_slot = in_scope(&p.bob, || {
- DeliverySlot::claim(&DispatchCtx::ambient(), &p.thread, router_b.clone())
- });
- assert!(
- bob_slot.is_some(),
- "alice's delivery does not hold bob's slot"
- );
- drop((held, bob_slot));
-
- // End to end: while Alice's delivery turn is in flight, Bob's runs and
- // lands in Bob's thread; then Alice's lands in hers.
- let delivered: Arc<Mutex<Vec<(String, String)>>> = Arc::default();
- let log = |who: &'static str| {
- let delivered = Arc::clone(&delivered);
- move |thread: String, notice: String| {
- delivered
- .lock()
- .unwrap()
- .push((format!("{who}:{thread}"), notice));
- async { Ok::<_, String>("ok".to_string()) }
- }
- };
- let bob = Arc::clone(&p.bob);
- let (thread, router_bob, bob_log) = (p.thread.clone(), router_b.clone(), log("bob"));
- let alice_deliver = move |thread_a: String, notice: String| {
- let bob = Arc::clone(&bob);
- let (thread, router_bob, bob_log) = (thread.clone(), router_bob.clone(), bob_log.clone());
- let alice_log = log("alice");
- async move {
- let retry = {
- let scope_ctx = crate::core::runtime::DispatchCtx::new(bob);
- try_deliver_with(&scope_ctx, thread, router_bob, bob_log, |_t, _n| async {
- unreachable!("bob's delivery succeeds")
- })
- .await
- };
- assert_eq!(retry, None);
- alice_log(thread_a, notice).await
- }
- };
- let retry = {
- let scope_ctx = crate::core::runtime::DispatchCtx::new(Arc::clone(&p.alice));
- try_deliver_with(
- &scope_ctx,
- p.thread.clone(),
- router_a,
- alice_deliver,
- |_t, _n| async { unreachable!("alice's delivery succeeds") },
- )
- .await
- };
- assert_eq!(retry, None);
-
- let delivered = delivered.lock().unwrap().clone();
- assert_eq!(delivered.len(), 2, "both profiles got their result");
- assert_eq!(delivered[0].0, format!("bob:{}", p.thread));
- assert!(delivered[0].1.contains("task-bob") && !delivered[0].1.contains("task-alice"));
- assert_eq!(delivered[1].0, format!("alice:{}", p.thread));
- assert!(delivered[1].1.contains("task-alice") && !delivered[1].1.contains("task-bob"));
- assert!(pending_for_ctx(&alice_ctx, p.ws_a.path(), &p.thread).is_empty());
- assert!(pending_for_ctx(&bob_ctx, p.ws_b.path(), &p.thread).is_empty());
-}
-
-#[tokio::test]
-async fn desktop_keys_are_the_bare_ids() {
- let _guard = crate::config::TEST_ENV_LOCK.lock().await;
- let desktop = CoreContext::for_test(DomainSet::full(), None);
- let ws = TestWorkspace::new();
- let thread = unique("t-desk");
- let session = unique("desk-session");
- in_scope(&desktop, || {
- background_completions::note_session_thread(&DispatchCtx::ambient(), &session, &thread)
- });
-
- let turn = in_scope(&desktop, || {
- TurnBusy::start(&DispatchCtx::ambient(), &session)
- });
- assert!(in_scope(&desktop, || is_busy(
- &DispatchCtx::ambient(),
- &thread
- )));
- drop(turn);
- assert!(!in_scope(&desktop, || is_busy(
- &DispatchCtx::ambient(),
- &thread
- )));
-
- let router = router_for_workspace(ws.path());
- let slot = in_scope(&desktop, || {
- DeliverySlot::claim(&DispatchCtx::ambient(), &thread, router)
- })
- .expect("free");
- assert_eq!(slot.key, thread);
- assert!(delivering().lock().unwrap().contains(&thread));
- drop(slot);
- assert!(!delivering().lock().unwrap().contains(&thread));
-}
-
-#[tokio::test]
-async fn an_older_guard_dropping_keeps_a_replacement_turn_busy() {
- let _guard = crate::config::TEST_ENV_LOCK.lock().await;
- let desktop = CoreContext::for_test(DomainSet::full(), None);
- let thread = unique("t-interrupt");
- let session = web_session(&thread);
-
- let old = in_scope(&desktop, || {
- TurnBusy::start(&DispatchCtx::ambient(), &session)
- });
- let new = in_scope(&desktop, || {
- TurnBusy::start(&DispatchCtx::ambient(), &session)
- });
- drop(old);
- assert!(
- in_scope(&desktop, || is_busy(&DispatchCtx::ambient(), &thread)),
- "replacement still running"
- );
- drop(new);
- assert!(!in_scope(&desktop, || is_busy(
- &DispatchCtx::ambient(),
- &thread
- )));
-}
diff --git a/crates/openhuman-core/src/agent/orchestration/background_delivery_tenant_tests.rs b/crates/openhuman-core/src/agent/orchestration/background_delivery_tenant_tests.rs
index c72e47df3c..a36f17ce61 100644
--- a/crates/openhuman-core/src/agent/orchestration/background_delivery_tenant_tests.rs
+++ b/crates/openhuman-core/src/agent/orchestration/background_delivery_tenant_tests.rs
@@ -9,7 +9,7 @@ use std::path::Path;
use std::sync::Arc;
use super::background_completions::*;
-use super::background_delivery::{drain_schedule_in, Drain};
+use super::background_delivery::{Drain, drain_schedule_in};
use crate::core::events::DomainEvent;
use crate::core::runtime::{ContextOverlay, CoreContext, DomainSet};
use crate::tools::toolpacks::ToolGroups;
@@ -151,7 +151,7 @@ fn an_unknown_session_is_dropped_in_saas() {
/// its own marks.
#[tokio::test]
async fn busy_marks_are_keyed_by_the_explicit_context() {
- use super::busy_guard::{clear_busy_for_thread, is_busy, TurnBusy};
+ use super::busy_guard::{TurnBusy, clear_busy_for_thread, is_busy};
let _guard = crate::config::TEST_ENV_LOCK.lock().await;
let (ws_a, ws_b) = (TestWorkspace::new(), TestWorkspace::new());
let alice = DispatchCtx::new(profile(&unique("b-alice"), ws_a.path()));
diff --git a/crates/openhuman-core/src/agent/orchestration/delivery_drain.rs b/crates/openhuman-core/src/agent/orchestration/delivery_drain.rs
index 310f563404..f7d809f0fa 100644
--- a/crates/openhuman-core/src/agent/orchestration/delivery_drain.rs
+++ b/crates/openhuman-core/src/agent/orchestration/delivery_drain.rs
@@ -1,36 +1,15 @@
-//! When a scheduled delivery drain may run: after its delay, once its thread
-//! is idle, and only while this node still holds the owning profile's lease.
-//!
-//! A drain is scheduled (debounced, a retry backoff, or a recovered record
-//! after a profile open) and runs later. In SaaS the profile's lease can be
-//! lost in between, and another node may already have opened the profile and
-//! started delivering the same records. A drain therefore carries the
-//! [`LeaseFence`] of the profile that scheduled it ([`DrainFence`]) and checks
-//! it after its delay, on every busy poll, and before the delivery turn; a
-//! latched fence also cuts the delay short. A fenced drain is dropped, not
-//! retried: its records stay pending in the profile's completion log, which
-//! the new holder recovers when it opens the profile.
-//!
-//! Outside SaaS, and for work that serves no hosted profile, a drain has no
-//! fence and always runs.
+//! The ownership fence applied to an asynchronous completion delivery.
-use crate::core::runtime::DispatchCtx;
use std::future::Future;
use std::pin::Pin;
use std::sync::Arc;
use std::time::Duration;
+use crate::core::runtime::DispatchCtx;
use crate::storage::fence::LeaseFence;
use tinyagents_tasks::DeliveryFence;
-use super::busy_guard::is_busy;
-
-/// Poll interval and ceiling while a recovered drain waits for a busy thread.
-pub(super) const RECOVERY_BUSY_POLL: Duration = Duration::from_secs(5);
-pub(super) const RECOVERY_BUSY_POLLS: u32 = 360;
-
-/// The lease fence a drain runs under: the owning profile's, or none (the
-/// desktop, embedded agents).
+/// The profile lease a delivery drain runs under, or no fence for local work.
#[derive(Clone, Debug, Default)]
pub(crate) struct DrainFence(Option<Arc<LeaseFence>>);
@@ -47,12 +26,11 @@ impl DrainFence {
/// The fence of the profile the calling task serves, when this node hosts
/// it. `None` outside SaaS or outside a profile.
- pub(crate) fn current(ctx: &crate::core::runtime::DispatchCtx) -> Self {
+ pub(crate) fn current(ctx: &DispatchCtx) -> Self {
Self(crate::profiles::host::current(ctx).map(|profile| Arc::clone(profile.lease_fence())))
}
- /// Whether this node may still act for the profile: the fence is not
- /// latched and the grant has not run out by this node's clock.
+ /// Whether this node may still act for the profile.
pub(crate) fn admits(&self) -> bool {
self.0.as_ref().is_none_or(|fence| {
fence
@@ -61,8 +39,8 @@ impl DrainFence {
})
}
- /// Sleep for `delay`, cut short when the fence latches. `true` when the
- /// drain may go on.
+ /// Sleep for `delay`, cut short when the fence latches. Returns whether
+ /// the drain may continue.
pub(crate) async fn sleep(&self, delay: Duration) -> bool {
match &self.0 {
None => {
@@ -91,68 +69,3 @@ impl DeliveryFence for DrainFence {
}
}
}
-
-/// How a drain ended.
-#[derive(Debug, PartialEq, Eq)]
-pub(super) enum DrainOutcome {
- /// `deliver` ran.
- Ran,
- /// The profile's lease was lost; nothing ran.
- Fenced,
- /// The thread stayed busy past the poll ceiling; nothing ran.
- StillBusy,
-}
-
-/// Wait `delay`, then (with `wait_idle`) for `thread_id` to go idle, then run
-/// `deliver`, unless `fence` stops admitting the profile at any point.
-///
-/// `wait_idle` serves a recovered record: it has no owner noted (that table is
-/// process-local), so the busy turn's completion event would not wake it.
-pub(super) async fn run_drain<F, Fut>(
- ctx: &DispatchCtx,
- thread_id: &str,
- delay: Duration,
- wait_idle: bool,
- fence: &DrainFence,
- poll: Duration,
- deliver: F,
-) -> DrainOutcome
-where
- F: FnOnce() -> Fut,
- Fut: Future<Output = ()>,
-{
- if !fence.sleep(delay).await {
- log::info!(
- "[background_delivery] profile lease lost; dropping the drain thread_id={thread_id}"
- );
- return DrainOutcome::Fenced;
- }
- if wait_idle {
- let mut polls = 0;
- while is_busy(ctx, thread_id) && polls < RECOVERY_BUSY_POLLS {
- polls += 1;
- if !fence.sleep(poll).await {
- log::info!(
- "[background_delivery] profile lease lost while waiting for an idle \
- thread; dropping the drain thread_id={thread_id}"
- );
- return DrainOutcome::Fenced;
- }
- }
- if is_busy(ctx, thread_id) {
- return DrainOutcome::StillBusy;
- }
- }
- if !fence.admits() {
- log::info!(
- "[background_delivery] profile lease lost; dropping the drain thread_id={thread_id}"
- );
- return DrainOutcome::Fenced;
- }
- deliver().await;
- DrainOutcome::Ran
-}
-
-#[cfg(test)]
-#[path = "delivery_drain_tests.rs"]
-mod tests;
diff --git a/crates/openhuman-core/src/agent/orchestration/delivery_drain_tests.rs b/crates/openhuman-core/src/agent/orchestration/delivery_drain_tests.rs
deleted file mode 100644
index 215339fcd7..0000000000
--- a/crates/openhuman-core/src/agent/orchestration/delivery_drain_tests.rs
+++ /dev/null
@@ -1,201 +0,0 @@
-use super::*;
-use crate::agent::orchestration::background_completions::{
- pending_for, record_completion, router_for_workspace, TestWorkspace,
-};
-use crate::agent::orchestration::busy_guard::TurnBusy;
-use crate::core::runtime::DispatchCtx;
-use crate::storage::fence::{system_now_ms, ScopeMatch};
-use crate::storage::lease::{DocumentLeases, LeaseStore};
-use crate::storage::MemoryStorage;
-use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
-
-const TTL: Duration = Duration::from_secs(30);
-const SHORT: Duration = Duration::from_millis(20);
-
-/// A profile's fence for a grant that is good for the TTL, plus the lease
-/// store it verifies against (kept alive by the caller).
-fn live_fence() -> (Arc<dyn LeaseStore>, Arc<LeaseFence>) {
- let backend = MemoryStorage::new();
- let leases: Arc<dyn LeaseStore> =
- Arc::new(DocumentLeases::cluster(&backend, "node-a", None, TTL).unwrap());
- let fence = Arc::new(LeaseFence::new(
- Arc::clone(&leases),
- "alice",
- "node-a",
- 1,
- system_now_ms() + 30_000,
- 5_000,
- vec![ScopeMatch::Exact("profile:alice".into())],
- ));
- (leases, fence)
-}
-
-/// Run a drain whose `deliver` only notes that it ran.
-async fn drain(
- thread: &str,
- delay: Duration,
- wait_idle: bool,
- fence: &DrainFence,
-) -> (DrainOutcome, bool) {
- let ran = Arc::new(AtomicBool::new(false));
- let flag = Arc::clone(&ran);
- let outcome = run_drain(
- &DispatchCtx::ambient(),
- thread,
- delay,
- wait_idle,
- fence,
- SHORT,
- || async move {
- flag.store(true, Ordering::SeqCst);
- },
- )
- .await;
- (outcome, ran.load(Ordering::SeqCst))
-}
-
-#[tokio::test]
-async fn a_drain_for_a_fenced_profile_delivers_nothing() {
- let (_leases, fence) = live_fence();
- fence.fence();
- let (outcome, ran) = drain("dd-fenced", SHORT, true, &DrainFence::of(fence)).await;
- assert_eq!(outcome, DrainOutcome::Fenced);
- assert!(!ran, "a node that lost the lease must not deliver");
-}
-
-#[tokio::test]
-async fn a_drain_for_a_live_profile_still_delivers() {
- let (_leases, fence) = live_fence();
- let (outcome, ran) = drain("dd-live", SHORT, true, &DrainFence::of(fence)).await;
- assert_eq!(outcome, DrainOutcome::Ran);
- assert!(ran);
- let (outcome, ran) = drain("dd-unfenced", SHORT, false, &DrainFence::none()).await;
- assert_eq!(outcome, DrainOutcome::Ran, "no fence (desktop) always runs");
- assert!(ran);
-}
-
-#[tokio::test]
-async fn losing_the_lease_during_the_delay_cuts_the_drain_short() {
- let (_leases, fence) = live_fence();
- let latch = Arc::clone(&fence);
- tokio::spawn(async move {
- tokio::time::sleep(SHORT).await;
- latch.fence();
- });
- let started = std::time::Instant::now();
- let (outcome, ran) = drain(
- "dd-mid-delay",
- Duration::from_secs(60),
- false,
- &DrainFence::of(fence),
- )
- .await;
- assert_eq!(outcome, DrainOutcome::Fenced);
- assert!(!ran);
- assert!(
- started.elapsed() < Duration::from_secs(10),
- "aborted, not slept out"
- );
-}
-
-#[tokio::test]
-async fn an_expired_grant_fences_the_drain_without_a_latch() {
- let backend = MemoryStorage::new();
- let leases: Arc<dyn LeaseStore> =
- Arc::new(DocumentLeases::cluster(&backend, "node-a", None, TTL).unwrap());
- // Renewals stopped landing: the grant ran out by this node's clock.
- let fence = Arc::new(LeaseFence::new(
- Arc::clone(&leases),
- "alice",
- "node-a",
- 1,
- system_now_ms().saturating_sub(1),
- 0,
- vec![],
- ));
- let (outcome, ran) = drain("dd-expired", SHORT, false, &DrainFence::of(fence)).await;
- assert_eq!(outcome, DrainOutcome::Fenced);
- assert!(!ran);
-}
-
-#[tokio::test]
-async fn losing_the_lease_while_waiting_for_an_idle_thread_drops_the_drain() {
- let _g = crate::config::TEST_ENV_LOCK.lock().await;
- let (_leases, fence) = live_fence();
- let turn = TurnBusy::start_on_thread(&DispatchCtx::ambient(), "dd-busy-fenced");
- let latch = Arc::clone(&fence);
- tokio::spawn(async move {
- tokio::time::sleep(SHORT * 3).await;
- latch.fence();
- });
- let (outcome, ran) = drain("dd-busy-fenced", SHORT, true, &DrainFence::of(fence)).await;
- drop(turn);
- assert_eq!(outcome, DrainOutcome::Fenced);
- assert!(!ran);
-}
-
-#[tokio::test]
-async fn a_recovered_drain_waits_for_a_relayed_turn_on_its_thread() {
- let _g = crate::config::TEST_ENV_LOCK.lock().await;
- let thread = "dd-relayed-thread";
- let turn = TurnBusy::start_on_thread(&DispatchCtx::ambient(), thread);
- assert!(
- is_busy(&DispatchCtx::ambient(), thread),
- "a relayed turn marks its thread busy"
- );
- let release = tokio::spawn(async move {
- tokio::time::sleep(SHORT * 3).await;
- drop(turn);
- });
- let (outcome, ran) = drain(thread, SHORT, true, &DrainFence::none()).await;
- release.await.unwrap();
- assert_eq!(outcome, DrainOutcome::Ran);
- assert!(ran);
- assert!(!is_busy(&DispatchCtx::ambient(), thread));
-}
-
-#[tokio::test]
-async fn a_delivery_waits_while_a_relayed_turn_runs_on_its_thread() {
- let _g = crate::config::TEST_ENV_LOCK.lock().await;
- let ws = TestWorkspace::new();
- let w = ws.path();
- let thread = "dd-relay-deliver";
- record_completion(
- &DispatchCtx::ambient(),
- w,
- "dd-relay-session",
- "sub-relay",
- "researcher",
- "finished during the relayed turn",
- Some(thread.to_owned()),
- )
- .await;
- let router = router_for_workspace(w);
- let turns = Arc::new(AtomicU32::new(0));
- let attempt = |turns: Arc<AtomicU32>| {
- super::super::background_delivery::try_deliver_for_test(
- DispatchCtx::ambient(),
- thread.to_string(),
- Arc::clone(&router),
- turns,
- )
- };
-
- let relayed = TurnBusy::start_on_thread(&DispatchCtx::ambient(), thread);
- attempt(Arc::clone(&turns)).await;
- assert_eq!(
- turns.load(Ordering::SeqCst),
- 0,
- "no delivery over a running relayed turn"
- );
- assert_eq!(pending_for(w, thread).len(), 1, "the result stays pending");
-
- drop(relayed);
- attempt(Arc::clone(&turns)).await;
- assert_eq!(
- turns.load(Ordering::SeqCst),
- 1,
- "delivered once the turn ends"
- );
- assert!(pending_for(w, thread).is_empty());
-}