The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.
Rust agent-orchestrationai-agentsai-assistantdesktopllmlocal-firstmcppersonal-aiprivacyrustsecond-braintauri
refactor(orchestration): remove delivery slot and drain recovery helpers
Steven Enamakel4d930c8e4def
diff --git a/crates/openhuman-core/src/agent/orchestration/background_delivery.rs b/crates/openhuman-core/src/agent/orchestration/background_delivery.rsindex 37c8421dda..b34d0cc1bb 100644--- a/crates/openhuman-core/src/agent/orchestration/background_delivery.rs+++ b/crates/openhuman-core/src/agent/orchestration/background_delivery.rs@@ -48,14 +48,14 @@ use super::background_completions;use super::busy_guard::is_busy;use super::completion_notice::build_undelivered_notice;use super::completion_owners;-use super::delivery_drain::{DrainFence, RECOVERY_BUSY_POLL};+use super::delivery_drain::DrainFence;#[path = "delivery_events.rs"]mod delivery_events;use crate::core::runtime::CoreContext;#[cfg(test)]-pub(super) use delivery_events::drain_schedule;-#[cfg(test)]pub(super) use delivery_events::Drain;+#[cfg(test)]+pub(super) use delivery_events::drain_schedule;pub(super) use delivery_events::{drain_schedule_in, owning_profiles};/// Coalesce completions landing within this window into one delivery turn.@@ -138,8 +138,8 @@ fn schedule_delivery_inner(.push((thread_id.clone(), delay));#[cfg(not(test))]let _ = delay; // The owner applies debounce/recovery timing inside its delivery API.- // Scoped: the thread -> workspace table is keyed per profile, so the- // delayed task must run under the profile that scheduled it.+ // Scoped: the thread -> workspace table is keyed per profile, so the+ // delayed task must run under the profile that scheduled it.ctx.clone().spawn(async move {let outcome = try_deliver(ctx.clone(), thread_id.clone(), fence.clone(), wait_idle).await;if outcome == DeliveryOutcome::ParentBusy {@@ -149,7 +149,13 @@ fn schedule_delivery_inner("[background_delivery] recovered drain still busy; waiting again \thread_id={thread_id}");- schedule_delivery_inner(ctx, thread_id, RECOVERY_BUSY_POLL, true, fence);+ schedule_delivery_inner(+ ctx,+ thread_id,+ CompletionDeliveryConfig::default().busy_poll,+ true,+ fence,+ );}});}@@ -248,7 +254,7 @@ async fn try_deliver(recovery_delay: RECOVERY_DELAY,retry_backoff: Duration::from_secs(2),busy_poll: RECOVERY_BUSY_POLL,- max_busy_polls: super::delivery_drain::RECOVERY_BUSY_POLLS,+ max_busy_polls: CompletionDeliveryConfig::default().max_busy_polls,batch_size: usize::MAX,},delivery_leases().clone(),@@ -480,4 +486,3 @@ pub(crate) fn register_background_delivery() {#[cfg(test)]#[path = "background_delivery_tests.rs"]mod tests;-diff --git a/crates/openhuman-core/src/agent/orchestration/background_delivery/slot.rs b/crates/openhuman-core/src/agent/orchestration/background_delivery/slot.rsdeleted file mode 100644index 70fb5ab9eb..0000000000--- a/crates/openhuman-core/src/agent/orchestration/background_delivery/slot.rs+++ /dev/null@@ -1,54 +0,0 @@-//! The per-thread (per profile) delivery slot: at most one delivery turn is in-//! flight for a thread, and the batch it claimed is leased until it settles.--use std::sync::Arc;--use tinyagents_tasks::{CompletionReceipt, CompletionRouter};--use crate::core::runtime::tenant;-use crate::core::runtime::DispatchCtx;--/// The per-thread (per profile) delivery slot, plus the batch lease.-/// Dropping it frees both: `release` on an already-settled record is a no-op,-/// so a settled batch is untouched and an abandoned one is claimable again.-pub(super) struct DeliverySlot {- /// [`tenant::profile_key_in`] of the thread id.- pub(super) key: String,- router: Arc<CompletionRouter>,- held: Vec<CompletionReceipt>,-}--impl DeliverySlot {- /// `None` when a delivery is already in flight for the thread.- pub(super) fn claim(- ctx: &DispatchCtx,- thread_id: &str,- router: Arc<CompletionRouter>,- ) -> Option<Self> {- let key = tenant::profile_key_in(&super::super::busy_guard::caller(ctx), thread_id);- let mut d = super::delivering().lock().expect("delivering poisoned");- if !d.insert(key.clone()) {- return None;- }- Some(Self {- key,- router,- held: Vec::new(),- })- }-- pub(super) fn hold(&mut self, receipts: &[CompletionReceipt]) {- self.held = receipts.to_vec();- }-}--impl Drop for DeliverySlot {- fn drop(&mut self) {- if !self.held.is_empty() {- self.router.release(&self.held);- }- if let Ok(mut d) = super::delivering().lock() {- d.remove(&self.key);- }- }-}diff --git a/crates/openhuman-core/src/agent/orchestration/background_delivery_recovery_tests.rs b/crates/openhuman-core/src/agent/orchestration/background_delivery_recovery_tests.rsdeleted file mode 100644index 09eeeb39ff..0000000000--- a/crates/openhuman-core/src/agent/orchestration/background_delivery_recovery_tests.rs+++ /dev/null@@ -1,81 +0,0 @@-use super::*;-use crate::core::runtime::DispatchCtx;--#[tokio::test]-async fn an_unexpired_completion_claim_stays_leased_after_router_reopen() {- let _g = test_guard().await;- let ws = workspace();- let w = ws.path();- record(- w,- "bd-claimed-restart",- "sub-1",- "claimed before restart",- "thread-claimed-restart",- )- .await;-- let router = router_for_workspace(w);- let claimed = router- .claim_pending("thread-claimed-restart", usize::MAX)- .unwrap();- assert_eq!(claimed.len(), 1, "the first owner holds the completion");- drop(claimed);- drop(router);- forget_workspace_for_test(w);-- let reopened = router_for_workspace(w);- assert!(- reopened- .claim_pending("thread-claimed-restart", usize::MAX)- .unwrap()- .is_empty(),- "an unexpired durable claim prevents duplicate delivery after reopening"- );- assert_eq!(- pending_ids(w, "thread-claimed-restart"),- ["sub-1"],- "the leased completion remains pending for delivery after its visibility lease expires"- );-}--#[tokio::test]-async fn a_mixed_batch_that_partly_gives_up_still_retries_the_rest() {- let _g = test_guard().await;- let ws = workspace();- let w = ws.path();- record(w, "bd-mixed", "sub-old", "old", "thread-mixed").await;- let router = router_for_workspace(w);- for _ in 0..(DEFAULT_MAX_ATTEMPTS - 1) {- try_deliver_with(- &DispatchCtx::ambient(),- "thread-mixed".into(),- router.clone(),- |_t, _n| async move { Err::<String, String>("checkout failed".to_string()) },- |_t, _n| async move { unreachable!("below the ceiling") },- )- .await;- }- // A newer result joins the old one, which is on its last attempt.- record(w, "bd-mixed", "sub-new", "new", "thread-mixed").await;-- let gave_up = Arc::new(Mutex::new(String::new()));- let sink = Arc::clone(&gave_up);- let retry = try_deliver_with(- &DispatchCtx::ambient(),- "thread-mixed".into(),- router,- |_t, _n| async move { Err::<String, String>("checkout failed".to_string()) },- move |_t, notice| async move {- *sink.lock().expect("sink") = notice;- },- )- .await;-- assert!(gave_up.lock().expect("sink").contains("sub-old"));- assert!(- retry.is_some(),- "the newer record must still be rescheduled"- );- assert_eq!(pending_ids(w, "thread-mixed"), ["sub-new"]);-}diff --git a/crates/openhuman-core/src/agent/orchestration/background_delivery_slots_tests.rs b/crates/openhuman-core/src/agent/orchestration/background_delivery_slots_tests.rsdeleted file mode 100644index 6104f6d2a1..0000000000--- a/crates/openhuman-core/src/agent/orchestration/background_delivery_slots_tests.rs+++ /dev/null@@ -1,276 +0,0 @@-//! Busy state and delivery slots are per profile. In SaaS two profiles can use-//! the same thread id and the same web-chat session id (`{client_id,-//! thread_id}`); one profile's in-flight turn or delivery must neither hold-//! back nor stand in for the other's. Desktop keys stay the bare ids.--use super::*;-use crate::agent::orchestration::background_completions::{- TestWorkspace, pending_for_ctx, record_completion, router_for_workspace,- router_for_workspace_ctx,-};-use crate::core::runtime::DispatchCtx;-use crate::core::runtime::{ContextOverlay, DomainSet};-use crate::tools::toolpacks::ToolGroups;--fn profile(name: &str, workspace: &Path) -> Arc<CoreContext> {- let config = crate::config::Config {- workspace_dir: workspace.to_path_buf(),- ..crate::config::Config::default()- };- CoreContext::for_test(DomainSet::full(), None).derive_with(- ContextOverlay::new(config, DomainSet::kernel(), ToolGroups::none())- .profile(name)- .session_agent(name),- )-}--fn unique(prefix: &str) -> String {- format!("{prefix}-{}", uuid::Uuid::new_v4().simple())-}--fn web_session(thread: &str) -> String {- format!(r#"{{"client_id":"c","thread_id":"{thread}"}}"#)-}--/// Two profiles, each with one pending result on the same thread id.-struct Pair {- alice: Arc<CoreContext>,- bob: Arc<CoreContext>,- ws_a: TestWorkspace,- ws_b: TestWorkspace,- thread: String,- session: String,-}--async fn pair() -> Pair {- let (ws_a, ws_b) = (TestWorkspace::new(), TestWorkspace::new());- let alice = profile(&unique("u-alice"), ws_a.path());- let bob = profile(&unique("u-bob"), ws_b.path());- let thread = unique("t");- let session = web_session(&thread);- for (ctx, ws, task) in [(&alice, &ws_a, "task-alice"), (&bob, &ws_b, "task-bob")] {- {- let scope_ctx = crate::core::runtime::DispatchCtx::new(Arc::clone(ctx));- record_completion(- &scope_ctx,- ws.path(),- &session,- task,- "researcher",- "done",- Some(thread.clone()),- )- .await- };- }- Pair {- alice,- bob,- ws_a,- ws_b,- thread,- session,- }-}--fn in_scope<R>(ctx: &Arc<CoreContext>, f: impl FnOnce() -> R) -> R {- CoreContext::sync_scope(Arc::clone(ctx), f)-}--#[tokio::test]-async fn one_profiles_busy_turn_does_not_block_anothers_thread() {- let _guard = crate::config::TEST_ENV_LOCK.lock().await;- let p = pair().await;- let turn = in_scope(&p.alice, || {- TurnBusy::start(&DispatchCtx::ambient(), &p.session)- });-- assert!(in_scope(&p.alice, || is_busy(- &DispatchCtx::ambient(),- &p.thread- )));- assert!(!in_scope(&p.bob, || is_busy(- &DispatchCtx::ambient(),- &p.thread- )));- let bob_ctx = DispatchCtx::new(Arc::clone(&p.bob));- let router_b = router_for_workspace_ctx(&bob_ctx, p.ws_b.path());- let batch = in_scope(&p.bob, || {- claim_ready(&DispatchCtx::ambient(), &router_b, &p.thread)- })- .expect("bob's thread is idle while alice's turn runs");- assert_eq!(batch[0].task_id, "task-bob");- let receipts: Vec<_> = batch- .iter()- .map(|record| record.receipt().expect("claimed record has receipt"))- .collect();- router_b.release(&receipts);-- // Bob's Stop clears only Bob's turns; Alice stays busy.- assert_eq!(- in_scope(&p.bob, || clear_busy_for_thread(- &DispatchCtx::ambient(),- &p.thread- )),- 0- );- assert!(in_scope(&p.alice, || is_busy(- &DispatchCtx::ambient(),- &p.thread- )));-- drop(turn);- assert!(!in_scope(&p.alice, || is_busy(- &DispatchCtx::ambient(),- &p.thread- )));-}--#[tokio::test]-async fn a_delivery_in_flight_for_one_profile_does_not_take_anothers_slot() {- let _guard = crate::config::TEST_ENV_LOCK.lock().await;- let p = pair().await;- let (alice_ctx, bob_ctx) = (- DispatchCtx::new(Arc::clone(&p.alice)),- DispatchCtx::new(Arc::clone(&p.bob)),- );- let (router_a, router_b) = (- router_for_workspace_ctx(&alice_ctx, p.ws_a.path()),- router_for_workspace_ctx(&bob_ctx, p.ws_b.path()),- );-- let held = in_scope(&p.alice, || {- DeliverySlot::claim(&DispatchCtx::ambient(), &p.thread, router_a.clone())- })- .expect("alice claims her slot");- assert!(- in_scope(&p.alice, || DeliverySlot::claim(- &DispatchCtx::ambient(),- &p.thread,- router_a.clone()- ))- .is_none(),- "one delivery per thread within a profile"- );- let bob_slot = in_scope(&p.bob, || {- DeliverySlot::claim(&DispatchCtx::ambient(), &p.thread, router_b.clone())- });- assert!(- bob_slot.is_some(),- "alice's delivery does not hold bob's slot"- );- drop((held, bob_slot));-- // End to end: while Alice's delivery turn is in flight, Bob's runs and- // lands in Bob's thread; then Alice's lands in hers.- let delivered: Arc<Mutex<Vec<(String, String)>>> = Arc::default();- let log = |who: &'static str| {- let delivered = Arc::clone(&delivered);- move |thread: String, notice: String| {- delivered- .lock()- .unwrap()- .push((format!("{who}:{thread}"), notice));- async { Ok::<_, String>("ok".to_string()) }- }- };- let bob = Arc::clone(&p.bob);- let (thread, router_bob, bob_log) = (p.thread.clone(), router_b.clone(), log("bob"));- let alice_deliver = move |thread_a: String, notice: String| {- let bob = Arc::clone(&bob);- let (thread, router_bob, bob_log) = (thread.clone(), router_bob.clone(), bob_log.clone());- let alice_log = log("alice");- async move {- let retry = {- let scope_ctx = crate::core::runtime::DispatchCtx::new(bob);- try_deliver_with(&scope_ctx, thread, router_bob, bob_log, |_t, _n| async {- unreachable!("bob's delivery succeeds")- })- .await- };- assert_eq!(retry, None);- alice_log(thread_a, notice).await- }- };- let retry = {- let scope_ctx = crate::core::runtime::DispatchCtx::new(Arc::clone(&p.alice));- try_deliver_with(- &scope_ctx,- p.thread.clone(),- router_a,- alice_deliver,- |_t, _n| async { unreachable!("alice's delivery succeeds") },- )- .await- };- assert_eq!(retry, None);-- let delivered = delivered.lock().unwrap().clone();- assert_eq!(delivered.len(), 2, "both profiles got their result");- assert_eq!(delivered[0].0, format!("bob:{}", p.thread));- assert!(delivered[0].1.contains("task-bob") && !delivered[0].1.contains("task-alice"));- assert_eq!(delivered[1].0, format!("alice:{}", p.thread));- assert!(delivered[1].1.contains("task-alice") && !delivered[1].1.contains("task-bob"));- assert!(pending_for_ctx(&alice_ctx, p.ws_a.path(), &p.thread).is_empty());- assert!(pending_for_ctx(&bob_ctx, p.ws_b.path(), &p.thread).is_empty());-}--#[tokio::test]-async fn desktop_keys_are_the_bare_ids() {- let _guard = crate::config::TEST_ENV_LOCK.lock().await;- let desktop = CoreContext::for_test(DomainSet::full(), None);- let ws = TestWorkspace::new();- let thread = unique("t-desk");- let session = unique("desk-session");- in_scope(&desktop, || {- background_completions::note_session_thread(&DispatchCtx::ambient(), &session, &thread)- });-- let turn = in_scope(&desktop, || {- TurnBusy::start(&DispatchCtx::ambient(), &session)- });- assert!(in_scope(&desktop, || is_busy(- &DispatchCtx::ambient(),- &thread- )));- drop(turn);- assert!(!in_scope(&desktop, || is_busy(- &DispatchCtx::ambient(),- &thread- )));-- let router = router_for_workspace(ws.path());- let slot = in_scope(&desktop, || {- DeliverySlot::claim(&DispatchCtx::ambient(), &thread, router)- })- .expect("free");- assert_eq!(slot.key, thread);- assert!(delivering().lock().unwrap().contains(&thread));- drop(slot);- assert!(!delivering().lock().unwrap().contains(&thread));-}--#[tokio::test]-async fn an_older_guard_dropping_keeps_a_replacement_turn_busy() {- let _guard = crate::config::TEST_ENV_LOCK.lock().await;- let desktop = CoreContext::for_test(DomainSet::full(), None);- let thread = unique("t-interrupt");- let session = web_session(&thread);-- let old = in_scope(&desktop, || {- TurnBusy::start(&DispatchCtx::ambient(), &session)- });- let new = in_scope(&desktop, || {- TurnBusy::start(&DispatchCtx::ambient(), &session)- });- drop(old);- assert!(- in_scope(&desktop, || is_busy(&DispatchCtx::ambient(), &thread)),- "replacement still running"- );- drop(new);- assert!(!in_scope(&desktop, || is_busy(- &DispatchCtx::ambient(),- &thread- )));-}diff --git a/crates/openhuman-core/src/agent/orchestration/background_delivery_tenant_tests.rs b/crates/openhuman-core/src/agent/orchestration/background_delivery_tenant_tests.rsindex c72e47df3c..a36f17ce61 100644--- a/crates/openhuman-core/src/agent/orchestration/background_delivery_tenant_tests.rs+++ b/crates/openhuman-core/src/agent/orchestration/background_delivery_tenant_tests.rs@@ -9,7 +9,7 @@ use std::path::Path;use std::sync::Arc;use super::background_completions::*;-use super::background_delivery::{drain_schedule_in, Drain};+use super::background_delivery::{Drain, drain_schedule_in};use crate::core::events::DomainEvent;use crate::core::runtime::{ContextOverlay, CoreContext, DomainSet};use crate::tools::toolpacks::ToolGroups;@@ -151,7 +151,7 @@ fn an_unknown_session_is_dropped_in_saas() {/// its own marks.#[tokio::test]async fn busy_marks_are_keyed_by_the_explicit_context() {- use super::busy_guard::{clear_busy_for_thread, is_busy, TurnBusy};+ use super::busy_guard::{TurnBusy, clear_busy_for_thread, is_busy};let _guard = crate::config::TEST_ENV_LOCK.lock().await;let (ws_a, ws_b) = (TestWorkspace::new(), TestWorkspace::new());let alice = DispatchCtx::new(profile(&unique("b-alice"), ws_a.path()));diff --git a/crates/openhuman-core/src/agent/orchestration/delivery_drain.rs b/crates/openhuman-core/src/agent/orchestration/delivery_drain.rsindex 310f563404..f7d809f0fa 100644--- a/crates/openhuman-core/src/agent/orchestration/delivery_drain.rs+++ b/crates/openhuman-core/src/agent/orchestration/delivery_drain.rs@@ -1,36 +1,15 @@-//! When a scheduled delivery drain may run: after its delay, once its thread-//! is idle, and only while this node still holds the owning profile's lease.-//!-//! A drain is scheduled (debounced, a retry backoff, or a recovered record-//! after a profile open) and runs later. In SaaS the profile's lease can be-//! lost in between, and another node may already have opened the profile and-//! started delivering the same records. A drain therefore carries the-//! [`LeaseFence`] of the profile that scheduled it ([`DrainFence`]) and checks-//! it after its delay, on every busy poll, and before the delivery turn; a-//! latched fence also cuts the delay short. A fenced drain is dropped, not-//! retried: its records stay pending in the profile's completion log, which-//! the new holder recovers when it opens the profile.-//!-//! Outside SaaS, and for work that serves no hosted profile, a drain has no-//! fence and always runs.+//! The ownership fence applied to an asynchronous completion delivery.-use crate::core::runtime::DispatchCtx;use std::future::Future;use std::pin::Pin;use std::sync::Arc;use std::time::Duration;+use crate::core::runtime::DispatchCtx;use crate::storage::fence::LeaseFence;use tinyagents_tasks::DeliveryFence;-use super::busy_guard::is_busy;--/// Poll interval and ceiling while a recovered drain waits for a busy thread.-pub(super) const RECOVERY_BUSY_POLL: Duration = Duration::from_secs(5);-pub(super) const RECOVERY_BUSY_POLLS: u32 = 360;--/// The lease fence a drain runs under: the owning profile's, or none (the-/// desktop, embedded agents).+/// The profile lease a delivery drain runs under, or no fence for local work.#[derive(Clone, Debug, Default)]pub(crate) struct DrainFence(Option<Arc<LeaseFence>>);@@ -47,12 +26,11 @@ impl DrainFence {/// The fence of the profile the calling task serves, when this node hosts/// it. `None` outside SaaS or outside a profile.- pub(crate) fn current(ctx: &crate::core::runtime::DispatchCtx) -> Self {+ pub(crate) fn current(ctx: &DispatchCtx) -> Self {Self(crate::profiles::host::current(ctx).map(|profile| Arc::clone(profile.lease_fence())))}- /// Whether this node may still act for the profile: the fence is not- /// latched and the grant has not run out by this node's clock.+ /// Whether this node may still act for the profile.pub(crate) fn admits(&self) -> bool {self.0.as_ref().is_none_or(|fence| {fence@@ -61,8 +39,8 @@ impl DrainFence {})}- /// Sleep for `delay`, cut short when the fence latches. `true` when the- /// drain may go on.+ /// Sleep for `delay`, cut short when the fence latches. Returns whether+ /// the drain may continue.pub(crate) async fn sleep(&self, delay: Duration) -> bool {match &self.0 {None => {@@ -91,68 +69,3 @@ impl DeliveryFence for DrainFence {}}}--/// How a drain ended.-#[derive(Debug, PartialEq, Eq)]-pub(super) enum DrainOutcome {- /// `deliver` ran.- Ran,- /// The profile's lease was lost; nothing ran.- Fenced,- /// The thread stayed busy past the poll ceiling; nothing ran.- StillBusy,-}--/// Wait `delay`, then (with `wait_idle`) for `thread_id` to go idle, then run-/// `deliver`, unless `fence` stops admitting the profile at any point.-///-/// `wait_idle` serves a recovered record: it has no owner noted (that table is-/// process-local), so the busy turn's completion event would not wake it.-pub(super) async fn run_drain<F, Fut>(- ctx: &DispatchCtx,- thread_id: &str,- delay: Duration,- wait_idle: bool,- fence: &DrainFence,- poll: Duration,- deliver: F,-) -> DrainOutcome-where- F: FnOnce() -> Fut,- Fut: Future<Output = ()>,-{- if !fence.sleep(delay).await {- log::info!(- "[background_delivery] profile lease lost; dropping the drain thread_id={thread_id}"- );- return DrainOutcome::Fenced;- }- if wait_idle {- let mut polls = 0;- while is_busy(ctx, thread_id) && polls < RECOVERY_BUSY_POLLS {- polls += 1;- if !fence.sleep(poll).await {- log::info!(- "[background_delivery] profile lease lost while waiting for an idle \- thread; dropping the drain thread_id={thread_id}"- );- return DrainOutcome::Fenced;- }- }- if is_busy(ctx, thread_id) {- return DrainOutcome::StillBusy;- }- }- if !fence.admits() {- log::info!(- "[background_delivery] profile lease lost; dropping the drain thread_id={thread_id}"- );- return DrainOutcome::Fenced;- }- deliver().await;- DrainOutcome::Ran-}--#[cfg(test)]-#[path = "delivery_drain_tests.rs"]-mod tests;diff --git a/crates/openhuman-core/src/agent/orchestration/delivery_drain_tests.rs b/crates/openhuman-core/src/agent/orchestration/delivery_drain_tests.rsdeleted file mode 100644index 215339fcd7..0000000000--- a/crates/openhuman-core/src/agent/orchestration/delivery_drain_tests.rs+++ /dev/null@@ -1,201 +0,0 @@-use super::*;-use crate::agent::orchestration::background_completions::{- pending_for, record_completion, router_for_workspace, TestWorkspace,-};-use crate::agent::orchestration::busy_guard::TurnBusy;-use crate::core::runtime::DispatchCtx;-use crate::storage::fence::{system_now_ms, ScopeMatch};-use crate::storage::lease::{DocumentLeases, LeaseStore};-use crate::storage::MemoryStorage;-use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};--const TTL: Duration = Duration::from_secs(30);-const SHORT: Duration = Duration::from_millis(20);--/// A profile's fence for a grant that is good for the TTL, plus the lease-/// store it verifies against (kept alive by the caller).-fn live_fence() -> (Arc<dyn LeaseStore>, Arc<LeaseFence>) {- let backend = MemoryStorage::new();- let leases: Arc<dyn LeaseStore> =- Arc::new(DocumentLeases::cluster(&backend, "node-a", None, TTL).unwrap());- let fence = Arc::new(LeaseFence::new(- Arc::clone(&leases),- "alice",- "node-a",- 1,- system_now_ms() + 30_000,- 5_000,- vec![ScopeMatch::Exact("profile:alice".into())],- ));- (leases, fence)-}--/// Run a drain whose `deliver` only notes that it ran.-async fn drain(- thread: &str,- delay: Duration,- wait_idle: bool,- fence: &DrainFence,-) -> (DrainOutcome, bool) {- let ran = Arc::new(AtomicBool::new(false));- let flag = Arc::clone(&ran);- let outcome = run_drain(- &DispatchCtx::ambient(),- thread,- delay,- wait_idle,- fence,- SHORT,- || async move {- flag.store(true, Ordering::SeqCst);- },- )- .await;- (outcome, ran.load(Ordering::SeqCst))-}--#[tokio::test]-async fn a_drain_for_a_fenced_profile_delivers_nothing() {- let (_leases, fence) = live_fence();- fence.fence();- let (outcome, ran) = drain("dd-fenced", SHORT, true, &DrainFence::of(fence)).await;- assert_eq!(outcome, DrainOutcome::Fenced);- assert!(!ran, "a node that lost the lease must not deliver");-}--#[tokio::test]-async fn a_drain_for_a_live_profile_still_delivers() {- let (_leases, fence) = live_fence();- let (outcome, ran) = drain("dd-live", SHORT, true, &DrainFence::of(fence)).await;- assert_eq!(outcome, DrainOutcome::Ran);- assert!(ran);- let (outcome, ran) = drain("dd-unfenced", SHORT, false, &DrainFence::none()).await;- assert_eq!(outcome, DrainOutcome::Ran, "no fence (desktop) always runs");- assert!(ran);-}--#[tokio::test]-async fn losing_the_lease_during_the_delay_cuts_the_drain_short() {- let (_leases, fence) = live_fence();- let latch = Arc::clone(&fence);- tokio::spawn(async move {- tokio::time::sleep(SHORT).await;- latch.fence();- });- let started = std::time::Instant::now();- let (outcome, ran) = drain(- "dd-mid-delay",- Duration::from_secs(60),- false,- &DrainFence::of(fence),- )- .await;- assert_eq!(outcome, DrainOutcome::Fenced);- assert!(!ran);- assert!(- started.elapsed() < Duration::from_secs(10),- "aborted, not slept out"- );-}--#[tokio::test]-async fn an_expired_grant_fences_the_drain_without_a_latch() {- let backend = MemoryStorage::new();- let leases: Arc<dyn LeaseStore> =- Arc::new(DocumentLeases::cluster(&backend, "node-a", None, TTL).unwrap());- // Renewals stopped landing: the grant ran out by this node's clock.- let fence = Arc::new(LeaseFence::new(- Arc::clone(&leases),- "alice",- "node-a",- 1,- system_now_ms().saturating_sub(1),- 0,- vec![],- ));- let (outcome, ran) = drain("dd-expired", SHORT, false, &DrainFence::of(fence)).await;- assert_eq!(outcome, DrainOutcome::Fenced);- assert!(!ran);-}--#[tokio::test]-async fn losing_the_lease_while_waiting_for_an_idle_thread_drops_the_drain() {- let _g = crate::config::TEST_ENV_LOCK.lock().await;- let (_leases, fence) = live_fence();- let turn = TurnBusy::start_on_thread(&DispatchCtx::ambient(), "dd-busy-fenced");- let latch = Arc::clone(&fence);- tokio::spawn(async move {- tokio::time::sleep(SHORT * 3).await;- latch.fence();- });- let (outcome, ran) = drain("dd-busy-fenced", SHORT, true, &DrainFence::of(fence)).await;- drop(turn);- assert_eq!(outcome, DrainOutcome::Fenced);- assert!(!ran);-}--#[tokio::test]-async fn a_recovered_drain_waits_for_a_relayed_turn_on_its_thread() {- let _g = crate::config::TEST_ENV_LOCK.lock().await;- let thread = "dd-relayed-thread";- let turn = TurnBusy::start_on_thread(&DispatchCtx::ambient(), thread);- assert!(- is_busy(&DispatchCtx::ambient(), thread),- "a relayed turn marks its thread busy"- );- let release = tokio::spawn(async move {- tokio::time::sleep(SHORT * 3).await;- drop(turn);- });- let (outcome, ran) = drain(thread, SHORT, true, &DrainFence::none()).await;- release.await.unwrap();- assert_eq!(outcome, DrainOutcome::Ran);- assert!(ran);- assert!(!is_busy(&DispatchCtx::ambient(), thread));-}--#[tokio::test]-async fn a_delivery_waits_while_a_relayed_turn_runs_on_its_thread() {- let _g = crate::config::TEST_ENV_LOCK.lock().await;- let ws = TestWorkspace::new();- let w = ws.path();- let thread = "dd-relay-deliver";- record_completion(- &DispatchCtx::ambient(),- w,- "dd-relay-session",- "sub-relay",- "researcher",- "finished during the relayed turn",- Some(thread.to_owned()),- )- .await;- let router = router_for_workspace(w);- let turns = Arc::new(AtomicU32::new(0));- let attempt = |turns: Arc<AtomicU32>| {- super::super::background_delivery::try_deliver_for_test(- DispatchCtx::ambient(),- thread.to_string(),- Arc::clone(&router),- turns,- )- };-- let relayed = TurnBusy::start_on_thread(&DispatchCtx::ambient(), thread);- attempt(Arc::clone(&turns)).await;- assert_eq!(- turns.load(Ordering::SeqCst),- 0,- "no delivery over a running relayed turn"- );- assert_eq!(pending_for(w, thread).len(), 1, "the result stays pending");-- drop(relayed);- attempt(Arc::clone(&turns)).await;- assert_eq!(- turns.load(Ordering::SeqCst),- 1,- "delivered once the turn ends"- );- assert!(pending_for(w, thread).is_empty());-}