Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

test(inbound): add pinned source delivery tests and fixtures

Steven Enamakel46405cf9ecf7
diff --git a/crates/openhuman-core/src/inbound/deliver_tests.rs b/crates/openhuman-core/src/inbound/deliver_tests.rs
index 44af0528f0..85213f17d2 100644
--- a/crates/openhuman-core/src/inbound/deliver_tests.rs
+++ b/crates/openhuman-core/src/inbound/deliver_tests.rs
@@ -339,3 +339,151 @@ async fn a_configured_deliverer_reads_the_triage_timeout_on_first_use() {
assert_eq!(response.status, DeliverStatus::Accepted);
assert!(deliverer.timeout().await <= Duration::from_secs(25));
}
+
+// ── Pinned sources ─────────────────────────────────────────────────────────
+
+use crate::inbound::fixture::{
+ memory_results, mentions_inbound, until_finished, Reply, ScriptedAgent,
+};
+use crate::inbound::pinned::{Pinned, PinnedAgent};
+use crate::inbound::results::{ResultStatus, Results};
+
+fn mention(id: &str) -> InboundEvent {
+ events::external(
+ "x:mentions",
+ "mentions",
+ id,
+ &format!("hello {id}"),
+ json!({ "author": "@a" }),
+ 5,
+ )
+ .unwrap()
+}
+
+fn pinned_deliverer(
+ triage: &Arc<ScriptedTriage>,
+ agent: &Arc<ScriptedAgent>,
+ results: &Results,
+) -> Deliverer {
+ deliverer(triage)
+ .with_inbound(mentions_inbound(10, 25))
+ .with_pinned(Pinned::new(
+ agent.clone() as Arc<dyn PinnedAgent>,
+ results.clone(),
+ ))
+}
+
+#[tokio::test]
+async fn a_pinned_source_runs_its_agent_without_triage_and_keeps_the_text() {
+ let triage = ScriptedTriage::new(Vec::<Step>::new());
+ let agent = ScriptedAgent::new([Reply::Text("draft")]);
+ let results = memory_results();
+ let response = pinned_deliverer(&triage, &agent, &results)
+ .deliver(turn(vec![mention("t1"), mention("t2")], Lane::Interactive, 1))
+ .await;
+ assert_eq!(response.status, DeliverStatus::Accepted);
+ assert_eq!(triage.runs(), 0, "a pinned source was triaged");
+ let runs = agent.runs();
+ assert_eq!(runs.len(), 1);
+ assert_eq!(runs[0].agent_id, "teeny-mentions");
+ assert_eq!(runs[0].source, "x:mentions");
+ assert_eq!(runs[0].key, "mentions");
+ assert!(runs[0].message.contains("hello t1") && runs[0].message.contains("hello t2"));
+ let stored = results.get("turn-1").unwrap().unwrap();
+ assert_eq!(stored.result.status, ResultStatus::Completed);
+ assert_eq!(stored.result.text.as_deref(), Some("draft"));
+ assert_eq!(stored.result.event_ids, ["t1", "t2"]);
+}
+
+#[tokio::test]
+async fn a_slow_pinned_run_is_acknowledged_at_the_budget_and_stored_when_it_ends() {
+ let triage = ScriptedTriage::new(Vec::<Step>::new());
+ let agent = ScriptedAgent::new([Reply::Slow(Duration::from_millis(300), "late")]);
+ let results = memory_results();
+ let deliverer =
+ pinned_deliverer(&triage, &agent, &results).with_budget(Duration::from_millis(50));
+ let started = std::time::Instant::now();
+ let response = deliverer
+ .deliver(turn(vec![mention("t1")], Lane::Interactive, 1))
+ .await;
+ assert_eq!(response.status, DeliverStatus::Accepted);
+ assert!(started.elapsed() < Duration::from_millis(300));
+ assert_eq!(
+ results.get("turn-1").unwrap().unwrap().result.status,
+ ResultStatus::Running
+ );
+ let done = until_finished(&results, "turn-1", Duration::from_secs(5))
+ .await
+ .unwrap();
+ assert_eq!(done.result.text.as_deref(), Some("late"));
+}
+
+#[tokio::test]
+async fn a_slow_pinned_run_that_fails_after_the_budget_is_stored_as_failed() {
+ let triage = ScriptedTriage::new(Vec::<Step>::new());
+ let agent = ScriptedAgent::new([Reply::SlowFail(Duration::from_millis(200), "late boom")]);
+ let results = memory_results();
+ let response = pinned_deliverer(&triage, &agent, &results)
+ .with_budget(Duration::from_millis(30))
+ .deliver(turn(vec![mention("t1")], Lane::Interactive, 1))
+ .await;
+ assert_eq!(response.status, DeliverStatus::Accepted);
+ let done = until_finished(&results, "turn-1", Duration::from_secs(5))
+ .await
+ .unwrap();
+ assert_eq!(done.result.status, ResultStatus::Failed);
+ assert_eq!(done.result.error.as_deref(), Some("late boom"));
+}
+
+#[tokio::test]
+async fn a_failed_pinned_run_is_retried_and_stored_as_failed_on_its_last_attempt() {
+ let triage = ScriptedTriage::new(Vec::<Step>::new());
+ let agent = ScriptedAgent::new([Reply::Fail("down"), Reply::Fail("still down")]);
+ let results = memory_results();
+ let deliverer = pinned_deliverer(&triage, &agent, &results);
+ let first = deliverer
+ .deliver(turn(vec![mention("t1")], Lane::Interactive, 1))
+ .await;
+ assert_eq!(first.status, DeliverStatus::Retry { after_ms: None });
+ assert_eq!(
+ results.get("turn-1").unwrap().unwrap().result.status,
+ ResultStatus::Running
+ );
+ let last = deliverer
+ .deliver(turn(vec![mention("t1")], Lane::Interactive, 3))
+ .await;
+ assert_eq!(last.status, DeliverStatus::Retry { after_ms: None });
+ let stored = results.get("turn-1").unwrap().unwrap();
+ assert_eq!(stored.result.status, ResultStatus::Failed);
+ assert_eq!(stored.result.error.as_deref(), Some("still down"));
+ assert_eq!(agent.runs().len(), 2);
+}
+
+#[tokio::test]
+async fn a_finished_pinned_turn_is_not_run_again() {
+ let triage = ScriptedTriage::new(Vec::<Step>::new());
+ let agent = ScriptedAgent::new([Reply::Text("once")]);
+ let results = memory_results();
+ let first = pinned_deliverer(&triage, &agent, &results)
+ .deliver(turn(vec![mention("t1")], Lane::Interactive, 1))
+ .await;
+ assert_eq!(first.status, DeliverStatus::Accepted);
+ // A new process (nothing remembered) is handed the same turn again.
+ let again = pinned_deliverer(&triage, &agent, &results)
+ .deliver(turn(vec![mention("t1")], Lane::Interactive, 2))
+ .await;
+ assert_eq!(again.status, DeliverStatus::Accepted);
+ assert_eq!(agent.runs().len(), 1, "a finished turn ran again");
+}
+
+#[tokio::test]
+async fn a_pinned_event_is_classified_actionable_without_triage() {
+ let triage = ScriptedTriage::new(Vec::<Step>::new());
+ let agent = ScriptedAgent::new([]);
+ let classified = pinned_deliverer(&triage, &agent, &memory_results())
+ .classify(ClassifySalienceRequest::new(mention("t1"), 35))
+ .await
+ .unwrap();
+ assert_eq!(classified.class, SalienceClass::Actionable);
+ assert_eq!(triage.runs(), 0);
+}
diff --git a/crates/openhuman-core/src/inbound/linked_fixture.rs b/crates/openhuman-core/src/inbound/linked_fixture.rs
index 1c99d264bb..050828e77d 100644
--- a/crates/openhuman-core/src/inbound/linked_fixture.rs
+++ b/crates/openhuman-core/src/inbound/linked_fixture.rs
@@ -24,6 +24,9 @@ use tinyeventqueues_bus::storage::SHIPPED_PIPELINE_JSON;
use super::deliver::Deliverer;
use super::host::InboundHost;
+use super::pinned::{Pinned, PinnedAgent, PinnedRun};
+use super::results::Results;
+use crate::config::schema::{InboundConfig, InboundSourceConfig};
use crate::core::runtime::DispatchCtx;
use crate::triage::{
TriageAction, TriageDecision, TriageOutcome, TriageResolutionPath, TriageRun, TriageRunner,
@@ -178,6 +181,110 @@ impl TriageRunner for ScriptedTriage {
}
}
+/// One scripted pinned-agent answer.
+#[derive(Debug, Clone)]
+pub(crate) enum Reply {
+ /// Answer this text.
+ Text(&'static str),
+ /// Fail with this message.
+ Fail(&'static str),
+ /// Answer this text after this long.
+ Slow(Duration, &'static str),
+ /// Fail with this message after this long.
+ SlowFail(Duration, &'static str),
+}
+
+/// A pinned agent that answers from a script (then `"ok"`) and records its
+/// runs.
+#[derive(Debug, Default)]
+pub(crate) struct ScriptedAgent {
+ script: Mutex<VecDeque<Reply>>,
+ runs: Mutex<Vec<PinnedRun>>,
+}
+
+impl ScriptedAgent {
+ pub(crate) fn new(replies: impl IntoIterator<Item = Reply>) -> Arc<Self> {
+ Arc::new(Self {
+ script: Mutex::new(replies.into_iter().collect()),
+ ..Self::default()
+ })
+ }
+
+ /// The runs so far.
+ pub(crate) fn runs(&self) -> Vec<PinnedRun> {
+ self.runs.lock().unwrap().clone()
+ }
+}
+
+#[async_trait]
+impl PinnedAgent for ScriptedAgent {
+ async fn run(&self, _ctx: &DispatchCtx, run: &PinnedRun) -> anyhow::Result<String> {
+ self.runs.lock().unwrap().push(run.clone());
+ let next = self.script.lock().unwrap().pop_front();
+ match next {
+ Some(Reply::Text(text)) => Ok(text.to_owned()),
+ Some(Reply::Fail(message)) => Err(anyhow::anyhow!(message)),
+ Some(Reply::Slow(delay, text)) => {
+ tokio::time::sleep(delay).await;
+ Ok(text.to_owned())
+ }
+ Some(Reply::SlowFail(delay, message)) => {
+ tokio::time::sleep(delay).await;
+ Err(anyhow::anyhow!(message))
+ }
+ None => Ok("ok".to_owned()),
+ }
+ }
+}
+
+/// A results store in fresh memory storage.
+pub(crate) fn memory_results() -> Results {
+ use crate::storage::StorageBackend;
+ let storage = crate::storage::MemoryStorage::new();
+ Results::over(
+ &storage
+ .for_scope(&crate::storage::Scope::new("local").unwrap())
+ .unwrap(),
+ )
+}
+
+/// `[inbound]` with `x:mentions` pinned to `teeny-mentions`, flushing after
+/// `debounce_ms` and carrying at most `max_batch` events a turn.
+pub(crate) fn mentions_inbound(debounce_ms: u64, max_batch: u32) -> InboundConfig {
+ let mut inbound = InboundConfig::default();
+ inbound.sources.insert(
+ "x:mentions".to_owned(),
+ InboundSourceConfig {
+ agent_id: "teeny-mentions".to_owned(),
+ debounce_ms,
+ max_wait_ms: debounce_ms * 20,
+ max_batch,
+ ..InboundSourceConfig::default()
+ },
+ );
+ inbound
+}
+
+/// Wait until `results` holds a finished result for `turn_id`, or `within`
+/// passed.
+pub(crate) async fn until_finished(
+ results: &Results,
+ turn_id: &str,
+ within: Duration,
+) -> Option<super::results::Stored> {
+ let deadline = tokio::time::Instant::now() + within;
+ loop {
+ let found = results.get(turn_id).unwrap();
+ let finished = found
+ .as_ref()
+ .is_some_and(|stored| stored.result.status != super::results::ResultStatus::Running);
+ if finished || tokio::time::Instant::now() >= deadline {
+ return found;
+ }
+ tokio::time::sleep(Duration::from_millis(10)).await;
+ }
+}
+
/// The shipped pipeline with a short debounce, so a test does not wait the
/// production 1.5 s per flush.
fn fast_pipeline() -> Value {
@@ -200,6 +307,16 @@ pub(crate) struct Rig {
impl Rig {
/// Attach the module and serve the host with `triage`.
pub(crate) async fn new(triage: Arc<ScriptedTriage>) -> Self {
+ Self::build(triage, None).await
+ }
+
+ /// Attach the module with `inbound`'s sources in its pipeline, and serve
+ /// the host with `pinned` running them.
+ pub(crate) async fn pinned(inbound: InboundConfig, pinned: Pinned) -> Self {
+ Self::build(ScriptedTriage::new(Vec::<Step>::new()), Some((inbound, pinned))).await
+ }
+
+ async fn build(triage: Arc<ScriptedTriage>, pinned: Option<(InboundConfig, Pinned)>) -> Self {
let bus = MemoryBus::new();
let broker = Broker::new();
broker.clone().spawn(bus.clone());
@@ -212,6 +329,14 @@ impl Rig {
Duration::from_secs(5),
)
.with_max_attempts(|_| Some(5));
+ let mut pipeline = fast_pipeline();
+ let deliverer = match pinned {
+ Some((inbound, pinned)) => {
+ pipeline = super::settings::with_sources(pipeline, &inbound);
+ deliverer.with_inbound(inbound).with_pinned(pinned)
+ }
+ None => deliverer,
+ };
host.serve_at(
ObjectPath::new(names::host::OBJECT_PATH).unwrap(),
InboundHost::new(Arc::new(deliverer)),
@@ -225,7 +350,7 @@ impl Rig {
.attach_linked_with_config(
tinyeventqueues::linked_module().unwrap(),
json!({
- "pipeline": fast_pipeline(),
+ "pipeline": pipeline,
"timer": { "min_interval_ms": 5, "max_interval_ms": 25 },
}),
)
diff --git a/crates/openhuman-core/src/inbound/pinned_tests.rs b/crates/openhuman-core/src/inbound/pinned_tests.rs
new file mode 100644
index 0000000000..a88de9570d
--- /dev/null
+++ b/crates/openhuman-core/src/inbound/pinned_tests.rs
@@ -0,0 +1,75 @@
+//! Tests for running a pinned source's batch: the prompt, and resuming the
+//! runs a previous process left unfinished.
+
+use std::sync::Arc;
+use std::time::Duration;
+
+use serde_json::json;
+
+use super::*;
+use crate::inbound::fixture::{memory_results, until_finished, Reply, ScriptedAgent};
+use crate::inbound::results::{ResultStatus, Stored};
+
+fn item(id: &str, text: &str) -> ExternalItem {
+ ExternalItem {
+ id: id.to_owned(),
+ text: text.to_owned(),
+ payload: json!({ "author": "@a" }),
+ received_at_ms: 7,
+ }
+}
+
+#[test]
+fn the_default_prompt_is_an_instruction_and_the_batch_as_json() {
+ let source = InboundSourceConfig::default();
+ let prompt = render_prompt("x:mentions", "mentions", &source, &[item("1", "hi")]);
+ assert!(prompt.contains("x:mentions"), "{prompt}");
+ let json_start = prompt.find('[').unwrap();
+ let events: serde_json::Value = serde_json::from_str(&prompt[json_start..]).unwrap();
+ assert_eq!(
+ events,
+ json!([{ "id": "1", "text": "hi", "payload": { "author": "@a" }, "received_at_ms": 7 }])
+ );
+}
+
+#[test]
+fn a_configured_prompt_takes_the_batch_in_place_of_its_placeholder() {
+ let source = InboundSourceConfig {
+ prompt: Some("Draft replies.\n{{events}}\nBe brief.".to_owned()),
+ ..InboundSourceConfig::default()
+ };
+ let prompt = render_prompt("x:mentions", "mentions", &source, &[item("1", "a"), item("2", "b")]);
+ assert!(prompt.starts_with("Draft replies.\n[{"), "{prompt}");
+ assert!(prompt.ends_with("}]\nBe brief."), "{prompt}");
+ assert!(!prompt.contains("{{events}}"));
+}
+
+#[tokio::test]
+async fn runs_left_running_by_a_previous_process_are_run_again() {
+ let results = memory_results();
+ results
+ .start(&Stored::running(
+ "t-left",
+ "x:mentions",
+ "mentions",
+ "teeny-mentions",
+ vec!["1".into()],
+ "the stored message",
+ 5,
+ ))
+ .unwrap();
+ let agent = ScriptedAgent::new([Reply::Text("resumed")]);
+ let pinned = Pinned::new(agent.clone() as Arc<dyn PinnedAgent>, results.clone());
+ assert_eq!(pinned.resume(&DispatchCtx::host_default()), 1);
+ let done = until_finished(&results, "t-left", Duration::from_secs(5))
+ .await
+ .unwrap();
+ assert_eq!(done.result.status, ResultStatus::Completed);
+ assert_eq!(done.result.text.as_deref(), Some("resumed"));
+ let runs = agent.runs();
+ assert_eq!(runs.len(), 1);
+ assert_eq!(runs[0].message, "the stored message");
+ assert_eq!(runs[0].agent_id, "teeny-mentions");
+ // Nothing is left to resume.
+ assert_eq!(pinned.resume(&DispatchCtx::host_default()), 0);
+}