The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.
Rust agent-orchestrationai-agentsai-assistantdesktopllmlocal-firstmcppersonal-aiprivacyrustsecond-braintauri
feat(inbound): add external source offer, results, and ack operations
Steven Enamakel6b038aff35df
diff --git a/crates/openhuman-core/src/inbound/ops.rs b/crates/openhuman-core/src/inbound/ops.rsindex 356c8e7711..1fa3f03b34 100644--- a/crates/openhuman-core/src/inbound/ops.rs+++ b/crates/openhuman-core/src/inbound/ops.rs@@ -1,7 +1,10 @@//! Operator surface of the inbound event queues: depths and counts, the-//! dead-letter queue, and redriving one item.+//! dead-letter queue, and redriving one item; and the external-source+//! surface: offering an event, reading pinned turns' results, acknowledging+//! one.-use serde::Serialize;+use serde::{Deserialize, Serialize};+use serde_json::Value;use tinybus::module::ModuleClient;use tinyeventqueues_bus::{DeadItem, StatsResponse};@@ -9,6 +12,8 @@ use crate::config::Config;use crate::core::Outcome;use crate::modules::event_queues;+use super::results::{InboundResult, ResultQuery, ResultStatus, Results};+/// The most dead letters one `inbound.dead` call returns.pub const MAX_DEAD_LIMIT: u32 = 500;@@ -97,6 +102,183 @@ pub async fn redrive())}+/// The most results one `inbound.results` call returns.+pub const MAX_RESULTS_LIMIT: usize = 500;++/// The parameters of `inbound.offer`.+#[derive(Debug, Clone, Deserialize)]+#[serde(deny_unknown_fields)]+pub struct OfferParams {+ /// A configured `[inbound.sources.*]` name.+ pub source: String,+ /// What the queues batch the source's events by.+ pub key: String,+ /// The source's own id; a redelivery is a `duplicate`.+ pub event_id: String,+ /// The event's text.+ pub text: String,+ /// The source's payload, handed to the agent as is.+ #[serde(default)]+ pub payload: Value,+ /// When the source received it; now when absent.+ #[serde(default)]+ pub received_at_ms: Option<u64>,+}++/// The result of `inbound.offer`.+#[derive(Debug, Clone, Serialize)]+pub struct Offer {+ /// What the queues did with the event: `accepted`, `duplicate`, ...+ pub disposition: String,+}++/// The parameters of `inbound.results`.+#[derive(Debug, Clone, Default, Deserialize)]+#[serde(deny_unknown_fields)]+pub struct ResultsParams {+ /// Only this source's results.+ #[serde(default)]+ pub source: Option<String>,+ /// Only results started at or after this time.+ #[serde(default)]+ pub since_ms: Option<u64>,+ /// At most this many (default 50, at most 500).+ #[serde(default)]+ pub limit: Option<usize>,+ /// Only results with this status.+ #[serde(default)]+ pub status: Option<ResultStatus>,+}++/// The result of `inbound.results`.+#[derive(Debug, Clone, Serialize)]+pub struct ResultList {+ /// Unacknowledged results, oldest first.+ pub results: Vec<InboundResult>,+}++/// The result of `inbound.result_ack`.+#[derive(Debug, Clone, Serialize)]+pub struct Acked {+ /// Always `true`: an unknown turn is an error.+ pub acked: bool,+}++pub(crate) fn now_ms() -> u64 {+ u64::try_from(chrono::Utc::now().timestamp_millis()).unwrap_or(0)+}++/// Offer one event of a configured external source to the queues. A+/// redelivery of an event already taken answers `duplicate`, a success.+///+/// # Errors+///+/// The component is off, the source is not configured (or unusable), a+/// field is empty, the event is too large, or the queues did not take it.+pub async fn offer(+ config: &Config,+ client: &ModuleClient,+ params: OfferParams,+) -> Result<Outcome<Offer>, String> {+ require(config)?;+ config+ .inbound+ .source(¶ms.source)+ .map_err(|error| format!("inbound.offer: {error}"))?;+ for (field, value) in [("key", ¶ms.key), ("event_id", ¶ms.event_id)] {+ if value.trim().is_empty() {+ return Err(format!("inbound.offer: {field} is required"));+ }+ }+ let event = super::events::external(+ ¶ms.source,+ ¶ms.key,+ ¶ms.event_id,+ ¶ms.text,+ params.payload,+ params.received_at_ms.unwrap_or_else(now_ms),+ )+ .map_err(|error| format!("inbound.offer: {error}"))?;+ match super::route::offer(config, client, event).await {+ super::route::Offered::Queued(disposition) => {+ let disposition = serde_json::to_value(disposition)+ .ok()+ .and_then(|value| value.as_str().map(str::to_owned))+ .unwrap_or_else(|| format!("{disposition:?}").to_lowercase());+ tracing::debug!(+ source = %params.source,+ event_id = %params.event_id,+ disposition = %disposition,+ "[inbound] offered"+ );+ Ok(Outcome::new(Offer { disposition }, Vec::new()))+ }+ super::route::Offered::Direct(why) => Err(format!(+ "inbound.offer: the inbound queues did not take the event: {why}"+ )),+ }+}++/// Unacknowledged results of pinned turns, oldest first, after pruning those+/// past their source's `result_ttl_days`.+///+/// # Errors+///+/// The component is off, or the store failed.+pub fn results(+ config: &Config,+ store: &Results,+ params: ResultsParams,+) -> Result<Outcome<ResultList>, String> {+ require(config)?;+ let ttl_days: std::collections::HashMap<String, u32> = config+ .inbound+ .sources+ .iter()+ .map(|(name, source)| (name.clone(), source.result_ttl_days))+ .collect();+ let default_days = crate::config::schema::InboundSourceConfig::default().result_ttl_days;+ let pruned = store+ .prune(now_ms(), move |source| {+ u64::from(*ttl_days.get(source).unwrap_or(&default_days)) * 24 * 60 * 60 * 1_000+ })+ .map_err(|error| format!("inbound.results: {error:#}"))?;+ if pruned > 0 {+ tracing::debug!(pruned, "[inbound] expired results pruned");+ }+ let query = ResultQuery {+ source: params.source,+ since_ms: params.since_ms,+ status: params.status,+ limit: params.limit.unwrap_or(50).min(MAX_RESULTS_LIMIT),+ };+ let results = store+ .list(&query)+ .map_err(|error| format!("inbound.results: {error:#}"))?;+ Ok(Outcome::new(ResultList { results }, Vec::new()))+}++/// Acknowledge pinned turn `turn_id`'s result: it is no longer listed.+///+/// # Errors+///+/// The component is off, the turn is unknown, or the store failed.+pub fn result_ack(+ config: &Config,+ store: &Results,+ turn_id: &str,+) -> Result<Outcome<Acked>, String> {+ require(config)?;+ let acked = store+ .ack(turn_id)+ .map_err(|error| format!("inbound.result_ack: {error:#}"))?;+ if !acked {+ return Err(format!("inbound.result_ack: unknown turn `{turn_id}`"));+ }+ tracing::debug!(turn = %turn_id, "[inbound] result acknowledged");+ Ok(Outcome::new(Acked { acked }, Vec::new()))+}+#[cfg(test)]#[path = "ops_tests.rs"]mod tests;diff --git a/crates/openhuman-core/src/inbound/schemas.rs b/crates/openhuman-core/src/inbound/schemas.rsindex 8ddbc7dbd5..408f825727 100644--- a/crates/openhuman-core/src/inbound/schemas.rs+++ b/crates/openhuman-core/src/inbound/schemas.rs@@ -1,6 +1,7 @@//! The `inbound` RPC namespace: `inbound.stats`, `inbound.dead`,-//! `inbound.redrive`. Each answers `COMPONENT_DISABLED:eventqueues` when the-//! component is off.+//! `inbound.redrive`, and for external sources `inbound.offer`,+//! `inbound.results` and `inbound.result_ack`. Each answers+//! `COMPONENT_DISABLED:eventqueues` when the component is off.use serde::de::DeserializeOwned;use serde::Deserialize;@@ -21,9 +22,17 @@ struct RedriveParams {id: String,}+#[derive(Debug, Deserialize)]+#[serde(deny_unknown_fields)]+struct ResultAckParams {+ turn_id: String,+}++const FUNCTIONS: [&str; 6] = ["stats", "dead", "redrive", "offer", "results", "result_ack"];+/// Every `inbound.*` schema.pub fn all_controller_schemas() -> Vec<ControllerSchema> {- vec![schemas("stats"), schemas("dead"), schemas("redrive")]+ FUNCTIONS.iter().map(|function| schemas(function)).collect()}/// Every `inbound.*` controller.@@ -41,6 +50,18 @@ pub fn all_registered_controllers() -> Vec<RegisteredController> {schema: schemas("redrive"),handler: handle_redrive,},+ RegisteredController {+ schema: schemas("offer"),+ handler: handle_offer,+ },+ RegisteredController {+ schema: schemas("results"),+ handler: handle_results,+ },+ RegisteredController {+ schema: schemas("result_ack"),+ handler: handle_result_ack,+ },]}@@ -82,6 +103,58 @@ pub fn schemas(function: &str) -> ControllerSchema {}],outputs: vec![json_output("result", "Whether the item was redriven.")],},+ "offer" => ControllerSchema {+ namespace: "inbound",+ function: "offer",+ description: "Offer one event of a configured external source \+ (`[inbound.sources.<source>]`) to the inbound queues. Events are \+ debounced per key into batches, each run as one turn of the \+ source's pinned agent. A redelivered `event_id` answers `duplicate`.",+ inputs: vec![+ string_input("source", "A configured source name, e.g. `x:mentions`.", true),+ string_input("key", "What the source's events are batched by.", true),+ string_input("event_id", "The source's own id; a redelivery is a duplicate.", true),+ string_input("text", "The event's text.", true),+ FieldSchema {+ name: "payload",+ ty: TypeSchema::Option(Box::new(TypeSchema::Json)),+ comment: "The source's payload, handed to the agent as is.",+ required: false,+ },+ u64_input("received_at_ms", "When the source received it (default now)."),+ ],+ outputs: vec![json_output(+ "result",+ "`{disposition}`: `accepted`, `duplicate`, or why it ended at ingress.",+ )],+ },+ "results" => ControllerSchema {+ namespace: "inbound",+ function: "results",+ description: "List unacknowledged results of pinned-agent turns, oldest first.",+ inputs: vec![+ string_input("source", "Only this source's results.", false),+ u64_input("since_ms", "Only results started at or after this time."),+ u64_input("limit", "Maximum number of results (default 50, at most 500)."),+ string_input("status", "`running`, `completed` or `failed`.", false),+ ],+ outputs: vec![json_output(+ "result",+ "`{results: [{turn_id, source, key, agent_id, status, text?, error?, \+ event_ids, created_ms, completed_ms?}]}`.",+ )],+ },+ "result_ack" => ControllerSchema {+ namespace: "inbound",+ function: "result_ack",+ description: "Acknowledge one pinned-agent result; it is no longer listed.",+ inputs: vec![string_input(+ "turn_id",+ "The result's turn id, as `inbound.results` lists it.",+ true,+ )],+ outputs: vec![json_output("result", "`{acked: true}`.")],+ },_ => ControllerSchema {namespace: "inbound",function: "unknown",@@ -123,6 +196,62 @@ fn handle_redrive(ctx: DispatchCtx, params: Map<String, Value>) -> ControllerFut})}+fn handle_offer(ctx: DispatchCtx, params: Map<String, Value>) -> ControllerFuture {+ Box::pin(async move {+ let params = deserialize_params::<super::ops::OfferParams>(params)?;+ let (config, client) = config_and_client(&ctx).await?;+ to_json(super::ops::offer(&config, &client, params).await?)+ })+}++fn handle_results(ctx: DispatchCtx, params: Map<String, Value>) -> ControllerFuture {+ Box::pin(async move {+ let params = deserialize_params::<super::ops::ResultsParams>(params)?;+ let config = ctx.load_config().await?;+ let store = results_store(&config)?;+ to_json(super::ops::results(&config, &store, params)?)+ })+}++fn handle_result_ack(ctx: DispatchCtx, params: Map<String, Value>) -> ControllerFuture {+ Box::pin(async move {+ let params = deserialize_params::<ResultAckParams>(params)?;+ let config = ctx.load_config().await?;+ let store = results_store(&config)?;+ to_json(super::ops::result_ack(&config, &store, ¶ms.turn_id)?)+ })+}++fn results_store(config: &crate::config::Config) -> Result<super::results::Results, String> {+ crate::core::runtime::components::require_config(config, super::settings::COMPONENT)?;+ super::results::Results::open(config).map_err(|error| format!("inbound results: {error:#}"))+}++fn string_input(name: &'static str, comment: &'static str, required: bool) -> FieldSchema {+ FieldSchema {+ name,+ ty: if required {+ TypeSchema::String+ } else {+ TypeSchema::Option(Box::new(TypeSchema::String))+ },+ comment,+ required,+ }+}++fn u64_input(name: &'static str, comment: &'static str) -> FieldSchema {+ FieldSchema {+ name,+ ty: TypeSchema::Option(Box::new(TypeSchema::BoundedU64 {+ min: 0,+ max: u64::MAX,+ })),+ comment,+ required: false,+ }+}+fn deserialize_params<T: DeserializeOwned>(params: Map<String, Value>) -> Result<T, String> {serde_json::from_value(Value::Object(params)).map_err(|e| format!("invalid params: {e}"))}diff --git a/crates/openhuman-core/src/modules/boot.rs b/crates/openhuman-core/src/modules/boot.rsindex 6dae3b227f..248d693b8c 100644--- a/crates/openhuman-core/src/modules/boot.rs+++ b/crates/openhuman-core/src/modules/boot.rs@@ -100,6 +100,9 @@ fn warm_inbound_queues(config: &Config) {// Loading a module is process-wide; the host's own context carries it.crate::core::runtime::DispatchCtx::host_default().spawn(async move {log::debug!("[modules] inbound event queues warm-up starting");+ // Pinned turns a previous process left running are run again; the+ // queues acknowledged them, so nothing else would.+ crate::inbound::pinned::resume_at_boot(&config);if let Err(reason) = ops::ensure_loaded(&config, super::event_queues::MODULE_ID).await {log::warn!("[modules] inbound event queues did not load at boot: {reason}");}diff --git a/crates/openhuman-core/src/storage/schema.rs b/crates/openhuman-core/src/storage/schema.rsindex 6ab5580b56..50d4b0ea71 100644--- a/crates/openhuman-core/src/storage/schema.rs+++ b/crates/openhuman-core/src/storage/schema.rs@@ -56,7 +56,10 @@ pub fn owner_schemas() -> Vec<Schema> {#[cfg(feature = "tinymemes")]schemas.push(tinymemes_bus::storage::storage_schema());#[cfg(feature = "inbound-queues")]- schemas.push(tinyeventqueues_bus::storage::storage_schema());+ {+ schemas.push(tinyeventqueues_bus::storage::storage_schema());+ schemas.push(crate::inbound::results::storage_schema());+ }schemas}