Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

feat(inbound): add external source offer, results, and ack operations

Steven Enamakel6b038aff35df
diff --git a/crates/openhuman-core/src/inbound/ops.rs b/crates/openhuman-core/src/inbound/ops.rs
index 356c8e7711..1fa3f03b34 100644
--- a/crates/openhuman-core/src/inbound/ops.rs
+++ b/crates/openhuman-core/src/inbound/ops.rs
@@ -1,7 +1,10 @@
//! Operator surface of the inbound event queues: depths and counts, the
-//! dead-letter queue, and redriving one item.
+//! dead-letter queue, and redriving one item; and the external-source
+//! surface: offering an event, reading pinned turns' results, acknowledging
+//! one.
-use serde::Serialize;
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
use tinybus::module::ModuleClient;
use tinyeventqueues_bus::{DeadItem, StatsResponse};
@@ -9,6 +12,8 @@ use crate::config::Config;
use crate::core::Outcome;
use crate::modules::event_queues;
+use super::results::{InboundResult, ResultQuery, ResultStatus, Results};
+
/// The most dead letters one `inbound.dead` call returns.
pub const MAX_DEAD_LIMIT: u32 = 500;
@@ -97,6 +102,183 @@ pub async fn redrive(
))
}
+/// The most results one `inbound.results` call returns.
+pub const MAX_RESULTS_LIMIT: usize = 500;
+
+/// The parameters of `inbound.offer`.
+#[derive(Debug, Clone, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct OfferParams {
+ /// A configured `[inbound.sources.*]` name.
+ pub source: String,
+ /// What the queues batch the source's events by.
+ pub key: String,
+ /// The source's own id; a redelivery is a `duplicate`.
+ pub event_id: String,
+ /// The event's text.
+ pub text: String,
+ /// The source's payload, handed to the agent as is.
+ #[serde(default)]
+ pub payload: Value,
+ /// When the source received it; now when absent.
+ #[serde(default)]
+ pub received_at_ms: Option<u64>,
+}
+
+/// The result of `inbound.offer`.
+#[derive(Debug, Clone, Serialize)]
+pub struct Offer {
+ /// What the queues did with the event: `accepted`, `duplicate`, ...
+ pub disposition: String,
+}
+
+/// The parameters of `inbound.results`.
+#[derive(Debug, Clone, Default, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct ResultsParams {
+ /// Only this source's results.
+ #[serde(default)]
+ pub source: Option<String>,
+ /// Only results started at or after this time.
+ #[serde(default)]
+ pub since_ms: Option<u64>,
+ /// At most this many (default 50, at most 500).
+ #[serde(default)]
+ pub limit: Option<usize>,
+ /// Only results with this status.
+ #[serde(default)]
+ pub status: Option<ResultStatus>,
+}
+
+/// The result of `inbound.results`.
+#[derive(Debug, Clone, Serialize)]
+pub struct ResultList {
+ /// Unacknowledged results, oldest first.
+ pub results: Vec<InboundResult>,
+}
+
+/// The result of `inbound.result_ack`.
+#[derive(Debug, Clone, Serialize)]
+pub struct Acked {
+ /// Always `true`: an unknown turn is an error.
+ pub acked: bool,
+}
+
+pub(crate) fn now_ms() -> u64 {
+ u64::try_from(chrono::Utc::now().timestamp_millis()).unwrap_or(0)
+}
+
+/// Offer one event of a configured external source to the queues. A
+/// redelivery of an event already taken answers `duplicate`, a success.
+///
+/// # Errors
+///
+/// The component is off, the source is not configured (or unusable), a
+/// field is empty, the event is too large, or the queues did not take it.
+pub async fn offer(
+ config: &Config,
+ client: &ModuleClient,
+ params: OfferParams,
+) -> Result<Outcome<Offer>, String> {
+ require(config)?;
+ config
+ .inbound
+ .source(&params.source)
+ .map_err(|error| format!("inbound.offer: {error}"))?;
+ for (field, value) in [("key", &params.key), ("event_id", &params.event_id)] {
+ if value.trim().is_empty() {
+ return Err(format!("inbound.offer: {field} is required"));
+ }
+ }
+ let event = super::events::external(
+ &params.source,
+ &params.key,
+ &params.event_id,
+ &params.text,
+ params.payload,
+ params.received_at_ms.unwrap_or_else(now_ms),
+ )
+ .map_err(|error| format!("inbound.offer: {error}"))?;
+ match super::route::offer(config, client, event).await {
+ super::route::Offered::Queued(disposition) => {
+ let disposition = serde_json::to_value(disposition)
+ .ok()
+ .and_then(|value| value.as_str().map(str::to_owned))
+ .unwrap_or_else(|| format!("{disposition:?}").to_lowercase());
+ tracing::debug!(
+ source = %params.source,
+ event_id = %params.event_id,
+ disposition = %disposition,
+ "[inbound] offered"
+ );
+ Ok(Outcome::new(Offer { disposition }, Vec::new()))
+ }
+ super::route::Offered::Direct(why) => Err(format!(
+ "inbound.offer: the inbound queues did not take the event: {why}"
+ )),
+ }
+}
+
+/// Unacknowledged results of pinned turns, oldest first, after pruning those
+/// past their source's `result_ttl_days`.
+///
+/// # Errors
+///
+/// The component is off, or the store failed.
+pub fn results(
+ config: &Config,
+ store: &Results,
+ params: ResultsParams,
+) -> Result<Outcome<ResultList>, String> {
+ require(config)?;
+ let ttl_days: std::collections::HashMap<String, u32> = config
+ .inbound
+ .sources
+ .iter()
+ .map(|(name, source)| (name.clone(), source.result_ttl_days))
+ .collect();
+ let default_days = crate::config::schema::InboundSourceConfig::default().result_ttl_days;
+ let pruned = store
+ .prune(now_ms(), move |source| {
+ u64::from(*ttl_days.get(source).unwrap_or(&default_days)) * 24 * 60 * 60 * 1_000
+ })
+ .map_err(|error| format!("inbound.results: {error:#}"))?;
+ if pruned > 0 {
+ tracing::debug!(pruned, "[inbound] expired results pruned");
+ }
+ let query = ResultQuery {
+ source: params.source,
+ since_ms: params.since_ms,
+ status: params.status,
+ limit: params.limit.unwrap_or(50).min(MAX_RESULTS_LIMIT),
+ };
+ let results = store
+ .list(&query)
+ .map_err(|error| format!("inbound.results: {error:#}"))?;
+ Ok(Outcome::new(ResultList { results }, Vec::new()))
+}
+
+/// Acknowledge pinned turn `turn_id`'s result: it is no longer listed.
+///
+/// # Errors
+///
+/// The component is off, the turn is unknown, or the store failed.
+pub fn result_ack(
+ config: &Config,
+ store: &Results,
+ turn_id: &str,
+) -> Result<Outcome<Acked>, String> {
+ require(config)?;
+ let acked = store
+ .ack(turn_id)
+ .map_err(|error| format!("inbound.result_ack: {error:#}"))?;
+ if !acked {
+ return Err(format!("inbound.result_ack: unknown turn `{turn_id}`"));
+ }
+ tracing::debug!(turn = %turn_id, "[inbound] result acknowledged");
+ Ok(Outcome::new(Acked { acked }, Vec::new()))
+}
+
#[cfg(test)]
#[path = "ops_tests.rs"]
mod tests;
diff --git a/crates/openhuman-core/src/inbound/schemas.rs b/crates/openhuman-core/src/inbound/schemas.rs
index 8ddbc7dbd5..408f825727 100644
--- a/crates/openhuman-core/src/inbound/schemas.rs
+++ b/crates/openhuman-core/src/inbound/schemas.rs
@@ -1,6 +1,7 @@
//! The `inbound` RPC namespace: `inbound.stats`, `inbound.dead`,
-//! `inbound.redrive`. Each answers `COMPONENT_DISABLED:eventqueues` when the
-//! component is off.
+//! `inbound.redrive`, and for external sources `inbound.offer`,
+//! `inbound.results` and `inbound.result_ack`. Each answers
+//! `COMPONENT_DISABLED:eventqueues` when the component is off.
use serde::de::DeserializeOwned;
use serde::Deserialize;
@@ -21,9 +22,17 @@ struct RedriveParams {
id: String,
}
+#[derive(Debug, Deserialize)]
+#[serde(deny_unknown_fields)]
+struct ResultAckParams {
+ turn_id: String,
+}
+
+const FUNCTIONS: [&str; 6] = ["stats", "dead", "redrive", "offer", "results", "result_ack"];
+
/// Every `inbound.*` schema.
pub fn all_controller_schemas() -> Vec<ControllerSchema> {
- vec![schemas("stats"), schemas("dead"), schemas("redrive")]
+ FUNCTIONS.iter().map(|function| schemas(function)).collect()
}
/// Every `inbound.*` controller.
@@ -41,6 +50,18 @@ pub fn all_registered_controllers() -> Vec<RegisteredController> {
schema: schemas("redrive"),
handler: handle_redrive,
},
+ RegisteredController {
+ schema: schemas("offer"),
+ handler: handle_offer,
+ },
+ RegisteredController {
+ schema: schemas("results"),
+ handler: handle_results,
+ },
+ RegisteredController {
+ schema: schemas("result_ack"),
+ handler: handle_result_ack,
+ },
]
}
@@ -82,6 +103,58 @@ pub fn schemas(function: &str) -> ControllerSchema {
}],
outputs: vec![json_output("result", "Whether the item was redriven.")],
},
+ "offer" => ControllerSchema {
+ namespace: "inbound",
+ function: "offer",
+ description: "Offer one event of a configured external source \
+ (`[inbound.sources.<source>]`) to the inbound queues. Events are \
+ debounced per key into batches, each run as one turn of the \
+ source's pinned agent. A redelivered `event_id` answers `duplicate`.",
+ inputs: vec![
+ string_input("source", "A configured source name, e.g. `x:mentions`.", true),
+ string_input("key", "What the source's events are batched by.", true),
+ string_input("event_id", "The source's own id; a redelivery is a duplicate.", true),
+ string_input("text", "The event's text.", true),
+ FieldSchema {
+ name: "payload",
+ ty: TypeSchema::Option(Box::new(TypeSchema::Json)),
+ comment: "The source's payload, handed to the agent as is.",
+ required: false,
+ },
+ u64_input("received_at_ms", "When the source received it (default now)."),
+ ],
+ outputs: vec![json_output(
+ "result",
+ "`{disposition}`: `accepted`, `duplicate`, or why it ended at ingress.",
+ )],
+ },
+ "results" => ControllerSchema {
+ namespace: "inbound",
+ function: "results",
+ description: "List unacknowledged results of pinned-agent turns, oldest first.",
+ inputs: vec![
+ string_input("source", "Only this source's results.", false),
+ u64_input("since_ms", "Only results started at or after this time."),
+ u64_input("limit", "Maximum number of results (default 50, at most 500)."),
+ string_input("status", "`running`, `completed` or `failed`.", false),
+ ],
+ outputs: vec![json_output(
+ "result",
+ "`{results: [{turn_id, source, key, agent_id, status, text?, error?, \
+ event_ids, created_ms, completed_ms?}]}`.",
+ )],
+ },
+ "result_ack" => ControllerSchema {
+ namespace: "inbound",
+ function: "result_ack",
+ description: "Acknowledge one pinned-agent result; it is no longer listed.",
+ inputs: vec![string_input(
+ "turn_id",
+ "The result's turn id, as `inbound.results` lists it.",
+ true,
+ )],
+ outputs: vec![json_output("result", "`{acked: true}`.")],
+ },
_ => ControllerSchema {
namespace: "inbound",
function: "unknown",
@@ -123,6 +196,62 @@ fn handle_redrive(ctx: DispatchCtx, params: Map<String, Value>) -> ControllerFut
})
}
+fn handle_offer(ctx: DispatchCtx, params: Map<String, Value>) -> ControllerFuture {
+ Box::pin(async move {
+ let params = deserialize_params::<super::ops::OfferParams>(params)?;
+ let (config, client) = config_and_client(&ctx).await?;
+ to_json(super::ops::offer(&config, &client, params).await?)
+ })
+}
+
+fn handle_results(ctx: DispatchCtx, params: Map<String, Value>) -> ControllerFuture {
+ Box::pin(async move {
+ let params = deserialize_params::<super::ops::ResultsParams>(params)?;
+ let config = ctx.load_config().await?;
+ let store = results_store(&config)?;
+ to_json(super::ops::results(&config, &store, params)?)
+ })
+}
+
+fn handle_result_ack(ctx: DispatchCtx, params: Map<String, Value>) -> ControllerFuture {
+ Box::pin(async move {
+ let params = deserialize_params::<ResultAckParams>(params)?;
+ let config = ctx.load_config().await?;
+ let store = results_store(&config)?;
+ to_json(super::ops::result_ack(&config, &store, &params.turn_id)?)
+ })
+}
+
+fn results_store(config: &crate::config::Config) -> Result<super::results::Results, String> {
+ crate::core::runtime::components::require_config(config, super::settings::COMPONENT)?;
+ super::results::Results::open(config).map_err(|error| format!("inbound results: {error:#}"))
+}
+
+fn string_input(name: &'static str, comment: &'static str, required: bool) -> FieldSchema {
+ FieldSchema {
+ name,
+ ty: if required {
+ TypeSchema::String
+ } else {
+ TypeSchema::Option(Box::new(TypeSchema::String))
+ },
+ comment,
+ required,
+ }
+}
+
+fn u64_input(name: &'static str, comment: &'static str) -> FieldSchema {
+ FieldSchema {
+ name,
+ ty: TypeSchema::Option(Box::new(TypeSchema::BoundedU64 {
+ min: 0,
+ max: u64::MAX,
+ })),
+ comment,
+ required: false,
+ }
+}
+
fn deserialize_params<T: DeserializeOwned>(params: Map<String, Value>) -> Result<T, String> {
serde_json::from_value(Value::Object(params)).map_err(|e| format!("invalid params: {e}"))
}
diff --git a/crates/openhuman-core/src/modules/boot.rs b/crates/openhuman-core/src/modules/boot.rs
index 6dae3b227f..248d693b8c 100644
--- a/crates/openhuman-core/src/modules/boot.rs
+++ b/crates/openhuman-core/src/modules/boot.rs
@@ -100,6 +100,9 @@ fn warm_inbound_queues(config: &Config) {
// Loading a module is process-wide; the host's own context carries it.
crate::core::runtime::DispatchCtx::host_default().spawn(async move {
log::debug!("[modules] inbound event queues warm-up starting");
+ // Pinned turns a previous process left running are run again; the
+ // queues acknowledged them, so nothing else would.
+ crate::inbound::pinned::resume_at_boot(&config);
if let Err(reason) = ops::ensure_loaded(&config, super::event_queues::MODULE_ID).await {
log::warn!("[modules] inbound event queues did not load at boot: {reason}");
}
diff --git a/crates/openhuman-core/src/storage/schema.rs b/crates/openhuman-core/src/storage/schema.rs
index 6ab5580b56..50d4b0ea71 100644
--- a/crates/openhuman-core/src/storage/schema.rs
+++ b/crates/openhuman-core/src/storage/schema.rs
@@ -56,7 +56,10 @@ pub fn owner_schemas() -> Vec<Schema> {
#[cfg(feature = "tinymemes")]
schemas.push(tinymemes_bus::storage::storage_schema());
#[cfg(feature = "inbound-queues")]
- schemas.push(tinyeventqueues_bus::storage::storage_schema());
+ {
+ schemas.push(tinyeventqueues_bus::storage::storage_schema());
+ schemas.push(crate::inbound::results::storage_schema());
+ }
schemas
}