Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

feat(config): add inbound source configuration support

Steven Enamakel69dfe15f7dbf
diff --git a/crates/openhuman-core/src/config/schema/inbound.rs b/crates/openhuman-core/src/config/schema/inbound.rs
index 425a53c3de..c36e766609 100644
--- a/crates/openhuman-core/src/config/schema/inbound.rs
+++ b/crates/openhuman-core/src/config/schema/inbound.rs
@@ -16,12 +16,16 @@
//! | `pipeline_path` | a `pipeline.json` replacing the shipped one |
//! | `queue_url` | where the queues live: `storage` (default) or a broker URL |
//! | `triage_timeout_ms` | how long one delivered event's triage may take (at most 25 s) |
+//! | `sources.<name>` | an external source pushed over `inbound.offer`, batched and run on a pinned agent ([`InboundSourceConfig`]) |
+use std::collections::BTreeMap;
use std::path::PathBuf;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
+pub use super::inbound_sources::InboundSourceConfig;
+
/// The longest one delivered event's triage may take. The module waits 30 s
/// for a host callback before it treats the delivery as failed and retries
/// it; a triage still running then would decide (and act) a second time on
@@ -51,6 +55,10 @@ pub struct InboundConfig {
/// before the delivery is retried. Capped at [`MAX_TRIAGE_TIMEOUT_MS`];
/// read when the host callbacks are served (the next core start).
pub triage_timeout_ms: u64,
+ /// External sources pushed over `inbound.offer`, by source name
+ /// (`<kind>:<name>`, e.g. `x:mentions`): each is debounced into batches
+ /// and run on its pinned agent, without triage.
+ pub sources: BTreeMap<String, InboundSourceConfig>,
}
impl InboundConfig {
@@ -74,6 +82,7 @@ impl std::fmt::Debug for InboundConfig {
&self.queue_url.as_deref().map(super::storage::redact_url),
)
.field("triage_timeout_ms", &self.triage_timeout_ms)
+ .field("sources", &self.sources)
.finish()
}
}
@@ -86,6 +95,7 @@ impl Default for InboundConfig {
pipeline_path: None,
queue_url: None,
triage_timeout_ms: DEFAULT_TRIAGE_TIMEOUT_MS,
+ sources: BTreeMap::new(),
}
}
}
diff --git a/crates/openhuman-core/src/config/schema/inbound_sources.rs b/crates/openhuman-core/src/config/schema/inbound_sources.rs
new file mode 100644
index 0000000000..758a5b7f40
--- /dev/null
+++ b/crates/openhuman-core/src/config/schema/inbound_sources.rs
@@ -0,0 +1,147 @@
+//! `[inbound.sources.<name>]` — an external source pushed into the inbound
+//! event queues over `inbound.offer` and handled by a pinned agent.
+//!
+//! The queues debounce a source's events per key into batches; each batch is
+//! one turn of `agent_id`, with no triage, and the agent's final text is kept
+//! as a result the pusher reads over `inbound.results` and acknowledges with
+//! `inbound.result_ack`.
+//!
+//! | setting | default | meaning |
+//! | --- | --- | --- |
+//! | `agent_id` | required | the agent every batch runs on |
+//! | `debounce_ms` | 5000 | flush a batch this long after its last event |
+//! | `max_wait_ms` | 60000 | flush a batch this long after its first event at the latest |
+//! | `max_batch` | 25 | the most events one turn carries |
+//! | `prompt` | a short instruction | the turn's message; `{{events}}` becomes the batch as JSON |
+//! | `result_ttl_days` | 7 | how long a result is kept |
+//!
+//! A source name is `<kind>:<name>` (`^[a-z0-9_-]+:[a-z0-9_.*-]+$`); the
+//! prefixes the host's own inbound paths use (`composio:`, `webhook:`, `web:`,
+//! `channel:`) are reserved.
+
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+
+use super::InboundConfig;
+
+pub const DEFAULT_DEBOUNCE_MS: u64 = 5_000;
+pub const DEFAULT_MAX_WAIT_MS: u64 = 60_000;
+pub const DEFAULT_MAX_BATCH: u32 = 25;
+pub const DEFAULT_RESULT_TTL_DAYS: u32 = 7;
+
+/// Source prefixes the host's own inbound paths own.
+const RESERVED_PREFIXES: [&str; 4] = ["composio:", "webhook:", "web:", "channel:"];
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
+#[serde(default)]
+pub struct InboundSourceConfig {
+ /// The agent every batch of this source runs on. Required.
+ pub agent_id: String,
+ /// Flush a batch this long after its last event, in milliseconds.
+ pub debounce_ms: u64,
+ /// Flush a batch this long after its first event at the latest.
+ pub max_wait_ms: u64,
+ /// The most events one turn carries.
+ pub max_batch: u32,
+ /// The turn's message; `{{events}}` is replaced with the batch as a JSON
+ /// array of `{id, text, payload, received_at_ms}`. Unset, a short
+ /// instruction followed by the JSON.
+ pub prompt: Option<String>,
+ /// How long a turn's result is kept, in days.
+ pub result_ttl_days: u32,
+}
+
+impl Default for InboundSourceConfig {
+ fn default() -> Self {
+ Self {
+ agent_id: String::new(),
+ debounce_ms: DEFAULT_DEBOUNCE_MS,
+ max_wait_ms: DEFAULT_MAX_WAIT_MS,
+ max_batch: DEFAULT_MAX_BATCH,
+ prompt: None,
+ result_ttl_days: DEFAULT_RESULT_TTL_DAYS,
+ }
+ }
+}
+
+impl InboundSourceConfig {
+ /// `Ok` when the source can run.
+ ///
+ /// # Errors
+ ///
+ /// The first unusable setting, by name.
+ pub fn check(&self) -> Result<(), String> {
+ if self.agent_id.trim().is_empty() {
+ return Err("agent_id is required".to_owned());
+ }
+ if self.max_batch == 0 {
+ return Err("max_batch must be at least 1".to_owned());
+ }
+ Ok(())
+ }
+}
+
+/// `Ok` when `name` may name an external source.
+///
+/// # Errors
+///
+/// A name outside `^[a-z0-9_-]+:[a-z0-9_.*-]+$`, or under a reserved prefix.
+pub fn check_source_name(name: &str) -> Result<(), String> {
+ let kind_char = |c: char| c.is_ascii_lowercase() || c.is_ascii_digit() || matches!(c, '_' | '-');
+ let well_formed = name.split_once(':').is_some_and(|(kind, rest)| {
+ !kind.is_empty()
+ && !rest.is_empty()
+ && kind.chars().all(kind_char)
+ && rest.chars().all(|c| kind_char(c) || matches!(c, '.' | '*'))
+ });
+ if !well_formed {
+ return Err(format!(
+ "source name `{name}` must match ^[a-z0-9_-]+:[a-z0-9_.*-]+$"
+ ));
+ }
+ if let Some(prefix) = RESERVED_PREFIXES.iter().find(|p| name.starts_with(**p)) {
+ return Err(format!("source prefix `{prefix}` is reserved"));
+ }
+ Ok(())
+}
+
+impl InboundConfig {
+ /// The configuration of external source `name`.
+ ///
+ /// # Errors
+ ///
+ /// `name` is not configured, or its name or settings are unusable.
+ pub fn source(&self, name: &str) -> Result<&InboundSourceConfig, String> {
+ check_source_name(name)?;
+ let source = self
+ .sources
+ .get(name)
+ .ok_or_else(|| format!("inbound source `{name}` is not configured"))?;
+ source
+ .check()
+ .map_err(|error| format!("inbound source `{name}`: {error}"))?;
+ Ok(source)
+ }
+
+ /// The configured sources that can run, by name; an unusable one is
+ /// logged and left out.
+ pub fn usable_sources(&self) -> impl Iterator<Item = (&str, &InboundSourceConfig)> {
+ self.sources.keys().filter_map(|name| match self.source(name) {
+ Ok(source) => Some((name.as_str(), source)),
+ Err(error) => {
+ tracing::warn!(error = %error, "[inbound] unusable source left out");
+ None
+ }
+ })
+ }
+
+ /// Whether `source` is a usable external source, run on a pinned agent.
+ #[must_use]
+ pub fn is_pinned(&self, source: &str) -> bool {
+ self.source(source).is_ok()
+ }
+}
+
+#[cfg(test)]
+#[path = "inbound_sources_tests.rs"]
+mod tests;
diff --git a/crates/openhuman-core/src/config/schema/inbound_sources_tests.rs b/crates/openhuman-core/src/config/schema/inbound_sources_tests.rs
new file mode 100644
index 0000000000..08a5a045ef
--- /dev/null
+++ b/crates/openhuman-core/src/config/schema/inbound_sources_tests.rs
@@ -0,0 +1,85 @@
+use super::*;
+
+fn parsed(toml_text: &str) -> crate::config::schema::InboundConfig {
+ toml::from_str(toml_text).unwrap()
+}
+
+#[test]
+fn a_source_needs_only_its_agent() {
+ let inbound = parsed("[sources.\"x:mentions\"]\nagent_id = \"teeny-mentions\"\n");
+ let source = inbound.source("x:mentions").unwrap();
+ assert_eq!(source.agent_id, "teeny-mentions");
+ assert_eq!(source.debounce_ms, DEFAULT_DEBOUNCE_MS);
+ assert_eq!(source.max_wait_ms, DEFAULT_MAX_WAIT_MS);
+ assert_eq!(source.max_batch, DEFAULT_MAX_BATCH);
+ assert_eq!(source.result_ttl_days, DEFAULT_RESULT_TTL_DAYS);
+ assert_eq!(source.prompt, None);
+ assert_eq!(
+ (5_000, 60_000, 25, 7),
+ (
+ DEFAULT_DEBOUNCE_MS,
+ DEFAULT_MAX_WAIT_MS,
+ DEFAULT_MAX_BATCH,
+ DEFAULT_RESULT_TTL_DAYS
+ )
+ );
+}
+
+#[test]
+fn every_setting_is_read() {
+ let inbound = parsed(
+ "[sources.\"x:mentions\"]\nagent_id = \"a\"\ndebounce_ms = 30000\nmax_wait_ms = 300000\n\
+ max_batch = 20\nprompt = \"go {{events}}\"\nresult_ttl_days = 3\n",
+ );
+ let source = inbound.source("x:mentions").unwrap();
+ assert_eq!(
+ (source.debounce_ms, source.max_wait_ms, source.max_batch),
+ (30_000, 300_000, 20)
+ );
+ assert_eq!(source.prompt.as_deref(), Some("go {{events}}"));
+ assert_eq!(source.result_ttl_days, 3);
+}
+
+#[test]
+fn an_unconfigured_source_is_refused_by_name() {
+ let inbound = parsed("");
+ let refused = inbound.source("x:mentions").unwrap_err();
+ assert!(refused.contains("x:mentions"), "{refused}");
+ assert!(refused.contains("not configured"), "{refused}");
+}
+
+#[test]
+fn source_names_are_checked() {
+ for good in ["x:mentions", "x_1:a.b", "a-b:c*", "rss:feed-2"] {
+ assert!(check_source_name(good).is_ok(), "{good}");
+ }
+ for bad in [
+ "", "mentions", "X:mentions", "x:", ":m", "x:m:n", "x y:m", "x:m/n", "x:é",
+ ] {
+ assert!(check_source_name(bad).is_err(), "{bad}");
+ }
+ for reserved in ["composio:gmail", "webhook:t1", "web:chat", "channel:slack"] {
+ let refused = check_source_name(reserved).unwrap_err();
+ assert!(refused.contains("reserved"), "{reserved}: {refused}");
+ }
+}
+
+#[test]
+fn an_unusable_source_is_refused_and_left_out() {
+ let inbound = parsed(
+ "[sources.\"x:mentions\"]\nagent_id = \"a\"\n\
+ [sources.\"x:dms\"]\nagent_id = \" \"\n\
+ [sources.\"x:zero\"]\nagent_id = \"a\"\nmax_batch = 0\n\
+ [sources.\"composio:gmail\"]\nagent_id = \"a\"\n",
+ );
+ assert!(inbound.source("x:dms").unwrap_err().contains("agent_id"));
+ assert!(inbound.source("x:zero").unwrap_err().contains("max_batch"));
+ assert!(inbound
+ .source("composio:gmail")
+ .unwrap_err()
+ .contains("reserved"));
+ let usable: Vec<&str> = inbound.usable_sources().map(|(name, _)| name).collect();
+ assert_eq!(usable, ["x:mentions"]);
+ assert!(inbound.is_pinned("x:mentions"));
+ assert!(!inbound.is_pinned("x:dms"));
+}
diff --git a/crates/openhuman-core/src/config/schema/mod.rs b/crates/openhuman-core/src/config/schema/mod.rs
index 14feb3eac4..6e71e2204f 100644
--- a/crates/openhuman-core/src/config/schema/mod.rs
+++ b/crates/openhuman-core/src/config/schema/mod.rs
@@ -61,6 +61,7 @@ pub(crate) use load::CONFIG_OWNER_MISMATCH_MARKER;
pub mod claude_agent_sdk;
pub use claude_agent_sdk::ClaudeAgentSdkConfig;
mod inbound;
+mod inbound_sources;
mod local_ai;
mod memes;
pub(crate) mod memory;
@@ -97,7 +98,7 @@ pub use dashboard::{DashboardConfig, DiagramViewerConfig, EventStreamConfig, Mod
pub use identity_cost::{
BudgetAction, BudgetPeriod, BudgetPolicy, BudgetScope, CostConfig, ModelPricing,
};
-pub use inbound::InboundConfig;
+pub use inbound::{InboundConfig, InboundSourceConfig};
pub use inbound_debounce::{InboundDebounceConfig, InboundDebounceOverride};
pub use local_ai::{LocalAiConfig, LocalAiUsage};
pub use memes::MemesConfig;