The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.
Rust agent-orchestrationai-agentsai-assistantdesktopllmlocal-firstmcppersonal-aiprivacyrustsecond-braintauri
feat(modules): load verified module artifacts with host callbacks
Steven Enamakelac8cb3a05a03
diff --git a/crates/openhuman-core/src/inbound/linked_fixture.rs b/crates/openhuman-core/src/inbound/linked_fixture.rsindex 2e1a066541..0d2b8472d9 100644--- a/crates/openhuman-core/src/inbound/linked_fixture.rs+++ b/crates/openhuman-core/src/inbound/linked_fixture.rs@@ -1,8 +1,8 @@-//! The `tinyeventqueues` module linked into the test binary on a private bus,-//! with the host callbacks served by a scripted triage runner.+//! The staged `tinyeventqueues` artifact on a private bus, with the host+//! callbacks served by a scripted triage runner.//!-//! Each [`Rig`] is its own broker and module host: the real module (its-//! `static-link` build) is attached with fast timers, the host's+//! Each [`Rig`] is its own broker and module host: the verified release module+//! is loaded with fast timers and a per-module storage callback, the host's//! `ai.tinyhumans.tinyeventqueues.Host` is served by an [`InboundHost`] whose//! [`Deliverer`] triages with a [`ScriptedTriage`], and [`Rig::client`] is a//! module client bound to the module's object, as `route::offer` takes it.@@ -299,6 +299,8 @@ pub(crate) struct Rig {_host: Connection,_modules: ModuleHost,_storage: crate::storage::test_support::InstalledBackendGuard,+ _callback_binding: Arc<tinybus::module::HostCallbackBinding>,+ _callback_storage: tinystoragedrivers_busproxy::ServedStorage,}impl Rig {@@ -353,15 +355,20 @@ impl Rig {crate::modules::storage_host::install(&host, &modules).await.unwrap();+ let artifact = crate::modules::ops::local_artifact_for_test("tinyeventqueues")+ .expect("the staged tinyeventqueues artifact is required for callback tests");+ let (binding, callback_storage) =+ crate::modules::storage_host::prepare_callbacks(&host, &modules)+ .await+ .unwrap();+ let config = json!({+ "pipeline": pipeline,+ "timer": { "min_interval_ms": 5, "max_interval_ms": 25 },+ "storage_callback": binding.unique_name().as_str(),+ });modules- .attach_linked_with_config(- tinyeventqueues::linked_module().unwrap(),- json!({- "pipeline": pipeline,- "timer": { "min_interval_ms": 5, "max_interval_ms": 25 },- "storage_callback": tinystoragedrivers_bus::STORAGE_INTERFACE,- }),- )+ .load_file_with_host_callbacks(&artifact, config, &binding)+ .await.unwrap();tokio::time::timeout(Duration::from_secs(10), async {loop {@@ -383,6 +390,8 @@ impl Rig {_host: host,_modules: modules,_storage: storage,+ _callback_binding: binding,+ _callback_storage: callback_storage,}}}diff --git a/crates/openhuman-core/src/inbound/settings.rs b/crates/openhuman-core/src/inbound/settings.rsindex 00d15569f5..421e23d921 100644--- a/crates/openhuman-core/src/inbound/settings.rs+++ b/crates/openhuman-core/src/inbound/settings.rs@@ -1,12 +1,12 @@//! What `[inbound]` resolves to: the queue manifest and pipeline configuration//! the module runs, where its queues live, and whether routing is on.//!-//! The host hands the module its configuration (`queues`, `pipeline`, and the-//! storage callback) in the module's private blob when it loads it-//! ([`module_config`]), and declares the same manifest on the module's storage-//! scope when the storage host binds it ([`queue_grant`]). Both read the last-//! resolution, so the queues the module asks for are exactly the queues the-//! host serves.+//! The host hands the module its queue and pipeline configuration in the+//! module's private blob when it loads it ([`module_config`]), and declares the+//! same manifest on the module's storage scope when the storage host binds it+//! ([`queue_grant`]). TinyBus injects the unique storage callback destination+//! during verified module loading. Both read the last resolution, so the queues+//! the module asks for are exactly the queues the host serves.use std::path::Path;use std::sync::{Arc, RwLock};@@ -221,7 +221,6 @@ pub fn module_config(config: &Config) -> Value {json!({"queues": resolved.queues_json,"pipeline": resolved.pipeline_json,- "storage_callback": tinystoragedrivers_bus::STORAGE_INTERFACE,})}diff --git a/crates/openhuman-core/src/inbound/settings_tests.rs b/crates/openhuman-core/src/inbound/settings_tests.rsindex 705bb27af4..0c909e2fd2 100644--- a/crates/openhuman-core/src/inbound/settings_tests.rs+++ b/crates/openhuman-core/src/inbound/settings_tests.rs@@ -112,10 +112,7 @@ async fn the_module_config_is_what_the_storage_grant_declares() {let blob = module_config(&config);assert_eq!(blob["queues"]["queues"][0]["max_attempts"], 4);assert_eq!(blob["pipeline"]["version"], 1);- assert_eq!(- blob["storage_callback"],- tinystoragedrivers_bus::STORAGE_INTERFACE- );+ assert!(blob.get("storage_callback").is_none());let grant = queue_grant().unwrap();assert_eq!(grantdiff --git a/crates/openhuman-core/src/modules/ops.rs b/crates/openhuman-core/src/modules/ops.rsindex d2ff5ed43e..8ca7bd2224 100644--- a/crates/openhuman-core/src/modules/ops.rs+++ b/crates/openhuman-core/src/modules/ops.rs@@ -33,8 +33,10 @@ use super::types::{ModuleRecord, ModuleSource, ModuleState, ModuleStatus};use super::{linked, registry, report, runtime, source};use crate::config::Config;use tinybus::module::{- Availability, LoadError, ModuleClient, ModuleSource as ResolverSource, Resolver, ResolverInputs,+ AccessFuture, Availability, FailureReporter, LoadError, ModuleAccess, ModuleClient,+ ModuleSource as ResolverSource, ProxyFailure, ReleaseRecord, Resolver, ResolverInputs,};+use tinybus::Proxy;/// Environment variable naming a directory of bundled release archives, for/// headless hosts (the Docker image, the CLI tarball, bench bundles).@@ -141,10 +143,11 @@ pub fn resolver(config: &Config) -> Resolver {linked_factories.insert(record.id.to_string(), factory);}}- ModuleSource::Release if record.id == "tinymcp" => {- // TinyMCP requires a host storage callback at initialization.- // Search-path loading has no config slot, so route a staged- // artifact through the resolver's configured local-file rung.+ ModuleSource::Release if record.id == "tinymcp" || record.id == "tinyeventqueues" => {+ // These modules require a per-module storage callback, which+ // TinyBus can bind only on its verified local-file path.+ // Route a staged release through that rung; the callback-aware+ // client below supplies the unique destination at load time.if let Some(path) = source::local_artifact(record.id) {sources.insert(record.id.to_string(),@@ -185,6 +188,12 @@ pub fn resolver(config: &Config) -> Resolver {})}+/// The staged artifact the callback-bound module fixture loads in isolation.+#[cfg(test)]+pub(crate) fn local_artifact_for_test(id: &str) -> Option<std::path::PathBuf> {+ source::local_artifact(id)+}+/// A `tinybus::module::ModuleClient` that loads modules lazily through/// [`resolver`].///@@ -193,7 +202,96 @@ pub fn resolver(config: &Config) -> Resolver {/// facade before the first call when artifacts ship with the host.#[must_use]pub fn client(config: Config) -> ModuleClient {- ModuleClient::new(resolver(&config))+ ModuleClient::new(CallbackAccess {+ resolver: resolver(&config),+ config,+ })+}++/// The two modules that connect to storage through a capability scoped to+/// their exact verified module instance.+fn needs_storage_callback(id: &str) -> bool {+ id == "tinymcp" || id == "tinyeventqueues"+}++/// Serializes creation and retention of module-scoped storage callback bindings.+static STORAGE_CALLBACK_LOADS: tokio::sync::Mutex<+ std::collections::HashMap<+ String,+ (+ std::sync::Arc<tinybus::module::HostCallbackBinding>,+ tinystoragedrivers_busproxy::ServedStorage,+ ),+ >,+> = tokio::sync::Mutex::const_new(std::collections::HashMap::new());++async fn ensure_loaded_with_storage_callback(+ resolver: &Resolver,+ config: &Config,+ id: &str,+) -> Result<(), String> {+ let mut loads = STORAGE_CALLBACK_LOADS.lock().await;+ if let Some((binding, _served)) = loads.get(id) {+ let mut module_config = module_config(config, id);+ module_config["storage_callback"] = serde_json::json!(binding.unique_name().as_str());+ return resolver+ .ensure_loaded_with_host_callbacks(id, std::sync::Arc::clone(binding), module_config)+ .await;+ }++ let runtime = resolver+ .runtime()+ .await+ .map_err(|error| error.to_string())?;+ let (binding, served) = super::storage_host::prepare_module_callbacks(runtime)+ .await+ .map_err(|error| error.to_string())?;+ let mut module_config = module_config(config, id);+ module_config["storage_callback"] = serde_json::json!(binding.unique_name().as_str());+ resolver+ .ensure_loaded_with_host_callbacks(id, std::sync::Arc::clone(&binding), module_config)+ .await?;+ loads.insert(id.to_owned(), (binding, served));+ Ok(())+}++/// Module access that grants storage callbacks only while loading the two+/// modules whose contracts require them.+struct CallbackAccess {+ resolver: Resolver,+ config: Config,+}++impl ModuleAccess for CallbackAccess {+ fn find(&self, id: &str) -> Option<&'static ReleaseRecord> {+ self.resolver.find(id)+ }++ fn enabled(&self) -> bool {+ self.resolver.enabled()+ }++ fn reporter(&self) -> &FailureReporter {+ self.resolver.reporter()+ }++ fn ensure_loaded<'a>(&'a self, id: &'a str) -> AccessFuture<'a, Result<(), String>> {+ Box::pin(async move {+ if needs_storage_callback(id) {+ ensure_loaded(&self.config, id).await+ } else {+ self.resolver.ensure_loaded(id).await+ }+ })+ }++ fn proxy<'a>(+ &'a self,+ bus_name: &'a str,+ object_path: &'a str,+ ) -> AccessFuture<'a, Result<Proxy, ProxyFailure>> {+ self.resolver.proxy(bus_name, object_path)+ }}/// A client whose every call goes to `proxy` instead of loading a module. The@@ -291,7 +389,23 @@ pub async fn ensure_loaded_within(id: &str,within: Option<Duration>,) -> Result<(), LoadError> {- let result = resolver(config).ensure_loaded_within(id, within).await;+ let resolver = resolver(config);+ let result = if needs_storage_callback(id) {+ if let Some(component) = disabled_component(config, id) {+ Err(LoadError::Disabled(component))+ } else {+ let load = ensure_loaded_with_storage_callback(&resolver, config, id);+ match within {+ Some(within) => tokio::time::timeout(within, load)+ .await+ .map_err(|_| LoadError::StillLoading)?+ .map_err(LoadError::Failed),+ None => load.await.map_err(LoadError::Failed),+ }+ }+ } else {+ resolver.ensure_loaded_within(id, within).await+ };if id == super::connectors::MODULE_ID && result.is_ok() {super::connectors::note_load_resolution_ready();}@@ -312,15 +426,10 @@ pub fn state_of(id: &str) -> ModuleState {/// Configuration crossing into a first-party compiled module.///-/// Credentials are intentionally absent. TinyMemory calls back into the host-/// for embedding and chat compute, and TinyMCP receives the shared host-/// storage service name; the other modules need no host config.+/// Credentials and callback destinations are intentionally absent. TinyMemory+/// receives only its compute configuration; TinyMCP and TinyEventQueues get a+/// unique storage callback destination from the verified loader.pub(super) fn module_config(config: &Config, id: &str) -> serde_json::Value {- if id == "tinymcp" {- return serde_json::json!({- "storage_callback": tinystoragedrivers_bus::STORAGE_INTERFACE,- });- }if id == super::search::MODULE_ID {return super::search::load_configuration(config);}@@ -348,16 +457,6 @@ pub(super) fn module_config(config: &Config, id: &str) -> serde_json::Value {serde_json::json!({ "offered_toolkits": super::connectors::offered_toolkits(config) })});}- // TinyMCP keeps its servers, secrets and audit trail in host storage and- // refuses to load without the bus name that serves it. That name is the- // storage host's interface, attached before any module loads- // ([`super::storage_host::install`]); the host still derives the scope and- // grant from the authenticated caller, never from this value.- if id == super::storage_grant_schemas::tinymcp::MODULE_ID {- return serde_json::json!({- "storage_callback": tinystoragedrivers_bus::STORAGE_INTERFACE.to_string()- });- }serde_json::json!({})}diff --git a/crates/openhuman-core/src/modules/ops_tests.rs b/crates/openhuman-core/src/modules/ops_tests.rsindex 245c348be2..eb1654865c 100644--- a/crates/openhuman-core/src/modules/ops_tests.rs+++ b/crates/openhuman-core/src/modules/ops_tests.rs@@ -115,14 +115,11 @@ fn the_default_config_enables_modules_and_downloads() {}#[test]-fn tinymcp_receives_the_host_storage_service_name() {+fn tinymcp_storage_destination_is_not_a_generic_module_config_value() {let config = Config::default();let module_config = ops::module_config_for_test(&config, "tinymcp");- assert_eq!(- module_config["storage_callback"],- tinystoragedrivers_bus::STORAGE_INTERFACE- );+ assert!(module_config.get("storage_callback").is_none());}#[test]@@ -719,14 +716,11 @@ fn the_ahead_warning_is_silent_for_released_modules_other_ids_and_release_builds}#[test]-fn tinymcp_is_told_which_bus_name_serves_its_storage() {- // TinyMCP refuses to load without a storage destination, so an empty config- // faults the module at startup and every MCP call reports "unavailable".+fn tinymcp_storage_destination_is_bound_by_the_verified_loader() {+ // A generic config cannot choose the callback destination. The verified+ // loader injects the unique name bound to this exact module instance.let config = ops::module_config_for_test(&Config::default(), "tinymcp");- assert_eq!(- config,- serde_json::json!({ "storage_callback": "ai.tinyhumans.tinystorage.Storage" })- );+ assert_eq!(config, serde_json::json!({}));assert_eq!(ops::module_config_for_test(&Config::default(), "tinybox"),serde_json::json!({})diff --git a/crates/openhuman-core/src/modules/storage_host.rs b/crates/openhuman-core/src/modules/storage_host.rsindex 8100afcf14..f23d13880e 100644--- a/crates/openhuman-core/src/modules/storage_host.rs+++ b/crates/openhuman-core/src/modules/storage_host.rs@@ -686,6 +686,38 @@ pub(crate) async fn install(}}+/// Prepare the exact per-module storage capability used by TinyMCP and+/// TinyEventQueues. The callback endpoint reuses the live storage policy but+/// is reachable only by the verified module instance loaded with this binding.+pub(super) async fn prepare_module_callbacks(+ runtime: &tinybus::module::ModuleRuntime,+) -> tinybus::Result<(+ Arc<tinybus::module::HostCallbackBinding>,+ tinystoragedrivers_busproxy::ServedStorage,+)> {+ let connection = runtime.connection();+ prepare_callbacks(&connection, runtime.host()).await+}++/// Prepare a callback storage endpoint for a module host on `connection`.+pub(super) async fn prepare_callbacks(+ connection: &tinybus::Connection,+ modules: &tinybus::module::ModuleHost,+) -> tinybus::Result<(+ Arc<tinybus::module::HostCallbackBinding>,+ tinystoragedrivers_busproxy::ServedStorage,+)> {+ let binding = tinystoragedrivers_busproxy::prepare_storage_host_callbacks(modules).await?;+ let endpoint = Arc::new(HostEndpoint::new(+ live_host(connection),+ Some(tokio::runtime::Handle::current()),+ ));+ let served =+ tinystoragedrivers_busproxy::serve_with_host_callbacks(Arc::clone(&binding), endpoint)+ .await?;+ Ok((binding, served))+}+#[path = "storage_host_endpoint.rs"]mod endpoint;diff --git a/vendor/tinybus b/vendor/tinybusindex 2b2ac16550..9793c29dba 160000--- a/vendor/tinybus+++ b/vendor/tinybus@@ -1 +1 @@-Subproject commit 2b2ac165505bdf3dc19c26e0b93c54fb4257bd3c+Subproject commit 9793c29dbae9d14bf191742ed6da078b9f687af3