Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

feat(modules): load verified module artifacts with host callbacks

Steven Enamakelac8cb3a05a03
diff --git a/crates/openhuman-core/src/inbound/linked_fixture.rs b/crates/openhuman-core/src/inbound/linked_fixture.rs
index 2e1a066541..0d2b8472d9 100644
--- a/crates/openhuman-core/src/inbound/linked_fixture.rs
+++ b/crates/openhuman-core/src/inbound/linked_fixture.rs
@@ -1,8 +1,8 @@
-//! The `tinyeventqueues` module linked into the test binary on a private bus,
-//! with the host callbacks served by a scripted triage runner.
+//! The staged `tinyeventqueues` artifact on a private bus, with the host
+//! callbacks served by a scripted triage runner.
//!
-//! Each [`Rig`] is its own broker and module host: the real module (its
-//! `static-link` build) is attached with fast timers, the host's
+//! Each [`Rig`] is its own broker and module host: the verified release module
+//! is loaded with fast timers and a per-module storage callback, the host's
//! `ai.tinyhumans.tinyeventqueues.Host` is served by an [`InboundHost`] whose
//! [`Deliverer`] triages with a [`ScriptedTriage`], and [`Rig::client`] is a
//! module client bound to the module's object, as `route::offer` takes it.
@@ -299,6 +299,8 @@ pub(crate) struct Rig {
_host: Connection,
_modules: ModuleHost,
_storage: crate::storage::test_support::InstalledBackendGuard,
+ _callback_binding: Arc<tinybus::module::HostCallbackBinding>,
+ _callback_storage: tinystoragedrivers_busproxy::ServedStorage,
}
impl Rig {
@@ -353,15 +355,20 @@ impl Rig {
crate::modules::storage_host::install(&host, &modules)
.await
.unwrap();
+ let artifact = crate::modules::ops::local_artifact_for_test("tinyeventqueues")
+ .expect("the staged tinyeventqueues artifact is required for callback tests");
+ let (binding, callback_storage) =
+ crate::modules::storage_host::prepare_callbacks(&host, &modules)
+ .await
+ .unwrap();
+ let config = json!({
+ "pipeline": pipeline,
+ "timer": { "min_interval_ms": 5, "max_interval_ms": 25 },
+ "storage_callback": binding.unique_name().as_str(),
+ });
modules
- .attach_linked_with_config(
- tinyeventqueues::linked_module().unwrap(),
- json!({
- "pipeline": pipeline,
- "timer": { "min_interval_ms": 5, "max_interval_ms": 25 },
- "storage_callback": tinystoragedrivers_bus::STORAGE_INTERFACE,
- }),
- )
+ .load_file_with_host_callbacks(&artifact, config, &binding)
+ .await
.unwrap();
tokio::time::timeout(Duration::from_secs(10), async {
loop {
@@ -383,6 +390,8 @@ impl Rig {
_host: host,
_modules: modules,
_storage: storage,
+ _callback_binding: binding,
+ _callback_storage: callback_storage,
}
}
}
diff --git a/crates/openhuman-core/src/inbound/settings.rs b/crates/openhuman-core/src/inbound/settings.rs
index 00d15569f5..421e23d921 100644
--- a/crates/openhuman-core/src/inbound/settings.rs
+++ b/crates/openhuman-core/src/inbound/settings.rs
@@ -1,12 +1,12 @@
//! What `[inbound]` resolves to: the queue manifest and pipeline configuration
//! the module runs, where its queues live, and whether routing is on.
//!
-//! The host hands the module its configuration (`queues`, `pipeline`, and the
-//! storage callback) in the module's private blob when it loads it
-//! ([`module_config`]), and declares the same manifest on the module's storage
-//! scope when the storage host binds it ([`queue_grant`]). Both read the last
-//! resolution, so the queues the module asks for are exactly the queues the
-//! host serves.
+//! The host hands the module its queue and pipeline configuration in the
+//! module's private blob when it loads it ([`module_config`]), and declares the
+//! same manifest on the module's storage scope when the storage host binds it
+//! ([`queue_grant`]). TinyBus injects the unique storage callback destination
+//! during verified module loading. Both read the last resolution, so the queues
+//! the module asks for are exactly the queues the host serves.
use std::path::Path;
use std::sync::{Arc, RwLock};
@@ -221,7 +221,6 @@ pub fn module_config(config: &Config) -> Value {
json!({
"queues": resolved.queues_json,
"pipeline": resolved.pipeline_json,
- "storage_callback": tinystoragedrivers_bus::STORAGE_INTERFACE,
})
}
diff --git a/crates/openhuman-core/src/inbound/settings_tests.rs b/crates/openhuman-core/src/inbound/settings_tests.rs
index 705bb27af4..0c909e2fd2 100644
--- a/crates/openhuman-core/src/inbound/settings_tests.rs
+++ b/crates/openhuman-core/src/inbound/settings_tests.rs
@@ -112,10 +112,7 @@ async fn the_module_config_is_what_the_storage_grant_declares() {
let blob = module_config(&config);
assert_eq!(blob["queues"]["queues"][0]["max_attempts"], 4);
assert_eq!(blob["pipeline"]["version"], 1);
- assert_eq!(
- blob["storage_callback"],
- tinystoragedrivers_bus::STORAGE_INTERFACE
- );
+ assert!(blob.get("storage_callback").is_none());
let grant = queue_grant().unwrap();
assert_eq!(
grant
diff --git a/crates/openhuman-core/src/modules/ops.rs b/crates/openhuman-core/src/modules/ops.rs
index d2ff5ed43e..8ca7bd2224 100644
--- a/crates/openhuman-core/src/modules/ops.rs
+++ b/crates/openhuman-core/src/modules/ops.rs
@@ -33,8 +33,10 @@ use super::types::{ModuleRecord, ModuleSource, ModuleState, ModuleStatus};
use super::{linked, registry, report, runtime, source};
use crate::config::Config;
use tinybus::module::{
- Availability, LoadError, ModuleClient, ModuleSource as ResolverSource, Resolver, ResolverInputs,
+ AccessFuture, Availability, FailureReporter, LoadError, ModuleAccess, ModuleClient,
+ ModuleSource as ResolverSource, ProxyFailure, ReleaseRecord, Resolver, ResolverInputs,
};
+use tinybus::Proxy;
/// Environment variable naming a directory of bundled release archives, for
/// headless hosts (the Docker image, the CLI tarball, bench bundles).
@@ -141,10 +143,11 @@ pub fn resolver(config: &Config) -> Resolver {
linked_factories.insert(record.id.to_string(), factory);
}
}
- ModuleSource::Release if record.id == "tinymcp" => {
- // TinyMCP requires a host storage callback at initialization.
- // Search-path loading has no config slot, so route a staged
- // artifact through the resolver's configured local-file rung.
+ ModuleSource::Release if record.id == "tinymcp" || record.id == "tinyeventqueues" => {
+ // These modules require a per-module storage callback, which
+ // TinyBus can bind only on its verified local-file path.
+ // Route a staged release through that rung; the callback-aware
+ // client below supplies the unique destination at load time.
if let Some(path) = source::local_artifact(record.id) {
sources.insert(
record.id.to_string(),
@@ -185,6 +188,12 @@ pub fn resolver(config: &Config) -> Resolver {
})
}
+/// The staged artifact the callback-bound module fixture loads in isolation.
+#[cfg(test)]
+pub(crate) fn local_artifact_for_test(id: &str) -> Option<std::path::PathBuf> {
+ source::local_artifact(id)
+}
+
/// A `tinybus::module::ModuleClient` that loads modules lazily through
/// [`resolver`].
///
@@ -193,7 +202,96 @@ pub fn resolver(config: &Config) -> Resolver {
/// facade before the first call when artifacts ship with the host.
#[must_use]
pub fn client(config: Config) -> ModuleClient {
- ModuleClient::new(resolver(&config))
+ ModuleClient::new(CallbackAccess {
+ resolver: resolver(&config),
+ config,
+ })
+}
+
+/// The two modules that connect to storage through a capability scoped to
+/// their exact verified module instance.
+fn needs_storage_callback(id: &str) -> bool {
+ id == "tinymcp" || id == "tinyeventqueues"
+}
+
+/// Serializes creation and retention of module-scoped storage callback bindings.
+static STORAGE_CALLBACK_LOADS: tokio::sync::Mutex<
+ std::collections::HashMap<
+ String,
+ (
+ std::sync::Arc<tinybus::module::HostCallbackBinding>,
+ tinystoragedrivers_busproxy::ServedStorage,
+ ),
+ >,
+> = tokio::sync::Mutex::const_new(std::collections::HashMap::new());
+
+async fn ensure_loaded_with_storage_callback(
+ resolver: &Resolver,
+ config: &Config,
+ id: &str,
+) -> Result<(), String> {
+ let mut loads = STORAGE_CALLBACK_LOADS.lock().await;
+ if let Some((binding, _served)) = loads.get(id) {
+ let mut module_config = module_config(config, id);
+ module_config["storage_callback"] = serde_json::json!(binding.unique_name().as_str());
+ return resolver
+ .ensure_loaded_with_host_callbacks(id, std::sync::Arc::clone(binding), module_config)
+ .await;
+ }
+
+ let runtime = resolver
+ .runtime()
+ .await
+ .map_err(|error| error.to_string())?;
+ let (binding, served) = super::storage_host::prepare_module_callbacks(runtime)
+ .await
+ .map_err(|error| error.to_string())?;
+ let mut module_config = module_config(config, id);
+ module_config["storage_callback"] = serde_json::json!(binding.unique_name().as_str());
+ resolver
+ .ensure_loaded_with_host_callbacks(id, std::sync::Arc::clone(&binding), module_config)
+ .await?;
+ loads.insert(id.to_owned(), (binding, served));
+ Ok(())
+}
+
+/// Module access that grants storage callbacks only while loading the two
+/// modules whose contracts require them.
+struct CallbackAccess {
+ resolver: Resolver,
+ config: Config,
+}
+
+impl ModuleAccess for CallbackAccess {
+ fn find(&self, id: &str) -> Option<&'static ReleaseRecord> {
+ self.resolver.find(id)
+ }
+
+ fn enabled(&self) -> bool {
+ self.resolver.enabled()
+ }
+
+ fn reporter(&self) -> &FailureReporter {
+ self.resolver.reporter()
+ }
+
+ fn ensure_loaded<'a>(&'a self, id: &'a str) -> AccessFuture<'a, Result<(), String>> {
+ Box::pin(async move {
+ if needs_storage_callback(id) {
+ ensure_loaded(&self.config, id).await
+ } else {
+ self.resolver.ensure_loaded(id).await
+ }
+ })
+ }
+
+ fn proxy<'a>(
+ &'a self,
+ bus_name: &'a str,
+ object_path: &'a str,
+ ) -> AccessFuture<'a, Result<Proxy, ProxyFailure>> {
+ self.resolver.proxy(bus_name, object_path)
+ }
}
/// A client whose every call goes to `proxy` instead of loading a module. The
@@ -291,7 +389,23 @@ pub async fn ensure_loaded_within(
id: &str,
within: Option<Duration>,
) -> Result<(), LoadError> {
- let result = resolver(config).ensure_loaded_within(id, within).await;
+ let resolver = resolver(config);
+ let result = if needs_storage_callback(id) {
+ if let Some(component) = disabled_component(config, id) {
+ Err(LoadError::Disabled(component))
+ } else {
+ let load = ensure_loaded_with_storage_callback(&resolver, config, id);
+ match within {
+ Some(within) => tokio::time::timeout(within, load)
+ .await
+ .map_err(|_| LoadError::StillLoading)?
+ .map_err(LoadError::Failed),
+ None => load.await.map_err(LoadError::Failed),
+ }
+ }
+ } else {
+ resolver.ensure_loaded_within(id, within).await
+ };
if id == super::connectors::MODULE_ID && result.is_ok() {
super::connectors::note_load_resolution_ready();
}
@@ -312,15 +426,10 @@ pub fn state_of(id: &str) -> ModuleState {
/// Configuration crossing into a first-party compiled module.
///
-/// Credentials are intentionally absent. TinyMemory calls back into the host
-/// for embedding and chat compute, and TinyMCP receives the shared host
-/// storage service name; the other modules need no host config.
+/// Credentials and callback destinations are intentionally absent. TinyMemory
+/// receives only its compute configuration; TinyMCP and TinyEventQueues get a
+/// unique storage callback destination from the verified loader.
pub(super) fn module_config(config: &Config, id: &str) -> serde_json::Value {
- if id == "tinymcp" {
- return serde_json::json!({
- "storage_callback": tinystoragedrivers_bus::STORAGE_INTERFACE,
- });
- }
if id == super::search::MODULE_ID {
return super::search::load_configuration(config);
}
@@ -348,16 +457,6 @@ pub(super) fn module_config(config: &Config, id: &str) -> serde_json::Value {
serde_json::json!({ "offered_toolkits": super::connectors::offered_toolkits(config) })
});
}
- // TinyMCP keeps its servers, secrets and audit trail in host storage and
- // refuses to load without the bus name that serves it. That name is the
- // storage host's interface, attached before any module loads
- // ([`super::storage_host::install`]); the host still derives the scope and
- // grant from the authenticated caller, never from this value.
- if id == super::storage_grant_schemas::tinymcp::MODULE_ID {
- return serde_json::json!({
- "storage_callback": tinystoragedrivers_bus::STORAGE_INTERFACE.to_string()
- });
- }
serde_json::json!({})
}
diff --git a/crates/openhuman-core/src/modules/ops_tests.rs b/crates/openhuman-core/src/modules/ops_tests.rs
index 245c348be2..eb1654865c 100644
--- a/crates/openhuman-core/src/modules/ops_tests.rs
+++ b/crates/openhuman-core/src/modules/ops_tests.rs
@@ -115,14 +115,11 @@ fn the_default_config_enables_modules_and_downloads() {
}
#[test]
-fn tinymcp_receives_the_host_storage_service_name() {
+fn tinymcp_storage_destination_is_not_a_generic_module_config_value() {
let config = Config::default();
let module_config = ops::module_config_for_test(&config, "tinymcp");
- assert_eq!(
- module_config["storage_callback"],
- tinystoragedrivers_bus::STORAGE_INTERFACE
- );
+ assert!(module_config.get("storage_callback").is_none());
}
#[test]
@@ -719,14 +716,11 @@ fn the_ahead_warning_is_silent_for_released_modules_other_ids_and_release_builds
}
#[test]
-fn tinymcp_is_told_which_bus_name_serves_its_storage() {
- // TinyMCP refuses to load without a storage destination, so an empty config
- // faults the module at startup and every MCP call reports "unavailable".
+fn tinymcp_storage_destination_is_bound_by_the_verified_loader() {
+ // A generic config cannot choose the callback destination. The verified
+ // loader injects the unique name bound to this exact module instance.
let config = ops::module_config_for_test(&Config::default(), "tinymcp");
- assert_eq!(
- config,
- serde_json::json!({ "storage_callback": "ai.tinyhumans.tinystorage.Storage" })
- );
+ assert_eq!(config, serde_json::json!({}));
assert_eq!(
ops::module_config_for_test(&Config::default(), "tinybox"),
serde_json::json!({})
diff --git a/crates/openhuman-core/src/modules/storage_host.rs b/crates/openhuman-core/src/modules/storage_host.rs
index 8100afcf14..f23d13880e 100644
--- a/crates/openhuman-core/src/modules/storage_host.rs
+++ b/crates/openhuman-core/src/modules/storage_host.rs
@@ -686,6 +686,38 @@ pub(crate) async fn install(
}
}
+/// Prepare the exact per-module storage capability used by TinyMCP and
+/// TinyEventQueues. The callback endpoint reuses the live storage policy but
+/// is reachable only by the verified module instance loaded with this binding.
+pub(super) async fn prepare_module_callbacks(
+ runtime: &tinybus::module::ModuleRuntime,
+) -> tinybus::Result<(
+ Arc<tinybus::module::HostCallbackBinding>,
+ tinystoragedrivers_busproxy::ServedStorage,
+)> {
+ let connection = runtime.connection();
+ prepare_callbacks(&connection, runtime.host()).await
+}
+
+/// Prepare a callback storage endpoint for a module host on `connection`.
+pub(super) async fn prepare_callbacks(
+ connection: &tinybus::Connection,
+ modules: &tinybus::module::ModuleHost,
+) -> tinybus::Result<(
+ Arc<tinybus::module::HostCallbackBinding>,
+ tinystoragedrivers_busproxy::ServedStorage,
+)> {
+ let binding = tinystoragedrivers_busproxy::prepare_storage_host_callbacks(modules).await?;
+ let endpoint = Arc::new(HostEndpoint::new(
+ live_host(connection),
+ Some(tokio::runtime::Handle::current()),
+ ));
+ let served =
+ tinystoragedrivers_busproxy::serve_with_host_callbacks(Arc::clone(&binding), endpoint)
+ .await?;
+ Ok((binding, served))
+}
+
#[path = "storage_host_endpoint.rs"]
mod endpoint;
diff --git a/vendor/tinybus b/vendor/tinybus
index 2b2ac16550..9793c29dba 160000
--- a/vendor/tinybus
+++ b/vendor/tinybus
@@ -1 +1 @@
-Subproject commit 2b2ac165505bdf3dc19c26e0b93c54fb4257bd3c
+Subproject commit 9793c29dbae9d14bf191742ed6da078b9f687af3