Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

refactor(storage): improve storage grant and host logic

Steven Enamakel099c93361120
diff --git a/crates/openhuman-core/src/modules/storage_grant_schemas_tests.rs b/crates/openhuman-core/src/modules/storage_grant_schemas_tests.rs
index 6c4d30cc6b..ffdc2538f1 100644
--- a/crates/openhuman-core/src/modules/storage_grant_schemas_tests.rs
+++ b/crates/openhuman-core/src/modules/storage_grant_schemas_tests.rs
@@ -1,5 +1,5 @@
use super::*;
-use crate::modules::storage_grants::{authorize, declared, Denied, Object, Plan};
+use crate::modules::storage_grants::{Denied, Object, Plan, authorize, declared};
use serde_json::json;
fn allowed(
diff --git a/crates/openhuman-core/src/modules/storage_grants.rs b/crates/openhuman-core/src/modules/storage_grants.rs
index 7295a5af80..f27cb7e0f0 100644
--- a/crates/openhuman-core/src/modules/storage_grants.rs
+++ b/crates/openhuman-core/src/modules/storage_grants.rs
@@ -508,7 +508,9 @@ fn queues(grant: &StorageGrant, selector: &str, request: &Value) -> Result<Plan,
let spec = request
.get("spec")
.cloned()
- .and_then(|value| serde_json::from_value::<tinystoragedrivers::QueueSpec>(value).ok())
+ .and_then(|value| {
+ serde_json::from_value::<tinystoragedrivers::QueueSpec>(value).ok()
+ })
.ok_or(Denied::Malformed)?;
let declared = declaration.spec();
if spec.max_attempts != declared.max_attempts
diff --git a/crates/openhuman-core/src/modules/storage_grants_tests.rs b/crates/openhuman-core/src/modules/storage_grants_tests.rs
index 357b316cca..f9a768fc81 100644
--- a/crates/openhuman-core/src/modules/storage_grants_tests.rs
+++ b/crates/openhuman-core/src/modules/storage_grants_tests.rs
@@ -168,11 +168,21 @@ fn capabilities_are_answered_for_an_object_the_grant_does_not_hold() {
Err(Denied::Object)
);
assert_eq!(
- authorize(&documents_only, Object::Streams, "Len", &json!({ "stream": "x" })),
+ authorize(
+ &documents_only,
+ Object::Streams,
+ "Len",
+ &json!({ "stream": "x" })
+ ),
Err(Denied::Object)
);
assert_eq!(
- authorize(&documents_only, Object::Secrets, "Get", &json!({ "name": "x" })),
+ authorize(
+ &documents_only,
+ Object::Secrets,
+ "Get",
+ &json!({ "name": "x" })
+ ),
Err(Denied::Object)
);
}
diff --git a/crates/openhuman-core/src/modules/storage_host.rs b/crates/openhuman-core/src/modules/storage_host.rs
index 1ebef726bb..157a1990cd 100644
--- a/crates/openhuman-core/src/modules/storage_host.rs
+++ b/crates/openhuman-core/src/modules/storage_host.rs
@@ -45,15 +45,15 @@ use std::sync::Arc;
use async_trait::async_trait;
use serde_json::Value;
use tinybus::BusName;
-use tinystoragedrivers::secrets::{DocumentSecrets, KeyProvider};
use tinystoragedrivers::BlobReader;
+use tinystoragedrivers::secrets::{DocumentSecrets, KeyProvider};
use tinystoragedrivers_bus::{
BlobKeyRequest, BlobStreamHead, PutBlobStreamRequest, StorageFailure, StorageKind,
};
use tinystoragedrivers_busproxy::{Admission, HostFailure, StorageServer};
use self::endpoint::HostEndpoint;
-use super::storage_grants::{authorize, declared, filter_reply, Object, Plan, StorageGrant};
+use super::storage_grants::{Object, Plan, StorageGrant, authorize, declared, filter_reply};
use crate::storage::{Scope, StorageBackend, StorageError};
/// The largest blob a module may send by `PutStream`. The body is bounded per
@@ -443,14 +443,14 @@ impl StorageHost {
.ensure_queue(&declaration.name, &declaration.spec())
.await
.map_err(|error| {
- tracing::warn!(
- target: "openhuman::storage_host",
- module,
- queue = %declaration.name,
- %error,
- "[storage_host] could not declare a granted queue"
- );
- BindError::Backend(StorageKind::Backend)
+ tracing::warn!(
+ target: "openhuman::storage_host",
+ module,
+ queue = %declaration.name,
+ %error,
+ "[storage_host] could not declare a granted queue"
+ );
+ BindError::Backend(StorageKind::Backend)
})?;
}
tracing::debug!(
diff --git a/crates/openhuman-core/src/modules/storage_host_bus_tests.rs b/crates/openhuman-core/src/modules/storage_host_bus_tests.rs
index b14b761a44..1aa954c273 100644
--- a/crates/openhuman-core/src/modules/storage_host_bus_tests.rs
+++ b/crates/openhuman-core/src/modules/storage_host_bus_tests.rs
@@ -115,9 +115,9 @@ async fn a_module_write_under_a_stale_lease_is_fenced_by_the_host() {
#[tokio::test]
async fn the_bus_hands_the_host_the_brokers_sender_and_nothing_else() {
+ use tinybus::Connection;
use tinybus::broker::Broker;
use tinybus::transport::memory::MemoryBus;
- use tinybus::Connection;
let bus = MemoryBus::new();
Broker::new().spawn(bus.clone());
@@ -180,10 +180,12 @@ async fn the_bus_hands_the_host_the_brokers_sender_and_nothing_else() {
// A forged scope is a refused request, not a different scope.
let mut forged = get("notes", "a");
forged["scope"] = json!("local");
- assert!(documents(&module)
- .call::<Value>("Get", forged)
- .await
- .is_err());
+ assert!(
+ documents(&module)
+ .call::<Value>("Get", forged)
+ .await
+ .is_err()
+ );
// Secrets are on a confidential route: an ordinary call never reaches the host.
let secrets = module
@@ -193,15 +195,18 @@ async fn the_bus_hands_the_host_the_brokers_sender_and_nothing_else() {
STORAGE_INTERFACE,
)
.unwrap();
- assert!(secrets
- .call::<Value>("Get", json!({"name": "token.a"}))
- .await
- .is_err());
- assert!(rig
- .trail
- .0
- .lock()
- .unwrap()
- .iter()
- .all(|record| record.object != "Secrets"));
+ assert!(
+ secrets
+ .call::<Value>("Get", json!({"name": "token.a"}))
+ .await
+ .is_err()
+ );
+ assert!(
+ rig.trail
+ .0
+ .lock()
+ .unwrap()
+ .iter()
+ .all(|record| record.object != "Secrets")
+ );
}
diff --git a/crates/openhuman-core/src/modules/storage_host_cdylib_stream_tests.rs b/crates/openhuman-core/src/modules/storage_host_cdylib_stream_tests.rs
index 2b7dc40865..f2823cd8e5 100644
--- a/crates/openhuman-core/src/modules/storage_host_cdylib_stream_tests.rs
+++ b/crates/openhuman-core/src/modules/storage_host_cdylib_stream_tests.rs
@@ -61,12 +61,16 @@ async fn a_blob_over_the_bus_frame_round_trips_through_a_peer_bound_stream() {
"{missing}"
);
let trail = rig.trail.0.lock().unwrap().clone();
- assert!(trail
- .iter()
- .any(|r| r.op == "PutStream" && r.outcome == "ok" && r.module == "tinyjuice"));
- assert!(trail
- .iter()
- .any(|r| r.op == "GetStream" && r.outcome == "ok" && r.module == "tinyjuice"));
+ assert!(
+ trail
+ .iter()
+ .any(|r| r.op == "PutStream" && r.outcome == "ok" && r.module == "tinyjuice")
+ );
+ assert!(
+ trail
+ .iter()
+ .any(|r| r.op == "GetStream" && r.outcome == "ok" && r.module == "tinyjuice")
+ );
}
#[tokio::test(flavor = "multi_thread")]
@@ -77,12 +81,14 @@ async fn a_streamed_blob_is_still_bound_by_the_grant_and_the_length() {
let outside = rig.member("StreamPut", ("juice/other/x", 2 * MIB)).await;
assert!(outside.get("err").is_some(), "{outside}");
let handle = rig.backend.for_scope(&rig.scope("tinyjuice")).unwrap();
- assert!(handle
- .blobs()
- .head("juice/other/x")
- .await
- .unwrap()
- .is_none());
+ assert!(
+ handle
+ .blobs()
+ .head("juice/other/x")
+ .await
+ .unwrap()
+ .is_none()
+ );
let denied_get = rig.member("StreamGet", ("juice/other/x",)).await;
assert!(
denied_get["err"].to_string().contains("target-not-granted"),
@@ -113,12 +119,14 @@ async fn an_inline_blob_no_frame_can_carry_fails_fast_instead_of_hanging() {
started.elapsed()
);
let handle = rig.backend.for_scope(&rig.scope("tinyjuice")).unwrap();
- assert!(handle
- .blobs()
- .head("juice/ccr/over-frame")
- .await
- .unwrap()
- .is_none());
+ assert!(
+ handle
+ .blobs()
+ .head("juice/ccr/over-frame")
+ .await
+ .unwrap()
+ .is_none()
+ );
let after: Value = rig.member("PutBlob", ("juice/ccr/after", 2048usize)).await;
assert!(
after.get("ok").is_some(),
@@ -142,9 +150,11 @@ async fn a_watch_signal_reaches_the_module_that_opened_the_watch() {
.await;
assert!(denied(&refused, "target-not-granted"), "{refused}");
let trail = rig.trail.0.lock().unwrap().clone();
- assert!(trail
- .iter()
- .any(|r| r.op == "WatchOpen" && r.outcome == "ok" && r.module == "tinymcp"));
+ assert!(
+ trail
+ .iter()
+ .any(|r| r.op == "WatchOpen" && r.outcome == "ok" && r.module == "tinymcp")
+ );
}
#[tokio::test(flavor = "multi_thread")]
@@ -249,12 +259,14 @@ async fn a_second_module_cannot_read_or_take_the_first_modules_streams() {
"a peer took another peer's stream: {hijack:?}"
);
let handle = rig.backend.for_scope(&rig.scope("tinyjuice")).unwrap();
- assert!(handle
- .blobs()
- .head("juice/ccr/hijacked")
- .await
- .unwrap()
- .is_none());
+ assert!(
+ handle
+ .blobs()
+ .head("juice/ccr/hijacked")
+ .await
+ .unwrap()
+ .is_none()
+ );
// The refused attempt did not spend the stream: its owner still uses it.
let used = rig.member("UseStream", ("juice/ccr/owned", MIB)).await;
assert!(
diff --git a/crates/openhuman-core/src/modules/storage_host_cdylib_tests.rs b/crates/openhuman-core/src/modules/storage_host_cdylib_tests.rs
index 8a7c7bc2f7..54609b0e44 100644
--- a/crates/openhuman-core/src/modules/storage_host_cdylib_tests.rs
+++ b/crates/openhuman-core/src/modules/storage_host_cdylib_tests.rs
@@ -23,7 +23,7 @@
use std::path::{Path, PathBuf};
use std::sync::{Mutex, OnceLock};
-use serde_json::{json, Value};
+use serde_json::{Value, json};
use tinybus::broker::Broker;
use tinybus::module::ModuleHost;
use tinybus::transport::memory::MemoryBus;
@@ -31,9 +31,9 @@ use tinybus::{Connection, Proxy};
use tinystoragedrivers::secrets::{DerivedKeys, KeyProvider};
use zeroize::Zeroizing;
-use crate::modules::storage_grants::{declared, StorageGrant};
+use crate::modules::storage_grants::{StorageGrant, declared};
use crate::modules::storage_host::{
- module_scope, AuditRecord, AuditSink, CallerDirectory, Environment, StorageHost,
+ AuditRecord, AuditSink, CallerDirectory, Environment, StorageHost, module_scope,
};
use crate::modules::storage_host_linked::attach;
use crate::storage::{Scope, StorageBackend, StorageError};
@@ -300,12 +300,13 @@ async fn a_loaded_module_reads_and_writes_its_own_granted_collection() {
assert!(read.to_string().contains("qualified_name"), "{read}");
// It landed in the module's own scope, in the backend the host opened.
let own = rig.backend.for_scope(&rig.scope("tinymcp")).unwrap();
- assert!(own
- .documents()
- .get("mcp.servers", "files")
- .await
- .unwrap()
- .is_some());
+ assert!(
+ own.documents()
+ .get("mcp.servers", "files")
+ .await
+ .unwrap()
+ .is_some()
+ );
}
#[tokio::test(flavor = "multi_thread")]
@@ -366,19 +367,22 @@ async fn a_loaded_module_cannot_reach_another_modules_collections() {
}
// Untouched: the refusal came before the backend.
let after = victim.documents().get(&theirs, "session").await.unwrap();
- assert!(after
- .expect("the other module's document survives")
- .doc
- .to_string()
- .contains("OTHER-MODULES-SECRET"));
+ assert!(
+ after
+ .expect("the other module's document survives")
+ .doc
+ .to_string()
+ .contains("OTHER-MODULES-SECRET")
+ );
// And its own scope never got the collection.
let own = rig.backend.for_scope(&rig.scope("tinymcp")).unwrap();
- assert!(!own
- .documents()
- .collections("")
- .await
- .unwrap()
- .contains(&theirs));
+ assert!(
+ !own.documents()
+ .collections("")
+ .await
+ .unwrap()
+ .contains(&theirs)
+ );
}
#[tokio::test(flavor = "multi_thread")]
@@ -432,12 +436,14 @@ async fn a_loaded_module_cannot_name_or_reach_another_scope() {
assert!(wrote.get("ok").is_some(), "{wrote}");
for scope in [Scope::local(), Scope::new("someone-else").unwrap()] {
let handle = rig.backend.for_scope(&scope).unwrap();
- assert!(handle
- .documents()
- .get("mcp.servers", "mine")
- .await
- .unwrap()
- .is_none());
+ assert!(
+ handle
+ .documents()
+ .get("mcp.servers", "mine")
+ .await
+ .unwrap()
+ .is_none()
+ );
}
}
@@ -568,12 +574,14 @@ async fn a_blob_over_the_inline_cap_is_refused_and_stores_nothing() {
"an oversized inline blob must fail: {big}"
);
let handle = rig.backend.for_scope(&rig.scope("tinyjuice")).unwrap();
- assert!(handle
- .blobs()
- .head("juice/ccr/over-cap")
- .await
- .unwrap()
- .is_none());
+ assert!(
+ handle
+ .blobs()
+ .head("juice/ccr/over-cap")
+ .await
+ .unwrap()
+ .is_none()
+ );
// The refusal does not take the module's connection down with it.
let after: Value = rig
.probe
@@ -611,12 +619,16 @@ async fn every_call_is_audited_under_the_module_id_without_a_payload() {
.await;
let trail = rig.trail.0.lock().unwrap().clone();
assert!(trail.iter().all(|record| record.module == "tinymcp"));
- assert!(trail
- .iter()
- .any(|record| record.op == "Put" && record.outcome == "ok"));
- assert!(trail
- .iter()
- .any(|record| record.outcome == "target-not-granted"));
+ assert!(
+ trail
+ .iter()
+ .any(|record| record.op == "Put" && record.outcome == "ok")
+ );
+ assert!(
+ trail
+ .iter()
+ .any(|record| record.outcome == "target-not-granted")
+ );
assert!(!format!("{trail:?}").contains("PAYLOAD-MARKER"));
}
diff --git a/crates/openhuman-core/src/modules/storage_host_queues_tests.rs b/crates/openhuman-core/src/modules/storage_host_queues_tests.rs
index 5b64f9a90d..88ca08163f 100644
--- a/crates/openhuman-core/src/modules/storage_host_queues_tests.rs
+++ b/crates/openhuman-core/src/modules/storage_host_queues_tests.rs
@@ -4,12 +4,12 @@
//! holds no queues at all, while that module still connects.
use super::*;
+use crate::modules::storage_queue_manifest::QueueManifest;
+use tinybus::Connection;
use tinybus::broker::Broker;
use tinybus::transport::memory::MemoryBus;
-use tinybus::Connection;
-use crate::modules::storage_queue_manifest::QueueManifest;
-use tinystoragedrivers_types::{NewMessage, QueueSpec};
use tinystoragedrivers::ErrorKind;
+use tinystoragedrivers_types::{NewMessage, QueueSpec};
fn manifest() -> Arc<QueueManifest> {
Arc::new(
@@ -106,11 +106,13 @@ async fn a_module_granted_a_manifest_enqueues_receives_and_acks_over_the_bus() {
assert_eq!(leased.len(), 1);
assert_eq!(leased[0].payload, json!({"n": 1}));
assert!(queues.ack("inbox", &leased[0].receipt).await.unwrap());
- assert!(queues
- .receive("inbox", 4, 30_000, "test")
- .await
- .unwrap()
- .is_empty());
+ assert!(
+ queues
+ .receive("inbox", 4, 30_000, "test")
+ .await
+ .unwrap()
+ .is_empty()
+ );
// The messages live in the module's own scope on the host backend.
let scope = module_scope(None, "alpha").unwrap();
diff --git a/crates/openhuman-core/src/modules/storage_host_stream_tests.rs b/crates/openhuman-core/src/modules/storage_host_stream_tests.rs
index 6b821d25a5..11b528a694 100644
--- a/crates/openhuman-core/src/modules/storage_host_stream_tests.rs
+++ b/crates/openhuman-core/src/modules/storage_host_stream_tests.rs
@@ -50,9 +50,11 @@ async fn a_streamed_blob_is_stored_in_the_callers_scope_and_read_back() {
.unwrap();
assert!(other.is_none());
let trail = rig.trail.0.lock().unwrap().clone();
- assert!(trail
- .iter()
- .any(|r| r.op == "PutStream" && r.outcome == "ok" && r.module == "alpha"));
+ assert!(
+ trail
+ .iter()
+ .any(|r| r.op == "PutStream" && r.outcome == "ok" && r.module == "alpha")
+ );
}
#[tokio::test]
@@ -94,15 +96,16 @@ async fn a_streamed_blob_is_decided_like_any_other_call() {
));
// Nothing was stored by any of the refusals.
let scope = rig.env.module_scope("alpha").unwrap();
- assert!(rig
- .backend
- .for_scope(&scope)
- .unwrap()
- .blobs()
- .head("files/a")
- .await
- .unwrap()
- .is_none());
+ assert!(
+ rig.backend
+ .for_scope(&scope)
+ .unwrap()
+ .blobs()
+ .head("files/a")
+ .await
+ .unwrap()
+ .is_none()
+ );
}
#[tokio::test]
diff --git a/crates/openhuman-core/src/modules/storage_host_tests.rs b/crates/openhuman-core/src/modules/storage_host_tests.rs
index f837c18eb4..4bf15d36a0 100644
--- a/crates/openhuman-core/src/modules/storage_host_tests.rs
+++ b/crates/openhuman-core/src/modules/storage_host_tests.rs
@@ -1,6 +1,6 @@
use super::*;
-use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Mutex;
+use std::sync::atomic::{AtomicBool, Ordering};
use serde_json::json;
use tinystoragedrivers::secrets::DerivedKeys;
diff --git a/crates/openhuman-core/src/modules/storage_queue_manifest.rs b/crates/openhuman-core/src/modules/storage_queue_manifest.rs
index a2faf4e77c..8999e6689c 100644
--- a/crates/openhuman-core/src/modules/storage_queue_manifest.rs
+++ b/crates/openhuman-core/src/modules/storage_queue_manifest.rs
@@ -53,14 +53,19 @@ impl QueueManifest {
pub(crate) fn from_json(text: &str) -> Result<Self, String> {
let mut manifest: Self = serde_json::from_str(text).map_err(|error| error.to_string())?;
manifest.validate()?;
- manifest.queues.sort_by(|left, right| left.name.cmp(&right.name));
+ manifest
+ .queues
+ .sort_by(|left, right| left.name.cmp(&right.name));
Ok(manifest)
}
/// Validate a value assembled by host code.
pub(crate) fn validate(&self) -> Result<(), String> {
if self.version != 1 {
- return Err(format!("unsupported queue manifest version {}", self.version));
+ return Err(format!(
+ "unsupported queue manifest version {}",
+ self.version
+ ));
}
if self.schema.as_ref().is_some_and(String::is_empty) {
return Err("queue manifest $schema must not be empty".to_owned());
@@ -68,14 +73,24 @@ impl QueueManifest {
let mut names = BTreeMap::new();
for queue in &self.queues {
validate_name(&queue.name)?;
- if queue.owner.as_ref().is_some_and(|owner| owner.trim().is_empty()) {
+ if queue
+ .owner
+ .as_ref()
+ .is_some_and(|owner| owner.trim().is_empty())
+ {
return Err(format!("queue {:?} owner must not be empty", queue.name));
}
if queue.max_attempts == 0 {
- return Err(format!("queue {:?} must allow at least one attempt", queue.name));
+ return Err(format!(
+ "queue {:?} must allow at least one attempt",
+ queue.name
+ ));
}
if queue.visibility_ms == Some(0) {
- return Err(format!("queue {:?} visibility_ms must be positive", queue.name));
+ return Err(format!(
+ "queue {:?} visibility_ms must be positive",
+ queue.name
+ ));
}
if names.insert(queue.name.as_str(), ()).is_some() {
return Err(format!("queue {:?} is declared more than once", queue.name));
@@ -98,9 +113,9 @@ impl QueueManifest {
fn validate_name(name: &str) -> Result<(), String> {
if name.is_empty()
|| name.len() > 64
- || !name
- .bytes()
- .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || b"._-".contains(&byte))
+ || !name.bytes().all(|byte| {
+ byte.is_ascii_lowercase() || byte.is_ascii_digit() || b"._-".contains(&byte)
+ })
{
return Err(format!("invalid queue name {name:?}"));
}
diff --git a/crates/openhuman-core/src/modules/storage_queue_manifest_tests.rs b/crates/openhuman-core/src/modules/storage_queue_manifest_tests.rs
index b1018fb0f3..d6e4056676 100644
--- a/crates/openhuman-core/src/modules/storage_queue_manifest_tests.rs
+++ b/crates/openhuman-core/src/modules/storage_queue_manifest_tests.rs
@@ -26,8 +26,8 @@ fn rejects_unknown_fields_duplicate_names_and_invalid_future_versions() {
#[test]
fn allows_old_manifest_defaults_but_rejects_zero_attempts_or_visibility() {
- let manifest = QueueManifest::from_json(r#"{"version":1,"queues":[{"name":"legacy"}]}"#)
- .unwrap();
+ let manifest =
+ QueueManifest::from_json(r#"{"version":1,"queues":[{"name":"legacy"}]}"#).unwrap();
assert_eq!(manifest.get("legacy").unwrap().spec().max_attempts, 8);
for text in [
r#"{"version":1,"queues":[{"name":"q","max_attempts":0}]}"#,
diff --git a/crates/openhuman-core/src/storage/blob_sweep_tests.rs b/crates/openhuman-core/src/storage/blob_sweep_tests.rs
index 9b6a5b47ec..aa8426622f 100644
--- a/crates/openhuman-core/src/storage/blob_sweep_tests.rs
+++ b/crates/openhuman-core/src/storage/blob_sweep_tests.rs
@@ -95,10 +95,12 @@ async fn module_blob_scopes_are_swept_alongside_the_acting_scope() {
#[cfg(not(feature = "modules"))]
#[test]
fn no_module_blob_scopes_without_the_modules_feature() {
- assert!(crate::modules::expiring_blob_module_scopes(
- &crate::core::runtime::DispatchCtx::host_default()
- )
- .is_empty());
+ assert!(
+ crate::modules::expiring_blob_module_scopes(
+ &crate::core::runtime::DispatchCtx::host_default()
+ )
+ .is_empty()
+ );
}
#[tokio::test]