Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

refactor(modules): update storage module imports and logic

Steven Enamakel2ca01a95b342
diff --git a/crates/openhuman-core/src/inbound/settings.rs b/crates/openhuman-core/src/inbound/settings.rs
index cc5dd5a744..5727daaf7f 100644
--- a/crates/openhuman-core/src/inbound/settings.rs
+++ b/crates/openhuman-core/src/inbound/settings.rs
@@ -13,10 +13,10 @@ use std::sync::{Arc, RwLock};
use serde_json::{json, Value};
use tinyeventqueues_bus::storage::{SHIPPED_PIPELINE_JSON, SHIPPED_QUEUES_JSON};
use tinyeventqueues_bus::{Lane, PipelineConfig};
-use tinystoragedrivers::queue::QueueManifest;
use tinystoragedrivers::QueueConfig;
use crate::config::Config;
+use crate::modules::storage_queue_manifest::QueueManifest;
use crate::modules::storage_grant_schemas::QueueGrant;
/// The `[components]` switch that owns the inbound event queues.
diff --git a/crates/openhuman-core/src/modules/mod.rs b/crates/openhuman-core/src/modules/mod.rs
index d50be833d6..3e33235899 100644
--- a/crates/openhuman-core/src/modules/mod.rs
+++ b/crates/openhuman-core/src/modules/mod.rs
@@ -98,6 +98,8 @@ pub(crate) mod storage_grant_schemas;
#[cfg(feature = "modules")]
pub mod storage_grants;
#[cfg(feature = "modules")]
+pub(crate) mod storage_queue_manifest;
+#[cfg(feature = "modules")]
pub(crate) mod storage_host;
#[cfg(feature = "modules")]
mod storage_host_linked;
diff --git a/crates/openhuman-core/src/modules/storage_grant_schemas.rs b/crates/openhuman-core/src/modules/storage_grant_schemas.rs
index ed10e446aa..660424b943 100644
--- a/crates/openhuman-core/src/modules/storage_grant_schemas.rs
+++ b/crates/openhuman-core/src/modules/storage_grant_schemas.rs
@@ -20,9 +20,8 @@
use std::sync::Arc;
-use tinystoragedrivers::queue::QueueManifest;
-
use super::storage_grants::StorageGrant;
+use super::storage_queue_manifest::QueueManifest;
use crate::storage::schema::module_items;
/// One module's row: its registry id, the schema owner that declares its
diff --git a/crates/openhuman-core/src/modules/storage_grants.rs b/crates/openhuman-core/src/modules/storage_grants.rs
index 70bef2c6dc..9b2fbe030a 100644
--- a/crates/openhuman-core/src/modules/storage_grants.rs
+++ b/crates/openhuman-core/src/modules/storage_grants.rs
@@ -29,10 +29,11 @@ use std::collections::BTreeSet;
use std::sync::Arc;
use serde_json::Value;
-use tinystoragedrivers::queue::QueueManifest;
use tinystoragedrivers::CollectionSpec;
-pub(crate) use tinystoragedrivers_client::Object;
+use super::storage_queue_manifest::QueueManifest;
+
+pub(crate) use tinystoragedrivers_busproxy::Object;
/// Whether the contract defines `selector` for `object`.
pub(crate) fn is_selector(object: Object, selector: &str) -> bool {
@@ -43,6 +44,7 @@ pub(crate) fn is_selector(object: Object, selector: &str) -> bool {
Object::Blobs => wire::is_blobs_selector(selector),
Object::Secrets => wire::is_secrets_selector(selector),
Object::Queues => wire::is_queues_selector(selector),
+ Object::Kv => wire::is_kv_selector(selector),
}
}
@@ -59,6 +61,10 @@ pub(crate) enum Denied {
Schema,
/// The selector is not part of the contract, or carries no usable target.
Malformed,
+ /// A queue is absent from the host-owned manifest.
+ QueueNotGranted,
+ /// A queue declaration differs from the host-owned manifest.
+ QueueSpec,
}
impl Denied {
@@ -70,6 +76,8 @@ impl Denied {
Self::ReadOnly => "read-only-grant",
Self::Schema => "schema-is-host-only",
Self::Malformed => "malformed-request",
+ Self::QueueNotGranted => "queue-not-granted",
+ Self::QueueSpec => "queue-spec-is-host-only",
}
}
}
@@ -180,6 +188,7 @@ impl StorageGrant {
Object::Blobs => !self.blob_prefixes.is_empty(),
Object::Secrets => !self.secret_prefixes.is_empty(),
Object::Queues => self.queues.is_some(),
+ Object::Kv => false,
}
}
@@ -254,7 +263,9 @@ pub(crate) fn authorize(
"ttl": false,
"max_key_len": 0,
}))),
- Object::Streams | Object::Secrets | Object::Queues => Err(Denied::Object),
+ Object::Streams | Object::Secrets | Object::Queues | Object::Kv => {
+ Err(Denied::Object)
+ }
};
}
// `BusStorage::connect` asks every module's host for queue features, so a
@@ -266,6 +277,11 @@ pub(crate) fn authorize(
.map_err(|_| Denied::Malformed)?,
));
}
+ // The client probes each port during connection setup. KV has no module
+ // policy yet, so report an empty capability set and deny data selectors.
+ if object == Object::Kv && selector == "Capabilities" {
+ return Ok(Plan::Answer(serde_json::json!({ "capabilities": [] })));
+ }
if !grant.has_object(object) {
return Err(Denied::Object);
}
@@ -279,6 +295,7 @@ pub(crate) fn authorize(
Object::Blobs => blobs(grant, selector, request),
Object::Secrets => secrets(grant, selector, request),
Object::Queues => queues(selector, request),
+ Object::Kv => Err(Denied::Object),
}?;
Ok(match plan {
Plan::Forward { target, filter, .. } => Plan::Forward {
@@ -475,10 +492,30 @@ pub(crate) fn filter_reply(grant: &StorageGrant, filter: ReplyFilter, reply: Val
/// `Queues` selectors. The target is the request's queue; the bound server
/// (holding the grant's manifest) refuses a queue the manifest does not name.
-fn queues(selector: &str, request: &Value) -> Result<Plan, Denied> {
+fn queues(grant: &StorageGrant, selector: &str, request: &Value) -> Result<Plan, Denied> {
match selector {
"Features" => Ok(forward("")),
- _ => Ok(forward(text(request, "queue")?)),
+ "EnsureQueue" | "Enqueue" | "Receive" | "Ack" | "Nack" | "Extend" | "Stats"
+ | "ListDead" | "Redrive" | "Purge" => {
+ let name = text(request, "queue")?;
+ let declaration = grant
+ .queues
+ .as_ref()
+ .and_then(|manifest| manifest.get(name))
+ .ok_or(Denied::QueueNotGranted)?;
+ if selector == "EnsureQueue" {
+ let spec = request
+ .get("spec")
+ .cloned()
+ .and_then(|value| serde_json::from_value::<tinystoragedrivers::QueueSpec>(value).ok())
+ .ok_or(Denied::Malformed)?;
+ if spec != declaration.spec() {
+ return Err(Denied::QueueSpec);
+ }
+ }
+ Ok(forward(name))
+ }
+ _ => Err(Denied::Malformed),
}
}
diff --git a/crates/openhuman-core/src/modules/storage_queue_manifest.rs b/crates/openhuman-core/src/modules/storage_queue_manifest.rs
new file mode 100644
index 0000000000..ff3c4ed188
--- /dev/null
+++ b/crates/openhuman-core/src/modules/storage_queue_manifest.rs
@@ -0,0 +1,98 @@
+//! Host-owned parser and policy for module queue grants.
+//!
+//! Queue declarations are deployment policy, not a storage-driver primitive.
+
+use std::collections::BTreeMap;
+
+use serde::{Deserialize, Serialize};
+use tinystoragedrivers::queue::{QueueSpec, validate_queue_name};
+
+/// A validated version-1 queue manifest supplied by the host.
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub(crate) struct QueueManifest {
+ #[serde(default, rename = "$schema", skip_serializing_if = "Option::is_none")]
+ schema: Option<String>,
+ version: u32,
+ queues: Vec<QueueDeclaration>,
+}
+
+/// Host policy for one queue.
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub(crate) struct QueueDeclaration {
+ pub(crate) name: String,
+ #[serde(default, skip_serializing_if = "String::is_empty")]
+ pub(crate) description: String,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub(crate) owner: Option<String>,
+ #[serde(default = "default_max_attempts")]
+ pub(crate) max_attempts: u32,
+ #[serde(default)]
+ pub(crate) dedupe_window_ms: u64,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub(crate) visibility_ms: Option<u64>,
+}
+
+fn default_max_attempts() -> u32 {
+ QueueSpec::default().max_attempts
+}
+
+impl QueueDeclaration {
+ /// The storage-port queue behavior declared by this host policy.
+ pub(crate) fn spec(&self) -> QueueSpec {
+ QueueSpec {
+ max_attempts: self.max_attempts,
+ dedupe_window_ms: self.dedupe_window_ms,
+ }
+ }
+}
+
+impl QueueManifest {
+ /// Parse and validate a manifest. Unknown fields and names fail closed.
+ pub(crate) fn from_json(text: &str) -> Result<Self, String> {
+ let mut manifest: Self = serde_json::from_str(text).map_err(|error| error.to_string())?;
+ manifest.validate()?;
+ manifest.queues.sort_by(|left, right| left.name.cmp(&right.name));
+ Ok(manifest)
+ }
+
+ /// Validate a value assembled by host code.
+ pub(crate) fn validate(&self) -> Result<(), String> {
+ if self.version != 1 {
+ return Err(format!("unsupported queue manifest version {}", self.version));
+ }
+ if self.schema.as_ref().is_some_and(String::is_empty) {
+ return Err("queue manifest $schema must not be empty".to_owned());
+ }
+ let mut names = BTreeMap::new();
+ for queue in &self.queues {
+ validate_queue_name(&queue.name).map_err(|error| error.to_string())?;
+ if queue.owner.as_ref().is_some_and(|owner| owner.trim().is_empty()) {
+ return Err(format!("queue {:?} owner must not be empty", queue.name));
+ }
+ queue.spec().validate().map_err(|error| error.to_string())?;
+ if queue.visibility_ms == Some(0) {
+ return Err(format!("queue {:?} visibility_ms must be positive", queue.name));
+ }
+ if names.insert(queue.name.as_str(), ()).is_some() {
+ return Err(format!("queue {:?} is declared more than once", queue.name));
+ }
+ }
+ Ok(())
+ }
+
+ /// Look up a declaration by exact queue name.
+ pub(crate) fn get(&self, name: &str) -> Option<&QueueDeclaration> {
+ self.queues.iter().find(|queue| queue.name == name)
+ }
+
+ /// Declarations for one-time host setup.
+ pub(crate) fn queues(&self) -> &[QueueDeclaration] {
+ &self.queues
+ }
+}
+
+#[cfg(test)]
+#[path = "storage_queue_manifest_tests.rs"]
+mod tests;