The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.
Rust agent-orchestrationai-agentsai-assistantdesktopllmlocal-firstmcppersonal-aiprivacyrustsecond-braintauri
chore: files changed crates/openhuman-core/src/modules/storage_host.rs,crates/openhuman-core/src/mod
Steven Enamakel705c43f3fd69
diff --git a/crates/openhuman-core/src/modules/storage_host.rs b/crates/openhuman-core/src/modules/storage_host.rsindex 8ad61c6556..9c9a154e5d 100644--- a/crates/openhuman-core/src/modules/storage_host.rs+++ b/crates/openhuman-core/src/modules/storage_host.rs@@ -45,8 +45,8 @@ use std::sync::Arc;use async_trait::async_trait;use serde_json::Value;use tinybus::BusName;-use tinystoragedrivers_api::BlobReader;use tinystoragedrivers::secrets::{DocumentSecrets, KeyProvider};+use tinystoragedrivers_api::BlobReader;use tinystoragedrivers_bus::{BlobKeyRequest, BlobStreamHead, PutBlobStreamRequest, StorageFailure, StorageKind,};diff --git a/crates/openhuman-core/src/modules/storage_host_linked_tests.rs b/crates/openhuman-core/src/modules/storage_host_linked_tests.rsindex 7b76c0151a..8ee5fad2ad 100644--- a/crates/openhuman-core/src/modules/storage_host_linked_tests.rs+++ b/crates/openhuman-core/src/modules/storage_host_linked_tests.rs@@ -6,8 +6,8 @@ use tinybus::transport::memory::MemoryBus;use tinybus::{BusName, Connection};use tinystoragedrivers::secrets::{DerivedKeys, KeyProvider};use tinystoragedrivers::{Scope, StorageError};-use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage;use tinystoragedrivers_bus::{SecretNameRequest, SecretValue, SetSecretRequest};+use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage;use zeroize::Zeroizing;use super::super::storage_grants::StorageGrant;diff --git a/crates/openhuman-core/src/modules/storage_host_queues_tests.rs b/crates/openhuman-core/src/modules/storage_host_queues_tests.rsindex 626709a581..74785ab6b1 100644--- a/crates/openhuman-core/src/modules/storage_host_queues_tests.rs+++ b/crates/openhuman-core/src/modules/storage_host_queues_tests.rs@@ -142,10 +142,7 @@ async fn a_queue_outside_the_manifest_is_not_found_and_a_divergent_spec_conflictasync fn a_module_without_a_queue_grant_connects_but_is_denied_queues() {let peers = peers().await;let queues = queues_of(&peers.plain).await;- assert_eq!(- queues.features(),- QueueFeatures::default()- );+ assert_eq!(queues.features(), QueueFeatures::default());let error = queues.enqueue("inbox", &[NewMessage::new(json!({}))]).awaitdiff --git a/crates/openhuman-core/src/modules/storage_host_watching.rs b/crates/openhuman-core/src/modules/storage_host_watching.rsindex 7ecf41e125..78497d2afd 100644--- a/crates/openhuman-core/src/modules/storage_host_watching.rs+++ b/crates/openhuman-core/src/modules/storage_host_watching.rs@@ -10,7 +10,7 @@ use std::collections::HashMap;use std::sync::{Arc, Mutex};use serde_json::Value;-use tinystoragedrivers_api::{DocumentStore, DocumentWatch, Storage, StorageBackend, async_trait};+use tinystoragedrivers_api::{async_trait, DocumentStore, DocumentWatch, Storage, StorageBackend};use tinystoragedrivers_testkit::MemoryBackend;use tinystoragedrivers_types::{Capabilities, Capability, CollectionSpec, Filter, Page, Precondition, Query, Result, Scope,diff --git a/crates/openhuman-core/src/storage/README.md b/crates/openhuman-core/src/storage/README.mdindex acadc8d980..04d3737c36 100644--- a/crates/openhuman-core/src/storage/README.md+++ b/crates/openhuman-core/src/storage/README.md@@ -104,13 +104,20 @@ it with the core stopped.need. `driver_has_cross_process_cas(driver)` is the same answer from aname, for the boot guard, which judges a URL before opening anything.Memory and file drivers coordinate only within one process.-- `blob_sweep`: reclaims expired blobs (`StorageBackend::sweep_expired_blobs`)+- `blob_sweep`: reclaims expired blobs (`BlobStore::sweep_expired`)for `local`, each known agent scope and each provisioned SaaS profile(`u:<id>` plus the blob-owning modules' scopes for it) every hour,spawned with the cron `ServiceSet` switch (`core::runtime::services`; notgated on `[cron] enabled`, since it reclaims storage rather than runningjobs). Each scope's sweep is bounded by `SCOPE_SWEEP_TIMEOUT`; a timeoutcounts as a failed sweep and is retried on the next tick.+- `sweep_unreferenced_blobs`: an explicit maintenance helper that pages+ physical blob metadata, keeps unknown or cutoff-equal ages, asks the caller+ whether each old key is referenced, and stops on the first error while+ preserving earlier deletions. It is not activated by the periodic sweeper:+ reference checking and deletion are separate operations, so a caller must+ coordinate blob writers/reference creation with cleanup or use immutable+ keys, with a grace period longer than the write-to-reference delay.- `journal_maintenance`: invokes TinyAgents' `maintain_run_journals` with thehost's configured retention and clock. It visits local and known agentscopes plus module scopes, and only profiles currently open under thisdiff --git a/crates/openhuman-core/src/storage/blob_sweep.rs b/crates/openhuman-core/src/storage/blob_sweep.rsindex 31976af9b9..81c4a2ebbd 100644--- a/crates/openhuman-core/src/storage/blob_sweep.rs+++ b/crates/openhuman-core/src/storage/blob_sweep.rs@@ -15,6 +15,11 @@ use std::sync::Arc;use std::time::Duration;use tinystoragedrivers::{Scope, StorageBackend};+use tinystoragedrivers_api::BlobStore;+use tinystoragedrivers_types::{+ is_reserved_blob_key, BlobListPage, Key, Result as StorageResult, StorageError,+ MAX_BLOB_LIST_PAGE_ITEMS, SEALED_BLOB_PREFIX,+};/// How often the sweeper wakes. Lapsed blobs are already invisible to readers,/// so this only bounds how long their bytes linger.@@ -30,6 +35,118 @@ pub const FIRST_SWEEP_DELAY: Duration = Duration::from_secs(5 * 60);/// retried on the next tick.pub const SCOPE_SWEEP_TIMEOUT: Duration = Duration::from_secs(5 * 60);+/// Delete old user blobs after the caller confirms each key is unreferenced.+///+/// This is a maintenance primitive, not an automatic policy: the caller must+/// arrange that blob writers and reference creation cannot race the check and+/// delete (or use immutable keys). The age grace period must exceed the maximum+/// time from writing a blob until its durable reference is visible. This+/// function is deliberately not called by [`sweep_all`].+///+/// Only blobs with a known `written_at_ms` strictly less than `older_than_ms`+/// are considered. Unknown ages and entries at the cutoff are retained. Pages+/// are bounded and key ordered. An error stops the sweep; deletions from prior+/// entries remain applied and the caller can retry after resolving the error.+///+/// # Errors+/// Returns an error for an invalid or engine-owned prefix, invalid listing+/// data, unsupported paging, a listing/deletion error, or a reference-check+/// error.+pub async fn sweep_unreferenced_blobs<F, Fut>(+ blobs: &dyn BlobStore,+ prefix: &str,+ older_than_ms: u64,+ mut is_referenced: F,+) -> StorageResult<u64>+where+ F: FnMut(String) -> Fut,+ Fut: std::future::Future<Output = StorageResult<bool>>,+{+ validate_orphan_sweep_prefix(prefix)?;+ let mut after: Option<String> = None;+ let mut removed = 0_u64;+ loop {+ let page = blobs+ .list_page(prefix, after.as_deref(), MAX_BLOB_LIST_PAGE_ITEMS)+ .await?;+ validate_orphan_sweep_page(prefix, after.as_deref(), &page)?;+ for entry in page.items {+ if !orphan_age_is_eligible(entry.written_at_ms, older_than_ms)+ || is_referenced(entry.meta.key.clone()).await?+ {+ continue;+ }+ if blobs.delete(&entry.meta.key).await? {+ removed = removed.saturating_add(1);+ }+ }+ let Some(next_after) = page.next_after else {+ return Ok(removed);+ };+ after = Some(next_after);+ }+}++fn orphan_age_is_eligible(written_at_ms: Option<u64>, older_than_ms: u64) -> bool {+ written_at_ms.is_some_and(|written_at_ms| written_at_ms < older_than_ms)+}++fn validate_orphan_sweep_prefix(prefix: &str) -> StorageResult<()> {+ let key = Key::new(prefix.to_owned())?;+ if key.as_str().is_empty()+ || SEALED_BLOB_PREFIX.starts_with(prefix)+ || prefix.starts_with(SEALED_BLOB_PREFIX)+ || prefix+ .trim_end_matches('/')+ .split('/')+ .any(|segment| segment.is_empty() || segment == "." || segment == "..")+ {+ return Err(StorageError::invalid_input(+ "orphan sweep requires a nonempty user blob prefix",+ ));+ }+ Ok(())+}++fn validate_orphan_sweep_page(+ prefix: &str,+ after: Option<&str>,+ page: &BlobListPage,+) -> StorageResult<()> {+ if page.items.len() > MAX_BLOB_LIST_PAGE_ITEMS {+ return Err(StorageError::backend(+ "orphan sweep listing page exceeded its requested bound",+ ));+ }+ let mut previous = after;+ for entry in &page.items {+ let key = entry.meta.key.as_str();+ if !key.starts_with(prefix)+ || Key::new(key.to_owned()).is_err()+ || is_reserved_blob_key(key)+ || previous.is_some_and(|previous| key <= previous)+ {+ return Err(StorageError::backend(+ "orphan sweep listing returned an invalid key",+ ));+ }+ previous = Some(key);+ }+ if let Some(cursor) = &page.next_after {+ if page+ .items+ .last()+ .is_none_or(|entry| entry.meta.key != *cursor)+ || after.is_some_and(|after| cursor.as_str() <= after)+ {+ return Err(StorageError::backend(+ "orphan sweep listing cursor did not advance",+ ));+ }+ }+ Ok(())+}+/// Reclaims the lapsed blobs of `scope`, returning how many went. A failure/// or a sweep past [`SCOPE_SWEEP_TIMEOUT`] is logged and counts as zero.pub async fn sweep_scope(backend: &dyn StorageBackend, scope: &Scope) -> u64 {diff --git a/crates/openhuman-core/src/storage/blob_sweep_tests.rs b/crates/openhuman-core/src/storage/blob_sweep_tests.rsindex 9b6a5b47ec..941bdd13bc 100644--- a/crates/openhuman-core/src/storage/blob_sweep_tests.rs+++ b/crates/openhuman-core/src/storage/blob_sweep_tests.rs@@ -211,3 +211,154 @@ fn module_blob_scopes_follow_the_handed_context_and_never_cross_tenants() {assert!(alice.iter().all(|s| s.as_str().contains("blob-alice")));assert!(bob.iter().all(|s| s.as_str().contains("blob-bob")));}++#[test]+fn orphan_sweep_age_is_strict_and_unknown_ages_are_retained() {+ assert!(!orphan_age_is_eligible(None, 100));+ assert!(!orphan_age_is_eligible(Some(100), 100));+ assert!(orphan_age_is_eligible(Some(99), 100));+}++#[test]+fn orphan_sweep_refuses_empty_and_engine_owned_prefixes() {+ for prefix in ["", "_tsd/", "_tsd/sealed", "_tsd/sealed/stream/"] {+ assert_eq!(+ validate_orphan_sweep_prefix(prefix).unwrap_err().kind(),+ tinystoragedrivers_types::ErrorKind::InvalidInput,+ "prefix {prefix:?}"+ );+ }+ assert!(validate_orphan_sweep_prefix("user/attachments/").is_ok());+}++#[test]+fn orphan_sweep_rejects_malformed_or_nonadvancing_listing_pages() {+ use tinystoragedrivers_types::{BlobListEntry, BlobListPage, BlobMeta};++ let page = BlobListPage {+ items: vec![BlobListEntry {+ meta: BlobMeta {+ key: "user/b".into(),+ len: 1,+ content_type: None,+ },+ written_at_ms: Some(1),+ }],+ next_after: Some("user/a".into()),+ };+ assert_eq!(+ validate_orphan_sweep_page("user/", None, &page)+ .unwrap_err()+ .kind(),+ tinystoragedrivers_types::ErrorKind::Backend+ );+ assert_eq!(+ validate_orphan_sweep_page("user/", Some("user/b"), &page)+ .unwrap_err()+ .kind(),+ tinystoragedrivers_types::ErrorKind::Backend+ );+}++#[tokio::test]+async fn orphan_sweep_pages_and_preserves_referenced_blobs() {+ use tinystoragedrivers_api::BlobStore;+ use tinystoragedrivers_testkit::memory_storage;+ use tinystoragedrivers_types::Scope;++ let storage = memory_storage(&Scope::local());+ let blobs = storage.blobs();+ for index in 0..=MAX_BLOB_LIST_PAGE_ITEMS {+ blobs+ .put(&format!("cleanup/{index:04}"), vec![index as u8], None)+ .await+ .unwrap();+ }+ let kept = format!("cleanup/{:04}", MAX_BLOB_LIST_PAGE_ITEMS);+ let removed = sweep_unreferenced_blobs(blobs.as_ref(), "cleanup/", u64::MAX, |key| {+ let kept = kept.clone();+ async move { Ok(key == kept) }+ })+ .await+ .unwrap();++ assert_eq!(removed, MAX_BLOB_LIST_PAGE_ITEMS as u64);+ assert!(blobs.get(&kept).await.unwrap().is_some());+ assert!(blobs.get("cleanup/0000").await.unwrap().is_none());+}++#[tokio::test]+async fn orphan_sweep_stops_on_reference_error_after_preserving_prior_progress() {+ use tinystoragedrivers_api::BlobStore;+ use tinystoragedrivers_testkit::memory_storage;+ use tinystoragedrivers_types::Scope;++ let storage = memory_storage(&Scope::local());+ let blobs = storage.blobs();+ for key in ["cleanup/a", "cleanup/b", "cleanup/c"] {+ blobs.put(key, vec![1], None).await.unwrap();+ }++ let error = sweep_unreferenced_blobs(blobs.as_ref(), "cleanup/", u64::MAX, |key| async move {+ if key == "cleanup/b" {+ Err(StorageError::backend("reference lookup failed"))+ } else {+ Ok(false)+ }+ })+ .await+ .unwrap_err();++ assert_eq!(error.kind(), tinystoragedrivers_types::ErrorKind::Backend);+ assert!(blobs.get("cleanup/a").await.unwrap().is_none());+ assert!(blobs.get("cleanup/b").await.unwrap().is_some());+ assert!(blobs.get("cleanup/c").await.unwrap().is_some());+}++#[tokio::test]+async fn orphan_sweep_stops_on_listing_error_before_touching_blobs() {+ use std::sync::Arc;+ use tinystoragedrivers_api::BlobStore;+ use tinystoragedrivers_testkit::{memory_storage, Failing, FaultPlan};+ use tinystoragedrivers_types::{ErrorKind, Scope};++ let storage = memory_storage(&Scope::local());+ let blobs = Arc::clone(storage.blobs());+ blobs.put("cleanup/a", vec![1], None).await.unwrap();+ let faults = FaultPlan::new();+ faults.fail_operation_nth("blob.list_page", 1, ErrorKind::Unavailable);+ let failing = Failing::with_plan(blobs.clone(), faults);++ let error = sweep_unreferenced_blobs(&failing, "cleanup/", u64::MAX, |_| async { Ok(false) })+ .await+ .unwrap_err();++ assert_eq!(error.kind(), ErrorKind::Unavailable);+ assert!(blobs.get("cleanup/a").await.unwrap().is_some());+}++#[tokio::test]+async fn orphan_sweep_stops_on_delete_error_after_preserving_prior_progress() {+ use std::sync::Arc;+ use tinystoragedrivers_api::BlobStore;+ use tinystoragedrivers_testkit::{memory_storage, Failing, FaultPlan};+ use tinystoragedrivers_types::{ErrorKind, Scope};++ let storage = memory_storage(&Scope::local());+ let blobs = Arc::clone(storage.blobs());+ for key in ["cleanup/a", "cleanup/b", "cleanup/c"] {+ blobs.put(key, vec![1], None).await.unwrap();+ }+ let faults = FaultPlan::new();+ faults.fail_operation_nth("blob.delete", 2, ErrorKind::Unavailable);+ let failing = Failing::with_plan(blobs.clone(), faults);++ let error = sweep_unreferenced_blobs(&failing, "cleanup/", u64::MAX, |_| async { Ok(false) })+ .await+ .unwrap_err();++ assert_eq!(error.kind(), ErrorKind::Unavailable);+ assert!(blobs.get("cleanup/a").await.unwrap().is_none());+ assert!(blobs.get("cleanup/b").await.unwrap().is_some());+ assert!(blobs.get("cleanup/c").await.unwrap().is_some());+}diff --git a/crates/openhuman-core/src/storage/fenced_backend_tests.rs b/crates/openhuman-core/src/storage/fenced_backend_tests.rsindex f872801a7d..f0ea8c2004 100644--- a/crates/openhuman-core/src/storage/fenced_backend_tests.rs+++ b/crates/openhuman-core/src/storage/fenced_backend_tests.rs@@ -6,8 +6,8 @@ use crate::storage::fence::{FenceError, LeaseFence, ScopeMatch};use crate::storage::lease::{DocumentLeases, LeaseGrant, LeaseStore};use serde_json::json;use std::sync::{- Arc,atomic::{AtomicU64, Ordering},+ Arc,};use std::time::Duration;use tinystoragedrivers_api::StorageBackend as _;@@ -112,14 +112,12 @@ async fn cached_handles_enforce_local_expiry_margin_and_accept_same_epoch_renewafence_error(&error),Some(FenceError::Expired { .. })));- assert!(- storage- .documents()- .get("items", "too-late")- .await- .unwrap()- .is_none()- );+ assert!(storage+ .documents()+ .get("items", "too-late")+ .await+ .unwrap()+ .is_none());}#[tokio::test]@@ -162,45 +160,39 @@ async fn a_cached_handle_cannot_mutate_after_cross_process_takeover_on_any_port(// A's local clock is deliberately behind. The local precheck succeeds;// the driver's atomic condition is what refuses each cached-handle write.a.set_now(1_000);- assert!(- storage- .documents()- .put("items", "late", json!({}), Precondition::Absent)- .await- .is_err()- );+ assert!(storage+ .documents()+ .put("items", "late", json!({}), Precondition::Absent)+ .await+ .is_err());assert!(storage.streams().append("events", json!(1)).await.is_err());- assert!(- storage- .blobs()- .put("blob", b"x".to_vec(), None)- .await- .is_err()- );+ assert!(storage+ .blobs()+ .put("blob", b"x".to_vec(), None)+ .await+ .is_err());assert!(storage.kv().put("key", json!(1)).await.is_err());assert!(storage.queues().ack("work", &receipt).await.is_err());let raw = shared.for_scope(&Scope::new("profile:alice").unwrap()).unwrap();- assert!(- raw.documents()- .get("items", "late")- .await- .unwrap()- .is_none()- );+ assert!(raw+ .documents()+ .get("items", "late")+ .await+ .unwrap()+ .is_none());assert_eq!(raw.streams().len("events").await.unwrap(), 0);assert!(raw.blobs().get("blob").await.unwrap().is_none());assert!(raw.kv().get("key").await.unwrap().is_none());assert_eq!(raw.queues().stats("work").await.unwrap().leased, 1);- assert!(- raw.documents()- .get("items", "kept")- .await- .unwrap()- .is_some()- );+ assert!(raw+ .documents()+ .get("items", "kept")+ .await+ .unwrap()+ .is_some());}#[tokio::test]@@ -215,23 +207,19 @@ async fn a_manually_latched_cached_fence_refuses_mutations_before_driver_io() {.await.unwrap();fence.fence();- assert!(- storage- .documents()- .put("items", "late", json!({}), Precondition::Absent)- .await- .is_err()- );- assert!(- shared- .for_scope(&Scope::new("profile:alice").unwrap())- .unwrap()- .documents()- .get("items", "late")- .await- .unwrap()- .is_none()- );+ assert!(storage+ .documents()+ .put("items", "late", json!({}), Precondition::Absent)+ .await+ .is_err());+ assert!(shared+ .for_scope(&Scope::new("profile:alice").unwrap())+ .unwrap()+ .documents()+ .get("items", "late")+ .await+ .unwrap()+ .is_none());}#[tokio::test]diff --git a/crates/openhuman-core/src/storage/fenced_ports.rs b/crates/openhuman-core/src/storage/fenced_ports.rsindex 598aabd497..34ccb4d1ac 100644--- a/crates/openhuman-core/src/storage/fenced_ports.rs+++ b/crates/openhuman-core/src/storage/fenced_ports.rs@@ -20,8 +20,8 @@ use tinystoragedrivers_types::{};use super::{- StorageError,fence::{FenceRegistry, LeaseFence},+ StorageError,};fn check(fence: &LeaseFence, clock: &FenceRegistry) -> Result<()> {