Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

chore: files changed crates/openhuman-core/src/modules/storage_host.rs,crates/openhuman-core/src/mod

Steven Enamakel705c43f3fd69
diff --git a/crates/openhuman-core/src/modules/storage_host.rs b/crates/openhuman-core/src/modules/storage_host.rs
index 8ad61c6556..9c9a154e5d 100644
--- a/crates/openhuman-core/src/modules/storage_host.rs
+++ b/crates/openhuman-core/src/modules/storage_host.rs
@@ -45,8 +45,8 @@ use std::sync::Arc;
use async_trait::async_trait;
use serde_json::Value;
use tinybus::BusName;
-use tinystoragedrivers_api::BlobReader;
use tinystoragedrivers::secrets::{DocumentSecrets, KeyProvider};
+use tinystoragedrivers_api::BlobReader;
use tinystoragedrivers_bus::{
BlobKeyRequest, BlobStreamHead, PutBlobStreamRequest, StorageFailure, StorageKind,
};
diff --git a/crates/openhuman-core/src/modules/storage_host_linked_tests.rs b/crates/openhuman-core/src/modules/storage_host_linked_tests.rs
index 7b76c0151a..8ee5fad2ad 100644
--- a/crates/openhuman-core/src/modules/storage_host_linked_tests.rs
+++ b/crates/openhuman-core/src/modules/storage_host_linked_tests.rs
@@ -6,8 +6,8 @@ use tinybus::transport::memory::MemoryBus;
use tinybus::{BusName, Connection};
use tinystoragedrivers::secrets::{DerivedKeys, KeyProvider};
use tinystoragedrivers::{Scope, StorageError};
-use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage;
use tinystoragedrivers_bus::{SecretNameRequest, SecretValue, SetSecretRequest};
+use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage;
use zeroize::Zeroizing;
use super::super::storage_grants::StorageGrant;
diff --git a/crates/openhuman-core/src/modules/storage_host_queues_tests.rs b/crates/openhuman-core/src/modules/storage_host_queues_tests.rs
index 626709a581..74785ab6b1 100644
--- a/crates/openhuman-core/src/modules/storage_host_queues_tests.rs
+++ b/crates/openhuman-core/src/modules/storage_host_queues_tests.rs
@@ -142,10 +142,7 @@ async fn a_queue_outside_the_manifest_is_not_found_and_a_divergent_spec_conflict
async fn a_module_without_a_queue_grant_connects_but_is_denied_queues() {
let peers = peers().await;
let queues = queues_of(&peers.plain).await;
- assert_eq!(
- queues.features(),
- QueueFeatures::default()
- );
+ assert_eq!(queues.features(), QueueFeatures::default());
let error = queues
.enqueue("inbox", &[NewMessage::new(json!({}))])
.await
diff --git a/crates/openhuman-core/src/modules/storage_host_watching.rs b/crates/openhuman-core/src/modules/storage_host_watching.rs
index 7ecf41e125..78497d2afd 100644
--- a/crates/openhuman-core/src/modules/storage_host_watching.rs
+++ b/crates/openhuman-core/src/modules/storage_host_watching.rs
@@ -10,7 +10,7 @@ use std::collections::HashMap;
use std::sync::{Arc, Mutex};
use serde_json::Value;
-use tinystoragedrivers_api::{DocumentStore, DocumentWatch, Storage, StorageBackend, async_trait};
+use tinystoragedrivers_api::{async_trait, DocumentStore, DocumentWatch, Storage, StorageBackend};
use tinystoragedrivers_testkit::MemoryBackend;
use tinystoragedrivers_types::{
Capabilities, Capability, CollectionSpec, Filter, Page, Precondition, Query, Result, Scope,
diff --git a/crates/openhuman-core/src/storage/README.md b/crates/openhuman-core/src/storage/README.md
index acadc8d980..04d3737c36 100644
--- a/crates/openhuman-core/src/storage/README.md
+++ b/crates/openhuman-core/src/storage/README.md
@@ -104,13 +104,20 @@ it with the core stopped.
need. `driver_has_cross_process_cas(driver)` is the same answer from a
name, for the boot guard, which judges a URL before opening anything.
Memory and file drivers coordinate only within one process.
-- `blob_sweep`: reclaims expired blobs (`StorageBackend::sweep_expired_blobs`)
+- `blob_sweep`: reclaims expired blobs (`BlobStore::sweep_expired`)
for `local`, each known agent scope and each provisioned SaaS profile
(`u:<id>` plus the blob-owning modules' scopes for it) every hour,
spawned with the cron `ServiceSet` switch (`core::runtime::services`; not
gated on `[cron] enabled`, since it reclaims storage rather than running
jobs). Each scope's sweep is bounded by `SCOPE_SWEEP_TIMEOUT`; a timeout
counts as a failed sweep and is retried on the next tick.
+- `sweep_unreferenced_blobs`: an explicit maintenance helper that pages
+ physical blob metadata, keeps unknown or cutoff-equal ages, asks the caller
+ whether each old key is referenced, and stops on the first error while
+ preserving earlier deletions. It is not activated by the periodic sweeper:
+ reference checking and deletion are separate operations, so a caller must
+ coordinate blob writers/reference creation with cleanup or use immutable
+ keys, with a grace period longer than the write-to-reference delay.
- `journal_maintenance`: invokes TinyAgents' `maintain_run_journals` with the
host's configured retention and clock. It visits local and known agent
scopes plus module scopes, and only profiles currently open under this
diff --git a/crates/openhuman-core/src/storage/blob_sweep.rs b/crates/openhuman-core/src/storage/blob_sweep.rs
index 31976af9b9..81c4a2ebbd 100644
--- a/crates/openhuman-core/src/storage/blob_sweep.rs
+++ b/crates/openhuman-core/src/storage/blob_sweep.rs
@@ -15,6 +15,11 @@ use std::sync::Arc;
use std::time::Duration;
use tinystoragedrivers::{Scope, StorageBackend};
+use tinystoragedrivers_api::BlobStore;
+use tinystoragedrivers_types::{
+ is_reserved_blob_key, BlobListPage, Key, Result as StorageResult, StorageError,
+ MAX_BLOB_LIST_PAGE_ITEMS, SEALED_BLOB_PREFIX,
+};
/// How often the sweeper wakes. Lapsed blobs are already invisible to readers,
/// so this only bounds how long their bytes linger.
@@ -30,6 +35,118 @@ pub const FIRST_SWEEP_DELAY: Duration = Duration::from_secs(5 * 60);
/// retried on the next tick.
pub const SCOPE_SWEEP_TIMEOUT: Duration = Duration::from_secs(5 * 60);
+/// Delete old user blobs after the caller confirms each key is unreferenced.
+///
+/// This is a maintenance primitive, not an automatic policy: the caller must
+/// arrange that blob writers and reference creation cannot race the check and
+/// delete (or use immutable keys). The age grace period must exceed the maximum
+/// time from writing a blob until its durable reference is visible. This
+/// function is deliberately not called by [`sweep_all`].
+///
+/// Only blobs with a known `written_at_ms` strictly less than `older_than_ms`
+/// are considered. Unknown ages and entries at the cutoff are retained. Pages
+/// are bounded and key ordered. An error stops the sweep; deletions from prior
+/// entries remain applied and the caller can retry after resolving the error.
+///
+/// # Errors
+/// Returns an error for an invalid or engine-owned prefix, invalid listing
+/// data, unsupported paging, a listing/deletion error, or a reference-check
+/// error.
+pub async fn sweep_unreferenced_blobs<F, Fut>(
+ blobs: &dyn BlobStore,
+ prefix: &str,
+ older_than_ms: u64,
+ mut is_referenced: F,
+) -> StorageResult<u64>
+where
+ F: FnMut(String) -> Fut,
+ Fut: std::future::Future<Output = StorageResult<bool>>,
+{
+ validate_orphan_sweep_prefix(prefix)?;
+ let mut after: Option<String> = None;
+ let mut removed = 0_u64;
+ loop {
+ let page = blobs
+ .list_page(prefix, after.as_deref(), MAX_BLOB_LIST_PAGE_ITEMS)
+ .await?;
+ validate_orphan_sweep_page(prefix, after.as_deref(), &page)?;
+ for entry in page.items {
+ if !orphan_age_is_eligible(entry.written_at_ms, older_than_ms)
+ || is_referenced(entry.meta.key.clone()).await?
+ {
+ continue;
+ }
+ if blobs.delete(&entry.meta.key).await? {
+ removed = removed.saturating_add(1);
+ }
+ }
+ let Some(next_after) = page.next_after else {
+ return Ok(removed);
+ };
+ after = Some(next_after);
+ }
+}
+
+fn orphan_age_is_eligible(written_at_ms: Option<u64>, older_than_ms: u64) -> bool {
+ written_at_ms.is_some_and(|written_at_ms| written_at_ms < older_than_ms)
+}
+
+fn validate_orphan_sweep_prefix(prefix: &str) -> StorageResult<()> {
+ let key = Key::new(prefix.to_owned())?;
+ if key.as_str().is_empty()
+ || SEALED_BLOB_PREFIX.starts_with(prefix)
+ || prefix.starts_with(SEALED_BLOB_PREFIX)
+ || prefix
+ .trim_end_matches('/')
+ .split('/')
+ .any(|segment| segment.is_empty() || segment == "." || segment == "..")
+ {
+ return Err(StorageError::invalid_input(
+ "orphan sweep requires a nonempty user blob prefix",
+ ));
+ }
+ Ok(())
+}
+
+fn validate_orphan_sweep_page(
+ prefix: &str,
+ after: Option<&str>,
+ page: &BlobListPage,
+) -> StorageResult<()> {
+ if page.items.len() > MAX_BLOB_LIST_PAGE_ITEMS {
+ return Err(StorageError::backend(
+ "orphan sweep listing page exceeded its requested bound",
+ ));
+ }
+ let mut previous = after;
+ for entry in &page.items {
+ let key = entry.meta.key.as_str();
+ if !key.starts_with(prefix)
+ || Key::new(key.to_owned()).is_err()
+ || is_reserved_blob_key(key)
+ || previous.is_some_and(|previous| key <= previous)
+ {
+ return Err(StorageError::backend(
+ "orphan sweep listing returned an invalid key",
+ ));
+ }
+ previous = Some(key);
+ }
+ if let Some(cursor) = &page.next_after {
+ if page
+ .items
+ .last()
+ .is_none_or(|entry| entry.meta.key != *cursor)
+ || after.is_some_and(|after| cursor.as_str() <= after)
+ {
+ return Err(StorageError::backend(
+ "orphan sweep listing cursor did not advance",
+ ));
+ }
+ }
+ Ok(())
+}
+
/// Reclaims the lapsed blobs of `scope`, returning how many went. A failure
/// or a sweep past [`SCOPE_SWEEP_TIMEOUT`] is logged and counts as zero.
pub async fn sweep_scope(backend: &dyn StorageBackend, scope: &Scope) -> u64 {
diff --git a/crates/openhuman-core/src/storage/blob_sweep_tests.rs b/crates/openhuman-core/src/storage/blob_sweep_tests.rs
index 9b6a5b47ec..941bdd13bc 100644
--- a/crates/openhuman-core/src/storage/blob_sweep_tests.rs
+++ b/crates/openhuman-core/src/storage/blob_sweep_tests.rs
@@ -211,3 +211,154 @@ fn module_blob_scopes_follow_the_handed_context_and_never_cross_tenants() {
assert!(alice.iter().all(|s| s.as_str().contains("blob-alice")));
assert!(bob.iter().all(|s| s.as_str().contains("blob-bob")));
}
+
+#[test]
+fn orphan_sweep_age_is_strict_and_unknown_ages_are_retained() {
+ assert!(!orphan_age_is_eligible(None, 100));
+ assert!(!orphan_age_is_eligible(Some(100), 100));
+ assert!(orphan_age_is_eligible(Some(99), 100));
+}
+
+#[test]
+fn orphan_sweep_refuses_empty_and_engine_owned_prefixes() {
+ for prefix in ["", "_tsd/", "_tsd/sealed", "_tsd/sealed/stream/"] {
+ assert_eq!(
+ validate_orphan_sweep_prefix(prefix).unwrap_err().kind(),
+ tinystoragedrivers_types::ErrorKind::InvalidInput,
+ "prefix {prefix:?}"
+ );
+ }
+ assert!(validate_orphan_sweep_prefix("user/attachments/").is_ok());
+}
+
+#[test]
+fn orphan_sweep_rejects_malformed_or_nonadvancing_listing_pages() {
+ use tinystoragedrivers_types::{BlobListEntry, BlobListPage, BlobMeta};
+
+ let page = BlobListPage {
+ items: vec![BlobListEntry {
+ meta: BlobMeta {
+ key: "user/b".into(),
+ len: 1,
+ content_type: None,
+ },
+ written_at_ms: Some(1),
+ }],
+ next_after: Some("user/a".into()),
+ };
+ assert_eq!(
+ validate_orphan_sweep_page("user/", None, &page)
+ .unwrap_err()
+ .kind(),
+ tinystoragedrivers_types::ErrorKind::Backend
+ );
+ assert_eq!(
+ validate_orphan_sweep_page("user/", Some("user/b"), &page)
+ .unwrap_err()
+ .kind(),
+ tinystoragedrivers_types::ErrorKind::Backend
+ );
+}
+
+#[tokio::test]
+async fn orphan_sweep_pages_and_preserves_referenced_blobs() {
+ use tinystoragedrivers_api::BlobStore;
+ use tinystoragedrivers_testkit::memory_storage;
+ use tinystoragedrivers_types::Scope;
+
+ let storage = memory_storage(&Scope::local());
+ let blobs = storage.blobs();
+ for index in 0..=MAX_BLOB_LIST_PAGE_ITEMS {
+ blobs
+ .put(&format!("cleanup/{index:04}"), vec![index as u8], None)
+ .await
+ .unwrap();
+ }
+ let kept = format!("cleanup/{:04}", MAX_BLOB_LIST_PAGE_ITEMS);
+ let removed = sweep_unreferenced_blobs(blobs.as_ref(), "cleanup/", u64::MAX, |key| {
+ let kept = kept.clone();
+ async move { Ok(key == kept) }
+ })
+ .await
+ .unwrap();
+
+ assert_eq!(removed, MAX_BLOB_LIST_PAGE_ITEMS as u64);
+ assert!(blobs.get(&kept).await.unwrap().is_some());
+ assert!(blobs.get("cleanup/0000").await.unwrap().is_none());
+}
+
+#[tokio::test]
+async fn orphan_sweep_stops_on_reference_error_after_preserving_prior_progress() {
+ use tinystoragedrivers_api::BlobStore;
+ use tinystoragedrivers_testkit::memory_storage;
+ use tinystoragedrivers_types::Scope;
+
+ let storage = memory_storage(&Scope::local());
+ let blobs = storage.blobs();
+ for key in ["cleanup/a", "cleanup/b", "cleanup/c"] {
+ blobs.put(key, vec![1], None).await.unwrap();
+ }
+
+ let error = sweep_unreferenced_blobs(blobs.as_ref(), "cleanup/", u64::MAX, |key| async move {
+ if key == "cleanup/b" {
+ Err(StorageError::backend("reference lookup failed"))
+ } else {
+ Ok(false)
+ }
+ })
+ .await
+ .unwrap_err();
+
+ assert_eq!(error.kind(), tinystoragedrivers_types::ErrorKind::Backend);
+ assert!(blobs.get("cleanup/a").await.unwrap().is_none());
+ assert!(blobs.get("cleanup/b").await.unwrap().is_some());
+ assert!(blobs.get("cleanup/c").await.unwrap().is_some());
+}
+
+#[tokio::test]
+async fn orphan_sweep_stops_on_listing_error_before_touching_blobs() {
+ use std::sync::Arc;
+ use tinystoragedrivers_api::BlobStore;
+ use tinystoragedrivers_testkit::{memory_storage, Failing, FaultPlan};
+ use tinystoragedrivers_types::{ErrorKind, Scope};
+
+ let storage = memory_storage(&Scope::local());
+ let blobs = Arc::clone(storage.blobs());
+ blobs.put("cleanup/a", vec![1], None).await.unwrap();
+ let faults = FaultPlan::new();
+ faults.fail_operation_nth("blob.list_page", 1, ErrorKind::Unavailable);
+ let failing = Failing::with_plan(blobs.clone(), faults);
+
+ let error = sweep_unreferenced_blobs(&failing, "cleanup/", u64::MAX, |_| async { Ok(false) })
+ .await
+ .unwrap_err();
+
+ assert_eq!(error.kind(), ErrorKind::Unavailable);
+ assert!(blobs.get("cleanup/a").await.unwrap().is_some());
+}
+
+#[tokio::test]
+async fn orphan_sweep_stops_on_delete_error_after_preserving_prior_progress() {
+ use std::sync::Arc;
+ use tinystoragedrivers_api::BlobStore;
+ use tinystoragedrivers_testkit::{memory_storage, Failing, FaultPlan};
+ use tinystoragedrivers_types::{ErrorKind, Scope};
+
+ let storage = memory_storage(&Scope::local());
+ let blobs = Arc::clone(storage.blobs());
+ for key in ["cleanup/a", "cleanup/b", "cleanup/c"] {
+ blobs.put(key, vec![1], None).await.unwrap();
+ }
+ let faults = FaultPlan::new();
+ faults.fail_operation_nth("blob.delete", 2, ErrorKind::Unavailable);
+ let failing = Failing::with_plan(blobs.clone(), faults);
+
+ let error = sweep_unreferenced_blobs(&failing, "cleanup/", u64::MAX, |_| async { Ok(false) })
+ .await
+ .unwrap_err();
+
+ assert_eq!(error.kind(), ErrorKind::Unavailable);
+ assert!(blobs.get("cleanup/a").await.unwrap().is_none());
+ assert!(blobs.get("cleanup/b").await.unwrap().is_some());
+ assert!(blobs.get("cleanup/c").await.unwrap().is_some());
+}
diff --git a/crates/openhuman-core/src/storage/fenced_backend_tests.rs b/crates/openhuman-core/src/storage/fenced_backend_tests.rs
index f872801a7d..f0ea8c2004 100644
--- a/crates/openhuman-core/src/storage/fenced_backend_tests.rs
+++ b/crates/openhuman-core/src/storage/fenced_backend_tests.rs
@@ -6,8 +6,8 @@ use crate::storage::fence::{FenceError, LeaseFence, ScopeMatch};
use crate::storage::lease::{DocumentLeases, LeaseGrant, LeaseStore};
use serde_json::json;
use std::sync::{
- Arc,
atomic::{AtomicU64, Ordering},
+ Arc,
};
use std::time::Duration;
use tinystoragedrivers_api::StorageBackend as _;
@@ -112,14 +112,12 @@ async fn cached_handles_enforce_local_expiry_margin_and_accept_same_epoch_renewa
fence_error(&error),
Some(FenceError::Expired { .. })
));
- assert!(
- storage
- .documents()
- .get("items", "too-late")
- .await
- .unwrap()
- .is_none()
- );
+ assert!(storage
+ .documents()
+ .get("items", "too-late")
+ .await
+ .unwrap()
+ .is_none());
}
#[tokio::test]
@@ -162,45 +160,39 @@ async fn a_cached_handle_cannot_mutate_after_cross_process_takeover_on_any_port(
// A's local clock is deliberately behind. The local precheck succeeds;
// the driver's atomic condition is what refuses each cached-handle write.
a.set_now(1_000);
- assert!(
- storage
- .documents()
- .put("items", "late", json!({}), Precondition::Absent)
- .await
- .is_err()
- );
+ assert!(storage
+ .documents()
+ .put("items", "late", json!({}), Precondition::Absent)
+ .await
+ .is_err());
assert!(storage.streams().append("events", json!(1)).await.is_err());
- assert!(
- storage
- .blobs()
- .put("blob", b"x".to_vec(), None)
- .await
- .is_err()
- );
+ assert!(storage
+ .blobs()
+ .put("blob", b"x".to_vec(), None)
+ .await
+ .is_err());
assert!(storage.kv().put("key", json!(1)).await.is_err());
assert!(storage.queues().ack("work", &receipt).await.is_err());
let raw = shared
.for_scope(&Scope::new("profile:alice").unwrap())
.unwrap();
- assert!(
- raw.documents()
- .get("items", "late")
- .await
- .unwrap()
- .is_none()
- );
+ assert!(raw
+ .documents()
+ .get("items", "late")
+ .await
+ .unwrap()
+ .is_none());
assert_eq!(raw.streams().len("events").await.unwrap(), 0);
assert!(raw.blobs().get("blob").await.unwrap().is_none());
assert!(raw.kv().get("key").await.unwrap().is_none());
assert_eq!(raw.queues().stats("work").await.unwrap().leased, 1);
- assert!(
- raw.documents()
- .get("items", "kept")
- .await
- .unwrap()
- .is_some()
- );
+ assert!(raw
+ .documents()
+ .get("items", "kept")
+ .await
+ .unwrap()
+ .is_some());
}
#[tokio::test]
@@ -215,23 +207,19 @@ async fn a_manually_latched_cached_fence_refuses_mutations_before_driver_io() {
.await
.unwrap();
fence.fence();
- assert!(
- storage
- .documents()
- .put("items", "late", json!({}), Precondition::Absent)
- .await
- .is_err()
- );
- assert!(
- shared
- .for_scope(&Scope::new("profile:alice").unwrap())
- .unwrap()
- .documents()
- .get("items", "late")
- .await
- .unwrap()
- .is_none()
- );
+ assert!(storage
+ .documents()
+ .put("items", "late", json!({}), Precondition::Absent)
+ .await
+ .is_err());
+ assert!(shared
+ .for_scope(&Scope::new("profile:alice").unwrap())
+ .unwrap()
+ .documents()
+ .get("items", "late")
+ .await
+ .unwrap()
+ .is_none());
}
#[tokio::test]
diff --git a/crates/openhuman-core/src/storage/fenced_ports.rs b/crates/openhuman-core/src/storage/fenced_ports.rs
index 598aabd497..34ccb4d1ac 100644
--- a/crates/openhuman-core/src/storage/fenced_ports.rs
+++ b/crates/openhuman-core/src/storage/fenced_ports.rs
@@ -20,8 +20,8 @@ use tinystoragedrivers_types::{
};
use super::{
- StorageError,
fence::{FenceRegistry, LeaseFence},
+ StorageError,
};
fn check(fence: &LeaseFence, clock: &FenceRegistry) -> Result<()> {