Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

fix(storage): validate sweep keys against backend max key length

Steven Enamakel5992b05c3e2a
diff --git a/crates/openhuman-core/src/storage/blob_sweep.rs b/crates/openhuman-core/src/storage/blob_sweep.rs
index 81c4a2ebbd..c4ec0ff2b0 100644
--- a/crates/openhuman-core/src/storage/blob_sweep.rs
+++ b/crates/openhuman-core/src/storage/blob_sweep.rs
@@ -17,7 +17,7 @@ use std::time::Duration;
use tinystoragedrivers::{Scope, StorageBackend};
use tinystoragedrivers_api::BlobStore;
use tinystoragedrivers_types::{
- is_reserved_blob_key, BlobListPage, Key, Result as StorageResult, StorageError,
+ is_reserved_blob_key, BlobListPage, Result as StorageResult, StorageError,
MAX_BLOB_LIST_PAGE_ITEMS, SEALED_BLOB_PREFIX,
};
@@ -62,14 +62,14 @@ where
F: FnMut(String) -> Fut,
Fut: std::future::Future<Output = StorageResult<bool>>,
{
- validate_orphan_sweep_prefix(prefix)?;
+ validate_orphan_sweep_prefix(prefix, blobs.max_key_len())?;
let mut after: Option<String> = None;
let mut removed = 0_u64;
loop {
let page = blobs
.list_page(prefix, after.as_deref(), MAX_BLOB_LIST_PAGE_ITEMS)
.await?;
- validate_orphan_sweep_page(prefix, after.as_deref(), &page)?;
+ validate_orphan_sweep_page(prefix, after.as_deref(), &page, blobs.max_key_len())?;
for entry in page.items {
if !orphan_age_is_eligible(entry.written_at_ms, older_than_ms)
|| is_referenced(entry.meta.key.clone()).await?
@@ -91,9 +91,10 @@ fn orphan_age_is_eligible(written_at_ms: Option<u64>, older_than_ms: u64) -> boo
written_at_ms.is_some_and(|written_at_ms| written_at_ms < older_than_ms)
}
-fn validate_orphan_sweep_prefix(prefix: &str) -> StorageResult<()> {
- let key = Key::new(prefix.to_owned())?;
- if key.as_str().is_empty()
+fn validate_orphan_sweep_prefix(prefix: &str, max_key_len: usize) -> StorageResult<()> {
+ if prefix.is_empty()
+ || prefix.len() > max_key_len
+ || prefix.bytes().any(|byte| byte == 0 || byte.is_ascii_control())
|| SEALED_BLOB_PREFIX.starts_with(prefix)
|| prefix.starts_with(SEALED_BLOB_PREFIX)
|| prefix
@@ -112,6 +113,7 @@ fn validate_orphan_sweep_page(
prefix: &str,
after: Option<&str>,
page: &BlobListPage,
+ max_key_len: usize,
) -> StorageResult<()> {
if page.items.len() > MAX_BLOB_LIST_PAGE_ITEMS {
return Err(StorageError::backend(
@@ -122,7 +124,9 @@ fn validate_orphan_sweep_page(
for entry in &page.items {
let key = entry.meta.key.as_str();
if !key.starts_with(prefix)
- || Key::new(key.to_owned()).is_err()
+ || key.is_empty()
+ || key.len() > max_key_len
+ || key.bytes().any(|byte| byte == 0 || byte.is_ascii_control())
|| is_reserved_blob_key(key)
|| previous.is_some_and(|previous| key <= previous)
{
diff --git a/crates/openhuman-core/src/storage/blob_sweep_tests.rs b/crates/openhuman-core/src/storage/blob_sweep_tests.rs
index 941bdd13bc..bc218ddcdd 100644
--- a/crates/openhuman-core/src/storage/blob_sweep_tests.rs
+++ b/crates/openhuman-core/src/storage/blob_sweep_tests.rs
@@ -223,12 +223,18 @@ fn orphan_sweep_age_is_strict_and_unknown_ages_are_retained() {
fn orphan_sweep_refuses_empty_and_engine_owned_prefixes() {
for prefix in ["", "_tsd/", "_tsd/sealed", "_tsd/sealed/stream/"] {
assert_eq!(
- validate_orphan_sweep_prefix(prefix).unwrap_err().kind(),
+ validate_orphan_sweep_prefix(prefix, 1024).unwrap_err().kind(),
tinystoragedrivers_types::ErrorKind::InvalidInput,
"prefix {prefix:?}"
);
}
- assert!(validate_orphan_sweep_prefix("user/attachments/").is_ok());
+ assert!(validate_orphan_sweep_prefix("user/attachments/", 1024).is_ok());
+ assert_eq!(
+ validate_orphan_sweep_prefix("a longer prefix", 3)
+ .unwrap_err()
+ .kind(),
+ tinystoragedrivers_types::ErrorKind::InvalidInput
+ );
}
#[test]
@@ -247,13 +253,13 @@ fn orphan_sweep_rejects_malformed_or_nonadvancing_listing_pages() {
next_after: Some("user/a".into()),
};
assert_eq!(
- validate_orphan_sweep_page("user/", None, &page)
+ validate_orphan_sweep_page("user/", None, &page, 1024)
.unwrap_err()
.kind(),
tinystoragedrivers_types::ErrorKind::Backend
);
assert_eq!(
- validate_orphan_sweep_page("user/", Some("user/b"), &page)
+ validate_orphan_sweep_page("user/", Some("user/b"), &page, 1024)
.unwrap_err()
.kind(),
tinystoragedrivers_types::ErrorKind::Backend