The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.
Rust agent-orchestrationai-agentsai-assistantdesktopllmlocal-firstmcppersonal-aiprivacyrustsecond-braintauri
feat(storage): add host known profiles and journal maintenance logic
Steven Enamakelaa35c7bfc712
diff --git a/crates/openhuman-core/src/profiles/host.rs b/crates/openhuman-core/src/profiles/host.rsindex bfee511279..51559cef2a 100644--- a/crates/openhuman-core/src/profiles/host.rs+++ b/crates/openhuman-core/src/profiles/host.rs@@ -157,6 +157,19 @@ impl ProfileHost {&self.node}+ /// Profile identities known to the host, including profiles that are not+ /// currently open. Background maintenance uses this inventory so dormant+ /// profiles are not skipped between requests.+ pub async fn known_profiles(&self) -> Result<Vec<String>, String> {+ Ok(self+ .registry+ .list()+ .await?+ .into_iter()+ .map(|profile| profile.profile_id.to_string())+ .collect())+ }+pub(crate) fn leases(&self) -> &dyn LeaseStore {self.leases.as_ref()}diff --git a/crates/openhuman-core/src/storage/journal_maintenance.rs b/crates/openhuman-core/src/storage/journal_maintenance.rsindex dd517ccce2..e87981c557 100644--- a/crates/openhuman-core/src/storage/journal_maintenance.rs+++ b/crates/openhuman-core/src/storage/journal_maintenance.rs@@ -1,201 +1,202 @@-//! Run-journal sealing and periodic storage maintenance (epic #7973, #7995).+//! Host scheduling and per-scope backfill state for harness journal maintenance.//!-//! A turn seals its journal stream (`harness.journal/<run_id>`) when it ends-//! (`agent::tinyagents::journal`). This module owns the rest of the lifecycle:-//!-//! - a **sweep** on a fixed tick: `Registry::maintain_all` applies every-//! owner's seal and retention policy (deleting journals sealed longer ago-//! than `[storage] journal_retention_days`), then `StorageBackend::maintain`-//! returns the freed space;-//! - a **one-shot backfill** per scope that seals the unsealed journal of-//! every run that is no longer active. It is recorded in that scope's-//! `tsd.migrations` (`harness` / [`BACKFILL_STEP`]); sealing is idempotent,-//! so a crash part-way simply re-runs the pass at the next sweep.-//!-//! Best-effort: a failure is logged and retried on the next tick.+//! Journal lifecycle rules belong to TinyAgents. This module supplies the+//! host's retention setting and clock, visits scopes the host knows about,+//! and records a durable per-scope marker after the owner's one-shot inactive+//! journal backfill succeeds.-use std::collections::HashSet;+use std::collections::BTreeSet;use std::sync::Arc;use std::time::Duration;-use tinyagents_harness::observability::DriverStatusStore;+use tinystoragedrivers::{CollectionSpec, ErrorKind, Precondition, Scope, StorageBackend};+use tinyagents_harness::observability::{DriverStatusStore, JournalMaintenanceReport};use tinyagents_harness::HarnessStatusStore;-use tinystoragedrivers::schema::{Item, Migrations, Registry};-use tinystoragedrivers::{- Capability, CollectionSpec, MaintenanceBudget, Scope, StorageBackend, StorageError,- MIGRATIONS_COLLECTION,-};--/// The journal stream family the harness declares.-pub const JOURNAL_STREAM: &str = "harness.journal";--/// Owner and step of the backfill's `tsd.migrations` record.-pub const BACKFILL_OWNER: &str = "harness";-/// Step id of the backfill's `tsd.migrations` record.-pub const BACKFILL_STEP: &str = "journal_seal_backfill_v1";/// How often the sweep wakes.pub const SWEEP_INTERVAL: Duration = Duration::from_secs(6 * 60 * 60);-/// Delay before the first sweep, after the boot reaper and migrations.+/// Delay before the first sweep, after boot preparation and migrations.pub const FIRST_SWEEP_DELAY: Duration = Duration::from_secs(10 * 60);+/// Host-owned durable marker collection for the one-shot journal backfill.+pub const BACKFILL_COLLECTION: &str = "openhuman.journal_backfill";++const BACKFILL_VERSION: u64 = 1;const LOG: &str = "[storage::journal_maintenance]";const DAY_MS: u64 = 24 * 60 * 60 * 1000;-/// The composed registry with the journal retention set to `days` (`0` keeps-/// sealed journals forever).-///-/// # Errors-///-/// The composition error, as [`super::schema::registry`].-pub fn registry_with_retention(days: u32) -> Result<Registry, StorageError> {- let mut schemas = super::schema::owner_schemas();- for schema in &mut schemas {- for item in &mut schema.items {- if let Item::Stream { name, seal, .. } = item {- if name == JOURNAL_STREAM {- seal.retain_ms = (days > 0).then(|| u64::from(days) * DAY_MS);- }- }- }- }- Registry::compose(schemas).map_err(|error| StorageError::invalid_input(error.to_string()))+/// Host collection storing the successful one-shot backfill marker per scope.+pub(crate) fn collection_specs() -> Vec<CollectionSpec> {+ vec![CollectionSpec::new(BACKFILL_COLLECTION)]}-/// What one maintenance pass did.+/// One pass over the scopes known to this host.#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]pub struct MaintenanceSummary {- /// Journals sealed by the backfill in this pass.+ /// Journals sealed by the inactive-run backfill.pub backfilled: u64,- /// Streams the policies deleted (retention passed).+ /// Expired sealed journals removed by TinyAgents.pub deleted: u64,- /// Streams the policies sealed.- pub sealed: u64,+ /// Eligible journals that changed before conditional deletion.+ pub conflicts: u64,+ /// Scopes whose maintenance pass completed.+ pub scopes: u64,}-/// One pass at `now_ms`: backfill, policies, then the backend's own-/// maintenance.+/// Runs owner journal maintenance once in every scope known to the host.///-/// # Errors-///-/// The policy or backend error; the backfill logs its own failures.+/// Owner policy stays in TinyAgents; OpenHuman supplies a retention duration,+/// timestamp, and a per-scope one-shot backfill decision. A failed owner pass+/// does not write the marker, so it is retried on the next sweep.pub async fn maintain_once(backend: &Arc<dyn StorageBackend>,retention_days: u32,now_ms: i64,-) -> Result<MaintenanceSummary, StorageError> {+) -> Result<MaintenanceSummary, crate::storage::StorageError> {+ let retention_ms = u64::from(retention_days).saturating_mul(DAY_MS);let mut summary = MaintenanceSummary::default();- for scope in scopes(backend).await {- match backfill_scope(backend.as_ref(), &scope).await {- Ok(sealed) => summary.backfilled += sealed,+ for scope in known_scopes().await? {+ match maintain_scope(backend, &scope, retention_ms, now_ms).await {+ Ok(report) => {+ summary.scopes += 1;+ summary.backfilled += report.backfilled as u64;+ summary.deleted += report.deleted as u64;+ summary.conflicts += report.conflicts as u64;+ }Err(error) => tracing::warn!(scope = scope.as_str(),%error,- "{LOG} journal seal backfill failed; retrying on the next sweep"+ "{LOG} owner journal maintenance failed; retrying on the next sweep"),}}- let registry = registry_with_retention(retention_days)?;- let budget = MaintenanceBudget::unlimited();- let report = if backend- .capabilities()- .contains(Capability::ScopeEnumeration)- {- registry- .maintain_all(backend.as_ref(), now_ms, budget)- .await?- } else {- registry- .maintain(&backend.for_scope(&Scope::local())?, now_ms, budget)- .await?- };- summary.deleted = report.deleted;- summary.sealed = report.sealed;- backend.maintain(budget).await?;tracing::debug!(+ scopes = summary.scopes,backfilled = summary.backfilled,- sealed = summary.sealed,deleted = summary.deleted,+ conflicts = summary.conflicts,"{LOG} pass complete");Ok(summary)}-/// `local` plus every scope the backend can list.-async fn scopes(backend: &Arc<dyn StorageBackend>) -> Vec<Scope> {- let mut scopes = vec![Scope::local()];- if backend- .capabilities()- .contains(Capability::ScopeEnumeration)- {- if let Ok(found) = backend.scopes("").await {- for scope in found {- if scope.parsed().is_ok() && !scope.is_cluster() && !scopes.contains(&scope) {- scopes.push(scope);- }+async fn maintain_scope(+ backend: &Arc<dyn StorageBackend>,+ scope: &Scope,+ retention_ms: u64,+ now_ms: i64,+) -> Result<JournalMaintenanceReport, crate::storage::StorageError> {+ let storage = backend.for_scope(scope)?;+ let docs = Arc::clone(storage.documents());+ docs.ensure_collection(&CollectionSpec::new(BACKFILL_COLLECTION))+ .await?;+ let key = crate::storage::schema_support::key_of("inactive-journal-backfill-v1")?;+ let backfill_done = match docs.get(BACKFILL_COLLECTION, &key).await? {+ None => false,+ Some(record) => {+ let version = record.doc.get("version").and_then(serde_json::Value::as_u64);+ if version != Some(BACKFILL_VERSION) {+ return Err(crate::storage::StorageError::invalid_input(format!(+ "invalid journal backfill marker in scope {}",+ scope.as_str()+ )));}+ true+ }+ };++ let statuses = DriverStatusStore::new(docs);+ let report = tinyagents_harness::observability::maintain_run_journals(+ &statuses,+ storage.streams().as_ref(),+ retention_ms,+ now_ms,+ !backfill_done,+ )+ .await+ .map_err(|error| crate::storage::StorageError::backend(error.to_string()))?;++ if !backfill_done {+ match storage+ .documents()+ .put_with(+ BACKFILL_COLLECTION,+ &key,+ serde_json::json!({ "version": BACKFILL_VERSION }),+ Precondition::Absent,+ Default::default(),+ )+ .await+ {+ Ok(_) => {}+ Err(error) if error.kind() == ErrorKind::Conflict => {}+ Err(error) => return Err(error),}}- scopes+ Ok(report)}-/// Seals every unsealed run journal in `scope` whose run is not active, once:-/// the pass is recorded in the scope's `tsd.migrations`. Returns how many-/// journals it sealed (0 when already done).-///-/// # Errors+/// Scopes represented by the host's own registries and compiled module table.///-/// A storage error; the record is written only after the whole pass, so the-/// next call resumes by skipping what is already sealed.-pub async fn backfill_scope(- backend: &dyn StorageBackend,- scope: &Scope,-) -> Result<u64, StorageError> {- let storage = backend.for_scope(scope)?;- storage- .documents()- .ensure_collection(&CollectionSpec::new(MIGRATIONS_COLLECTION))- .await?;- let migrations = Migrations::new(&storage);- if migrations.done(BACKFILL_OWNER, BACKFILL_STEP).await? {- return Ok(0);+/// This deliberately does not ask the backend to enumerate arbitrary scopes:+/// it includes local/cluster, registered profiles, known agent contexts, and+/// compiled-in module scopes for local and registered profiles.+async fn known_scopes() -> Result<Vec<Scope>, crate::storage::StorageError> {+ let mut scopes = BTreeSet::new();+ scopes.insert(Scope::cluster());+ if !crate::core::runtime::mode::is_saas() {+ scopes.insert(Scope::local());+ }++ let mut profiles = BTreeSet::new();+ if let Some(host) = crate::profiles::host::host() {+ let listed = host+ .known_profiles()+ .await+ .map_err(crate::storage::StorageError::backend)?;+ for profile in listed {+ profiles.insert(profile);+ }}- let active: HashSet<String> = DriverStatusStore::new(Arc::clone(storage.documents()))- .list_active()- .await- .map_err(|error| StorageError::invalid_input(error.to_string()))?- .into_iter()- .map(|status| status.run_id.as_str().to_string())- .collect();- let streams = storage.streams();- let family = format!("{JOURNAL_STREAM}/");- let mut sealed = 0;- for name in streams.streams(JOURNAL_STREAM).await? {- let Some(run) = name.strip_prefix(&family) else {- continue;- };- if active.contains(run) {- continue;++ for (agent, context) in crate::storage::agents::contexts() {+ let dispatch = crate::core::runtime::DispatchCtx::new(context);+ if let Ok(tenant) = dispatch.tenant() {+ if let Some(profile) = tenant.profile {+ profiles.insert(profile);+ } else if let Some(agent) = tenant.agent.or(Some(agent)) {+ scopes.insert(crate::storage::scope_for_agent(&agent));+ }}- match streams.info(&name).await? {- Some(info) if info.sealed_ms.is_none() => {- streams.seal(&name).await?;- sealed += 1;+ }++ let bases: Vec<Scope> = if crate::core::runtime::mode::is_saas() {+ profiles+ .iter()+ .map(|profile| crate::storage::scope_for_profile(profile))+ .collect()+ } else {+ std::iter::once(Scope::local())+ .chain(+ profiles+ .iter()+ .map(|profile| crate::storage::scope_for_profile(profile)),+ )+ .collect()+ };+ scopes.extend(bases.iter().cloned());+ for base in bases {+ for module in crate::modules::registry::ALL {+ if let Ok(scope) = base.module(module.id) {+ scopes.insert(scope);}- _ => {}}}- migrations.mark_done(BACKFILL_OWNER, BACKFILL_STEP).await?;- tracing::info!(- scope = scope.as_str(),- sealed,- "{LOG} journal seal backfill done"- );- Ok(sealed)+ Ok(scopes.into_iter().collect())}-/// Retention days from the current config; the default when it cannot load.+/// Retention days from the current config; default when it cannot load.async fn configured_retention_days(ctx: &crate::core::runtime::DispatchCtx) -> u32 {match crate::config::ops::load_current_or_init(ctx).await {Ok(config) => config.storage.journal_retention_days,diff --git a/crates/openhuman-core/src/storage/journal_maintenance_tests.rs b/crates/openhuman-core/src/storage/journal_maintenance_tests.rsindex 5971746343..5fdc5a335d 100644--- a/crates/openhuman-core/src/storage/journal_maintenance_tests.rs+++ b/crates/openhuman-core/src/storage/journal_maintenance_tests.rs@@ -1,21 +1,14 @@use super::*;-use std::sync::atomic::{AtomicU64, Ordering};use tinyagents_harness::events::{AgentEvent, EventSink, HarnessRunStatus};use tinyagents_harness::ids::{ComponentId, HarnessPhase, RunId};use tinyagents_harness::observability::replay::read_run_events_page;use tinyagents_harness::observability::{AgentObservation, HarnessEventJournal, StoreEventJournal};use tinyagents_harness::store::{AppendStore, DriverAppendStore};use tinyagents_harness::HarnessStatusStore;-use tinystoragedrivers::MemoryStorage;+use tinystoragedrivers_testkit::memory_storage;-fn backend_with_clock() -> (Arc<AtomicU64>, Arc<dyn StorageBackend>) {- let now = Arc::new(AtomicU64::new(1_000));- let clock = Arc::clone(&now);- let backend: Arc<dyn StorageBackend> =- Arc::new(MemoryStorage::with_clock(Arc::new(move || {- clock.load(Ordering::SeqCst)- })));- (now, backend)+fn backend() -> Arc<dyn StorageBackend> {+ memory_storage(&Scope::local())}fn journal(backend: &Arc<dyn StorageBackend>) -> StoreEventJournal<Arc<dyn AppendStore>> {@@ -77,7 +70,7 @@ async fn pages(#[tokio::test]async fn a_sealed_journal_serves_the_same_pages_as_an_unsealed_one() {- let (_, backend) = backend_with_clock();+ let backend = backend();let journal = journal(&backend);seed(&journal, "run.a", 7).await;let before = pages(&journal, "run.a").await;@@ -88,52 +81,59 @@ async fn a_sealed_journal_serves_the_same_pages_as_an_unsealed_one() {#[tokio::test]async fn backfill_seals_finished_runs_but_not_active_ones_and_records_itself() {- let (_, backend) = backend_with_clock();+ let backend = backend();let journal = journal(&backend);seed(&journal, "run.done", 2).await;seed(&journal, "run.live", 2).await;mark_active(&backend, "run.live").await;- assert_eq!(- backfill_scope(backend.as_ref(), &Scope::local())- .await- .unwrap(),- 1- );+ let report = maintain_scope(&backend, &Scope::local(), 0, 1_000)+ .await+ .unwrap();+ assert_eq!(report.backfilled, 1);assert!(sealed(&backend, "run.done").await);assert!(!sealed(&backend, "run.live").await);// Recorded: a later pass does nothing, even for a run that has since ended.seed(&journal, "run.late", 1).await;- assert_eq!(- backfill_scope(backend.as_ref(), &Scope::local())- .await- .unwrap(),- 0- );+ let report = maintain_scope(&backend, &Scope::local(), 0, 1_000)+ .await+ .unwrap();+ assert_eq!(report.backfilled, 0);assert!(!sealed(&backend, "run.late").await);let storage = backend.for_scope(&Scope::local()).unwrap();- assert!(Migrations::new(&storage)- .done(BACKFILL_OWNER, BACKFILL_STEP)+ let marker = storage+ .documents()+ .get(BACKFILL_COLLECTION, &crate::storage::schema_support::key_of("inactive-journal-backfill-v1").unwrap()).await- .unwrap());+ .unwrap()+ .unwrap();+ assert_eq!(marker.doc["version"], BACKFILL_VERSION);}#[tokio::test]async fn maintenance_deletes_a_sealed_journal_after_the_configured_retention() {- let (now, backend) = backend_with_clock();+ let backend = backend();let journal = journal(&backend);seed(&journal, "run.old", 2).await;journal.seal_run("run.old").await.unwrap();+ let storage = backend.for_scope(&Scope::local()).unwrap();+ let sealed_at = storage+ .streams()+ .info("harness.journal/run.old")+ .await+ .unwrap()+ .unwrap()+ .sealed_ms+ .unwrap();// Inside the window: kept.- let early = maintain_once(&backend, 1, 1_000 + DAY_MS as i64 / 2)+ let early = maintain_scope(&backend, &Scope::local(), DAY_MS, sealed_at + DAY_MS as i64 / 2).await.unwrap();assert_eq!(early.deleted, 0);// Past one day, the configured retention (not the schema's 30).- now.store(1_000 + DAY_MS + 10, Ordering::SeqCst);- let late = maintain_once(&backend, 1, 1_000 + DAY_MS as i64 + 10)+ let late = maintain_scope(&backend, &Scope::local(), DAY_MS, sealed_at + DAY_MS as i64 + 10).await.unwrap();assert_eq!(late.deleted, 1);@@ -148,11 +148,20 @@ async fn maintenance_deletes_a_sealed_journal_after_the_configured_retention() {#[tokio::test]async fn zero_retention_days_keeps_sealed_journals_forever() {- let (_, backend) = backend_with_clock();+ let backend = backend();let journal = journal(&backend);seed(&journal, "run.keep", 1).await;journal.seal_run("run.keep").await.unwrap();- let summary = maintain_once(&backend, 0, 1_000 + 400 * DAY_MS as i64)+ let storage = backend.for_scope(&Scope::local()).unwrap();+ let sealed_at = storage+ .streams()+ .info("harness.journal/run.keep")+ .await+ .unwrap()+ .unwrap()+ .sealed_ms+ .unwrap();+ let summary = maintain_scope(&backend, &Scope::local(), 0, sealed_at + 400 * DAY_MS as i64).await.unwrap();assert_eq!(summary.deleted, 0);diff --git a/crates/openhuman-core/src/storage/schema.rs b/crates/openhuman-core/src/storage/schema.rsindex dac89787a8..dfbedfbcec 100644--- a/crates/openhuman-core/src/storage/schema.rs+++ b/crates/openhuman-core/src/storage/schema.rs@@ -43,6 +43,7 @@ fn declared_collections() -> Vec<CollectionSpec> {collections.extend(crate::threads::store::collection_specs());collections.extend(crate::agent::orchestration::subagent_sessions::collection_specs());collections.extend(super::agents::collection_specs());+ collections.extend(super::journal_maintenance::collection_specs());#[cfg(feature = "skills")]collections.extend(tinyskills_drivers::collections());