Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

feat(storage): add host known profiles and journal maintenance logic

Steven Enamakelaa35c7bfc712
diff --git a/crates/openhuman-core/src/profiles/host.rs b/crates/openhuman-core/src/profiles/host.rs
index bfee511279..51559cef2a 100644
--- a/crates/openhuman-core/src/profiles/host.rs
+++ b/crates/openhuman-core/src/profiles/host.rs
@@ -157,6 +157,19 @@ impl ProfileHost {
&self.node
}
+ /// Profile identities known to the host, including profiles that are not
+ /// currently open. Background maintenance uses this inventory so dormant
+ /// profiles are not skipped between requests.
+ pub async fn known_profiles(&self) -> Result<Vec<String>, String> {
+ Ok(self
+ .registry
+ .list()
+ .await?
+ .into_iter()
+ .map(|profile| profile.profile_id.to_string())
+ .collect())
+ }
+
pub(crate) fn leases(&self) -> &dyn LeaseStore {
self.leases.as_ref()
}
diff --git a/crates/openhuman-core/src/storage/journal_maintenance.rs b/crates/openhuman-core/src/storage/journal_maintenance.rs
index dd517ccce2..e87981c557 100644
--- a/crates/openhuman-core/src/storage/journal_maintenance.rs
+++ b/crates/openhuman-core/src/storage/journal_maintenance.rs
@@ -1,201 +1,202 @@
-//! Run-journal sealing and periodic storage maintenance (epic #7973, #7995).
+//! Host scheduling and per-scope backfill state for harness journal maintenance.
//!
-//! A turn seals its journal stream (`harness.journal/<run_id>`) when it ends
-//! (`agent::tinyagents::journal`). This module owns the rest of the lifecycle:
-//!
-//! - a **sweep** on a fixed tick: `Registry::maintain_all` applies every
-//! owner's seal and retention policy (deleting journals sealed longer ago
-//! than `[storage] journal_retention_days`), then `StorageBackend::maintain`
-//! returns the freed space;
-//! - a **one-shot backfill** per scope that seals the unsealed journal of
-//! every run that is no longer active. It is recorded in that scope's
-//! `tsd.migrations` (`harness` / [`BACKFILL_STEP`]); sealing is idempotent,
-//! so a crash part-way simply re-runs the pass at the next sweep.
-//!
-//! Best-effort: a failure is logged and retried on the next tick.
+//! Journal lifecycle rules belong to TinyAgents. This module supplies the
+//! host's retention setting and clock, visits scopes the host knows about,
+//! and records a durable per-scope marker after the owner's one-shot inactive
+//! journal backfill succeeds.
-use std::collections::HashSet;
+use std::collections::BTreeSet;
use std::sync::Arc;
use std::time::Duration;
-use tinyagents_harness::observability::DriverStatusStore;
+use tinystoragedrivers::{CollectionSpec, ErrorKind, Precondition, Scope, StorageBackend};
+use tinyagents_harness::observability::{DriverStatusStore, JournalMaintenanceReport};
use tinyagents_harness::HarnessStatusStore;
-use tinystoragedrivers::schema::{Item, Migrations, Registry};
-use tinystoragedrivers::{
- Capability, CollectionSpec, MaintenanceBudget, Scope, StorageBackend, StorageError,
- MIGRATIONS_COLLECTION,
-};
-
-/// The journal stream family the harness declares.
-pub const JOURNAL_STREAM: &str = "harness.journal";
-
-/// Owner and step of the backfill's `tsd.migrations` record.
-pub const BACKFILL_OWNER: &str = "harness";
-/// Step id of the backfill's `tsd.migrations` record.
-pub const BACKFILL_STEP: &str = "journal_seal_backfill_v1";
/// How often the sweep wakes.
pub const SWEEP_INTERVAL: Duration = Duration::from_secs(6 * 60 * 60);
-/// Delay before the first sweep, after the boot reaper and migrations.
+/// Delay before the first sweep, after boot preparation and migrations.
pub const FIRST_SWEEP_DELAY: Duration = Duration::from_secs(10 * 60);
+/// Host-owned durable marker collection for the one-shot journal backfill.
+pub const BACKFILL_COLLECTION: &str = "openhuman.journal_backfill";
+
+const BACKFILL_VERSION: u64 = 1;
const LOG: &str = "[storage::journal_maintenance]";
const DAY_MS: u64 = 24 * 60 * 60 * 1000;
-/// The composed registry with the journal retention set to `days` (`0` keeps
-/// sealed journals forever).
-///
-/// # Errors
-///
-/// The composition error, as [`super::schema::registry`].
-pub fn registry_with_retention(days: u32) -> Result<Registry, StorageError> {
- let mut schemas = super::schema::owner_schemas();
- for schema in &mut schemas {
- for item in &mut schema.items {
- if let Item::Stream { name, seal, .. } = item {
- if name == JOURNAL_STREAM {
- seal.retain_ms = (days > 0).then(|| u64::from(days) * DAY_MS);
- }
- }
- }
- }
- Registry::compose(schemas).map_err(|error| StorageError::invalid_input(error.to_string()))
+/// Host collection storing the successful one-shot backfill marker per scope.
+pub(crate) fn collection_specs() -> Vec<CollectionSpec> {
+ vec![CollectionSpec::new(BACKFILL_COLLECTION)]
}
-/// What one maintenance pass did.
+/// One pass over the scopes known to this host.
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
pub struct MaintenanceSummary {
- /// Journals sealed by the backfill in this pass.
+ /// Journals sealed by the inactive-run backfill.
pub backfilled: u64,
- /// Streams the policies deleted (retention passed).
+ /// Expired sealed journals removed by TinyAgents.
pub deleted: u64,
- /// Streams the policies sealed.
- pub sealed: u64,
+ /// Eligible journals that changed before conditional deletion.
+ pub conflicts: u64,
+ /// Scopes whose maintenance pass completed.
+ pub scopes: u64,
}
-/// One pass at `now_ms`: backfill, policies, then the backend's own
-/// maintenance.
+/// Runs owner journal maintenance once in every scope known to the host.
///
-/// # Errors
-///
-/// The policy or backend error; the backfill logs its own failures.
+/// Owner policy stays in TinyAgents; OpenHuman supplies a retention duration,
+/// timestamp, and a per-scope one-shot backfill decision. A failed owner pass
+/// does not write the marker, so it is retried on the next sweep.
pub async fn maintain_once(
backend: &Arc<dyn StorageBackend>,
retention_days: u32,
now_ms: i64,
-) -> Result<MaintenanceSummary, StorageError> {
+) -> Result<MaintenanceSummary, crate::storage::StorageError> {
+ let retention_ms = u64::from(retention_days).saturating_mul(DAY_MS);
let mut summary = MaintenanceSummary::default();
- for scope in scopes(backend).await {
- match backfill_scope(backend.as_ref(), &scope).await {
- Ok(sealed) => summary.backfilled += sealed,
+ for scope in known_scopes().await? {
+ match maintain_scope(backend, &scope, retention_ms, now_ms).await {
+ Ok(report) => {
+ summary.scopes += 1;
+ summary.backfilled += report.backfilled as u64;
+ summary.deleted += report.deleted as u64;
+ summary.conflicts += report.conflicts as u64;
+ }
Err(error) => tracing::warn!(
scope = scope.as_str(),
%error,
- "{LOG} journal seal backfill failed; retrying on the next sweep"
+ "{LOG} owner journal maintenance failed; retrying on the next sweep"
),
}
}
- let registry = registry_with_retention(retention_days)?;
- let budget = MaintenanceBudget::unlimited();
- let report = if backend
- .capabilities()
- .contains(Capability::ScopeEnumeration)
- {
- registry
- .maintain_all(backend.as_ref(), now_ms, budget)
- .await?
- } else {
- registry
- .maintain(&backend.for_scope(&Scope::local())?, now_ms, budget)
- .await?
- };
- summary.deleted = report.deleted;
- summary.sealed = report.sealed;
- backend.maintain(budget).await?;
tracing::debug!(
+ scopes = summary.scopes,
backfilled = summary.backfilled,
- sealed = summary.sealed,
deleted = summary.deleted,
+ conflicts = summary.conflicts,
"{LOG} pass complete"
);
Ok(summary)
}
-/// `local` plus every scope the backend can list.
-async fn scopes(backend: &Arc<dyn StorageBackend>) -> Vec<Scope> {
- let mut scopes = vec![Scope::local()];
- if backend
- .capabilities()
- .contains(Capability::ScopeEnumeration)
- {
- if let Ok(found) = backend.scopes("").await {
- for scope in found {
- if scope.parsed().is_ok() && !scope.is_cluster() && !scopes.contains(&scope) {
- scopes.push(scope);
- }
+async fn maintain_scope(
+ backend: &Arc<dyn StorageBackend>,
+ scope: &Scope,
+ retention_ms: u64,
+ now_ms: i64,
+) -> Result<JournalMaintenanceReport, crate::storage::StorageError> {
+ let storage = backend.for_scope(scope)?;
+ let docs = Arc::clone(storage.documents());
+ docs.ensure_collection(&CollectionSpec::new(BACKFILL_COLLECTION))
+ .await?;
+ let key = crate::storage::schema_support::key_of("inactive-journal-backfill-v1")?;
+ let backfill_done = match docs.get(BACKFILL_COLLECTION, &key).await? {
+ None => false,
+ Some(record) => {
+ let version = record.doc.get("version").and_then(serde_json::Value::as_u64);
+ if version != Some(BACKFILL_VERSION) {
+ return Err(crate::storage::StorageError::invalid_input(format!(
+ "invalid journal backfill marker in scope {}",
+ scope.as_str()
+ )));
}
+ true
+ }
+ };
+
+ let statuses = DriverStatusStore::new(docs);
+ let report = tinyagents_harness::observability::maintain_run_journals(
+ &statuses,
+ storage.streams().as_ref(),
+ retention_ms,
+ now_ms,
+ !backfill_done,
+ )
+ .await
+ .map_err(|error| crate::storage::StorageError::backend(error.to_string()))?;
+
+ if !backfill_done {
+ match storage
+ .documents()
+ .put_with(
+ BACKFILL_COLLECTION,
+ &key,
+ serde_json::json!({ "version": BACKFILL_VERSION }),
+ Precondition::Absent,
+ Default::default(),
+ )
+ .await
+ {
+ Ok(_) => {}
+ Err(error) if error.kind() == ErrorKind::Conflict => {}
+ Err(error) => return Err(error),
}
}
- scopes
+ Ok(report)
}
-/// Seals every unsealed run journal in `scope` whose run is not active, once:
-/// the pass is recorded in the scope's `tsd.migrations`. Returns how many
-/// journals it sealed (0 when already done).
-///
-/// # Errors
+/// Scopes represented by the host's own registries and compiled module table.
///
-/// A storage error; the record is written only after the whole pass, so the
-/// next call resumes by skipping what is already sealed.
-pub async fn backfill_scope(
- backend: &dyn StorageBackend,
- scope: &Scope,
-) -> Result<u64, StorageError> {
- let storage = backend.for_scope(scope)?;
- storage
- .documents()
- .ensure_collection(&CollectionSpec::new(MIGRATIONS_COLLECTION))
- .await?;
- let migrations = Migrations::new(&storage);
- if migrations.done(BACKFILL_OWNER, BACKFILL_STEP).await? {
- return Ok(0);
+/// This deliberately does not ask the backend to enumerate arbitrary scopes:
+/// it includes local/cluster, registered profiles, known agent contexts, and
+/// compiled-in module scopes for local and registered profiles.
+async fn known_scopes() -> Result<Vec<Scope>, crate::storage::StorageError> {
+ let mut scopes = BTreeSet::new();
+ scopes.insert(Scope::cluster());
+ if !crate::core::runtime::mode::is_saas() {
+ scopes.insert(Scope::local());
+ }
+
+ let mut profiles = BTreeSet::new();
+ if let Some(host) = crate::profiles::host::host() {
+ let listed = host
+ .known_profiles()
+ .await
+ .map_err(crate::storage::StorageError::backend)?;
+ for profile in listed {
+ profiles.insert(profile);
+ }
}
- let active: HashSet<String> = DriverStatusStore::new(Arc::clone(storage.documents()))
- .list_active()
- .await
- .map_err(|error| StorageError::invalid_input(error.to_string()))?
- .into_iter()
- .map(|status| status.run_id.as_str().to_string())
- .collect();
- let streams = storage.streams();
- let family = format!("{JOURNAL_STREAM}/");
- let mut sealed = 0;
- for name in streams.streams(JOURNAL_STREAM).await? {
- let Some(run) = name.strip_prefix(&family) else {
- continue;
- };
- if active.contains(run) {
- continue;
+
+ for (agent, context) in crate::storage::agents::contexts() {
+ let dispatch = crate::core::runtime::DispatchCtx::new(context);
+ if let Ok(tenant) = dispatch.tenant() {
+ if let Some(profile) = tenant.profile {
+ profiles.insert(profile);
+ } else if let Some(agent) = tenant.agent.or(Some(agent)) {
+ scopes.insert(crate::storage::scope_for_agent(&agent));
+ }
}
- match streams.info(&name).await? {
- Some(info) if info.sealed_ms.is_none() => {
- streams.seal(&name).await?;
- sealed += 1;
+ }
+
+ let bases: Vec<Scope> = if crate::core::runtime::mode::is_saas() {
+ profiles
+ .iter()
+ .map(|profile| crate::storage::scope_for_profile(profile))
+ .collect()
+ } else {
+ std::iter::once(Scope::local())
+ .chain(
+ profiles
+ .iter()
+ .map(|profile| crate::storage::scope_for_profile(profile)),
+ )
+ .collect()
+ };
+ scopes.extend(bases.iter().cloned());
+ for base in bases {
+ for module in crate::modules::registry::ALL {
+ if let Ok(scope) = base.module(module.id) {
+ scopes.insert(scope);
}
- _ => {}
}
}
- migrations.mark_done(BACKFILL_OWNER, BACKFILL_STEP).await?;
- tracing::info!(
- scope = scope.as_str(),
- sealed,
- "{LOG} journal seal backfill done"
- );
- Ok(sealed)
+ Ok(scopes.into_iter().collect())
}
-/// Retention days from the current config; the default when it cannot load.
+/// Retention days from the current config; default when it cannot load.
async fn configured_retention_days(ctx: &crate::core::runtime::DispatchCtx) -> u32 {
match crate::config::ops::load_current_or_init(ctx).await {
Ok(config) => config.storage.journal_retention_days,
diff --git a/crates/openhuman-core/src/storage/journal_maintenance_tests.rs b/crates/openhuman-core/src/storage/journal_maintenance_tests.rs
index 5971746343..5fdc5a335d 100644
--- a/crates/openhuman-core/src/storage/journal_maintenance_tests.rs
+++ b/crates/openhuman-core/src/storage/journal_maintenance_tests.rs
@@ -1,21 +1,14 @@
use super::*;
-use std::sync::atomic::{AtomicU64, Ordering};
use tinyagents_harness::events::{AgentEvent, EventSink, HarnessRunStatus};
use tinyagents_harness::ids::{ComponentId, HarnessPhase, RunId};
use tinyagents_harness::observability::replay::read_run_events_page;
use tinyagents_harness::observability::{AgentObservation, HarnessEventJournal, StoreEventJournal};
use tinyagents_harness::store::{AppendStore, DriverAppendStore};
use tinyagents_harness::HarnessStatusStore;
-use tinystoragedrivers::MemoryStorage;
+use tinystoragedrivers_testkit::memory_storage;
-fn backend_with_clock() -> (Arc<AtomicU64>, Arc<dyn StorageBackend>) {
- let now = Arc::new(AtomicU64::new(1_000));
- let clock = Arc::clone(&now);
- let backend: Arc<dyn StorageBackend> =
- Arc::new(MemoryStorage::with_clock(Arc::new(move || {
- clock.load(Ordering::SeqCst)
- })));
- (now, backend)
+fn backend() -> Arc<dyn StorageBackend> {
+ memory_storage(&Scope::local())
}
fn journal(backend: &Arc<dyn StorageBackend>) -> StoreEventJournal<Arc<dyn AppendStore>> {
@@ -77,7 +70,7 @@ async fn pages(
#[tokio::test]
async fn a_sealed_journal_serves_the_same_pages_as_an_unsealed_one() {
- let (_, backend) = backend_with_clock();
+ let backend = backend();
let journal = journal(&backend);
seed(&journal, "run.a", 7).await;
let before = pages(&journal, "run.a").await;
@@ -88,52 +81,59 @@ async fn a_sealed_journal_serves_the_same_pages_as_an_unsealed_one() {
#[tokio::test]
async fn backfill_seals_finished_runs_but_not_active_ones_and_records_itself() {
- let (_, backend) = backend_with_clock();
+ let backend = backend();
let journal = journal(&backend);
seed(&journal, "run.done", 2).await;
seed(&journal, "run.live", 2).await;
mark_active(&backend, "run.live").await;
- assert_eq!(
- backfill_scope(backend.as_ref(), &Scope::local())
- .await
- .unwrap(),
- 1
- );
+ let report = maintain_scope(&backend, &Scope::local(), 0, 1_000)
+ .await
+ .unwrap();
+ assert_eq!(report.backfilled, 1);
assert!(sealed(&backend, "run.done").await);
assert!(!sealed(&backend, "run.live").await);
// Recorded: a later pass does nothing, even for a run that has since ended.
seed(&journal, "run.late", 1).await;
- assert_eq!(
- backfill_scope(backend.as_ref(), &Scope::local())
- .await
- .unwrap(),
- 0
- );
+ let report = maintain_scope(&backend, &Scope::local(), 0, 1_000)
+ .await
+ .unwrap();
+ assert_eq!(report.backfilled, 0);
assert!(!sealed(&backend, "run.late").await);
let storage = backend.for_scope(&Scope::local()).unwrap();
- assert!(Migrations::new(&storage)
- .done(BACKFILL_OWNER, BACKFILL_STEP)
+ let marker = storage
+ .documents()
+ .get(BACKFILL_COLLECTION, &crate::storage::schema_support::key_of("inactive-journal-backfill-v1").unwrap())
.await
- .unwrap());
+ .unwrap()
+ .unwrap();
+ assert_eq!(marker.doc["version"], BACKFILL_VERSION);
}
#[tokio::test]
async fn maintenance_deletes_a_sealed_journal_after_the_configured_retention() {
- let (now, backend) = backend_with_clock();
+ let backend = backend();
let journal = journal(&backend);
seed(&journal, "run.old", 2).await;
journal.seal_run("run.old").await.unwrap();
+ let storage = backend.for_scope(&Scope::local()).unwrap();
+ let sealed_at = storage
+ .streams()
+ .info("harness.journal/run.old")
+ .await
+ .unwrap()
+ .unwrap()
+ .sealed_ms
+ .unwrap();
// Inside the window: kept.
- let early = maintain_once(&backend, 1, 1_000 + DAY_MS as i64 / 2)
+ let early = maintain_scope(&backend, &Scope::local(), DAY_MS, sealed_at + DAY_MS as i64 / 2)
.await
.unwrap();
assert_eq!(early.deleted, 0);
// Past one day, the configured retention (not the schema's 30).
- now.store(1_000 + DAY_MS + 10, Ordering::SeqCst);
- let late = maintain_once(&backend, 1, 1_000 + DAY_MS as i64 + 10)
+ let late = maintain_scope(&backend, &Scope::local(), DAY_MS, sealed_at + DAY_MS as i64 + 10)
.await
.unwrap();
assert_eq!(late.deleted, 1);
@@ -148,11 +148,20 @@ async fn maintenance_deletes_a_sealed_journal_after_the_configured_retention() {
#[tokio::test]
async fn zero_retention_days_keeps_sealed_journals_forever() {
- let (_, backend) = backend_with_clock();
+ let backend = backend();
let journal = journal(&backend);
seed(&journal, "run.keep", 1).await;
journal.seal_run("run.keep").await.unwrap();
- let summary = maintain_once(&backend, 0, 1_000 + 400 * DAY_MS as i64)
+ let storage = backend.for_scope(&Scope::local()).unwrap();
+ let sealed_at = storage
+ .streams()
+ .info("harness.journal/run.keep")
+ .await
+ .unwrap()
+ .unwrap()
+ .sealed_ms
+ .unwrap();
+ let summary = maintain_scope(&backend, &Scope::local(), 0, sealed_at + 400 * DAY_MS as i64)
.await
.unwrap();
assert_eq!(summary.deleted, 0);
diff --git a/crates/openhuman-core/src/storage/schema.rs b/crates/openhuman-core/src/storage/schema.rs
index dac89787a8..dfbedfbcec 100644
--- a/crates/openhuman-core/src/storage/schema.rs
+++ b/crates/openhuman-core/src/storage/schema.rs
@@ -43,6 +43,7 @@ fn declared_collections() -> Vec<CollectionSpec> {
collections.extend(crate::threads::store::collection_specs());
collections.extend(crate::agent::orchestration::subagent_sessions::collection_specs());
collections.extend(super::agents::collection_specs());
+ collections.extend(super::journal_maintenance::collection_specs());
#[cfg(feature = "skills")]
collections.extend(tinyskills_drivers::collections());