Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

refactor(storage): introduce record upgrade functions for schema migration

Steven Enamakela56c6111d42a
diff --git a/crates/openhuman-core/src/agent/orchestration/subagent_sessions/storage_schema.rs b/crates/openhuman-core/src/agent/orchestration/subagent_sessions/storage_schema.rs
index 2e82e04e49..23c24cc6ad 100644
--- a/crates/openhuman-core/src/agent/orchestration/subagent_sessions/storage_schema.rs
+++ b/crates/openhuman-core/src/agent/orchestration/subagent_sessions/storage_schema.rs
@@ -12,21 +12,21 @@
//! session's last use.
use serde_json::{Map, Value};
-use tinystoragedrivers::schema::{Schema, ScopeClass, SourceRecord, Step, TargetRecord};
-use tinystoragedrivers::{CollectionSpec, Result};
+use anyhow::Result;
+use tinystoragedrivers::CollectionSpec;
-use crate::storage::schema_support::{key_of, owner, time_ms};
+use crate::storage::schema_support::{
+ key_of, map_legacy_collection, time_ms, LegacySourceRecord as SourceRecord,
+ LegacyTargetRecord as TargetRecord,
+};
pub(super) const SESSIONS: &str = "subagents.sessions";
pub(super) const LEGACY_SESSIONS: &str = "subagent_sessions";
-const SCHEMA: u32 = 2;
-
-/// What the session store keeps and the step from the v1 collection.
-#[must_use]
-pub fn storage_schema() -> Schema {
- Schema::new(owner("subagents"))
- .collection_with(SCHEMA, spec(), ScopeClass::Agent)
- .step(Step::new("sessions_v2", SESSIONS, 1, SCHEMA, sessions_v2).source(LEGACY_SESSIONS))
+/// Converts legacy sub-agent session rows into the current owner shape.
+pub(crate) fn upgrade_records(
+ records: Vec<crate::storage::import::Record>,
+) -> Result<Vec<crate::storage::import::Record>> {
+ map_legacy_collection(records, LEGACY_SESSIONS, sessions_v2)
}
pub(super) fn spec() -> CollectionSpec {
diff --git a/crates/openhuman-core/src/artifacts/storage_schema.rs b/crates/openhuman-core/src/artifacts/storage_schema.rs
index 1292248d03..4ba7587c8a 100644
--- a/crates/openhuman-core/src/artifacts/storage_schema.rs
+++ b/crates/openhuman-core/src/artifacts/storage_schema.rs
@@ -13,11 +13,14 @@
//! [`args_of_body`] reads either form back.
use serde_json::{json, Value};
-use tinystoragedrivers::schema::{Schema, ScopeClass, SourceRecord, Step, TargetRecord};
-use tinystoragedrivers::{CollectionSpec, IndexSpec, Result};
+use anyhow::Result;
+use tinystoragedrivers::{CollectionSpec, IndexSpec};
use super::types::ArtifactMeta;
-use crate::storage::schema_support::{camel_keys, key_of, owner, snake_keys, time_ms};
+use crate::storage::schema_support::{
+ camel_keys, key_of, map_legacy_collection, snake_keys, time_ms,
+ LegacySourceRecord as SourceRecord, LegacyTargetRecord as TargetRecord,
+};
pub(super) const RECORDS: &str = "artifacts.records";
pub(super) const ARGS: &str = "artifacts.args";
@@ -25,7 +28,6 @@ pub(super) const VERSIONS: &str = "artifacts.versions";
pub(super) const LEGACY_RECORDS: &str = "artifacts";
pub(super) const LEGACY_ARGS: &str = "artifact_args";
pub(super) const LEGACY_VERSIONS: &str = "artifact_versions";
-const SCHEMA: u32 = 2;
/// The v2 names of the `*AtMillis` fields of `tiny_artifacts::ArtifactRecord`.
const RECORD_TIME_KEYS: [(&str, &str); 4] = [
@@ -35,16 +37,13 @@ const RECORD_TIME_KEYS: [(&str, &str); 4] = [
("publishedAtMillis", "published_ms"),
];
-/// What the artifact stores keep and the steps from the v1 collections.
-#[must_use]
-pub fn storage_schema() -> Schema {
- Schema::new(owner("artifacts"))
- .collection_with(SCHEMA, records_spec(), ScopeClass::Tenant)
- .collection_with(SCHEMA, CollectionSpec::new(ARGS), ScopeClass::Tenant)
- .collection_with(SCHEMA, CollectionSpec::new(VERSIONS), ScopeClass::Tenant)
- .step(Step::new("records_v2", RECORDS, 1, SCHEMA, records_v2).source(LEGACY_RECORDS))
- .step(Step::new("args_v2", ARGS, 1, SCHEMA, args_v2).source(LEGACY_ARGS))
- .step(Step::new("versions_v2", VERSIONS, 1, SCHEMA, versions_v2).source(LEGACY_VERSIONS))
+/// Converts artifact legacy documents into the current owner shapes.
+pub(crate) fn upgrade_records(
+ records: Vec<crate::storage::import::Record>,
+) -> Result<Vec<crate::storage::import::Record>> {
+ let records = map_legacy_collection(records, LEGACY_RECORDS, records_v2)?;
+ let records = map_legacy_collection(records, LEGACY_ARGS, args_v2)?;
+ map_legacy_collection(records, LEGACY_VERSIONS, versions_v2)
}
pub(super) fn records_spec() -> CollectionSpec {
diff --git a/crates/openhuman-core/src/profiles/storage_schema.rs b/crates/openhuman-core/src/profiles/storage_schema.rs
index 150b0c06af..b763f854d7 100644
--- a/crates/openhuman-core/src/profiles/storage_schema.rs
+++ b/crates/openhuman-core/src/profiles/storage_schema.rs
@@ -14,37 +14,29 @@
//! time a registry opens the shared backend.
use serde_json::Value;
-use tinystoragedrivers::schema::{
- MigrateOptions, Registry, Schema, ScopeClass, SourceRecord, Step, TargetRecord,
-};
-use tinystoragedrivers::{CollectionSpec, Result, ScopedStorage};
+use anyhow::Result;
+use tinystoragedrivers::{CollectionSpec, Storage};
-use crate::storage::schema_support::{key_of, owner, Body};
+use crate::storage::schema_support::{
+ key_of, map_legacy_collection, Body, LegacySourceRecord as SourceRecord,
+ LegacyTargetRecord as TargetRecord,
+};
/// The v2 collection: one document per profile.
pub const REGISTRY_COLLECTION: &str = "profiles.registry";
/// The v1 name.
pub(super) const LEGACY_REGISTRY_COLLECTION: &str = "profiles";
-const SCHEMA: u32 = 2;
/// Collections owned by the profile registry.
pub(crate) fn collection_specs() -> Vec<CollectionSpec> {
vec![CollectionSpec::new(REGISTRY_COLLECTION)]
}
-/// What the profile registry keeps and the step from the v1 collection.
-#[must_use]
-pub fn storage_schema() -> Schema {
- Schema::new(owner("profiles"))
- .collection_with(
- SCHEMA,
- CollectionSpec::new(REGISTRY_COLLECTION),
- ScopeClass::Cluster,
- )
- .step(
- Step::new("registry_v2", REGISTRY_COLLECTION, 1, SCHEMA, registry_v2)
- .source(LEGACY_REGISTRY_COLLECTION),
- )
+/// Converts legacy profile records into the current registry shape.
+pub(crate) fn upgrade_records(
+ records: Vec<crate::storage::import::Record>,
+) -> Result<Vec<crate::storage::import::Record>> {
+ map_legacy_collection(records, LEGACY_REGISTRY_COLLECTION, registry_v2)
}
fn registry_v2(batch: Vec<SourceRecord>) -> Result<Vec<TargetRecord>> {
@@ -81,13 +73,11 @@ fn registry_v2(batch: Vec<SourceRecord>) -> Result<Vec<TargetRecord>> {
///
/// A storage error; the registry then reads the unmigrated records as absent
/// until the next open retries.
-pub async fn migrate_cluster(scoped: &ScopedStorage) -> Result<()> {
- let registry = Registry::compose([storage_schema()])?;
+pub async fn migrate_cluster(scoped: &Storage) -> Result<()> {
scoped
.documents()
.ensure_collection(&CollectionSpec::new(REGISTRY_COLLECTION))
.await?;
- registry.migrate(scoped, &MigrateOptions::new()).await?;
migrate_legacy_leases(scoped).await?;
Ok(())
}
@@ -128,7 +118,7 @@ fn lease_v2(old: &Value) -> Option<Value> {
/// # Errors
///
/// A storage error; the records not yet moved stay for the next open.
-pub async fn migrate_legacy_leases(scoped: &ScopedStorage) -> Result<usize> {
+pub async fn migrate_legacy_leases(scoped: &Storage) -> Result<usize> {
use tinystoragedrivers::{Capability, DocumentStoreExt, Key, Precondition, Query};
let docs = scoped.documents();
// A driver that cannot list collections is asked for the records
diff --git a/crates/openhuman-core/src/threads/store/storage_schema.rs b/crates/openhuman-core/src/threads/store/storage_schema.rs
index 188e54ebef..366b0be46d 100644
--- a/crates/openhuman-core/src/threads/store/storage_schema.rs
+++ b/crates/openhuman-core/src/threads/store/storage_schema.rs
@@ -10,30 +10,21 @@
//! legacy JSONL import is a `tsd.migrations` entry
//! ([`crate::storage::import_state`]).
-use tinystoragedrivers::schema::{Schema, ScopeClass, SourceRecord, Step, TargetRecord};
-use tinystoragedrivers::{CollectionSpec, Result};
+use anyhow::Result;
+use tinystoragedrivers::CollectionSpec;
-use crate::storage::schema_support::{key_of, owner};
+use crate::storage::schema_support::{
+ key_of, map_legacy_collection, LegacySourceRecord as SourceRecord,
+ LegacyTargetRecord as TargetRecord,
+};
pub(super) const TRANSCRIPT_KEYS: &str = "threads.transcript_keys";
pub(super) const LEGACY_TRANSCRIPT_KEYS: &str = "threads_transcript_keys";
-const SCHEMA: u32 = 2;
-
-/// What the thread host keeps and the step from the v1 collection.
-#[must_use]
-pub fn storage_schema() -> Schema {
- Schema::new(owner("threads"))
- .collection_with(SCHEMA, spec(), ScopeClass::Tenant)
- .step(
- Step::new(
- "transcript_keys_v2",
- TRANSCRIPT_KEYS,
- 1,
- SCHEMA,
- transcript_keys_v2,
- )
- .source(LEGACY_TRANSCRIPT_KEYS),
- )
+/// Converts legacy transcript-key rows into the current owner shape.
+pub(crate) fn upgrade_records(
+ records: Vec<crate::storage::import::Record>,
+) -> Result<Vec<crate::storage::import::Record>> {
+ map_legacy_collection(records, LEGACY_TRANSCRIPT_KEYS, transcript_keys_v2)
}
pub(super) fn spec() -> CollectionSpec {