Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

refactor(storage): update import state management to host-owned records

Steven Enamakelc8e6cb3f47de
diff --git a/crates/openhuman-core/src/storage/import/layout_db_tests.rs b/crates/openhuman-core/src/storage/import/layout_db_tests.rs
index 607816cccc..ff291263fa 100644
--- a/crates/openhuman-core/src/storage/import/layout_db_tests.rs
+++ b/crates/openhuman-core/src/storage/import/layout_db_tests.rs
@@ -1,6 +1,6 @@
use super::*;
use crate::storage::layout::test_fixtures::{fixture_docs, v1_profile};
-use crate::storage::layout::{legacy, Target};
+use crate::storage::layout::{Target, legacy};
use tempfile::TempDir;
#[test]
diff --git a/crates/openhuman-core/src/storage/import_state.rs b/crates/openhuman-core/src/storage/import_state.rs
index 09669d8feb..dc4fcbb1d5 100644
--- a/crates/openhuman-core/src/storage/import_state.rs
+++ b/crates/openhuman-core/src/storage/import_state.rs
@@ -1,52 +1,30 @@
-//! Where a one-time legacy importer records that it finished, and the claim
-//! that keeps two processes from importing at once.
+//! Host-owned completion markers and expiring claims for one-time imports.
//!
-//! Both are tsd primitives, not host collections:
-//!
-//! - **Completion** is a `tsd.migrations` entry ([`Migrations::mark_done`]) in
-//! the scope the importer writes to, keyed `(owner, step)`. This replaces
-//! the per-domain marker collections (`storage_import`, `threads_import`,
-//! `subagent_sessions_import`, `composio_state_import` and the module
-//! `*_host_import` ones). A marker row an older build wrote is honoured by
-//! [`migrate_legacy_markers`], which turns each into the same entry at boot,
-//! so an installed profile is never imported twice.
-//! - **The claim** is a [`Leases`] lease on `(owner, step)` in `tsd.leases`
-//! (schema-v2 section 5). The process that wins it copies the legacy
-//! records; the others wait it out, then re-check completion. The owner
-//! renews the lease while it works, and a lease that lapses (its owner
-//! crashed) is taken over by the next acquirer, so a dead process never
-//! blocks an import.
-//!
-//! A step is named `<kind>/<id>`: `kind` is the old marker collection name
-//! (`storage_import`) and `id` the old marker id (`approval@<workspace
-//! digest>`), so both the new check and the migrated entry derive from the
-//! same pair.
+//! Import progress is ordinary owner data in `storage.import_state`. The
+//! collection is deliberately separate from Tiny Storage Drivers' schemas:
+//! the host owns importer names, completion records, and claim policy.
use std::sync::Arc;
use std::time::{Duration, Instant};
use anyhow::Result;
-use tinystoragedrivers::schema::Migrations;
+use serde::{Deserialize, Serialize};
use tinystoragedrivers::{
- Capability, DocumentStore, DocumentStoreExt, Leases, Query, ScopedStorage, StorageError,
+ CollectionSpec, DocumentStore, DocumentStoreExt, ErrorKind, ExpectedDocument, Precondition,
+ ReadCheck, Storage, StorageError, Versioned, WriteOp,
};
use super::documents::Repo;
-/// How long a claim blocks other processes before it counts as abandoned.
+/// How long a claim blocks other processes before it can be taken over.
pub const CLAIM_TTL_MS: i64 = 10 * 60 * 1000;
-/// The `tsd.leases` owner of import claims.
-const CLAIM_OWNER: &str = "import";
-
+const COLLECTION: &str = "storage.import_state";
const LOG_PREFIX: &str = "[storage::import_state]";
-
-/// How often a waiting process re-tries a claim another process holds.
const WAIT_POLL: Duration = Duration::from_millis(100);
+const CAS_RETRIES: usize = 32;
-/// The legacy marker collections and the `tsd.migrations` owner each moves
-/// under. A marker lived in the scope its importer wrote to, so every scope
-/// is swept.
+/// The legacy marker collections whose completion rows are moved at boot.
pub const LEGACY_MARKERS: &[(&str, &str)] = &[
("storage_import", "storage"),
("threads_import", "threads"),
@@ -60,136 +38,182 @@ pub const LEGACY_MARKERS: &[(&str, &str)] = &[
("mcp_host_import", "host"),
];
-/// The step name of the importer that wrote marker `id` into `kind`.
+/// Declares the host-owned import progress collection.
+#[must_use]
+pub fn collection_specs() -> Vec<CollectionSpec> {
+ vec![CollectionSpec::new(COLLECTION)]
+}
+
+/// The stable step key for a marker in an older collection.
#[must_use]
pub fn step_id(kind: &str, id: &str) -> String {
format!("{kind}/{id}")
}
-/// Declares the primitives an import state needs, for a backend that did not
-/// go through the boot `prepare` (a test's, an embedder's).
-async fn declare(docs: &Arc<dyn DocumentStore>) -> Result<(), StorageError> {
- for spec in tinystoragedrivers::primitives::collection_specs() {
- docs.ensure_collection(&spec).await?;
+/// The owner name used by module importers.
+pub const HOST_OWNER: &str = "host";
+
+fn record_key(owner: &str, step: &str, kind: &str) -> String {
+ // Hex encoding keeps this key injective even when either source component
+ // contains separators used by the document-id grammar.
+ format!(
+ "{kind}/{}/{}",
+ hex::encode(owner.as_bytes()),
+ hex::encode(step.as_bytes())
+ )
+}
+
+async fn ensure(scoped: &Storage) -> Result<(), StorageError> {
+ for spec in collection_specs() {
+ scoped.ensure_collection(&spec).await?;
}
Ok(())
}
-/// Whether importer `step` of `owner` finished in `scoped`'s scope.
-///
-/// # Errors
-///
-/// A storage error.
-pub async fn done(scoped: &ScopedStorage, owner: &str, step: &str) -> Result<bool, StorageError> {
- declare(scoped.documents()).await?;
- Migrations::new(scoped).done(owner, step).await
+#[derive(Debug, Clone, Serialize, Deserialize)]
+struct Completion {
+ record_type: String,
+ owner: String,
+ step: String,
+ completed_at_ms: i64,
}
-/// Records importer `step` of `owner` as finished in `scoped`'s scope.
-///
-/// # Errors
-///
-/// A storage error.
-pub async fn mark_done(
- scoped: &ScopedStorage,
+#[derive(Debug, Clone, Serialize, Deserialize)]
+struct Claim {
+ record_type: String,
+ owner: String,
+ step: String,
+ holder: Option<String>,
+ epoch: u64,
+ valid_until_ms: i64,
+}
+
+fn decode<T: for<'de> Deserialize<'de>>(
+ row: Versioned<serde_json::Value>,
+ expected_owner: &str,
+ expected_step: &str,
+) -> Result<T, StorageError> {
+ let owner = row.doc.get("owner").and_then(serde_json::Value::as_str);
+ let step = row.doc.get("step").and_then(serde_json::Value::as_str);
+ if owner != Some(expected_owner) || step != Some(expected_step) {
+ return Err(StorageError::serialization(
+ "import state key does not match its owner and step",
+ ));
+ }
+ serde_json::from_value(row.doc)
+ .map_err(|error| StorageError::serialization(format!("invalid import state: {error}")))
+}
+
+/// Checks whether the import `step` for `owner` is complete in `scoped`.
+pub async fn done(scoped: &Storage, owner: &str, step: &str) -> Result<bool, StorageError> {
+ ensure(scoped).await?;
+ let id = record_key(owner, step, "done");
+ let Some(row) = scoped.get(COLLECTION, &id).await? else {
+ return Ok(false);
+ };
+ let _: Completion = decode(row, owner, step)?;
+ Ok(true)
+}
+
+/// Records completion of an import step. Existing completion is immutable.
+pub async fn mark_done(scoped: &Storage, owner: &str, step: &str) -> Result<(), StorageError> {
+ mark_done_at(scoped, owner, step, chrono::Utc::now().timestamp_millis()).await
+}
+
+async fn mark_done_at(
+ scoped: &Storage,
owner: &str,
step: &str,
+ completed_at_ms: i64,
) -> Result<(), StorageError> {
- declare(scoped.documents()).await?;
- Migrations::new(scoped).mark_done(owner, step).await
+ ensure(scoped).await?;
+ let id = record_key(owner, step, "done");
+ let body = serde_json::to_value(Completion {
+ record_type: "completion".to_owned(),
+ owner: owner.to_owned(),
+ step: step.to_owned(),
+ completed_at_ms,
+ })
+ .map_err(|error| StorageError::serialization(error.to_string()))?;
+ for _ in 0..CAS_RETRIES {
+ if let Some(current) = scoped.get(COLLECTION, &id).await? {
+ let _: Completion = decode(current, owner, step)?;
+ return Ok(());
+ }
+ match scoped
+ .put(COLLECTION, &id, body.clone(), Precondition::Absent)
+ .await
+ {
+ Ok(_) => return Ok(()),
+ Err(error) if error.kind() == ErrorKind::Conflict => continue,
+ Err(error) => return Err(error),
+ }
+ }
+ Err(StorageError::conflict(
+ "import completion kept changing during its retry budget",
+ ))
}
-/// The `tsd.migrations` owner of the module importers, which run in a module's
-/// scope on behalf of the host (`kind` is the old marker collection name).
-pub const HOST_OWNER: &str = "host";
-
-/// Whether the module importer `kind` already finished for marker `id` in
-/// `scoped`'s (module) scope.
-///
-/// # Errors
-///
-/// A storage error.
+/// Whether the module importer has completed a legacy marker.
pub async fn host_import_done(
- scoped: &ScopedStorage,
+ scoped: &Storage,
kind: &str,
id: &str,
) -> Result<bool, StorageError> {
done(scoped, HOST_OWNER, &step_id(kind, id)).await
}
-/// Records the module importer `kind` as finished for marker `id`.
-///
-/// # Errors
-///
-/// A storage error.
-pub async fn host_import_mark(
- scoped: &ScopedStorage,
- kind: &str,
- id: &str,
-) -> Result<(), StorageError> {
+/// Records completion of a module import marker.
+pub async fn host_import_mark(scoped: &Storage, kind: &str, id: &str) -> Result<(), StorageError> {
mark_done(scoped, HOST_OWNER, &step_id(kind, id)).await
}
-/// [`done`] from synchronous code, over the scope `repo` was opened on.
-///
-/// # Errors
-///
-/// A storage error.
+/// Synchronous completion lookup for existing synchronous import APIs.
pub fn is_done(repo: &Repo, owner: &str, step: &str) -> Result<bool> {
let scoped = repo.scoped().clone();
let (owner, step) = (owner.to_owned(), step.to_owned());
repo.run(move |_| async move { done(&scoped, &owner, &step).await })
}
-/// [`mark_done`] from synchronous code.
-///
-/// # Errors
-///
-/// A storage error.
+/// Synchronous completion write for existing synchronous import APIs.
pub fn set_done(repo: &Repo, owner: &str, step: &str) -> Result<()> {
let scoped = repo.scoped().clone();
let (owner, step) = (owner.to_owned(), step.to_owned());
repo.run(move |_| async move { mark_done(&scoped, &owner, &step).await })
}
-/// Runs `body` while holding the claim for `(owner, step)` in `repo`'s scope.
-///
-/// A live claim held by another process is waited out (up to
-/// [`CLAIM_TTL_MS`]); once it is free `body` runs, and it must re-check its
-/// completion because the winner has usually finished by then. Returns `None`
-/// only when the claim stays held for the whole wait. The claim is renewed
-/// while `body` runs, so a long import is not mistaken for a crashed one.
-///
-/// # Errors
+/// Runs `body` while holding a claim for this importer step.
///
-/// A storage error while claiming, or whatever `body` returns. The claim is
-/// released either way.
+/// A competing live claim is polled until release or expiry, bounded by the
+/// claim TTL. The body must re-check completion after acquiring the claim.
pub fn with_claim<T>(
repo: &Repo,
owner: &str,
step: &str,
body: impl FnOnce() -> Result<T>,
) -> Result<Option<T>> {
- // `with_claim_at` only runs the closure once it holds the claim.
- let mut body = Some(body);
let started = Instant::now();
+ let mut body = Some(body);
loop {
let now = chrono::Utc::now().timestamp_millis();
- if let Some(ran) = with_claim_at(repo, owner, step, now, CLAIM_TTL_MS, || {
- (body.take().expect("import body runs once"))()
- })? {
- return Ok(Some(ran));
+ if let Some(token) = acquire(repo, owner, step, now, CLAIM_TTL_MS)? {
+ let (stop_tx, stop_rx) = std::sync::mpsc::channel();
+ let heartbeat = start_heartbeat(repo.clone(), token.clone(), CLAIM_TTL_MS, stop_rx);
+ let result = body.take().expect("import body runs once")();
+ let _ = stop_tx.send(());
+ let _ = heartbeat.join();
+ release(repo, &token)?;
+ return result.map(Some);
}
- if started.elapsed().as_millis() as i64 >= CLAIM_TTL_MS {
+ if started.elapsed() >= Duration::from_millis(CLAIM_TTL_MS.max(0) as u64) {
return Ok(None);
}
std::thread::sleep(WAIT_POLL);
}
}
-/// One attempt of [`with_claim`] with an explicit clock and lease length, for
-/// tests: returns `None` without running `body` or waiting when another live
-/// process holds the claim.
+/// One bounded claim attempt with an explicit clock and TTL, primarily for
+/// deterministic concurrency tests.
pub fn with_claim_at<T>(
repo: &Repo,
owner: &str,
@@ -198,112 +222,207 @@ pub fn with_claim_at<T>(
ttl_ms: i64,
body: impl FnOnce() -> Result<T>,
) -> Result<Option<T>> {
- let started = Instant::now();
- let clock: tinystoragedrivers::Clock = Arc::new(move || {
- (now_ms.max(0) as u64).saturating_add(started.elapsed().as_millis() as u64)
- });
- // A holder name unique to this attempt, so two threads of one process do
- // not both "re-acquire" the same live lease.
- let holder = format!("pid-{}-{}", std::process::id(), uuid::Uuid::new_v4());
- let (claim_owner, claim_key) = (owner.to_owned(), step.to_owned());
- let acquired = {
- let (clock, holder) = (clock.clone(), holder.clone());
- let (owner, key) = (claim_owner.clone(), claim_key.clone());
- repo.run(move |docs| async move {
- declare(&docs).await?;
- Leases::new(&docs)
- .with_clock(clock)
- .acquire(CLAIM_OWNER, &lease_key(&owner, &key), &holder, ttl_ms)
- .await
- })?
- };
- let Some(held) = acquired else {
- tracing::debug!(
- target: "openhuman::storage_import",
- owner,
- step,
- "{LOG_PREFIX} another process holds the import claim"
- );
+ let Some(token) = acquire(repo, owner, step, now_ms, ttl_ms)? else {
return Ok(None);
};
- let epoch = held.epoch;
- let (stop_tx, stop_rx) = std::sync::mpsc::channel::<()>();
- let heartbeat = {
- let (repo, clock, holder) = (repo.clone(), clock.clone(), holder.clone());
- let key = lease_key(&claim_owner, &claim_key);
- let interval = Duration::from_millis((ttl_ms / 3).max(1) as u64);
- // A bare OS thread on purpose (baselined in the SaaS ambient ratchet):
- // the heartbeat reads no ambient state, only the `repo` handed to it,
- // and is joined below before this returns. `with_claim` is synchronous
- // and may run outside a tokio runtime, so `DispatchCtx::spawn_blocking`
- // (which needs `Handle::current()`) is not available here.
- std::thread::spawn(move || {
- while let Err(std::sync::mpsc::RecvTimeoutError::Timeout) =
- stop_rx.recv_timeout(interval)
- {
- let (clock, holder, key) = (clock.clone(), holder.clone(), key.clone());
- let renewed = repo.run(move |docs| async move {
- Leases::new(&docs)
- .with_clock(clock)
- .renew(CLAIM_OWNER, &key, &holder, epoch, ttl_ms)
- .await
- });
- match renewed {
- Ok(Some(_)) => {}
- Ok(None) | Err(_) => {
- tracing::warn!(
- target: "openhuman::storage_import",
- "{LOG_PREFIX} could not renew the import claim"
- );
- break;
+ let result = body();
+ release(repo, &token)?;
+ result.map(Some)
+}
+
+#[derive(Clone)]
+struct ClaimToken {
+ owner: String,
+ step: String,
+ holder: String,
+ epoch: u64,
+ ttl_ms: i64,
+ clock_started: Instant,
+ clock_start_ms: i64,
+}
+
+fn acquire(
+ repo: &Repo,
+ owner: &str,
+ step: &str,
+ now_ms: i64,
+ ttl_ms: i64,
+) -> Result<Option<ClaimToken>> {
+ if ttl_ms <= 0 {
+ return Err(anyhow::anyhow!("import claim TTL must be positive"));
+ }
+ let holder = format!("pid-{}-{}", std::process::id(), uuid::Uuid::new_v4());
+ let (owner_owned, step_owned, holder_owned) =
+ (owner.to_owned(), step.to_owned(), holder.clone());
+ let scoped = repo.scoped().clone();
+ let outcome = repo.run(move |docs| async move {
+ ensure(&scoped).await?;
+ let id = record_key(&owner_owned, &step_owned, "claim");
+ for _ in 0..CAS_RETRIES {
+ let current = docs.get(COLLECTION, &id).await?;
+ let (expected, precondition, epoch) = match current {
+ Some(row) => {
+ let claim: Claim = decode(row.clone(), &owner_owned, &step_owned)?;
+ if claim.holder.is_some() && claim.valid_until_ms > now_ms {
+ return Ok(None);
}
+ let epoch = claim
+ .epoch
+ .checked_add(1)
+ .ok_or_else(|| StorageError::backend("import claim epoch exhausted"))?;
+ (
+ ExpectedDocument::Version(row.version),
+ Precondition::Version(row.version),
+ epoch,
+ )
}
+ None => (ExpectedDocument::Absent, Precondition::Absent, 1),
+ };
+ let valid_until_ms = now_ms.saturating_add(ttl_ms);
+ let body = serde_json::to_value(Claim {
+ record_type: "claim".to_owned(),
+ owner: owner_owned.clone(),
+ step: step_owned.clone(),
+ holder: Some(holder_owned.clone()),
+ epoch,
+ valid_until_ms,
+ })
+ .map_err(|error| StorageError::serialization(error.to_string()))?;
+ let check = ReadCheck {
+ collection: COLLECTION.to_owned(),
+ id: id.clone(),
+ expected,
+ };
+ let write = WriteOp::Put {
+ collection: COLLECTION.to_owned(),
+ id: id.clone(),
+ doc: body,
+ precondition,
+ };
+ match docs.atomic_batch_checked(vec![check], vec![write]).await {
+ Ok(_) => return Ok(Some(epoch)),
+ Err(error) if error.kind() == ErrorKind::Conflict => continue,
+ Err(error) => return Err(error),
}
- })
- };
- let result = body();
- let _ = stop_tx.send(());
- let _ = heartbeat.join();
- let released = {
- let key = lease_key(&claim_owner, &claim_key);
- repo.run(move |docs| async move {
- Leases::new(&docs)
- .with_clock(clock)
- .release(CLAIM_OWNER, &key, &holder, epoch)
- .await
- })
- };
- if let Err(error) = released {
- // The lease expires on its own; releasing is an optimisation.
- tracing::warn!(
- target: "openhuman::storage_import",
- error = %format!("{error:#}"),
- "{LOG_PREFIX} could not release the import claim; it expires on its own"
- );
- }
- result.map(Some)
+ }
+ Err(StorageError::conflict(
+ "import claim kept changing during its retry budget",
+ ))
+ })?;
+ Ok(outcome.map(|epoch| ClaimToken {
+ owner: owner.to_owned(),
+ step: step.to_owned(),
+ holder,
+ epoch,
+ ttl_ms,
+ clock_started: Instant::now(),
+ clock_start_ms: now_ms,
+ }))
}
-fn lease_key(owner: &str, step: &str) -> String {
- format!("{owner}/{step}")
+fn start_heartbeat(
+ repo: Repo,
+ token: ClaimToken,
+ ttl_ms: i64,
+ stop: std::sync::mpsc::Receiver<()>,
+) -> std::thread::JoinHandle<()> {
+ let interval = Duration::from_millis((ttl_ms / 3).max(1) as u64);
+ std::thread::spawn(move || {
+ while let Err(std::sync::mpsc::RecvTimeoutError::Timeout) = stop.recv_timeout(interval) {
+ let now_ms = token
+ .clock_start_ms
+ .saturating_add(token.clock_started.elapsed().as_millis() as i64);
+ match renew(&repo, &token, now_ms) {
+ Ok(true) => {}
+ Ok(false) | Err(_) => {
+ tracing::warn!(target: "openhuman::storage_import", "{LOG_PREFIX} could not renew the import claim");
+ break;
+ }
+ }
+ }
+ })
}
-/// Turns the marker rows older builds wrote ([`LEGACY_MARKERS`]) in `scoped`'s
-/// scope into `tsd.migrations` entries and removes the old collections, so an
-/// already imported profile is not imported again. Returns the markers moved.
-///
-/// Idempotent: a marker already moved is simply marked again, and a collection
-/// that is gone is skipped. A claim row (`<id>#claim`) is dropped with its
-/// collection.
-///
-/// # Errors
-///
-/// A storage error; the markers not yet moved stay in place for the next boot.
-pub async fn migrate_legacy_markers(scoped: &ScopedStorage) -> Result<usize, StorageError> {
+fn renew(repo: &Repo, token: &ClaimToken, now_ms: i64) -> Result<bool> {
+ let scoped = repo.scoped().clone();
+ let token = token.clone();
+ repo.run(move |docs| async move {
+ ensure(&scoped).await?;
+ let id = record_key(&token.owner, &token.step, "claim");
+ let Some(row) = docs.get(COLLECTION, &id).await? else {
+ return Ok(false);
+ };
+ let mut claim: Claim = decode(row.clone(), &token.owner, &token.step)?;
+ if claim.holder.as_deref() != Some(&token.holder)
+ || claim.epoch != token.epoch
+ || claim.valid_until_ms <= now_ms
+ {
+ return Ok(false);
+ }
+ claim.valid_until_ms = now_ms.saturating_add(token.ttl_ms);
+ let body = serde_json::to_value(claim)
+ .map_err(|error| StorageError::serialization(error.to_string()))?;
+ let check = ReadCheck {
+ collection: COLLECTION.to_owned(),
+ id: id.clone(),
+ expected: ExpectedDocument::Version(row.version),
+ };
+ let write = WriteOp::Put {
+ collection: COLLECTION.to_owned(),
+ id,
+ doc: body,
+ precondition: Precondition::Version(row.version),
+ };
+ match docs.atomic_batch_checked(vec![check], vec![write]).await {
+ Ok(_) => Ok(true),
+ Err(error) if error.kind() == ErrorKind::Conflict => Ok(false),
+ Err(error) => Err(error),
+ }
+ })
+}
+
+fn release(repo: &Repo, token: &ClaimToken) -> Result<()> {
+ let scoped = repo.scoped().clone();
+ let token = token.clone();
+ repo.run(move |docs| async move {
+ ensure(&scoped).await?;
+ let id = record_key(&token.owner, &token.step, "claim");
+ let Some(row) = docs.get(COLLECTION, &id).await? else {
+ return Ok(());
+ };
+ let mut claim: Claim = decode(row.clone(), &token.owner, &token.step)?;
+ if claim.holder.as_deref() != Some(&token.holder) || claim.epoch != token.epoch {
+ return Ok(());
+ }
+ claim.holder = None;
+ claim.valid_until_ms = 0;
+ let body = serde_json::to_value(claim)
+ .map_err(|error| StorageError::serialization(error.to_string()))?;
+ let check = ReadCheck {
+ collection: COLLECTION.to_owned(),
+ id: id.clone(),
+ expected: ExpectedDocument::Version(row.version),
+ };
+ let write = WriteOp::Put {
+ collection: COLLECTION.to_owned(),
+ id,
+ doc: body,
+ precondition: Precondition::Version(row.version),
+ };
+ match docs.atomic_batch_checked(vec![check], vec![write]).await {
+ Ok(_) => Ok(()),
+ Err(error) if error.kind() == ErrorKind::Conflict => Ok(()),
+ Err(error) => Err(error),
+ }
+ })
+}
+
+/// Migrates older import marker documents to host-owned completion rows.
+pub async fn migrate_legacy_markers(scoped: &Storage) -> Result<usize, StorageError> {
let docs = scoped.documents();
if !docs
.capabilities()
- .contains(Capability::CollectionEnumeration)
+ .contains(tinystoragedrivers::Capability::CollectionEnumeration)
{
return Ok(0);
}
@@ -313,21 +432,30 @@ pub async fn migrate_legacy_markers(scoped: &ScopedStorage) -> Result<usize, Sto
if !present.iter().any(|name| name == collection) {
continue;
}
- declare(docs).await?;
- for marker in docs.query_all(collection, &Query::all()).await? {
+ ensure(scoped).await?;
+ for marker in docs
+ .query_all(collection, &tinystoragedrivers::Query::all())
+ .await?
+ {
if marker.id.ends_with("#claim") {
continue;
}
- mark_done(scoped, owner, &step_id(collection, &marker.id)).await?;
+ let completed_at_ms = if marker.meta.created_ms > 0 {
+ marker.meta.created_ms
+ } else {
+ chrono::Utc::now().timestamp_millis()
+ };
+ mark_done_at(
+ scoped,
+ owner,
+ &step_id(collection, &marker.id),
+ completed_at_ms,
+ )
+ .await?;
moved += 1;
}
docs.drop_collection(collection).await?;
- tracing::info!(
- target: "openhuman::storage",
- collection,
- owner,
- "{LOG_PREFIX} legacy import markers moved to tsd.migrations"
- );
+ tracing::info!(target: "openhuman::storage", collection, owner, "{LOG_PREFIX} legacy import markers moved to host-owned state");
}
Ok(moved)
}
diff --git a/crates/openhuman-core/src/storage/import_state_tests.rs b/crates/openhuman-core/src/storage/import_state_tests.rs
index b96bf5bc52..514efd7787 100644
--- a/crates/openhuman-core/src/storage/import_state_tests.rs
+++ b/crates/openhuman-core/src/storage/import_state_tests.rs
@@ -1,8 +1,8 @@
use super::*;
use crate::storage::{MemoryStorage, Scope, StorageBackend};
-use std::sync::mpsc;
use std::sync::Arc;
-use tinystoragedrivers::{CollectionSpec, Leases};
+use std::sync::mpsc;
+use tinystoragedrivers::CollectionSpec;
const OWNER: &str = "claims_test";
@@ -15,22 +15,30 @@ fn repo() -> Repo {
Repo::on(&backend, &Scope::local(), "claim-test", spec).unwrap()
}
-fn claim_held(repo: &Repo, step: &str) -> bool {
- let key = lease_key(OWNER, step);
+fn load_claim(repo: &Repo, owner: &str, step: &str) -> Option<Claim> {
+ let scoped = repo.scoped().clone();
+ let (owner, step) = (owner.to_owned(), step.to_owned());
repo.run(move |docs| async move {
- declare(&docs).await?;
- Leases::new(&docs).current(CLAIM_OWNER, &key).await
+ ensure(&scoped).await?;
+ let id = record_key(&owner, &step, "claim");
+ match docs.get(COLLECTION, &id).await? {
+ Some(row) => decode(row, &owner, &step).map(Some),
+ None => Ok(None),
+ }
})
.unwrap()
- .is_some()
}
#[test]
-fn claim_runs_the_body_and_releases() {
+fn claim_runs_the_body_and_releases_while_retaining_its_epoch() {
let repo = repo();
let ran = with_claim(&repo, OWNER, "m", || Ok(7)).unwrap();
assert_eq!(ran, Some(7));
- assert!(!claim_held(&repo, "m"));
+ let released = load_claim(&repo, OWNER, "m").unwrap();
+ assert_eq!(released.epoch, 1);
+ assert!(released.holder.is_none());
+ assert_eq!(with_claim(&repo, OWNER, "m", || Ok(8)).unwrap(), Some(8));
+ assert_eq!(load_claim(&repo, OWNER, "m").unwrap().epoch, 2);
}
#[test]
@@ -38,7 +46,7 @@ fn claim_is_released_when_the_body_fails() {
let repo = repo();
let result: Result<Option<()>> = with_claim(&repo, OWNER, "m", || Err(anyhow::anyhow!("boom")));
assert!(result.is_err());
- assert!(!claim_held(&repo, "m"));
+ assert!(load_claim(&repo, OWNER, "m").unwrap().holder.is_none());
assert_eq!(with_claim(&repo, OWNER, "m", || Ok(1)).unwrap(), Some(1));
}
@@ -97,7 +105,7 @@ fn many_racing_processes_run_the_body_once_at_a_time() {
if active.fetch_add(1, SeqCst) != 0 {
overlap.store(true, SeqCst);
}
- std::thread::sleep(std::time::Duration::from_millis(30));
+ std::thread::sleep(Duration::from_millis(30));
active.fetch_sub(1, SeqCst);
Ok(())
})
@@ -108,41 +116,24 @@ fn many_racing_processes_run_the_body_once_at_a_time() {
.collect();
let ran = handles
.into_iter()
- .map(|h| h.join().unwrap())
- .filter(|r| *r)
+ .filter(|handle| handle.join().unwrap())
.count();
assert!(ran >= 1, "somebody imports");
- assert!(
- !overlap.load(std::sync::atomic::Ordering::SeqCst),
- "two bodies held the claim at once"
- );
+ assert!(!overlap.load(std::sync::atomic::Ordering::SeqCst));
}
#[test]
-fn a_stale_claim_is_taken_over() {
+fn a_stale_claim_is_taken_over_with_a_new_epoch() {
let repo = repo();
- let key = lease_key(OWNER, "m");
- repo.run(move |docs| async move {
- declare(&docs).await?;
- let clock: tinystoragedrivers::Clock = Arc::new(|| 1_000);
- Leases::new(&docs)
- .with_clock(clock)
- .acquire(CLAIM_OWNER, &key, "dead-process", 100)
- .await
- .map(|_| ())
- })
- .unwrap();
-
- // Fresh: still held.
- let held = with_claim_at(&repo, OWNER, "m", 1_000 + 10, 100, || Ok(())).unwrap();
- assert_eq!(held, None);
- // Past the lease: taken over, run, released.
- let taken = with_claim_at(&repo, OWNER, "m", 1_000 + 200, 100, || Ok(())).unwrap();
- assert_eq!(taken, Some(()));
+ let first = acquire(&repo, OWNER, "m", 1_000, 100).unwrap().unwrap();
+ assert!(acquire(&repo, OWNER, "m", 1_010, 100).unwrap().is_none());
+ let second = acquire(&repo, OWNER, "m", 1_200, 100).unwrap().unwrap();
+ assert_eq!(second.epoch, first.epoch + 1);
+ release(&repo, &second).unwrap();
}
#[test]
-fn completion_is_a_migrations_entry_in_the_scope() {
+fn completion_is_an_immutable_host_owned_record_in_the_scope() {
let repo = repo();
assert!(!is_done(&repo, "threads", "legacy-conversations").unwrap());
set_done(&repo, "threads", "legacy-conversations").unwrap();
@@ -157,46 +148,43 @@ fn a_legacy_marker_row_is_honoured_and_removed() {
let scoped = backend.for_scope(&Scope::local()).unwrap();
let owned = scoped.clone();
crate::storage::block_on(async move {
- let scoped = owned;
- let docs = scoped.documents();
+ let docs = owned.documents();
docs.ensure_collection(&CollectionSpec::new("storage_import"))
.await?;
docs.put(
"storage_import",
"approval@abcd",
serde_json::json!({ "version": 1, "domain": "approval" }),
- tinystoragedrivers::Precondition::Absent,
+ Precondition::Absent,
)
.await?;
docs.put(
"storage_import",
"approval@abcd#claim",
serde_json::json!({ "claimed_at_ms": 1 }),
- tinystoragedrivers::Precondition::Absent,
+ Precondition::Absent,
)
.await?;
- let moved = migrate_legacy_markers(&scoped).await?;
+ let moved = migrate_legacy_markers(&owned).await?;
assert_eq!(moved, 1);
assert!(
done(
- &scoped,
+ &owned,
"storage",
&step_id("storage_import", "approval@abcd")
)
.await?
);
- assert!(!done(&scoped, "storage", &step_id("storage_import", "other@x")).await?);
+ assert!(!done(&owned, "storage", &step_id("storage_import", "other@x")).await?);
assert!(
docs.query_all("storage_import", &tinystoragedrivers::Query::all())
- .await
- .map_or(true, |rows| rows.is_empty()),
- "the marker rows are gone"
+ .await?
+ .is_empty()
);
- // A second sweep has nothing to do and keeps the entry.
- assert_eq!(migrate_legacy_markers(&scoped).await?, 0);
+ assert_eq!(migrate_legacy_markers(&owned).await?, 0);
assert!(
done(
- &scoped,
+ &owned,
"storage",
&step_id("storage_import", "approval@abcd")
)
@@ -215,16 +203,16 @@ fn a_waiting_process_runs_once_the_holder_releases() {
let holder = std::thread::spawn(move || {
with_claim(&holder_repo, OWNER, "m", || {
started_tx.send(()).unwrap();
- std::thread::sleep(std::time::Duration::from_millis(400));
+ std::thread::sleep(Duration::from_millis(400));
Ok("first")
})
.unwrap()
});
started_rx.recv().unwrap();
- let began = std::time::Instant::now();
+ let began = Instant::now();
let second = with_claim(&repo, OWNER, "m", || Ok("second")).unwrap();
assert_eq!(second, Some("second"));
- assert!(began.elapsed() >= std::time::Duration::from_millis(200));
+ assert!(began.elapsed() >= Duration::from_millis(200));
assert_eq!(holder.join().unwrap(), Some("first"));
}
@@ -232,12 +220,11 @@ fn a_waiting_process_runs_once_the_holder_releases() {
fn a_long_import_keeps_its_claim_alive() {
let repo = repo();
let now = chrono::Utc::now().timestamp_millis();
- // 90ms lease, body runs 400ms: without renewal a rival would take over.
let rival_repo = repo.clone();
let rival_ran = with_claim_at(&repo, OWNER, "m", now, 90, || {
let mut took_over = false;
for _ in 0..8 {
- std::thread::sleep(std::time::Duration::from_millis(50));
+ std::thread::sleep(Duration::from_millis(50));
let at = chrono::Utc::now().timestamp_millis();
took_over |= with_claim_at(&rival_repo, OWNER, "m", at, 90, || Ok(()))
.unwrap()