The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.
Rust agent-orchestrationai-agentsai-assistantdesktopllmlocal-firstmcppersonal-aiprivacyrustsecond-braintauri
refactor(storage): update import state management to host-owned records
Steven Enamakelc8e6cb3f47de
diff --git a/crates/openhuman-core/src/storage/import/layout_db_tests.rs b/crates/openhuman-core/src/storage/import/layout_db_tests.rsindex 607816cccc..ff291263fa 100644--- a/crates/openhuman-core/src/storage/import/layout_db_tests.rs+++ b/crates/openhuman-core/src/storage/import/layout_db_tests.rs@@ -1,6 +1,6 @@use super::*;use crate::storage::layout::test_fixtures::{fixture_docs, v1_profile};-use crate::storage::layout::{legacy, Target};+use crate::storage::layout::{Target, legacy};use tempfile::TempDir;#[test]diff --git a/crates/openhuman-core/src/storage/import_state.rs b/crates/openhuman-core/src/storage/import_state.rsindex 09669d8feb..dc4fcbb1d5 100644--- a/crates/openhuman-core/src/storage/import_state.rs+++ b/crates/openhuman-core/src/storage/import_state.rs@@ -1,52 +1,30 @@-//! Where a one-time legacy importer records that it finished, and the claim-//! that keeps two processes from importing at once.+//! Host-owned completion markers and expiring claims for one-time imports.//!-//! Both are tsd primitives, not host collections:-//!-//! - **Completion** is a `tsd.migrations` entry ([`Migrations::mark_done`]) in-//! the scope the importer writes to, keyed `(owner, step)`. This replaces-//! the per-domain marker collections (`storage_import`, `threads_import`,-//! `subagent_sessions_import`, `composio_state_import` and the module-//! `*_host_import` ones). A marker row an older build wrote is honoured by-//! [`migrate_legacy_markers`], which turns each into the same entry at boot,-//! so an installed profile is never imported twice.-//! - **The claim** is a [`Leases`] lease on `(owner, step)` in `tsd.leases`-//! (schema-v2 section 5). The process that wins it copies the legacy-//! records; the others wait it out, then re-check completion. The owner-//! renews the lease while it works, and a lease that lapses (its owner-//! crashed) is taken over by the next acquirer, so a dead process never-//! blocks an import.-//!-//! A step is named `<kind>/<id>`: `kind` is the old marker collection name-//! (`storage_import`) and `id` the old marker id (`approval@<workspace-//! digest>`), so both the new check and the migrated entry derive from the-//! same pair.+//! Import progress is ordinary owner data in `storage.import_state`. The+//! collection is deliberately separate from Tiny Storage Drivers' schemas:+//! the host owns importer names, completion records, and claim policy.use std::sync::Arc;use std::time::{Duration, Instant};use anyhow::Result;-use tinystoragedrivers::schema::Migrations;+use serde::{Deserialize, Serialize};use tinystoragedrivers::{- Capability, DocumentStore, DocumentStoreExt, Leases, Query, ScopedStorage, StorageError,+ CollectionSpec, DocumentStore, DocumentStoreExt, ErrorKind, ExpectedDocument, Precondition,+ ReadCheck, Storage, StorageError, Versioned, WriteOp,};use super::documents::Repo;-/// How long a claim blocks other processes before it counts as abandoned.+/// How long a claim blocks other processes before it can be taken over.pub const CLAIM_TTL_MS: i64 = 10 * 60 * 1000;-/// The `tsd.leases` owner of import claims.-const CLAIM_OWNER: &str = "import";-+const COLLECTION: &str = "storage.import_state";const LOG_PREFIX: &str = "[storage::import_state]";--/// How often a waiting process re-tries a claim another process holds.const WAIT_POLL: Duration = Duration::from_millis(100);+const CAS_RETRIES: usize = 32;-/// The legacy marker collections and the `tsd.migrations` owner each moves-/// under. A marker lived in the scope its importer wrote to, so every scope-/// is swept.+/// The legacy marker collections whose completion rows are moved at boot.pub const LEGACY_MARKERS: &[(&str, &str)] = &[("storage_import", "storage"),("threads_import", "threads"),@@ -60,136 +38,182 @@ pub const LEGACY_MARKERS: &[(&str, &str)] = &[("mcp_host_import", "host"),];-/// The step name of the importer that wrote marker `id` into `kind`.+/// Declares the host-owned import progress collection.+#[must_use]+pub fn collection_specs() -> Vec<CollectionSpec> {+ vec![CollectionSpec::new(COLLECTION)]+}++/// The stable step key for a marker in an older collection.#[must_use]pub fn step_id(kind: &str, id: &str) -> String {format!("{kind}/{id}")}-/// Declares the primitives an import state needs, for a backend that did not-/// go through the boot `prepare` (a test's, an embedder's).-async fn declare(docs: &Arc<dyn DocumentStore>) -> Result<(), StorageError> {- for spec in tinystoragedrivers::primitives::collection_specs() {- docs.ensure_collection(&spec).await?;+/// The owner name used by module importers.+pub const HOST_OWNER: &str = "host";++fn record_key(owner: &str, step: &str, kind: &str) -> String {+ // Hex encoding keeps this key injective even when either source component+ // contains separators used by the document-id grammar.+ format!(+ "{kind}/{}/{}",+ hex::encode(owner.as_bytes()),+ hex::encode(step.as_bytes())+ )+}++async fn ensure(scoped: &Storage) -> Result<(), StorageError> {+ for spec in collection_specs() {+ scoped.ensure_collection(&spec).await?;}Ok(())}-/// Whether importer `step` of `owner` finished in `scoped`'s scope.-///-/// # Errors-///-/// A storage error.-pub async fn done(scoped: &ScopedStorage, owner: &str, step: &str) -> Result<bool, StorageError> {- declare(scoped.documents()).await?;- Migrations::new(scoped).done(owner, step).await+#[derive(Debug, Clone, Serialize, Deserialize)]+struct Completion {+ record_type: String,+ owner: String,+ step: String,+ completed_at_ms: i64,}-/// Records importer `step` of `owner` as finished in `scoped`'s scope.-///-/// # Errors-///-/// A storage error.-pub async fn mark_done(- scoped: &ScopedStorage,+#[derive(Debug, Clone, Serialize, Deserialize)]+struct Claim {+ record_type: String,+ owner: String,+ step: String,+ holder: Option<String>,+ epoch: u64,+ valid_until_ms: i64,+}++fn decode<T: for<'de> Deserialize<'de>>(+ row: Versioned<serde_json::Value>,+ expected_owner: &str,+ expected_step: &str,+) -> Result<T, StorageError> {+ let owner = row.doc.get("owner").and_then(serde_json::Value::as_str);+ let step = row.doc.get("step").and_then(serde_json::Value::as_str);+ if owner != Some(expected_owner) || step != Some(expected_step) {+ return Err(StorageError::serialization(+ "import state key does not match its owner and step",+ ));+ }+ serde_json::from_value(row.doc)+ .map_err(|error| StorageError::serialization(format!("invalid import state: {error}")))+}++/// Checks whether the import `step` for `owner` is complete in `scoped`.+pub async fn done(scoped: &Storage, owner: &str, step: &str) -> Result<bool, StorageError> {+ ensure(scoped).await?;+ let id = record_key(owner, step, "done");+ let Some(row) = scoped.get(COLLECTION, &id).await? else {+ return Ok(false);+ };+ let _: Completion = decode(row, owner, step)?;+ Ok(true)+}++/// Records completion of an import step. Existing completion is immutable.+pub async fn mark_done(scoped: &Storage, owner: &str, step: &str) -> Result<(), StorageError> {+ mark_done_at(scoped, owner, step, chrono::Utc::now().timestamp_millis()).await+}++async fn mark_done_at(+ scoped: &Storage,owner: &str,step: &str,+ completed_at_ms: i64,) -> Result<(), StorageError> {- declare(scoped.documents()).await?;- Migrations::new(scoped).mark_done(owner, step).await+ ensure(scoped).await?;+ let id = record_key(owner, step, "done");+ let body = serde_json::to_value(Completion {+ record_type: "completion".to_owned(),+ owner: owner.to_owned(),+ step: step.to_owned(),+ completed_at_ms,+ })+ .map_err(|error| StorageError::serialization(error.to_string()))?;+ for _ in 0..CAS_RETRIES {+ if let Some(current) = scoped.get(COLLECTION, &id).await? {+ let _: Completion = decode(current, owner, step)?;+ return Ok(());+ }+ match scoped+ .put(COLLECTION, &id, body.clone(), Precondition::Absent)+ .await+ {+ Ok(_) => return Ok(()),+ Err(error) if error.kind() == ErrorKind::Conflict => continue,+ Err(error) => return Err(error),+ }+ }+ Err(StorageError::conflict(+ "import completion kept changing during its retry budget",+ ))}-/// The `tsd.migrations` owner of the module importers, which run in a module's-/// scope on behalf of the host (`kind` is the old marker collection name).-pub const HOST_OWNER: &str = "host";--/// Whether the module importer `kind` already finished for marker `id` in-/// `scoped`'s (module) scope.-///-/// # Errors-///-/// A storage error.+/// Whether the module importer has completed a legacy marker.pub async fn host_import_done(- scoped: &ScopedStorage,+ scoped: &Storage,kind: &str,id: &str,) -> Result<bool, StorageError> {done(scoped, HOST_OWNER, &step_id(kind, id)).await}-/// Records the module importer `kind` as finished for marker `id`.-///-/// # Errors-///-/// A storage error.-pub async fn host_import_mark(- scoped: &ScopedStorage,- kind: &str,- id: &str,-) -> Result<(), StorageError> {+/// Records completion of a module import marker.+pub async fn host_import_mark(scoped: &Storage, kind: &str, id: &str) -> Result<(), StorageError> {mark_done(scoped, HOST_OWNER, &step_id(kind, id)).await}-/// [`done`] from synchronous code, over the scope `repo` was opened on.-///-/// # Errors-///-/// A storage error.+/// Synchronous completion lookup for existing synchronous import APIs.pub fn is_done(repo: &Repo, owner: &str, step: &str) -> Result<bool> {let scoped = repo.scoped().clone();let (owner, step) = (owner.to_owned(), step.to_owned());repo.run(move |_| async move { done(&scoped, &owner, &step).await })}-/// [`mark_done`] from synchronous code.-///-/// # Errors-///-/// A storage error.+/// Synchronous completion write for existing synchronous import APIs.pub fn set_done(repo: &Repo, owner: &str, step: &str) -> Result<()> {let scoped = repo.scoped().clone();let (owner, step) = (owner.to_owned(), step.to_owned());repo.run(move |_| async move { mark_done(&scoped, &owner, &step).await })}-/// Runs `body` while holding the claim for `(owner, step)` in `repo`'s scope.-///-/// A live claim held by another process is waited out (up to-/// [`CLAIM_TTL_MS`]); once it is free `body` runs, and it must re-check its-/// completion because the winner has usually finished by then. Returns `None`-/// only when the claim stays held for the whole wait. The claim is renewed-/// while `body` runs, so a long import is not mistaken for a crashed one.-///-/// # Errors+/// Runs `body` while holding a claim for this importer step.///-/// A storage error while claiming, or whatever `body` returns. The claim is-/// released either way.+/// A competing live claim is polled until release or expiry, bounded by the+/// claim TTL. The body must re-check completion after acquiring the claim.pub fn with_claim<T>(repo: &Repo,owner: &str,step: &str,body: impl FnOnce() -> Result<T>,) -> Result<Option<T>> {- // `with_claim_at` only runs the closure once it holds the claim.- let mut body = Some(body);let started = Instant::now();+ let mut body = Some(body);loop {let now = chrono::Utc::now().timestamp_millis();- if let Some(ran) = with_claim_at(repo, owner, step, now, CLAIM_TTL_MS, || {- (body.take().expect("import body runs once"))()- })? {- return Ok(Some(ran));+ if let Some(token) = acquire(repo, owner, step, now, CLAIM_TTL_MS)? {+ let (stop_tx, stop_rx) = std::sync::mpsc::channel();+ let heartbeat = start_heartbeat(repo.clone(), token.clone(), CLAIM_TTL_MS, stop_rx);+ let result = body.take().expect("import body runs once")();+ let _ = stop_tx.send(());+ let _ = heartbeat.join();+ release(repo, &token)?;+ return result.map(Some);}- if started.elapsed().as_millis() as i64 >= CLAIM_TTL_MS {+ if started.elapsed() >= Duration::from_millis(CLAIM_TTL_MS.max(0) as u64) {return Ok(None);}std::thread::sleep(WAIT_POLL);}}-/// One attempt of [`with_claim`] with an explicit clock and lease length, for-/// tests: returns `None` without running `body` or waiting when another live-/// process holds the claim.+/// One bounded claim attempt with an explicit clock and TTL, primarily for+/// deterministic concurrency tests.pub fn with_claim_at<T>(repo: &Repo,owner: &str,@@ -198,112 +222,207 @@ pub fn with_claim_at<T>(ttl_ms: i64,body: impl FnOnce() -> Result<T>,) -> Result<Option<T>> {- let started = Instant::now();- let clock: tinystoragedrivers::Clock = Arc::new(move || {- (now_ms.max(0) as u64).saturating_add(started.elapsed().as_millis() as u64)- });- // A holder name unique to this attempt, so two threads of one process do- // not both "re-acquire" the same live lease.- let holder = format!("pid-{}-{}", std::process::id(), uuid::Uuid::new_v4());- let (claim_owner, claim_key) = (owner.to_owned(), step.to_owned());- let acquired = {- let (clock, holder) = (clock.clone(), holder.clone());- let (owner, key) = (claim_owner.clone(), claim_key.clone());- repo.run(move |docs| async move {- declare(&docs).await?;- Leases::new(&docs)- .with_clock(clock)- .acquire(CLAIM_OWNER, &lease_key(&owner, &key), &holder, ttl_ms)- .await- })?- };- let Some(held) = acquired else {- tracing::debug!(- target: "openhuman::storage_import",- owner,- step,- "{LOG_PREFIX} another process holds the import claim"- );+ let Some(token) = acquire(repo, owner, step, now_ms, ttl_ms)? else {return Ok(None);};- let epoch = held.epoch;- let (stop_tx, stop_rx) = std::sync::mpsc::channel::<()>();- let heartbeat = {- let (repo, clock, holder) = (repo.clone(), clock.clone(), holder.clone());- let key = lease_key(&claim_owner, &claim_key);- let interval = Duration::from_millis((ttl_ms / 3).max(1) as u64);- // A bare OS thread on purpose (baselined in the SaaS ambient ratchet):- // the heartbeat reads no ambient state, only the `repo` handed to it,- // and is joined below before this returns. `with_claim` is synchronous- // and may run outside a tokio runtime, so `DispatchCtx::spawn_blocking`- // (which needs `Handle::current()`) is not available here.- std::thread::spawn(move || {- while let Err(std::sync::mpsc::RecvTimeoutError::Timeout) =- stop_rx.recv_timeout(interval)- {- let (clock, holder, key) = (clock.clone(), holder.clone(), key.clone());- let renewed = repo.run(move |docs| async move {- Leases::new(&docs)- .with_clock(clock)- .renew(CLAIM_OWNER, &key, &holder, epoch, ttl_ms)- .await- });- match renewed {- Ok(Some(_)) => {}- Ok(None) | Err(_) => {- tracing::warn!(- target: "openhuman::storage_import",- "{LOG_PREFIX} could not renew the import claim"- );- break;+ let result = body();+ release(repo, &token)?;+ result.map(Some)+}++#[derive(Clone)]+struct ClaimToken {+ owner: String,+ step: String,+ holder: String,+ epoch: u64,+ ttl_ms: i64,+ clock_started: Instant,+ clock_start_ms: i64,+}++fn acquire(+ repo: &Repo,+ owner: &str,+ step: &str,+ now_ms: i64,+ ttl_ms: i64,+) -> Result<Option<ClaimToken>> {+ if ttl_ms <= 0 {+ return Err(anyhow::anyhow!("import claim TTL must be positive"));+ }+ let holder = format!("pid-{}-{}", std::process::id(), uuid::Uuid::new_v4());+ let (owner_owned, step_owned, holder_owned) =+ (owner.to_owned(), step.to_owned(), holder.clone());+ let scoped = repo.scoped().clone();+ let outcome = repo.run(move |docs| async move {+ ensure(&scoped).await?;+ let id = record_key(&owner_owned, &step_owned, "claim");+ for _ in 0..CAS_RETRIES {+ let current = docs.get(COLLECTION, &id).await?;+ let (expected, precondition, epoch) = match current {+ Some(row) => {+ let claim: Claim = decode(row.clone(), &owner_owned, &step_owned)?;+ if claim.holder.is_some() && claim.valid_until_ms > now_ms {+ return Ok(None);}+ let epoch = claim+ .epoch+ .checked_add(1)+ .ok_or_else(|| StorageError::backend("import claim epoch exhausted"))?;+ (+ ExpectedDocument::Version(row.version),+ Precondition::Version(row.version),+ epoch,+ )}+ None => (ExpectedDocument::Absent, Precondition::Absent, 1),+ };+ let valid_until_ms = now_ms.saturating_add(ttl_ms);+ let body = serde_json::to_value(Claim {+ record_type: "claim".to_owned(),+ owner: owner_owned.clone(),+ step: step_owned.clone(),+ holder: Some(holder_owned.clone()),+ epoch,+ valid_until_ms,+ })+ .map_err(|error| StorageError::serialization(error.to_string()))?;+ let check = ReadCheck {+ collection: COLLECTION.to_owned(),+ id: id.clone(),+ expected,+ };+ let write = WriteOp::Put {+ collection: COLLECTION.to_owned(),+ id: id.clone(),+ doc: body,+ precondition,+ };+ match docs.atomic_batch_checked(vec![check], vec![write]).await {+ Ok(_) => return Ok(Some(epoch)),+ Err(error) if error.kind() == ErrorKind::Conflict => continue,+ Err(error) => return Err(error),}- })- };- let result = body();- let _ = stop_tx.send(());- let _ = heartbeat.join();- let released = {- let key = lease_key(&claim_owner, &claim_key);- repo.run(move |docs| async move {- Leases::new(&docs)- .with_clock(clock)- .release(CLAIM_OWNER, &key, &holder, epoch)- .await- })- };- if let Err(error) = released {- // The lease expires on its own; releasing is an optimisation.- tracing::warn!(- target: "openhuman::storage_import",- error = %format!("{error:#}"),- "{LOG_PREFIX} could not release the import claim; it expires on its own"- );- }- result.map(Some)+ }+ Err(StorageError::conflict(+ "import claim kept changing during its retry budget",+ ))+ })?;+ Ok(outcome.map(|epoch| ClaimToken {+ owner: owner.to_owned(),+ step: step.to_owned(),+ holder,+ epoch,+ ttl_ms,+ clock_started: Instant::now(),+ clock_start_ms: now_ms,+ }))}-fn lease_key(owner: &str, step: &str) -> String {- format!("{owner}/{step}")+fn start_heartbeat(+ repo: Repo,+ token: ClaimToken,+ ttl_ms: i64,+ stop: std::sync::mpsc::Receiver<()>,+) -> std::thread::JoinHandle<()> {+ let interval = Duration::from_millis((ttl_ms / 3).max(1) as u64);+ std::thread::spawn(move || {+ while let Err(std::sync::mpsc::RecvTimeoutError::Timeout) = stop.recv_timeout(interval) {+ let now_ms = token+ .clock_start_ms+ .saturating_add(token.clock_started.elapsed().as_millis() as i64);+ match renew(&repo, &token, now_ms) {+ Ok(true) => {}+ Ok(false) | Err(_) => {+ tracing::warn!(target: "openhuman::storage_import", "{LOG_PREFIX} could not renew the import claim");+ break;+ }+ }+ }+ })}-/// Turns the marker rows older builds wrote ([`LEGACY_MARKERS`]) in `scoped`'s-/// scope into `tsd.migrations` entries and removes the old collections, so an-/// already imported profile is not imported again. Returns the markers moved.-///-/// Idempotent: a marker already moved is simply marked again, and a collection-/// that is gone is skipped. A claim row (`<id>#claim`) is dropped with its-/// collection.-///-/// # Errors-///-/// A storage error; the markers not yet moved stay in place for the next boot.-pub async fn migrate_legacy_markers(scoped: &ScopedStorage) -> Result<usize, StorageError> {+fn renew(repo: &Repo, token: &ClaimToken, now_ms: i64) -> Result<bool> {+ let scoped = repo.scoped().clone();+ let token = token.clone();+ repo.run(move |docs| async move {+ ensure(&scoped).await?;+ let id = record_key(&token.owner, &token.step, "claim");+ let Some(row) = docs.get(COLLECTION, &id).await? else {+ return Ok(false);+ };+ let mut claim: Claim = decode(row.clone(), &token.owner, &token.step)?;+ if claim.holder.as_deref() != Some(&token.holder)+ || claim.epoch != token.epoch+ || claim.valid_until_ms <= now_ms+ {+ return Ok(false);+ }+ claim.valid_until_ms = now_ms.saturating_add(token.ttl_ms);+ let body = serde_json::to_value(claim)+ .map_err(|error| StorageError::serialization(error.to_string()))?;+ let check = ReadCheck {+ collection: COLLECTION.to_owned(),+ id: id.clone(),+ expected: ExpectedDocument::Version(row.version),+ };+ let write = WriteOp::Put {+ collection: COLLECTION.to_owned(),+ id,+ doc: body,+ precondition: Precondition::Version(row.version),+ };+ match docs.atomic_batch_checked(vec![check], vec![write]).await {+ Ok(_) => Ok(true),+ Err(error) if error.kind() == ErrorKind::Conflict => Ok(false),+ Err(error) => Err(error),+ }+ })+}++fn release(repo: &Repo, token: &ClaimToken) -> Result<()> {+ let scoped = repo.scoped().clone();+ let token = token.clone();+ repo.run(move |docs| async move {+ ensure(&scoped).await?;+ let id = record_key(&token.owner, &token.step, "claim");+ let Some(row) = docs.get(COLLECTION, &id).await? else {+ return Ok(());+ };+ let mut claim: Claim = decode(row.clone(), &token.owner, &token.step)?;+ if claim.holder.as_deref() != Some(&token.holder) || claim.epoch != token.epoch {+ return Ok(());+ }+ claim.holder = None;+ claim.valid_until_ms = 0;+ let body = serde_json::to_value(claim)+ .map_err(|error| StorageError::serialization(error.to_string()))?;+ let check = ReadCheck {+ collection: COLLECTION.to_owned(),+ id: id.clone(),+ expected: ExpectedDocument::Version(row.version),+ };+ let write = WriteOp::Put {+ collection: COLLECTION.to_owned(),+ id,+ doc: body,+ precondition: Precondition::Version(row.version),+ };+ match docs.atomic_batch_checked(vec![check], vec![write]).await {+ Ok(_) => Ok(()),+ Err(error) if error.kind() == ErrorKind::Conflict => Ok(()),+ Err(error) => Err(error),+ }+ })+}++/// Migrates older import marker documents to host-owned completion rows.+pub async fn migrate_legacy_markers(scoped: &Storage) -> Result<usize, StorageError> {let docs = scoped.documents();if !docs.capabilities()- .contains(Capability::CollectionEnumeration)+ .contains(tinystoragedrivers::Capability::CollectionEnumeration){return Ok(0);}@@ -313,21 +432,30 @@ pub async fn migrate_legacy_markers(scoped: &ScopedStorage) -> Result<usize, Stoif !present.iter().any(|name| name == collection) {continue;}- declare(docs).await?;- for marker in docs.query_all(collection, &Query::all()).await? {+ ensure(scoped).await?;+ for marker in docs+ .query_all(collection, &tinystoragedrivers::Query::all())+ .await?+ {if marker.id.ends_with("#claim") {continue;}- mark_done(scoped, owner, &step_id(collection, &marker.id)).await?;+ let completed_at_ms = if marker.meta.created_ms > 0 {+ marker.meta.created_ms+ } else {+ chrono::Utc::now().timestamp_millis()+ };+ mark_done_at(+ scoped,+ owner,+ &step_id(collection, &marker.id),+ completed_at_ms,+ )+ .await?;moved += 1;}docs.drop_collection(collection).await?;- tracing::info!(- target: "openhuman::storage",- collection,- owner,- "{LOG_PREFIX} legacy import markers moved to tsd.migrations"- );+ tracing::info!(target: "openhuman::storage", collection, owner, "{LOG_PREFIX} legacy import markers moved to host-owned state");}Ok(moved)}diff --git a/crates/openhuman-core/src/storage/import_state_tests.rs b/crates/openhuman-core/src/storage/import_state_tests.rsindex b96bf5bc52..514efd7787 100644--- a/crates/openhuman-core/src/storage/import_state_tests.rs+++ b/crates/openhuman-core/src/storage/import_state_tests.rs@@ -1,8 +1,8 @@use super::*;use crate::storage::{MemoryStorage, Scope, StorageBackend};-use std::sync::mpsc;use std::sync::Arc;-use tinystoragedrivers::{CollectionSpec, Leases};+use std::sync::mpsc;+use tinystoragedrivers::CollectionSpec;const OWNER: &str = "claims_test";@@ -15,22 +15,30 @@ fn repo() -> Repo {Repo::on(&backend, &Scope::local(), "claim-test", spec).unwrap()}-fn claim_held(repo: &Repo, step: &str) -> bool {- let key = lease_key(OWNER, step);+fn load_claim(repo: &Repo, owner: &str, step: &str) -> Option<Claim> {+ let scoped = repo.scoped().clone();+ let (owner, step) = (owner.to_owned(), step.to_owned());repo.run(move |docs| async move {- declare(&docs).await?;- Leases::new(&docs).current(CLAIM_OWNER, &key).await+ ensure(&scoped).await?;+ let id = record_key(&owner, &step, "claim");+ match docs.get(COLLECTION, &id).await? {+ Some(row) => decode(row, &owner, &step).map(Some),+ None => Ok(None),+ }}).unwrap()- .is_some()}#[test]-fn claim_runs_the_body_and_releases() {+fn claim_runs_the_body_and_releases_while_retaining_its_epoch() {let repo = repo();let ran = with_claim(&repo, OWNER, "m", || Ok(7)).unwrap();assert_eq!(ran, Some(7));- assert!(!claim_held(&repo, "m"));+ let released = load_claim(&repo, OWNER, "m").unwrap();+ assert_eq!(released.epoch, 1);+ assert!(released.holder.is_none());+ assert_eq!(with_claim(&repo, OWNER, "m", || Ok(8)).unwrap(), Some(8));+ assert_eq!(load_claim(&repo, OWNER, "m").unwrap().epoch, 2);}#[test]@@ -38,7 +46,7 @@ fn claim_is_released_when_the_body_fails() {let repo = repo();let result: Result<Option<()>> = with_claim(&repo, OWNER, "m", || Err(anyhow::anyhow!("boom")));assert!(result.is_err());- assert!(!claim_held(&repo, "m"));+ assert!(load_claim(&repo, OWNER, "m").unwrap().holder.is_none());assert_eq!(with_claim(&repo, OWNER, "m", || Ok(1)).unwrap(), Some(1));}@@ -97,7 +105,7 @@ fn many_racing_processes_run_the_body_once_at_a_time() {if active.fetch_add(1, SeqCst) != 0 {overlap.store(true, SeqCst);}- std::thread::sleep(std::time::Duration::from_millis(30));+ std::thread::sleep(Duration::from_millis(30));active.fetch_sub(1, SeqCst);Ok(())})@@ -108,41 +116,24 @@ fn many_racing_processes_run_the_body_once_at_a_time() {.collect();let ran = handles.into_iter()- .map(|h| h.join().unwrap())- .filter(|r| *r)+ .filter(|handle| handle.join().unwrap()).count();assert!(ran >= 1, "somebody imports");- assert!(- !overlap.load(std::sync::atomic::Ordering::SeqCst),- "two bodies held the claim at once"- );+ assert!(!overlap.load(std::sync::atomic::Ordering::SeqCst));}#[test]-fn a_stale_claim_is_taken_over() {+fn a_stale_claim_is_taken_over_with_a_new_epoch() {let repo = repo();- let key = lease_key(OWNER, "m");- repo.run(move |docs| async move {- declare(&docs).await?;- let clock: tinystoragedrivers::Clock = Arc::new(|| 1_000);- Leases::new(&docs)- .with_clock(clock)- .acquire(CLAIM_OWNER, &key, "dead-process", 100)- .await- .map(|_| ())- })- .unwrap();-- // Fresh: still held.- let held = with_claim_at(&repo, OWNER, "m", 1_000 + 10, 100, || Ok(())).unwrap();- assert_eq!(held, None);- // Past the lease: taken over, run, released.- let taken = with_claim_at(&repo, OWNER, "m", 1_000 + 200, 100, || Ok(())).unwrap();- assert_eq!(taken, Some(()));+ let first = acquire(&repo, OWNER, "m", 1_000, 100).unwrap().unwrap();+ assert!(acquire(&repo, OWNER, "m", 1_010, 100).unwrap().is_none());+ let second = acquire(&repo, OWNER, "m", 1_200, 100).unwrap().unwrap();+ assert_eq!(second.epoch, first.epoch + 1);+ release(&repo, &second).unwrap();}#[test]-fn completion_is_a_migrations_entry_in_the_scope() {+fn completion_is_an_immutable_host_owned_record_in_the_scope() {let repo = repo();assert!(!is_done(&repo, "threads", "legacy-conversations").unwrap());set_done(&repo, "threads", "legacy-conversations").unwrap();@@ -157,46 +148,43 @@ fn a_legacy_marker_row_is_honoured_and_removed() {let scoped = backend.for_scope(&Scope::local()).unwrap();let owned = scoped.clone();crate::storage::block_on(async move {- let scoped = owned;- let docs = scoped.documents();+ let docs = owned.documents();docs.ensure_collection(&CollectionSpec::new("storage_import")).await?;docs.put("storage_import","approval@abcd",serde_json::json!({ "version": 1, "domain": "approval" }),- tinystoragedrivers::Precondition::Absent,+ Precondition::Absent,).await?;docs.put("storage_import","approval@abcd#claim",serde_json::json!({ "claimed_at_ms": 1 }),- tinystoragedrivers::Precondition::Absent,+ Precondition::Absent,).await?;- let moved = migrate_legacy_markers(&scoped).await?;+ let moved = migrate_legacy_markers(&owned).await?;assert_eq!(moved, 1);assert!(done(- &scoped,+ &owned,"storage",&step_id("storage_import", "approval@abcd")).await?);- assert!(!done(&scoped, "storage", &step_id("storage_import", "other@x")).await?);+ assert!(!done(&owned, "storage", &step_id("storage_import", "other@x")).await?);assert!(docs.query_all("storage_import", &tinystoragedrivers::Query::all())- .await- .map_or(true, |rows| rows.is_empty()),- "the marker rows are gone"+ .await?+ .is_empty());- // A second sweep has nothing to do and keeps the entry.- assert_eq!(migrate_legacy_markers(&scoped).await?, 0);+ assert_eq!(migrate_legacy_markers(&owned).await?, 0);assert!(done(- &scoped,+ &owned,"storage",&step_id("storage_import", "approval@abcd"))@@ -215,16 +203,16 @@ fn a_waiting_process_runs_once_the_holder_releases() {let holder = std::thread::spawn(move || {with_claim(&holder_repo, OWNER, "m", || {started_tx.send(()).unwrap();- std::thread::sleep(std::time::Duration::from_millis(400));+ std::thread::sleep(Duration::from_millis(400));Ok("first")}).unwrap()});started_rx.recv().unwrap();- let began = std::time::Instant::now();+ let began = Instant::now();let second = with_claim(&repo, OWNER, "m", || Ok("second")).unwrap();assert_eq!(second, Some("second"));- assert!(began.elapsed() >= std::time::Duration::from_millis(200));+ assert!(began.elapsed() >= Duration::from_millis(200));assert_eq!(holder.join().unwrap(), Some("first"));}@@ -232,12 +220,11 @@ fn a_waiting_process_runs_once_the_holder_releases() {fn a_long_import_keeps_its_claim_alive() {let repo = repo();let now = chrono::Utc::now().timestamp_millis();- // 90ms lease, body runs 400ms: without renewal a rival would take over.let rival_repo = repo.clone();let rival_ran = with_claim_at(&repo, OWNER, "m", now, 90, || {let mut took_over = false;for _ in 0..8 {- std::thread::sleep(std::time::Duration::from_millis(50));+ std::thread::sleep(Duration::from_millis(50));let at = chrono::Utc::now().timestamp_millis();took_over |= with_claim_at(&rival_repo, OWNER, "m", at, 90, || Ok(())).unwrap()