Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(volume): replicate the Volume across the account's machines (#6280)

ddupont74657cf5a9b4
diff --git a/libs/cua/Cargo.lock b/libs/cua/Cargo.lock
index f7500f3e52..3060411e4b 100644
--- a/libs/cua/Cargo.lock
+++ b/libs/cua/Cargo.lock
@@ -3633,6 +3633,7 @@ dependencies = [
"thiserror 2.0.20",
"tokio",
"tracing",
+ "tracing-subscriber",
]
[[package]]
diff --git a/libs/cua/crates/cua-volume/Cargo.toml b/libs/cua/crates/cua-volume/Cargo.toml
index 782bbbf515..2ff6e54196 100644
--- a/libs/cua/crates/cua-volume/Cargo.toml
+++ b/libs/cua/crates/cua-volume/Cargo.toml
@@ -45,6 +45,7 @@ fuser = { version = "0.18", default-features = false, optional = true }
[dev-dependencies]
tempfile.workspace = true
+tracing-subscriber.workspace = true
tokio = { workspace = true, features = ["full"] }
[[example]]
diff --git a/libs/cua/crates/cua-volume/src/fs.rs b/libs/cua/crates/cua-volume/src/fs.rs
index ea42c84796..6fc9490c63 100644
--- a/libs/cua/crates/cua-volume/src/fs.rs
+++ b/libs/cua/crates/cua-volume/src/fs.rs
@@ -148,7 +148,8 @@ impl FsBackend {
Ok(())
}
- fn new_version() -> String {
+ /// A fresh version id (time-ordered, unique).
+ pub fn new_version() -> String {
// Strictly increasing within the process: two writes in the same
// millisecond must still sort in write order, so the random suffix
// only keeps ids apart across processes.
@@ -246,21 +247,20 @@ impl FsBackend {
Ok(out)
}
- fn delete_blocking(&self, key: &str, cond: Condition) -> Result<()> {
+ fn delete_blocking(&self, key: &str, cond: Condition) -> Result<String> {
let _lock = self.lock()?;
let current = self.read_head(key)?;
if current.is_none() && cond == Condition::None {
return Err(Error::NotFound(key.to_string()));
}
Self::check(&cond, current.as_ref(), key)?;
- let marker = self
- .versions_dir(key)?
- .join(format!("{}.del", Self::new_version()));
+ let version = Self::new_version();
+ let marker = self.versions_dir(key)?.join(format!("{version}.del"));
Self::atomic_write(&marker, b"")?;
let head = self.head_path(key)?;
cua_home::guard_write(&head)?;
fs::remove_file(head)?;
- Ok(())
+ Ok(version)
}
fn get_blocking(&self, key: &str, version: Option<&str>) -> Result<(Vec<u8>, ObjectMeta)> {
@@ -534,6 +534,209 @@ impl FsBackend {
}
}
+/// Where the bytes of an applied version come from.
+pub enum ApplySource {
+ /// In memory.
+ Bytes(Vec<u8>),
+ /// A finished file the store takes: moved (same file system) or copied.
+ /// With `link` the file stays where it is and the store gets a hard link
+ /// (a copy where hard links are not possible).
+ File { path: PathBuf, link: bool },
+}
+
+/// What replication needs from the local store: writes that keep the
+/// owner's version ids (so history, etags and preconditions mean the same
+/// on every machine), heads moved back to an earlier version, and the paths
+/// of version files. None of this changes the [`Backend`] contract.
+impl FsBackend {
+ /// Deletes `key` like [`Backend::delete`] and returns the delete marker's
+ /// version id (what replication ships).
+ pub async fn delete_versioned(&self, key: &str, cond: Condition) -> Result<String> {
+ let (me, key) = (self.clone(), key.to_string());
+ blocking(move || me.delete_blocking(&key, cond)).await
+ }
+
+ /// The file holding `version` of `key`.
+ pub fn version_file(&self, key: &str, version: &str) -> Result<PathBuf> {
+ self.version_path(key, version)
+ }
+
+ fn install_version(
+ &self,
+ key: &str,
+ version: &str,
+ src: ApplySource,
+ full: bool,
+ ) -> Result<()> {
+ let dst = self.version_path(key, version)?;
+ if dst.is_file() {
+ // Already there (a replayed record).
+ if let ApplySource::File { path, link: false } = src {
+ let _ = fs::remove_file(path);
+ }
+ return Ok(());
+ }
+ let dir = dst
+ .parent()
+ .ok_or_else(|| Error::Invalid("no parent".into()))?;
+ self.ensure_root()?;
+ cua_home::guard_write(&dst)?;
+ fs::create_dir_all(dir)?;
+ match src {
+ ApplySource::Bytes(b) => Self::atomic_write_with(&dst, &b, full)?,
+ ApplySource::File { path, link } => {
+ let tmp = dir.join(format!(".tmp-{}", crate::new_id()));
+ if link {
+ if fs::hard_link(&path, &tmp).is_err() {
+ fs::copy(&path, &tmp)?;
+ }
+ } else if fs::rename(&path, &tmp).is_err() {
+ fs::copy(&path, &tmp)?;
+ let _ = fs::remove_file(&path);
+ }
+ File::open(&tmp)?.sync_all()?;
+ fs::rename(&tmp, &dst).inspect_err(|_| {
+ let _ = fs::remove_file(&tmp);
+ })?;
+ }
+ }
+ Ok(())
+ }
+
+ /// Makes `meta` (a version the owner assigned) the head of its key,
+ /// storing its bytes under that version. Idempotent.
+ pub async fn apply_put(&self, meta: ObjectMeta, src: ApplySource) -> Result<ObjectMeta> {
+ let me = self.clone();
+ blocking(move || {
+ let _lock = me.lock()?;
+ me.install_version(&meta.key, &meta.version, src, true)?;
+ Self::atomic_write(&me.head_path(&meta.key)?, &serde_json::to_vec(&meta)?)?;
+ Ok(meta)
+ })
+ .await
+ }
+
+ /// [`FsBackend::apply_put`] for many small objects at once: one lock, one
+ /// disk barrier at the end (each file is still written and renamed whole,
+ /// so a crash leaves each either old or new).
+ pub async fn apply_many(&self, items: Vec<(ObjectMeta, Vec<u8>)>) -> Result<()> {
+ let me = self.clone();
+ blocking(move || {
+ let _lock = me.lock()?;
+ for (meta, bytes) in &items {
+ me.install_version(
+ &meta.key,
+ &meta.version,
+ ApplySource::Bytes(bytes.clone()),
+ false,
+ )?;
+ Self::atomic_write_with(
+ &me.head_path(&meta.key)?,
+ &serde_json::to_vec(meta)?,
+ false,
+ )?;
+ }
+ if !items.is_empty() {
+ OpenOptions::new()
+ .write(true)
+ .open(me.root.join(".lock"))?
+ .sync_all()?;
+ }
+ Ok(())
+ })
+ .await
+ }
+
+ /// Records the owner's delete marker `version` and removes the head.
+ /// Idempotent.
+ pub async fn apply_delete(&self, key: &str, version: &str) -> Result<()> {
+ let (me, key, version) = (self.clone(), key.to_string(), version.to_string());
+ blocking(move || {
+ let _lock = me.lock()?;
+ let marker = me.versions_dir(&key)?.join(format!("{version}.del"));
+ if !marker.is_file() {
+ Self::atomic_write(&marker, b"")?;
+ }
+ let head = me.head_path(&key)?;
+ match fs::remove_file(head) {
+ Ok(()) => Ok(()),
+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
+ Err(e) => Err(e.into()),
+ }
+ })
+ .await
+ }
+
+ /// Moves the head of `key` to `to` (a version this store still holds;
+ /// `None`: no head). Versions are never removed, so nothing is lost.
+ pub async fn set_head(&self, key: &str, to: Option<ObjectMeta>) -> Result<()> {
+ let (me, key) = (self.clone(), key.to_string());
+ blocking(move || {
+ let _lock = me.lock()?;
+ let head = me.head_path(&key)?;
+ match to {
+ Some(m) => {
+ if !me.version_path(&key, &m.version)?.is_file() {
+ return Err(Error::NotFound(format!("{key} version {}", m.version)));
+ }
+ Self::atomic_write(&head, &serde_json::to_vec(&m)?)
+ }
+ None => match fs::remove_file(head) {
+ Ok(()) => Ok(()),
+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
+ Err(e) => Err(e.into()),
+ },
+ }
+ })
+ .await
+ }
+
+ /// Removes one version's bytes (a finished spool entry; never used on
+ /// the data store).
+ pub async fn purge_version(&self, key: &str, version: &str) -> Result<()> {
+ let (me, key, version) = (self.clone(), key.to_string(), version.to_string());
+ blocking(move || {
+ let p = me.version_path(&key, &version)?;
+ match fs::remove_file(&p) {
+ Ok(()) => {}
+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
+ Err(e) => return Err(e.into()),
+ }
+ if let Ok(Some(h)) = me.read_head(&key)
+ && h.version == version
+ {
+ let _ = fs::remove_file(me.head_path(&key)?);
+ }
+ Ok(())
+ })
+ .await
+ }
+
+ /// Free bytes on the file system holding the store (best effort).
+ pub fn free_bytes(&self) -> Option<u64> {
+ #[cfg(unix)]
+ {
+ let probe = if self.root.exists() {
+ self.root.clone()
+ } else {
+ self.root.parent()?.to_path_buf()
+ };
+ let c = std::ffi::CString::new(probe.to_string_lossy().as_bytes()).ok()?;
+ // SAFETY: an all-zero statvfs is a valid value to pass as an out pointer.
+ let mut st: libc::statvfs = unsafe { std::mem::zeroed() };
+ // SAFETY: `c` is a valid C string and `st` a valid out pointer.
+ if unsafe { libc::statvfs(c.as_ptr(), &mut st) } == 0 {
+ return Some((st.f_bavail as u64).saturating_mul(st.f_frsize as u64));
+ }
+ None
+ }
+ #[cfg(not(unix))]
+ {
+ None
+ }
+ }
+}
+
#[cfg(unix)]
fn read_at(f: &File, buf: &mut [u8], offset: u64) -> std::io::Result<usize> {
std::os::unix::fs::FileExt::read_at(f, buf, offset)
@@ -601,8 +804,7 @@ impl Backend for FsBackend {
}
async fn delete(&self, key: &str, cond: Condition) -> Result<()> {
- let (me, key) = (self.clone(), key.to_string());
- blocking(move || me.delete_blocking(&key, cond)).await
+ self.delete_versioned(key, cond).await.map(|_| ())
}
async fn versions(&self, key: &str) -> Result<Vec<VersionInfo>> {
diff --git a/libs/cua/crates/cua-volume/src/lib.rs b/libs/cua/crates/cua-volume/src/lib.rs
index c37326690f..7d20b06bdc 100644
--- a/libs/cua/crates/cua-volume/src/lib.rs
+++ b/libs/cua/crates/cua-volume/src/lib.rs
@@ -60,6 +60,7 @@ pub mod lease;
pub mod nfs;
pub mod path;
pub mod remote;
+pub mod replica;
#[cfg(feature = "s3")]
pub mod s3;
pub mod scan;
diff --git a/libs/cua/crates/cua-volume/src/replica/apply.rs b/libs/cua/crates/cua-volume/src/replica/apply.rs
new file mode 100644
index 0000000000..9fa311a2c6
--- /dev/null
+++ b/libs/cua/crates/cua-volume/src/replica/apply.rs
@@ -0,0 +1,523 @@
+// SPDX-License-Identifier: FSL-1.1-MIT
+// Copyright (c) 2026 Cua AI, Inc.
+
+//! The replica side: apply the owner's records in order, catch up from a
+//! snapshot, and put right a copy that was on the wrong side of a takeover.
+
+use std::collections::HashSet;
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use super::era::{Eras, Pos};
+use super::link::Link;
+use super::migrate::{self, Unlogged};
+use super::node::{Inner, save_eras};
+use super::oplog::{Op, Record};
+use super::wire::{Cond, Hello, Msg, Pending, Rep, Req, Src};
+use crate::backend::{Backend, ObjectMeta};
+use crate::fs::ApplySource;
+use crate::{Error, Result};
+
+impl Inner {
+ /// Where the bytes of `meta` can come from without the network, if
+ /// anywhere on this machine.
+ async fn local_source(&self, meta: &ObjectMeta) -> Option<ApplySource> {
+ if meta.size == 0 {
+ return Some(ApplySource::Bytes(vec![]));
+ }
+ if let Ok(Some(h)) = self.store.head(&meta.key).await
+ && h.etag == meta.etag
+ && let Ok(path) = self.store.version_file(&meta.key, &h.version)
+ && path.is_file()
+ {
+ return Some(ApplySource::File { path, link: true });
+ }
+ let pending: Vec<Pending> = {
+ let st = self.st.lock().await;
+ st.outbox
+ .items()
+ .iter()
+ .filter(|p| {
+ p.meta
+ .as_ref()
+ .is_some_and(|m| m.etag == meta.etag && m.size == meta.size)
+ })
+ .cloned()
+ .collect()
+ };
+ for p in pending {
+ let path = match &p.src {
+ Src::Spool { key, version } => self.spool.version_file(key, version),
+ Src::Data { key, version } => self.store.version_file(key, version),
+ };
+ if let Ok(path) = path
+ && path.is_file()
+ {
+ return Some(ApplySource::File { path, link: true });
+ }
+ }
+ None
+ }
+
+ /// The bytes of `meta`, from this machine if it has them, else pulled
+ /// from `from_peer`.
+ pub(super) async fn acquire(
+ self: &Arc<Self>,
+ from_peer: &str,
+ meta: &ObjectMeta,
+ inline: Option<Vec<u8>>,
+ ) -> Result<ApplySource> {
+ if let Some(b) = inline {
+ return Ok(ApplySource::Bytes(b));
+ }
+ if let Some(s) = self.local_source(meta).await {
+ return Ok(s);
+ }
+ let dl = self
+ .data_link(from_peer)
+ .await
+ .ok_or_else(|| Error::Backend("no data link to the owner".into()))?;
+ let seed = self.store.head(&meta.key).await.ok().flatten();
+ let path = self
+ .pull(
+ &dl,
+ Src::Data {
+ key: meta.key.clone(),
+ version: meta.version.clone(),
+ },
+ meta,
+ seed.as_ref(),
+ )
+ .await?;
+ Ok(ApplySource::File { path, link: false })
+ }
+
+ pub(super) async fn handle_append(
+ self: &Arc<Self>,
+ link: &Arc<Link>,
+ epoch: u64,
+ records: Vec<super::wire::RecordW>,
+ data: Vec<u8>,
+ ) -> Msg {
+ let t0 = std::time::Instant::now();
+ let _g = self.gate.lock().await;
+ tracing::debug!(n = records.len(), gate_wait = ?t0.elapsed(), "replica: append received");
+ {
+ let mut st = self.st.lock().await;
+ if link.peer != st.eras.owner() || epoch < st.eras.epoch() {
+ st.fenced_writes += records.len() as u64;
+ return Msg::Fenced {
+ eras: st.eras.recs.clone(),
+ };
+ }
+ if epoch > st.eras.epoch() {
+ return Msg::Behind {
+ applied: st.log.tail(),
+ };
+ }
+ }
+ // Runs of small puts that came with their bytes are applied as one
+ // batch (one disk barrier, one run of log records).
+ let mut records = records.into_iter().peekable();
+ while let Some(rw) = records.next() {
+ if let (Op::Put { .. }, Some(_)) = (&rw.op, rw.body) {
+ let mut run = vec![rw];
+ while run.len() < 256
+ && records
+ .peek()
+ .is_some_and(|n| matches!(n.op, Op::Put { .. }) && n.body.is_some())
+ {
+ run.push(records.next().expect("peeked"));
+ }
+ let tail = self.st.lock().await.log.tail();
+ let mut expect = tail;
+ let mut items = vec![];
+ let mut recs = vec![];
+ let mut gap = false;
+ for r in run {
+ if r.pos <= tail {
+ continue;
+ }
+ let contiguous = if r.pos.epoch == expect.epoch {
+ r.pos.seq == expect.seq + 1
+ } else {
+ r.pos.epoch > expect.epoch && r.pos.seq == 1
+ };
+ if !contiguous {
+ gap = true;
+ break;
+ }
+ expect = r.pos;
+ let (o, l) = r.body.expect("a body");
+ let Some(bytes) = data.get(o as usize..(o + l) as usize) else {
+ gap = true;
+ break;
+ };
+ if let Op::Put { meta, .. } = &r.op {
+ items.push((meta.clone(), bytes.to_vec()));
+ }
+ recs.push(Record {
+ pos: r.pos,
+ op: r.op,
+ at_ms: r.at_ms,
+ });
+ }
+ if !items.is_empty() {
+ if let Err(e) = self.store.apply_many(items).await {
+ if e.to_string().contains("No space")
+ || e.to_string().contains("disk is full")
+ {
+ self.st.lock().await.healthy = false;
+ }
+ return Msg::Error {
+ kind: e.tag().into(),
+ message: e.to_string(),
+ };
+ }
+ let last = recs.last().map(|r| r.pos);
+ let mut st = self.st.lock().await;
+ st.healthy = true;
+ if let Err(e) = st.log.append_batch(recs) {
+ return Msg::Error {
+ kind: e.tag().into(),
+ message: e.to_string(),
+ };
+ }
+ if let Some(p) = last {
+ self.applied_tx.send_replace(p);
+ }
+ }
+ if gap {
+ let t = self.st.lock().await.log.tail();
+ return Msg::Behind { applied: t };
+ }
+ continue;
+ }
+ let tail = self.st.lock().await.log.tail();
+ if rw.pos <= tail {
+ tracing::debug!(me = %self.me, pos = %rw.pos, %tail, key = %rw.op.key(), "replica: skipped (not after tail)");
+ continue;
+ }
+ let contiguous = if rw.pos.epoch == tail.epoch {
+ rw.pos.seq == tail.seq + 1
+ } else {
+ rw.pos.epoch > tail.epoch && rw.pos.seq == 1
+ };
+ if !contiguous {
+ return Msg::Behind { applied: tail };
+ }
+ let inline = rw
+ .body
+ .and_then(|(o, l)| data.get(o as usize..(o + l) as usize).map(<[u8]>::to_vec));
+ let r: Result<()> = async {
+ match &rw.op {
+ Op::Put { meta, .. } => {
+ let src = self.acquire(&link.peer, meta, inline).await?;
+ self.store.apply_put(meta.clone(), src).await?;
+ }
+ Op::Delete { key, version, .. } => {
+ self.store.apply_delete(key, version).await?;
+ }
+ }
+ Ok(())
+ }
+ .await;
+ if let Err(e) = r {
+ let full =
+ e.to_string().contains("disk is full") || e.to_string().contains("No space");
+ if full {
+ self.st.lock().await.healthy = false;
+ }
+ return Msg::Error {
+ kind: e.tag().into(),
+ message: e.to_string(),
+ };
+ }
+ let mut st = self.st.lock().await;
+ st.healthy = true;
+ tracing::debug!(me = %self.me, pos = %rw.pos, key = %rw.op.key(), "replica: applied");
+ if let Err(e) = st.log.append(Record {
+ pos: rw.pos,
+ op: rw.op,
+ at_ms: rw.at_ms,
+ }) {
+ return Msg::Error {
+ kind: e.tag().into(),
+ message: e.to_string(),
+ };
+ }
+ self.applied_tx.send_replace(rw.pos);
+ }
+ let st = self.st.lock().await;
+ Msg::Ack {
+ applied: st.log.tail(),
+ }
+ }
+
+ pub(super) async fn handle_snapshot(self: &Arc<Self>, link: &Arc<Link>, at: Pos) -> Msg {
+ let _g = self.gate.lock().await;
+ match self.pull_snapshot(link, at).await {
+ Ok(()) => Msg::Ack { applied: at },
+ Err(e) => Msg::Error {
+ kind: e.tag().into(),
+ message: e.to_string(),
+ },
+ }
+ }
+
+ async fn pull_snapshot(self: &Arc<Self>, link: &Arc<Link>, at: Pos) -> Result<()> {
+ let peer = link.peer.clone();
+ let dl = self
+ .data_link(&peer)
+ .await
+ .ok_or_else(|| Error::Backend("no data link to the owner".into()))?;
+ let mut seen: HashSet<String> = HashSet::new();
+ let mut after = String::new();
+ loop {
+ let (m, _) = dl
+ .call(
+ Msg::Req(Req::Heads {
+ after_key: after.clone(),
+ max: 512,
+ }),
+ &[],
+ self.cfg.data_timeout,
+ )
+ .await
+ .map_err(|e| Error::Backend(e.to_string()))?;
+ let (items, done) = match m {
+ Msg::Rep(Rep::Heads { items, done, .. }) => (items, done),
+ Msg::Rep(Rep::Fail { kind, message }) => {
+ return Err(Rep::into_error(&kind, message));
+ }
+ _ => return Err(Error::Backend("unexpected reply".into())),
+ };
+ for meta in &items {
+ seen.insert(meta.key.clone());
+ if let Some(local) = self.store.head(&meta.key).await?
+ && local.version == meta.version
+ {
+ continue;
+ }
+ if self.st.lock().await.outbox.has_key(&meta.key) {
+ continue;
+ }
+ let src = self.acquire(&peer, meta, None).await?;
+ self.store.apply_put(meta.clone(), src).await?;
+ }
+ if let Some(last) = items.last() {
+ after = last.key.clone();
+ }
+ if done || items.is_empty() {
+ break;
+ }
+ }
+ // Heads that are not the owner's and not waiting to be sent are stale.
+ for local in self.store.list("").await? {
+ if !seen.contains(&local.key) && !self.st.lock().await.outbox.has_key(&local.key) {
+ self.store.set_head(&local.key, None).await?;
+ }
+ }
+ let mut st = self.st.lock().await;
+ st.log.reset_to(at)?;
+ drop(st);
+ self.applied_tx.send_replace(at);
+ Ok(())
+ }
+
+ /// This machine holds records past a later era's base (it was the owner
+ /// while another took over, or it was ahead): move those heads back,
+ /// keep every version, and queue the changes for the new owner.
+ pub(super) async fn reconcile_eras(self: &Arc<Self>) {
+ let _g = self.gate.lock().await;
+ if let Err(e) = self.reconcile_locked().await {
+ tracing::warn!(error = %e, "replica: could not hand over this machine's divergent changes yet");
+ }
+ }
+
+ async fn reconcile_locked(self: &Arc<Self>) -> Result<()> {
+ let tail_records = {
+ let mut st = self.st.lock().await;
+ let Some(keep) = st.eras.divergence(st.log.tail()) else {
+ return Ok(());
+ };
+ tracing::info!(%keep, "replica: this machine was ahead of the new owner; handing over its tail");
+ st.log.cut_after(keep)?
+ };
+ tracing::debug!(n = tail_records.len(), first = ?tail_records.last().map(|r| r.op.key().to_string()), "replica: tail cut");
+ // Newest first: rewind each head to what it was before.
+ for rec in &tail_records {
+ match &rec.op {
+ Op::Put { meta, prev } => {
+ if self
+ .store
+ .head(&meta.key)
+ .await?
+ .is_some_and(|h| h.version == meta.version)
+ {
+ self.store.set_head(&meta.key, prev.clone()).await?;
+ }
+ }
+ Op::Delete { key, prev, .. } => {
+ if self.store.head(key).await?.is_none() {
+ self.store.set_head(key, prev.clone()).await?;
+ }
+ }
+ }
+ }
+ // Oldest first: they go to the new owner as changes against what
+ // each key was before.
+ let mut st = self.st.lock().await;
+ for rec in tail_records.iter().rev() {
+ let (key, meta, prev) = match &rec.op {
+ Op::Put { meta, prev } => (meta.key.clone(), Some(meta.clone()), prev.clone()),
+ Op::Delete { key, prev, .. } => (key.clone(), None, prev.clone()),
+ };
+ let src = Src::Data {
+ key: key.clone(),
+ version: meta.as_ref().map(|m| m.version.clone()).unwrap_or_default(),
+ };
+ st.outbox.add(Pending {
+ id: crate::new_id(),
+ key,
+ meta,
+ base: prev.map(|p| p.version),
+ cond: Cond::None,
+ machine: self.me.clone(),
+ name: self.name.clone(),
+ ts_ms: self.wall_ms(),
+ src,
+ body: None,
+ })?;
+ }
+ let tail = st.log.tail();
+ for p in st.peers.values_mut() {
+ if p.acked > tail {
+ p.acked = tail;
+ }
+ }
+ Ok(())
+ }
+
+ /// Two volumes that started apart met; this machine is the less
+ /// reliable one and joins the other's: everything it has goes to the new
+ /// owner as changes against nothing (identical files merge, different
+ /// ones become conflict copies).
+ pub(super) async fn join_lineage(self: &Arc<Self>, h: Hello) {
+ let _g = self.gate.lock().await;
+ let mut st = self.st.lock().await;
+ if st.eras.lineage == h.lineage {
+ return;
+ }
+ tracing::info!(other = %h.machine, "replica: joining another machine's volume");
+ let heads = match self.store.list("").await {
+ Ok(h) => h,
+ Err(_) => return,
+ };
+ for meta in heads {
+ if st.outbox.has_key(&meta.key) {
+ continue;
+ }
+ let _ = st.outbox.add(Pending {
+ id: crate::new_id(),
+ key: meta.key.clone(),
+ meta: Some(meta.clone()),
+ base: None,
+ cond: Cond::None,
+ machine: self.me.clone(),
+ name: self.name.clone(),
+ ts_ms: self.wall_ms(),
+ src: Src::Data {
+ key: meta.key.clone(),
+ version: meta.version.clone(),
+ },
+ body: None,
+ });
+ }
+ st.eras = Eras {
+ lineage: h.lineage.clone(),
+ recs: h.eras.clone(),
+ };
+ let _ = save_eras(&self.dir, &st.eras);
+ let _ = st.log.reset_to(Pos::ZERO);
+ self.applied_tx.send_replace(Pos::ZERO);
+ for p in st.peers.values_mut() {
+ p.acked = Pos::ZERO;
+ }
+ }
+
+ #[allow(dead_code)]
+ pub(super) fn incoming_dir(&self) -> PathBuf {
+ self.dir.join("incoming")
+ }
+}
+
+impl Inner {
+ /// Changes made to the store while replication was not looking (an older
+ /// build on the same home) go into the log (this machine owns the
+ /// Volume) or to the owner as changes (it does not).
+ pub(super) async fn adopt_unlogged(&self) -> Result<()> {
+ let unlogged = {
+ let st = self.st.lock().await;
+ migrate::find_unlogged(&self.dir, &self.store, &st.log).await?
+ };
+ if unlogged.is_empty() {
+ return Ok(());
+ }
+ tracing::info!(
+ n = unlogged.len(),
+ "replica: found changes made while replication was off; handing them over"
+ );
+ let mut st = self.st.lock().await;
+ let owner = st.eras.owner() == self.me;
+ for u in unlogged {
+ if owner {
+ let op = match u {
+ Unlogged::Put { meta, prev } => super::oplog::Op::Put { meta, prev },
+ Unlogged::Delete { key, version, prev } => {
+ super::oplog::Op::Delete { key, version, prev }
+ }
+ };
+ self.commit_locked(&mut st, op)?;
+ } else {
+ let (key, meta, base, src) = match u {
+ Unlogged::Put { meta, prev } => {
+ let src = Src::Data {
+ key: meta.key.clone(),
+ version: meta.version.clone(),
+ };
+ (meta.key.clone(), Some(meta), prev.map(|p| p.version), src)
+ }
+ Unlogged::Delete { key, prev, .. } => {
+ let src = Src::Data {
+ key: key.clone(),
+ version: String::new(),
+ };
+ (key, None, prev.map(|p| p.version), src)
+ }
+ };
+ st.outbox.add(Pending {
+ id: crate::new_id(),
+ key,
+ meta,
+ base,
+ cond: Cond::None,
+ machine: self.me.clone(),
+ name: self.name.clone(),
+ ts_ms: self.wall_ms(),
+ src,
+ body: None,
+ })?;
+ }
+ }
+ Ok(())
+ }
+
+ /// Records what the heads are now (see [`migrate`]).
+ pub(super) async fn save_snapshot(&self) {
+ let pos = self.st.lock().await.log.tail();
+ if let Err(e) = migrate::save_snap(&self.dir, &self.store, pos).await {
+ tracing::warn!(error = %e, "replica: could not save the heads snapshot");
+ }
+ }
+}
diff --git a/libs/cua/crates/cua-volume/src/replica/backend.rs b/libs/cua/crates/cua-volume/src/replica/backend.rs
new file mode 100644
index 0000000000..b4caae8141
--- /dev/null
+++ b/libs/cua/crates/cua-volume/src/replica/backend.rs
@@ -0,0 +1,252 @@
+// SPDX-License-Identifier: FSL-1.1-MIT
+// Copyright (c) 2026 Cua AI, Inc.
+
+//! The `Backend` the drive sees: one volume across your machines.
+//!
+//! Machines that hold a copy answer reads from it; a machine that does not
+//! (a client) asks one that does, and the block cache sits in front of it.
+//! Writes go to the owner, which orders them, logs them and ships them;
+//! unconditional writes made while the owner is away are saved and sent
+//! when it is back.
+
+use std::path::Path;
+use std::sync::Arc;
+use std::sync::atomic::Ordering;
+
+use super::node::Inner;
+use super::owner::Body;
+use super::roles::Role;
+use super::status::Durability;
+use super::wire::{Msg, Rep, Req};
+use crate::backend::{Backend, Condition, ObjectMeta, VersionInfo};
+use crate::{Error, Result};
+
+pub struct ReplicatedBackend {
+ inner: Arc<Inner>,
+}
+
+impl ReplicatedBackend {
+ pub(super) fn new(inner: Arc<Inner>) -> ReplicatedBackend {
+ ReplicatedBackend { inner }
+ }
+
+ /// Writes and says how many machines hold the result.
+ pub async fn put_durable(
+ &self,
+ key: &str,
+ bytes: Vec<u8>,
+ cond: Condition,
+ ) -> Result<(ObjectMeta, Durability)> {
+ self.inner.write_put(key, Body::Bytes(bytes), cond).await
+ }
+
+ /// The machines to ask for a read, best first: the owner, then the
+ /// other machines that hold a copy.
+ async fn read_targets(&self) -> Vec<String> {
+ let st = self.inner.st.lock().await;
+ let layout = self.inner.layout_locked(&st);
+ let owner = st.eras.owner().to_string();
+ let mut v = vec![];
+ if owner != self.inner.me {
+ v.push(owner.clone());
+ }
+ for r in layout.replicas {
+ if r != self.inner.me && r != owner {
+ v.push(r);
+ }
+ }
+ v
+ }
+
+ async fn ask(&self, req: Req, big: bool) -> Result<(Rep, Vec<u8>)> {
+ let targets = self.read_targets().await;
+ let mut last = Error::Backend("no machine that holds the volume can be reached".into());
+ for t in targets {
+ let link = if big {
+ self.inner.data_link(&t).await
+ } else {
+ self.inner.ctl_link(&t)
+ };
+ let Some(link) = link else { continue };
+ let to = if big {
+ self.inner.cfg.data_timeout
+ } else {
+ self.inner.cfg.call_timeout
+ };
+ let r = if big {
+ link.call_strict(Msg::Req(req.clone()), &[], to).await
+ } else {
+ link.call(Msg::Req(req.clone()), &[], to).await
+ };
+ match r {
+ Ok((Msg::Rep(Rep::Fail { kind, message }), _)) => {
+ return Err(Rep::into_error(&kind, message));
+ }
+ Ok((Msg::Rep(r), d)) => return Ok((r, d)),
+ Ok(_) => last = Error::Backend("unexpected reply".into()),
+ Err(e) => last = Error::Backend(e.to_string()),
+ }
+ }
+ Err(last)
+ }
+
+ fn holds_copy(&self) -> bool {
+ !self.inner.is_client.load(Ordering::SeqCst)
+ }
+}
+
+#[async_trait::async_trait]
+impl Backend for ReplicatedBackend {
+ fn kind(&self) -> &'static str {
+ "mesh"
+ }
+
+ async fn put(&self, key: &str, bytes: Vec<u8>, cond: Condition) -> Result<ObjectMeta> {
+ Ok(self.inner.write_put(key, Body::Bytes(bytes), cond).await?.0)
+ }
+
+ async fn put_file(&self, key: &str, file: &Path, cond: Condition) -> Result<ObjectMeta> {
+ Ok(self
+ .inner
+ .write_put(key, Body::File(file.to_path_buf()), cond)
+ .await?
+ .0)
+ }
+
+ async fn put_many(&self, items: Vec<(String, Vec<u8>)>) -> Result<Vec<ObjectMeta>> {
+ self.inner.write_many(items).await
+ }
+
+ async fn delete(&self, key: &str, cond: Condition) -> Result<()> {
+ self.inner.write_delete(key, cond).await.map(|_| ())
+ }
+
+ async fn get(&self, key: &str, version: Option<&str>) -> Result<(Vec<u8>, ObjectMeta)> {
+ if self.holds_copy() {
+ match self.inner.store.get(key, version).await {
+ Err(Error::NotFound(_)) if version.is_some() => {}
+ r => return r,
+ }
+ }
+ match self
+ .ask(
+ Req::Get {
+ key: key.to_string(),
+ version: version.map(str::to_string),
+ },
+ true,
+ )
+ .await?
+ {
+ (Rep::Bytes { meta }, bytes) => Ok((bytes, meta)),
+ _ => Err(Error::Backend("unexpected reply".into())),
+ }
+ }
+
+ async fn head(&self, key: &str) -> Result<Option<ObjectMeta>> {
+ if self.holds_copy() {
+ return self.inner.store.head(key).await;
+ }
+ match self
+ .ask(
+ Req::Head {
+ key: key.to_string(),
+ },
+ false,
+ )
+ .await?
+ {
+ (Rep::Head { meta }, _) => Ok(meta),
+ _ => Err(Error::Backend("unexpected reply".into())),
+ }
+ }
+
+ async fn list(&self, prefix: &str) -> Result<Vec<ObjectMeta>> {
+ if self.holds_copy() {
+ return self.inner.store.list(prefix).await;
+ }
+ match self
+ .ask(
+ Req::List {
+ prefix: prefix.to_string(),
+ },
+ true,
+ )
+ .await?
+ {
+ (Rep::List { items }, _) => Ok(items),
+ _ => Err(Error::Backend("unexpected reply".into())),
+ }
+ }
+
+ async fn list_dir(&self, folder: &str) -> Result<(Vec<ObjectMeta>, Vec<String>)> {
+ if self.holds_copy() {
+ return self.inner.store.list_dir(folder).await;
+ }
+ match self
+ .ask(
+ Req::ListDir {
+ folder: folder.to_string(),
+ },
+ false,
+ )
+ .await?
+ {
+ (Rep::ListDir { files, folders }, _) => Ok((files, folders)),
+ _ => Err(Error::Backend("unexpected reply".into())),
+ }
+ }
+
+ async fn versions(&self, key: &str) -> Result<Vec<VersionInfo>> {
+ if self.holds_copy() {
+ return self.inner.store.versions(key).await;
+ }
+ match self
+ .ask(
+ Req::Versions {
+ key: key.to_string(),
+ },
+ false,
+ )
+ .await?
+ {
+ (Rep::Versions { items }, _) => Ok(items),
+ _ => Err(Error::Backend("unexpected reply".into())),
+ }
+ }
+
+ async fn get_range(&self, key: &str, version: &str, offset: u64, len: u64) -> Result<Vec<u8>> {
+ if self.holds_copy() {
+ match self.inner.store.get_range(key, version, offset, len).await {
+ Err(Error::NotFound(_)) => {}
+ r => return r,
+ }
+ }
+ match self
+ .ask(
+ Req::GetRange {
+ key: key.to_string(),
+ version: version.to_string(),
+ offset,
+ len,
+ },
+ true,
+ )
+ .await?
+ {
+ (Rep::Range, bytes) => Ok(bytes),
+ _ => Err(Error::Backend("unexpected reply".into())),
+ }
+ }
+
+ fn remote(&self) -> bool {
+ !self.holds_copy()
+ }
+
+ fn identity(&self) -> String {
+ format!("mesh:{}", self.inner.store.root().display())
+ }
+}
+
+#[allow(dead_code)]
+fn _role(_: Role) {}
diff --git a/libs/cua/crates/cua-volume/src/replica/detector.rs b/libs/cua/crates/cua-volume/src/replica/detector.rs
new file mode 100644
index 0000000000..149dcf0eef
--- /dev/null
+++ b/libs/cua/crates/cua-volume/src/replica/detector.rs
@@ -0,0 +1,324 @@
+// SPDX-License-Identifier: FSL-1.1-MIT
+// Copyright (c) 2026 Cua AI, Inc.
+
+//! Is that machine still there? One detector per peer, fed by heartbeat
+//! arrivals on the control link.
+//!
+//! The rule is built so a bad network never causes a takeover and a closed
+//! laptop is noticed quickly:
+//!
+//! - *Adaptive threshold.* The silence that makes a peer suspect is
+//! `max(floor, mean + 8 * deviation)` of the heartbeat gaps seen so far
+//! (an exponentially weighted mean, like TCP's retransmit timer). Jitter
+//! and loss widen the gaps and so widen the threshold.
+//! - *Confirmation.* A suspect peer is declared dead only after the silence
+//! has lasted `confirm` more. Any heartbeat before that clears the
+//! suspicion.
+//! - *Recovery needs a run.* A dead peer is alive again after `recover`
+//! heartbeats in a row, each within the threshold of the one before, so a
+//! link that flaps does not bounce ownership back and forth.
+//! - *A goodbye is believed at once.* A machine going to sleep says so
+//! (`Bye`); it counts as away immediately, without waiting.
+//! - *Our own pauses are not theirs.* If this process was frozen (our lid
+//! was closed, a SIGSTOP), every peer looks silent. [`Detector::resume`]
+//! forgives a gap in our own clock so a waking laptop never declares the
+//! studio dead because it slept.
+
+use std::time::{Duration, Instant};
+
+/// Timing knobs (all derived from the heartbeat interval by default).
+#[derive(Clone, Copy, Debug)]
+pub struct Timing {
+ /// Heartbeat interval.
+ pub interval: Duration,
+ /// The least silence that raises suspicion.
+ pub floor: Duration,
+ /// How long suspicion must last before dead.
+ pub confirm: Duration,
+ /// Heartbeats in a row that revive a dead peer.
+ pub recover: u32,
+}
+
+impl Timing {
+ /// 2 s heartbeats, suspect after 6 s, dead 12 s later, 3 in a row to
+ /// come back.
+ pub fn standard() -> Timing {
+ Timing {
+ interval: Duration::from_secs(2),
+ floor: Duration::from_secs(6),
+ confirm: Duration::from_secs(12),
+ recover: 3,
+ }
+ }
+}
+
+/// What the detector believes.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum Health {
+ /// Heartbeats arrive.
+ Alive,
+ /// Silent longer than the threshold; not yet declared dead.
+ Suspect,
+ /// Silent through the confirmation window.
+ Dead,
+ /// Said goodbye (asleep, shutting down, network gone).
+ Away,
+}
+
+impl Health {
+ /// Dead or away: nothing to wait for.
+ pub fn is_down(self) -> bool {
+ matches!(self, Health::Dead | Health::Away)
+ }
+}
+
+/// One peer's detector.
+#[derive(Clone, Debug)]
+pub struct Detector {
+ t: Timing,
+ last: Option<Instant>,
+ mean_ms: f64,
+ dev_ms: f64,
+ state: Health,
+ /// Down on the machine's own word (goodbye) or on refusals, not on silence.
+ explicit: bool,
+ suspect_since: Option<Instant>,
+ run: u32,
+ /// How many times it went from alive to dead (a flap counter for the
+ /// reliability score).
+ pub downs: u32,
+}
+
+impl Detector {
+ pub fn new(t: Timing, now: Instant) -> Detector {
+ Detector {
+ t,
+ last: Some(now),
+ mean_ms: t.interval.as_millis() as f64,
+ dev_ms: 0.0,
+ state: Health::Alive,
+ explicit: false,
+ suspect_since: None,
+ run: 0,
+ downs: 0,
+ }
+ }
+
+ /// The silence that makes this peer suspect now.
+ pub fn threshold(&self) -> Duration {
+ let adaptive = self.mean_ms + 8.0 * self.dev_ms;
+ Duration::from_millis(adaptive as u64).max(self.t.floor)
+ }
+
+ /// A heartbeat (or any control message) arrived.
+ pub fn arrival(&mut self, now: Instant) {
+ if let Some(last) = self.last {
+ let gap = now.saturating_duration_since(last).as_millis() as f64;
+ // A gap far past the threshold is an outage, not a sample.
+ if gap < 6.0 * self.threshold().as_millis() as f64 {
+ let err = gap - self.mean_ms;
+ self.mean_ms += 0.125 * err;
+ self.dev_ms += 0.25 * (err.abs() - self.dev_ms);
+ }
+ let within = now.saturating_duration_since(last) <= self.threshold();
+ match self.state {
+ Health::Dead | Health::Away => {
+ self.run = if within || self.run == 0 {
+ self.run + 1
+ } else {
+ 1
+ };
+ if self.run >= self.t.recover {
+ self.state = Health::Alive;
+ self.explicit = false;
+ self.suspect_since = None;
+ self.run = 0;
+ }
+ }
+ Health::Suspect => {
+ self.state = Health::Alive;
+ self.suspect_since = None;
+ }
+ Health::Alive => {}
+ }
+ }
+ self.last = Some(now);
+ }
+
+ /// The peer said goodbye.
+ pub fn bye(&mut self, now: Instant) {
+ if !self.state.is_down() {
+ self.downs += 1;
+ }
+ self.state = Health::Away;
+ self.explicit = true;
+ self.run = 0;
+ self.last = Some(now);
+ }
+
+ /// The machine answered a dial with a refusal, or accepted and hung up
+ /// before saying hello, several times in a row: someone is there to say
+ /// no, and nobody is running the node. That is far better evidence than
+ /// silence, so it skips the confirmation window (never the first
+ /// refusal alone: the caller counts).
+ pub fn refused(&mut self, now: Instant) {
+ if !self.state.is_down() {
+ self.downs += 1;
+ }
+ self.state = Health::Dead;
+ self.explicit = true;
+ self.run = 0;
+ self.last = Some(now);
+ }
+
+ /// This process was paused: forgive the gap.
+ pub fn resume(&mut self, now: Instant) {
+ if matches!(self.state, Health::Alive | Health::Suspect) {
+ self.state = Health::Alive;
+ self.suspect_since = None;
+ }
+ self.last = Some(now);
+ }
+
+ /// What it believes at `now`.
+ pub fn poll(&mut self, now: Instant) -> Health {
+ let Some(last) = self.last else {
+ return self.state;
+ };
+ let silence = now.saturating_duration_since(last);
+ match self.state {
+ Health::Alive if silence > self.threshold() => {
+ self.state = Health::Suspect;
+ self.suspect_since = Some(now);
+ }
+ Health::Suspect => {
+ if let Some(since) = self.suspect_since
+ && now.saturating_duration_since(since) >= self.t.confirm
+ && silence > self.threshold()
+ {
+ self.state = Health::Dead;
+ self.explicit = false;
+ self.downs += 1;
+ self.run = 0;
+ }
+ }
+ _ => {}
+ }
+ self.state
+ }
+
+ /// How long since the peer was last heard from.
+ pub fn silence(&self, now: Instant) -> Duration {
+ self.last
+ .map_or(Duration::MAX, |l| now.saturating_duration_since(l))
+ }
+
+ /// Down by the detector's verdict *and* still silent: a machine that
+ /// has just started talking again is not down, whatever the verdict
+ /// says until it has been heard from a few times (a link that comes
+ /// back must not start a takeover on the strength of an old verdict).
+ pub fn down_and_silent(&self, now: Instant) -> bool {
+ self.state.is_down()
+ && (self.explicit || self.silence(now) >= self.threshold().min(Duration::from_secs(30)))
+ }
+
+ pub fn state(&self) -> Health {
+ self.state
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn t() -> Timing {
+ Timing::standard()
+ }
+
+ fn at(base: Instant, ms: u64) -> Instant {
+ base + Duration::from_millis(ms)
+ }
+
+ #[test]
+ fn silence_becomes_suspect_then_dead_after_confirmation() {
+ let b = Instant::now();
+ let mut d = Detector::new(t(), b);
+ assert_eq!(d.poll(at(b, 5_000)), Health::Alive);
+ assert_eq!(d.poll(at(b, 7_000)), Health::Suspect);
+ assert_eq!(d.poll(at(b, 18_000)), Health::Suspect, "confirm window");
+ assert_eq!(d.poll(at(b, 19_100)), Health::Dead);
+ assert_eq!(d.downs, 1);
+ }
+
+ #[test]
+ fn a_late_heartbeat_clears_suspicion() {
+ let b = Instant::now();
+ let mut d = Detector::new(t(), b);
+ assert_eq!(d.poll(at(b, 7_000)), Health::Suspect);
+ d.arrival(at(b, 10_000));
+ assert_eq!(d.poll(at(b, 10_100)), Health::Alive);
+ assert_eq!(d.downs, 0);
+ }
+
+ #[test]
+ fn jitter_widens_the_threshold_so_no_false_death() {
+ // Heartbeats whose gaps swing between 1 and 9 seconds: a fixed 6 s
+ // rule would suspect constantly; the adaptive rule must never
+ // declare dead.
+ let b = Instant::now();
+ let mut d = Detector::new(t(), b);
+ let mut now = 0u64;
+ let gaps = [1_000, 9_000, 2_000, 8_000, 1_500, 9_500, 3_000, 7_000];
+ for round in 0..40 {
+ now += gaps[round % gaps.len()];
+ // Poll just before each arrival (the worst moment).
+ let h = d.poll(at(b, now - 1));
+ assert_ne!(h, Health::Dead, "round {round}");
+ d.arrival(at(b, now));
+ }
+ assert_eq!(d.downs, 0);
+ assert!(d.threshold() > Duration::from_secs(6));
+ }
+
+ #[test]
+ fn a_dead_peer_needs_a_run_of_heartbeats_to_return() {
+ let b = Instant::now();
+ let mut d = Detector::new(t(), b);
+ d.poll(at(b, 7_000));
+ d.poll(at(b, 19_500));
+ assert_eq!(d.state(), Health::Dead);
+ d.arrival(at(b, 30_000));
+ assert_eq!(d.state(), Health::Dead);
+ d.arrival(at(b, 32_000));
+ assert_eq!(d.state(), Health::Dead);
+ d.arrival(at(b, 34_000));
+ assert_eq!(d.state(), Health::Alive);
+ }
+
+ #[test]
+ fn repeated_refusals_skip_the_confirmation_window() {
+ let b = Instant::now();
+ let mut d = Detector::new(t(), b);
+ d.refused(at(b, 1_000));
+ assert_eq!(d.poll(at(b, 1_001)), Health::Dead);
+ assert_eq!(d.downs, 1);
+ // It comes back by the usual run of heartbeats.
+ d.arrival(at(b, 3_000));
+ d.arrival(at(b, 5_000));
+ d.arrival(at(b, 7_000));
+ assert_eq!(d.state(), Health::Alive);
+ }
+
+ #[test]
+ fn a_goodbye_counts_at_once_and_resume_forgives_our_own_sleep() {
+ let b = Instant::now();
+ let mut d = Detector::new(t(), b);
+ d.bye(at(b, 100));
+ assert_eq!(d.poll(at(b, 101)), Health::Away);
+ assert!(d.state().is_down());
+ // We slept for an hour: nothing is suspect after waking.
+ let mut e = Detector::new(t(), b);
+ e.resume(at(b, 3_600_000));
+ assert_eq!(e.poll(at(b, 3_600_100)), Health::Alive);
+ }
+}
diff --git a/libs/cua/crates/cua-volume/src/replica/election.rs b/libs/cua/crates/cua-volume/src/replica/election.rs
new file mode 100644
index 0000000000..05a7d92962
--- /dev/null
+++ b/libs/cua/crates/cua-volume/src/replica/election.rs
@@ -0,0 +1,329 @@
+// SPDX-License-Identifier: FSL-1.1-MIT
+// Copyright (c) 2026 Cua AI, Inc.
+
+//! Who owns the volume: takeover when the owner is gone, hand-back when a
+//! more reliable machine returns.
+
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+use super::detector::Health;
+use super::era::{EraRec, Pos};
+use super::node::{Inner, save_eras};
+use super::roles::{self, Role};
+use super::wire::Msg;
+
+impl Inner {
+ /// A machine holding a copy takes the volume when its owner is down.
+ pub(super) async fn maybe_takeover(self: &Arc<Self>) {
+ struct Plan {
+ epoch: u64,
+ base: Pos,
+ need: usize,
+ owner: String,
+ }
+ let plan = {
+ let mut st = self.st.lock().await;
+ let owner = st.eras.owner().to_string();
+ if owner == self.me {
+ return;
+ }
+ if let Some(t) = st.last_takeover_try
+ && t.elapsed() < self.cfg.takeover_cooldown
+ {
+ return;
+ }
+ let layout = self.layout_locked(&st);
+ // A machine that keeps no copy never takes the volume.
+ if layout.role_of(&self.me) != Role::Replica {
+ return;
+ }
+ let Some(op) = st.peers.get(&owner) else {
+ return;
+ };
+ if !op.last_health.is_down() || !op.det.down_and_silent(Instant::now()) {
+ return;
+ }
+ // A machine that does not need to order writes leaves a flaky
+ // owner alone (its changes wait in the outbox); one that does,
+ // or one whose owner announced it was leaving, or has been gone
+ // a long time, takes over.
+ let wanted = st.authority_wanted.is_some_and(|t| {
+ t.elapsed() < self.cfg.timing.confirm * 8 + Duration::from_secs(5)
+ });
+ let gone_long = op
+ .down_since
+ .is_some_and(|t| t.elapsed() >= self.cfg.idle_takeover);
+ if !(wanted || op.intentional || gone_long) {
+ return;
+ }
+ // The best replica that is up takes it; the others wait.
+ let my = self.my_facts();
+ for r in &layout.replicas {
+ if *r == self.me || *r == owner {
+ continue;
+ }
+ let alive = st
+ .peers
+ .get(r)
+ .is_some_and(|p| p.last_health == Health::Alive);
+ if !alive {
+ continue;
+ }
+ let theirs = self.member_facts(&st, r);
+ if (theirs.rank, std::cmp::Reverse(r)) > (my.rank, std::cmp::Reverse(&self.me)) {
+ return;
+ }
+ }
+ let n = self.active_members(&st).len();
+ let need = if n >= 3 { n / 2 + 1 } else { 1 };
+ st.last_takeover_try = Some(Instant::now());
+ Plan {
+ epoch: st.eras.epoch() + 1,
+ base: st.log.tail(),
+ need,
+ owner,
+ }
+ };
+ if plan.need > 1 {
+ let peers: Vec<_> = {
+ let st = self.st.lock().await;
+ st.peers
+ .iter()
+ .filter(|(k, p)| **k != plan.owner && p.last_health == Health::Alive)
+ .map(|(k, _)| k.clone())
+ .collect()
+ };
+ let mut set = tokio::task::JoinSet::new();
+ for p in peers {
+ if let Some(l) = self.ctl_link(&p) {
+ let (epoch, base) = (plan.epoch, plan.base);
+ let to = self.cfg.call_timeout.min(Duration::from_secs(5));
+ set.spawn(async move {
+ matches!(
+ l.call(Msg::Prepare { epoch, base }, &[], to).await,
+ Ok((Msg::Promise { ok: true, .. }, _))
+ )
+ });
+ }
+ }
+ let mut yes = 1;
+ while let Some(r) = set.join_next().await {
+ if r.unwrap_or(false) {
+ yes += 1;
+ }
+ }
+ if yes < plan.need {
+ tracing::info!(
+ yes,
+ need = plan.need,
+ "replica: not enough machines agree to take over"
+ );
+ return;
+ }
+ }
+ let recs = {
+ let mut st = self.st.lock().await;
+ if st.eras.epoch() >= plan.epoch || st.eras.owner() != plan.owner {
+ return;
+ }
+ let base = st.log.tail();
+ st.eras.adopt(EraRec {
+ epoch: plan.epoch,
+ owner: self.me.clone(),
+ base,
+ at_ms: self.wall_ms(),
+ });
+ let _ = save_eras(&self.dir, &st.eras);
+ st.takeovers += 1;
+ tracing::info!(
+ epoch = plan.epoch,
+ "replica: this machine took over the volume"
+ );
+ st.eras
+ .recs
+ .iter()
+ .rev()
+ .take(4)
+ .rev()
+ .cloned()
+ .collect::<Vec<_>>()
+ };
+ self.announce(recs).await;
+ self.wake.notify_one();
+ }
+
+ pub(super) async fn announce(self: &Arc<Self>, recs: Vec<EraRec>) {
+ let links: Vec<_> = {
+ let l = self.links.lock().expect("links");
+ l.values().filter_map(|p| p.ctl.clone()).collect()
+ };
+ for l in links {
+ let recs = recs.clone();
+ let to = self.cfg.call_timeout.min(Duration::from_secs(5));
+ tokio::spawn(async move {
+ let _ = l.call(Msg::Era { recs }, &[], to).await;
+ });
+ }
+ }
+
+ pub(super) async fn handle_prepare(&self, from: &str, epoch: u64, _base: Pos) -> Msg {
+ let mut st = self.st.lock().await;
+ let no = |reason: &str| Msg::Promise {
+ ok: false,
+ reason: reason.into(),
+ };
+ if epoch <= st.eras.epoch() {
+ return no("that epoch has passed");
+ }
+ let owner = st.eras.owner().to_string();
+ if owner == from {
+ return no("the candidate is the owner");
+ }
+ if owner != self.me
+ && !st
+ .peers
+ .get(&owner)
+ .is_some_and(|p| p.last_health.is_down() && p.det.down_and_silent(Instant::now()))
+ {
+ return no("the owner is up from here");
+ }
+ let now = Instant::now();
+ for (k, p) in &st.peers {
+ if k != from
+ && let Some((e, until)) = p.promised
+ && e >= epoch
+ && until > now
+ {
+ return no("already promised another machine");
+ }
+ }
+ let until = now + self.cfg.timing.confirm * 4;
+ self.peer_state(&mut st, from).promised = Some((epoch, until));
+ Msg::Promise {
+ ok: true,
+ reason: String::new(),
+ }
+ }
+
+ pub(super) async fn handle_era(self: &Arc<Self>, _from: &str, recs: Vec<EraRec>) {
+ let advanced = {
+ let mut st = self.st.lock().await;
+ let a = st.eras.adopt_from(&recs);
+ if a {
+ let _ = save_eras(&self.dir, &st.eras);
+ self.refresh_roles(&mut st);
+ }
+ a
+ };
+ if advanced {
+ let me = self.clone();
+ tokio::spawn(async move { me.reconcile_eras().await });
+ self.wake.notify_one();
+ }
+ }
+
+ /// The owner gives the volume back to a clearly more reliable machine
+ /// that has been up and caught up for a while.
+ pub(super) async fn maybe_handoff(self: &Arc<Self>) {
+ let mut st = self.st.lock().await;
+ if st.eras.owner() != self.me {
+ return;
+ }
+ if let Some(t) = st.last_takeover_try
+ && t.elapsed() < self.cfg.takeover_cooldown
+ {
+ return;
+ }
+ let layout = self.layout_locked(&st);
+ let target = layout.owner.clone();
+ if target == self.me || target.is_empty() {
+ return;
+ }
+ let mine = self.my_facts();
+ let Some(p) = st.peers.get(&target) else {
+ return;
+ };
+ let stable = p
+ .alive_since
+ .is_some_and(|t| t.elapsed() >= self.cfg.handback_after);
+ let tail = st.log.tail();
+ if p.last_health != Health::Alive
+ || !stable
+ || p.acked < tail
+ || !roles::clearly_better(&p.facts, &mine)
+ {
+ return;
+ }
+ let Some(link) = self.ctl_link(&target) else {
+ return;
+ };
+ let epoch = st.eras.epoch() + 1;
+ st.last_takeover_try = Some(Instant::now());
+ // Nothing is written while the offer is out (the lock is held), so
+ // the base is exact.
+ let r = link
+ .call(
+ Msg::Handoff {
+ epoch,
+ to: target.clone(),
+ base: tail,
+ },
+ &[],
+ self.cfg.call_timeout.min(Duration::from_secs(5)),
+ )
+ .await;
+ if !matches!(r, Ok((Msg::HandoffReply { ok: true, .. }, _))) {
+ return;
+ }
+ st.eras.adopt(EraRec {
+ epoch,
+ owner: target.clone(),
+ base: tail,
+ at_ms: self.wall_ms(),
+ });
+ let _ = save_eras(&self.dir, &st.eras);
+ st.handbacks += 1;
+ tracing::info!(to = %target, epoch, "replica: the volume went back to a more reliable machine");
+ let recs: Vec<EraRec> = st.eras.recs.iter().rev().take(4).rev().cloned().collect();
+ drop(st);
+ self.announce(recs).await;
+ self.wake.notify_one();
+ }
+
+ pub(super) async fn handle_handoff(
+ &self,
+ from: &str,
+ epoch: u64,
+ to: String,
+ base: Pos,
+ ) -> Msg {
+ let mut st = self.st.lock().await;
+ let no = |reason: &str| Msg::HandoffReply {
+ ok: false,
+ reason: reason.into(),
+ };
+ if to != self.me {
+ return no("not for this machine");
+ }
+ if from != st.eras.owner() || epoch != st.eras.epoch() + 1 {
+ return no("the offer is stale");
+ }
+ if st.log.tail() != base {
+ return no("this machine is not caught up");
+ }
+ st.eras.adopt(EraRec {
+ epoch,
+ owner: self.me.clone(),
+ base,
+ at_ms: self.wall_ms(),
+ });
+ let _ = save_eras(&self.dir, &st.eras);
+ st.handbacks += 1;
+ self.wake.notify_one();
+ Msg::HandoffReply {
+ ok: true,
+ reason: String::new(),
+ }
+ }
+}
diff --git a/libs/cua/crates/cua-volume/src/replica/era.rs b/libs/cua/crates/cua-volume/src/replica/era.rs
new file mode 100644
index 0000000000..2cda2a55bf
--- /dev/null
+++ b/libs/cua/crates/cua-volume/src/replica/era.rs
@@ -0,0 +1,162 @@
+// SPDX-License-Identifier: FSL-1.1-MIT
+// Copyright (c) 2026 Cua AI, Inc.
+
+//! Positions in the replication log and the history of who owned it.
+//!
+//! An *era* is a stretch of the log with one owner. A takeover or a
+//! hand-back starts a new era at `epoch + 1` and records its `base`: the
+//! newest position the new owner had applied. Whatever another machine holds
+//! past a later era's base was written by an owner that era did not see (the
+//! divergent tail): it is rebased, never replayed.
+
+use serde::{Deserialize, Serialize};
+
+use super::MachineId;
+
+/// A position in the log: records are ordered by `(epoch, seq)`.
+#[derive(
+ Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize,
+)]
+pub struct Pos {
+ pub epoch: u64,
+ pub seq: u64,
+}
+
+impl Pos {
+ pub const ZERO: Pos = Pos { epoch: 0, seq: 0 };
+}
+
+impl std::fmt::Display for Pos {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ write!(f, "{}.{}", self.epoch, self.seq)
+ }
+}
+
+/// One era.
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct EraRec {
+ pub epoch: u64,
+ pub owner: MachineId,
+ /// What the owner had applied when it took the era (zero for the first).
+ pub base: Pos,
+ /// Wall time it started (display only).
+ pub at_ms: u64,
+}
+
+/// The eras a machine knows, oldest first, of one lineage (the id of the
+/// volume's origin: two machines that each started a volume alone have
+/// different lineages until one joins the other).
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Eras {
+ pub lineage: String,
+ pub recs: Vec<EraRec>,
+}
+
+impl Eras {
+ /// A volume that starts here, owned by `owner`.
+ pub fn genesis(owner: &str, now_ms: u64) -> Eras {
+ Eras {
+ lineage: crate::new_id(),
+ recs: vec![EraRec {
+ epoch: 1,
+ owner: owner.to_string(),
+ base: Pos::ZERO,
+ at_ms: now_ms,
+ }],
+ }
+ }
+
+ pub fn current(&self) -> &EraRec {
+ self.recs.last().expect("an era always exists")
+ }
+
+ pub fn epoch(&self) -> u64 {
+ self.current().epoch
+ }
+
+ pub fn owner(&self) -> &str {
+ &self.current().owner
+ }
+
+ /// Adds `rec` when it is newer. Returns whether it was added.
+ pub fn adopt(&mut self, rec: EraRec) -> bool {
+ if rec.epoch <= self.epoch() {
+ return false;
+ }
+ self.recs.push(rec);
+ true
+ }
+
+ /// Adopts the history `other` carries when it extends ours (same
+ /// lineage): every era newer than ours.
+ pub fn adopt_from(&mut self, other: &[EraRec]) -> bool {
+ let mut any = false;
+ for r in other {
+ if self.adopt(r.clone()) {
+ any = true;
+ }
+ }
+ any
+ }
+
+ /// Where a machine whose newest applied record is `p` stops agreeing
+ /// with this history: `Some(keep)` when it holds records past the base
+ /// of an era that came after it (those are its divergent tail; it keeps
+ /// everything up to `keep`).
+ pub fn divergence(&self, p: Pos) -> Option<Pos> {
+ self.recs
+ .iter()
+ .filter(|r| r.epoch > p.epoch && r.epoch > 1)
+ .filter(|r| p > r.base)
+ .map(|r| r.base)
+ .min()
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn rec(epoch: u64, owner: &str, base: (u64, u64)) -> EraRec {
+ EraRec {
+ epoch,
+ owner: owner.into(),
+ base: Pos {
+ epoch: base.0,
+ seq: base.1,
+ },
+ at_ms: 0,
+ }
+ }
+
+ #[test]
+ fn a_machine_ahead_of_a_later_eras_base_diverges() {
+ let mut e = Eras::genesis("laptop", 0);
+ assert_eq!(e.divergence(Pos { epoch: 1, seq: 50 }), None);
+ // The studio took over having applied up to 1.40.
+ assert!(e.adopt(rec(2, "studio", (1, 40))));
+ assert!(!e.adopt(rec(2, "other", (1, 1))), "an old epoch is ignored");
+ // The laptop applied 1.50 (ten records the studio never saw).
+ assert_eq!(
+ e.divergence(Pos { epoch: 1, seq: 50 }),
+ Some(Pos { epoch: 1, seq: 40 })
+ );
+ // One that stopped at 1.40 or earlier agrees.
+ assert_eq!(e.divergence(Pos { epoch: 1, seq: 40 }), None);
+ assert_eq!(e.divergence(Pos { epoch: 1, seq: 7 }), None);
+ // A machine already in the new era never diverges.
+ assert_eq!(e.divergence(Pos { epoch: 2, seq: 9 }), None);
+ }
+
+ #[test]
+ fn the_earliest_base_wins_across_several_eras() {
+ let mut e = Eras::genesis("a", 0);
+ e.adopt(rec(2, "b", (1, 40)));
+ e.adopt(rec(3, "c", (1, 30)));
+ assert_eq!(
+ e.divergence(Pos { epoch: 1, seq: 45 }),
+ Some(Pos { epoch: 1, seq: 30 })
+ );
+ assert_eq!(e.divergence(Pos { epoch: 1, seq: 30 }), None);
+ }
+}
diff --git a/libs/cua/crates/cua-volume/src/replica/link.rs b/libs/cua/crates/cua-volume/src/replica/link.rs
new file mode 100644
index 0000000000..6797597a1a
--- /dev/null
+++ b/libs/cua/crates/cua-volume/src/replica/link.rs
@@ -0,0 +1,361 @@
+// SPDX-License-Identifier: FSL-1.1-MIT
+// Copyright (c) 2026 Cua AI, Inc.
+
+//! One framed connection to a peer, with request/reply correlation.
+//!
+//! A [`Link`] owns a writer task and a reader task over any duplex stream
+//! (a mesh stream in the product, TCP in tests, an in-memory pipe in
+//! simulations). Messages that answer a request (`re` set) go to the waiting
+//! [`Link::call`]; everything else goes to the handler, each on its own task
+//! so a slow handler never blocks the reader (and so a heartbeat is always
+//! read).
+
+use std::collections::HashMap;
+use std::future::Future;
+use std::pin::Pin;
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
+use std::sync::{Arc, Mutex};
+use std::time::Duration;
+
+use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};
+use tokio::sync::{Notify, mpsc, oneshot};
+
+use super::MachineId;
+use super::wire::{self, Chan, Frame, Msg};
+
+/// Any duplex byte stream.
+pub trait Duplex: AsyncRead + AsyncWrite + Send + Unpin + 'static {}
+impl<T: AsyncRead + AsyncWrite + Send + Unpin + 'static> Duplex for T {}
+
+/// A boxed stream.
+pub type Stream = Box<dyn Duplex>;
+
+/// Bytes crossing links (frames, as sent and received).
+#[derive(Debug, Default)]
+pub struct Counters {
+ pub sent: AtomicU64,
+ pub received: AtomicU64,
+}
+
+/// What a handler receives.
+pub type Handler = Arc<
+ dyn Fn(Arc<Link>, Frame, Vec<u8>) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync,
+>;
+
+/// What the writer task is asked to do.
+enum Out {
+ Bytes(Vec<u8>),
+ /// Fires once everything queued before it has been written.
+ Flush(oneshot::Sender<()>),
+}
+
+type Waiter = oneshot::Sender<(Frame, Vec<u8>)>;
+
+/// One connection.
+pub struct Link {
+ pub peer: MachineId,
+ pub peer_name: String,
+ pub chan: Chan,
+ /// This side dialed.
+ pub initiated: bool,
+ /// When the connection came up.
+ pub created: std::time::Instant,
+ out: mpsc::UnboundedSender<Out>,
+ calls: Mutex<HashMap<u64, Waiter>>,
+ next: AtomicU64,
+ closed: AtomicBool,
+ closed_notify: Notify,
+ counters: Arc<Counters>,
+ /// Peer facts from its Hello (the node reads them).
+ pub hello: Mutex<Option<wire::Hello>>,
+ tasks: Mutex<Vec<tokio::task::AbortHandle>>,
+}
+
+impl std::fmt::Debug for Link {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("Link")
+ .field("peer", &self.peer)
+ .field("chan", &self.chan)
+ .field("closed", &self.is_closed())
+ .finish()
+ }
+}
+
+/// Why a call failed.
+#[derive(Debug)]
+pub enum CallError {
+ Closed,
+ Timeout,
+ Io(std::io::Error),
+}
+
+impl std::fmt::Display for CallError {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ match self {
+ CallError::Closed => f.write_str("the link to the machine closed"),
+ CallError::Timeout => f.write_str("the machine did not answer in time"),
+ CallError::Io(e) => write!(f, "{e}"),
+ }
+ }
+}
+
+impl Link {
+ /// Starts the reader and writer tasks over `stream`.
+ #[allow(clippy::too_many_arguments)]
+ pub fn spawn(
+ stream: Stream,
+ peer: MachineId,
+ peer_name: String,
+ chan: Chan,
+ initiated: bool,
+ counters: Arc<Counters>,
+ handler: Handler,
+ on_close: impl FnOnce(Arc<Link>) + Send + 'static,
+ ) -> Arc<Link> {
+ let (mut rd, mut wr) = tokio::io::split(stream);
+ let (tx, mut rx) = mpsc::unbounded_channel::<Out>();
+ let link = Arc::new(Link {
+ peer,
+ peer_name,
+ chan,
+ initiated,
+ created: std::time::Instant::now(),
+ out: tx,
+ calls: Mutex::new(HashMap::new()),
+ next: AtomicU64::new(1),
+ closed: AtomicBool::new(false),
+ closed_notify: Notify::new(),
+ counters: counters.clone(),
+ hello: Mutex::new(None),
+ tasks: Mutex::new(vec![]),
+ });
+ let l = link.clone();
+ let c = counters.clone();
+ let wtask = tokio::spawn(async move {
+ while let Some(out) = rx.recv().await {
+ match out {
+ Out::Bytes(bytes) => {
+ c.sent.fetch_add(bytes.len() as u64, Ordering::Relaxed);
+ if wr.write_all(&bytes).await.is_err() || wr.flush().await.is_err() {
+ break;
+ }
+ }
+ Out::Flush(done) => {
+ let _ = done.send(());
+ }
+ }
+ }
+ let _ = wr.shutdown().await;
+ l.close();
+ });
+ let l = link.clone();
+ let rtask = tokio::spawn(async move {
+ #[allow(clippy::while_let_loop)]
+ loop {
+ match wire::read(&mut rd).await {
+ Ok(Some((frame, data))) => {
+ l.counters
+ .received
+ .fetch_add((data.len() + 64) as u64, Ordering::Relaxed);
+ if let Some(re) = frame.re {
+ let waiter = l.calls.lock().expect("calls").remove(&re);
+ if let Some(w) = waiter {
+ let _ = w.send((frame, data));
+ }
+ } else {
+ let h = handler.clone();
+ let l2 = l.clone();
+ tokio::spawn(async move { h(l2, frame, data).await });
+ }
+ }
+ Ok(None) | Err(_) => break,
+ }
+ }
+ l.close();
+ on_close(l);
+ });
+ link.tasks
+ .lock()
+ .expect("tasks")
+ .extend([wtask.abort_handle(), rtask.abort_handle()]);
+ link
+ }
+
+ /// Waits (briefly) until what was sent has been handed to the stream.
+ pub async fn flush(&self, within: Duration) {
+ let (tx, rx) = oneshot::channel();
+ if self.out.send(Out::Flush(tx)).is_ok() {
+ let _ = tokio::time::timeout(within, rx).await;
+ }
+ }
+
+ /// Marks the link closed and fails every waiting call.
+ pub fn close(&self) {
+ if !self.closed.swap(true, Ordering::SeqCst) {
+ self.calls.lock().expect("calls").clear();
+ self.closed_notify.notify_waiters();
+ // A dead connection must not keep its tasks (and so its socket)
+ // alive: the peer sees the end of the stream and redials.
+ for t in self.tasks.lock().expect("tasks").drain(..) {
+ t.abort();
+ }
+ }
+ }
+
+ pub fn is_closed(&self) -> bool {
+ self.closed.load(Ordering::SeqCst)
+ }
+
+ /// Resolves when the link closes.
+ pub async fn closed(&self) {
+ loop {
+ let n = self.closed_notify.notified();
+ if self.is_closed() {
+ return;
+ }
+ n.await;
+ }
+ }
+
+ /// Sends a message with no reply expected.
+ pub fn send(&self, msg: Msg, data: &[u8]) -> bool {
+ self.send_frame(
+ Frame {
+ id: self.next.fetch_add(1, Ordering::Relaxed),
+ re: None,
+ msg,
+ },
+ data,
+ )
+ }
+
+ /// Answers request `id`.
+ pub fn reply(&self, id: u64, msg: Msg, data: &[u8]) -> bool {
+ self.send_frame(
+ Frame {
+ id: self.next.fetch_add(1, Ordering::Relaxed),
+ re: Some(id),
+ msg,
+ },
+ data,
+ )
+ }
+
+ fn send_frame(&self, f: Frame, data: &[u8]) -> bool {
+ if self.is_closed() {
+ return false;
+ }
+ match wire::encode(&f, data) {
+ Ok(b) => self.out.send(Out::Bytes(b)).is_ok(),
+ Err(_) => false,
+ }
+ }
+
+ /// Like [`Link::call`], and a timeout closes the link: a bulk request
+ /// that gets no answer means the path is dead (a half-open connection
+ /// looks exactly like a slow one until something gives up on it), and the
+ /// next attempt must dial afresh.
+ pub async fn call_strict(
+ &self,
+ msg: Msg,
+ data: &[u8],
+ timeout: Duration,
+ ) -> Result<(Msg, Vec<u8>), CallError> {
+ let r = self.call(msg, data, timeout).await;
+ if matches!(r, Err(CallError::Timeout)) {
+ self.close();
+ }
+ r
+ }
+
+ /// Sends a request and waits for its reply.
+ pub async fn call(
+ &self,
+ msg: Msg,
+ data: &[u8],
+ timeout: Duration,
+ ) -> Result<(Msg, Vec<u8>), CallError> {
+ if self.is_closed() {
+ return Err(CallError::Closed);
+ }
+ let id = self.next.fetch_add(1, Ordering::Relaxed);
+ let (tx, rx) = oneshot::channel();
+ self.calls.lock().expect("calls").insert(id, tx);
+ if !self.send_frame(Frame { id, re: None, msg }, data) {
+ self.calls.lock().expect("calls").remove(&id);
+ return Err(CallError::Closed);
+ }
+ match tokio::time::timeout(timeout, rx).await {
+ Ok(Ok((f, d))) => Ok((f.msg, d)),
+ Ok(Err(_)) => Err(CallError::Closed),
+ Err(_) => {
+ self.calls.lock().expect("calls").remove(&id);
+ Err(CallError::Timeout)
+ }
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::replica::wire::Rep;
+
+ fn echo() -> Handler {
+ Arc::new(|link, frame, data| {
+ Box::pin(async move {
+ if let Msg::Req(_) = frame.msg {
+ link.reply(frame.id, Msg::Rep(Rep::Range), &data);
+ }
+ })
+ })
+ }
+
+ #[tokio::test]
+ async fn calls_get_their_replies_and_a_closed_link_fails_them() {
+ let (a, b) = tokio::io::duplex(1 << 16);
+ let ca = Arc::new(Counters::default());
+ let la = Link::spawn(
+ Box::new(a),
+ "b".into(),
+ "B".into(),
+ Chan::Control,
+ true,
+ ca.clone(),
+ echo(),
+ |_| {},
+ );
+ let lb = Link::spawn(
+ Box::new(b),
+ "a".into(),
+ "A".into(),
+ Chan::Control,
+ false,
+ Arc::new(Counters::default()),
+ echo(),
+ |_| {},
+ );
+ let (m, d) = la
+ .call(
+ Msg::Req(wire::Req::Head { key: "k".into() }),
+ b"hi",
+ Duration::from_secs(2),
+ )
+ .await
+ .unwrap();
+ assert_eq!(m, Msg::Rep(Rep::Range));
+ assert_eq!(d, b"hi");
+ assert!(ca.sent.load(Ordering::Relaxed) > 0);
+ lb.close();
+ drop(lb);
+ // The other side's writer ends when its handle is gone; the call fails.
+ let r = la
+ .call(
+ Msg::Req(wire::Req::Head { key: "k".into() }),
+ b"",
+ Duration::from_millis(300),
+ )
+ .await;
+ assert!(r.is_err() || la.is_closed());
+ }
+}
diff --git a/libs/cua/crates/cua-volume/src/replica/merge.rs b/libs/cua/crates/cua-volume/src/replica/merge.rs
new file mode 100644
index 0000000000..68eea7c36c
--- /dev/null
+++ b/libs/cua/crates/cua-volume/src/replica/merge.rs
@@ -0,0 +1,160 @@
+// SPDX-License-Identifier: FSL-1.1-MIT
+// Copyright (c) 2026 Cua AI, Inc.
+
+//! What to do when two machines changed one key.
+//!
+//! Almost always: keep the owner's version at the key and store the other
+//! machine's content next to it as `name (conflict from <machine> <time>).ext`
+//! (the same naming the sync feed uses). Nothing is overwritten and nothing
+//! is lost. Control objects whose meaning is an order merge by that meaning
+//! instead: the session fence is a monotonic epoch, so the higher epoch
+//! wins and a stale driver is fenced the moment the machines meet again.
+
+use crate::feed::conflict_path;
+use crate::journal::FENCE;
+
+/// How a conflict on `key` resolves.
+#[derive(Debug, PartialEq, Eq)]
+pub enum Resolution {
+ /// Keep the head already at the key, drop the incoming content (it is
+ /// identical or lower).
+ KeepHead,
+ /// Replace the head with the incoming content (it is higher).
+ TakeIncoming,
+ /// Keep the head; store the incoming content at this key.
+ Copy(String),
+}
+
+/// A session fence key (`.../sessions/<session>/fence`).
+fn is_fence(key: &str) -> bool {
+ key.rsplit('/').next() == Some(FENCE) && key.contains("/sessions/")
+}
+
+/// A durable run's supervisor lease (`agents/<agent>/.cua-supervisor`).
+fn is_supervisor(key: &str) -> bool {
+ key.rsplit('/').next() == Some(crate::replica::supervision::LEASE)
+}
+
+/// A run's record (`.cua-runs/<agent>.json`).
+fn is_run_record(key: &str) -> bool {
+ key.starts_with(crate::replica::supervision::RUNS_DIR)
+}
+
+/// Keys whose value is an order (fence, supervisor lease, run record): a
+/// change to one is always judged against the head by that order, even when
+/// it was written against the head's own version, because a stale writer's
+/// later changes chain onto its earlier ones.
+pub fn is_ordered(key: &str) -> bool {
+ is_fence(key) || is_supervisor(key) || is_run_record(key)
+}
+
+fn json_field(bytes: &[u8], field: &str) -> Option<serde_json::Value> {
+ serde_json::from_slice::<serde_json::Value>(bytes)
+ .ok()?
+ .get(field)
+ .cloned()
+}
+
+fn fence_epoch(bytes: &[u8]) -> Option<u64> {
+ serde_json::from_slice::<serde_json::Value>(bytes)
+ .ok()?
+ .get("epoch")?
+ .as_u64()
+}
+
+/// Resolves a clash on `key` between the head's bytes and the incoming
+/// bytes from `machine` at `ts_ms`.
+pub fn resolve(key: &str, head: &[u8], incoming: &[u8], machine: &str, ts_ms: u64) -> Resolution {
+ if head == incoming {
+ return Resolution::KeepHead;
+ }
+ if is_fence(key)
+ && let (Some(h), Some(i)) = (fence_epoch(head), fence_epoch(incoming))
+ {
+ return if i > h {
+ Resolution::TakeIncoming
+ } else {
+ Resolution::KeepHead
+ };
+ }
+ // The supervisor lease is an order: the higher epoch holds the run, and
+ // within an epoch the later beat.
+ if is_supervisor(key) {
+ let order = |b: &[u8]| {
+ (
+ json_field(b, "epoch").and_then(|v| v.as_u64()).unwrap_or(0),
+ json_field(b, "beat").and_then(|v| v.as_u64()).unwrap_or(0),
+ )
+ };
+ return if order(incoming) > order(head) {
+ Resolution::TakeIncoming
+ } else {
+ Resolution::KeepHead
+ };
+ }
+ // A finished run stays finished.
+ if is_run_record(key) {
+ let done = |b: &[u8]| {
+ json_field(b, "state")
+ .and_then(|v| v.as_str().map(|s| s == "done"))
+ .unwrap_or(false)
+ };
+ return if done(incoming) && !done(head) {
+ Resolution::TakeIncoming
+ } else {
+ Resolution::KeepHead
+ };
+ }
+ Resolution::Copy(conflict_path(key, machine, ts_ms))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn plain_files_become_conflict_copies() {
+ let r = resolve("agents/ada/notes.md", b"a", b"b", "MacBook", 0);
+ match r {
+ Resolution::Copy(k) => {
+ assert!(
+ k.starts_with("agents/ada/notes (conflict from MacBook "),
+ "{k}"
+ );
+ assert!(k.ends_with(").md"));
+ }
+ other => panic!("{other:?}"),
+ }
+ assert_eq!(
+ resolve("agents/ada/notes.md", b"same", b"same", "m", 0),
+ Resolution::KeepHead
+ );
+ }
+
+ #[test]
+ fn a_supervisor_lease_goes_to_the_higher_epoch_and_a_done_run_stays_done() {
+ let k = "agents/ada/.cua-supervisor";
+ let a = br#"{"epoch":2,"beat":9}"#;
+ let b = br#"{"epoch":3,"beat":0}"#;
+ assert_eq!(resolve(k, a, b, "m", 0), Resolution::TakeIncoming);
+ assert_eq!(resolve(k, b, a, "m", 0), Resolution::KeepHead);
+ let r = ".cua-runs/ada.json";
+ assert_eq!(
+ resolve(r, br#"{"state":"running"}"#, br#"{"state":"done"}"#, "m", 0),
+ Resolution::TakeIncoming
+ );
+ assert_eq!(
+ resolve(r, br#"{"state":"done"}"#, br#"{"state":"running"}"#, "m", 0),
+ Resolution::KeepHead
+ );
+ }
+
+ #[test]
+ fn a_fence_merges_to_the_higher_epoch() {
+ let k = "spaces/dev/sessions/s1/fence";
+ let low = br#"{"epoch":2,"holder":"a"}"#;
+ let high = br#"{"epoch":5,"holder":"b"}"#;
+ assert_eq!(resolve(k, low, high, "m", 0), Resolution::TakeIncoming);
+ assert_eq!(resolve(k, high, low, "m", 0), Resolution::KeepHead);
+ }
+}
diff --git a/libs/cua/crates/cua-volume/src/replica/migrate.rs b/libs/cua/crates/cua-volume/src/replica/migrate.rs
new file mode 100644
index 0000000000..b994ef07e3
--- /dev/null
+++ b/libs/cua/crates/cua-volume/src/replica/migrate.rs
@@ -0,0 +1,371 @@
+// SPDX-License-Identifier: FSL-1.1-MIT
+// Copyright (c) 2026 Cua AI, Inc.
+
+//! Turning replication on for a Volume that already exists, and living with
+//! builds that do not know about it.
+//!
+//! Nothing about the Volume's own files changes: `data/` keeps its layout
+//! (versions and heads, written the way every release has), so a build that
+//! predates replication reads it exactly as before. Everything replication
+//! adds is new and lives under `replica/` next to it:
+//!
+//! - `replica/format.json`: the format version of that directory. A build
+//! that finds a newer one than it understands refuses with a plain
+//! message instead of guessing (never corruption).
+//! - `replica/backup/heads-<time>/`: a copy of the heads taken the first time
+//! replication is enabled on a Volume that has files, before anything
+//! moves. Versions are never deleted, so the heads are all a restore needs.
+//! - `replica/heads.snap`: what the heads were at the last clean stop. A
+//! build that does not know about replication may write to `data/` while
+//! replication is off for it (an older Cua Spaces on the same home); on the
+//! next start the differences are found and handed to the log (or, on a
+//! machine that does not own the Volume, to the owner as changes), so they
+//! reach the other machines instead of silently diverging.
+//!
+//! Replication starts only when this account's machines list has another
+//! machine and no bucket is configured ([`decide`]).
+
+use std::collections::BTreeMap;
+use std::path::{Path, PathBuf};
+
+use serde::{Deserialize, Serialize};
+
+use super::era::Pos;
+use super::oplog::{Op, OpLog};
+use crate::backend::{Backend, ObjectMeta};
+use crate::fs::FsBackend;
+use crate::{Error, Result};
+
+/// The format of the `replica/` directory this build writes and reads.
+pub const FORMAT: u32 = 1;
+
+#[derive(Debug, Serialize, Deserialize)]
+struct Format {
+ format: u32,
+ created_ms: u64,
+}
+
+/// Reads or creates `replica/format.json`. A directory written by a newer
+/// format is refused.
+pub fn check_format(dir: &Path) -> Result<()> {
+ let p = dir.join("format.json");
+ match std::fs::read(&p) {
+ Ok(b) => {
+ let f: Format = serde_json::from_slice(&b)
+ .map_err(|e| Error::Invalid(format!("replica/format.json: {e}")))?;
+ if f.format > FORMAT {
+ return Err(Error::Invalid(format!(
+ "this Volume was prepared by a newer Cua Spaces (format {}, this one reads up to {FORMAT}); update Cua Spaces on this machine",
+ f.format
+ )));
+ }
+ Ok(())
+ }
+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
+ std::fs::create_dir_all(dir)?;
+ cua_home::guard_write(&p)?;
+ std::fs::write(
+ &p,
+ serde_json::to_vec(&Format {
+ format: FORMAT,
+ created_ms: crate::now_ms(),
+ })?,
+ )?;
+ Ok(())
+ }
+ Err(e) => Err(e.into()),
+ }
+}
+
+fn copy_tree(from: &Path, to: &Path) -> std::io::Result<u64> {
+ let mut n = 0;
+ std::fs::create_dir_all(to)?;
+ for e in std::fs::read_dir(from)? {
+ let e = e?;
+ let (src, dst) = (e.path(), to.join(e.file_name()));
+ if e.file_type()?.is_dir() {
+ n += copy_tree(&src, &dst)?;
+ } else {
+ std::fs::copy(&src, &dst)?;
+ n += 1;
+ }
+ }
+ Ok(n)
+}
+
+/// Copies the store's heads to `dir/backup/heads-<ms>/` (when it has any).
+/// Returns where, and how many head files.
+pub fn backup_heads(dir: &Path, store_root: &Path) -> Result<Option<(PathBuf, u64)>> {
+ let heads = store_root.join("heads");
+ if !heads.is_dir() {
+ return Ok(None);
+ }
+ let to = dir
+ .join("backup")
+ .join(format!("heads-{}", crate::now_ms()));
+ cua_home::guard_write(&to.join("x"))?;
+ let n = copy_tree(&heads, &to)?;
+ Ok(Some((to, n)))
+}
+
+/// What the heads were.
+#[derive(Debug, Default, Serialize, Deserialize)]
+pub struct HeadsSnap {
+ pub pos: Pos,
+ pub heads: BTreeMap<String, ObjectMeta>,
+}
+
+fn snap_path(dir: &Path) -> PathBuf {
+ dir.join("heads.snap")
+}
+
+/// Saves the heads as of `pos`.
+pub async fn save_snap(dir: &Path, store: &FsBackend, pos: Pos) -> Result<()> {
+ let heads: BTreeMap<String, ObjectMeta> = store
+ .list("")
+ .await?
+ .into_iter()
+ .map(|m| (m.key.clone(), m))
+ .collect();
+ let p = snap_path(dir);
+ cua_home::guard_write(&p)?;
+ let tmp = p.with_extension("tmp");
+ std::fs::write(&tmp, serde_json::to_vec(&HeadsSnap { pos, heads })?)?;
+ std::fs::rename(&tmp, &p)?;
+ Ok(())
+}
+
+/// One change made behind replication's back.
+#[derive(Debug, PartialEq, Eq)]
+pub enum Unlogged {
+ /// `meta` is the head now; `prev` was the head at the snapshot.
+ Put {
+ meta: ObjectMeta,
+ prev: Option<ObjectMeta>,
+ },
+ /// `key` was deleted (a marker with `version`); `prev` was the head.
+ Delete {
+ key: String,
+ version: String,
+ prev: Option<ObjectMeta>,
+ },
+}
+
+/// The changes to `store` since the snapshot that its log does not account
+/// for. Empty when there is no snapshot (the Volume predates replication or
+/// was never stopped cleanly) or the log no longer reaches back to it.
+pub async fn find_unlogged(dir: &Path, store: &FsBackend, log: &OpLog) -> Result<Vec<Unlogged>> {
+ let snap = std::fs::read(snap_path(dir))
+ .ok()
+ .and_then(|b| serde_json::from_slice::<HeadsSnap>(&b).ok());
+ let Some(snap) = snap else {
+ return find_unlogged_by_time(store, log).await;
+ };
+ let Some(after) = log.after(snap.pos, usize::MAX) else {
+ // The log was trimmed past the snapshot: expectations cannot be rebuilt.
+ return Ok(vec![]);
+ };
+ // What the heads should be: the snapshot plus what the log did since.
+ let mut expect = snap.heads;
+ for r in after {
+ match r.op {
+ Op::Put { meta, .. } => {
+ expect.insert(meta.key.clone(), meta);
+ }
+ Op::Delete { key, .. } => {
+ expect.remove(&key);
+ }
+ }
+ }
+ let actual: BTreeMap<String, ObjectMeta> = store
+ .list("")
+ .await?
+ .into_iter()
+ .map(|m| (m.key.clone(), m))
+ .collect();
+ let mut out = vec![];
+ for (k, a) in &actual {
+ match expect.get(k) {
+ Some(e) if e.version == a.version => {}
+ other => out.push(Unlogged::Put {
+ meta: a.clone(),
+ prev: other.cloned(),
+ }),
+ }
+ }
+ for (k, e) in &expect {
+ if !actual.contains_key(k) {
+ let version = store
+ .versions(k)
+ .await
+ .ok()
+ .and_then(|v| v.into_iter().find(|v| v.deleted).map(|v| v.version))
+ .unwrap_or_default();
+ if !version.is_empty() {
+ out.push(Unlogged::Delete {
+ key: k.clone(),
+ version,
+ prev: Some(e.clone()),
+ });
+ }
+ }
+ }
+ Ok(out)
+}
+
+/// Without a snapshot (the machine was stopped hard, or never stopped
+/// cleanly): a head written after the newest log record, by anyone but the
+/// log, is a change made behind its back. Only for stores that have a log.
+async fn find_unlogged_by_time(store: &FsBackend, log: &OpLog) -> Result<Vec<Unlogged>> {
+ let Some(newest) = log.newest_at_ms() else {
+ return Ok(vec![]);
+ };
+ let known: std::collections::HashSet<(String, String)> = log
+ .range(Pos::ZERO, log.tail())
+ .into_iter()
+ .filter_map(|r| match r.op {
+ Op::Put { meta, .. } => Some((meta.key, meta.version)),
+ _ => None,
+ })
+ .collect();
+ let mut out = vec![];
+ for m in store.list("").await? {
+ if m.modified_ms > newest + 1000 && !known.contains(&(m.key.clone(), m.version.clone())) {
+ out.push(Unlogged::Put {
+ meta: m,
+ prev: None,
+ });
+ }
+ }
+ Ok(out)
+}
+
+/// Whether replication should run at all.
+#[derive(Debug, PartialEq, Eq)]
+pub enum Decision {
+ On,
+ /// Off, and why (for a log line and a status).
+ Off(&'static str),
+}
+
+/// Replication starts only when the user has signed in (a machines list
+/// exists), has another machine on it, and has not pointed the Volume at a
+/// bucket. Anything else leaves the Volume exactly as an earlier release
+/// served it.
+pub fn decide(bucket_configured: bool, signed_in: bool, other_machines: usize) -> Decision {
+ if bucket_configured {
+ Decision::Off("the Volume is on your own bucket")
+ } else if !signed_in {
+ Decision::Off("not signed in")
+ } else if other_machines == 0 {
+ Decision::Off("this is the only machine")
+ } else {
+ Decision::On
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn a_newer_format_is_refused_and_an_older_or_missing_one_is_fine() {
+ let d = tempfile::tempdir().unwrap();
+ let dir = d.path().join("replica");
+ check_format(&dir).unwrap();
+ check_format(&dir).unwrap();
+ std::fs::write(dir.join("format.json"), br#"{"format":99,"created_ms":1}"#).unwrap();
+ let e = check_format(&dir).unwrap_err();
+ assert!(e.to_string().contains("newer Cua Spaces"), "{e}");
+ std::fs::write(dir.join("format.json"), br#"{"format":0,"created_ms":1}"#).unwrap();
+ check_format(&dir).unwrap();
+ }
+
+ #[test]
+ fn replication_starts_only_after_signing_in_with_another_machine_and_no_bucket() {
+ assert_eq!(
+ decide(true, true, 3),
+ Decision::Off("the Volume is on your own bucket")
+ );
+ assert_eq!(decide(false, false, 3), Decision::Off("not signed in"));
+ assert_eq!(
+ decide(false, true, 0),
+ Decision::Off("this is the only machine")
+ );
+ assert_eq!(decide(false, true, 1), Decision::On);
+ }
+
+ #[tokio::test]
+ async fn without_a_snapshot_a_head_newer_than_the_log_is_found() {
+ use crate::backend::Condition;
+ use crate::replica::oplog::Record;
+ let d = tempfile::tempdir().unwrap();
+ let store = FsBackend::new(d.path().join("data"));
+ let state = d.path().join("replica");
+ let mut log = OpLog::open(&state, 100).unwrap();
+ let m = store
+ .put("a/old", b"1".to_vec(), Condition::None)
+ .await
+ .unwrap();
+ log.append(Record {
+ pos: Pos { epoch: 1, seq: 1 },
+ op: Op::Put {
+ meta: m.clone(),
+ prev: None,
+ },
+ at_ms: m.modified_ms + 1,
+ })
+ .unwrap();
+ // The log knows a/old; a head made well after the last record is not its doing.
+ tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
+ store
+ .put("a/new", b"2".to_vec(), Condition::None)
+ .await
+ .unwrap();
+ let found = find_unlogged(&state, &store, &log).await.unwrap();
+ assert_eq!(found.len(), 1, "{found:?}");
+ assert!(matches!(&found[0], Unlogged::Put { meta, .. } if meta.key == "a/new"));
+ }
+
+ #[tokio::test]
+ async fn heads_are_backed_up_and_a_backdoor_write_is_found() {
+ use crate::backend::Condition;
+ let d = tempfile::tempdir().unwrap();
+ let store = FsBackend::new(d.path().join("data"));
+ store
+ .put("a/one", b"1".to_vec(), Condition::None)
+ .await
+ .unwrap();
+ store
+ .put("a/two", b"2".to_vec(), Condition::None)
+ .await
+ .unwrap();
+ let state = d.path().join("replica");
+ let (to, n) = backup_heads(&state, store.root()).unwrap().unwrap();
+ assert_eq!(n, 2);
+ assert!(to.join("a.d").is_dir());
+ let log = OpLog::open(&state, 100).unwrap();
+ save_snap(&state, &store, log.tail()).await.unwrap();
+ // An older build writes straight to the store.
+ store
+ .put("a/one", b"1 changed".to_vec(), Condition::None)
+ .await
+ .unwrap();
+ store
+ .put("a/three", b"3".to_vec(), Condition::None)
+ .await
+ .unwrap();
+ store.delete("a/two", Condition::None).await.unwrap();
+ let mut found = find_unlogged(&state, &store, &log).await.unwrap();
+ found.sort_by_key(|u| match u {
+ Unlogged::Put { meta, .. } => meta.key.clone(),
+ Unlogged::Delete { key, .. } => key.clone(),
+ });
+ assert_eq!(found.len(), 3, "{found:?}");
+ assert!(
+ matches!(&found[0], Unlogged::Put { meta, prev: Some(p) } if meta.key == "a/one" && p.size == 1)
+ );
+ assert!(matches!(&found[1], Unlogged::Put { meta, prev: None } if meta.key == "a/three"));
+ assert!(matches!(&found[2], Unlogged::Delete { key, prev: Some(_), .. } if key == "a/two"));
+ }
+}
diff --git a/libs/cua/crates/cua-volume/src/replica/mod.rs b/libs/cua/crates/cua-volume/src/replica/mod.rs
new file mode 100644
index 0000000000..6ee8c4c2c5
--- /dev/null
+++ b/libs/cua/crates/cua-volume/src/replica/mod.rs
@@ -0,0 +1,58 @@
+// SPDX-License-Identifier: FSL-1.1-MIT
+// Copyright (c) 2026 Cua AI, Inc.
+
+//! A Volume that lives on your machines: no bucket, no server.
+//!
+//! With no bucket configured and two or more machines signed in, a durable
+//! agent's Volume is kept on the most reliable machine (the owner) and one
+//! more (a replica), so closing the laptop loses nothing and the agent keeps
+//! its files. See `DESIGN.md` of the just-works-machines work for the
+//! reasoning; this is the map.
+//!
+//! | Module | What it owns |
+//! |---|---|
+//! | [`era`] | log positions and the history of owners (epochs, bases, divergence) |
+//! | [`oplog`] | the replication log (what the owner did, in order) |
+//! | [`detector`] | is that machine still there: adaptive threshold, hysteresis, goodbyes |
+//! | [`roles`] | machines, reliability facts, who owns and who holds copies |
+//! | [`merge`] | what to do when two machines changed one key |
+//! | [`migrate`] | turning replication on for an existing Volume: format marker, backup, changes made behind its back |
+//! | [`wire`], [`link`], [`transport`] | the peer protocol, one connection, how machines reach each other |
+//! | [`outbox`] | changes saved here that the owner has not taken yet |
+//! | [`node`] | the runtime: owner writes, log shipping, catch-up, takeover, hand-back |
+//! | [`words`], [`status`] | what the app and the CLI say |
+//! | [`supervision`] | which machine runs a durable agent, and who takes it over |
+//! | [`simnet`] | an in-memory network with latency, loss, partitions and held bytes (tests, chaos) |
+
+mod apply;
+pub mod backend;
+pub mod detector;
+mod election;
+pub mod era;
+pub mod link;
+pub mod merge;
+pub mod migrate;
+pub mod node;
+pub mod oplog;
+pub mod outbox;
+mod owner;
+pub mod roles;
+mod ship;
+pub mod simnet;
+pub mod status;
+pub mod supervision;
+pub mod transport;
+pub mod wire;
+pub mod words;
+mod xfer;
+
+pub use backend::ReplicatedBackend;
+pub use detector::{Health, Timing};
+pub use era::{EraRec, Eras, Pos};
+pub use node::{Config, MachineView, PowerEvent, ReplicaNode};
+pub use roles::{Facts, Layout, Member, Role};
+pub use status::{Durability, ReplicaStatus};
+pub use transport::{TcpTransport, Transport};
+
+/// A machine's id: its machine id on the account.
+pub type MachineId = String;
diff --git a/libs/cua/crates/cua-volume/src/replica/node.rs b/libs/cua/crates/cua-volume/src/replica/node.rs
new file mode 100644
index 0000000000..4e94fe4e09
--- /dev/null
+++ b/libs/cua/crates/cua-volume/src/replica/node.rs
@@ -0,0 +1,1600 @@
+// SPDX-License-Identifier: FSL-1.1-MIT
+// Copyright (c) 2026 Cua AI, Inc.
+
+//! The replication runtime of one machine: who it talks to, what it knows,
+//! and the loop that keeps the volume converged. The pieces live in
+//! sibling modules (`owner`, `apply`, `election`, `backend`); this file has
+//! the state, the lifecycle, the links, and the tick.
+
+use std::collections::{BTreeMap, HashMap};
+use std::path::{Path, PathBuf};
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
+use std::sync::{Arc, Mutex};
+use std::time::{Duration, Instant};
+
+use tokio::sync::{Notify, watch};
+
+use super::detector::{Detector, Health, Timing};
+use super::era::{EraRec, Eras, Pos};
+use super::link::{Counters, Handler, Link, Stream};
+use super::oplog::OpLog;
+use super::outbox::Outbox;
+use super::roles::{self, Facts, Layout, Member, Role};
+use super::ship::Shipping;
+use super::status::{PeerView, PendingView, ReplicaStatus};
+use super::transport::Transport;
+use super::wire::{self, Beat, Chan, Frame, Hello, Msg};
+use super::words::Summary;
+use super::{MachineId, backend::ReplicatedBackend};
+use crate::backend::Backend as _;
+use crate::fs::FsBackend;
+use crate::{Error, Result, now_ms};
+
+/// Timing and size knobs. Defaults are for real networks; [`Config::fast`]
+/// scales time down for simulations.
+#[derive(Clone, Debug)]
+pub struct Config {
+ pub timing: Timing,
+ /// How long a write waits for a second copy before it is reported as
+ /// held on one machine (the write itself never fails for this).
+ pub ack_wait: Duration,
+ /// A control request that gets no answer in this long fails.
+ pub call_timeout: Duration,
+ /// A bulk request (a block) that gets no answer in this long fails.
+ pub data_timeout: Duration,
+ /// A more reliable machine must have been up and caught up this long
+ /// before the volume moves back to it.
+ pub handback_after: Duration,
+ /// Bodies up to this size travel inside the control message.
+ pub inline_max: usize,
+ /// A write by a non-owner above this size is spooled and pulled by the
+ /// owner in blocks instead of pushed in one request.
+ pub push_max: usize,
+ /// Bytes per transfer block.
+ pub block: u64,
+ /// Blocks in flight per transfer.
+ pub window: usize,
+ /// Records per log message.
+ pub batch: usize,
+ /// Log records kept.
+ pub log_keep: usize,
+ pub dial_backoff_min: Duration,
+ pub dial_backoff_max: Duration,
+ /// A machine not heard from in this long is not counted for takeover
+ /// quorums.
+ pub active_window: Duration,
+ /// After a takeover attempt, wait this long before trying again.
+ pub takeover_cooldown: Duration,
+ /// A machine that does not need to order writes takes over from an
+ /// owner only after it has been down this long.
+ pub idle_takeover: Duration,
+ /// How often the heads are recorded (see `migrate`).
+ pub snapshot_every: Duration,
+ /// Added to this node's wall clock (simulating skew).
+ pub wall_skew_ms: i64,
+}
+
+impl Config {
+ pub fn standard() -> Config {
+ Config {
+ timing: Timing::standard(),
+ ack_wait: Duration::from_secs(2),
+ call_timeout: Duration::from_secs(30),
+ data_timeout: Duration::from_secs(60),
+ handback_after: Duration::from_secs(120),
+ inline_max: 16 * 1024,
+ push_max: 1 << 20,
+ block: 1 << 20,
+ window: 8,
+ batch: 256,
+ log_keep: 200_000,
+ dial_backoff_min: Duration::from_millis(250),
+ dial_backoff_max: Duration::from_secs(15),
+ active_window: Duration::from_secs(14 * 24 * 3600),
+ takeover_cooldown: Duration::from_secs(10),
+ idle_takeover: Duration::from_secs(300),
+ snapshot_every: Duration::from_secs(600),
+ wall_skew_ms: 0,
+ }
+ }
+
+ /// Time scaled down by about 40 (heartbeat 50 ms): for simulations.
+ pub fn fast() -> Config {
+ Config {
+ timing: Timing {
+ interval: Duration::from_millis(50),
+ floor: Duration::from_millis(250),
+ confirm: Duration::from_millis(400),
+ recover: 3,
+ },
+ ack_wait: Duration::from_millis(400),
+ call_timeout: Duration::from_secs(4),
+ data_timeout: Duration::from_secs(6),
+ handback_after: Duration::from_millis(1500),
+ dial_backoff_min: Duration::from_millis(20),
+ dial_backoff_max: Duration::from_millis(250),
+ takeover_cooldown: Duration::from_millis(300),
+ idle_takeover: Duration::from_millis(2500),
+ snapshot_every: Duration::from_secs(5),
+ ..Config::standard()
+ }
+ }
+}
+
+/// One machine of the group as the supervisor sees it.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct MachineView {
+ pub id: MachineId,
+ pub name: String,
+ pub is_me: bool,
+ /// Reachable now (this machine counts as up unless it is going to sleep).
+ pub alive: bool,
+ /// Confirmed down: silent through the whole confirmation window, said
+ /// goodbye, or refused connections. Not merely slow.
+ pub down: bool,
+ /// Up without a break for at least the hand-back time.
+ pub stable: bool,
+ /// Holds a full copy of the volume.
+ pub replica: bool,
+ pub facts: Facts,
+}
+
+/// What a machine's power state does to the volume.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum PowerEvent {
+ /// The machine is about to sleep (lid closed, idle sleep): tell peers.
+ WillSleep,
+ /// It woke.
+ DidWake,
+ /// The network went away.
+ NetworkLost,
+ /// The network is back (or changed address): dial again now.
+ NetworkUp,
+}
+
+pub(super) struct PeerState {
+ pub det: Detector,
+ pub facts: Facts,
+ pub applied: Pos,
+ pub tail: Pos,
+ /// Owner's view: the replica has applied through this.
+ pub acked: Pos,
+ pub pending: u64,
+ pub problem: Option<String>,
+ /// When the detector last turned alive and stayed so.
+ pub alive_since: Option<Instant>,
+ pub last_health: Health,
+ pub next_dial: Instant,
+ pub backoff: Duration,
+ pub dialing: bool,
+ /// A promise given to a takeover candidate: (epoch, until).
+ pub promised: Option<(u64, Instant)>,
+ pub heard_ms: u64,
+ /// Since when the detector says it is down.
+ pub down_since: Option<Instant>,
+ /// It said goodbye on purpose (sleep, shutdown): no flapping to fear.
+ pub intentional: bool,
+ /// Dials in a row the machine refused (or hung up on before hello).
+ pub refusals: u32,
+ /// When that run of refusals began.
+ pub first_refusal: Option<Instant>,
+ /// How long its acknowledgements take (a moving average, ms): a machine
+ /// that takes longer than half the wait is not waited for.
+ pub ack_rtt_ms: f64,
+ /// Writes in a row that waited for its acknowledgement in vain: a
+ /// machine that does not answer is not waited for until it does.
+ pub missed_acks: u32,
+}
+
+pub(super) struct State {
+ pub members: BTreeMap<MachineId, Member>,
+ pub eras: Eras,
+ pub log: OpLog,
+ pub outbox: Outbox,
+ pub peers: HashMap<MachineId, PeerState>,
+ pub volume_bytes: u64,
+ pub last_takeover_try: Option<Instant>,
+ pub takeovers: u64,
+ pub handbacks: u64,
+ pub fenced_writes: u64,
+ pub conflict_copies: u64,
+ /// Our own recent writes by position, for the read-your-writes wait.
+ pub healthy: bool,
+ /// When a local operation last needed to order a write itself.
+ pub authority_wanted: Option<Instant>,
+ /// When each recent record was written here (to time acknowledgements).
+ pub written_at: std::collections::BTreeMap<Pos, Instant>,
+}
+
+#[derive(Default)]
+pub(super) struct PeerLinks {
+ pub ctl: Option<Arc<Link>>,
+ pub data: Option<Arc<Link>>,
+}
+
+pub(super) struct Inner {
+ pub me: MachineId,
+ pub name: String,
+ pub cfg: Config,
+ pub store: FsBackend,
+ pub spool: FsBackend,
+ pub dir: PathBuf,
+ pub transport: Arc<dyn Transport>,
+ pub counters: Arc<Counters>,
+ pub st: tokio::sync::Mutex<State>,
+ /// One applier of records, snapshots and rebases at a time.
+ pub gate: tokio::sync::Mutex<()>,
+ pub links: Mutex<HashMap<MachineId, PeerLinks>>,
+ pub facts: Mutex<Facts>,
+ pub wake: Notify,
+ /// Wakes the tick loop early (a link dropped, authority is wanted, a
+ /// machine said goodbye).
+ pub tick_wake: Notify,
+ pub acks: watch::Sender<u64>,
+ pub applied_tx: watch::Sender<Pos>,
+ pub closed: AtomicBool,
+ pub frozen: watch::Sender<bool>,
+ pub is_client: AtomicBool,
+ pub started: AtomicBool,
+ pub hashes: Mutex<HashMap<String, Arc<Vec<String>>>>,
+ pub seq_ack_counter: AtomicU64,
+ pub self_away: AtomicBool,
+ pub last_tick: Mutex<Instant>,
+ pub submitting: AtomicBool,
+ pub shipping: Shipping,
+ pub takeover_busy: AtomicBool,
+ pub handoff_busy: AtomicBool,
+}
+
+/// One machine's replicated volume.
+#[derive(Clone)]
+pub struct ReplicaNode {
+ pub(super) inner: Arc<Inner>,
+}
+
+impl std::fmt::Debug for ReplicaNode {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ write!(f, "ReplicaNode({})", self.inner.me)
+ }
+}
+
+/// A dial that failed because the machine said no, not because nothing
+/// answered.
+fn is_refusal(e: &Error) -> bool {
+ let m = e.to_string();
+ m.contains("refused") || m.contains("did not say hello") || m.contains("Connection reset")
+}
+
+fn load_eras(dir: &Path) -> Option<Eras> {
+ serde_json::from_slice(&std::fs::read(dir.join("era.json")).ok()?).ok()
+}
+
+pub(super) fn save_eras(dir: &Path, e: &Eras) -> Result<()> {
+ let p = dir.join("era.json");
+ cua_home::guard_write(&p)?;
+ let tmp = p.with_extension("tmp");
+ std::fs::write(&tmp, serde_json::to_vec(e)?)?;
+ std::fs::rename(&tmp, &p)?;
+ Ok(())
+}
+
+impl ReplicaNode {
+ /// Opens this machine's replica state under `state_dir/replica` around
+ /// `store` (the volume's local copy). `members` is the machines list,
+ /// this machine included.
+ pub async fn open(
+ cfg: Config,
+ me: &str,
+ name: &str,
+ state_dir: &Path,
+ store: FsBackend,
+ transport: Arc<dyn Transport>,
+ members: Vec<Member>,
+ ) -> Result<ReplicaNode> {
+ let dir = state_dir.join("replica");
+ std::fs::create_dir_all(&dir)?;
+ // A directory a newer Cua Spaces prepared is refused, not guessed at.
+ super::migrate::check_format(&dir)?;
+ let mut log = OpLog::open(&dir, cfg.log_keep)?;
+ let eras = match load_eras(&dir) {
+ Some(e) => e,
+ None => {
+ let e = Eras::genesis(me, now_ms());
+ // The first time replication touches a Volume that has
+ // files: keep a copy of its heads before anything moves.
+ let has_files = !store.list("").await?.is_empty();
+ if has_files {
+ super::migrate::backup_heads(&dir, store.root())?;
+ }
+ save_eras(&dir, &e)?;
+ // A volume that already holds files starts its log after
+ // them: a machine that joins gets a snapshot, not a replay
+ // of history this log never recorded.
+ if has_files {
+ log.reset_to(Pos { epoch: 1, seq: 0 })?;
+ }
+ e
+ }
+ };
+ let outbox = Outbox::open(&dir)?;
+ let spool = FsBackend::new(dir.join("spool"));
+ let (acks, _) = watch::channel(0u64);
+ let (applied_tx, _) = watch::channel(log.tail());
+ let (frozen, _) = watch::channel(false);
+ let mut m: BTreeMap<MachineId, Member> =
+ members.into_iter().map(|x| (x.id.clone(), x)).collect();
+ m.entry(me.to_string()).or_insert_with(|| Member {
+ id: me.to_string(),
+ name: name.to_string(),
+ ..Default::default()
+ });
+ let inner = Arc::new(Inner {
+ me: me.to_string(),
+ name: name.to_string(),
+ cfg,
+ store,
+ spool,
+ dir,
+ transport,
+ counters: Arc::new(Counters::default()),
+ st: tokio::sync::Mutex::new(State {
+ members: m,
+ eras,
+ log,
+ outbox,
+ peers: HashMap::new(),
+ volume_bytes: 0,
+ last_takeover_try: None,
+ takeovers: 0,
+ handbacks: 0,
+ fenced_writes: 0,
+ conflict_copies: 0,
+ healthy: true,
+ authority_wanted: None,
+ written_at: Default::default(),
+ }),
+ gate: tokio::sync::Mutex::new(()),
+ links: Mutex::new(HashMap::new()),
+ facts: Mutex::new(Facts::default()),
+ wake: Notify::new(),
+ tick_wake: Notify::new(),
+ acks,
+ applied_tx,
+ closed: AtomicBool::new(false),
+ frozen,
+ is_client: AtomicBool::new(false),
+ started: AtomicBool::new(false),
+ hashes: Mutex::new(HashMap::new()),
+ seq_ack_counter: AtomicU64::new(0),
+ self_away: AtomicBool::new(false),
+ last_tick: Mutex::new(Instant::now()),
+ submitting: AtomicBool::new(false),
+ shipping: Mutex::new(HashMap::new()),
+ takeover_busy: AtomicBool::new(false),
+ handoff_busy: AtomicBool::new(false),
+ });
+ inner.recover_intent().await?;
+ inner.adopt_unlogged().await?;
+ {
+ let mut st = inner.st.lock().await;
+ inner.refresh_roles(&mut st);
+ }
+ Ok(ReplicaNode { inner })
+ }
+
+ /// The `Backend` the drive uses.
+ pub fn backend(&self) -> Arc<ReplicatedBackend> {
+ Arc::new(ReplicatedBackend::new(self.inner.clone()))
+ }
+
+ /// Starts the tick loop and the shippers.
+ pub fn start(&self) {
+ if self.inner.started.swap(true, Ordering::SeqCst) {
+ return;
+ }
+ let i = self.inner.clone();
+ tokio::spawn(async move { i.tick_loop().await });
+ let i = self.inner.clone();
+ tokio::spawn(async move { i.ship_all_loop().await });
+ }
+
+ /// Says goodbye to every peer and stops.
+ pub async fn shutdown(&self) {
+ let first = !self.inner.closed.load(Ordering::SeqCst);
+ self.inner.say_bye_and_flush("shutdown").await;
+ // Only the first stop records the heads: a second one (after something
+ // else wrote to the store) would hide those writes.
+ if first {
+ self.inner.save_snapshot().await;
+ }
+ self.inner.closed.store(true, Ordering::SeqCst);
+ self.inner.wake.notify_waiters();
+ let links: Vec<Arc<Link>> = {
+ let mut l = self.inner.links.lock().expect("links");
+ let v = l
+ .values()
+ .flat_map(|p| [p.ctl.clone(), p.data.clone()])
+ .flatten()
+ .collect();
+ l.clear();
+ v
+ };
+ tokio::time::sleep(Duration::from_millis(20)).await;
+ for l in links {
+ l.close();
+ }
+ }
+
+ /// A stream arrived for this machine.
+ pub async fn accept(&self, stream: Stream) {
+ let i = self.inner.clone();
+ tokio::spawn(async move {
+ let _ = i.accept_stream(stream).await;
+ });
+ }
+
+ /// The machines list changed.
+ pub async fn set_members(&self, members: Vec<Member>) {
+ let mut st = self.inner.st.lock().await;
+ for m in members {
+ let cur = st.members.entry(m.id.clone()).or_default();
+ cur.id = m.id.clone();
+ cur.name = m.name;
+ if m.facts != Facts::default() {
+ cur.facts = m.facts;
+ }
+ }
+ self.inner.refresh_roles(&mut st);
+ }
+
+ /// This machine's reliability facts changed.
+ pub fn set_facts(&self, f: Facts) {
+ *self.inner.facts.lock().expect("facts") = f;
+ }
+
+ /// A power or network event from the operating system.
+ pub async fn notify_power(&self, ev: PowerEvent) {
+ match ev {
+ PowerEvent::WillSleep => {
+ self.inner.say_bye_and_flush("sleep").await;
+ self.inner.self_away.store(true, Ordering::SeqCst);
+ }
+ PowerEvent::NetworkLost => {
+ self.inner.say_bye_and_flush("network_lost").await;
+ self.inner.self_away.store(true, Ordering::SeqCst);
+ self.inner.drop_links();
+ }
+ PowerEvent::DidWake | PowerEvent::NetworkUp => {
+ self.inner.self_away.store(false, Ordering::SeqCst);
+ let now = Instant::now();
+ let mut st = self.inner.st.lock().await;
+ for p in st.peers.values_mut() {
+ p.det.resume(now);
+ p.next_dial = now;
+ p.backoff = self.inner.cfg.dial_backoff_min;
+ }
+ drop(st);
+ self.inner.drop_links();
+ *self.inner.last_tick.lock().expect("tick") = now;
+ }
+ }
+ }
+
+ /// Freezes this node (a stopped process: nothing is read, written or
+ /// timed until it thaws). Simulations only; the real test uses SIGSTOP.
+ pub fn freeze(&self, on: bool) {
+ self.inner.frozen.send_replace(on);
+ if !on {
+ self.inner.wake.notify_waiters();
+ }
+ }
+
+ pub fn machine(&self) -> &str {
+ &self.inner.me
+ }
+
+ /// This machine's name in the machines list.
+ pub fn name(&self) -> &str {
+ &self.inner.name
+ }
+
+ /// Every machine of the group with what is known about it now.
+ pub async fn machines(&self) -> Vec<MachineView> {
+ let st = self.inner.st.lock().await;
+ let layout = self.inner.layout_locked(&st);
+ let handback = self.inner.cfg.handback_after;
+ let mut out = vec![];
+ for (id, m) in &st.members {
+ let facts = self.inner.member_facts(&st, id);
+ if *id == self.inner.me {
+ out.push(MachineView {
+ id: id.clone(),
+ name: self.inner.name.clone(),
+ is_me: true,
+ alive: !self.inner.self_away.load(Ordering::SeqCst),
+ down: false,
+ stable: true,
+ replica: layout.role_of(id) == Role::Replica,
+ facts,
+ });
+ continue;
+ }
+ let p = st.peers.get(id);
+ let alive = p.is_some_and(|p| p.last_health == Health::Alive)
+ && self.inner.ctl_link(id).is_some();
+ let stable =
+ alive && p.is_some_and(|p| p.alive_since.is_some_and(|t| t.elapsed() >= handback));
+ out.push(MachineView {
+ id: id.clone(),
+ name: if m.name.is_empty() {
+ id.clone()
+ } else {
+ m.name.clone()
+ },
+ is_me: false,
+ alive,
+ down: p.is_none_or(|p| {
+ p.last_health.is_down() && p.det.down_and_silent(Instant::now())
+ }),
+ stable,
+ replica: layout.role_of(id) == Role::Replica,
+ facts,
+ });
+ }
+ out
+ }
+
+ /// Whether this machine owns the volume now.
+ pub async fn is_owner(&self) -> bool {
+ self.inner.st.lock().await.eras.owner() == self.inner.me
+ }
+
+ /// Bytes the links carried (sent, received).
+ pub fn wire_bytes(&self) -> (u64, u64) {
+ (
+ self.inner.counters.sent.load(Ordering::Relaxed),
+ self.inner.counters.received.load(Ordering::Relaxed),
+ )
+ }
+
+ /// What this machine's log and outbox know about `key` (for a failure's
+ /// report).
+ pub async fn debug_key(&self, key: &str) -> String {
+ let st = self.inner.st.lock().await;
+ let mut out = String::new();
+ for r in st.log.range(Pos::ZERO, st.log.tail()) {
+ if r.op.key() == key {
+ let what = match &r.op {
+ super::oplog::Op::Put { meta, .. } => format!("put {}", meta.version),
+ super::oplog::Op::Delete { version, .. } => format!("delete {version}"),
+ };
+ out.push_str(&format!(" log@{}:{what}", r.pos));
+ }
+ }
+ for p in st.outbox.items().iter().filter(|p| p.key == key) {
+ out.push_str(&format!(
+ " pending({})",
+ p.meta
+ .as_ref()
+ .map_or("delete".into(), |m| m.version.clone())
+ ));
+ }
+ out
+ }
+
+ pub async fn status(&self) -> ReplicaStatus {
+ self.inner.status().await
+ }
+
+ /// The facts for the one plain paragraph (see `words`).
+ pub async fn summary(&self, run_on: Option<&str>) -> Summary {
+ self.inner.summary(run_on).await
+ }
+}
+
+impl Inner {
+ pub(super) fn wall_ms(&self) -> u64 {
+ (now_ms() as i64 + self.cfg.wall_skew_ms).max(0) as u64
+ }
+
+ pub(super) fn my_facts(&self) -> Facts {
+ let mut f = self.facts.lock().expect("facts").clone();
+ if f.free_disk == 0 {
+ f.free_disk = self.store.free_bytes().unwrap_or(0);
+ }
+ f
+ }
+
+ pub(super) async fn wait_unfrozen(&self) {
+ let mut rx = self.frozen.subscribe();
+ while *rx.borrow() {
+ if rx.changed().await.is_err() {
+ return;
+ }
+ }
+ }
+
+ pub(super) fn handler(self: &Arc<Self>) -> Handler {
+ let me = self.clone();
+ Arc::new(move |link, frame, data| {
+ let me = me.clone();
+ Box::pin(async move {
+ me.wait_unfrozen().await;
+ me.handle(link, frame, data).await;
+ })
+ })
+ }
+
+ /// Recomputes who should hold copies and whether this machine is a
+ /// client (state lock held).
+ pub(super) fn refresh_roles(&self, st: &mut State) {
+ let l = self.layout_locked(st);
+ self.is_client
+ .store(l.role_of(&self.me) == Role::Client, Ordering::SeqCst);
+ }
+
+ pub(super) fn member_facts(&self, st: &State, id: &str) -> Facts {
+ if id == self.me {
+ return self.my_facts();
+ }
+ st.peers
+ .get(id)
+ .map(|p| p.facts.clone())
+ .filter(|f| *f != Facts::default())
+ .or_else(|| st.members.get(id).map(|m| m.facts.clone()))
+ .unwrap_or_default()
+ }
+
+ /// Members that count (heard from recently, or never heard but listed).
+ pub(super) fn active_members(&self, st: &State) -> Vec<MachineId> {
+ let now = self.wall_ms();
+ let win = self.cfg.active_window.as_millis() as u64;
+ st.members
+ .values()
+ .filter(|m| {
+ m.id == self.me
+ || st
+ .peers
+ .get(&m.id)
+ .map_or(m.last_seen_ms, |p| p.heard_ms.max(m.last_seen_ms))
+ == 0
+ || now.saturating_sub(
+ st.peers
+ .get(&m.id)
+ .map_or(m.last_seen_ms, |p| p.heard_ms.max(m.last_seen_ms)),
+ ) < win
+ })
+ .map(|m| m.id.clone())
+ .collect()
+ }
+
+ pub(super) fn layout_locked(&self, st: &State) -> Layout {
+ let active = self.active_members(st);
+ let members: Vec<Member> = active
+ .iter()
+ .filter_map(|id| st.members.get(id))
+ .map(|m| Member {
+ facts: self.member_facts(st, &m.id),
+ ..m.clone()
+ })
+ .collect();
+ roles::layout(
+ &members,
+ roles::replica_count(members.len()),
+ st.volume_bytes,
+ )
+ }
+
+ fn beat(&self, st: &State) -> Beat {
+ let eras: Vec<EraRec> = st.eras.recs.iter().rev().take(4).rev().cloned().collect();
+ Beat {
+ epoch: st.eras.epoch(),
+ owner: st.eras.owner().to_string(),
+ lineage: st.eras.lineage.clone(),
+ eras,
+ problem: if st.healthy {
+ None
+ } else {
+ Some("this machine cannot store more of the volume".into())
+ },
+ applied: st.log.tail(),
+ tail: st.log.tail(),
+ facts: self.my_facts(),
+ pending: st.outbox.len() as u64,
+ }
+ }
+
+ fn hello(&self, st: &State, chan: Chan) -> Hello {
+ Hello {
+ machine: self.me.clone(),
+ name: self.name.clone(),
+ chan,
+ lineage: st.eras.lineage.clone(),
+ eras: st.eras.recs.iter().rev().take(8).rev().cloned().collect(),
+ applied: st.log.tail(),
+ tail: st.log.tail(),
+ facts: self.my_facts(),
+ v: wire::VERSION,
+ }
+ }
+
+ pub(super) fn peer_state<'a>(&self, st: &'a mut State, id: &str) -> &'a mut PeerState {
+ let timing = self.cfg.timing;
+ let min = self.cfg.dial_backoff_min;
+ st.peers.entry(id.to_string()).or_insert_with(|| PeerState {
+ det: Detector::new(timing, Instant::now()),
+ facts: Facts::default(),
+ applied: Pos::ZERO,
+ tail: Pos::ZERO,
+ acked: Pos::ZERO,
+ pending: 0,
+ problem: None,
+ alive_since: None,
+ last_health: Health::Alive,
+ next_dial: Instant::now(),
+ backoff: min,
+ dialing: false,
+ promised: None,
+ heard_ms: 0,
+ down_since: None,
+ intentional: false,
+ missed_acks: 0,
+ refusals: 0,
+ first_refusal: None,
+ ack_rtt_ms: 0.0,
+ })
+ }
+
+ pub(super) fn say_bye(&self, reason: &str) -> Vec<Arc<Link>> {
+ let links = self.links.lock().expect("links");
+ let mut sent = vec![];
+ for p in links.values() {
+ if let Some(l) = &p.ctl {
+ l.send(
+ Msg::Bye {
+ reason: reason.into(),
+ },
+ &[],
+ );
+ sent.push(l.clone());
+ }
+ }
+ sent
+ }
+
+ /// Says goodbye and waits (a moment) for it to leave: a machine about
+ /// to sleep has only a moment.
+ pub(super) async fn say_bye_and_flush(&self, reason: &str) {
+ for l in self.say_bye(reason) {
+ l.flush(Duration::from_millis(500)).await;
+ }
+ }
+
+ pub(super) fn drop_links(&self) {
+ let links: Vec<Arc<Link>> = {
+ let mut l = self.links.lock().expect("links");
+ let v = l
+ .values()
+ .flat_map(|p| [p.ctl.clone(), p.data.clone()])
+ .flatten()
+ .collect();
+ l.clear();
+ v
+ };
+ for l in links {
+ l.close();
+ }
+ }
+
+ pub(super) fn ctl_link(&self, peer: &str) -> Option<Arc<Link>> {
+ self.links
+ .lock()
+ .expect("links")
+ .get(peer)
+ .and_then(|p| p.ctl.clone())
+ .filter(|l| !l.is_closed())
+ }
+
+ pub(super) fn data_link_now(&self, peer: &str) -> Option<Arc<Link>> {
+ self.links
+ .lock()
+ .expect("links")
+ .get(peer)
+ .and_then(|p| p.data.clone())
+ .filter(|l| !l.is_closed())
+ }
+
+ /// The data link to `peer`, dialing it if this machine has none.
+ pub(super) async fn data_link(self: &Arc<Self>, peer: &str) -> Option<Arc<Link>> {
+ if let Some(l) = self.data_link_now(peer) {
+ return Some(l);
+ }
+ self.dial_chan(peer, Chan::Data).await.ok()
+ }
+
+ fn register(&self, link: &Arc<Link>) {
+ let mut links = self.links.lock().expect("links");
+ let e = links.entry(link.peer.clone()).or_default();
+ match link.chan {
+ Chan::Control => e.ctl = Some(link.clone()),
+ Chan::Data => e.data = Some(link.clone()),
+ }
+ }
+
+ fn unregister(&self, link: &Arc<Link>) {
+ let mut links = self.links.lock().expect("links");
+ if let Some(e) = links.get_mut(&link.peer) {
+ let slot = match link.chan {
+ Chan::Control => &mut e.ctl,
+ Chan::Data => &mut e.data,
+ };
+ if slot.as_ref().is_some_and(|l| Arc::ptr_eq(l, link)) {
+ *slot = None;
+ }
+ }
+ drop(links);
+ // A link that dropped is redialed now, not at the next tick.
+ self.tick_wake.notify_one();
+ }
+
+ async fn handshake(
+ self: &Arc<Self>,
+ mut stream: Stream,
+ chan: Chan,
+ initiated: bool,
+ expect: Option<&str>,
+ ) -> Result<Arc<Link>> {
+ use tokio::io::AsyncWriteExt as _;
+ let hello = {
+ let st = self.st.lock().await;
+ self.hello(&st, chan)
+ };
+ let frame = Frame {
+ id: 0,
+ re: None,
+ msg: Msg::Hello(hello),
+ };
+ let bytes = wire::encode(&frame, &[])?;
+ let to = self.cfg.call_timeout;
+ let exchange = async {
+ stream.write_all(&bytes).await?;
+ stream.flush().await?;
+ wire::read(&mut stream).await
+ };
+ let got = tokio::time::timeout(to, exchange)
+ .await
+ .map_err(|_| Error::Backend("handshake timed out".into()))??;
+ let Some((
+ Frame {
+ msg: Msg::Hello(h), ..
+ },
+ _,
+ )) = got
+ else {
+ return Err(Error::Backend("the machine did not say hello".into()));
+ };
+ if let Some(e) = expect
+ && e != h.machine
+ {
+ return Err(Error::Backend(format!(
+ "dialed {e} and {} answered",
+ h.machine
+ )));
+ }
+ {
+ let st = self.st.lock().await;
+ if !st.members.contains_key(&h.machine) {
+ return Err(Error::Forbidden(format!(
+ "{} is not one of this account's machines",
+ h.machine
+ )));
+ }
+ }
+ let me = self.clone();
+ let me2 = self.clone();
+ let link = Link::spawn(
+ stream,
+ h.machine.clone(),
+ h.name.clone(),
+ chan,
+ initiated,
+ self.counters.clone(),
+ self.handler(),
+ move |l| {
+ me2.unregister(&l);
+ },
+ );
+ *link.hello.lock().expect("hello") = Some(h.clone());
+ me.register(&link);
+ if chan == Chan::Control {
+ me.peer_up(&link, h).await;
+ }
+ Ok(link)
+ }
+
+ async fn accept_stream(self: &Arc<Self>, stream: Stream) -> Result<()> {
+ if self.closed.load(Ordering::SeqCst) {
+ return Ok(());
+ }
+ // The chan is in the peer's hello; read it from a peek of our own
+ // handshake: both sides send first, so the chan we answer with is
+ // the one the dialer chose. Read first, then answer.
+ use tokio::io::AsyncWriteExt as _;
+ let mut stream = stream;
+ let first = tokio::time::timeout(self.cfg.call_timeout, wire::read(&mut stream))
+ .await
+ .map_err(|_| Error::Backend("handshake timed out".into()))??;
+ let Some((
+ Frame {
+ msg: Msg::Hello(h), ..
+ },
+ _,
+ )) = first
+ else {
+ return Err(Error::Backend("no hello".into()));
+ };
+ {
+ let st = self.st.lock().await;
+ if !st.members.contains_key(&h.machine) {
+ return Err(Error::Forbidden(format!(
+ "{} is not one of this account's machines",
+ h.machine
+ )));
+ }
+ }
+ let mine = {
+ let st = self.st.lock().await;
+ self.hello(&st, h.chan)
+ };
+ let bytes = wire::encode(
+ &Frame {
+ id: 0,
+ re: None,
+ msg: Msg::Hello(mine),
+ },
+ &[],
+ )?;
+ stream.write_all(&bytes).await?;
+ stream.flush().await?;
+ let me2 = self.clone();
+ let link = Link::spawn(
+ stream,
+ h.machine.clone(),
+ h.name.clone(),
+ h.chan,
+ false,
+ self.counters.clone(),
+ self.handler(),
+ move |l| me2.unregister(&l),
+ );
+ *link.hello.lock().expect("hello") = Some(h.clone());
+ self.register(&link);
+ if h.chan == Chan::Control {
+ self.peer_up(&link, h).await;
+ }
+ Ok(())
+ }
+
+ pub(super) async fn dial_chan(self: &Arc<Self>, peer: &str, chan: Chan) -> Result<Arc<Link>> {
+ let stream = self.transport.dial(peer).await.map_err(|e| {
+ if e.kind() == std::io::ErrorKind::ConnectionRefused {
+ Error::Backend(format!("connection refused: {e}"))
+ } else {
+ Error::from(e)
+ }
+ })?;
+ self.handshake(stream, chan, true, Some(peer)).await
+ }
+
+ /// A control link came up (either direction).
+ async fn peer_up(self: &Arc<Self>, _link: &Arc<Link>, h: Hello) {
+ let now = Instant::now();
+ let mut reconcile = false;
+ let mut join = false;
+ {
+ let mut st = self.st.lock().await;
+ if let Some(m) = st.members.get_mut(&h.machine) {
+ m.name = h.name.clone();
+ m.last_seen_ms = self.wall_ms();
+ }
+ if h.lineage != st.eras.lineage {
+ // Two volumes that started apart: the less reliable machine
+ // joins the other's.
+ let mine = self.my_facts();
+ let theirs = h.facts.clone();
+ join = (mine.rank, &self.me) < (theirs.rank, &h.machine);
+ // Equal rank: the lower id joins (a deterministic pick).
+ let _ = mine;
+ } else {
+ let advanced = st.eras.adopt_from(&h.eras);
+ if advanced {
+ let _ = save_eras(&self.dir, &st.eras);
+ reconcile = true;
+ }
+ }
+ let owner_now = st.eras.owner() == self.me;
+ let other_lineage = h.lineage != st.eras.lineage;
+ let divergence = st.eras.divergence(h.applied);
+ let p = self.peer_state(&mut st, &h.machine);
+ p.det.arrival(now);
+ p.facts = h.facts.clone();
+ p.heard_ms = self.wall_ms();
+ p.applied = h.applied;
+ p.tail = h.tail;
+ // What a machine of another volume has applied says nothing
+ // about this one.
+ p.acked = if other_lineage {
+ Pos::ZERO
+ } else {
+ divergence.unwrap_or(h.applied)
+ };
+ p.missed_acks = 0;
+ p.backoff = self.cfg.dial_backoff_min;
+ p.dialing = false;
+ let _ = owner_now;
+ if reconcile || st.eras.divergence(st.log.tail()).is_some() {
+ reconcile = true;
+ }
+ self.refresh_roles(&mut st);
+ }
+ if join {
+ let me = self.clone();
+ let h2 = h.clone();
+ tokio::spawn(async move { me.join_lineage(h2).await });
+ } else if reconcile {
+ let me = self.clone();
+ tokio::spawn(async move { me.reconcile_eras().await });
+ }
+ self.wake.notify_one();
+ }
+
+ /// Frame dispatch.
+ async fn handle(self: &Arc<Self>, link: Arc<Link>, frame: Frame, data: Vec<u8>) {
+ if self.closed.load(Ordering::SeqCst) {
+ return;
+ }
+ let id = frame.id;
+ match frame.msg {
+ Msg::Ping(b) => {
+ self.on_beat(&link.peer, b).await;
+ let mine = {
+ let st = self.st.lock().await;
+ self.beat(&st)
+ };
+ link.reply(id, Msg::Pong(mine), &[]);
+ }
+ Msg::Bye { reason } => self.on_bye(&link.peer, &reason).await,
+ Msg::Append { epoch, records } => {
+ let reply = self.handle_append(&link, epoch, records, data).await;
+ link.reply(id, reply, &[]);
+ }
+ Msg::Snapshot { at } => {
+ let reply = self.handle_snapshot(&link, at).await;
+ link.reply(id, reply, &[]);
+ }
+ Msg::Prepare { epoch, base } => {
+ let reply = self.handle_prepare(&link.peer, epoch, base).await;
+ link.reply(id, reply, &[]);
+ }
+ Msg::Era { recs } => {
+ self.handle_era(&link.peer, recs).await;
+ link.reply(id, Msg::Ack { applied: Pos::ZERO }, &[]);
+ }
+ Msg::Handoff { epoch, to, base } => {
+ let reply = self.handle_handoff(&link.peer, epoch, to, base).await;
+ link.reply(id, reply, &[]);
+ }
+ Msg::Submit { ops } => {
+ let reply = self.handle_submit(&link, ops, data).await;
+ link.reply(id, reply, &[]);
+ }
+ Msg::Req(r) => {
+ let (rep, out) = self.handle_req(&link, r, data).await;
+ link.reply(id, Msg::Rep(rep), &out);
+ }
+ Msg::Hashes {
+ src,
+ block,
+ from,
+ count,
+ } => {
+ let reply = self.handle_hashes(src, block, from, count).await;
+ link.reply(id, reply, &[]);
+ }
+ Msg::Block { src, block, idx } => match self.handle_block(src, block, idx).await {
+ Ok(bytes) => {
+ link.reply(id, Msg::BlockIs, &bytes);
+ }
+ Err(e) => {
+ link.reply(
+ id,
+ Msg::Error {
+ kind: e.tag().into(),
+ message: e.to_string(),
+ },
+ &[],
+ );
+ }
+ },
+ // Replies and handshakes are never dispatched.
+ _ => {}
+ }
+ }
+
+ pub(super) async fn on_beat(self: &Arc<Self>, peer: &str, b: Beat) {
+ let now = Instant::now();
+ let mut reconcile = false;
+ let mut join: Option<Hello> = None;
+ {
+ let mut st = self.st.lock().await;
+ if !b.lineage.is_empty() && b.lineage != st.eras.lineage {
+ // The peer belongs to another volume now (it joined one while
+ // we were connected): the less reliable of the two joins.
+ let mine = self.my_facts();
+ if (mine.rank, &self.me) < (b.facts.rank, &peer.to_string()) {
+ join = Some(Hello {
+ machine: peer.to_string(),
+ name: st
+ .members
+ .get(peer)
+ .map(|m| m.name.clone())
+ .unwrap_or_default(),
+ chan: Chan::Control,
+ lineage: b.lineage.clone(),
+ eras: b.eras.clone(),
+ applied: b.applied,
+ tail: b.tail,
+ facts: b.facts.clone(),
+ v: wire::VERSION,
+ });
+ }
+ } else if b.epoch > st.eras.epoch() && st.eras.adopt_from(&b.eras) {
+ let _ = save_eras(&self.dir, &st.eras);
+ reconcile = true;
+ }
+ if let Some(m) = st.members.get_mut(peer) {
+ m.last_seen_ms = self.wall_ms();
+ }
+ let wall = self.wall_ms();
+ let p = self.peer_state(&mut st, peer);
+ p.det.arrival(now);
+ p.facts = b.facts;
+ p.applied = b.applied;
+ p.tail = b.tail;
+ p.pending = b.pending;
+ p.problem = b.problem;
+ p.heard_ms = wall;
+ // The beat is the machine's own account of how far it got
+ // (it moves back when the machine puts itself right after a
+ // takeover, or starts over after joining another volume).
+ if b.applied > p.acked {
+ p.missed_acks = 0;
+ }
+ p.acked = b.applied;
+ }
+ if let Some(h) = join {
+ let me = self.clone();
+ tokio::spawn(async move { me.join_lineage(h).await });
+ } else if reconcile {
+ let me = self.clone();
+ tokio::spawn(async move { me.reconcile_eras().await });
+ }
+ self.acks
+ .send_replace(self.seq_ack_counter.fetch_add(1, Ordering::Relaxed));
+ }
+
+ pub(super) async fn on_bye(&self, peer: &str, reason: &str) {
+ let now = Instant::now();
+ let mut st = self.st.lock().await;
+ let p = self.peer_state(&mut st, peer);
+ p.det.bye(now);
+ p.intentional = matches!(reason, "sleep" | "shutdown");
+ tracing::info!(peer, reason, "replica: a machine said goodbye");
+ drop(st);
+ self.tick_wake.notify_one();
+ self.wake.notify_one();
+ }
+}
+
+impl Inner {
+ /// The tick: heartbeats, health, dialing, takeover, hand-back, outbox.
+ pub(super) async fn tick_loop(self: Arc<Self>) {
+ let period = (self.cfg.timing.interval / 2).max(Duration::from_millis(10));
+ let mut last_ping = Instant::now() - self.cfg.timing.interval;
+ let mut last_size = Instant::now() - Duration::from_secs(3600);
+ let mut last_snap = Instant::now();
+ let mut snapped = Pos::ZERO;
+ loop {
+ tokio::select! {
+ _ = tokio::time::sleep(period) => {}
+ _ = self.tick_wake.notified() => {}
+ }
+ if self.closed.load(Ordering::SeqCst) {
+ return;
+ }
+ self.wait_unfrozen().await;
+ let now = Instant::now();
+ // Were we paused (lid closed, process stopped)? Then every
+ // peer looks silent through no fault of theirs.
+ let gap = {
+ let mut lt = self.last_tick.lock().expect("tick");
+ let g = now.saturating_duration_since(*lt);
+ *lt = now;
+ g
+ };
+ if gap > (self.cfg.timing.interval * 4).max(period * 6) {
+ let mut st = self.st.lock().await;
+ for p in st.peers.values_mut() {
+ p.det.resume(now);
+ }
+ tracing::info!(?gap, "replica: this machine was paused; forgiving silence");
+ }
+ if self.self_away.load(Ordering::SeqCst) {
+ continue;
+ }
+ if now.saturating_duration_since(last_ping) >= self.cfg.timing.interval {
+ last_ping = now;
+ self.ping_all().await;
+ }
+ if now.saturating_duration_since(last_size) >= Duration::from_secs(60) {
+ last_size = now;
+ self.measure_volume().await;
+ }
+ // A hard stop leaves the last snapshot, so keep it fresh.
+ if now.saturating_duration_since(last_snap) >= self.cfg.snapshot_every {
+ last_snap = now;
+ let tail = self.st.lock().await.log.tail();
+ if tail != snapped {
+ snapped = tail;
+ self.save_snapshot().await;
+ }
+ }
+ self.poll_health(now).await;
+ self.dial_missing().await;
+ // None of these may hold the tick: each can wait on the network
+ // (a vote, an offer, the outbox), and a tick that waits is a
+ // machine that stops noticing its peers.
+ if !self.takeover_busy.swap(true, Ordering::SeqCst) {
+ let me = self.clone();
+ tokio::spawn(async move {
+ me.maybe_takeover().await;
+ me.takeover_busy.store(false, Ordering::SeqCst);
+ });
+ }
+ if !self.handoff_busy.swap(true, Ordering::SeqCst) {
+ let me = self.clone();
+ tokio::spawn(async move {
+ me.maybe_handoff().await;
+ me.handoff_busy.store(false, Ordering::SeqCst);
+ });
+ }
+ let me = self.clone();
+ tokio::spawn(async move { me.submit_outbox().await });
+ }
+ }
+
+ async fn measure_volume(&self) {
+ if let Ok(all) = crate::backend::Backend::list(&self.store, "").await {
+ let total: u64 = all.iter().map(|m| m.size).sum();
+ let mut st = self.st.lock().await;
+ st.volume_bytes = total;
+ self.refresh_roles(&mut st);
+ }
+ }
+
+ async fn ping_all(self: &Arc<Self>) {
+ let peers: Vec<(MachineId, Arc<Link>)> = {
+ let l = self.links.lock().expect("links");
+ l.iter()
+ .filter_map(|(k, v)| {
+ v.ctl
+ .clone()
+ .filter(|c| !c.is_closed())
+ .map(|c| (k.clone(), c))
+ })
+ .collect()
+ };
+ for (peer, link) in peers {
+ let me = self.clone();
+ tokio::spawn(async move {
+ let beat = {
+ let st = me.st.lock().await;
+ me.beat(&st)
+ };
+ let to = me.cfg.timing.floor.max(me.cfg.timing.interval * 3);
+ if let Ok((Msg::Pong(b), _)) = link.call(Msg::Ping(beat), &[], to).await {
+ me.on_beat(&peer, b).await;
+ }
+ });
+ }
+ }
+
+ async fn poll_health(&self, now: Instant) {
+ let mut st = self.st.lock().await;
+ let ids: Vec<MachineId> = st.peers.keys().cloned().collect();
+ let mut changed = false;
+ for id in ids {
+ let h = {
+ let p = st.peers.get_mut(&id).expect("peer");
+ let h = p.det.poll(now);
+ if h != p.last_health {
+ tracing::info!(me = %self.me, peer = %id, from = p.last_health.word(), to = h.word(), "replica: peer health");
+ if h.is_down() && p.down_since.is_none() {
+ p.down_since = Some(now);
+ }
+ if h == Health::Alive {
+ p.down_since = None;
+ p.intentional = false;
+ }
+ if h == Health::Alive && p.last_health.is_down() {
+ p.alive_since = Some(now);
+ }
+ if h != Health::Alive {
+ p.alive_since = None;
+ }
+ p.last_health = h;
+ changed = true;
+ }
+ if h == Health::Alive && p.alive_since.is_none() {
+ p.alive_since = Some(now);
+ }
+ h
+ };
+ // A silent link may be half open: drop it so a new one is dialed.
+ // A link that has only just come up has not had time to be
+ // silent: it is the way back, not the problem.
+ if matches!(h, Health::Suspect | Health::Dead) {
+ let threshold = st
+ .peers
+ .get(&id)
+ .map_or(Duration::ZERO, |p| p.det.threshold());
+ let l = {
+ let links = self.links.lock().expect("links");
+ links.get(&id).and_then(|p| p.ctl.clone())
+ };
+ if let Some(l) = l
+ && l.created.elapsed() > threshold
+ {
+ l.close();
+ self.unregister(&l);
+ }
+ }
+ }
+ if changed {
+ self.acks
+ .send_replace(self.seq_ack_counter.fetch_add(1, Ordering::Relaxed));
+ self.wake.notify_one();
+ }
+ }
+
+ async fn dial_missing(self: &Arc<Self>) {
+ let want: Vec<MachineId> = {
+ let st = self.st.lock().await;
+ st.members
+ .keys()
+ .filter(|k| **k != self.me)
+ .cloned()
+ .collect()
+ };
+ let now = Instant::now();
+ for id in want {
+ if self.ctl_link(&id).is_some() {
+ continue;
+ }
+ let go = {
+ let mut st = self.st.lock().await;
+ let p = self.peer_state(&mut st, &id);
+ if p.dialing || now < p.next_dial {
+ false
+ } else {
+ p.dialing = true;
+ true
+ }
+ };
+ if !go {
+ continue;
+ }
+ let me = self.clone();
+ tokio::spawn(async move {
+ let r = me.dial_chan(&id, Chan::Control).await;
+ let mut st = me.st.lock().await;
+ let max = me.cfg.dial_backoff_max;
+ let p = me.peer_state(&mut st, &id);
+ p.dialing = false;
+ if let Err(e) = &r {
+ // Right after a link drops the first few redials come
+ // fast (is the machine there?); only then does the wait
+ // grow.
+ let refusal = is_refusal(e);
+ if !(refusal && p.refusals < 3) {
+ p.backoff = (p.backoff * 2).min(max);
+ }
+ let jitter = Duration::from_millis(rand::random::<u64>() % 40);
+ p.next_dial = Instant::now() + p.backoff + jitter;
+ // Look again when the backoff ends, not at the next tick.
+ let (me2, after) = (me.clone(), p.backoff + jitter);
+ tokio::spawn(async move {
+ tokio::time::sleep(after).await;
+ me2.tick_wake.notify_one();
+ });
+ // Someone answered to say no (or hung up before hello):
+ // after three in a row, nobody is running the node.
+ if refusal {
+ p.refusals += 1;
+ let began = *p.first_refusal.get_or_insert_with(Instant::now);
+ // Three refusals in a row, over at least the silence
+ // floor: a link that flaps for a few seconds gives
+ // the same refusals and must not count as a death.
+ if p.refusals >= 3
+ && began.elapsed() >= me.cfg.timing.floor
+ && !p.last_health.is_down()
+ {
+ tracing::info!(peer = %id, "replica: the machine refuses connections; counting it as down");
+ p.det.refused(Instant::now());
+ p.last_health = Health::Dead;
+ p.down_since = Some(Instant::now());
+ drop(st);
+ me.wake.notify_one();
+ }
+ } else {
+ p.refusals = 0;
+ p.first_refusal = None;
+ }
+ } else {
+ p.refusals = 0;
+ p.first_refusal = None;
+ }
+ });
+ }
+ }
+
+ pub(super) async fn status(&self) -> ReplicaStatus {
+ let st = self.st.lock().await;
+ let layout = self.layout_locked(&st);
+ let owner = st.eras.owner().to_string();
+ let tail = st.log.tail();
+ let name_of = |id: &str| {
+ st.members
+ .get(id)
+ .map(|m| m.name.clone())
+ .filter(|n| !n.is_empty())
+ .unwrap_or_else(|| id.to_string())
+ };
+ let mut peers = vec![];
+ let mut copies_now = 1usize;
+ let owner_tail = if owner == self.me {
+ tail
+ } else {
+ st.peers.get(&owner).map_or(tail, |p| p.tail)
+ };
+ for (id, m) in &st.members {
+ if *id == self.me {
+ continue;
+ }
+ let p = st.peers.get(id);
+ let health = p.map_or(Health::Dead, |p| p.last_health);
+ let applied = p.map_or(Pos::ZERO, |p| {
+ if owner == self.me { p.acked } else { p.applied }
+ });
+ let role = layout.role_of(id);
+ let lag = if applied.epoch == owner_tail.epoch {
+ owner_tail.seq.saturating_sub(applied.seq)
+ } else if applied < owner_tail {
+ owner_tail.seq.max(1)
+ } else {
+ 0
+ };
+ if role == Role::Replica && !health.is_down() && lag == 0 && applied >= owner_tail {
+ copies_now += 1;
+ }
+ peers.push(PeerView {
+ id: id.clone(),
+ name: m.name.clone(),
+ role,
+ health: health.word().into(),
+ connected: self.ctl_link(id).is_some(),
+ owner: *id == owner,
+ applied,
+ lag,
+ problem: p.and_then(|p| p.problem.clone()),
+ });
+ }
+ let wanted = layout.replicas.len().max(1);
+ let unreplicated = if owner == self.me {
+ let mut acks: Vec<Pos> = layout
+ .replicas
+ .iter()
+ .filter(|r| **r != self.me)
+ .map(|r| st.peers.get(r).map_or(Pos::ZERO, |p| p.acked))
+ .collect();
+ acks.sort();
+ if wanted <= 1 {
+ 0
+ } else {
+ let best = acks.last().copied().unwrap_or(Pos::ZERO);
+ if best.epoch == tail.epoch {
+ tail.seq.saturating_sub(best.seq)
+ } else {
+ tail.seq
+ }
+ }
+ } else {
+ 0
+ };
+ ReplicaStatus {
+ me: self.me.clone(),
+ name: self.name.clone(),
+ role: layout.role_of(&self.me),
+ owner: owner.clone(),
+ owner_name: name_of(&owner),
+ i_own: owner == self.me,
+ epoch: st.eras.epoch(),
+ applied: tail,
+ tail,
+ peers,
+ pending: st
+ .outbox
+ .items()
+ .iter()
+ .map(|p| PendingView {
+ key: p.key.clone(),
+ size: p.meta.as_ref().map_or(0, |m| m.size),
+ since_ms: p.ts_ms,
+ deleted: p.meta.is_none(),
+ })
+ .collect(),
+ pending_bytes: st.outbox.bytes(),
+ copies_now,
+ copies_wanted: wanted,
+ unreplicated,
+ volume_bytes: st.volume_bytes,
+ bytes_sent: self.counters.sent.load(Ordering::Relaxed),
+ bytes_received: self.counters.received.load(Ordering::Relaxed),
+ takeovers: st.takeovers,
+ handbacks: st.handbacks,
+ fenced_writes: st.fenced_writes,
+ conflict_copies: st.conflict_copies,
+ }
+ }
+
+ pub(super) async fn summary(&self, run_on: Option<&str>) -> Summary {
+ let s = self.status().await;
+ let st = self.st.lock().await;
+ let facts = self.my_facts();
+ let layout = self.layout_locked(&st);
+ let mut holders_up = vec![];
+ let mut holders_away = vec![];
+ let mut hold = |name: String, up: bool| {
+ if up {
+ holders_up.push(name)
+ } else {
+ holders_away.push(name)
+ }
+ };
+ if layout.role_of(&self.me) == Role::Replica {
+ hold(self.name.clone(), true);
+ }
+ for p in &s.peers {
+ if p.role == Role::Replica {
+ hold(p.name.clone(), p.health == "up" || p.health == "slow");
+ }
+ }
+ let owner_name = s.owner_name.clone();
+ let pending_for = if s.pending.is_empty() {
+ None
+ } else {
+ Some(owner_name)
+ };
+ Summary {
+ this_machine: self.name.clone(),
+ run_on: run_on.map(str::to_string),
+ machines: st.members.len(),
+ holders_up,
+ holders_away,
+ pending_here: s.pending.len() as u64,
+ pending_for,
+ this_sleeps: facts.sleep_prone,
+ sleep_blocked: facts.sleep_prevented,
+ }
+ }
+}
diff --git a/libs/cua/crates/cua-volume/src/replica/oplog.rs b/libs/cua/crates/cua-volume/src/replica/oplog.rs
new file mode 100644
index 0000000000..56af9f13c0
--- /dev/null
+++ b/libs/cua/crates/cua-volume/src/replica/oplog.rs
@@ -0,0 +1,325 @@
+// SPDX-License-Identifier: FSL-1.1-MIT
+// Copyright (c) 2026 Cua AI, Inc.
+
+//! The replication log: what the owner did, in order.
+//!
+//! One JSON object per line in `oplog.jsonl`. Every machine that holds a
+//! copy keeps the records it applied (the owner the ones it made), so any
+//! of them can become the owner and replay history to the others. A record
+//! carries the previous head of its key (`prev`), so a machine that finds
+//! itself on the wrong side of a takeover can move its heads back without
+//! losing a version.
+
+use std::io::Write as _;
+use std::path::{Path, PathBuf};
+
+use serde::{Deserialize, Serialize};
+
+use super::era::Pos;
+use crate::backend::ObjectMeta;
+use crate::{Error, Result};
+
+/// What happened to one key.
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(tag = "op", rename_all = "snake_case")]
+pub enum Op {
+ /// `meta` became the head of its key.
+ Put {
+ meta: ObjectMeta,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ prev: Option<ObjectMeta>,
+ },
+ /// `key` was deleted (a marker with `version`).
+ Delete {
+ key: String,
+ version: String,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ prev: Option<ObjectMeta>,
+ },
+}
+
+impl Op {
+ pub fn key(&self) -> &str {
+ match self {
+ Op::Put { meta, .. } => &meta.key,
+ Op::Delete { key, .. } => key,
+ }
+ }
+
+ pub fn prev(&self) -> Option<&ObjectMeta> {
+ match self {
+ Op::Put { prev, .. } | Op::Delete { prev, .. } => prev.as_ref(),
+ }
+ }
+}
+
+/// One log entry.
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Record {
+ pub pos: Pos,
+ pub op: Op,
+ /// Wall time at the owner (display only).
+ #[serde(default)]
+ pub at_ms: u64,
+}
+
+#[derive(Serialize, Deserialize)]
+struct Header {
+ floor: Pos,
+}
+
+/// The log on disk plus the records kept in memory.
+#[derive(Debug)]
+pub struct OpLog {
+ path: PathBuf,
+ recs: Vec<Record>,
+ keep: usize,
+ /// Everything at or before this position is gone (trimmed, or the
+ /// machine joined by snapshot): a replica behind it needs a snapshot.
+ floor: Pos,
+}
+
+impl OpLog {
+ /// Opens (creating) `dir/oplog.jsonl`. A torn last line (a crash while
+ /// appending) is dropped.
+ pub fn open(dir: &Path, keep: usize) -> Result<OpLog> {
+ std::fs::create_dir_all(dir)?;
+ let path = dir.join("oplog.jsonl");
+ let mut recs = vec![];
+ let mut floor = Pos::ZERO;
+ if let Ok(text) = std::fs::read_to_string(&path) {
+ for line in text.lines() {
+ if let Ok(h) = serde_json::from_str::<Header>(line) {
+ floor = h.floor;
+ continue;
+ }
+ match serde_json::from_str::<Record>(line) {
+ Ok(r) => recs.push(r),
+ Err(_) => break,
+ }
+ }
+ }
+ let log = OpLog {
+ path,
+ recs,
+ keep,
+ floor,
+ };
+ log.rewrite()?;
+ Ok(log)
+ }
+
+ fn rewrite(&self) -> Result<()> {
+ let mut out = Vec::new();
+ serde_json::to_writer(&mut out, &Header { floor: self.floor })?;
+ out.push(b'\n');
+ for r in &self.recs {
+ serde_json::to_writer(&mut out, r)?;
+ out.push(b'\n');
+ }
+ cua_home::guard_write(&self.path)?;
+ let tmp = self.path.with_extension("tmp");
+ {
+ let mut f = std::fs::File::create(&tmp)?;
+ f.write_all(&out)?;
+ f.sync_all()?;
+ }
+ std::fs::rename(&tmp, &self.path)?;
+ Ok(())
+ }
+
+ /// The newest record's position (zero when empty).
+ pub fn tail(&self) -> Pos {
+ self.recs.last().map_or(self.floor, |r| r.pos)
+ }
+
+ /// Starts an empty log at `pos` (a machine that joined by snapshot has
+ /// no records before it).
+ pub fn reset_to(&mut self, pos: Pos) -> Result<()> {
+ self.recs.clear();
+ self.floor = pos;
+ self.rewrite()
+ }
+
+ /// Wall time of the newest record (what it recorded as `at_ms`).
+ pub fn newest_at_ms(&self) -> Option<u64> {
+ self.recs.iter().map(|r| r.at_ms).max()
+ }
+
+ /// The oldest record kept.
+ pub fn head(&self) -> Option<Pos> {
+ self.recs.first().map(|r| r.pos)
+ }
+
+ pub fn len(&self) -> usize {
+ self.recs.len()
+ }
+
+ pub fn is_empty(&self) -> bool {
+ self.recs.is_empty()
+ }
+
+ /// Appends `rec` (its position must be after the tail) and makes it
+ /// durable.
+ pub fn append(&mut self, rec: Record) -> Result<()> {
+ if rec.pos <= self.tail() {
+ return Err(Error::Backend(format!(
+ "oplog: {} is not after {}",
+ rec.pos,
+ self.tail()
+ )));
+ }
+ let mut line = serde_json::to_vec(&rec)?;
+ line.push(b'\n');
+ cua_home::guard_write(&self.path)?;
+ let mut f = std::fs::OpenOptions::new().append(true).open(&self.path)?;
+ f.write_all(&line)?;
+ f.sync_data()?;
+ self.recs.push(rec);
+ if self.recs.len() > self.keep.saturating_mul(2) {
+ let drop = self.recs.len() - self.keep;
+ if let Some(last) = self.recs[drop - 1..drop].first() {
+ self.floor = last.pos;
+ }
+ self.recs.drain(..drop);
+ self.rewrite()?;
+ }
+ Ok(())
+ }
+
+ /// Appends many records with one disk barrier (a batch of writes).
+ pub fn append_batch(&mut self, recs: Vec<Record>) -> Result<()> {
+ if recs.is_empty() {
+ return Ok(());
+ }
+ let mut prev = self.tail();
+ let mut bytes = Vec::new();
+ for r in &recs {
+ if r.pos <= prev {
+ return Err(Error::Backend(format!(
+ "oplog: {} is not after {prev}",
+ r.pos
+ )));
+ }
+ prev = r.pos;
+ serde_json::to_writer(&mut bytes, r)?;
+ bytes.push(b'\n');
+ }
+ cua_home::guard_write(&self.path)?;
+ let mut f = std::fs::OpenOptions::new().append(true).open(&self.path)?;
+ f.write_all(&bytes)?;
+ f.sync_data()?;
+ self.recs.extend(recs);
+ if self.recs.len() > self.keep.saturating_mul(2) {
+ let drop = self.recs.len() - self.keep;
+ if let Some(last) = self.recs[drop - 1..drop].first() {
+ self.floor = last.pos;
+ }
+ self.recs.drain(..drop);
+ self.rewrite()?;
+ }
+ Ok(())
+ }
+
+ /// Records after `pos`, at most `max`. `None` when the log no longer
+ /// reaches back to `pos` (the caller sends a snapshot instead).
+ pub fn after(&self, pos: Pos, max: usize) -> Option<Vec<Record>> {
+ if pos < self.floor {
+ return None;
+ }
+ let i = self.recs.partition_point(|r| r.pos <= pos);
+ Some(self.recs[i..].iter().take(max).cloned().collect())
+ }
+
+ /// Records in `(after, up_to]`.
+ pub fn range(&self, after: Pos, up_to: Pos) -> Vec<Record> {
+ self.recs
+ .iter()
+ .filter(|r| r.pos > after && r.pos <= up_to)
+ .cloned()
+ .collect()
+ }
+
+ /// Drops every record after `keep` and returns them (newest first):
+ /// the divergent tail.
+ pub fn cut_after(&mut self, keep: Pos) -> Result<Vec<Record>> {
+ let i = self.recs.partition_point(|r| r.pos <= keep);
+ let mut tail: Vec<Record> = self.recs.drain(i..).collect();
+ tail.reverse();
+ if !tail.is_empty() {
+ self.rewrite()?;
+ }
+ Ok(tail)
+ }
+
+ /// Whether the record at `pos` is held.
+ pub fn has(&self, pos: Pos) -> bool {
+ self.recs.binary_search_by(|r| r.pos.cmp(&pos)).is_ok()
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn put(epoch: u64, seq: u64, key: &str) -> Record {
+ Record {
+ pos: Pos { epoch, seq },
+ op: Op::Put {
+ meta: ObjectMeta {
+ key: key.into(),
+ size: 1,
+ etag: "e".into(),
+ version: format!("v{epoch}.{seq}"),
+ modified_ms: 0,
+ },
+ prev: None,
+ },
+ at_ms: 0,
+ }
+ }
+
+ #[test]
+ fn append_replay_cut_and_reopen() {
+ let d = tempfile::tempdir().unwrap();
+ let mut l = OpLog::open(d.path(), 100).unwrap();
+ for s in 1..=5 {
+ l.append(put(1, s, &format!("k{s}"))).unwrap();
+ }
+ assert!(l.append(put(1, 5, "dup")).is_err());
+ assert_eq!(l.tail(), Pos { epoch: 1, seq: 5 });
+ assert_eq!(l.after(Pos { epoch: 1, seq: 3 }, 10).unwrap().len(), 2);
+ assert_eq!(l.after(Pos::ZERO, 10).unwrap().len(), 5);
+ let tail = l.cut_after(Pos { epoch: 1, seq: 3 }).unwrap();
+ assert_eq!(tail.len(), 2);
+ assert_eq!(tail[0].pos.seq, 5, "newest first");
+ l.append(put(2, 1, "later")).unwrap();
+ let again = OpLog::open(d.path(), 100).unwrap();
+ assert_eq!(again.tail(), Pos { epoch: 2, seq: 1 });
+ assert_eq!(again.len(), 4);
+ }
+
+ #[test]
+ fn a_torn_last_line_is_dropped_and_trimming_asks_for_a_snapshot() {
+ let d = tempfile::tempdir().unwrap();
+ {
+ let mut l = OpLog::open(d.path(), 4).unwrap();
+ for s in 1..=3 {
+ l.append(put(1, s, "k")).unwrap();
+ }
+ }
+ let p = d.path().join("oplog.jsonl");
+ let mut f = std::fs::OpenOptions::new().append(true).open(&p).unwrap();
+ f.write_all(b"{\"pos\":{\"epoch\":1,\"se").unwrap();
+ let mut l = OpLog::open(d.path(), 4).unwrap();
+ assert_eq!(l.len(), 3);
+ for s in 4..=20 {
+ l.append(put(1, s, "k")).unwrap();
+ }
+ // The log kept only the newest; a new replica needs a snapshot.
+ assert!(l.after(Pos::ZERO, 10).is_none());
+ let head = l.head().unwrap();
+ assert!(l.after(head, 100).is_some());
+ assert!(l.after(Pos { epoch: 1, seq: 1 }, 100).is_none());
+ }
+}
diff --git a/libs/cua/crates/cua-volume/src/replica/outbox.rs b/libs/cua/crates/cua-volume/src/replica/outbox.rs
new file mode 100644
index 0000000000..0f01315086
--- /dev/null
+++ b/libs/cua/crates/cua-volume/src/replica/outbox.rs
@@ -0,0 +1,104 @@
+// SPDX-License-Identifier: FSL-1.1-MIT
+// Copyright (c) 2026 Cua AI, Inc.
+
+//! Changes this machine saved that the owner has not taken yet.
+//!
+//! One JSON file per change in `<state>/outbox/`. A change survives a
+//! restart; it leaves the outbox only when the owner says what it did with
+//! it. The bytes of a pending write are a version in the spool store (or, for
+//! the changes a machine hands over when it steps down, a version in its data
+//! store), so history is never rewritten.
+
+use std::path::{Path, PathBuf};
+
+use super::wire::Pending;
+use crate::{Error, Result};
+
+#[derive(Debug)]
+pub struct Outbox {
+ dir: PathBuf,
+ items: Vec<Pending>,
+}
+
+impl Outbox {
+ pub fn open(state: &Path) -> Result<Outbox> {
+ let dir = state.join("outbox");
+ std::fs::create_dir_all(&dir)?;
+ let mut items = vec![];
+ for e in std::fs::read_dir(&dir)? {
+ let p = e?.path();
+ if p.extension().is_some_and(|x| x == "json")
+ && let Ok(b) = std::fs::read(&p)
+ && let Ok(item) = serde_json::from_slice::<Pending>(&b)
+ {
+ items.push(item);
+ }
+ }
+ items.sort_by(|a, b| (a.ts_ms, &a.id).cmp(&(b.ts_ms, &b.id)));
+ Ok(Outbox { dir, items })
+ }
+
+ fn file(&self, id: &str) -> PathBuf {
+ self.dir.join(format!("{id}.json"))
+ }
+
+ pub fn add(&mut self, p: Pending) -> Result<()> {
+ let f = self.file(&p.id);
+ cua_home::guard_write(&f)?;
+ let tmp = f.with_extension("tmp");
+ std::fs::write(&tmp, serde_json::to_vec(&p)?)?;
+ std::fs::rename(&tmp, &f)?;
+ self.items.push(p);
+ Ok(())
+ }
+
+ pub fn remove(&mut self, id: &str) -> Result<Option<Pending>> {
+ let Some(i) = self.items.iter().position(|p| p.id == id) else {
+ return Ok(None);
+ };
+ let p = self.items.remove(i);
+ match std::fs::remove_file(self.file(id)) {
+ Ok(()) => {}
+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
+ Err(e) => return Err(Error::from(e)),
+ }
+ Ok(Some(p))
+ }
+
+ /// Pending changes of the same writer chain on after `old` (a version
+ /// of ours the owner stored as `new`): they now follow `new`.
+ pub fn rebase(&mut self, old: &str, new: &str) -> Result<()> {
+ for i in 0..self.items.len() {
+ if self.items[i].base.as_deref() == Some(old) {
+ self.items[i].base = Some(new.to_string());
+ let f = self.file(&self.items[i].id);
+ std::fs::write(&f, serde_json::to_vec(&self.items[i])?)?;
+ }
+ }
+ Ok(())
+ }
+
+ pub fn items(&self) -> &[Pending] {
+ &self.items
+ }
+
+ pub fn len(&self) -> usize {
+ self.items.len()
+ }
+
+ pub fn is_empty(&self) -> bool {
+ self.items.is_empty()
+ }
+
+ pub fn has_key(&self, key: &str) -> bool {
+ self.items.iter().any(|p| p.key == key)
+ }
+
+ /// Bytes of the writes waiting (what a status line shows).
+ pub fn bytes(&self) -> u64 {
+ self.items
+ .iter()
+ .filter_map(|p| p.meta.as_ref().map(|m| m.size))
+ .sum()
+ }
+}
diff --git a/libs/cua/crates/cua-volume/src/replica/owner.rs b/libs/cua/crates/cua-volume/src/replica/owner.rs
new file mode 100644
index 0000000000..4b92e8b568
--- /dev/null
+++ b/libs/cua/crates/cua-volume/src/replica/owner.rs
@@ -0,0 +1,1270 @@
+// SPDX-License-Identifier: FSL-1.1-MIT
+// Copyright (c) 2026 Cua AI, Inc.
+
+//! Writes: at the owner (serialized, logged, shipped), forwarded from other
+//! machines, and saved for later when the owner is away.
+
+use std::collections::HashMap;
+use std::path::PathBuf;
+use std::sync::Arc;
+use std::sync::atomic::Ordering;
+use std::time::{Duration, Instant};
+
+use super::era::Pos;
+use super::link::{CallError, Link};
+use super::merge::{self, Resolution};
+use super::node::{Inner, State};
+use super::oplog::{Op, Record};
+use super::status::Durability;
+use super::wire::{Chan, Cond, Msg, Outcome, Pending, Rep, Req, Src};
+use crate::backend::{Backend, Condition, ObjectMeta};
+use crate::fs::{ApplySource, FsBackend};
+use crate::{Error, Result};
+
+/// What the owner says when asked to write and it is not the owner (the
+/// caller looks up the new owner and retries).
+pub(super) const NOT_OWNER: &str = "this machine does not keep the volume now";
+
+pub(super) fn is_not_owner(e: &Error) -> bool {
+ matches!(e, Error::Backend(m) if m == NOT_OWNER)
+}
+
+/// The bytes of a write.
+pub(super) enum Body {
+ Bytes(Vec<u8>),
+ File(PathBuf),
+}
+
+impl Body {
+ pub fn size(&self) -> u64 {
+ match self {
+ Body::Bytes(b) => b.len() as u64,
+ Body::File(p) => std::fs::metadata(p).map_or(0, |m| m.len()),
+ }
+ }
+}
+
+/// Why a forward did not happen.
+pub(super) enum Fwd {
+ /// No answer from the owner.
+ Unreachable(String),
+ /// The owner answered with an error.
+ Remote(Error),
+}
+
+fn unreachable_msg(owner: &str) -> String {
+ format!("{owner}, which keeps this volume in order, cannot be reached right now")
+}
+
+#[derive(serde::Serialize, serde::Deserialize, Default)]
+struct Intent {
+ keys: Vec<(String, bool)>,
+}
+
+impl Inner {
+ fn intent_path(&self) -> PathBuf {
+ self.dir.join("intent.json")
+ }
+
+ fn write_intent(&self, keys: &[(&str, bool)]) -> Result<()> {
+ let i = Intent {
+ keys: keys.iter().map(|(k, d)| (k.to_string(), *d)).collect(),
+ };
+ let p = self.intent_path();
+ cua_home::guard_write(&p)?;
+ std::fs::write(p, serde_json::to_vec(&i)?)?;
+ Ok(())
+ }
+
+ fn clear_intent(&self) {
+ let _ = std::fs::remove_file(self.intent_path());
+ }
+
+ /// A crash between the store write and the log append leaves a head
+ /// the log does not know. Make it known.
+ pub(super) async fn recover_intent(&self) -> Result<()> {
+ let Ok(b) = std::fs::read(self.intent_path()) else {
+ return Ok(());
+ };
+ let i: Intent = serde_json::from_slice(&b).unwrap_or_default();
+ let mut st = self.st.lock().await;
+ if st.eras.owner() == self.me {
+ for (key, deleted) in i.keys {
+ let head = self.store.head(&key).await?;
+ let known = st.log.range(Pos::ZERO, st.log.tail());
+ let last = known.iter().rev().find(|r| r.op.key() == key);
+ match (deleted, head) {
+ (false, Some(h)) => {
+ let logged = matches!(last.map(|r| &r.op), Some(Op::Put { meta, .. }) if meta.version == h.version);
+ if !logged {
+ self.commit_locked(
+ &mut st,
+ Op::Put {
+ meta: h,
+ prev: None,
+ },
+ )?;
+ }
+ }
+ (true, None) => {
+ let logged = matches!(last.map(|r| &r.op), Some(Op::Delete { .. }));
+ if !logged
+ && let Ok(vs) = self.store.versions(&key).await
+ && let Some(v) = vs.iter().find(|v| v.deleted)
+ {
+ self.commit_locked(
+ &mut st,
+ Op::Delete {
+ key: key.clone(),
+ version: v.version.clone(),
+ prev: None,
+ },
+ )?;
+ }
+ }
+ _ => {}
+ }
+ }
+ }
+ self.clear_intent();
+ Ok(())
+ }
+
+ pub(super) fn next_pos(&self, st: &State) -> Pos {
+ let epoch = st.eras.epoch();
+ let t = st.log.tail();
+ Pos {
+ epoch,
+ seq: if t.epoch == epoch { t.seq + 1 } else { 1 },
+ }
+ }
+
+ /// Appends `op` to the log at the next position (state lock held).
+ pub(super) fn commit_locked(&self, st: &mut State, op: Op) -> Result<Pos> {
+ let pos = self.next_pos(st);
+ st.log.append(Record {
+ pos,
+ op,
+ at_ms: self.wall_ms(),
+ })?;
+ st.written_at.insert(pos, Instant::now());
+ while st.written_at.len() > 4096 {
+ st.written_at.pop_first();
+ }
+ self.applied_tx.send_replace(pos);
+ Ok(pos)
+ }
+
+ /// A write at the owner.
+ pub(super) async fn owner_put(
+ &self,
+ key: &str,
+ body: Body,
+ cond: Condition,
+ ) -> Result<(ObjectMeta, Pos)> {
+ match self.owner_put_try(key, body, cond).await {
+ Ok(r) => r,
+ Err(_) => Err(Error::Backend(NOT_OWNER.into())),
+ }
+ }
+
+ /// A write at the owner; hands the body back when this machine is not
+ /// the owner (any more).
+ pub(super) async fn owner_put_try(
+ &self,
+ key: &str,
+ body: Body,
+ cond: Condition,
+ ) -> std::result::Result<Result<(ObjectMeta, Pos)>, Body> {
+ let mut st = self.st.lock().await;
+ if st.eras.owner() != self.me {
+ return Err(body);
+ }
+ let r = self.owner_put_locked(&mut st, key, body, cond).await;
+ drop(st);
+ self.wake.notify_one();
+ Ok(r)
+ }
+
+ async fn owner_put_locked(
+ &self,
+ st: &mut State,
+ key: &str,
+ body: Body,
+ cond: Condition,
+ ) -> Result<(ObjectMeta, Pos)> {
+ let prev = self.store.head(key).await?;
+ self.write_intent(&[(key, false)])?;
+ let meta = match body {
+ Body::Bytes(b) => self.store.put(key, b, cond).await,
+ Body::File(p) => self.store.put_file(key, &p, cond).await,
+ };
+ let meta = match meta {
+ Ok(m) => m,
+ Err(e) => {
+ self.clear_intent();
+ return Err(e);
+ }
+ };
+ let pos = self.commit_locked(
+ st,
+ Op::Put {
+ meta: meta.clone(),
+ prev,
+ },
+ )?;
+ self.clear_intent();
+ Ok((meta, pos))
+ }
+
+ pub(super) async fn owner_delete(&self, key: &str, cond: Condition) -> Result<Pos> {
+ let mut st = self.st.lock().await;
+ if st.eras.owner() != self.me {
+ return Err(Error::Backend(NOT_OWNER.into()));
+ }
+ let r = self.owner_delete_locked(&mut st, key, cond).await;
+ drop(st);
+ self.wake.notify_one();
+ r
+ }
+
+ async fn owner_delete_locked(&self, st: &mut State, key: &str, cond: Condition) -> Result<Pos> {
+ let prev = self.store.head(key).await?;
+ self.write_intent(&[(key, true)])?;
+ let version = match self.store.delete_versioned(key, cond).await {
+ Ok(v) => v,
+ Err(e) => {
+ self.clear_intent();
+ return Err(e);
+ }
+ };
+ let pos = self.commit_locked(
+ st,
+ Op::Delete {
+ key: key.to_string(),
+ version,
+ prev,
+ },
+ )?;
+ self.clear_intent();
+ Ok(pos)
+ }
+
+ /// Copies holding `pos`, copies wanted, and copies that could still
+ /// hold it (machines holding it, plus holders that are up and have not
+ /// acknowledged yet) (state lock held).
+ fn copies_at(&self, st: &State, pos: Pos) -> (usize, usize, usize) {
+ let layout = self.layout_locked(st);
+ let want = layout.replicas.len().max(1);
+ let mut copies = 1;
+ let mut possible = 1;
+ for r in &layout.replicas {
+ if *r == self.me {
+ continue;
+ }
+ if let Some(p) = st.peers.get(r)
+ && !p.last_health.is_down()
+ {
+ if p.acked >= pos {
+ copies += 1;
+ possible += 1;
+ } else if self.ctl_link(r).is_some()
+ && p.missed_acks < 2
+ && p.ack_rtt_ms * 2.0 < self.cfg.ack_wait.as_millis() as f64
+ {
+ possible += 1;
+ }
+ }
+ }
+ (copies, want, possible)
+ }
+
+ /// Waits up to `ack_wait` for enough copies of `pos`; the write never
+ /// fails because a machine is away.
+ pub(super) async fn wait_durable(&self, pos: Pos) -> (usize, bool) {
+ let deadline = Instant::now() + self.cfg.ack_wait;
+ let mut rx = self.acks.subscribe();
+ loop {
+ let (copies, want, possible) = {
+ let st = self.st.lock().await;
+ self.copies_at(&st, pos)
+ };
+ if want >= 2 && copies >= want {
+ return (copies, true);
+ }
+ // One machine, or the other holder is known to be away: there is
+ // nobody to wait for, so a write never waits (a closed laptop
+ // costs the studio nothing).
+ if want < 2 || possible < want {
+ return (copies, false);
+ }
+ let left = deadline.saturating_duration_since(Instant::now());
+ if left.is_zero() {
+ // Whoever did not answer in time is not waited for again
+ // until it answers.
+ let mut st = self.st.lock().await;
+ let layout = self.layout_locked(&st);
+ for r in layout.replicas.iter().filter(|r| **r != self.me) {
+ if let Some(p) = st.peers.get_mut(r)
+ && p.acked < pos
+ {
+ p.missed_acks = p.missed_acks.saturating_add(1);
+ }
+ }
+ return (copies, false);
+ }
+ tokio::select! {
+ _ = rx.changed() => {}
+ _ = tokio::time::sleep(left) => {}
+ }
+ }
+ }
+
+ pub(super) fn durability(copies: usize, durable: bool) -> Durability {
+ if durable {
+ Durability::Replicated(copies)
+ } else {
+ Durability::Local(copies)
+ }
+ }
+
+ /// A forwarded `Backend` call, at the owner or at any machine holding a
+ /// copy (reads).
+ pub(super) async fn handle_req(
+ self: &Arc<Self>,
+ _link: &Arc<Link>,
+ req: Req,
+ data: Vec<u8>,
+ ) -> (Rep, Vec<u8>) {
+ let fail = |e: Error| (Rep::fail(&e), vec![]);
+ match req {
+ Req::Put { key, cond } => {
+ match self.owner_put(&key, Body::Bytes(data), cond.into()).await {
+ Ok((meta, pos)) => {
+ let (copies, durable) = self.wait_durable(pos).await;
+ (
+ Rep::Meta {
+ meta,
+ pos,
+ copies: copies as u32,
+ durable,
+ },
+ vec![],
+ )
+ }
+ Err(e) => fail(e),
+ }
+ }
+ Req::Delete { key, cond } => match self.owner_delete(&key, cond.into()).await {
+ Ok(pos) => {
+ let (copies, durable) = self.wait_durable(pos).await;
+ (
+ Rep::Done {
+ pos,
+ copies: copies as u32,
+ durable,
+ },
+ vec![],
+ )
+ }
+ Err(e) => fail(e),
+ },
+ Req::Head { key } => match self.store.head(&key).await {
+ Ok(meta) => (Rep::Head { meta }, vec![]),
+ Err(e) => fail(e),
+ },
+ Req::Get { key, version } => match self.store.get(&key, version.as_deref()).await {
+ Ok((bytes, meta)) => (Rep::Bytes { meta }, bytes),
+ Err(e) => fail(e),
+ },
+ Req::GetRange {
+ key,
+ version,
+ offset,
+ len,
+ } => match self.store.get_range(&key, &version, offset, len).await {
+ Ok(bytes) => (Rep::Range, bytes),
+ Err(e) => fail(e),
+ },
+ Req::List { prefix } => match self.store.list(&prefix).await {
+ Ok(items) => (Rep::List { items }, vec![]),
+ Err(e) => fail(e),
+ },
+ Req::ListDir { folder } => match self.store.list_dir(&folder).await {
+ Ok((files, folders)) => (Rep::ListDir { files, folders }, vec![]),
+ Err(e) => fail(e),
+ },
+ Req::Versions { key } => match self.store.versions(&key).await {
+ Ok(items) => (Rep::Versions { items }, vec![]),
+ Err(e) => fail(e),
+ },
+ Req::Heads { after_key, max } => {
+ let at = self.st.lock().await.log.tail();
+ match self.store.list_after("", &after_key, max).await {
+ Ok(items) => {
+ let done = items.len() < max;
+ (Rep::Heads { items, done, at }, vec![])
+ }
+ Err(e) => fail(e),
+ }
+ }
+ }
+ }
+
+ // ---- the owner takes pending changes --------------------------------
+
+ pub(super) async fn handle_submit(
+ self: &Arc<Self>,
+ link: &Arc<Link>,
+ ops: Vec<Pending>,
+ data: Vec<u8>,
+ ) -> Msg {
+ if !self.st.lock().await.eras.owner().eq(&self.me) {
+ return Msg::Error {
+ kind: "not_owner".into(),
+ message: NOT_OWNER.into(),
+ };
+ }
+ let mut remap: HashMap<String, String> = HashMap::new();
+ let mut outcomes = Vec::with_capacity(ops.len());
+ for op in &ops {
+ let o = self.apply_pending(Some(link), op, &data, &mut remap).await;
+ outcomes.push(o);
+ }
+ self.wake.notify_one();
+ Msg::Submitted { outcomes }
+ }
+
+ async fn fetch_body(
+ self: &Arc<Self>,
+ link: Option<&Arc<Link>>,
+ op: &Pending,
+ data: &[u8],
+ ) -> Result<Body> {
+ let meta = op.meta.as_ref().expect("a put");
+ // The owner's own outbox: the bytes are on this machine.
+ let Some(link) = link else {
+ let path = match &op.src {
+ Src::Spool { key, version } => self.spool.version_file(key, version)?,
+ Src::Data { key, version } => self.store.version_file(key, version)?,
+ };
+ return Ok(Body::File(path));
+ };
+ if let Some((off, len)) = op.body {
+ let (o, l) = (off as usize, len as usize);
+ let bytes = data
+ .get(o..o + l)
+ .ok_or_else(|| Error::Backend("a body outside its frame".into()))?;
+ return Ok(Body::Bytes(bytes.to_vec()));
+ }
+ let dl = match self.data_link(&link.peer).await {
+ Some(l) => l,
+ None => return Err(Error::Backend("no data link to the writer".into())),
+ };
+ let seed = self.store.head(&op.key).await.ok().flatten();
+ let path = self.pull(&dl, op.src.clone(), meta, seed.as_ref()).await?;
+ Ok(Body::File(path))
+ }
+
+ async fn apply_pending(
+ self: &Arc<Self>,
+ link: Option<&Arc<Link>>,
+ op: &Pending,
+ data: &[u8],
+ remap: &mut HashMap<String, String>,
+ ) -> Outcome {
+ let base = op
+ .base
+ .as_ref()
+ .map(|b| remap.get(b).cloned().unwrap_or_else(|| b.clone()));
+ let Some(meta) = op.meta.as_ref() else {
+ // A delete.
+ let mut st = self.st.lock().await;
+ if st.eras.owner() != self.me {
+ return Outcome::Failed {
+ message: NOT_OWNER.into(),
+ };
+ }
+ let head = match self.store.head(&op.key).await {
+ Ok(h) => h,
+ Err(e) => {
+ return Outcome::Failed {
+ message: e.to_string(),
+ };
+ }
+ };
+ return match head {
+ None => Outcome::Merged,
+ Some(h) if Some(&h.version) == base.as_ref() => {
+ match self
+ .owner_delete_locked(&mut st, &op.key, Condition::None)
+ .await
+ {
+ Ok(pos) => Outcome::Deleted { pos },
+ Err(e) => Outcome::Failed {
+ message: e.to_string(),
+ },
+ }
+ }
+ // Someone wrote after the deleter last looked: the write stays.
+ Some(_) => Outcome::Kept,
+ };
+ };
+ // The head may be this very change, made visible here while it
+ // waited (a version of its own, never written to the log). The owner
+ // must log it, not take it for content it already holds.
+ if matches!(Condition::from(op.cond.clone()), Condition::None) {
+ let mut st = self.st.lock().await;
+ if st.eras.owner() == self.me
+ && let Ok(Some(h)) = self.store.head(&op.key).await
+ && h.version == meta.version
+ {
+ return match self.commit_locked(
+ &mut st,
+ Op::Put {
+ meta: h.clone(),
+ prev: None,
+ },
+ ) {
+ Ok(pos) => {
+ drop(st);
+ self.wake.notify_one();
+ Outcome::Applied { meta: h, pos }
+ }
+ Err(e) => Outcome::Failed {
+ message: e.to_string(),
+ },
+ };
+ }
+ }
+ // Content first, outside the lock.
+ let cond: Condition = op.cond.clone().into();
+ let head0 = self.store.head(&op.key).await.ok().flatten();
+ let have_same = head0.as_ref().is_some_and(|h| h.etag == meta.etag);
+ let body = if have_same {
+ None
+ } else {
+ match self.fetch_body(link, op, data).await {
+ Ok(b) => Some(b),
+ Err(e) => {
+ return Outcome::Failed {
+ message: e.to_string(),
+ };
+ }
+ }
+ };
+ let mut st = self.st.lock().await;
+ if st.eras.owner() != self.me {
+ return Outcome::Failed {
+ message: NOT_OWNER.into(),
+ };
+ }
+ let head = match self.store.head(&op.key).await {
+ Ok(h) => h,
+ Err(e) => {
+ return Outcome::Failed {
+ message: e.to_string(),
+ };
+ }
+ };
+ if let Some(h) = &head
+ && h.etag == meta.etag
+ && matches!(cond, Condition::None)
+ {
+ remap.insert(meta.version.clone(), h.version.clone());
+ return Outcome::Merged;
+ }
+ let Some(body) = body else {
+ // The head had this content when we looked, and changed since.
+ return Outcome::Failed {
+ message: "the key changed while it was being merged; it will be tried again".into(),
+ };
+ };
+ let clean = match (&head, &base) {
+ (None, _) => true,
+ (Some(h), Some(b)) => &h.version == b,
+ (Some(_), None) => false,
+ };
+ if !matches!(cond, Condition::None) {
+ // A conditional write is evaluated as written; no merging.
+ return match self.owner_put_locked(&mut st, &op.key, body, cond).await {
+ Ok((m, pos)) => {
+ remap.insert(meta.version.clone(), m.version.clone());
+ Outcome::Applied { meta: m, pos }
+ }
+ Err(Error::Precondition(m)) => Outcome::Precondition { message: m },
+ Err(e) => Outcome::Failed {
+ message: e.to_string(),
+ },
+ };
+ }
+ // An order (fence, lease, run record) is judged by its order even
+ // when the change chains onto a version the owner just kept.
+ if clean
+ && merge::is_ordered(&op.key)
+ && let Some(h) = &head
+ && h.size <= self.cfg.inline_max as u64 * 4
+ && meta.size <= self.cfg.inline_max as u64 * 4
+ {
+ let head_bytes = self.store.get(&op.key, None).await.map(|(b, _)| b);
+ let in_bytes = match &body {
+ Body::Bytes(b) => Ok(b.clone()),
+ Body::File(p) => std::fs::read(p).map_err(Error::from),
+ };
+ if let (Ok(hb), Ok(ib)) = (head_bytes, in_bytes)
+ && merge::resolve(&op.key, &hb, &ib, &op.name, op.ts_ms) == Resolution::KeepHead
+ {
+ remap.insert(meta.version.clone(), h.version.clone());
+ return Outcome::Merged;
+ }
+ }
+ if clean {
+ return match self
+ .owner_put_locked(&mut st, &op.key, body, Condition::None)
+ .await
+ {
+ Ok((m, pos)) => {
+ remap.insert(meta.version.clone(), m.version.clone());
+ Outcome::Applied { meta: m, pos }
+ }
+ Err(e) => Outcome::Failed {
+ message: e.to_string(),
+ },
+ };
+ }
+ // The key changed on the owner since the writer looked.
+ let head = head.expect("not clean implies a head");
+ let incoming_small = meta.size <= self.cfg.inline_max as u64 * 4
+ && head.size <= self.cfg.inline_max as u64 * 4;
+ let resolution = if incoming_small {
+ let head_bytes = self.store.get(&op.key, None).await.map(|(b, _)| b);
+ let in_bytes = match &body {
+ Body::Bytes(b) => Ok(b.clone()),
+ Body::File(p) => std::fs::read(p).map_err(Error::from),
+ };
+ match (head_bytes, in_bytes) {
+ (Ok(h), Ok(i)) => merge::resolve(&op.key, &h, &i, &op.name, op.ts_ms),
+ _ => Resolution::Copy(crate::feed::conflict_path(&op.key, &op.name, op.ts_ms)),
+ }
+ } else {
+ Resolution::Copy(crate::feed::conflict_path(&op.key, &op.name, op.ts_ms))
+ };
+ match resolution {
+ Resolution::KeepHead => {
+ remap.insert(meta.version.clone(), head.version.clone());
+ Outcome::Merged
+ }
+ Resolution::TakeIncoming => {
+ match self
+ .owner_put_locked(&mut st, &op.key, body, Condition::None)
+ .await
+ {
+ Ok((m, pos)) => {
+ remap.insert(meta.version.clone(), m.version.clone());
+ Outcome::Applied { meta: m, pos }
+ }
+ Err(e) => Outcome::Failed {
+ message: e.to_string(),
+ },
+ }
+ }
+ Resolution::Copy(ckey) => {
+ st.conflict_copies += 1;
+ match self
+ .owner_put_locked(&mut st, &ckey, body, Condition::None)
+ .await
+ {
+ Ok((m, pos)) => {
+ remap.insert(meta.version.clone(), head.version.clone());
+ Outcome::Copy { key: m.key, pos }
+ }
+ Err(e) => Outcome::Failed {
+ message: e.to_string(),
+ },
+ }
+ }
+ }
+ }
+
+ // ---- writes from any machine ----------------------------------------
+
+ /// Reaches the owner's control link, dialing once if there is none.
+ async fn owner_ctl(self: &Arc<Self>, owner: &str) -> Option<Arc<Link>> {
+ if let Some(l) = self.ctl_link(owner) {
+ return Some(l);
+ }
+ // One quick try; the tick keeps trying in the background.
+ match tokio::time::timeout(Duration::from_secs(3), self.dial_chan(owner, Chan::Control))
+ .await
+ {
+ Ok(Ok(l)) => Some(l),
+ _ => self.ctl_link(owner),
+ }
+ }
+
+ pub(super) async fn write_put(
+ self: &Arc<Self>,
+ key: &str,
+ body: Body,
+ cond: Condition,
+ ) -> Result<(ObjectMeta, Durability)> {
+ let mut body = body;
+ for attempt in 0..6u32 {
+ let owner = {
+ let st = self.st.lock().await;
+ st.eras.owner().to_string()
+ };
+ if owner == self.me {
+ match self
+ .owner_put_try(
+ key,
+ std::mem::replace(&mut body, Body::Bytes(vec![])),
+ cond.clone(),
+ )
+ .await
+ {
+ Ok(Ok((meta, pos))) => {
+ let (copies, durable) = self.wait_durable(pos).await;
+ return Ok((meta, Self::durability(copies, durable)));
+ }
+ Ok(Err(e)) => return Err(e),
+ // The era moved between looking and writing: look again.
+ Err(b) => {
+ body = b;
+ continue;
+ }
+ }
+ }
+ let t0 = Instant::now();
+ match self.forward_put(&owner, key, &body, &cond).await {
+ Ok((meta, pos, copies, durable)) => {
+ let t1 = t0.elapsed();
+ self.wait_applied(pos).await;
+ tracing::debug!(%key, ?t1, applied_wait = ?t0.elapsed() - t1, %pos, "replica: forwarded write");
+ return Ok((meta, Self::durability(copies as usize, durable)));
+ }
+ Err(Fwd::Remote(e)) if is_not_owner(&e) && attempt < 5 => {
+ tokio::time::sleep(self.cfg.timing.interval).await;
+ continue;
+ }
+ Err(Fwd::Remote(e)) => return Err(e),
+ Err(Fwd::Unreachable(why)) => {
+ if matches!(cond, Condition::None) {
+ let meta = self.save_pending_put(key, body).await?;
+ return Ok((meta, Durability::Pending));
+ }
+ // An ordered write cannot wait in an outbox: this
+ // machine needs the authority, and takes it as soon as
+ // the owner is confirmed gone.
+ self.want_authority().await;
+ let name = self.owner_name(&owner).await;
+ let _ = why;
+ return Err(Error::Backend(unreachable_msg(&name)));
+ }
+ }
+ }
+ Err(Error::Backend(
+ "the volume's owner keeps changing; try again".into(),
+ ))
+ }
+
+ /// Records that this machine needs to order writes itself.
+ pub(super) async fn want_authority(&self) {
+ let mut st = self.st.lock().await;
+ st.authority_wanted = Some(Instant::now());
+ drop(st);
+ self.tick_wake.notify_one();
+ }
+
+ pub(super) async fn owner_name(&self, id: &str) -> String {
+ let st = self.st.lock().await;
+ st.members
+ .get(id)
+ .map(|m| m.name.clone())
+ .filter(|n| !n.is_empty())
+ .unwrap_or_else(|| id.to_string())
+ }
+
+ /// Waits (briefly) for our own copy to apply `pos`, so a read right
+ /// after a forwarded write sees it.
+ pub(super) async fn wait_applied(&self, pos: Pos) {
+ if self.is_client.load(Ordering::SeqCst) {
+ return;
+ }
+ let mut rx = self.applied_tx.subscribe();
+ let deadline = Instant::now() + self.cfg.ack_wait * 2;
+ while *rx.borrow() < pos {
+ let left = deadline.saturating_duration_since(Instant::now());
+ if left.is_zero() {
+ return;
+ }
+ tokio::select! {
+ r = rx.changed() => { if r.is_err() { return; } }
+ _ = tokio::time::sleep(left) => return,
+ }
+ }
+ }
+
+ async fn forward_put(
+ self: &Arc<Self>,
+ owner: &str,
+ key: &str,
+ body: &Body,
+ cond: &Condition,
+ ) -> std::result::Result<(ObjectMeta, Pos, u32, bool), Fwd> {
+ let size = body.size();
+ if size > self.cfg.push_max as u64 {
+ return self.forward_big(owner, key, body, cond).await;
+ }
+ let bytes = match body {
+ Body::Bytes(b) => b.clone(),
+ Body::File(p) => std::fs::read(p).map_err(|e| Fwd::Remote(e.into()))?,
+ };
+ let link = if bytes.len() <= self.cfg.inline_max {
+ self.owner_ctl(owner).await
+ } else {
+ self.data_link(owner).await
+ };
+ let Some(link) = link else {
+ return Err(Fwd::Unreachable("no link".into()));
+ };
+ let to = self.cfg.data_timeout;
+ match link
+ .call_strict(
+ Msg::Req(Req::Put {
+ key: key.to_string(),
+ cond: Cond::from(cond.clone()),
+ }),
+ &bytes,
+ to,
+ )
+ .await
+ {
+ Ok((
+ Msg::Rep(Rep::Meta {
+ meta,
+ pos,
+ copies,
+ durable,
+ }),
+ _,
+ )) => Ok((meta, pos, copies, durable)),
+ Ok((Msg::Rep(Rep::Fail { kind, message }), _)) => {
+ Err(Fwd::Remote(Rep::into_error(&kind, message)))
+ }
+ Ok(_) => Err(Fwd::Remote(Error::Backend("unexpected reply".into()))),
+ Err(CallError::Closed | CallError::Timeout) => {
+ Err(Fwd::Unreachable("no answer".into()))
+ }
+ Err(CallError::Io(e)) => Err(Fwd::Unreachable(e.to_string())),
+ }
+ }
+
+ /// A large write by a non-owner: spool it, let the owner pull it in
+ /// blocks (resumable), and wait for what it did.
+ async fn forward_big(
+ self: &Arc<Self>,
+ owner: &str,
+ key: &str,
+ body: &Body,
+ cond: &Condition,
+ ) -> std::result::Result<(ObjectMeta, Pos, u32, bool), Fwd> {
+ let Some(link) = self.owner_ctl(owner).await else {
+ return Err(Fwd::Unreachable("no link".into()));
+ };
+ let meta = match body {
+ Body::Bytes(b) => self.spool.put(key, b.clone(), Condition::None).await,
+ Body::File(p) => self.spool.put_file(key, p, Condition::None).await,
+ }
+ .map_err(Fwd::Remote)?;
+ let p = Pending {
+ id: crate::new_id(),
+ key: key.to_string(),
+ meta: Some(meta.clone()),
+ base: None,
+ cond: Cond::from(cond.clone()),
+ machine: self.me.clone(),
+ name: self.name.clone(),
+ ts_ms: self.wall_ms(),
+ src: Src::Spool {
+ key: key.to_string(),
+ version: meta.version.clone(),
+ },
+ body: None,
+ };
+ let r = link
+ .call(Msg::Submit { ops: vec![p] }, &[], self.cfg.data_timeout * 4)
+ .await;
+ let _ = self.spool.purge_version(key, &meta.version).await;
+ match r {
+ Ok((Msg::Submitted { outcomes }, _)) => match outcomes.into_iter().next() {
+ Some(Outcome::Applied { meta, pos }) => Ok((meta, pos, 1, false)),
+ Some(Outcome::Precondition { message }) => {
+ Err(Fwd::Remote(Error::Precondition(message)))
+ }
+ Some(Outcome::Failed { message }) => Err(Fwd::Remote(Error::Backend(message))),
+ _ => Err(Fwd::Remote(Error::Backend("unexpected outcome".into()))),
+ },
+ Ok((Msg::Error { kind, message }, _)) => {
+ Err(Fwd::Remote(Rep::into_error(&kind, message)))
+ }
+ Ok(_) => Err(Fwd::Remote(Error::Backend("unexpected reply".into()))),
+ Err(_) => Err(Fwd::Unreachable("no answer".into())),
+ }
+ }
+
+ /// Many unconditional writes: at the owner they are one batch (one disk
+ /// barrier, one run of log records, one wait for the second copy).
+ pub(super) async fn write_many(
+ self: &Arc<Self>,
+ items: Vec<(String, Vec<u8>)>,
+ ) -> Result<Vec<ObjectMeta>> {
+ let unique = items
+ .iter()
+ .map(|(k, _)| k.as_str())
+ .collect::<std::collections::HashSet<_>>()
+ .len()
+ == items.len();
+ if unique && !items.is_empty() {
+ let mut st = self.st.lock().await;
+ if st.eras.owner() == self.me {
+ let mut prevs = Vec::with_capacity(items.len());
+ for (k, _) in &items {
+ prevs.push(self.store.head(k).await?);
+ }
+ let keys: Vec<(&str, bool)> =
+ items.iter().map(|(k, _)| (k.as_str(), false)).collect();
+ self.write_intent(&keys)?;
+ let metas = match self.store.put_many(items).await {
+ Ok(m) => m,
+ Err(e) => {
+ self.clear_intent();
+ return Err(e);
+ }
+ };
+ let mut recs = Vec::with_capacity(metas.len());
+ let mut pos = self.next_pos(&st);
+ let at_ms = self.wall_ms();
+ for (meta, prev) in metas.iter().zip(prevs) {
+ recs.push(Record {
+ pos,
+ op: Op::Put {
+ meta: meta.clone(),
+ prev,
+ },
+ at_ms,
+ });
+ pos.seq += 1;
+ }
+ let last = recs.last().map_or(Pos::ZERO, |r| r.pos);
+ let now = Instant::now();
+ for r in &recs {
+ st.written_at.insert(r.pos, now);
+ }
+ while st.written_at.len() > 4096 {
+ st.written_at.pop_first();
+ }
+ st.log.append_batch(recs)?;
+ self.applied_tx.send_replace(last);
+ self.clear_intent();
+ drop(st);
+ self.wake.notify_one();
+ self.wait_durable(last).await;
+ return Ok(metas);
+ }
+ }
+ let mut out = Vec::with_capacity(items.len());
+ for (k, b) in items {
+ out.push(self.write_put(&k, Body::Bytes(b), Condition::None).await?.0);
+ }
+ Ok(out)
+ }
+
+ pub(super) async fn write_delete(
+ self: &Arc<Self>,
+ key: &str,
+ cond: Condition,
+ ) -> Result<Durability> {
+ for attempt in 0..6u32 {
+ let owner = self.st.lock().await.eras.owner().to_string();
+ if owner == self.me {
+ match self.owner_delete(key, cond.clone()).await {
+ Ok(pos) => {
+ let (copies, durable) = self.wait_durable(pos).await;
+ return Ok(Self::durability(copies, durable));
+ }
+ Err(e) if is_not_owner(&e) => continue,
+ Err(e) => return Err(e),
+ }
+ }
+ let link = self.owner_ctl(&owner).await;
+ let r = match link {
+ Some(l) => {
+ l.call(
+ Msg::Req(Req::Delete {
+ key: key.to_string(),
+ cond: Cond::from(cond.clone()),
+ }),
+ &[],
+ self.cfg.call_timeout,
+ )
+ .await
+ }
+ None => Err(CallError::Closed),
+ };
+ match r {
+ Ok((
+ Msg::Rep(Rep::Done {
+ pos,
+ copies,
+ durable,
+ }),
+ _,
+ )) => {
+ self.wait_applied(pos).await;
+ return Ok(Self::durability(copies as usize, durable));
+ }
+ Ok((Msg::Rep(Rep::Fail { kind, message }), _)) => {
+ let e = Rep::into_error(&kind, message);
+ if is_not_owner(&e) && attempt < 5 {
+ tokio::time::sleep(self.cfg.timing.interval).await;
+ continue;
+ }
+ return Err(e);
+ }
+ Ok(_) => return Err(Error::Backend("unexpected reply".into())),
+ Err(_) => {
+ if matches!(cond, Condition::None) {
+ self.save_pending_delete(key).await?;
+ return Ok(Durability::Pending);
+ }
+ self.want_authority().await;
+ let name = self.owner_name(&owner).await;
+ return Err(Error::Backend(unreachable_msg(&name)));
+ }
+ }
+ }
+ Err(Error::Backend(
+ "the volume's owner keeps changing; try again".into(),
+ ))
+ }
+
+ // ---- saved for later --------------------------------------------------
+
+ async fn pending_base(&self, key: &str) -> Option<String> {
+ {
+ let st = self.st.lock().await;
+ if let Some(p) = st.outbox.items().iter().rev().find(|p| p.key == key) {
+ return p.meta.as_ref().map(|m| m.version.clone());
+ }
+ }
+ if self.is_client.load(Ordering::SeqCst) {
+ return None;
+ }
+ self.store.head(key).await.ok().flatten().map(|h| h.version)
+ }
+
+ pub(super) async fn save_pending_put(&self, key: &str, body: Body) -> Result<ObjectMeta> {
+ let base = self.pending_base(key).await;
+ let meta = match body {
+ Body::Bytes(b) => self.spool.put(key, b, Condition::None).await?,
+ Body::File(p) => self.spool.put_file(key, &p, Condition::None).await?,
+ };
+ let mut st = self.st.lock().await;
+ st.outbox.add(Pending {
+ id: crate::new_id(),
+ key: key.to_string(),
+ meta: Some(meta.clone()),
+ base,
+ cond: Cond::None,
+ machine: self.me.clone(),
+ name: self.name.clone(),
+ ts_ms: self.wall_ms(),
+ src: Src::Spool {
+ key: key.to_string(),
+ version: meta.version.clone(),
+ },
+ body: None,
+ })?;
+ if !self.is_client.load(Ordering::SeqCst) {
+ let path = self.spool.version_file(key, &meta.version)?;
+ self.store
+ .apply_put(meta.clone(), ApplySource::File { path, link: true })
+ .await?;
+ }
+ Ok(meta)
+ }
+
+ pub(super) async fn save_pending_delete(&self, key: &str) -> Result<()> {
+ let base = self.pending_base(key).await;
+ let mut st = self.st.lock().await;
+ st.outbox.add(Pending {
+ id: crate::new_id(),
+ key: key.to_string(),
+ meta: None,
+ base,
+ cond: Cond::None,
+ machine: self.me.clone(),
+ name: self.name.clone(),
+ ts_ms: self.wall_ms(),
+ src: Src::Data {
+ key: key.to_string(),
+ version: String::new(),
+ },
+ body: None,
+ })?;
+ if !self.is_client.load(Ordering::SeqCst) {
+ self.store
+ .apply_delete(key, &FsBackend::new_version())
+ .await?;
+ }
+ Ok(())
+ }
+
+ /// Hands the outbox to the owner once this machine is caught up with it.
+ pub(super) async fn submit_outbox(self: &Arc<Self>) {
+ if self.drain_own_outbox().await {
+ return;
+ }
+ let (owner, ops, caught_up) = {
+ let st = self.st.lock().await;
+ if st.outbox.is_empty() || st.eras.owner() == self.me {
+ return;
+ }
+ let owner = st.eras.owner().to_string();
+ let owner_tail = st.peers.get(&owner).map_or(Pos::ZERO, |p| p.tail);
+ let client = self.is_client.load(Ordering::SeqCst);
+ (
+ owner,
+ st.outbox
+ .items()
+ .iter()
+ .take(200)
+ .cloned()
+ .collect::<Vec<_>>(),
+ client || st.log.tail() >= owner_tail,
+ )
+ };
+ if !caught_up {
+ return;
+ }
+ if self
+ .submitting
+ .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
+ .is_err()
+ {
+ return;
+ }
+ let r = self.submit_ops(&owner, ops).await;
+ self.submitting.store(false, Ordering::SeqCst);
+ if let Err(e) = r {
+ tracing::debug!(error = %e, "replica: the outbox waits");
+ }
+ }
+
+ /// This machine owns the volume and holds changes it saved while it
+ /// did not (it stepped down, then got the volume back): they are applied
+ /// here, in order, by the same rules the owner applies anyone's. Returns
+ /// whether this machine is the owner (nothing is sent to anyone).
+ async fn drain_own_outbox(self: &Arc<Self>) -> bool {
+ if self.st.lock().await.eras.owner() != self.me {
+ return false;
+ }
+ let ops: Vec<Pending> = {
+ let st = self.st.lock().await;
+ st.outbox.items().iter().take(200).cloned().collect()
+ };
+ if ops.is_empty() {
+ return true;
+ }
+ if self
+ .submitting
+ .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
+ .is_err()
+ {
+ return true;
+ }
+ let mut remap: HashMap<String, String> = HashMap::new();
+ for op in &ops {
+ let o = self.apply_pending(None, op, &[], &mut remap).await;
+ tracing::debug!(key = %op.key, detail = ?o, "replica: applied own saved change");
+ if let Outcome::Failed { message } = &o {
+ tracing::warn!(key = %op.key, %message, "replica: could not apply a saved change here yet");
+ continue;
+ }
+ let mut st = self.st.lock().await;
+ let _ = st.outbox.remove(&op.id);
+ if let (Outcome::Applied { meta: new, .. }, Some(old)) = (&o, &op.meta) {
+ let _ = st.outbox.rebase(&old.version, &new.version);
+ }
+ drop(st);
+ if let (Some(_), Src::Spool { key, version }) = (&op.meta, &op.src) {
+ let _ = self.spool.purge_version(key, version).await;
+ }
+ }
+ self.submitting.store(false, Ordering::SeqCst);
+ self.wake.notify_one();
+ true
+ }
+
+ async fn submit_ops(self: &Arc<Self>, owner: &str, mut ops: Vec<Pending>) -> Result<()> {
+ let Some(link) = self.ctl_link(owner) else {
+ return Err(Error::Backend("owner not connected".into()));
+ };
+ // Small bodies travel in the frame; large ones are pulled.
+ let mut data: Vec<u8> = vec![];
+ for op in &mut ops {
+ op.body = None;
+ if let Some(m) = &op.meta
+ && m.size <= self.cfg.inline_max as u64
+ && data.len() + (m.size as usize) < (1 << 20)
+ {
+ let r = match &op.src {
+ Src::Spool { key, version } => self.spool.get(key, Some(version)).await,
+ Src::Data { key, version } => self.store.get(key, Some(version)).await,
+ };
+ if let Ok((bytes, _)) = r {
+ op.body = Some((data.len() as u64, bytes.len() as u64));
+ data.extend_from_slice(&bytes);
+ }
+ }
+ }
+ let n = ops.len();
+ let to = self.cfg.data_timeout * 4;
+ let (reply, _) = link
+ .call(Msg::Submit { ops: ops.clone() }, &data, to)
+ .await
+ .map_err(|e| Error::Backend(e.to_string()))?;
+ let outcomes = match reply {
+ Msg::Submitted { outcomes } => outcomes,
+ Msg::Error { kind, message } => return Err(Rep::into_error(&kind, message)),
+ _ => return Err(Error::Backend("unexpected reply".into())),
+ };
+ if outcomes.len() != n {
+ return Err(Error::Backend("short answer".into()));
+ }
+ let mut done = vec![];
+ let mut st = self.st.lock().await;
+ for (op, o) in ops.iter().zip(outcomes) {
+ tracing::debug!(key = %op.key, outcome = ?std::mem::discriminant(&o), detail = ?o, "replica: owner took a change");
+ if let Outcome::Failed { message } = &o {
+ tracing::warn!(key = %op.key, %message, "replica: the owner could not take a change yet");
+ continue;
+ }
+ st.outbox.remove(&op.id)?;
+ if let (Outcome::Applied { meta: new, .. }, Some(old)) = (&o, &op.meta) {
+ st.outbox.rebase(&old.version, &new.version)?;
+ }
+ done.push(op.clone());
+ }
+ drop(st);
+ for op in done {
+ if let (Some(m), Src::Spool { key, version }) = (&op.meta, &op.src) {
+ let _ = m;
+ let _ = self.spool.purge_version(key, version).await;
+ }
+ }
+ self.wake.notify_one();
+ Ok(())
+ }
+}
diff --git a/libs/cua/crates/cua-volume/src/replica/roles.rs b/libs/cua/crates/cua-volume/src/replica/roles.rs
new file mode 100644
index 0000000000..2c4b24b2fc
--- /dev/null
+++ b/libs/cua/crates/cua-volume/src/replica/roles.rs
@@ -0,0 +1,274 @@
+// SPDX-License-Identifier: FSL-1.1-MIT
+// Copyright (c) 2026 Cua AI, Inc.
+
+//! Who keeps the volume: the machines list, their reliability, and the
+//! roles that follow from them.
+//!
+//! The machines list is the account's (every machine signed in); this module
+//! only reads it. Reliability is a number each machine computes about itself
+//! (`cua_cluster::reliability::score`: a desktop beats a plugged-in laptop
+//! beats a laptop on battery; the machine the user is sitting at loses ties)
+//! and shares in its heartbeat. The *same* number places a durable drive, so
+//! the drive and its volume land on the same machine.
+
+use serde::{Deserialize, Serialize};
+
+use super::MachineId;
+
+/// What a machine says about itself (in `Hello` and every heartbeat).
+#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Facts {
+ /// Reliability score; higher keeps the volume. Categories are 1000
+ /// apart, so a hysteresis margin of 1000 means "a whole step better".
+ pub rank: u64,
+ /// Free bytes where the volume's copy lives.
+ pub free_disk: u64,
+ /// Running on battery.
+ pub on_battery: bool,
+ /// Can sleep when its lid closes (a laptop).
+ pub sleep_prone: bool,
+ /// The user is working at this machine right now.
+ pub user_here: bool,
+ /// A durable run holds a sleep assertion here.
+ pub sleep_prevented: bool,
+ /// CPU architecture (`arm64`, `amd64`; empty: unknown). Replication is
+ /// the same on every architecture; this only tells a takeover across
+ /// architectures to say that built programs are not reused.
+ #[serde(default)]
+ pub arch: String,
+}
+
+/// `arm64` or `amd64` for the names the platforms use.
+pub fn normalize_arch(a: &str) -> String {
+ match a.trim().to_ascii_lowercase().as_str() {
+ "aarch64" | "arm64" => "arm64".into(),
+ "x86_64" | "amd64" | "x64" => "amd64".into(),
+ other => other.to_string(),
+ }
+}
+
+/// Paths in a workspace that hold programs built for one architecture
+/// (build folders and native libraries), as short folder or file names; at
+/// most `max`. The rest of a workspace (sources, notes, journals, data) is
+/// the same on every architecture.
+pub fn arch_specific(keys: impl IntoIterator<Item = String>, max: usize) -> Vec<String> {
+ const DIRS: &[&str] = &[
+ "target",
+ "node_modules",
+ ".venv",
+ "venv",
+ "build",
+ "dist",
+ "cmake-build-debug",
+ ".gradle",
+ "Pods",
+ ];
+ const EXTS: &[&str] = &["o", "so", "dylib", "a", "node", "rlib", "dll", "exe", "pyd"];
+ let mut out: Vec<String> = vec![];
+ for k in keys {
+ let parts: Vec<&str> = k.split('/').collect();
+ let hit = parts
+ .iter()
+ .enumerate()
+ .find(|(i, p)| *i + 1 < parts.len() && DIRS.contains(p))
+ .map(|(i, _)| format!("{}/", parts[..=i].join("/")))
+ .or_else(|| {
+ let ext = k.rsplit('.').next()?;
+ (k.contains('.') && EXTS.contains(&ext)).then(|| k.clone())
+ });
+ if let Some(h) = hit {
+ // Show the folder after the agent's home, not the whole key.
+ let short = h.splitn(3, '/').nth(2).map(str::to_string).unwrap_or(h);
+ if !out.contains(&short) {
+ out.push(short);
+ if out.len() >= max {
+ break;
+ }
+ }
+ }
+ }
+ out
+}
+
+/// One machine of the group.
+#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Member {
+ pub id: MachineId,
+ /// The name the machines list shows ("Mac Studio").
+ pub name: String,
+ pub facts: Facts,
+ /// Wall time this machine was last heard from (0: never; display and
+ /// the active-member window only).
+ #[serde(default)]
+ pub last_seen_ms: u64,
+}
+
+/// What a machine does for the volume.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum Role {
+ /// Holds a full copy and acknowledges writes (the owner is one).
+ Replica,
+ /// Keeps a cache only and sends its writes to a replica.
+ Client,
+}
+
+/// The preferred layout.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct Layout {
+ /// The machine that should own the volume.
+ pub owner: MachineId,
+ /// Machines that hold a full copy, owner first.
+ pub replicas: Vec<MachineId>,
+}
+
+impl Layout {
+ pub fn role_of(&self, id: &str) -> Role {
+ if self.replicas.iter().any(|r| r == id) {
+ Role::Replica
+ } else {
+ Role::Client
+ }
+ }
+}
+
+/// Disk a machine needs free to hold a copy of `volume_bytes`: the volume
+/// again plus a margin (10 percent, at least 1 GiB).
+pub fn disk_needed(volume_bytes: u64) -> u64 {
+ volume_bytes.saturating_add((volume_bytes / 10).max(1 << 30))
+}
+
+/// The preferred layout for `members`: the `replicas` most reliable
+/// machines that have room for a copy, the best of them owning. Ties break
+/// by id so every machine computes the same answer.
+pub fn layout(members: &[Member], replicas: usize, volume_bytes: u64) -> Layout {
+ let need = disk_needed(volume_bytes);
+ let mut able: Vec<&Member> = members
+ .iter()
+ .filter(|m| m.facts.free_disk >= need)
+ .collect();
+ if able.is_empty() {
+ // Nobody reports room: the most reliable machines hold it anyway
+ // (a store that is full fails loudly; a refusal helps nobody).
+ able = members.iter().collect();
+ }
+ able.sort_by(|a, b| {
+ b.facts
+ .rank
+ .cmp(&a.facts.rank)
+ .then_with(|| a.id.cmp(&b.id))
+ });
+ let replicas: Vec<MachineId> = able
+ .iter()
+ .take(replicas.max(1))
+ .map(|m| m.id.clone())
+ .collect();
+ Layout {
+ owner: replicas.first().cloned().unwrap_or_default(),
+ replicas,
+ }
+}
+
+/// One class step of reliability (see `cua_cluster::reliability::STEP`).
+pub const STEP_RANK: u64 = 1000;
+
+/// Copies to keep: 2 when two or more machines exist, else 1.
+pub fn replica_count(machines: usize) -> usize {
+ if machines >= 2 { 2 } else { 1 }
+}
+
+/// Whether `candidate` is better than the current owner by a whole step.
+pub fn clearly_better(candidate: &Facts, owner: &Facts) -> bool {
+ candidate.rank >= owner.rank.saturating_add(STEP_RANK)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn m(id: &str, rank: u64, disk_gib: u64) -> Member {
+ Member {
+ id: id.into(),
+ name: id.into(),
+ facts: Facts {
+ rank,
+ free_disk: disk_gib << 30,
+ ..Default::default()
+ },
+ last_seen_ms: 0,
+ }
+ }
+
+ #[test]
+ fn the_most_reliable_machine_owns_and_the_next_holds_a_copy() {
+ let ms = [
+ m("laptop", 2000, 200),
+ m("studio", 9000, 800),
+ m("mini", 7000, 500),
+ ];
+ let l = layout(&ms, 2, 10 << 30);
+ assert_eq!(l.owner, "studio");
+ assert_eq!(l.replicas, ["studio", "mini"]);
+ assert_eq!(l.role_of("laptop"), Role::Client);
+ assert_eq!(l.role_of("mini"), Role::Replica);
+ }
+
+ #[test]
+ fn architecture_names_normalize_and_built_programs_are_found() {
+ assert_eq!(normalize_arch("aarch64"), "arm64");
+ assert_eq!(normalize_arch("x86_64"), "amd64");
+ let keys = [
+ "agents/ada/src/main.rs",
+ "agents/ada/target/debug/ada",
+ "agents/ada/target/release/libx.rlib",
+ "agents/ada/notes.md",
+ "agents/ada/vendor/libfoo.so",
+ "agents/ada/node_modules/x/y.js",
+ ]
+ .map(String::from);
+ let found = arch_specific(keys, 5);
+ assert_eq!(found, ["target/", "vendor/libfoo.so", "node_modules/"]);
+ }
+
+ #[test]
+ fn a_machine_without_room_becomes_a_client() {
+ let ms = [m("studio", 9000, 2), m("laptop", 2000, 400)];
+ // A 100 GiB volume does not fit the studio's 2 GiB.
+ let l = layout(&ms, 2, 100 << 30);
+ assert_eq!(l.owner, "laptop");
+ assert_eq!(l.replicas, ["laptop"]);
+ assert_eq!(l.role_of("studio"), Role::Client);
+ // Everyone is full: the most reliable still hold it.
+ let l = layout(&[m("a", 1, 0), m("b", 2, 0)], 2, 100 << 30);
+ assert_eq!(l.replicas, ["b", "a"]);
+ }
+
+ #[test]
+ fn equal_ranks_break_by_id_and_counts_follow_the_machines() {
+ let ms = [m("b", 5000, 100), m("a", 5000, 100)];
+ assert_eq!(layout(&ms, 2, 0).owner, "a");
+ assert_eq!(replica_count(1), 1);
+ assert_eq!(replica_count(2), 2);
+ assert_eq!(replica_count(9), 2);
+ assert!(clearly_better(
+ &Facts {
+ rank: 9000,
+ ..Default::default()
+ },
+ &Facts {
+ rank: 7999,
+ ..Default::default()
+ }
+ ));
+ assert!(!clearly_better(
+ &Facts {
+ rank: 8500,
+ ..Default::default()
+ },
+ &Facts {
+ rank: 8000,
+ ..Default::default()
+ }
+ ));
+ }
+}
diff --git a/libs/cua/crates/cua-volume/src/replica/ship.rs b/libs/cua/crates/cua-volume/src/replica/ship.rs
new file mode 100644
index 0000000000..b5f3454f78
--- /dev/null
+++ b/libs/cua/crates/cua-volume/src/replica/ship.rs
@@ -0,0 +1,221 @@
+// SPDX-License-Identifier: FSL-1.1-MIT
+// Copyright (c) 2026 Cua AI, Inc.
+
+//! The owner side of replication: ship the log to every machine that holds
+//! a copy, resuming from what each has applied.
+
+use std::collections::HashMap;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::time::Duration;
+
+use super::node::{Inner, save_eras};
+use super::oplog::Op;
+use super::roles::Role;
+use super::wire::{Msg, RecordW};
+use crate::backend::Backend;
+
+pub(super) type Shipping = std::sync::Mutex<HashMap<String, Arc<AtomicBool>>>;
+
+impl Inner {
+ pub(super) async fn ship_all_loop(self: Arc<Self>) {
+ loop {
+ tokio::select! {
+ _ = self.wake.notified() => {}
+ _ = tokio::time::sleep(Duration::from_millis(100)) => {}
+ }
+ if self.closed.load(Ordering::SeqCst) {
+ return;
+ }
+ self.wait_unfrozen().await;
+ if self.self_away.load(Ordering::SeqCst) {
+ continue;
+ }
+ let targets: Vec<String> = {
+ let st = self.st.lock().await;
+ if st.eras.owner() != self.me {
+ continue;
+ }
+ let layout = self.layout_locked(&st);
+ layout
+ .replicas
+ .iter()
+ .filter(|r| **r != self.me && layout.role_of(r) == Role::Replica)
+ .cloned()
+ .collect()
+ };
+ for peer in targets {
+ if self.ctl_link(&peer).is_none() {
+ continue;
+ }
+ let flag = {
+ let mut m = self.shipping.lock().expect("shipping");
+ m.entry(peer.clone()).or_default().clone()
+ };
+ if flag.swap(true, Ordering::SeqCst) {
+ continue;
+ }
+ let me = self.clone();
+ tokio::spawn(async move {
+ me.ship_until_idle(&peer).await;
+ flag.store(false, Ordering::SeqCst);
+ });
+ }
+ }
+ }
+
+ async fn ship_until_idle(self: &Arc<Self>, peer: &str) {
+ let mut last_acked = None;
+ loop {
+ if self.closed.load(Ordering::SeqCst) {
+ return;
+ }
+ let Some(link) = self.ctl_link(peer) else {
+ return;
+ };
+ let (epoch, acked, recs, tail) = {
+ let st = self.st.lock().await;
+ if st.eras.owner() != self.me {
+ return;
+ }
+ let acked = st
+ .peers
+ .get(peer)
+ .map_or(super::era::Pos::ZERO, |p| p.acked);
+ (
+ st.eras.epoch(),
+ acked,
+ st.log.after(acked, self.cfg.batch),
+ st.log.tail(),
+ )
+ };
+ if last_acked == Some(acked) {
+ // No progress since the last round: wait for news.
+ return;
+ }
+ let Some(recs) = recs else {
+ // The log no longer reaches this replica: it pulls a snapshot.
+ let r = link
+ .call(Msg::Snapshot { at: tail }, &[], self.cfg.data_timeout * 30)
+ .await;
+ match r {
+ Ok((Msg::Ack { applied }, _)) => self.set_acked(peer, applied).await,
+ Ok((Msg::Error { message, .. }, _)) => {
+ tracing::warn!(peer, %message, "replica: a snapshot could not be applied");
+ return;
+ }
+ _ => return,
+ }
+ last_acked = Some(acked);
+ continue;
+ };
+ if recs.is_empty() {
+ return;
+ }
+ let mut data: Vec<u8> = vec![];
+ let mut out = Vec::with_capacity(recs.len());
+ for r in recs {
+ let mut body = None;
+ if let Op::Put { meta, .. } = &r.op
+ && meta.size <= self.cfg.inline_max as u64
+ && data.len() + (meta.size as usize) < (1 << 20)
+ && let Ok((bytes, _)) = self.store.get(&meta.key, Some(&meta.version)).await
+ {
+ body = Some((data.len() as u64, bytes.len() as u64));
+ data.extend_from_slice(&bytes);
+ }
+ out.push(RecordW {
+ pos: r.pos,
+ op: r.op,
+ at_ms: r.at_ms,
+ body,
+ });
+ }
+ let t0 = std::time::Instant::now();
+ let n_out = out.len();
+ let reply = link
+ .call(
+ Msg::Append {
+ epoch,
+ records: out,
+ },
+ &data,
+ self.cfg.call_timeout * 10,
+ )
+ .await;
+ tracing::debug!(peer, n = n_out, took = ?t0.elapsed(), ok = reply.is_ok(), "replica: append sent");
+ last_acked = Some(acked);
+ match reply {
+ Ok((Msg::Ack { applied }, _)) => {
+ self.set_acked(peer, applied).await;
+ if applied > acked {
+ last_acked = None;
+ }
+ }
+ Ok((Msg::Behind { applied }, _)) => {
+ {
+ let mut st = self.st.lock().await;
+ if let Some(p) = st.peers.get_mut(peer) {
+ p.acked = applied;
+ }
+ }
+ if applied != acked {
+ last_acked = None;
+ }
+ }
+ Ok((Msg::Fenced { eras }, _)) => {
+ let advanced = {
+ let mut st = self.st.lock().await;
+ let a = st.eras.adopt_from(&eras);
+ if a {
+ let _ = save_eras(&self.dir, &st.eras);
+ }
+ a
+ };
+ if advanced {
+ self.reconcile_eras().await;
+ }
+ return;
+ }
+ Ok((Msg::Error { kind, message }, _)) => {
+ let _ = kind;
+ {
+ let mut st = self.st.lock().await;
+ if let Some(p) = st.peers.get_mut(peer) {
+ p.problem = Some(message);
+ }
+ }
+ return;
+ }
+ _ => return,
+ }
+ }
+ }
+
+ async fn set_acked(&self, peer: &str, applied: super::era::Pos) {
+ {
+ let mut st = self.st.lock().await;
+ let sample = st
+ .written_at
+ .get(&applied)
+ .map(|t| t.elapsed().as_millis() as f64);
+ if let Some(p) = st.peers.get_mut(peer) {
+ if let Some(ms) = sample {
+ p.ack_rtt_ms = if p.ack_rtt_ms == 0.0 {
+ ms
+ } else {
+ 0.7 * p.ack_rtt_ms + 0.3 * ms
+ };
+ }
+ if applied > p.acked {
+ p.acked = applied;
+ }
+ p.applied = applied;
+ p.problem = None;
+ p.missed_acks = 0;
+ }
+ }
+ self.acks
+ .send_replace(self.seq_ack_counter.fetch_add(1, Ordering::Relaxed));
+ }
+}
diff --git a/libs/cua/crates/cua-volume/src/replica/simnet.rs b/libs/cua/crates/cua-volume/src/replica/simnet.rs
new file mode 100644
index 0000000000..e8c407b40f
--- /dev/null
+++ b/libs/cua/crates/cua-volume/src/replica/simnet.rs
@@ -0,0 +1,261 @@
+// SPDX-License-Identifier: FSL-1.1-MIT
+// Copyright (c) 2026 Cua AI, Inc.
+
+//! An in-memory network between named machines, for simulations and the
+//! chaos suite. Streams behave like TCP: reliable and ordered, so loss shows
+//! up the way it does on a real connection, as delay. What it can do to a
+//! pair of machines, per direction:
+//!
+//! - latency, jitter, and a bandwidth cap;
+//! - *hold*: bytes stop arriving (a blackholed path) and flow again when the
+//! hold ends;
+//! - *sever*: every stream between the pair is reset and new dials fail
+//! until it is lifted (a long partition or a changed address);
+//! - an asymmetric version of any of these.
+
+use std::collections::HashMap;
+use std::sync::{Arc, Mutex};
+use std::time::Duration;
+
+use tokio::io::{AsyncReadExt, AsyncWriteExt};
+
+use super::link::Stream;
+use super::transport::Transport;
+
+/// What one direction of a pair does to traffic.
+#[derive(Clone, Debug, Default)]
+pub struct Impair {
+ pub latency: Duration,
+ pub jitter: Duration,
+ /// Bytes per second (0: unlimited).
+ pub bandwidth: u64,
+ /// Bytes stop arriving until this is cleared.
+ pub hold: bool,
+ /// Streams are reset and dials fail.
+ pub sever: bool,
+}
+
+type Pair = (String, String);
+
+#[derive(Default)]
+struct Net {
+ impair: HashMap<Pair, Impair>,
+ /// Accept queues by machine.
+ inbox: HashMap<String, tokio::sync::mpsc::UnboundedSender<(String, Stream)>>,
+ bytes: HashMap<Pair, u64>,
+ /// Machines whose process is gone: their connections are reset.
+ dead: std::collections::HashSet<String>,
+}
+
+/// The network.
+#[derive(Clone, Default)]
+pub struct SimNet {
+ net: Arc<Mutex<Net>>,
+}
+
+impl SimNet {
+ pub fn new() -> SimNet {
+ SimNet::default()
+ }
+
+ /// Registers `machine` and returns its transport and the streams that
+ /// arrive for it as `(dialer, stream)`.
+ pub fn join(
+ &self,
+ machine: &str,
+ ) -> (
+ SimTransport,
+ tokio::sync::mpsc::UnboundedReceiver<(String, Stream)>,
+ ) {
+ let (tx, rx) = tokio::sync::mpsc::unbounded_channel();
+ self.net
+ .lock()
+ .expect("net")
+ .inbox
+ .insert(machine.to_string(), tx);
+ (
+ SimTransport {
+ net: self.clone(),
+ me: machine.to_string(),
+ },
+ rx,
+ )
+ }
+
+ /// The machine's process is gone: its connections are reset and dials to
+ /// it are refused, as when a process dies on a machine that stays up.
+ pub fn leave(&self, machine: &str) {
+ let mut n = self.net.lock().expect("net");
+ n.inbox.remove(machine);
+ n.dead.insert(machine.to_string());
+ }
+
+ fn is_dead(&self, machine: &str) -> bool {
+ self.net.lock().expect("net").dead.contains(machine)
+ }
+
+ /// Sets what traffic from `from` to `to` suffers.
+ pub fn set(&self, from: &str, to: &str, i: Impair) {
+ self.net
+ .lock()
+ .expect("net")
+ .impair
+ .insert((from.into(), to.into()), i);
+ }
+
+ /// Both directions at once.
+ pub fn set_both(&self, a: &str, b: &str, i: Impair) {
+ self.set(a, b, i.clone());
+ self.set(b, a, i);
+ }
+
+ /// Clears everything between the pair.
+ pub fn heal(&self, a: &str, b: &str) {
+ self.set_both(a, b, Impair::default());
+ }
+
+ /// Bytes that crossed from `from` to `to` so far.
+ pub fn bytes(&self, from: &str, to: &str) -> u64 {
+ *self
+ .net
+ .lock()
+ .expect("net")
+ .bytes
+ .get(&(from.to_string(), to.to_string()))
+ .unwrap_or(&0)
+ }
+
+ fn get(&self, from: &str, to: &str) -> Impair {
+ self.net
+ .lock()
+ .expect("net")
+ .impair
+ .get(&(from.to_string(), to.to_string()))
+ .cloned()
+ .unwrap_or_default()
+ }
+
+ fn count(&self, from: &str, to: &str, n: u64) {
+ *self
+ .net
+ .lock()
+ .expect("net")
+ .bytes
+ .entry((from.to_string(), to.to_string()))
+ .or_default() += n;
+ }
+
+ /// One direction: reads from `rd`, delivers to `wr` after the pair's
+ /// current impairment. Delays are release times, not sleeps in the read
+ /// loop, so a delayed stream still pipelines like a real long path.
+ fn pump(
+ &self,
+ from: String,
+ to: String,
+ mut rd: tokio::io::ReadHalf<tokio::io::DuplexStream>,
+ mut wr: tokio::io::WriteHalf<tokio::io::DuplexStream>,
+ ) {
+ let net = self.clone();
+ let (tx, mut rx) =
+ tokio::sync::mpsc::unbounded_channel::<(tokio::time::Instant, Vec<u8>)>();
+ let (f2, t2, n2) = (from.clone(), to.clone(), net.clone());
+ tokio::spawn(async move {
+ while let Some((at, bytes)) = rx.recv().await {
+ tokio::time::sleep_until(at).await;
+ n2.count(&f2, &t2, bytes.len() as u64);
+ if wr.write_all(&bytes).await.is_err() {
+ break;
+ }
+ }
+ let _ = wr.shutdown().await;
+ });
+ tokio::spawn(async move {
+ let mut buf = vec![0u8; 16 * 1024];
+ let mut next_free = tokio::time::Instant::now();
+ loop {
+ let n = match rd.read(&mut buf).await {
+ Ok(0) | Err(_) => break,
+ Ok(n) => n,
+ };
+ loop {
+ let i = net.get(&from, &to);
+ if i.sever || net.is_dead(&from) || net.is_dead(&to) {
+ return;
+ }
+ if !i.hold {
+ break;
+ }
+ tokio::time::sleep(Duration::from_millis(5)).await;
+ }
+ let i = net.get(&from, &to);
+ let now = tokio::time::Instant::now();
+ let start = now.max(next_free);
+ let ser = if i.bandwidth > 0 {
+ Duration::from_secs_f64(n as f64 / i.bandwidth as f64)
+ } else {
+ Duration::ZERO
+ };
+ next_free = start + ser;
+ let mut delay = i.latency;
+ if !i.jitter.is_zero() {
+ let j = rand::random::<u64>() % (i.jitter.as_millis() as u64 + 1);
+ delay += Duration::from_millis(j);
+ }
+ let at = (start + ser + delay).max(now);
+ if tx.send((at, buf[..n].to_vec())).is_err() {
+ break;
+ }
+ }
+ });
+ }
+}
+
+/// One machine's dialer.
+#[derive(Clone)]
+pub struct SimTransport {
+ net: SimNet,
+ me: String,
+}
+
+#[async_trait::async_trait]
+impl Transport for SimTransport {
+ async fn dial(&self, machine: &str) -> std::io::Result<Stream> {
+ let i = self.net.get(&self.me, machine);
+ let back = self.net.get(machine, &self.me);
+ if i.sever || back.sever || i.hold || back.hold {
+ // A dial into a dead path takes a while to fail.
+ tokio::time::sleep(Duration::from_millis(50)).await;
+ return Err(std::io::Error::new(
+ std::io::ErrorKind::TimedOut,
+ "no route to machine",
+ ));
+ }
+ let tx = self
+ .net
+ .net
+ .lock()
+ .expect("net")
+ .inbox
+ .get(machine)
+ .cloned()
+ .ok_or_else(|| {
+ std::io::Error::new(std::io::ErrorKind::ConnectionRefused, "machine is off")
+ })?;
+ // Two pipes and a pump per direction make one connection.
+ let (dialer_end, a_inner) = tokio::io::duplex(256 * 1024);
+ let (acceptor_end, b_inner) = tokio::io::duplex(256 * 1024);
+ let (a_rd, a_wr) = tokio::io::split(a_inner);
+ let (b_rd, b_wr) = tokio::io::split(b_inner);
+ // The dial itself costs a round trip.
+ tokio::time::sleep(i.latency + back.latency).await;
+ self.net
+ .pump(self.me.clone(), machine.to_string(), a_rd, b_wr);
+ self.net
+ .pump(machine.to_string(), self.me.clone(), b_rd, a_wr);
+ tx.send((self.me.clone(), Box::new(acceptor_end)))
+ .map_err(|_| {
+ std::io::Error::new(std::io::ErrorKind::ConnectionRefused, "machine is off")
+ })?;
+ Ok(Box::new(dialer_end))
+ }
+}
diff --git a/libs/cua/crates/cua-volume/src/replica/status.rs b/libs/cua/crates/cua-volume/src/replica/status.rs
new file mode 100644
index 0000000000..bd2507c9ef
--- /dev/null
+++ b/libs/cua/crates/cua-volume/src/replica/status.rs
@@ -0,0 +1,88 @@
+// SPDX-License-Identifier: FSL-1.1-MIT
+// Copyright (c) 2026 Cua AI, Inc.
+
+//! What the app and the CLI show about the replicated volume.
+
+use serde::{Deserialize, Serialize};
+
+use super::detector::Health;
+use super::era::Pos;
+use super::roles::Role;
+
+/// How a write stands.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(tag = "kind", content = "copies", rename_all = "snake_case")]
+pub enum Durability {
+ /// This many machines hold it (2 or more when replication is on).
+ Replicated(usize),
+ /// Only this many (1): the other machine was away or slow.
+ Local(usize),
+ /// Saved on this machine only, waiting for the owner.
+ Pending,
+}
+
+/// A machine as the status shows it.
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct PeerView {
+ pub id: String,
+ pub name: String,
+ pub role: Role,
+ pub health: String,
+ pub connected: bool,
+ pub owner: bool,
+ /// Records it has applied that this machine knows of.
+ pub applied: Pos,
+ /// Records behind the owner's tail (0: caught up).
+ pub lag: u64,
+ pub problem: Option<String>,
+}
+
+/// A change waiting for another machine.
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct PendingView {
+ pub key: String,
+ pub size: u64,
+ pub since_ms: u64,
+ pub deleted: bool,
+}
+
+/// The whole picture.
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct ReplicaStatus {
+ pub me: String,
+ pub name: String,
+ pub role: Role,
+ pub owner: String,
+ pub owner_name: String,
+ pub i_own: bool,
+ pub epoch: u64,
+ pub applied: Pos,
+ pub tail: Pos,
+ pub peers: Vec<PeerView>,
+ pub pending: Vec<PendingView>,
+ pub pending_bytes: u64,
+ /// Machines (this one included) holding everything up to the owner's tail.
+ pub copies_now: usize,
+ /// Copies wanted.
+ pub copies_wanted: usize,
+ /// Records the owner holds that fewer than `copies_wanted` machines hold.
+ pub unreplicated: u64,
+ pub volume_bytes: u64,
+ pub bytes_sent: u64,
+ pub bytes_received: u64,
+ pub takeovers: u64,
+ pub handbacks: u64,
+ pub fenced_writes: u64,
+ pub conflict_copies: u64,
+}
+
+impl Health {
+ pub fn word(self) -> &'static str {
+ match self {
+ Health::Alive => "up",
+ Health::Suspect => "slow",
+ Health::Dead => "down",
+ Health::Away => "away",
+ }
+ }
+}
diff --git a/libs/cua/crates/cua-volume/src/replica/supervision.rs b/libs/cua/crates/cua-volume/src/replica/supervision.rs
new file mode 100644
index 0000000000..0876849758
--- /dev/null
+++ b/libs/cua/crates/cua-volume/src/replica/supervision.rs
@@ -0,0 +1,823 @@
+// SPDX-License-Identifier: FSL-1.1-MIT
+// Copyright (c) 2026 Cua AI, Inc.
+
+//! Who runs a durable agent, and who takes over when that machine goes.
+//!
+//! A durable agent's *driver* (the loop that starts the agent, watches it
+//! and re-drives it) must not live on the machine the user will close. It
+//! runs on the most reliable machine that holds the agent's files, and moves
+//! when that machine goes away. This module is that rule, built on the
+//! Volume's ordered writes (they are atomic across machines when the
+//! replicated store is on):
+//!
+//! ```text
+//! .cua-runs/<agent>.json the run: its task, and whether it is done
+//! agents/<agent>/.cua-supervisor the lease: holder, epoch, beat, progress
+//! ```
+//!
+//! Every machine runs the same small loop. The *preferred* machine is the
+//! most reliable one that is up and holds a copy (the same score placement
+//! uses; ties by id). It takes a free lease, or a lease whose holder is
+//! down, by compare-and-swap with `epoch + 1`; the old holder, if it wakes,
+//! finds the epoch moved on and stops (it is fenced), and anything it wrote
+//! in between merges by the order of the lease, not by luck. A laptop that
+//! holds a run yields it to a clearly more reliable machine that has been
+//! up and caught up for a while. Nothing here needs a clock: a holder is
+//! alive while its beat keeps changing as the observer sees it.
+//!
+//! What a takeover says is told from facts: the new holder compares the
+//! journal steps it holds with the progress the old holder last reported,
+//! and says "resumed" or "restarted from step N, steps M to K are redone".
+
+use std::collections::HashMap;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
+use std::time::{Duration, Instant};
+
+use serde::{Deserialize, Serialize};
+
+use super::node::{MachineView, ReplicaNode};
+use super::words::{Summary, closing_summary};
+use crate::backend::{Backend, Condition};
+use crate::{Error, Result};
+
+/// The lease object's name inside an agent's home.
+pub const LEASE: &str = ".cua-supervisor";
+/// The folder of run records.
+pub const RUNS_DIR: &str = ".cua-runs/";
+
+/// A run.
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct RunRecord {
+ pub agent: String,
+ /// What to run (the driver reads it).
+ pub spec: serde_json::Value,
+ /// `running` or `done`.
+ pub state: String,
+ pub created_by: String,
+ pub created_ms: u64,
+ /// A line for people when it is done.
+ #[serde(default)]
+ pub outcome: String,
+}
+
+/// The lease.
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct LeaseRec {
+ pub holder: String,
+ pub name: String,
+ pub epoch: u64,
+ /// Changes at every renewal: a holder is alive while this keeps moving.
+ pub beat: u64,
+ /// Steps of the run the holder says are done.
+ pub progress: u64,
+ /// The holder before this one (for the takeover sentence).
+ #[serde(default)]
+ pub previous: String,
+ /// The holder's CPU architecture.
+ #[serde(default)]
+ pub arch: String,
+ /// The previous holder's.
+ #[serde(default)]
+ pub previous_arch: String,
+}
+
+fn lease_key(agent: &str) -> Result<String> {
+ Ok(format!("{}{LEASE}", crate::path::agent_home(agent)?))
+}
+
+fn run_key(agent: &str) -> String {
+ format!("{RUNS_DIR}{agent}.json")
+}
+
+/// What the new holder knows about where the run stands.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct Resume {
+ /// The first holder (nothing to resume).
+ pub fresh: bool,
+ /// The step the previous holder last reported.
+ pub reported: u64,
+ /// The step this machine's copy of the journal reaches (the driver
+ /// fills it in; the supervisor leaves it at the reported step until then).
+ pub held: u64,
+}
+
+impl Resume {
+ /// The sentence for a takeover.
+ pub fn words(&self, to: &str, from: &str) -> String {
+ if self.fresh {
+ return format!("Started on {to}.");
+ }
+ if self.held >= self.reported {
+ format!(
+ "Resumed on {to} where {from} left off (step {}).",
+ self.held
+ )
+ } else {
+ format!(
+ "Resumed on {to} from step {}; steps {} to {} had not reached it and are done again.",
+ self.held,
+ self.held + 1,
+ self.reported
+ )
+ }
+ }
+}
+
+/// Handed to the driver while it holds a run.
+pub struct RunCtx {
+ pub agent: String,
+ pub epoch: u64,
+ pub spec: serde_json::Value,
+ pub machine: String,
+ pub backend: Arc<dyn Backend>,
+ pub resume: Resume,
+ fenced: Arc<AtomicBool>,
+ progress: Arc<AtomicU64>,
+ held: Arc<AtomicU64>,
+}
+
+impl RunCtx {
+ /// The run was taken from this machine (a later epoch holds it, or it
+ /// was handed to a better machine): stop.
+ pub fn fenced(&self) -> bool {
+ self.fenced.load(Ordering::SeqCst)
+ }
+
+ /// Reports how far the run is (in steps); renewals carry it.
+ pub fn report(&self, step: u64) {
+ self.progress.store(step, Ordering::SeqCst);
+ }
+
+ /// Sets what this machine's journal copy reaches (call once at start).
+ pub fn held_reaches(&mut self, step: u64) {
+ self.resume.held = step;
+ self.held.store(step, Ordering::SeqCst);
+ }
+}
+
+/// How a drive ended.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum DriveEnd {
+ /// The run finished; the line says how.
+ Done(String),
+ /// The driver stopped because it was fenced.
+ Fenced,
+ /// It failed; the supervisor leaves the lease to be adopted.
+ Failed(String),
+}
+
+/// What actually runs the agent.
+#[async_trait::async_trait]
+pub trait Driver: Send + Sync + 'static {
+ async fn drive(&self, ctx: RunCtx) -> DriveEnd;
+}
+
+/// Held while a run is active on this machine (a sleep assertion): dropping
+/// it releases whatever it holds.
+pub type AwakeToken = Box<dyn std::any::Any + Send>;
+
+/// Keeps this machine awake for a reason people can read.
+pub type Awake = Arc<dyn Fn(&str) -> AwakeToken + Send + Sync>;
+
+/// Timing.
+#[derive(Clone, Debug)]
+pub struct SupervisionConfig {
+ /// A lease whose beat has not changed this long is dead.
+ pub ttl: Duration,
+ pub tick: Duration,
+ /// A machine must have been up this long (and the run's files with it)
+ /// before a laptop yields the run to it.
+ pub yield_after: Duration,
+ /// A machine does not adopt a run of a holder it still sees up until the
+ /// beat stalls; a holder it cannot see (down) is adopted after this.
+ pub down_grace: Duration,
+}
+
+impl SupervisionConfig {
+ pub fn standard() -> Self {
+ SupervisionConfig {
+ ttl: Duration::from_secs(20),
+ tick: Duration::from_millis(500),
+ yield_after: Duration::from_secs(120),
+ down_grace: Duration::from_secs(1),
+ }
+ }
+
+ pub fn fast() -> Self {
+ SupervisionConfig {
+ ttl: Duration::from_millis(1800),
+ tick: Duration::from_millis(60),
+ yield_after: Duration::from_millis(1500),
+ down_grace: Duration::from_millis(100),
+ }
+ }
+}
+
+/// What a run looks like from here.
+#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
+pub struct RunView {
+ pub agent: String,
+ pub state: String,
+ pub holder: String,
+ pub holder_name: String,
+ pub epoch: u64,
+ pub progress: u64,
+ pub running_here: bool,
+ pub outcome: String,
+ /// What happened at the last takeover, in plain words.
+ pub note: String,
+}
+
+/// What `start_run` says.
+#[derive(Clone, Debug, PartialEq)]
+pub struct StartInfo {
+ /// Where the run will go (a name from the machines list).
+ pub run_on: String,
+ /// The one plain paragraph about closing this machine.
+ pub words: String,
+}
+
+/// What a takeover said, kept until the run ends (the words are made when
+/// asked, so they use what the driver found in its copy of the journal).
+struct Note {
+ fresh: bool,
+ reported: u64,
+ held: Arc<AtomicU64>,
+ to: String,
+ from: String,
+ /// Set when the run moved between architectures: what in the workspace
+ /// was built for the old one.
+ arch_change: Option<(String, Vec<String>)>,
+}
+
+impl Note {
+ fn words(&self) -> String {
+ Resume {
+ fresh: self.fresh,
+ reported: self.reported,
+ held: self.held.load(Ordering::SeqCst),
+ }
+ .words(&self.to, &self.from)
+ + &self.arch_sentence()
+ }
+
+ fn arch_sentence(&self) -> String {
+ let Some((from_arch, built)) = &self.arch_change else {
+ return String::new();
+ };
+ let what = if built.is_empty() {
+ String::new()
+ } else {
+ format!(" ({})", built.join(", "))
+ };
+ format!(
+ " {} is {from_arch}, so programs built there{what} are not reused here and should be built again; files, notes and the run's journal carry over as they are.",
+ self.from
+ )
+ }
+}
+
+struct Live {
+ fenced: Arc<AtomicBool>,
+ task: tokio::task::JoinHandle<()>,
+ _awake: Option<AwakeToken>,
+}
+
+#[derive(Default)]
+struct Seen {
+ etag: String,
+ since: Option<Instant>,
+}
+
+struct Inner {
+ node: ReplicaNode,
+ backend: Arc<dyn Backend>,
+ cfg: SupervisionConfig,
+ driver: Arc<dyn Driver>,
+ awake: Option<Awake>,
+ live: tokio::sync::Mutex<HashMap<String, Live>>,
+ seen: tokio::sync::Mutex<HashMap<String, Seen>>,
+ notes: std::sync::Mutex<HashMap<String, Note>>,
+ progress: std::sync::Mutex<HashMap<String, Arc<AtomicU64>>>,
+ stop: AtomicBool,
+ stable_since: tokio::sync::Mutex<HashMap<String, Instant>>,
+}
+
+/// The supervisor of one machine.
+#[derive(Clone)]
+pub struct Supervision {
+ inner: Arc<Inner>,
+}
+
+/// The machine that should run durable work now: the most reliable machine
+/// that is up and holds a copy (any up machine when none does), ties by id.
+pub fn preferred(machines: &[MachineView]) -> Option<&MachineView> {
+ let pick = |only_replicas: bool| {
+ machines
+ .iter()
+ .filter(|m| m.alive && (!only_replicas || m.replica))
+ .max_by(|a, b| {
+ a.facts
+ .rank
+ .cmp(&b.facts.rank)
+ .then_with(|| b.id.cmp(&a.id))
+ })
+ };
+ pick(true).or_else(|| pick(false))
+}
+
+impl Supervision {
+ pub fn new(
+ node: ReplicaNode,
+ backend: Arc<dyn Backend>,
+ cfg: SupervisionConfig,
+ driver: Arc<dyn Driver>,
+ awake: Option<Awake>,
+ ) -> Supervision {
+ Supervision {
+ inner: Arc::new(Inner {
+ node,
+ backend,
+ cfg,
+ driver,
+ awake,
+ live: Default::default(),
+ seen: Default::default(),
+ notes: Default::default(),
+ progress: Default::default(),
+ stop: AtomicBool::new(false),
+ stable_since: Default::default(),
+ }),
+ }
+ }
+
+ /// Starts the loop.
+ pub fn spawn(&self) {
+ let me = self.inner.clone();
+ tokio::spawn(async move {
+ while !me.stop.load(Ordering::SeqCst) {
+ tokio::time::sleep(me.cfg.tick).await;
+ if let Err(e) = me.tick().await {
+ tracing::debug!(error = %e, "supervision tick");
+ }
+ }
+ });
+ }
+
+ /// Stops the loop and the runs here (they are not fenced: another
+ /// machine takes them when the lease goes stale).
+ pub async fn stop(&self) {
+ self.inner.stop.store(true, Ordering::SeqCst);
+ let all: Vec<Live> = self
+ .inner
+ .live
+ .lock()
+ .await
+ .drain()
+ .map(|(_, v)| v)
+ .collect();
+ for l in all {
+ l.task.abort();
+ }
+ }
+
+ /// Records a run. The most reliable machine takes it on its next look;
+ /// this machine does if it is the only one.
+ pub async fn start_run(&self, agent: &str, spec: serde_json::Value) -> Result<StartInfo> {
+ // A name the Volume would refuse fails here, not silently later.
+ crate::path::agent_home(agent)?;
+ let rec = RunRecord {
+ agent: agent.to_string(),
+ spec,
+ state: "running".into(),
+ created_by: self.inner.node.machine().to_string(),
+ created_ms: crate::now_ms(),
+ outcome: String::new(),
+ };
+ self.inner
+ .backend
+ .put(&run_key(agent), serde_json::to_vec(&rec)?, Condition::None)
+ .await?;
+ let machines = self.inner.node.machines().await;
+ let run_on = preferred(&machines).map(|m| m.name.clone());
+ let words = self.words(run_on.as_deref()).await;
+ Ok(StartInfo {
+ run_on: run_on.unwrap_or_default(),
+ words,
+ })
+ }
+
+ /// The paragraph about closing this machine for a run on `run_on`.
+ pub async fn words(&self, run_on: Option<&str>) -> String {
+ let s: Summary = self.inner.node.summary(run_on).await;
+ closing_summary(&s)
+ }
+
+ /// Every run, as this machine sees it.
+ pub async fn runs(&self) -> Result<Vec<RunView>> {
+ let mut out = vec![];
+ for m in self.inner.backend.list(RUNS_DIR).await? {
+ let Ok((b, _)) = self.inner.backend.get(&m.key, None).await else {
+ continue;
+ };
+ let Ok(rec) = serde_json::from_slice::<RunRecord>(&b) else {
+ continue;
+ };
+ let lease = self.inner.read_lease(&rec.agent).await.ok().flatten();
+ let running_here = self.inner.live.lock().await.contains_key(&rec.agent);
+ out.push(RunView {
+ agent: rec.agent.clone(),
+ state: rec.state,
+ holder: lease
+ .as_ref()
+ .map(|l| l.0.holder.clone())
+ .unwrap_or_default(),
+ holder_name: lease.as_ref().map(|l| l.0.name.clone()).unwrap_or_default(),
+ epoch: lease.as_ref().map_or(0, |l| l.0.epoch),
+ progress: lease.as_ref().map_or(0, |l| l.0.progress),
+ running_here,
+ outcome: rec.outcome,
+ note: self
+ .inner
+ .notes
+ .lock()
+ .expect("notes")
+ .get(&rec.agent)
+ .map(Note::words)
+ .unwrap_or_default(),
+ });
+ }
+ Ok(out)
+ }
+}
+
+/// The name of machine `id` in the machines list.
+async fn machines_name(node: &ReplicaNode, id: &str) -> String {
+ node.machines()
+ .await
+ .into_iter()
+ .find(|m| m.id == id)
+ .map(|m| m.name)
+ .unwrap_or_else(|| id.to_string())
+}
+
+impl Inner {
+ async fn read_lease(&self, agent: &str) -> Result<Option<(LeaseRec, String)>> {
+ let k = lease_key(agent)?;
+ match self.backend.get(&k, None).await {
+ Ok((b, meta)) => {
+ let l: LeaseRec = serde_json::from_slice(&b)?;
+ Ok(Some((l, meta.etag)))
+ }
+ Err(Error::NotFound(_)) => Ok(None),
+ Err(e) => Err(e),
+ }
+ }
+
+ async fn tick(self: &Arc<Self>) -> Result<()> {
+ let machines = self.node.machines().await;
+ let me = self.node.machine().to_string();
+ let pref = preferred(&machines).map(|m| m.id.clone());
+ for m in self.backend.list(RUNS_DIR).await? {
+ let Ok((b, _)) = self.backend.get(&m.key, None).await else {
+ continue;
+ };
+ let Ok(rec) = serde_json::from_slice::<RunRecord>(&b) else {
+ continue;
+ };
+ if rec.state == "done" {
+ self.drop_live(&rec.agent, true).await;
+ continue;
+ }
+ self.look(&rec, &machines, &me, pref.as_deref()).await;
+ }
+ Ok(())
+ }
+
+ async fn drop_live(&self, agent: &str, fence: bool) {
+ if let Some(l) = self.live.lock().await.remove(agent) {
+ if fence {
+ l.fenced.store(true, Ordering::SeqCst);
+ }
+ // Give the driver a moment to stop at a step boundary.
+ let t = l.task;
+ let _ = tokio::time::timeout(self.cfg.ttl, t).await;
+ }
+ }
+
+ async fn look(
+ self: &Arc<Self>,
+ rec: &RunRecord,
+ machines: &[MachineView],
+ me: &str,
+ pref: Option<&str>,
+ ) {
+ let agent = &rec.agent;
+ let lease = match self.read_lease(agent).await {
+ Ok(l) => l,
+ Err(_) => return,
+ };
+ let running_here = self.live.lock().await.contains_key(agent);
+ match lease {
+ None => {
+ if pref == Some(me) {
+ let l = LeaseRec {
+ holder: me.to_string(),
+ name: self.node.name().to_string(),
+ epoch: 1,
+ beat: 0,
+ progress: 0,
+ previous: String::new(),
+ arch: String::new(),
+ previous_arch: String::new(),
+ };
+ self.claim(rec, l, Condition::IfNoneMatch, true).await;
+ }
+ }
+ Some((l, etag)) => {
+ // Observe the beat on this machine's own clock.
+ let (dead, quiet_for) = {
+ let mut seen = self.seen.lock().await;
+ let s = seen.entry(agent.clone()).or_default();
+ if s.etag != etag {
+ s.etag = etag.clone();
+ s.since = Some(Instant::now());
+ }
+ let q = s.since.map_or(Duration::ZERO, |t| t.elapsed());
+ (q >= self.cfg.ttl, q)
+ };
+ if l.holder == me {
+ if !running_here {
+ // A restart of this machine's process while it held
+ // the run: take it again at the next epoch.
+ let next = LeaseRec {
+ epoch: l.epoch + 1,
+ beat: 0,
+ previous: me.to_string(),
+ previous_arch: l.arch.clone(),
+ ..l.clone()
+ };
+ self.claim(rec, next, Condition::IfMatch(etag), false).await;
+ return;
+ }
+ self.renew(rec, &l, &etag, machines, me, pref).await;
+ return;
+ }
+ // Someone else holds it. If this machine runs it, it is
+ // fenced.
+ if running_here {
+ tracing::info!(
+ agent,
+ epoch = l.epoch,
+ "supervision: this machine was fenced"
+ );
+ self.fence_here(agent).await;
+ return;
+ }
+ if pref != Some(me) {
+ return;
+ }
+ // Confirmed down, not merely slow: a holder that is only
+ // late keeps its run until its beat stalls for the whole ttl.
+ let holder_down = machines
+ .iter()
+ .find(|m| m.id == l.holder)
+ .is_none_or(|m| m.down);
+ // The preferred machine takes a run whose holder is down
+ // (after a short grace) or whose beat stopped.
+ let take = dead || (holder_down && quiet_for >= self.cfg.down_grace);
+ // Or the holder handed it to this machine.
+ let handed = l.holder == me;
+ if take || handed {
+ let next = LeaseRec {
+ holder: me.to_string(),
+ name: self.node.name().to_string(),
+ epoch: l.epoch + 1,
+ beat: 0,
+ progress: l.progress,
+ previous: l.holder.clone(),
+ arch: String::new(),
+ previous_arch: l.arch.clone(),
+ };
+ self.claim(rec, next, Condition::IfMatch(etag), false).await;
+ }
+ }
+ }
+ }
+
+ /// Takes the lease with `cond` and starts the driver.
+ async fn claim(
+ self: &Arc<Self>,
+ rec: &RunRecord,
+ mut l: LeaseRec,
+ cond: Condition,
+ fresh: bool,
+ ) {
+ // The holder's own architecture goes into the lease.
+ l.arch = self
+ .node
+ .machines()
+ .await
+ .into_iter()
+ .find(|m| m.is_me)
+ .map(|m| super::roles::normalize_arch(&m.facts.arch))
+ .unwrap_or_default();
+ let Ok(k) = lease_key(&rec.agent) else { return };
+ let Ok(bytes) = serde_json::to_vec(&l) else {
+ return;
+ };
+ match self.backend.put(&k, bytes, cond).await {
+ Ok(_) => {}
+ Err(_) => return,
+ }
+ let note = Resume {
+ fresh,
+ reported: l.progress,
+ held: l.progress,
+ };
+ tracing::info!(agent = %rec.agent, epoch = l.epoch, "supervision: this machine runs the agent now");
+ self.start_driver(rec, l, note).await;
+ }
+
+ async fn start_driver(self: &Arc<Self>, rec: &RunRecord, l: LeaseRec, resume: Resume) {
+ let fenced = Arc::new(AtomicBool::new(false));
+ let progress = Arc::new(AtomicU64::new(l.progress));
+ self.progress
+ .lock()
+ .expect("progress")
+ .insert(rec.agent.clone(), progress.clone());
+ let awake = self
+ .awake
+ .as_ref()
+ .map(|f| f(&format!("Cua: the durable agent {} is running", rec.agent)));
+ let held = Arc::new(AtomicU64::new(resume.held));
+ let ctx = RunCtx {
+ agent: rec.agent.clone(),
+ epoch: l.epoch,
+ spec: rec.spec.clone(),
+ machine: self.node.machine().to_string(),
+ backend: self.backend.clone(),
+ resume: resume.clone(),
+ fenced: fenced.clone(),
+ progress,
+ held: held.clone(),
+ };
+ let driver = self.driver.clone();
+ let me = self.clone();
+ let agent = rec.agent.clone();
+ let from = l.previous.clone();
+ let to = self.node.name().to_string();
+ let from_name = machines_name(&self.node, &from).await;
+ let arch_change =
+ if !l.previous_arch.is_empty() && !l.arch.is_empty() && l.previous_arch != l.arch {
+ let keys = self
+ .backend
+ .list(&crate::path::agent_home(&rec.agent).unwrap_or_default())
+ .await
+ .map(|v| v.into_iter().map(|m| m.key).collect::<Vec<_>>())
+ .unwrap_or_default();
+ Some((
+ l.previous_arch.clone(),
+ super::roles::arch_specific(keys, 4),
+ ))
+ } else {
+ None
+ };
+ self.notes.lock().expect("notes").insert(
+ agent.clone(),
+ Note {
+ fresh: resume.fresh,
+ reported: resume.reported,
+ held,
+ to,
+ from: from_name,
+ arch_change,
+ },
+ );
+ let rec2 = rec.clone();
+ let task = tokio::spawn(async move {
+ let end = driver.drive(ctx).await;
+ match end {
+ DriveEnd::Done(outcome) => {
+ let mut done = rec2.clone();
+ done.state = "done".into();
+ done.outcome = outcome;
+ if let Ok(b) = serde_json::to_vec(&done) {
+ let _ = me.backend.put(&run_key(&agent), b, Condition::None).await;
+ }
+ }
+ DriveEnd::Fenced => {}
+ DriveEnd::Failed(why) => {
+ tracing::warn!(agent = %agent, %why, "supervision: the driver failed");
+ // The lease goes stale and the preferred machine re-drives.
+ me.live.lock().await.remove(&agent);
+ }
+ }
+ });
+ self.live.lock().await.insert(
+ rec.agent.clone(),
+ Live {
+ fenced,
+ task,
+ _awake: awake,
+ },
+ );
+ }
+
+ async fn fence_here(&self, agent: &str) {
+ if let Some(l) = self.live.lock().await.remove(agent) {
+ l.fenced.store(true, Ordering::SeqCst);
+ let t = l.task;
+ let _ = tokio::time::timeout(self.cfg.ttl, t).await;
+ }
+ }
+
+ /// The holder's tick: renew by compare-and-swap, and hand the run to a
+ /// clearly better machine when this one is a laptop.
+ async fn renew(
+ self: &Arc<Self>,
+ rec: &RunRecord,
+ l: &LeaseRec,
+ etag: &str,
+ machines: &[MachineView],
+ me: &str,
+ pref: Option<&str>,
+ ) {
+ // A driver that ended (failed) is not running here any more: handled
+ // by the caller through `running_here`.
+ let progress = self
+ .progress
+ .lock()
+ .expect("progress")
+ .get(&rec.agent)
+ .map_or(l.progress, |p| p.load(Ordering::SeqCst));
+ let mut next = LeaseRec {
+ beat: l.beat + 1,
+ progress,
+ ..l.clone()
+ };
+ // Yield to a clearly better machine that has been up for a while.
+ if let (Some(p), Some(mine)) = (pref, machines.iter().find(|m| m.id == me))
+ && p != me
+ && mine.facts.rank + super::roles::STEP_RANK
+ <= machines
+ .iter()
+ .find(|m| m.id == p)
+ .map_or(0, |m| m.facts.rank)
+ && machines.iter().any(|m| m.id == p && m.stable)
+ {
+ let since = {
+ let mut s = self.stable_since.lock().await;
+ *s.entry(rec.agent.clone()).or_insert_with(Instant::now)
+ };
+ if since.elapsed() >= self.cfg.yield_after {
+ let target = machines.iter().find(|m| m.id == p).expect("pref");
+ next = LeaseRec {
+ holder: target.id.clone(),
+ name: target.name.clone(),
+ epoch: l.epoch + 1,
+ beat: 0,
+ progress,
+ previous: me.to_string(),
+ arch: target.facts.arch.clone(),
+ previous_arch: l.arch.clone(),
+ };
+ if let Ok(k) = lease_key(&rec.agent)
+ && let Ok(b) = serde_json::to_vec(&next)
+ && self
+ .backend
+ .put(&k, b, Condition::IfMatch(etag.to_string()))
+ .await
+ .is_ok()
+ {
+ tracing::info!(agent = %rec.agent, to = %target.name, "supervision: handed the run to a more reliable machine");
+ self.stable_since.lock().await.remove(&rec.agent);
+ self.fence_here(&rec.agent).await;
+ }
+ return;
+ }
+ } else {
+ self.stable_since.lock().await.remove(&rec.agent);
+ }
+ let Ok(k) = lease_key(&rec.agent) else { return };
+ let Ok(b) = serde_json::to_vec(&next) else {
+ return;
+ };
+ match self
+ .backend
+ .put(&k, b, Condition::IfMatch(etag.to_string()))
+ .await
+ {
+ Ok(_) => {}
+ Err(Error::Precondition(_)) => {
+ // Someone changed the lease: if it is not ours any more, we
+ // are fenced; the next look decides.
+ }
+ Err(_) => {
+ // The owner is out of reach: the run goes on (its files are
+ // saved here and sync later); the beat just does not move.
+ }
+ }
+ }
+}
diff --git a/libs/cua/crates/cua-volume/src/replica/transport.rs b/libs/cua/crates/cua-volume/src/replica/transport.rs
new file mode 100644
index 0000000000..adbe30db79
--- /dev/null
+++ b/libs/cua/crates/cua-volume/src/replica/transport.rs
@@ -0,0 +1,75 @@
+// SPDX-License-Identifier: FSL-1.1-MIT
+// Copyright (c) 2026 Cua AI, Inc.
+
+//! How machines reach each other.
+//!
+//! In the product a peer link is a stream over the Cua mesh (WireGuard
+//! between the account's node keys; the relay forwards ciphertext), so data
+//! only ever moves between the user's own signed-in machines and is
+//! encrypted. [`Transport`] is the seam: the node asks it for a stream to a
+//! machine id and hands it every stream that arrives. [`TcpTransport`] is
+//! the same thing over plain TCP for LANs and tests, and must be wrapped in
+//! something that encrypts (an SSH tunnel, the mesh) outside a test.
+
+use std::collections::HashMap;
+use std::net::SocketAddr;
+use std::sync::{Arc, Mutex};
+use std::time::Duration;
+
+use super::link::Stream;
+
+/// Opens streams to machines.
+#[async_trait::async_trait]
+pub trait Transport: Send + Sync + 'static {
+ /// A stream to `machine` (its machine id).
+ async fn dial(&self, machine: &str) -> std::io::Result<Stream>;
+}
+
+/// Plain TCP, addresses by machine id.
+#[derive(Clone, Default)]
+pub struct TcpTransport {
+ addrs: Arc<Mutex<HashMap<String, SocketAddr>>>,
+ timeout: Duration,
+}
+
+impl TcpTransport {
+ pub fn new() -> TcpTransport {
+ TcpTransport {
+ addrs: Arc::default(),
+ timeout: Duration::from_secs(5),
+ }
+ }
+
+ /// Where `machine` listens.
+ pub fn set(&self, machine: &str, addr: SocketAddr) {
+ self.addrs
+ .lock()
+ .expect("addrs")
+ .insert(machine.into(), addr);
+ }
+}
+
+#[async_trait::async_trait]
+impl Transport for TcpTransport {
+ async fn dial(&self, machine: &str) -> std::io::Result<Stream> {
+ let addr = self
+ .addrs
+ .lock()
+ .expect("addrs")
+ .get(machine)
+ .copied()
+ .ok_or_else(|| {
+ std::io::Error::new(
+ std::io::ErrorKind::NotFound,
+ format!("no address for {machine}"),
+ )
+ })?;
+ let s = tokio::time::timeout(self.timeout, tokio::net::TcpStream::connect(addr))
+ .await
+ .map_err(|_| {
+ std::io::Error::new(std::io::ErrorKind::TimedOut, "connect timed out")
+ })??;
+ s.set_nodelay(true)?;
+ Ok(Box::new(s))
+ }
+}
diff --git a/libs/cua/crates/cua-volume/src/replica/wire.rs b/libs/cua/crates/cua-volume/src/replica/wire.rs
new file mode 100644
index 0000000000..502ee7ecd6
--- /dev/null
+++ b/libs/cua/crates/cua-volume/src/replica/wire.rs
@@ -0,0 +1,479 @@
+// SPDX-License-Identifier: FSL-1.1-MIT
+// Copyright (c) 2026 Cua AI, Inc.
+
+//! The peer protocol: framed JSON with an optional binary tail.
+//!
+//! ```text
+//! [len: u32 BE][json_len: u32 BE][json][data] len = 4 + json_len + data
+//! ```
+//!
+//! Two links join a pair of machines. The *control* link carries heartbeats,
+//! era changes, log records and small requests; it never carries bulk bytes,
+//! so a slow link cannot delay a heartbeat behind a file. The *data* link
+//! carries block transfers and large writes.
+
+use serde::{Deserialize, Serialize};
+
+use super::MachineId;
+use super::era::{EraRec, Pos};
+use super::oplog::Op;
+use super::roles::Facts;
+use crate::Error;
+use crate::backend::{Condition, ObjectMeta, VersionInfo};
+
+/// Largest frame accepted.
+pub const MAX_FRAME: usize = 8 * 1024 * 1024;
+
+/// Which link of the pair.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum Chan {
+ Control,
+ Data,
+}
+
+/// The first message each side sends on a new link.
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Hello {
+ pub machine: MachineId,
+ pub name: String,
+ pub chan: Chan,
+ pub lineage: String,
+ pub eras: Vec<EraRec>,
+ pub applied: Pos,
+ pub tail: Pos,
+ pub facts: Facts,
+ /// Protocol version.
+ pub v: u32,
+}
+
+/// Protocol version.
+pub const VERSION: u32 = 1;
+
+/// A heartbeat.
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Beat {
+ pub epoch: u64,
+ pub owner: MachineId,
+ /// Which volume this machine belongs to (two machines of different
+ /// lineages that find each other merge, the less reliable one joining).
+ #[serde(default)]
+ pub lineage: String,
+ /// The newest eras the sender knows (so a machine that missed a
+ /// takeover learns it from any heartbeat).
+ #[serde(default)]
+ pub eras: Vec<EraRec>,
+ /// Why the sender cannot keep a copy (a full disk), when it cannot.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub problem: Option<String>,
+ pub applied: Pos,
+ pub tail: Pos,
+ pub facts: Facts,
+ /// Changes waiting in the sender's outbox.
+ #[serde(default)]
+ pub pending: u64,
+}
+
+/// A precondition on the wire.
+#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(tag = "c", content = "etag", rename_all = "snake_case")]
+pub enum Cond {
+ #[default]
+ None,
+ IfNoneMatch,
+ IfMatch(String),
+}
+
+impl From<Condition> for Cond {
+ fn from(c: Condition) -> Cond {
+ match c {
+ Condition::None => Cond::None,
+ Condition::IfNoneMatch => Cond::IfNoneMatch,
+ Condition::IfMatch(e) => Cond::IfMatch(e),
+ }
+ }
+}
+
+impl From<Cond> for Condition {
+ fn from(c: Cond) -> Condition {
+ match c {
+ Cond::None => Condition::None,
+ Cond::IfNoneMatch => Condition::IfNoneMatch,
+ Cond::IfMatch(e) => Condition::IfMatch(e),
+ }
+ }
+}
+
+/// Where the bytes of a version a peer wants live on the sender.
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(tag = "s", rename_all = "snake_case")]
+pub enum Src {
+ /// The sender's data store (history is kept).
+ Data { key: String, version: String },
+ /// The sender's spool of changes waiting for the owner.
+ Spool { key: String, version: String },
+}
+
+/// A change waiting for the owner, as the owner sees it.
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Pending {
+ pub id: String,
+ pub key: String,
+ /// `None` for a delete.
+ pub meta: Option<ObjectMeta>,
+ /// The head version the writer saw before its change.
+ pub base: Option<String>,
+ pub cond: Cond,
+ pub machine: MachineId,
+ pub name: String,
+ pub ts_ms: u64,
+ /// Where the content is on the sender.
+ pub src: Src,
+ /// Offset and length of an inline body in the frame's data.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub body: Option<(u64, u64)>,
+}
+
+/// What the owner did with one pending change.
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(tag = "r", rename_all = "snake_case")]
+pub enum Outcome {
+ /// Applied as the new head.
+ Applied { meta: ObjectMeta, pos: Pos },
+ /// A delete was applied.
+ Deleted { pos: Pos },
+ /// The owner already had this content at the key.
+ Merged,
+ /// The key changed meanwhile; the content is at `key` as a conflict copy.
+ Copy { key: String, pos: Pos },
+ /// A delete lost to a newer write; the write stays.
+ Kept,
+ /// A precondition failed.
+ Precondition { message: String },
+ /// Could not apply.
+ Failed { message: String },
+}
+
+/// A forwarded `Backend` call.
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(tag = "q", rename_all = "snake_case")]
+pub enum Req {
+ /// Body in `data`.
+ Put {
+ key: String,
+ cond: Cond,
+ },
+ Delete {
+ key: String,
+ cond: Cond,
+ },
+ Head {
+ key: String,
+ },
+ Get {
+ key: String,
+ version: Option<String>,
+ },
+ GetRange {
+ key: String,
+ version: String,
+ offset: u64,
+ len: u64,
+ },
+ List {
+ prefix: String,
+ },
+ ListDir {
+ folder: String,
+ },
+ Versions {
+ key: String,
+ },
+ /// Heads after `after_key`, at most `max` (a snapshot's pages).
+ Heads {
+ after_key: String,
+ max: usize,
+ },
+}
+
+/// The answer to a [`Req`].
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(tag = "p", rename_all = "snake_case")]
+pub enum Rep {
+ Meta {
+ meta: ObjectMeta,
+ pos: Pos,
+ /// Machines holding it when the owner answered.
+ copies: u32,
+ /// The owner counted it durable (enough copies).
+ durable: bool,
+ },
+ Done {
+ pos: Pos,
+ copies: u32,
+ durable: bool,
+ },
+ Head {
+ meta: Option<ObjectMeta>,
+ },
+ /// The bytes are in `data`.
+ Bytes {
+ meta: ObjectMeta,
+ },
+ Range,
+ List {
+ items: Vec<ObjectMeta>,
+ },
+ ListDir {
+ files: Vec<ObjectMeta>,
+ folders: Vec<String>,
+ },
+ Versions {
+ items: Vec<VersionInfo>,
+ },
+ Heads {
+ items: Vec<ObjectMeta>,
+ done: bool,
+ at: Pos,
+ },
+ Fail {
+ kind: String,
+ message: String,
+ },
+}
+
+impl Rep {
+ pub fn fail(e: &Error) -> Rep {
+ Rep::Fail {
+ kind: e.tag().to_string(),
+ message: match e {
+ Error::Invalid(m)
+ | Error::NotFound(m)
+ | Error::Forbidden(m)
+ | Error::Precondition(m)
+ | Error::Backend(m)
+ | Error::NotConfirmed(m) => m.clone(),
+ other => other.to_string(),
+ },
+ }
+ }
+
+ /// The error a `Fail` stands for.
+ pub fn into_error(kind: &str, message: String) -> Error {
+ match kind {
+ "invalid_argument" => Error::Invalid(message),
+ "not_found" => Error::NotFound(message),
+ "forbidden" => Error::Forbidden(message),
+ "precondition_failed" => Error::Precondition(message),
+ _ => Error::Backend(message),
+ }
+ }
+}
+
+/// A record on the wire; inline bodies are slices of the frame's data.
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct RecordW {
+ pub pos: Pos,
+ pub op: Op,
+ pub at_ms: u64,
+ /// Offset and length of the body in the frame's data (small puts).
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub body: Option<(u64, u64)>,
+}
+
+/// Everything that crosses a link.
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(tag = "t", rename_all = "snake_case")]
+pub enum Msg {
+ Hello(Hello),
+ Ping(Beat),
+ Pong(Beat),
+ /// The sender is going away (sleep, shutdown, network lost).
+ Bye {
+ reason: String,
+ },
+ /// Log records from the owner.
+ Append {
+ epoch: u64,
+ records: Vec<RecordW>,
+ },
+ /// Replica to owner: applied through `applied`.
+ Ack {
+ applied: Pos,
+ },
+ /// The sender is below the receiver's epoch; here is the history.
+ Fenced {
+ eras: Vec<EraRec>,
+ },
+ /// The receiver holds records past this era's base: keep up to `keep`.
+ Diverged {
+ keep: Pos,
+ eras: Vec<EraRec>,
+ },
+ /// Replica to owner: a gap; resend after `applied`.
+ Behind {
+ applied: Pos,
+ },
+ /// Owner to replica: the log no longer reaches you; pull a snapshot at `at`.
+ Snapshot {
+ at: Pos,
+ },
+ /// A candidate asks to take over at `epoch`.
+ Prepare {
+ epoch: u64,
+ base: Pos,
+ },
+ Promise {
+ ok: bool,
+ reason: String,
+ },
+ /// A new era (takeover or hand-back).
+ Era {
+ recs: Vec<EraRec>,
+ },
+ /// The owner offers the era to `to` at `epoch`, having logged through `base`.
+ Handoff {
+ epoch: u64,
+ to: MachineId,
+ base: Pos,
+ },
+ HandoffReply {
+ ok: bool,
+ reason: String,
+ },
+ /// Changes waiting for the owner.
+ Submit {
+ ops: Vec<Pending>,
+ },
+ Submitted {
+ outcomes: Vec<Outcome>,
+ },
+ Req(Req),
+ Rep(Rep),
+ /// Block hashes of a source: `count` hashes from block `from`.
+ Hashes {
+ src: Src,
+ block: u64,
+ from: u64,
+ count: u64,
+ },
+ HashesAre {
+ hashes: Vec<String>,
+ size: u64,
+ },
+ /// One block of a source, bytes in `data`.
+ Block {
+ src: Src,
+ block: u64,
+ idx: u64,
+ },
+ BlockIs,
+ /// A refusal or failure of any request.
+ Error {
+ kind: String,
+ message: String,
+ },
+}
+
+/// One frame.
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Frame {
+ pub id: u64,
+ /// The id of the request this answers.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub re: Option<u64>,
+ pub msg: Msg,
+}
+
+/// Encodes a frame and its data.
+pub fn encode(f: &Frame, data: &[u8]) -> std::io::Result<Vec<u8>> {
+ let json = serde_json::to_vec(f)
+ .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
+ let len = 4 + json.len() + data.len();
+ if len > MAX_FRAME {
+ return Err(std::io::Error::new(
+ std::io::ErrorKind::InvalidInput,
+ format!("frame of {len} bytes"),
+ ));
+ }
+ let mut out = Vec::with_capacity(8 + json.len() + data.len());
+ out.extend_from_slice(&(len as u32).to_be_bytes());
+ out.extend_from_slice(&(json.len() as u32).to_be_bytes());
+ out.extend_from_slice(&json);
+ out.extend_from_slice(data);
+ Ok(out)
+}
+
+/// Reads one frame; `None` at a clean end of stream.
+pub async fn read<R: tokio::io::AsyncRead + Unpin>(
+ r: &mut R,
+) -> std::io::Result<Option<(Frame, Vec<u8>)>> {
+ use tokio::io::AsyncReadExt as _;
+ let mut len = [0u8; 4];
+ match r.read_exact(&mut len).await {
+ Ok(_) => {}
+ Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => return Ok(None),
+ Err(e) => return Err(e),
+ }
+ let len = u32::from_be_bytes(len) as usize;
+ if !(4..=MAX_FRAME).contains(&len) {
+ return Err(std::io::Error::new(
+ std::io::ErrorKind::InvalidData,
+ format!("frame of {len} bytes"),
+ ));
+ }
+ let mut body = vec![0u8; len];
+ r.read_exact(&mut body).await?;
+ let json_len = u32::from_be_bytes([body[0], body[1], body[2], body[3]]) as usize;
+ if 4 + json_len > len {
+ return Err(std::io::Error::new(
+ std::io::ErrorKind::InvalidData,
+ "json longer than its frame",
+ ));
+ }
+ let data = body.split_off(4 + json_len);
+ let frame: Frame = serde_json::from_slice(&body[4..])
+ .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
+ Ok(Some((frame, data)))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[tokio::test]
+ async fn frames_round_trip_with_data() {
+ let f = Frame {
+ id: 7,
+ re: Some(3),
+ msg: Msg::Block {
+ src: Src::Data {
+ key: "a/b".into(),
+ version: "v".into(),
+ },
+ block: 1 << 20,
+ idx: 4,
+ },
+ };
+ let bytes = encode(&f, b"payload").unwrap();
+ let mut r = std::io::Cursor::new(bytes);
+ let (g, d) = read(&mut r).await.unwrap().unwrap();
+ assert_eq!(f, g);
+ assert_eq!(d, b"payload");
+ assert!(read(&mut r).await.unwrap().is_none());
+ }
+
+ #[test]
+ fn errors_keep_their_kind_across_the_wire() {
+ for e in [
+ Error::NotFound("k".into()),
+ Error::Precondition("changed".into()),
+ Error::Invalid("x".into()),
+ ] {
+ let Rep::Fail { kind, message } = Rep::fail(&e) else {
+ panic!()
+ };
+ assert_eq!(Rep::into_error(&kind, message), e);
+ }
+ }
+}
diff --git a/libs/cua/crates/cua-volume/src/replica/words.rs b/libs/cua/crates/cua-volume/src/replica/words.rs
new file mode 100644
index 0000000000..2d27eb3d06
--- /dev/null
+++ b/libs/cua/crates/cua-volume/src/replica/words.rs
@@ -0,0 +1,206 @@
+// SPDX-License-Identifier: FSL-1.1-MIT
+// Copyright (c) 2026 Cua AI, Inc.
+
+//! The one place that says, in plain words, what happens when you close the
+//! laptop. Shown when a durable agent starts and in the app. It uses the
+//! names from the machines list and nothing else: no settings, no new
+//! nouns, no promise the facts do not back.
+
+use serde::{Deserialize, Serialize};
+
+/// The facts the sentence is made of.
+#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Summary {
+ /// This machine's name ("MacBook Pro").
+ pub this_machine: String,
+ /// The machine running the durable agent's driver, when known.
+ pub run_on: Option<String>,
+ /// Machines signed in to the account, including this one.
+ pub machines: usize,
+ /// Names of machines that hold a copy of the files and are up now.
+ pub holders_up: Vec<String>,
+ /// Names of machines that hold a copy and are away now.
+ pub holders_away: Vec<String>,
+ /// Changes saved only on this machine, waiting for another one.
+ pub pending_here: u64,
+ /// The machine the changes are waiting for, when known.
+ pub pending_for: Option<String>,
+ /// This machine can sleep when its lid closes.
+ pub this_sleeps: bool,
+ /// A sleep assertion is held here for the run.
+ pub sleep_blocked: bool,
+}
+
+fn machines(n: usize) -> String {
+ if n == 1 {
+ "1 machine".into()
+ } else {
+ format!("{n} machines")
+ }
+}
+
+fn list(names: &[String]) -> String {
+ match names {
+ [] => String::new(),
+ [a] => a.clone(),
+ [a, b] => format!("{a} and {b}"),
+ [rest @ .., last] => format!("{}, and {last}", rest.join(", ")),
+ }
+}
+
+/// What will happen when this machine closes, in one short paragraph.
+pub fn closing_summary(s: &Summary) -> String {
+ let here = s.this_machine.as_str();
+ let on_other = s
+ .run_on
+ .as_deref()
+ .filter(|r| !r.is_empty() && *r != here)
+ .map(str::to_string);
+ let mut out = String::new();
+ match (&on_other, s.machines) {
+ (Some(other), _) => {
+ out.push_str(&format!("Your agent keeps running on {other}. "));
+ let copies = s.holders_up.len();
+ let other_holds = s.holders_up.iter().any(|h| h == other);
+ if copies >= 2 {
+ out.push_str(&format!(
+ "Files are on {}. You can close this machine.",
+ machines(copies)
+ ));
+ } else if other_holds {
+ let missing = if s.holders_away.is_empty() {
+ here.to_string()
+ } else {
+ list(&s.holders_away)
+ };
+ out.push_str(&format!(
+ "Files are on 1 machine until {missing} is back. You can close this machine; nothing is lost, there is just one copy until then."
+ ));
+ } else {
+ out.push_str("Files are not on that machine yet; wait for them to copy before you close this machine.");
+ }
+ }
+ (None, n) if n <= 1 => {
+ out.push_str(&format!(
+ "Your agent runs on {here} and will pause when it sleeps. "
+ ));
+ if s.sleep_blocked {
+ out.push_str("It stays awake while the agent works, but closing the lid still puts a laptop to sleep unless it is plugged in with a display attached. ");
+ }
+ out.push_str("Add another machine and it keeps running when you close this one.");
+ }
+ (None, _) => {
+ out.push_str(&format!("Your agent runs on {here}"));
+ if s.this_sleeps {
+ out.push_str(" and will pause when it sleeps. ");
+ } else {
+ out.push_str(". ");
+ }
+ if s.holders_away.is_empty() {
+ out.push_str("Your other machines are not available to take over right now.");
+ } else {
+ out.push_str(&format!("{} is away right now.", list(&s.holders_away)));
+ }
+ }
+ }
+ if s.pending_here > 0 {
+ let who = s.pending_for.as_deref().unwrap_or("another machine");
+ out.push_str(&format!(
+ " {} saved only on {here} will sync when {who} is back.",
+ if s.pending_here == 1 {
+ "1 change".to_string()
+ } else {
+ format!("{} changes", s.pending_here)
+ }
+ ));
+ }
+ out
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn base() -> Summary {
+ Summary {
+ this_machine: "MacBook".into(),
+ machines: 2,
+ ..Default::default()
+ }
+ }
+
+ #[test]
+ fn two_machines_both_holding_files() {
+ let s = Summary {
+ run_on: Some("Mac Studio".into()),
+ holders_up: vec!["Mac Studio".into(), "MacBook".into()],
+ ..base()
+ };
+ assert_eq!(
+ closing_summary(&s),
+ "Your agent keeps running on Mac Studio. Files are on 2 machines. You can close this machine."
+ );
+ }
+
+ #[test]
+ fn only_the_studio_holds_files_until_the_laptop_is_back() {
+ let s = Summary {
+ run_on: Some("Mac Studio".into()),
+ holders_up: vec!["Mac Studio".into()],
+ holders_away: vec!["MacBook".into()],
+ ..base()
+ };
+ let t = closing_summary(&s);
+ assert!(t.starts_with("Your agent keeps running on Mac Studio. Files are on 1 machine until MacBook is back."), "{t}");
+ assert!(!t.contains("2 machines"));
+ }
+
+ #[test]
+ fn one_machine_says_it_will_pause_and_is_honest_about_the_lid() {
+ let s = Summary {
+ machines: 1,
+ run_on: Some("MacBook".into()),
+ sleep_blocked: true,
+ ..base()
+ };
+ let t = closing_summary(&s);
+ assert!(t.contains("will pause when it sleeps"), "{t}");
+ assert!(
+ t.contains("closing the lid still puts a laptop to sleep"),
+ "{t}"
+ );
+ assert!(t.contains("Add another machine"), "{t}");
+ }
+
+ #[test]
+ fn pending_changes_name_who_they_wait_for() {
+ let s = Summary {
+ run_on: Some("Mac Studio".into()),
+ holders_up: vec!["Mac Studio".into(), "MacBook".into()],
+ pending_here: 3,
+ pending_for: Some("Mac Studio".into()),
+ ..base()
+ };
+ let t = closing_summary(&s);
+ assert!(
+ t.ends_with("3 changes saved only on MacBook will sync when Mac Studio is back."),
+ "{t}"
+ );
+ }
+
+ #[test]
+ fn the_agent_here_with_a_studio_away_does_not_promise_takeover() {
+ let s = Summary {
+ run_on: Some("MacBook".into()),
+ this_sleeps: true,
+ holders_away: vec!["Mac Studio".into()],
+ ..base()
+ };
+ let t = closing_summary(&s);
+ assert!(
+ t.starts_with("Your agent runs on MacBook and will pause when it sleeps."),
+ "{t}"
+ );
+ assert!(t.contains("Mac Studio is away right now."), "{t}");
+ }
+}
diff --git a/libs/cua/crates/cua-volume/src/replica/xfer.rs b/libs/cua/crates/cua-volume/src/replica/xfer.rs
new file mode 100644
index 0000000000..679b27e35d
--- /dev/null
+++ b/libs/cua/crates/cua-volume/src/replica/xfer.rs
@@ -0,0 +1,343 @@
+// SPDX-License-Identifier: FSL-1.1-MIT
+// Copyright (c) 2026 Cua AI, Inc.
+
+//! Moving big objects between machines in blocks.
+//!
+//! The receiver pulls. It asks the sender for the SHA-256 of every 1 MiB
+//! block of a version, keeps the blocks it already has (from a partial file
+//! of an earlier try, or from its own previous version of the key at the
+//! same offsets), fetches the rest a few at a time, checks each block's
+//! hash on arrival, and checks the whole object's hash before it is used.
+//! So a link that drops and returns costs the blocks in flight, not the file,
+//! and a file that only changed at its end costs its end.
+
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use tokio::task::JoinSet;
+
+use super::link::Link;
+use super::node::Inner;
+use super::wire::{Msg, Src};
+use crate::backend::ObjectMeta;
+use crate::{Error, Result, sha256_hex};
+
+#[cfg(unix)]
+fn read_at(f: &std::fs::File, buf: &mut [u8], offset: u64) -> std::io::Result<usize> {
+ std::os::unix::fs::FileExt::read_at(f, buf, offset)
+}
+
+#[cfg(windows)]
+fn read_at(f: &std::fs::File, buf: &mut [u8], offset: u64) -> std::io::Result<usize> {
+ std::os::windows::fs::FileExt::seek_read(f, buf, offset)
+}
+
+fn read_block(path: &std::path::Path, offset: u64, len: usize) -> std::io::Result<Vec<u8>> {
+ let f = std::fs::File::open(path)?;
+ let size = f.metadata()?.len();
+ if offset >= size {
+ return Ok(vec![]);
+ }
+ let want = len.min((size - offset) as usize);
+ let mut buf = vec![0u8; want];
+ let mut done = 0;
+ while done < want {
+ let n = read_at(&f, &mut buf[done..], offset + done as u64)?;
+ if n == 0 {
+ break;
+ }
+ done += n;
+ }
+ buf.truncate(done);
+ Ok(buf)
+}
+
+fn io_err(e: std::io::Error) -> Error {
+ if e.raw_os_error() == Some(28) {
+ Error::Backend("the disk is full".into())
+ } else {
+ Error::from(e)
+ }
+}
+
+impl Inner {
+ fn src_path(&self, src: &Src) -> Result<PathBuf> {
+ match src {
+ Src::Data { key, version } => self.store.version_file(key, version),
+ Src::Spool { key, version } => self.spool.version_file(key, version),
+ }
+ }
+
+ async fn hashes_of(&self, path: PathBuf, block: u64) -> Result<(Arc<Vec<String>>, u64)> {
+ let ck = format!("{}:{block}", path.display());
+ if let Some(h) = self.hashes.lock().expect("hashes").get(&ck).cloned() {
+ let size = std::fs::metadata(&path)?.len();
+ return Ok((h, size));
+ }
+ let p = path.clone();
+ let (hashes, size) = tokio::task::spawn_blocking(move || -> Result<(Vec<String>, u64)> {
+ use std::io::Read as _;
+ let mut f = std::fs::File::open(&p)?;
+ let size = f.metadata()?.len();
+ let mut out = vec![];
+ let mut buf = vec![0u8; block as usize];
+ loop {
+ let mut n = 0;
+ while n < buf.len() {
+ let r = f.read(&mut buf[n..])?;
+ if r == 0 {
+ break;
+ }
+ n += r;
+ }
+ if n == 0 {
+ break;
+ }
+ out.push(sha256_hex(&buf[..n]));
+ if n < buf.len() {
+ break;
+ }
+ }
+ Ok((out, size))
+ })
+ .await
+ .map_err(|e| Error::Backend(e.to_string()))??;
+ let arc = Arc::new(hashes);
+ let mut c = self.hashes.lock().expect("hashes");
+ if c.len() > 32
+ && let Some(k) = c.keys().next().cloned()
+ {
+ c.remove(&k);
+ }
+ c.insert(ck, arc.clone());
+ Ok((arc, size))
+ }
+
+ pub(super) async fn handle_hashes(&self, src: Src, block: u64, from: u64, count: u64) -> Msg {
+ let r = async {
+ let path = self.src_path(&src)?;
+ let (all, size) = self.hashes_of(path, block).await?;
+ let from = (from as usize).min(all.len());
+ let to = from.saturating_add(count as usize).min(all.len());
+ Ok::<_, Error>(Msg::HashesAre {
+ hashes: all[from..to].to_vec(),
+ size,
+ })
+ }
+ .await;
+ r.unwrap_or_else(|e| Msg::Error {
+ kind: e.tag().into(),
+ message: e.to_string(),
+ })
+ }
+
+ pub(super) async fn handle_block(&self, src: Src, block: u64, idx: u64) -> Result<Vec<u8>> {
+ let path = self.src_path(&src)?;
+ tokio::task::spawn_blocking(move || read_block(&path, idx * block, block as usize))
+ .await
+ .map_err(|e| Error::Backend(e.to_string()))?
+ .map_err(|e| match e.kind() {
+ std::io::ErrorKind::NotFound => Error::NotFound("that version is not here".into()),
+ _ => Error::from(e),
+ })
+ }
+
+ /// Fetches `meta` from `link`'s machine into a file in the incoming
+ /// directory, verified, and returns its path. `seed` is this machine's
+ /// own previous version of the key, whose unchanged blocks are reused.
+ pub(super) async fn pull(
+ self: &Arc<Self>,
+ link: &Arc<Link>,
+ src: Src,
+ meta: &ObjectMeta,
+ seed: Option<&ObjectMeta>,
+ ) -> Result<PathBuf> {
+ let block = self.cfg.block;
+ let size = meta.size;
+ let nblocks = size.div_ceil(block);
+ let inc = self.dir.join("incoming");
+ std::fs::create_dir_all(&inc)?;
+ let stem = format!("{}-{}", meta.etag, size);
+ let part = inc.join(format!("{stem}.part"));
+ let havef = inc.join(format!("{stem}.have"));
+ cua_home::guard_write(&part)?;
+ // What an earlier attempt left.
+ let mut have: Vec<bool> = match (std::fs::read(&havef), part.is_file()) {
+ (Ok(b), true) if b.len() as u64 == nblocks => b.iter().map(|x| *x == 1).collect(),
+ _ => vec![false; nblocks as usize],
+ };
+ if !part.is_file() {
+ have = vec![false; nblocks as usize];
+ }
+ {
+ let f = std::fs::OpenOptions::new()
+ .create(true)
+ .truncate(false)
+ .write(true)
+ .open(&part)
+ .map_err(io_err)?;
+ f.set_len(size).map_err(io_err)?;
+ }
+ // The hashes, in pages.
+ let mut hashes: Vec<String> = Vec::with_capacity(nblocks as usize);
+ let to = self.cfg.data_timeout;
+ while (hashes.len() as u64) < nblocks {
+ let from = hashes.len() as u64;
+ let (m, _) = link
+ .call_strict(
+ Msg::Hashes {
+ src: src.clone(),
+ block,
+ from,
+ count: 4096,
+ },
+ &[],
+ to,
+ )
+ .await
+ .map_err(|e| Error::Backend(e.to_string()))?;
+ match m {
+ Msg::HashesAre { hashes: h, size: s } => {
+ if s != size || h.is_empty() {
+ return Err(Error::Backend(
+ "the sender's version is not the one announced".into(),
+ ));
+ }
+ hashes.extend(h);
+ }
+ Msg::Error { kind, message } => {
+ return Err(super::wire::Rep::into_error(&kind, message));
+ }
+ _ => return Err(Error::Backend("unexpected reply".into())),
+ }
+ }
+ // Blocks this machine already holds from its own previous version.
+ let seed_path = seed.and_then(|h| self.store.version_file(&h.key, &h.version).ok());
+ let mut need: Vec<u64> = vec![];
+ for idx in 0..nblocks {
+ if have[idx as usize] {
+ continue;
+ }
+ if let Some(sp) = &seed_path
+ && let Ok(b) = read_block(sp, idx * block, block as usize)
+ && !b.is_empty()
+ && sha256_hex(&b) == hashes[idx as usize]
+ {
+ write_at(&part, idx * block, &b).map_err(io_err)?;
+ have[idx as usize] = true;
+ continue;
+ }
+ need.push(idx);
+ }
+ persist_have(&havef, &have);
+ // The rest, a window at a time.
+ let hashes = Arc::new(hashes);
+ let mut set: JoinSet<Result<(u64, Vec<u8>)>> = JoinSet::new();
+ let mut next = 0usize;
+ let mut since_persist = 0;
+ let window = self.cfg.window.max(1);
+ let mut failed: Option<Error> = None;
+ while next < need.len() || !set.is_empty() {
+ while failed.is_none() && next < need.len() && set.len() < window {
+ let idx = need[next];
+ next += 1;
+ let (link, src, hashes) = (link.clone(), src.clone(), hashes.clone());
+ set.spawn(async move {
+ let (m, bytes) = link
+ .call_strict(Msg::Block { src, block, idx }, &[], to)
+ .await
+ .map_err(|e| Error::Backend(e.to_string()))?;
+ match m {
+ Msg::BlockIs => {}
+ Msg::Error { kind, message } => {
+ return Err(super::wire::Rep::into_error(&kind, message));
+ }
+ _ => return Err(Error::Backend("unexpected reply".into())),
+ }
+ if sha256_hex(&bytes) != hashes[idx as usize] {
+ return Err(Error::Backend(format!("block {idx} arrived damaged")));
+ }
+ Ok((idx, bytes))
+ });
+ }
+ let Some(joined) = set.join_next().await else {
+ break;
+ };
+ match joined {
+ Ok(Ok((idx, bytes))) => {
+ write_at(&part, idx * block, &bytes).map_err(io_err)?;
+ have[idx as usize] = true;
+ since_persist += 1;
+ if since_persist >= 16 {
+ since_persist = 0;
+ persist_have(&havef, &have);
+ }
+ }
+ Ok(Err(e)) => failed = Some(e),
+ Err(e) => failed = Some(Error::Backend(e.to_string())),
+ }
+ }
+ persist_have(&havef, &have);
+ if let Some(e) = failed {
+ return Err(e);
+ }
+ // The whole object, once.
+ let p = part.clone();
+ let etag = meta.etag.clone();
+ let ok = tokio::task::spawn_blocking(move || -> std::io::Result<bool> {
+ use sha2::Digest as _;
+ use std::io::Read as _;
+ let mut f = std::fs::File::open(&p)?;
+ let mut h = sha2::Sha256::new();
+ let mut buf = vec![0u8; 1 << 20];
+ loop {
+ let n = f.read(&mut buf)?;
+ if n == 0 {
+ break;
+ }
+ h.update(&buf[..n]);
+ }
+ Ok(hex::encode(h.finalize()) == etag)
+ })
+ .await
+ .map_err(|e| Error::Backend(e.to_string()))?
+ .map_err(io_err)?;
+ if !ok {
+ let _ = std::fs::remove_file(&part);
+ let _ = std::fs::remove_file(&havef);
+ return Err(Error::Backend(
+ "the object did not match its hash; it will be fetched again".into(),
+ ));
+ }
+ let _ = std::fs::remove_file(&havef);
+ Ok(part)
+ }
+}
+
+fn write_at(path: &std::path::Path, offset: u64, bytes: &[u8]) -> std::io::Result<()> {
+ let f = std::fs::OpenOptions::new().write(true).open(path)?;
+ #[cfg(unix)]
+ {
+ std::os::unix::fs::FileExt::write_all_at(&f, bytes, offset)
+ }
+ #[cfg(windows)]
+ {
+ let mut done = 0;
+ while done < bytes.len() {
+ done += std::os::windows::fs::FileExt::seek_write(
+ &f,
+ &bytes[done..],
+ offset + done as u64,
+ )?;
+ }
+ Ok(())
+ }
+}
+
+fn persist_have(path: &std::path::Path, have: &[bool]) {
+ let bytes: Vec<u8> = have.iter().map(|b| u8::from(*b)).collect();
+ let tmp = path.with_extension("tmp");
+ if std::fs::write(&tmp, bytes).is_ok() {
+ let _ = std::fs::rename(&tmp, path);
+ }
+}
diff --git a/libs/cua/crates/cua-volume/tests/common/mod.rs b/libs/cua/crates/cua-volume/tests/common/mod.rs
new file mode 100644
index 0000000000..66afed040b
--- /dev/null
+++ b/libs/cua/crates/cua-volume/tests/common/mod.rs
@@ -0,0 +1,255 @@
+// SPDX-License-Identifier: FSL-1.1-MIT
+// Copyright (c) 2026 Cua AI, Inc.
+
+//! Shared by the replication simulations: a group of real nodes on a
+//! simulated network.
+
+#![allow(dead_code)]
+
+use std::collections::HashMap;
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+use cua_volume::backend::{Backend, Condition};
+use cua_volume::fs::FsBackend;
+use cua_volume::replica::simnet::{Impair, SimNet};
+use cua_volume::replica::{Config, Facts, Member, ReplicaNode, ReplicatedBackend};
+
+pub struct M {
+ #[allow(dead_code)]
+ pub id: String,
+ pub node: ReplicaNode,
+ pub backend: Arc<ReplicatedBackend>,
+ pub store: FsBackend,
+ pub dir: tempfile::TempDir,
+}
+
+pub struct Sim {
+ pub net: SimNet,
+ pub m: HashMap<String, M>,
+ pub members: Vec<Member>,
+ pub cfg: Config,
+}
+
+pub fn member(id: &str, rank: u64) -> Member {
+ Member {
+ id: id.into(),
+ name: id.to_string(),
+ facts: Facts {
+ rank,
+ free_disk: 500 << 30,
+ ..Default::default()
+ },
+ last_seen_ms: 0,
+ }
+}
+
+impl Sim {
+ pub async fn new(specs: &[(&str, u64)]) -> Sim {
+ Sim::with(specs, Config::fast()).await
+ }
+
+ pub async fn with(specs: &[(&str, u64)], cfg: Config) -> Sim {
+ // CUA_TEST_LOG=cua_volume=debug shows what the nodes decide.
+ if let Ok(f) = std::env::var("CUA_TEST_LOG") {
+ let _ = tracing_subscriber::fmt()
+ .with_env_filter(f)
+ .with_test_writer()
+ .try_init();
+ }
+ let net = SimNet::new();
+ let members: Vec<Member> = specs.iter().map(|(i, r)| member(i, *r)).collect();
+ let mut sim = Sim {
+ net,
+ m: HashMap::new(),
+ members,
+ cfg,
+ };
+ for (id, _) in specs {
+ sim.boot(id, tempfile::tempdir().unwrap()).await;
+ }
+ sim
+ }
+
+ pub async fn boot(&mut self, id: &str, dir: tempfile::TempDir) {
+ let (tr, mut rx) = self.net.join(id);
+ let store = FsBackend::new(dir.path().join("data"));
+ let node = ReplicaNode::open(
+ self.cfg.clone(),
+ id,
+ id,
+ dir.path(),
+ store.clone(),
+ Arc::new(tr),
+ self.members.clone(),
+ )
+ .await
+ .unwrap();
+ let facts = self
+ .members
+ .iter()
+ .find(|m| m.id == id)
+ .unwrap()
+ .facts
+ .clone();
+ node.set_facts(facts);
+ let n2 = node.clone();
+ tokio::spawn(async move {
+ while let Some((_from, s)) = rx.recv().await {
+ n2.accept(s).await;
+ }
+ });
+ node.start();
+ let backend = node.backend();
+ self.m.insert(
+ id.to_string(),
+ M {
+ id: id.to_string(),
+ node,
+ backend,
+ store,
+ dir,
+ },
+ );
+ }
+
+ pub fn get(&self, id: &str) -> &M {
+ &self.m[id]
+ }
+
+ /// A process restart: shut down, keep the directory, boot again.
+ pub async fn restart(&mut self, id: &str) {
+ let old = self.m.remove(id).unwrap();
+ old.node.shutdown().await;
+ let dir = old.dir;
+ drop(old.backend);
+ self.boot(id, dir).await;
+ }
+
+ pub fn cut(&self, a: &str, b: &str) {
+ self.net.set_both(
+ a,
+ b,
+ Impair {
+ sever: true,
+ ..Default::default()
+ },
+ );
+ }
+
+ pub fn heal(&self, a: &str, b: &str) {
+ self.net.heal(a, b);
+ }
+
+ pub async fn shutdown(&self) {
+ for m in self.m.values() {
+ m.node.shutdown().await;
+ }
+ }
+}
+
+pub async fn eventually<F, Fut>(what: &str, secs: u64, mut f: F)
+where
+ F: FnMut() -> Fut,
+ Fut: std::future::Future<Output = bool>,
+{
+ let end = Instant::now() + Duration::from_secs(secs);
+ loop {
+ if f().await {
+ return;
+ }
+ if Instant::now() > end {
+ panic!("timed out waiting for: {what}");
+ }
+ tokio::time::sleep(Duration::from_millis(20)).await;
+ }
+}
+
+/// A machine that needs an ordered write (a lease, a fence) keeps asking
+/// until it can have one; that is what makes it take over from a dead owner.
+pub async fn needs_authority(m: &M, key: &str) {
+ eventually("an ordered write succeeds", 30, || async {
+ m.backend
+ .put(key, b"x".to_vec(), Condition::IfNoneMatch)
+ .await
+ .is_ok()
+ })
+ .await;
+}
+
+pub async fn head_etag(m: &M, key: &str) -> Option<String> {
+ m.store.head(key).await.unwrap().map(|h| h.etag)
+}
+
+pub async fn heads(m: &M) -> Vec<(String, String)> {
+ m.store
+ .list("")
+ .await
+ .unwrap()
+ .into_iter()
+ .map(|h| (h.key, h.etag))
+ .collect()
+}
+
+pub async fn converged(sim: &Sim, ids: &[&str]) -> bool {
+ let first = heads(sim.get(ids[0])).await;
+ for id in &ids[1..] {
+ if heads(sim.get(id)).await != first {
+ return false;
+ }
+ }
+ true
+}
+
+pub async fn group_ready(sim: &Sim, owner: &str) {
+ let dump = || async {
+ for m in sim.m.values() {
+ let s = m.node.status().await;
+ eprintln!(
+ "{}: owner={} epoch={} peers={:?}",
+ m.id,
+ s.owner,
+ s.epoch,
+ s.peers
+ .iter()
+ .map(|p| (p.id.clone(), p.health.clone(), p.connected))
+ .collect::<Vec<_>>()
+ );
+ }
+ };
+ let end = Instant::now() + Duration::from_secs(14);
+ while Instant::now() < end {
+ let mut ok = true;
+ for m in sim.m.values() {
+ if m.node.status().await.owner != owner {
+ ok = false;
+ }
+ }
+ if ok {
+ break;
+ }
+ tokio::time::sleep(Duration::from_millis(20)).await;
+ }
+ dump().await;
+ eventually("the group agrees on an owner", 5, || async {
+ for m in sim.m.values() {
+ let s = m.node.status().await;
+ if s.owner != owner {
+ return false;
+ }
+ }
+ true
+ })
+ .await;
+ // Everyone is connected to everyone.
+ eventually("the machines are connected", 15, || async {
+ for m in sim.m.values() {
+ let s = m.node.status().await;
+ if s.peers.iter().any(|p| !p.connected) {
+ return false;
+ }
+ }
+ true
+ })
+ .await;
+}
diff --git a/libs/cua/crates/cua-volume/tests/replica_sim.rs b/libs/cua/crates/cua-volume/tests/replica_sim.rs
new file mode 100644
index 0000000000..76e156bd67
--- /dev/null
+++ b/libs/cua/crates/cua-volume/tests/replica_sim.rs
@@ -0,0 +1,1122 @@
+// SPDX-License-Identifier: FSL-1.1-MIT
+// Copyright (c) 2026 Cua AI, Inc.
+
+//! Simulations of the replicated volume across machines: partitions, a
+//! closed laptop, a takeover with the old owner waking up, jitter, big files
+//! on a flapping link, a quorum of three. Time is scaled down (heartbeat
+//! 50 ms) and the network is `SimNet`; the nodes are the real ones.
+
+mod common;
+
+use std::time::{Duration, Instant};
+
+use common::*;
+use cua_volume::backend::{Backend, Condition};
+use cua_volume::replica::simnet::Impair;
+use cua_volume::replica::{Config, Durability, PowerEvent};
+
+#[tokio::test]
+async fn two_machines_agree_on_an_owner_and_hold_two_copies() {
+ let sim = Sim::new(&[("studio", 9000), ("laptop", 2000)]).await;
+ // Both started alone with a volume of their own; the less reliable one
+ // joins the other's and the more reliable owns it.
+ group_ready(&sim, "studio").await;
+ let studio = sim.get("studio");
+ let (meta, d) = studio
+ .backend
+ .put_durable("agents/ada/notes.md", b"hello".to_vec(), Condition::None)
+ .await
+ .unwrap();
+ assert_eq!(
+ d,
+ Durability::Replicated(2),
+ "the write waits for the second copy"
+ );
+ let laptop = sim.get("laptop");
+ // The second copy holds the same version and bytes.
+ let h = laptop
+ .store
+ .head("agents/ada/notes.md")
+ .await
+ .unwrap()
+ .unwrap();
+ assert_eq!(h.version, meta.version);
+ assert_eq!(
+ laptop
+ .store
+ .get("agents/ada/notes.md", None)
+ .await
+ .unwrap()
+ .0,
+ b"hello"
+ );
+ let s = laptop.node.status().await;
+ assert!(!s.i_own && s.owner == "studio" && s.copies_now == 2 && s.copies_wanted == 2);
+ sim.shutdown().await;
+}
+
+#[tokio::test]
+async fn a_write_on_the_laptop_goes_through_the_owner_and_both_see_it() {
+ let sim = Sim::new(&[("studio", 9000), ("laptop", 2000)]).await;
+ group_ready(&sim, "studio").await;
+ let laptop = sim.get("laptop");
+ let (meta, d) = laptop
+ .backend
+ .put_durable("public/a.txt", b"from the laptop".to_vec(), Condition::None)
+ .await
+ .unwrap();
+ assert_eq!(d, Durability::Replicated(2));
+ // Read-your-writes on the laptop, and the owner has it.
+ assert_eq!(
+ laptop
+ .backend
+ .get("public/a.txt", None)
+ .await
+ .unwrap()
+ .1
+ .version,
+ meta.version
+ );
+ assert_eq!(
+ sim.get("studio")
+ .store
+ .head("public/a.txt")
+ .await
+ .unwrap()
+ .unwrap()
+ .version,
+ meta.version
+ );
+ // A conditional write from the laptop is atomic across machines: a
+ // second create-only loses.
+ laptop
+ .backend
+ .put(
+ "agents/ada/.cua-lease",
+ b"laptop".to_vec(),
+ Condition::IfNoneMatch,
+ )
+ .await
+ .unwrap();
+ let e = sim
+ .get("studio")
+ .backend
+ .put(
+ "agents/ada/.cua-lease",
+ b"studio".to_vec(),
+ Condition::IfNoneMatch,
+ )
+ .await
+ .unwrap_err();
+ assert_eq!(e.tag(), "precondition_failed");
+ sim.shutdown().await;
+}
+
+#[tokio::test]
+async fn a_closed_laptop_never_blocks_the_studio_and_catches_up_when_it_returns() {
+ let sim = Sim::new(&[("studio", 9000), ("laptop", 2000)]).await;
+ group_ready(&sim, "studio").await;
+ let studio = sim.get("studio");
+ studio
+ .backend
+ .put("agents/ada/before.txt", b"1".to_vec(), Condition::None)
+ .await
+ .unwrap();
+ // The lid closes: the laptop says goodbye and goes dark.
+ sim.get("laptop")
+ .node
+ .notify_power(PowerEvent::WillSleep)
+ .await;
+ // The OS gives the machine a moment between "will sleep" and sleep.
+ tokio::time::sleep(Duration::from_millis(50)).await;
+ sim.get("laptop").node.freeze(true);
+ sim.cut("studio", "laptop");
+ let t0 = Instant::now();
+ let mut last = Durability::Pending;
+ for i in 0..20 {
+ let (_, d) = studio
+ .backend
+ .put_durable(
+ &format!("agents/ada/out-{i}.txt"),
+ format!("line {i}").into_bytes(),
+ Condition::None,
+ )
+ .await
+ .unwrap();
+ last = d;
+ }
+ // 20 writes while the second machine is away took 20 short waits at
+ // most, and each says there is one copy.
+ assert_eq!(last, Durability::Local(1));
+ assert!(
+ t0.elapsed() < Duration::from_secs(20 * 400 / 1000 + 4),
+ "writes were held up: {:?}",
+ t0.elapsed()
+ );
+ let s = studio.node.status().await;
+ assert!(
+ s.unreplicated > 0,
+ "the status says files are on one machine"
+ );
+ let words = studio.node.summary(Some("studio")).await;
+ assert_eq!(words.holders_up, vec!["studio".to_string()]);
+ assert_eq!(words.holders_away, vec!["laptop".to_string()]);
+ // The lid opens.
+ sim.heal("studio", "laptop");
+ sim.get("laptop").node.freeze(false);
+ sim.get("laptop")
+ .node
+ .notify_power(PowerEvent::DidWake)
+ .await;
+ eventually("the laptop catches up", 15, || async {
+ converged(&sim, &["studio", "laptop"]).await
+ && sim.get("studio").node.status().await.unreplicated == 0
+ })
+ .await;
+ assert_eq!(heads(sim.get("laptop")).await.len(), 21);
+ sim.shutdown().await;
+}
+
+#[tokio::test]
+async fn changes_made_on_a_laptop_with_no_owner_are_saved_and_sync_later() {
+ let sim = Sim::new(&[("studio", 9000), ("laptop", 2000)]).await;
+ group_ready(&sim, "studio").await;
+ sim.cut("studio", "laptop");
+ let laptop = sim.get("laptop");
+ eventually("the laptop sees the studio go", 10, || async {
+ laptop.node.status().await.peers[0].health != "up"
+ })
+ .await;
+ let (_, d) = laptop
+ .backend
+ .put_durable(
+ "agents/ada/draft.md",
+ b"written on a plane".to_vec(),
+ Condition::None,
+ )
+ .await
+ .unwrap();
+ assert_eq!(d, Durability::Pending);
+ // The laptop shows its own write, and says what is waiting.
+ assert_eq!(
+ laptop
+ .backend
+ .get("agents/ada/draft.md", None)
+ .await
+ .unwrap()
+ .0,
+ b"written on a plane"
+ );
+ let s = laptop.node.status().await;
+ assert_eq!(s.pending.len(), 1);
+ let words = laptop.node.summary(Some("studio")).await;
+ assert_eq!(words.pending_here, 1);
+ // A conditional write cannot be ordered without the owner: it fails
+ // plainly instead of pretending.
+ let e = laptop
+ .backend
+ .put(
+ "agents/ada/.cua-lease",
+ b"x".to_vec(),
+ Condition::IfNoneMatch,
+ )
+ .await
+ .unwrap_err();
+ assert!(e.to_string().contains("cannot be reached"), "{e}");
+ sim.heal("studio", "laptop");
+ eventually("the studio takes the change", 15, || async {
+ head_etag(sim.get("studio"), "agents/ada/draft.md")
+ .await
+ .is_some()
+ && laptop.node.status().await.pending.is_empty()
+ })
+ .await;
+ assert_eq!(
+ sim.get("studio")
+ .store
+ .get("agents/ada/draft.md", None)
+ .await
+ .unwrap()
+ .0,
+ b"written on a plane"
+ );
+ eventually("both agree", 10, || async {
+ converged(&sim, &["studio", "laptop"]).await
+ })
+ .await;
+ sim.shutdown().await;
+}
+
+#[tokio::test]
+async fn two_machines_changing_one_file_keep_both_versions() {
+ let sim = Sim::new(&[("studio", 9000), ("laptop", 2000)]).await;
+ group_ready(&sim, "studio").await;
+ sim.get("studio")
+ .backend
+ .put("agents/ada/plan.md", b"v0".to_vec(), Condition::None)
+ .await
+ .unwrap();
+ eventually("replicated", 10, || async {
+ head_etag(sim.get("laptop"), "agents/ada/plan.md")
+ .await
+ .is_some()
+ })
+ .await;
+ sim.cut("studio", "laptop");
+ let laptop = sim.get("laptop");
+ eventually("laptop notices", 10, || async {
+ laptop.node.status().await.peers[0].health != "up"
+ })
+ .await;
+ laptop
+ .backend
+ .put(
+ "agents/ada/plan.md",
+ b"laptop edit".to_vec(),
+ Condition::None,
+ )
+ .await
+ .unwrap();
+ sim.get("studio")
+ .backend
+ .put(
+ "agents/ada/plan.md",
+ b"studio edit".to_vec(),
+ Condition::None,
+ )
+ .await
+ .unwrap();
+ sim.heal("studio", "laptop");
+ eventually("both converge", 20, || async {
+ converged(&sim, &["studio", "laptop"]).await
+ && laptop.node.status().await.pending.is_empty()
+ })
+ .await;
+ // The owner's version stays at the key; the laptop's edit is a copy.
+ let st = sim.get("studio");
+ assert_eq!(
+ st.store.get("agents/ada/plan.md", None).await.unwrap().0,
+ b"studio edit"
+ );
+ let all = heads(st).await;
+ let copies: Vec<_> = all
+ .iter()
+ .filter(|(k, _)| k.contains("(conflict from laptop"))
+ .collect();
+ assert_eq!(copies.len(), 1, "{all:?}");
+ assert_eq!(
+ st.store.get(&copies[0].0, None).await.unwrap().0,
+ b"laptop edit"
+ );
+ sim.shutdown().await;
+}
+
+/// The laptop owns the volume, is cut off, and writes; the studio takes
+/// over and writes; the laptop wakes. Nothing is lost, there is one owner,
+/// the laptop's stale records are refused, and both end up equal.
+#[tokio::test]
+async fn the_old_owner_waking_after_a_takeover_is_fenced_and_loses_nothing() {
+ // The laptop is the more reliable one here, so it owns at the start and
+ // gets the volume back at the end.
+ let sim = Sim::new(&[("laptop", 9000), ("studio", 2000)]).await;
+ group_ready(&sim, "laptop").await;
+ let laptop = sim.get("laptop");
+ let studio = sim.get("studio");
+ laptop
+ .backend
+ .put("agents/ada/base.txt", b"base".to_vec(), Condition::None)
+ .await
+ .unwrap();
+ laptop
+ .backend
+ .put("agents/ada/shared.txt", b"s0".to_vec(), Condition::None)
+ .await
+ .unwrap();
+ eventually("studio holds the base", 10, || async {
+ head_etag(studio, "agents/ada/shared.txt").await.is_some()
+ })
+ .await;
+ // A partition: the laptop keeps writing as owner; nobody hears it.
+ sim.cut("laptop", "studio");
+ laptop
+ .backend
+ .put("agents/ada/lonely-1.txt", b"L1".to_vec(), Condition::None)
+ .await
+ .unwrap();
+ laptop
+ .backend
+ .put(
+ "agents/ada/shared.txt",
+ b"laptop shared".to_vec(),
+ Condition::None,
+ )
+ .await
+ .unwrap();
+ // The studio needs to order a write (an agent's fence): it takes over.
+ needs_authority(studio, "spaces/dev/sessions/s1/fence-x").await;
+ assert!(studio.node.status().await.i_own);
+ studio
+ .backend
+ .put("agents/ada/studio-1.txt", b"S1".to_vec(), Condition::None)
+ .await
+ .unwrap();
+ studio
+ .backend
+ .put(
+ "agents/ada/shared.txt",
+ b"studio shared".to_vec(),
+ Condition::None,
+ )
+ .await
+ .unwrap();
+ assert!(!laptop.node.status().await.peers.is_empty());
+ // The laptop still thinks it owns the volume and keeps writing.
+ assert!(laptop.node.status().await.i_own);
+ laptop
+ .backend
+ .put("agents/ada/lonely-2.txt", b"L2".to_vec(), Condition::None)
+ .await
+ .unwrap();
+ sim.heal("laptop", "studio");
+ eventually("everything converges", 30, || async {
+ converged(&sim, &["laptop", "studio"]).await
+ && laptop.node.status().await.pending.is_empty()
+ && head_etag(studio, "agents/ada/lonely-2.txt").await.is_some()
+ })
+ .await;
+ // The studio's era stood; nothing from the laptop's divergent tail was
+ // applied as is.
+ let st = studio.node.status().await;
+ assert!(st.epoch >= 2, "{st:?}");
+ // Nothing is lost: every written value is a head or a conflict copy.
+ let mut values: Vec<Vec<u8>> = vec![];
+ for (k, _) in heads(studio).await {
+ values.push(studio.store.get(&k, None).await.unwrap().0);
+ }
+ for want in [
+ b"base".to_vec(),
+ b"L1".to_vec(),
+ b"L2".to_vec(),
+ b"S1".to_vec(),
+ b"studio shared".to_vec(),
+ b"laptop shared".to_vec(),
+ ] {
+ assert!(
+ values.contains(&want),
+ "lost {:?}: {:?}",
+ String::from_utf8_lossy(&want),
+ values
+ .iter()
+ .map(|v| String::from_utf8_lossy(v).to_string())
+ .collect::<Vec<_>>()
+ );
+ }
+ // The key both sides changed has the owner's value and a copy of the other.
+ let owner_now = if studio.node.status().await.i_own {
+ studio
+ } else {
+ laptop
+ };
+ let v = owner_now
+ .store
+ .get("agents/ada/shared.txt", None)
+ .await
+ .unwrap()
+ .0;
+ assert!(v == b"studio shared" || v == b"laptop shared");
+ // The more reliable laptop gets the volume back, without anyone losing it.
+ eventually("the volume goes back to the laptop", 20, || async {
+ laptop.node.status().await.i_own && studio.node.status().await.owner == "laptop"
+ })
+ .await;
+ let s = studio.node.status().await;
+ assert!(s.handbacks >= 1 || laptop.node.status().await.handbacks >= 1);
+ sim.shutdown().await;
+}
+
+#[tokio::test]
+async fn a_goodbye_hands_the_volume_over_at_once() {
+ let sim = Sim::new(&[("laptop", 9000), ("studio", 2000)]).await;
+ group_ready(&sim, "laptop").await;
+ sim.get("laptop")
+ .backend
+ .put("agents/ada/a.txt", b"a".to_vec(), Condition::None)
+ .await
+ .unwrap();
+ eventually("replicated", 10, || async {
+ head_etag(sim.get("studio"), "agents/ada/a.txt")
+ .await
+ .is_some()
+ })
+ .await;
+ let t0 = Instant::now();
+ sim.get("laptop")
+ .node
+ .notify_power(PowerEvent::WillSleep)
+ .await;
+ eventually("the studio owns the volume", 10, || async {
+ sim.get("studio").node.status().await.i_own
+ })
+ .await;
+ let took = t0.elapsed();
+ // Silence alone needs the floor (250 ms) plus confirmation (400 ms).
+ assert!(took < Duration::from_millis(600), "took {took:?}");
+ sim.shutdown().await;
+}
+
+#[tokio::test]
+async fn jitter_and_a_skewed_clock_cause_no_takeover() {
+ let mut cfg = Config::fast();
+ cfg.wall_skew_ms = 3_600_000; // the studio's clock is an hour ahead
+ let sim = Sim::with(&[("laptop", 9000), ("studio", 2000)], cfg).await;
+ group_ready(&sim, "laptop").await;
+ sim.net.set_both(
+ "laptop",
+ "studio",
+ Impair {
+ latency: Duration::from_millis(30),
+ jitter: Duration::from_millis(140),
+ ..Default::default()
+ },
+ );
+ let laptop = sim.get("laptop");
+ for i in 0..40 {
+ laptop
+ .backend
+ .put(
+ &format!("agents/ada/j-{i}.txt"),
+ vec![b'x'; 2000],
+ Condition::None,
+ )
+ .await
+ .unwrap();
+ tokio::time::sleep(Duration::from_millis(50)).await;
+ }
+ eventually("replicated under jitter", 20, || async {
+ converged(&sim, &["laptop", "studio"]).await
+ })
+ .await;
+ for id in ["laptop", "studio"] {
+ let s = sim.get(id).node.status().await;
+ assert_eq!(s.takeovers, 0, "{id} took over on jitter");
+ assert_eq!(s.epoch, 1, "{id}");
+ }
+ sim.shutdown().await;
+}
+
+fn sample(n: usize, seed: u8) -> Vec<u8> {
+ (0..n)
+ .map(|i| ((i as u64 * 2654435761 + seed as u64) >> 7) as u8)
+ .collect()
+}
+
+#[tokio::test]
+async fn a_big_file_survives_a_flapping_link_without_being_sent_again() {
+ let mut cfg = Config::fast();
+ cfg.block = 256 * 1024;
+ cfg.data_timeout = Duration::from_secs(3);
+ let sim = Sim::with(&[("studio", 9000), ("laptop", 2000)], cfg).await;
+ group_ready(&sim, "studio").await;
+ let size = 6 * 1024 * 1024;
+ let body = sample(size, 3);
+ // 100 kB per second each way: the file takes a minute without help, so
+ // flap it.
+ sim.net.set_both(
+ "studio",
+ "laptop",
+ Impair {
+ bandwidth: 2 * 1024 * 1024,
+ latency: Duration::from_millis(5),
+ ..Default::default()
+ },
+ );
+ let before = sim.net.bytes("studio", "laptop");
+ let studio = sim.get("studio");
+ let w = {
+ let b = studio.backend.clone();
+ let body = body.clone();
+ tokio::spawn(async move { b.put("agents/ada/big.bin", body, Condition::None).await })
+ };
+ // Cut the link a few times while it moves.
+ for _ in 0..4 {
+ tokio::time::sleep(Duration::from_millis(400)).await;
+ sim.cut("studio", "laptop");
+ tokio::time::sleep(Duration::from_millis(300)).await;
+ sim.heal("studio", "laptop");
+ }
+ w.await.unwrap().unwrap();
+ let t = Instant::now();
+ eventually("the big file arrives intact", 25, || async {
+ if t.elapsed().as_secs() % 5 == 4 {
+ eprintln!(
+ "laptop {:?}\nstudio {:?}",
+ sim.get("laptop").node.status().await,
+ sim.get("studio").node.status().await
+ );
+ }
+ sim.get("laptop")
+ .store
+ .head("agents/ada/big.bin")
+ .await
+ .unwrap()
+ .is_some()
+ })
+ .await;
+ let got = sim
+ .get("laptop")
+ .store
+ .get("agents/ada/big.bin", None)
+ .await
+ .unwrap()
+ .0;
+ assert_eq!(got.len(), size);
+ assert_eq!(
+ cua_volume::sha256_hex(&got),
+ cua_volume::sha256_hex(&body),
+ "bytes differ"
+ );
+ let moved = sim.net.bytes("studio", "laptop") - before;
+ assert!(
+ moved < (size as u64) * 2,
+ "{moved} bytes crossed for a {size} byte file: it was sent again"
+ );
+ // A new version that only grows at the end costs its end, not the file.
+ let mut longer = body.clone();
+ longer.extend(sample(100 * 1024, 9));
+ let b2 = sim.net.bytes("studio", "laptop");
+ studio
+ .backend
+ .put("agents/ada/big.bin", longer.clone(), Condition::None)
+ .await
+ .unwrap();
+ eventually("the longer version arrives", 30, || async {
+ sim.get("laptop")
+ .store
+ .head("agents/ada/big.bin")
+ .await
+ .unwrap()
+ .is_some_and(|h| h.size == longer.len() as u64)
+ })
+ .await;
+ let moved2 = sim.net.bytes("studio", "laptop") - b2;
+ assert!(
+ moved2 < 1024 * 1024,
+ "{moved2} bytes crossed for an append of 100 KiB"
+ );
+ sim.shutdown().await;
+}
+
+#[tokio::test]
+async fn with_three_machines_a_lone_minority_does_not_take_over() {
+ let sim = Sim::new(&[("studio", 9000), ("mini", 7000), ("laptop", 2000)]).await;
+ group_ready(&sim, "studio").await;
+ let mini = sim.get("mini");
+ // The laptop keeps no copy (two copies: studio and mini).
+ let s = sim.get("laptop").node.status().await;
+ assert_eq!(s.role, cua_volume::replica::Role::Client);
+ // The mini is cut off from both: it is a minority of one.
+ sim.cut("mini", "studio");
+ sim.cut("mini", "laptop");
+ tokio::time::sleep(Duration::from_millis(2500)).await;
+ let s = mini.node.status().await;
+ assert_eq!(s.takeovers, 0, "a minority must not take over");
+ assert_ne!(s.owner, "mini");
+ // The studio goes away; the mini, back with the laptop, now has a
+ // majority (itself and the laptop) and takes over.
+ sim.heal("mini", "laptop");
+ sim.heal("mini", "studio");
+ sim.cut("studio", "mini");
+ sim.cut("studio", "laptop");
+ needs_authority(mini, "agents/ada/.cua-lease").await;
+ assert!(mini.node.status().await.i_own);
+ sim.shutdown().await;
+}
+
+#[tokio::test]
+async fn a_machine_that_keeps_no_copy_reads_through_its_peers() {
+ let sim = Sim::new(&[("studio", 9000), ("mini", 7000), ("laptop", 2000)]).await;
+ group_ready(&sim, "studio").await;
+ let laptop = sim.get("laptop");
+ assert!(
+ laptop.backend.remote(),
+ "a client sits behind the block cache"
+ );
+ sim.get("studio")
+ .backend
+ .put("agents/ada/r.txt", b"read me".to_vec(), Condition::None)
+ .await
+ .unwrap();
+ let (bytes, meta) = laptop.backend.get("agents/ada/r.txt", None).await.unwrap();
+ assert_eq!(bytes, b"read me");
+ assert_eq!(
+ laptop
+ .backend
+ .get_range("agents/ada/r.txt", &meta.version, 5, 2)
+ .await
+ .unwrap(),
+ b"me"
+ );
+ assert_eq!(laptop.backend.list("agents/").await.unwrap().len(), 1);
+ // The owner goes away: the other copy serves.
+ sim.cut("studio", "laptop");
+ sim.cut("studio", "mini");
+ eventually("the laptop reads from the mini", 15, || async {
+ laptop.backend.get("agents/ada/r.txt", None).await.is_ok()
+ })
+ .await;
+ // Someone needs an ordered write: the mini takes over (with the laptop's
+ // vote), and a write from the client goes to it.
+ needs_authority(sim.get("mini"), "agents/ada/.cua-lease").await;
+ let (_, d) = laptop
+ .backend
+ .put_durable("agents/ada/w.txt", b"w".to_vec(), Condition::None)
+ .await
+ .unwrap();
+ assert!(matches!(
+ d,
+ Durability::Local(_) | Durability::Replicated(_)
+ ));
+ sim.shutdown().await;
+}
+
+#[tokio::test]
+async fn a_restart_keeps_the_log_and_the_outbox() {
+ let mut sim = Sim::new(&[("studio", 9000), ("laptop", 2000)]).await;
+ group_ready(&sim, "studio").await;
+ sim.cut("studio", "laptop");
+ let laptop_id = "laptop";
+ eventually("laptop sees the studio go", 10, || async {
+ sim.get(laptop_id).node.status().await.peers[0].health != "up"
+ })
+ .await;
+ sim.get("laptop")
+ .backend
+ .put(
+ "agents/ada/kept.txt",
+ b"survives a restart".to_vec(),
+ Condition::None,
+ )
+ .await
+ .unwrap();
+ sim.restart("laptop").await;
+ assert_eq!(sim.get("laptop").node.status().await.pending.len(), 1);
+ sim.heal("studio", "laptop");
+ eventually("the studio takes it after the restart", 20, || async {
+ head_etag(sim.get("studio"), "agents/ada/kept.txt")
+ .await
+ .is_some()
+ && sim.get("laptop").node.status().await.pending.is_empty()
+ })
+ .await;
+ sim.shutdown().await;
+}
+
+/// A laptop on bad wifi comes and goes. It must never take the volume away
+/// from the studio that runs the agent (its changes wait in the outbox), and
+/// the studio's ordered writes (leases, fences) never wait for it.
+#[tokio::test]
+async fn a_flaky_laptop_never_takes_the_volume_from_the_studio() {
+ let sim = Sim::new(&[("studio", 9000), ("laptop", 2000)]).await;
+ group_ready(&sim, "studio").await;
+ let studio = sim.get("studio");
+ let laptop = sim.get("laptop");
+ for round in 0..6 {
+ sim.cut("studio", "laptop");
+ tokio::time::sleep(Duration::from_millis(900)).await; // longer than detection
+ laptop
+ .backend
+ .put(
+ &format!("agents/ada/plane-{round}.txt"),
+ vec![b'p'; 100],
+ Condition::None,
+ )
+ .await
+ .unwrap();
+ // The studio's ordered writes are immediate.
+ let t = Instant::now();
+ studio
+ .backend
+ .put(
+ &format!("agents/ada/.fence-{round}"),
+ b"1".to_vec(),
+ Condition::IfNoneMatch,
+ )
+ .await
+ .unwrap();
+ assert!(
+ t.elapsed() < Duration::from_millis(1200),
+ "the studio waited for the laptop"
+ );
+ sim.heal("studio", "laptop");
+ tokio::time::sleep(Duration::from_millis(300)).await;
+ }
+ eventually("everything lands", 30, || async {
+ converged(&sim, &["studio", "laptop"]).await
+ && laptop.node.status().await.pending.is_empty()
+ })
+ .await;
+ let s = laptop.node.status().await;
+ assert_eq!(s.takeovers, 0, "the laptop took the volume");
+ assert_eq!(studio.node.status().await.epoch, 1);
+ assert!(studio.node.status().await.i_own);
+ assert_eq!(heads(studio).await.len(), 12);
+ sim.shutdown().await;
+}
+
+/// The usual way a second machine arrives: the laptop already has a volume
+/// with files in it, and the studio is new. The studio, the more reliable
+/// one, ends up owning it; the laptop's files are not lost and are not
+/// duplicated; writes from either side stay quick afterwards.
+#[tokio::test]
+async fn a_new_studio_joins_a_laptop_that_already_has_files() {
+ let mut sim = Sim::new(&[("laptop", 2000)]).await;
+ let laptop_files = [
+ ("agents/ada/old-1.txt", "one"),
+ ("agents/ada/old-2.txt", "two"),
+ ("public/readme.md", "hello"),
+ ];
+ for (k, v) in laptop_files {
+ sim.get("laptop")
+ .backend
+ .put(k, v.as_bytes().to_vec(), Condition::None)
+ .await
+ .unwrap();
+ }
+ // The studio signs in: the laptop's machines list gains it.
+ sim.members.push(member("studio", 9000));
+ let studio_member = member("studio", 9000);
+ sim.get("laptop")
+ .node
+ .set_members(vec![studio_member])
+ .await;
+ let dir = tempfile::tempdir().unwrap();
+ sim.boot("studio", dir).await;
+ group_ready(&sim, "studio").await;
+ eventually("the laptop's files are on the studio, once", 20, || async {
+ heads(sim.get("studio")).await.len() == 3
+ && converged(&sim, &["studio", "laptop"]).await
+ && sim.get("laptop").node.status().await.pending.is_empty()
+ })
+ .await;
+ for (k, v) in laptop_files {
+ assert_eq!(
+ sim.get("studio").store.get(k, None).await.unwrap().0,
+ v.as_bytes()
+ );
+ }
+ // A write from the laptop is not held up by the arrival.
+ let t = Instant::now();
+ let (_, d) = sim
+ .get("laptop")
+ .backend
+ .put_durable("agents/ada/new.txt", b"n".to_vec(), Condition::None)
+ .await
+ .unwrap();
+ assert_eq!(d, Durability::Replicated(2));
+ assert!(
+ t.elapsed() < Duration::from_millis(600),
+ "slow after joining: {:?}",
+ t.elapsed()
+ );
+ // And in the other direction.
+ let t = Instant::now();
+ sim.get("studio")
+ .backend
+ .put("agents/ada/new2.txt", b"n".to_vec(), Condition::None)
+ .await
+ .unwrap();
+ assert!(t.elapsed() < Duration::from_millis(600));
+ sim.shutdown().await;
+}
+
+/// A laptop that vanishes without a goodbye (a dead battery, a dropped
+/// link) is waited for at most twice: after two writes whose second copy
+/// never came, the studio stops waiting until the laptop answers again.
+#[tokio::test]
+async fn a_laptop_that_vanishes_without_a_goodbye_costs_the_studio_two_waits() {
+ let sim = Sim::new(&[("studio", 9000), ("laptop", 2000)]).await;
+ group_ready(&sim, "studio").await;
+ let studio = sim.get("studio");
+ studio
+ .backend
+ .put("agents/ada/warm.txt", b"w".to_vec(), Condition::None)
+ .await
+ .unwrap();
+ sim.get("laptop").node.freeze(true);
+ // A stopped process behind an open connection: nothing arrives, nothing
+ // is reset (the link stays open until the detector gives up on it).
+ sim.net.set_both(
+ "studio",
+ "laptop",
+ Impair {
+ hold: true,
+ ..Default::default()
+ },
+ );
+ let t = Instant::now();
+ let mut durs = vec![];
+ for i in 0..12 {
+ let t1 = Instant::now();
+ studio
+ .backend
+ .put(
+ &format!("agents/ada/v-{i}.txt"),
+ vec![b'v'; 100],
+ Condition::None,
+ )
+ .await
+ .unwrap();
+ durs.push(t1.elapsed());
+ }
+ let ack_wait = Config::fast().ack_wait;
+ assert!(
+ t.elapsed() < ack_wait * 2 + Duration::from_millis(1500),
+ "12 writes took {:?} ({durs:?})",
+ t.elapsed()
+ );
+ assert!(
+ durs[4..].iter().all(|d| *d < Duration::from_millis(200)),
+ "{durs:?}"
+ );
+ sim.shutdown().await;
+}
+
+/// A machine whose process is gone answers dials with a refusal. That is
+/// far better evidence than silence, so the volume does not wait out the
+/// silence window before a machine that needs authority takes over.
+#[tokio::test]
+async fn a_machine_that_refuses_connections_is_replaced_faster_than_a_silent_one() {
+ // Silence would need 1.5 s of quiet and 3 s of confirmation.
+ let mut cfg = Config::fast();
+ cfg.timing.floor = Duration::from_millis(1500);
+ cfg.timing.confirm = Duration::from_millis(3000);
+ let sim = Sim::with(&[("laptop", 9000), ("studio", 2000)], cfg.clone()).await;
+ group_ready(&sim, "laptop").await;
+ // The owner's process dies: its connections break, nothing answers.
+ sim.get("laptop").node.freeze(true);
+ sim.net.leave("laptop");
+ let t = Instant::now();
+ needs_authority(sim.get("studio"), "agents/ada/.cua-lease").await;
+ let took = t.elapsed();
+ assert!(
+ took < cfg.timing.floor + cfg.timing.confirm - Duration::from_millis(1500),
+ "took {took:?}, as long as silence would"
+ );
+ assert!(sim.get("studio").node.status().await.i_own);
+ sim.shutdown().await;
+}
+
+/// An older Cua Spaces build on the same home writes to the store directly
+/// (it does not know about replication). When the new build starts again
+/// those changes are found and reach the other machine; nothing is lost and
+/// nothing diverges silently.
+#[tokio::test]
+async fn changes_an_older_build_made_behind_replications_back_still_arrive() {
+ let mut sim = Sim::new(&[("studio", 9000), ("laptop", 2000)]).await;
+ group_ready(&sim, "studio").await;
+ sim.get("studio")
+ .backend
+ .put("agents/ada/a.txt", b"a".to_vec(), Condition::None)
+ .await
+ .unwrap();
+ sim.get("studio")
+ .backend
+ .put("agents/ada/b.txt", b"b".to_vec(), Condition::None)
+ .await
+ .unwrap();
+ eventually("replicated", 10, || async {
+ converged(&sim, &["studio", "laptop"]).await && heads(sim.get("laptop")).await.len() == 2
+ })
+ .await;
+ // Both machines stop cleanly (the app is quit); an older build then
+ // writes on each home.
+ let (studio_store, laptop_store) = (
+ sim.get("studio").store.clone(),
+ sim.get("laptop").store.clone(),
+ );
+ sim.get("studio").node.shutdown().await;
+ sim.get("laptop").node.shutdown().await;
+ studio_store
+ .put(
+ "agents/ada/a.txt",
+ b"a, edited by the old build".to_vec(),
+ Condition::None,
+ )
+ .await
+ .unwrap();
+ studio_store
+ .put(
+ "agents/ada/from-old-build.txt",
+ b"new".to_vec(),
+ Condition::None,
+ )
+ .await
+ .unwrap();
+ laptop_store
+ .delete("agents/ada/b.txt", Condition::None)
+ .await
+ .unwrap();
+ // The new build starts again on both.
+ sim.restart("studio").await;
+ sim.restart("laptop").await;
+ group_ready(&sim, "studio").await;
+ eventually("the changes reach both machines", 30, || async {
+ converged(&sim, &["studio", "laptop"]).await
+ && sim.get("laptop").node.status().await.pending.is_empty()
+ && head_etag(sim.get("laptop"), "agents/ada/from-old-build.txt")
+ .await
+ .is_some()
+ })
+ .await;
+ assert_eq!(
+ sim.get("laptop")
+ .store
+ .get("agents/ada/a.txt", None)
+ .await
+ .unwrap()
+ .0,
+ b"a, edited by the old build"
+ );
+ assert!(
+ head_etag(sim.get("studio"), "agents/ada/b.txt")
+ .await
+ .is_none(),
+ "the delete arrived"
+ );
+ sim.shutdown().await;
+}
+
+/// The first time replication touches a Volume that has files, a copy of its
+/// heads is taken; a directory prepared by a newer format is refused.
+#[tokio::test]
+async fn turning_replication_on_backs_up_the_heads_and_refuses_a_newer_format() {
+ let dir = tempfile::tempdir().unwrap();
+ let store = cua_volume::fs::FsBackend::new(dir.path().join("data"));
+ for i in 0..5 {
+ store
+ .put(
+ &format!("agents/ada/f{i}.txt"),
+ vec![b'x'; 10 + i],
+ Condition::None,
+ )
+ .await
+ .unwrap();
+ }
+ let net = cua_volume::replica::simnet::SimNet::new();
+ let (tr, _rx) = net.join("m");
+ let node = cua_volume::replica::ReplicaNode::open(
+ Config::fast(),
+ "m",
+ "m",
+ dir.path(),
+ store.clone(),
+ std::sync::Arc::new(tr),
+ vec![common::member("m", 5000)],
+ )
+ .await
+ .unwrap();
+ let backups: Vec<_> = std::fs::read_dir(dir.path().join("replica/backup"))
+ .unwrap()
+ .collect();
+ assert_eq!(backups.len(), 1, "one backup of the heads");
+ // The Volume's own files are untouched.
+ assert_eq!(store.list("").await.unwrap().len(), 5);
+ node.shutdown().await;
+ drop(node);
+ // A newer format is refused with a plain message.
+ std::fs::write(
+ dir.path().join("replica/format.json"),
+ br#"{"format":99,"created_ms":1}"#,
+ )
+ .unwrap();
+ let (tr2, _rx2) = net.join("m2");
+ let e = cua_volume::replica::ReplicaNode::open(
+ Config::fast(),
+ "m",
+ "m",
+ dir.path(),
+ store,
+ std::sync::Arc::new(tr2),
+ vec![common::member("m", 5000)],
+ )
+ .await
+ .unwrap_err();
+ assert!(e.to_string().contains("newer Cua Spaces"), "{e}");
+}
+
+/// A machine that wrote while the owner was away and then became the owner
+/// itself must put those writes in the log so the other machine gets them
+/// (they were visible on the writing machine as heads of their own).
+#[tokio::test]
+async fn writes_made_while_apart_by_the_machine_that_takes_over_reach_the_other() {
+ let sim = Sim::new(&[("studio", 9000), ("laptop", 2000)]).await;
+ group_ready(&sim, "studio").await;
+ sim.cut("studio", "laptop");
+ let laptop = sim.get("laptop");
+ eventually("the laptop sees the studio go", 10, || async {
+ laptop.node.status().await.peers[0].health != "up"
+ })
+ .await;
+ // Written while the owner is away: saved, visible here, not in any log.
+ laptop
+ .backend
+ .put(
+ "agents/laptop-w/p1",
+ b"typed on a plane".to_vec(),
+ Condition::None,
+ )
+ .await
+ .unwrap();
+ // The laptop then needs an ordered write and becomes the owner.
+ needs_authority(laptop, "agents/ada/.cua-lease").await;
+ assert!(laptop.node.status().await.i_own);
+ sim.heal("studio", "laptop");
+ eventually("the studio has it too", 30, || async {
+ converged(&sim, &["studio", "laptop"]).await
+ && head_etag(sim.get("studio"), "agents/laptop-w/p1")
+ .await
+ .is_some()
+ })
+ .await;
+ assert_eq!(
+ sim.get("studio")
+ .store
+ .get("agents/laptop-w/p1", None)
+ .await
+ .unwrap()
+ .0,
+ b"typed on a plane"
+ );
+ sim.shutdown().await;
+}
+
+/// A folder copy-in is one batch at the owner (one disk barrier, one run of
+/// log records) and reaches the other machine whole.
+#[tokio::test]
+async fn a_batch_of_files_is_one_write_at_the_owner_and_reaches_the_other_machine() {
+ let sim = Sim::new(&[("studio", 9000), ("laptop", 2000)]).await;
+ group_ready(&sim, "studio").await;
+ let items: Vec<(String, Vec<u8>)> = (0..300)
+ .map(|i| {
+ (
+ format!("agents/ada/batch/f{i}"),
+ format!("file {i}").into_bytes(),
+ )
+ })
+ .collect();
+ let metas = sim
+ .get("studio")
+ .backend
+ .put_many(items.clone())
+ .await
+ .unwrap();
+ assert_eq!(metas.len(), 300);
+ eventually("all of it is on the laptop", 20, || async {
+ converged(&sim, &["studio", "laptop"]).await && heads(sim.get("laptop")).await.len() == 300
+ })
+ .await;
+ for (k, v) in items.iter().step_by(37) {
+ assert_eq!(&sim.get("laptop").store.get(k, None).await.unwrap().0, v);
+ }
+ sim.shutdown().await;
+}
diff --git a/libs/cua/crates/cua-volume/tests/replica_supervision.rs b/libs/cua/crates/cua-volume/tests/replica_supervision.rs
new file mode 100644
index 0000000000..1631d80ca2
--- /dev/null
+++ b/libs/cua/crates/cua-volume/tests/replica_supervision.rs
@@ -0,0 +1,525 @@
+// SPDX-License-Identifier: FSL-1.1-MIT
+// Copyright (c) 2026 Cua AI, Inc.
+
+//! Durable agents across machines on the simulated network: the run goes to
+//! the most reliable machine, survives a closed laptop, moves when its
+//! machine dies, and a holder that wakes after being replaced is fenced.
+
+mod common;
+
+use std::collections::HashMap;
+use std::sync::{Arc, Mutex};
+use std::time::{Duration, Instant};
+
+use common::*;
+use cua_volume::backend::{Backend, Condition};
+use cua_volume::replica::PowerEvent;
+use cua_volume::replica::supervision::{DriveEnd, Driver, RunCtx, Supervision, SupervisionConfig};
+
+type StepLog = Arc<Mutex<Vec<(String, u64, u64, Instant)>>>;
+
+/// An agent made of N steps; step i writes `out/step-i` (the same bytes in
+/// any epoch) and a journal marker, and the end writes the checksum of every
+/// step. It resumes from the steps its machine's journal holds.
+struct Steps {
+ n: u64,
+ step_ms: u64,
+ log: StepLog,
+}
+
+fn content(i: u64) -> Vec<u8> {
+ format!("step {i}: {}", cua_volume::sha256_hex(&i.to_le_bytes())).into_bytes()
+}
+
+fn expected_sum(n: u64) -> String {
+ let mut all = vec![];
+ for i in 0..n {
+ all.extend(content(i));
+ }
+ cua_volume::sha256_hex(&all)
+}
+
+#[async_trait::async_trait]
+impl Driver for Steps {
+ async fn drive(&self, mut ctx: RunCtx) -> DriveEnd {
+ let prefix = format!("agents/{}/journal/", ctx.agent);
+ // How far this machine's copy of the journal reaches (contiguous).
+ let mut held = 0u64;
+ while ctx
+ .backend
+ .head(&format!("{prefix}{held:06}"))
+ .await
+ .ok()
+ .flatten()
+ .is_some()
+ {
+ held += 1;
+ }
+ ctx.held_reaches(held);
+ let mut i = held;
+ while i < self.n {
+ if ctx.fenced() {
+ return DriveEnd::Fenced;
+ }
+ self.log
+ .lock()
+ .unwrap()
+ .push((ctx.machine.clone(), ctx.epoch, i, Instant::now()));
+ let out = format!("agents/{}/out/step-{i}", ctx.agent);
+ if ctx
+ .backend
+ .put(&out, content(i), Condition::None)
+ .await
+ .is_err()
+ {
+ return DriveEnd::Failed("write".into());
+ }
+ let _ = ctx
+ .backend
+ .put(&format!("{prefix}{i:06}"), vec![b'.'], Condition::None)
+ .await;
+ ctx.report(i + 1);
+ i += 1;
+ tokio::time::sleep(Duration::from_millis(self.step_ms)).await;
+ }
+ if ctx.fenced() {
+ return DriveEnd::Fenced;
+ }
+ // The result: the checksum of every step as the Volume holds it.
+ let mut all = vec![];
+ for s in 0..self.n {
+ match ctx
+ .backend
+ .get(&format!("agents/{}/out/step-{s}", ctx.agent), None)
+ .await
+ {
+ Ok((b, _)) => all.extend(b),
+ Err(_) => return DriveEnd::Failed("a step is missing".into()),
+ }
+ }
+ let sum = cua_volume::sha256_hex(&all);
+ let _ = ctx
+ .backend
+ .put(
+ &format!("agents/{}/result.txt", ctx.agent),
+ sum.clone().into_bytes(),
+ Condition::None,
+ )
+ .await;
+ DriveEnd::Done(sum)
+ }
+}
+
+struct World {
+ sim: Sim,
+ sups: HashMap<String, Supervision>,
+ log: StepLog,
+ awake: Arc<Mutex<Vec<String>>>,
+}
+
+async fn world(specs: &[(&str, u64)], n: u64, step_ms: u64) -> World {
+ let sim = Sim::new(specs).await;
+ let mut w = World {
+ sim,
+ sups: HashMap::new(),
+ log: Arc::new(Mutex::new(vec![])),
+ awake: Arc::new(Mutex::new(vec![])),
+ };
+ for (id, _) in specs {
+ w.supervise(id, n, step_ms);
+ }
+ w
+}
+
+impl World {
+ fn supervise(&mut self, id: &str, n: u64, step_ms: u64) {
+ let m = self.sim.get(id);
+ let awake = self.awake.clone();
+ let s = Supervision::new(
+ m.node.clone(),
+ m.backend.clone(),
+ SupervisionConfig::fast(),
+ Arc::new(Steps {
+ n,
+ step_ms,
+ log: self.log.clone(),
+ }),
+ Some(Arc::new(move |why: &str| {
+ awake.lock().unwrap().push(why.to_string());
+ Box::new(()) as Box<dyn std::any::Any + Send>
+ })),
+ );
+ s.spawn();
+ self.sups.insert(id.to_string(), s);
+ }
+
+ fn who_ran(&self) -> Vec<String> {
+ let mut v: Vec<String> = self
+ .log
+ .lock()
+ .unwrap()
+ .iter()
+ .map(|e| e.0.clone())
+ .collect();
+ v.sort();
+ v.dedup();
+ v
+ }
+
+ async fn result_on(&self, id: &str, agent: &str) -> Option<String> {
+ self.sim
+ .get(id)
+ .store
+ .get(&format!("agents/{agent}/result.txt"), None)
+ .await
+ .ok()
+ .map(|(b, _)| String::from_utf8(b).unwrap())
+ }
+
+ async fn stop_all(&self) {
+ for s in self.sups.values() {
+ s.stop().await;
+ }
+ self.sim.shutdown().await;
+ }
+}
+
+#[tokio::test]
+async fn a_run_started_on_the_laptop_runs_on_the_studio_and_says_so() {
+ let w = world(&[("studio", 9000), ("laptop", 2000)], 20, 30).await;
+ group_ready(&w.sim, "studio").await;
+ let info = w.sups["laptop"]
+ .start_run("ada", serde_json::json!({"task": "demo"}))
+ .await
+ .unwrap();
+ assert_eq!(info.run_on, "studio");
+ assert_eq!(
+ info.words,
+ "Your agent keeps running on studio. Files are on 2 machines. You can close this machine."
+ );
+ eventually("the run finishes", 30, || async {
+ w.result_on("laptop", "ada").await.is_some()
+ })
+ .await;
+ assert_eq!(
+ w.who_ran(),
+ vec!["studio".to_string()],
+ "the laptop never drove it"
+ );
+ assert_eq!(
+ w.result_on("laptop", "ada").await.unwrap(),
+ expected_sum(20)
+ );
+ assert_eq!(
+ w.result_on("studio", "ada").await.unwrap(),
+ expected_sum(20)
+ );
+ // The studio held a sleep assertion for it with a readable reason.
+ assert_eq!(
+ w.awake.lock().unwrap().as_slice(),
+ ["Cua: the durable agent ada is running".to_string()]
+ );
+ w.stop_all().await;
+}
+
+#[tokio::test]
+async fn closing_the_laptop_does_not_stop_the_run_or_lose_files() {
+ let w = world(&[("studio", 9000), ("laptop", 2000)], 60, 40).await;
+ group_ready(&w.sim, "studio").await;
+ w.sups["laptop"]
+ .start_run("ada", serde_json::json!({}))
+ .await
+ .unwrap();
+ eventually("some steps ran", 20, || async {
+ w.log.lock().unwrap().len() >= 8
+ })
+ .await;
+ // The lid closes.
+ w.sim
+ .get("laptop")
+ .node
+ .notify_power(PowerEvent::WillSleep)
+ .await;
+ tokio::time::sleep(Duration::from_millis(50)).await;
+ w.sim.get("laptop").node.freeze(true);
+ w.sim.cut("studio", "laptop");
+ let at_close = w.log.lock().unwrap().len();
+ eventually(
+ "the run finishes while the laptop is closed",
+ 30,
+ || async { w.result_on("studio", "ada").await.is_some() },
+ )
+ .await;
+ assert!(w.log.lock().unwrap().len() > at_close);
+ assert_eq!(w.who_ran(), vec!["studio".to_string()]);
+ // The lid opens; the laptop has every file.
+ w.sim.heal("studio", "laptop");
+ w.sim.get("laptop").node.freeze(false);
+ w.sim
+ .get("laptop")
+ .node
+ .notify_power(PowerEvent::DidWake)
+ .await;
+ eventually("the laptop has the result and every step", 20, || async {
+ converged(&w.sim, &["studio", "laptop"]).await
+ && w.result_on("laptop", "ada").await.is_some()
+ })
+ .await;
+ assert_eq!(
+ w.result_on("laptop", "ada").await.unwrap(),
+ expected_sum(60)
+ );
+ assert_eq!(w.sim.get("studio").node.status().await.takeovers, 0);
+ w.stop_all().await;
+}
+
+#[tokio::test]
+async fn when_the_studio_dies_the_laptop_takes_the_run_and_resumes_it() {
+ let w = world(&[("studio", 9000), ("laptop", 2000)], 80, 40).await;
+ group_ready(&w.sim, "studio").await;
+ w.sups["laptop"]
+ .start_run("ada", serde_json::json!({}))
+ .await
+ .unwrap();
+ eventually("steps are running on the studio", 20, || async {
+ w.log.lock().unwrap().len() >= 15
+ })
+ .await;
+ // The studio's process dies.
+ let dead_at = Instant::now();
+ w.sups["studio"].stop().await;
+ w.sim.get("studio").node.shutdown().await;
+ w.sim.cut("studio", "laptop");
+ let ran_on_studio = w.log.lock().unwrap().len();
+ eventually("the laptop runs it", 30, || async {
+ w.log.lock().unwrap().iter().any(|e| e.0 == "laptop")
+ })
+ .await;
+ let took = w
+ .log
+ .lock()
+ .unwrap()
+ .iter()
+ .find(|e| e.0 == "laptop")
+ .map(|e| e.3.duration_since(dead_at))
+ .unwrap();
+ assert!(took < Duration::from_secs(8), "took over in {took:?}");
+ eventually("it finishes on the laptop", 30, || async {
+ w.result_on("laptop", "ada").await.is_some()
+ })
+ .await;
+ assert_eq!(
+ w.result_on("laptop", "ada").await.unwrap(),
+ expected_sum(80)
+ );
+ // It did not start again from the beginning: the laptop's first step is
+ // after what the studio had done (its journal copy was there).
+ let first_laptop_step = w
+ .log
+ .lock()
+ .unwrap()
+ .iter()
+ .find(|e| e.0 == "laptop")
+ .map(|e| e.2)
+ .unwrap();
+ assert!(
+ first_laptop_step >= 10,
+ "restarted at step {first_laptop_step}"
+ );
+ assert!(ran_on_studio >= 15);
+ let runs = w.sups["laptop"].runs().await.unwrap();
+ let note = &runs[0].note;
+ assert!(note.starts_with("Resumed on laptop"), "{note}");
+ w.stop_all().await;
+}
+
+#[tokio::test]
+async fn a_frozen_holder_that_wakes_after_a_takeover_is_fenced() {
+ let w = world(&[("studio", 9000), ("laptop", 2000)], 100, 40).await;
+ group_ready(&w.sim, "studio").await;
+ w.sups["laptop"]
+ .start_run("ada", serde_json::json!({}))
+ .await
+ .unwrap();
+ eventually("running on the studio", 20, || async {
+ w.log.lock().unwrap().len() >= 10
+ })
+ .await;
+ // The studio freezes (a stopped process) and loses the network.
+ w.sim.get("studio").node.freeze(true);
+ w.sim.cut("studio", "laptop");
+ eventually("the laptop takes the run", 30, || async {
+ w.log.lock().unwrap().iter().any(|e| e.0 == "laptop")
+ })
+ .await;
+ // The studio thaws and the network returns.
+ tokio::time::sleep(Duration::from_millis(300)).await;
+ w.sim.heal("studio", "laptop");
+ w.sim.get("studio").node.freeze(false);
+ w.sim
+ .get("studio")
+ .node
+ .notify_power(PowerEvent::DidWake)
+ .await;
+ eventually("the run finishes", 60, || async {
+ w.result_on("laptop", "ada").await.is_some()
+ && converged(&w.sim, &["studio", "laptop"]).await
+ })
+ .await;
+ // The output is right everywhere, whatever ran twice in between.
+ assert_eq!(
+ w.result_on("laptop", "ada").await.unwrap(),
+ expected_sum(100)
+ );
+ assert_eq!(
+ w.result_on("studio", "ada").await.unwrap(),
+ expected_sum(100)
+ );
+ for i in 0..100 {
+ let b = w
+ .sim
+ .get("studio")
+ .store
+ .get(&format!("agents/ada/out/step-{i}"), None)
+ .await
+ .unwrap()
+ .0;
+ assert_eq!(b, content(i), "step {i} is wrong or missing");
+ }
+ // Exactly one epoch holds the lease at the end, the later one; the
+ // stopped holder's driver stopped.
+ let runs = w.sups["studio"].runs().await.unwrap();
+ assert!(runs[0].epoch >= 2, "{runs:?}");
+ eventually("the stopped holder's driver is stopped", 30, || async {
+ let r = w.sups["studio"].runs().await.unwrap();
+ !r[0].running_here
+ })
+ .await;
+ w.stop_all().await;
+}
+
+#[tokio::test]
+async fn a_run_on_a_laptop_goes_back_to_the_studio_when_it_returns() {
+ // The studio starts later than the laptop: the run begins on the laptop.
+ let sim = Sim::new(&[("studio", 9000), ("laptop", 2000)]).await;
+ group_ready(&sim, "studio").await;
+ let mut w = World {
+ sim,
+ sups: HashMap::new(),
+ log: Arc::new(Mutex::new(vec![])),
+ awake: Arc::new(Mutex::new(vec![])),
+ };
+ // Only the laptop supervises at first, and the studio is cut off.
+ w.supervise("laptop", 120, 40);
+ w.sim.cut("studio", "laptop");
+ eventually("the laptop sees the studio go", 10, || async {
+ w.sim.get("laptop").node.status().await.peers[0].health != "up"
+ })
+ .await;
+ w.sups["laptop"]
+ .start_run("ada", serde_json::json!({}))
+ .await
+ .unwrap();
+ // (The write cannot reach the owner, so the laptop needs authority.)
+ needs_authority(w.sim.get("laptop"), "agents/ada/.cua-lease").await;
+ let started = w.sups["laptop"]
+ .start_run("ada", serde_json::json!({}))
+ .await;
+ assert!(started.is_ok());
+ eventually("the laptop runs it", 20, || async {
+ w.log.lock().unwrap().iter().any(|e| e.0 == "laptop")
+ })
+ .await;
+ // The studio comes back and supervises.
+ w.sim.heal("studio", "laptop");
+ w.supervise("studio", 120, 40);
+ eventually("the studio gets the run", 40, || async {
+ w.log.lock().unwrap().iter().any(|e| e.0 == "studio")
+ })
+ .await;
+ eventually("it finishes", 60, || async {
+ w.result_on("laptop", "ada").await.is_some()
+ })
+ .await;
+ assert_eq!(
+ w.result_on("laptop", "ada").await.unwrap(),
+ expected_sum(120)
+ );
+ w.stop_all().await;
+}
+
+#[tokio::test]
+async fn one_machine_says_it_will_pause_and_keeps_itself_awake() {
+ let w = world(&[("macbook", 2000)], 5, 20).await;
+ let info = w.sups["macbook"]
+ .start_run("ada", serde_json::json!({}))
+ .await
+ .unwrap();
+ assert!(
+ info.words.contains("will pause when it sleeps"),
+ "{}",
+ info.words
+ );
+ assert!(info.words.contains("Add another machine"), "{}", info.words);
+ eventually("done", 20, || async {
+ w.result_on("macbook", "ada").await.is_some()
+ })
+ .await;
+ assert_eq!(w.who_ran(), vec!["macbook".to_string()]);
+ assert_eq!(w.awake.lock().unwrap().len(), 1);
+ w.stop_all().await;
+}
+
+/// The run moves between architectures: replication and the journal are the
+/// same, built programs are not, and the words say so.
+#[tokio::test]
+async fn a_takeover_across_architectures_says_built_programs_are_not_reused() {
+ let w = world(&[("studio", 9000), ("laptop", 2000)], 60, 40).await;
+ for (id, rank, arch) in [("studio", 9000, "x86_64"), ("laptop", 2000, "aarch64")] {
+ w.sim.get(id).node.set_facts(cua_volume::replica::Facts {
+ rank,
+ free_disk: 500 << 30,
+ arch: arch.into(),
+ ..Default::default()
+ });
+ }
+ group_ready(&w.sim, "studio").await;
+ w.sim
+ .get("studio")
+ .backend
+ .put(
+ "agents/ada/target/debug/ada",
+ b"ELF".to_vec(),
+ Condition::None,
+ )
+ .await
+ .unwrap();
+ w.sups["laptop"]
+ .start_run("ada", serde_json::json!({}))
+ .await
+ .unwrap();
+ eventually("steps run on the studio", 20, || async {
+ w.log.lock().unwrap().len() >= 10
+ })
+ .await;
+ w.sups["studio"].stop().await;
+ w.sim.get("studio").node.shutdown().await;
+ w.sim.cut("studio", "laptop");
+ eventually("it finishes on the laptop", 40, || async {
+ w.result_on("laptop", "ada").await.is_some()
+ })
+ .await;
+ assert_eq!(
+ w.result_on("laptop", "ada").await.unwrap(),
+ expected_sum(60)
+ );
+ let note = w.sups["laptop"].runs().await.unwrap()[0].note.clone();
+ assert!(note.starts_with("Resumed on laptop"), "{note}");
+ assert!(
+ note.contains("studio is amd64")
+ && note.contains("target/")
+ && note.contains("built again"),
+ "{note}"
+ );
+ w.stop_all().await;
+}