Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(volume): replicate the Volume across the account's machines (#6280)
ddupont74657cf5a9b4
diff --git a/libs/cua/Cargo.lock b/libs/cua/Cargo.lockindex f7500f3e52..3060411e4b 100644--- a/libs/cua/Cargo.lock+++ b/libs/cua/Cargo.lock@@ -3633,6 +3633,7 @@ dependencies = ["thiserror 2.0.20","tokio","tracing",+ "tracing-subscriber",][[package]]diff --git a/libs/cua/crates/cua-volume/Cargo.toml b/libs/cua/crates/cua-volume/Cargo.tomlindex 782bbbf515..2ff6e54196 100644--- a/libs/cua/crates/cua-volume/Cargo.toml+++ b/libs/cua/crates/cua-volume/Cargo.toml@@ -45,6 +45,7 @@ fuser = { version = "0.18", default-features = false, optional = true }[dev-dependencies]tempfile.workspace = true+tracing-subscriber.workspace = truetokio = { workspace = true, features = ["full"] }[[example]]diff --git a/libs/cua/crates/cua-volume/src/fs.rs b/libs/cua/crates/cua-volume/src/fs.rsindex ea42c84796..6fc9490c63 100644--- a/libs/cua/crates/cua-volume/src/fs.rs+++ b/libs/cua/crates/cua-volume/src/fs.rs@@ -148,7 +148,8 @@ impl FsBackend {Ok(())}- fn new_version() -> String {+ /// A fresh version id (time-ordered, unique).+ pub fn new_version() -> String {// Strictly increasing within the process: two writes in the same// millisecond must still sort in write order, so the random suffix// only keeps ids apart across processes.@@ -246,21 +247,20 @@ impl FsBackend {Ok(out)}- fn delete_blocking(&self, key: &str, cond: Condition) -> Result<()> {+ fn delete_blocking(&self, key: &str, cond: Condition) -> Result<String> {let _lock = self.lock()?;let current = self.read_head(key)?;if current.is_none() && cond == Condition::None {return Err(Error::NotFound(key.to_string()));}Self::check(&cond, current.as_ref(), key)?;- let marker = self- .versions_dir(key)?- .join(format!("{}.del", Self::new_version()));+ let version = Self::new_version();+ let marker = self.versions_dir(key)?.join(format!("{version}.del"));Self::atomic_write(&marker, b"")?;let head = self.head_path(key)?;cua_home::guard_write(&head)?;fs::remove_file(head)?;- Ok(())+ Ok(version)}fn get_blocking(&self, key: &str, version: Option<&str>) -> Result<(Vec<u8>, ObjectMeta)> {@@ -534,6 +534,209 @@ impl FsBackend {}}+/// Where the bytes of an applied version come from.+pub enum ApplySource {+ /// In memory.+ Bytes(Vec<u8>),+ /// A finished file the store takes: moved (same file system) or copied.+ /// With `link` the file stays where it is and the store gets a hard link+ /// (a copy where hard links are not possible).+ File { path: PathBuf, link: bool },+}++/// What replication needs from the local store: writes that keep the+/// owner's version ids (so history, etags and preconditions mean the same+/// on every machine), heads moved back to an earlier version, and the paths+/// of version files. None of this changes the [`Backend`] contract.+impl FsBackend {+ /// Deletes `key` like [`Backend::delete`] and returns the delete marker's+ /// version id (what replication ships).+ pub async fn delete_versioned(&self, key: &str, cond: Condition) -> Result<String> {+ let (me, key) = (self.clone(), key.to_string());+ blocking(move || me.delete_blocking(&key, cond)).await+ }++ /// The file holding `version` of `key`.+ pub fn version_file(&self, key: &str, version: &str) -> Result<PathBuf> {+ self.version_path(key, version)+ }++ fn install_version(+ &self,+ key: &str,+ version: &str,+ src: ApplySource,+ full: bool,+ ) -> Result<()> {+ let dst = self.version_path(key, version)?;+ if dst.is_file() {+ // Already there (a replayed record).+ if let ApplySource::File { path, link: false } = src {+ let _ = fs::remove_file(path);+ }+ return Ok(());+ }+ let dir = dst+ .parent()+ .ok_or_else(|| Error::Invalid("no parent".into()))?;+ self.ensure_root()?;+ cua_home::guard_write(&dst)?;+ fs::create_dir_all(dir)?;+ match src {+ ApplySource::Bytes(b) => Self::atomic_write_with(&dst, &b, full)?,+ ApplySource::File { path, link } => {+ let tmp = dir.join(format!(".tmp-{}", crate::new_id()));+ if link {+ if fs::hard_link(&path, &tmp).is_err() {+ fs::copy(&path, &tmp)?;+ }+ } else if fs::rename(&path, &tmp).is_err() {+ fs::copy(&path, &tmp)?;+ let _ = fs::remove_file(&path);+ }+ File::open(&tmp)?.sync_all()?;+ fs::rename(&tmp, &dst).inspect_err(|_| {+ let _ = fs::remove_file(&tmp);+ })?;+ }+ }+ Ok(())+ }++ /// Makes `meta` (a version the owner assigned) the head of its key,+ /// storing its bytes under that version. Idempotent.+ pub async fn apply_put(&self, meta: ObjectMeta, src: ApplySource) -> Result<ObjectMeta> {+ let me = self.clone();+ blocking(move || {+ let _lock = me.lock()?;+ me.install_version(&meta.key, &meta.version, src, true)?;+ Self::atomic_write(&me.head_path(&meta.key)?, &serde_json::to_vec(&meta)?)?;+ Ok(meta)+ })+ .await+ }++ /// [`FsBackend::apply_put`] for many small objects at once: one lock, one+ /// disk barrier at the end (each file is still written and renamed whole,+ /// so a crash leaves each either old or new).+ pub async fn apply_many(&self, items: Vec<(ObjectMeta, Vec<u8>)>) -> Result<()> {+ let me = self.clone();+ blocking(move || {+ let _lock = me.lock()?;+ for (meta, bytes) in &items {+ me.install_version(+ &meta.key,+ &meta.version,+ ApplySource::Bytes(bytes.clone()),+ false,+ )?;+ Self::atomic_write_with(+ &me.head_path(&meta.key)?,+ &serde_json::to_vec(meta)?,+ false,+ )?;+ }+ if !items.is_empty() {+ OpenOptions::new()+ .write(true)+ .open(me.root.join(".lock"))?+ .sync_all()?;+ }+ Ok(())+ })+ .await+ }++ /// Records the owner's delete marker `version` and removes the head.+ /// Idempotent.+ pub async fn apply_delete(&self, key: &str, version: &str) -> Result<()> {+ let (me, key, version) = (self.clone(), key.to_string(), version.to_string());+ blocking(move || {+ let _lock = me.lock()?;+ let marker = me.versions_dir(&key)?.join(format!("{version}.del"));+ if !marker.is_file() {+ Self::atomic_write(&marker, b"")?;+ }+ let head = me.head_path(&key)?;+ match fs::remove_file(head) {+ Ok(()) => Ok(()),+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),+ Err(e) => Err(e.into()),+ }+ })+ .await+ }++ /// Moves the head of `key` to `to` (a version this store still holds;+ /// `None`: no head). Versions are never removed, so nothing is lost.+ pub async fn set_head(&self, key: &str, to: Option<ObjectMeta>) -> Result<()> {+ let (me, key) = (self.clone(), key.to_string());+ blocking(move || {+ let _lock = me.lock()?;+ let head = me.head_path(&key)?;+ match to {+ Some(m) => {+ if !me.version_path(&key, &m.version)?.is_file() {+ return Err(Error::NotFound(format!("{key} version {}", m.version)));+ }+ Self::atomic_write(&head, &serde_json::to_vec(&m)?)+ }+ None => match fs::remove_file(head) {+ Ok(()) => Ok(()),+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),+ Err(e) => Err(e.into()),+ },+ }+ })+ .await+ }++ /// Removes one version's bytes (a finished spool entry; never used on+ /// the data store).+ pub async fn purge_version(&self, key: &str, version: &str) -> Result<()> {+ let (me, key, version) = (self.clone(), key.to_string(), version.to_string());+ blocking(move || {+ let p = me.version_path(&key, &version)?;+ match fs::remove_file(&p) {+ Ok(()) => {}+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}+ Err(e) => return Err(e.into()),+ }+ if let Ok(Some(h)) = me.read_head(&key)+ && h.version == version+ {+ let _ = fs::remove_file(me.head_path(&key)?);+ }+ Ok(())+ })+ .await+ }++ /// Free bytes on the file system holding the store (best effort).+ pub fn free_bytes(&self) -> Option<u64> {+ #[cfg(unix)]+ {+ let probe = if self.root.exists() {+ self.root.clone()+ } else {+ self.root.parent()?.to_path_buf()+ };+ let c = std::ffi::CString::new(probe.to_string_lossy().as_bytes()).ok()?;+ // SAFETY: an all-zero statvfs is a valid value to pass as an out pointer.+ let mut st: libc::statvfs = unsafe { std::mem::zeroed() };+ // SAFETY: `c` is a valid C string and `st` a valid out pointer.+ if unsafe { libc::statvfs(c.as_ptr(), &mut st) } == 0 {+ return Some((st.f_bavail as u64).saturating_mul(st.f_frsize as u64));+ }+ None+ }+ #[cfg(not(unix))]+ {+ None+ }+ }+}+#[cfg(unix)]fn read_at(f: &File, buf: &mut [u8], offset: u64) -> std::io::Result<usize> {std::os::unix::fs::FileExt::read_at(f, buf, offset)@@ -601,8 +804,7 @@ impl Backend for FsBackend {}async fn delete(&self, key: &str, cond: Condition) -> Result<()> {- let (me, key) = (self.clone(), key.to_string());- blocking(move || me.delete_blocking(&key, cond)).await+ self.delete_versioned(key, cond).await.map(|_| ())}async fn versions(&self, key: &str) -> Result<Vec<VersionInfo>> {diff --git a/libs/cua/crates/cua-volume/src/lib.rs b/libs/cua/crates/cua-volume/src/lib.rsindex c37326690f..7d20b06bdc 100644--- a/libs/cua/crates/cua-volume/src/lib.rs+++ b/libs/cua/crates/cua-volume/src/lib.rs@@ -60,6 +60,7 @@ pub mod lease;pub mod nfs;pub mod path;pub mod remote;+pub mod replica;#[cfg(feature = "s3")]pub mod s3;pub mod scan;diff --git a/libs/cua/crates/cua-volume/src/replica/apply.rs b/libs/cua/crates/cua-volume/src/replica/apply.rsnew file mode 100644index 0000000000..9fa311a2c6--- /dev/null+++ b/libs/cua/crates/cua-volume/src/replica/apply.rs@@ -0,0 +1,523 @@+// SPDX-License-Identifier: FSL-1.1-MIT+// Copyright (c) 2026 Cua AI, Inc.++//! The replica side: apply the owner's records in order, catch up from a+//! snapshot, and put right a copy that was on the wrong side of a takeover.++use std::collections::HashSet;+use std::path::PathBuf;+use std::sync::Arc;++use super::era::{Eras, Pos};+use super::link::Link;+use super::migrate::{self, Unlogged};+use super::node::{Inner, save_eras};+use super::oplog::{Op, Record};+use super::wire::{Cond, Hello, Msg, Pending, Rep, Req, Src};+use crate::backend::{Backend, ObjectMeta};+use crate::fs::ApplySource;+use crate::{Error, Result};++impl Inner {+ /// Where the bytes of `meta` can come from without the network, if+ /// anywhere on this machine.+ async fn local_source(&self, meta: &ObjectMeta) -> Option<ApplySource> {+ if meta.size == 0 {+ return Some(ApplySource::Bytes(vec![]));+ }+ if let Ok(Some(h)) = self.store.head(&meta.key).await+ && h.etag == meta.etag+ && let Ok(path) = self.store.version_file(&meta.key, &h.version)+ && path.is_file()+ {+ return Some(ApplySource::File { path, link: true });+ }+ let pending: Vec<Pending> = {+ let st = self.st.lock().await;+ st.outbox+ .items()+ .iter()+ .filter(|p| {+ p.meta+ .as_ref()+ .is_some_and(|m| m.etag == meta.etag && m.size == meta.size)+ })+ .cloned()+ .collect()+ };+ for p in pending {+ let path = match &p.src {+ Src::Spool { key, version } => self.spool.version_file(key, version),+ Src::Data { key, version } => self.store.version_file(key, version),+ };+ if let Ok(path) = path+ && path.is_file()+ {+ return Some(ApplySource::File { path, link: true });+ }+ }+ None+ }++ /// The bytes of `meta`, from this machine if it has them, else pulled+ /// from `from_peer`.+ pub(super) async fn acquire(+ self: &Arc<Self>,+ from_peer: &str,+ meta: &ObjectMeta,+ inline: Option<Vec<u8>>,+ ) -> Result<ApplySource> {+ if let Some(b) = inline {+ return Ok(ApplySource::Bytes(b));+ }+ if let Some(s) = self.local_source(meta).await {+ return Ok(s);+ }+ let dl = self+ .data_link(from_peer)+ .await+ .ok_or_else(|| Error::Backend("no data link to the owner".into()))?;+ let seed = self.store.head(&meta.key).await.ok().flatten();+ let path = self+ .pull(+ &dl,+ Src::Data {+ key: meta.key.clone(),+ version: meta.version.clone(),+ },+ meta,+ seed.as_ref(),+ )+ .await?;+ Ok(ApplySource::File { path, link: false })+ }++ pub(super) async fn handle_append(+ self: &Arc<Self>,+ link: &Arc<Link>,+ epoch: u64,+ records: Vec<super::wire::RecordW>,+ data: Vec<u8>,+ ) -> Msg {+ let t0 = std::time::Instant::now();+ let _g = self.gate.lock().await;+ tracing::debug!(n = records.len(), gate_wait = ?t0.elapsed(), "replica: append received");+ {+ let mut st = self.st.lock().await;+ if link.peer != st.eras.owner() || epoch < st.eras.epoch() {+ st.fenced_writes += records.len() as u64;+ return Msg::Fenced {+ eras: st.eras.recs.clone(),+ };+ }+ if epoch > st.eras.epoch() {+ return Msg::Behind {+ applied: st.log.tail(),+ };+ }+ }+ // Runs of small puts that came with their bytes are applied as one+ // batch (one disk barrier, one run of log records).+ let mut records = records.into_iter().peekable();+ while let Some(rw) = records.next() {+ if let (Op::Put { .. }, Some(_)) = (&rw.op, rw.body) {+ let mut run = vec![rw];+ while run.len() < 256+ && records+ .peek()+ .is_some_and(|n| matches!(n.op, Op::Put { .. }) && n.body.is_some())+ {+ run.push(records.next().expect("peeked"));+ }+ let tail = self.st.lock().await.log.tail();+ let mut expect = tail;+ let mut items = vec![];+ let mut recs = vec![];+ let mut gap = false;+ for r in run {+ if r.pos <= tail {+ continue;+ }+ let contiguous = if r.pos.epoch == expect.epoch {+ r.pos.seq == expect.seq + 1+ } else {+ r.pos.epoch > expect.epoch && r.pos.seq == 1+ };+ if !contiguous {+ gap = true;+ break;+ }+ expect = r.pos;+ let (o, l) = r.body.expect("a body");+ let Some(bytes) = data.get(o as usize..(o + l) as usize) else {+ gap = true;+ break;+ };+ if let Op::Put { meta, .. } = &r.op {+ items.push((meta.clone(), bytes.to_vec()));+ }+ recs.push(Record {+ pos: r.pos,+ op: r.op,+ at_ms: r.at_ms,+ });+ }+ if !items.is_empty() {+ if let Err(e) = self.store.apply_many(items).await {+ if e.to_string().contains("No space")+ || e.to_string().contains("disk is full")+ {+ self.st.lock().await.healthy = false;+ }+ return Msg::Error {+ kind: e.tag().into(),+ message: e.to_string(),+ };+ }+ let last = recs.last().map(|r| r.pos);+ let mut st = self.st.lock().await;+ st.healthy = true;+ if let Err(e) = st.log.append_batch(recs) {+ return Msg::Error {+ kind: e.tag().into(),+ message: e.to_string(),+ };+ }+ if let Some(p) = last {+ self.applied_tx.send_replace(p);+ }+ }+ if gap {+ let t = self.st.lock().await.log.tail();+ return Msg::Behind { applied: t };+ }+ continue;+ }+ let tail = self.st.lock().await.log.tail();+ if rw.pos <= tail {+ tracing::debug!(me = %self.me, pos = %rw.pos, %tail, key = %rw.op.key(), "replica: skipped (not after tail)");+ continue;+ }+ let contiguous = if rw.pos.epoch == tail.epoch {+ rw.pos.seq == tail.seq + 1+ } else {+ rw.pos.epoch > tail.epoch && rw.pos.seq == 1+ };+ if !contiguous {+ return Msg::Behind { applied: tail };+ }+ let inline = rw+ .body+ .and_then(|(o, l)| data.get(o as usize..(o + l) as usize).map(<[u8]>::to_vec));+ let r: Result<()> = async {+ match &rw.op {+ Op::Put { meta, .. } => {+ let src = self.acquire(&link.peer, meta, inline).await?;+ self.store.apply_put(meta.clone(), src).await?;+ }+ Op::Delete { key, version, .. } => {+ self.store.apply_delete(key, version).await?;+ }+ }+ Ok(())+ }+ .await;+ if let Err(e) = r {+ let full =+ e.to_string().contains("disk is full") || e.to_string().contains("No space");+ if full {+ self.st.lock().await.healthy = false;+ }+ return Msg::Error {+ kind: e.tag().into(),+ message: e.to_string(),+ };+ }+ let mut st = self.st.lock().await;+ st.healthy = true;+ tracing::debug!(me = %self.me, pos = %rw.pos, key = %rw.op.key(), "replica: applied");+ if let Err(e) = st.log.append(Record {+ pos: rw.pos,+ op: rw.op,+ at_ms: rw.at_ms,+ }) {+ return Msg::Error {+ kind: e.tag().into(),+ message: e.to_string(),+ };+ }+ self.applied_tx.send_replace(rw.pos);+ }+ let st = self.st.lock().await;+ Msg::Ack {+ applied: st.log.tail(),+ }+ }++ pub(super) async fn handle_snapshot(self: &Arc<Self>, link: &Arc<Link>, at: Pos) -> Msg {+ let _g = self.gate.lock().await;+ match self.pull_snapshot(link, at).await {+ Ok(()) => Msg::Ack { applied: at },+ Err(e) => Msg::Error {+ kind: e.tag().into(),+ message: e.to_string(),+ },+ }+ }++ async fn pull_snapshot(self: &Arc<Self>, link: &Arc<Link>, at: Pos) -> Result<()> {+ let peer = link.peer.clone();+ let dl = self+ .data_link(&peer)+ .await+ .ok_or_else(|| Error::Backend("no data link to the owner".into()))?;+ let mut seen: HashSet<String> = HashSet::new();+ let mut after = String::new();+ loop {+ let (m, _) = dl+ .call(+ Msg::Req(Req::Heads {+ after_key: after.clone(),+ max: 512,+ }),+ &[],+ self.cfg.data_timeout,+ )+ .await+ .map_err(|e| Error::Backend(e.to_string()))?;+ let (items, done) = match m {+ Msg::Rep(Rep::Heads { items, done, .. }) => (items, done),+ Msg::Rep(Rep::Fail { kind, message }) => {+ return Err(Rep::into_error(&kind, message));+ }+ _ => return Err(Error::Backend("unexpected reply".into())),+ };+ for meta in &items {+ seen.insert(meta.key.clone());+ if let Some(local) = self.store.head(&meta.key).await?+ && local.version == meta.version+ {+ continue;+ }+ if self.st.lock().await.outbox.has_key(&meta.key) {+ continue;+ }+ let src = self.acquire(&peer, meta, None).await?;+ self.store.apply_put(meta.clone(), src).await?;+ }+ if let Some(last) = items.last() {+ after = last.key.clone();+ }+ if done || items.is_empty() {+ break;+ }+ }+ // Heads that are not the owner's and not waiting to be sent are stale.+ for local in self.store.list("").await? {+ if !seen.contains(&local.key) && !self.st.lock().await.outbox.has_key(&local.key) {+ self.store.set_head(&local.key, None).await?;+ }+ }+ let mut st = self.st.lock().await;+ st.log.reset_to(at)?;+ drop(st);+ self.applied_tx.send_replace(at);+ Ok(())+ }++ /// This machine holds records past a later era's base (it was the owner+ /// while another took over, or it was ahead): move those heads back,+ /// keep every version, and queue the changes for the new owner.+ pub(super) async fn reconcile_eras(self: &Arc<Self>) {+ let _g = self.gate.lock().await;+ if let Err(e) = self.reconcile_locked().await {+ tracing::warn!(error = %e, "replica: could not hand over this machine's divergent changes yet");+ }+ }++ async fn reconcile_locked(self: &Arc<Self>) -> Result<()> {+ let tail_records = {+ let mut st = self.st.lock().await;+ let Some(keep) = st.eras.divergence(st.log.tail()) else {+ return Ok(());+ };+ tracing::info!(%keep, "replica: this machine was ahead of the new owner; handing over its tail");+ st.log.cut_after(keep)?+ };+ tracing::debug!(n = tail_records.len(), first = ?tail_records.last().map(|r| r.op.key().to_string()), "replica: tail cut");+ // Newest first: rewind each head to what it was before.+ for rec in &tail_records {+ match &rec.op {+ Op::Put { meta, prev } => {+ if self+ .store+ .head(&meta.key)+ .await?+ .is_some_and(|h| h.version == meta.version)+ {+ self.store.set_head(&meta.key, prev.clone()).await?;+ }+ }+ Op::Delete { key, prev, .. } => {+ if self.store.head(key).await?.is_none() {+ self.store.set_head(key, prev.clone()).await?;+ }+ }+ }+ }+ // Oldest first: they go to the new owner as changes against what+ // each key was before.+ let mut st = self.st.lock().await;+ for rec in tail_records.iter().rev() {+ let (key, meta, prev) = match &rec.op {+ Op::Put { meta, prev } => (meta.key.clone(), Some(meta.clone()), prev.clone()),+ Op::Delete { key, prev, .. } => (key.clone(), None, prev.clone()),+ };+ let src = Src::Data {+ key: key.clone(),+ version: meta.as_ref().map(|m| m.version.clone()).unwrap_or_default(),+ };+ st.outbox.add(Pending {+ id: crate::new_id(),+ key,+ meta,+ base: prev.map(|p| p.version),+ cond: Cond::None,+ machine: self.me.clone(),+ name: self.name.clone(),+ ts_ms: self.wall_ms(),+ src,+ body: None,+ })?;+ }+ let tail = st.log.tail();+ for p in st.peers.values_mut() {+ if p.acked > tail {+ p.acked = tail;+ }+ }+ Ok(())+ }++ /// Two volumes that started apart met; this machine is the less+ /// reliable one and joins the other's: everything it has goes to the new+ /// owner as changes against nothing (identical files merge, different+ /// ones become conflict copies).+ pub(super) async fn join_lineage(self: &Arc<Self>, h: Hello) {+ let _g = self.gate.lock().await;+ let mut st = self.st.lock().await;+ if st.eras.lineage == h.lineage {+ return;+ }+ tracing::info!(other = %h.machine, "replica: joining another machine's volume");+ let heads = match self.store.list("").await {+ Ok(h) => h,+ Err(_) => return,+ };+ for meta in heads {+ if st.outbox.has_key(&meta.key) {+ continue;+ }+ let _ = st.outbox.add(Pending {+ id: crate::new_id(),+ key: meta.key.clone(),+ meta: Some(meta.clone()),+ base: None,+ cond: Cond::None,+ machine: self.me.clone(),+ name: self.name.clone(),+ ts_ms: self.wall_ms(),+ src: Src::Data {+ key: meta.key.clone(),+ version: meta.version.clone(),+ },+ body: None,+ });+ }+ st.eras = Eras {+ lineage: h.lineage.clone(),+ recs: h.eras.clone(),+ };+ let _ = save_eras(&self.dir, &st.eras);+ let _ = st.log.reset_to(Pos::ZERO);+ self.applied_tx.send_replace(Pos::ZERO);+ for p in st.peers.values_mut() {+ p.acked = Pos::ZERO;+ }+ }++ #[allow(dead_code)]+ pub(super) fn incoming_dir(&self) -> PathBuf {+ self.dir.join("incoming")+ }+}++impl Inner {+ /// Changes made to the store while replication was not looking (an older+ /// build on the same home) go into the log (this machine owns the+ /// Volume) or to the owner as changes (it does not).+ pub(super) async fn adopt_unlogged(&self) -> Result<()> {+ let unlogged = {+ let st = self.st.lock().await;+ migrate::find_unlogged(&self.dir, &self.store, &st.log).await?+ };+ if unlogged.is_empty() {+ return Ok(());+ }+ tracing::info!(+ n = unlogged.len(),+ "replica: found changes made while replication was off; handing them over"+ );+ let mut st = self.st.lock().await;+ let owner = st.eras.owner() == self.me;+ for u in unlogged {+ if owner {+ let op = match u {+ Unlogged::Put { meta, prev } => super::oplog::Op::Put { meta, prev },+ Unlogged::Delete { key, version, prev } => {+ super::oplog::Op::Delete { key, version, prev }+ }+ };+ self.commit_locked(&mut st, op)?;+ } else {+ let (key, meta, base, src) = match u {+ Unlogged::Put { meta, prev } => {+ let src = Src::Data {+ key: meta.key.clone(),+ version: meta.version.clone(),+ };+ (meta.key.clone(), Some(meta), prev.map(|p| p.version), src)+ }+ Unlogged::Delete { key, prev, .. } => {+ let src = Src::Data {+ key: key.clone(),+ version: String::new(),+ };+ (key, None, prev.map(|p| p.version), src)+ }+ };+ st.outbox.add(Pending {+ id: crate::new_id(),+ key,+ meta,+ base,+ cond: Cond::None,+ machine: self.me.clone(),+ name: self.name.clone(),+ ts_ms: self.wall_ms(),+ src,+ body: None,+ })?;+ }+ }+ Ok(())+ }++ /// Records what the heads are now (see [`migrate`]).+ pub(super) async fn save_snapshot(&self) {+ let pos = self.st.lock().await.log.tail();+ if let Err(e) = migrate::save_snap(&self.dir, &self.store, pos).await {+ tracing::warn!(error = %e, "replica: could not save the heads snapshot");+ }+ }+}diff --git a/libs/cua/crates/cua-volume/src/replica/backend.rs b/libs/cua/crates/cua-volume/src/replica/backend.rsnew file mode 100644index 0000000000..b4caae8141--- /dev/null+++ b/libs/cua/crates/cua-volume/src/replica/backend.rs@@ -0,0 +1,252 @@+// SPDX-License-Identifier: FSL-1.1-MIT+// Copyright (c) 2026 Cua AI, Inc.++//! The `Backend` the drive sees: one volume across your machines.+//!+//! Machines that hold a copy answer reads from it; a machine that does not+//! (a client) asks one that does, and the block cache sits in front of it.+//! Writes go to the owner, which orders them, logs them and ships them;+//! unconditional writes made while the owner is away are saved and sent+//! when it is back.++use std::path::Path;+use std::sync::Arc;+use std::sync::atomic::Ordering;++use super::node::Inner;+use super::owner::Body;+use super::roles::Role;+use super::status::Durability;+use super::wire::{Msg, Rep, Req};+use crate::backend::{Backend, Condition, ObjectMeta, VersionInfo};+use crate::{Error, Result};++pub struct ReplicatedBackend {+ inner: Arc<Inner>,+}++impl ReplicatedBackend {+ pub(super) fn new(inner: Arc<Inner>) -> ReplicatedBackend {+ ReplicatedBackend { inner }+ }++ /// Writes and says how many machines hold the result.+ pub async fn put_durable(+ &self,+ key: &str,+ bytes: Vec<u8>,+ cond: Condition,+ ) -> Result<(ObjectMeta, Durability)> {+ self.inner.write_put(key, Body::Bytes(bytes), cond).await+ }++ /// The machines to ask for a read, best first: the owner, then the+ /// other machines that hold a copy.+ async fn read_targets(&self) -> Vec<String> {+ let st = self.inner.st.lock().await;+ let layout = self.inner.layout_locked(&st);+ let owner = st.eras.owner().to_string();+ let mut v = vec![];+ if owner != self.inner.me {+ v.push(owner.clone());+ }+ for r in layout.replicas {+ if r != self.inner.me && r != owner {+ v.push(r);+ }+ }+ v+ }++ async fn ask(&self, req: Req, big: bool) -> Result<(Rep, Vec<u8>)> {+ let targets = self.read_targets().await;+ let mut last = Error::Backend("no machine that holds the volume can be reached".into());+ for t in targets {+ let link = if big {+ self.inner.data_link(&t).await+ } else {+ self.inner.ctl_link(&t)+ };+ let Some(link) = link else { continue };+ let to = if big {+ self.inner.cfg.data_timeout+ } else {+ self.inner.cfg.call_timeout+ };+ let r = if big {+ link.call_strict(Msg::Req(req.clone()), &[], to).await+ } else {+ link.call(Msg::Req(req.clone()), &[], to).await+ };+ match r {+ Ok((Msg::Rep(Rep::Fail { kind, message }), _)) => {+ return Err(Rep::into_error(&kind, message));+ }+ Ok((Msg::Rep(r), d)) => return Ok((r, d)),+ Ok(_) => last = Error::Backend("unexpected reply".into()),+ Err(e) => last = Error::Backend(e.to_string()),+ }+ }+ Err(last)+ }++ fn holds_copy(&self) -> bool {+ !self.inner.is_client.load(Ordering::SeqCst)+ }+}++#[async_trait::async_trait]+impl Backend for ReplicatedBackend {+ fn kind(&self) -> &'static str {+ "mesh"+ }++ async fn put(&self, key: &str, bytes: Vec<u8>, cond: Condition) -> Result<ObjectMeta> {+ Ok(self.inner.write_put(key, Body::Bytes(bytes), cond).await?.0)+ }++ async fn put_file(&self, key: &str, file: &Path, cond: Condition) -> Result<ObjectMeta> {+ Ok(self+ .inner+ .write_put(key, Body::File(file.to_path_buf()), cond)+ .await?+ .0)+ }++ async fn put_many(&self, items: Vec<(String, Vec<u8>)>) -> Result<Vec<ObjectMeta>> {+ self.inner.write_many(items).await+ }++ async fn delete(&self, key: &str, cond: Condition) -> Result<()> {+ self.inner.write_delete(key, cond).await.map(|_| ())+ }++ async fn get(&self, key: &str, version: Option<&str>) -> Result<(Vec<u8>, ObjectMeta)> {+ if self.holds_copy() {+ match self.inner.store.get(key, version).await {+ Err(Error::NotFound(_)) if version.is_some() => {}+ r => return r,+ }+ }+ match self+ .ask(+ Req::Get {+ key: key.to_string(),+ version: version.map(str::to_string),+ },+ true,+ )+ .await?+ {+ (Rep::Bytes { meta }, bytes) => Ok((bytes, meta)),+ _ => Err(Error::Backend("unexpected reply".into())),+ }+ }++ async fn head(&self, key: &str) -> Result<Option<ObjectMeta>> {+ if self.holds_copy() {+ return self.inner.store.head(key).await;+ }+ match self+ .ask(+ Req::Head {+ key: key.to_string(),+ },+ false,+ )+ .await?+ {+ (Rep::Head { meta }, _) => Ok(meta),+ _ => Err(Error::Backend("unexpected reply".into())),+ }+ }++ async fn list(&self, prefix: &str) -> Result<Vec<ObjectMeta>> {+ if self.holds_copy() {+ return self.inner.store.list(prefix).await;+ }+ match self+ .ask(+ Req::List {+ prefix: prefix.to_string(),+ },+ true,+ )+ .await?+ {+ (Rep::List { items }, _) => Ok(items),+ _ => Err(Error::Backend("unexpected reply".into())),+ }+ }++ async fn list_dir(&self, folder: &str) -> Result<(Vec<ObjectMeta>, Vec<String>)> {+ if self.holds_copy() {+ return self.inner.store.list_dir(folder).await;+ }+ match self+ .ask(+ Req::ListDir {+ folder: folder.to_string(),+ },+ false,+ )+ .await?+ {+ (Rep::ListDir { files, folders }, _) => Ok((files, folders)),+ _ => Err(Error::Backend("unexpected reply".into())),+ }+ }++ async fn versions(&self, key: &str) -> Result<Vec<VersionInfo>> {+ if self.holds_copy() {+ return self.inner.store.versions(key).await;+ }+ match self+ .ask(+ Req::Versions {+ key: key.to_string(),+ },+ false,+ )+ .await?+ {+ (Rep::Versions { items }, _) => Ok(items),+ _ => Err(Error::Backend("unexpected reply".into())),+ }+ }++ async fn get_range(&self, key: &str, version: &str, offset: u64, len: u64) -> Result<Vec<u8>> {+ if self.holds_copy() {+ match self.inner.store.get_range(key, version, offset, len).await {+ Err(Error::NotFound(_)) => {}+ r => return r,+ }+ }+ match self+ .ask(+ Req::GetRange {+ key: key.to_string(),+ version: version.to_string(),+ offset,+ len,+ },+ true,+ )+ .await?+ {+ (Rep::Range, bytes) => Ok(bytes),+ _ => Err(Error::Backend("unexpected reply".into())),+ }+ }++ fn remote(&self) -> bool {+ !self.holds_copy()+ }++ fn identity(&self) -> String {+ format!("mesh:{}", self.inner.store.root().display())+ }+}++#[allow(dead_code)]+fn _role(_: Role) {}diff --git a/libs/cua/crates/cua-volume/src/replica/detector.rs b/libs/cua/crates/cua-volume/src/replica/detector.rsnew file mode 100644index 0000000000..149dcf0eef--- /dev/null+++ b/libs/cua/crates/cua-volume/src/replica/detector.rs@@ -0,0 +1,324 @@+// SPDX-License-Identifier: FSL-1.1-MIT+// Copyright (c) 2026 Cua AI, Inc.++//! Is that machine still there? One detector per peer, fed by heartbeat+//! arrivals on the control link.+//!+//! The rule is built so a bad network never causes a takeover and a closed+//! laptop is noticed quickly:+//!+//! - *Adaptive threshold.* The silence that makes a peer suspect is+//! `max(floor, mean + 8 * deviation)` of the heartbeat gaps seen so far+//! (an exponentially weighted mean, like TCP's retransmit timer). Jitter+//! and loss widen the gaps and so widen the threshold.+//! - *Confirmation.* A suspect peer is declared dead only after the silence+//! has lasted `confirm` more. Any heartbeat before that clears the+//! suspicion.+//! - *Recovery needs a run.* A dead peer is alive again after `recover`+//! heartbeats in a row, each within the threshold of the one before, so a+//! link that flaps does not bounce ownership back and forth.+//! - *A goodbye is believed at once.* A machine going to sleep says so+//! (`Bye`); it counts as away immediately, without waiting.+//! - *Our own pauses are not theirs.* If this process was frozen (our lid+//! was closed, a SIGSTOP), every peer looks silent. [`Detector::resume`]+//! forgives a gap in our own clock so a waking laptop never declares the+//! studio dead because it slept.++use std::time::{Duration, Instant};++/// Timing knobs (all derived from the heartbeat interval by default).+#[derive(Clone, Copy, Debug)]+pub struct Timing {+ /// Heartbeat interval.+ pub interval: Duration,+ /// The least silence that raises suspicion.+ pub floor: Duration,+ /// How long suspicion must last before dead.+ pub confirm: Duration,+ /// Heartbeats in a row that revive a dead peer.+ pub recover: u32,+}++impl Timing {+ /// 2 s heartbeats, suspect after 6 s, dead 12 s later, 3 in a row to+ /// come back.+ pub fn standard() -> Timing {+ Timing {+ interval: Duration::from_secs(2),+ floor: Duration::from_secs(6),+ confirm: Duration::from_secs(12),+ recover: 3,+ }+ }+}++/// What the detector believes.+#[derive(Clone, Copy, Debug, PartialEq, Eq)]+pub enum Health {+ /// Heartbeats arrive.+ Alive,+ /// Silent longer than the threshold; not yet declared dead.+ Suspect,+ /// Silent through the confirmation window.+ Dead,+ /// Said goodbye (asleep, shutting down, network gone).+ Away,+}++impl Health {+ /// Dead or away: nothing to wait for.+ pub fn is_down(self) -> bool {+ matches!(self, Health::Dead | Health::Away)+ }+}++/// One peer's detector.+#[derive(Clone, Debug)]+pub struct Detector {+ t: Timing,+ last: Option<Instant>,+ mean_ms: f64,+ dev_ms: f64,+ state: Health,+ /// Down on the machine's own word (goodbye) or on refusals, not on silence.+ explicit: bool,+ suspect_since: Option<Instant>,+ run: u32,+ /// How many times it went from alive to dead (a flap counter for the+ /// reliability score).+ pub downs: u32,+}++impl Detector {+ pub fn new(t: Timing, now: Instant) -> Detector {+ Detector {+ t,+ last: Some(now),+ mean_ms: t.interval.as_millis() as f64,+ dev_ms: 0.0,+ state: Health::Alive,+ explicit: false,+ suspect_since: None,+ run: 0,+ downs: 0,+ }+ }++ /// The silence that makes this peer suspect now.+ pub fn threshold(&self) -> Duration {+ let adaptive = self.mean_ms + 8.0 * self.dev_ms;+ Duration::from_millis(adaptive as u64).max(self.t.floor)+ }++ /// A heartbeat (or any control message) arrived.+ pub fn arrival(&mut self, now: Instant) {+ if let Some(last) = self.last {+ let gap = now.saturating_duration_since(last).as_millis() as f64;+ // A gap far past the threshold is an outage, not a sample.+ if gap < 6.0 * self.threshold().as_millis() as f64 {+ let err = gap - self.mean_ms;+ self.mean_ms += 0.125 * err;+ self.dev_ms += 0.25 * (err.abs() - self.dev_ms);+ }+ let within = now.saturating_duration_since(last) <= self.threshold();+ match self.state {+ Health::Dead | Health::Away => {+ self.run = if within || self.run == 0 {+ self.run + 1+ } else {+ 1+ };+ if self.run >= self.t.recover {+ self.state = Health::Alive;+ self.explicit = false;+ self.suspect_since = None;+ self.run = 0;+ }+ }+ Health::Suspect => {+ self.state = Health::Alive;+ self.suspect_since = None;+ }+ Health::Alive => {}+ }+ }+ self.last = Some(now);+ }++ /// The peer said goodbye.+ pub fn bye(&mut self, now: Instant) {+ if !self.state.is_down() {+ self.downs += 1;+ }+ self.state = Health::Away;+ self.explicit = true;+ self.run = 0;+ self.last = Some(now);+ }++ /// The machine answered a dial with a refusal, or accepted and hung up+ /// before saying hello, several times in a row: someone is there to say+ /// no, and nobody is running the node. That is far better evidence than+ /// silence, so it skips the confirmation window (never the first+ /// refusal alone: the caller counts).+ pub fn refused(&mut self, now: Instant) {+ if !self.state.is_down() {+ self.downs += 1;+ }+ self.state = Health::Dead;+ self.explicit = true;+ self.run = 0;+ self.last = Some(now);+ }++ /// This process was paused: forgive the gap.+ pub fn resume(&mut self, now: Instant) {+ if matches!(self.state, Health::Alive | Health::Suspect) {+ self.state = Health::Alive;+ self.suspect_since = None;+ }+ self.last = Some(now);+ }++ /// What it believes at `now`.+ pub fn poll(&mut self, now: Instant) -> Health {+ let Some(last) = self.last else {+ return self.state;+ };+ let silence = now.saturating_duration_since(last);+ match self.state {+ Health::Alive if silence > self.threshold() => {+ self.state = Health::Suspect;+ self.suspect_since = Some(now);+ }+ Health::Suspect => {+ if let Some(since) = self.suspect_since+ && now.saturating_duration_since(since) >= self.t.confirm+ && silence > self.threshold()+ {+ self.state = Health::Dead;+ self.explicit = false;+ self.downs += 1;+ self.run = 0;+ }+ }+ _ => {}+ }+ self.state+ }++ /// How long since the peer was last heard from.+ pub fn silence(&self, now: Instant) -> Duration {+ self.last+ .map_or(Duration::MAX, |l| now.saturating_duration_since(l))+ }++ /// Down by the detector's verdict *and* still silent: a machine that+ /// has just started talking again is not down, whatever the verdict+ /// says until it has been heard from a few times (a link that comes+ /// back must not start a takeover on the strength of an old verdict).+ pub fn down_and_silent(&self, now: Instant) -> bool {+ self.state.is_down()+ && (self.explicit || self.silence(now) >= self.threshold().min(Duration::from_secs(30)))+ }++ pub fn state(&self) -> Health {+ self.state+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ fn t() -> Timing {+ Timing::standard()+ }++ fn at(base: Instant, ms: u64) -> Instant {+ base + Duration::from_millis(ms)+ }++ #[test]+ fn silence_becomes_suspect_then_dead_after_confirmation() {+ let b = Instant::now();+ let mut d = Detector::new(t(), b);+ assert_eq!(d.poll(at(b, 5_000)), Health::Alive);+ assert_eq!(d.poll(at(b, 7_000)), Health::Suspect);+ assert_eq!(d.poll(at(b, 18_000)), Health::Suspect, "confirm window");+ assert_eq!(d.poll(at(b, 19_100)), Health::Dead);+ assert_eq!(d.downs, 1);+ }++ #[test]+ fn a_late_heartbeat_clears_suspicion() {+ let b = Instant::now();+ let mut d = Detector::new(t(), b);+ assert_eq!(d.poll(at(b, 7_000)), Health::Suspect);+ d.arrival(at(b, 10_000));+ assert_eq!(d.poll(at(b, 10_100)), Health::Alive);+ assert_eq!(d.downs, 0);+ }++ #[test]+ fn jitter_widens_the_threshold_so_no_false_death() {+ // Heartbeats whose gaps swing between 1 and 9 seconds: a fixed 6 s+ // rule would suspect constantly; the adaptive rule must never+ // declare dead.+ let b = Instant::now();+ let mut d = Detector::new(t(), b);+ let mut now = 0u64;+ let gaps = [1_000, 9_000, 2_000, 8_000, 1_500, 9_500, 3_000, 7_000];+ for round in 0..40 {+ now += gaps[round % gaps.len()];+ // Poll just before each arrival (the worst moment).+ let h = d.poll(at(b, now - 1));+ assert_ne!(h, Health::Dead, "round {round}");+ d.arrival(at(b, now));+ }+ assert_eq!(d.downs, 0);+ assert!(d.threshold() > Duration::from_secs(6));+ }++ #[test]+ fn a_dead_peer_needs_a_run_of_heartbeats_to_return() {+ let b = Instant::now();+ let mut d = Detector::new(t(), b);+ d.poll(at(b, 7_000));+ d.poll(at(b, 19_500));+ assert_eq!(d.state(), Health::Dead);+ d.arrival(at(b, 30_000));+ assert_eq!(d.state(), Health::Dead);+ d.arrival(at(b, 32_000));+ assert_eq!(d.state(), Health::Dead);+ d.arrival(at(b, 34_000));+ assert_eq!(d.state(), Health::Alive);+ }++ #[test]+ fn repeated_refusals_skip_the_confirmation_window() {+ let b = Instant::now();+ let mut d = Detector::new(t(), b);+ d.refused(at(b, 1_000));+ assert_eq!(d.poll(at(b, 1_001)), Health::Dead);+ assert_eq!(d.downs, 1);+ // It comes back by the usual run of heartbeats.+ d.arrival(at(b, 3_000));+ d.arrival(at(b, 5_000));+ d.arrival(at(b, 7_000));+ assert_eq!(d.state(), Health::Alive);+ }++ #[test]+ fn a_goodbye_counts_at_once_and_resume_forgives_our_own_sleep() {+ let b = Instant::now();+ let mut d = Detector::new(t(), b);+ d.bye(at(b, 100));+ assert_eq!(d.poll(at(b, 101)), Health::Away);+ assert!(d.state().is_down());+ // We slept for an hour: nothing is suspect after waking.+ let mut e = Detector::new(t(), b);+ e.resume(at(b, 3_600_000));+ assert_eq!(e.poll(at(b, 3_600_100)), Health::Alive);+ }+}diff --git a/libs/cua/crates/cua-volume/src/replica/election.rs b/libs/cua/crates/cua-volume/src/replica/election.rsnew file mode 100644index 0000000000..05a7d92962--- /dev/null+++ b/libs/cua/crates/cua-volume/src/replica/election.rs@@ -0,0 +1,329 @@+// SPDX-License-Identifier: FSL-1.1-MIT+// Copyright (c) 2026 Cua AI, Inc.++//! Who owns the volume: takeover when the owner is gone, hand-back when a+//! more reliable machine returns.++use std::sync::Arc;+use std::time::{Duration, Instant};++use super::detector::Health;+use super::era::{EraRec, Pos};+use super::node::{Inner, save_eras};+use super::roles::{self, Role};+use super::wire::Msg;++impl Inner {+ /// A machine holding a copy takes the volume when its owner is down.+ pub(super) async fn maybe_takeover(self: &Arc<Self>) {+ struct Plan {+ epoch: u64,+ base: Pos,+ need: usize,+ owner: String,+ }+ let plan = {+ let mut st = self.st.lock().await;+ let owner = st.eras.owner().to_string();+ if owner == self.me {+ return;+ }+ if let Some(t) = st.last_takeover_try+ && t.elapsed() < self.cfg.takeover_cooldown+ {+ return;+ }+ let layout = self.layout_locked(&st);+ // A machine that keeps no copy never takes the volume.+ if layout.role_of(&self.me) != Role::Replica {+ return;+ }+ let Some(op) = st.peers.get(&owner) else {+ return;+ };+ if !op.last_health.is_down() || !op.det.down_and_silent(Instant::now()) {+ return;+ }+ // A machine that does not need to order writes leaves a flaky+ // owner alone (its changes wait in the outbox); one that does,+ // or one whose owner announced it was leaving, or has been gone+ // a long time, takes over.+ let wanted = st.authority_wanted.is_some_and(|t| {+ t.elapsed() < self.cfg.timing.confirm * 8 + Duration::from_secs(5)+ });+ let gone_long = op+ .down_since+ .is_some_and(|t| t.elapsed() >= self.cfg.idle_takeover);+ if !(wanted || op.intentional || gone_long) {+ return;+ }+ // The best replica that is up takes it; the others wait.+ let my = self.my_facts();+ for r in &layout.replicas {+ if *r == self.me || *r == owner {+ continue;+ }+ let alive = st+ .peers+ .get(r)+ .is_some_and(|p| p.last_health == Health::Alive);+ if !alive {+ continue;+ }+ let theirs = self.member_facts(&st, r);+ if (theirs.rank, std::cmp::Reverse(r)) > (my.rank, std::cmp::Reverse(&self.me)) {+ return;+ }+ }+ let n = self.active_members(&st).len();+ let need = if n >= 3 { n / 2 + 1 } else { 1 };+ st.last_takeover_try = Some(Instant::now());+ Plan {+ epoch: st.eras.epoch() + 1,+ base: st.log.tail(),+ need,+ owner,+ }+ };+ if plan.need > 1 {+ let peers: Vec<_> = {+ let st = self.st.lock().await;+ st.peers+ .iter()+ .filter(|(k, p)| **k != plan.owner && p.last_health == Health::Alive)+ .map(|(k, _)| k.clone())+ .collect()+ };+ let mut set = tokio::task::JoinSet::new();+ for p in peers {+ if let Some(l) = self.ctl_link(&p) {+ let (epoch, base) = (plan.epoch, plan.base);+ let to = self.cfg.call_timeout.min(Duration::from_secs(5));+ set.spawn(async move {+ matches!(+ l.call(Msg::Prepare { epoch, base }, &[], to).await,+ Ok((Msg::Promise { ok: true, .. }, _))+ )+ });+ }+ }+ let mut yes = 1;+ while let Some(r) = set.join_next().await {+ if r.unwrap_or(false) {+ yes += 1;+ }+ }+ if yes < plan.need {+ tracing::info!(+ yes,+ need = plan.need,+ "replica: not enough machines agree to take over"+ );+ return;+ }+ }+ let recs = {+ let mut st = self.st.lock().await;+ if st.eras.epoch() >= plan.epoch || st.eras.owner() != plan.owner {+ return;+ }+ let base = st.log.tail();+ st.eras.adopt(EraRec {+ epoch: plan.epoch,+ owner: self.me.clone(),+ base,+ at_ms: self.wall_ms(),+ });+ let _ = save_eras(&self.dir, &st.eras);+ st.takeovers += 1;+ tracing::info!(+ epoch = plan.epoch,+ "replica: this machine took over the volume"+ );+ st.eras+ .recs+ .iter()+ .rev()+ .take(4)+ .rev()+ .cloned()+ .collect::<Vec<_>>()+ };+ self.announce(recs).await;+ self.wake.notify_one();+ }++ pub(super) async fn announce(self: &Arc<Self>, recs: Vec<EraRec>) {+ let links: Vec<_> = {+ let l = self.links.lock().expect("links");+ l.values().filter_map(|p| p.ctl.clone()).collect()+ };+ for l in links {+ let recs = recs.clone();+ let to = self.cfg.call_timeout.min(Duration::from_secs(5));+ tokio::spawn(async move {+ let _ = l.call(Msg::Era { recs }, &[], to).await;+ });+ }+ }++ pub(super) async fn handle_prepare(&self, from: &str, epoch: u64, _base: Pos) -> Msg {+ let mut st = self.st.lock().await;+ let no = |reason: &str| Msg::Promise {+ ok: false,+ reason: reason.into(),+ };+ if epoch <= st.eras.epoch() {+ return no("that epoch has passed");+ }+ let owner = st.eras.owner().to_string();+ if owner == from {+ return no("the candidate is the owner");+ }+ if owner != self.me+ && !st+ .peers+ .get(&owner)+ .is_some_and(|p| p.last_health.is_down() && p.det.down_and_silent(Instant::now()))+ {+ return no("the owner is up from here");+ }+ let now = Instant::now();+ for (k, p) in &st.peers {+ if k != from+ && let Some((e, until)) = p.promised+ && e >= epoch+ && until > now+ {+ return no("already promised another machine");+ }+ }+ let until = now + self.cfg.timing.confirm * 4;+ self.peer_state(&mut st, from).promised = Some((epoch, until));+ Msg::Promise {+ ok: true,+ reason: String::new(),+ }+ }++ pub(super) async fn handle_era(self: &Arc<Self>, _from: &str, recs: Vec<EraRec>) {+ let advanced = {+ let mut st = self.st.lock().await;+ let a = st.eras.adopt_from(&recs);+ if a {+ let _ = save_eras(&self.dir, &st.eras);+ self.refresh_roles(&mut st);+ }+ a+ };+ if advanced {+ let me = self.clone();+ tokio::spawn(async move { me.reconcile_eras().await });+ self.wake.notify_one();+ }+ }++ /// The owner gives the volume back to a clearly more reliable machine+ /// that has been up and caught up for a while.+ pub(super) async fn maybe_handoff(self: &Arc<Self>) {+ let mut st = self.st.lock().await;+ if st.eras.owner() != self.me {+ return;+ }+ if let Some(t) = st.last_takeover_try+ && t.elapsed() < self.cfg.takeover_cooldown+ {+ return;+ }+ let layout = self.layout_locked(&st);+ let target = layout.owner.clone();+ if target == self.me || target.is_empty() {+ return;+ }+ let mine = self.my_facts();+ let Some(p) = st.peers.get(&target) else {+ return;+ };+ let stable = p+ .alive_since+ .is_some_and(|t| t.elapsed() >= self.cfg.handback_after);+ let tail = st.log.tail();+ if p.last_health != Health::Alive+ || !stable+ || p.acked < tail+ || !roles::clearly_better(&p.facts, &mine)+ {+ return;+ }+ let Some(link) = self.ctl_link(&target) else {+ return;+ };+ let epoch = st.eras.epoch() + 1;+ st.last_takeover_try = Some(Instant::now());+ // Nothing is written while the offer is out (the lock is held), so+ // the base is exact.+ let r = link+ .call(+ Msg::Handoff {+ epoch,+ to: target.clone(),+ base: tail,+ },+ &[],+ self.cfg.call_timeout.min(Duration::from_secs(5)),+ )+ .await;+ if !matches!(r, Ok((Msg::HandoffReply { ok: true, .. }, _))) {+ return;+ }+ st.eras.adopt(EraRec {+ epoch,+ owner: target.clone(),+ base: tail,+ at_ms: self.wall_ms(),+ });+ let _ = save_eras(&self.dir, &st.eras);+ st.handbacks += 1;+ tracing::info!(to = %target, epoch, "replica: the volume went back to a more reliable machine");+ let recs: Vec<EraRec> = st.eras.recs.iter().rev().take(4).rev().cloned().collect();+ drop(st);+ self.announce(recs).await;+ self.wake.notify_one();+ }++ pub(super) async fn handle_handoff(+ &self,+ from: &str,+ epoch: u64,+ to: String,+ base: Pos,+ ) -> Msg {+ let mut st = self.st.lock().await;+ let no = |reason: &str| Msg::HandoffReply {+ ok: false,+ reason: reason.into(),+ };+ if to != self.me {+ return no("not for this machine");+ }+ if from != st.eras.owner() || epoch != st.eras.epoch() + 1 {+ return no("the offer is stale");+ }+ if st.log.tail() != base {+ return no("this machine is not caught up");+ }+ st.eras.adopt(EraRec {+ epoch,+ owner: self.me.clone(),+ base,+ at_ms: self.wall_ms(),+ });+ let _ = save_eras(&self.dir, &st.eras);+ st.handbacks += 1;+ self.wake.notify_one();+ Msg::HandoffReply {+ ok: true,+ reason: String::new(),+ }+ }+}diff --git a/libs/cua/crates/cua-volume/src/replica/era.rs b/libs/cua/crates/cua-volume/src/replica/era.rsnew file mode 100644index 0000000000..2cda2a55bf--- /dev/null+++ b/libs/cua/crates/cua-volume/src/replica/era.rs@@ -0,0 +1,162 @@+// SPDX-License-Identifier: FSL-1.1-MIT+// Copyright (c) 2026 Cua AI, Inc.++//! Positions in the replication log and the history of who owned it.+//!+//! An *era* is a stretch of the log with one owner. A takeover or a+//! hand-back starts a new era at `epoch + 1` and records its `base`: the+//! newest position the new owner had applied. Whatever another machine holds+//! past a later era's base was written by an owner that era did not see (the+//! divergent tail): it is rebased, never replayed.++use serde::{Deserialize, Serialize};++use super::MachineId;++/// A position in the log: records are ordered by `(epoch, seq)`.+#[derive(+ Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize,+)]+pub struct Pos {+ pub epoch: u64,+ pub seq: u64,+}++impl Pos {+ pub const ZERO: Pos = Pos { epoch: 0, seq: 0 };+}++impl std::fmt::Display for Pos {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(f, "{}.{}", self.epoch, self.seq)+ }+}++/// One era.+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]+pub struct EraRec {+ pub epoch: u64,+ pub owner: MachineId,+ /// What the owner had applied when it took the era (zero for the first).+ pub base: Pos,+ /// Wall time it started (display only).+ pub at_ms: u64,+}++/// The eras a machine knows, oldest first, of one lineage (the id of the+/// volume's origin: two machines that each started a volume alone have+/// different lineages until one joins the other).+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]+pub struct Eras {+ pub lineage: String,+ pub recs: Vec<EraRec>,+}++impl Eras {+ /// A volume that starts here, owned by `owner`.+ pub fn genesis(owner: &str, now_ms: u64) -> Eras {+ Eras {+ lineage: crate::new_id(),+ recs: vec![EraRec {+ epoch: 1,+ owner: owner.to_string(),+ base: Pos::ZERO,+ at_ms: now_ms,+ }],+ }+ }++ pub fn current(&self) -> &EraRec {+ self.recs.last().expect("an era always exists")+ }++ pub fn epoch(&self) -> u64 {+ self.current().epoch+ }++ pub fn owner(&self) -> &str {+ &self.current().owner+ }++ /// Adds `rec` when it is newer. Returns whether it was added.+ pub fn adopt(&mut self, rec: EraRec) -> bool {+ if rec.epoch <= self.epoch() {+ return false;+ }+ self.recs.push(rec);+ true+ }++ /// Adopts the history `other` carries when it extends ours (same+ /// lineage): every era newer than ours.+ pub fn adopt_from(&mut self, other: &[EraRec]) -> bool {+ let mut any = false;+ for r in other {+ if self.adopt(r.clone()) {+ any = true;+ }+ }+ any+ }++ /// Where a machine whose newest applied record is `p` stops agreeing+ /// with this history: `Some(keep)` when it holds records past the base+ /// of an era that came after it (those are its divergent tail; it keeps+ /// everything up to `keep`).+ pub fn divergence(&self, p: Pos) -> Option<Pos> {+ self.recs+ .iter()+ .filter(|r| r.epoch > p.epoch && r.epoch > 1)+ .filter(|r| p > r.base)+ .map(|r| r.base)+ .min()+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ fn rec(epoch: u64, owner: &str, base: (u64, u64)) -> EraRec {+ EraRec {+ epoch,+ owner: owner.into(),+ base: Pos {+ epoch: base.0,+ seq: base.1,+ },+ at_ms: 0,+ }+ }++ #[test]+ fn a_machine_ahead_of_a_later_eras_base_diverges() {+ let mut e = Eras::genesis("laptop", 0);+ assert_eq!(e.divergence(Pos { epoch: 1, seq: 50 }), None);+ // The studio took over having applied up to 1.40.+ assert!(e.adopt(rec(2, "studio", (1, 40))));+ assert!(!e.adopt(rec(2, "other", (1, 1))), "an old epoch is ignored");+ // The laptop applied 1.50 (ten records the studio never saw).+ assert_eq!(+ e.divergence(Pos { epoch: 1, seq: 50 }),+ Some(Pos { epoch: 1, seq: 40 })+ );+ // One that stopped at 1.40 or earlier agrees.+ assert_eq!(e.divergence(Pos { epoch: 1, seq: 40 }), None);+ assert_eq!(e.divergence(Pos { epoch: 1, seq: 7 }), None);+ // A machine already in the new era never diverges.+ assert_eq!(e.divergence(Pos { epoch: 2, seq: 9 }), None);+ }++ #[test]+ fn the_earliest_base_wins_across_several_eras() {+ let mut e = Eras::genesis("a", 0);+ e.adopt(rec(2, "b", (1, 40)));+ e.adopt(rec(3, "c", (1, 30)));+ assert_eq!(+ e.divergence(Pos { epoch: 1, seq: 45 }),+ Some(Pos { epoch: 1, seq: 30 })+ );+ assert_eq!(e.divergence(Pos { epoch: 1, seq: 30 }), None);+ }+}diff --git a/libs/cua/crates/cua-volume/src/replica/link.rs b/libs/cua/crates/cua-volume/src/replica/link.rsnew file mode 100644index 0000000000..6797597a1a--- /dev/null+++ b/libs/cua/crates/cua-volume/src/replica/link.rs@@ -0,0 +1,361 @@+// SPDX-License-Identifier: FSL-1.1-MIT+// Copyright (c) 2026 Cua AI, Inc.++//! One framed connection to a peer, with request/reply correlation.+//!+//! A [`Link`] owns a writer task and a reader task over any duplex stream+//! (a mesh stream in the product, TCP in tests, an in-memory pipe in+//! simulations). Messages that answer a request (`re` set) go to the waiting+//! [`Link::call`]; everything else goes to the handler, each on its own task+//! so a slow handler never blocks the reader (and so a heartbeat is always+//! read).++use std::collections::HashMap;+use std::future::Future;+use std::pin::Pin;+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};+use std::sync::{Arc, Mutex};+use std::time::Duration;++use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};+use tokio::sync::{Notify, mpsc, oneshot};++use super::MachineId;+use super::wire::{self, Chan, Frame, Msg};++/// Any duplex byte stream.+pub trait Duplex: AsyncRead + AsyncWrite + Send + Unpin + 'static {}+impl<T: AsyncRead + AsyncWrite + Send + Unpin + 'static> Duplex for T {}++/// A boxed stream.+pub type Stream = Box<dyn Duplex>;++/// Bytes crossing links (frames, as sent and received).+#[derive(Debug, Default)]+pub struct Counters {+ pub sent: AtomicU64,+ pub received: AtomicU64,+}++/// What a handler receives.+pub type Handler = Arc<+ dyn Fn(Arc<Link>, Frame, Vec<u8>) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync,+>;++/// What the writer task is asked to do.+enum Out {+ Bytes(Vec<u8>),+ /// Fires once everything queued before it has been written.+ Flush(oneshot::Sender<()>),+}++type Waiter = oneshot::Sender<(Frame, Vec<u8>)>;++/// One connection.+pub struct Link {+ pub peer: MachineId,+ pub peer_name: String,+ pub chan: Chan,+ /// This side dialed.+ pub initiated: bool,+ /// When the connection came up.+ pub created: std::time::Instant,+ out: mpsc::UnboundedSender<Out>,+ calls: Mutex<HashMap<u64, Waiter>>,+ next: AtomicU64,+ closed: AtomicBool,+ closed_notify: Notify,+ counters: Arc<Counters>,+ /// Peer facts from its Hello (the node reads them).+ pub hello: Mutex<Option<wire::Hello>>,+ tasks: Mutex<Vec<tokio::task::AbortHandle>>,+}++impl std::fmt::Debug for Link {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ f.debug_struct("Link")+ .field("peer", &self.peer)+ .field("chan", &self.chan)+ .field("closed", &self.is_closed())+ .finish()+ }+}++/// Why a call failed.+#[derive(Debug)]+pub enum CallError {+ Closed,+ Timeout,+ Io(std::io::Error),+}++impl std::fmt::Display for CallError {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ match self {+ CallError::Closed => f.write_str("the link to the machine closed"),+ CallError::Timeout => f.write_str("the machine did not answer in time"),+ CallError::Io(e) => write!(f, "{e}"),+ }+ }+}++impl Link {+ /// Starts the reader and writer tasks over `stream`.+ #[allow(clippy::too_many_arguments)]+ pub fn spawn(+ stream: Stream,+ peer: MachineId,+ peer_name: String,+ chan: Chan,+ initiated: bool,+ counters: Arc<Counters>,+ handler: Handler,+ on_close: impl FnOnce(Arc<Link>) + Send + 'static,+ ) -> Arc<Link> {+ let (mut rd, mut wr) = tokio::io::split(stream);+ let (tx, mut rx) = mpsc::unbounded_channel::<Out>();+ let link = Arc::new(Link {+ peer,+ peer_name,+ chan,+ initiated,+ created: std::time::Instant::now(),+ out: tx,+ calls: Mutex::new(HashMap::new()),+ next: AtomicU64::new(1),+ closed: AtomicBool::new(false),+ closed_notify: Notify::new(),+ counters: counters.clone(),+ hello: Mutex::new(None),+ tasks: Mutex::new(vec![]),+ });+ let l = link.clone();+ let c = counters.clone();+ let wtask = tokio::spawn(async move {+ while let Some(out) = rx.recv().await {+ match out {+ Out::Bytes(bytes) => {+ c.sent.fetch_add(bytes.len() as u64, Ordering::Relaxed);+ if wr.write_all(&bytes).await.is_err() || wr.flush().await.is_err() {+ break;+ }+ }+ Out::Flush(done) => {+ let _ = done.send(());+ }+ }+ }+ let _ = wr.shutdown().await;+ l.close();+ });+ let l = link.clone();+ let rtask = tokio::spawn(async move {+ #[allow(clippy::while_let_loop)]+ loop {+ match wire::read(&mut rd).await {+ Ok(Some((frame, data))) => {+ l.counters+ .received+ .fetch_add((data.len() + 64) as u64, Ordering::Relaxed);+ if let Some(re) = frame.re {+ let waiter = l.calls.lock().expect("calls").remove(&re);+ if let Some(w) = waiter {+ let _ = w.send((frame, data));+ }+ } else {+ let h = handler.clone();+ let l2 = l.clone();+ tokio::spawn(async move { h(l2, frame, data).await });+ }+ }+ Ok(None) | Err(_) => break,+ }+ }+ l.close();+ on_close(l);+ });+ link.tasks+ .lock()+ .expect("tasks")+ .extend([wtask.abort_handle(), rtask.abort_handle()]);+ link+ }++ /// Waits (briefly) until what was sent has been handed to the stream.+ pub async fn flush(&self, within: Duration) {+ let (tx, rx) = oneshot::channel();+ if self.out.send(Out::Flush(tx)).is_ok() {+ let _ = tokio::time::timeout(within, rx).await;+ }+ }++ /// Marks the link closed and fails every waiting call.+ pub fn close(&self) {+ if !self.closed.swap(true, Ordering::SeqCst) {+ self.calls.lock().expect("calls").clear();+ self.closed_notify.notify_waiters();+ // A dead connection must not keep its tasks (and so its socket)+ // alive: the peer sees the end of the stream and redials.+ for t in self.tasks.lock().expect("tasks").drain(..) {+ t.abort();+ }+ }+ }++ pub fn is_closed(&self) -> bool {+ self.closed.load(Ordering::SeqCst)+ }++ /// Resolves when the link closes.+ pub async fn closed(&self) {+ loop {+ let n = self.closed_notify.notified();+ if self.is_closed() {+ return;+ }+ n.await;+ }+ }++ /// Sends a message with no reply expected.+ pub fn send(&self, msg: Msg, data: &[u8]) -> bool {+ self.send_frame(+ Frame {+ id: self.next.fetch_add(1, Ordering::Relaxed),+ re: None,+ msg,+ },+ data,+ )+ }++ /// Answers request `id`.+ pub fn reply(&self, id: u64, msg: Msg, data: &[u8]) -> bool {+ self.send_frame(+ Frame {+ id: self.next.fetch_add(1, Ordering::Relaxed),+ re: Some(id),+ msg,+ },+ data,+ )+ }++ fn send_frame(&self, f: Frame, data: &[u8]) -> bool {+ if self.is_closed() {+ return false;+ }+ match wire::encode(&f, data) {+ Ok(b) => self.out.send(Out::Bytes(b)).is_ok(),+ Err(_) => false,+ }+ }++ /// Like [`Link::call`], and a timeout closes the link: a bulk request+ /// that gets no answer means the path is dead (a half-open connection+ /// looks exactly like a slow one until something gives up on it), and the+ /// next attempt must dial afresh.+ pub async fn call_strict(+ &self,+ msg: Msg,+ data: &[u8],+ timeout: Duration,+ ) -> Result<(Msg, Vec<u8>), CallError> {+ let r = self.call(msg, data, timeout).await;+ if matches!(r, Err(CallError::Timeout)) {+ self.close();+ }+ r+ }++ /// Sends a request and waits for its reply.+ pub async fn call(+ &self,+ msg: Msg,+ data: &[u8],+ timeout: Duration,+ ) -> Result<(Msg, Vec<u8>), CallError> {+ if self.is_closed() {+ return Err(CallError::Closed);+ }+ let id = self.next.fetch_add(1, Ordering::Relaxed);+ let (tx, rx) = oneshot::channel();+ self.calls.lock().expect("calls").insert(id, tx);+ if !self.send_frame(Frame { id, re: None, msg }, data) {+ self.calls.lock().expect("calls").remove(&id);+ return Err(CallError::Closed);+ }+ match tokio::time::timeout(timeout, rx).await {+ Ok(Ok((f, d))) => Ok((f.msg, d)),+ Ok(Err(_)) => Err(CallError::Closed),+ Err(_) => {+ self.calls.lock().expect("calls").remove(&id);+ Err(CallError::Timeout)+ }+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::replica::wire::Rep;++ fn echo() -> Handler {+ Arc::new(|link, frame, data| {+ Box::pin(async move {+ if let Msg::Req(_) = frame.msg {+ link.reply(frame.id, Msg::Rep(Rep::Range), &data);+ }+ })+ })+ }++ #[tokio::test]+ async fn calls_get_their_replies_and_a_closed_link_fails_them() {+ let (a, b) = tokio::io::duplex(1 << 16);+ let ca = Arc::new(Counters::default());+ let la = Link::spawn(+ Box::new(a),+ "b".into(),+ "B".into(),+ Chan::Control,+ true,+ ca.clone(),+ echo(),+ |_| {},+ );+ let lb = Link::spawn(+ Box::new(b),+ "a".into(),+ "A".into(),+ Chan::Control,+ false,+ Arc::new(Counters::default()),+ echo(),+ |_| {},+ );+ let (m, d) = la+ .call(+ Msg::Req(wire::Req::Head { key: "k".into() }),+ b"hi",+ Duration::from_secs(2),+ )+ .await+ .unwrap();+ assert_eq!(m, Msg::Rep(Rep::Range));+ assert_eq!(d, b"hi");+ assert!(ca.sent.load(Ordering::Relaxed) > 0);+ lb.close();+ drop(lb);+ // The other side's writer ends when its handle is gone; the call fails.+ let r = la+ .call(+ Msg::Req(wire::Req::Head { key: "k".into() }),+ b"",+ Duration::from_millis(300),+ )+ .await;+ assert!(r.is_err() || la.is_closed());+ }+}diff --git a/libs/cua/crates/cua-volume/src/replica/merge.rs b/libs/cua/crates/cua-volume/src/replica/merge.rsnew file mode 100644index 0000000000..68eea7c36c--- /dev/null+++ b/libs/cua/crates/cua-volume/src/replica/merge.rs@@ -0,0 +1,160 @@+// SPDX-License-Identifier: FSL-1.1-MIT+// Copyright (c) 2026 Cua AI, Inc.++//! What to do when two machines changed one key.+//!+//! Almost always: keep the owner's version at the key and store the other+//! machine's content next to it as `name (conflict from <machine> <time>).ext`+//! (the same naming the sync feed uses). Nothing is overwritten and nothing+//! is lost. Control objects whose meaning is an order merge by that meaning+//! instead: the session fence is a monotonic epoch, so the higher epoch+//! wins and a stale driver is fenced the moment the machines meet again.++use crate::feed::conflict_path;+use crate::journal::FENCE;++/// How a conflict on `key` resolves.+#[derive(Debug, PartialEq, Eq)]+pub enum Resolution {+ /// Keep the head already at the key, drop the incoming content (it is+ /// identical or lower).+ KeepHead,+ /// Replace the head with the incoming content (it is higher).+ TakeIncoming,+ /// Keep the head; store the incoming content at this key.+ Copy(String),+}++/// A session fence key (`.../sessions/<session>/fence`).+fn is_fence(key: &str) -> bool {+ key.rsplit('/').next() == Some(FENCE) && key.contains("/sessions/")+}++/// A durable run's supervisor lease (`agents/<agent>/.cua-supervisor`).+fn is_supervisor(key: &str) -> bool {+ key.rsplit('/').next() == Some(crate::replica::supervision::LEASE)+}++/// A run's record (`.cua-runs/<agent>.json`).+fn is_run_record(key: &str) -> bool {+ key.starts_with(crate::replica::supervision::RUNS_DIR)+}++/// Keys whose value is an order (fence, supervisor lease, run record): a+/// change to one is always judged against the head by that order, even when+/// it was written against the head's own version, because a stale writer's+/// later changes chain onto its earlier ones.+pub fn is_ordered(key: &str) -> bool {+ is_fence(key) || is_supervisor(key) || is_run_record(key)+}++fn json_field(bytes: &[u8], field: &str) -> Option<serde_json::Value> {+ serde_json::from_slice::<serde_json::Value>(bytes)+ .ok()?+ .get(field)+ .cloned()+}++fn fence_epoch(bytes: &[u8]) -> Option<u64> {+ serde_json::from_slice::<serde_json::Value>(bytes)+ .ok()?+ .get("epoch")?+ .as_u64()+}++/// Resolves a clash on `key` between the head's bytes and the incoming+/// bytes from `machine` at `ts_ms`.+pub fn resolve(key: &str, head: &[u8], incoming: &[u8], machine: &str, ts_ms: u64) -> Resolution {+ if head == incoming {+ return Resolution::KeepHead;+ }+ if is_fence(key)+ && let (Some(h), Some(i)) = (fence_epoch(head), fence_epoch(incoming))+ {+ return if i > h {+ Resolution::TakeIncoming+ } else {+ Resolution::KeepHead+ };+ }+ // The supervisor lease is an order: the higher epoch holds the run, and+ // within an epoch the later beat.+ if is_supervisor(key) {+ let order = |b: &[u8]| {+ (+ json_field(b, "epoch").and_then(|v| v.as_u64()).unwrap_or(0),+ json_field(b, "beat").and_then(|v| v.as_u64()).unwrap_or(0),+ )+ };+ return if order(incoming) > order(head) {+ Resolution::TakeIncoming+ } else {+ Resolution::KeepHead+ };+ }+ // A finished run stays finished.+ if is_run_record(key) {+ let done = |b: &[u8]| {+ json_field(b, "state")+ .and_then(|v| v.as_str().map(|s| s == "done"))+ .unwrap_or(false)+ };+ return if done(incoming) && !done(head) {+ Resolution::TakeIncoming+ } else {+ Resolution::KeepHead+ };+ }+ Resolution::Copy(conflict_path(key, machine, ts_ms))+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn plain_files_become_conflict_copies() {+ let r = resolve("agents/ada/notes.md", b"a", b"b", "MacBook", 0);+ match r {+ Resolution::Copy(k) => {+ assert!(+ k.starts_with("agents/ada/notes (conflict from MacBook "),+ "{k}"+ );+ assert!(k.ends_with(").md"));+ }+ other => panic!("{other:?}"),+ }+ assert_eq!(+ resolve("agents/ada/notes.md", b"same", b"same", "m", 0),+ Resolution::KeepHead+ );+ }++ #[test]+ fn a_supervisor_lease_goes_to_the_higher_epoch_and_a_done_run_stays_done() {+ let k = "agents/ada/.cua-supervisor";+ let a = br#"{"epoch":2,"beat":9}"#;+ let b = br#"{"epoch":3,"beat":0}"#;+ assert_eq!(resolve(k, a, b, "m", 0), Resolution::TakeIncoming);+ assert_eq!(resolve(k, b, a, "m", 0), Resolution::KeepHead);+ let r = ".cua-runs/ada.json";+ assert_eq!(+ resolve(r, br#"{"state":"running"}"#, br#"{"state":"done"}"#, "m", 0),+ Resolution::TakeIncoming+ );+ assert_eq!(+ resolve(r, br#"{"state":"done"}"#, br#"{"state":"running"}"#, "m", 0),+ Resolution::KeepHead+ );+ }++ #[test]+ fn a_fence_merges_to_the_higher_epoch() {+ let k = "spaces/dev/sessions/s1/fence";+ let low = br#"{"epoch":2,"holder":"a"}"#;+ let high = br#"{"epoch":5,"holder":"b"}"#;+ assert_eq!(resolve(k, low, high, "m", 0), Resolution::TakeIncoming);+ assert_eq!(resolve(k, high, low, "m", 0), Resolution::KeepHead);+ }+}diff --git a/libs/cua/crates/cua-volume/src/replica/migrate.rs b/libs/cua/crates/cua-volume/src/replica/migrate.rsnew file mode 100644index 0000000000..b994ef07e3--- /dev/null+++ b/libs/cua/crates/cua-volume/src/replica/migrate.rs@@ -0,0 +1,371 @@+// SPDX-License-Identifier: FSL-1.1-MIT+// Copyright (c) 2026 Cua AI, Inc.++//! Turning replication on for a Volume that already exists, and living with+//! builds that do not know about it.+//!+//! Nothing about the Volume's own files changes: `data/` keeps its layout+//! (versions and heads, written the way every release has), so a build that+//! predates replication reads it exactly as before. Everything replication+//! adds is new and lives under `replica/` next to it:+//!+//! - `replica/format.json`: the format version of that directory. A build+//! that finds a newer one than it understands refuses with a plain+//! message instead of guessing (never corruption).+//! - `replica/backup/heads-<time>/`: a copy of the heads taken the first time+//! replication is enabled on a Volume that has files, before anything+//! moves. Versions are never deleted, so the heads are all a restore needs.+//! - `replica/heads.snap`: what the heads were at the last clean stop. A+//! build that does not know about replication may write to `data/` while+//! replication is off for it (an older Cua Spaces on the same home); on the+//! next start the differences are found and handed to the log (or, on a+//! machine that does not own the Volume, to the owner as changes), so they+//! reach the other machines instead of silently diverging.+//!+//! Replication starts only when this account's machines list has another+//! machine and no bucket is configured ([`decide`]).++use std::collections::BTreeMap;+use std::path::{Path, PathBuf};++use serde::{Deserialize, Serialize};++use super::era::Pos;+use super::oplog::{Op, OpLog};+use crate::backend::{Backend, ObjectMeta};+use crate::fs::FsBackend;+use crate::{Error, Result};++/// The format of the `replica/` directory this build writes and reads.+pub const FORMAT: u32 = 1;++#[derive(Debug, Serialize, Deserialize)]+struct Format {+ format: u32,+ created_ms: u64,+}++/// Reads or creates `replica/format.json`. A directory written by a newer+/// format is refused.+pub fn check_format(dir: &Path) -> Result<()> {+ let p = dir.join("format.json");+ match std::fs::read(&p) {+ Ok(b) => {+ let f: Format = serde_json::from_slice(&b)+ .map_err(|e| Error::Invalid(format!("replica/format.json: {e}")))?;+ if f.format > FORMAT {+ return Err(Error::Invalid(format!(+ "this Volume was prepared by a newer Cua Spaces (format {}, this one reads up to {FORMAT}); update Cua Spaces on this machine",+ f.format+ )));+ }+ Ok(())+ }+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => {+ std::fs::create_dir_all(dir)?;+ cua_home::guard_write(&p)?;+ std::fs::write(+ &p,+ serde_json::to_vec(&Format {+ format: FORMAT,+ created_ms: crate::now_ms(),+ })?,+ )?;+ Ok(())+ }+ Err(e) => Err(e.into()),+ }+}++fn copy_tree(from: &Path, to: &Path) -> std::io::Result<u64> {+ let mut n = 0;+ std::fs::create_dir_all(to)?;+ for e in std::fs::read_dir(from)? {+ let e = e?;+ let (src, dst) = (e.path(), to.join(e.file_name()));+ if e.file_type()?.is_dir() {+ n += copy_tree(&src, &dst)?;+ } else {+ std::fs::copy(&src, &dst)?;+ n += 1;+ }+ }+ Ok(n)+}++/// Copies the store's heads to `dir/backup/heads-<ms>/` (when it has any).+/// Returns where, and how many head files.+pub fn backup_heads(dir: &Path, store_root: &Path) -> Result<Option<(PathBuf, u64)>> {+ let heads = store_root.join("heads");+ if !heads.is_dir() {+ return Ok(None);+ }+ let to = dir+ .join("backup")+ .join(format!("heads-{}", crate::now_ms()));+ cua_home::guard_write(&to.join("x"))?;+ let n = copy_tree(&heads, &to)?;+ Ok(Some((to, n)))+}++/// What the heads were.+#[derive(Debug, Default, Serialize, Deserialize)]+pub struct HeadsSnap {+ pub pos: Pos,+ pub heads: BTreeMap<String, ObjectMeta>,+}++fn snap_path(dir: &Path) -> PathBuf {+ dir.join("heads.snap")+}++/// Saves the heads as of `pos`.+pub async fn save_snap(dir: &Path, store: &FsBackend, pos: Pos) -> Result<()> {+ let heads: BTreeMap<String, ObjectMeta> = store+ .list("")+ .await?+ .into_iter()+ .map(|m| (m.key.clone(), m))+ .collect();+ let p = snap_path(dir);+ cua_home::guard_write(&p)?;+ let tmp = p.with_extension("tmp");+ std::fs::write(&tmp, serde_json::to_vec(&HeadsSnap { pos, heads })?)?;+ std::fs::rename(&tmp, &p)?;+ Ok(())+}++/// One change made behind replication's back.+#[derive(Debug, PartialEq, Eq)]+pub enum Unlogged {+ /// `meta` is the head now; `prev` was the head at the snapshot.+ Put {+ meta: ObjectMeta,+ prev: Option<ObjectMeta>,+ },+ /// `key` was deleted (a marker with `version`); `prev` was the head.+ Delete {+ key: String,+ version: String,+ prev: Option<ObjectMeta>,+ },+}++/// The changes to `store` since the snapshot that its log does not account+/// for. Empty when there is no snapshot (the Volume predates replication or+/// was never stopped cleanly) or the log no longer reaches back to it.+pub async fn find_unlogged(dir: &Path, store: &FsBackend, log: &OpLog) -> Result<Vec<Unlogged>> {+ let snap = std::fs::read(snap_path(dir))+ .ok()+ .and_then(|b| serde_json::from_slice::<HeadsSnap>(&b).ok());+ let Some(snap) = snap else {+ return find_unlogged_by_time(store, log).await;+ };+ let Some(after) = log.after(snap.pos, usize::MAX) else {+ // The log was trimmed past the snapshot: expectations cannot be rebuilt.+ return Ok(vec![]);+ };+ // What the heads should be: the snapshot plus what the log did since.+ let mut expect = snap.heads;+ for r in after {+ match r.op {+ Op::Put { meta, .. } => {+ expect.insert(meta.key.clone(), meta);+ }+ Op::Delete { key, .. } => {+ expect.remove(&key);+ }+ }+ }+ let actual: BTreeMap<String, ObjectMeta> = store+ .list("")+ .await?+ .into_iter()+ .map(|m| (m.key.clone(), m))+ .collect();+ let mut out = vec![];+ for (k, a) in &actual {+ match expect.get(k) {+ Some(e) if e.version == a.version => {}+ other => out.push(Unlogged::Put {+ meta: a.clone(),+ prev: other.cloned(),+ }),+ }+ }+ for (k, e) in &expect {+ if !actual.contains_key(k) {+ let version = store+ .versions(k)+ .await+ .ok()+ .and_then(|v| v.into_iter().find(|v| v.deleted).map(|v| v.version))+ .unwrap_or_default();+ if !version.is_empty() {+ out.push(Unlogged::Delete {+ key: k.clone(),+ version,+ prev: Some(e.clone()),+ });+ }+ }+ }+ Ok(out)+}++/// Without a snapshot (the machine was stopped hard, or never stopped+/// cleanly): a head written after the newest log record, by anyone but the+/// log, is a change made behind its back. Only for stores that have a log.+async fn find_unlogged_by_time(store: &FsBackend, log: &OpLog) -> Result<Vec<Unlogged>> {+ let Some(newest) = log.newest_at_ms() else {+ return Ok(vec![]);+ };+ let known: std::collections::HashSet<(String, String)> = log+ .range(Pos::ZERO, log.tail())+ .into_iter()+ .filter_map(|r| match r.op {+ Op::Put { meta, .. } => Some((meta.key, meta.version)),+ _ => None,+ })+ .collect();+ let mut out = vec![];+ for m in store.list("").await? {+ if m.modified_ms > newest + 1000 && !known.contains(&(m.key.clone(), m.version.clone())) {+ out.push(Unlogged::Put {+ meta: m,+ prev: None,+ });+ }+ }+ Ok(out)+}++/// Whether replication should run at all.+#[derive(Debug, PartialEq, Eq)]+pub enum Decision {+ On,+ /// Off, and why (for a log line and a status).+ Off(&'static str),+}++/// Replication starts only when the user has signed in (a machines list+/// exists), has another machine on it, and has not pointed the Volume at a+/// bucket. Anything else leaves the Volume exactly as an earlier release+/// served it.+pub fn decide(bucket_configured: bool, signed_in: bool, other_machines: usize) -> Decision {+ if bucket_configured {+ Decision::Off("the Volume is on your own bucket")+ } else if !signed_in {+ Decision::Off("not signed in")+ } else if other_machines == 0 {+ Decision::Off("this is the only machine")+ } else {+ Decision::On+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn a_newer_format_is_refused_and_an_older_or_missing_one_is_fine() {+ let d = tempfile::tempdir().unwrap();+ let dir = d.path().join("replica");+ check_format(&dir).unwrap();+ check_format(&dir).unwrap();+ std::fs::write(dir.join("format.json"), br#"{"format":99,"created_ms":1}"#).unwrap();+ let e = check_format(&dir).unwrap_err();+ assert!(e.to_string().contains("newer Cua Spaces"), "{e}");+ std::fs::write(dir.join("format.json"), br#"{"format":0,"created_ms":1}"#).unwrap();+ check_format(&dir).unwrap();+ }++ #[test]+ fn replication_starts_only_after_signing_in_with_another_machine_and_no_bucket() {+ assert_eq!(+ decide(true, true, 3),+ Decision::Off("the Volume is on your own bucket")+ );+ assert_eq!(decide(false, false, 3), Decision::Off("not signed in"));+ assert_eq!(+ decide(false, true, 0),+ Decision::Off("this is the only machine")+ );+ assert_eq!(decide(false, true, 1), Decision::On);+ }++ #[tokio::test]+ async fn without_a_snapshot_a_head_newer_than_the_log_is_found() {+ use crate::backend::Condition;+ use crate::replica::oplog::Record;+ let d = tempfile::tempdir().unwrap();+ let store = FsBackend::new(d.path().join("data"));+ let state = d.path().join("replica");+ let mut log = OpLog::open(&state, 100).unwrap();+ let m = store+ .put("a/old", b"1".to_vec(), Condition::None)+ .await+ .unwrap();+ log.append(Record {+ pos: Pos { epoch: 1, seq: 1 },+ op: Op::Put {+ meta: m.clone(),+ prev: None,+ },+ at_ms: m.modified_ms + 1,+ })+ .unwrap();+ // The log knows a/old; a head made well after the last record is not its doing.+ tokio::time::sleep(std::time::Duration::from_millis(1100)).await;+ store+ .put("a/new", b"2".to_vec(), Condition::None)+ .await+ .unwrap();+ let found = find_unlogged(&state, &store, &log).await.unwrap();+ assert_eq!(found.len(), 1, "{found:?}");+ assert!(matches!(&found[0], Unlogged::Put { meta, .. } if meta.key == "a/new"));+ }++ #[tokio::test]+ async fn heads_are_backed_up_and_a_backdoor_write_is_found() {+ use crate::backend::Condition;+ let d = tempfile::tempdir().unwrap();+ let store = FsBackend::new(d.path().join("data"));+ store+ .put("a/one", b"1".to_vec(), Condition::None)+ .await+ .unwrap();+ store+ .put("a/two", b"2".to_vec(), Condition::None)+ .await+ .unwrap();+ let state = d.path().join("replica");+ let (to, n) = backup_heads(&state, store.root()).unwrap().unwrap();+ assert_eq!(n, 2);+ assert!(to.join("a.d").is_dir());+ let log = OpLog::open(&state, 100).unwrap();+ save_snap(&state, &store, log.tail()).await.unwrap();+ // An older build writes straight to the store.+ store+ .put("a/one", b"1 changed".to_vec(), Condition::None)+ .await+ .unwrap();+ store+ .put("a/three", b"3".to_vec(), Condition::None)+ .await+ .unwrap();+ store.delete("a/two", Condition::None).await.unwrap();+ let mut found = find_unlogged(&state, &store, &log).await.unwrap();+ found.sort_by_key(|u| match u {+ Unlogged::Put { meta, .. } => meta.key.clone(),+ Unlogged::Delete { key, .. } => key.clone(),+ });+ assert_eq!(found.len(), 3, "{found:?}");+ assert!(+ matches!(&found[0], Unlogged::Put { meta, prev: Some(p) } if meta.key == "a/one" && p.size == 1)+ );+ assert!(matches!(&found[1], Unlogged::Put { meta, prev: None } if meta.key == "a/three"));+ assert!(matches!(&found[2], Unlogged::Delete { key, prev: Some(_), .. } if key == "a/two"));+ }+}diff --git a/libs/cua/crates/cua-volume/src/replica/mod.rs b/libs/cua/crates/cua-volume/src/replica/mod.rsnew file mode 100644index 0000000000..6ee8c4c2c5--- /dev/null+++ b/libs/cua/crates/cua-volume/src/replica/mod.rs@@ -0,0 +1,58 @@+// SPDX-License-Identifier: FSL-1.1-MIT+// Copyright (c) 2026 Cua AI, Inc.++//! A Volume that lives on your machines: no bucket, no server.+//!+//! With no bucket configured and two or more machines signed in, a durable+//! agent's Volume is kept on the most reliable machine (the owner) and one+//! more (a replica), so closing the laptop loses nothing and the agent keeps+//! its files. See `DESIGN.md` of the just-works-machines work for the+//! reasoning; this is the map.+//!+//! | Module | What it owns |+//! |---|---|+//! | [`era`] | log positions and the history of owners (epochs, bases, divergence) |+//! | [`oplog`] | the replication log (what the owner did, in order) |+//! | [`detector`] | is that machine still there: adaptive threshold, hysteresis, goodbyes |+//! | [`roles`] | machines, reliability facts, who owns and who holds copies |+//! | [`merge`] | what to do when two machines changed one key |+//! | [`migrate`] | turning replication on for an existing Volume: format marker, backup, changes made behind its back |+//! | [`wire`], [`link`], [`transport`] | the peer protocol, one connection, how machines reach each other |+//! | [`outbox`] | changes saved here that the owner has not taken yet |+//! | [`node`] | the runtime: owner writes, log shipping, catch-up, takeover, hand-back |+//! | [`words`], [`status`] | what the app and the CLI say |+//! | [`supervision`] | which machine runs a durable agent, and who takes it over |+//! | [`simnet`] | an in-memory network with latency, loss, partitions and held bytes (tests, chaos) |++mod apply;+pub mod backend;+pub mod detector;+mod election;+pub mod era;+pub mod link;+pub mod merge;+pub mod migrate;+pub mod node;+pub mod oplog;+pub mod outbox;+mod owner;+pub mod roles;+mod ship;+pub mod simnet;+pub mod status;+pub mod supervision;+pub mod transport;+pub mod wire;+pub mod words;+mod xfer;++pub use backend::ReplicatedBackend;+pub use detector::{Health, Timing};+pub use era::{EraRec, Eras, Pos};+pub use node::{Config, MachineView, PowerEvent, ReplicaNode};+pub use roles::{Facts, Layout, Member, Role};+pub use status::{Durability, ReplicaStatus};+pub use transport::{TcpTransport, Transport};++/// A machine's id: its machine id on the account.+pub type MachineId = String;diff --git a/libs/cua/crates/cua-volume/src/replica/node.rs b/libs/cua/crates/cua-volume/src/replica/node.rsnew file mode 100644index 0000000000..4e94fe4e09--- /dev/null+++ b/libs/cua/crates/cua-volume/src/replica/node.rs@@ -0,0 +1,1600 @@+// SPDX-License-Identifier: FSL-1.1-MIT+// Copyright (c) 2026 Cua AI, Inc.++//! The replication runtime of one machine: who it talks to, what it knows,+//! and the loop that keeps the volume converged. The pieces live in+//! sibling modules (`owner`, `apply`, `election`, `backend`); this file has+//! the state, the lifecycle, the links, and the tick.++use std::collections::{BTreeMap, HashMap};+use std::path::{Path, PathBuf};+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};+use std::sync::{Arc, Mutex};+use std::time::{Duration, Instant};++use tokio::sync::{Notify, watch};++use super::detector::{Detector, Health, Timing};+use super::era::{EraRec, Eras, Pos};+use super::link::{Counters, Handler, Link, Stream};+use super::oplog::OpLog;+use super::outbox::Outbox;+use super::roles::{self, Facts, Layout, Member, Role};+use super::ship::Shipping;+use super::status::{PeerView, PendingView, ReplicaStatus};+use super::transport::Transport;+use super::wire::{self, Beat, Chan, Frame, Hello, Msg};+use super::words::Summary;+use super::{MachineId, backend::ReplicatedBackend};+use crate::backend::Backend as _;+use crate::fs::FsBackend;+use crate::{Error, Result, now_ms};++/// Timing and size knobs. Defaults are for real networks; [`Config::fast`]+/// scales time down for simulations.+#[derive(Clone, Debug)]+pub struct Config {+ pub timing: Timing,+ /// How long a write waits for a second copy before it is reported as+ /// held on one machine (the write itself never fails for this).+ pub ack_wait: Duration,+ /// A control request that gets no answer in this long fails.+ pub call_timeout: Duration,+ /// A bulk request (a block) that gets no answer in this long fails.+ pub data_timeout: Duration,+ /// A more reliable machine must have been up and caught up this long+ /// before the volume moves back to it.+ pub handback_after: Duration,+ /// Bodies up to this size travel inside the control message.+ pub inline_max: usize,+ /// A write by a non-owner above this size is spooled and pulled by the+ /// owner in blocks instead of pushed in one request.+ pub push_max: usize,+ /// Bytes per transfer block.+ pub block: u64,+ /// Blocks in flight per transfer.+ pub window: usize,+ /// Records per log message.+ pub batch: usize,+ /// Log records kept.+ pub log_keep: usize,+ pub dial_backoff_min: Duration,+ pub dial_backoff_max: Duration,+ /// A machine not heard from in this long is not counted for takeover+ /// quorums.+ pub active_window: Duration,+ /// After a takeover attempt, wait this long before trying again.+ pub takeover_cooldown: Duration,+ /// A machine that does not need to order writes takes over from an+ /// owner only after it has been down this long.+ pub idle_takeover: Duration,+ /// How often the heads are recorded (see `migrate`).+ pub snapshot_every: Duration,+ /// Added to this node's wall clock (simulating skew).+ pub wall_skew_ms: i64,+}++impl Config {+ pub fn standard() -> Config {+ Config {+ timing: Timing::standard(),+ ack_wait: Duration::from_secs(2),+ call_timeout: Duration::from_secs(30),+ data_timeout: Duration::from_secs(60),+ handback_after: Duration::from_secs(120),+ inline_max: 16 * 1024,+ push_max: 1 << 20,+ block: 1 << 20,+ window: 8,+ batch: 256,+ log_keep: 200_000,+ dial_backoff_min: Duration::from_millis(250),+ dial_backoff_max: Duration::from_secs(15),+ active_window: Duration::from_secs(14 * 24 * 3600),+ takeover_cooldown: Duration::from_secs(10),+ idle_takeover: Duration::from_secs(300),+ snapshot_every: Duration::from_secs(600),+ wall_skew_ms: 0,+ }+ }++ /// Time scaled down by about 40 (heartbeat 50 ms): for simulations.+ pub fn fast() -> Config {+ Config {+ timing: Timing {+ interval: Duration::from_millis(50),+ floor: Duration::from_millis(250),+ confirm: Duration::from_millis(400),+ recover: 3,+ },+ ack_wait: Duration::from_millis(400),+ call_timeout: Duration::from_secs(4),+ data_timeout: Duration::from_secs(6),+ handback_after: Duration::from_millis(1500),+ dial_backoff_min: Duration::from_millis(20),+ dial_backoff_max: Duration::from_millis(250),+ takeover_cooldown: Duration::from_millis(300),+ idle_takeover: Duration::from_millis(2500),+ snapshot_every: Duration::from_secs(5),+ ..Config::standard()+ }+ }+}++/// One machine of the group as the supervisor sees it.+#[derive(Clone, Debug, PartialEq, Eq)]+pub struct MachineView {+ pub id: MachineId,+ pub name: String,+ pub is_me: bool,+ /// Reachable now (this machine counts as up unless it is going to sleep).+ pub alive: bool,+ /// Confirmed down: silent through the whole confirmation window, said+ /// goodbye, or refused connections. Not merely slow.+ pub down: bool,+ /// Up without a break for at least the hand-back time.+ pub stable: bool,+ /// Holds a full copy of the volume.+ pub replica: bool,+ pub facts: Facts,+}++/// What a machine's power state does to the volume.+#[derive(Clone, Copy, Debug, PartialEq, Eq)]+pub enum PowerEvent {+ /// The machine is about to sleep (lid closed, idle sleep): tell peers.+ WillSleep,+ /// It woke.+ DidWake,+ /// The network went away.+ NetworkLost,+ /// The network is back (or changed address): dial again now.+ NetworkUp,+}++pub(super) struct PeerState {+ pub det: Detector,+ pub facts: Facts,+ pub applied: Pos,+ pub tail: Pos,+ /// Owner's view: the replica has applied through this.+ pub acked: Pos,+ pub pending: u64,+ pub problem: Option<String>,+ /// When the detector last turned alive and stayed so.+ pub alive_since: Option<Instant>,+ pub last_health: Health,+ pub next_dial: Instant,+ pub backoff: Duration,+ pub dialing: bool,+ /// A promise given to a takeover candidate: (epoch, until).+ pub promised: Option<(u64, Instant)>,+ pub heard_ms: u64,+ /// Since when the detector says it is down.+ pub down_since: Option<Instant>,+ /// It said goodbye on purpose (sleep, shutdown): no flapping to fear.+ pub intentional: bool,+ /// Dials in a row the machine refused (or hung up on before hello).+ pub refusals: u32,+ /// When that run of refusals began.+ pub first_refusal: Option<Instant>,+ /// How long its acknowledgements take (a moving average, ms): a machine+ /// that takes longer than half the wait is not waited for.+ pub ack_rtt_ms: f64,+ /// Writes in a row that waited for its acknowledgement in vain: a+ /// machine that does not answer is not waited for until it does.+ pub missed_acks: u32,+}++pub(super) struct State {+ pub members: BTreeMap<MachineId, Member>,+ pub eras: Eras,+ pub log: OpLog,+ pub outbox: Outbox,+ pub peers: HashMap<MachineId, PeerState>,+ pub volume_bytes: u64,+ pub last_takeover_try: Option<Instant>,+ pub takeovers: u64,+ pub handbacks: u64,+ pub fenced_writes: u64,+ pub conflict_copies: u64,+ /// Our own recent writes by position, for the read-your-writes wait.+ pub healthy: bool,+ /// When a local operation last needed to order a write itself.+ pub authority_wanted: Option<Instant>,+ /// When each recent record was written here (to time acknowledgements).+ pub written_at: std::collections::BTreeMap<Pos, Instant>,+}++#[derive(Default)]+pub(super) struct PeerLinks {+ pub ctl: Option<Arc<Link>>,+ pub data: Option<Arc<Link>>,+}++pub(super) struct Inner {+ pub me: MachineId,+ pub name: String,+ pub cfg: Config,+ pub store: FsBackend,+ pub spool: FsBackend,+ pub dir: PathBuf,+ pub transport: Arc<dyn Transport>,+ pub counters: Arc<Counters>,+ pub st: tokio::sync::Mutex<State>,+ /// One applier of records, snapshots and rebases at a time.+ pub gate: tokio::sync::Mutex<()>,+ pub links: Mutex<HashMap<MachineId, PeerLinks>>,+ pub facts: Mutex<Facts>,+ pub wake: Notify,+ /// Wakes the tick loop early (a link dropped, authority is wanted, a+ /// machine said goodbye).+ pub tick_wake: Notify,+ pub acks: watch::Sender<u64>,+ pub applied_tx: watch::Sender<Pos>,+ pub closed: AtomicBool,+ pub frozen: watch::Sender<bool>,+ pub is_client: AtomicBool,+ pub started: AtomicBool,+ pub hashes: Mutex<HashMap<String, Arc<Vec<String>>>>,+ pub seq_ack_counter: AtomicU64,+ pub self_away: AtomicBool,+ pub last_tick: Mutex<Instant>,+ pub submitting: AtomicBool,+ pub shipping: Shipping,+ pub takeover_busy: AtomicBool,+ pub handoff_busy: AtomicBool,+}++/// One machine's replicated volume.+#[derive(Clone)]+pub struct ReplicaNode {+ pub(super) inner: Arc<Inner>,+}++impl std::fmt::Debug for ReplicaNode {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(f, "ReplicaNode({})", self.inner.me)+ }+}++/// A dial that failed because the machine said no, not because nothing+/// answered.+fn is_refusal(e: &Error) -> bool {+ let m = e.to_string();+ m.contains("refused") || m.contains("did not say hello") || m.contains("Connection reset")+}++fn load_eras(dir: &Path) -> Option<Eras> {+ serde_json::from_slice(&std::fs::read(dir.join("era.json")).ok()?).ok()+}++pub(super) fn save_eras(dir: &Path, e: &Eras) -> Result<()> {+ let p = dir.join("era.json");+ cua_home::guard_write(&p)?;+ let tmp = p.with_extension("tmp");+ std::fs::write(&tmp, serde_json::to_vec(e)?)?;+ std::fs::rename(&tmp, &p)?;+ Ok(())+}++impl ReplicaNode {+ /// Opens this machine's replica state under `state_dir/replica` around+ /// `store` (the volume's local copy). `members` is the machines list,+ /// this machine included.+ pub async fn open(+ cfg: Config,+ me: &str,+ name: &str,+ state_dir: &Path,+ store: FsBackend,+ transport: Arc<dyn Transport>,+ members: Vec<Member>,+ ) -> Result<ReplicaNode> {+ let dir = state_dir.join("replica");+ std::fs::create_dir_all(&dir)?;+ // A directory a newer Cua Spaces prepared is refused, not guessed at.+ super::migrate::check_format(&dir)?;+ let mut log = OpLog::open(&dir, cfg.log_keep)?;+ let eras = match load_eras(&dir) {+ Some(e) => e,+ None => {+ let e = Eras::genesis(me, now_ms());+ // The first time replication touches a Volume that has+ // files: keep a copy of its heads before anything moves.+ let has_files = !store.list("").await?.is_empty();+ if has_files {+ super::migrate::backup_heads(&dir, store.root())?;+ }+ save_eras(&dir, &e)?;+ // A volume that already holds files starts its log after+ // them: a machine that joins gets a snapshot, not a replay+ // of history this log never recorded.+ if has_files {+ log.reset_to(Pos { epoch: 1, seq: 0 })?;+ }+ e+ }+ };+ let outbox = Outbox::open(&dir)?;+ let spool = FsBackend::new(dir.join("spool"));+ let (acks, _) = watch::channel(0u64);+ let (applied_tx, _) = watch::channel(log.tail());+ let (frozen, _) = watch::channel(false);+ let mut m: BTreeMap<MachineId, Member> =+ members.into_iter().map(|x| (x.id.clone(), x)).collect();+ m.entry(me.to_string()).or_insert_with(|| Member {+ id: me.to_string(),+ name: name.to_string(),+ ..Default::default()+ });+ let inner = Arc::new(Inner {+ me: me.to_string(),+ name: name.to_string(),+ cfg,+ store,+ spool,+ dir,+ transport,+ counters: Arc::new(Counters::default()),+ st: tokio::sync::Mutex::new(State {+ members: m,+ eras,+ log,+ outbox,+ peers: HashMap::new(),+ volume_bytes: 0,+ last_takeover_try: None,+ takeovers: 0,+ handbacks: 0,+ fenced_writes: 0,+ conflict_copies: 0,+ healthy: true,+ authority_wanted: None,+ written_at: Default::default(),+ }),+ gate: tokio::sync::Mutex::new(()),+ links: Mutex::new(HashMap::new()),+ facts: Mutex::new(Facts::default()),+ wake: Notify::new(),+ tick_wake: Notify::new(),+ acks,+ applied_tx,+ closed: AtomicBool::new(false),+ frozen,+ is_client: AtomicBool::new(false),+ started: AtomicBool::new(false),+ hashes: Mutex::new(HashMap::new()),+ seq_ack_counter: AtomicU64::new(0),+ self_away: AtomicBool::new(false),+ last_tick: Mutex::new(Instant::now()),+ submitting: AtomicBool::new(false),+ shipping: Mutex::new(HashMap::new()),+ takeover_busy: AtomicBool::new(false),+ handoff_busy: AtomicBool::new(false),+ });+ inner.recover_intent().await?;+ inner.adopt_unlogged().await?;+ {+ let mut st = inner.st.lock().await;+ inner.refresh_roles(&mut st);+ }+ Ok(ReplicaNode { inner })+ }++ /// The `Backend` the drive uses.+ pub fn backend(&self) -> Arc<ReplicatedBackend> {+ Arc::new(ReplicatedBackend::new(self.inner.clone()))+ }++ /// Starts the tick loop and the shippers.+ pub fn start(&self) {+ if self.inner.started.swap(true, Ordering::SeqCst) {+ return;+ }+ let i = self.inner.clone();+ tokio::spawn(async move { i.tick_loop().await });+ let i = self.inner.clone();+ tokio::spawn(async move { i.ship_all_loop().await });+ }++ /// Says goodbye to every peer and stops.+ pub async fn shutdown(&self) {+ let first = !self.inner.closed.load(Ordering::SeqCst);+ self.inner.say_bye_and_flush("shutdown").await;+ // Only the first stop records the heads: a second one (after something+ // else wrote to the store) would hide those writes.+ if first {+ self.inner.save_snapshot().await;+ }+ self.inner.closed.store(true, Ordering::SeqCst);+ self.inner.wake.notify_waiters();+ let links: Vec<Arc<Link>> = {+ let mut l = self.inner.links.lock().expect("links");+ let v = l+ .values()+ .flat_map(|p| [p.ctl.clone(), p.data.clone()])+ .flatten()+ .collect();+ l.clear();+ v+ };+ tokio::time::sleep(Duration::from_millis(20)).await;+ for l in links {+ l.close();+ }+ }++ /// A stream arrived for this machine.+ pub async fn accept(&self, stream: Stream) {+ let i = self.inner.clone();+ tokio::spawn(async move {+ let _ = i.accept_stream(stream).await;+ });+ }++ /// The machines list changed.+ pub async fn set_members(&self, members: Vec<Member>) {+ let mut st = self.inner.st.lock().await;+ for m in members {+ let cur = st.members.entry(m.id.clone()).or_default();+ cur.id = m.id.clone();+ cur.name = m.name;+ if m.facts != Facts::default() {+ cur.facts = m.facts;+ }+ }+ self.inner.refresh_roles(&mut st);+ }++ /// This machine's reliability facts changed.+ pub fn set_facts(&self, f: Facts) {+ *self.inner.facts.lock().expect("facts") = f;+ }++ /// A power or network event from the operating system.+ pub async fn notify_power(&self, ev: PowerEvent) {+ match ev {+ PowerEvent::WillSleep => {+ self.inner.say_bye_and_flush("sleep").await;+ self.inner.self_away.store(true, Ordering::SeqCst);+ }+ PowerEvent::NetworkLost => {+ self.inner.say_bye_and_flush("network_lost").await;+ self.inner.self_away.store(true, Ordering::SeqCst);+ self.inner.drop_links();+ }+ PowerEvent::DidWake | PowerEvent::NetworkUp => {+ self.inner.self_away.store(false, Ordering::SeqCst);+ let now = Instant::now();+ let mut st = self.inner.st.lock().await;+ for p in st.peers.values_mut() {+ p.det.resume(now);+ p.next_dial = now;+ p.backoff = self.inner.cfg.dial_backoff_min;+ }+ drop(st);+ self.inner.drop_links();+ *self.inner.last_tick.lock().expect("tick") = now;+ }+ }+ }++ /// Freezes this node (a stopped process: nothing is read, written or+ /// timed until it thaws). Simulations only; the real test uses SIGSTOP.+ pub fn freeze(&self, on: bool) {+ self.inner.frozen.send_replace(on);+ if !on {+ self.inner.wake.notify_waiters();+ }+ }++ pub fn machine(&self) -> &str {+ &self.inner.me+ }++ /// This machine's name in the machines list.+ pub fn name(&self) -> &str {+ &self.inner.name+ }++ /// Every machine of the group with what is known about it now.+ pub async fn machines(&self) -> Vec<MachineView> {+ let st = self.inner.st.lock().await;+ let layout = self.inner.layout_locked(&st);+ let handback = self.inner.cfg.handback_after;+ let mut out = vec![];+ for (id, m) in &st.members {+ let facts = self.inner.member_facts(&st, id);+ if *id == self.inner.me {+ out.push(MachineView {+ id: id.clone(),+ name: self.inner.name.clone(),+ is_me: true,+ alive: !self.inner.self_away.load(Ordering::SeqCst),+ down: false,+ stable: true,+ replica: layout.role_of(id) == Role::Replica,+ facts,+ });+ continue;+ }+ let p = st.peers.get(id);+ let alive = p.is_some_and(|p| p.last_health == Health::Alive)+ && self.inner.ctl_link(id).is_some();+ let stable =+ alive && p.is_some_and(|p| p.alive_since.is_some_and(|t| t.elapsed() >= handback));+ out.push(MachineView {+ id: id.clone(),+ name: if m.name.is_empty() {+ id.clone()+ } else {+ m.name.clone()+ },+ is_me: false,+ alive,+ down: p.is_none_or(|p| {+ p.last_health.is_down() && p.det.down_and_silent(Instant::now())+ }),+ stable,+ replica: layout.role_of(id) == Role::Replica,+ facts,+ });+ }+ out+ }++ /// Whether this machine owns the volume now.+ pub async fn is_owner(&self) -> bool {+ self.inner.st.lock().await.eras.owner() == self.inner.me+ }++ /// Bytes the links carried (sent, received).+ pub fn wire_bytes(&self) -> (u64, u64) {+ (+ self.inner.counters.sent.load(Ordering::Relaxed),+ self.inner.counters.received.load(Ordering::Relaxed),+ )+ }++ /// What this machine's log and outbox know about `key` (for a failure's+ /// report).+ pub async fn debug_key(&self, key: &str) -> String {+ let st = self.inner.st.lock().await;+ let mut out = String::new();+ for r in st.log.range(Pos::ZERO, st.log.tail()) {+ if r.op.key() == key {+ let what = match &r.op {+ super::oplog::Op::Put { meta, .. } => format!("put {}", meta.version),+ super::oplog::Op::Delete { version, .. } => format!("delete {version}"),+ };+ out.push_str(&format!(" log@{}:{what}", r.pos));+ }+ }+ for p in st.outbox.items().iter().filter(|p| p.key == key) {+ out.push_str(&format!(+ " pending({})",+ p.meta+ .as_ref()+ .map_or("delete".into(), |m| m.version.clone())+ ));+ }+ out+ }++ pub async fn status(&self) -> ReplicaStatus {+ self.inner.status().await+ }++ /// The facts for the one plain paragraph (see `words`).+ pub async fn summary(&self, run_on: Option<&str>) -> Summary {+ self.inner.summary(run_on).await+ }+}++impl Inner {+ pub(super) fn wall_ms(&self) -> u64 {+ (now_ms() as i64 + self.cfg.wall_skew_ms).max(0) as u64+ }++ pub(super) fn my_facts(&self) -> Facts {+ let mut f = self.facts.lock().expect("facts").clone();+ if f.free_disk == 0 {+ f.free_disk = self.store.free_bytes().unwrap_or(0);+ }+ f+ }++ pub(super) async fn wait_unfrozen(&self) {+ let mut rx = self.frozen.subscribe();+ while *rx.borrow() {+ if rx.changed().await.is_err() {+ return;+ }+ }+ }++ pub(super) fn handler(self: &Arc<Self>) -> Handler {+ let me = self.clone();+ Arc::new(move |link, frame, data| {+ let me = me.clone();+ Box::pin(async move {+ me.wait_unfrozen().await;+ me.handle(link, frame, data).await;+ })+ })+ }++ /// Recomputes who should hold copies and whether this machine is a+ /// client (state lock held).+ pub(super) fn refresh_roles(&self, st: &mut State) {+ let l = self.layout_locked(st);+ self.is_client+ .store(l.role_of(&self.me) == Role::Client, Ordering::SeqCst);+ }++ pub(super) fn member_facts(&self, st: &State, id: &str) -> Facts {+ if id == self.me {+ return self.my_facts();+ }+ st.peers+ .get(id)+ .map(|p| p.facts.clone())+ .filter(|f| *f != Facts::default())+ .or_else(|| st.members.get(id).map(|m| m.facts.clone()))+ .unwrap_or_default()+ }++ /// Members that count (heard from recently, or never heard but listed).+ pub(super) fn active_members(&self, st: &State) -> Vec<MachineId> {+ let now = self.wall_ms();+ let win = self.cfg.active_window.as_millis() as u64;+ st.members+ .values()+ .filter(|m| {+ m.id == self.me+ || st+ .peers+ .get(&m.id)+ .map_or(m.last_seen_ms, |p| p.heard_ms.max(m.last_seen_ms))+ == 0+ || now.saturating_sub(+ st.peers+ .get(&m.id)+ .map_or(m.last_seen_ms, |p| p.heard_ms.max(m.last_seen_ms)),+ ) < win+ })+ .map(|m| m.id.clone())+ .collect()+ }++ pub(super) fn layout_locked(&self, st: &State) -> Layout {+ let active = self.active_members(st);+ let members: Vec<Member> = active+ .iter()+ .filter_map(|id| st.members.get(id))+ .map(|m| Member {+ facts: self.member_facts(st, &m.id),+ ..m.clone()+ })+ .collect();+ roles::layout(+ &members,+ roles::replica_count(members.len()),+ st.volume_bytes,+ )+ }++ fn beat(&self, st: &State) -> Beat {+ let eras: Vec<EraRec> = st.eras.recs.iter().rev().take(4).rev().cloned().collect();+ Beat {+ epoch: st.eras.epoch(),+ owner: st.eras.owner().to_string(),+ lineage: st.eras.lineage.clone(),+ eras,+ problem: if st.healthy {+ None+ } else {+ Some("this machine cannot store more of the volume".into())+ },+ applied: st.log.tail(),+ tail: st.log.tail(),+ facts: self.my_facts(),+ pending: st.outbox.len() as u64,+ }+ }++ fn hello(&self, st: &State, chan: Chan) -> Hello {+ Hello {+ machine: self.me.clone(),+ name: self.name.clone(),+ chan,+ lineage: st.eras.lineage.clone(),+ eras: st.eras.recs.iter().rev().take(8).rev().cloned().collect(),+ applied: st.log.tail(),+ tail: st.log.tail(),+ facts: self.my_facts(),+ v: wire::VERSION,+ }+ }++ pub(super) fn peer_state<'a>(&self, st: &'a mut State, id: &str) -> &'a mut PeerState {+ let timing = self.cfg.timing;+ let min = self.cfg.dial_backoff_min;+ st.peers.entry(id.to_string()).or_insert_with(|| PeerState {+ det: Detector::new(timing, Instant::now()),+ facts: Facts::default(),+ applied: Pos::ZERO,+ tail: Pos::ZERO,+ acked: Pos::ZERO,+ pending: 0,+ problem: None,+ alive_since: None,+ last_health: Health::Alive,+ next_dial: Instant::now(),+ backoff: min,+ dialing: false,+ promised: None,+ heard_ms: 0,+ down_since: None,+ intentional: false,+ missed_acks: 0,+ refusals: 0,+ first_refusal: None,+ ack_rtt_ms: 0.0,+ })+ }++ pub(super) fn say_bye(&self, reason: &str) -> Vec<Arc<Link>> {+ let links = self.links.lock().expect("links");+ let mut sent = vec![];+ for p in links.values() {+ if let Some(l) = &p.ctl {+ l.send(+ Msg::Bye {+ reason: reason.into(),+ },+ &[],+ );+ sent.push(l.clone());+ }+ }+ sent+ }++ /// Says goodbye and waits (a moment) for it to leave: a machine about+ /// to sleep has only a moment.+ pub(super) async fn say_bye_and_flush(&self, reason: &str) {+ for l in self.say_bye(reason) {+ l.flush(Duration::from_millis(500)).await;+ }+ }++ pub(super) fn drop_links(&self) {+ let links: Vec<Arc<Link>> = {+ let mut l = self.links.lock().expect("links");+ let v = l+ .values()+ .flat_map(|p| [p.ctl.clone(), p.data.clone()])+ .flatten()+ .collect();+ l.clear();+ v+ };+ for l in links {+ l.close();+ }+ }++ pub(super) fn ctl_link(&self, peer: &str) -> Option<Arc<Link>> {+ self.links+ .lock()+ .expect("links")+ .get(peer)+ .and_then(|p| p.ctl.clone())+ .filter(|l| !l.is_closed())+ }++ pub(super) fn data_link_now(&self, peer: &str) -> Option<Arc<Link>> {+ self.links+ .lock()+ .expect("links")+ .get(peer)+ .and_then(|p| p.data.clone())+ .filter(|l| !l.is_closed())+ }++ /// The data link to `peer`, dialing it if this machine has none.+ pub(super) async fn data_link(self: &Arc<Self>, peer: &str) -> Option<Arc<Link>> {+ if let Some(l) = self.data_link_now(peer) {+ return Some(l);+ }+ self.dial_chan(peer, Chan::Data).await.ok()+ }++ fn register(&self, link: &Arc<Link>) {+ let mut links = self.links.lock().expect("links");+ let e = links.entry(link.peer.clone()).or_default();+ match link.chan {+ Chan::Control => e.ctl = Some(link.clone()),+ Chan::Data => e.data = Some(link.clone()),+ }+ }++ fn unregister(&self, link: &Arc<Link>) {+ let mut links = self.links.lock().expect("links");+ if let Some(e) = links.get_mut(&link.peer) {+ let slot = match link.chan {+ Chan::Control => &mut e.ctl,+ Chan::Data => &mut e.data,+ };+ if slot.as_ref().is_some_and(|l| Arc::ptr_eq(l, link)) {+ *slot = None;+ }+ }+ drop(links);+ // A link that dropped is redialed now, not at the next tick.+ self.tick_wake.notify_one();+ }++ async fn handshake(+ self: &Arc<Self>,+ mut stream: Stream,+ chan: Chan,+ initiated: bool,+ expect: Option<&str>,+ ) -> Result<Arc<Link>> {+ use tokio::io::AsyncWriteExt as _;+ let hello = {+ let st = self.st.lock().await;+ self.hello(&st, chan)+ };+ let frame = Frame {+ id: 0,+ re: None,+ msg: Msg::Hello(hello),+ };+ let bytes = wire::encode(&frame, &[])?;+ let to = self.cfg.call_timeout;+ let exchange = async {+ stream.write_all(&bytes).await?;+ stream.flush().await?;+ wire::read(&mut stream).await+ };+ let got = tokio::time::timeout(to, exchange)+ .await+ .map_err(|_| Error::Backend("handshake timed out".into()))??;+ let Some((+ Frame {+ msg: Msg::Hello(h), ..+ },+ _,+ )) = got+ else {+ return Err(Error::Backend("the machine did not say hello".into()));+ };+ if let Some(e) = expect+ && e != h.machine+ {+ return Err(Error::Backend(format!(+ "dialed {e} and {} answered",+ h.machine+ )));+ }+ {+ let st = self.st.lock().await;+ if !st.members.contains_key(&h.machine) {+ return Err(Error::Forbidden(format!(+ "{} is not one of this account's machines",+ h.machine+ )));+ }+ }+ let me = self.clone();+ let me2 = self.clone();+ let link = Link::spawn(+ stream,+ h.machine.clone(),+ h.name.clone(),+ chan,+ initiated,+ self.counters.clone(),+ self.handler(),+ move |l| {+ me2.unregister(&l);+ },+ );+ *link.hello.lock().expect("hello") = Some(h.clone());+ me.register(&link);+ if chan == Chan::Control {+ me.peer_up(&link, h).await;+ }+ Ok(link)+ }++ async fn accept_stream(self: &Arc<Self>, stream: Stream) -> Result<()> {+ if self.closed.load(Ordering::SeqCst) {+ return Ok(());+ }+ // The chan is in the peer's hello; read it from a peek of our own+ // handshake: both sides send first, so the chan we answer with is+ // the one the dialer chose. Read first, then answer.+ use tokio::io::AsyncWriteExt as _;+ let mut stream = stream;+ let first = tokio::time::timeout(self.cfg.call_timeout, wire::read(&mut stream))+ .await+ .map_err(|_| Error::Backend("handshake timed out".into()))??;+ let Some((+ Frame {+ msg: Msg::Hello(h), ..+ },+ _,+ )) = first+ else {+ return Err(Error::Backend("no hello".into()));+ };+ {+ let st = self.st.lock().await;+ if !st.members.contains_key(&h.machine) {+ return Err(Error::Forbidden(format!(+ "{} is not one of this account's machines",+ h.machine+ )));+ }+ }+ let mine = {+ let st = self.st.lock().await;+ self.hello(&st, h.chan)+ };+ let bytes = wire::encode(+ &Frame {+ id: 0,+ re: None,+ msg: Msg::Hello(mine),+ },+ &[],+ )?;+ stream.write_all(&bytes).await?;+ stream.flush().await?;+ let me2 = self.clone();+ let link = Link::spawn(+ stream,+ h.machine.clone(),+ h.name.clone(),+ h.chan,+ false,+ self.counters.clone(),+ self.handler(),+ move |l| me2.unregister(&l),+ );+ *link.hello.lock().expect("hello") = Some(h.clone());+ self.register(&link);+ if h.chan == Chan::Control {+ self.peer_up(&link, h).await;+ }+ Ok(())+ }++ pub(super) async fn dial_chan(self: &Arc<Self>, peer: &str, chan: Chan) -> Result<Arc<Link>> {+ let stream = self.transport.dial(peer).await.map_err(|e| {+ if e.kind() == std::io::ErrorKind::ConnectionRefused {+ Error::Backend(format!("connection refused: {e}"))+ } else {+ Error::from(e)+ }+ })?;+ self.handshake(stream, chan, true, Some(peer)).await+ }++ /// A control link came up (either direction).+ async fn peer_up(self: &Arc<Self>, _link: &Arc<Link>, h: Hello) {+ let now = Instant::now();+ let mut reconcile = false;+ let mut join = false;+ {+ let mut st = self.st.lock().await;+ if let Some(m) = st.members.get_mut(&h.machine) {+ m.name = h.name.clone();+ m.last_seen_ms = self.wall_ms();+ }+ if h.lineage != st.eras.lineage {+ // Two volumes that started apart: the less reliable machine+ // joins the other's.+ let mine = self.my_facts();+ let theirs = h.facts.clone();+ join = (mine.rank, &self.me) < (theirs.rank, &h.machine);+ // Equal rank: the lower id joins (a deterministic pick).+ let _ = mine;+ } else {+ let advanced = st.eras.adopt_from(&h.eras);+ if advanced {+ let _ = save_eras(&self.dir, &st.eras);+ reconcile = true;+ }+ }+ let owner_now = st.eras.owner() == self.me;+ let other_lineage = h.lineage != st.eras.lineage;+ let divergence = st.eras.divergence(h.applied);+ let p = self.peer_state(&mut st, &h.machine);+ p.det.arrival(now);+ p.facts = h.facts.clone();+ p.heard_ms = self.wall_ms();+ p.applied = h.applied;+ p.tail = h.tail;+ // What a machine of another volume has applied says nothing+ // about this one.+ p.acked = if other_lineage {+ Pos::ZERO+ } else {+ divergence.unwrap_or(h.applied)+ };+ p.missed_acks = 0;+ p.backoff = self.cfg.dial_backoff_min;+ p.dialing = false;+ let _ = owner_now;+ if reconcile || st.eras.divergence(st.log.tail()).is_some() {+ reconcile = true;+ }+ self.refresh_roles(&mut st);+ }+ if join {+ let me = self.clone();+ let h2 = h.clone();+ tokio::spawn(async move { me.join_lineage(h2).await });+ } else if reconcile {+ let me = self.clone();+ tokio::spawn(async move { me.reconcile_eras().await });+ }+ self.wake.notify_one();+ }++ /// Frame dispatch.+ async fn handle(self: &Arc<Self>, link: Arc<Link>, frame: Frame, data: Vec<u8>) {+ if self.closed.load(Ordering::SeqCst) {+ return;+ }+ let id = frame.id;+ match frame.msg {+ Msg::Ping(b) => {+ self.on_beat(&link.peer, b).await;+ let mine = {+ let st = self.st.lock().await;+ self.beat(&st)+ };+ link.reply(id, Msg::Pong(mine), &[]);+ }+ Msg::Bye { reason } => self.on_bye(&link.peer, &reason).await,+ Msg::Append { epoch, records } => {+ let reply = self.handle_append(&link, epoch, records, data).await;+ link.reply(id, reply, &[]);+ }+ Msg::Snapshot { at } => {+ let reply = self.handle_snapshot(&link, at).await;+ link.reply(id, reply, &[]);+ }+ Msg::Prepare { epoch, base } => {+ let reply = self.handle_prepare(&link.peer, epoch, base).await;+ link.reply(id, reply, &[]);+ }+ Msg::Era { recs } => {+ self.handle_era(&link.peer, recs).await;+ link.reply(id, Msg::Ack { applied: Pos::ZERO }, &[]);+ }+ Msg::Handoff { epoch, to, base } => {+ let reply = self.handle_handoff(&link.peer, epoch, to, base).await;+ link.reply(id, reply, &[]);+ }+ Msg::Submit { ops } => {+ let reply = self.handle_submit(&link, ops, data).await;+ link.reply(id, reply, &[]);+ }+ Msg::Req(r) => {+ let (rep, out) = self.handle_req(&link, r, data).await;+ link.reply(id, Msg::Rep(rep), &out);+ }+ Msg::Hashes {+ src,+ block,+ from,+ count,+ } => {+ let reply = self.handle_hashes(src, block, from, count).await;+ link.reply(id, reply, &[]);+ }+ Msg::Block { src, block, idx } => match self.handle_block(src, block, idx).await {+ Ok(bytes) => {+ link.reply(id, Msg::BlockIs, &bytes);+ }+ Err(e) => {+ link.reply(+ id,+ Msg::Error {+ kind: e.tag().into(),+ message: e.to_string(),+ },+ &[],+ );+ }+ },+ // Replies and handshakes are never dispatched.+ _ => {}+ }+ }++ pub(super) async fn on_beat(self: &Arc<Self>, peer: &str, b: Beat) {+ let now = Instant::now();+ let mut reconcile = false;+ let mut join: Option<Hello> = None;+ {+ let mut st = self.st.lock().await;+ if !b.lineage.is_empty() && b.lineage != st.eras.lineage {+ // The peer belongs to another volume now (it joined one while+ // we were connected): the less reliable of the two joins.+ let mine = self.my_facts();+ if (mine.rank, &self.me) < (b.facts.rank, &peer.to_string()) {+ join = Some(Hello {+ machine: peer.to_string(),+ name: st+ .members+ .get(peer)+ .map(|m| m.name.clone())+ .unwrap_or_default(),+ chan: Chan::Control,+ lineage: b.lineage.clone(),+ eras: b.eras.clone(),+ applied: b.applied,+ tail: b.tail,+ facts: b.facts.clone(),+ v: wire::VERSION,+ });+ }+ } else if b.epoch > st.eras.epoch() && st.eras.adopt_from(&b.eras) {+ let _ = save_eras(&self.dir, &st.eras);+ reconcile = true;+ }+ if let Some(m) = st.members.get_mut(peer) {+ m.last_seen_ms = self.wall_ms();+ }+ let wall = self.wall_ms();+ let p = self.peer_state(&mut st, peer);+ p.det.arrival(now);+ p.facts = b.facts;+ p.applied = b.applied;+ p.tail = b.tail;+ p.pending = b.pending;+ p.problem = b.problem;+ p.heard_ms = wall;+ // The beat is the machine's own account of how far it got+ // (it moves back when the machine puts itself right after a+ // takeover, or starts over after joining another volume).+ if b.applied > p.acked {+ p.missed_acks = 0;+ }+ p.acked = b.applied;+ }+ if let Some(h) = join {+ let me = self.clone();+ tokio::spawn(async move { me.join_lineage(h).await });+ } else if reconcile {+ let me = self.clone();+ tokio::spawn(async move { me.reconcile_eras().await });+ }+ self.acks+ .send_replace(self.seq_ack_counter.fetch_add(1, Ordering::Relaxed));+ }++ pub(super) async fn on_bye(&self, peer: &str, reason: &str) {+ let now = Instant::now();+ let mut st = self.st.lock().await;+ let p = self.peer_state(&mut st, peer);+ p.det.bye(now);+ p.intentional = matches!(reason, "sleep" | "shutdown");+ tracing::info!(peer, reason, "replica: a machine said goodbye");+ drop(st);+ self.tick_wake.notify_one();+ self.wake.notify_one();+ }+}++impl Inner {+ /// The tick: heartbeats, health, dialing, takeover, hand-back, outbox.+ pub(super) async fn tick_loop(self: Arc<Self>) {+ let period = (self.cfg.timing.interval / 2).max(Duration::from_millis(10));+ let mut last_ping = Instant::now() - self.cfg.timing.interval;+ let mut last_size = Instant::now() - Duration::from_secs(3600);+ let mut last_snap = Instant::now();+ let mut snapped = Pos::ZERO;+ loop {+ tokio::select! {+ _ = tokio::time::sleep(period) => {}+ _ = self.tick_wake.notified() => {}+ }+ if self.closed.load(Ordering::SeqCst) {+ return;+ }+ self.wait_unfrozen().await;+ let now = Instant::now();+ // Were we paused (lid closed, process stopped)? Then every+ // peer looks silent through no fault of theirs.+ let gap = {+ let mut lt = self.last_tick.lock().expect("tick");+ let g = now.saturating_duration_since(*lt);+ *lt = now;+ g+ };+ if gap > (self.cfg.timing.interval * 4).max(period * 6) {+ let mut st = self.st.lock().await;+ for p in st.peers.values_mut() {+ p.det.resume(now);+ }+ tracing::info!(?gap, "replica: this machine was paused; forgiving silence");+ }+ if self.self_away.load(Ordering::SeqCst) {+ continue;+ }+ if now.saturating_duration_since(last_ping) >= self.cfg.timing.interval {+ last_ping = now;+ self.ping_all().await;+ }+ if now.saturating_duration_since(last_size) >= Duration::from_secs(60) {+ last_size = now;+ self.measure_volume().await;+ }+ // A hard stop leaves the last snapshot, so keep it fresh.+ if now.saturating_duration_since(last_snap) >= self.cfg.snapshot_every {+ last_snap = now;+ let tail = self.st.lock().await.log.tail();+ if tail != snapped {+ snapped = tail;+ self.save_snapshot().await;+ }+ }+ self.poll_health(now).await;+ self.dial_missing().await;+ // None of these may hold the tick: each can wait on the network+ // (a vote, an offer, the outbox), and a tick that waits is a+ // machine that stops noticing its peers.+ if !self.takeover_busy.swap(true, Ordering::SeqCst) {+ let me = self.clone();+ tokio::spawn(async move {+ me.maybe_takeover().await;+ me.takeover_busy.store(false, Ordering::SeqCst);+ });+ }+ if !self.handoff_busy.swap(true, Ordering::SeqCst) {+ let me = self.clone();+ tokio::spawn(async move {+ me.maybe_handoff().await;+ me.handoff_busy.store(false, Ordering::SeqCst);+ });+ }+ let me = self.clone();+ tokio::spawn(async move { me.submit_outbox().await });+ }+ }++ async fn measure_volume(&self) {+ if let Ok(all) = crate::backend::Backend::list(&self.store, "").await {+ let total: u64 = all.iter().map(|m| m.size).sum();+ let mut st = self.st.lock().await;+ st.volume_bytes = total;+ self.refresh_roles(&mut st);+ }+ }++ async fn ping_all(self: &Arc<Self>) {+ let peers: Vec<(MachineId, Arc<Link>)> = {+ let l = self.links.lock().expect("links");+ l.iter()+ .filter_map(|(k, v)| {+ v.ctl+ .clone()+ .filter(|c| !c.is_closed())+ .map(|c| (k.clone(), c))+ })+ .collect()+ };+ for (peer, link) in peers {+ let me = self.clone();+ tokio::spawn(async move {+ let beat = {+ let st = me.st.lock().await;+ me.beat(&st)+ };+ let to = me.cfg.timing.floor.max(me.cfg.timing.interval * 3);+ if let Ok((Msg::Pong(b), _)) = link.call(Msg::Ping(beat), &[], to).await {+ me.on_beat(&peer, b).await;+ }+ });+ }+ }++ async fn poll_health(&self, now: Instant) {+ let mut st = self.st.lock().await;+ let ids: Vec<MachineId> = st.peers.keys().cloned().collect();+ let mut changed = false;+ for id in ids {+ let h = {+ let p = st.peers.get_mut(&id).expect("peer");+ let h = p.det.poll(now);+ if h != p.last_health {+ tracing::info!(me = %self.me, peer = %id, from = p.last_health.word(), to = h.word(), "replica: peer health");+ if h.is_down() && p.down_since.is_none() {+ p.down_since = Some(now);+ }+ if h == Health::Alive {+ p.down_since = None;+ p.intentional = false;+ }+ if h == Health::Alive && p.last_health.is_down() {+ p.alive_since = Some(now);+ }+ if h != Health::Alive {+ p.alive_since = None;+ }+ p.last_health = h;+ changed = true;+ }+ if h == Health::Alive && p.alive_since.is_none() {+ p.alive_since = Some(now);+ }+ h+ };+ // A silent link may be half open: drop it so a new one is dialed.+ // A link that has only just come up has not had time to be+ // silent: it is the way back, not the problem.+ if matches!(h, Health::Suspect | Health::Dead) {+ let threshold = st+ .peers+ .get(&id)+ .map_or(Duration::ZERO, |p| p.det.threshold());+ let l = {+ let links = self.links.lock().expect("links");+ links.get(&id).and_then(|p| p.ctl.clone())+ };+ if let Some(l) = l+ && l.created.elapsed() > threshold+ {+ l.close();+ self.unregister(&l);+ }+ }+ }+ if changed {+ self.acks+ .send_replace(self.seq_ack_counter.fetch_add(1, Ordering::Relaxed));+ self.wake.notify_one();+ }+ }++ async fn dial_missing(self: &Arc<Self>) {+ let want: Vec<MachineId> = {+ let st = self.st.lock().await;+ st.members+ .keys()+ .filter(|k| **k != self.me)+ .cloned()+ .collect()+ };+ let now = Instant::now();+ for id in want {+ if self.ctl_link(&id).is_some() {+ continue;+ }+ let go = {+ let mut st = self.st.lock().await;+ let p = self.peer_state(&mut st, &id);+ if p.dialing || now < p.next_dial {+ false+ } else {+ p.dialing = true;+ true+ }+ };+ if !go {+ continue;+ }+ let me = self.clone();+ tokio::spawn(async move {+ let r = me.dial_chan(&id, Chan::Control).await;+ let mut st = me.st.lock().await;+ let max = me.cfg.dial_backoff_max;+ let p = me.peer_state(&mut st, &id);+ p.dialing = false;+ if let Err(e) = &r {+ // Right after a link drops the first few redials come+ // fast (is the machine there?); only then does the wait+ // grow.+ let refusal = is_refusal(e);+ if !(refusal && p.refusals < 3) {+ p.backoff = (p.backoff * 2).min(max);+ }+ let jitter = Duration::from_millis(rand::random::<u64>() % 40);+ p.next_dial = Instant::now() + p.backoff + jitter;+ // Look again when the backoff ends, not at the next tick.+ let (me2, after) = (me.clone(), p.backoff + jitter);+ tokio::spawn(async move {+ tokio::time::sleep(after).await;+ me2.tick_wake.notify_one();+ });+ // Someone answered to say no (or hung up before hello):+ // after three in a row, nobody is running the node.+ if refusal {+ p.refusals += 1;+ let began = *p.first_refusal.get_or_insert_with(Instant::now);+ // Three refusals in a row, over at least the silence+ // floor: a link that flaps for a few seconds gives+ // the same refusals and must not count as a death.+ if p.refusals >= 3+ && began.elapsed() >= me.cfg.timing.floor+ && !p.last_health.is_down()+ {+ tracing::info!(peer = %id, "replica: the machine refuses connections; counting it as down");+ p.det.refused(Instant::now());+ p.last_health = Health::Dead;+ p.down_since = Some(Instant::now());+ drop(st);+ me.wake.notify_one();+ }+ } else {+ p.refusals = 0;+ p.first_refusal = None;+ }+ } else {+ p.refusals = 0;+ p.first_refusal = None;+ }+ });+ }+ }++ pub(super) async fn status(&self) -> ReplicaStatus {+ let st = self.st.lock().await;+ let layout = self.layout_locked(&st);+ let owner = st.eras.owner().to_string();+ let tail = st.log.tail();+ let name_of = |id: &str| {+ st.members+ .get(id)+ .map(|m| m.name.clone())+ .filter(|n| !n.is_empty())+ .unwrap_or_else(|| id.to_string())+ };+ let mut peers = vec![];+ let mut copies_now = 1usize;+ let owner_tail = if owner == self.me {+ tail+ } else {+ st.peers.get(&owner).map_or(tail, |p| p.tail)+ };+ for (id, m) in &st.members {+ if *id == self.me {+ continue;+ }+ let p = st.peers.get(id);+ let health = p.map_or(Health::Dead, |p| p.last_health);+ let applied = p.map_or(Pos::ZERO, |p| {+ if owner == self.me { p.acked } else { p.applied }+ });+ let role = layout.role_of(id);+ let lag = if applied.epoch == owner_tail.epoch {+ owner_tail.seq.saturating_sub(applied.seq)+ } else if applied < owner_tail {+ owner_tail.seq.max(1)+ } else {+ 0+ };+ if role == Role::Replica && !health.is_down() && lag == 0 && applied >= owner_tail {+ copies_now += 1;+ }+ peers.push(PeerView {+ id: id.clone(),+ name: m.name.clone(),+ role,+ health: health.word().into(),+ connected: self.ctl_link(id).is_some(),+ owner: *id == owner,+ applied,+ lag,+ problem: p.and_then(|p| p.problem.clone()),+ });+ }+ let wanted = layout.replicas.len().max(1);+ let unreplicated = if owner == self.me {+ let mut acks: Vec<Pos> = layout+ .replicas+ .iter()+ .filter(|r| **r != self.me)+ .map(|r| st.peers.get(r).map_or(Pos::ZERO, |p| p.acked))+ .collect();+ acks.sort();+ if wanted <= 1 {+ 0+ } else {+ let best = acks.last().copied().unwrap_or(Pos::ZERO);+ if best.epoch == tail.epoch {+ tail.seq.saturating_sub(best.seq)+ } else {+ tail.seq+ }+ }+ } else {+ 0+ };+ ReplicaStatus {+ me: self.me.clone(),+ name: self.name.clone(),+ role: layout.role_of(&self.me),+ owner: owner.clone(),+ owner_name: name_of(&owner),+ i_own: owner == self.me,+ epoch: st.eras.epoch(),+ applied: tail,+ tail,+ peers,+ pending: st+ .outbox+ .items()+ .iter()+ .map(|p| PendingView {+ key: p.key.clone(),+ size: p.meta.as_ref().map_or(0, |m| m.size),+ since_ms: p.ts_ms,+ deleted: p.meta.is_none(),+ })+ .collect(),+ pending_bytes: st.outbox.bytes(),+ copies_now,+ copies_wanted: wanted,+ unreplicated,+ volume_bytes: st.volume_bytes,+ bytes_sent: self.counters.sent.load(Ordering::Relaxed),+ bytes_received: self.counters.received.load(Ordering::Relaxed),+ takeovers: st.takeovers,+ handbacks: st.handbacks,+ fenced_writes: st.fenced_writes,+ conflict_copies: st.conflict_copies,+ }+ }++ pub(super) async fn summary(&self, run_on: Option<&str>) -> Summary {+ let s = self.status().await;+ let st = self.st.lock().await;+ let facts = self.my_facts();+ let layout = self.layout_locked(&st);+ let mut holders_up = vec![];+ let mut holders_away = vec![];+ let mut hold = |name: String, up: bool| {+ if up {+ holders_up.push(name)+ } else {+ holders_away.push(name)+ }+ };+ if layout.role_of(&self.me) == Role::Replica {+ hold(self.name.clone(), true);+ }+ for p in &s.peers {+ if p.role == Role::Replica {+ hold(p.name.clone(), p.health == "up" || p.health == "slow");+ }+ }+ let owner_name = s.owner_name.clone();+ let pending_for = if s.pending.is_empty() {+ None+ } else {+ Some(owner_name)+ };+ Summary {+ this_machine: self.name.clone(),+ run_on: run_on.map(str::to_string),+ machines: st.members.len(),+ holders_up,+ holders_away,+ pending_here: s.pending.len() as u64,+ pending_for,+ this_sleeps: facts.sleep_prone,+ sleep_blocked: facts.sleep_prevented,+ }+ }+}diff --git a/libs/cua/crates/cua-volume/src/replica/oplog.rs b/libs/cua/crates/cua-volume/src/replica/oplog.rsnew file mode 100644index 0000000000..56af9f13c0--- /dev/null+++ b/libs/cua/crates/cua-volume/src/replica/oplog.rs@@ -0,0 +1,325 @@+// SPDX-License-Identifier: FSL-1.1-MIT+// Copyright (c) 2026 Cua AI, Inc.++//! The replication log: what the owner did, in order.+//!+//! One JSON object per line in `oplog.jsonl`. Every machine that holds a+//! copy keeps the records it applied (the owner the ones it made), so any+//! of them can become the owner and replay history to the others. A record+//! carries the previous head of its key (`prev`), so a machine that finds+//! itself on the wrong side of a takeover can move its heads back without+//! losing a version.++use std::io::Write as _;+use std::path::{Path, PathBuf};++use serde::{Deserialize, Serialize};++use super::era::Pos;+use crate::backend::ObjectMeta;+use crate::{Error, Result};++/// What happened to one key.+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]+#[serde(tag = "op", rename_all = "snake_case")]+pub enum Op {+ /// `meta` became the head of its key.+ Put {+ meta: ObjectMeta,+ #[serde(default, skip_serializing_if = "Option::is_none")]+ prev: Option<ObjectMeta>,+ },+ /// `key` was deleted (a marker with `version`).+ Delete {+ key: String,+ version: String,+ #[serde(default, skip_serializing_if = "Option::is_none")]+ prev: Option<ObjectMeta>,+ },+}++impl Op {+ pub fn key(&self) -> &str {+ match self {+ Op::Put { meta, .. } => &meta.key,+ Op::Delete { key, .. } => key,+ }+ }++ pub fn prev(&self) -> Option<&ObjectMeta> {+ match self {+ Op::Put { prev, .. } | Op::Delete { prev, .. } => prev.as_ref(),+ }+ }+}++/// One log entry.+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]+pub struct Record {+ pub pos: Pos,+ pub op: Op,+ /// Wall time at the owner (display only).+ #[serde(default)]+ pub at_ms: u64,+}++#[derive(Serialize, Deserialize)]+struct Header {+ floor: Pos,+}++/// The log on disk plus the records kept in memory.+#[derive(Debug)]+pub struct OpLog {+ path: PathBuf,+ recs: Vec<Record>,+ keep: usize,+ /// Everything at or before this position is gone (trimmed, or the+ /// machine joined by snapshot): a replica behind it needs a snapshot.+ floor: Pos,+}++impl OpLog {+ /// Opens (creating) `dir/oplog.jsonl`. A torn last line (a crash while+ /// appending) is dropped.+ pub fn open(dir: &Path, keep: usize) -> Result<OpLog> {+ std::fs::create_dir_all(dir)?;+ let path = dir.join("oplog.jsonl");+ let mut recs = vec![];+ let mut floor = Pos::ZERO;+ if let Ok(text) = std::fs::read_to_string(&path) {+ for line in text.lines() {+ if let Ok(h) = serde_json::from_str::<Header>(line) {+ floor = h.floor;+ continue;+ }+ match serde_json::from_str::<Record>(line) {+ Ok(r) => recs.push(r),+ Err(_) => break,+ }+ }+ }+ let log = OpLog {+ path,+ recs,+ keep,+ floor,+ };+ log.rewrite()?;+ Ok(log)+ }++ fn rewrite(&self) -> Result<()> {+ let mut out = Vec::new();+ serde_json::to_writer(&mut out, &Header { floor: self.floor })?;+ out.push(b'\n');+ for r in &self.recs {+ serde_json::to_writer(&mut out, r)?;+ out.push(b'\n');+ }+ cua_home::guard_write(&self.path)?;+ let tmp = self.path.with_extension("tmp");+ {+ let mut f = std::fs::File::create(&tmp)?;+ f.write_all(&out)?;+ f.sync_all()?;+ }+ std::fs::rename(&tmp, &self.path)?;+ Ok(())+ }++ /// The newest record's position (zero when empty).+ pub fn tail(&self) -> Pos {+ self.recs.last().map_or(self.floor, |r| r.pos)+ }++ /// Starts an empty log at `pos` (a machine that joined by snapshot has+ /// no records before it).+ pub fn reset_to(&mut self, pos: Pos) -> Result<()> {+ self.recs.clear();+ self.floor = pos;+ self.rewrite()+ }++ /// Wall time of the newest record (what it recorded as `at_ms`).+ pub fn newest_at_ms(&self) -> Option<u64> {+ self.recs.iter().map(|r| r.at_ms).max()+ }++ /// The oldest record kept.+ pub fn head(&self) -> Option<Pos> {+ self.recs.first().map(|r| r.pos)+ }++ pub fn len(&self) -> usize {+ self.recs.len()+ }++ pub fn is_empty(&self) -> bool {+ self.recs.is_empty()+ }++ /// Appends `rec` (its position must be after the tail) and makes it+ /// durable.+ pub fn append(&mut self, rec: Record) -> Result<()> {+ if rec.pos <= self.tail() {+ return Err(Error::Backend(format!(+ "oplog: {} is not after {}",+ rec.pos,+ self.tail()+ )));+ }+ let mut line = serde_json::to_vec(&rec)?;+ line.push(b'\n');+ cua_home::guard_write(&self.path)?;+ let mut f = std::fs::OpenOptions::new().append(true).open(&self.path)?;+ f.write_all(&line)?;+ f.sync_data()?;+ self.recs.push(rec);+ if self.recs.len() > self.keep.saturating_mul(2) {+ let drop = self.recs.len() - self.keep;+ if let Some(last) = self.recs[drop - 1..drop].first() {+ self.floor = last.pos;+ }+ self.recs.drain(..drop);+ self.rewrite()?;+ }+ Ok(())+ }++ /// Appends many records with one disk barrier (a batch of writes).+ pub fn append_batch(&mut self, recs: Vec<Record>) -> Result<()> {+ if recs.is_empty() {+ return Ok(());+ }+ let mut prev = self.tail();+ let mut bytes = Vec::new();+ for r in &recs {+ if r.pos <= prev {+ return Err(Error::Backend(format!(+ "oplog: {} is not after {prev}",+ r.pos+ )));+ }+ prev = r.pos;+ serde_json::to_writer(&mut bytes, r)?;+ bytes.push(b'\n');+ }+ cua_home::guard_write(&self.path)?;+ let mut f = std::fs::OpenOptions::new().append(true).open(&self.path)?;+ f.write_all(&bytes)?;+ f.sync_data()?;+ self.recs.extend(recs);+ if self.recs.len() > self.keep.saturating_mul(2) {+ let drop = self.recs.len() - self.keep;+ if let Some(last) = self.recs[drop - 1..drop].first() {+ self.floor = last.pos;+ }+ self.recs.drain(..drop);+ self.rewrite()?;+ }+ Ok(())+ }++ /// Records after `pos`, at most `max`. `None` when the log no longer+ /// reaches back to `pos` (the caller sends a snapshot instead).+ pub fn after(&self, pos: Pos, max: usize) -> Option<Vec<Record>> {+ if pos < self.floor {+ return None;+ }+ let i = self.recs.partition_point(|r| r.pos <= pos);+ Some(self.recs[i..].iter().take(max).cloned().collect())+ }++ /// Records in `(after, up_to]`.+ pub fn range(&self, after: Pos, up_to: Pos) -> Vec<Record> {+ self.recs+ .iter()+ .filter(|r| r.pos > after && r.pos <= up_to)+ .cloned()+ .collect()+ }++ /// Drops every record after `keep` and returns them (newest first):+ /// the divergent tail.+ pub fn cut_after(&mut self, keep: Pos) -> Result<Vec<Record>> {+ let i = self.recs.partition_point(|r| r.pos <= keep);+ let mut tail: Vec<Record> = self.recs.drain(i..).collect();+ tail.reverse();+ if !tail.is_empty() {+ self.rewrite()?;+ }+ Ok(tail)+ }++ /// Whether the record at `pos` is held.+ pub fn has(&self, pos: Pos) -> bool {+ self.recs.binary_search_by(|r| r.pos.cmp(&pos)).is_ok()+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ fn put(epoch: u64, seq: u64, key: &str) -> Record {+ Record {+ pos: Pos { epoch, seq },+ op: Op::Put {+ meta: ObjectMeta {+ key: key.into(),+ size: 1,+ etag: "e".into(),+ version: format!("v{epoch}.{seq}"),+ modified_ms: 0,+ },+ prev: None,+ },+ at_ms: 0,+ }+ }++ #[test]+ fn append_replay_cut_and_reopen() {+ let d = tempfile::tempdir().unwrap();+ let mut l = OpLog::open(d.path(), 100).unwrap();+ for s in 1..=5 {+ l.append(put(1, s, &format!("k{s}"))).unwrap();+ }+ assert!(l.append(put(1, 5, "dup")).is_err());+ assert_eq!(l.tail(), Pos { epoch: 1, seq: 5 });+ assert_eq!(l.after(Pos { epoch: 1, seq: 3 }, 10).unwrap().len(), 2);+ assert_eq!(l.after(Pos::ZERO, 10).unwrap().len(), 5);+ let tail = l.cut_after(Pos { epoch: 1, seq: 3 }).unwrap();+ assert_eq!(tail.len(), 2);+ assert_eq!(tail[0].pos.seq, 5, "newest first");+ l.append(put(2, 1, "later")).unwrap();+ let again = OpLog::open(d.path(), 100).unwrap();+ assert_eq!(again.tail(), Pos { epoch: 2, seq: 1 });+ assert_eq!(again.len(), 4);+ }++ #[test]+ fn a_torn_last_line_is_dropped_and_trimming_asks_for_a_snapshot() {+ let d = tempfile::tempdir().unwrap();+ {+ let mut l = OpLog::open(d.path(), 4).unwrap();+ for s in 1..=3 {+ l.append(put(1, s, "k")).unwrap();+ }+ }+ let p = d.path().join("oplog.jsonl");+ let mut f = std::fs::OpenOptions::new().append(true).open(&p).unwrap();+ f.write_all(b"{\"pos\":{\"epoch\":1,\"se").unwrap();+ let mut l = OpLog::open(d.path(), 4).unwrap();+ assert_eq!(l.len(), 3);+ for s in 4..=20 {+ l.append(put(1, s, "k")).unwrap();+ }+ // The log kept only the newest; a new replica needs a snapshot.+ assert!(l.after(Pos::ZERO, 10).is_none());+ let head = l.head().unwrap();+ assert!(l.after(head, 100).is_some());+ assert!(l.after(Pos { epoch: 1, seq: 1 }, 100).is_none());+ }+}diff --git a/libs/cua/crates/cua-volume/src/replica/outbox.rs b/libs/cua/crates/cua-volume/src/replica/outbox.rsnew file mode 100644index 0000000000..0f01315086--- /dev/null+++ b/libs/cua/crates/cua-volume/src/replica/outbox.rs@@ -0,0 +1,104 @@+// SPDX-License-Identifier: FSL-1.1-MIT+// Copyright (c) 2026 Cua AI, Inc.++//! Changes this machine saved that the owner has not taken yet.+//!+//! One JSON file per change in `<state>/outbox/`. A change survives a+//! restart; it leaves the outbox only when the owner says what it did with+//! it. The bytes of a pending write are a version in the spool store (or, for+//! the changes a machine hands over when it steps down, a version in its data+//! store), so history is never rewritten.++use std::path::{Path, PathBuf};++use super::wire::Pending;+use crate::{Error, Result};++#[derive(Debug)]+pub struct Outbox {+ dir: PathBuf,+ items: Vec<Pending>,+}++impl Outbox {+ pub fn open(state: &Path) -> Result<Outbox> {+ let dir = state.join("outbox");+ std::fs::create_dir_all(&dir)?;+ let mut items = vec![];+ for e in std::fs::read_dir(&dir)? {+ let p = e?.path();+ if p.extension().is_some_and(|x| x == "json")+ && let Ok(b) = std::fs::read(&p)+ && let Ok(item) = serde_json::from_slice::<Pending>(&b)+ {+ items.push(item);+ }+ }+ items.sort_by(|a, b| (a.ts_ms, &a.id).cmp(&(b.ts_ms, &b.id)));+ Ok(Outbox { dir, items })+ }++ fn file(&self, id: &str) -> PathBuf {+ self.dir.join(format!("{id}.json"))+ }++ pub fn add(&mut self, p: Pending) -> Result<()> {+ let f = self.file(&p.id);+ cua_home::guard_write(&f)?;+ let tmp = f.with_extension("tmp");+ std::fs::write(&tmp, serde_json::to_vec(&p)?)?;+ std::fs::rename(&tmp, &f)?;+ self.items.push(p);+ Ok(())+ }++ pub fn remove(&mut self, id: &str) -> Result<Option<Pending>> {+ let Some(i) = self.items.iter().position(|p| p.id == id) else {+ return Ok(None);+ };+ let p = self.items.remove(i);+ match std::fs::remove_file(self.file(id)) {+ Ok(()) => {}+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}+ Err(e) => return Err(Error::from(e)),+ }+ Ok(Some(p))+ }++ /// Pending changes of the same writer chain on after `old` (a version+ /// of ours the owner stored as `new`): they now follow `new`.+ pub fn rebase(&mut self, old: &str, new: &str) -> Result<()> {+ for i in 0..self.items.len() {+ if self.items[i].base.as_deref() == Some(old) {+ self.items[i].base = Some(new.to_string());+ let f = self.file(&self.items[i].id);+ std::fs::write(&f, serde_json::to_vec(&self.items[i])?)?;+ }+ }+ Ok(())+ }++ pub fn items(&self) -> &[Pending] {+ &self.items+ }++ pub fn len(&self) -> usize {+ self.items.len()+ }++ pub fn is_empty(&self) -> bool {+ self.items.is_empty()+ }++ pub fn has_key(&self, key: &str) -> bool {+ self.items.iter().any(|p| p.key == key)+ }++ /// Bytes of the writes waiting (what a status line shows).+ pub fn bytes(&self) -> u64 {+ self.items+ .iter()+ .filter_map(|p| p.meta.as_ref().map(|m| m.size))+ .sum()+ }+}diff --git a/libs/cua/crates/cua-volume/src/replica/owner.rs b/libs/cua/crates/cua-volume/src/replica/owner.rsnew file mode 100644index 0000000000..4b92e8b568--- /dev/null+++ b/libs/cua/crates/cua-volume/src/replica/owner.rs@@ -0,0 +1,1270 @@+// SPDX-License-Identifier: FSL-1.1-MIT+// Copyright (c) 2026 Cua AI, Inc.++//! Writes: at the owner (serialized, logged, shipped), forwarded from other+//! machines, and saved for later when the owner is away.++use std::collections::HashMap;+use std::path::PathBuf;+use std::sync::Arc;+use std::sync::atomic::Ordering;+use std::time::{Duration, Instant};++use super::era::Pos;+use super::link::{CallError, Link};+use super::merge::{self, Resolution};+use super::node::{Inner, State};+use super::oplog::{Op, Record};+use super::status::Durability;+use super::wire::{Chan, Cond, Msg, Outcome, Pending, Rep, Req, Src};+use crate::backend::{Backend, Condition, ObjectMeta};+use crate::fs::{ApplySource, FsBackend};+use crate::{Error, Result};++/// What the owner says when asked to write and it is not the owner (the+/// caller looks up the new owner and retries).+pub(super) const NOT_OWNER: &str = "this machine does not keep the volume now";++pub(super) fn is_not_owner(e: &Error) -> bool {+ matches!(e, Error::Backend(m) if m == NOT_OWNER)+}++/// The bytes of a write.+pub(super) enum Body {+ Bytes(Vec<u8>),+ File(PathBuf),+}++impl Body {+ pub fn size(&self) -> u64 {+ match self {+ Body::Bytes(b) => b.len() as u64,+ Body::File(p) => std::fs::metadata(p).map_or(0, |m| m.len()),+ }+ }+}++/// Why a forward did not happen.+pub(super) enum Fwd {+ /// No answer from the owner.+ Unreachable(String),+ /// The owner answered with an error.+ Remote(Error),+}++fn unreachable_msg(owner: &str) -> String {+ format!("{owner}, which keeps this volume in order, cannot be reached right now")+}++#[derive(serde::Serialize, serde::Deserialize, Default)]+struct Intent {+ keys: Vec<(String, bool)>,+}++impl Inner {+ fn intent_path(&self) -> PathBuf {+ self.dir.join("intent.json")+ }++ fn write_intent(&self, keys: &[(&str, bool)]) -> Result<()> {+ let i = Intent {+ keys: keys.iter().map(|(k, d)| (k.to_string(), *d)).collect(),+ };+ let p = self.intent_path();+ cua_home::guard_write(&p)?;+ std::fs::write(p, serde_json::to_vec(&i)?)?;+ Ok(())+ }++ fn clear_intent(&self) {+ let _ = std::fs::remove_file(self.intent_path());+ }++ /// A crash between the store write and the log append leaves a head+ /// the log does not know. Make it known.+ pub(super) async fn recover_intent(&self) -> Result<()> {+ let Ok(b) = std::fs::read(self.intent_path()) else {+ return Ok(());+ };+ let i: Intent = serde_json::from_slice(&b).unwrap_or_default();+ let mut st = self.st.lock().await;+ if st.eras.owner() == self.me {+ for (key, deleted) in i.keys {+ let head = self.store.head(&key).await?;+ let known = st.log.range(Pos::ZERO, st.log.tail());+ let last = known.iter().rev().find(|r| r.op.key() == key);+ match (deleted, head) {+ (false, Some(h)) => {+ let logged = matches!(last.map(|r| &r.op), Some(Op::Put { meta, .. }) if meta.version == h.version);+ if !logged {+ self.commit_locked(+ &mut st,+ Op::Put {+ meta: h,+ prev: None,+ },+ )?;+ }+ }+ (true, None) => {+ let logged = matches!(last.map(|r| &r.op), Some(Op::Delete { .. }));+ if !logged+ && let Ok(vs) = self.store.versions(&key).await+ && let Some(v) = vs.iter().find(|v| v.deleted)+ {+ self.commit_locked(+ &mut st,+ Op::Delete {+ key: key.clone(),+ version: v.version.clone(),+ prev: None,+ },+ )?;+ }+ }+ _ => {}+ }+ }+ }+ self.clear_intent();+ Ok(())+ }++ pub(super) fn next_pos(&self, st: &State) -> Pos {+ let epoch = st.eras.epoch();+ let t = st.log.tail();+ Pos {+ epoch,+ seq: if t.epoch == epoch { t.seq + 1 } else { 1 },+ }+ }++ /// Appends `op` to the log at the next position (state lock held).+ pub(super) fn commit_locked(&self, st: &mut State, op: Op) -> Result<Pos> {+ let pos = self.next_pos(st);+ st.log.append(Record {+ pos,+ op,+ at_ms: self.wall_ms(),+ })?;+ st.written_at.insert(pos, Instant::now());+ while st.written_at.len() > 4096 {+ st.written_at.pop_first();+ }+ self.applied_tx.send_replace(pos);+ Ok(pos)+ }++ /// A write at the owner.+ pub(super) async fn owner_put(+ &self,+ key: &str,+ body: Body,+ cond: Condition,+ ) -> Result<(ObjectMeta, Pos)> {+ match self.owner_put_try(key, body, cond).await {+ Ok(r) => r,+ Err(_) => Err(Error::Backend(NOT_OWNER.into())),+ }+ }++ /// A write at the owner; hands the body back when this machine is not+ /// the owner (any more).+ pub(super) async fn owner_put_try(+ &self,+ key: &str,+ body: Body,+ cond: Condition,+ ) -> std::result::Result<Result<(ObjectMeta, Pos)>, Body> {+ let mut st = self.st.lock().await;+ if st.eras.owner() != self.me {+ return Err(body);+ }+ let r = self.owner_put_locked(&mut st, key, body, cond).await;+ drop(st);+ self.wake.notify_one();+ Ok(r)+ }++ async fn owner_put_locked(+ &self,+ st: &mut State,+ key: &str,+ body: Body,+ cond: Condition,+ ) -> Result<(ObjectMeta, Pos)> {+ let prev = self.store.head(key).await?;+ self.write_intent(&[(key, false)])?;+ let meta = match body {+ Body::Bytes(b) => self.store.put(key, b, cond).await,+ Body::File(p) => self.store.put_file(key, &p, cond).await,+ };+ let meta = match meta {+ Ok(m) => m,+ Err(e) => {+ self.clear_intent();+ return Err(e);+ }+ };+ let pos = self.commit_locked(+ st,+ Op::Put {+ meta: meta.clone(),+ prev,+ },+ )?;+ self.clear_intent();+ Ok((meta, pos))+ }++ pub(super) async fn owner_delete(&self, key: &str, cond: Condition) -> Result<Pos> {+ let mut st = self.st.lock().await;+ if st.eras.owner() != self.me {+ return Err(Error::Backend(NOT_OWNER.into()));+ }+ let r = self.owner_delete_locked(&mut st, key, cond).await;+ drop(st);+ self.wake.notify_one();+ r+ }++ async fn owner_delete_locked(&self, st: &mut State, key: &str, cond: Condition) -> Result<Pos> {+ let prev = self.store.head(key).await?;+ self.write_intent(&[(key, true)])?;+ let version = match self.store.delete_versioned(key, cond).await {+ Ok(v) => v,+ Err(e) => {+ self.clear_intent();+ return Err(e);+ }+ };+ let pos = self.commit_locked(+ st,+ Op::Delete {+ key: key.to_string(),+ version,+ prev,+ },+ )?;+ self.clear_intent();+ Ok(pos)+ }++ /// Copies holding `pos`, copies wanted, and copies that could still+ /// hold it (machines holding it, plus holders that are up and have not+ /// acknowledged yet) (state lock held).+ fn copies_at(&self, st: &State, pos: Pos) -> (usize, usize, usize) {+ let layout = self.layout_locked(st);+ let want = layout.replicas.len().max(1);+ let mut copies = 1;+ let mut possible = 1;+ for r in &layout.replicas {+ if *r == self.me {+ continue;+ }+ if let Some(p) = st.peers.get(r)+ && !p.last_health.is_down()+ {+ if p.acked >= pos {+ copies += 1;+ possible += 1;+ } else if self.ctl_link(r).is_some()+ && p.missed_acks < 2+ && p.ack_rtt_ms * 2.0 < self.cfg.ack_wait.as_millis() as f64+ {+ possible += 1;+ }+ }+ }+ (copies, want, possible)+ }++ /// Waits up to `ack_wait` for enough copies of `pos`; the write never+ /// fails because a machine is away.+ pub(super) async fn wait_durable(&self, pos: Pos) -> (usize, bool) {+ let deadline = Instant::now() + self.cfg.ack_wait;+ let mut rx = self.acks.subscribe();+ loop {+ let (copies, want, possible) = {+ let st = self.st.lock().await;+ self.copies_at(&st, pos)+ };+ if want >= 2 && copies >= want {+ return (copies, true);+ }+ // One machine, or the other holder is known to be away: there is+ // nobody to wait for, so a write never waits (a closed laptop+ // costs the studio nothing).+ if want < 2 || possible < want {+ return (copies, false);+ }+ let left = deadline.saturating_duration_since(Instant::now());+ if left.is_zero() {+ // Whoever did not answer in time is not waited for again+ // until it answers.+ let mut st = self.st.lock().await;+ let layout = self.layout_locked(&st);+ for r in layout.replicas.iter().filter(|r| **r != self.me) {+ if let Some(p) = st.peers.get_mut(r)+ && p.acked < pos+ {+ p.missed_acks = p.missed_acks.saturating_add(1);+ }+ }+ return (copies, false);+ }+ tokio::select! {+ _ = rx.changed() => {}+ _ = tokio::time::sleep(left) => {}+ }+ }+ }++ pub(super) fn durability(copies: usize, durable: bool) -> Durability {+ if durable {+ Durability::Replicated(copies)+ } else {+ Durability::Local(copies)+ }+ }++ /// A forwarded `Backend` call, at the owner or at any machine holding a+ /// copy (reads).+ pub(super) async fn handle_req(+ self: &Arc<Self>,+ _link: &Arc<Link>,+ req: Req,+ data: Vec<u8>,+ ) -> (Rep, Vec<u8>) {+ let fail = |e: Error| (Rep::fail(&e), vec![]);+ match req {+ Req::Put { key, cond } => {+ match self.owner_put(&key, Body::Bytes(data), cond.into()).await {+ Ok((meta, pos)) => {+ let (copies, durable) = self.wait_durable(pos).await;+ (+ Rep::Meta {+ meta,+ pos,+ copies: copies as u32,+ durable,+ },+ vec![],+ )+ }+ Err(e) => fail(e),+ }+ }+ Req::Delete { key, cond } => match self.owner_delete(&key, cond.into()).await {+ Ok(pos) => {+ let (copies, durable) = self.wait_durable(pos).await;+ (+ Rep::Done {+ pos,+ copies: copies as u32,+ durable,+ },+ vec![],+ )+ }+ Err(e) => fail(e),+ },+ Req::Head { key } => match self.store.head(&key).await {+ Ok(meta) => (Rep::Head { meta }, vec![]),+ Err(e) => fail(e),+ },+ Req::Get { key, version } => match self.store.get(&key, version.as_deref()).await {+ Ok((bytes, meta)) => (Rep::Bytes { meta }, bytes),+ Err(e) => fail(e),+ },+ Req::GetRange {+ key,+ version,+ offset,+ len,+ } => match self.store.get_range(&key, &version, offset, len).await {+ Ok(bytes) => (Rep::Range, bytes),+ Err(e) => fail(e),+ },+ Req::List { prefix } => match self.store.list(&prefix).await {+ Ok(items) => (Rep::List { items }, vec![]),+ Err(e) => fail(e),+ },+ Req::ListDir { folder } => match self.store.list_dir(&folder).await {+ Ok((files, folders)) => (Rep::ListDir { files, folders }, vec![]),+ Err(e) => fail(e),+ },+ Req::Versions { key } => match self.store.versions(&key).await {+ Ok(items) => (Rep::Versions { items }, vec![]),+ Err(e) => fail(e),+ },+ Req::Heads { after_key, max } => {+ let at = self.st.lock().await.log.tail();+ match self.store.list_after("", &after_key, max).await {+ Ok(items) => {+ let done = items.len() < max;+ (Rep::Heads { items, done, at }, vec![])+ }+ Err(e) => fail(e),+ }+ }+ }+ }++ // ---- the owner takes pending changes --------------------------------++ pub(super) async fn handle_submit(+ self: &Arc<Self>,+ link: &Arc<Link>,+ ops: Vec<Pending>,+ data: Vec<u8>,+ ) -> Msg {+ if !self.st.lock().await.eras.owner().eq(&self.me) {+ return Msg::Error {+ kind: "not_owner".into(),+ message: NOT_OWNER.into(),+ };+ }+ let mut remap: HashMap<String, String> = HashMap::new();+ let mut outcomes = Vec::with_capacity(ops.len());+ for op in &ops {+ let o = self.apply_pending(Some(link), op, &data, &mut remap).await;+ outcomes.push(o);+ }+ self.wake.notify_one();+ Msg::Submitted { outcomes }+ }++ async fn fetch_body(+ self: &Arc<Self>,+ link: Option<&Arc<Link>>,+ op: &Pending,+ data: &[u8],+ ) -> Result<Body> {+ let meta = op.meta.as_ref().expect("a put");+ // The owner's own outbox: the bytes are on this machine.+ let Some(link) = link else {+ let path = match &op.src {+ Src::Spool { key, version } => self.spool.version_file(key, version)?,+ Src::Data { key, version } => self.store.version_file(key, version)?,+ };+ return Ok(Body::File(path));+ };+ if let Some((off, len)) = op.body {+ let (o, l) = (off as usize, len as usize);+ let bytes = data+ .get(o..o + l)+ .ok_or_else(|| Error::Backend("a body outside its frame".into()))?;+ return Ok(Body::Bytes(bytes.to_vec()));+ }+ let dl = match self.data_link(&link.peer).await {+ Some(l) => l,+ None => return Err(Error::Backend("no data link to the writer".into())),+ };+ let seed = self.store.head(&op.key).await.ok().flatten();+ let path = self.pull(&dl, op.src.clone(), meta, seed.as_ref()).await?;+ Ok(Body::File(path))+ }++ async fn apply_pending(+ self: &Arc<Self>,+ link: Option<&Arc<Link>>,+ op: &Pending,+ data: &[u8],+ remap: &mut HashMap<String, String>,+ ) -> Outcome {+ let base = op+ .base+ .as_ref()+ .map(|b| remap.get(b).cloned().unwrap_or_else(|| b.clone()));+ let Some(meta) = op.meta.as_ref() else {+ // A delete.+ let mut st = self.st.lock().await;+ if st.eras.owner() != self.me {+ return Outcome::Failed {+ message: NOT_OWNER.into(),+ };+ }+ let head = match self.store.head(&op.key).await {+ Ok(h) => h,+ Err(e) => {+ return Outcome::Failed {+ message: e.to_string(),+ };+ }+ };+ return match head {+ None => Outcome::Merged,+ Some(h) if Some(&h.version) == base.as_ref() => {+ match self+ .owner_delete_locked(&mut st, &op.key, Condition::None)+ .await+ {+ Ok(pos) => Outcome::Deleted { pos },+ Err(e) => Outcome::Failed {+ message: e.to_string(),+ },+ }+ }+ // Someone wrote after the deleter last looked: the write stays.+ Some(_) => Outcome::Kept,+ };+ };+ // The head may be this very change, made visible here while it+ // waited (a version of its own, never written to the log). The owner+ // must log it, not take it for content it already holds.+ if matches!(Condition::from(op.cond.clone()), Condition::None) {+ let mut st = self.st.lock().await;+ if st.eras.owner() == self.me+ && let Ok(Some(h)) = self.store.head(&op.key).await+ && h.version == meta.version+ {+ return match self.commit_locked(+ &mut st,+ Op::Put {+ meta: h.clone(),+ prev: None,+ },+ ) {+ Ok(pos) => {+ drop(st);+ self.wake.notify_one();+ Outcome::Applied { meta: h, pos }+ }+ Err(e) => Outcome::Failed {+ message: e.to_string(),+ },+ };+ }+ }+ // Content first, outside the lock.+ let cond: Condition = op.cond.clone().into();+ let head0 = self.store.head(&op.key).await.ok().flatten();+ let have_same = head0.as_ref().is_some_and(|h| h.etag == meta.etag);+ let body = if have_same {+ None+ } else {+ match self.fetch_body(link, op, data).await {+ Ok(b) => Some(b),+ Err(e) => {+ return Outcome::Failed {+ message: e.to_string(),+ };+ }+ }+ };+ let mut st = self.st.lock().await;+ if st.eras.owner() != self.me {+ return Outcome::Failed {+ message: NOT_OWNER.into(),+ };+ }+ let head = match self.store.head(&op.key).await {+ Ok(h) => h,+ Err(e) => {+ return Outcome::Failed {+ message: e.to_string(),+ };+ }+ };+ if let Some(h) = &head+ && h.etag == meta.etag+ && matches!(cond, Condition::None)+ {+ remap.insert(meta.version.clone(), h.version.clone());+ return Outcome::Merged;+ }+ let Some(body) = body else {+ // The head had this content when we looked, and changed since.+ return Outcome::Failed {+ message: "the key changed while it was being merged; it will be tried again".into(),+ };+ };+ let clean = match (&head, &base) {+ (None, _) => true,+ (Some(h), Some(b)) => &h.version == b,+ (Some(_), None) => false,+ };+ if !matches!(cond, Condition::None) {+ // A conditional write is evaluated as written; no merging.+ return match self.owner_put_locked(&mut st, &op.key, body, cond).await {+ Ok((m, pos)) => {+ remap.insert(meta.version.clone(), m.version.clone());+ Outcome::Applied { meta: m, pos }+ }+ Err(Error::Precondition(m)) => Outcome::Precondition { message: m },+ Err(e) => Outcome::Failed {+ message: e.to_string(),+ },+ };+ }+ // An order (fence, lease, run record) is judged by its order even+ // when the change chains onto a version the owner just kept.+ if clean+ && merge::is_ordered(&op.key)+ && let Some(h) = &head+ && h.size <= self.cfg.inline_max as u64 * 4+ && meta.size <= self.cfg.inline_max as u64 * 4+ {+ let head_bytes = self.store.get(&op.key, None).await.map(|(b, _)| b);+ let in_bytes = match &body {+ Body::Bytes(b) => Ok(b.clone()),+ Body::File(p) => std::fs::read(p).map_err(Error::from),+ };+ if let (Ok(hb), Ok(ib)) = (head_bytes, in_bytes)+ && merge::resolve(&op.key, &hb, &ib, &op.name, op.ts_ms) == Resolution::KeepHead+ {+ remap.insert(meta.version.clone(), h.version.clone());+ return Outcome::Merged;+ }+ }+ if clean {+ return match self+ .owner_put_locked(&mut st, &op.key, body, Condition::None)+ .await+ {+ Ok((m, pos)) => {+ remap.insert(meta.version.clone(), m.version.clone());+ Outcome::Applied { meta: m, pos }+ }+ Err(e) => Outcome::Failed {+ message: e.to_string(),+ },+ };+ }+ // The key changed on the owner since the writer looked.+ let head = head.expect("not clean implies a head");+ let incoming_small = meta.size <= self.cfg.inline_max as u64 * 4+ && head.size <= self.cfg.inline_max as u64 * 4;+ let resolution = if incoming_small {+ let head_bytes = self.store.get(&op.key, None).await.map(|(b, _)| b);+ let in_bytes = match &body {+ Body::Bytes(b) => Ok(b.clone()),+ Body::File(p) => std::fs::read(p).map_err(Error::from),+ };+ match (head_bytes, in_bytes) {+ (Ok(h), Ok(i)) => merge::resolve(&op.key, &h, &i, &op.name, op.ts_ms),+ _ => Resolution::Copy(crate::feed::conflict_path(&op.key, &op.name, op.ts_ms)),+ }+ } else {+ Resolution::Copy(crate::feed::conflict_path(&op.key, &op.name, op.ts_ms))+ };+ match resolution {+ Resolution::KeepHead => {+ remap.insert(meta.version.clone(), head.version.clone());+ Outcome::Merged+ }+ Resolution::TakeIncoming => {+ match self+ .owner_put_locked(&mut st, &op.key, body, Condition::None)+ .await+ {+ Ok((m, pos)) => {+ remap.insert(meta.version.clone(), m.version.clone());+ Outcome::Applied { meta: m, pos }+ }+ Err(e) => Outcome::Failed {+ message: e.to_string(),+ },+ }+ }+ Resolution::Copy(ckey) => {+ st.conflict_copies += 1;+ match self+ .owner_put_locked(&mut st, &ckey, body, Condition::None)+ .await+ {+ Ok((m, pos)) => {+ remap.insert(meta.version.clone(), head.version.clone());+ Outcome::Copy { key: m.key, pos }+ }+ Err(e) => Outcome::Failed {+ message: e.to_string(),+ },+ }+ }+ }+ }++ // ---- writes from any machine ----------------------------------------++ /// Reaches the owner's control link, dialing once if there is none.+ async fn owner_ctl(self: &Arc<Self>, owner: &str) -> Option<Arc<Link>> {+ if let Some(l) = self.ctl_link(owner) {+ return Some(l);+ }+ // One quick try; the tick keeps trying in the background.+ match tokio::time::timeout(Duration::from_secs(3), self.dial_chan(owner, Chan::Control))+ .await+ {+ Ok(Ok(l)) => Some(l),+ _ => self.ctl_link(owner),+ }+ }++ pub(super) async fn write_put(+ self: &Arc<Self>,+ key: &str,+ body: Body,+ cond: Condition,+ ) -> Result<(ObjectMeta, Durability)> {+ let mut body = body;+ for attempt in 0..6u32 {+ let owner = {+ let st = self.st.lock().await;+ st.eras.owner().to_string()+ };+ if owner == self.me {+ match self+ .owner_put_try(+ key,+ std::mem::replace(&mut body, Body::Bytes(vec![])),+ cond.clone(),+ )+ .await+ {+ Ok(Ok((meta, pos))) => {+ let (copies, durable) = self.wait_durable(pos).await;+ return Ok((meta, Self::durability(copies, durable)));+ }+ Ok(Err(e)) => return Err(e),+ // The era moved between looking and writing: look again.+ Err(b) => {+ body = b;+ continue;+ }+ }+ }+ let t0 = Instant::now();+ match self.forward_put(&owner, key, &body, &cond).await {+ Ok((meta, pos, copies, durable)) => {+ let t1 = t0.elapsed();+ self.wait_applied(pos).await;+ tracing::debug!(%key, ?t1, applied_wait = ?t0.elapsed() - t1, %pos, "replica: forwarded write");+ return Ok((meta, Self::durability(copies as usize, durable)));+ }+ Err(Fwd::Remote(e)) if is_not_owner(&e) && attempt < 5 => {+ tokio::time::sleep(self.cfg.timing.interval).await;+ continue;+ }+ Err(Fwd::Remote(e)) => return Err(e),+ Err(Fwd::Unreachable(why)) => {+ if matches!(cond, Condition::None) {+ let meta = self.save_pending_put(key, body).await?;+ return Ok((meta, Durability::Pending));+ }+ // An ordered write cannot wait in an outbox: this+ // machine needs the authority, and takes it as soon as+ // the owner is confirmed gone.+ self.want_authority().await;+ let name = self.owner_name(&owner).await;+ let _ = why;+ return Err(Error::Backend(unreachable_msg(&name)));+ }+ }+ }+ Err(Error::Backend(+ "the volume's owner keeps changing; try again".into(),+ ))+ }++ /// Records that this machine needs to order writes itself.+ pub(super) async fn want_authority(&self) {+ let mut st = self.st.lock().await;+ st.authority_wanted = Some(Instant::now());+ drop(st);+ self.tick_wake.notify_one();+ }++ pub(super) async fn owner_name(&self, id: &str) -> String {+ let st = self.st.lock().await;+ st.members+ .get(id)+ .map(|m| m.name.clone())+ .filter(|n| !n.is_empty())+ .unwrap_or_else(|| id.to_string())+ }++ /// Waits (briefly) for our own copy to apply `pos`, so a read right+ /// after a forwarded write sees it.+ pub(super) async fn wait_applied(&self, pos: Pos) {+ if self.is_client.load(Ordering::SeqCst) {+ return;+ }+ let mut rx = self.applied_tx.subscribe();+ let deadline = Instant::now() + self.cfg.ack_wait * 2;+ while *rx.borrow() < pos {+ let left = deadline.saturating_duration_since(Instant::now());+ if left.is_zero() {+ return;+ }+ tokio::select! {+ r = rx.changed() => { if r.is_err() { return; } }+ _ = tokio::time::sleep(left) => return,+ }+ }+ }++ async fn forward_put(+ self: &Arc<Self>,+ owner: &str,+ key: &str,+ body: &Body,+ cond: &Condition,+ ) -> std::result::Result<(ObjectMeta, Pos, u32, bool), Fwd> {+ let size = body.size();+ if size > self.cfg.push_max as u64 {+ return self.forward_big(owner, key, body, cond).await;+ }+ let bytes = match body {+ Body::Bytes(b) => b.clone(),+ Body::File(p) => std::fs::read(p).map_err(|e| Fwd::Remote(e.into()))?,+ };+ let link = if bytes.len() <= self.cfg.inline_max {+ self.owner_ctl(owner).await+ } else {+ self.data_link(owner).await+ };+ let Some(link) = link else {+ return Err(Fwd::Unreachable("no link".into()));+ };+ let to = self.cfg.data_timeout;+ match link+ .call_strict(+ Msg::Req(Req::Put {+ key: key.to_string(),+ cond: Cond::from(cond.clone()),+ }),+ &bytes,+ to,+ )+ .await+ {+ Ok((+ Msg::Rep(Rep::Meta {+ meta,+ pos,+ copies,+ durable,+ }),+ _,+ )) => Ok((meta, pos, copies, durable)),+ Ok((Msg::Rep(Rep::Fail { kind, message }), _)) => {+ Err(Fwd::Remote(Rep::into_error(&kind, message)))+ }+ Ok(_) => Err(Fwd::Remote(Error::Backend("unexpected reply".into()))),+ Err(CallError::Closed | CallError::Timeout) => {+ Err(Fwd::Unreachable("no answer".into()))+ }+ Err(CallError::Io(e)) => Err(Fwd::Unreachable(e.to_string())),+ }+ }++ /// A large write by a non-owner: spool it, let the owner pull it in+ /// blocks (resumable), and wait for what it did.+ async fn forward_big(+ self: &Arc<Self>,+ owner: &str,+ key: &str,+ body: &Body,+ cond: &Condition,+ ) -> std::result::Result<(ObjectMeta, Pos, u32, bool), Fwd> {+ let Some(link) = self.owner_ctl(owner).await else {+ return Err(Fwd::Unreachable("no link".into()));+ };+ let meta = match body {+ Body::Bytes(b) => self.spool.put(key, b.clone(), Condition::None).await,+ Body::File(p) => self.spool.put_file(key, p, Condition::None).await,+ }+ .map_err(Fwd::Remote)?;+ let p = Pending {+ id: crate::new_id(),+ key: key.to_string(),+ meta: Some(meta.clone()),+ base: None,+ cond: Cond::from(cond.clone()),+ machine: self.me.clone(),+ name: self.name.clone(),+ ts_ms: self.wall_ms(),+ src: Src::Spool {+ key: key.to_string(),+ version: meta.version.clone(),+ },+ body: None,+ };+ let r = link+ .call(Msg::Submit { ops: vec![p] }, &[], self.cfg.data_timeout * 4)+ .await;+ let _ = self.spool.purge_version(key, &meta.version).await;+ match r {+ Ok((Msg::Submitted { outcomes }, _)) => match outcomes.into_iter().next() {+ Some(Outcome::Applied { meta, pos }) => Ok((meta, pos, 1, false)),+ Some(Outcome::Precondition { message }) => {+ Err(Fwd::Remote(Error::Precondition(message)))+ }+ Some(Outcome::Failed { message }) => Err(Fwd::Remote(Error::Backend(message))),+ _ => Err(Fwd::Remote(Error::Backend("unexpected outcome".into()))),+ },+ Ok((Msg::Error { kind, message }, _)) => {+ Err(Fwd::Remote(Rep::into_error(&kind, message)))+ }+ Ok(_) => Err(Fwd::Remote(Error::Backend("unexpected reply".into()))),+ Err(_) => Err(Fwd::Unreachable("no answer".into())),+ }+ }++ /// Many unconditional writes: at the owner they are one batch (one disk+ /// barrier, one run of log records, one wait for the second copy).+ pub(super) async fn write_many(+ self: &Arc<Self>,+ items: Vec<(String, Vec<u8>)>,+ ) -> Result<Vec<ObjectMeta>> {+ let unique = items+ .iter()+ .map(|(k, _)| k.as_str())+ .collect::<std::collections::HashSet<_>>()+ .len()+ == items.len();+ if unique && !items.is_empty() {+ let mut st = self.st.lock().await;+ if st.eras.owner() == self.me {+ let mut prevs = Vec::with_capacity(items.len());+ for (k, _) in &items {+ prevs.push(self.store.head(k).await?);+ }+ let keys: Vec<(&str, bool)> =+ items.iter().map(|(k, _)| (k.as_str(), false)).collect();+ self.write_intent(&keys)?;+ let metas = match self.store.put_many(items).await {+ Ok(m) => m,+ Err(e) => {+ self.clear_intent();+ return Err(e);+ }+ };+ let mut recs = Vec::with_capacity(metas.len());+ let mut pos = self.next_pos(&st);+ let at_ms = self.wall_ms();+ for (meta, prev) in metas.iter().zip(prevs) {+ recs.push(Record {+ pos,+ op: Op::Put {+ meta: meta.clone(),+ prev,+ },+ at_ms,+ });+ pos.seq += 1;+ }+ let last = recs.last().map_or(Pos::ZERO, |r| r.pos);+ let now = Instant::now();+ for r in &recs {+ st.written_at.insert(r.pos, now);+ }+ while st.written_at.len() > 4096 {+ st.written_at.pop_first();+ }+ st.log.append_batch(recs)?;+ self.applied_tx.send_replace(last);+ self.clear_intent();+ drop(st);+ self.wake.notify_one();+ self.wait_durable(last).await;+ return Ok(metas);+ }+ }+ let mut out = Vec::with_capacity(items.len());+ for (k, b) in items {+ out.push(self.write_put(&k, Body::Bytes(b), Condition::None).await?.0);+ }+ Ok(out)+ }++ pub(super) async fn write_delete(+ self: &Arc<Self>,+ key: &str,+ cond: Condition,+ ) -> Result<Durability> {+ for attempt in 0..6u32 {+ let owner = self.st.lock().await.eras.owner().to_string();+ if owner == self.me {+ match self.owner_delete(key, cond.clone()).await {+ Ok(pos) => {+ let (copies, durable) = self.wait_durable(pos).await;+ return Ok(Self::durability(copies, durable));+ }+ Err(e) if is_not_owner(&e) => continue,+ Err(e) => return Err(e),+ }+ }+ let link = self.owner_ctl(&owner).await;+ let r = match link {+ Some(l) => {+ l.call(+ Msg::Req(Req::Delete {+ key: key.to_string(),+ cond: Cond::from(cond.clone()),+ }),+ &[],+ self.cfg.call_timeout,+ )+ .await+ }+ None => Err(CallError::Closed),+ };+ match r {+ Ok((+ Msg::Rep(Rep::Done {+ pos,+ copies,+ durable,+ }),+ _,+ )) => {+ self.wait_applied(pos).await;+ return Ok(Self::durability(copies as usize, durable));+ }+ Ok((Msg::Rep(Rep::Fail { kind, message }), _)) => {+ let e = Rep::into_error(&kind, message);+ if is_not_owner(&e) && attempt < 5 {+ tokio::time::sleep(self.cfg.timing.interval).await;+ continue;+ }+ return Err(e);+ }+ Ok(_) => return Err(Error::Backend("unexpected reply".into())),+ Err(_) => {+ if matches!(cond, Condition::None) {+ self.save_pending_delete(key).await?;+ return Ok(Durability::Pending);+ }+ self.want_authority().await;+ let name = self.owner_name(&owner).await;+ return Err(Error::Backend(unreachable_msg(&name)));+ }+ }+ }+ Err(Error::Backend(+ "the volume's owner keeps changing; try again".into(),+ ))+ }++ // ---- saved for later --------------------------------------------------++ async fn pending_base(&self, key: &str) -> Option<String> {+ {+ let st = self.st.lock().await;+ if let Some(p) = st.outbox.items().iter().rev().find(|p| p.key == key) {+ return p.meta.as_ref().map(|m| m.version.clone());+ }+ }+ if self.is_client.load(Ordering::SeqCst) {+ return None;+ }+ self.store.head(key).await.ok().flatten().map(|h| h.version)+ }++ pub(super) async fn save_pending_put(&self, key: &str, body: Body) -> Result<ObjectMeta> {+ let base = self.pending_base(key).await;+ let meta = match body {+ Body::Bytes(b) => self.spool.put(key, b, Condition::None).await?,+ Body::File(p) => self.spool.put_file(key, &p, Condition::None).await?,+ };+ let mut st = self.st.lock().await;+ st.outbox.add(Pending {+ id: crate::new_id(),+ key: key.to_string(),+ meta: Some(meta.clone()),+ base,+ cond: Cond::None,+ machine: self.me.clone(),+ name: self.name.clone(),+ ts_ms: self.wall_ms(),+ src: Src::Spool {+ key: key.to_string(),+ version: meta.version.clone(),+ },+ body: None,+ })?;+ if !self.is_client.load(Ordering::SeqCst) {+ let path = self.spool.version_file(key, &meta.version)?;+ self.store+ .apply_put(meta.clone(), ApplySource::File { path, link: true })+ .await?;+ }+ Ok(meta)+ }++ pub(super) async fn save_pending_delete(&self, key: &str) -> Result<()> {+ let base = self.pending_base(key).await;+ let mut st = self.st.lock().await;+ st.outbox.add(Pending {+ id: crate::new_id(),+ key: key.to_string(),+ meta: None,+ base,+ cond: Cond::None,+ machine: self.me.clone(),+ name: self.name.clone(),+ ts_ms: self.wall_ms(),+ src: Src::Data {+ key: key.to_string(),+ version: String::new(),+ },+ body: None,+ })?;+ if !self.is_client.load(Ordering::SeqCst) {+ self.store+ .apply_delete(key, &FsBackend::new_version())+ .await?;+ }+ Ok(())+ }++ /// Hands the outbox to the owner once this machine is caught up with it.+ pub(super) async fn submit_outbox(self: &Arc<Self>) {+ if self.drain_own_outbox().await {+ return;+ }+ let (owner, ops, caught_up) = {+ let st = self.st.lock().await;+ if st.outbox.is_empty() || st.eras.owner() == self.me {+ return;+ }+ let owner = st.eras.owner().to_string();+ let owner_tail = st.peers.get(&owner).map_or(Pos::ZERO, |p| p.tail);+ let client = self.is_client.load(Ordering::SeqCst);+ (+ owner,+ st.outbox+ .items()+ .iter()+ .take(200)+ .cloned()+ .collect::<Vec<_>>(),+ client || st.log.tail() >= owner_tail,+ )+ };+ if !caught_up {+ return;+ }+ if self+ .submitting+ .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)+ .is_err()+ {+ return;+ }+ let r = self.submit_ops(&owner, ops).await;+ self.submitting.store(false, Ordering::SeqCst);+ if let Err(e) = r {+ tracing::debug!(error = %e, "replica: the outbox waits");+ }+ }++ /// This machine owns the volume and holds changes it saved while it+ /// did not (it stepped down, then got the volume back): they are applied+ /// here, in order, by the same rules the owner applies anyone's. Returns+ /// whether this machine is the owner (nothing is sent to anyone).+ async fn drain_own_outbox(self: &Arc<Self>) -> bool {+ if self.st.lock().await.eras.owner() != self.me {+ return false;+ }+ let ops: Vec<Pending> = {+ let st = self.st.lock().await;+ st.outbox.items().iter().take(200).cloned().collect()+ };+ if ops.is_empty() {+ return true;+ }+ if self+ .submitting+ .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)+ .is_err()+ {+ return true;+ }+ let mut remap: HashMap<String, String> = HashMap::new();+ for op in &ops {+ let o = self.apply_pending(None, op, &[], &mut remap).await;+ tracing::debug!(key = %op.key, detail = ?o, "replica: applied own saved change");+ if let Outcome::Failed { message } = &o {+ tracing::warn!(key = %op.key, %message, "replica: could not apply a saved change here yet");+ continue;+ }+ let mut st = self.st.lock().await;+ let _ = st.outbox.remove(&op.id);+ if let (Outcome::Applied { meta: new, .. }, Some(old)) = (&o, &op.meta) {+ let _ = st.outbox.rebase(&old.version, &new.version);+ }+ drop(st);+ if let (Some(_), Src::Spool { key, version }) = (&op.meta, &op.src) {+ let _ = self.spool.purge_version(key, version).await;+ }+ }+ self.submitting.store(false, Ordering::SeqCst);+ self.wake.notify_one();+ true+ }++ async fn submit_ops(self: &Arc<Self>, owner: &str, mut ops: Vec<Pending>) -> Result<()> {+ let Some(link) = self.ctl_link(owner) else {+ return Err(Error::Backend("owner not connected".into()));+ };+ // Small bodies travel in the frame; large ones are pulled.+ let mut data: Vec<u8> = vec![];+ for op in &mut ops {+ op.body = None;+ if let Some(m) = &op.meta+ && m.size <= self.cfg.inline_max as u64+ && data.len() + (m.size as usize) < (1 << 20)+ {+ let r = match &op.src {+ Src::Spool { key, version } => self.spool.get(key, Some(version)).await,+ Src::Data { key, version } => self.store.get(key, Some(version)).await,+ };+ if let Ok((bytes, _)) = r {+ op.body = Some((data.len() as u64, bytes.len() as u64));+ data.extend_from_slice(&bytes);+ }+ }+ }+ let n = ops.len();+ let to = self.cfg.data_timeout * 4;+ let (reply, _) = link+ .call(Msg::Submit { ops: ops.clone() }, &data, to)+ .await+ .map_err(|e| Error::Backend(e.to_string()))?;+ let outcomes = match reply {+ Msg::Submitted { outcomes } => outcomes,+ Msg::Error { kind, message } => return Err(Rep::into_error(&kind, message)),+ _ => return Err(Error::Backend("unexpected reply".into())),+ };+ if outcomes.len() != n {+ return Err(Error::Backend("short answer".into()));+ }+ let mut done = vec![];+ let mut st = self.st.lock().await;+ for (op, o) in ops.iter().zip(outcomes) {+ tracing::debug!(key = %op.key, outcome = ?std::mem::discriminant(&o), detail = ?o, "replica: owner took a change");+ if let Outcome::Failed { message } = &o {+ tracing::warn!(key = %op.key, %message, "replica: the owner could not take a change yet");+ continue;+ }+ st.outbox.remove(&op.id)?;+ if let (Outcome::Applied { meta: new, .. }, Some(old)) = (&o, &op.meta) {+ st.outbox.rebase(&old.version, &new.version)?;+ }+ done.push(op.clone());+ }+ drop(st);+ for op in done {+ if let (Some(m), Src::Spool { key, version }) = (&op.meta, &op.src) {+ let _ = m;+ let _ = self.spool.purge_version(key, version).await;+ }+ }+ self.wake.notify_one();+ Ok(())+ }+}diff --git a/libs/cua/crates/cua-volume/src/replica/roles.rs b/libs/cua/crates/cua-volume/src/replica/roles.rsnew file mode 100644index 0000000000..2c4b24b2fc--- /dev/null+++ b/libs/cua/crates/cua-volume/src/replica/roles.rs@@ -0,0 +1,274 @@+// SPDX-License-Identifier: FSL-1.1-MIT+// Copyright (c) 2026 Cua AI, Inc.++//! Who keeps the volume: the machines list, their reliability, and the+//! roles that follow from them.+//!+//! The machines list is the account's (every machine signed in); this module+//! only reads it. Reliability is a number each machine computes about itself+//! (`cua_cluster::reliability::score`: a desktop beats a plugged-in laptop+//! beats a laptop on battery; the machine the user is sitting at loses ties)+//! and shares in its heartbeat. The *same* number places a durable drive, so+//! the drive and its volume land on the same machine.++use serde::{Deserialize, Serialize};++use super::MachineId;++/// What a machine says about itself (in `Hello` and every heartbeat).+#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]+pub struct Facts {+ /// Reliability score; higher keeps the volume. Categories are 1000+ /// apart, so a hysteresis margin of 1000 means "a whole step better".+ pub rank: u64,+ /// Free bytes where the volume's copy lives.+ pub free_disk: u64,+ /// Running on battery.+ pub on_battery: bool,+ /// Can sleep when its lid closes (a laptop).+ pub sleep_prone: bool,+ /// The user is working at this machine right now.+ pub user_here: bool,+ /// A durable run holds a sleep assertion here.+ pub sleep_prevented: bool,+ /// CPU architecture (`arm64`, `amd64`; empty: unknown). Replication is+ /// the same on every architecture; this only tells a takeover across+ /// architectures to say that built programs are not reused.+ #[serde(default)]+ pub arch: String,+}++/// `arm64` or `amd64` for the names the platforms use.+pub fn normalize_arch(a: &str) -> String {+ match a.trim().to_ascii_lowercase().as_str() {+ "aarch64" | "arm64" => "arm64".into(),+ "x86_64" | "amd64" | "x64" => "amd64".into(),+ other => other.to_string(),+ }+}++/// Paths in a workspace that hold programs built for one architecture+/// (build folders and native libraries), as short folder or file names; at+/// most `max`. The rest of a workspace (sources, notes, journals, data) is+/// the same on every architecture.+pub fn arch_specific(keys: impl IntoIterator<Item = String>, max: usize) -> Vec<String> {+ const DIRS: &[&str] = &[+ "target",+ "node_modules",+ ".venv",+ "venv",+ "build",+ "dist",+ "cmake-build-debug",+ ".gradle",+ "Pods",+ ];+ const EXTS: &[&str] = &["o", "so", "dylib", "a", "node", "rlib", "dll", "exe", "pyd"];+ let mut out: Vec<String> = vec![];+ for k in keys {+ let parts: Vec<&str> = k.split('/').collect();+ let hit = parts+ .iter()+ .enumerate()+ .find(|(i, p)| *i + 1 < parts.len() && DIRS.contains(p))+ .map(|(i, _)| format!("{}/", parts[..=i].join("/")))+ .or_else(|| {+ let ext = k.rsplit('.').next()?;+ (k.contains('.') && EXTS.contains(&ext)).then(|| k.clone())+ });+ if let Some(h) = hit {+ // Show the folder after the agent's home, not the whole key.+ let short = h.splitn(3, '/').nth(2).map(str::to_string).unwrap_or(h);+ if !out.contains(&short) {+ out.push(short);+ if out.len() >= max {+ break;+ }+ }+ }+ }+ out+}++/// One machine of the group.+#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]+pub struct Member {+ pub id: MachineId,+ /// The name the machines list shows ("Mac Studio").+ pub name: String,+ pub facts: Facts,+ /// Wall time this machine was last heard from (0: never; display and+ /// the active-member window only).+ #[serde(default)]+ pub last_seen_ms: u64,+}++/// What a machine does for the volume.+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]+#[serde(rename_all = "snake_case")]+pub enum Role {+ /// Holds a full copy and acknowledges writes (the owner is one).+ Replica,+ /// Keeps a cache only and sends its writes to a replica.+ Client,+}++/// The preferred layout.+#[derive(Clone, Debug, PartialEq, Eq)]+pub struct Layout {+ /// The machine that should own the volume.+ pub owner: MachineId,+ /// Machines that hold a full copy, owner first.+ pub replicas: Vec<MachineId>,+}++impl Layout {+ pub fn role_of(&self, id: &str) -> Role {+ if self.replicas.iter().any(|r| r == id) {+ Role::Replica+ } else {+ Role::Client+ }+ }+}++/// Disk a machine needs free to hold a copy of `volume_bytes`: the volume+/// again plus a margin (10 percent, at least 1 GiB).+pub fn disk_needed(volume_bytes: u64) -> u64 {+ volume_bytes.saturating_add((volume_bytes / 10).max(1 << 30))+}++/// The preferred layout for `members`: the `replicas` most reliable+/// machines that have room for a copy, the best of them owning. Ties break+/// by id so every machine computes the same answer.+pub fn layout(members: &[Member], replicas: usize, volume_bytes: u64) -> Layout {+ let need = disk_needed(volume_bytes);+ let mut able: Vec<&Member> = members+ .iter()+ .filter(|m| m.facts.free_disk >= need)+ .collect();+ if able.is_empty() {+ // Nobody reports room: the most reliable machines hold it anyway+ // (a store that is full fails loudly; a refusal helps nobody).+ able = members.iter().collect();+ }+ able.sort_by(|a, b| {+ b.facts+ .rank+ .cmp(&a.facts.rank)+ .then_with(|| a.id.cmp(&b.id))+ });+ let replicas: Vec<MachineId> = able+ .iter()+ .take(replicas.max(1))+ .map(|m| m.id.clone())+ .collect();+ Layout {+ owner: replicas.first().cloned().unwrap_or_default(),+ replicas,+ }+}++/// One class step of reliability (see `cua_cluster::reliability::STEP`).+pub const STEP_RANK: u64 = 1000;++/// Copies to keep: 2 when two or more machines exist, else 1.+pub fn replica_count(machines: usize) -> usize {+ if machines >= 2 { 2 } else { 1 }+}++/// Whether `candidate` is better than the current owner by a whole step.+pub fn clearly_better(candidate: &Facts, owner: &Facts) -> bool {+ candidate.rank >= owner.rank.saturating_add(STEP_RANK)+}++#[cfg(test)]+mod tests {+ use super::*;++ fn m(id: &str, rank: u64, disk_gib: u64) -> Member {+ Member {+ id: id.into(),+ name: id.into(),+ facts: Facts {+ rank,+ free_disk: disk_gib << 30,+ ..Default::default()+ },+ last_seen_ms: 0,+ }+ }++ #[test]+ fn the_most_reliable_machine_owns_and_the_next_holds_a_copy() {+ let ms = [+ m("laptop", 2000, 200),+ m("studio", 9000, 800),+ m("mini", 7000, 500),+ ];+ let l = layout(&ms, 2, 10 << 30);+ assert_eq!(l.owner, "studio");+ assert_eq!(l.replicas, ["studio", "mini"]);+ assert_eq!(l.role_of("laptop"), Role::Client);+ assert_eq!(l.role_of("mini"), Role::Replica);+ }++ #[test]+ fn architecture_names_normalize_and_built_programs_are_found() {+ assert_eq!(normalize_arch("aarch64"), "arm64");+ assert_eq!(normalize_arch("x86_64"), "amd64");+ let keys = [+ "agents/ada/src/main.rs",+ "agents/ada/target/debug/ada",+ "agents/ada/target/release/libx.rlib",+ "agents/ada/notes.md",+ "agents/ada/vendor/libfoo.so",+ "agents/ada/node_modules/x/y.js",+ ]+ .map(String::from);+ let found = arch_specific(keys, 5);+ assert_eq!(found, ["target/", "vendor/libfoo.so", "node_modules/"]);+ }++ #[test]+ fn a_machine_without_room_becomes_a_client() {+ let ms = [m("studio", 9000, 2), m("laptop", 2000, 400)];+ // A 100 GiB volume does not fit the studio's 2 GiB.+ let l = layout(&ms, 2, 100 << 30);+ assert_eq!(l.owner, "laptop");+ assert_eq!(l.replicas, ["laptop"]);+ assert_eq!(l.role_of("studio"), Role::Client);+ // Everyone is full: the most reliable still hold it.+ let l = layout(&[m("a", 1, 0), m("b", 2, 0)], 2, 100 << 30);+ assert_eq!(l.replicas, ["b", "a"]);+ }++ #[test]+ fn equal_ranks_break_by_id_and_counts_follow_the_machines() {+ let ms = [m("b", 5000, 100), m("a", 5000, 100)];+ assert_eq!(layout(&ms, 2, 0).owner, "a");+ assert_eq!(replica_count(1), 1);+ assert_eq!(replica_count(2), 2);+ assert_eq!(replica_count(9), 2);+ assert!(clearly_better(+ &Facts {+ rank: 9000,+ ..Default::default()+ },+ &Facts {+ rank: 7999,+ ..Default::default()+ }+ ));+ assert!(!clearly_better(+ &Facts {+ rank: 8500,+ ..Default::default()+ },+ &Facts {+ rank: 8000,+ ..Default::default()+ }+ ));+ }+}diff --git a/libs/cua/crates/cua-volume/src/replica/ship.rs b/libs/cua/crates/cua-volume/src/replica/ship.rsnew file mode 100644index 0000000000..b5f3454f78--- /dev/null+++ b/libs/cua/crates/cua-volume/src/replica/ship.rs@@ -0,0 +1,221 @@+// SPDX-License-Identifier: FSL-1.1-MIT+// Copyright (c) 2026 Cua AI, Inc.++//! The owner side of replication: ship the log to every machine that holds+//! a copy, resuming from what each has applied.++use std::collections::HashMap;+use std::sync::Arc;+use std::sync::atomic::{AtomicBool, Ordering};+use std::time::Duration;++use super::node::{Inner, save_eras};+use super::oplog::Op;+use super::roles::Role;+use super::wire::{Msg, RecordW};+use crate::backend::Backend;++pub(super) type Shipping = std::sync::Mutex<HashMap<String, Arc<AtomicBool>>>;++impl Inner {+ pub(super) async fn ship_all_loop(self: Arc<Self>) {+ loop {+ tokio::select! {+ _ = self.wake.notified() => {}+ _ = tokio::time::sleep(Duration::from_millis(100)) => {}+ }+ if self.closed.load(Ordering::SeqCst) {+ return;+ }+ self.wait_unfrozen().await;+ if self.self_away.load(Ordering::SeqCst) {+ continue;+ }+ let targets: Vec<String> = {+ let st = self.st.lock().await;+ if st.eras.owner() != self.me {+ continue;+ }+ let layout = self.layout_locked(&st);+ layout+ .replicas+ .iter()+ .filter(|r| **r != self.me && layout.role_of(r) == Role::Replica)+ .cloned()+ .collect()+ };+ for peer in targets {+ if self.ctl_link(&peer).is_none() {+ continue;+ }+ let flag = {+ let mut m = self.shipping.lock().expect("shipping");+ m.entry(peer.clone()).or_default().clone()+ };+ if flag.swap(true, Ordering::SeqCst) {+ continue;+ }+ let me = self.clone();+ tokio::spawn(async move {+ me.ship_until_idle(&peer).await;+ flag.store(false, Ordering::SeqCst);+ });+ }+ }+ }++ async fn ship_until_idle(self: &Arc<Self>, peer: &str) {+ let mut last_acked = None;+ loop {+ if self.closed.load(Ordering::SeqCst) {+ return;+ }+ let Some(link) = self.ctl_link(peer) else {+ return;+ };+ let (epoch, acked, recs, tail) = {+ let st = self.st.lock().await;+ if st.eras.owner() != self.me {+ return;+ }+ let acked = st+ .peers+ .get(peer)+ .map_or(super::era::Pos::ZERO, |p| p.acked);+ (+ st.eras.epoch(),+ acked,+ st.log.after(acked, self.cfg.batch),+ st.log.tail(),+ )+ };+ if last_acked == Some(acked) {+ // No progress since the last round: wait for news.+ return;+ }+ let Some(recs) = recs else {+ // The log no longer reaches this replica: it pulls a snapshot.+ let r = link+ .call(Msg::Snapshot { at: tail }, &[], self.cfg.data_timeout * 30)+ .await;+ match r {+ Ok((Msg::Ack { applied }, _)) => self.set_acked(peer, applied).await,+ Ok((Msg::Error { message, .. }, _)) => {+ tracing::warn!(peer, %message, "replica: a snapshot could not be applied");+ return;+ }+ _ => return,+ }+ last_acked = Some(acked);+ continue;+ };+ if recs.is_empty() {+ return;+ }+ let mut data: Vec<u8> = vec![];+ let mut out = Vec::with_capacity(recs.len());+ for r in recs {+ let mut body = None;+ if let Op::Put { meta, .. } = &r.op+ && meta.size <= self.cfg.inline_max as u64+ && data.len() + (meta.size as usize) < (1 << 20)+ && let Ok((bytes, _)) = self.store.get(&meta.key, Some(&meta.version)).await+ {+ body = Some((data.len() as u64, bytes.len() as u64));+ data.extend_from_slice(&bytes);+ }+ out.push(RecordW {+ pos: r.pos,+ op: r.op,+ at_ms: r.at_ms,+ body,+ });+ }+ let t0 = std::time::Instant::now();+ let n_out = out.len();+ let reply = link+ .call(+ Msg::Append {+ epoch,+ records: out,+ },+ &data,+ self.cfg.call_timeout * 10,+ )+ .await;+ tracing::debug!(peer, n = n_out, took = ?t0.elapsed(), ok = reply.is_ok(), "replica: append sent");+ last_acked = Some(acked);+ match reply {+ Ok((Msg::Ack { applied }, _)) => {+ self.set_acked(peer, applied).await;+ if applied > acked {+ last_acked = None;+ }+ }+ Ok((Msg::Behind { applied }, _)) => {+ {+ let mut st = self.st.lock().await;+ if let Some(p) = st.peers.get_mut(peer) {+ p.acked = applied;+ }+ }+ if applied != acked {+ last_acked = None;+ }+ }+ Ok((Msg::Fenced { eras }, _)) => {+ let advanced = {+ let mut st = self.st.lock().await;+ let a = st.eras.adopt_from(&eras);+ if a {+ let _ = save_eras(&self.dir, &st.eras);+ }+ a+ };+ if advanced {+ self.reconcile_eras().await;+ }+ return;+ }+ Ok((Msg::Error { kind, message }, _)) => {+ let _ = kind;+ {+ let mut st = self.st.lock().await;+ if let Some(p) = st.peers.get_mut(peer) {+ p.problem = Some(message);+ }+ }+ return;+ }+ _ => return,+ }+ }+ }++ async fn set_acked(&self, peer: &str, applied: super::era::Pos) {+ {+ let mut st = self.st.lock().await;+ let sample = st+ .written_at+ .get(&applied)+ .map(|t| t.elapsed().as_millis() as f64);+ if let Some(p) = st.peers.get_mut(peer) {+ if let Some(ms) = sample {+ p.ack_rtt_ms = if p.ack_rtt_ms == 0.0 {+ ms+ } else {+ 0.7 * p.ack_rtt_ms + 0.3 * ms+ };+ }+ if applied > p.acked {+ p.acked = applied;+ }+ p.applied = applied;+ p.problem = None;+ p.missed_acks = 0;+ }+ }+ self.acks+ .send_replace(self.seq_ack_counter.fetch_add(1, Ordering::Relaxed));+ }+}diff --git a/libs/cua/crates/cua-volume/src/replica/simnet.rs b/libs/cua/crates/cua-volume/src/replica/simnet.rsnew file mode 100644index 0000000000..e8c407b40f--- /dev/null+++ b/libs/cua/crates/cua-volume/src/replica/simnet.rs@@ -0,0 +1,261 @@+// SPDX-License-Identifier: FSL-1.1-MIT+// Copyright (c) 2026 Cua AI, Inc.++//! An in-memory network between named machines, for simulations and the+//! chaos suite. Streams behave like TCP: reliable and ordered, so loss shows+//! up the way it does on a real connection, as delay. What it can do to a+//! pair of machines, per direction:+//!+//! - latency, jitter, and a bandwidth cap;+//! - *hold*: bytes stop arriving (a blackholed path) and flow again when the+//! hold ends;+//! - *sever*: every stream between the pair is reset and new dials fail+//! until it is lifted (a long partition or a changed address);+//! - an asymmetric version of any of these.++use std::collections::HashMap;+use std::sync::{Arc, Mutex};+use std::time::Duration;++use tokio::io::{AsyncReadExt, AsyncWriteExt};++use super::link::Stream;+use super::transport::Transport;++/// What one direction of a pair does to traffic.+#[derive(Clone, Debug, Default)]+pub struct Impair {+ pub latency: Duration,+ pub jitter: Duration,+ /// Bytes per second (0: unlimited).+ pub bandwidth: u64,+ /// Bytes stop arriving until this is cleared.+ pub hold: bool,+ /// Streams are reset and dials fail.+ pub sever: bool,+}++type Pair = (String, String);++#[derive(Default)]+struct Net {+ impair: HashMap<Pair, Impair>,+ /// Accept queues by machine.+ inbox: HashMap<String, tokio::sync::mpsc::UnboundedSender<(String, Stream)>>,+ bytes: HashMap<Pair, u64>,+ /// Machines whose process is gone: their connections are reset.+ dead: std::collections::HashSet<String>,+}++/// The network.+#[derive(Clone, Default)]+pub struct SimNet {+ net: Arc<Mutex<Net>>,+}++impl SimNet {+ pub fn new() -> SimNet {+ SimNet::default()+ }++ /// Registers `machine` and returns its transport and the streams that+ /// arrive for it as `(dialer, stream)`.+ pub fn join(+ &self,+ machine: &str,+ ) -> (+ SimTransport,+ tokio::sync::mpsc::UnboundedReceiver<(String, Stream)>,+ ) {+ let (tx, rx) = tokio::sync::mpsc::unbounded_channel();+ self.net+ .lock()+ .expect("net")+ .inbox+ .insert(machine.to_string(), tx);+ (+ SimTransport {+ net: self.clone(),+ me: machine.to_string(),+ },+ rx,+ )+ }++ /// The machine's process is gone: its connections are reset and dials to+ /// it are refused, as when a process dies on a machine that stays up.+ pub fn leave(&self, machine: &str) {+ let mut n = self.net.lock().expect("net");+ n.inbox.remove(machine);+ n.dead.insert(machine.to_string());+ }++ fn is_dead(&self, machine: &str) -> bool {+ self.net.lock().expect("net").dead.contains(machine)+ }++ /// Sets what traffic from `from` to `to` suffers.+ pub fn set(&self, from: &str, to: &str, i: Impair) {+ self.net+ .lock()+ .expect("net")+ .impair+ .insert((from.into(), to.into()), i);+ }++ /// Both directions at once.+ pub fn set_both(&self, a: &str, b: &str, i: Impair) {+ self.set(a, b, i.clone());+ self.set(b, a, i);+ }++ /// Clears everything between the pair.+ pub fn heal(&self, a: &str, b: &str) {+ self.set_both(a, b, Impair::default());+ }++ /// Bytes that crossed from `from` to `to` so far.+ pub fn bytes(&self, from: &str, to: &str) -> u64 {+ *self+ .net+ .lock()+ .expect("net")+ .bytes+ .get(&(from.to_string(), to.to_string()))+ .unwrap_or(&0)+ }++ fn get(&self, from: &str, to: &str) -> Impair {+ self.net+ .lock()+ .expect("net")+ .impair+ .get(&(from.to_string(), to.to_string()))+ .cloned()+ .unwrap_or_default()+ }++ fn count(&self, from: &str, to: &str, n: u64) {+ *self+ .net+ .lock()+ .expect("net")+ .bytes+ .entry((from.to_string(), to.to_string()))+ .or_default() += n;+ }++ /// One direction: reads from `rd`, delivers to `wr` after the pair's+ /// current impairment. Delays are release times, not sleeps in the read+ /// loop, so a delayed stream still pipelines like a real long path.+ fn pump(+ &self,+ from: String,+ to: String,+ mut rd: tokio::io::ReadHalf<tokio::io::DuplexStream>,+ mut wr: tokio::io::WriteHalf<tokio::io::DuplexStream>,+ ) {+ let net = self.clone();+ let (tx, mut rx) =+ tokio::sync::mpsc::unbounded_channel::<(tokio::time::Instant, Vec<u8>)>();+ let (f2, t2, n2) = (from.clone(), to.clone(), net.clone());+ tokio::spawn(async move {+ while let Some((at, bytes)) = rx.recv().await {+ tokio::time::sleep_until(at).await;+ n2.count(&f2, &t2, bytes.len() as u64);+ if wr.write_all(&bytes).await.is_err() {+ break;+ }+ }+ let _ = wr.shutdown().await;+ });+ tokio::spawn(async move {+ let mut buf = vec![0u8; 16 * 1024];+ let mut next_free = tokio::time::Instant::now();+ loop {+ let n = match rd.read(&mut buf).await {+ Ok(0) | Err(_) => break,+ Ok(n) => n,+ };+ loop {+ let i = net.get(&from, &to);+ if i.sever || net.is_dead(&from) || net.is_dead(&to) {+ return;+ }+ if !i.hold {+ break;+ }+ tokio::time::sleep(Duration::from_millis(5)).await;+ }+ let i = net.get(&from, &to);+ let now = tokio::time::Instant::now();+ let start = now.max(next_free);+ let ser = if i.bandwidth > 0 {+ Duration::from_secs_f64(n as f64 / i.bandwidth as f64)+ } else {+ Duration::ZERO+ };+ next_free = start + ser;+ let mut delay = i.latency;+ if !i.jitter.is_zero() {+ let j = rand::random::<u64>() % (i.jitter.as_millis() as u64 + 1);+ delay += Duration::from_millis(j);+ }+ let at = (start + ser + delay).max(now);+ if tx.send((at, buf[..n].to_vec())).is_err() {+ break;+ }+ }+ });+ }+}++/// One machine's dialer.+#[derive(Clone)]+pub struct SimTransport {+ net: SimNet,+ me: String,+}++#[async_trait::async_trait]+impl Transport for SimTransport {+ async fn dial(&self, machine: &str) -> std::io::Result<Stream> {+ let i = self.net.get(&self.me, machine);+ let back = self.net.get(machine, &self.me);+ if i.sever || back.sever || i.hold || back.hold {+ // A dial into a dead path takes a while to fail.+ tokio::time::sleep(Duration::from_millis(50)).await;+ return Err(std::io::Error::new(+ std::io::ErrorKind::TimedOut,+ "no route to machine",+ ));+ }+ let tx = self+ .net+ .net+ .lock()+ .expect("net")+ .inbox+ .get(machine)+ .cloned()+ .ok_or_else(|| {+ std::io::Error::new(std::io::ErrorKind::ConnectionRefused, "machine is off")+ })?;+ // Two pipes and a pump per direction make one connection.+ let (dialer_end, a_inner) = tokio::io::duplex(256 * 1024);+ let (acceptor_end, b_inner) = tokio::io::duplex(256 * 1024);+ let (a_rd, a_wr) = tokio::io::split(a_inner);+ let (b_rd, b_wr) = tokio::io::split(b_inner);+ // The dial itself costs a round trip.+ tokio::time::sleep(i.latency + back.latency).await;+ self.net+ .pump(self.me.clone(), machine.to_string(), a_rd, b_wr);+ self.net+ .pump(machine.to_string(), self.me.clone(), b_rd, a_wr);+ tx.send((self.me.clone(), Box::new(acceptor_end)))+ .map_err(|_| {+ std::io::Error::new(std::io::ErrorKind::ConnectionRefused, "machine is off")+ })?;+ Ok(Box::new(dialer_end))+ }+}diff --git a/libs/cua/crates/cua-volume/src/replica/status.rs b/libs/cua/crates/cua-volume/src/replica/status.rsnew file mode 100644index 0000000000..bd2507c9ef--- /dev/null+++ b/libs/cua/crates/cua-volume/src/replica/status.rs@@ -0,0 +1,88 @@+// SPDX-License-Identifier: FSL-1.1-MIT+// Copyright (c) 2026 Cua AI, Inc.++//! What the app and the CLI show about the replicated volume.++use serde::{Deserialize, Serialize};++use super::detector::Health;+use super::era::Pos;+use super::roles::Role;++/// How a write stands.+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]+#[serde(tag = "kind", content = "copies", rename_all = "snake_case")]+pub enum Durability {+ /// This many machines hold it (2 or more when replication is on).+ Replicated(usize),+ /// Only this many (1): the other machine was away or slow.+ Local(usize),+ /// Saved on this machine only, waiting for the owner.+ Pending,+}++/// A machine as the status shows it.+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]+pub struct PeerView {+ pub id: String,+ pub name: String,+ pub role: Role,+ pub health: String,+ pub connected: bool,+ pub owner: bool,+ /// Records it has applied that this machine knows of.+ pub applied: Pos,+ /// Records behind the owner's tail (0: caught up).+ pub lag: u64,+ pub problem: Option<String>,+}++/// A change waiting for another machine.+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]+pub struct PendingView {+ pub key: String,+ pub size: u64,+ pub since_ms: u64,+ pub deleted: bool,+}++/// The whole picture.+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]+pub struct ReplicaStatus {+ pub me: String,+ pub name: String,+ pub role: Role,+ pub owner: String,+ pub owner_name: String,+ pub i_own: bool,+ pub epoch: u64,+ pub applied: Pos,+ pub tail: Pos,+ pub peers: Vec<PeerView>,+ pub pending: Vec<PendingView>,+ pub pending_bytes: u64,+ /// Machines (this one included) holding everything up to the owner's tail.+ pub copies_now: usize,+ /// Copies wanted.+ pub copies_wanted: usize,+ /// Records the owner holds that fewer than `copies_wanted` machines hold.+ pub unreplicated: u64,+ pub volume_bytes: u64,+ pub bytes_sent: u64,+ pub bytes_received: u64,+ pub takeovers: u64,+ pub handbacks: u64,+ pub fenced_writes: u64,+ pub conflict_copies: u64,+}++impl Health {+ pub fn word(self) -> &'static str {+ match self {+ Health::Alive => "up",+ Health::Suspect => "slow",+ Health::Dead => "down",+ Health::Away => "away",+ }+ }+}diff --git a/libs/cua/crates/cua-volume/src/replica/supervision.rs b/libs/cua/crates/cua-volume/src/replica/supervision.rsnew file mode 100644index 0000000000..0876849758--- /dev/null+++ b/libs/cua/crates/cua-volume/src/replica/supervision.rs@@ -0,0 +1,823 @@+// SPDX-License-Identifier: FSL-1.1-MIT+// Copyright (c) 2026 Cua AI, Inc.++//! Who runs a durable agent, and who takes over when that machine goes.+//!+//! A durable agent's *driver* (the loop that starts the agent, watches it+//! and re-drives it) must not live on the machine the user will close. It+//! runs on the most reliable machine that holds the agent's files, and moves+//! when that machine goes away. This module is that rule, built on the+//! Volume's ordered writes (they are atomic across machines when the+//! replicated store is on):+//!+//! ```text+//! .cua-runs/<agent>.json the run: its task, and whether it is done+//! agents/<agent>/.cua-supervisor the lease: holder, epoch, beat, progress+//! ```+//!+//! Every machine runs the same small loop. The *preferred* machine is the+//! most reliable one that is up and holds a copy (the same score placement+//! uses; ties by id). It takes a free lease, or a lease whose holder is+//! down, by compare-and-swap with `epoch + 1`; the old holder, if it wakes,+//! finds the epoch moved on and stops (it is fenced), and anything it wrote+//! in between merges by the order of the lease, not by luck. A laptop that+//! holds a run yields it to a clearly more reliable machine that has been+//! up and caught up for a while. Nothing here needs a clock: a holder is+//! alive while its beat keeps changing as the observer sees it.+//!+//! What a takeover says is told from facts: the new holder compares the+//! journal steps it holds with the progress the old holder last reported,+//! and says "resumed" or "restarted from step N, steps M to K are redone".++use std::collections::HashMap;+use std::sync::Arc;+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};+use std::time::{Duration, Instant};++use serde::{Deserialize, Serialize};++use super::node::{MachineView, ReplicaNode};+use super::words::{Summary, closing_summary};+use crate::backend::{Backend, Condition};+use crate::{Error, Result};++/// The lease object's name inside an agent's home.+pub const LEASE: &str = ".cua-supervisor";+/// The folder of run records.+pub const RUNS_DIR: &str = ".cua-runs/";++/// A run.+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]+pub struct RunRecord {+ pub agent: String,+ /// What to run (the driver reads it).+ pub spec: serde_json::Value,+ /// `running` or `done`.+ pub state: String,+ pub created_by: String,+ pub created_ms: u64,+ /// A line for people when it is done.+ #[serde(default)]+ pub outcome: String,+}++/// The lease.+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]+pub struct LeaseRec {+ pub holder: String,+ pub name: String,+ pub epoch: u64,+ /// Changes at every renewal: a holder is alive while this keeps moving.+ pub beat: u64,+ /// Steps of the run the holder says are done.+ pub progress: u64,+ /// The holder before this one (for the takeover sentence).+ #[serde(default)]+ pub previous: String,+ /// The holder's CPU architecture.+ #[serde(default)]+ pub arch: String,+ /// The previous holder's.+ #[serde(default)]+ pub previous_arch: String,+}++fn lease_key(agent: &str) -> Result<String> {+ Ok(format!("{}{LEASE}", crate::path::agent_home(agent)?))+}++fn run_key(agent: &str) -> String {+ format!("{RUNS_DIR}{agent}.json")+}++/// What the new holder knows about where the run stands.+#[derive(Clone, Debug, PartialEq, Eq)]+pub struct Resume {+ /// The first holder (nothing to resume).+ pub fresh: bool,+ /// The step the previous holder last reported.+ pub reported: u64,+ /// The step this machine's copy of the journal reaches (the driver+ /// fills it in; the supervisor leaves it at the reported step until then).+ pub held: u64,+}++impl Resume {+ /// The sentence for a takeover.+ pub fn words(&self, to: &str, from: &str) -> String {+ if self.fresh {+ return format!("Started on {to}.");+ }+ if self.held >= self.reported {+ format!(+ "Resumed on {to} where {from} left off (step {}).",+ self.held+ )+ } else {+ format!(+ "Resumed on {to} from step {}; steps {} to {} had not reached it and are done again.",+ self.held,+ self.held + 1,+ self.reported+ )+ }+ }+}++/// Handed to the driver while it holds a run.+pub struct RunCtx {+ pub agent: String,+ pub epoch: u64,+ pub spec: serde_json::Value,+ pub machine: String,+ pub backend: Arc<dyn Backend>,+ pub resume: Resume,+ fenced: Arc<AtomicBool>,+ progress: Arc<AtomicU64>,+ held: Arc<AtomicU64>,+}++impl RunCtx {+ /// The run was taken from this machine (a later epoch holds it, or it+ /// was handed to a better machine): stop.+ pub fn fenced(&self) -> bool {+ self.fenced.load(Ordering::SeqCst)+ }++ /// Reports how far the run is (in steps); renewals carry it.+ pub fn report(&self, step: u64) {+ self.progress.store(step, Ordering::SeqCst);+ }++ /// Sets what this machine's journal copy reaches (call once at start).+ pub fn held_reaches(&mut self, step: u64) {+ self.resume.held = step;+ self.held.store(step, Ordering::SeqCst);+ }+}++/// How a drive ended.+#[derive(Clone, Debug, PartialEq, Eq)]+pub enum DriveEnd {+ /// The run finished; the line says how.+ Done(String),+ /// The driver stopped because it was fenced.+ Fenced,+ /// It failed; the supervisor leaves the lease to be adopted.+ Failed(String),+}++/// What actually runs the agent.+#[async_trait::async_trait]+pub trait Driver: Send + Sync + 'static {+ async fn drive(&self, ctx: RunCtx) -> DriveEnd;+}++/// Held while a run is active on this machine (a sleep assertion): dropping+/// it releases whatever it holds.+pub type AwakeToken = Box<dyn std::any::Any + Send>;++/// Keeps this machine awake for a reason people can read.+pub type Awake = Arc<dyn Fn(&str) -> AwakeToken + Send + Sync>;++/// Timing.+#[derive(Clone, Debug)]+pub struct SupervisionConfig {+ /// A lease whose beat has not changed this long is dead.+ pub ttl: Duration,+ pub tick: Duration,+ /// A machine must have been up this long (and the run's files with it)+ /// before a laptop yields the run to it.+ pub yield_after: Duration,+ /// A machine does not adopt a run of a holder it still sees up until the+ /// beat stalls; a holder it cannot see (down) is adopted after this.+ pub down_grace: Duration,+}++impl SupervisionConfig {+ pub fn standard() -> Self {+ SupervisionConfig {+ ttl: Duration::from_secs(20),+ tick: Duration::from_millis(500),+ yield_after: Duration::from_secs(120),+ down_grace: Duration::from_secs(1),+ }+ }++ pub fn fast() -> Self {+ SupervisionConfig {+ ttl: Duration::from_millis(1800),+ tick: Duration::from_millis(60),+ yield_after: Duration::from_millis(1500),+ down_grace: Duration::from_millis(100),+ }+ }+}++/// What a run looks like from here.+#[derive(Clone, Debug, PartialEq, Eq, Serialize)]+pub struct RunView {+ pub agent: String,+ pub state: String,+ pub holder: String,+ pub holder_name: String,+ pub epoch: u64,+ pub progress: u64,+ pub running_here: bool,+ pub outcome: String,+ /// What happened at the last takeover, in plain words.+ pub note: String,+}++/// What `start_run` says.+#[derive(Clone, Debug, PartialEq)]+pub struct StartInfo {+ /// Where the run will go (a name from the machines list).+ pub run_on: String,+ /// The one plain paragraph about closing this machine.+ pub words: String,+}++/// What a takeover said, kept until the run ends (the words are made when+/// asked, so they use what the driver found in its copy of the journal).+struct Note {+ fresh: bool,+ reported: u64,+ held: Arc<AtomicU64>,+ to: String,+ from: String,+ /// Set when the run moved between architectures: what in the workspace+ /// was built for the old one.+ arch_change: Option<(String, Vec<String>)>,+}++impl Note {+ fn words(&self) -> String {+ Resume {+ fresh: self.fresh,+ reported: self.reported,+ held: self.held.load(Ordering::SeqCst),+ }+ .words(&self.to, &self.from)+ + &self.arch_sentence()+ }++ fn arch_sentence(&self) -> String {+ let Some((from_arch, built)) = &self.arch_change else {+ return String::new();+ };+ let what = if built.is_empty() {+ String::new()+ } else {+ format!(" ({})", built.join(", "))+ };+ format!(+ " {} is {from_arch}, so programs built there{what} are not reused here and should be built again; files, notes and the run's journal carry over as they are.",+ self.from+ )+ }+}++struct Live {+ fenced: Arc<AtomicBool>,+ task: tokio::task::JoinHandle<()>,+ _awake: Option<AwakeToken>,+}++#[derive(Default)]+struct Seen {+ etag: String,+ since: Option<Instant>,+}++struct Inner {+ node: ReplicaNode,+ backend: Arc<dyn Backend>,+ cfg: SupervisionConfig,+ driver: Arc<dyn Driver>,+ awake: Option<Awake>,+ live: tokio::sync::Mutex<HashMap<String, Live>>,+ seen: tokio::sync::Mutex<HashMap<String, Seen>>,+ notes: std::sync::Mutex<HashMap<String, Note>>,+ progress: std::sync::Mutex<HashMap<String, Arc<AtomicU64>>>,+ stop: AtomicBool,+ stable_since: tokio::sync::Mutex<HashMap<String, Instant>>,+}++/// The supervisor of one machine.+#[derive(Clone)]+pub struct Supervision {+ inner: Arc<Inner>,+}++/// The machine that should run durable work now: the most reliable machine+/// that is up and holds a copy (any up machine when none does), ties by id.+pub fn preferred(machines: &[MachineView]) -> Option<&MachineView> {+ let pick = |only_replicas: bool| {+ machines+ .iter()+ .filter(|m| m.alive && (!only_replicas || m.replica))+ .max_by(|a, b| {+ a.facts+ .rank+ .cmp(&b.facts.rank)+ .then_with(|| b.id.cmp(&a.id))+ })+ };+ pick(true).or_else(|| pick(false))+}++impl Supervision {+ pub fn new(+ node: ReplicaNode,+ backend: Arc<dyn Backend>,+ cfg: SupervisionConfig,+ driver: Arc<dyn Driver>,+ awake: Option<Awake>,+ ) -> Supervision {+ Supervision {+ inner: Arc::new(Inner {+ node,+ backend,+ cfg,+ driver,+ awake,+ live: Default::default(),+ seen: Default::default(),+ notes: Default::default(),+ progress: Default::default(),+ stop: AtomicBool::new(false),+ stable_since: Default::default(),+ }),+ }+ }++ /// Starts the loop.+ pub fn spawn(&self) {+ let me = self.inner.clone();+ tokio::spawn(async move {+ while !me.stop.load(Ordering::SeqCst) {+ tokio::time::sleep(me.cfg.tick).await;+ if let Err(e) = me.tick().await {+ tracing::debug!(error = %e, "supervision tick");+ }+ }+ });+ }++ /// Stops the loop and the runs here (they are not fenced: another+ /// machine takes them when the lease goes stale).+ pub async fn stop(&self) {+ self.inner.stop.store(true, Ordering::SeqCst);+ let all: Vec<Live> = self+ .inner+ .live+ .lock()+ .await+ .drain()+ .map(|(_, v)| v)+ .collect();+ for l in all {+ l.task.abort();+ }+ }++ /// Records a run. The most reliable machine takes it on its next look;+ /// this machine does if it is the only one.+ pub async fn start_run(&self, agent: &str, spec: serde_json::Value) -> Result<StartInfo> {+ // A name the Volume would refuse fails here, not silently later.+ crate::path::agent_home(agent)?;+ let rec = RunRecord {+ agent: agent.to_string(),+ spec,+ state: "running".into(),+ created_by: self.inner.node.machine().to_string(),+ created_ms: crate::now_ms(),+ outcome: String::new(),+ };+ self.inner+ .backend+ .put(&run_key(agent), serde_json::to_vec(&rec)?, Condition::None)+ .await?;+ let machines = self.inner.node.machines().await;+ let run_on = preferred(&machines).map(|m| m.name.clone());+ let words = self.words(run_on.as_deref()).await;+ Ok(StartInfo {+ run_on: run_on.unwrap_or_default(),+ words,+ })+ }++ /// The paragraph about closing this machine for a run on `run_on`.+ pub async fn words(&self, run_on: Option<&str>) -> String {+ let s: Summary = self.inner.node.summary(run_on).await;+ closing_summary(&s)+ }++ /// Every run, as this machine sees it.+ pub async fn runs(&self) -> Result<Vec<RunView>> {+ let mut out = vec![];+ for m in self.inner.backend.list(RUNS_DIR).await? {+ let Ok((b, _)) = self.inner.backend.get(&m.key, None).await else {+ continue;+ };+ let Ok(rec) = serde_json::from_slice::<RunRecord>(&b) else {+ continue;+ };+ let lease = self.inner.read_lease(&rec.agent).await.ok().flatten();+ let running_here = self.inner.live.lock().await.contains_key(&rec.agent);+ out.push(RunView {+ agent: rec.agent.clone(),+ state: rec.state,+ holder: lease+ .as_ref()+ .map(|l| l.0.holder.clone())+ .unwrap_or_default(),+ holder_name: lease.as_ref().map(|l| l.0.name.clone()).unwrap_or_default(),+ epoch: lease.as_ref().map_or(0, |l| l.0.epoch),+ progress: lease.as_ref().map_or(0, |l| l.0.progress),+ running_here,+ outcome: rec.outcome,+ note: self+ .inner+ .notes+ .lock()+ .expect("notes")+ .get(&rec.agent)+ .map(Note::words)+ .unwrap_or_default(),+ });+ }+ Ok(out)+ }+}++/// The name of machine `id` in the machines list.+async fn machines_name(node: &ReplicaNode, id: &str) -> String {+ node.machines()+ .await+ .into_iter()+ .find(|m| m.id == id)+ .map(|m| m.name)+ .unwrap_or_else(|| id.to_string())+}++impl Inner {+ async fn read_lease(&self, agent: &str) -> Result<Option<(LeaseRec, String)>> {+ let k = lease_key(agent)?;+ match self.backend.get(&k, None).await {+ Ok((b, meta)) => {+ let l: LeaseRec = serde_json::from_slice(&b)?;+ Ok(Some((l, meta.etag)))+ }+ Err(Error::NotFound(_)) => Ok(None),+ Err(e) => Err(e),+ }+ }++ async fn tick(self: &Arc<Self>) -> Result<()> {+ let machines = self.node.machines().await;+ let me = self.node.machine().to_string();+ let pref = preferred(&machines).map(|m| m.id.clone());+ for m in self.backend.list(RUNS_DIR).await? {+ let Ok((b, _)) = self.backend.get(&m.key, None).await else {+ continue;+ };+ let Ok(rec) = serde_json::from_slice::<RunRecord>(&b) else {+ continue;+ };+ if rec.state == "done" {+ self.drop_live(&rec.agent, true).await;+ continue;+ }+ self.look(&rec, &machines, &me, pref.as_deref()).await;+ }+ Ok(())+ }++ async fn drop_live(&self, agent: &str, fence: bool) {+ if let Some(l) = self.live.lock().await.remove(agent) {+ if fence {+ l.fenced.store(true, Ordering::SeqCst);+ }+ // Give the driver a moment to stop at a step boundary.+ let t = l.task;+ let _ = tokio::time::timeout(self.cfg.ttl, t).await;+ }+ }++ async fn look(+ self: &Arc<Self>,+ rec: &RunRecord,+ machines: &[MachineView],+ me: &str,+ pref: Option<&str>,+ ) {+ let agent = &rec.agent;+ let lease = match self.read_lease(agent).await {+ Ok(l) => l,+ Err(_) => return,+ };+ let running_here = self.live.lock().await.contains_key(agent);+ match lease {+ None => {+ if pref == Some(me) {+ let l = LeaseRec {+ holder: me.to_string(),+ name: self.node.name().to_string(),+ epoch: 1,+ beat: 0,+ progress: 0,+ previous: String::new(),+ arch: String::new(),+ previous_arch: String::new(),+ };+ self.claim(rec, l, Condition::IfNoneMatch, true).await;+ }+ }+ Some((l, etag)) => {+ // Observe the beat on this machine's own clock.+ let (dead, quiet_for) = {+ let mut seen = self.seen.lock().await;+ let s = seen.entry(agent.clone()).or_default();+ if s.etag != etag {+ s.etag = etag.clone();+ s.since = Some(Instant::now());+ }+ let q = s.since.map_or(Duration::ZERO, |t| t.elapsed());+ (q >= self.cfg.ttl, q)+ };+ if l.holder == me {+ if !running_here {+ // A restart of this machine's process while it held+ // the run: take it again at the next epoch.+ let next = LeaseRec {+ epoch: l.epoch + 1,+ beat: 0,+ previous: me.to_string(),+ previous_arch: l.arch.clone(),+ ..l.clone()+ };+ self.claim(rec, next, Condition::IfMatch(etag), false).await;+ return;+ }+ self.renew(rec, &l, &etag, machines, me, pref).await;+ return;+ }+ // Someone else holds it. If this machine runs it, it is+ // fenced.+ if running_here {+ tracing::info!(+ agent,+ epoch = l.epoch,+ "supervision: this machine was fenced"+ );+ self.fence_here(agent).await;+ return;+ }+ if pref != Some(me) {+ return;+ }+ // Confirmed down, not merely slow: a holder that is only+ // late keeps its run until its beat stalls for the whole ttl.+ let holder_down = machines+ .iter()+ .find(|m| m.id == l.holder)+ .is_none_or(|m| m.down);+ // The preferred machine takes a run whose holder is down+ // (after a short grace) or whose beat stopped.+ let take = dead || (holder_down && quiet_for >= self.cfg.down_grace);+ // Or the holder handed it to this machine.+ let handed = l.holder == me;+ if take || handed {+ let next = LeaseRec {+ holder: me.to_string(),+ name: self.node.name().to_string(),+ epoch: l.epoch + 1,+ beat: 0,+ progress: l.progress,+ previous: l.holder.clone(),+ arch: String::new(),+ previous_arch: l.arch.clone(),+ };+ self.claim(rec, next, Condition::IfMatch(etag), false).await;+ }+ }+ }+ }++ /// Takes the lease with `cond` and starts the driver.+ async fn claim(+ self: &Arc<Self>,+ rec: &RunRecord,+ mut l: LeaseRec,+ cond: Condition,+ fresh: bool,+ ) {+ // The holder's own architecture goes into the lease.+ l.arch = self+ .node+ .machines()+ .await+ .into_iter()+ .find(|m| m.is_me)+ .map(|m| super::roles::normalize_arch(&m.facts.arch))+ .unwrap_or_default();+ let Ok(k) = lease_key(&rec.agent) else { return };+ let Ok(bytes) = serde_json::to_vec(&l) else {+ return;+ };+ match self.backend.put(&k, bytes, cond).await {+ Ok(_) => {}+ Err(_) => return,+ }+ let note = Resume {+ fresh,+ reported: l.progress,+ held: l.progress,+ };+ tracing::info!(agent = %rec.agent, epoch = l.epoch, "supervision: this machine runs the agent now");+ self.start_driver(rec, l, note).await;+ }++ async fn start_driver(self: &Arc<Self>, rec: &RunRecord, l: LeaseRec, resume: Resume) {+ let fenced = Arc::new(AtomicBool::new(false));+ let progress = Arc::new(AtomicU64::new(l.progress));+ self.progress+ .lock()+ .expect("progress")+ .insert(rec.agent.clone(), progress.clone());+ let awake = self+ .awake+ .as_ref()+ .map(|f| f(&format!("Cua: the durable agent {} is running", rec.agent)));+ let held = Arc::new(AtomicU64::new(resume.held));+ let ctx = RunCtx {+ agent: rec.agent.clone(),+ epoch: l.epoch,+ spec: rec.spec.clone(),+ machine: self.node.machine().to_string(),+ backend: self.backend.clone(),+ resume: resume.clone(),+ fenced: fenced.clone(),+ progress,+ held: held.clone(),+ };+ let driver = self.driver.clone();+ let me = self.clone();+ let agent = rec.agent.clone();+ let from = l.previous.clone();+ let to = self.node.name().to_string();+ let from_name = machines_name(&self.node, &from).await;+ let arch_change =+ if !l.previous_arch.is_empty() && !l.arch.is_empty() && l.previous_arch != l.arch {+ let keys = self+ .backend+ .list(&crate::path::agent_home(&rec.agent).unwrap_or_default())+ .await+ .map(|v| v.into_iter().map(|m| m.key).collect::<Vec<_>>())+ .unwrap_or_default();+ Some((+ l.previous_arch.clone(),+ super::roles::arch_specific(keys, 4),+ ))+ } else {+ None+ };+ self.notes.lock().expect("notes").insert(+ agent.clone(),+ Note {+ fresh: resume.fresh,+ reported: resume.reported,+ held,+ to,+ from: from_name,+ arch_change,+ },+ );+ let rec2 = rec.clone();+ let task = tokio::spawn(async move {+ let end = driver.drive(ctx).await;+ match end {+ DriveEnd::Done(outcome) => {+ let mut done = rec2.clone();+ done.state = "done".into();+ done.outcome = outcome;+ if let Ok(b) = serde_json::to_vec(&done) {+ let _ = me.backend.put(&run_key(&agent), b, Condition::None).await;+ }+ }+ DriveEnd::Fenced => {}+ DriveEnd::Failed(why) => {+ tracing::warn!(agent = %agent, %why, "supervision: the driver failed");+ // The lease goes stale and the preferred machine re-drives.+ me.live.lock().await.remove(&agent);+ }+ }+ });+ self.live.lock().await.insert(+ rec.agent.clone(),+ Live {+ fenced,+ task,+ _awake: awake,+ },+ );+ }++ async fn fence_here(&self, agent: &str) {+ if let Some(l) = self.live.lock().await.remove(agent) {+ l.fenced.store(true, Ordering::SeqCst);+ let t = l.task;+ let _ = tokio::time::timeout(self.cfg.ttl, t).await;+ }+ }++ /// The holder's tick: renew by compare-and-swap, and hand the run to a+ /// clearly better machine when this one is a laptop.+ async fn renew(+ self: &Arc<Self>,+ rec: &RunRecord,+ l: &LeaseRec,+ etag: &str,+ machines: &[MachineView],+ me: &str,+ pref: Option<&str>,+ ) {+ // A driver that ended (failed) is not running here any more: handled+ // by the caller through `running_here`.+ let progress = self+ .progress+ .lock()+ .expect("progress")+ .get(&rec.agent)+ .map_or(l.progress, |p| p.load(Ordering::SeqCst));+ let mut next = LeaseRec {+ beat: l.beat + 1,+ progress,+ ..l.clone()+ };+ // Yield to a clearly better machine that has been up for a while.+ if let (Some(p), Some(mine)) = (pref, machines.iter().find(|m| m.id == me))+ && p != me+ && mine.facts.rank + super::roles::STEP_RANK+ <= machines+ .iter()+ .find(|m| m.id == p)+ .map_or(0, |m| m.facts.rank)+ && machines.iter().any(|m| m.id == p && m.stable)+ {+ let since = {+ let mut s = self.stable_since.lock().await;+ *s.entry(rec.agent.clone()).or_insert_with(Instant::now)+ };+ if since.elapsed() >= self.cfg.yield_after {+ let target = machines.iter().find(|m| m.id == p).expect("pref");+ next = LeaseRec {+ holder: target.id.clone(),+ name: target.name.clone(),+ epoch: l.epoch + 1,+ beat: 0,+ progress,+ previous: me.to_string(),+ arch: target.facts.arch.clone(),+ previous_arch: l.arch.clone(),+ };+ if let Ok(k) = lease_key(&rec.agent)+ && let Ok(b) = serde_json::to_vec(&next)+ && self+ .backend+ .put(&k, b, Condition::IfMatch(etag.to_string()))+ .await+ .is_ok()+ {+ tracing::info!(agent = %rec.agent, to = %target.name, "supervision: handed the run to a more reliable machine");+ self.stable_since.lock().await.remove(&rec.agent);+ self.fence_here(&rec.agent).await;+ }+ return;+ }+ } else {+ self.stable_since.lock().await.remove(&rec.agent);+ }+ let Ok(k) = lease_key(&rec.agent) else { return };+ let Ok(b) = serde_json::to_vec(&next) else {+ return;+ };+ match self+ .backend+ .put(&k, b, Condition::IfMatch(etag.to_string()))+ .await+ {+ Ok(_) => {}+ Err(Error::Precondition(_)) => {+ // Someone changed the lease: if it is not ours any more, we+ // are fenced; the next look decides.+ }+ Err(_) => {+ // The owner is out of reach: the run goes on (its files are+ // saved here and sync later); the beat just does not move.+ }+ }+ }+}diff --git a/libs/cua/crates/cua-volume/src/replica/transport.rs b/libs/cua/crates/cua-volume/src/replica/transport.rsnew file mode 100644index 0000000000..adbe30db79--- /dev/null+++ b/libs/cua/crates/cua-volume/src/replica/transport.rs@@ -0,0 +1,75 @@+// SPDX-License-Identifier: FSL-1.1-MIT+// Copyright (c) 2026 Cua AI, Inc.++//! How machines reach each other.+//!+//! In the product a peer link is a stream over the Cua mesh (WireGuard+//! between the account's node keys; the relay forwards ciphertext), so data+//! only ever moves between the user's own signed-in machines and is+//! encrypted. [`Transport`] is the seam: the node asks it for a stream to a+//! machine id and hands it every stream that arrives. [`TcpTransport`] is+//! the same thing over plain TCP for LANs and tests, and must be wrapped in+//! something that encrypts (an SSH tunnel, the mesh) outside a test.++use std::collections::HashMap;+use std::net::SocketAddr;+use std::sync::{Arc, Mutex};+use std::time::Duration;++use super::link::Stream;++/// Opens streams to machines.+#[async_trait::async_trait]+pub trait Transport: Send + Sync + 'static {+ /// A stream to `machine` (its machine id).+ async fn dial(&self, machine: &str) -> std::io::Result<Stream>;+}++/// Plain TCP, addresses by machine id.+#[derive(Clone, Default)]+pub struct TcpTransport {+ addrs: Arc<Mutex<HashMap<String, SocketAddr>>>,+ timeout: Duration,+}++impl TcpTransport {+ pub fn new() -> TcpTransport {+ TcpTransport {+ addrs: Arc::default(),+ timeout: Duration::from_secs(5),+ }+ }++ /// Where `machine` listens.+ pub fn set(&self, machine: &str, addr: SocketAddr) {+ self.addrs+ .lock()+ .expect("addrs")+ .insert(machine.into(), addr);+ }+}++#[async_trait::async_trait]+impl Transport for TcpTransport {+ async fn dial(&self, machine: &str) -> std::io::Result<Stream> {+ let addr = self+ .addrs+ .lock()+ .expect("addrs")+ .get(machine)+ .copied()+ .ok_or_else(|| {+ std::io::Error::new(+ std::io::ErrorKind::NotFound,+ format!("no address for {machine}"),+ )+ })?;+ let s = tokio::time::timeout(self.timeout, tokio::net::TcpStream::connect(addr))+ .await+ .map_err(|_| {+ std::io::Error::new(std::io::ErrorKind::TimedOut, "connect timed out")+ })??;+ s.set_nodelay(true)?;+ Ok(Box::new(s))+ }+}diff --git a/libs/cua/crates/cua-volume/src/replica/wire.rs b/libs/cua/crates/cua-volume/src/replica/wire.rsnew file mode 100644index 0000000000..502ee7ecd6--- /dev/null+++ b/libs/cua/crates/cua-volume/src/replica/wire.rs@@ -0,0 +1,479 @@+// SPDX-License-Identifier: FSL-1.1-MIT+// Copyright (c) 2026 Cua AI, Inc.++//! The peer protocol: framed JSON with an optional binary tail.+//!+//! ```text+//! [len: u32 BE][json_len: u32 BE][json][data] len = 4 + json_len + data+//! ```+//!+//! Two links join a pair of machines. The *control* link carries heartbeats,+//! era changes, log records and small requests; it never carries bulk bytes,+//! so a slow link cannot delay a heartbeat behind a file. The *data* link+//! carries block transfers and large writes.++use serde::{Deserialize, Serialize};++use super::MachineId;+use super::era::{EraRec, Pos};+use super::oplog::Op;+use super::roles::Facts;+use crate::Error;+use crate::backend::{Condition, ObjectMeta, VersionInfo};++/// Largest frame accepted.+pub const MAX_FRAME: usize = 8 * 1024 * 1024;++/// Which link of the pair.+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]+#[serde(rename_all = "snake_case")]+pub enum Chan {+ Control,+ Data,+}++/// The first message each side sends on a new link.+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]+pub struct Hello {+ pub machine: MachineId,+ pub name: String,+ pub chan: Chan,+ pub lineage: String,+ pub eras: Vec<EraRec>,+ pub applied: Pos,+ pub tail: Pos,+ pub facts: Facts,+ /// Protocol version.+ pub v: u32,+}++/// Protocol version.+pub const VERSION: u32 = 1;++/// A heartbeat.+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]+pub struct Beat {+ pub epoch: u64,+ pub owner: MachineId,+ /// Which volume this machine belongs to (two machines of different+ /// lineages that find each other merge, the less reliable one joining).+ #[serde(default)]+ pub lineage: String,+ /// The newest eras the sender knows (so a machine that missed a+ /// takeover learns it from any heartbeat).+ #[serde(default)]+ pub eras: Vec<EraRec>,+ /// Why the sender cannot keep a copy (a full disk), when it cannot.+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub problem: Option<String>,+ pub applied: Pos,+ pub tail: Pos,+ pub facts: Facts,+ /// Changes waiting in the sender's outbox.+ #[serde(default)]+ pub pending: u64,+}++/// A precondition on the wire.+#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]+#[serde(tag = "c", content = "etag", rename_all = "snake_case")]+pub enum Cond {+ #[default]+ None,+ IfNoneMatch,+ IfMatch(String),+}++impl From<Condition> for Cond {+ fn from(c: Condition) -> Cond {+ match c {+ Condition::None => Cond::None,+ Condition::IfNoneMatch => Cond::IfNoneMatch,+ Condition::IfMatch(e) => Cond::IfMatch(e),+ }+ }+}++impl From<Cond> for Condition {+ fn from(c: Cond) -> Condition {+ match c {+ Cond::None => Condition::None,+ Cond::IfNoneMatch => Condition::IfNoneMatch,+ Cond::IfMatch(e) => Condition::IfMatch(e),+ }+ }+}++/// Where the bytes of a version a peer wants live on the sender.+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]+#[serde(tag = "s", rename_all = "snake_case")]+pub enum Src {+ /// The sender's data store (history is kept).+ Data { key: String, version: String },+ /// The sender's spool of changes waiting for the owner.+ Spool { key: String, version: String },+}++/// A change waiting for the owner, as the owner sees it.+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]+pub struct Pending {+ pub id: String,+ pub key: String,+ /// `None` for a delete.+ pub meta: Option<ObjectMeta>,+ /// The head version the writer saw before its change.+ pub base: Option<String>,+ pub cond: Cond,+ pub machine: MachineId,+ pub name: String,+ pub ts_ms: u64,+ /// Where the content is on the sender.+ pub src: Src,+ /// Offset and length of an inline body in the frame's data.+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub body: Option<(u64, u64)>,+}++/// What the owner did with one pending change.+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]+#[serde(tag = "r", rename_all = "snake_case")]+pub enum Outcome {+ /// Applied as the new head.+ Applied { meta: ObjectMeta, pos: Pos },+ /// A delete was applied.+ Deleted { pos: Pos },+ /// The owner already had this content at the key.+ Merged,+ /// The key changed meanwhile; the content is at `key` as a conflict copy.+ Copy { key: String, pos: Pos },+ /// A delete lost to a newer write; the write stays.+ Kept,+ /// A precondition failed.+ Precondition { message: String },+ /// Could not apply.+ Failed { message: String },+}++/// A forwarded `Backend` call.+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]+#[serde(tag = "q", rename_all = "snake_case")]+pub enum Req {+ /// Body in `data`.+ Put {+ key: String,+ cond: Cond,+ },+ Delete {+ key: String,+ cond: Cond,+ },+ Head {+ key: String,+ },+ Get {+ key: String,+ version: Option<String>,+ },+ GetRange {+ key: String,+ version: String,+ offset: u64,+ len: u64,+ },+ List {+ prefix: String,+ },+ ListDir {+ folder: String,+ },+ Versions {+ key: String,+ },+ /// Heads after `after_key`, at most `max` (a snapshot's pages).+ Heads {+ after_key: String,+ max: usize,+ },+}++/// The answer to a [`Req`].+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]+#[serde(tag = "p", rename_all = "snake_case")]+pub enum Rep {+ Meta {+ meta: ObjectMeta,+ pos: Pos,+ /// Machines holding it when the owner answered.+ copies: u32,+ /// The owner counted it durable (enough copies).+ durable: bool,+ },+ Done {+ pos: Pos,+ copies: u32,+ durable: bool,+ },+ Head {+ meta: Option<ObjectMeta>,+ },+ /// The bytes are in `data`.+ Bytes {+ meta: ObjectMeta,+ },+ Range,+ List {+ items: Vec<ObjectMeta>,+ },+ ListDir {+ files: Vec<ObjectMeta>,+ folders: Vec<String>,+ },+ Versions {+ items: Vec<VersionInfo>,+ },+ Heads {+ items: Vec<ObjectMeta>,+ done: bool,+ at: Pos,+ },+ Fail {+ kind: String,+ message: String,+ },+}++impl Rep {+ pub fn fail(e: &Error) -> Rep {+ Rep::Fail {+ kind: e.tag().to_string(),+ message: match e {+ Error::Invalid(m)+ | Error::NotFound(m)+ | Error::Forbidden(m)+ | Error::Precondition(m)+ | Error::Backend(m)+ | Error::NotConfirmed(m) => m.clone(),+ other => other.to_string(),+ },+ }+ }++ /// The error a `Fail` stands for.+ pub fn into_error(kind: &str, message: String) -> Error {+ match kind {+ "invalid_argument" => Error::Invalid(message),+ "not_found" => Error::NotFound(message),+ "forbidden" => Error::Forbidden(message),+ "precondition_failed" => Error::Precondition(message),+ _ => Error::Backend(message),+ }+ }+}++/// A record on the wire; inline bodies are slices of the frame's data.+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]+pub struct RecordW {+ pub pos: Pos,+ pub op: Op,+ pub at_ms: u64,+ /// Offset and length of the body in the frame's data (small puts).+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub body: Option<(u64, u64)>,+}++/// Everything that crosses a link.+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]+#[serde(tag = "t", rename_all = "snake_case")]+pub enum Msg {+ Hello(Hello),+ Ping(Beat),+ Pong(Beat),+ /// The sender is going away (sleep, shutdown, network lost).+ Bye {+ reason: String,+ },+ /// Log records from the owner.+ Append {+ epoch: u64,+ records: Vec<RecordW>,+ },+ /// Replica to owner: applied through `applied`.+ Ack {+ applied: Pos,+ },+ /// The sender is below the receiver's epoch; here is the history.+ Fenced {+ eras: Vec<EraRec>,+ },+ /// The receiver holds records past this era's base: keep up to `keep`.+ Diverged {+ keep: Pos,+ eras: Vec<EraRec>,+ },+ /// Replica to owner: a gap; resend after `applied`.+ Behind {+ applied: Pos,+ },+ /// Owner to replica: the log no longer reaches you; pull a snapshot at `at`.+ Snapshot {+ at: Pos,+ },+ /// A candidate asks to take over at `epoch`.+ Prepare {+ epoch: u64,+ base: Pos,+ },+ Promise {+ ok: bool,+ reason: String,+ },+ /// A new era (takeover or hand-back).+ Era {+ recs: Vec<EraRec>,+ },+ /// The owner offers the era to `to` at `epoch`, having logged through `base`.+ Handoff {+ epoch: u64,+ to: MachineId,+ base: Pos,+ },+ HandoffReply {+ ok: bool,+ reason: String,+ },+ /// Changes waiting for the owner.+ Submit {+ ops: Vec<Pending>,+ },+ Submitted {+ outcomes: Vec<Outcome>,+ },+ Req(Req),+ Rep(Rep),+ /// Block hashes of a source: `count` hashes from block `from`.+ Hashes {+ src: Src,+ block: u64,+ from: u64,+ count: u64,+ },+ HashesAre {+ hashes: Vec<String>,+ size: u64,+ },+ /// One block of a source, bytes in `data`.+ Block {+ src: Src,+ block: u64,+ idx: u64,+ },+ BlockIs,+ /// A refusal or failure of any request.+ Error {+ kind: String,+ message: String,+ },+}++/// One frame.+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]+pub struct Frame {+ pub id: u64,+ /// The id of the request this answers.+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub re: Option<u64>,+ pub msg: Msg,+}++/// Encodes a frame and its data.+pub fn encode(f: &Frame, data: &[u8]) -> std::io::Result<Vec<u8>> {+ let json = serde_json::to_vec(f)+ .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;+ let len = 4 + json.len() + data.len();+ if len > MAX_FRAME {+ return Err(std::io::Error::new(+ std::io::ErrorKind::InvalidInput,+ format!("frame of {len} bytes"),+ ));+ }+ let mut out = Vec::with_capacity(8 + json.len() + data.len());+ out.extend_from_slice(&(len as u32).to_be_bytes());+ out.extend_from_slice(&(json.len() as u32).to_be_bytes());+ out.extend_from_slice(&json);+ out.extend_from_slice(data);+ Ok(out)+}++/// Reads one frame; `None` at a clean end of stream.+pub async fn read<R: tokio::io::AsyncRead + Unpin>(+ r: &mut R,+) -> std::io::Result<Option<(Frame, Vec<u8>)>> {+ use tokio::io::AsyncReadExt as _;+ let mut len = [0u8; 4];+ match r.read_exact(&mut len).await {+ Ok(_) => {}+ Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => return Ok(None),+ Err(e) => return Err(e),+ }+ let len = u32::from_be_bytes(len) as usize;+ if !(4..=MAX_FRAME).contains(&len) {+ return Err(std::io::Error::new(+ std::io::ErrorKind::InvalidData,+ format!("frame of {len} bytes"),+ ));+ }+ let mut body = vec![0u8; len];+ r.read_exact(&mut body).await?;+ let json_len = u32::from_be_bytes([body[0], body[1], body[2], body[3]]) as usize;+ if 4 + json_len > len {+ return Err(std::io::Error::new(+ std::io::ErrorKind::InvalidData,+ "json longer than its frame",+ ));+ }+ let data = body.split_off(4 + json_len);+ let frame: Frame = serde_json::from_slice(&body[4..])+ .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;+ Ok(Some((frame, data)))+}++#[cfg(test)]+mod tests {+ use super::*;++ #[tokio::test]+ async fn frames_round_trip_with_data() {+ let f = Frame {+ id: 7,+ re: Some(3),+ msg: Msg::Block {+ src: Src::Data {+ key: "a/b".into(),+ version: "v".into(),+ },+ block: 1 << 20,+ idx: 4,+ },+ };+ let bytes = encode(&f, b"payload").unwrap();+ let mut r = std::io::Cursor::new(bytes);+ let (g, d) = read(&mut r).await.unwrap().unwrap();+ assert_eq!(f, g);+ assert_eq!(d, b"payload");+ assert!(read(&mut r).await.unwrap().is_none());+ }++ #[test]+ fn errors_keep_their_kind_across_the_wire() {+ for e in [+ Error::NotFound("k".into()),+ Error::Precondition("changed".into()),+ Error::Invalid("x".into()),+ ] {+ let Rep::Fail { kind, message } = Rep::fail(&e) else {+ panic!()+ };+ assert_eq!(Rep::into_error(&kind, message), e);+ }+ }+}diff --git a/libs/cua/crates/cua-volume/src/replica/words.rs b/libs/cua/crates/cua-volume/src/replica/words.rsnew file mode 100644index 0000000000..2d27eb3d06--- /dev/null+++ b/libs/cua/crates/cua-volume/src/replica/words.rs@@ -0,0 +1,206 @@+// SPDX-License-Identifier: FSL-1.1-MIT+// Copyright (c) 2026 Cua AI, Inc.++//! The one place that says, in plain words, what happens when you close the+//! laptop. Shown when a durable agent starts and in the app. It uses the+//! names from the machines list and nothing else: no settings, no new+//! nouns, no promise the facts do not back.++use serde::{Deserialize, Serialize};++/// The facts the sentence is made of.+#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]+pub struct Summary {+ /// This machine's name ("MacBook Pro").+ pub this_machine: String,+ /// The machine running the durable agent's driver, when known.+ pub run_on: Option<String>,+ /// Machines signed in to the account, including this one.+ pub machines: usize,+ /// Names of machines that hold a copy of the files and are up now.+ pub holders_up: Vec<String>,+ /// Names of machines that hold a copy and are away now.+ pub holders_away: Vec<String>,+ /// Changes saved only on this machine, waiting for another one.+ pub pending_here: u64,+ /// The machine the changes are waiting for, when known.+ pub pending_for: Option<String>,+ /// This machine can sleep when its lid closes.+ pub this_sleeps: bool,+ /// A sleep assertion is held here for the run.+ pub sleep_blocked: bool,+}++fn machines(n: usize) -> String {+ if n == 1 {+ "1 machine".into()+ } else {+ format!("{n} machines")+ }+}++fn list(names: &[String]) -> String {+ match names {+ [] => String::new(),+ [a] => a.clone(),+ [a, b] => format!("{a} and {b}"),+ [rest @ .., last] => format!("{}, and {last}", rest.join(", ")),+ }+}++/// What will happen when this machine closes, in one short paragraph.+pub fn closing_summary(s: &Summary) -> String {+ let here = s.this_machine.as_str();+ let on_other = s+ .run_on+ .as_deref()+ .filter(|r| !r.is_empty() && *r != here)+ .map(str::to_string);+ let mut out = String::new();+ match (&on_other, s.machines) {+ (Some(other), _) => {+ out.push_str(&format!("Your agent keeps running on {other}. "));+ let copies = s.holders_up.len();+ let other_holds = s.holders_up.iter().any(|h| h == other);+ if copies >= 2 {+ out.push_str(&format!(+ "Files are on {}. You can close this machine.",+ machines(copies)+ ));+ } else if other_holds {+ let missing = if s.holders_away.is_empty() {+ here.to_string()+ } else {+ list(&s.holders_away)+ };+ out.push_str(&format!(+ "Files are on 1 machine until {missing} is back. You can close this machine; nothing is lost, there is just one copy until then."+ ));+ } else {+ out.push_str("Files are not on that machine yet; wait for them to copy before you close this machine.");+ }+ }+ (None, n) if n <= 1 => {+ out.push_str(&format!(+ "Your agent runs on {here} and will pause when it sleeps. "+ ));+ if s.sleep_blocked {+ out.push_str("It stays awake while the agent works, but closing the lid still puts a laptop to sleep unless it is plugged in with a display attached. ");+ }+ out.push_str("Add another machine and it keeps running when you close this one.");+ }+ (None, _) => {+ out.push_str(&format!("Your agent runs on {here}"));+ if s.this_sleeps {+ out.push_str(" and will pause when it sleeps. ");+ } else {+ out.push_str(". ");+ }+ if s.holders_away.is_empty() {+ out.push_str("Your other machines are not available to take over right now.");+ } else {+ out.push_str(&format!("{} is away right now.", list(&s.holders_away)));+ }+ }+ }+ if s.pending_here > 0 {+ let who = s.pending_for.as_deref().unwrap_or("another machine");+ out.push_str(&format!(+ " {} saved only on {here} will sync when {who} is back.",+ if s.pending_here == 1 {+ "1 change".to_string()+ } else {+ format!("{} changes", s.pending_here)+ }+ ));+ }+ out+}++#[cfg(test)]+mod tests {+ use super::*;++ fn base() -> Summary {+ Summary {+ this_machine: "MacBook".into(),+ machines: 2,+ ..Default::default()+ }+ }++ #[test]+ fn two_machines_both_holding_files() {+ let s = Summary {+ run_on: Some("Mac Studio".into()),+ holders_up: vec!["Mac Studio".into(), "MacBook".into()],+ ..base()+ };+ assert_eq!(+ closing_summary(&s),+ "Your agent keeps running on Mac Studio. Files are on 2 machines. You can close this machine."+ );+ }++ #[test]+ fn only_the_studio_holds_files_until_the_laptop_is_back() {+ let s = Summary {+ run_on: Some("Mac Studio".into()),+ holders_up: vec!["Mac Studio".into()],+ holders_away: vec!["MacBook".into()],+ ..base()+ };+ let t = closing_summary(&s);+ assert!(t.starts_with("Your agent keeps running on Mac Studio. Files are on 1 machine until MacBook is back."), "{t}");+ assert!(!t.contains("2 machines"));+ }++ #[test]+ fn one_machine_says_it_will_pause_and_is_honest_about_the_lid() {+ let s = Summary {+ machines: 1,+ run_on: Some("MacBook".into()),+ sleep_blocked: true,+ ..base()+ };+ let t = closing_summary(&s);+ assert!(t.contains("will pause when it sleeps"), "{t}");+ assert!(+ t.contains("closing the lid still puts a laptop to sleep"),+ "{t}"+ );+ assert!(t.contains("Add another machine"), "{t}");+ }++ #[test]+ fn pending_changes_name_who_they_wait_for() {+ let s = Summary {+ run_on: Some("Mac Studio".into()),+ holders_up: vec!["Mac Studio".into(), "MacBook".into()],+ pending_here: 3,+ pending_for: Some("Mac Studio".into()),+ ..base()+ };+ let t = closing_summary(&s);+ assert!(+ t.ends_with("3 changes saved only on MacBook will sync when Mac Studio is back."),+ "{t}"+ );+ }++ #[test]+ fn the_agent_here_with_a_studio_away_does_not_promise_takeover() {+ let s = Summary {+ run_on: Some("MacBook".into()),+ this_sleeps: true,+ holders_away: vec!["Mac Studio".into()],+ ..base()+ };+ let t = closing_summary(&s);+ assert!(+ t.starts_with("Your agent runs on MacBook and will pause when it sleeps."),+ "{t}"+ );+ assert!(t.contains("Mac Studio is away right now."), "{t}");+ }+}diff --git a/libs/cua/crates/cua-volume/src/replica/xfer.rs b/libs/cua/crates/cua-volume/src/replica/xfer.rsnew file mode 100644index 0000000000..679b27e35d--- /dev/null+++ b/libs/cua/crates/cua-volume/src/replica/xfer.rs@@ -0,0 +1,343 @@+// SPDX-License-Identifier: FSL-1.1-MIT+// Copyright (c) 2026 Cua AI, Inc.++//! Moving big objects between machines in blocks.+//!+//! The receiver pulls. It asks the sender for the SHA-256 of every 1 MiB+//! block of a version, keeps the blocks it already has (from a partial file+//! of an earlier try, or from its own previous version of the key at the+//! same offsets), fetches the rest a few at a time, checks each block's+//! hash on arrival, and checks the whole object's hash before it is used.+//! So a link that drops and returns costs the blocks in flight, not the file,+//! and a file that only changed at its end costs its end.++use std::path::PathBuf;+use std::sync::Arc;++use tokio::task::JoinSet;++use super::link::Link;+use super::node::Inner;+use super::wire::{Msg, Src};+use crate::backend::ObjectMeta;+use crate::{Error, Result, sha256_hex};++#[cfg(unix)]+fn read_at(f: &std::fs::File, buf: &mut [u8], offset: u64) -> std::io::Result<usize> {+ std::os::unix::fs::FileExt::read_at(f, buf, offset)+}++#[cfg(windows)]+fn read_at(f: &std::fs::File, buf: &mut [u8], offset: u64) -> std::io::Result<usize> {+ std::os::windows::fs::FileExt::seek_read(f, buf, offset)+}++fn read_block(path: &std::path::Path, offset: u64, len: usize) -> std::io::Result<Vec<u8>> {+ let f = std::fs::File::open(path)?;+ let size = f.metadata()?.len();+ if offset >= size {+ return Ok(vec![]);+ }+ let want = len.min((size - offset) as usize);+ let mut buf = vec![0u8; want];+ let mut done = 0;+ while done < want {+ let n = read_at(&f, &mut buf[done..], offset + done as u64)?;+ if n == 0 {+ break;+ }+ done += n;+ }+ buf.truncate(done);+ Ok(buf)+}++fn io_err(e: std::io::Error) -> Error {+ if e.raw_os_error() == Some(28) {+ Error::Backend("the disk is full".into())+ } else {+ Error::from(e)+ }+}++impl Inner {+ fn src_path(&self, src: &Src) -> Result<PathBuf> {+ match src {+ Src::Data { key, version } => self.store.version_file(key, version),+ Src::Spool { key, version } => self.spool.version_file(key, version),+ }+ }++ async fn hashes_of(&self, path: PathBuf, block: u64) -> Result<(Arc<Vec<String>>, u64)> {+ let ck = format!("{}:{block}", path.display());+ if let Some(h) = self.hashes.lock().expect("hashes").get(&ck).cloned() {+ let size = std::fs::metadata(&path)?.len();+ return Ok((h, size));+ }+ let p = path.clone();+ let (hashes, size) = tokio::task::spawn_blocking(move || -> Result<(Vec<String>, u64)> {+ use std::io::Read as _;+ let mut f = std::fs::File::open(&p)?;+ let size = f.metadata()?.len();+ let mut out = vec![];+ let mut buf = vec![0u8; block as usize];+ loop {+ let mut n = 0;+ while n < buf.len() {+ let r = f.read(&mut buf[n..])?;+ if r == 0 {+ break;+ }+ n += r;+ }+ if n == 0 {+ break;+ }+ out.push(sha256_hex(&buf[..n]));+ if n < buf.len() {+ break;+ }+ }+ Ok((out, size))+ })+ .await+ .map_err(|e| Error::Backend(e.to_string()))??;+ let arc = Arc::new(hashes);+ let mut c = self.hashes.lock().expect("hashes");+ if c.len() > 32+ && let Some(k) = c.keys().next().cloned()+ {+ c.remove(&k);+ }+ c.insert(ck, arc.clone());+ Ok((arc, size))+ }++ pub(super) async fn handle_hashes(&self, src: Src, block: u64, from: u64, count: u64) -> Msg {+ let r = async {+ let path = self.src_path(&src)?;+ let (all, size) = self.hashes_of(path, block).await?;+ let from = (from as usize).min(all.len());+ let to = from.saturating_add(count as usize).min(all.len());+ Ok::<_, Error>(Msg::HashesAre {+ hashes: all[from..to].to_vec(),+ size,+ })+ }+ .await;+ r.unwrap_or_else(|e| Msg::Error {+ kind: e.tag().into(),+ message: e.to_string(),+ })+ }++ pub(super) async fn handle_block(&self, src: Src, block: u64, idx: u64) -> Result<Vec<u8>> {+ let path = self.src_path(&src)?;+ tokio::task::spawn_blocking(move || read_block(&path, idx * block, block as usize))+ .await+ .map_err(|e| Error::Backend(e.to_string()))?+ .map_err(|e| match e.kind() {+ std::io::ErrorKind::NotFound => Error::NotFound("that version is not here".into()),+ _ => Error::from(e),+ })+ }++ /// Fetches `meta` from `link`'s machine into a file in the incoming+ /// directory, verified, and returns its path. `seed` is this machine's+ /// own previous version of the key, whose unchanged blocks are reused.+ pub(super) async fn pull(+ self: &Arc<Self>,+ link: &Arc<Link>,+ src: Src,+ meta: &ObjectMeta,+ seed: Option<&ObjectMeta>,+ ) -> Result<PathBuf> {+ let block = self.cfg.block;+ let size = meta.size;+ let nblocks = size.div_ceil(block);+ let inc = self.dir.join("incoming");+ std::fs::create_dir_all(&inc)?;+ let stem = format!("{}-{}", meta.etag, size);+ let part = inc.join(format!("{stem}.part"));+ let havef = inc.join(format!("{stem}.have"));+ cua_home::guard_write(&part)?;+ // What an earlier attempt left.+ let mut have: Vec<bool> = match (std::fs::read(&havef), part.is_file()) {+ (Ok(b), true) if b.len() as u64 == nblocks => b.iter().map(|x| *x == 1).collect(),+ _ => vec![false; nblocks as usize],+ };+ if !part.is_file() {+ have = vec![false; nblocks as usize];+ }+ {+ let f = std::fs::OpenOptions::new()+ .create(true)+ .truncate(false)+ .write(true)+ .open(&part)+ .map_err(io_err)?;+ f.set_len(size).map_err(io_err)?;+ }+ // The hashes, in pages.+ let mut hashes: Vec<String> = Vec::with_capacity(nblocks as usize);+ let to = self.cfg.data_timeout;+ while (hashes.len() as u64) < nblocks {+ let from = hashes.len() as u64;+ let (m, _) = link+ .call_strict(+ Msg::Hashes {+ src: src.clone(),+ block,+ from,+ count: 4096,+ },+ &[],+ to,+ )+ .await+ .map_err(|e| Error::Backend(e.to_string()))?;+ match m {+ Msg::HashesAre { hashes: h, size: s } => {+ if s != size || h.is_empty() {+ return Err(Error::Backend(+ "the sender's version is not the one announced".into(),+ ));+ }+ hashes.extend(h);+ }+ Msg::Error { kind, message } => {+ return Err(super::wire::Rep::into_error(&kind, message));+ }+ _ => return Err(Error::Backend("unexpected reply".into())),+ }+ }+ // Blocks this machine already holds from its own previous version.+ let seed_path = seed.and_then(|h| self.store.version_file(&h.key, &h.version).ok());+ let mut need: Vec<u64> = vec![];+ for idx in 0..nblocks {+ if have[idx as usize] {+ continue;+ }+ if let Some(sp) = &seed_path+ && let Ok(b) = read_block(sp, idx * block, block as usize)+ && !b.is_empty()+ && sha256_hex(&b) == hashes[idx as usize]+ {+ write_at(&part, idx * block, &b).map_err(io_err)?;+ have[idx as usize] = true;+ continue;+ }+ need.push(idx);+ }+ persist_have(&havef, &have);+ // The rest, a window at a time.+ let hashes = Arc::new(hashes);+ let mut set: JoinSet<Result<(u64, Vec<u8>)>> = JoinSet::new();+ let mut next = 0usize;+ let mut since_persist = 0;+ let window = self.cfg.window.max(1);+ let mut failed: Option<Error> = None;+ while next < need.len() || !set.is_empty() {+ while failed.is_none() && next < need.len() && set.len() < window {+ let idx = need[next];+ next += 1;+ let (link, src, hashes) = (link.clone(), src.clone(), hashes.clone());+ set.spawn(async move {+ let (m, bytes) = link+ .call_strict(Msg::Block { src, block, idx }, &[], to)+ .await+ .map_err(|e| Error::Backend(e.to_string()))?;+ match m {+ Msg::BlockIs => {}+ Msg::Error { kind, message } => {+ return Err(super::wire::Rep::into_error(&kind, message));+ }+ _ => return Err(Error::Backend("unexpected reply".into())),+ }+ if sha256_hex(&bytes) != hashes[idx as usize] {+ return Err(Error::Backend(format!("block {idx} arrived damaged")));+ }+ Ok((idx, bytes))+ });+ }+ let Some(joined) = set.join_next().await else {+ break;+ };+ match joined {+ Ok(Ok((idx, bytes))) => {+ write_at(&part, idx * block, &bytes).map_err(io_err)?;+ have[idx as usize] = true;+ since_persist += 1;+ if since_persist >= 16 {+ since_persist = 0;+ persist_have(&havef, &have);+ }+ }+ Ok(Err(e)) => failed = Some(e),+ Err(e) => failed = Some(Error::Backend(e.to_string())),+ }+ }+ persist_have(&havef, &have);+ if let Some(e) = failed {+ return Err(e);+ }+ // The whole object, once.+ let p = part.clone();+ let etag = meta.etag.clone();+ let ok = tokio::task::spawn_blocking(move || -> std::io::Result<bool> {+ use sha2::Digest as _;+ use std::io::Read as _;+ let mut f = std::fs::File::open(&p)?;+ let mut h = sha2::Sha256::new();+ let mut buf = vec![0u8; 1 << 20];+ loop {+ let n = f.read(&mut buf)?;+ if n == 0 {+ break;+ }+ h.update(&buf[..n]);+ }+ Ok(hex::encode(h.finalize()) == etag)+ })+ .await+ .map_err(|e| Error::Backend(e.to_string()))?+ .map_err(io_err)?;+ if !ok {+ let _ = std::fs::remove_file(&part);+ let _ = std::fs::remove_file(&havef);+ return Err(Error::Backend(+ "the object did not match its hash; it will be fetched again".into(),+ ));+ }+ let _ = std::fs::remove_file(&havef);+ Ok(part)+ }+}++fn write_at(path: &std::path::Path, offset: u64, bytes: &[u8]) -> std::io::Result<()> {+ let f = std::fs::OpenOptions::new().write(true).open(path)?;+ #[cfg(unix)]+ {+ std::os::unix::fs::FileExt::write_all_at(&f, bytes, offset)+ }+ #[cfg(windows)]+ {+ let mut done = 0;+ while done < bytes.len() {+ done += std::os::windows::fs::FileExt::seek_write(+ &f,+ &bytes[done..],+ offset + done as u64,+ )?;+ }+ Ok(())+ }+}++fn persist_have(path: &std::path::Path, have: &[bool]) {+ let bytes: Vec<u8> = have.iter().map(|b| u8::from(*b)).collect();+ let tmp = path.with_extension("tmp");+ if std::fs::write(&tmp, bytes).is_ok() {+ let _ = std::fs::rename(&tmp, path);+ }+}diff --git a/libs/cua/crates/cua-volume/tests/common/mod.rs b/libs/cua/crates/cua-volume/tests/common/mod.rsnew file mode 100644index 0000000000..66afed040b--- /dev/null+++ b/libs/cua/crates/cua-volume/tests/common/mod.rs@@ -0,0 +1,255 @@+// SPDX-License-Identifier: FSL-1.1-MIT+// Copyright (c) 2026 Cua AI, Inc.++//! Shared by the replication simulations: a group of real nodes on a+//! simulated network.++#![allow(dead_code)]++use std::collections::HashMap;+use std::sync::Arc;+use std::time::{Duration, Instant};++use cua_volume::backend::{Backend, Condition};+use cua_volume::fs::FsBackend;+use cua_volume::replica::simnet::{Impair, SimNet};+use cua_volume::replica::{Config, Facts, Member, ReplicaNode, ReplicatedBackend};++pub struct M {+ #[allow(dead_code)]+ pub id: String,+ pub node: ReplicaNode,+ pub backend: Arc<ReplicatedBackend>,+ pub store: FsBackend,+ pub dir: tempfile::TempDir,+}++pub struct Sim {+ pub net: SimNet,+ pub m: HashMap<String, M>,+ pub members: Vec<Member>,+ pub cfg: Config,+}++pub fn member(id: &str, rank: u64) -> Member {+ Member {+ id: id.into(),+ name: id.to_string(),+ facts: Facts {+ rank,+ free_disk: 500 << 30,+ ..Default::default()+ },+ last_seen_ms: 0,+ }+}++impl Sim {+ pub async fn new(specs: &[(&str, u64)]) -> Sim {+ Sim::with(specs, Config::fast()).await+ }++ pub async fn with(specs: &[(&str, u64)], cfg: Config) -> Sim {+ // CUA_TEST_LOG=cua_volume=debug shows what the nodes decide.+ if let Ok(f) = std::env::var("CUA_TEST_LOG") {+ let _ = tracing_subscriber::fmt()+ .with_env_filter(f)+ .with_test_writer()+ .try_init();+ }+ let net = SimNet::new();+ let members: Vec<Member> = specs.iter().map(|(i, r)| member(i, *r)).collect();+ let mut sim = Sim {+ net,+ m: HashMap::new(),+ members,+ cfg,+ };+ for (id, _) in specs {+ sim.boot(id, tempfile::tempdir().unwrap()).await;+ }+ sim+ }++ pub async fn boot(&mut self, id: &str, dir: tempfile::TempDir) {+ let (tr, mut rx) = self.net.join(id);+ let store = FsBackend::new(dir.path().join("data"));+ let node = ReplicaNode::open(+ self.cfg.clone(),+ id,+ id,+ dir.path(),+ store.clone(),+ Arc::new(tr),+ self.members.clone(),+ )+ .await+ .unwrap();+ let facts = self+ .members+ .iter()+ .find(|m| m.id == id)+ .unwrap()+ .facts+ .clone();+ node.set_facts(facts);+ let n2 = node.clone();+ tokio::spawn(async move {+ while let Some((_from, s)) = rx.recv().await {+ n2.accept(s).await;+ }+ });+ node.start();+ let backend = node.backend();+ self.m.insert(+ id.to_string(),+ M {+ id: id.to_string(),+ node,+ backend,+ store,+ dir,+ },+ );+ }++ pub fn get(&self, id: &str) -> &M {+ &self.m[id]+ }++ /// A process restart: shut down, keep the directory, boot again.+ pub async fn restart(&mut self, id: &str) {+ let old = self.m.remove(id).unwrap();+ old.node.shutdown().await;+ let dir = old.dir;+ drop(old.backend);+ self.boot(id, dir).await;+ }++ pub fn cut(&self, a: &str, b: &str) {+ self.net.set_both(+ a,+ b,+ Impair {+ sever: true,+ ..Default::default()+ },+ );+ }++ pub fn heal(&self, a: &str, b: &str) {+ self.net.heal(a, b);+ }++ pub async fn shutdown(&self) {+ for m in self.m.values() {+ m.node.shutdown().await;+ }+ }+}++pub async fn eventually<F, Fut>(what: &str, secs: u64, mut f: F)+where+ F: FnMut() -> Fut,+ Fut: std::future::Future<Output = bool>,+{+ let end = Instant::now() + Duration::from_secs(secs);+ loop {+ if f().await {+ return;+ }+ if Instant::now() > end {+ panic!("timed out waiting for: {what}");+ }+ tokio::time::sleep(Duration::from_millis(20)).await;+ }+}++/// A machine that needs an ordered write (a lease, a fence) keeps asking+/// until it can have one; that is what makes it take over from a dead owner.+pub async fn needs_authority(m: &M, key: &str) {+ eventually("an ordered write succeeds", 30, || async {+ m.backend+ .put(key, b"x".to_vec(), Condition::IfNoneMatch)+ .await+ .is_ok()+ })+ .await;+}++pub async fn head_etag(m: &M, key: &str) -> Option<String> {+ m.store.head(key).await.unwrap().map(|h| h.etag)+}++pub async fn heads(m: &M) -> Vec<(String, String)> {+ m.store+ .list("")+ .await+ .unwrap()+ .into_iter()+ .map(|h| (h.key, h.etag))+ .collect()+}++pub async fn converged(sim: &Sim, ids: &[&str]) -> bool {+ let first = heads(sim.get(ids[0])).await;+ for id in &ids[1..] {+ if heads(sim.get(id)).await != first {+ return false;+ }+ }+ true+}++pub async fn group_ready(sim: &Sim, owner: &str) {+ let dump = || async {+ for m in sim.m.values() {+ let s = m.node.status().await;+ eprintln!(+ "{}: owner={} epoch={} peers={:?}",+ m.id,+ s.owner,+ s.epoch,+ s.peers+ .iter()+ .map(|p| (p.id.clone(), p.health.clone(), p.connected))+ .collect::<Vec<_>>()+ );+ }+ };+ let end = Instant::now() + Duration::from_secs(14);+ while Instant::now() < end {+ let mut ok = true;+ for m in sim.m.values() {+ if m.node.status().await.owner != owner {+ ok = false;+ }+ }+ if ok {+ break;+ }+ tokio::time::sleep(Duration::from_millis(20)).await;+ }+ dump().await;+ eventually("the group agrees on an owner", 5, || async {+ for m in sim.m.values() {+ let s = m.node.status().await;+ if s.owner != owner {+ return false;+ }+ }+ true+ })+ .await;+ // Everyone is connected to everyone.+ eventually("the machines are connected", 15, || async {+ for m in sim.m.values() {+ let s = m.node.status().await;+ if s.peers.iter().any(|p| !p.connected) {+ return false;+ }+ }+ true+ })+ .await;+}diff --git a/libs/cua/crates/cua-volume/tests/replica_sim.rs b/libs/cua/crates/cua-volume/tests/replica_sim.rsnew file mode 100644index 0000000000..76e156bd67--- /dev/null+++ b/libs/cua/crates/cua-volume/tests/replica_sim.rs@@ -0,0 +1,1122 @@+// SPDX-License-Identifier: FSL-1.1-MIT+// Copyright (c) 2026 Cua AI, Inc.++//! Simulations of the replicated volume across machines: partitions, a+//! closed laptop, a takeover with the old owner waking up, jitter, big files+//! on a flapping link, a quorum of three. Time is scaled down (heartbeat+//! 50 ms) and the network is `SimNet`; the nodes are the real ones.++mod common;++use std::time::{Duration, Instant};++use common::*;+use cua_volume::backend::{Backend, Condition};+use cua_volume::replica::simnet::Impair;+use cua_volume::replica::{Config, Durability, PowerEvent};++#[tokio::test]+async fn two_machines_agree_on_an_owner_and_hold_two_copies() {+ let sim = Sim::new(&[("studio", 9000), ("laptop", 2000)]).await;+ // Both started alone with a volume of their own; the less reliable one+ // joins the other's and the more reliable owns it.+ group_ready(&sim, "studio").await;+ let studio = sim.get("studio");+ let (meta, d) = studio+ .backend+ .put_durable("agents/ada/notes.md", b"hello".to_vec(), Condition::None)+ .await+ .unwrap();+ assert_eq!(+ d,+ Durability::Replicated(2),+ "the write waits for the second copy"+ );+ let laptop = sim.get("laptop");+ // The second copy holds the same version and bytes.+ let h = laptop+ .store+ .head("agents/ada/notes.md")+ .await+ .unwrap()+ .unwrap();+ assert_eq!(h.version, meta.version);+ assert_eq!(+ laptop+ .store+ .get("agents/ada/notes.md", None)+ .await+ .unwrap()+ .0,+ b"hello"+ );+ let s = laptop.node.status().await;+ assert!(!s.i_own && s.owner == "studio" && s.copies_now == 2 && s.copies_wanted == 2);+ sim.shutdown().await;+}++#[tokio::test]+async fn a_write_on_the_laptop_goes_through_the_owner_and_both_see_it() {+ let sim = Sim::new(&[("studio", 9000), ("laptop", 2000)]).await;+ group_ready(&sim, "studio").await;+ let laptop = sim.get("laptop");+ let (meta, d) = laptop+ .backend+ .put_durable("public/a.txt", b"from the laptop".to_vec(), Condition::None)+ .await+ .unwrap();+ assert_eq!(d, Durability::Replicated(2));+ // Read-your-writes on the laptop, and the owner has it.+ assert_eq!(+ laptop+ .backend+ .get("public/a.txt", None)+ .await+ .unwrap()+ .1+ .version,+ meta.version+ );+ assert_eq!(+ sim.get("studio")+ .store+ .head("public/a.txt")+ .await+ .unwrap()+ .unwrap()+ .version,+ meta.version+ );+ // A conditional write from the laptop is atomic across machines: a+ // second create-only loses.+ laptop+ .backend+ .put(+ "agents/ada/.cua-lease",+ b"laptop".to_vec(),+ Condition::IfNoneMatch,+ )+ .await+ .unwrap();+ let e = sim+ .get("studio")+ .backend+ .put(+ "agents/ada/.cua-lease",+ b"studio".to_vec(),+ Condition::IfNoneMatch,+ )+ .await+ .unwrap_err();+ assert_eq!(e.tag(), "precondition_failed");+ sim.shutdown().await;+}++#[tokio::test]+async fn a_closed_laptop_never_blocks_the_studio_and_catches_up_when_it_returns() {+ let sim = Sim::new(&[("studio", 9000), ("laptop", 2000)]).await;+ group_ready(&sim, "studio").await;+ let studio = sim.get("studio");+ studio+ .backend+ .put("agents/ada/before.txt", b"1".to_vec(), Condition::None)+ .await+ .unwrap();+ // The lid closes: the laptop says goodbye and goes dark.+ sim.get("laptop")+ .node+ .notify_power(PowerEvent::WillSleep)+ .await;+ // The OS gives the machine a moment between "will sleep" and sleep.+ tokio::time::sleep(Duration::from_millis(50)).await;+ sim.get("laptop").node.freeze(true);+ sim.cut("studio", "laptop");+ let t0 = Instant::now();+ let mut last = Durability::Pending;+ for i in 0..20 {+ let (_, d) = studio+ .backend+ .put_durable(+ &format!("agents/ada/out-{i}.txt"),+ format!("line {i}").into_bytes(),+ Condition::None,+ )+ .await+ .unwrap();+ last = d;+ }+ // 20 writes while the second machine is away took 20 short waits at+ // most, and each says there is one copy.+ assert_eq!(last, Durability::Local(1));+ assert!(+ t0.elapsed() < Duration::from_secs(20 * 400 / 1000 + 4),+ "writes were held up: {:?}",+ t0.elapsed()+ );+ let s = studio.node.status().await;+ assert!(+ s.unreplicated > 0,+ "the status says files are on one machine"+ );+ let words = studio.node.summary(Some("studio")).await;+ assert_eq!(words.holders_up, vec!["studio".to_string()]);+ assert_eq!(words.holders_away, vec!["laptop".to_string()]);+ // The lid opens.+ sim.heal("studio", "laptop");+ sim.get("laptop").node.freeze(false);+ sim.get("laptop")+ .node+ .notify_power(PowerEvent::DidWake)+ .await;+ eventually("the laptop catches up", 15, || async {+ converged(&sim, &["studio", "laptop"]).await+ && sim.get("studio").node.status().await.unreplicated == 0+ })+ .await;+ assert_eq!(heads(sim.get("laptop")).await.len(), 21);+ sim.shutdown().await;+}++#[tokio::test]+async fn changes_made_on_a_laptop_with_no_owner_are_saved_and_sync_later() {+ let sim = Sim::new(&[("studio", 9000), ("laptop", 2000)]).await;+ group_ready(&sim, "studio").await;+ sim.cut("studio", "laptop");+ let laptop = sim.get("laptop");+ eventually("the laptop sees the studio go", 10, || async {+ laptop.node.status().await.peers[0].health != "up"+ })+ .await;+ let (_, d) = laptop+ .backend+ .put_durable(+ "agents/ada/draft.md",+ b"written on a plane".to_vec(),+ Condition::None,+ )+ .await+ .unwrap();+ assert_eq!(d, Durability::Pending);+ // The laptop shows its own write, and says what is waiting.+ assert_eq!(+ laptop+ .backend+ .get("agents/ada/draft.md", None)+ .await+ .unwrap()+ .0,+ b"written on a plane"+ );+ let s = laptop.node.status().await;+ assert_eq!(s.pending.len(), 1);+ let words = laptop.node.summary(Some("studio")).await;+ assert_eq!(words.pending_here, 1);+ // A conditional write cannot be ordered without the owner: it fails+ // plainly instead of pretending.+ let e = laptop+ .backend+ .put(+ "agents/ada/.cua-lease",+ b"x".to_vec(),+ Condition::IfNoneMatch,+ )+ .await+ .unwrap_err();+ assert!(e.to_string().contains("cannot be reached"), "{e}");+ sim.heal("studio", "laptop");+ eventually("the studio takes the change", 15, || async {+ head_etag(sim.get("studio"), "agents/ada/draft.md")+ .await+ .is_some()+ && laptop.node.status().await.pending.is_empty()+ })+ .await;+ assert_eq!(+ sim.get("studio")+ .store+ .get("agents/ada/draft.md", None)+ .await+ .unwrap()+ .0,+ b"written on a plane"+ );+ eventually("both agree", 10, || async {+ converged(&sim, &["studio", "laptop"]).await+ })+ .await;+ sim.shutdown().await;+}++#[tokio::test]+async fn two_machines_changing_one_file_keep_both_versions() {+ let sim = Sim::new(&[("studio", 9000), ("laptop", 2000)]).await;+ group_ready(&sim, "studio").await;+ sim.get("studio")+ .backend+ .put("agents/ada/plan.md", b"v0".to_vec(), Condition::None)+ .await+ .unwrap();+ eventually("replicated", 10, || async {+ head_etag(sim.get("laptop"), "agents/ada/plan.md")+ .await+ .is_some()+ })+ .await;+ sim.cut("studio", "laptop");+ let laptop = sim.get("laptop");+ eventually("laptop notices", 10, || async {+ laptop.node.status().await.peers[0].health != "up"+ })+ .await;+ laptop+ .backend+ .put(+ "agents/ada/plan.md",+ b"laptop edit".to_vec(),+ Condition::None,+ )+ .await+ .unwrap();+ sim.get("studio")+ .backend+ .put(+ "agents/ada/plan.md",+ b"studio edit".to_vec(),+ Condition::None,+ )+ .await+ .unwrap();+ sim.heal("studio", "laptop");+ eventually("both converge", 20, || async {+ converged(&sim, &["studio", "laptop"]).await+ && laptop.node.status().await.pending.is_empty()+ })+ .await;+ // The owner's version stays at the key; the laptop's edit is a copy.+ let st = sim.get("studio");+ assert_eq!(+ st.store.get("agents/ada/plan.md", None).await.unwrap().0,+ b"studio edit"+ );+ let all = heads(st).await;+ let copies: Vec<_> = all+ .iter()+ .filter(|(k, _)| k.contains("(conflict from laptop"))+ .collect();+ assert_eq!(copies.len(), 1, "{all:?}");+ assert_eq!(+ st.store.get(&copies[0].0, None).await.unwrap().0,+ b"laptop edit"+ );+ sim.shutdown().await;+}++/// The laptop owns the volume, is cut off, and writes; the studio takes+/// over and writes; the laptop wakes. Nothing is lost, there is one owner,+/// the laptop's stale records are refused, and both end up equal.+#[tokio::test]+async fn the_old_owner_waking_after_a_takeover_is_fenced_and_loses_nothing() {+ // The laptop is the more reliable one here, so it owns at the start and+ // gets the volume back at the end.+ let sim = Sim::new(&[("laptop", 9000), ("studio", 2000)]).await;+ group_ready(&sim, "laptop").await;+ let laptop = sim.get("laptop");+ let studio = sim.get("studio");+ laptop+ .backend+ .put("agents/ada/base.txt", b"base".to_vec(), Condition::None)+ .await+ .unwrap();+ laptop+ .backend+ .put("agents/ada/shared.txt", b"s0".to_vec(), Condition::None)+ .await+ .unwrap();+ eventually("studio holds the base", 10, || async {+ head_etag(studio, "agents/ada/shared.txt").await.is_some()+ })+ .await;+ // A partition: the laptop keeps writing as owner; nobody hears it.+ sim.cut("laptop", "studio");+ laptop+ .backend+ .put("agents/ada/lonely-1.txt", b"L1".to_vec(), Condition::None)+ .await+ .unwrap();+ laptop+ .backend+ .put(+ "agents/ada/shared.txt",+ b"laptop shared".to_vec(),+ Condition::None,+ )+ .await+ .unwrap();+ // The studio needs to order a write (an agent's fence): it takes over.+ needs_authority(studio, "spaces/dev/sessions/s1/fence-x").await;+ assert!(studio.node.status().await.i_own);+ studio+ .backend+ .put("agents/ada/studio-1.txt", b"S1".to_vec(), Condition::None)+ .await+ .unwrap();+ studio+ .backend+ .put(+ "agents/ada/shared.txt",+ b"studio shared".to_vec(),+ Condition::None,+ )+ .await+ .unwrap();+ assert!(!laptop.node.status().await.peers.is_empty());+ // The laptop still thinks it owns the volume and keeps writing.+ assert!(laptop.node.status().await.i_own);+ laptop+ .backend+ .put("agents/ada/lonely-2.txt", b"L2".to_vec(), Condition::None)+ .await+ .unwrap();+ sim.heal("laptop", "studio");+ eventually("everything converges", 30, || async {+ converged(&sim, &["laptop", "studio"]).await+ && laptop.node.status().await.pending.is_empty()+ && head_etag(studio, "agents/ada/lonely-2.txt").await.is_some()+ })+ .await;+ // The studio's era stood; nothing from the laptop's divergent tail was+ // applied as is.+ let st = studio.node.status().await;+ assert!(st.epoch >= 2, "{st:?}");+ // Nothing is lost: every written value is a head or a conflict copy.+ let mut values: Vec<Vec<u8>> = vec![];+ for (k, _) in heads(studio).await {+ values.push(studio.store.get(&k, None).await.unwrap().0);+ }+ for want in [+ b"base".to_vec(),+ b"L1".to_vec(),+ b"L2".to_vec(),+ b"S1".to_vec(),+ b"studio shared".to_vec(),+ b"laptop shared".to_vec(),+ ] {+ assert!(+ values.contains(&want),+ "lost {:?}: {:?}",+ String::from_utf8_lossy(&want),+ values+ .iter()+ .map(|v| String::from_utf8_lossy(v).to_string())+ .collect::<Vec<_>>()+ );+ }+ // The key both sides changed has the owner's value and a copy of the other.+ let owner_now = if studio.node.status().await.i_own {+ studio+ } else {+ laptop+ };+ let v = owner_now+ .store+ .get("agents/ada/shared.txt", None)+ .await+ .unwrap()+ .0;+ assert!(v == b"studio shared" || v == b"laptop shared");+ // The more reliable laptop gets the volume back, without anyone losing it.+ eventually("the volume goes back to the laptop", 20, || async {+ laptop.node.status().await.i_own && studio.node.status().await.owner == "laptop"+ })+ .await;+ let s = studio.node.status().await;+ assert!(s.handbacks >= 1 || laptop.node.status().await.handbacks >= 1);+ sim.shutdown().await;+}++#[tokio::test]+async fn a_goodbye_hands_the_volume_over_at_once() {+ let sim = Sim::new(&[("laptop", 9000), ("studio", 2000)]).await;+ group_ready(&sim, "laptop").await;+ sim.get("laptop")+ .backend+ .put("agents/ada/a.txt", b"a".to_vec(), Condition::None)+ .await+ .unwrap();+ eventually("replicated", 10, || async {+ head_etag(sim.get("studio"), "agents/ada/a.txt")+ .await+ .is_some()+ })+ .await;+ let t0 = Instant::now();+ sim.get("laptop")+ .node+ .notify_power(PowerEvent::WillSleep)+ .await;+ eventually("the studio owns the volume", 10, || async {+ sim.get("studio").node.status().await.i_own+ })+ .await;+ let took = t0.elapsed();+ // Silence alone needs the floor (250 ms) plus confirmation (400 ms).+ assert!(took < Duration::from_millis(600), "took {took:?}");+ sim.shutdown().await;+}++#[tokio::test]+async fn jitter_and_a_skewed_clock_cause_no_takeover() {+ let mut cfg = Config::fast();+ cfg.wall_skew_ms = 3_600_000; // the studio's clock is an hour ahead+ let sim = Sim::with(&[("laptop", 9000), ("studio", 2000)], cfg).await;+ group_ready(&sim, "laptop").await;+ sim.net.set_both(+ "laptop",+ "studio",+ Impair {+ latency: Duration::from_millis(30),+ jitter: Duration::from_millis(140),+ ..Default::default()+ },+ );+ let laptop = sim.get("laptop");+ for i in 0..40 {+ laptop+ .backend+ .put(+ &format!("agents/ada/j-{i}.txt"),+ vec![b'x'; 2000],+ Condition::None,+ )+ .await+ .unwrap();+ tokio::time::sleep(Duration::from_millis(50)).await;+ }+ eventually("replicated under jitter", 20, || async {+ converged(&sim, &["laptop", "studio"]).await+ })+ .await;+ for id in ["laptop", "studio"] {+ let s = sim.get(id).node.status().await;+ assert_eq!(s.takeovers, 0, "{id} took over on jitter");+ assert_eq!(s.epoch, 1, "{id}");+ }+ sim.shutdown().await;+}++fn sample(n: usize, seed: u8) -> Vec<u8> {+ (0..n)+ .map(|i| ((i as u64 * 2654435761 + seed as u64) >> 7) as u8)+ .collect()+}++#[tokio::test]+async fn a_big_file_survives_a_flapping_link_without_being_sent_again() {+ let mut cfg = Config::fast();+ cfg.block = 256 * 1024;+ cfg.data_timeout = Duration::from_secs(3);+ let sim = Sim::with(&[("studio", 9000), ("laptop", 2000)], cfg).await;+ group_ready(&sim, "studio").await;+ let size = 6 * 1024 * 1024;+ let body = sample(size, 3);+ // 100 kB per second each way: the file takes a minute without help, so+ // flap it.+ sim.net.set_both(+ "studio",+ "laptop",+ Impair {+ bandwidth: 2 * 1024 * 1024,+ latency: Duration::from_millis(5),+ ..Default::default()+ },+ );+ let before = sim.net.bytes("studio", "laptop");+ let studio = sim.get("studio");+ let w = {+ let b = studio.backend.clone();+ let body = body.clone();+ tokio::spawn(async move { b.put("agents/ada/big.bin", body, Condition::None).await })+ };+ // Cut the link a few times while it moves.+ for _ in 0..4 {+ tokio::time::sleep(Duration::from_millis(400)).await;+ sim.cut("studio", "laptop");+ tokio::time::sleep(Duration::from_millis(300)).await;+ sim.heal("studio", "laptop");+ }+ w.await.unwrap().unwrap();+ let t = Instant::now();+ eventually("the big file arrives intact", 25, || async {+ if t.elapsed().as_secs() % 5 == 4 {+ eprintln!(+ "laptop {:?}\nstudio {:?}",+ sim.get("laptop").node.status().await,+ sim.get("studio").node.status().await+ );+ }+ sim.get("laptop")+ .store+ .head("agents/ada/big.bin")+ .await+ .unwrap()+ .is_some()+ })+ .await;+ let got = sim+ .get("laptop")+ .store+ .get("agents/ada/big.bin", None)+ .await+ .unwrap()+ .0;+ assert_eq!(got.len(), size);+ assert_eq!(+ cua_volume::sha256_hex(&got),+ cua_volume::sha256_hex(&body),+ "bytes differ"+ );+ let moved = sim.net.bytes("studio", "laptop") - before;+ assert!(+ moved < (size as u64) * 2,+ "{moved} bytes crossed for a {size} byte file: it was sent again"+ );+ // A new version that only grows at the end costs its end, not the file.+ let mut longer = body.clone();+ longer.extend(sample(100 * 1024, 9));+ let b2 = sim.net.bytes("studio", "laptop");+ studio+ .backend+ .put("agents/ada/big.bin", longer.clone(), Condition::None)+ .await+ .unwrap();+ eventually("the longer version arrives", 30, || async {+ sim.get("laptop")+ .store+ .head("agents/ada/big.bin")+ .await+ .unwrap()+ .is_some_and(|h| h.size == longer.len() as u64)+ })+ .await;+ let moved2 = sim.net.bytes("studio", "laptop") - b2;+ assert!(+ moved2 < 1024 * 1024,+ "{moved2} bytes crossed for an append of 100 KiB"+ );+ sim.shutdown().await;+}++#[tokio::test]+async fn with_three_machines_a_lone_minority_does_not_take_over() {+ let sim = Sim::new(&[("studio", 9000), ("mini", 7000), ("laptop", 2000)]).await;+ group_ready(&sim, "studio").await;+ let mini = sim.get("mini");+ // The laptop keeps no copy (two copies: studio and mini).+ let s = sim.get("laptop").node.status().await;+ assert_eq!(s.role, cua_volume::replica::Role::Client);+ // The mini is cut off from both: it is a minority of one.+ sim.cut("mini", "studio");+ sim.cut("mini", "laptop");+ tokio::time::sleep(Duration::from_millis(2500)).await;+ let s = mini.node.status().await;+ assert_eq!(s.takeovers, 0, "a minority must not take over");+ assert_ne!(s.owner, "mini");+ // The studio goes away; the mini, back with the laptop, now has a+ // majority (itself and the laptop) and takes over.+ sim.heal("mini", "laptop");+ sim.heal("mini", "studio");+ sim.cut("studio", "mini");+ sim.cut("studio", "laptop");+ needs_authority(mini, "agents/ada/.cua-lease").await;+ assert!(mini.node.status().await.i_own);+ sim.shutdown().await;+}++#[tokio::test]+async fn a_machine_that_keeps_no_copy_reads_through_its_peers() {+ let sim = Sim::new(&[("studio", 9000), ("mini", 7000), ("laptop", 2000)]).await;+ group_ready(&sim, "studio").await;+ let laptop = sim.get("laptop");+ assert!(+ laptop.backend.remote(),+ "a client sits behind the block cache"+ );+ sim.get("studio")+ .backend+ .put("agents/ada/r.txt", b"read me".to_vec(), Condition::None)+ .await+ .unwrap();+ let (bytes, meta) = laptop.backend.get("agents/ada/r.txt", None).await.unwrap();+ assert_eq!(bytes, b"read me");+ assert_eq!(+ laptop+ .backend+ .get_range("agents/ada/r.txt", &meta.version, 5, 2)+ .await+ .unwrap(),+ b"me"+ );+ assert_eq!(laptop.backend.list("agents/").await.unwrap().len(), 1);+ // The owner goes away: the other copy serves.+ sim.cut("studio", "laptop");+ sim.cut("studio", "mini");+ eventually("the laptop reads from the mini", 15, || async {+ laptop.backend.get("agents/ada/r.txt", None).await.is_ok()+ })+ .await;+ // Someone needs an ordered write: the mini takes over (with the laptop's+ // vote), and a write from the client goes to it.+ needs_authority(sim.get("mini"), "agents/ada/.cua-lease").await;+ let (_, d) = laptop+ .backend+ .put_durable("agents/ada/w.txt", b"w".to_vec(), Condition::None)+ .await+ .unwrap();+ assert!(matches!(+ d,+ Durability::Local(_) | Durability::Replicated(_)+ ));+ sim.shutdown().await;+}++#[tokio::test]+async fn a_restart_keeps_the_log_and_the_outbox() {+ let mut sim = Sim::new(&[("studio", 9000), ("laptop", 2000)]).await;+ group_ready(&sim, "studio").await;+ sim.cut("studio", "laptop");+ let laptop_id = "laptop";+ eventually("laptop sees the studio go", 10, || async {+ sim.get(laptop_id).node.status().await.peers[0].health != "up"+ })+ .await;+ sim.get("laptop")+ .backend+ .put(+ "agents/ada/kept.txt",+ b"survives a restart".to_vec(),+ Condition::None,+ )+ .await+ .unwrap();+ sim.restart("laptop").await;+ assert_eq!(sim.get("laptop").node.status().await.pending.len(), 1);+ sim.heal("studio", "laptop");+ eventually("the studio takes it after the restart", 20, || async {+ head_etag(sim.get("studio"), "agents/ada/kept.txt")+ .await+ .is_some()+ && sim.get("laptop").node.status().await.pending.is_empty()+ })+ .await;+ sim.shutdown().await;+}++/// A laptop on bad wifi comes and goes. It must never take the volume away+/// from the studio that runs the agent (its changes wait in the outbox), and+/// the studio's ordered writes (leases, fences) never wait for it.+#[tokio::test]+async fn a_flaky_laptop_never_takes_the_volume_from_the_studio() {+ let sim = Sim::new(&[("studio", 9000), ("laptop", 2000)]).await;+ group_ready(&sim, "studio").await;+ let studio = sim.get("studio");+ let laptop = sim.get("laptop");+ for round in 0..6 {+ sim.cut("studio", "laptop");+ tokio::time::sleep(Duration::from_millis(900)).await; // longer than detection+ laptop+ .backend+ .put(+ &format!("agents/ada/plane-{round}.txt"),+ vec![b'p'; 100],+ Condition::None,+ )+ .await+ .unwrap();+ // The studio's ordered writes are immediate.+ let t = Instant::now();+ studio+ .backend+ .put(+ &format!("agents/ada/.fence-{round}"),+ b"1".to_vec(),+ Condition::IfNoneMatch,+ )+ .await+ .unwrap();+ assert!(+ t.elapsed() < Duration::from_millis(1200),+ "the studio waited for the laptop"+ );+ sim.heal("studio", "laptop");+ tokio::time::sleep(Duration::from_millis(300)).await;+ }+ eventually("everything lands", 30, || async {+ converged(&sim, &["studio", "laptop"]).await+ && laptop.node.status().await.pending.is_empty()+ })+ .await;+ let s = laptop.node.status().await;+ assert_eq!(s.takeovers, 0, "the laptop took the volume");+ assert_eq!(studio.node.status().await.epoch, 1);+ assert!(studio.node.status().await.i_own);+ assert_eq!(heads(studio).await.len(), 12);+ sim.shutdown().await;+}++/// The usual way a second machine arrives: the laptop already has a volume+/// with files in it, and the studio is new. The studio, the more reliable+/// one, ends up owning it; the laptop's files are not lost and are not+/// duplicated; writes from either side stay quick afterwards.+#[tokio::test]+async fn a_new_studio_joins_a_laptop_that_already_has_files() {+ let mut sim = Sim::new(&[("laptop", 2000)]).await;+ let laptop_files = [+ ("agents/ada/old-1.txt", "one"),+ ("agents/ada/old-2.txt", "two"),+ ("public/readme.md", "hello"),+ ];+ for (k, v) in laptop_files {+ sim.get("laptop")+ .backend+ .put(k, v.as_bytes().to_vec(), Condition::None)+ .await+ .unwrap();+ }+ // The studio signs in: the laptop's machines list gains it.+ sim.members.push(member("studio", 9000));+ let studio_member = member("studio", 9000);+ sim.get("laptop")+ .node+ .set_members(vec![studio_member])+ .await;+ let dir = tempfile::tempdir().unwrap();+ sim.boot("studio", dir).await;+ group_ready(&sim, "studio").await;+ eventually("the laptop's files are on the studio, once", 20, || async {+ heads(sim.get("studio")).await.len() == 3+ && converged(&sim, &["studio", "laptop"]).await+ && sim.get("laptop").node.status().await.pending.is_empty()+ })+ .await;+ for (k, v) in laptop_files {+ assert_eq!(+ sim.get("studio").store.get(k, None).await.unwrap().0,+ v.as_bytes()+ );+ }+ // A write from the laptop is not held up by the arrival.+ let t = Instant::now();+ let (_, d) = sim+ .get("laptop")+ .backend+ .put_durable("agents/ada/new.txt", b"n".to_vec(), Condition::None)+ .await+ .unwrap();+ assert_eq!(d, Durability::Replicated(2));+ assert!(+ t.elapsed() < Duration::from_millis(600),+ "slow after joining: {:?}",+ t.elapsed()+ );+ // And in the other direction.+ let t = Instant::now();+ sim.get("studio")+ .backend+ .put("agents/ada/new2.txt", b"n".to_vec(), Condition::None)+ .await+ .unwrap();+ assert!(t.elapsed() < Duration::from_millis(600));+ sim.shutdown().await;+}++/// A laptop that vanishes without a goodbye (a dead battery, a dropped+/// link) is waited for at most twice: after two writes whose second copy+/// never came, the studio stops waiting until the laptop answers again.+#[tokio::test]+async fn a_laptop_that_vanishes_without_a_goodbye_costs_the_studio_two_waits() {+ let sim = Sim::new(&[("studio", 9000), ("laptop", 2000)]).await;+ group_ready(&sim, "studio").await;+ let studio = sim.get("studio");+ studio+ .backend+ .put("agents/ada/warm.txt", b"w".to_vec(), Condition::None)+ .await+ .unwrap();+ sim.get("laptop").node.freeze(true);+ // A stopped process behind an open connection: nothing arrives, nothing+ // is reset (the link stays open until the detector gives up on it).+ sim.net.set_both(+ "studio",+ "laptop",+ Impair {+ hold: true,+ ..Default::default()+ },+ );+ let t = Instant::now();+ let mut durs = vec![];+ for i in 0..12 {+ let t1 = Instant::now();+ studio+ .backend+ .put(+ &format!("agents/ada/v-{i}.txt"),+ vec![b'v'; 100],+ Condition::None,+ )+ .await+ .unwrap();+ durs.push(t1.elapsed());+ }+ let ack_wait = Config::fast().ack_wait;+ assert!(+ t.elapsed() < ack_wait * 2 + Duration::from_millis(1500),+ "12 writes took {:?} ({durs:?})",+ t.elapsed()+ );+ assert!(+ durs[4..].iter().all(|d| *d < Duration::from_millis(200)),+ "{durs:?}"+ );+ sim.shutdown().await;+}++/// A machine whose process is gone answers dials with a refusal. That is+/// far better evidence than silence, so the volume does not wait out the+/// silence window before a machine that needs authority takes over.+#[tokio::test]+async fn a_machine_that_refuses_connections_is_replaced_faster_than_a_silent_one() {+ // Silence would need 1.5 s of quiet and 3 s of confirmation.+ let mut cfg = Config::fast();+ cfg.timing.floor = Duration::from_millis(1500);+ cfg.timing.confirm = Duration::from_millis(3000);+ let sim = Sim::with(&[("laptop", 9000), ("studio", 2000)], cfg.clone()).await;+ group_ready(&sim, "laptop").await;+ // The owner's process dies: its connections break, nothing answers.+ sim.get("laptop").node.freeze(true);+ sim.net.leave("laptop");+ let t = Instant::now();+ needs_authority(sim.get("studio"), "agents/ada/.cua-lease").await;+ let took = t.elapsed();+ assert!(+ took < cfg.timing.floor + cfg.timing.confirm - Duration::from_millis(1500),+ "took {took:?}, as long as silence would"+ );+ assert!(sim.get("studio").node.status().await.i_own);+ sim.shutdown().await;+}++/// An older Cua Spaces build on the same home writes to the store directly+/// (it does not know about replication). When the new build starts again+/// those changes are found and reach the other machine; nothing is lost and+/// nothing diverges silently.+#[tokio::test]+async fn changes_an_older_build_made_behind_replications_back_still_arrive() {+ let mut sim = Sim::new(&[("studio", 9000), ("laptop", 2000)]).await;+ group_ready(&sim, "studio").await;+ sim.get("studio")+ .backend+ .put("agents/ada/a.txt", b"a".to_vec(), Condition::None)+ .await+ .unwrap();+ sim.get("studio")+ .backend+ .put("agents/ada/b.txt", b"b".to_vec(), Condition::None)+ .await+ .unwrap();+ eventually("replicated", 10, || async {+ converged(&sim, &["studio", "laptop"]).await && heads(sim.get("laptop")).await.len() == 2+ })+ .await;+ // Both machines stop cleanly (the app is quit); an older build then+ // writes on each home.+ let (studio_store, laptop_store) = (+ sim.get("studio").store.clone(),+ sim.get("laptop").store.clone(),+ );+ sim.get("studio").node.shutdown().await;+ sim.get("laptop").node.shutdown().await;+ studio_store+ .put(+ "agents/ada/a.txt",+ b"a, edited by the old build".to_vec(),+ Condition::None,+ )+ .await+ .unwrap();+ studio_store+ .put(+ "agents/ada/from-old-build.txt",+ b"new".to_vec(),+ Condition::None,+ )+ .await+ .unwrap();+ laptop_store+ .delete("agents/ada/b.txt", Condition::None)+ .await+ .unwrap();+ // The new build starts again on both.+ sim.restart("studio").await;+ sim.restart("laptop").await;+ group_ready(&sim, "studio").await;+ eventually("the changes reach both machines", 30, || async {+ converged(&sim, &["studio", "laptop"]).await+ && sim.get("laptop").node.status().await.pending.is_empty()+ && head_etag(sim.get("laptop"), "agents/ada/from-old-build.txt")+ .await+ .is_some()+ })+ .await;+ assert_eq!(+ sim.get("laptop")+ .store+ .get("agents/ada/a.txt", None)+ .await+ .unwrap()+ .0,+ b"a, edited by the old build"+ );+ assert!(+ head_etag(sim.get("studio"), "agents/ada/b.txt")+ .await+ .is_none(),+ "the delete arrived"+ );+ sim.shutdown().await;+}++/// The first time replication touches a Volume that has files, a copy of its+/// heads is taken; a directory prepared by a newer format is refused.+#[tokio::test]+async fn turning_replication_on_backs_up_the_heads_and_refuses_a_newer_format() {+ let dir = tempfile::tempdir().unwrap();+ let store = cua_volume::fs::FsBackend::new(dir.path().join("data"));+ for i in 0..5 {+ store+ .put(+ &format!("agents/ada/f{i}.txt"),+ vec![b'x'; 10 + i],+ Condition::None,+ )+ .await+ .unwrap();+ }+ let net = cua_volume::replica::simnet::SimNet::new();+ let (tr, _rx) = net.join("m");+ let node = cua_volume::replica::ReplicaNode::open(+ Config::fast(),+ "m",+ "m",+ dir.path(),+ store.clone(),+ std::sync::Arc::new(tr),+ vec![common::member("m", 5000)],+ )+ .await+ .unwrap();+ let backups: Vec<_> = std::fs::read_dir(dir.path().join("replica/backup"))+ .unwrap()+ .collect();+ assert_eq!(backups.len(), 1, "one backup of the heads");+ // The Volume's own files are untouched.+ assert_eq!(store.list("").await.unwrap().len(), 5);+ node.shutdown().await;+ drop(node);+ // A newer format is refused with a plain message.+ std::fs::write(+ dir.path().join("replica/format.json"),+ br#"{"format":99,"created_ms":1}"#,+ )+ .unwrap();+ let (tr2, _rx2) = net.join("m2");+ let e = cua_volume::replica::ReplicaNode::open(+ Config::fast(),+ "m",+ "m",+ dir.path(),+ store,+ std::sync::Arc::new(tr2),+ vec![common::member("m", 5000)],+ )+ .await+ .unwrap_err();+ assert!(e.to_string().contains("newer Cua Spaces"), "{e}");+}++/// A machine that wrote while the owner was away and then became the owner+/// itself must put those writes in the log so the other machine gets them+/// (they were visible on the writing machine as heads of their own).+#[tokio::test]+async fn writes_made_while_apart_by_the_machine_that_takes_over_reach_the_other() {+ let sim = Sim::new(&[("studio", 9000), ("laptop", 2000)]).await;+ group_ready(&sim, "studio").await;+ sim.cut("studio", "laptop");+ let laptop = sim.get("laptop");+ eventually("the laptop sees the studio go", 10, || async {+ laptop.node.status().await.peers[0].health != "up"+ })+ .await;+ // Written while the owner is away: saved, visible here, not in any log.+ laptop+ .backend+ .put(+ "agents/laptop-w/p1",+ b"typed on a plane".to_vec(),+ Condition::None,+ )+ .await+ .unwrap();+ // The laptop then needs an ordered write and becomes the owner.+ needs_authority(laptop, "agents/ada/.cua-lease").await;+ assert!(laptop.node.status().await.i_own);+ sim.heal("studio", "laptop");+ eventually("the studio has it too", 30, || async {+ converged(&sim, &["studio", "laptop"]).await+ && head_etag(sim.get("studio"), "agents/laptop-w/p1")+ .await+ .is_some()+ })+ .await;+ assert_eq!(+ sim.get("studio")+ .store+ .get("agents/laptop-w/p1", None)+ .await+ .unwrap()+ .0,+ b"typed on a plane"+ );+ sim.shutdown().await;+}++/// A folder copy-in is one batch at the owner (one disk barrier, one run of+/// log records) and reaches the other machine whole.+#[tokio::test]+async fn a_batch_of_files_is_one_write_at_the_owner_and_reaches_the_other_machine() {+ let sim = Sim::new(&[("studio", 9000), ("laptop", 2000)]).await;+ group_ready(&sim, "studio").await;+ let items: Vec<(String, Vec<u8>)> = (0..300)+ .map(|i| {+ (+ format!("agents/ada/batch/f{i}"),+ format!("file {i}").into_bytes(),+ )+ })+ .collect();+ let metas = sim+ .get("studio")+ .backend+ .put_many(items.clone())+ .await+ .unwrap();+ assert_eq!(metas.len(), 300);+ eventually("all of it is on the laptop", 20, || async {+ converged(&sim, &["studio", "laptop"]).await && heads(sim.get("laptop")).await.len() == 300+ })+ .await;+ for (k, v) in items.iter().step_by(37) {+ assert_eq!(&sim.get("laptop").store.get(k, None).await.unwrap().0, v);+ }+ sim.shutdown().await;+}diff --git a/libs/cua/crates/cua-volume/tests/replica_supervision.rs b/libs/cua/crates/cua-volume/tests/replica_supervision.rsnew file mode 100644index 0000000000..1631d80ca2--- /dev/null+++ b/libs/cua/crates/cua-volume/tests/replica_supervision.rs@@ -0,0 +1,525 @@+// SPDX-License-Identifier: FSL-1.1-MIT+// Copyright (c) 2026 Cua AI, Inc.++//! Durable agents across machines on the simulated network: the run goes to+//! the most reliable machine, survives a closed laptop, moves when its+//! machine dies, and a holder that wakes after being replaced is fenced.++mod common;++use std::collections::HashMap;+use std::sync::{Arc, Mutex};+use std::time::{Duration, Instant};++use common::*;+use cua_volume::backend::{Backend, Condition};+use cua_volume::replica::PowerEvent;+use cua_volume::replica::supervision::{DriveEnd, Driver, RunCtx, Supervision, SupervisionConfig};++type StepLog = Arc<Mutex<Vec<(String, u64, u64, Instant)>>>;++/// An agent made of N steps; step i writes `out/step-i` (the same bytes in+/// any epoch) and a journal marker, and the end writes the checksum of every+/// step. It resumes from the steps its machine's journal holds.+struct Steps {+ n: u64,+ step_ms: u64,+ log: StepLog,+}++fn content(i: u64) -> Vec<u8> {+ format!("step {i}: {}", cua_volume::sha256_hex(&i.to_le_bytes())).into_bytes()+}++fn expected_sum(n: u64) -> String {+ let mut all = vec![];+ for i in 0..n {+ all.extend(content(i));+ }+ cua_volume::sha256_hex(&all)+}++#[async_trait::async_trait]+impl Driver for Steps {+ async fn drive(&self, mut ctx: RunCtx) -> DriveEnd {+ let prefix = format!("agents/{}/journal/", ctx.agent);+ // How far this machine's copy of the journal reaches (contiguous).+ let mut held = 0u64;+ while ctx+ .backend+ .head(&format!("{prefix}{held:06}"))+ .await+ .ok()+ .flatten()+ .is_some()+ {+ held += 1;+ }+ ctx.held_reaches(held);+ let mut i = held;+ while i < self.n {+ if ctx.fenced() {+ return DriveEnd::Fenced;+ }+ self.log+ .lock()+ .unwrap()+ .push((ctx.machine.clone(), ctx.epoch, i, Instant::now()));+ let out = format!("agents/{}/out/step-{i}", ctx.agent);+ if ctx+ .backend+ .put(&out, content(i), Condition::None)+ .await+ .is_err()+ {+ return DriveEnd::Failed("write".into());+ }+ let _ = ctx+ .backend+ .put(&format!("{prefix}{i:06}"), vec![b'.'], Condition::None)+ .await;+ ctx.report(i + 1);+ i += 1;+ tokio::time::sleep(Duration::from_millis(self.step_ms)).await;+ }+ if ctx.fenced() {+ return DriveEnd::Fenced;+ }+ // The result: the checksum of every step as the Volume holds it.+ let mut all = vec![];+ for s in 0..self.n {+ match ctx+ .backend+ .get(&format!("agents/{}/out/step-{s}", ctx.agent), None)+ .await+ {+ Ok((b, _)) => all.extend(b),+ Err(_) => return DriveEnd::Failed("a step is missing".into()),+ }+ }+ let sum = cua_volume::sha256_hex(&all);+ let _ = ctx+ .backend+ .put(+ &format!("agents/{}/result.txt", ctx.agent),+ sum.clone().into_bytes(),+ Condition::None,+ )+ .await;+ DriveEnd::Done(sum)+ }+}++struct World {+ sim: Sim,+ sups: HashMap<String, Supervision>,+ log: StepLog,+ awake: Arc<Mutex<Vec<String>>>,+}++async fn world(specs: &[(&str, u64)], n: u64, step_ms: u64) -> World {+ let sim = Sim::new(specs).await;+ let mut w = World {+ sim,+ sups: HashMap::new(),+ log: Arc::new(Mutex::new(vec![])),+ awake: Arc::new(Mutex::new(vec![])),+ };+ for (id, _) in specs {+ w.supervise(id, n, step_ms);+ }+ w+}++impl World {+ fn supervise(&mut self, id: &str, n: u64, step_ms: u64) {+ let m = self.sim.get(id);+ let awake = self.awake.clone();+ let s = Supervision::new(+ m.node.clone(),+ m.backend.clone(),+ SupervisionConfig::fast(),+ Arc::new(Steps {+ n,+ step_ms,+ log: self.log.clone(),+ }),+ Some(Arc::new(move |why: &str| {+ awake.lock().unwrap().push(why.to_string());+ Box::new(()) as Box<dyn std::any::Any + Send>+ })),+ );+ s.spawn();+ self.sups.insert(id.to_string(), s);+ }++ fn who_ran(&self) -> Vec<String> {+ let mut v: Vec<String> = self+ .log+ .lock()+ .unwrap()+ .iter()+ .map(|e| e.0.clone())+ .collect();+ v.sort();+ v.dedup();+ v+ }++ async fn result_on(&self, id: &str, agent: &str) -> Option<String> {+ self.sim+ .get(id)+ .store+ .get(&format!("agents/{agent}/result.txt"), None)+ .await+ .ok()+ .map(|(b, _)| String::from_utf8(b).unwrap())+ }++ async fn stop_all(&self) {+ for s in self.sups.values() {+ s.stop().await;+ }+ self.sim.shutdown().await;+ }+}++#[tokio::test]+async fn a_run_started_on_the_laptop_runs_on_the_studio_and_says_so() {+ let w = world(&[("studio", 9000), ("laptop", 2000)], 20, 30).await;+ group_ready(&w.sim, "studio").await;+ let info = w.sups["laptop"]+ .start_run("ada", serde_json::json!({"task": "demo"}))+ .await+ .unwrap();+ assert_eq!(info.run_on, "studio");+ assert_eq!(+ info.words,+ "Your agent keeps running on studio. Files are on 2 machines. You can close this machine."+ );+ eventually("the run finishes", 30, || async {+ w.result_on("laptop", "ada").await.is_some()+ })+ .await;+ assert_eq!(+ w.who_ran(),+ vec!["studio".to_string()],+ "the laptop never drove it"+ );+ assert_eq!(+ w.result_on("laptop", "ada").await.unwrap(),+ expected_sum(20)+ );+ assert_eq!(+ w.result_on("studio", "ada").await.unwrap(),+ expected_sum(20)+ );+ // The studio held a sleep assertion for it with a readable reason.+ assert_eq!(+ w.awake.lock().unwrap().as_slice(),+ ["Cua: the durable agent ada is running".to_string()]+ );+ w.stop_all().await;+}++#[tokio::test]+async fn closing_the_laptop_does_not_stop_the_run_or_lose_files() {+ let w = world(&[("studio", 9000), ("laptop", 2000)], 60, 40).await;+ group_ready(&w.sim, "studio").await;+ w.sups["laptop"]+ .start_run("ada", serde_json::json!({}))+ .await+ .unwrap();+ eventually("some steps ran", 20, || async {+ w.log.lock().unwrap().len() >= 8+ })+ .await;+ // The lid closes.+ w.sim+ .get("laptop")+ .node+ .notify_power(PowerEvent::WillSleep)+ .await;+ tokio::time::sleep(Duration::from_millis(50)).await;+ w.sim.get("laptop").node.freeze(true);+ w.sim.cut("studio", "laptop");+ let at_close = w.log.lock().unwrap().len();+ eventually(+ "the run finishes while the laptop is closed",+ 30,+ || async { w.result_on("studio", "ada").await.is_some() },+ )+ .await;+ assert!(w.log.lock().unwrap().len() > at_close);+ assert_eq!(w.who_ran(), vec!["studio".to_string()]);+ // The lid opens; the laptop has every file.+ w.sim.heal("studio", "laptop");+ w.sim.get("laptop").node.freeze(false);+ w.sim+ .get("laptop")+ .node+ .notify_power(PowerEvent::DidWake)+ .await;+ eventually("the laptop has the result and every step", 20, || async {+ converged(&w.sim, &["studio", "laptop"]).await+ && w.result_on("laptop", "ada").await.is_some()+ })+ .await;+ assert_eq!(+ w.result_on("laptop", "ada").await.unwrap(),+ expected_sum(60)+ );+ assert_eq!(w.sim.get("studio").node.status().await.takeovers, 0);+ w.stop_all().await;+}++#[tokio::test]+async fn when_the_studio_dies_the_laptop_takes_the_run_and_resumes_it() {+ let w = world(&[("studio", 9000), ("laptop", 2000)], 80, 40).await;+ group_ready(&w.sim, "studio").await;+ w.sups["laptop"]+ .start_run("ada", serde_json::json!({}))+ .await+ .unwrap();+ eventually("steps are running on the studio", 20, || async {+ w.log.lock().unwrap().len() >= 15+ })+ .await;+ // The studio's process dies.+ let dead_at = Instant::now();+ w.sups["studio"].stop().await;+ w.sim.get("studio").node.shutdown().await;+ w.sim.cut("studio", "laptop");+ let ran_on_studio = w.log.lock().unwrap().len();+ eventually("the laptop runs it", 30, || async {+ w.log.lock().unwrap().iter().any(|e| e.0 == "laptop")+ })+ .await;+ let took = w+ .log+ .lock()+ .unwrap()+ .iter()+ .find(|e| e.0 == "laptop")+ .map(|e| e.3.duration_since(dead_at))+ .unwrap();+ assert!(took < Duration::from_secs(8), "took over in {took:?}");+ eventually("it finishes on the laptop", 30, || async {+ w.result_on("laptop", "ada").await.is_some()+ })+ .await;+ assert_eq!(+ w.result_on("laptop", "ada").await.unwrap(),+ expected_sum(80)+ );+ // It did not start again from the beginning: the laptop's first step is+ // after what the studio had done (its journal copy was there).+ let first_laptop_step = w+ .log+ .lock()+ .unwrap()+ .iter()+ .find(|e| e.0 == "laptop")+ .map(|e| e.2)+ .unwrap();+ assert!(+ first_laptop_step >= 10,+ "restarted at step {first_laptop_step}"+ );+ assert!(ran_on_studio >= 15);+ let runs = w.sups["laptop"].runs().await.unwrap();+ let note = &runs[0].note;+ assert!(note.starts_with("Resumed on laptop"), "{note}");+ w.stop_all().await;+}++#[tokio::test]+async fn a_frozen_holder_that_wakes_after_a_takeover_is_fenced() {+ let w = world(&[("studio", 9000), ("laptop", 2000)], 100, 40).await;+ group_ready(&w.sim, "studio").await;+ w.sups["laptop"]+ .start_run("ada", serde_json::json!({}))+ .await+ .unwrap();+ eventually("running on the studio", 20, || async {+ w.log.lock().unwrap().len() >= 10+ })+ .await;+ // The studio freezes (a stopped process) and loses the network.+ w.sim.get("studio").node.freeze(true);+ w.sim.cut("studio", "laptop");+ eventually("the laptop takes the run", 30, || async {+ w.log.lock().unwrap().iter().any(|e| e.0 == "laptop")+ })+ .await;+ // The studio thaws and the network returns.+ tokio::time::sleep(Duration::from_millis(300)).await;+ w.sim.heal("studio", "laptop");+ w.sim.get("studio").node.freeze(false);+ w.sim+ .get("studio")+ .node+ .notify_power(PowerEvent::DidWake)+ .await;+ eventually("the run finishes", 60, || async {+ w.result_on("laptop", "ada").await.is_some()+ && converged(&w.sim, &["studio", "laptop"]).await+ })+ .await;+ // The output is right everywhere, whatever ran twice in between.+ assert_eq!(+ w.result_on("laptop", "ada").await.unwrap(),+ expected_sum(100)+ );+ assert_eq!(+ w.result_on("studio", "ada").await.unwrap(),+ expected_sum(100)+ );+ for i in 0..100 {+ let b = w+ .sim+ .get("studio")+ .store+ .get(&format!("agents/ada/out/step-{i}"), None)+ .await+ .unwrap()+ .0;+ assert_eq!(b, content(i), "step {i} is wrong or missing");+ }+ // Exactly one epoch holds the lease at the end, the later one; the+ // stopped holder's driver stopped.+ let runs = w.sups["studio"].runs().await.unwrap();+ assert!(runs[0].epoch >= 2, "{runs:?}");+ eventually("the stopped holder's driver is stopped", 30, || async {+ let r = w.sups["studio"].runs().await.unwrap();+ !r[0].running_here+ })+ .await;+ w.stop_all().await;+}++#[tokio::test]+async fn a_run_on_a_laptop_goes_back_to_the_studio_when_it_returns() {+ // The studio starts later than the laptop: the run begins on the laptop.+ let sim = Sim::new(&[("studio", 9000), ("laptop", 2000)]).await;+ group_ready(&sim, "studio").await;+ let mut w = World {+ sim,+ sups: HashMap::new(),+ log: Arc::new(Mutex::new(vec![])),+ awake: Arc::new(Mutex::new(vec![])),+ };+ // Only the laptop supervises at first, and the studio is cut off.+ w.supervise("laptop", 120, 40);+ w.sim.cut("studio", "laptop");+ eventually("the laptop sees the studio go", 10, || async {+ w.sim.get("laptop").node.status().await.peers[0].health != "up"+ })+ .await;+ w.sups["laptop"]+ .start_run("ada", serde_json::json!({}))+ .await+ .unwrap();+ // (The write cannot reach the owner, so the laptop needs authority.)+ needs_authority(w.sim.get("laptop"), "agents/ada/.cua-lease").await;+ let started = w.sups["laptop"]+ .start_run("ada", serde_json::json!({}))+ .await;+ assert!(started.is_ok());+ eventually("the laptop runs it", 20, || async {+ w.log.lock().unwrap().iter().any(|e| e.0 == "laptop")+ })+ .await;+ // The studio comes back and supervises.+ w.sim.heal("studio", "laptop");+ w.supervise("studio", 120, 40);+ eventually("the studio gets the run", 40, || async {+ w.log.lock().unwrap().iter().any(|e| e.0 == "studio")+ })+ .await;+ eventually("it finishes", 60, || async {+ w.result_on("laptop", "ada").await.is_some()+ })+ .await;+ assert_eq!(+ w.result_on("laptop", "ada").await.unwrap(),+ expected_sum(120)+ );+ w.stop_all().await;+}++#[tokio::test]+async fn one_machine_says_it_will_pause_and_keeps_itself_awake() {+ let w = world(&[("macbook", 2000)], 5, 20).await;+ let info = w.sups["macbook"]+ .start_run("ada", serde_json::json!({}))+ .await+ .unwrap();+ assert!(+ info.words.contains("will pause when it sleeps"),+ "{}",+ info.words+ );+ assert!(info.words.contains("Add another machine"), "{}", info.words);+ eventually("done", 20, || async {+ w.result_on("macbook", "ada").await.is_some()+ })+ .await;+ assert_eq!(w.who_ran(), vec!["macbook".to_string()]);+ assert_eq!(w.awake.lock().unwrap().len(), 1);+ w.stop_all().await;+}++/// The run moves between architectures: replication and the journal are the+/// same, built programs are not, and the words say so.+#[tokio::test]+async fn a_takeover_across_architectures_says_built_programs_are_not_reused() {+ let w = world(&[("studio", 9000), ("laptop", 2000)], 60, 40).await;+ for (id, rank, arch) in [("studio", 9000, "x86_64"), ("laptop", 2000, "aarch64")] {+ w.sim.get(id).node.set_facts(cua_volume::replica::Facts {+ rank,+ free_disk: 500 << 30,+ arch: arch.into(),+ ..Default::default()+ });+ }+ group_ready(&w.sim, "studio").await;+ w.sim+ .get("studio")+ .backend+ .put(+ "agents/ada/target/debug/ada",+ b"ELF".to_vec(),+ Condition::None,+ )+ .await+ .unwrap();+ w.sups["laptop"]+ .start_run("ada", serde_json::json!({}))+ .await+ .unwrap();+ eventually("steps run on the studio", 20, || async {+ w.log.lock().unwrap().len() >= 10+ })+ .await;+ w.sups["studio"].stop().await;+ w.sim.get("studio").node.shutdown().await;+ w.sim.cut("studio", "laptop");+ eventually("it finishes on the laptop", 40, || async {+ w.result_on("laptop", "ada").await.is_some()+ })+ .await;+ assert_eq!(+ w.result_on("laptop", "ada").await.unwrap(),+ expected_sum(60)+ );+ let note = w.sups["laptop"].runs().await.unwrap()[0].note.clone();+ assert!(note.starts_with("Resumed on laptop"), "{note}");+ assert!(+ note.contains("studio is amd64")+ && note.contains("target/")+ && note.contains("built again"),+ "{note}"+ );+ w.stop_all().await;+}