Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-vmm): reserve overlay-sized space for QEMU creates and check it before pulling (#5042)

injaneity40f82b4b61d3
diff --git a/libs/cua/crates/cua-daemon/src/local.rs b/libs/cua/crates/cua-daemon/src/local.rs
index b5540caba4..e9f6083f6b 100644
--- a/libs/cua/crates/cua-daemon/src/local.rs
+++ b/libs/cua/crates/cua-daemon/src/local.rs
@@ -1352,7 +1352,7 @@ pub async fn pull_image(local: &VmmLocal, reference: &str) -> Result<LocalImageI
(BackendKind::Qemu, ImageSource::Oci { reference: r }) => {
let client = cua_image::RegistryClient::default();
let cache = cua_image::ImageCache::default();
- let disk = cua_image::containerdisk::pull(&client, &cache, &r, arch.oci(), false)
+ let disk = cua_image::containerdisk::pull(&client, &cache, &r, arch.oci())
.await
.map_err(image_error)?;
let size = std::fs::metadata(&disk).map(|m| m.len()).unwrap_or(0);
@@ -1475,7 +1475,6 @@ async fn build_with(
&cua_image::ImageCache::default(),
&reference,
opts.arch.oci(),
- false,
)
.await
.map_err(image_error)?,
diff --git a/libs/cua/crates/cua-image/src/containerdisk.rs b/libs/cua/crates/cua-image/src/containerdisk.rs
index f498338718..0961f32bf0 100644
--- a/libs/cua/crates/cua-image/src/containerdisk.rs
+++ b/libs/cua/crates/cua-image/src/containerdisk.rs
@@ -154,14 +154,12 @@ pub fn pack(disk: &Path, arch: &str, out_dir: &Path) -> Result<PackedImage> {
/// Pull a containerDisk for `linux/<arch>` and return the cached qcow2.
///
/// Cached per platform-manifest digest at `~/.cua/images/disks/<hex>/disk.qcow2`
-/// (so a moving tag re-pulls only when it changes). The compressed layer blob
-/// is deleted after extraction unless `keep_blobs` (disks are GBs).
+/// (so a moving tag re-pulls only when it changes).
pub async fn pull(
client: &RegistryClient,
cache: &ImageCache,
reference: &str,
arch: &str,
- keep_blobs: bool,
) -> Result<PathBuf> {
let (pinned, manifest, digest) = client.resolve_platform(reference, arch).await?;
let dir = cache.disk_dir(&digest)?;
@@ -229,11 +227,11 @@ pub async fn pull(
}
}
}
- .map_err(|e| ImageError::Registry(e.to_string()))??;
+ .map_err(|e| ImageError::Registry(e.to_string()));
+ let _ = std::fs::remove_file(&blob);
+ let _ = std::fs::remove_file(disk.with_extension("partial"));
+ let found = found??;
progress.extract_done(layer.size);
- if !keep_blobs {
- let _ = std::fs::remove_file(&blob);
- }
if found {
cache.record_ref(reference, &digest)?;
cua_vmm::disk::mark_used(&dir);
@@ -345,7 +343,7 @@ impl ContainerDiskResolver {
#[async_trait::async_trait]
impl cua_vmm::DiskResolver for ContainerDiskResolver {
async fn resolve(&self, reference: &str, arch: cua_vmm::Arch) -> cua_vmm::Result<PathBuf> {
- Ok(pull(&self.client, &self.cache, reference, arch.oci(), false).await?)
+ Ok(pull(&self.client, &self.cache, reference, arch.oci()).await?)
}
/// Pulls a private containerDisk with the caller's credentials (a
@@ -360,7 +358,7 @@ impl cua_vmm::DiskResolver for ContainerDiskResolver {
None => self.resolve(reference, arch).await,
Some(c) => {
let client = RegistryClient::with_credentials(c.clone());
- Ok(pull(&client, &self.cache, reference, arch.oci(), false).await?)
+ Ok(pull(&client, &self.cache, reference, arch.oci()).await?)
}
}
}
diff --git a/libs/cua/crates/cua-image/tests/low_disk.rs b/libs/cua/crates/cua-image/tests/low_disk.rs
index 0b6acc9ade..39ed65d57b 100644
--- a/libs/cua/crates/cua-image/tests/low_disk.rs
+++ b/libs/cua/crates/cua-image/tests/low_disk.rs
@@ -71,6 +71,16 @@ async fn serve(listener: TcpListener, docs: Arc<HashMap<String, (String, Vec<u8>
}
}
+fn walk(p: &Path) -> Vec<std::path::PathBuf> {
+ std::fs::read_dir(p)
+ .unwrap()
+ .flat_map(|e| {
+ let p = e.unwrap().path();
+ if p.is_dir() { walk(&p) } else { vec![p] }
+ })
+ .collect()
+}
+
struct Fake(u64);
impl SpaceProbe for Fake {
@@ -82,23 +92,19 @@ impl SpaceProbe for Fake {
}
}
-#[tokio::test]
-async fn pull_fails_typed_before_writing_when_the_disk_is_nearly_full() {
- let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
- let host = format!("127.0.0.1:{}", listener.local_addr().unwrap().port());
- let dir = tempfile::tempdir().unwrap();
- let disk = dir.path().join("in.qcow2");
- std::fs::write(&disk, b"qcow2-bytes-for-the-test").unwrap();
- let img = cua_image::containerdisk::pack(&disk, "arm64", &dir.path().join("packed")).unwrap();
- let mut docs = HashMap::new();
+fn publish(
+ docs: &mut HashMap<String, (String, Vec<u8>)>,
+ repo: &str,
+ img: &cua_image::PackedImage,
+) {
let cfg = img.config_descriptor();
docs.insert(
- format!("/v2/team/disk/blobs/{}", cfg.digest),
+ format!("/v2/{repo}/blobs/{}", cfg.digest),
("application/octet-stream".to_string(), img.config.clone()),
);
for (d, path) in &img.layers {
docs.insert(
- format!("/v2/team/disk/blobs/{}", d.digest),
+ format!("/v2/{repo}/blobs/{}", d.digest),
(
"application/octet-stream".to_string(),
std::fs::read(path).unwrap(),
@@ -108,13 +114,53 @@ async fn pull_fails_typed_before_writing_when_the_disk_is_nearly_full() {
let m = serde_json::to_vec(&img.manifest()).unwrap();
let md = sha256_bytes(&m);
docs.insert(
- "/v2/team/disk/manifests/1".to_string(),
+ format!("/v2/{repo}/manifests/1"),
(OCI_MANIFEST.to_string(), m.clone()),
);
docs.insert(
- format!("/v2/team/disk/manifests/{md}"),
+ format!("/v2/{repo}/manifests/{md}"),
(OCI_MANIFEST.to_string(), m),
);
+}
+
+fn truncated_disk_image(dir: &Path) -> cua_image::PackedImage {
+ std::fs::create_dir_all(dir).unwrap();
+ let disk = dir.join("in.qcow2");
+ let mut x: u32 = 1;
+ let noise: Vec<u8> = (0..1 << 20)
+ .map(|_| {
+ x = x.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
+ (x >> 24) as u8
+ })
+ .collect();
+ std::fs::write(&disk, noise).unwrap();
+ let img = cua_image::containerdisk::pack(&disk, "arm64", &dir.join("packed")).unwrap();
+ let (desc, layer) = &img.layers[0];
+ let mut bytes = std::fs::read(layer).unwrap();
+ bytes.truncate(bytes.len() / 2);
+ let path = dir.join("layer.tar.gz");
+ std::fs::write(&path, &bytes).unwrap();
+ let desc = cua_image::manifest::Descriptor {
+ digest: sha256_bytes(&bytes),
+ size: bytes.len() as u64,
+ ..desc.clone()
+ };
+ let config = serde_json::from_slice(&img.config).unwrap();
+ cua_image::PackedImage::new(&img.arch, config, vec![(desc, path)]).unwrap()
+}
+
+#[tokio::test]
+async fn pull_fails_typed_before_writing_when_the_disk_is_nearly_full() {
+ let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let host = format!("127.0.0.1:{}", listener.local_addr().unwrap().port());
+ let dir = tempfile::tempdir().unwrap();
+ let disk = dir.path().join("in.qcow2");
+ std::fs::write(&disk, b"qcow2-bytes-for-the-test").unwrap();
+ let img = cua_image::containerdisk::pack(&disk, "arm64", &dir.path().join("packed")).unwrap();
+ let mut docs = HashMap::new();
+ publish(&mut docs, "team/disk", &img);
+ let broken = truncated_disk_image(&dir.path().join("broken"));
+ publish(&mut docs, "team/broken", &broken);
let server = tokio::spawn(serve(listener, Arc::new(docs)));
let empty = tempfile::tempdir().unwrap();
@@ -139,7 +185,7 @@ async fn pull_fails_typed_before_writing_when_the_disk_is_nearly_full() {
// 5 GiB is the default floor: 5 GiB available leaves no room.
cua_vmm::disk::set_probe(Some(Arc::new(Fake(5 * GIB))));
- let err = cua_image::containerdisk::pull(&client, &cache, &reference, "arm64", false)
+ let err = cua_image::containerdisk::pull(&client, &cache, &reference, "arm64")
.await
.unwrap_err();
let ImageError::Vmm(cua_vmm::VmmError::InsufficientDisk(e)) = &err else {
@@ -163,12 +209,28 @@ async fn pull_fails_typed_before_writing_when_the_disk_is_nearly_full() {
}
.write(&dir.path().join("home/cache.json"))
.unwrap();
- let got = cua_image::containerdisk::pull(&client, &cache, &reference, "arm64", false)
+ let got = cua_image::containerdisk::pull(&client, &cache, &reference, "arm64")
.await
.unwrap();
assert_eq!(std::fs::read(&got).unwrap(), b"qcow2-bytes-for-the-test");
// The entry is marked used for the LRU.
assert!(got.parent().unwrap().join(".last-used").exists());
+
+ let err =
+ cua_image::containerdisk::pull(&client, &cache, &format!("{host}/team/broken:1"), "arm64")
+ .await
+ .unwrap_err();
+ assert!(!matches!(err, ImageError::Vmm(_)), "{err:?}");
+ let blobs_left: Vec<_> = walk(&blobs);
+ assert!(blobs_left.is_empty(), "{blobs_left:?} {err:?}");
+ for d in std::fs::read_dir(dir.path().join("cache/disks")).unwrap() {
+ let leftovers: Vec<_> = std::fs::read_dir(d.unwrap().path())
+ .unwrap()
+ .map(|e| e.unwrap().file_name())
+ .filter(|n| n != ".lock" && n != ".last-used" && n != "disk.qcow2")
+ .collect();
+ assert!(leftovers.is_empty(), "{leftovers:?}");
+ }
cua_vmm::disk::set_probe(None);
server.abort();
}
diff --git a/libs/cua/crates/cua-vmm/src/qemu/mod.rs b/libs/cua/crates/cua-vmm/src/qemu/mod.rs
index 6696fbba55..785007626f 100644
--- a/libs/cua/crates/cua-vmm/src/qemu/mod.rs
+++ b/libs/cua/crates/cua-vmm/src/qemu/mod.rs
@@ -382,7 +382,7 @@ impl QemuRuntime {
) -> Result<QemuState> {
crate::disk::ensure_space(
&self.cfg.root,
- crate::disk::VM_CREATE_ESTIMATE,
+ backing.map_or(crate::disk::VM_CREATE_ESTIMATE, |_| crate::disk::GIB),
&format!("create VM {name}"),
)?;
let dir = self.dir(name);
@@ -1021,6 +1021,11 @@ impl Runtime for QemuRuntime {
st
}
Err(VmmError::NotFound(_)) => {
+ crate::disk::ensure_space(
+ &self.cfg.root,
+ crate::disk::GIB,
+ &format!("create VM {}", spec.name),
+ )?;
let (backing, iso) = match &spec.image {
ImageSource::Existing => return Err(VmmError::NotFound(spec.name.clone())),
ImageSource::Disk { path }
@@ -1569,6 +1574,76 @@ mod tests {
);
}
+ struct LowUnder(PathBuf, u64);
+
+ impl crate::disk::SpaceProbe for LowUnder {
+ fn space(&self, path: &Path) -> std::io::Result<crate::disk::Space> {
+ let available = if path.starts_with(&self.0) {
+ self.1
+ } else {
+ 500 * crate::disk::GIB
+ };
+ Ok(crate::disk::Space {
+ available,
+ total: 500 * crate::disk::GIB,
+ })
+ }
+ }
+
+ struct CountingResolver(std::sync::atomic::AtomicUsize);
+
+ #[async_trait]
+ impl DiskResolver for CountingResolver {
+ async fn resolve(&self, _: &str, _: Arch) -> Result<PathBuf> {
+ self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+ Err(VmmError::invalid("stop here"))
+ }
+ }
+
+ #[tokio::test]
+ async fn a_full_disk_fails_before_the_pull_and_overlays_need_less_than_blank_disks() {
+ let tmp = tempfile::tempdir().unwrap();
+ let resolver = Arc::new(CountingResolver(Default::default()));
+ let r = QemuRuntime::new(QemuConfig {
+ root: tmp.path().to_path_buf(),
+ resolver: Some(resolver.clone()),
+ ..Default::default()
+ });
+ let min_free = crate::disk::CacheConfig::load().min_free;
+ let set_available = |bytes| {
+ crate::disk::set_probe(Some(Arc::new(LowUnder(tmp.path().to_path_buf(), bytes))))
+ };
+
+ set_available(min_free + crate::disk::GIB / 2);
+ let err = r
+ .start(&StartSpec::new("full", ImageSource::oci("ghcr.io/x/y:1")))
+ .await
+ .unwrap_err();
+ let pulls = resolver.0.load(std::sync::atomic::Ordering::SeqCst);
+
+ set_available(min_free + 2 * crate::disk::GIB);
+ let backing = tmp.path().join("base.qcow2");
+ std::fs::write(&backing, b"not really a disk").unwrap();
+ let spec = StartSpec::new("o1", ImageSource::disk(&backing));
+ let overlay = r
+ .create_instance("o1", Some(&backing), None, &spec, None)
+ .await
+ .err();
+ let blank = r.create_instance("b1", None, None, &spec, None).await.err();
+ crate::disk::set_probe(None);
+
+ assert!(matches!(err, VmmError::InsufficientDisk(_)), "{err}");
+ assert_eq!(pulls, 0, "nothing is pulled when the overlay cannot fit");
+ assert!(
+ !matches!(overlay, Some(VmmError::InsufficientDisk(_))),
+ "{overlay:?}"
+ );
+ assert!(
+ matches!(blank, Some(VmmError::InsufficientDisk(_))),
+ "{blank:?}"
+ );
+ }
+
#[test]
fn missing_qemu_hints_show_the_install_plan() {
assert_eq!(