Skip to content

armgit / kiri

Public Imported from GitHub
0

A small, fast container runtime for macOS, Windows and Linux

Rust

Keep unpacked layers whole across VM stops (#2)

Precious Okwu76ccc11b8d7d
diff --git a/crates/kiri-init/src/linux/agent.rs b/crates/kiri-init/src/linux/agent.rs
index a01ddb8..24ff1cf 100644
--- a/crates/kiri-init/src/linux/agent.rs
+++ b/crates/kiri-init/src/linux/agent.rs
@@ -459,6 +459,10 @@ pub fn serve(mut conn: File, paths: &Paths) -> io::Result<()> {
container::retain(paths, keep, &running);
continue;
}
+ Ok(Msg::RetainLayers(keep)) => {
+ layers::retain(&paths.layers, keep.iter());
+ continue;
+ }
Ok(Msg::SetTime(ns)) => {
let ts = libc::timespec { tv_sec: (ns / 1_000_000_000) as _, tv_nsec: (ns % 1_000_000_000) as _ };
// A host process shares the host's clock; only the VM needs telling.
diff --git a/crates/kiri-init/src/linux/layers.rs b/crates/kiri-init/src/linux/layers.rs
index 17ac3f2..31e49a6 100644
--- a/crates/kiri-init/src/linux/layers.rs
+++ b/crates/kiri-init/src/linux/layers.rs
@@ -1,6 +1,9 @@
//! The guest's layer store: compressed layer blobs arrive from the host over
//! vsock and are unpacked once into `layers/<hex>/`, ready to be stacked by
-//! overlayfs. A layer directory only appears, by rename, once it is complete.
+//! overlayfs. A layer directory only appears, by rename, once it is complete,
+//! and is only used once `<hex>.done` beside it says its files are on disk:
+//! a VM that stops before the guest flushes can keep a layer's names but
+//! lose its contents, leaving every file empty.
use std::fs;
use std::io::{self, BufReader, Read};
@@ -24,8 +27,12 @@ pub fn dir(root: &Path, digest: &str) -> io::Result<PathBuf> {
Ok(root.join(hex(digest)?))
}
+fn marker(root: &Path, digest: &str) -> io::Result<PathBuf> {
+ Ok(root.join(format!("{}.done", hex(digest)?)))
+}
+
pub fn present(root: &Path, digest: &str) -> bool {
- dir(root, digest).is_ok_and(|d| d.is_dir())
+ dir(root, digest).is_ok_and(|d| d.is_dir()) && marker(root, digest).is_ok_and(|m| m.is_file())
}
/// Reads exactly `begin.size` bytes of blob from `stream` and unpacks them.
@@ -42,15 +49,8 @@ pub fn receive(stream: &mut impl Read, begin: &LayerBegin<'_>, root: &Path) -> i
// Whatever happened, consume the rest of the blob so the next frame lines up.
io::copy(&mut limited, &mut io::sink())?;
- match result {
- Ok(unpacked_bytes) => {
- if final_dir.exists() {
- let _ = fs::remove_dir_all(&tmp);
- } else {
- fs::rename(&tmp, &final_dir)?;
- }
- Ok(LayerStored { unpacked_bytes, elapsed_ns: started.elapsed().as_nanos() as u64 })
- }
+ match result.and_then(|unpacked_bytes| install(root, begin.digest, &tmp, &final_dir).map(|()| unpacked_bytes)) {
+ Ok(unpacked_bytes) => Ok(LayerStored { unpacked_bytes, elapsed_ns: started.elapsed().as_nanos() as u64 }),
Err(e) => {
let _ = fs::remove_dir_all(&tmp);
Err(e)
@@ -58,6 +58,60 @@ pub fn receive(stream: &mut impl Read, begin: &LayerBegin<'_>, root: &Path) -> i
}
}
+/// Puts an unpacked layer in place: its files reach the disk first, then
+/// the directory is renamed in, then the marker says it is whole.
+fn install(root: &Path, digest: &str, tmp: &Path, final_dir: &Path) -> io::Result<()> {
+ if present(root, digest) {
+ let _ = fs::remove_dir_all(tmp);
+ return Ok(());
+ }
+ sync_fs(tmp)?;
+ if final_dir.exists() {
+ // A copy without a marker was cut short and may hold empty files.
+ // Moved aside rather than deleted here, which can take a while;
+ // the next RetainLayers clears it.
+ let nanos = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0, |d| d.as_nanos());
+ fs::rename(final_dir, root.join(format!(".stale-{}-{nanos}", hex(digest)?)))?;
+ }
+ fs::rename(tmp, final_dir)?;
+ fs::File::create(marker(root, digest)?)?.sync_all()?;
+ fs::File::open(root)?.sync_all()
+}
+
+/// Flushes the filesystem holding `path` to disk.
+fn sync_fs(path: &Path) -> io::Result<()> {
+ let f = fs::File::open(path)?;
+ // SAFETY: a valid descriptor that outlives the call.
+ sys::check(unsafe { libc::syncfs(std::os::fd::AsRawFd::as_raw_fd(&f)) } as _, "syncfs")?;
+ Ok(())
+}
+
+/// Deletes every layer not in `keep` (blob digests), every layer without
+/// its marker, and whatever an unpack left half-done. Anything that isn't
+/// one of those names is left alone.
+pub fn retain<'a>(root: &Path, keep: impl Iterator<Item = &'a str>) {
+ let keep: std::collections::HashSet<&str> = keep.filter_map(|d| hex(d).ok()).collect();
+ let Ok(entries) = fs::read_dir(root) else { return };
+ for entry in entries.flatten() {
+ let name = entry.file_name();
+ let Some(name) = name.to_str() else { continue };
+ let leftover = name.starts_with(".tmp-") || name.starts_with(".stale-");
+ let (h, is_marker) = match name.strip_suffix(".done") {
+ Some(h) => (h, true),
+ None => (name, false),
+ };
+ let layer = h.len() == 64 && h.bytes().all(|b| b.is_ascii_hexdigit());
+ if !leftover && !layer {
+ continue;
+ }
+ let whole = layer && root.join(h).is_dir() && root.join(format!("{h}.done")).is_file();
+ if leftover || !keep.contains(h) || !whole {
+ let path = entry.path();
+ let _ = if is_marker || !path.is_dir() { fs::remove_file(&path) } else { fs::remove_dir_all(&path) };
+ }
+ }
+}
+
/// The most a layer may expand to. The blob's digest covers only its
/// compressed bytes, so a tiny, digest-valid layer could otherwise expand
/// without limit onto the disk (on a Linux host or in WSL, the host's own).
@@ -314,6 +368,65 @@ mod tests {
let _ = fs::remove_dir_all(&dest);
}
+ fn layer_blob(files: &[(&str, &[u8])]) -> Vec<u8> {
+ let mut b = tar::Builder::new(Vec::new());
+ for (name, data) in files {
+ b.append_data(&mut header(data.len() as u64, 0o644), name, *data).unwrap();
+ }
+ b.into_inner().unwrap()
+ }
+
+ const D1: &str = "sha256:1111111111111111111111111111111111111111111111111111111111111111";
+ const D2: &str = "sha256:2222222222222222222222222222222222222222222222222222222222222222";
+
+ fn send(root: &Path, digest: &str, blob: &[u8]) -> LayerStored {
+ let begin = LayerBegin { digest, media_type: "application/vnd.oci.image.layer.v1.tar", size: blob.len() as u64 };
+ receive(&mut &blob[..], &begin, root).unwrap()
+ }
+
+ /// A stored layer is present only with its marker; one cut short
+ /// (directory, no marker) is not, and is replaced when sent again.
+ #[test]
+ fn layer_without_marker_is_replaced() {
+ let root = scratch("marker");
+ let blob = layer_blob(&[("bin/sh", b"real")]);
+ send(&root, D1, &blob);
+ assert!(present(&root, D1));
+ assert_eq!(fs::read(dir(&root, D1).unwrap().join("bin/sh")).unwrap(), b"real");
+
+ // What a VM stopped before the flush leaves: names, empty contents.
+ fs::remove_file(marker(&root, D1).unwrap()).unwrap();
+ fs::write(dir(&root, D1).unwrap().join("bin/sh"), b"").unwrap();
+ assert!(!present(&root, D1));
+ send(&root, D1, &blob);
+ assert!(present(&root, D1));
+ assert_eq!(fs::read(dir(&root, D1).unwrap().join("bin/sh")).unwrap(), b"real");
+ let _ = fs::remove_dir_all(&root);
+ }
+
+ /// Retain keeps whole layers it is told to and deletes the rest:
+ /// unlisted layers, unmarked ones, leftovers. Other names stay.
+ #[test]
+ fn retain_deletes_unused_and_broken_layers() {
+ let root = scratch("retain");
+ let blob = layer_blob(&[("f", b"x")]);
+ send(&root, D1, &blob);
+ send(&root, D2, &blob);
+ let broken = "3".repeat(64);
+ fs::create_dir(root.join(&broken)).unwrap();
+ fs::create_dir(root.join(format!(".stale-{broken}-1"))).unwrap();
+ fs::create_dir(root.join(format!(".tmp-{broken}"))).unwrap();
+ fs::write(root.join("other"), "keep").unwrap();
+
+ retain(&root, [D1, &format!("sha256:{broken}")].into_iter());
+ assert!(present(&root, D1));
+ assert!(!dir(&root, D2).unwrap().exists() && !marker(&root, D2).unwrap().exists());
+ assert!(!root.join(&broken).exists(), "unmarked layer kept");
+ assert!(!root.join(format!(".stale-{broken}-1")).exists() && !root.join(format!(".tmp-{broken}")).exists());
+ assert!(root.join("other").is_file());
+ let _ = fs::remove_dir_all(&root);
+ }
+
/// A layer that expands past its cap is refused.
#[test]
fn expansion_is_capped() {
diff --git a/crates/kiri-proto/src/lib.rs b/crates/kiri-proto/src/lib.rs
index 582395e..58ae559 100644
--- a/crates/kiri-proto/src/lib.rs
+++ b/crates/kiri-proto/src/lib.rs
@@ -94,6 +94,10 @@ pub enum Msg<'a> {
/// Host → guest: delete every stopped container's files except these.
/// Sent once after connecting. No reply.
Retain(StrList<'a>),
+ /// Host → guest: delete every unpacked layer except these (by blob
+ /// digest), and any left half-written. Sent once after connecting,
+ /// before anything runs. No reply.
+ RetainLayers(StrList<'a>),
/// Host → guest: metadata of a path inside a running container's root
/// (symlinks are not followed). Answered by `FsStat` or `Error`.
FsStatRequest { id: &'a str, path: &'a str },
@@ -635,6 +639,7 @@ mod kind {
pub const UPLINK_QUERY: u16 = 70;
pub const UPLINK: u16 = 71;
pub const REMOVE_BRIDGE: u16 = 72;
+ pub const RETAIN_LAYERS: u16 = 73;
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -718,6 +723,11 @@ pub fn encode_retain<S: AsRef<str>>(out: &mut Vec<u8>, ids: &[S]) {
frame(out, kind::RETAIN, |out| StrList::put(out, ids));
}
+/// Encodes a `RetainLayers` frame from the sender's layer digests.
+pub fn encode_retain_layers<S: AsRef<str>>(out: &mut Vec<u8>, digests: &[S]) {
+ frame(out, kind::RETAIN_LAYERS, |out| StrList::put(out, digests));
+}
+
/// A container's network, from the sender's side.
#[derive(Debug, Clone, Copy)]
pub struct Net<'a, S> {
@@ -883,6 +893,10 @@ impl Msg<'_> {
put_u32(o, ids.count);
o.extend_from_slice(ids.raw);
}),
+ Msg::RetainLayers(ids) => frame(out, kind::RETAIN_LAYERS, |o| {
+ put_u32(o, ids.count);
+ o.extend_from_slice(ids.raw);
+ }),
Msg::FsStatRequest { id, path } => frame(out, kind::FS_STAT_REQUEST, |o| {
put_str(o, id);
put_str(o, path);
@@ -1162,6 +1176,7 @@ pub fn decode(frame: &[u8]) -> Result<Msg<'_>, DecodeError> {
opts: if r.remaining() >= 4 { r.list()? } else { StrList { count: 0, raw: &[] } },
}),
kind::RETAIN => Msg::Retain(r.list()?),
+ kind::RETAIN_LAYERS => Msg::RetainLayers(r.list()?),
kind::FS_STAT_REQUEST => Msg::FsStatRequest { id: r.str()?, path: r.str()? },
kind::FS_STAT => Msg::FsStat(read_entry(&mut r)?),
kind::FS_LIST_REQUEST => Msg::FsListRequest { id: r.str()?, path: r.str()? },
@@ -1319,6 +1334,14 @@ mod tests {
encode_retain(&mut buf, &["a", "b"]);
let Msg::Retain(ids) = decode(&buf).unwrap() else { panic!() };
assert_eq!(ids.iter().collect::<Vec<_>>(), ["a", "b"]);
+
+ buf.clear();
+ encode_retain_layers(&mut buf, &["sha256:aa"]);
+ let Msg::RetainLayers(ids) = decode(&buf).unwrap() else { panic!() };
+ assert_eq!(ids.iter().collect::<Vec<_>>(), ["sha256:aa"]);
+ let mut again = Vec::new();
+ Msg::RetainLayers(ids).encode(&mut again);
+ assert_eq!(buf, again);
}
#[test]
diff --git a/crates/kiri/src/daemon/engine.rs b/crates/kiri/src/daemon/engine.rs
index b80a6d8..77dd8de 100644
--- a/crates/kiri/src/daemon/engine.rs
+++ b/crates/kiri/src/daemon/engine.rs
@@ -671,6 +671,16 @@ impl Engine {
// Everything else on the guest's disk belongs to containers that are gone.
let keep: Vec<String> = self.lock().containers.keys().cloned().collect();
vm.retain(&keep)?;
+ // And every unpacked layer no image or container uses. Skipped if
+ // the image list can't be read, rather than deleting everything.
+ match self.store.images() {
+ Ok(images) => {
+ let mut layers: std::collections::BTreeSet<String> = images.iter().flat_map(|i| i.layers.iter().map(|l| l.digest.clone())).collect();
+ layers.extend(self.lock().containers.values().flat_map(|c| c.layers.iter().cloned()));
+ vm.retain_layers(&layers.into_iter().collect::<Vec<_>>())?;
+ }
+ Err(e) => crate::diag::warn!("kept every unpacked layer: images unreadable", error = e),
+ }
*self.memory_target.lock().unwrap_or_else(|e| e.into_inner()) = ceiling;
self.sleeping.store(false, std::sync::atomic::Ordering::SeqCst);
*link = Some(vm.clone());
@@ -884,6 +894,10 @@ impl Engine {
self.memory_used.store(0, std::sync::atomic::Ordering::Relaxed);
self.memory_stall.store(0, std::sync::atomic::Ordering::Relaxed);
vm.shutdown();
+ // Let the guest flush its disk and power off before the VM is dropped.
+ if !vm.wait_stopped(Duration::from_secs(10)) {
+ crate::diag::warn!("the VM did not power off in 10 s; stopped it");
+ }
// Dropping the last reference stops the VM and frees the data disk
// before anyone may boot a new one (they wait on `link`).
drop(vm);
@@ -2829,9 +2843,8 @@ impl Engine {
}
if let Some(vm) = self.awake() {
vm.shutdown();
- #[cfg(target_os = "linux")]
if !vm.wait_stopped(Duration::from_secs(10)) {
- crate::diag::warn!("the agent did not stop in 10 s; killed it");
+ crate::diag::warn!("the VM did not power off in 10 s; stopped it");
}
}
}
diff --git a/crates/kiri/src/daemon/vmlink.rs b/crates/kiri/src/daemon/vmlink.rs
index e449102..2cb7c7c 100644
--- a/crates/kiri/src/daemon/vmlink.rs
+++ b/crates/kiri/src/daemon/vmlink.rs
@@ -426,6 +426,11 @@ impl VmLink {
self.send(|o| kiri_proto::encode_retain(o, ids))
}
+ /// Tells the guest which unpacked layers are still used; it deletes the rest.
+ pub fn retain_layers(&self, digests: &[String]) -> io::Result<()> {
+ self.send(|o| kiri_proto::encode_retain_layers(o, digests))
+ }
+
pub fn kill(&self, id: &str, signal: u32) -> io::Result<()> {
self.send(|o| Msg::Kill { id, signal }.encode(o))
}
@@ -482,12 +487,15 @@ impl VmLink {
let _ = self.send(|o| Msg::Shutdown.encode(o));
}
- /// Linux host: waits for the agent process to exit after `shutdown`
- /// (it kills what is left and removes its bridges and nftables table
- /// first), and kills it if it has not within `timeout`. The daemon
- /// must not exit before it: an agent left behind kept its containers,
- /// ports and sockets, and the next daemon could not start.
- #[cfg(target_os = "linux")]
+ /// Waits for the guest to stop after `shutdown`, and forces it if it has
+ /// not within `timeout`. False if it had to be forced.
+ ///
+ /// In a VM the guest flushes its disk before powering off; stopping the
+ /// VM before then (as dropping it does) loses what was still cached, and
+ /// files written shortly before come back empty. On a Linux host the
+ /// agent kills what is left and removes its bridges and nftables table
+ /// first; an agent left behind kept its containers, ports and sockets,
+ /// and the next daemon could not start.
pub fn wait_stopped(&self, timeout: std::time::Duration) -> bool {
let deadline = std::time::Instant::now() + timeout;
let vm = self._vm.lock().unwrap_or_else(|e| e.into_inner());