A small, fast container runtime for macOS, Windows and Linux
Rust
Container DNS survives the Mac's VM resolver going down; a deleted Kiri hands docker build back (#11)
Precious Okwude47ac0d91e7
diff --git a/crates/kiri-init/src/linux/agent.rs b/crates/kiri-init/src/linux/agent.rsindex 24ff1cf..1c4d4d6 100644--- a/crates/kiri-init/src/linux/agent.rs+++ b/crates/kiri-init/src/linux/agent.rs@@ -62,7 +62,9 @@ pub fn serve(mut conn: File, paths: &Paths) -> io::Result<()> {// On a Linux host, SIGTERM or SIGINT stops the agent as Shutdown does.let term = if host_process { Some(term_signals()?) } else { None };let mut bridges = Bridges::default();+ // In the VM, lookups the gateway's resolver fails go through the host.let mut dns = super::dns::Dns::default();+ dns.relay = !host_process;let forward = super::relay::listen()?;// host.docker.internal's relay to the Mac (VM only). A Linux host maps// that name to the bridge gateway and redirects nothing here, so the@@ -130,6 +132,10 @@ pub fn serve(mut conn: File, paths: &Paths) -> io::Result<()> {let wait = TTY_HUP_WAIT.as_millis() as i32;timeout = if timeout < 0 { wait } else { timeout.min(wait) };}+ let wait = dns.timeout_ms();+ if wait >= 0 {+ timeout = if timeout < 0 { wait } else { timeout.min(wait) };+ }// SAFETY: `fds` is a valid array of pollfd of the given length.let n = unsafe { libc::poll(fds.as_mut_ptr(), fds.len() as libc::nfds_t, timeout) };if n < 0 {@@ -157,6 +163,8 @@ pub fn serve(mut conn: File, paths: &Paths) -> io::Result<()> {}}+ dns.on_timer();+// Output and input first, so a process's last words go out before its exit.for pfd in &fds[3..] {if pfd.revents == 0 {diff --git a/crates/kiri-init/src/linux/container.rs b/crates/kiri-init/src/linux/container.rsindex 6a89529..cc93bd2 100644--- a/crates/kiri-init/src/linux/container.rs+++ b/crates/kiri-init/src/linux/container.rs@@ -257,7 +257,7 @@ pub fn spawn(spec: &RunSpec<'_>, paths: &Paths, net: NetCtx<'_>) -> io::Result<Pprepared.host = HostApplied { rlimits: host.rlimits.clone(), groups: host.groups.clone(), nodes };// Small, short-lived files: kept in memory (/run is tmpfs), not on the data disk.let etc_dir = PathBuf::from("/run/kiri").join(spec.id);- let embedded_dns = embedded_dns(spec, net.upstream_dns);+ let embedded_dns = embedded_dns(spec, net.upstream_dns, net.dns.relay);prepared.etc_files = write_etc_files(spec, &etc_dir, net.upstream_dns, embedded_dns).map_err(cleanup)?;prepared.mounts = parse_mounts(spec.mounts).map_err(cleanup)?;let bridged = !spec.net.bridge.is_empty() && spec.flags & (run_flags::NET_HOST | run_flags::NET_NONE) == 0 && prepared.net_join.is_none();@@ -736,16 +736,18 @@ fn write_etc_files(spec: &RunSpec<'_>, base: &std::path::Path, upstream: &[[u8;/// `--dns-search` and `--dns-option` apply either way./// Whether the container's DNS goes to the agent's resolver (127.0.0.11):/// on a user network, as Docker's embedded DNS; and on the default bridge-/// when the host's resolver is one of its own addresses (net::local_resolver),-/// unless the container names its own servers (`--dns`).-fn embedded_dns(spec: &RunSpec<'_>, upstream: &[[u8; 4]]) -> bool {+/// when the host's resolver is one of its own addresses (net::local_resolver)+/// or, in the VM (`relay`), so that lookups the Mac's gateway resolver fails+/// can go through the host (dns.rs), unless the container names its own+/// servers (`--dns`).+fn embedded_dns(spec: &RunSpec<'_>, upstream: &[[u8; 4]], relay: bool) -> bool {if spec.flags & run_flags::DNS_GATEWAY != 0 {return true;}let bridged = !spec.net.bridge.is_empty()&& spec.flags & (run_flags::NET_HOST | run_flags::NET_NONE) == 0&& net_opts::values(spec.net.opts, net_opts::NETNS).next().is_none();- bridged && net_opts::values(spec.net.opts, net_opts::NAMESERVER).next().is_none() && net::local_resolver(upstream)+ bridged && net_opts::values(spec.net.opts, net_opts::NAMESERVER).next().is_none() && (relay || net::local_resolver(upstream))}fn resolv_conf(spec: &RunSpec<'_>, upstream: &[[u8; 4]], embedded_dns: bool) -> String {@@ -1380,8 +1382,23 @@ mod tests {encode_run(&mut buf, "c1", &[], &["sh"], &[], "/", "", "c1", 0, net, &[], &Limits::default(), 0, &[] as &[&str]);let Ok(Msg::Run(spec)) = decode(&buf) else { panic!() };// TEST-NET-3: no machine running the tests has it.- assert!(!super::embedded_dns(&spec, &[[203, 0, 113, 53]]), "a resolver elsewhere is asked directly");- assert!(!super::embedded_dns(&spec, &[]), "no resolver: the public fallback, not the agent");+ assert!(!super::embedded_dns(&spec, &[[203, 0, 113, 53]], false), "a resolver elsewhere is asked directly");+ assert!(!super::embedded_dns(&spec, &[], false), "no resolver: the public fallback, not the agent");+ }++ #[test]+ fn in_the_vm_the_default_bridge_goes_through_the_agent() {+ let spec_with = |flags: u32, opts: &[&str], f: &dyn Fn(&kiri_proto::RunSpec<'_>)| {+ let mut buf = Vec::new();+ let net = Net { bridge: "kiri0", ip: [172, 17, 0, 2], prefix: 16, gateway: [172, 17, 0, 1], extra_hosts: &[], dns_names: &[], opts };+ encode_run(&mut buf, "c1", &[], &["sh"], &[], "/", "", "c1", flags, net, &[], &Limits::default(), 0, &[] as &[&str]);+ let Ok(Msg::Run(spec)) = decode(&buf) else { panic!() };+ f(&spec);+ };+ let gateway = [[192, 168, 64, 1]];+ spec_with(0, &[], &|s| assert!(super::embedded_dns(s, &gateway, true), "so a refused gateway can fall back to the host"));+ spec_with(0, &["nameserver=8.8.8.8"], &|s| assert!(!super::embedded_dns(s, &gateway, true), "--dns servers are asked directly"));+ spec_with(run_flags::NET_HOST, &[], &|s| assert!(!super::embedded_dns(s, &gateway, true), "host networking keeps the VM's resolv.conf"));}#[test]diff --git a/crates/kiri-init/src/linux/dns.rs b/crates/kiri-init/src/linux/dns.rsindex 51ed059..ddf507a 100644--- a/crates/kiri-init/src/linux/dns.rs+++ b/crates/kiri-init/src/linux/dns.rs@@ -6,6 +6,15 @@//! forwarded to the VM's upstream resolver (or the container's own `--dns`//! servers) and the answer relayed back. The 127.0.0.11 socket is opened in//! the container's namespace by this agent, so no process runs in there.+//!+//! In the VM the upstream is the Mac's resolver for the NAT network, at its+//! gateway (192.168.64.1). macOS doesn't always answer there: after a+//! network change, sleep or a VPN, lookups are refused or go unanswered+//! while the Mac itself resolves fine. So a lookup the gateway refuses, or+//! leaves unanswered for `FALLBACK_AFTER`, is asked again through the Kiri+//! daemon on the Mac, over vsock, which asks the Mac's own resolvers+//! (crates/kiri/src/daemon/dns.rs). A container's own `--dns` servers are+//! never second-guessed.use std::collections::HashMap;use std::io::{self, Read};@@ -24,6 +33,12 @@ const FORWARD_TIMEOUT: Duration = Duration::from_secs(5);/// At most this many forwarded queries in flight (each has its own socket);/// more are dropped, and the client asks again.const MAX_PENDING: usize = 256;+/// A lookup the VM's resolver hasn't answered within this goes through the+/// host (VM only). Gateway answers take milliseconds; a slow recursive one+/// that loses the race is simply replaced by the host's.+const FALLBACK_AFTER: Duration = Duration::from_secs(1);+/// Largest reply the host relay may send (a DNS over TCP frame's limit).+const MAX_RELAYED: usize = 2 + 65_535;/// Docker's embedded DNS address, inside each container.pub const EMBEDDED: [u8; 4] = [127, 0, 0, 11];@@ -42,7 +57,8 @@ struct Pending {/// Its own socket, connected to the resolver: a fresh source port for/// every query, as Docker's embedded DNS does, so one lost or blocked/// flow doesn't take the resolver away from every later query, and an- /// answer has to guess the port as well as the ID.+ /// answer has to guess the port as well as the ID. Once the query goes+ /// through the host, the vsock stream to it instead.sock: OwnedFd,/// The ID it went out with.our: u16,@@ -52,6 +68,13 @@ struct Pending {id: u16,question: Vec<u8>,sent: Instant,+ /// The query as sent (with `our` ID), to send again through the host.+ query: Vec<u8>,+ /// It may still go through the host: in the VM, asked of the VM's own+ /// resolver (not `--dns` servers), and not tried there yet.+ may_relay: bool,+ /// Sent through the host: the reply's bytes so far, `u16 len | reply`.+ relayed: Option<Vec<u8>>,}/// One name's IPv6 addresses, each with its container's IPv4 address.@@ -68,6 +91,9 @@ pub struct Dns {/// Container address → its `--dns` servers, used instead of the VM's.clients: HashMap<[u8; 4], Vec<[u8; 4]>>,pending: Vec<Pending>,+ /// Lookups the VM's resolver fails go through the host (in the VM; a+ /// Linux host's agent asks the host's resolvers itself).+ pub relay: bool,}fn udp_socket() -> io::Result<OwnedFd> {@@ -220,21 +246,27 @@ impl Dns {pub fn on_readable(&mut self, fd: RawFd, upstream: &[[u8; 4]]) {let mut buf = [0u8; 1500];if let Some(at) = self.pending.iter().position(|p| p.sock.as_raw_fd() == fd) {+ if self.pending[at].relayed.is_some() {+ self.read_relayed(at);+ return;+ }// A forwarded answer: hand it back to whoever asked.loop {// SAFETY: receiving into a buffer we own.let n = unsafe { libc::recv(fd, buf.as_mut_ptr().cast(), buf.len(), 0) };+ if n < 0 && io::Error::last_os_error().raw_os_error() == Some(libc::ECONNREFUSED) {+ // Nothing listens at the resolver: ask the host now+ // rather than wait for the client to give up.+ self.relay(at);+ break;+ }if n < 12 {break;}let msg = &mut buf[..n as usize];let p = &self.pending[at];if u16::from_be_bytes([msg[0], msg[1]]) == p.our && response_matches(msg, p) {- let p = self.pending.swap_remove(at);- msg[0..2].copy_from_slice(&p.id.to_be_bytes());- if let Some(s) = self.servers.iter().find(|s| s.sock.as_raw_fd() == p.server) {- send_to(s.sock.as_raw_fd(), msg, &p.client);- }+ self.answered(at, msg);break;}}@@ -309,31 +341,169 @@ impl Dns {let now = Instant::now();self.pending.retain(|p| now.duration_since(p.sent) < FORWARD_TIMEOUT);let key = self.servers[server].container.map_or(client.sin_addr.s_addr.to_ne_bytes(), |(_, owner)| owner);- let own = self.clients.get(&key).and_then(|v| v.first());- let Some(dest) = own.or(upstream.first()).copied() else { return };+ let own = self.clients.get(&key).and_then(|v| v.first()).copied();if self.pending.len() >= MAX_PENDING {return;}let Some((_, _, end)) = parse_question(q) else {return;};- let Ok(sock) = udp_socket() else { return };- let to = sockaddr(dest, 53);- // Connected, so it only receives datagrams from this resolver.- // SAFETY: connecting to a valid sockaddr_in.- if unsafe { libc::connect(sock.as_raw_fd(), (&to as *const libc::sockaddr_in).cast(), std::mem::size_of::<libc::sockaddr_in>() as u32) } != 0 {- return;- }let Some(our) = random_id() else { return };let mut msg = q.to_vec();let id = u16::from_be_bytes([msg[0], msg[1]]);msg[0..2].copy_from_slice(&our.to_be_bytes());- // SAFETY: sending a buffer we own on a connected socket.- if unsafe { libc::send(sock.as_raw_fd(), msg.as_ptr().cast(), msg.len(), 0) } < 0 {+ let may_relay = self.relay && own.is_none();+ let dest = own.or(upstream.first().copied());+ // No resolver yet, or it can't be reached: the host, if allowed.+ let (sock, relayed) = match dest.and_then(|d| ask(d, &msg)) {+ Some(s) => (s, false),+ None if may_relay => match relay_stream(&msg) {+ Some(s) => (s, true),+ None => return,+ },+ None => return,+ };+ let p = Pending {+ sock,+ our,+ server: self.servers[server].sock.as_raw_fd(),+ client,+ id,+ question: q[12..end].to_vec(),+ sent: now,+ query: msg,+ may_relay: may_relay && !relayed,+ relayed: relayed.then(Vec::new),+ };+ self.pending.push(p);+ }++ /// Hands a matching answer to whoever asked, and forgets the query.+ fn answered(&mut self, at: usize, msg: &mut [u8]) {+ let p = self.pending.swap_remove(at);+ msg[0..2].copy_from_slice(&p.id.to_be_bytes());+ if let Some(s) = self.servers.iter().find(|s| s.sock.as_raw_fd() == p.server) {+ send_to(s.sock.as_raw_fd(), msg, &p.client);+ }+ }++ /// Sends pending query `at` again, through the host, once.+ fn relay(&mut self, at: usize) {+ let p = &mut self.pending[at];+ if !p.may_relay {return;}- self.pending.push(Pending { sock, our, server: self.servers[server].sock.as_raw_fd(), client, id, question: q[12..end].to_vec(), sent: now });+ p.may_relay = false;+ if let Some(stream) = relay_stream(&p.query) {+ p.sock = stream;+ p.relayed = Some(Vec::new());+ p.sent = Instant::now();+ }}++ /// Reads the host's reply to query `at` as it arrives.+ fn read_relayed(&mut self, at: usize) {+ let fd = self.pending[at].sock.as_raw_fd();+ let mut buf = [0u8; 4096];+ let mut eof = false;+ loop {+ // SAFETY: reading into a buffer we own.+ let n = unsafe { libc::read(fd, buf.as_mut_ptr().cast(), buf.len()) };+ if n > 0 {+ let got = self.pending[at].relayed.get_or_insert_with(Vec::new);+ got.extend_from_slice(&buf[..n as usize]);+ if got.len() > MAX_RELAYED {+ eof = true;+ break;+ }+ continue;+ }+ if n < 0 && io::Error::last_os_error().kind() == io::ErrorKind::WouldBlock {+ break;+ }+ eof = true;+ break;+ }+ let got = self.pending[at].relayed.take().unwrap_or_default();+ match framed_reply(&got) {+ Some(reply) => {+ let mut msg = reply.to_vec();+ let p = &self.pending[at];+ if msg.len() >= 12 && u16::from_be_bytes([msg[0], msg[1]]) == p.our && response_matches(&msg, p) {+ self.answered(at, &mut msg);+ } else {+ self.pending.swap_remove(at);+ }+ }+ // Gone before a whole reply: the client asks again.+ None if eof => {+ self.pending.swap_remove(at);+ }+ None => self.pending[at].relayed = Some(got),+ }+ }++ /// Sends lookups the VM's resolver has left unanswered too long+ /// through the host. Called on every turn of the agent's loop.+ pub fn on_timer(&mut self) {+ let now = Instant::now();+ for at in 0..self.pending.len() {+ let p = &self.pending[at];+ if p.may_relay && now.duration_since(p.sent) >= FALLBACK_AFTER {+ self.relay(at);+ }+ }+ }++ /// Milliseconds until `on_timer` has something to do; -1 for never.+ pub fn timeout_ms(&self) -> i32 {+ let now = Instant::now();+ self.pending+ .iter()+ .filter(|p| p.may_relay)+ .map(|p| FALLBACK_AFTER.saturating_sub(now.duration_since(p.sent)).as_millis().min(i32::MAX as u128) as i32)+ .min()+ .unwrap_or(-1)+ }+}++/// A UDP socket connected to `dest`:53 with `msg` sent on it, or None if+/// it couldn't be sent (no route, the network down).+fn ask(dest: [u8; 4], msg: &[u8]) -> Option<OwnedFd> {+ let sock = udp_socket().ok()?;+ let to = sockaddr(dest, 53);+ // Connected, so it only receives datagrams from this resolver.+ // SAFETY: connecting to a valid sockaddr_in.+ if unsafe { libc::connect(sock.as_raw_fd(), (&to as *const libc::sockaddr_in).cast(), std::mem::size_of::<libc::sockaddr_in>() as u32) } != 0 {+ return None;+ }+ // SAFETY: sending a buffer we own on a connected socket.+ if unsafe { libc::send(sock.as_raw_fd(), msg.as_ptr().cast(), msg.len(), 0) } < 0 {+ return None;+ }+ Some(sock)+}++/// A stream to the host's DNS relay with `msg` sent on it, non-blocking+/// for the reply.+fn relay_stream(msg: &[u8]) -> Option<OwnedFd> {+ let stream = super::transport::dial(kiri_proto::DNS_RELAY_PORT).ok()?;+ let mut frame = Vec::with_capacity(2 + msg.len());+ frame.extend_from_slice(&u16::try_from(msg.len()).ok()?.to_be_bytes());+ frame.extend_from_slice(msg);+ // A few hundred bytes on a fresh stream: one write, never short.+ // SAFETY: writing a buffer we own to a socket we own.+ if unsafe { libc::write(stream.as_raw_fd(), frame.as_ptr().cast(), frame.len()) } != frame.len() as isize {+ return None;+ }+ sys::set_nonblocking(stream.as_raw_fd()).ok()?;+ Some(stream)+}++/// The reply in `u16 len | reply`, once all of it is there.+fn framed_reply(got: &[u8]) -> Option<&[u8]> {+ let len = u16::from_be_bytes([*got.first()?, *got.get(1)?]) as usize;+ got.get(2..2 + len)}/// A UDP socket bound to `ip`:53.@@ -560,9 +730,84 @@ mod tests {id: 7,question: b"\x03www\x07example\x03com\x00\x00\x01\x00\x01".to_vec(),sent: Instant::now(),+ query: Vec::new(),+ may_relay: false,+ relayed: None,}}+ #[test]+ fn a_relayed_reply_is_whole_only_once_its_length_is_there() {+ use super::framed_reply;+ assert_eq!(framed_reply(&[]), None);+ assert_eq!(framed_reply(&[0]), None);+ assert_eq!(framed_reply(&[0, 3, 1, 2]), None);+ assert_eq!(framed_reply(&[0, 3, 1, 2, 3]), Some(&[1u8, 2, 3][..]));+ assert_eq!(framed_reply(&[0, 0]), Some(&[][..]));+ }++ #[test]+ fn only_lookups_the_vm_resolver_may_miss_wait_for_the_host() {+ let mut dns = Dns::default();+ assert_eq!(dns.timeout_ms(), -1, "nothing pending");+ dns.pending.push(pending());+ assert_eq!(dns.timeout_ms(), -1, "--dns servers are not second-guessed");+ let mut p = pending();+ p.may_relay = true;+ dns.pending.push(p);+ let t = dns.timeout_ms();+ assert!((0..=1000).contains(&t), "{t}");+ // Past the wait it is tried through the host once, whatever comes of it.+ dns.pending[1].sent = Instant::now() - super::FALLBACK_AFTER;+ assert_eq!(dns.timeout_ms(), 0);+ dns.on_timer();+ assert!(!dns.pending[1].may_relay);+ assert_eq!(dns.timeout_ms(), -1);+ }++ #[test]+ fn the_hosts_reply_goes_back_to_whoever_asked() {+ use std::net::UdpSocket;+ use std::os::fd::{AsRawFd, FromRawFd, OwnedFd};+ let client = UdpSocket::bind("127.0.0.1:0").unwrap();+ client.set_read_timeout(Some(std::time::Duration::from_secs(2))).unwrap();+ let std::net::SocketAddr::V4(addr) = client.local_addr().unwrap() else { panic!() };+ let mut dns = Dns::default();+ dns.servers.push(Server { bridges: vec!["kiri1".into()], sock: udp_socket().unwrap(), container: None });+ let mut fds = [0; 2];+ // SAFETY: `fds` has room for the pair.+ assert_eq!(unsafe { libc::socketpair(libc::AF_UNIX, libc::SOCK_STREAM | libc::SOCK_NONBLOCK, 0, fds.as_mut_ptr()) }, 0);+ // SAFETY: fresh descriptors from socketpair.+ let (ours, host) = unsafe { (OwnedFd::from_raw_fd(fds[0]), OwnedFd::from_raw_fd(fds[1])) };+ let mut p = pending();+ p.sock = ours;+ p.server = dns.servers[0].sock.as_raw_fd();+ p.client = super::sockaddr(addr.ip().octets(), addr.port());+ p.relayed = Some(Vec::new());+ let question = p.question.clone();+ dns.pending.push(p);+ let mut reply = vec![0, 1, 0x81, 0x80, 0, 1, 0, 0, 0, 0, 0, 0];+ reply.extend_from_slice(&question);+ let mut frame = (reply.len() as u16).to_be_bytes().to_vec();+ frame.extend_from_slice(&reply);+ let send = |b: &[u8]| {+ // SAFETY: writing a buffer we own to our end of the pair.+ assert_eq!(unsafe { libc::write(host.as_raw_fd(), b.as_ptr().cast(), b.len()) }, b.len() as isize)+ };+ // Half a reply waits for the rest.+ send(&frame[..5]);+ let fd = dns.pending[0].sock.as_raw_fd();+ dns.on_readable(fd, &[]);+ assert_eq!(dns.pending.len(), 1);+ send(&frame[5..]);+ dns.on_readable(fd, &[]);+ assert!(dns.pending.is_empty());+ let mut buf = [0u8; 512];+ let n = client.recv(&mut buf).unwrap();+ assert_eq!(&buf[..2], &[0, 7], "the client's own ID");+ assert_eq!(&buf[2..n], &reply[2..]);+ }+#[test]fn response_must_match_the_whole_question() {let p = pending();diff --git a/crates/kiri-proto/src/lib.rs b/crates/kiri-proto/src/lib.rsindex 58ae559..25e2742 100644--- a/crates/kiri-proto/src/lib.rs+++ b/crates/kiri-proto/src/lib.rs@@ -167,6 +167,11 @@ pub enum Msg<'a> {/// vsock port of the guest's copy service: one connection per copy.pub const COPY_PORT: u32 = 1028;+/// vsock port of the host's DNS relay (VM only). The agent sends a lookup+/// the VM's gateway resolver didn't answer as `u16 len | query`, as DNS over+/// TCP frames it, and gets `u16 len | reply` back: one lookup per connection.+pub const DNS_RELAY_PORT: u32 = 1037;+pub mod copy_op {pub const IN: u8 = 1;pub const OUT: u8 = 2;diff --git a/crates/kiri-win/src/main.rs b/crates/kiri-win/src/main.rsindex c4bfcdb..b78100a 100644--- a/crates/kiri-win/src/main.rs+++ b/crates/kiri-win/src/main.rs@@ -46,6 +46,11 @@ mod builds;#[cfg(windows)]#[path = "../../kiri/src/buildwith.rs"]mod buildwith;+// Kiri's program moved or deleted from where it started: the Mac's and+// Linux's own crates/kiri/src/appwatch.rs.+#[cfg(windows)]+#[path = "../../kiri/src/appwatch.rs"]+mod appwatch;// An image's Layers and Security tabs, asked of the engine in WSL.#[cfg(windows)]mod scan;@@ -139,6 +144,22 @@ fn main() -> std::process::ExitCode {serve::log(&format!("dns on 127.0.0.1:53: {e}"));}engine::start(engine::Env { distro: cfg.distro.clone(), pipe: cfg.pipe.clone() });+ // Its folder renamed, moved or deleted around it (Windows won't delete+ // a running program itself): docker build handed back as it was before+ // Kiri took it, the choice kept for when Kiri runs again, and Kiri quits.+ appwatch::watch(|exe| {+ serve::log(&format!("{} is gone from where Kiri started; handing docker build back and quitting", exe.display()));+ if let Err(e) = buildwith::places().and_then(|p| p.give_back()) {+ serve::log(&format!("docker build not handed back: {e}"));+ }+ app::Handle.run(|_| {+ // SAFETY: ends the window's message loop; run on its thread.+ unsafe { windows::Win32::UI::WindowsAndMessaging::PostQuitMessage(0) }+ });+ // No window to quit (it never opened): quit from here.+ std::thread::sleep(std::time::Duration::from_secs(5));+ std::process::exit(0);+ });// The engine starts when the page says `ready`, and with the window// hidden (--background, as from the login shortcut) no page loads: after// kiri-win restarted, docker got no answer until Start was pressed.diff --git a/crates/kiri/src/appwatch.rs b/crates/kiri/src/appwatch.rsnew file mode 100644index 0000000..aa9763f--- /dev/null+++ b/crates/kiri/src/appwatch.rs@@ -0,0 +1,79 @@+//! Noticing that Kiri's own program is gone from where it started: the app+//! dragged to the Trash, a development build's folder deleted, or (on+//! Windows, where a running program can't be deleted) its folder renamed+//! or moved. What's left running can't do its job any more (on the Mac the+//! VM's kernel went with the app), yet it still holds what "Build with+//! Kiri" took, buildx's builder for every context, and every `docker+//! build` fails on it. The daemon (daemon/mod.rs) and kiri-win both watch,+//! hand the builder back and stop.+//!+//! The same file serves the Mac and Linux (crates/kiri) and Windows+//! (kiri-win includes it).++use std::path::{Path, PathBuf};+use std::time::Duration;++/// How often the program's path is looked at.+pub const EVERY: Duration = Duration::from_secs(30);++/// Calls `on_gone` once, on its own thread, when the program this process+/// started from has been missing from its path on two looks in a row (an+/// update swaps the app in well under that). The path is the one it started+/// with: on a Linux host a package upgrade puts a new file there, which+/// counts as still there.+pub fn watch(on_gone: impl FnOnce(&Path) + Send + 'static) {+ if let Ok(exe) = std::env::current_exe() {+ watch_path(exe, EVERY, on_gone);+ }+}++fn watch_path(path: PathBuf, every: Duration, on_gone: impl FnOnce(&Path) + Send + 'static) {+ let _ = std::thread::Builder::new().name("kiri-app-watch".into()).spawn(move || {+ let mut missing = 0;+ while missing < 2 {+ std::thread::sleep(every);+ missing = if path.exists() { 0 } else { missing + 1 };+ }+ on_gone(&path);+ });+}++#[cfg(test)]+mod tests {+ use std::sync::mpsc;+ use std::time::Duration;++ #[test]+ fn a_program_moved_away_is_noticed_once_and_one_put_back_is_not() {+ let dir = std::env::temp_dir().join(format!("kiri-appwatch-{}", std::process::id()));+ let _ = std::fs::remove_dir_all(&dir);+ std::fs::create_dir_all(dir.join("app")).unwrap();+ let exe = dir.join("app").join("kiri");+ std::fs::write(&exe, b"program").unwrap();+ let (tx, rx) = mpsc::channel();+ super::watch_path(exe.clone(), Duration::from_millis(20), move |p| tx.send(p.to_owned()).unwrap());+ // There all along: nothing.+ assert!(rx.recv_timeout(Duration::from_millis(150)).is_err());+ // Its folder renamed, as Windows allows for a running program.+ std::fs::rename(dir.join("app"), dir.join("moved")).unwrap();+ assert_eq!(rx.recv_timeout(Duration::from_secs(5)).unwrap(), exe);+ std::fs::remove_dir_all(&dir).unwrap();+ }++ #[test]+ fn a_brief_swap_is_not_taken_for_gone() {+ let dir = std::env::temp_dir().join(format!("kiri-appwatch-swap-{}", std::process::id()));+ let _ = std::fs::remove_dir_all(&dir);+ std::fs::create_dir_all(&dir).unwrap();+ let exe = dir.join("kiri");+ std::fs::write(&exe, b"old").unwrap();+ let (tx, rx) = mpsc::channel();+ super::watch_path(exe.clone(), Duration::from_millis(400), move |p| tx.send(p.to_owned()).unwrap());+ // Gone for the first look (at 400 ms), back before the second (800 ms).+ std::fs::remove_file(&exe).unwrap();+ std::thread::sleep(Duration::from_millis(600));+ std::fs::write(&exe, b"new").unwrap();+ assert!(rx.recv_timeout(Duration::from_millis(1500)).is_err());+ std::fs::remove_dir_all(&dir).unwrap();+ }+}diff --git a/crates/kiri/src/buildwith.rs b/crates/kiri/src/buildwith.rsindex d3f3c04..87832d1 100644--- a/crates/kiri/src/buildwith.rs+++ b/crates/kiri/src/buildwith.rs@@ -209,8 +209,9 @@ impl Places {}/// Build with Docker: puts back what `take` kept, where it is still- /// as Kiri left it.- fn give_back(&self) -> io::Result<()> {+ /// as Kiri left it. Also when Kiri's program is gone (appwatch.rs):+ /// the choice itself is kept, so `apply` takes it again later.+ pub fn give_back(&self) -> io::Result<()> {let mut s = self.state();let before = s["before"].take();if before.is_null() {@@ -493,6 +494,27 @@ mod tests {assert!(p.instance("kiri").is_file());}+ #[test]+ fn a_second_kiri_whose_app_is_gone_hands_builds_back_to_the_first() {+ let d = scratch("second");+ let main = places(&d);+ builder(&main);+ main.choose(With::Kiri).unwrap();+ // A development build of Kiri, its own state and builder, takes them.+ let dev = Places { state: d.join(".kiri-layers/build.json"), builder: "kiri-layers-build".into(), context: "kiri-layers".into(), ..places(&d) };+ let i = json!({"Name": "kiri-layers-build", "Driver": "docker-container", "Nodes": [{"Name": "kiri-layers-build0", "Endpoint": "kiri-layers"}], "Dynamic": false});+ write_bytes(&dev.instance("kiri-layers-build"), i.to_string().as_bytes()).unwrap();+ dev.choose(With::Kiri).unwrap();+ assert!(dev.taken() && !main.taken());+ // Its app deleted: the daemon hands back (daemon/mod.rs).+ dev.give_back().unwrap();+ assert!(main.taken(), "builds go to the first Kiri again");+ assert_eq!(dev.saved(), Some(With::Kiri), "the choice stays");+ // Taken up again, it takes them again.+ dev.apply(None).unwrap();+ assert!(dev.taken());+ }+#[test]fn switching_back_restores_exactly() {let d = scratch("restore");diff --git a/crates/kiri/src/daemon/dns.rs b/crates/kiri/src/daemon/dns.rsindex c0fb52f..28dbb6d 100644--- a/crates/kiri/src/daemon/dns.rs+++ b/crates/kiri/src/daemon/dns.rs@@ -3,9 +3,16 @@//! picks the container from the Host header. macOS sends these lookups here//! because of `/etc/resolver/kiri.local`, which `kiri setup` installs once//! (it needs an administrator). Anything else is refused.+//!+//! Also the relay for container lookups the VM's gateway resolver fails+//! (`serve_relay`): the guest agent sends them here over vsock, and they're+//! asked of the Mac's own resolvers.-use std::io;-use std::net::UdpSocket;+use std::io::{self, Read, Write};+use std::net::{IpAddr, SocketAddr, UdpSocket};+use std::path::Path;+use std::sync::atomic::{AtomicBool, Ordering};+use std::time::{Duration, Instant};/// The resolver file's port; unprivileged, so the daemon needs no root.pub const PORT: u16 = 53535;@@ -73,9 +80,165 @@ fn answer(q: &[u8]) -> Option<Vec<u8>> {Some(r)}+/// How long one resolver gets to answer a relayed lookup.+const RELAY_TRY: Duration = Duration::from_millis(1500);+/// All the resolvers together: the agent stops waiting at five seconds.+const RELAY_BUDGET: Duration = Duration::from_millis(3500);++/// One lookup from the guest agent: `u16 len | query` in, `u16 len | reply`+/// out (kiri_proto::DNS_RELAY_PORT). The agent sends here only what the+/// VM's gateway resolver refused or left unanswered, so this is the path+/// while macOS's resolver for the VM network is down.+pub fn serve_relay(stream: kiri_vmm::VsockStream) {+ let _ = std::thread::Builder::new().name("kiri-dns-relay".into()).spawn(move || {+ let Ok(fd) = stream.fd.try_clone() else { return };+ let mut conn = std::fs::File::from(fd);+ let mut len = [0u8; 2];+ if conn.read_exact(&mut len).is_err() {+ return;+ }+ let mut query = vec![0u8; u16::from_be_bytes(len) as usize];+ if query.len() < 12 || conn.read_exact(&mut query).is_err() {+ return;+ }+ static SAID: AtomicBool = AtomicBool::new(false);+ if !SAID.swap(true, Ordering::Relaxed) {+ crate::diag::warn!("the VM's DNS at its gateway did not answer; container lookups go through the Mac");+ }+ let name = question_name(&query).unwrap_or_default();+ let servers = mac_resolvers(&name, Path::new("/etc/resolv.conf"), Path::new("/etc/resolver"));+ let reply = relay_query(&query, &servers).unwrap_or_else(|| servfail(&query));+ let mut out = Vec::with_capacity(2 + reply.len());+ out.extend_from_slice(&(reply.len().min(u16::MAX as usize) as u16).to_be_bytes());+ out.extend_from_slice(&reply[..reply.len().min(u16::MAX as usize)]);+ let _ = conn.write_all(&out);+ drop(stream);+ });+}++/// The first answer to `query` from `servers`, tried in order.+fn relay_query(query: &[u8], servers: &[SocketAddr]) -> Option<Vec<u8>> {+ let start = Instant::now();+ let mut buf = vec![0u8; 65_535];+ for server in servers {+ let left = RELAY_BUDGET.checked_sub(start.elapsed())?;+ let bind: SocketAddr = if server.is_ipv4() { ([0, 0, 0, 0], 0).into() } else { (std::net::Ipv6Addr::UNSPECIFIED, 0).into() };+ let Ok(sock) = UdpSocket::bind(bind) else { continue };+ if sock.connect(server).is_err() || sock.send(query).is_err() {+ continue;+ }+ let until = Instant::now() + RELAY_TRY.min(left);+ // Read past stray datagrams until this query's answer or the deadline.+ while let Some(wait) = until.checked_duration_since(Instant::now()).filter(|d| !d.is_zero()) {+ if sock.set_read_timeout(Some(wait)).is_err() {+ break;+ }+ let Ok(n) = sock.recv(&mut buf) else { break };+ let r = &buf[..n];+ if n >= 12 && r[..2] == query[..2] && r[2] & 0x80 != 0 {+ return Some(r.to_vec());+ }+ }+ }+ None+}++/// SERVFAIL for `query`: no resolver on the Mac answered either.+fn servfail(query: &[u8]) -> Vec<u8> {+ let end = question_end(query).unwrap_or(12).min(query.len());+ let mut r = query[..end].to_vec();+ r[2] = 0x80 | (query[2] & 0x79); // QR, opcode and RD kept+ r[3] = 0x80 | 2; // RA, SERVFAIL+ r[6..12].fill(0);+ if end > 12 {+ r[5] = 1;+ } else {+ r[4..6].fill(0);+ }+ r+}++/// Offset just past the first question, if there is one.+fn question_end(q: &[u8]) -> Option<usize> {+ if q.len() < 12 || u16::from_be_bytes([q[4], q[5]]) == 0 {+ return None;+ }+ let mut i = 12;+ loop {+ let len = *q.get(i)? as usize;+ if len == 0 {+ break;+ }+ if len > 63 {+ return None;+ }+ i += 1 + len;+ }+ (i + 5 <= q.len()).then_some(i + 5)+}++/// The first question's name, lowercase, without the trailing dot.+fn question_name(q: &[u8]) -> Option<String> {+ question_end(q)?;+ let mut i = 12;+ let mut name = String::new();+ loop {+ let len = q[i] as usize;+ if len == 0 {+ return Some(name);+ }+ if !name.is_empty() {+ name.push('.');+ }+ name.push_str(&String::from_utf8_lossy(&q[i + 1..i + 1 + len]).to_ascii_lowercase());+ i += 1 + len;+ }+}++/// Where the Mac would send a lookup for `name`: the servers of the+/// `/etc/resolver/<domain>` file with the longest domain that `name` is in+/// (VPNs and tools such as dnsmasq set these up), else `/etc/resolv.conf`'s,+/// which macOS keeps to its current primary DNS servers. Read on every+/// lookup, so network changes are followed. Kiri's own `kiri.local` file is+/// skipped: its server answers 127.0.0.1, the Mac's loopback, which means+/// nothing in a container.+fn mac_resolvers(name: &str, resolv_conf: &Path, resolver_dir: &Path) -> Vec<SocketAddr> {+ let mut best: Option<(usize, Vec<SocketAddr>)> = None;+ for entry in std::fs::read_dir(resolver_dir).into_iter().flatten().flatten() {+ let Ok(text) = std::fs::read_to_string(entry.path()) else { continue };+ let file = entry.file_name().to_string_lossy().to_ascii_lowercase();+ let domain = text+ .lines()+ .find_map(|l| l.trim().strip_prefix("domain").map(|d| d.trim().trim_end_matches('.').to_ascii_lowercase()))+ .filter(|d| !d.is_empty())+ .unwrap_or(file);+ if domain == SUFFIX {+ continue;+ }+ let inside = name == domain || name.ends_with(&format!(".{domain}"));+ if !inside || best.as_ref().is_some_and(|(len, _)| *len >= domain.len()) {+ continue;+ }+ let servers = parse_servers(&text);+ if !servers.is_empty() {+ best = Some((domain.len(), servers));+ }+ }+ best.map(|(_, s)| s).unwrap_or_else(|| parse_servers(&std::fs::read_to_string(resolv_conf).unwrap_or_default()))+}++/// `nameserver` lines (and a `port` line) of a resolv.conf-style file.+/// Link-local IPv6 servers, which need an interface named, are skipped.+fn parse_servers(text: &str) -> Vec<SocketAddr> {+ let field = |key: &'static str| text.lines().filter_map(move |l| l.trim().strip_prefix(key).filter(|v| v.starts_with([' ', '\t'])).map(str::trim));+ let port = field("port").find_map(|p| p.parse::<u16>().ok()).unwrap_or(53);+ field("nameserver").filter_map(|ip| ip.parse::<IpAddr>().ok()).map(|ip| SocketAddr::new(ip, port)).collect()+}+#[cfg(test)]mod tests {use super::answer;+ use std::net::UdpSocket;fn query(name: &str, qtype: u16) -> Vec<u8> {let mut q = vec![0x12, 0x34, 0x01, 0x00, 0, 1, 0, 0, 0, 0, 0, 0];@@ -100,4 +263,54 @@ mod tests {assert_eq!(answer(&query("notkiri.local", 1)).unwrap()[3], 5);assert!(answer(&[0; 5]).is_none());}++ #[test]+ fn relayed_lookups_go_where_the_mac_would_send_them() {+ let dir = std::env::temp_dir().join(format!("kiri-dns-test-{}", std::process::id()));+ let resolvers = dir.join("resolver");+ std::fs::create_dir_all(&resolvers).unwrap();+ let conf = dir.join("resolv.conf");+ std::fs::write(&conf, "# macOS\nsearch home\nnameserver 192.168.1.1\nnameserver fe80::1%en0\nnameserver 2001:db8::53\n").unwrap();+ std::fs::write(resolvers.join("corp.example"), "nameserver 10.0.0.53\n").unwrap();+ std::fs::write(resolvers.join("eu.corp.example"), "nameserver 10.1.0.53\nport 5353\n").unwrap();+ std::fs::write(resolvers.join("kiri.local"), "nameserver 127.0.0.1\nport 53535\n").unwrap();+ let r = |name: &str| super::mac_resolvers(name, &conf, &resolvers).iter().map(ToString::to_string).collect::<Vec<_>>();+ assert_eq!(r("registry-1.docker.io"), ["192.168.1.1:53", "[2001:db8::53]:53"]);+ assert_eq!(r("git.corp.example"), ["10.0.0.53:53"]);+ assert_eq!(r("corp.example"), ["10.0.0.53:53"]);+ assert_eq!(r("db.eu.corp.example"), ["10.1.0.53:5353"], "the longest domain wins");+ assert_eq!(r("notcorp.example"), ["192.168.1.1:53", "[2001:db8::53]:53"]);+ assert_eq!(r("web.kiri.local"), ["192.168.1.1:53", "[2001:db8::53]:53"], "Kiri's own resolver is the Mac's loopback");+ std::fs::remove_dir_all(&dir).unwrap();+ }++ #[test]+ fn a_relayed_lookup_takes_the_first_resolver_that_answers() {+ let up = UdpSocket::bind("127.0.0.1:0").unwrap();+ let addr = up.local_addr().unwrap();+ let resolver = std::thread::spawn(move || {+ let mut buf = [0u8; 512];+ let (n, from) = up.recv_from(&mut buf).unwrap();+ // A stray datagram first, then the answer.+ up.send_to(&[9; 12], from).unwrap();+ buf[2] |= 0x80;+ up.send_to(&buf[..n], from).unwrap();+ });+ // A closed port first: refused, then the next one is asked.+ let closed = UdpSocket::bind("127.0.0.1:0").unwrap().local_addr().unwrap();+ let q = query("registry-1.docker.io", 1);+ let r = super::relay_query(&q, &[closed, addr]).expect("answered");+ resolver.join().unwrap();+ assert_eq!((&r[..2], r[2] & 0x80, &r[12..]), (&q[..2], 0x80, &q[12..]));+ assert_eq!(super::question_name(&q).as_deref(), Some("registry-1.docker.io"));+ }++ #[test]+ fn servfail_keeps_the_id_and_question() {+ let q = query("example.com", 1);+ let r = super::servfail(&q);+ assert_eq!((&r[..2], r[2] & 0x80, r[3] & 0x0f), (&q[..2], 0x80, 2));+ assert_eq!((&r[4..12], &r[12..]), (&[0, 1, 0, 0, 0, 0, 0, 0][..], &q[12..]));+ assert_eq!(super::servfail(&[0x12, 0x34, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])[3], 0x82);+ }}diff --git a/crates/kiri/src/daemon/engine.rs b/crates/kiri/src/daemon/engine.rsindex 77dd8de..d01554a 100644--- a/crates/kiri/src/daemon/engine.rs+++ b/crates/kiri/src/daemon/engine.rs@@ -668,6 +668,11 @@ impl Engine {if cfg!(target_os = "macos") {vm.listen(super::ports::ENGINE_API_PORT, Box::new(super::ports::serve_engine_api))?;}+ // Container lookups the VM's gateway resolver fails, asked of the+ // Mac's own (VM only: a Linux host's agent asks the host's itself).+ if cfg!(target_os = "macos") {+ vm.listen(kiri_proto::DNS_RELAY_PORT, Box::new(super::dns::serve_relay))?;+ }// Everything else on the guest's disk belongs to containers that are gone.let keep: Vec<String> = self.lock().containers.keys().cloned().collect();vm.retain(&keep)?;diff --git a/crates/kiri/src/daemon/mod.rs b/crates/kiri/src/daemon/mod.rsindex ac3daa4..7d3aea6 100644--- a/crates/kiri/src/daemon/mod.rs+++ b/crates/kiri/src/daemon/mod.rs@@ -151,6 +151,8 @@ pub fn run(_args: &[String]) -> io::Result<()> {}}+ watch_own_app(engine.clone(), socket.clone());+let on_signal = engine.clone();let sock = socket.clone();std::thread::Builder::new().name("kiri-signals".into()).spawn(move || {@@ -180,6 +182,26 @@ pub fn run(_args: &[String]) -> io::Result<()> {Ok(())}+/// When the app this daemon runs from is deleted (appwatch.rs): docker+/// build handed back (buildwith.rs: what was there before; the choice+/// stays, so the app taken up again takes it again), and the engine stops.+fn watch_own_app(engine: std::sync::Arc<engine::Engine>, socket: std::path::PathBuf) {+ crate::appwatch::watch(move |exe| {+ diag::error!("the app this engine runs from was deleted; stopping and handing docker build back", path = exe.display());+ if let Err(e) = crate::buildwith::places().and_then(|p| p.give_back()) {+ diag::warn!("docker build not handed back", error = e);+ }+ #[cfg(target_os = "macos")]+ nfs::unmount();+ let _ = std::fs::remove_file(&socket);+ engine.shutdown();+ std::thread::sleep(Duration::from_millis(50));+ // 0: launchd restarts the engine on failure only, and there is+ // nothing left to restart.+ std::process::exit(0);+ });+}+/// Serves container files over NFS and mounts them at ~/Kiri./// KIRI_NO_FINDER=1 skips the mount.#[cfg(target_os = "macos")]diff --git a/crates/kiri/src/guest.rs b/crates/kiri/src/guest.rsindex 0244021..a60582f 100644--- a/crates/kiri/src/guest.rs+++ b/crates/kiri/src/guest.rs@@ -141,6 +141,9 @@ impl Guest {share: opts.data_disk.then(|| (PathBuf::from("/"), "kiri-mac".to_owned())),rosetta_tag: opts.data_disk.then(|| "rosetta".to_owned()),};+ if !cfg.kernel.is_file() {+ return Err(missing_kernel(&cfg.kernel, std::env::current_exe().ok().as_deref()));+ }Ok(cfg)}@@ -404,6 +407,17 @@ fn translated() -> bool {on == 1}+/// The error for a guest kernel that isn't there. Mostly the app was+/// deleted or moved while its engine ran: say that, not a bare path.+#[cfg(target_os = "macos")]+fn missing_kernel(kernel: &std::path::Path, exe: Option<&std::path::Path>) -> io::Error {+ let msg = match exe {+ Some(exe) if !exe.exists() => format!("{} was deleted or moved while it ran: open Kiri again from where it is now", crate::flavor::APP),+ _ => format!("the guest kernel {} is missing: reinstall Kiri", kernel.display()),+ };+ io::Error::new(io::ErrorKind::NotFound, msg)+}+#[cfg(target_os = "macos")]/// Where the kernel and initramfs live: `KIRI_GUEST_DIR`, else/// `Kiri.app/Contents/Resources/guest` when running from the app, else thediff --git a/crates/kiri/src/main.rs b/crates/kiri/src/main.rsindex ca27446..3e896bd 100644--- a/crates/kiri/src/main.rs+++ b/crates/kiri/src/main.rs@@ -12,6 +12,10 @@ mod boot;#[cfg(unix)]#[cfg_attr(not(target_os = "macos"), allow(dead_code))]mod builds;+// The engine stops when its app is deleted under it (kiri-win includes it+// for Windows).+#[cfg(unix)]+mod appwatch;// Which builder plain `docker build` uses: Docker's or Kiri's.#[cfg(unix)]#[cfg_attr(not(target_os = "macos"), allow(dead_code))]