Skip to content

armgit / kiri

Public Imported from GitHub
0

A small, fast container runtime for macOS, Windows and Linux

Rust

Container DNS survives the Mac's VM resolver going down; a deleted Kiri hands docker build back (#11)

Precious Okwude47ac0d91e7
diff --git a/crates/kiri-init/src/linux/agent.rs b/crates/kiri-init/src/linux/agent.rs
index 24ff1cf..1c4d4d6 100644
--- a/crates/kiri-init/src/linux/agent.rs
+++ b/crates/kiri-init/src/linux/agent.rs
@@ -62,7 +62,9 @@ pub fn serve(mut conn: File, paths: &Paths) -> io::Result<()> {
// On a Linux host, SIGTERM or SIGINT stops the agent as Shutdown does.
let term = if host_process { Some(term_signals()?) } else { None };
let mut bridges = Bridges::default();
+ // In the VM, lookups the gateway's resolver fails go through the host.
let mut dns = super::dns::Dns::default();
+ dns.relay = !host_process;
let forward = super::relay::listen()?;
// host.docker.internal's relay to the Mac (VM only). A Linux host maps
// that name to the bridge gateway and redirects nothing here, so the
@@ -130,6 +132,10 @@ pub fn serve(mut conn: File, paths: &Paths) -> io::Result<()> {
let wait = TTY_HUP_WAIT.as_millis() as i32;
timeout = if timeout < 0 { wait } else { timeout.min(wait) };
}
+ let wait = dns.timeout_ms();
+ if wait >= 0 {
+ timeout = if timeout < 0 { wait } else { timeout.min(wait) };
+ }
// SAFETY: `fds` is a valid array of pollfd of the given length.
let n = unsafe { libc::poll(fds.as_mut_ptr(), fds.len() as libc::nfds_t, timeout) };
if n < 0 {
@@ -157,6 +163,8 @@ pub fn serve(mut conn: File, paths: &Paths) -> io::Result<()> {
}
}
+ dns.on_timer();
+
// Output and input first, so a process's last words go out before its exit.
for pfd in &fds[3..] {
if pfd.revents == 0 {
diff --git a/crates/kiri-init/src/linux/container.rs b/crates/kiri-init/src/linux/container.rs
index 6a89529..cc93bd2 100644
--- a/crates/kiri-init/src/linux/container.rs
+++ b/crates/kiri-init/src/linux/container.rs
@@ -257,7 +257,7 @@ pub fn spawn(spec: &RunSpec<'_>, paths: &Paths, net: NetCtx<'_>) -> io::Result<P
prepared.host = HostApplied { rlimits: host.rlimits.clone(), groups: host.groups.clone(), nodes };
// Small, short-lived files: kept in memory (/run is tmpfs), not on the data disk.
let etc_dir = PathBuf::from("/run/kiri").join(spec.id);
- let embedded_dns = embedded_dns(spec, net.upstream_dns);
+ let embedded_dns = embedded_dns(spec, net.upstream_dns, net.dns.relay);
prepared.etc_files = write_etc_files(spec, &etc_dir, net.upstream_dns, embedded_dns).map_err(cleanup)?;
prepared.mounts = parse_mounts(spec.mounts).map_err(cleanup)?;
let bridged = !spec.net.bridge.is_empty() && spec.flags & (run_flags::NET_HOST | run_flags::NET_NONE) == 0 && prepared.net_join.is_none();
@@ -736,16 +736,18 @@ fn write_etc_files(spec: &RunSpec<'_>, base: &std::path::Path, upstream: &[[u8;
/// `--dns-search` and `--dns-option` apply either way.
/// Whether the container's DNS goes to the agent's resolver (127.0.0.11):
/// on a user network, as Docker's embedded DNS; and on the default bridge
-/// when the host's resolver is one of its own addresses (net::local_resolver),
-/// unless the container names its own servers (`--dns`).
-fn embedded_dns(spec: &RunSpec<'_>, upstream: &[[u8; 4]]) -> bool {
+/// when the host's resolver is one of its own addresses (net::local_resolver)
+/// or, in the VM (`relay`), so that lookups the Mac's gateway resolver fails
+/// can go through the host (dns.rs), unless the container names its own
+/// servers (`--dns`).
+fn embedded_dns(spec: &RunSpec<'_>, upstream: &[[u8; 4]], relay: bool) -> bool {
if spec.flags & run_flags::DNS_GATEWAY != 0 {
return true;
}
let bridged = !spec.net.bridge.is_empty()
&& spec.flags & (run_flags::NET_HOST | run_flags::NET_NONE) == 0
&& net_opts::values(spec.net.opts, net_opts::NETNS).next().is_none();
- bridged && net_opts::values(spec.net.opts, net_opts::NAMESERVER).next().is_none() && net::local_resolver(upstream)
+ bridged && net_opts::values(spec.net.opts, net_opts::NAMESERVER).next().is_none() && (relay || net::local_resolver(upstream))
}
fn resolv_conf(spec: &RunSpec<'_>, upstream: &[[u8; 4]], embedded_dns: bool) -> String {
@@ -1380,8 +1382,23 @@ mod tests {
encode_run(&mut buf, "c1", &[], &["sh"], &[], "/", "", "c1", 0, net, &[], &Limits::default(), 0, &[] as &[&str]);
let Ok(Msg::Run(spec)) = decode(&buf) else { panic!() };
// TEST-NET-3: no machine running the tests has it.
- assert!(!super::embedded_dns(&spec, &[[203, 0, 113, 53]]), "a resolver elsewhere is asked directly");
- assert!(!super::embedded_dns(&spec, &[]), "no resolver: the public fallback, not the agent");
+ assert!(!super::embedded_dns(&spec, &[[203, 0, 113, 53]], false), "a resolver elsewhere is asked directly");
+ assert!(!super::embedded_dns(&spec, &[], false), "no resolver: the public fallback, not the agent");
+ }
+
+ #[test]
+ fn in_the_vm_the_default_bridge_goes_through_the_agent() {
+ let spec_with = |flags: u32, opts: &[&str], f: &dyn Fn(&kiri_proto::RunSpec<'_>)| {
+ let mut buf = Vec::new();
+ let net = Net { bridge: "kiri0", ip: [172, 17, 0, 2], prefix: 16, gateway: [172, 17, 0, 1], extra_hosts: &[], dns_names: &[], opts };
+ encode_run(&mut buf, "c1", &[], &["sh"], &[], "/", "", "c1", flags, net, &[], &Limits::default(), 0, &[] as &[&str]);
+ let Ok(Msg::Run(spec)) = decode(&buf) else { panic!() };
+ f(&spec);
+ };
+ let gateway = [[192, 168, 64, 1]];
+ spec_with(0, &[], &|s| assert!(super::embedded_dns(s, &gateway, true), "so a refused gateway can fall back to the host"));
+ spec_with(0, &["nameserver=8.8.8.8"], &|s| assert!(!super::embedded_dns(s, &gateway, true), "--dns servers are asked directly"));
+ spec_with(run_flags::NET_HOST, &[], &|s| assert!(!super::embedded_dns(s, &gateway, true), "host networking keeps the VM's resolv.conf"));
}
#[test]
diff --git a/crates/kiri-init/src/linux/dns.rs b/crates/kiri-init/src/linux/dns.rs
index 51ed059..ddf507a 100644
--- a/crates/kiri-init/src/linux/dns.rs
+++ b/crates/kiri-init/src/linux/dns.rs
@@ -6,6 +6,15 @@
//! forwarded to the VM's upstream resolver (or the container's own `--dns`
//! servers) and the answer relayed back. The 127.0.0.11 socket is opened in
//! the container's namespace by this agent, so no process runs in there.
+//!
+//! In the VM the upstream is the Mac's resolver for the NAT network, at its
+//! gateway (192.168.64.1). macOS doesn't always answer there: after a
+//! network change, sleep or a VPN, lookups are refused or go unanswered
+//! while the Mac itself resolves fine. So a lookup the gateway refuses, or
+//! leaves unanswered for `FALLBACK_AFTER`, is asked again through the Kiri
+//! daemon on the Mac, over vsock, which asks the Mac's own resolvers
+//! (crates/kiri/src/daemon/dns.rs). A container's own `--dns` servers are
+//! never second-guessed.
use std::collections::HashMap;
use std::io::{self, Read};
@@ -24,6 +33,12 @@ const FORWARD_TIMEOUT: Duration = Duration::from_secs(5);
/// At most this many forwarded queries in flight (each has its own socket);
/// more are dropped, and the client asks again.
const MAX_PENDING: usize = 256;
+/// A lookup the VM's resolver hasn't answered within this goes through the
+/// host (VM only). Gateway answers take milliseconds; a slow recursive one
+/// that loses the race is simply replaced by the host's.
+const FALLBACK_AFTER: Duration = Duration::from_secs(1);
+/// Largest reply the host relay may send (a DNS over TCP frame's limit).
+const MAX_RELAYED: usize = 2 + 65_535;
/// Docker's embedded DNS address, inside each container.
pub const EMBEDDED: [u8; 4] = [127, 0, 0, 11];
@@ -42,7 +57,8 @@ struct Pending {
/// Its own socket, connected to the resolver: a fresh source port for
/// every query, as Docker's embedded DNS does, so one lost or blocked
/// flow doesn't take the resolver away from every later query, and an
- /// answer has to guess the port as well as the ID.
+ /// answer has to guess the port as well as the ID. Once the query goes
+ /// through the host, the vsock stream to it instead.
sock: OwnedFd,
/// The ID it went out with.
our: u16,
@@ -52,6 +68,13 @@ struct Pending {
id: u16,
question: Vec<u8>,
sent: Instant,
+ /// The query as sent (with `our` ID), to send again through the host.
+ query: Vec<u8>,
+ /// It may still go through the host: in the VM, asked of the VM's own
+ /// resolver (not `--dns` servers), and not tried there yet.
+ may_relay: bool,
+ /// Sent through the host: the reply's bytes so far, `u16 len | reply`.
+ relayed: Option<Vec<u8>>,
}
/// One name's IPv6 addresses, each with its container's IPv4 address.
@@ -68,6 +91,9 @@ pub struct Dns {
/// Container address → its `--dns` servers, used instead of the VM's.
clients: HashMap<[u8; 4], Vec<[u8; 4]>>,
pending: Vec<Pending>,
+ /// Lookups the VM's resolver fails go through the host (in the VM; a
+ /// Linux host's agent asks the host's resolvers itself).
+ pub relay: bool,
}
fn udp_socket() -> io::Result<OwnedFd> {
@@ -220,21 +246,27 @@ impl Dns {
pub fn on_readable(&mut self, fd: RawFd, upstream: &[[u8; 4]]) {
let mut buf = [0u8; 1500];
if let Some(at) = self.pending.iter().position(|p| p.sock.as_raw_fd() == fd) {
+ if self.pending[at].relayed.is_some() {
+ self.read_relayed(at);
+ return;
+ }
// A forwarded answer: hand it back to whoever asked.
loop {
// SAFETY: receiving into a buffer we own.
let n = unsafe { libc::recv(fd, buf.as_mut_ptr().cast(), buf.len(), 0) };
+ if n < 0 && io::Error::last_os_error().raw_os_error() == Some(libc::ECONNREFUSED) {
+ // Nothing listens at the resolver: ask the host now
+ // rather than wait for the client to give up.
+ self.relay(at);
+ break;
+ }
if n < 12 {
break;
}
let msg = &mut buf[..n as usize];
let p = &self.pending[at];
if u16::from_be_bytes([msg[0], msg[1]]) == p.our && response_matches(msg, p) {
- let p = self.pending.swap_remove(at);
- msg[0..2].copy_from_slice(&p.id.to_be_bytes());
- if let Some(s) = self.servers.iter().find(|s| s.sock.as_raw_fd() == p.server) {
- send_to(s.sock.as_raw_fd(), msg, &p.client);
- }
+ self.answered(at, msg);
break;
}
}
@@ -309,31 +341,169 @@ impl Dns {
let now = Instant::now();
self.pending.retain(|p| now.duration_since(p.sent) < FORWARD_TIMEOUT);
let key = self.servers[server].container.map_or(client.sin_addr.s_addr.to_ne_bytes(), |(_, owner)| owner);
- let own = self.clients.get(&key).and_then(|v| v.first());
- let Some(dest) = own.or(upstream.first()).copied() else { return };
+ let own = self.clients.get(&key).and_then(|v| v.first()).copied();
if self.pending.len() >= MAX_PENDING {
return;
}
let Some((_, _, end)) = parse_question(q) else {
return;
};
- let Ok(sock) = udp_socket() else { return };
- let to = sockaddr(dest, 53);
- // Connected, so it only receives datagrams from this resolver.
- // SAFETY: connecting to a valid sockaddr_in.
- if unsafe { libc::connect(sock.as_raw_fd(), (&to as *const libc::sockaddr_in).cast(), std::mem::size_of::<libc::sockaddr_in>() as u32) } != 0 {
- return;
- }
let Some(our) = random_id() else { return };
let mut msg = q.to_vec();
let id = u16::from_be_bytes([msg[0], msg[1]]);
msg[0..2].copy_from_slice(&our.to_be_bytes());
- // SAFETY: sending a buffer we own on a connected socket.
- if unsafe { libc::send(sock.as_raw_fd(), msg.as_ptr().cast(), msg.len(), 0) } < 0 {
+ let may_relay = self.relay && own.is_none();
+ let dest = own.or(upstream.first().copied());
+ // No resolver yet, or it can't be reached: the host, if allowed.
+ let (sock, relayed) = match dest.and_then(|d| ask(d, &msg)) {
+ Some(s) => (s, false),
+ None if may_relay => match relay_stream(&msg) {
+ Some(s) => (s, true),
+ None => return,
+ },
+ None => return,
+ };
+ let p = Pending {
+ sock,
+ our,
+ server: self.servers[server].sock.as_raw_fd(),
+ client,
+ id,
+ question: q[12..end].to_vec(),
+ sent: now,
+ query: msg,
+ may_relay: may_relay && !relayed,
+ relayed: relayed.then(Vec::new),
+ };
+ self.pending.push(p);
+ }
+
+ /// Hands a matching answer to whoever asked, and forgets the query.
+ fn answered(&mut self, at: usize, msg: &mut [u8]) {
+ let p = self.pending.swap_remove(at);
+ msg[0..2].copy_from_slice(&p.id.to_be_bytes());
+ if let Some(s) = self.servers.iter().find(|s| s.sock.as_raw_fd() == p.server) {
+ send_to(s.sock.as_raw_fd(), msg, &p.client);
+ }
+ }
+
+ /// Sends pending query `at` again, through the host, once.
+ fn relay(&mut self, at: usize) {
+ let p = &mut self.pending[at];
+ if !p.may_relay {
return;
}
- self.pending.push(Pending { sock, our, server: self.servers[server].sock.as_raw_fd(), client, id, question: q[12..end].to_vec(), sent: now });
+ p.may_relay = false;
+ if let Some(stream) = relay_stream(&p.query) {
+ p.sock = stream;
+ p.relayed = Some(Vec::new());
+ p.sent = Instant::now();
+ }
}
+
+ /// Reads the host's reply to query `at` as it arrives.
+ fn read_relayed(&mut self, at: usize) {
+ let fd = self.pending[at].sock.as_raw_fd();
+ let mut buf = [0u8; 4096];
+ let mut eof = false;
+ loop {
+ // SAFETY: reading into a buffer we own.
+ let n = unsafe { libc::read(fd, buf.as_mut_ptr().cast(), buf.len()) };
+ if n > 0 {
+ let got = self.pending[at].relayed.get_or_insert_with(Vec::new);
+ got.extend_from_slice(&buf[..n as usize]);
+ if got.len() > MAX_RELAYED {
+ eof = true;
+ break;
+ }
+ continue;
+ }
+ if n < 0 && io::Error::last_os_error().kind() == io::ErrorKind::WouldBlock {
+ break;
+ }
+ eof = true;
+ break;
+ }
+ let got = self.pending[at].relayed.take().unwrap_or_default();
+ match framed_reply(&got) {
+ Some(reply) => {
+ let mut msg = reply.to_vec();
+ let p = &self.pending[at];
+ if msg.len() >= 12 && u16::from_be_bytes([msg[0], msg[1]]) == p.our && response_matches(&msg, p) {
+ self.answered(at, &mut msg);
+ } else {
+ self.pending.swap_remove(at);
+ }
+ }
+ // Gone before a whole reply: the client asks again.
+ None if eof => {
+ self.pending.swap_remove(at);
+ }
+ None => self.pending[at].relayed = Some(got),
+ }
+ }
+
+ /// Sends lookups the VM's resolver has left unanswered too long
+ /// through the host. Called on every turn of the agent's loop.
+ pub fn on_timer(&mut self) {
+ let now = Instant::now();
+ for at in 0..self.pending.len() {
+ let p = &self.pending[at];
+ if p.may_relay && now.duration_since(p.sent) >= FALLBACK_AFTER {
+ self.relay(at);
+ }
+ }
+ }
+
+ /// Milliseconds until `on_timer` has something to do; -1 for never.
+ pub fn timeout_ms(&self) -> i32 {
+ let now = Instant::now();
+ self.pending
+ .iter()
+ .filter(|p| p.may_relay)
+ .map(|p| FALLBACK_AFTER.saturating_sub(now.duration_since(p.sent)).as_millis().min(i32::MAX as u128) as i32)
+ .min()
+ .unwrap_or(-1)
+ }
+}
+
+/// A UDP socket connected to `dest`:53 with `msg` sent on it, or None if
+/// it couldn't be sent (no route, the network down).
+fn ask(dest: [u8; 4], msg: &[u8]) -> Option<OwnedFd> {
+ let sock = udp_socket().ok()?;
+ let to = sockaddr(dest, 53);
+ // Connected, so it only receives datagrams from this resolver.
+ // SAFETY: connecting to a valid sockaddr_in.
+ if unsafe { libc::connect(sock.as_raw_fd(), (&to as *const libc::sockaddr_in).cast(), std::mem::size_of::<libc::sockaddr_in>() as u32) } != 0 {
+ return None;
+ }
+ // SAFETY: sending a buffer we own on a connected socket.
+ if unsafe { libc::send(sock.as_raw_fd(), msg.as_ptr().cast(), msg.len(), 0) } < 0 {
+ return None;
+ }
+ Some(sock)
+}
+
+/// A stream to the host's DNS relay with `msg` sent on it, non-blocking
+/// for the reply.
+fn relay_stream(msg: &[u8]) -> Option<OwnedFd> {
+ let stream = super::transport::dial(kiri_proto::DNS_RELAY_PORT).ok()?;
+ let mut frame = Vec::with_capacity(2 + msg.len());
+ frame.extend_from_slice(&u16::try_from(msg.len()).ok()?.to_be_bytes());
+ frame.extend_from_slice(msg);
+ // A few hundred bytes on a fresh stream: one write, never short.
+ // SAFETY: writing a buffer we own to a socket we own.
+ if unsafe { libc::write(stream.as_raw_fd(), frame.as_ptr().cast(), frame.len()) } != frame.len() as isize {
+ return None;
+ }
+ sys::set_nonblocking(stream.as_raw_fd()).ok()?;
+ Some(stream)
+}
+
+/// The reply in `u16 len | reply`, once all of it is there.
+fn framed_reply(got: &[u8]) -> Option<&[u8]> {
+ let len = u16::from_be_bytes([*got.first()?, *got.get(1)?]) as usize;
+ got.get(2..2 + len)
}
/// A UDP socket bound to `ip`:53.
@@ -560,9 +730,84 @@ mod tests {
id: 7,
question: b"\x03www\x07example\x03com\x00\x00\x01\x00\x01".to_vec(),
sent: Instant::now(),
+ query: Vec::new(),
+ may_relay: false,
+ relayed: None,
}
}
+ #[test]
+ fn a_relayed_reply_is_whole_only_once_its_length_is_there() {
+ use super::framed_reply;
+ assert_eq!(framed_reply(&[]), None);
+ assert_eq!(framed_reply(&[0]), None);
+ assert_eq!(framed_reply(&[0, 3, 1, 2]), None);
+ assert_eq!(framed_reply(&[0, 3, 1, 2, 3]), Some(&[1u8, 2, 3][..]));
+ assert_eq!(framed_reply(&[0, 0]), Some(&[][..]));
+ }
+
+ #[test]
+ fn only_lookups_the_vm_resolver_may_miss_wait_for_the_host() {
+ let mut dns = Dns::default();
+ assert_eq!(dns.timeout_ms(), -1, "nothing pending");
+ dns.pending.push(pending());
+ assert_eq!(dns.timeout_ms(), -1, "--dns servers are not second-guessed");
+ let mut p = pending();
+ p.may_relay = true;
+ dns.pending.push(p);
+ let t = dns.timeout_ms();
+ assert!((0..=1000).contains(&t), "{t}");
+ // Past the wait it is tried through the host once, whatever comes of it.
+ dns.pending[1].sent = Instant::now() - super::FALLBACK_AFTER;
+ assert_eq!(dns.timeout_ms(), 0);
+ dns.on_timer();
+ assert!(!dns.pending[1].may_relay);
+ assert_eq!(dns.timeout_ms(), -1);
+ }
+
+ #[test]
+ fn the_hosts_reply_goes_back_to_whoever_asked() {
+ use std::net::UdpSocket;
+ use std::os::fd::{AsRawFd, FromRawFd, OwnedFd};
+ let client = UdpSocket::bind("127.0.0.1:0").unwrap();
+ client.set_read_timeout(Some(std::time::Duration::from_secs(2))).unwrap();
+ let std::net::SocketAddr::V4(addr) = client.local_addr().unwrap() else { panic!() };
+ let mut dns = Dns::default();
+ dns.servers.push(Server { bridges: vec!["kiri1".into()], sock: udp_socket().unwrap(), container: None });
+ let mut fds = [0; 2];
+ // SAFETY: `fds` has room for the pair.
+ assert_eq!(unsafe { libc::socketpair(libc::AF_UNIX, libc::SOCK_STREAM | libc::SOCK_NONBLOCK, 0, fds.as_mut_ptr()) }, 0);
+ // SAFETY: fresh descriptors from socketpair.
+ let (ours, host) = unsafe { (OwnedFd::from_raw_fd(fds[0]), OwnedFd::from_raw_fd(fds[1])) };
+ let mut p = pending();
+ p.sock = ours;
+ p.server = dns.servers[0].sock.as_raw_fd();
+ p.client = super::sockaddr(addr.ip().octets(), addr.port());
+ p.relayed = Some(Vec::new());
+ let question = p.question.clone();
+ dns.pending.push(p);
+ let mut reply = vec![0, 1, 0x81, 0x80, 0, 1, 0, 0, 0, 0, 0, 0];
+ reply.extend_from_slice(&question);
+ let mut frame = (reply.len() as u16).to_be_bytes().to_vec();
+ frame.extend_from_slice(&reply);
+ let send = |b: &[u8]| {
+ // SAFETY: writing a buffer we own to our end of the pair.
+ assert_eq!(unsafe { libc::write(host.as_raw_fd(), b.as_ptr().cast(), b.len()) }, b.len() as isize)
+ };
+ // Half a reply waits for the rest.
+ send(&frame[..5]);
+ let fd = dns.pending[0].sock.as_raw_fd();
+ dns.on_readable(fd, &[]);
+ assert_eq!(dns.pending.len(), 1);
+ send(&frame[5..]);
+ dns.on_readable(fd, &[]);
+ assert!(dns.pending.is_empty());
+ let mut buf = [0u8; 512];
+ let n = client.recv(&mut buf).unwrap();
+ assert_eq!(&buf[..2], &[0, 7], "the client's own ID");
+ assert_eq!(&buf[2..n], &reply[2..]);
+ }
+
#[test]
fn response_must_match_the_whole_question() {
let p = pending();
diff --git a/crates/kiri-proto/src/lib.rs b/crates/kiri-proto/src/lib.rs
index 58ae559..25e2742 100644
--- a/crates/kiri-proto/src/lib.rs
+++ b/crates/kiri-proto/src/lib.rs
@@ -167,6 +167,11 @@ pub enum Msg<'a> {
/// vsock port of the guest's copy service: one connection per copy.
pub const COPY_PORT: u32 = 1028;
+/// vsock port of the host's DNS relay (VM only). The agent sends a lookup
+/// the VM's gateway resolver didn't answer as `u16 len | query`, as DNS over
+/// TCP frames it, and gets `u16 len | reply` back: one lookup per connection.
+pub const DNS_RELAY_PORT: u32 = 1037;
+
pub mod copy_op {
pub const IN: u8 = 1;
pub const OUT: u8 = 2;
diff --git a/crates/kiri-win/src/main.rs b/crates/kiri-win/src/main.rs
index c4bfcdb..b78100a 100644
--- a/crates/kiri-win/src/main.rs
+++ b/crates/kiri-win/src/main.rs
@@ -46,6 +46,11 @@ mod builds;
#[cfg(windows)]
#[path = "../../kiri/src/buildwith.rs"]
mod buildwith;
+// Kiri's program moved or deleted from where it started: the Mac's and
+// Linux's own crates/kiri/src/appwatch.rs.
+#[cfg(windows)]
+#[path = "../../kiri/src/appwatch.rs"]
+mod appwatch;
// An image's Layers and Security tabs, asked of the engine in WSL.
#[cfg(windows)]
mod scan;
@@ -139,6 +144,22 @@ fn main() -> std::process::ExitCode {
serve::log(&format!("dns on 127.0.0.1:53: {e}"));
}
engine::start(engine::Env { distro: cfg.distro.clone(), pipe: cfg.pipe.clone() });
+ // Its folder renamed, moved or deleted around it (Windows won't delete
+ // a running program itself): docker build handed back as it was before
+ // Kiri took it, the choice kept for when Kiri runs again, and Kiri quits.
+ appwatch::watch(|exe| {
+ serve::log(&format!("{} is gone from where Kiri started; handing docker build back and quitting", exe.display()));
+ if let Err(e) = buildwith::places().and_then(|p| p.give_back()) {
+ serve::log(&format!("docker build not handed back: {e}"));
+ }
+ app::Handle.run(|_| {
+ // SAFETY: ends the window's message loop; run on its thread.
+ unsafe { windows::Win32::UI::WindowsAndMessaging::PostQuitMessage(0) }
+ });
+ // No window to quit (it never opened): quit from here.
+ std::thread::sleep(std::time::Duration::from_secs(5));
+ std::process::exit(0);
+ });
// The engine starts when the page says `ready`, and with the window
// hidden (--background, as from the login shortcut) no page loads: after
// kiri-win restarted, docker got no answer until Start was pressed.
diff --git a/crates/kiri/src/appwatch.rs b/crates/kiri/src/appwatch.rs
new file mode 100644
index 0000000..aa9763f
--- /dev/null
+++ b/crates/kiri/src/appwatch.rs
@@ -0,0 +1,79 @@
+//! Noticing that Kiri's own program is gone from where it started: the app
+//! dragged to the Trash, a development build's folder deleted, or (on
+//! Windows, where a running program can't be deleted) its folder renamed
+//! or moved. What's left running can't do its job any more (on the Mac the
+//! VM's kernel went with the app), yet it still holds what "Build with
+//! Kiri" took, buildx's builder for every context, and every `docker
+//! build` fails on it. The daemon (daemon/mod.rs) and kiri-win both watch,
+//! hand the builder back and stop.
+//!
+//! The same file serves the Mac and Linux (crates/kiri) and Windows
+//! (kiri-win includes it).
+
+use std::path::{Path, PathBuf};
+use std::time::Duration;
+
+/// How often the program's path is looked at.
+pub const EVERY: Duration = Duration::from_secs(30);
+
+/// Calls `on_gone` once, on its own thread, when the program this process
+/// started from has been missing from its path on two looks in a row (an
+/// update swaps the app in well under that). The path is the one it started
+/// with: on a Linux host a package upgrade puts a new file there, which
+/// counts as still there.
+pub fn watch(on_gone: impl FnOnce(&Path) + Send + 'static) {
+ if let Ok(exe) = std::env::current_exe() {
+ watch_path(exe, EVERY, on_gone);
+ }
+}
+
+fn watch_path(path: PathBuf, every: Duration, on_gone: impl FnOnce(&Path) + Send + 'static) {
+ let _ = std::thread::Builder::new().name("kiri-app-watch".into()).spawn(move || {
+ let mut missing = 0;
+ while missing < 2 {
+ std::thread::sleep(every);
+ missing = if path.exists() { 0 } else { missing + 1 };
+ }
+ on_gone(&path);
+ });
+}
+
+#[cfg(test)]
+mod tests {
+ use std::sync::mpsc;
+ use std::time::Duration;
+
+ #[test]
+ fn a_program_moved_away_is_noticed_once_and_one_put_back_is_not() {
+ let dir = std::env::temp_dir().join(format!("kiri-appwatch-{}", std::process::id()));
+ let _ = std::fs::remove_dir_all(&dir);
+ std::fs::create_dir_all(dir.join("app")).unwrap();
+ let exe = dir.join("app").join("kiri");
+ std::fs::write(&exe, b"program").unwrap();
+ let (tx, rx) = mpsc::channel();
+ super::watch_path(exe.clone(), Duration::from_millis(20), move |p| tx.send(p.to_owned()).unwrap());
+ // There all along: nothing.
+ assert!(rx.recv_timeout(Duration::from_millis(150)).is_err());
+ // Its folder renamed, as Windows allows for a running program.
+ std::fs::rename(dir.join("app"), dir.join("moved")).unwrap();
+ assert_eq!(rx.recv_timeout(Duration::from_secs(5)).unwrap(), exe);
+ std::fs::remove_dir_all(&dir).unwrap();
+ }
+
+ #[test]
+ fn a_brief_swap_is_not_taken_for_gone() {
+ let dir = std::env::temp_dir().join(format!("kiri-appwatch-swap-{}", std::process::id()));
+ let _ = std::fs::remove_dir_all(&dir);
+ std::fs::create_dir_all(&dir).unwrap();
+ let exe = dir.join("kiri");
+ std::fs::write(&exe, b"old").unwrap();
+ let (tx, rx) = mpsc::channel();
+ super::watch_path(exe.clone(), Duration::from_millis(400), move |p| tx.send(p.to_owned()).unwrap());
+ // Gone for the first look (at 400 ms), back before the second (800 ms).
+ std::fs::remove_file(&exe).unwrap();
+ std::thread::sleep(Duration::from_millis(600));
+ std::fs::write(&exe, b"new").unwrap();
+ assert!(rx.recv_timeout(Duration::from_millis(1500)).is_err());
+ std::fs::remove_dir_all(&dir).unwrap();
+ }
+}
diff --git a/crates/kiri/src/buildwith.rs b/crates/kiri/src/buildwith.rs
index d3f3c04..87832d1 100644
--- a/crates/kiri/src/buildwith.rs
+++ b/crates/kiri/src/buildwith.rs
@@ -209,8 +209,9 @@ impl Places {
}
/// Build with Docker: puts back what `take` kept, where it is still
- /// as Kiri left it.
- fn give_back(&self) -> io::Result<()> {
+ /// as Kiri left it. Also when Kiri's program is gone (appwatch.rs):
+ /// the choice itself is kept, so `apply` takes it again later.
+ pub fn give_back(&self) -> io::Result<()> {
let mut s = self.state();
let before = s["before"].take();
if before.is_null() {
@@ -493,6 +494,27 @@ mod tests {
assert!(p.instance("kiri").is_file());
}
+ #[test]
+ fn a_second_kiri_whose_app_is_gone_hands_builds_back_to_the_first() {
+ let d = scratch("second");
+ let main = places(&d);
+ builder(&main);
+ main.choose(With::Kiri).unwrap();
+ // A development build of Kiri, its own state and builder, takes them.
+ let dev = Places { state: d.join(".kiri-layers/build.json"), builder: "kiri-layers-build".into(), context: "kiri-layers".into(), ..places(&d) };
+ let i = json!({"Name": "kiri-layers-build", "Driver": "docker-container", "Nodes": [{"Name": "kiri-layers-build0", "Endpoint": "kiri-layers"}], "Dynamic": false});
+ write_bytes(&dev.instance("kiri-layers-build"), i.to_string().as_bytes()).unwrap();
+ dev.choose(With::Kiri).unwrap();
+ assert!(dev.taken() && !main.taken());
+ // Its app deleted: the daemon hands back (daemon/mod.rs).
+ dev.give_back().unwrap();
+ assert!(main.taken(), "builds go to the first Kiri again");
+ assert_eq!(dev.saved(), Some(With::Kiri), "the choice stays");
+ // Taken up again, it takes them again.
+ dev.apply(None).unwrap();
+ assert!(dev.taken());
+ }
+
#[test]
fn switching_back_restores_exactly() {
let d = scratch("restore");
diff --git a/crates/kiri/src/daemon/dns.rs b/crates/kiri/src/daemon/dns.rs
index c0fb52f..28dbb6d 100644
--- a/crates/kiri/src/daemon/dns.rs
+++ b/crates/kiri/src/daemon/dns.rs
@@ -3,9 +3,16 @@
//! picks the container from the Host header. macOS sends these lookups here
//! because of `/etc/resolver/kiri.local`, which `kiri setup` installs once
//! (it needs an administrator). Anything else is refused.
+//!
+//! Also the relay for container lookups the VM's gateway resolver fails
+//! (`serve_relay`): the guest agent sends them here over vsock, and they're
+//! asked of the Mac's own resolvers.
-use std::io;
-use std::net::UdpSocket;
+use std::io::{self, Read, Write};
+use std::net::{IpAddr, SocketAddr, UdpSocket};
+use std::path::Path;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::time::{Duration, Instant};
/// The resolver file's port; unprivileged, so the daemon needs no root.
pub const PORT: u16 = 53535;
@@ -73,9 +80,165 @@ fn answer(q: &[u8]) -> Option<Vec<u8>> {
Some(r)
}
+/// How long one resolver gets to answer a relayed lookup.
+const RELAY_TRY: Duration = Duration::from_millis(1500);
+/// All the resolvers together: the agent stops waiting at five seconds.
+const RELAY_BUDGET: Duration = Duration::from_millis(3500);
+
+/// One lookup from the guest agent: `u16 len | query` in, `u16 len | reply`
+/// out (kiri_proto::DNS_RELAY_PORT). The agent sends here only what the
+/// VM's gateway resolver refused or left unanswered, so this is the path
+/// while macOS's resolver for the VM network is down.
+pub fn serve_relay(stream: kiri_vmm::VsockStream) {
+ let _ = std::thread::Builder::new().name("kiri-dns-relay".into()).spawn(move || {
+ let Ok(fd) = stream.fd.try_clone() else { return };
+ let mut conn = std::fs::File::from(fd);
+ let mut len = [0u8; 2];
+ if conn.read_exact(&mut len).is_err() {
+ return;
+ }
+ let mut query = vec![0u8; u16::from_be_bytes(len) as usize];
+ if query.len() < 12 || conn.read_exact(&mut query).is_err() {
+ return;
+ }
+ static SAID: AtomicBool = AtomicBool::new(false);
+ if !SAID.swap(true, Ordering::Relaxed) {
+ crate::diag::warn!("the VM's DNS at its gateway did not answer; container lookups go through the Mac");
+ }
+ let name = question_name(&query).unwrap_or_default();
+ let servers = mac_resolvers(&name, Path::new("/etc/resolv.conf"), Path::new("/etc/resolver"));
+ let reply = relay_query(&query, &servers).unwrap_or_else(|| servfail(&query));
+ let mut out = Vec::with_capacity(2 + reply.len());
+ out.extend_from_slice(&(reply.len().min(u16::MAX as usize) as u16).to_be_bytes());
+ out.extend_from_slice(&reply[..reply.len().min(u16::MAX as usize)]);
+ let _ = conn.write_all(&out);
+ drop(stream);
+ });
+}
+
+/// The first answer to `query` from `servers`, tried in order.
+fn relay_query(query: &[u8], servers: &[SocketAddr]) -> Option<Vec<u8>> {
+ let start = Instant::now();
+ let mut buf = vec![0u8; 65_535];
+ for server in servers {
+ let left = RELAY_BUDGET.checked_sub(start.elapsed())?;
+ let bind: SocketAddr = if server.is_ipv4() { ([0, 0, 0, 0], 0).into() } else { (std::net::Ipv6Addr::UNSPECIFIED, 0).into() };
+ let Ok(sock) = UdpSocket::bind(bind) else { continue };
+ if sock.connect(server).is_err() || sock.send(query).is_err() {
+ continue;
+ }
+ let until = Instant::now() + RELAY_TRY.min(left);
+ // Read past stray datagrams until this query's answer or the deadline.
+ while let Some(wait) = until.checked_duration_since(Instant::now()).filter(|d| !d.is_zero()) {
+ if sock.set_read_timeout(Some(wait)).is_err() {
+ break;
+ }
+ let Ok(n) = sock.recv(&mut buf) else { break };
+ let r = &buf[..n];
+ if n >= 12 && r[..2] == query[..2] && r[2] & 0x80 != 0 {
+ return Some(r.to_vec());
+ }
+ }
+ }
+ None
+}
+
+/// SERVFAIL for `query`: no resolver on the Mac answered either.
+fn servfail(query: &[u8]) -> Vec<u8> {
+ let end = question_end(query).unwrap_or(12).min(query.len());
+ let mut r = query[..end].to_vec();
+ r[2] = 0x80 | (query[2] & 0x79); // QR, opcode and RD kept
+ r[3] = 0x80 | 2; // RA, SERVFAIL
+ r[6..12].fill(0);
+ if end > 12 {
+ r[5] = 1;
+ } else {
+ r[4..6].fill(0);
+ }
+ r
+}
+
+/// Offset just past the first question, if there is one.
+fn question_end(q: &[u8]) -> Option<usize> {
+ if q.len() < 12 || u16::from_be_bytes([q[4], q[5]]) == 0 {
+ return None;
+ }
+ let mut i = 12;
+ loop {
+ let len = *q.get(i)? as usize;
+ if len == 0 {
+ break;
+ }
+ if len > 63 {
+ return None;
+ }
+ i += 1 + len;
+ }
+ (i + 5 <= q.len()).then_some(i + 5)
+}
+
+/// The first question's name, lowercase, without the trailing dot.
+fn question_name(q: &[u8]) -> Option<String> {
+ question_end(q)?;
+ let mut i = 12;
+ let mut name = String::new();
+ loop {
+ let len = q[i] as usize;
+ if len == 0 {
+ return Some(name);
+ }
+ if !name.is_empty() {
+ name.push('.');
+ }
+ name.push_str(&String::from_utf8_lossy(&q[i + 1..i + 1 + len]).to_ascii_lowercase());
+ i += 1 + len;
+ }
+}
+
+/// Where the Mac would send a lookup for `name`: the servers of the
+/// `/etc/resolver/<domain>` file with the longest domain that `name` is in
+/// (VPNs and tools such as dnsmasq set these up), else `/etc/resolv.conf`'s,
+/// which macOS keeps to its current primary DNS servers. Read on every
+/// lookup, so network changes are followed. Kiri's own `kiri.local` file is
+/// skipped: its server answers 127.0.0.1, the Mac's loopback, which means
+/// nothing in a container.
+fn mac_resolvers(name: &str, resolv_conf: &Path, resolver_dir: &Path) -> Vec<SocketAddr> {
+ let mut best: Option<(usize, Vec<SocketAddr>)> = None;
+ for entry in std::fs::read_dir(resolver_dir).into_iter().flatten().flatten() {
+ let Ok(text) = std::fs::read_to_string(entry.path()) else { continue };
+ let file = entry.file_name().to_string_lossy().to_ascii_lowercase();
+ let domain = text
+ .lines()
+ .find_map(|l| l.trim().strip_prefix("domain").map(|d| d.trim().trim_end_matches('.').to_ascii_lowercase()))
+ .filter(|d| !d.is_empty())
+ .unwrap_or(file);
+ if domain == SUFFIX {
+ continue;
+ }
+ let inside = name == domain || name.ends_with(&format!(".{domain}"));
+ if !inside || best.as_ref().is_some_and(|(len, _)| *len >= domain.len()) {
+ continue;
+ }
+ let servers = parse_servers(&text);
+ if !servers.is_empty() {
+ best = Some((domain.len(), servers));
+ }
+ }
+ best.map(|(_, s)| s).unwrap_or_else(|| parse_servers(&std::fs::read_to_string(resolv_conf).unwrap_or_default()))
+}
+
+/// `nameserver` lines (and a `port` line) of a resolv.conf-style file.
+/// Link-local IPv6 servers, which need an interface named, are skipped.
+fn parse_servers(text: &str) -> Vec<SocketAddr> {
+ let field = |key: &'static str| text.lines().filter_map(move |l| l.trim().strip_prefix(key).filter(|v| v.starts_with([' ', '\t'])).map(str::trim));
+ let port = field("port").find_map(|p| p.parse::<u16>().ok()).unwrap_or(53);
+ field("nameserver").filter_map(|ip| ip.parse::<IpAddr>().ok()).map(|ip| SocketAddr::new(ip, port)).collect()
+}
+
#[cfg(test)]
mod tests {
use super::answer;
+ use std::net::UdpSocket;
fn query(name: &str, qtype: u16) -> Vec<u8> {
let mut q = vec![0x12, 0x34, 0x01, 0x00, 0, 1, 0, 0, 0, 0, 0, 0];
@@ -100,4 +263,54 @@ mod tests {
assert_eq!(answer(&query("notkiri.local", 1)).unwrap()[3], 5);
assert!(answer(&[0; 5]).is_none());
}
+
+ #[test]
+ fn relayed_lookups_go_where_the_mac_would_send_them() {
+ let dir = std::env::temp_dir().join(format!("kiri-dns-test-{}", std::process::id()));
+ let resolvers = dir.join("resolver");
+ std::fs::create_dir_all(&resolvers).unwrap();
+ let conf = dir.join("resolv.conf");
+ std::fs::write(&conf, "# macOS\nsearch home\nnameserver 192.168.1.1\nnameserver fe80::1%en0\nnameserver 2001:db8::53\n").unwrap();
+ std::fs::write(resolvers.join("corp.example"), "nameserver 10.0.0.53\n").unwrap();
+ std::fs::write(resolvers.join("eu.corp.example"), "nameserver 10.1.0.53\nport 5353\n").unwrap();
+ std::fs::write(resolvers.join("kiri.local"), "nameserver 127.0.0.1\nport 53535\n").unwrap();
+ let r = |name: &str| super::mac_resolvers(name, &conf, &resolvers).iter().map(ToString::to_string).collect::<Vec<_>>();
+ assert_eq!(r("registry-1.docker.io"), ["192.168.1.1:53", "[2001:db8::53]:53"]);
+ assert_eq!(r("git.corp.example"), ["10.0.0.53:53"]);
+ assert_eq!(r("corp.example"), ["10.0.0.53:53"]);
+ assert_eq!(r("db.eu.corp.example"), ["10.1.0.53:5353"], "the longest domain wins");
+ assert_eq!(r("notcorp.example"), ["192.168.1.1:53", "[2001:db8::53]:53"]);
+ assert_eq!(r("web.kiri.local"), ["192.168.1.1:53", "[2001:db8::53]:53"], "Kiri's own resolver is the Mac's loopback");
+ std::fs::remove_dir_all(&dir).unwrap();
+ }
+
+ #[test]
+ fn a_relayed_lookup_takes_the_first_resolver_that_answers() {
+ let up = UdpSocket::bind("127.0.0.1:0").unwrap();
+ let addr = up.local_addr().unwrap();
+ let resolver = std::thread::spawn(move || {
+ let mut buf = [0u8; 512];
+ let (n, from) = up.recv_from(&mut buf).unwrap();
+ // A stray datagram first, then the answer.
+ up.send_to(&[9; 12], from).unwrap();
+ buf[2] |= 0x80;
+ up.send_to(&buf[..n], from).unwrap();
+ });
+ // A closed port first: refused, then the next one is asked.
+ let closed = UdpSocket::bind("127.0.0.1:0").unwrap().local_addr().unwrap();
+ let q = query("registry-1.docker.io", 1);
+ let r = super::relay_query(&q, &[closed, addr]).expect("answered");
+ resolver.join().unwrap();
+ assert_eq!((&r[..2], r[2] & 0x80, &r[12..]), (&q[..2], 0x80, &q[12..]));
+ assert_eq!(super::question_name(&q).as_deref(), Some("registry-1.docker.io"));
+ }
+
+ #[test]
+ fn servfail_keeps_the_id_and_question() {
+ let q = query("example.com", 1);
+ let r = super::servfail(&q);
+ assert_eq!((&r[..2], r[2] & 0x80, r[3] & 0x0f), (&q[..2], 0x80, 2));
+ assert_eq!((&r[4..12], &r[12..]), (&[0, 1, 0, 0, 0, 0, 0, 0][..], &q[12..]));
+ assert_eq!(super::servfail(&[0x12, 0x34, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])[3], 0x82);
+ }
}
diff --git a/crates/kiri/src/daemon/engine.rs b/crates/kiri/src/daemon/engine.rs
index 77dd8de..d01554a 100644
--- a/crates/kiri/src/daemon/engine.rs
+++ b/crates/kiri/src/daemon/engine.rs
@@ -668,6 +668,11 @@ impl Engine {
if cfg!(target_os = "macos") {
vm.listen(super::ports::ENGINE_API_PORT, Box::new(super::ports::serve_engine_api))?;
}
+ // Container lookups the VM's gateway resolver fails, asked of the
+ // Mac's own (VM only: a Linux host's agent asks the host's itself).
+ if cfg!(target_os = "macos") {
+ vm.listen(kiri_proto::DNS_RELAY_PORT, Box::new(super::dns::serve_relay))?;
+ }
// Everything else on the guest's disk belongs to containers that are gone.
let keep: Vec<String> = self.lock().containers.keys().cloned().collect();
vm.retain(&keep)?;
diff --git a/crates/kiri/src/daemon/mod.rs b/crates/kiri/src/daemon/mod.rs
index ac3daa4..7d3aea6 100644
--- a/crates/kiri/src/daemon/mod.rs
+++ b/crates/kiri/src/daemon/mod.rs
@@ -151,6 +151,8 @@ pub fn run(_args: &[String]) -> io::Result<()> {
}
}
+ watch_own_app(engine.clone(), socket.clone());
+
let on_signal = engine.clone();
let sock = socket.clone();
std::thread::Builder::new().name("kiri-signals".into()).spawn(move || {
@@ -180,6 +182,26 @@ pub fn run(_args: &[String]) -> io::Result<()> {
Ok(())
}
+/// When the app this daemon runs from is deleted (appwatch.rs): docker
+/// build handed back (buildwith.rs: what was there before; the choice
+/// stays, so the app taken up again takes it again), and the engine stops.
+fn watch_own_app(engine: std::sync::Arc<engine::Engine>, socket: std::path::PathBuf) {
+ crate::appwatch::watch(move |exe| {
+ diag::error!("the app this engine runs from was deleted; stopping and handing docker build back", path = exe.display());
+ if let Err(e) = crate::buildwith::places().and_then(|p| p.give_back()) {
+ diag::warn!("docker build not handed back", error = e);
+ }
+ #[cfg(target_os = "macos")]
+ nfs::unmount();
+ let _ = std::fs::remove_file(&socket);
+ engine.shutdown();
+ std::thread::sleep(Duration::from_millis(50));
+ // 0: launchd restarts the engine on failure only, and there is
+ // nothing left to restart.
+ std::process::exit(0);
+ });
+}
+
/// Serves container files over NFS and mounts them at ~/Kiri.
/// KIRI_NO_FINDER=1 skips the mount.
#[cfg(target_os = "macos")]
diff --git a/crates/kiri/src/guest.rs b/crates/kiri/src/guest.rs
index 0244021..a60582f 100644
--- a/crates/kiri/src/guest.rs
+++ b/crates/kiri/src/guest.rs
@@ -141,6 +141,9 @@ impl Guest {
share: opts.data_disk.then(|| (PathBuf::from("/"), "kiri-mac".to_owned())),
rosetta_tag: opts.data_disk.then(|| "rosetta".to_owned()),
};
+ if !cfg.kernel.is_file() {
+ return Err(missing_kernel(&cfg.kernel, std::env::current_exe().ok().as_deref()));
+ }
Ok(cfg)
}
@@ -404,6 +407,17 @@ fn translated() -> bool {
on == 1
}
+/// The error for a guest kernel that isn't there. Mostly the app was
+/// deleted or moved while its engine ran: say that, not a bare path.
+#[cfg(target_os = "macos")]
+fn missing_kernel(kernel: &std::path::Path, exe: Option<&std::path::Path>) -> io::Error {
+ let msg = match exe {
+ Some(exe) if !exe.exists() => format!("{} was deleted or moved while it ran: open Kiri again from where it is now", crate::flavor::APP),
+ _ => format!("the guest kernel {} is missing: reinstall Kiri", kernel.display()),
+ };
+ io::Error::new(io::ErrorKind::NotFound, msg)
+}
+
#[cfg(target_os = "macos")]
/// Where the kernel and initramfs live: `KIRI_GUEST_DIR`, else
/// `Kiri.app/Contents/Resources/guest` when running from the app, else the
diff --git a/crates/kiri/src/main.rs b/crates/kiri/src/main.rs
index ca27446..3e896bd 100644
--- a/crates/kiri/src/main.rs
+++ b/crates/kiri/src/main.rs
@@ -12,6 +12,10 @@ mod boot;
#[cfg(unix)]
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
mod builds;
+// The engine stops when its app is deleted under it (kiri-win includes it
+// for Windows).
+#[cfg(unix)]
+mod appwatch;
// Which builder plain `docker build` uses: Docker's or Kiri's.
#[cfg(unix)]
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]