A small, fast container runtime for macOS, Windows and Linux
Rust
Containers' connections no longer break as others start and stop (#17)
Precious Okwud314d6b3c203
diff --git a/crates/kiri-init/src/linux/container.rs b/crates/kiri-init/src/linux/container.rsindex 4ff58b6..86cfc6a 100644--- a/crates/kiri-init/src/linux/container.rs+++ b/crates/kiri-init/src/linux/container.rs@@ -489,14 +489,15 @@ pub fn oom_killed(p: &Process) -> bool {/// `netns_shared`: another live process still uses the container's network/// namespace (a container started with `--network container:`). Only then/// are its veths deleted here, or they would live on with that namespace-/// and keep its address. Otherwise the kernel deletes them as the namespace-/// goes, off the agent's loop: deleting one here waits out RCU, 15-40 ms of-/// netlink that every request after the exit queued behind.+/// and keep its address. Otherwise the next container to be plugged in+/// deletes them (see `net::retire`).pub fn teardown(p: &Process, netns_shared: bool) {if netns_shared {for v in &p.veths {let _ = net::detach(v);}+ } else {+ net::retire(&p.veths);}if let Some(t) = &p.teardown {undo_now(t, t.remove);diff --git a/crates/kiri-init/src/linux/net.rs b/crates/kiri-init/src/linux/net.rsindex efc9ed6..de48a64 100644--- a/crates/kiri-init/src/linux/net.rs+++ b/crates/kiri-init/src/linux/net.rs@@ -659,6 +659,8 @@ pub fn attach(bridges: &mut Bridges, bridge: &str, host_ifname: &str, pid: libc:let bridge_index = bridges.ensure(bridge, (!no_ipv4).then_some(gateway), prefix, v6)?;let netns = std::fs::File::open(format!("/proc/{pid}/ns/net"))?;let mut host = Netlink::open(libc::NETLINK_ROUTE)?;+ // A stopped container's veth may still hold this address and MAC.+ delete_retired(&mut host);let ifname = format!("eth{n}");netlink::create_veth(&mut host, host_ifname, bridge_index, &netns, CONTAINER_MTU, &ifname, mac)?;// By name: a new namespace may have tunl0, gre0 … before it.@@ -686,6 +688,29 @@ pub fn attach(bridges: &mut Bridges, bridge: &str, host_ifname: &str, pid: libc:Ok(())}+/// Host-side veths of stopped containers, deleted by the next `attach`.+static RETIRED: std::sync::Mutex<Vec<String>> = std::sync::Mutex::new(Vec::new());++/// Notes a stopped container's host-side veths for the next `attach` to+/// delete. Deleting one waits out RCU, 15-40 ms that every request after+/// the exit would queue behind on the agent's loop; and the agent stays+/// single-threaded, as clone3 without CLONE_VM needs. Left to the kernel+/// they went only as the namespace did, a second or more later: by then the+/// next container had the same address and MAC, the bridge sent its replies+/// to the old namespace, and that answered with resets (a download that+/// broke mid-TLS or stalled).+pub fn retire(veths: &[String]) {+ RETIRED.lock().unwrap_or_else(|e| e.into_inner()).extend_from_slice(veths);+}++/// Deletes the veths `retire` noted. Gone already (the kernel took them+/// with their namespace) is fine.+fn delete_retired(nl: &mut Netlink) {+ for v in std::mem::take(&mut *RETIRED.lock().unwrap_or_else(|e| e.into_inner())) {+ let _ = netlink::delete_link(nl, &v);+ }+}+/// Unplugs a container's extra network by its host-side veth.pub fn detach(host_ifname: &str) -> io::Result<()> {let mut host = Netlink::open(libc::NETLINK_ROUTE)?;diff --git a/crates/kiri-init/src/linux/netlink.rs b/crates/kiri-init/src/linux/netlink.rsindex 6ac441c..40315f4 100644--- a/crates/kiri-init/src/linux/netlink.rs+++ b/crates/kiri-init/src/linux/netlink.rs@@ -181,16 +181,48 @@ fn ifinfo(index: i32, flags: u32, change: u32) -> [u8; 16] {b}+/// Creates bridge `name` with a MAC address of its own. Without one a Linux+/// bridge takes the lowest address among its ports, so the gateway's MAC+/// changed whenever a container joined or left: every other container's ARP+/// entry for it went stale, and their connections stalled, were reset or+/// broke mid-TLS until it was asked again. Docker gives its bridges an+/// address for the same reason. A bridge left by an earlier run without+/// one (a Linux host) gets one too.pub fn create_bridge(nl: &mut Netlink, name: &str) -> io::Result<()> {- let mut m = Msg::new(RTM_NEWLINK, NLM_F_ACK | NLM_F_CREATE | NLM_F_EXCL);- m.raw(&ifinfo(0, 0, 0)).attr_str(IFLA_IFNAME, name);- m.nest(IFLA_LINKINFO).attr_str(IFLA_INFO_KIND, "bridge").end();- match nl.request(&mut [m]) {- Err(e) if e.raw_os_error() == Some(libc::EEXIST) => Ok(()),+ match nl.request(&mut [new_bridge(name, bridge_mac(random_bytes()))]) {+ Err(e) if e.raw_os_error() == Some(libc::EEXIST) => {+ // NET_ADDR_SET (3): someone set it, so it no longer follows the ports.+ let kind = std::fs::read_to_string(format!("/sys/class/net/{name}/addr_assign_type")).unwrap_or_default();+ if kind.trim() == "3" {+ return Ok(());+ }+ let mut m = Msg::new(RTM_NEWLINK, NLM_F_ACK);+ m.raw(&ifinfo(index_of(name)?, 0, 0)).attr(IFLA_ADDRESS, &bridge_mac(random_bytes()));+ nl.request(&mut [m])+ }r => r,}}+fn new_bridge(name: &str, mac: [u8; 6]) -> Msg {+ let mut m = Msg::new(RTM_NEWLINK, NLM_F_ACK | NLM_F_CREATE | NLM_F_EXCL);+ m.raw(&ifinfo(0, 0, 0)).attr_str(IFLA_IFNAME, name).attr(IFLA_ADDRESS, &mac);+ m.nest(IFLA_LINKINFO).attr_str(IFLA_INFO_KIND, "bridge").end();+ m+}++/// `b` as a locally administered unicast MAC address.+fn bridge_mac(mut b: [u8; 6]) -> [u8; 6] {+ b[0] = (b[0] & 0xfc) | 0x02;+ b+}++fn random_bytes() -> [u8; 6] {+ let mut b = [0u8; 6];+ let _ = std::fs::File::open("/dev/urandom").and_then(|mut f| std::io::Read::read_exact(&mut f, &mut b));+ b+}+/// Creates a veth pair: `host` stays here, attached to `bridge` and up; its/// peer is created straight inside `netns` as `peer`. The kernel picks the/// peer's interface index: a new namespace is not always only lo (with ipip@@ -884,6 +916,18 @@ mod tests {u32::from_be_bytes(d.try_into().unwrap())}+ #[test]+ fn a_bridge_is_made_with_a_mac_address_of_its_own() {+ let m = new_bridge("kiri0", bridge_mac([0xff; 6]));+ let a = attrs(&m.buf[32..]); // nlmsghdr, ifinfomsg+ assert_eq!(get(&a, IFLA_ADDRESS), &[0xfe, 0xff, 0xff, 0xff, 0xff, 0xff]);+ assert_eq!(get(&a, IFLA_IFNAME), b"kiri0\0");+ // Locally administered, never multicast.+ for b in [[0u8; 6], [0x01; 6], [0xff; 6]] {+ assert_eq!(bridge_mac(b)[0] & 0x03, 0x02);+ }+ }+/// A rule's expressions: (name, its NFTA_EXPR_DATA attributes).fn exprs(m: &Msg) -> Vec<(String, Vec<(u16, &[u8])>)> {let top = attrs(&m.buf[20..]); // nlmsghdr, nfgenmsgdiff --git a/tests/e2e.sh b/tests/e2e.shindex 95380ad..9781fb8 100755--- a/tests/e2e.sh+++ b/tests/e2e.sh@@ -185,6 +185,15 @@ t internet-dns "Address" $D run --rm alpine nslookup example.comt dns-default-bridge-gateway "nameserver 172.17.0.1" $D run --rm alpine cat /etc/resolv.confcode dns-default-bridge-no-loopback 1 $D run --rm alpine grep -q 127.0.0.11 /etc/resolv.conft internet-https "200" $D run --rm curlimages/curl:latest -s -o /dev/null -w '%{http_code}' https://www.google.com+# Each container gets the last one's address and MAC: its first download+# broke mid-TLS or stalled while the old one's veth lingered, and one that+# keeps running lost its gateway whenever the bridge's MAC moved.+APKIDX=https://dl-cdn.alpinelinux.org/alpine/v3.22/main/aarch64/APKINDEX.tar.gz+t first-download-x8 "8" bash -c "n=0; for i in \$(seq 8); do $D run --rm curlimages/curl:latest -s -m 20 -o /dev/null -w '%{exitcode}' $APKIDX | grep -qx 0 && n=\$((n+1)); done; echo \$n"+$D run -d --name dl curlimages/curl:latest sh -c "while true; do curl -s -m 10 -o /dev/null -w '%{exitcode} ' $APKIDX; done" >/dev/null+for _ in $(seq 6); do $D run --rm alpine true; sleep 1; done+code download-while-others-start 1 bash -c "$D logs dl 2>&1 | grep -qE '(^| )[1-9][0-9]* '"+$D rm -f dl >/dev/nullt none-net "lo" $D run --rm --network none alpine ip linkcode none-no-internet 1 $D run --rm --network none alpine wget -qO- -T 2 http://example.comt host-net "192.168.64." $D run --rm --network host alpine ip -4 addr show eth0