Skip to content

armgit / kiri

Public Imported from GitHub
0

A small, fast container runtime for macOS, Windows and Linux

Rust

Containers' connections no longer break as others start and stop (#17)

Precious Okwud314d6b3c203
diff --git a/crates/kiri-init/src/linux/container.rs b/crates/kiri-init/src/linux/container.rs
index 4ff58b6..86cfc6a 100644
--- a/crates/kiri-init/src/linux/container.rs
+++ b/crates/kiri-init/src/linux/container.rs
@@ -489,14 +489,15 @@ pub fn oom_killed(p: &Process) -> bool {
/// `netns_shared`: another live process still uses the container's network
/// namespace (a container started with `--network container:`). Only then
/// are its veths deleted here, or they would live on with that namespace
-/// and keep its address. Otherwise the kernel deletes them as the namespace
-/// goes, off the agent's loop: deleting one here waits out RCU, 15-40 ms of
-/// netlink that every request after the exit queued behind.
+/// and keep its address. Otherwise the next container to be plugged in
+/// deletes them (see `net::retire`).
pub fn teardown(p: &Process, netns_shared: bool) {
if netns_shared {
for v in &p.veths {
let _ = net::detach(v);
}
+ } else {
+ net::retire(&p.veths);
}
if let Some(t) = &p.teardown {
undo_now(t, t.remove);
diff --git a/crates/kiri-init/src/linux/net.rs b/crates/kiri-init/src/linux/net.rs
index efc9ed6..de48a64 100644
--- a/crates/kiri-init/src/linux/net.rs
+++ b/crates/kiri-init/src/linux/net.rs
@@ -659,6 +659,8 @@ pub fn attach(bridges: &mut Bridges, bridge: &str, host_ifname: &str, pid: libc:
let bridge_index = bridges.ensure(bridge, (!no_ipv4).then_some(gateway), prefix, v6)?;
let netns = std::fs::File::open(format!("/proc/{pid}/ns/net"))?;
let mut host = Netlink::open(libc::NETLINK_ROUTE)?;
+ // A stopped container's veth may still hold this address and MAC.
+ delete_retired(&mut host);
let ifname = format!("eth{n}");
netlink::create_veth(&mut host, host_ifname, bridge_index, &netns, CONTAINER_MTU, &ifname, mac)?;
// By name: a new namespace may have tunl0, gre0 … before it.
@@ -686,6 +688,29 @@ pub fn attach(bridges: &mut Bridges, bridge: &str, host_ifname: &str, pid: libc:
Ok(())
}
+/// Host-side veths of stopped containers, deleted by the next `attach`.
+static RETIRED: std::sync::Mutex<Vec<String>> = std::sync::Mutex::new(Vec::new());
+
+/// Notes a stopped container's host-side veths for the next `attach` to
+/// delete. Deleting one waits out RCU, 15-40 ms that every request after
+/// the exit would queue behind on the agent's loop; and the agent stays
+/// single-threaded, as clone3 without CLONE_VM needs. Left to the kernel
+/// they went only as the namespace did, a second or more later: by then the
+/// next container had the same address and MAC, the bridge sent its replies
+/// to the old namespace, and that answered with resets (a download that
+/// broke mid-TLS or stalled).
+pub fn retire(veths: &[String]) {
+ RETIRED.lock().unwrap_or_else(|e| e.into_inner()).extend_from_slice(veths);
+}
+
+/// Deletes the veths `retire` noted. Gone already (the kernel took them
+/// with their namespace) is fine.
+fn delete_retired(nl: &mut Netlink) {
+ for v in std::mem::take(&mut *RETIRED.lock().unwrap_or_else(|e| e.into_inner())) {
+ let _ = netlink::delete_link(nl, &v);
+ }
+}
+
/// Unplugs a container's extra network by its host-side veth.
pub fn detach(host_ifname: &str) -> io::Result<()> {
let mut host = Netlink::open(libc::NETLINK_ROUTE)?;
diff --git a/crates/kiri-init/src/linux/netlink.rs b/crates/kiri-init/src/linux/netlink.rs
index 6ac441c..40315f4 100644
--- a/crates/kiri-init/src/linux/netlink.rs
+++ b/crates/kiri-init/src/linux/netlink.rs
@@ -181,16 +181,48 @@ fn ifinfo(index: i32, flags: u32, change: u32) -> [u8; 16] {
b
}
+/// Creates bridge `name` with a MAC address of its own. Without one a Linux
+/// bridge takes the lowest address among its ports, so the gateway's MAC
+/// changed whenever a container joined or left: every other container's ARP
+/// entry for it went stale, and their connections stalled, were reset or
+/// broke mid-TLS until it was asked again. Docker gives its bridges an
+/// address for the same reason. A bridge left by an earlier run without
+/// one (a Linux host) gets one too.
pub fn create_bridge(nl: &mut Netlink, name: &str) -> io::Result<()> {
- let mut m = Msg::new(RTM_NEWLINK, NLM_F_ACK | NLM_F_CREATE | NLM_F_EXCL);
- m.raw(&ifinfo(0, 0, 0)).attr_str(IFLA_IFNAME, name);
- m.nest(IFLA_LINKINFO).attr_str(IFLA_INFO_KIND, "bridge").end();
- match nl.request(&mut [m]) {
- Err(e) if e.raw_os_error() == Some(libc::EEXIST) => Ok(()),
+ match nl.request(&mut [new_bridge(name, bridge_mac(random_bytes()))]) {
+ Err(e) if e.raw_os_error() == Some(libc::EEXIST) => {
+ // NET_ADDR_SET (3): someone set it, so it no longer follows the ports.
+ let kind = std::fs::read_to_string(format!("/sys/class/net/{name}/addr_assign_type")).unwrap_or_default();
+ if kind.trim() == "3" {
+ return Ok(());
+ }
+ let mut m = Msg::new(RTM_NEWLINK, NLM_F_ACK);
+ m.raw(&ifinfo(index_of(name)?, 0, 0)).attr(IFLA_ADDRESS, &bridge_mac(random_bytes()));
+ nl.request(&mut [m])
+ }
r => r,
}
}
+fn new_bridge(name: &str, mac: [u8; 6]) -> Msg {
+ let mut m = Msg::new(RTM_NEWLINK, NLM_F_ACK | NLM_F_CREATE | NLM_F_EXCL);
+ m.raw(&ifinfo(0, 0, 0)).attr_str(IFLA_IFNAME, name).attr(IFLA_ADDRESS, &mac);
+ m.nest(IFLA_LINKINFO).attr_str(IFLA_INFO_KIND, "bridge").end();
+ m
+}
+
+/// `b` as a locally administered unicast MAC address.
+fn bridge_mac(mut b: [u8; 6]) -> [u8; 6] {
+ b[0] = (b[0] & 0xfc) | 0x02;
+ b
+}
+
+fn random_bytes() -> [u8; 6] {
+ let mut b = [0u8; 6];
+ let _ = std::fs::File::open("/dev/urandom").and_then(|mut f| std::io::Read::read_exact(&mut f, &mut b));
+ b
+}
+
/// Creates a veth pair: `host` stays here, attached to `bridge` and up; its
/// peer is created straight inside `netns` as `peer`. The kernel picks the
/// peer's interface index: a new namespace is not always only lo (with ipip
@@ -884,6 +916,18 @@ mod tests {
u32::from_be_bytes(d.try_into().unwrap())
}
+ #[test]
+ fn a_bridge_is_made_with_a_mac_address_of_its_own() {
+ let m = new_bridge("kiri0", bridge_mac([0xff; 6]));
+ let a = attrs(&m.buf[32..]); // nlmsghdr, ifinfomsg
+ assert_eq!(get(&a, IFLA_ADDRESS), &[0xfe, 0xff, 0xff, 0xff, 0xff, 0xff]);
+ assert_eq!(get(&a, IFLA_IFNAME), b"kiri0\0");
+ // Locally administered, never multicast.
+ for b in [[0u8; 6], [0x01; 6], [0xff; 6]] {
+ assert_eq!(bridge_mac(b)[0] & 0x03, 0x02);
+ }
+ }
+
/// A rule's expressions: (name, its NFTA_EXPR_DATA attributes).
fn exprs(m: &Msg) -> Vec<(String, Vec<(u16, &[u8])>)> {
let top = attrs(&m.buf[20..]); // nlmsghdr, nfgenmsg
diff --git a/tests/e2e.sh b/tests/e2e.sh
index 95380ad..9781fb8 100755
--- a/tests/e2e.sh
+++ b/tests/e2e.sh
@@ -185,6 +185,15 @@ t internet-dns "Address" $D run --rm alpine nslookup example.com
t dns-default-bridge-gateway "nameserver 172.17.0.1" $D run --rm alpine cat /etc/resolv.conf
code dns-default-bridge-no-loopback 1 $D run --rm alpine grep -q 127.0.0.11 /etc/resolv.conf
t internet-https "200" $D run --rm curlimages/curl:latest -s -o /dev/null -w '%{http_code}' https://www.google.com
+# Each container gets the last one's address and MAC: its first download
+# broke mid-TLS or stalled while the old one's veth lingered, and one that
+# keeps running lost its gateway whenever the bridge's MAC moved.
+APKIDX=https://dl-cdn.alpinelinux.org/alpine/v3.22/main/aarch64/APKINDEX.tar.gz
+t first-download-x8 "8" bash -c "n=0; for i in \$(seq 8); do $D run --rm curlimages/curl:latest -s -m 20 -o /dev/null -w '%{exitcode}' $APKIDX | grep -qx 0 && n=\$((n+1)); done; echo \$n"
+$D run -d --name dl curlimages/curl:latest sh -c "while true; do curl -s -m 10 -o /dev/null -w '%{exitcode} ' $APKIDX; done" >/dev/null
+for _ in $(seq 6); do $D run --rm alpine true; sleep 1; done
+code download-while-others-start 1 bash -c "$D logs dl 2>&1 | grep -qE '(^| )[1-9][0-9]* '"
+$D rm -f dl >/dev/null
t none-net "lo" $D run --rm --network none alpine ip link
code none-no-internet 1 $D run --rm --network none alpine wget -qO- -T 2 http://example.com
t host-net "192.168.64." $D run --rm --network host alpine ip -4 addr show eth0