Skip to content

A modern terminal with a built-in multiplexer: close it and your sessions keep running. Run shells and tons of AI agents across your machines, then pick them up from desktop, browser, TUI and more. Written in Rust with no UI framework: interface drawn straight on the GPU. No Electron, GPUI or webview.

refactor(tailnet): run the built-in node in a program of its own

RoversXcf8d266d94f8
diff --git a/Cargo.lock b/Cargo.lock
index e420fc79..ab0f2c3c 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -9595,6 +9595,17 @@ dependencies = [
"wezterm-ssh",
]
+[[package]]
+name = "thinkterm-tailnet-node"
+version = "0.1.0"
+dependencies = [
+ "config",
+ "log",
+ "nix 0.29.0",
+ "tempfile",
+ "thinkterm-tailnet",
+]
+
[[package]]
name = "thinkterm-tui"
version = "0.1.0"
diff --git a/Cargo.toml b/Cargo.toml
index 58529562..c4f5ca6f 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -8,6 +8,7 @@ members = [
"thinkterm-syntax",
"thinkterm-core",
"thinkterm-tailnet",
+ "thinkterm-tailnet-node",
"thinkterm-file-index",
"thinkterm-font-core",
"thinkterm-font-web",
diff --git a/deps/tailscale-rs/PATCHES.txt b/deps/tailscale-rs/PATCHES.txt
index b8473b0b..31f4ff03 100644
--- a/deps/tailscale-rs/PATCHES.txt
+++ b/deps/tailscale-rs/PATCHES.txt
@@ -191,6 +191,6 @@ Upgrading
3. Bump the version: the five entries under [patch.crates-io] in the root Cargo.toml and
the two in thinkterm-tailnet/Cargo.toml, then `cargo update` the other tailscale-rs
crates to the same version. All tailscale-rs crates move together.
-4. Build, run the tests above and thinkterm-tailnet's, then sign in to a real tailnet and
- connect to a machine by name and by address.
+4. Build, run the tests above and thinkterm-tailnet's and thinkterm-tailnet-node's, then
+ sign in to a real tailnet and connect to a machine by name and by address.
5. Update this file: the version above, and every group that changed or went away.
diff --git a/thinkterm-tailnet-node/Cargo.toml b/thinkterm-tailnet-node/Cargo.toml
new file mode 100644
index 00000000..b234567b
--- /dev/null
+++ b/thinkterm-tailnet-node/Cargo.toml
@@ -0,0 +1,27 @@
+[package]
+name = "thinkterm-tailnet-node"
+version = "0.1.0"
+edition = "2021"
+publish = false
+
+# `thinkterm-tailnet`: the machine's node in the tailnet, in a process apart
+# from the mux server, which keeps it running while the tailnet is on (see
+# thinkterm-tailnet's keeper). Whatever goes wrong in the library stays out
+# of the process the terminal sessions live in.
+
+[[bin]]
+name = "thinkterm-tailnet"
+path = "src/main.rs"
+# Its docs would land where the thinkterm-tailnet library's do.
+doc = false
+
+[dependencies]
+config.workspace = true
+log = { workspace = true, features = ["std"] }
+thinkterm-tailnet = { workspace = true, features = ["node"] }
+
+[dev-dependencies]
+tempfile.workspace = true
+
+[target.'cfg(unix)'.dev-dependencies]
+nix = { workspace = true, features = ["signal"] }
diff --git a/thinkterm-tailnet-node/src/main.rs b/thinkterm-tailnet-node/src/main.rs
new file mode 100644
index 00000000..2348495c
--- /dev/null
+++ b/thinkterm-tailnet-node/src/main.rs
@@ -0,0 +1,143 @@
+//! `thinkterm-tailnet`: the machine's node in the tailnet, in a process of its
+//! own. The mux server starts it while the settings turn the tailnet on (see
+//! `thinkterm_tailnet::keep_node`), and it holds the node the way the server
+//! once did itself (see `thinkterm_tailnet::follow`): by the node's lock, one
+//! process at a time, answering the machine's other processes on the
+//! tailnet's socket and writing the report they read.
+//!
+//! It ends with the process that started it, and once the settings turn the
+//! tailnet off; the server starts it again when the tailnet is on.
+
+#[cfg(any(target_os = "macos", target_os = "linux"))]
+use std::time::Duration;
+
+/// How often it looks whether to end.
+#[cfg(any(target_os = "macos", target_os = "linux"))]
+const WATCH: Duration = Duration::from_secs(1);
+
+#[cfg(any(target_os = "macos", target_os = "linux"))]
+fn main() {
+ log_to_file();
+ let parent = match parse_parent() {
+ Ok(parent) => parent,
+ Err(err) => {
+ log::error!("{err}");
+ eprintln!("thinkterm-tailnet: {err}");
+ std::process::exit(2);
+ }
+ };
+ log::info!("holding the tailnet's node for process {parent}");
+ thinkterm_tailnet::follow(|| {});
+ loop {
+ // Already gone by the time this looks is gone all the same: the
+ // process is handed to another parent as its own ends.
+ if std::os::unix::process::parent_id() != parent {
+ log::info!("process {parent}, which started this, has gone");
+ break;
+ }
+ if thinkterm_tailnet::is_off() {
+ log::info!("the tailnet is off");
+ break;
+ }
+ // Ended for good: nothing acts on the settings any more. Started
+ // anew by the mux server, it does again.
+ if !thinkterm_tailnet::is_following() {
+ log::error!("the thread that follows the settings has ended");
+ std::process::exit(1);
+ }
+ std::thread::sleep(WATCH);
+ }
+}
+
+#[cfg(not(any(target_os = "macos", target_os = "linux")))]
+fn main() {
+ eprintln!("thinkterm-tailnet: joining a tailnet is not supported on this platform");
+ std::process::exit(2);
+}
+
+/// The process it ends with: `--parent <pid>`, which the mux server passes,
+/// else whichever started it.
+#[cfg(any(target_os = "macos", target_os = "linux"))]
+fn parse_parent() -> Result<u32, String> {
+ let mut args = std::env::args().skip(1);
+ let mut parent = std::os::unix::process::parent_id();
+ while let Some(arg) = args.next() {
+ match arg.as_str() {
+ "--parent" => {
+ parent = args
+ .next()
+ .and_then(|pid| pid.parse().ok())
+ .ok_or("--parent needs a process id")?;
+ }
+ _ => return Err(format!("unknown argument {arg:?}")),
+ }
+ }
+ Ok(parent)
+}
+
+/// The log: a file of its own in the runtime folder, named as the mux
+/// server's are, so that the app prunes it with theirs once it is a week
+/// old. Readable by this user alone: a login link may be in it. Standard
+/// error when the file cannot be had. Panics go to it too.
+#[cfg(any(target_os = "macos", target_os = "linux"))]
+fn log_to_file() {
+ use std::io::Write;
+ use std::os::unix::fs::OpenOptionsExt;
+ use std::sync::Mutex;
+ use std::time::{SystemTime, UNIX_EPOCH};
+
+ struct Log(Mutex<Box<dyn Write + Send>>);
+
+ impl log::Log for Log {
+ fn enabled(&self, metadata: &log::Metadata) -> bool {
+ metadata.level() <= log::Level::Info
+ }
+
+ fn log(&self, record: &log::Record) {
+ if !self.enabled(record.metadata()) {
+ return;
+ }
+ let since = SystemTime::now()
+ .duration_since(UNIX_EPOCH)
+ .unwrap_or_default();
+ let line = format!(
+ "{}.{:03} [{}] {}: {}\n",
+ since.as_secs(),
+ since.subsec_millis(),
+ record.level(),
+ record.target(),
+ record.args()
+ );
+ if let Ok(mut out) = self.0.lock() {
+ let _ = out.write_all(line.as_bytes());
+ }
+ }
+
+ fn flush(&self) {}
+ }
+
+ let dir = &*config::RUNTIME_DIR;
+ let path = dir.join(format!("thinkterm-tailnet-log-{}.txt", std::process::id()));
+ let out: Box<dyn Write + Send> = match std::fs::create_dir_all(dir).and_then(|()| {
+ std::fs::OpenOptions::new()
+ .create(true)
+ .append(true)
+ .mode(0o600)
+ .open(&path)
+ }) {
+ Ok(file) => Box::new(file),
+ Err(err) => {
+ eprintln!("thinkterm-tailnet: logging to {}: {err}", path.display());
+ Box::new(std::io::stderr())
+ }
+ };
+ if log::set_boxed_logger(Box::new(Log(Mutex::new(out)))).is_ok() {
+ log::set_max_level(log::LevelFilter::Info);
+ }
+ std::panic::set_hook(Box::new(|info| {
+ log::error!(
+ "panic: {info}\n{}",
+ std::backtrace::Backtrace::force_capture()
+ );
+ }));
+}
diff --git a/thinkterm-tailnet-node/tests/lifecycle.rs b/thinkterm-tailnet-node/tests/lifecycle.rs
new file mode 100644
index 00000000..892c88ca
--- /dev/null
+++ b/thinkterm-tailnet-node/tests/lifecycle.rs
@@ -0,0 +1,133 @@
+//! When `thinkterm-tailnet` runs and when it ends, run as the mux server runs
+//! it, in a home of its own: its settings, lock, socket and log, and nothing
+//! of the machine's. None of these starts a node: the tailnet is off, or
+//! another process holds the node's lock.
+
+#![cfg(any(target_os = "macos", target_os = "linux"))]
+
+use std::fs::{File, OpenOptions};
+use std::path::{Path, PathBuf};
+use std::process::{Child, Command, ExitStatus, Stdio};
+use std::time::{Duration, Instant};
+
+const PROGRAM: &str = env!("CARGO_BIN_EXE_thinkterm-tailnet");
+
+/// Short: the node's socket goes in it.
+fn home() -> tempfile::TempDir {
+ tempfile::Builder::new()
+ .prefix("ttn-")
+ .tempdir_in("/tmp")
+ .unwrap()
+}
+
+fn command(program: &str, home: &Path) -> Command {
+ let mut command = Command::new(program);
+ command
+ .env("HOME", home)
+ .env_remove("XDG_RUNTIME_DIR")
+ .env_remove("XDG_DATA_HOME")
+ .stdin(Stdio::null())
+ .stdout(Stdio::null())
+ .stderr(Stdio::null());
+ command
+}
+
+/// Where the settings are in `home`, as the library finds them.
+fn settings_dir(home: &Path) -> PathBuf {
+ if cfg!(target_os = "macos") {
+ home.join("Library/Application Support/ThinkTerm")
+ } else {
+ home.join(".local/share/ThinkTerm")
+ }
+}
+
+/// Turns the tailnet on, with the node's lock held as another process on the
+/// machine would hold it: the program waits for it, and starts no node.
+fn on_and_held_elsewhere(home: &Path) -> File {
+ let dir = settings_dir(home);
+ std::fs::create_dir_all(&dir).unwrap();
+ std::fs::write(dir.join("tailnet.json"), r#"{"enabled":true}"#).unwrap();
+ let lock = OpenOptions::new()
+ .create(true)
+ .truncate(false)
+ .write(true)
+ .open(dir.join("tailnet-node.lock"))
+ .unwrap();
+ lock.try_lock().unwrap();
+ lock
+}
+
+fn ended_within(child: &mut Child, wait: Duration) -> Option<ExitStatus> {
+ let deadline = Instant::now() + wait;
+ while Instant::now() < deadline {
+ if let Some(status) = child.try_wait().unwrap() {
+ return Some(status);
+ }
+ std::thread::sleep(Duration::from_millis(50));
+ }
+ None
+}
+
+fn running(pid: i32) -> bool {
+ nix::sys::signal::kill(nix::unistd::Pid::from_raw(pid), None).is_ok()
+}
+
+#[test]
+fn it_ends_while_the_tailnet_is_off() {
+ let home = home();
+ let mut program = command(PROGRAM, home.path()).spawn().unwrap();
+ let ended = ended_within(&mut program, Duration::from_secs(10));
+ assert!(ended.is_some_and(|status| status.success()), "{ended:?}");
+}
+
+#[test]
+fn it_stays_while_the_tailnet_is_on() {
+ let home = home();
+ let _held = on_and_held_elsewhere(home.path());
+ let mut program = command(PROGRAM, home.path()).spawn().unwrap();
+ let ended = ended_within(&mut program, Duration::from_secs(3));
+ let _ = program.kill();
+ let _ = program.wait();
+ assert!(ended.is_none(), "it ended: {ended:?}");
+}
+
+#[test]
+fn it_ends_with_the_process_that_started_it() {
+ let home = home();
+ let _held = on_and_held_elsewhere(home.path());
+ // Started by a shell that leaves at once, as a server that exits leaves it.
+ let output = command("/bin/sh", home.path())
+ .arg("-c")
+ .arg(r#""$0" --parent $$ </dev/null >/dev/null 2>&1 & echo $!"#)
+ .arg(PROGRAM)
+ .stdout(Stdio::piped())
+ .output()
+ .unwrap();
+ let pid: i32 = String::from_utf8_lossy(&output.stdout)
+ .trim()
+ .parse()
+ .unwrap();
+ let deadline = Instant::now() + Duration::from_secs(10);
+ while running(pid) && Instant::now() < deadline {
+ std::thread::sleep(Duration::from_millis(50));
+ }
+ let outlived = running(pid);
+ if outlived {
+ let _ = nix::sys::signal::kill(
+ nix::unistd::Pid::from_raw(pid),
+ nix::sys::signal::Signal::SIGKILL,
+ );
+ }
+ assert!(!outlived, "it outlived the process that started it");
+}
+
+#[test]
+fn it_says_what_is_wrong_with_its_arguments() {
+ let home = home();
+ let mut program = command(PROGRAM, home.path())
+ .arg("--parent")
+ .spawn()
+ .unwrap();
+ let ended = ended_within(&mut program, Duration::from_secs(10));
+ assert_eq!(ended.and_then(|status| status.code()), Some(2));
+}
diff --git a/thinkterm-tailnet/Cargo.toml b/thinkterm-tailnet/Cargo.toml
index 3be7f6a0..acbde8a8 100644
--- a/thinkterm-tailnet/Cargo.toml
+++ b/thinkterm-tailnet/Cargo.toml
@@ -5,10 +5,11 @@ edition = "2021"
publish = false
[features]
-# The node itself, for the one process that holds it: the mux server.
-# Without it this crate reads and changes the settings, reads the server's
-# reports, and asks the server for connections -- all the app and
-# `thinkterm` need, without the library in them.
+# The node itself, for the one process that holds it: `thinkterm-tailnet`
+# (thinkterm-tailnet-node). Without it this crate reads and changes the
+# settings, reads the node's reports, asks for connections through it, and
+# keeps it running -- all the app, `thinkterm` and the mux server need,
+# without the library in them.
node = ["dep:tailscale", "dep:tokio", "dep:ts_control"]
[dependencies]
diff --git a/thinkterm-tailnet/src/holder.rs b/thinkterm-tailnet/src/holder.rs
index 339f73cb..f81d7fe6 100644
--- a/thinkterm-tailnet/src/holder.rs
+++ b/thinkterm-tailnet/src/holder.rs
@@ -1,12 +1,12 @@
-//! Holding the machine's node: what the mux server does, from the time it
-//! starts following the settings ([`follow`]). The node starts when the
+//! Holding the machine's node: what `thinkterm-tailnet` does, from the time
+//! it starts following the settings ([`follow`]). The node starts when the
//! settings turn the tailnet on, restarts when they change what it runs on,
//! and stops when they turn it off; what it does goes to the report the app
//! and `thinkterm tailnet` read.
use crate::node;
use crate::settings::{self, Settings};
-use crate::status::{self, read_live_report, read_report, Family, Peer, Report, Status};
+use crate::status::{self, read_live_report, read_report, Peer, Report, Status};
use anyhow::Result;
use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering};
use std::sync::mpsc;
@@ -14,17 +14,18 @@ use std::sync::{Arc, Mutex, OnceLock};
use std::time::{Duration, Instant};
/// How often the settings file is looked at for a change made by another
-/// process -- the app's switch, `thinkterm tailnet up` -- and how soon a
-/// process kept from the node by another one tries again.
+/// process -- the app's switch, `thinkterm tailnet up` -- and the longest a
+/// process kept from the node by another one waits to try again.
const POLL: Duration = Duration::from_secs(3);
+/// The first wait of a process kept from the node by another one, which
+/// doubles up to POLL: the other may be on its way out, as the
+/// predecessor's is when a mux server takes over.
+const HELD_FIRST: Duration = Duration::from_millis(250);
+
/// The longest wait before trying again at a node that would not start.
const RETRY_MAX: Duration = Duration::from_secs(60);
-/// How long a connection asked for while this process acts on the settings
-/// waits for it to finish.
-const SETTLE_WAIT: Duration = Duration::from_secs(5);
-
type Changed = Arc<dyn Fn() + Send + Sync>;
/// Why the settings thread looks again before its next poll.
@@ -44,28 +45,19 @@ static FOLLOWER: OnceLock<Follower> = OnceLock::new();
/// The status while no node runs here: Off, or why there is none.
static IDLE: Mutex<Status> = Mutex::new(Status::Off);
-/// How many looks at the settings this process asked its settings thread
-/// for -- the first one, and one after each change this process makes --
-/// and up to which of them it has acted on. Until it has acted on all, a
-/// connection selecting the tailnet waits for the node rather
-/// than go out without it: sessions taken over at a restart connect before
-/// the first look. Counted, not a flag: a look begun before a change does
-/// not act on it, so its end must not end the wait for it.
-static ASKED: AtomicU64 = AtomicU64::new(0);
-static ACTED_ON: AtomicU64 = AtomicU64::new(0);
-
-/// Whether the settings thread has settings to act on that it has not yet.
-fn settling() -> bool {
- ACTED_ON.load(Ordering::SeqCst) < ASKED.load(Ordering::SeqCst)
-}
+/// Whether the settings thread last found the tailnet off, and let go of
+/// the node for it (see [`is_off`]).
+static OFF: AtomicBool = AtomicBool::new(false);
-/// Ends every wait for the settings thread however it ends, a panic
-/// included: nothing would act on what is asked after that.
-struct SettlingEnds;
+/// Whether the settings thread runs (see [`is_following`]).
+static FOLLOWING: AtomicBool = AtomicBool::new(false);
-impl Drop for SettlingEnds {
+/// Says the settings thread has ended however it ends, a panic included.
+struct FollowingEnds;
+
+impl Drop for FollowingEnds {
fn drop(&mut self) {
- ACTED_ON.store(u64::MAX, Ordering::SeqCst);
+ FOLLOWING.store(false, Ordering::SeqCst);
}
}
@@ -84,23 +76,6 @@ static GIVEN_UP: AtomicU32 = AtomicU32::new(0);
/// node is gone and said to be.
static HOLD: Mutex<Option<std::fs::File>> = Mutex::new(None);
-/// Set from `will_hold` until the settings thread first acted on the
-/// settings: the process took over from a predecessor that holds the node
-/// until it exits.
-static TAKING_OVER: AtomicBool = AtomicBool::new(false);
-
-/// How long a connection selecting the tailnet
-/// waits for this process to take the node up from its predecessor.
-const TAKEOVER_WAIT: Duration = Duration::from_secs(15);
-
-/// This process takes over from a predecessor that holds the node until it
-/// exits, and starts following the settings (see `follow`) after that:
-/// connections to the tailnet's machines meanwhile wait for this process's
-/// own node, rather than go through the predecessor's and end with it.
-pub fn will_hold() {
- TAKING_OVER.store(true, Ordering::SeqCst);
-}
-
/// Holds the machine's node in this process for as long as the settings
/// file turns the tailnet on, starting, stopping and restarting it as the
/// file changes. `changed` is called, on a background thread, whenever the
@@ -117,26 +92,40 @@ pub fn follow(changed: impl Fn() + Send + Sync + 'static) {
{
return;
}
- ASKED.fetch_add(1, Ordering::SeqCst);
+ FOLLOWING.store(true, Ordering::SeqCst);
let spawned = std::thread::Builder::new()
.name("tailnet-settings".into())
.spawn(move || {
- let _ends = SettlingEnds;
+ let _ends = FollowingEnds;
follow_loop(changed, woken)
});
if let Err(err) = spawned {
- ACTED_ON.store(u64::MAX, Ordering::SeqCst);
+ FOLLOWING.store(false, Ordering::SeqCst);
log::error!("tailnet: starting the settings thread: {err}");
}
}
-/// This process changed the settings file: connections asked for from here
-/// on wait for the settings thread to act on it, while there is one to.
+/// Whether the settings thread runs: from [`follow`] until it ends, by a
+/// panic among others. Nothing acts on the settings after that, and the
+/// node, if there is one, goes on as it was: `thinkterm-tailnet` ends for
+/// the mux server to start it anew.
+pub fn is_following() -> bool {
+ FOLLOWING.load(Ordering::SeqCst)
+}
+
+/// Whether the settings thread last found the tailnet off and let go of the
+/// node for it: what `thinkterm-tailnet` ends at. False until it first
+/// looked, and while it cannot read the settings.
+pub fn is_off() -> bool {
+ OFF.load(Ordering::SeqCst)
+}
+
+/// This process changed the settings file: the settings thread looks again
+/// at once, rather than at its next poll.
pub(crate) fn settings_changed() {
if FOLLOWER.get().is_none() {
return;
}
- ASKED.fetch_add(1, Ordering::SeqCst);
send(Wake::Settings);
}
@@ -171,100 +160,6 @@ fn peers() -> Vec<Peer> {
vec![]
}
-/// A connection through this process's node (see crate::dial): Some with
-/// its answer while this process holds one, None for the caller to ask the
-/// process that does.
-pub(crate) fn dial(
- host: &str,
- port: u16,
- family: Family,
- timeout: Duration,
- cancelled: &dyn Fn() -> bool,
-) -> Option<Option<Result<socket2::Socket>>> {
- // A node stopped while this went through it -- restarted for new
- // settings, or given up -- took what it had with it: the one in its
- // place is asked again. Within the one `timeout` either way.
- let started = Instant::now();
- let left = || timeout.saturating_sub(started.elapsed());
- for _ in 0..2 {
- // The caller selected this node, including while settings restart it.
- if settling() {
- if let Err(err) = wait_for_settings(left(), cancelled, settling) {
- return Some(Some(Err(err)));
- }
- }
- let node = NODE.lock().unwrap().clone();
- let Some(node) = node else {
- return dial_once_held(host, port, family, left(), cancelled);
- };
- let dialed = node.dial(host, port, family, left(), cancelled);
- if !node.is_stopped() || cancelled() {
- return Some(dialed);
- }
- }
- Some(None)
-}
-
-/// While a predecessor still holds the node (see `will_hold`), the selected
-/// connection waits for this process's own node within `timeout`. None once this
-/// process is not taking over, or is but holds no node after all, for the
-/// caller to ask whichever process holds it.
-fn dial_once_held(
- host: &str,
- port: u16,
- family: Family,
- timeout: Duration,
- cancelled: &dyn Fn() -> bool,
-) -> Option<Option<Result<socket2::Socket>>> {
- if !TAKING_OVER.load(Ordering::SeqCst)
- || !settings::load_summary().is_ok_and(|settings| settings.enabled)
- {
- return None;
- }
- let started = Instant::now();
- let deadline = started + timeout.min(TAKEOVER_WAIT);
- loop {
- if let Some(node) = NODE.lock().unwrap().clone() {
- let left = timeout.saturating_sub(started.elapsed());
- return Some(node.dial(host, port, family, left, cancelled));
- }
- if !TAKING_OVER.load(Ordering::SeqCst) {
- return None;
- }
- if cancelled() || Instant::now() >= deadline {
- return Some(None);
- }
- std::thread::sleep(Duration::from_millis(100));
- }
-}
-
-/// Waits, for up to SETTLE_WAIT, for the settings thread to act on settings
-/// it has not yet: a node being started is not a node turned off. The
-/// connection goes on after that either way, with whatever node there is --
-/// a thread that is slow or gone must not fail every connection, the
-/// tailnet's or not. Err only when the session is closed meanwhile.
-fn wait_for_settings(
- timeout: Duration,
- cancelled: &dyn Fn() -> bool,
- settling: impl Fn() -> bool,
-) -> Result<()> {
- let deadline = Instant::now() + timeout.min(SETTLE_WAIT);
- loop {
- if cancelled() {
- anyhow::bail!("the session was closed while waiting for the tailnet settings");
- }
- if !settling() {
- return Ok(());
- }
- let remaining = deadline.saturating_duration_since(Instant::now());
- if remaining.is_zero() {
- log::warn!("tailnet: connecting before the settings thread acted on the settings");
- return Ok(());
- }
- std::thread::sleep(remaining.min(Duration::from_millis(50)));
- }
-}
-
/// What the settings thread does after acting on the settings.
enum Next {
/// Nothing until the settings change.
@@ -293,20 +188,17 @@ fn follow_loop(changed: Changed, woken: mpsc::Receiver<Wake>) {
let file_changed = look || stamp != seen;
if file_changed || retry.is_some_and(|(at, _)| Instant::now() >= at) {
seen = stamp;
- // The looks asked for so far: this one reads the file as they
- // left it.
- let asked = ASKED.load(Ordering::SeqCst);
retry = match reconcile() {
Next::Settled => None,
- Next::Held => Some((Instant::now() + POLL, POLL)),
+ Next::Held => {
+ let wait = held_wait(retry.map(|(_, wait)| wait));
+ Some((Instant::now() + wait, wait))
+ }
Next::Failed => {
- let wait = retry.map_or(POLL, |(_, wait)| (wait * 2).min(RETRY_MAX));
+ let wait = retry.map_or(POLL, |(_, wait)| (wait * 2).clamp(POLL, RETRY_MAX));
Some((Instant::now() + wait, wait))
}
};
- ACTED_ON.fetch_max(asked, Ordering::SeqCst);
- // Whatever came of it: holding the node now, or not to.
- TAKING_OVER.store(false, Ordering::SeqCst);
}
// Told when it changes, and after every change of the file even when
// the node goes on as it was: the page shows the file, and
@@ -321,7 +213,11 @@ fn follow_loop(changed: Changed, woken: mpsc::Receiver<Wake>) {
changed();
told = Some(now);
}
- look = match woken.recv_timeout(POLL) {
+ // Not past the next try, which may be sooner than the next poll.
+ let sleep = retry.map_or(POLL, |(at, _)| {
+ at.saturating_duration_since(Instant::now()).min(POLL)
+ });
+ look = match woken.recv_timeout(sleep) {
Ok(Wake::Settings) => true,
// A node that has to get in again may take an auth key saved
// while it was in, which did not restart it then; one given up
@@ -334,6 +230,12 @@ fn follow_loop(changed: Changed, woken: mpsc::Receiver<Wake>) {
}
}
+/// The wait before trying again at a node another process holds, after
+/// `previous`.
+fn held_wait(previous: Option<Duration>) -> Duration {
+ previous.map_or(HELD_FIRST, |wait| (wait * 2).min(POLL))
+}
+
/// Starts, stops or restarts this process's node to match the settings.
///
/// The slow parts -- writing new keys, building the node's runtime -- run
@@ -345,6 +247,7 @@ fn reconcile() -> Next {
Err(err) => return failed(err),
};
ACTED.store(summary.revision, Ordering::SeqCst);
+ OFF.store(false, Ordering::SeqCst);
if !summary.enabled {
let node = NODE.lock().unwrap().take();
let stopped = node.is_some();
@@ -359,6 +262,7 @@ fn reconcile() -> Next {
// Only after the report: the process that takes the node over can
// only start once this lock is free, so its report comes after.
HOLD.lock().unwrap().take();
+ OFF.store(true, Ordering::SeqCst);
return Next::Settled;
}
@@ -536,26 +440,28 @@ mod tests {
use super::*;
#[test]
- fn dialing_waits_for_settings_to_finish_initializing() {
- let settling = AtomicBool::new(true);
- std::thread::scope(|scope| {
- scope.spawn(|| {
- std::thread::sleep(Duration::from_millis(75));
- settling.store(false, Ordering::SeqCst);
- });
- wait_for_settings(Duration::from_secs(1), &|| false, || {
- settling.load(Ordering::SeqCst)
- })
- .unwrap();
- assert!(!settling.load(Ordering::SeqCst));
- });
+ fn a_settings_thread_that_panics_is_told_to_have_ended() {
+ FOLLOWING.store(true, Ordering::SeqCst);
+ let ended = std::thread::spawn(|| {
+ let _ends = FollowingEnds;
+ panic!("the settings thread failing on purpose");
+ })
+ .join();
+ assert!(ended.is_err());
+ assert!(!is_following());
}
#[test]
- fn a_closed_session_stops_waiting_and_a_long_wait_goes_on() {
- assert!(wait_for_settings(Duration::from_secs(1), &|| true, || true).is_err());
- // A settings thread that is slow or gone fails no connection.
- assert!(wait_for_settings(Duration::from_millis(20), &|| false, || true).is_ok());
- assert!(wait_for_settings(Duration::ZERO, &|| false, || false).is_ok());
+ fn a_node_held_elsewhere_is_tried_again_soon_at_first() {
+ let mut waits = vec![];
+ let mut wait = None;
+ for _ in 0..6 {
+ let next = held_wait(wait);
+ waits.push(next.as_millis());
+ wait = Some(next);
+ }
+ assert_eq!(waits, [250, 500, 1000, 2000, 3000, 3000]);
+ // After a start that failed, at most a poll away.
+ assert_eq!(held_wait(Some(RETRY_MAX)), POLL);
}
}
diff --git a/thinkterm-tailnet/src/keeper.rs b/thinkterm-tailnet/src/keeper.rs
new file mode 100644
index 00000000..271ea6c1
--- /dev/null
+++ b/thinkterm-tailnet/src/keeper.rs
@@ -0,0 +1,369 @@
+//! Keeping the machine's node running: what the mux server does. The node
+//! itself runs in `thinkterm-tailnet`, a program beside the server's own (see
+//! the crate's docs). The server starts it while the settings turn the
+//! tailnet on, and again when it dies; it ends by itself, with the server
+//! that started it or once the settings turn the tailnet off.
+//!
+//! Tied to the server that started it rather than kept across a takeover:
+//! macOS holds the server responsible, in privacy terms, for what it starts
+//! -- reaching the local network included -- and a program outliving its
+//! server would answer to one that is gone (see the server's
+//! `exec_as_own_tcc_identity`).
+
+use crate::settings;
+use std::path::{Path, PathBuf};
+use std::process::{Child, Command, Stdio};
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::OnceLock;
+use std::time::{Duration, Instant};
+
+/// The program, beside the server's executable.
+const PROGRAM: &str = "thinkterm-tailnet";
+
+/// How often the program, and whether the settings file changed, are
+/// looked at while the tailnet is on or the program runs; and while neither,
+/// as the settings were looked at before the node had a program of its own.
+const POLL: Duration = Duration::from_secs(1);
+const IDLE_POLL: Duration = Duration::from_secs(3);
+
+/// How long the program waits to be started again after it ended while the
+/// tailnet was on, at first and at most: the wait doubles each time it ends
+/// within RAN_LONG_ENOUGH of its start.
+const RESTART_FIRST: Duration = Duration::from_secs(1);
+const RESTART_MAX: Duration = Duration::from_secs(60);
+const RAN_LONG_ENOUGH: Duration = Duration::from_secs(60);
+
+/// How long a connection asked for during a takeover waits for the
+/// predecessor's node to have gone.
+const TAKEOVER_WAIT: Duration = Duration::from_secs(15);
+
+/// How long after the predecessor has exited its `thinkterm-tailnet` may
+/// take to end with it: it looks every second (see thinkterm-tailnet-node).
+const PREDECESSOR_ENDS: Duration = Duration::from_secs(2);
+
+/// Set from `will_hold` until the predecessor's node has had time to go,
+/// PREDECESSOR_ENDS after this server set out to keep its own, or the
+/// tailnet is off.
+static TAKING_OVER: AtomicBool = AtomicBool::new(false);
+
+static KEEPING: OnceLock<()> = OnceLock::new();
+
+/// This server takes over from a predecessor, whose `thinkterm-tailnet`
+/// holds the node until the predecessor exits, and calls [`keep_node`]
+/// after that: connections to the tailnet's machines meanwhile wait for the
+/// predecessor's node to go (see `wait_out_takeover`), rather than go
+/// through it and end with it. After that, they go through whichever
+/// process holds the node: this server's own, or another one's.
+pub fn will_hold() {
+ TAKING_OVER.store(true, Ordering::SeqCst);
+}
+
+/// Keeps `thinkterm-tailnet` running while the settings turn the tailnet on,
+/// from a thread of its own. Calling it again does nothing.
+pub fn keep_node() {
+ if KEEPING.set(()).is_err() {
+ return;
+ }
+ let program = match program() {
+ Ok(program) => program,
+ Err(err) => {
+ log::error!("tailnet: finding {PROGRAM}: {err}");
+ TAKING_OVER.store(false, Ordering::SeqCst);
+ return;
+ }
+ };
+ let path = match settings::path() {
+ Ok(path) => path,
+ Err(err) => {
+ log::error!("tailnet: {err:#}");
+ TAKING_OVER.store(false, Ordering::SeqCst);
+ return;
+ }
+ };
+ let spawned = std::thread::Builder::new()
+ .name("tailnet-keeper".into())
+ .spawn(move || {
+ let mut keeper = Keeper::new(program);
+ let started = Instant::now();
+ // Read again only once the file has changed.
+ let mut seen = None;
+ let mut enabled = false;
+ loop {
+ let stamp = settings::stamp(&path);
+ if seen != Some(stamp) {
+ seen = Some(stamp);
+ enabled = settings::load_summary().is_ok_and(|settings| settings.enabled);
+ }
+ keeper.step(enabled, Instant::now());
+ if !enabled || started.elapsed() >= PREDECESSOR_ENDS {
+ TAKING_OVER.store(false, Ordering::SeqCst);
+ }
+ let running = keeper.child.is_some();
+ std::thread::sleep(if enabled || running { POLL } else { IDLE_POLL });
+ }
+ });
+ if let Err(err) = spawned {
+ log::error!("tailnet: starting the thread that keeps {PROGRAM} running: {err}");
+ TAKING_OVER.store(false, Ordering::SeqCst);
+ }
+}
+
+/// While this server takes over (see [`will_hold`]), waits for the
+/// predecessor's node to have gone, for up to TAKEOVER_WAIT of `timeout`.
+/// False when the session is closed meanwhile, or the wait runs out: the
+/// connection fails rather than go through the predecessor's node.
+pub(crate) fn wait_out_takeover(timeout: Duration, cancelled: &dyn Fn() -> bool) -> bool {
+ wait_while(&TAKING_OVER, timeout.min(TAKEOVER_WAIT), cancelled)
+}
+
+/// Waits while `flag` is set, for up to `wait`: whether it was let go of.
+fn wait_while(flag: &AtomicBool, wait: Duration, cancelled: &dyn Fn() -> bool) -> bool {
+ let deadline = Instant::now() + wait;
+ while flag.load(Ordering::SeqCst) {
+ if cancelled() {
+ return false;
+ }
+ if Instant::now() >= deadline {
+ log::warn!("tailnet: the predecessor's node did not go in time");
+ return false;
+ }
+ std::thread::sleep(Duration::from_millis(100));
+ }
+ true
+}
+
+/// The program beside the server's executable: beside the file itself, not
+/// a link to it, as the command-line tools are linked into a bin directory
+/// from the app bundle.
+fn program() -> std::io::Result<PathBuf> {
+ let exe = std::env::current_exe()?;
+ let exe = exe.canonicalize().unwrap_or(exe);
+ Ok(exe.with_file_name(PROGRAM))
+}
+
+/// The program as this server runs it.
+struct Keeper {
+ program: PathBuf,
+ /// The program while it runs, and when it was started.
+ child: Option<(Child, Instant)>,
+ /// When it may be started next.
+ next_start: Instant,
+ /// The wait before the next start, should it end soon again.
+ wait: Duration,
+}
+
+impl Keeper {
+ fn new(program: PathBuf) -> Self {
+ Self {
+ program,
+ child: None,
+ next_start: Instant::now(),
+ wait: RESTART_FIRST,
+ }
+ }
+
+ /// Reaps the program once it has ended, and starts it while the tailnet
+ /// is `enabled`, it is not running and its wait is over.
+ fn step(&mut self, enabled: bool, now: Instant) {
+ if let Some((child, started)) = &mut self.child {
+ match child.try_wait() {
+ Ok(None) => {}
+ Ok(Some(status)) => {
+ let ran = now.saturating_duration_since(*started);
+ self.child = None;
+ if enabled {
+ if ran >= RAN_LONG_ENOUGH {
+ self.wait = RESTART_FIRST;
+ }
+ log::warn!(
+ "tailnet: {PROGRAM} ended ({status}) after {ran:?}; \
+ starting it again in {:?}",
+ self.wait
+ );
+ self.wait_before_next(now);
+ } else {
+ // It ended as the tailnet was turned off: on again,
+ // it starts at once.
+ log::info!("tailnet: {PROGRAM} ended ({status})");
+ self.wait = RESTART_FIRST;
+ self.next_start = now;
+ }
+ }
+ Err(err) => log::warn!("tailnet: waiting for {PROGRAM}: {err}"),
+ }
+ }
+ if !enabled || self.child.is_some() || now < self.next_start {
+ return;
+ }
+ match start(&self.program) {
+ Ok(child) => {
+ log::info!("tailnet: started {PROGRAM}, pid {}", child.id());
+ self.child = Some((child, now));
+ }
+ Err(err) => {
+ log::warn!(
+ "tailnet: starting {}: {err}; trying again in {:?}",
+ self.program.display(),
+ self.wait
+ );
+ self.wait_before_next(now);
+ }
+ }
+ }
+
+ /// The next start is not before the wait, and the one after that waits
+ /// twice as long.
+ fn wait_before_next(&mut self, now: Instant) {
+ self.next_start = now + self.wait;
+ self.wait = (self.wait * 2).min(RESTART_MAX);
+ }
+}
+
+/// Starts the program, on its own: no terminal, and a process group of its
+/// own, so that a signal meant for the server's group is not meant for it.
+/// It ends with this process all the same, which it is told is its parent
+/// (see thinkterm-tailnet-node).
+fn start(program: &Path) -> std::io::Result<Child> {
+ use std::os::unix::process::CommandExt;
+ Command::new(program)
+ .arg("--parent")
+ .arg(std::process::id().to_string())
+ .stdin(Stdio::null())
+ .stdout(Stdio::null())
+ .stderr(Stdio::null())
+ .process_group(0)
+ .spawn()
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ /// A program in `dir` that runs `script` as sh would.
+ fn program(dir: &Path, name: &str, script: &str) -> PathBuf {
+ use std::os::unix::fs::PermissionsExt;
+ let path = dir.join(name);
+ std::fs::write(&path, format!("#!/bin/sh\n{script}\n")).unwrap();
+ std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap();
+ path
+ }
+
+ /// Steps `keeper` at `now` until the program it ran has ended.
+ fn until_ended(keeper: &mut Keeper, enabled: bool, now: Instant) {
+ for _ in 0..100 {
+ keeper.step(enabled, now);
+ if keeper.child.is_none() {
+ return;
+ }
+ std::thread::sleep(Duration::from_millis(20));
+ }
+ panic!("the program did not end");
+ }
+
+ fn stop(keeper: &mut Keeper) {
+ if let Some((child, _)) = keeper.child.as_mut() {
+ let _ = child.kill();
+ let _ = child.wait();
+ }
+ }
+
+ #[test]
+ fn the_program_runs_while_the_tailnet_is_on() {
+ let dir = tempfile::tempdir().unwrap();
+ let mut keeper = Keeper::new(program(dir.path(), "stays", "sleep 5"));
+ let now = Instant::now();
+ keeper.step(false, now);
+ assert!(keeper.child.is_none(), "started while the tailnet is off");
+ keeper.step(true, now);
+ let pid = keeper.child.as_ref().expect("not started").0.id();
+ keeper.step(true, now);
+ assert_eq!(keeper.child.as_ref().unwrap().0.id(), pid, "started twice");
+ stop(&mut keeper);
+ }
+
+ #[test]
+ fn a_program_that_keeps_ending_waits_longer_each_time() {
+ let dir = tempfile::tempdir().unwrap();
+ let mut keeper = Keeper::new(program(dir.path(), "ends", "exit 3"));
+ let mut now = Instant::now();
+ let mut waits = vec![];
+ for _ in 0..8 {
+ keeper.step(true, now);
+ assert!(keeper.child.is_some(), "not started once its wait was over");
+ until_ended(&mut keeper, true, now);
+ let wait = keeper.next_start - now;
+ waits.push(wait.as_secs());
+ // Not before the wait.
+ keeper.step(true, now + wait - Duration::from_millis(1));
+ assert!(keeper.child.is_none(), "started before its wait was over");
+ now += wait;
+ }
+ assert_eq!(waits, [1, 2, 4, 8, 16, 32, 60, 60]);
+ }
+
+ #[test]
+ fn a_program_that_ran_long_enough_starts_again_after_the_first_wait() {
+ let dir = tempfile::tempdir().unwrap();
+ let mut keeper = Keeper::new(program(dir.path(), "ends", "exit 3"));
+ keeper.wait = RESTART_MAX;
+ let started = Instant::now();
+ keeper.step(true, started);
+ until_ended(&mut keeper, true, started + RAN_LONG_ENOUGH);
+ assert_eq!(
+ keeper.next_start - (started + RAN_LONG_ENOUGH),
+ RESTART_FIRST
+ );
+ }
+
+ #[test]
+ fn a_program_that_ended_as_the_tailnet_went_off_starts_at_once_when_it_is_on() {
+ let dir = tempfile::tempdir().unwrap();
+ let mut keeper = Keeper::new(program(dir.path(), "ends", "exit 0"));
+ keeper.wait = RESTART_MAX;
+ let now = Instant::now();
+ keeper.step(true, now);
+ until_ended(&mut keeper, false, now);
+ keeper.step(false, now);
+ assert!(keeper.child.is_none(), "started while the tailnet is off");
+ keeper.step(true, now);
+ assert!(keeper.child.is_some(), "not started at once");
+ until_ended(&mut keeper, false, now);
+ }
+
+ #[test]
+ fn a_missing_program_is_tried_again_later() {
+ let dir = tempfile::tempdir().unwrap();
+ let mut keeper = Keeper::new(dir.path().join("missing"));
+ let now = Instant::now();
+ keeper.step(true, now);
+ assert!(keeper.child.is_none());
+ assert_eq!(keeper.next_start - now, RESTART_FIRST);
+ keeper.step(true, now + RESTART_FIRST);
+ assert_eq!(keeper.next_start - (now + RESTART_FIRST), RESTART_FIRST * 2);
+ }
+
+ #[test]
+ fn a_connection_during_a_takeover_waits_for_the_predecessors_node_to_go() {
+ let taking_over = AtomicBool::new(true);
+ std::thread::scope(|scope| {
+ scope.spawn(|| {
+ std::thread::sleep(Duration::from_millis(150));
+ taking_over.store(false, Ordering::SeqCst);
+ });
+ let started = Instant::now();
+ assert!(wait_while(&taking_over, Duration::from_secs(2), &|| false));
+ assert!(started.elapsed() >= Duration::from_millis(150));
+ });
+ // Not for good, nor once the session is closed.
+ let taking_over = AtomicBool::new(true);
+ assert!(!wait_while(
+ &taking_over,
+ Duration::from_millis(50),
+ &|| false
+ ));
+ assert!(!wait_while(&taking_over, Duration::from_secs(2), &|| true));
+ assert!(wait_while(&AtomicBool::new(false), Duration::ZERO, &|| {
+ false
+ }));
+ }
+}
diff --git a/thinkterm-tailnet/src/lib.rs b/thinkterm-tailnet/src/lib.rs
index 5c144603..ee76310d 100644
--- a/thinkterm-tailnet/src/lib.rs
+++ b/thinkterm-tailnet/src/lib.rs
@@ -1,14 +1,19 @@
//! Joining a tailnet from inside ThinkTerm, without the Tailscale app: a node
-//! of Tailscale's own Rust implementation, held by the machine's mux server.
+//! of Tailscale's own Rust implementation, held by `thinkterm-tailnet`, a
+//! program the machine's mux server keeps running for it.
//!
-//! The settings file says whether the machine joins; the mux server, built
-//! with the `node` feature, holds the node while it does ([`follow`]) and
-//! reports what the node is doing to a file the app and `thinkterm tailnet`
-//! read. SSH connections to the tailnet's machines go through the node
-//! ([`use_for_ssh`]) from any ThinkTerm process on the machine: the others
-//! ask the server for them on a socket, and need nothing of the library. The node
-//! also takes SSH connections from the tailnet to this machine's own sshd
-//! when `thinkterm tailnet up --serve-ssh` said to ([`Settings::serve_ssh`]), and
+//! The settings file says whether the machine joins; while it does, the mux
+//! server keeps `thinkterm-tailnet` running ([`keep_node`]), which holds the
+//! node ([`follow`], with the `node` feature) and reports what the node is
+//! doing to a file the app and `thinkterm tailnet` read. The node runs in a
+//! process of its own so that whatever goes wrong in the library -- a panic
+//! that aborts, a loop that spins, memory that grows -- stays out of the
+//! process the terminal sessions live in. SSH connections to the tailnet's
+//! machines go through the node ([`use_for_ssh`]) from any ThinkTerm process
+//! on the machine, the mux server's included: they ask `thinkterm-tailnet`
+//! for them on a socket, and need nothing of the library. The node also
+//! takes SSH connections from the tailnet to this machine's own sshd when
+//! `thinkterm tailnet up --serve-ssh` said to ([`Settings::serve_ssh`]), and
//! nothing else.
mod settings;
@@ -16,13 +21,17 @@ mod status;
#[cfg(all(feature = "node", any(target_os = "macos", target_os = "linux")))]
mod holder;
+#[cfg(any(target_os = "macos", target_os = "linux"))]
+mod keeper;
#[cfg(all(feature = "node", any(target_os = "macos", target_os = "linux")))]
mod node;
#[cfg(any(target_os = "macos", target_os = "linux"))]
mod proxy;
#[cfg(all(feature = "node", any(target_os = "macos", target_os = "linux")))]
-pub use holder::{follow, will_hold};
+pub use holder::{follow, is_following, is_off};
+#[cfg(any(target_os = "macos", target_os = "linux"))]
+pub use keeper::{keep_node, will_hold};
pub use settings::{load, load_summary, Settings, Summary};
pub(crate) use status::Family;
pub use status::{read_live_report, Peer, Report, Status};
@@ -33,8 +42,24 @@ pub fn follow(changed: impl Fn() + Send + Sync + 'static) {
let _ = changed;
}
-/// Nothing to hold where the library does not run.
+/// Nothing is held where the library does not run, so nothing is to let go.
#[cfg(all(feature = "node", not(any(target_os = "macos", target_os = "linux"))))]
+pub fn is_off() -> bool {
+ true
+}
+
+/// Nothing follows the settings where the library does not run.
+#[cfg(all(feature = "node", not(any(target_os = "macos", target_os = "linux"))))]
+pub fn is_following() -> bool {
+ false
+}
+
+/// Nothing to keep running where the library does not run.
+#[cfg(not(any(target_os = "macos", target_os = "linux")))]
+pub fn keep_node() {}
+
+/// Nothing to take over where the library does not run.
+#[cfg(not(any(target_os = "macos", target_os = "linux")))]
pub fn will_hold() {}
use anyhow::{Context, Result};
@@ -48,9 +73,9 @@ pub const SUPPORTED: bool = cfg!(any(target_os = "macos", target_os = "linux"));
/// no limit of its own.
const DIAL_TIMEOUT: Duration = Duration::from_secs(30);
-/// How long a connection selecting the tailnet waits for the mux server
-/// to take up the node: just after the switch was
-/// turned on, or while the server starts at login.
+/// How long a connection selecting the tailnet waits for `thinkterm-tailnet`
+/// to take up the node: just after the switch was turned on, or while it
+/// starts with the mux server at login.
#[cfg(any(target_os = "macos", target_os = "linux"))]
const HOLDER_WAIT: Duration = Duration::from_secs(15);
@@ -177,7 +202,7 @@ fn fetch_control_key(_url: &url::Url) -> Result<()> {
}
/// Registers the tailnet for SSH connections that explicitly select it.
-/// For the processes that never hold the node: the app and `thinkterm`.
+/// For the app and `thinkterm`.
pub fn use_for_ssh() {
if !SUPPORTED {
return;
@@ -189,9 +214,9 @@ pub fn use_for_ssh() {
)));
}
-/// [`use_for_ssh`] for the mux server: through its own node while it holds
-/// one (see `server_dial`).
-#[cfg(feature = "node")]
+/// [`use_for_ssh`] for the mux server: through whichever process holds the
+/// node, once the predecessor's node has gone after a takeover (see
+/// `server_dial`).
pub fn use_for_ssh_in_server() {
if !SUPPORTED {
return;
@@ -208,9 +233,8 @@ pub fn use_for_ssh_in_server() {
/// the SSH dialer reports that as an error, without trying system DNS.
/// Blocks while connecting, and gives up soon after `cancelled` says so.
///
-/// For the processes that never hold the node, the app and `thinkterm`:
-/// it asks the server, and refers to nothing of the node itself, so that
-/// they link none of it even when built together with the server.
+/// For the app and `thinkterm`: it asks `thinkterm-tailnet`, and refers to
+/// nothing of the node itself, so that they link none of it.
pub(crate) fn dial(
host: &str,
port: u16,
@@ -235,9 +259,9 @@ pub(crate) fn dial(
}
}
-/// `dial` for the mux server: through its own node while it holds one,
-/// else through the process that does.
-#[cfg(feature = "node")]
+/// `dial` for the mux server. During a takeover, it waits for the
+/// predecessor's node to go, rather than go through it and end with it (see
+/// `keeper::wait_out_takeover`).
pub(crate) fn server_dial(
host: &str,
port: u16,
@@ -248,10 +272,12 @@ pub(crate) fn server_dial(
#[cfg(any(target_os = "macos", target_os = "linux"))]
{
let timeout = timeout.unwrap_or(DIAL_TIMEOUT);
- if let Some(dialed) = holder::dial(host, port, family, timeout, cancelled) {
- return dialed;
+ let started = std::time::Instant::now();
+ if !keeper::wait_out_takeover(timeout, cancelled) {
+ return None;
}
- through_holder(host, port, family, timeout, cancelled)
+ let left = timeout.saturating_sub(started.elapsed());
+ through_holder(host, port, family, left, cancelled)
}
#[cfg(not(any(target_os = "macos", target_os = "linux")))]
{
@@ -261,7 +287,8 @@ pub(crate) fn server_dial(
}
/// A connection through the process holding the selected node. While
-/// nobody holds it, the connection waits for a holder within its timeout.
+/// nobody holds it, or the node it holds is on its way out, the connection
+/// waits for a holder within its timeout.
#[cfg(any(target_os = "macos", target_os = "linux"))]
fn through_holder(
host: &str,
@@ -372,6 +399,40 @@ mod tests {
holder.join().unwrap();
}
+ #[cfg(any(target_os = "macos", target_os = "linux"))]
+ #[test]
+ fn a_connection_is_asked_again_of_the_node_in_the_place_of_one_that_went() {
+ use std::io::{Read, Write};
+ let (_dir, path) = temp_socket();
+ let listener = std::os::unix::net::UnixListener::bind(&path).unwrap();
+ let holder = std::thread::spawn(move || {
+ // The node on its way out takes the request and answers nothing.
+ let (mut going, _) = listener.accept().unwrap();
+ let mut byte = [0u8; 1];
+ while going.read(&mut byte).unwrap() == 1 && byte[0] != b'\n' {}
+ drop(going);
+ let (mut stream, _) = listener.accept().unwrap();
+ while stream.read(&mut byte).unwrap() == 1 && byte[0] != b'\n' {}
+ stream.write_all(b"ok\n").unwrap();
+ // The connection carries on until the asker is done with it.
+ let _ = stream.read(&mut byte);
+ });
+ let dialed = through_holder_at(
+ &path,
+ "server-a",
+ 22,
+ Family::Any,
+ Duration::from_secs(5),
+ &|| false,
+ );
+ match dialed {
+ Some(Ok(connection)) => drop(connection),
+ Some(Err(err)) => panic!("no connection: {err:#}"),
+ None => panic!("no connection"),
+ }
+ holder.join().unwrap();
+ }
+
#[cfg(any(target_os = "macos", target_os = "linux"))]
#[test]
fn a_closed_session_does_not_wait() {
diff --git a/thinkterm-tailnet/src/node.rs b/thinkterm-tailnet/src/node.rs
index 4be5efd8..8893a258 100644
--- a/thinkterm-tailnet/src/node.rs
+++ b/thinkterm-tailnet/src/node.rs
@@ -1,7 +1,6 @@
//! A node in the tailnet, on a Tokio runtime of its own: the library needs
-//! one, and nothing else in these processes runs on Tokio.
+//! one, and nothing else in `thinkterm-tailnet` runs on Tokio.
-use crate::proxy::widen;
use crate::status::{Family, Peer, Status};
use anyhow::{anyhow, bail, Context, Result};
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr};
@@ -90,7 +89,6 @@ impl NodeConfig {
pub(crate) struct Node {
config: NodeConfig,
- handle: tokio::runtime::Handle,
runtime: Mutex<Option<tokio::runtime::Runtime>>,
shared: Arc<Shared>,
/// Where the machine's other ThinkTerm processes reach this node (see
@@ -168,6 +166,13 @@ impl Shared {
self.state.lock().unwrap().0.clone()
}
+ /// Whether the node is on its way out: stopped, or given up to be
+ /// started anew. What is asked of it is better asked of the node in its
+ /// place.
+ pub fn is_going(&self) -> bool {
+ self.stopped.load(Ordering::SeqCst) || self.broken.load(Ordering::SeqCst)
+ }
+
fn peers(&self) -> Vec<Peer> {
self.state.lock().unwrap().1.clone()
}
@@ -370,7 +375,6 @@ impl Node {
};
Ok(Self {
config,
- handle: runtime.handle().clone(),
runtime: Mutex::new(Some(runtime)),
shared,
socket,
@@ -410,12 +414,6 @@ impl Node {
}
}
- /// Whether the node was stopped: a dial that found nothing on it may
- /// find it on the node started in its place.
- pub fn is_stopped(&self) -> bool {
- self.shared.stopped.load(Ordering::SeqCst)
- }
-
/// Whether the node was given up, to be started anew: it was stuck for
/// its `broken_after` (see `watchdog`), or one of its loops panicked.
/// The library tries the control server only a couple of times before it
@@ -429,87 +427,6 @@ impl Node {
pub fn was_in(&self) -> bool {
self.shared.was_in.load(Ordering::SeqCst)
}
-
- /// A connection to `host:port` through the tailnet (see crate::dial),
- /// else None. Blocks while it connects, so it is for the threads SSH
- /// sessions connect on; gives up soon after `cancelled` says so.
- pub fn dial(
- &self,
- host: &str,
- port: u16,
- family: Family,
- timeout: Duration,
- cancelled: &dyn Fn() -> bool,
- ) -> Option<Result<socket2::Socket>> {
- let started = Instant::now();
- let ip = self
- .shared
- .resolve(host, family, starting_wait(timeout), cancelled)?;
- // What is left of the time, for the connection itself.
- let left = timeout.saturating_sub(started.elapsed());
- Some(
- self.connect(SocketAddr::new(ip, port), left, cancelled)
- .with_context(|| format!("connecting to {host}:{port} through the tailnet")),
- )
- }
-
- fn connect(
- &self,
- to: SocketAddr,
- timeout: Duration,
- cancelled: &dyn Fn() -> bool,
- ) -> Result<socket2::Socket> {
- // One end goes to the SSH library as its socket; the other is
- // copied to and from the tailnet's stream.
- let (ours, theirs) =
- std::os::unix::net::UnixStream::pair().context("making a socket pair")?;
- widen(&ours);
- widen(&theirs);
- ours.set_nonblocking(true)?;
- let (done, connected) = std::sync::mpsc::sync_channel::<Result<()>>(1);
- let shared = Arc::clone(&self.shared);
- self.handle.spawn(async move {
- let stream = match shared.tcp_connect(to, timeout).await {
- Ok(stream) => stream,
- Err(err) => {
- let _ = done.send(Err(err));
- return;
- }
- };
- let ours = match tokio::net::UnixStream::from_std(ours) {
- Ok(ours) => ours,
- Err(err) => {
- let _ = done.send(Err(err.into()));
- return;
- }
- };
- let _ = done.send(Ok(()));
- // Ends at once if the caller gave up meanwhile: its end is gone.
- bridge(stream, ours).await;
- });
- // A ConnectTimeout too large to add to now waits as long as it takes.
- let deadline = Instant::now().checked_add(timeout.saturating_add(Duration::from_secs(1)));
- loop {
- match connected.recv_timeout(Duration::from_millis(100)) {
- Ok(result) => {
- result?;
- break;
- }
- Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {
- if cancelled() {
- bail!("the session was closed while connecting");
- }
- if deadline.is_some_and(|deadline| Instant::now() >= deadline) {
- bail!("no answer within {timeout:?}");
- }
- }
- Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => {
- bail!("the tailnet node stopped")
- }
- }
- }
- Ok(socket2::Socket::from(std::os::fd::OwnedFd::from(theirs)))
- }
}
async fn run(
diff --git a/thinkterm-tailnet/src/proxy.rs b/thinkterm-tailnet/src/proxy.rs
index a823e11a..f43d72c2 100644
--- a/thinkterm-tailnet/src/proxy.rs
+++ b/thinkterm-tailnet/src/proxy.rs
@@ -1,6 +1,6 @@
//! The machine's ThinkTerm processes reach the tailnet through the one
-//! holding the node, the mux server: the app, `thinkterm ssh`, and the
-//! server itself while another server holds it. A connection is asked for on
+//! holding the node, `thinkterm-tailnet`: the app, `thinkterm ssh` and the
+//! mux server. A connection is asked for on
//! a socket beside the settings that only this user can open, with a line
//! naming the host; the answer is a line too, and after "ok" the socket
//! carries the connection itself. Asking needs nothing of the library: only
@@ -61,11 +61,23 @@ pub(crate) fn widen(socket: &impl std::os::fd::AsFd) {
pub(crate) enum Asked {
/// Nobody listens at `path`: no process here holds a node.
Nobody,
+ /// The holder went without answering, as one does whose node is on its
+ /// way out (see `answer`): the node started in its place may answer.
+ Gone,
/// A connection, why there is none, or None when `host` is none of its
- /// machines or the holder went without answering.
+ /// machines or the holder kept its answer past any wait it could have.
Answered(Option<Result<socket2::Socket>>),
}
+/// What reading the holder's answer came to.
+enum Answer {
+ Line(String),
+ /// It went without one.
+ Gone,
+ /// It kept it past any wait it could have.
+ Late,
+}
+
/// A connection to `host:port` through the node of the process listening
/// at `path`, which has `timeout` to make it.
pub(crate) fn ask(
@@ -89,13 +101,15 @@ pub(crate) fn ask(
return Asked::Answered(None);
};
line.push(b'\n');
+ // Closed before it read the request: it went without answering.
if stream.write_all(&line).is_err() {
- return Asked::Answered(None);
+ return Asked::Gone;
}
let answer = match read_answer(&mut stream, timeout, cancelled) {
- Ok(Some(answer)) => answer,
- // It went without answering; the selected network cannot connect.
- Ok(None) => return Asked::Answered(None),
+ Ok(Answer::Line(answer)) => answer,
+ Ok(Answer::Gone) => return Asked::Gone,
+ // The selected network cannot connect.
+ Ok(Answer::Late) => return Asked::Answered(None),
Err(err) => return Asked::Answered(Some(Err(err))),
};
Asked::Answered(match answer.as_str() {
@@ -127,14 +141,20 @@ fn connect_within(path: &Path, wait: Duration) -> Option<UnixStream> {
}
/// The holder's answer, read a byte at a time: what follows it is the
-/// connection's own. None when the holder went without one, or kept it
-/// past any wait it could have.
+/// connection's own.
fn read_answer(
stream: &mut UnixStream,
timeout: Duration,
cancelled: &dyn Fn() -> bool,
-) -> Result<Option<String>> {
- stream.set_read_timeout(Some(Duration::from_millis(100)))?;
+) -> Result<Answer> {
+ // macOS sets no timeout on a connection whose other end has closed: it
+ // went without answering.
+ if stream
+ .set_read_timeout(Some(Duration::from_millis(100)))
+ .is_err()
+ {
+ return Ok(Answer::Gone);
+ }
// The holder waits for its node to get in and for the other machine
// within `timeout`.
let deadline = Instant::now().checked_add(timeout.saturating_add(ANSWER_SLACK));
@@ -142,9 +162,9 @@ fn read_answer(
loop {
let mut byte = [0u8; 1];
match stream.read(&mut byte) {
- Ok(0) => return Ok(None),
+ Ok(0) => return Ok(Answer::Gone),
Ok(_) if byte[0] == b'\n' => {
- return Ok(Some(String::from_utf8_lossy(&line).into_owned()))
+ return Ok(Answer::Line(String::from_utf8_lossy(&line).into_owned()))
}
Ok(_) if line.len() < LINE_MAX => line.push(byte[0]),
Ok(_) => bail!("the tailnet's node gave no answer it could read"),
@@ -160,10 +180,10 @@ fn read_answer(
// The caller turns an unanswered request into a failure.
if deadline.is_some_and(|deadline| Instant::now() >= deadline) {
log::warn!("tailnet: the process holding the node did not answer");
- return Ok(None);
+ return Ok(Answer::Late);
}
}
- Err(_) => return Ok(None),
+ Err(_) => return Ok(Answer::Gone),
}
}
}
@@ -237,8 +257,11 @@ async fn answer(shared: Arc<Shared>, mut stream: tokio::net::UnixStream) {
})
.await
};
+ // A node on its way out answers nothing, rather than that it knows no
+ // such machine or cannot reach it: the asker asks the node in its place.
let ip = match resolved {
Ok(Some(ip)) => ip,
+ Ok(None) if shared.is_going() => return,
Ok(None) => {
let _ = stream.write_all(b"none\n").await;
return;
@@ -257,6 +280,7 @@ async fn answer(shared: Arc<Shared>, mut stream: tokio::net::UnixStream) {
node::bridge(tailnet, stream).await;
}
}
+ Err(_) if shared.is_going() => {}
Err(err) => {
let err = err.context(format!(
"connecting to {}:{} through the tailnet",
@@ -322,6 +346,7 @@ mod tests {
match ask(path, host, 22, family, Duration::from_secs(5), &|| false) {
Asked::Answered(answered) => answered,
Asked::Nobody => panic!("nobody listened at {}", path.display()),
+ Asked::Gone => panic!("the holder at {} went without answering", path.display()),
}
}
diff --git a/thinkterm-tailnet/src/settings.rs b/thinkterm-tailnet/src/settings.rs
index 149393e8..d34bf113 100644
--- a/thinkterm-tailnet/src/settings.rs
+++ b/thinkterm-tailnet/src/settings.rs
@@ -15,7 +15,8 @@ use std::sync::atomic::{AtomicU64, Ordering};
/// the secrets out.
#[derive(Clone, Default, PartialEq, Eq)]
pub struct Settings {
- /// Whether the machine's mux server holds a node in the tailnet.
+ /// Whether the machine joins the tailnet: its mux server keeps a node
+ /// running while it does.
pub enabled: bool,
/// Whether the node takes SSH connections from the tailnet to this
/// machine's own sshd: what `thinkterm tailnet up` turns on, for a
@@ -167,7 +168,7 @@ pub(crate) fn lock_for_update() -> Result<File> {
/// Held for as long as a process holds the machine's node: two processes on
/// one node's keys would take the node from each other -- a debug build's
-/// mux server beside the release one's. None while another process holds
+/// `thinkterm-tailnet` beside the release one's. None while another process holds
/// it. Released when dropped, or when the process ends.
#[cfg(all(feature = "node", any(target_os = "macos", target_os = "linux")))]
pub(crate) fn try_hold_node() -> Result<Option<File>> {
@@ -213,10 +214,7 @@ fn open_lock(name: &str) -> Result<File> {
/// What tells one version of the file from the next: a save puts a new
/// file in place, so its inode tells even where the clock is too coarse to.
-#[cfg(any(
- all(feature = "node", any(target_os = "macos", target_os = "linux")),
- test
-))]
+#[cfg(any(target_os = "macos", target_os = "linux", test))]
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) struct Stamp {
modified: Option<std::time::SystemTime>,
@@ -226,10 +224,7 @@ pub(crate) struct Stamp {
}
/// None while there is no file.
-#[cfg(any(
- all(feature = "node", any(target_os = "macos", target_os = "linux")),
- test
-))]
+#[cfg(any(target_os = "macos", target_os = "linux", test))]
pub(crate) fn stamp(path: &Path) -> Option<Stamp> {
let meta = std::fs::metadata(path).ok()?;
Some(Stamp {
diff --git a/thinkterm-tailnet/src/status.rs b/thinkterm-tailnet/src/status.rs
index 044c6c9f..0410f122 100644
--- a/thinkterm-tailnet/src/status.rs
+++ b/thinkterm-tailnet/src/status.rs
@@ -1,6 +1,6 @@
-//! Where the node is, as the process holding it -- the mux server -- last
-//! saw. It writes that to a file in the runtime folder, which is how the app
-//! and `thinkterm tailnet` learn what the node is doing without asking it.
+//! Where the node is, as the process holding it -- `thinkterm-tailnet` --
+//! last saw. It writes that to a file in the runtime folder, which is how the
+//! app and `thinkterm tailnet` learn what the node is doing without asking it.
use serde::{Deserialize, Serialize};
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
@@ -125,7 +125,7 @@ impl Peer {
/// The status file's contents.
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct Report {
- /// The process that wrote it, to tell a report from a server that has
+ /// The process that wrote it, to tell a report from a process that has
/// since gone.
pub pid: u32,
pub status: Status,
@@ -149,10 +149,10 @@ pub(crate) fn read_report() -> Option<Report> {
}
/// The last report, if the process that wrote it is still running: one
-/// left behind by a server that has since gone describes nothing. One about
-/// a node in the tailnet also needs the node's socket to answer: the
-/// server's number may have gone to another process since, after a crash
-/// or a restart.
+/// left behind by a process that has since gone describes nothing. One
+/// about a node in the tailnet also needs the node's socket to answer: the
+/// process's number may have gone to another one since, after a crash or a
+/// restart.
pub fn read_live_report() -> Option<Report> {
read_report().filter(|report| {
process_is_running(report.pid) && (!holds_node(&report.status) || node_answers())
diff --git a/wezterm-mux-server/Cargo.toml b/wezterm-mux-server/Cargo.toml
index d919dfc0..8a39d224 100644
--- a/wezterm-mux-server/Cargo.toml
+++ b/wezterm-mux-server/Cargo.toml
@@ -18,8 +18,9 @@ path = "src/main.rs"
[features]
default = ["tailnet"]
# Joining a tailnet with no Tailscale app installed (thinkterm-tailnet): the
-# server is the process that holds the machine's node.
-tailnet = ["dep:thinkterm-tailnet", "thinkterm-tailnet/node"]
+# server keeps `thinkterm-tailnet` running, which holds the machine's node in
+# a process of its own (see thinkterm-tailnet-node).
+tailnet = ["dep:thinkterm-tailnet"]
[dependencies]
thinkterm-tailnet = { workspace = true, optional = true }
diff --git a/wezterm-mux-server/src/main.rs b/wezterm-mux-server/src/main.rs
index 9b823d0b..a4537545 100644
--- a/wezterm-mux-server/src/main.rs
+++ b/wezterm-mux-server/src/main.rs
@@ -452,11 +452,11 @@ async fn async_run(
let config = config::configuration();
let took_over = takeover.is_some();
// Before anything connects: a takeover's sessions reconnect at once.
- // Until the predecessor lets go of the node, theirs to the tailnet's
- // machines wait for this server's own (see follow_tailnet).
+ // Until the predecessor's node has gone with it, theirs to the tailnet's
+ // machines wait for this server's own (see keep_tailnet_node).
#[cfg(feature = "tailnet")]
{
- // Through its own node, or whichever process here holds it.
+ // Through the node of whichever process here holds it.
thinkterm_tailnet::use_for_ssh_in_server();
if took_over {
thinkterm_tailnet::will_hold();
@@ -519,10 +519,11 @@ async fn async_run(
let web_servers = config.web_servers.clone();
thread::spawn(move || {
wezterm_mux_server_impl::handoff::wait_for_predecessor_exit(stream);
- // The predecessor's node went with it: two processes on one
- // node's keys would take the node from each other.
+ // The predecessor's node goes with it, as its `thinkterm-tailnet`
+ // ends with it: two processes on one node's keys would take the
+ // node from each other.
#[cfg(feature = "tailnet")]
- follow_tailnet();
+ keep_tailnet_node();
// Its descriptors close one after another as it exits; the
// handoff socket may answer for a while after the stream did.
// A server that cannot be taken over is one whose sessions the
@@ -570,7 +571,7 @@ async fn async_run(
let _ = (takeover, daemonized);
#[cfg(feature = "tailnet")]
if !took_over {
- follow_tailnet();
+ keep_tailnet_node();
}
let _config_subscription = config::subscribe_to_config_reload(move || {
promise::spawn::spawn_into_main_thread(async move {
@@ -614,12 +615,14 @@ async fn async_run(
Ok(())
}
-/// Holds the machine's node while the tailnet is on -- the app's switch or
-/// `thinkterm tailnet up` (see thinkterm_tailnet) -- taking SSH connections
-/// from it to this machine's sshd when `thinkterm tailnet up` said to.
+/// Keeps the machine's node running while the tailnet is on -- the app's
+/// switch or `thinkterm tailnet up` (see thinkterm_tailnet) -- in
+/// `thinkterm-tailnet`, a program of its own, which also takes SSH
+/// connections from the tailnet to this machine's sshd when `thinkterm
+/// tailnet up` said to.
#[cfg(feature = "tailnet")]
-fn follow_tailnet() {
- thinkterm_tailnet::follow(|| {});
+fn keep_tailnet_node() {
+ thinkterm_tailnet::keep_node();
}
/// The pid file the predecessor handed over: same lock, our pid in it.
diff --git a/wezterm/src/tailnet.rs b/wezterm/src/tailnet.rs
index 987fe927..16e7548b 100644
--- a/wezterm/src/tailnet.rs
+++ b/wezterm/src/tailnet.rs
@@ -1,7 +1,8 @@
//! `thinkterm tailnet`: a tailnet joined without the Tailscale app, for a
-//! machine nobody sits at. The mux server holds the node and lets SSH from
-//! the tailnet through to this machine's sshd; this command only changes the
-//! settings the server follows and reads back what it reports.
+//! machine nobody sits at. The mux server keeps the node running, in
+//! `thinkterm-tailnet`, which lets SSH from the tailnet through to this
+//! machine's sshd; this command only changes the settings they follow and
+//! reads back what the node reports.
use anyhow::{bail, Context};
use clap::{Parser, Subcommand};
@@ -252,8 +253,9 @@ fn wait_until_up(revision: u64) -> anyhow::Result<()> {
let report = fresh_report(revision);
if report.is_none() && start.elapsed() > Duration::from_secs(10) {
println!(
- "Saved. No mux server has picked it up yet: it joins when one starts \
- (`thinkterm-mux-server --daemonize`)."
+ "Saved. Nothing has picked it up yet: it joins once a mux server runs \
+ (`thinkterm-mux-server --daemonize`) and has started thinkterm-tailnet, \
+ whose log says why if it did not."
);
return Ok(());
}
@@ -326,7 +328,7 @@ fn print_status(settings: &Settings) -> anyhow::Result<()> {
);
let Some(report) = report else {
if settings.enabled {
- println!("Node: not running (no mux server is)");
+ println!("Node: not running (no mux server is, or its thinkterm-tailnet is not)");
}
return Ok(());
};