Skip to content

A modern terminal with a built-in multiplexer: close it and your sessions keep running. Run shells and tons of AI agents across your machines, then pick them up from desktop, browser, TUI and more. Written in Rust with no UI framework: interface drawn straight on the GPU. No Electron, GPUI or webview.

Offer to update a remote mux server that no longer matches this client

RoversX57d08bc16c65
diff --git a/wezterm-client/Cargo.toml b/wezterm-client/Cargo.toml
index c9869523..f11690cb 100644
--- a/wezterm-client/Cargo.toml
+++ b/wezterm-client/Cargo.toml
@@ -32,6 +32,7 @@ smol.workspace = true
termwiz.workspace = true
textwrap.workspace = true
thinkterm-proto.workspace = true
+thinkterm-update.workspace = true
thiserror.workspace = true
umask.workspace = true
url.workspace = true
diff --git a/wezterm-client/src/client.rs b/wezterm-client/src/client.rs
index b307e0dc..65842915 100644
--- a/wezterm-client/src/client.rs
+++ b/wezterm-client/src/client.rs
@@ -1243,13 +1243,19 @@ impl Reconnectable {
/// This string is passed directly to SSH exec, so avoid shell-specific
/// fallback logic here; users can set remote_wezterm_path or
/// override_proxy_command when their remote binary has another name.
+ ///
+ /// `~/.local/bin` is tried first: it is where install.sh (and the remote
+ /// update that runs it) puts the binary, and a non-interactive ssh shell
+ /// does not source the startup files that would add it to PATH. A copy
+ /// the person installed there for their own user is also the more
+ /// deliberate choice over a system-wide one.
fn remote_mux_command(path: &Option<String>, args: &str) -> String {
match path {
Some(path) => format!("{path} {args}"),
- // The command runs via the remote user's shell: prefer a
- // thinkterm binary, fall back to a wezterm one.
None => format!(
- "if command -v thinkterm >/dev/null 2>&1; \
+ "if [ -x \"$HOME/.local/bin/thinkterm\" ]; \
+ then exec \"$HOME/.local/bin/thinkterm\" {args}; \
+ elif command -v thinkterm >/dev/null 2>&1; \
then exec thinkterm {args}; else exec wezterm {args}; fi"
),
}
@@ -1626,6 +1632,7 @@ impl Client {
let is_local = reconnectable.is_local();
let (sender, mut receiver) = unbounded();
let client_id = mux::client::generate_client_id();
+ let reader_client_id = client_id.clone();
let connection_phase = Arc::new(AtomicU8::new(ClientConnectionPhase::Registering as u8));
let reader_connection_phase = Arc::clone(&connection_phase);
let (resume_reconnect_tx, resume_reconnect_rx) = channel::<()>();
@@ -1905,7 +1912,7 @@ impl Client {
ui.close();
}
- async fn detach(local_domain_id: DomainId) -> anyhow::Result<()> {
+ async fn detach(local_domain_id: DomainId, client_id: ClientId) -> anyhow::Result<()> {
if let Some(mux) = Mux::try_get() {
let client_domain = mux
.get_domain(local_domain_id)
@@ -1916,13 +1923,30 @@ impl Client {
.ok_or_else(|| {
anyhow!("domain {} is not a ClientDomain instance", local_domain_id)
})?;
- client_domain.perform_detach();
+ // A client that was replaced while it was still alive --
+ // the attach path does that after updating a remote
+ // server -- must not tear down the domain its successor
+ // now owns.
+ let owned = client_domain
+ .inner()
+ .map(|inner| inner.client.client_id == client_id)
+ .unwrap_or(true);
+ if owned {
+ client_domain.perform_detach();
+ } else {
+ log::info!(
+ "domain {} is now attached through another client; \
+ the superseded client leaves it alone",
+ local_domain_id
+ );
+ }
}
Ok(())
}
if let Some(domain_id) = local_domain_id {
+ let client_id = reader_client_id;
promise::spawn::spawn_into_main_thread(async move {
- detach(domain_id).await.ok();
+ detach(domain_id, client_id).await.ok();
})
.detach();
}
diff --git a/wezterm-client/src/domain.rs b/wezterm-client/src/domain.rs
index 26c261d4..70f60f85 100644
--- a/wezterm-client/src/domain.rs
+++ b/wezterm-client/src/domain.rs
@@ -1904,7 +1904,7 @@ impl ClientDomain {
self.inner()?.client.remote_os_release()
}
- fn inner(&self) -> Option<Arc<ClientInner>> {
+ pub(crate) fn inner(&self) -> Option<Arc<ClientInner>> {
self.inner.lock().unwrap().as_ref().map(Arc::clone)
}
@@ -5021,26 +5021,67 @@ impl ClientDomain {
ui.async_run_and_log_error({
let ui = ui.clone();
async move {
- let mut cloned_ui = ui.clone();
- let client = spawn_into_new_thread(move || match &config {
- ClientDomainConfig::Unix(unix) => {
- let initial = true;
- let no_auto_start = false;
- Client::new_unix_domain(
- Some(domain_id),
- unix,
- initial,
- &mut cloned_ui,
- no_auto_start,
- )
- }
- ClientDomainConfig::Tls(tls) => Client::new_tls(domain_id, tls, &mut cloned_ui),
- ClientDomainConfig::Ssh(ssh) => Client::new_ssh(domain_id, ssh, &mut cloned_ui),
- })
- .await?;
+ let connect = |config: ClientDomainConfig, ui: ConnectionUI| {
+ spawn_into_new_thread(move || {
+ let mut cloned_ui = ui;
+ match &config {
+ ClientDomainConfig::Unix(unix) => {
+ let initial = true;
+ let no_auto_start = false;
+ Client::new_unix_domain(
+ Some(domain_id),
+ unix,
+ initial,
+ &mut cloned_ui,
+ no_auto_start,
+ )
+ }
+ ClientDomainConfig::Tls(tls) => {
+ Client::new_tls(domain_id, tls, &mut cloned_ui)
+ }
+ ClientDomainConfig::Ssh(ssh) => {
+ Client::new_ssh(domain_id, ssh, &mut cloned_ui)
+ }
+ }
+ })
+ };
+ let mut client = connect(config.clone(), ui.clone()).await?;
ui.output_str("Checking server version\n");
- client.verify_version_compat(&ui).await?;
+ if let Err(err) = client.verify_version_compat(&ui).await {
+ // A codec mismatch over ssh is the one connect failure
+ // this side can repair: the installer runs over the same
+ // hop, at this client's version. Anything else, and a
+ // declined offer, keeps the original error.
+ let mismatch = err.downcast_ref::<crate::client::IncompatibleVersionError>();
+ let (ClientDomainConfig::Ssh(ssh), Some(mismatch)) = (&config, mismatch)
+ else {
+ return Err(err);
+ };
+ let outcome = {
+ let ssh = ssh.clone();
+ let ui = ui.clone();
+ let mismatch = mismatch.clone();
+ spawn_into_new_thread(move || {
+ crate::remote_update::offer_remote_update(&ssh, &ui, &mismatch)
+ })
+ .await
+ };
+ match outcome {
+ Ok(crate::remote_update::RemoteUpdateOutcome::Updated {
+ restarted: true,
+ }) => {
+ ui.output_str("Reconnecting to the updated server\n");
+ client = connect(config.clone(), ui.clone()).await?;
+ client.verify_version_compat(&ui).await?;
+ }
+ Ok(_) => return Err(err),
+ Err(update_err) => {
+ ui.output_str(&format!("Remote update failed: {update_err:#}\n"));
+ return Err(err);
+ }
+ }
+ }
ui.output_str("Version check OK! Requesting pane list...\n");
let panes = client.list_panes().await?;
diff --git a/wezterm-client/src/lib.rs b/wezterm-client/src/lib.rs
index 07352115..e5753a8b 100644
--- a/wezterm-client/src/lib.rs
+++ b/wezterm-client/src/lib.rs
@@ -2,3 +2,4 @@ pub mod client;
pub mod discovery;
pub mod domain;
pub mod pane;
+pub mod remote_update;
diff --git a/wezterm-client/src/remote_update.rs b/wezterm-client/src/remote_update.rs
new file mode 100644
index 00000000..3da60a67
--- /dev/null
+++ b/wezterm-client/src/remote_update.rs
@@ -0,0 +1,282 @@
+//! Fixing a version mismatch with a remote mux server from this side of the
+//! ssh connection.
+//!
+//! The client and the server refuse to talk across a codec change, and the
+//! usual way that happens is the desktop being upgraded while the server on
+//! some host keeps running the build from last month. Asking the person to
+//! log in there and update by hand is the wrong answer when the client
+//! already holds an ssh session to the host: it can run the installer over
+//! that session, at the client's own version, so the two match again.
+//!
+//! What this deliberately does not do:
+//!
+//! - guess. It asks before installing anything and again before restarting
+//! the server, because a restart ends every session the server holds.
+//! - pick "the latest release". The server has to match *this* client, which
+//! may not be the latest; a development build has no release to match at
+//! all and is told so.
+//! - keep sessions alive across the restart. That needs the server to hand
+//! its ptys to its successor, which it cannot do yet.
+
+use crate::client::IncompatibleVersionError;
+use anyhow::Context;
+use config::SshDomain;
+use mux::connui::ConnectionUI;
+use portable_pty::Child as _;
+use std::io::Read;
+use wezterm_ssh::Session;
+
+#[derive(Debug, PartialEq, Eq)]
+pub enum RemoteUpdateOutcome {
+ /// Nothing was changed on the host: the person said no, or the client is
+ /// not a release build.
+ Declined,
+ /// The new version is installed. `restarted` says whether the old mux
+ /// server was stopped, so that the next connect starts the new one.
+ Updated { restarted: bool },
+}
+
+/// Offer to bring the mux server on `ssh_dom`'s host to this client's
+/// version, after the handshake reported `err`. Interactive: every question
+/// goes through `ui`, and a UI that cannot ask (a headless attach) simply
+/// declines.
+pub fn offer_remote_update(
+ ssh_dom: &SshDomain,
+ ui: &ConnectionUI,
+ err: &IncompatibleVersionError,
+) -> anyhow::Result<RemoteUpdateOutcome> {
+ let local = thinkterm_update::running_release_version();
+ let host = ssh_dom.remote_address.clone();
+
+ // A configured remote path is used verbatim for every connect, so an
+ // install under ~/.local would never be the binary that answers.
+ if let Some(path) = &ssh_dom.remote_wezterm_path {
+ ui.output_str(&format!(
+ "\nThis domain connects through the configured remote path {path}, so an \
+ install under ~/.local on {host} would not be used. Update that binary \
+ to {local} by hand.\n"
+ ));
+ return Ok(RemoteUpdateOutcome::Declined);
+ }
+
+ if !thinkterm_update::is_release_version(&local) {
+ ui.output_str(&format!(
+ "\nThis client is a development build ({local}), so there is no release \
+ to install on {host} that matches it. Build and install a matching \
+ thinkterm-mux-server there by hand.\n"
+ ));
+ return Ok(RemoteUpdateOutcome::Declined);
+ }
+
+ ui.output_str(&format!(
+ "\nThe mux server on {host} is {} (codec {}); this client is {local} (codec {}).\n",
+ err.version,
+ err.codec_vers,
+ codec::CODEC_VERSION
+ ));
+ // The ssh session comes first: the host has to be asked what is
+ // installed there before the question can say what will change. A
+ // workstation with the desktop variant keeps its GUI; --server would
+ // have removed it.
+ let ssh_config = mux::ssh::ssh_domain_to_ssh_config(ssh_dom)?;
+ let mut ui_for_auth = ui.clone();
+ let session = mux::ssh::ssh_connect_with_ui_and_password(
+ ssh_config,
+ &mut ui_for_auth,
+ ssh_dom.stored_password.clone(),
+ )
+ .with_context(|| format!("opening an ssh session to {host} for the update"))?;
+ // A host the release binaries cannot run on gets the answer straight
+ // away, rather than an installer run that fails at its first check.
+ if let Some(reason) = host_cannot_take_a_release(&session) {
+ ui.output_str(&format!(
+ "{reason}, so the release build of {local} cannot be installed on {host}.\n{}\n",
+ thinkterm_update::build_from_source_instructions(&local)
+ ));
+ return Ok(RemoteUpdateOutcome::Declined);
+ }
+
+ let variant = remote_variant(&session);
+ let command = thinkterm_update::install_command(variant, &local);
+
+ let answer = ui.input(&format!(
+ "ThinkTerm can install {local} there now ({variant} variant, as the ssh user, \
+ under ~/.local, with no root):\n \
+ {command}\n\
+ Install it? [y/N] ",
+ ));
+ let answer = match answer {
+ Ok(answer) => answer,
+ Err(err) => {
+ log::info!("remote update not offered: {err:#}");
+ return Ok(RemoteUpdateOutcome::Declined);
+ }
+ };
+ if !is_yes(&answer) {
+ ui.output_str(&format!(
+ "Not installing. To do it yourself, run this on {host}:\n {command}\n"
+ ));
+ return Ok(RemoteUpdateOutcome::Declined);
+ }
+
+ ui.output_str(&format!("\nInstalling ThinkTerm {local} on {host}...\n"));
+ let status = run_and_relay(&session, ui, &command)?;
+ if !status {
+ ui.output_str(&format!(
+ "\nThe installer on {host} failed; its output is above. Nothing was changed there.\n\
+ If it refused the host (glibc, musl, architecture), a release binary cannot run on it.\n{}\n",
+ thinkterm_update::build_from_source_instructions(&local)
+ ));
+ anyhow::bail!("the installer on {host} failed");
+ }
+
+ // The install is done whatever happens to this question: a UI that has
+ // gone away by now leaves the old server running, which is the safe
+ // answer, not a failure.
+ let answer = ui
+ .input(&format!(
+ "\nInstalled. The old mux server on {host} is still running and still \
+ answers with the old version.\n\
+ Restart it now? Every session running in it will end. [y/N] ",
+ ))
+ .unwrap_or_default();
+ if !is_yes(&answer) {
+ ui.output_str(&format!(
+ "Leaving it running. Stop it when you are ready (`{STOP_HINT}` on {host}) \
+ and reconnect; the next connect starts the new version.\n"
+ ));
+ return Ok(RemoteUpdateOutcome::Updated { restarted: false });
+ }
+
+ let stopped = run_and_relay(&session, ui, STOP_COMMAND)?;
+ if !stopped {
+ ui.output_str(&format!(
+ "No running mux server was found to stop on {host}; reconnecting anyway.\n"
+ ));
+ } else {
+ ui.output_str("Old server stopped; reconnecting.\n");
+ }
+ Ok(RemoteUpdateOutcome::Updated { restarted: true })
+}
+
+/// What a person types to stop the server themselves.
+const STOP_HINT: &str = "pkill -f thinkterm-mux-server";
+
+/// Stop the invoking user's mux server. Matched on the full command line,
+/// not the process name: Linux truncates the name to 15 characters, so
+/// `-x thinkterm-mux-server` can never match there. The bracket keeps the
+/// shell running this very command line from matching itself. Both binary
+/// names, because a host set up before the rename may still run a
+/// wezterm-mux-server under a thinkterm domain. Exit status is pkill's:
+/// 0 when something was signalled, 1 when nothing matched.
+const STOP_COMMAND: &str = "pkill -u \"$(id -u)\" -f '[t]hinkterm-mux-server' \
+ || pkill -u \"$(id -u)\" -f '[w]ezterm-mux-server'";
+
+/// Why the host cannot run a release binary, if it cannot: the same two
+/// checks `install.sh` makes first, made here so the answer comes before
+/// the question. An unreadable answer is not a refusal; the script will
+/// check again.
+fn host_cannot_take_a_release(session: &Session) -> Option<String> {
+ let probe = "uname -s; uname -m; getconf GNU_LIBC_VERSION 2>/dev/null || ldd --version 2>&1 | head -n1";
+ let out = run_capture(session, probe).ok()?;
+ let mut lines = out.lines().map(str::trim);
+ let os = lines.next()?;
+ let arch = lines.next()?;
+ let libc = lines.next().unwrap_or("");
+ if os == "Linux" && !matches!(arch, "x86_64" | "aarch64") {
+ return Some(format!("{host_arch} has no release build", host_arch = arch));
+ }
+ if os == "Linux" && libc.to_ascii_lowercase().contains("musl") {
+ return Some("The host uses musl libc".to_string());
+ }
+ if os == "Linux" {
+ if let Some(version) = thinkterm_update::parse_glibc_version(libc) {
+ if version < thinkterm_update::MIN_GLIBC {
+ return Some(format!(
+ "The host has glibc {}.{} and the release binaries need {}.{}",
+ version.0,
+ version.1,
+ thinkterm_update::MIN_GLIBC.0,
+ thinkterm_update::MIN_GLIBC.1
+ ));
+ }
+ }
+ }
+ None
+}
+
+/// Which variant the script installed on the host, read from its manifest.
+/// Falls back to the server variant, which is the only one that makes sense
+/// on a host with no install to preserve.
+fn remote_variant(session: &Session) -> &'static str {
+ let read = "sed -n 's/^variant=//p' \"$HOME/.local/share/thinkterm/install-manifest\" 2>/dev/null";
+ match run_capture(session, read).as_deref().map(str::trim) {
+ Ok("desktop") => "desktop",
+ _ => "server",
+ }
+}
+
+/// Run `command` and return its stdout, for the small questions asked of
+/// the host before anything is changed on it.
+fn run_capture(session: &Session, command: &str) -> anyhow::Result<String> {
+ let exec = smol::block_on(session.exec(command, None))
+ .with_context(|| format!("running `{command}` over ssh"))?;
+ let mut stdout = exec.stdout;
+ let mut child = exec.child;
+ drop(exec.stdin);
+ drop(exec.stderr);
+ let mut out = Vec::new();
+ stdout.read_to_end(&mut out)?;
+ let _ = child.wait();
+ Ok(String::from_utf8_lossy(&out).into_owned())
+}
+
+fn is_yes(answer: &str) -> bool {
+ matches!(answer.trim(), "y" | "Y" | "yes" | "YES")
+}
+
+/// Run `command` on the session, copying its stdout and stderr into the UI
+/// as they arrive, and return whether it exited successfully.
+fn run_and_relay(session: &Session, ui: &ConnectionUI, command: &str) -> anyhow::Result<bool> {
+ let exec = smol::block_on(session.exec(command, None))
+ .with_context(|| format!("running `{command}` over ssh"))?;
+ let mut stdout = exec.stdout;
+ let mut stderr = exec.stderr;
+ let mut child = exec.child;
+ drop(exec.stdin);
+
+ let ui_err = ui.clone();
+ let stderr_thread = std::thread::spawn(move || {
+ let mut buf = [0u8; 1024];
+ while let Ok(len) = stderr.read(&mut buf) {
+ if len == 0 {
+ break;
+ }
+ ui_err.output_str(&String::from_utf8_lossy(&buf[..len]));
+ }
+ });
+ let mut buf = [0u8; 1024];
+ while let Ok(len) = stdout.read(&mut buf) {
+ if len == 0 {
+ break;
+ }
+ ui.output_str(&String::from_utf8_lossy(&buf[..len]));
+ }
+ let _ = stderr_thread.join();
+ let status = child.wait().with_context(|| format!("waiting for `{command}`"))?;
+ Ok(status.success())
+}
+
+#[cfg(test)]
+mod tests {
+ use super::is_yes;
+
+ #[test]
+ fn only_an_explicit_yes_counts() {
+ assert!(is_yes("y"));
+ assert!(is_yes(" yes\n"));
+ assert!(!is_yes(""));
+ assert!(!is_yes("n"));
+ assert!(!is_yes("maybe"));
+ }
+}