Skip to content

A modern terminal with a built-in multiplexer: close it and your sessions keep running. Run shells and tons of AI agents across your machines, then pick them up from desktop, browser, TUI and more. Written in Rust with no UI framework: interface drawn straight on the GPU. No Electron, GPUI or webview.

56.8 KB
Raw Blame
use crate::authentication::AuthenticationInfo;
use crate::channelwrap::ChannelWrap;
use crate::config::ConfigMap;
use crate::dirwrap::DirWrap;
use crate::filewrap::FileWrap;
use crate::pty::*;
use crate::session::{
    Exec, ExecResult, SessionEvent, SessionRequest, SessionShutdown, SignalChannel,
};
use crate::sessionwrap::SessionWrap;
use crate::sftp::dir::{Dir, DirId, DirRequest};
use crate::sftp::file::{File, FileId, FileRequest};
use crate::sftp::{OpenWithMode, SftpChannelResult, SftpRequest};
use crate::sftpwrap::SftpWrap;
use anyhow::{anyhow, Context};
use camino::Utf8PathBuf;
use filedescriptor::{
    poll, pollfd, socketpair, AsRawSocketDescriptor, FileDescriptor, POLLIN, POLLOUT,
};
use portable_pty::ExitStatus;
use smol::channel::{bounded, Receiver, Sender, TryRecvError};
use socket2::Socket;
use std::collections::{HashMap, VecDeque};
use std::io::{Read, Write};
use std::sync::{Arc, OnceLock};
use std::time::{Duration, Instant};

#[derive(Debug)]
pub(crate) struct DescriptorState {
    pub fd: Option<FileDescriptor>,
    pub buf: VecDeque<u8>,
}

/// What one channel stream can hold on its way to or from the pipe. Each
/// loop iteration moves at most this much per stream, so it bounds the
/// throughput of a busy channel; 8 KiB made a mux client's output crawl.
pub(crate) const CHANNEL_BUFFER: usize = 64 * 1024;

/// Whether `address` is the host's own, as opposed to one of its others.
fn is_own(address: &(String, u16), hostname: &str, port: u16) -> bool {
    address.0 == hostname && address.1 == port
}

pub(crate) struct ChannelInfo {
    pub channel_id: ChannelId,
    pub channel: ChannelWrap,
    pub exit: Option<Sender<ExitStatus>>,
    pub exited: bool,
    pub descriptors: [DescriptorState; 3],
}

pub(crate) type ChannelId = usize;

pub(crate) struct SessionInner {
    pub config: ConfigMap,
    pub tx_event: Sender<SessionEvent>,
    pub rx_req: Receiver<SessionRequest>,
    pub channels: HashMap<ChannelId, ChannelInfo>,
    pub files: HashMap<FileId, FileWrap>,
    pub dirs: HashMap<DirId, DirWrap>,
    pub next_channel_id: ChannelId,
    pub next_file_id: FileId,
    pub sender_read: FileDescriptor,
    pub session_was_dropped: bool,
    pub shown_accept_env_error: bool,
    pub last_keep_alive: Instant,
    pub keep_alive: Option<Duration>,
    pub shutdown: Arc<SessionShutdown>,
    pub authentication: Arc<OnceLock<AuthenticationInfo>>,
}

impl Drop for SessionInner {
    fn drop(&mut self) {
        self.shutdown.clear();
        log::trace!("Dropping SessionInner");
    }
}

impl SessionInner {
    pub fn run(&mut self) {
        if let Err(err) = self.run_impl() {
            self.tx_event
                .try_send(SessionEvent::Error(format!("{:#}", err)))
                .ok();
        }
    }

    fn run_impl(&mut self) -> anyhow::Result<()> {
        self.shutdown.check()?;
        let backend = self
            .config
            .get("wezterm_ssh_backend")
            .map(|s| s.as_str())
            .unwrap_or(
                #[cfg(feature = "libssh-rs")]
                "libssh",
                #[cfg(not(feature = "libssh-rs"))]
                "ssh2",
            );
        match backend {
            #[cfg(feature = "ssh2")]
            "ssh2" => self.run_impl_ssh2(),

            #[cfg(not(feature = "ssh2"))]
            "ssh2" => anyhow::bail!(
                "invalid wezterm_ssh_backend value: {}, not compiled with `ssh2`",
                backend
            ),

            #[cfg(feature = "libssh-rs")]
            "libssh" => self.run_impl_libssh(),

            #[cfg(not(feature = "libssh-rs"))]
            "libssh" => anyhow::bail!(
                "invalid wezterm_ssh_backend value: {}, not compiled with `libssh`",
                backend
            ),

            _ => anyhow::bail!(
                "invalid wezterm_ssh_backend value: {}, expected either `ssh2` or `libssh`",
                backend
            ),
        }
    }

    #[cfg(feature = "libssh-rs")]
    fn run_impl_libssh(&mut self) -> anyhow::Result<()> {
        let hostname = self
            .config
            .get("hostname")
            .ok_or_else(|| anyhow!("hostname not present in config"))?
            .to_string();
        let user = self
            .config
            .get("user")
            .ok_or_else(|| anyhow!("username not present in config"))?
            .to_string();
        let port = self
            .config
            .get("port")
            .ok_or_else(|| anyhow!("port is always set in config loader"))?
            .parse::<u16>()?;

        self.tx_event
            .try_send(SessionEvent::Banner(Some(format!(
                "Using libssh-rs to connect to {}@{}:{}",
                user, hostname, port
            ))))
            .context("notifying user of banner")?;

        let verbose = self
            .config
            .get("wezterm_ssh_verbose")
            .map(|s| s.as_str())
            .unwrap_or("false")
            == "true";
        if verbose {
            /// libssh logs to stderr, but on Windows in the GUI there isn't a valid
            /// stderr for it to log to.
            /// So, we redirect logging via our own log callback and pipe it via
            /// the `log` crate.
            unsafe extern "C" fn log_callback(
                _priority: std::os::raw::c_int,
                function: *const std::os::raw::c_char,
                message: *const std::os::raw::c_char,
                _userdata: *mut std::os::raw::c_void,
            ) {
                use std::ffi::CStr;
                let function = CStr::from_ptr(function).to_string_lossy().to_string();
                let message = CStr::from_ptr(message).to_string_lossy().to_string();

                // The message typically has "function: message" prefixed, which
                // looks redundant when logged with the function prefix by the
                // logging crate.
                // Strip that off!
                let message = match message.strip_prefix(&format!("{}: ", function)) {
                    Some(m) => m,
                    None => &message,
                };

                log::logger().log(
                    &log::Record::builder()
                        .args(format_args!("{}", message))
                        .level(log::Level::Info)
                        .module_path(Some(&function))
                        .target(&format!("libssh::{}", function))
                        .build(),
                );
            }
            unsafe {
                libssh_rs::sys::ssh_set_log_callback(Some(log_callback));
            }
        }

        // A session per address tried: see `connect_to_host`.
        let mut not_this_host = vec![];
        let (sess, _child, reached) = loop {
            let sess = self.libssh_session(&hostname, &user, port, verbose)?;
            let (sock, child, reached) =
                self.connect_to_host(&hostname, port, verbose, &not_this_host)?;
            self.shutdown.watch(&sock)?;
            let raw = {
                #[cfg(unix)]
                {
                    use std::os::unix::io::IntoRawFd;
                    sock.into_raw_fd()
                }
                #[cfg(windows)]
                {
                    use std::os::windows::io::IntoRawSocket;
                    sock.into_raw_socket()
                }
            };

            sess.set_option(libssh_rs::SshOption::Socket(raw))?;

            sess.connect()
                .with_context(|| format!("Connecting to {hostname}:{port}"))?;

            if let Some(other) = reached.as_ref().filter(|at| !is_own(at, &hostname, port)) {
                if matches!(
                    sess.is_known_server()?,
                    libssh_rs::KnownHosts::Changed | libssh_rs::KnownHosts::Other
                ) {
                    self.pass_over(&hostname, other)?;
                    not_this_host.push(other.clone());
                    continue;
                }
            }
            break (sess, child, reached);
        };

        let banner = sess.get_server_banner()?;
        self.tx_event
            .try_send(SessionEvent::Banner(Some(banner)))
            .context("notifying user of banner")?;

        let host_verified = self.host_verification_libssh(&sess, &hostname, port)?;
        let authentication = self.authenticate_libssh(&sess)?.finish(host_verified);
        self.authentication
            .set(authentication)
            .map_err(|_| anyhow!("SSH authentication was already recorded"))?;
        if let Some(reached) = &reached {
            crate::addresses::remember_reached(&hostname, port, reached);
        }

        if let Ok(banner) = sess.get_issue_banner() {
            self.tx_event
                .try_send(SessionEvent::Banner(Some(banner)))
                .context("notifying user of banner")?;
        }

        self.tx_event
            .try_send(SessionEvent::Authenticated)
            .context("notifying user that session is authenticated")?;

        if let Some("yes") = self.config.get("forwardagent").map(|s| s.as_str()) {
            if self.identity_agent().is_some() {
                sess.enable_accept_agent_forward(true);
            } else {
                log::error!("ForwardAgent is set to yes, but IdentityAgent is not set");
            }
        }
        sess.set_blocking(false);
        let mut sess = SessionWrap::with_libssh(sess);
        let result = self.request_loop(&mut sess);
        self.close_sftp_handles(&mut sess);
        result
    }

    /// A libssh session for the host, with every option set except the
    /// socket.
    #[cfg(feature = "libssh-rs")]
    fn libssh_session(
        &self,
        hostname: &str,
        user: &str,
        port: u16,
        verbose: bool,
    ) -> anyhow::Result<libssh_rs::Session> {
        let sess = libssh_rs::Session::new()?;
        if verbose {
            sess.set_option(libssh_rs::SshOption::LogLevel(libssh_rs::LogLevel::Packet))?;
        }
        sess.set_option(libssh_rs::SshOption::Hostname(hostname.to_string()))?;
        sess.set_option(libssh_rs::SshOption::User(Some(user.to_string())))?;
        sess.set_option(libssh_rs::SshOption::Port(port))?;
        sess.options_parse_config(None)?; // FIXME: overridden config path?
        if let Some(agent) = self.config.get("identityagent") {
            sess.set_option(libssh_rs::SshOption::IdentityAgent(Some(agent.clone())))?;
        }
        if let Some(files) = self.config.get("identityfile") {
            for file in files.split_whitespace() {
                sess.set_option(libssh_rs::SshOption::AddIdentity(file.to_string()))?;
            }
        }
        if let Some(kh) = self.config.get("userknownhostsfile") {
            for file in kh.split_whitespace() {
                sess.set_option(libssh_rs::SshOption::KnownHosts(Some(file.to_string())))?;
                break;
            }
        }
        if let Some(types) = self.config.get("pubkeyacceptedtypes") {
            sess.set_option(libssh_rs::SshOption::PublicKeyAcceptedTypes(
                types.to_string(),
            ))?;
        }
        if let Some(bind_addr) = self.config.get("bindaddress") {
            sess.set_option(libssh_rs::SshOption::BindAddress(bind_addr.to_string()))?;
        }
        if let Some(host_key) = self.config.get("hostkeyalgorithms") {
            sess.set_option(libssh_rs::SshOption::HostKeys(host_key.to_string()))?;
        }
        Ok(sess)
    }

    #[cfg(feature = "ssh2")]
    fn run_impl_ssh2(&mut self) -> anyhow::Result<()> {
        let verbose = self
            .config
            .get("wezterm_ssh_verbose")
            .map(|s| s.as_str())
            .unwrap_or("false")
            == "true";

        let hostname = self
            .config
            .get("hostname")
            .ok_or_else(|| anyhow!("hostname not present in config"))?
            .to_string();
        let user = self
            .config
            .get("user")
            .ok_or_else(|| anyhow!("username not present in config"))?
            .to_string();
        let port = self
            .config
            .get("port")
            .ok_or_else(|| anyhow!("port is always set in config loader"))?
            .parse::<u16>()?;
        let remote_address = format!("{}:{}", hostname, port);

        self.tx_event
            .try_send(SessionEvent::Banner(Some(format!(
                "Using ssh2 to connect to {}@{}:{}",
                user, hostname, port
            ))))
            .context("notifying user of banner")?;

        // A session per address tried: see `connect_to_host`.
        let mut not_this_host = vec![];
        let (sess, _child, reached) = loop {
            let (sock, child, reached) =
                self.connect_to_host(&hostname, port, verbose, &not_this_host)?;
            self.shutdown.watch(&sock)?;

            let mut sess = ssh2::Session::new()?;
            if verbose {
                sess.trace(ssh2::TraceFlags::all());
            }
            sess.set_blocking(true);
            sess.set_tcp_stream(sock);
            sess.handshake()
                .with_context(|| format!("ssh handshake with {}", remote_address))?;

            if let Some(other) = reached.as_ref().filter(|at| !is_own(at, &hostname, port)) {
                if self.known_key_differs(&sess, &hostname, port)? {
                    self.pass_over(&hostname, other)?;
                    not_this_host.push(other.clone());
                    continue;
                }
            }
            break (sess, child, reached);
        };

        self.tx_event
            .try_send(SessionEvent::Banner(sess.banner().map(|s| s.to_string())))
            .context("notifying user of banner")?;

        let host_verified = self
            .host_verification(&sess, &hostname, port, &remote_address)
            .context("host verification")?;

        let authentication = self
            .authenticate(&sess, &user, &hostname)
            .context("authentication")?;
        self.authentication
            .set(authentication.finish(host_verified))
            .map_err(|_| anyhow!("SSH authentication was already recorded"))?;
        if let Some(reached) = &reached {
            crate::addresses::remember_reached(&hostname, port, reached);
        }

        self.tx_event
            .try_send(SessionEvent::Authenticated)
            .context("notifying user that session is authenticated")?;

        sess.set_blocking(false);

        let mut sess = SessionWrap::with_ssh2(sess);
        let result = self.request_loop(&mut sess);
        self.close_sftp_handles(&mut sess);
        result
    }

    /// Explicitly and directly connect to the requested host because
    /// neither libssh no libssh2 respect addressfamily, so we must
    /// handle it for ourselves.
    /// If proxy_command is set, then we execute that process for ourselves
    /// too, as proxy commands are not supported by libssh2 and are not supported
    /// on Windows in libssh.
    /// With other addresses for the host (see `crate::addresses`), also
    /// gives back the one that answered, and leaves out those in
    /// `not_this_host`: other addresses found serving some other machine.
    fn connect_to_host(
        &self,
        hostname: &str,
        port: u16,
        verbose: bool,
        not_this_host: &[(String, u16)],
    ) -> anyhow::Result<(Socket, Option<KillOnDropChild>, Option<(String, u16)>)> {
        self.shutdown.check()?;
        let dialer = crate::addresses::Dialer::from_config(&self.config)?;
        match self.config.get("proxycommand").map(|s| s.as_str()) {
            Some("none") | None => {}
            Some(proxy_command) => {
                let mut cmd;
                if cfg!(windows) {
                    let comspec = std::env::var("COMSPEC").unwrap_or_else(|_| "cmd".to_string());
                    cmd = std::process::Command::new(comspec);
                    cmd.args(["/c", proxy_command]);
                } else {
                    cmd = std::process::Command::new("sh");
                    cmd.args(["-c", &format!("exec {}", proxy_command)]);
                }

                let (a, b) = socketpair()?;

                cmd.stdin(b.as_stdio()?);
                cmd.stdout(b.as_stdio()?);
                cmd.stderr(std::process::Stdio::inherit());
                let child = KillOnDropChild(
                    cmd.spawn()
                        .with_context(|| format!("spawning ProxyCommand {}", proxy_command))?,
                );

                #[cfg(unix)]
                {
                    use std::os::fd::AsFd;
                    let socket = Socket::from(a.as_fd().try_clone_to_owned()?);
                    self.shutdown.watch(&socket)?;
                }

                #[cfg(unix)]
                unsafe {
                    use passfd::FdPassingExt;
                    use std::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd};

                    let dest = match self.config.get("proxyusefdpass").map(|s| s.as_str()) {
                        Some("yes") => a.as_raw_fd().recv_fd()?,
                        _ => a.into_raw_fd(),
                    };

                    return Ok((Socket::from_raw_fd(dest), Some(child), None));
                }
                #[cfg(windows)]
                unsafe {
                    use std::os::windows::io::{FromRawSocket, IntoRawSocket};
                    return Ok((
                        Socket::from_raw_socket(a.into_raw_socket()),
                        Some(child),
                        None,
                    ));
                }
            }
        }

        let mut candidates = crate::addresses::candidates(&self.config, hostname, port);
        candidates.retain(|address| !not_this_host.contains(address));
        if candidates.len() > 1 {
            if verbose {
                log::info!("trying {hostname}:{port} at {candidates:?}");
            }
            let raced = crate::addresses::race(&candidates, dialer, || self.shutdown.check());
            let (sock, index) = match raced {
                Ok(won) => won,
                Err(err) => {
                    crate::addresses::forget_reached(hostname, port);
                    return Err(err);
                }
            };
            let reached = candidates.swap_remove(index);
            if !is_own(&reached, hostname, port) {
                self.tx_event
                    .try_send(SessionEvent::Banner(Some(format!(
                        "Reached {hostname} at {}:{}",
                        reached.0, reached.1
                    ))))
                    .context("notifying user of banner")?;
            }
            self.shutdown.check()?;
            return Ok((sock, None, Some(reached)));
        }

        let closed = || self.shutdown.check().is_err();
        let sock = dialer.dial_with(hostname, port, verbose, &closed, |sock| {
            self.shutdown.watch(sock)
        })?;
        self.shutdown.check()?;
        Ok((sock, None, None))
    }

    /// An other address answered as some other machine: say so, and the
    /// caller tries the rest.
    fn pass_over(&self, hostname: &str, (host, port): &(String, u16)) -> anyhow::Result<()> {
        log::warn!(
            "{host}:{port} holds a different host key than {hostname}; trying its other addresses"
        );
        self.tx_event
            .try_send(SessionEvent::Banner(Some(format!(
                "{host}:{port} is not {hostname} (its host key differs); trying the others"
            ))))
            .context("notifying user of banner")
    }

    fn do_keepalive(&mut self, sess: &mut SessionWrap) -> anyhow::Result<()> {
        match sess {
            #[cfg(feature = "ssh2")]
            SessionWrap::Ssh2(_sess) => Ok(()),
            #[cfg(feature = "libssh-rs")]
            SessionWrap::LibSsh(sess) => {
                // We implement a very basic keep alive mechanism here;
                // every ServerAliveInterval seconds (if non-zero), we will
                // send an ignore packet.
                // Unlike the openssh client, we do not have a ServerAliveCountMax
                // limit (because it is not clear how we could correctly implement
                // that based on what we can see here in this crate), nor do we
                // explicitly trigger a disconnect if there is an error with
                // the ignore packet.
                if let Some(duration) = self.keep_alive {
                    if self.last_keep_alive.elapsed() >= duration {
                        log::trace!("sending keep alive");
                        self.last_keep_alive = Instant::now();
                        let ignore_me = [0x42; 128];
                        if let Err(err) = sess.sess.send_ignore(&ignore_me) {
                            log::warn!(
                                "Error sending IGNORE packet: {err:#}. Is peer disconnected?"
                            );
                        }
                    }
                }
                Ok(())
            }
        }
    }

    fn request_loop(&mut self, sess: &mut SessionWrap) -> anyhow::Result<()> {
        let mut sleep_delay = Duration::from_millis(100);
        let initial_sleep_delay = Duration::from_millis(100);
        let max_sleep_delay = Duration::from_secs(2);
        let immediate_wakeup = Duration::from_millis(10);
        // Every event that could wake poll without there being anything to
        // do is masked above; what remains is the odd spurious wakeup, and
        // a long nap for one of those is throughput thrown away.
        let busy_loop_throttle = Duration::from_millis(5);
        let busy_loop_warn_every = Duration::from_secs(30);
        let mut busy_loop_started: Option<Instant> = None;
        let mut busy_loop_last_warn: Option<Instant> = None;
        // Immediate wakeups in a row that moved nothing. One is ordinary:
        // libssh reports its socket readable while it holds half a packet,
        // and the read that completes the packet hands over no bytes yet.
        // Napping on every such turn cost a busy channel most of its time.
        let mut idle_wakes: u32 = 0;

        loop {
            self.shutdown.check()?;
            self.do_keepalive(sess)?;
            let mut made_progress = self.tick_io()?;
            made_progress |= self.drain_request_pipe();
            made_progress |= self.dispatch_pending_requests(sess)?;
            made_progress |= self.connect_pending_agent_forward_channels(sess);

            if self.channels.is_empty() && self.session_was_dropped {
                log::trace!(
                    "Stopping session loop as there are no more channels and Session was dropped"
                );
                return Ok(());
            }

            // Reading from the session only pays off while some open stream
            // has room for what may come. With every stream full (the
            // readers on the other side of their pipes are busy), a readable
            // session socket woke poll at once, nothing could be done, and
            // the loop was taken for a busy loop and sent to sleep for 50
            // milliseconds: one buffer per nap, a few hundred kilobytes a
            // second, for as long as the reader lagged. Waiting on the pipes
            // instead lets the next iteration run the moment one drains.
            //
            // One full stream among several is no reason to stop reading:
            // the session socket carries every channel, and a read made on
            // behalf of a channel with room moves the other channels'
            // packets into libssh's own buffers, where they wait until
            // their stream drains; once its window is spent the far end
            // stops sending on that channel alone. Not reading would have
            // held every pane of an ssh domain behind its busiest one. With
            // no channel at all the socket is read as it always was.
            let some_stream_has_room = self.channels.is_empty()
                || self.channels.values().any(|info| {
                    info.descriptors
                        .iter()
                        .skip(1)
                        .filter(|state| state.fd.is_some())
                        .any(|state| state.buf.len() < state.buf.capacity())
                });
            let session_events = if some_stream_has_room {
                sess.get_poll_flags()
            } else {
                sess.get_poll_flags() & !POLLIN
            };
            let mut poll_array = vec![
                pollfd {
                    fd: self.sender_read.as_socket_descriptor(),
                    events: POLLIN,
                    revents: 0,
                },
                pollfd {
                    fd: sess.as_socket_descriptor(),
                    events: session_events,
                    revents: 0,
                },
            ];
            let mut mapping = vec![];

            for info in self.channels.values() {
                for (fd_num, state) in info.descriptors.iter().enumerate() {
                    if let Some(fd) = state.fd.as_ref() {
                        poll_array.push(pollfd {
                            fd: fd.as_socket_descriptor(),
                            events: if fd_num == 0 {
                                // Only while there is room to read into:
                                // with the stdin buffer full (libssh is
                                // not taking writes yet), a readable pipe
                                // woke poll at once for nothing, and the
                                // loop took itself for a busy loop.
                                if state.buf.len() < state.buf.capacity() {
                                    POLLIN
                                } else {
                                    0
                                }
                            } else if !state.buf.is_empty() || info.exited {
                                POLLOUT
                            } else {
                                0
                            },
                            revents: 0,
                        });
                        mapping.push((info.channel_id, fd_num));
                    }
                }
            }

            let poll_started = Instant::now();
            let _ready = poll(&mut poll_array, Some(sleep_delay)).context("poll")?;
            let poll_elapsed = poll_started.elapsed();
            sleep_delay = sleep_delay
                .checked_mul(2)
                .map(|delay| delay.min(max_sleep_delay))
                .unwrap_or(max_sleep_delay);
            let mut saw_revents = false;

            for (idx, poll) in poll_array.iter().enumerate() {
                if poll.revents != 0 {
                    saw_revents = true;
                    sleep_delay = initial_sleep_delay;
                }
                if idx == 0 || idx == 1 {
                    // Dealt with at the top of the loop
                } else if poll.revents != 0 {
                    let (channel_id, fd_num) = mapping[idx - 2];
                    let info = self.channels.get_mut(&channel_id).unwrap();
                    let state = &mut info.descriptors[fd_num];
                    let fd = state.fd.as_mut().unwrap();

                    if fd_num == 0 {
                        // There's data we can read into the buffer
                        match read_into_buf(fd, &mut state.buf) {
                            Ok(progress) => made_progress |= progress,
                            Err(err) => {
                                log::debug!(
                                    "error reading from channel {channel_id} stdin pipe: {:#}",
                                    err
                                );
                                info.channel.close();
                                state.fd.take();
                                made_progress = true;
                            }
                        }
                    } else {
                        if info.exited && state.buf.is_empty() {
                            log::trace!("channel {channel_id} exited and we have no data to send to fd {fd_num}: close it!");
                            state.fd.take();
                            made_progress = true;
                        } else {
                            // We can write our buffered output
                            match write_from_buf(fd, &mut state.buf) {
                                Ok(progress) => made_progress |= progress,
                                Err(err) => {
                                    log::debug!(
                                        "error while writing to channel {} fd {}: {:#}",
                                        channel_id,
                                        fd_num,
                                        err
                                    );

                                    // Close it out
                                    state.fd.take();
                                    made_progress = true;
                                }
                            }
                        }
                    }
                }
            }

            if made_progress || !saw_revents || poll_elapsed >= immediate_wakeup {
                busy_loop_started = None;
                busy_loop_last_warn = None;
                idle_wakes = 0;
            } else if idle_wakes < 3 {
                idle_wakes += 1;
            } else {
                // Repeated immediate wakeups with no IO progress. The nap
                // below caps this at a couple of hundred iterations a
                // second, which keeps CPU usage negligible; don't kill the
                // session over it (protocol-only traffic such as keepalives
                // can legitimately look like this), just make the condition
                // visible in the logs.
                let started = *busy_loop_started.get_or_insert_with(Instant::now);
                let should_warn = match busy_loop_last_warn {
                    None => started.elapsed() >= busy_loop_warn_every,
                    Some(last) => last.elapsed() >= busy_loop_warn_every,
                };
                if should_warn {
                    busy_loop_last_warn = Some(Instant::now());
                    log::warn!(
                        "SSH session has made no IO progress despite repeated \
                         immediate poll wakeups for {:.0?}; throttling the \
                         session loop to avoid a busy loop",
                        started.elapsed()
                    );
                }
                std::thread::sleep(busy_loop_throttle);
            }
        }
    }

    /// Goal: if we have data to write to channels, try to send it.
    /// If we have room in our channel fd write buffers, try to fill it
    fn tick_io(&mut self) -> anyhow::Result<bool> {
        let mut made_progress = false;
        let mut dead = vec![];
        for (id, chan) in self.channels.iter_mut() {
            if chan.exit.is_some() {
                if let Some(status) = chan.channel.exit_status() {
                    log::trace!("channel {id} has exit status {status:?}");
                    chan.exited = true;
                    let exit = chan.exit.take().unwrap();
                    smol::block_on(exit.send(status)).ok();
                    made_progress = true;
                }
            }

            let stdin = &mut chan.descriptors[0];
            if stdin.fd.is_some() && !stdin.buf.is_empty() {
                match write_from_buf(&mut chan.channel.writer(), &mut stdin.buf)
                    .context("writing to channel")
                {
                    Ok(progress) => made_progress |= progress,
                    Err(err) => {
                        log::trace!(
                            "Failed to write data to channel {} stdin: {:#}, closing pipe",
                            id,
                            err
                        );
                        stdin.fd.take();
                        made_progress = true;
                    }
                }
            }

            for (idx, out) in chan
                .descriptors
                .get_mut(1..)
                .unwrap()
                .iter_mut()
                .enumerate()
            {
                if out.fd.is_none() {
                    continue;
                }
                // Drain what the channel holds while there is room. libssh
                // takes in a whole packet at a time, so after one read the
                // socket can show nothing more while the channel still holds
                // plenty; one read per wakeup left the rest to the next one,
                // which came only when the socket had more or poll timed out
                // -- and that timeout doubles, so a busy stream waited longer
                // at every turn.
                loop {
                    if out.buf.len() == out.buf.capacity() {
                        break;
                    }
                    let read = read_into_buf(&mut chan.channel.reader(idx), &mut out.buf);
                    match read {
                        Ok(true) => {
                            made_progress = true;
                            if !chan.channel.has_pending(idx) {
                                break;
                            }
                        }
                        Ok(false) => break,
                        Err(err) => {
                            if out.buf.is_empty() {
                                log::trace!(
                                    "Failed to read data from channel {} stream {}: {:#}, closing pipe",
                                    id,
                                    idx,
                                    err
                                );
                                out.fd.take();
                                made_progress = true;
                            } else {
                                log::trace!(
                                    "Failed to read data from channel {} stream {}: {:#}, but \
                                             still have some buffer to drain",
                                    id,
                                    idx,
                                    err
                                );
                            }
                            break;
                        }
                    }
                }
            }

            if chan
                .descriptors
                .iter()
                .all(|descriptor| descriptor.fd.is_none())
            {
                log::trace!("all descriptors on channel {} are closed", id);
                dead.push(*id);
            }
        }
        for id in dead {
            self.channels.remove(&id);
            made_progress = true;
        }
        Ok(made_progress)
    }

    fn drain_request_pipe(&mut self) -> bool {
        let mut buf = [0u8; 16];
        match self.sender_read.read(&mut buf) {
            Ok(len) => len > 0,
            Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => false,
            Err(_) => false,
        }
    }

    fn dispatch_pending_requests(&mut self, sess: &mut SessionWrap) -> anyhow::Result<bool> {
        let mut made_progress = false;
        while self.dispatch_one_request(sess)? {
            made_progress = true;
        }
        Ok(made_progress)
    }

    fn dispatch_one_request(&mut self, sess: &mut SessionWrap) -> anyhow::Result<bool> {
        self.shutdown.check()?;
        match self.rx_req.try_recv() {
            Err(TryRecvError::Closed) => anyhow::bail!("all clients are closed"),
            Err(TryRecvError::Empty) => Ok(false),
            Ok(req) => {
                sess.set_blocking(true);
                let res = match req {
                    SessionRequest::SessionDropped => {
                        self.session_was_dropped = true;
                        Ok(true)
                    }
                    SessionRequest::NewPty(newpty, reply) => {
                        dispatch(reply, || self.new_pty(sess, newpty), "NewPty")
                    }
                    SessionRequest::ResizePty(resize, Some(reply)) => {
                        dispatch(reply, || self.resize_pty(resize), "resize_pty")
                    }
                    SessionRequest::ResizePty(resize, None) => {
                        if let Err(err) = self.resize_pty(resize) {
                            log::error!("error in resize_pty: {:#}", err);
                        }
                        Ok(true)
                    }
                    SessionRequest::Exec(exec, reply) => {
                        dispatch(reply, || self.exec(sess, exec), "exec")
                    }
                    SessionRequest::SignalChannel(info) => {
                        if let Err(err) = self.signal_channel(&info) {
                            log::error!("{:?} -> error: {:#}", info, err);
                        }
                        Ok(true)
                    }
                    SessionRequest::Sftp(SftpRequest::OpenWithMode(msg, reply)) => {
                        dispatch(reply, || self.open_with_mode(sess, &msg), "OpenWithMode")
                    }
                    SessionRequest::Sftp(SftpRequest::OpenDir(path, reply)) => {
                        dispatch(reply, || self.open_dir(sess, path), "OpenDir")
                    }
                    SessionRequest::Sftp(SftpRequest::File(FileRequest::Write(msg, reply))) => {
                        dispatch(
                            reply,
                            || {
                                let file = self
                                    .files
                                    .get_mut(&msg.file_id)
                                    .ok_or_else(|| anyhow!("invalid file_id"))?;
                                file.writer().write_all(&msg.data)?;
                                Ok(())
                            },
                            "write_file",
                        )
                    }
                    SessionRequest::Sftp(SftpRequest::File(FileRequest::Read(msg, reply))) => {
                        dispatch(
                            reply,
                            || {
                                let file = self
                                    .files
                                    .get_mut(&msg.file_id)
                                    .ok_or_else(|| anyhow!("invalid file_id"))?;

                                // TODO: Move this somewhere to avoid re-allocating buffer
                                let mut buf = vec![0u8; msg.max_bytes];
                                let n = file.reader().read(&mut buf)?;
                                buf.truncate(n);
                                Ok(buf)
                            },
                            "read_file",
                        )
                    }
                    SessionRequest::Sftp(SftpRequest::File(FileRequest::Close(file_id, reply))) => {
                        dispatch(
                            reply,
                            || {
                                self.files.remove(&file_id);
                                Ok(())
                            },
                            "close_file",
                        )
                    }
                    SessionRequest::Sftp(SftpRequest::Dir(DirRequest::Close(dir_id, reply))) => {
                        dispatch(
                            reply,
                            || {
                                self.dirs
                                    .remove(&dir_id)
                                    .ok_or_else(|| anyhow!("invalid dir_id"))?;
                                Ok(())
                            },
                            "close_dir",
                        )
                    }
                    SessionRequest::Sftp(SftpRequest::Dir(DirRequest::ReadDir(dir_id, reply))) => {
                        dispatch(
                            reply,
                            || {
                                let dir = self
                                    .dirs
                                    .get_mut(&dir_id)
                                    .ok_or_else(|| anyhow!("invalid dir_id"))?;
                                dir.read_dir()
                            },
                            "read_dir",
                        )
                    }
                    SessionRequest::Sftp(SftpRequest::File(FileRequest::Flush(file_id, reply))) => {
                        dispatch(
                            reply,
                            || {
                                let file = self
                                    .files
                                    .get_mut(&file_id)
                                    .ok_or_else(|| anyhow!("invalid file_id"))?;
                                file.writer().flush()?;
                                Ok(())
                            },
                            "flush_file",
                        )
                    }
                    SessionRequest::Sftp(SftpRequest::File(FileRequest::SetMetadata(
                        msg,
                        reply,
                    ))) => dispatch(
                        reply,
                        || {
                            let file = self
                                .files
                                .get_mut(&msg.file_id)
                                .ok_or_else(|| anyhow!("invalid file_id"))?;
                            file.set_metadata(msg.metadata)
                        },
                        "set_metadata_file",
                    ),
                    SessionRequest::Sftp(SftpRequest::File(FileRequest::Metadata(
                        file_id,
                        reply,
                    ))) => dispatch(
                        reply,
                        || {
                            let file = self
                                .files
                                .get_mut(&file_id)
                                .ok_or_else(|| anyhow!("invalid file_id"))?;
                            file.metadata()
                        },
                        "metadata_file",
                    ),
                    SessionRequest::Sftp(SftpRequest::File(FileRequest::Fsync(file_id, reply))) => {
                        dispatch(
                            reply,
                            || {
                                let file = self
                                    .files
                                    .get_mut(&file_id)
                                    .ok_or_else(|| anyhow!("invalid file_id"))?;
                                file.fsync()
                            },
                            "fsync",
                        )
                    }
                    SessionRequest::Sftp(SftpRequest::File(FileRequest::Seek(msg, reply))) => {
                        dispatch(
                            reply,
                            || {
                                let file = self
                                    .files
                                    .get_mut(&msg.file_id)
                                    .ok_or_else(|| anyhow!("invalid file_id"))?;
                                file.seek(msg.position)
                            },
                            "seek",
                        )
                    }

                    SessionRequest::Sftp(SftpRequest::ReadDir(path, reply)) => {
                        dispatch(reply, || self.init_sftp(sess)?.read_dir(&path), "read_dir")
                    }
                    SessionRequest::Sftp(SftpRequest::CreateDir(msg, reply)) => dispatch(
                        reply,
                        || self.init_sftp(sess)?.create_dir(&msg.filename, msg.mode),
                        "create_dir",
                    ),
                    SessionRequest::Sftp(SftpRequest::RemoveDir(path, reply)) => dispatch(
                        reply,
                        || self.init_sftp(sess)?.remove_dir(&path),
                        "remove_dir",
                    ),
                    SessionRequest::Sftp(SftpRequest::Metadata(path, reply)) => {
                        dispatch(reply, || self.init_sftp(sess)?.metadata(&path), "metadata")
                    }
                    SessionRequest::Sftp(SftpRequest::SymlinkMetadata(path, reply)) => dispatch(
                        reply,
                        || self.init_sftp(sess)?.symlink_metadata(&path),
                        "symlink_metadata",
                    ),
                    SessionRequest::Sftp(SftpRequest::SetMetadata(msg, reply)) => dispatch(
                        reply,
                        || {
                            self.init_sftp(sess)?
                                .set_metadata(&msg.filename, msg.metadata)
                        },
                        "set_metadata",
                    ),
                    SessionRequest::Sftp(SftpRequest::Symlink(msg, reply)) => dispatch(
                        reply,
                        || self.init_sftp(sess)?.symlink(&msg.path, &msg.target),
                        "symlink",
                    ),
                    SessionRequest::Sftp(SftpRequest::ReadLink(path, reply)) => dispatch(
                        reply,
                        || self.init_sftp(sess)?.read_link(&path),
                        "read_link",
                    ),
                    SessionRequest::Sftp(SftpRequest::Canonicalize(path, reply)) => dispatch(
                        reply,
                        || self.init_sftp(sess)?.canonicalize(&path),
                        "canonicalize",
                    ),
                    SessionRequest::Sftp(SftpRequest::Rename(msg, reply)) => dispatch(
                        reply,
                        || self.init_sftp(sess)?.rename(&msg.src, &msg.dst, msg.opts),
                        "rename",
                    ),
                    SessionRequest::Sftp(SftpRequest::RemoveFile(path, reply)) => {
                        dispatch(reply, || self.init_sftp(sess)?.unlink(&path), "remove_file")
                    }
                };
                sess.set_blocking(false);
                res
            }
        }
    }

    fn connect_pending_agent_forward_channels(&mut self, sess: &mut SessionWrap) -> bool {
        let mut made_progress = false;
        fn process_one(sess: &mut SessionInner, channel: ChannelWrap) -> anyhow::Result<()> {
            let identity_agent = sess
                .identity_agent()
                .ok_or_else(|| anyhow!("no identity agent in config"))?;
            let mut fd = {
                use wezterm_uds::UnixStream;
                #[cfg(unix)]
                {
                    FileDescriptor::new(UnixStream::connect(&identity_agent)?)
                }
                #[cfg(windows)]
                unsafe {
                    use std::os::windows::io::{FromRawSocket, IntoRawSocket};
                    FileDescriptor::from_raw_socket(
                        UnixStream::connect(&identity_agent)?.into_raw_socket(),
                    )
                }
            };
            fd.set_non_blocking(true)?;

            let read_from_agent = fd;
            let write_to_agent = read_from_agent.try_clone()?;
            let channel_id = sess.next_channel_id;
            sess.next_channel_id += 1;
            let info = ChannelInfo {
                channel_id,
                channel,
                exit: None,
                exited: false,
                descriptors: [
                    DescriptorState {
                        fd: Some(read_from_agent),
                        buf: VecDeque::with_capacity(CHANNEL_BUFFER),
                    },
                    DescriptorState {
                        fd: Some(write_to_agent),
                        buf: VecDeque::with_capacity(CHANNEL_BUFFER),
                    },
                    DescriptorState {
                        fd: None,
                        buf: VecDeque::with_capacity(CHANNEL_BUFFER),
                    },
                ],
            };
            sess.channels.insert(channel_id, info);
            Ok(())
        }
        while let Some(channel) = sess.accept_agent_forward() {
            made_progress = true;
            if let Err(err) = process_one(self, channel) {
                log::error!("error connecting agent forward: {:#}", err);
            }
        }
        made_progress
    }

    pub fn signal_channel(&mut self, info: &SignalChannel) -> anyhow::Result<()> {
        let chan_info = self
            .channels
            .get_mut(&info.channel)
            .ok_or_else(|| anyhow::anyhow!("invalid channel id {}", info.channel))?;
        log::trace!("send SIG{} to channel {}", info.signame, info.channel);
        chan_info.channel.send_signal(info.signame)?;
        Ok(())
    }

    pub fn exec(&mut self, sess: &mut SessionWrap, exec: Exec) -> anyhow::Result<ExecResult> {
        let mut channel = sess.open_session()?;

        if let Some("yes") = self.config.get("forwardagent").map(|s| s.as_str()) {
            if self.identity_agent().is_some() {
                if let Err(err) = channel.request_auth_agent_forwarding() {
                    log::error!("Failed to request agent forwarding: {:#}", err);
                }
            }
        }

        if let Some(env) = &exec.env {
            for (key, val) in env {
                if let Err(err) = channel.request_env(key, val) {
                    // Depending on the server configuration, a given
                    // setenv request may not succeed, but that doesn't
                    // prevent the connection from being set up.
                    log::warn!(
                        "ssh: setenv {}={} failed: {}. \
                         Check the AcceptEnv setting on the ssh server side.",
                        key,
                        val,
                        err
                    );
                }
            }
        }

        channel.request_exec(&exec.command_line)?;

        let channel_id = self.next_channel_id;
        self.next_channel_id += 1;

        let (write_to_stdin, mut read_from_stdin) = socketpair()?;
        let (mut write_to_stdout, read_from_stdout) = socketpair()?;
        let (mut write_to_stderr, read_from_stderr) = socketpair()?;

        read_from_stdin.set_non_blocking(true)?;
        write_to_stdout.set_non_blocking(true)?;
        write_to_stderr.set_non_blocking(true)?;

        let (exit_tx, exit_rx) = bounded(1);

        let child = SshChildProcess {
            channel: channel_id,
            tx: None,
            exit: exit_rx,
            exited: None,
        };

        let result = ExecResult {
            stdin: write_to_stdin,
            stdout: read_from_stdout,
            stderr: read_from_stderr,
            child,
        };

        let info = ChannelInfo {
            channel_id,
            channel,
            exit: Some(exit_tx),
            exited: false,
            descriptors: [
                DescriptorState {
                    fd: Some(read_from_stdin),
                    buf: VecDeque::with_capacity(CHANNEL_BUFFER),
                },
                DescriptorState {
                    fd: Some(write_to_stdout),
                    buf: VecDeque::with_capacity(CHANNEL_BUFFER),
                },
                DescriptorState {
                    fd: Some(write_to_stderr),
                    buf: VecDeque::with_capacity(CHANNEL_BUFFER),
                },
            ],
        };

        self.channels.insert(channel_id, info);

        Ok(result)
    }

    /// Open a handle to a file.
    pub fn open_with_mode(
        &mut self,
        sess: &mut SessionWrap,
        msg: &OpenWithMode,
    ) -> SftpChannelResult<File> {
        let ssh_file = self.init_sftp(sess)?.open(&msg.filename, msg.opts)?;

        let file_id = self.next_file_id;
        self.next_file_id += 1;

        let file = File::new(file_id);

        self.files.insert(file_id, ssh_file);
        Ok(file)
    }

    /// Helper to open a directory for reading its contents.
    pub fn open_dir(
        &mut self,
        sess: &mut SessionWrap,
        path: Utf8PathBuf,
    ) -> SftpChannelResult<Dir> {
        let ssh_dir = self.init_sftp(sess)?.open_dir(&path)?;

        let dir_id = self.next_file_id;
        self.next_file_id += 1;

        let dir = Dir::new(dir_id);

        self.dirs.insert(dir_id, ssh_dir);
        Ok(dir)
    }

    /// Initialize the sftp channel if not already created, returning a mutable reference to it
    fn init_sftp<'a>(&mut self, sess: &'a mut SessionWrap) -> SftpChannelResult<&'a mut SftpWrap> {
        match sess {
            #[cfg(feature = "ssh2")]
            SessionWrap::Ssh2(sess) => {
                if sess.sftp.is_none() {
                    sess.sftp = Some(SftpWrap::Ssh2(sess.sess.sftp()?));
                }
                Ok(sess.sftp.as_mut().expect("sftp should have been set above"))
            }

            #[cfg(feature = "libssh-rs")]
            SessionWrap::LibSsh(sess) => {
                if sess.sftp.is_none() {
                    sess.sftp = Some(SftpWrap::LibSsh(sess.sess.sftp()?));
                }
                Ok(sess.sftp.as_mut().expect("sftp should have been set above"))
            }
        }
    }

    /// Closes the SFTP files and directories still open as the loop ends,
    /// while `sess` still holds the SFTP session they were opened on: libssh
    /// closes one through that session, and it goes with `sess`. Blocking, as
    /// a close the loop is asked for is.
    fn close_sftp_handles(&mut self, sess: &mut SessionWrap) {
        sess.set_blocking(true);
        self.files.clear();
        self.dirs.clear();
    }

    pub fn identity_agent(&self) -> Option<String> {
        self.config
            .get("identityagent")
            .map(|s| s.to_owned())
            .or_else(|| std::env::var("SSH_AUTH_SOCK").ok())
    }
}

fn write_from_buf<W: Write>(w: &mut W, buf: &mut VecDeque<u8>) -> std::io::Result<bool> {
    match w.write(buf.make_contiguous()) {
        Ok(len) => {
            buf.drain(0..len);
            Ok(len > 0)
        }
        Err(err) => {
            if err.kind() == std::io::ErrorKind::WouldBlock {
                return Ok(false);
            }
            Err(err)
        }
    }
}

fn read_into_buf<R: Read>(r: &mut R, buf: &mut VecDeque<u8>) -> std::io::Result<bool> {
    let current_len = buf.len();
    if current_len == buf.capacity() {
        // The channel buffer is full. Report no progress so the event loop
        // applies backpressure instead of reading an empty slice and treating
        // the resulting Ok(0) as EOF.
        return Ok(false);
    }
    buf.resize(buf.capacity(), 0);
    let target_buf = &mut buf.make_contiguous()[current_len..];
    match r.read(target_buf) {
        Ok(len) => {
            buf.resize(current_len + len, 0);
            if len == 0 {
                Err(std::io::Error::new(
                    std::io::ErrorKind::UnexpectedEof,
                    "EOF",
                ))
            } else {
                Ok(true)
            }
        }
        Err(err) => {
            buf.resize(current_len, 0);

            if err.kind() == std::io::ErrorKind::WouldBlock {
                return Ok(false);
            }
            Err(err)
        }
    }
}

/// A little helper to ensure that the Result returned by `f()`
/// is routed via a Sender
fn dispatch<T, F>(reply: Sender<T>, f: F, what: &str) -> anyhow::Result<bool>
where
    F: FnOnce() -> T,
    T: Send + Sync + 'static,
{
    if let Err(err) = reply.try_send(f()) {
        log::error!("{}: {:#}", what, err);
    }
    Ok(true)
}

/// A little helper to ensure the Child process is killed on Drop.
struct KillOnDropChild(std::process::Child);

impl Drop for KillOnDropChild {
    fn drop(&mut self) {
        if let Err(err) = self.0.kill() {
            log::error!("Error killing ProxyCommand: {}", err);
        }
        if let Err(err) = self.0.wait() {
            log::error!("Error waiting for ProxyCommand to finish: {}", err);
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::io::Cursor;

    struct CountingReader {
        calls: usize,
    }

    impl Read for CountingReader {
        fn read(&mut self, _buf: &mut [u8]) -> std::io::Result<usize> {
            self.calls += 1;
            panic!("full buffers must not call read");
        }
    }

    #[test]
    fn read_into_full_buf_reports_backpressure_without_reading() {
        let mut buf = VecDeque::with_capacity(3);
        while buf.len() < buf.capacity() {
            buf.push_back(0);
        }
        let mut reader = CountingReader { calls: 0 };

        assert_eq!(read_into_buf(&mut reader, &mut buf).unwrap(), false);
        assert_eq!(reader.calls, 0);
        assert_eq!(buf.len(), buf.capacity());
    }

    #[test]
    fn read_into_non_full_buf_keeps_eof_semantics() {
        let mut buf = VecDeque::with_capacity(8);
        let mut reader = Cursor::new(Vec::<u8>::new());

        let err = read_into_buf(&mut reader, &mut buf).unwrap_err();
        assert_eq!(err.kind(), std::io::ErrorKind::UnexpectedEof);
    }

    #[test]
    fn read_into_non_full_buf_reports_progress() {
        let mut buf = VecDeque::with_capacity(8);
        let mut reader = Cursor::new(b"abc".to_vec());

        assert_eq!(read_into_buf(&mut reader, &mut buf).unwrap(), true);
        assert_eq!(buf.iter().copied().collect::<Vec<_>>(), b"abc");
    }
}