Skip to content

Make Codex and Claude Code subscriptions go further: a local model router that gives routine agent work to economical models and keeps your strongest model for the hard parts.

16.7 KB
Raw Blame
//! Managed in-process lifetime for the production Pingora listener.

use std::io;
use std::net::SocketAddr;
use std::sync::{Arc, Mutex};
use std::time::Duration;

use thiserror::Error;

use super::GatewayLauncher;
#[cfg(unix)]
use super::ListenerRunStyle;
use super::{GatewayLauncherError, GatewayMode};
use hiroute_host_runtime::{connect_address, validate_gateway_listen};

const STARTUP_TIMEOUT: Duration = Duration::from_secs(10);
const STARTUP_CLEANUP_TIMEOUT: Duration = Duration::from_secs(5);

/// The externally meaningful phase of a managed Gateway listener.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum ManagedGatewayPhase {
    Starting,
    Ready,
    ShutdownRequested,
    Terminated,
    Failed,
}

/// A production Gateway listener owned by an embedding process.
///
/// `shutdown` is idempotent. The embedding owner supplies the bound for
/// `join`, so it can preserve its reverse shutdown order without exposing
/// Pingora runtime details.
pub struct ManagedGatewayHandle {
    listen: SocketAddr,
    phase: Arc<Mutex<ManagedGatewayPhase>>,
    shutdown: Arc<ShutdownRequest>,
    completion: std::sync::mpsc::Receiver<WorkerOutcome>,
    worker: Option<std::thread::JoinHandle<()>>,
    outcome: Option<WorkerOutcome>,
}

impl ManagedGatewayHandle {
    pub fn listen_address(&self) -> SocketAddr {
        self.listen
    }

    pub fn phase(&self) -> ManagedGatewayPhase {
        read_phase(&self.phase)
    }

    pub fn shutdown(&self) {
        self.shutdown.request();
        let mut phase = self
            .phase
            .lock()
            .unwrap_or_else(|poisoned| poisoned.into_inner());
        if matches!(
            *phase,
            ManagedGatewayPhase::Starting | ManagedGatewayPhase::Ready
        ) {
            *phase = ManagedGatewayPhase::ShutdownRequested;
        }
    }

    pub fn join(&mut self, timeout: Duration) -> Result<(), ManagedGatewayError> {
        if let Some(outcome) = self.outcome {
            return outcome.into_result(self.listen);
        }
        let outcome = match self.completion.recv_timeout(timeout) {
            Ok(outcome) => outcome,
            Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {
                return Err(ManagedGatewayError::JoinTimeout {
                    listen: self.listen,
                    timeout,
                });
            }
            Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => WorkerOutcome::Panicked,
        };
        self.finish_worker(outcome)
    }

    fn wait_until_ready(&mut self, timeout: Duration) -> Result<(), ManagedGatewayError> {
        let started = std::time::Instant::now();
        loop {
            if production_ready(self.listen) {
                set_phase(&self.phase, ManagedGatewayPhase::Ready);
                return Ok(());
            }
            match self.completion.try_recv() {
                Ok(WorkerOutcome::Terminated) => {
                    self.finish_worker(WorkerOutcome::Terminated)?;
                    return Err(ManagedGatewayError::StoppedBeforeReady(self.listen));
                }
                Ok(WorkerOutcome::Panicked) => {
                    return self.finish_worker(WorkerOutcome::Panicked);
                }
                Err(std::sync::mpsc::TryRecvError::Disconnected) => {
                    return self.finish_worker(WorkerOutcome::Panicked);
                }
                Err(std::sync::mpsc::TryRecvError::Empty) => {}
            }
            if started.elapsed() >= timeout {
                return Err(ManagedGatewayError::ReadyTimeout {
                    listen: self.listen,
                    timeout,
                });
            }
            std::thread::sleep(Duration::from_millis(20));
        }
    }

    fn finish_worker(&mut self, mut outcome: WorkerOutcome) -> Result<(), ManagedGatewayError> {
        if let Some(worker) = self.worker.take()
            && worker.join().is_err()
        {
            outcome = WorkerOutcome::Panicked;
            set_phase(&self.phase, ManagedGatewayPhase::Failed);
        }
        self.outcome = Some(outcome);
        outcome.into_result(self.listen)
    }
}

impl Drop for ManagedGatewayHandle {
    fn drop(&mut self) {
        self.shutdown();
    }
}

#[derive(Clone, Copy)]
enum WorkerOutcome {
    Terminated,
    Panicked,
}

impl WorkerOutcome {
    fn into_result(self, listen: SocketAddr) -> Result<(), ManagedGatewayError> {
        match self {
            Self::Terminated => Ok(()),
            Self::Panicked => Err(ManagedGatewayError::WorkerPanicked(listen)),
        }
    }
}

#[derive(Default)]
struct ShutdownRequest {
    requested: std::sync::atomic::AtomicBool,
    wake: tokio::sync::Notify,
}

impl ShutdownRequest {
    fn request(&self) {
        if !self
            .requested
            .swap(true, std::sync::atomic::Ordering::AcqRel)
        {
            self.wake.notify_one();
        }
    }

    async fn wait(&self) {
        loop {
            if self.requested.load(std::sync::atomic::Ordering::Acquire) {
                return;
            }
            self.wake.notified().await;
        }
    }
}

#[cfg(unix)]
struct ManagedShutdownSignal {
    request: Arc<ShutdownRequest>,
}

#[cfg(unix)]
#[async_trait::async_trait]
impl pingora_core::server::ShutdownSignalWatch for ManagedShutdownSignal {
    async fn recv(&self) -> pingora_core::server::ShutdownSignal {
        self.request.wait().await;
        pingora_core::server::ShutdownSignal::GracefulTerminate
    }
}

#[cfg(any(unix, windows))]
pub(super) fn start(
    launcher: GatewayLauncher,
) -> Result<ManagedGatewayHandle, ManagedGatewayError> {
    if !matches!(&launcher.mode, GatewayMode::Production { .. }) {
        return Err(ManagedGatewayError::ProductionOnly);
    }
    let listen = launcher.listen;
    if validate_gateway_listen(listen).is_err() {
        return Err(GatewayLauncherError::ListenerMustUseReservedAddress(listen).into());
    }
    let reservation = std::net::TcpListener::bind(listen)
        .map_err(|source| ManagedGatewayError::ListenerUnavailable { listen, source })?;
    drop(reservation);

    #[cfg(unix)]
    let server = launcher.into_server(ListenerRunStyle::Managed)?;
    #[cfg(windows)]
    let services = launcher.into_services();
    let phase = Arc::new(Mutex::new(ManagedGatewayPhase::Starting));
    let shutdown = Arc::new(ShutdownRequest::default());
    let (completion_tx, completion) = std::sync::mpsc::channel();
    let worker_phase = Arc::clone(&phase);
    let worker_shutdown = Arc::clone(&shutdown);
    let worker = std::thread::Builder::new()
        .name("hiroute-managed-gateway".to_owned())
        .spawn(move || {
            let outcome = if std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
                #[cfg(unix)]
                server.run(pingora_core::server::RunArgs {
                    shutdown_signal: Box::new(ManagedShutdownSignal {
                        request: worker_shutdown,
                    }),
                });
                #[cfg(windows)]
                run_windows_services(services, worker_shutdown);
            }))
            .is_ok()
            {
                set_phase(&worker_phase, ManagedGatewayPhase::Terminated);
                WorkerOutcome::Terminated
            } else {
                set_phase(&worker_phase, ManagedGatewayPhase::Failed);
                WorkerOutcome::Panicked
            };
            let _ = completion_tx.send(outcome);
        })
        .map_err(ManagedGatewayError::WorkerSpawn)?;
    let mut handle = ManagedGatewayHandle {
        listen,
        phase,
        shutdown,
        completion,
        worker: Some(worker),
        outcome: None,
    };
    if let Err(error) = handle.wait_until_ready(STARTUP_TIMEOUT) {
        handle.shutdown();
        if handle.worker.is_some() {
            handle.join(STARTUP_CLEANUP_TIMEOUT)?;
        }
        return Err(error);
    }
    Ok(handle)
}

#[cfg(not(any(unix, windows)))]
pub(super) fn start(
    _launcher: GatewayLauncher,
) -> Result<ManagedGatewayHandle, ManagedGatewayError> {
    Err(ManagedGatewayError::UnsupportedPlatform)
}

#[cfg(windows)]
fn run_windows_services(
    services: Vec<Box<dyn pingora_core::services::ServiceWithDependents>>,
    shutdown: Arc<ShutdownRequest>,
) {
    // Public Pingora services preserve the same GatewayHttpApp, listener and production ports.
    // Its Server Windows loop ignores RunArgs and waits for process Ctrl-C, so embedding it
    // would leave RoleAll.shutdown()/join() unable to stop this listener.
    let runtime = tokio::runtime::Builder::new_multi_thread()
        .worker_threads(super::LISTENER_WORKER_THREADS)
        .thread_name("hiroute-gateway-io")
        .enable_all()
        .build()
        .expect("cannot create managed Gateway runtime");
    runtime.block_on(async move {
        let (send_shutdown, receive_shutdown) = tokio::sync::watch::channel(false);
        let mut tasks = tokio::task::JoinSet::new();
        for mut service in services {
            let watch = receive_shutdown.clone();
            let (ready, _) = tokio::sync::watch::channel(false);
            let notifier = pingora_core::services::ServiceReadyNotifier::new(ready);
            tasks.spawn(async move { service.start_service(watch, 1, notifier).await });
        }
        tokio::select! {
            _ = shutdown.wait() => {}
            outcome = tasks.join_next() => {
                panic!("managed Gateway service stopped before shutdown: {outcome:?}");
            }
        }
        let _ = send_shutdown.send(true);
        let drained = tokio::time::timeout(Duration::from_secs(1), async {
            while let Some(outcome) = tasks.join_next().await {
                outcome.expect("managed Gateway service failed during shutdown");
            }
        })
        .await;
        if drained.is_err() {
            tasks.abort_all();
            while tasks.join_next().await.is_some() {}
            panic!("managed Gateway service exceeded its shutdown deadline");
        }
        // Match the Unix managed listener's one-second graceful connection window.
        // Runtime shutdown then cancels stalled connection tasks instead of waiting forever.
        tokio::time::sleep(Duration::from_secs(1)).await;
    });
    runtime.shutdown_timeout(Duration::from_secs(1));
}

fn production_ready(listen: SocketAddr) -> bool {
    use std::io::{Read, Write};

    let io_timeout = Duration::from_millis(200);
    let Ok(target) = connect_address(listen) else {
        return false;
    };
    let Ok(mut stream) = std::net::TcpStream::connect_timeout(&target, io_timeout) else {
        return false;
    };
    let _ = stream.set_read_timeout(Some(io_timeout));
    let _ = stream.set_write_timeout(Some(io_timeout));
    let request =
        format!("GET /_hiroute/ready HTTP/1.1\r\nHost: {target}\r\nConnection: close\r\n\r\n");
    if stream.write_all(request.as_bytes()).is_err() {
        return false;
    }
    let mut response = Vec::new();
    if stream.read_to_end(&mut response).is_err() {
        return false;
    }
    let Some(body_start) = response.windows(4).position(|window| window == b"\r\n\r\n") else {
        return false;
    };
    if !response.starts_with(b"HTTP/1.1 200 ") && !response.starts_with(b"HTTP/1.0 200 ") {
        return false;
    }
    serde_json::from_slice::<serde_json::Value>(&response[body_start + 4..]).is_ok_and(|body| {
        body["schema_version"] == "hiroute.gateway.ready/v2" && body["status"] == "ready"
    })
}

fn read_phase(phase: &Mutex<ManagedGatewayPhase>) -> ManagedGatewayPhase {
    *phase
        .lock()
        .unwrap_or_else(|poisoned| poisoned.into_inner())
}

fn set_phase(phase: &Mutex<ManagedGatewayPhase>, next: ManagedGatewayPhase) {
    *phase
        .lock()
        .unwrap_or_else(|poisoned| poisoned.into_inner()) = next;
}

#[derive(Debug, Error)]
pub enum ManagedGatewayError {
    #[error(transparent)]
    Launcher(#[from] GatewayLauncherError),
    #[error("managed Gateway startup requires a production launcher")]
    ProductionOnly,
    #[error("managed Gateway lifecycle is unavailable on this platform")]
    UnsupportedPlatform,
    #[error("cannot spawn managed Gateway worker: {0}")]
    WorkerSpawn(#[source] io::Error),
    #[error("managed Gateway listener {listen} is unavailable: {source}")]
    ListenerUnavailable {
        listen: SocketAddr,
        #[source]
        source: io::Error,
    },
    #[error("managed Gateway at {0} stopped before readiness")]
    StoppedBeforeReady(SocketAddr),
    #[error("managed Gateway at {listen} did not become ready within {timeout:?}")]
    ReadyTimeout {
        listen: SocketAddr,
        timeout: Duration,
    },
    #[error("managed Gateway at {listen} did not join within {timeout:?}")]
    JoinTimeout {
        listen: SocketAddr,
        timeout: Duration,
    },
    #[error("managed Gateway worker at {0} panicked")]
    WorkerPanicked(SocketAddr),
}

#[cfg(all(test, any(unix, windows)))]
mod tests {
    use super::*;

    fn reserved_loopback_address() -> SocketAddr {
        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
        listener.local_addr().unwrap()
    }

    #[test]
    fn production_listener_is_ready_and_releases_port_after_idempotent_shutdown() {
        let listen = reserved_loopback_address();
        let lkg_path = std::env::temp_dir().join(format!(
            "hiroute-managed-gateway-missing-lkg-{}-{listen}.json",
            std::process::id()
        ));
        let launcher = GatewayLauncher::production(listen, lkg_path).unwrap();

        let mut handle = launcher.start_managed().unwrap();
        assert_eq!(handle.listen_address(), listen);
        assert_eq!(handle.phase(), ManagedGatewayPhase::Ready);
        assert!(production_ready(listen));

        handle.shutdown();
        handle.shutdown();
        handle.join(Duration::from_secs(5)).unwrap();
        assert_eq!(handle.phase(), ManagedGatewayPhase::Terminated);

        std::net::TcpListener::bind(listen).expect("managed join must release the listener port");
    }

    #[test]
    fn wildcard_listener_uses_loopback_for_readiness_and_releases_the_bound_port() {
        let reservation = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
        let port = reservation.local_addr().unwrap().port();
        drop(reservation);
        let listen = SocketAddr::from(([0, 0, 0, 0], port));
        let lkg_path = std::env::temp_dir().join(format!(
            "hiroute-managed-gateway-wildcard-missing-lkg-{}-{port}.json",
            std::process::id()
        ));
        let launcher = GatewayLauncher::production(listen, lkg_path).unwrap();
        let mut handle = launcher.start_managed().unwrap();
        assert_eq!(handle.listen_address(), listen);
        assert!(production_ready(listen));
        handle.shutdown();
        handle.join(Duration::from_secs(5)).unwrap();
        std::net::TcpListener::bind(listen).expect("wildcard listener must be released");
    }

    #[test]
    fn shutdown_cancels_a_client_that_never_completes_its_http_request() {
        use std::io::{Read, Write};

        let listen = reserved_loopback_address();
        let lkg = std::env::temp_dir().join(format!(
            "hiroute-managed-gateway-stalled-missing-lkg-{}-{}.json",
            std::process::id(),
            listen.port()
        ));
        let mut handle = GatewayLauncher::production(listen, lkg)
            .unwrap()
            .start_managed()
            .unwrap();
        let mut client = std::net::TcpStream::connect(listen).unwrap();
        client
            .set_read_timeout(Some(Duration::from_secs(5)))
            .unwrap();
        client
            .write_all(b"GET /_hiroute/ready HTTP/1.1\r\nHost: localhost\r\n")
            .unwrap();
        handle.shutdown();
        handle.join(Duration::from_secs(5)).unwrap();
        let closed = client.read(&mut [0u8; 1]);
        assert!(
            matches!(closed, Ok(0))
                || closed.as_ref().is_err_and(|error| {
                    matches!(
                        error.kind(),
                        std::io::ErrorKind::ConnectionReset | std::io::ErrorKind::ConnectionAborted
                    )
                }),
            "stalled connection must close after managed join: {closed:?}"
        );
        std::net::TcpListener::bind(listen).expect("shutdown must release a stalled listener");
    }

    #[test]
    fn ipv6_listener_is_rejected_before_startup() {
        let listen = "[::1]:5837".parse().unwrap();
        let lkg_path = std::env::temp_dir().join(format!(
            "hiroute-managed-gateway-ipv6-missing-lkg-{}.json",
            std::process::id()
        ));
        let launcher = GatewayLauncher::production(listen, lkg_path).unwrap();
        assert!(matches!(
            launcher.start_managed(),
            Err(ManagedGatewayError::Launcher(
                GatewayLauncherError::ListenerMustUseReservedAddress(value)
            )) if value == listen
        ));
    }
}