🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
use std::{
future::{Future, IntoFuture},
io,
sync::{
Arc,
atomic::{AtomicBool, Ordering},
},
time::Duration,
};
use axum::{Json, Router, extract::State, http::StatusCode, routing::get};
use serde::Serialize;
use tokio::{net::TcpListener, sync::oneshot};
mod connection;
pub(crate) use connection::ConnectionId;
#[derive(Clone, Default)]
pub struct GatewayLifecycleState {
ready: Arc<AtomicBool>,
disabled: bool,
}
impl GatewayLifecycleState {
pub fn is_ready(&self) -> bool {
!self.disabled && self.ready.load(Ordering::Acquire)
}
/// Keep liveness available without advertising inference readiness.
pub fn unconfigured() -> Self {
Self {
disabled: true,
..Self::default()
}
}
}
#[derive(Serialize)]
struct Probe {
status: &'static str,
}
pub fn router(state: GatewayLifecycleState) -> Router {
Router::new()
.route("/health", get(|| async { Json(Probe { status: "ok" }) }))
.route("/ready", get(readiness))
.with_state(state)
}
async fn readiness(State(state): State<GatewayLifecycleState>) -> (StatusCode, Json<Probe>) {
if state.is_ready() {
(StatusCode::OK, Json(Probe { status: "ready" }))
} else {
(
StatusCode::SERVICE_UNAVAILABLE,
Json(Probe {
status: if state.disabled {
"unconfigured"
} else {
"draining"
},
}),
)
}
}
/// Serve an initialized router until shutdown, then bound connection draining.
/// A timeout must terminate the owning process/runtime to stop remaining tasks.
///
/// # Errors
/// Returns the server's I/O error, or [`io::ErrorKind::TimedOut`] if connections
/// do not finish draining within `connection_drain_timeout`.
pub async fn serve(
listener: TcpListener,
app: Router,
state: GatewayLifecycleState,
shutdown: impl Future<Output = ()> + Send + 'static,
connection_drain_timeout: Duration,
) -> io::Result<()> {
let (draining_tx, draining_rx) = oneshot::channel();
let shutdown_state = state.clone();
let server = axum::serve(
connection::ObservedListener(listener),
app.into_make_service_with_connect_info::<ConnectionId>(),
)
.with_graceful_shutdown(async move {
shutdown.await;
shutdown_state.ready.store(false, Ordering::Release);
tracing::info!("gateway draining");
let _ = draining_tx.send(());
})
.into_future();
tokio::pin!(server);
state.ready.store(true, Ordering::Release);
let result = tokio::select! {
result = &mut server => result,
() = async {
let _ = draining_rx.await;
tokio::time::sleep(connection_drain_timeout).await;
} => Err(io::Error::new(io::ErrorKind::TimedOut, "gateway drain deadline exceeded")),
};
state.ready.store(false, Ordering::Release);
result
}