🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(ai-gateway): HTTP/2 to Web and one immediate resolve retry (#19523)
Hassieb Pakzadf2196feee734
diff --git a/ai-gateway/Cargo.lock b/ai-gateway/Cargo.lockindex edaa93e555..52a1011201 100644--- a/ai-gateway/Cargo.lock+++ b/ai-gateway/Cargo.lock@@ -353,6 +353,12 @@ version = "1.18.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34"+[[package]]+name = "equivalent"+version = "1.0.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826"+[[package]]name = "errno"version = "0.3.14"@@ -380,6 +386,12 @@ dependencies = ["zlib-rs",]+[[package]]+name = "fnv"+version = "1.0.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"+[[package]]name = "form_urlencoded"version = "1.2.2"@@ -506,6 +518,31 @@ dependencies = ["wasm-bindgen",]+[[package]]+name = "h2"+version = "0.4.20"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7d29020232d6aa3fb1daca64c1127cf662cf97f254ae16c18c05b8ab635fc118"+dependencies = [+ "atomic-waker",+ "bytes",+ "fnv",+ "futures-core",+ "futures-sink",+ "http",+ "indexmap",+ "slab",+ "tokio",+ "tokio-util",+ "tracing",+]++[[package]]+name = "hashbrown"+version = "0.17.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"+[[package]]name = "hmac"version = "0.13.0"@@ -579,6 +616,7 @@ dependencies = ["bytes","futures-channel","futures-core",+ "h2","http","http-body","httparse",@@ -732,6 +770,16 @@ dependencies = ["icu_properties",]+[[package]]+name = "indexmap"+version = "2.14.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855"+dependencies = [+ "equivalent",+ "hashbrown",+]+[[package]]name = "ipnet"version = "2.12.2"@@ -1229,6 +1277,7 @@ dependencies = ["futures-channel","futures-core","futures-util",+ "h2","http","http-body","http-body-util",@@ -1742,6 +1791,7 @@ dependencies = ["bytes","futures-core","futures-sink",+ "libc","pin-project-lite","tokio",]diff --git a/ai-gateway/Cargo.toml b/ai-gateway/Cargo.tomlindex 9f3cf35ac7..3e490b418f 100644--- a/ai-gateway/Cargo.toml+++ b/ai-gateway/Cargo.toml@@ -17,7 +17,7 @@ opentelemetry-http = "0.33"opentelemetry = { version = "0.33", default-features = false, features = ["trace", "metrics", "internal-logs"] }opentelemetry_sdk = { version = "0.33", default-features = false, features = ["trace", "metrics", "internal-logs"] }opentelemetry-otlp = { version = "0.33", default-features = false, features = ["trace", "metrics", "http-proto", "reqwest-blocking-client", "internal-logs"] }-reqwest = { version = "0.13.5", default-features = false, features = ["rustls", "stream"] }+reqwest = { version = "0.13.5", default-features = false, features = ["http2", "rustls", "stream"] }reqwest-middleware = "0.5"reqwest-tracing = { version = "0.7", default-features = false }serde = { version = "1", features = ["derive"] }diff --git a/ai-gateway/scripts/smoke-image.sh b/ai-gateway/scripts/smoke-image.shindex b29acc7e2e..b6039b44d0 100644--- a/ai-gateway/scripts/smoke-image.sh+++ b/ai-gateway/scripts/smoke-image.sh@@ -29,6 +29,9 @@ docker exec "$container" test -s /usr/share/licenses/ai-gateway/THIRD_PARTY_LICE--max-time 2 -X POST "http://$port/openai/v1/responses")" == 401 ]]docker stop --time 5 "$container" >/dev/null[[ "$(docker inspect --format '{{.State.ExitCode}}' "$container")" == 0 ]]-docker logs "$container" 2>&1 | grep -q 'gateway draining'-docker logs "$container" 2>&1 | grep -q 'gateway stopped'+# Piping docker logs into grep -q fails under pipefail when grep exits before+# docker logs finishes writing (SIGPIPE, exit 141).+logs="$(docker logs "$container" 2>&1)"+grep -q 'gateway draining' <<<"$logs"+grep -q 'gateway stopped' <<<"$logs"printf '%s\n' 'Image smoke: non-root, license notices, probes, ECS health command, unauthenticated inference, SIGTERM passed'diff --git a/ai-gateway/src/inference.rs b/ai-gateway/src/inference.rsindex 1357e2fd9e..dcf088447d 100644--- a/ai-gateway/src/inference.rs+++ b/ai-gateway/src/inference.rs@@ -217,6 +217,7 @@ impl InferenceService {"resolution",otel.kind = "internal",gateway.outcome = tracing::field::Empty,+ gateway.resolution.retry = tracing::field::Empty,otel.status_code = tracing::field::Empty,error.type = tracing::field::Empty);diff --git a/ai-gateway/src/providers/mod.rs b/ai-gateway/src/providers/mod.rsindex 4dc340db7a..748f3ed210 100644--- a/ai-gateway/src/providers/mod.rs+++ b/ai-gateway/src/providers/mod.rs@@ -123,6 +123,8 @@ impl ProviderTransport {// provider generates before sending headers. The relay bounds stalls instead..connect_timeout(Duration::from_secs(5)).pool_max_idle_per_host(16)+ // reqwest's http2 feature is for Web-bound clients (resolver, telemetry); provider calls stay on HTTP/1.1.+ .http1_only().build().map_err(|_| ProviderError::Configuration)?;Ok(Self {diff --git a/ai-gateway/src/resolution/mod.rs b/ai-gateway/src/resolution/mod.rsindex 064cc71556..aea67710ed 100644--- a/ai-gateway/src/resolution/mod.rs+++ b/ai-gateway/src/resolution/mod.rs@@ -232,7 +232,7 @@ impl ControlPlaneClient {.header(CONTENT_TYPE, "application/json").body(body);let body = self- .execute(request, self.config.resolve_limits, &mut |_| Ok(()))+ .execute_retrying_once(request, self.config.resolve_limits).await?;contracts::decode(&body,@@ -286,6 +286,55 @@ impl ControlPlaneClient {.await.map_err(|_| ResolutionError::Timeout)?}++ /// Like [`Self::execute`], but resends once, immediately, when no usable response+ /// arrived. Resolve is read-only on Web, so a duplicate is harmless. Both attempts+ /// share one deadline, so a timeout is never retried.+ async fn execute_retrying_once(+ &self,+ request: RequestBuilder,+ limits: ControlPlaneLimits,+ ) -> Result<Vec<u8>, ResolutionError> {+ tokio::time::timeout(limits.timeout, async {+ let retry = request.try_clone();+ let mut sent = request.send().await;+ if let Some(retry) = retry+ && let Some(reason) = retry_reason(&sent)+ {+ tracing::warn!(reason, "gateway resolution retrying once");+ tracing::Span::current().record("gateway.resolution.retry", reason);+ sent = retry.send().await;+ }+ let response = sent.map_err(|_| ResolutionError::Transport)?;+ read_response(response, limits.max_response_bytes, &mut |_| Ok(())).await+ })+ .await+ .map_err(|_| ResolutionError::Timeout)?+ }+}++/// Retry only failures where Web never answered: connect and send errors (including a+/// connection closed or reset under the request, or an HTTP/2 GOAWAY or refused stream),+/// and 502, which Web never returns itself, so it comes from the load balancer. Web's+/// 503 asks callers to back off, and 504 or a timeout has already spent the budget.+fn retry_reason(+ sent: &Result<reqwest::Response, reqwest_middleware::Error>,+) -> Option<&'static str> {+ match sent {+ Ok(response) if response.status() == reqwest::StatusCode::BAD_GATEWAY => {+ Some("bad_gateway")+ }+ Err(reqwest_middleware::Error::Reqwest(error)) if !error.is_timeout() => {+ if error.is_connect() {+ Some("connect")+ } else if error.is_request() {+ Some("send")+ } else {+ None+ }+ }+ _ => None,+ }}async fn read_response(diff --git a/ai-gateway/src/resolution/tests.rs b/ai-gateway/src/resolution/tests.rsindex bec6e4728c..c1d55c6d26 100644--- a/ai-gateway/src/resolution/tests.rs+++ b/ai-gateway/src/resolution/tests.rs@@ -260,7 +260,7 @@ async fn model_catalogs_have_a_larger_response_limit_than_resolution() {#[tokio::test]async fn denied_and_failed_resolutions_are_classified_without_retries_or_body_leaks() {- for status in [400, 401, 403, 404, 500, 503] {+ for status in [400, 401, 403, 404, 429, 500, 503, 504] {let web = FakeWeb::start(move |_| async move {Response::builder().status(status)@@ -291,6 +291,118 @@ async fn denied_and_failed_resolutions_are_classified_without_retries_or_body_le}}+async fn resolve_against(web: &FakeWeb) -> Result<ResolvedRequestContext, ResolutionError> {+ web.control_plane()+ .resolve_at(+ GATEWAY_KEY,+ ApiFormat::OpenAiResponses,+ "primary:gpt-4.1",+ NOW_TIME,+ REQUEST_ID,+ )+ .await+}++#[tokio::test]+async fn retries_a_bad_gateway_once_and_then_gives_up() {+ let attempts = Arc::new(AtomicUsize::new(0));+ let seen = attempts.clone();+ let web = FakeWeb::start(move |_| {+ let attempt = seen.fetch_add(1, Ordering::SeqCst);+ async move {+ if attempt == 0 {+ Response::builder()+ .status(StatusCode::BAD_GATEWAY)+ .body(Body::empty())+ .unwrap()+ } else {+ response(success("project-1", "private-provider-token"))+ }+ }+ })+ .await;+ let context = resolve_against(&web).await.unwrap();+ assert_eq!(context.connection().id(), "connection-1");+ assert_eq!(web.calls(), 2);++ let web = FakeWeb::start(|_| async {+ Response::builder()+ .status(StatusCode::BAD_GATEWAY)+ .body(Body::empty())+ .unwrap()+ })+ .await;+ assert_eq!(+ resolve_against(&web).await.unwrap_err(),+ ResolutionError::Unavailable+ );+ assert_eq!(web.calls(), 2);+}++#[tokio::test]+async fn retries_once_when_web_closes_the_connection_without_answering() {+ use tokio::io::{AsyncReadExt, AsyncWriteExt};++ let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();+ let url = format!("http://{}", listener.local_addr().unwrap());+ let connections = Arc::new(AtomicUsize::new(0));+ let accepted = connections.clone();+ let body = success("project-1", "private-provider-token").to_string();+ let server = tokio::spawn(async move {+ loop {+ let (mut socket, _) = listener.accept().await.unwrap();+ let connection = accepted.fetch_add(1, Ordering::SeqCst);+ let mut request = vec![0; 16 * 1024];+ let _ = socket.read(&mut request).await.unwrap();+ if connection == 0 {+ continue;+ }+ let reply = format!(+ "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",+ body.len()+ );+ socket.write_all(reply.as_bytes()).await.unwrap();+ }+ });+ let config = ControlPlaneConfig::new(&url, SERVICE_KEY).unwrap();+ let context = ControlPlaneClient::new(config)+ .unwrap()+ .resolve_at(+ GATEWAY_KEY,+ ApiFormat::OpenAiResponses,+ "primary:gpt-4.1",+ NOW_TIME,+ REQUEST_ID,+ )+ .await+ .unwrap();+ server.abort();+ assert_eq!(context.connection().id(), "connection-1");+ assert_eq!(connections.load(Ordering::SeqCst), 2);+}++#[tokio::test]+async fn timeouts_are_not_retried() {+ let web = FakeWeb::start(|_| async {+ tokio::time::sleep(Duration::from_secs(1)).await;+ response(success("project-1", "private-provider-token"))+ })+ .await;+ let error = web+ .control_plane_with_resolve_limits(Duration::from_millis(100), 256 * 1024)+ .resolve_at(+ GATEWAY_KEY,+ ApiFormat::OpenAiResponses,+ "primary:gpt-4.1",+ NOW_TIME,+ REQUEST_ID,+ )+ .await+ .unwrap_err();+ assert_eq!(error, ResolutionError::Timeout);+ assert_eq!(web.calls(), 1);+}+#[tokio::test]async fn model_policy_rejections_are_distinct_from_other_forbidden_responses() {let web = FakeWeb::start(|_| async {