Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(ai-gateway): HTTP/2 to Web and one immediate resolve retry (#19523)

Hassieb Pakzadf2196feee734
diff --git a/ai-gateway/Cargo.lock b/ai-gateway/Cargo.lock
index edaa93e555..52a1011201 100644
--- a/ai-gateway/Cargo.lock
+++ b/ai-gateway/Cargo.lock
@@ -353,6 +353,12 @@ version = "1.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34"
+[[package]]
+name = "equivalent"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826"
+
[[package]]
name = "errno"
version = "0.3.14"
@@ -380,6 +386,12 @@ dependencies = [
"zlib-rs",
]
+[[package]]
+name = "fnv"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
+
[[package]]
name = "form_urlencoded"
version = "1.2.2"
@@ -506,6 +518,31 @@ dependencies = [
"wasm-bindgen",
]
+[[package]]
+name = "h2"
+version = "0.4.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7d29020232d6aa3fb1daca64c1127cf662cf97f254ae16c18c05b8ab635fc118"
+dependencies = [
+ "atomic-waker",
+ "bytes",
+ "fnv",
+ "futures-core",
+ "futures-sink",
+ "http",
+ "indexmap",
+ "slab",
+ "tokio",
+ "tokio-util",
+ "tracing",
+]
+
+[[package]]
+name = "hashbrown"
+version = "0.17.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
+
[[package]]
name = "hmac"
version = "0.13.0"
@@ -579,6 +616,7 @@ dependencies = [
"bytes",
"futures-channel",
"futures-core",
+ "h2",
"http",
"http-body",
"httparse",
@@ -732,6 +770,16 @@ dependencies = [
"icu_properties",
]
+[[package]]
+name = "indexmap"
+version = "2.14.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855"
+dependencies = [
+ "equivalent",
+ "hashbrown",
+]
+
[[package]]
name = "ipnet"
version = "2.12.2"
@@ -1229,6 +1277,7 @@ dependencies = [
"futures-channel",
"futures-core",
"futures-util",
+ "h2",
"http",
"http-body",
"http-body-util",
@@ -1742,6 +1791,7 @@ dependencies = [
"bytes",
"futures-core",
"futures-sink",
+ "libc",
"pin-project-lite",
"tokio",
]
diff --git a/ai-gateway/Cargo.toml b/ai-gateway/Cargo.toml
index 9f3cf35ac7..3e490b418f 100644
--- a/ai-gateway/Cargo.toml
+++ b/ai-gateway/Cargo.toml
@@ -17,7 +17,7 @@ opentelemetry-http = "0.33"
opentelemetry = { version = "0.33", default-features = false, features = ["trace", "metrics", "internal-logs"] }
opentelemetry_sdk = { version = "0.33", default-features = false, features = ["trace", "metrics", "internal-logs"] }
opentelemetry-otlp = { version = "0.33", default-features = false, features = ["trace", "metrics", "http-proto", "reqwest-blocking-client", "internal-logs"] }
-reqwest = { version = "0.13.5", default-features = false, features = ["rustls", "stream"] }
+reqwest = { version = "0.13.5", default-features = false, features = ["http2", "rustls", "stream"] }
reqwest-middleware = "0.5"
reqwest-tracing = { version = "0.7", default-features = false }
serde = { version = "1", features = ["derive"] }
diff --git a/ai-gateway/scripts/smoke-image.sh b/ai-gateway/scripts/smoke-image.sh
index b29acc7e2e..b6039b44d0 100644
--- a/ai-gateway/scripts/smoke-image.sh
+++ b/ai-gateway/scripts/smoke-image.sh
@@ -29,6 +29,9 @@ docker exec "$container" test -s /usr/share/licenses/ai-gateway/THIRD_PARTY_LICE
--max-time 2 -X POST "http://$port/openai/v1/responses")" == 401 ]]
docker stop --time 5 "$container" >/dev/null
[[ "$(docker inspect --format '{{.State.ExitCode}}' "$container")" == 0 ]]
-docker logs "$container" 2>&1 | grep -q 'gateway draining'
-docker logs "$container" 2>&1 | grep -q 'gateway stopped'
+# Piping docker logs into grep -q fails under pipefail when grep exits before
+# docker logs finishes writing (SIGPIPE, exit 141).
+logs="$(docker logs "$container" 2>&1)"
+grep -q 'gateway draining' <<<"$logs"
+grep -q 'gateway stopped' <<<"$logs"
printf '%s\n' 'Image smoke: non-root, license notices, probes, ECS health command, unauthenticated inference, SIGTERM passed'
diff --git a/ai-gateway/src/inference.rs b/ai-gateway/src/inference.rs
index 1357e2fd9e..dcf088447d 100644
--- a/ai-gateway/src/inference.rs
+++ b/ai-gateway/src/inference.rs
@@ -217,6 +217,7 @@ impl InferenceService {
"resolution",
otel.kind = "internal",
gateway.outcome = tracing::field::Empty,
+ gateway.resolution.retry = tracing::field::Empty,
otel.status_code = tracing::field::Empty,
error.type = tracing::field::Empty
);
diff --git a/ai-gateway/src/providers/mod.rs b/ai-gateway/src/providers/mod.rs
index 4dc340db7a..748f3ed210 100644
--- a/ai-gateway/src/providers/mod.rs
+++ b/ai-gateway/src/providers/mod.rs
@@ -123,6 +123,8 @@ impl ProviderTransport {
// provider generates before sending headers. The relay bounds stalls instead.
.connect_timeout(Duration::from_secs(5))
.pool_max_idle_per_host(16)
+ // reqwest's http2 feature is for Web-bound clients (resolver, telemetry); provider calls stay on HTTP/1.1.
+ .http1_only()
.build()
.map_err(|_| ProviderError::Configuration)?;
Ok(Self {
diff --git a/ai-gateway/src/resolution/mod.rs b/ai-gateway/src/resolution/mod.rs
index 064cc71556..aea67710ed 100644
--- a/ai-gateway/src/resolution/mod.rs
+++ b/ai-gateway/src/resolution/mod.rs
@@ -232,7 +232,7 @@ impl ControlPlaneClient {
.header(CONTENT_TYPE, "application/json")
.body(body);
let body = self
- .execute(request, self.config.resolve_limits, &mut |_| Ok(()))
+ .execute_retrying_once(request, self.config.resolve_limits)
.await?;
contracts::decode(
&body,
@@ -286,6 +286,55 @@ impl ControlPlaneClient {
.await
.map_err(|_| ResolutionError::Timeout)?
}
+
+ /// Like [`Self::execute`], but resends once, immediately, when no usable response
+ /// arrived. Resolve is read-only on Web, so a duplicate is harmless. Both attempts
+ /// share one deadline, so a timeout is never retried.
+ async fn execute_retrying_once(
+ &self,
+ request: RequestBuilder,
+ limits: ControlPlaneLimits,
+ ) -> Result<Vec<u8>, ResolutionError> {
+ tokio::time::timeout(limits.timeout, async {
+ let retry = request.try_clone();
+ let mut sent = request.send().await;
+ if let Some(retry) = retry
+ && let Some(reason) = retry_reason(&sent)
+ {
+ tracing::warn!(reason, "gateway resolution retrying once");
+ tracing::Span::current().record("gateway.resolution.retry", reason);
+ sent = retry.send().await;
+ }
+ let response = sent.map_err(|_| ResolutionError::Transport)?;
+ read_response(response, limits.max_response_bytes, &mut |_| Ok(())).await
+ })
+ .await
+ .map_err(|_| ResolutionError::Timeout)?
+ }
+}
+
+/// Retry only failures where Web never answered: connect and send errors (including a
+/// connection closed or reset under the request, or an HTTP/2 GOAWAY or refused stream),
+/// and 502, which Web never returns itself, so it comes from the load balancer. Web's
+/// 503 asks callers to back off, and 504 or a timeout has already spent the budget.
+fn retry_reason(
+ sent: &Result<reqwest::Response, reqwest_middleware::Error>,
+) -> Option<&'static str> {
+ match sent {
+ Ok(response) if response.status() == reqwest::StatusCode::BAD_GATEWAY => {
+ Some("bad_gateway")
+ }
+ Err(reqwest_middleware::Error::Reqwest(error)) if !error.is_timeout() => {
+ if error.is_connect() {
+ Some("connect")
+ } else if error.is_request() {
+ Some("send")
+ } else {
+ None
+ }
+ }
+ _ => None,
+ }
}
async fn read_response(
diff --git a/ai-gateway/src/resolution/tests.rs b/ai-gateway/src/resolution/tests.rs
index bec6e4728c..c1d55c6d26 100644
--- a/ai-gateway/src/resolution/tests.rs
+++ b/ai-gateway/src/resolution/tests.rs
@@ -260,7 +260,7 @@ async fn model_catalogs_have_a_larger_response_limit_than_resolution() {
#[tokio::test]
async fn denied_and_failed_resolutions_are_classified_without_retries_or_body_leaks() {
- for status in [400, 401, 403, 404, 500, 503] {
+ for status in [400, 401, 403, 404, 429, 500, 503, 504] {
let web = FakeWeb::start(move |_| async move {
Response::builder()
.status(status)
@@ -291,6 +291,118 @@ async fn denied_and_failed_resolutions_are_classified_without_retries_or_body_le
}
}
+async fn resolve_against(web: &FakeWeb) -> Result<ResolvedRequestContext, ResolutionError> {
+ web.control_plane()
+ .resolve_at(
+ GATEWAY_KEY,
+ ApiFormat::OpenAiResponses,
+ "primary:gpt-4.1",
+ NOW_TIME,
+ REQUEST_ID,
+ )
+ .await
+}
+
+#[tokio::test]
+async fn retries_a_bad_gateway_once_and_then_gives_up() {
+ let attempts = Arc::new(AtomicUsize::new(0));
+ let seen = attempts.clone();
+ let web = FakeWeb::start(move |_| {
+ let attempt = seen.fetch_add(1, Ordering::SeqCst);
+ async move {
+ if attempt == 0 {
+ Response::builder()
+ .status(StatusCode::BAD_GATEWAY)
+ .body(Body::empty())
+ .unwrap()
+ } else {
+ response(success("project-1", "private-provider-token"))
+ }
+ }
+ })
+ .await;
+ let context = resolve_against(&web).await.unwrap();
+ assert_eq!(context.connection().id(), "connection-1");
+ assert_eq!(web.calls(), 2);
+
+ let web = FakeWeb::start(|_| async {
+ Response::builder()
+ .status(StatusCode::BAD_GATEWAY)
+ .body(Body::empty())
+ .unwrap()
+ })
+ .await;
+ assert_eq!(
+ resolve_against(&web).await.unwrap_err(),
+ ResolutionError::Unavailable
+ );
+ assert_eq!(web.calls(), 2);
+}
+
+#[tokio::test]
+async fn retries_once_when_web_closes_the_connection_without_answering() {
+ use tokio::io::{AsyncReadExt, AsyncWriteExt};
+
+ let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let url = format!("http://{}", listener.local_addr().unwrap());
+ let connections = Arc::new(AtomicUsize::new(0));
+ let accepted = connections.clone();
+ let body = success("project-1", "private-provider-token").to_string();
+ let server = tokio::spawn(async move {
+ loop {
+ let (mut socket, _) = listener.accept().await.unwrap();
+ let connection = accepted.fetch_add(1, Ordering::SeqCst);
+ let mut request = vec![0; 16 * 1024];
+ let _ = socket.read(&mut request).await.unwrap();
+ if connection == 0 {
+ continue;
+ }
+ let reply = format!(
+ "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
+ body.len()
+ );
+ socket.write_all(reply.as_bytes()).await.unwrap();
+ }
+ });
+ let config = ControlPlaneConfig::new(&url, SERVICE_KEY).unwrap();
+ let context = ControlPlaneClient::new(config)
+ .unwrap()
+ .resolve_at(
+ GATEWAY_KEY,
+ ApiFormat::OpenAiResponses,
+ "primary:gpt-4.1",
+ NOW_TIME,
+ REQUEST_ID,
+ )
+ .await
+ .unwrap();
+ server.abort();
+ assert_eq!(context.connection().id(), "connection-1");
+ assert_eq!(connections.load(Ordering::SeqCst), 2);
+}
+
+#[tokio::test]
+async fn timeouts_are_not_retried() {
+ let web = FakeWeb::start(|_| async {
+ tokio::time::sleep(Duration::from_secs(1)).await;
+ response(success("project-1", "private-provider-token"))
+ })
+ .await;
+ let error = web
+ .control_plane_with_resolve_limits(Duration::from_millis(100), 256 * 1024)
+ .resolve_at(
+ GATEWAY_KEY,
+ ApiFormat::OpenAiResponses,
+ "primary:gpt-4.1",
+ NOW_TIME,
+ REQUEST_ID,
+ )
+ .await
+ .unwrap_err();
+ assert_eq!(error, ResolutionError::Timeout);
+ assert_eq!(web.calls(), 1);
+}
+
#[tokio::test]
async fn model_policy_rejections_are_distinct_from_other_forbidden_responses() {
let web = FakeWeb::start(|_| async {