Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(ai-gateway): relay Responses compact and GET models (#17594)

Hassieb Pakzad4cf14c7a14a3
diff --git a/ai-gateway/README.md b/ai-gateway/README.md
index ee8d38c39d..d2f5442729 100644
--- a/ai-gateway/README.md
+++ b/ai-gateway/README.md
@@ -1,9 +1,11 @@
# AI Gateway service
-A standalone Rust gateway for `POST /openai/v1/responses`. Web resolves the gateway
-key to a trusted provider connection; Rust relays native JSON or SSE without
-rewriting provider bytes. A bounded capture layer captures request/response facts and
-logs capture completeness at debug level. Each finalized execution is immediately
+A standalone Rust gateway for `POST /openai/v1/responses`, `POST /openai/v1/responses/compact`,
+and `GET /openai/v1/models`. Web resolves the gateway key to a trusted provider connection;
+Rust relays native JSON or SSE without rewriting provider bytes. Compact uses the same
+Responses capture path. Models listing is a GET proxy of OpenAI's catalog and is not ingested
+as a generation. A bounded capture layer captures request/response facts and
+logs capture completeness at debug level. Each finalized generation is immediately
uploaded as a Langfuse generation through OTLP. Building and testing need no real
Web, database or provider credentials.
@@ -165,6 +167,14 @@ curl http://localhost:8080/openai/v1/responses \
-H 'Content-Type: application/json' \
-d '{"model":"gpt-4.1-mini","input":"Say hello"}'
# Add "stream":true to the JSON and use curl -N for SSE.
+
+curl http://localhost:8080/openai/v1/responses/compact \
+ -H "Authorization: Bearer $LANGFUSE_GATEWAY_KEY" \
+ -H 'Content-Type: application/json' \
+ -d '{"model":"gpt-4.1","input":[]}'
+
+curl http://localhost:8080/openai/v1/models \
+ -H "Authorization: Bearer $LANGFUSE_GATEWAY_KEY"
```
```python
@@ -180,9 +190,11 @@ response = client.responses.create(model="gpt-4.1-mini", input="Say hello")
print(response.output_text)
```
-The gateway forwards inference to the official OpenAI Responses endpoint only. It does not
-retry, follow redirects, or accept client routing overrides. Request parsing is
-best-effort capture only; Web still selects the provider connection.
+The gateway forwards inference to official OpenAI v1 paths only: Responses, Responses compact,
+and GET `/models`. It does not retry, follow redirects, or accept client routing overrides.
+Request parsing is best-effort capture only; Web still selects the provider connection.
+Compact `encrypted_content` is relayed unchanged. GET `/models` returns OpenAI's list
+payload as-is, drops request query strings, and does not upload a generation.
Provider errors retain their status and body. Gateway errors use an OpenAI-style
`{"error":{"message":"...","type":"...","param":null,"code":"..."}}` envelope.
@@ -548,7 +560,7 @@ cargo test --locked
- `main.rs`: configuration, logging, signal registration and process exit.
- `http.rs`: credential extraction, bounded body reads and gateway error envelopes.
- `inference.rs`: `InferenceService` coordinates resolution, admission and forwarding.
-- `providers/openai.rs`: fixed destination, provider credentials and admission.
+- `providers/openai.rs`: fixed OpenAI v1 origin, route, provider credentials and admission.
- `transport/mod.rs`: header allowlists, bounded byte relay and response lifetime.
- `resolution/mod.rs`: trusted base URL configuration and bounded Web HTTP client.
- `resolution/contracts.rs`: strict Web response validation and immutable execution context.
diff --git a/ai-gateway/src/capture/mod.rs b/ai-gateway/src/capture/mod.rs
index b37d850b0e..d26710a8ab 100644
--- a/ai-gateway/src/capture/mod.rs
+++ b/ai-gateway/src/capture/mod.rs
@@ -67,6 +67,23 @@ pub(crate) struct ExecutionCapture {
}
impl ExecutionCapture {
+ pub fn unobserved() -> Self {
+ Self {
+ span: tracing::Span::current(),
+ protocol: None,
+ started: Instant::now(),
+ start_time_unix_ms: SystemTime::now()
+ .duration_since(UNIX_EPOCH)
+ .unwrap_or_default()
+ .as_millis(),
+ first_byte_ms: None,
+ completion_start_ms: None,
+ http_status: None,
+ metadata: json!({}),
+ delivery: None,
+ }
+ }
+
pub fn openai_responses(
context: &ResolvedRequestContext,
headers: &HeaderMap,
diff --git a/ai-gateway/src/capture/openai_responses/tests.rs b/ai-gateway/src/capture/openai_responses/tests.rs
index 1a137c2f56..7efbf55a92 100644
--- a/ai-gateway/src/capture/openai_responses/tests.rs
+++ b/ai-gateway/src/capture/openai_responses/tests.rs
@@ -454,7 +454,7 @@ async fn upstream_read_timeout_is_logged_as_timeout() {
.await;
let context = resolved_request_context_with_mode("provider-secret", "full").await;
let provider = OpenAiProvider::for_test(
- format!("{}/v1/responses", upstream.url),
+ format!("{}/v1", upstream.url),
ProviderLimits {
active: 1,
read_timeout: Duration::from_millis(50),
@@ -509,7 +509,7 @@ async fn native_http_relay_logs_once_on_eof_drop_and_unpolled_deadline() {
.await;
let context = resolved_request_context_with_mode("provider-secret", "full").await;
let provider = OpenAiProvider::for_test(
- format!("{}/v1/responses", upstream.url),
+ format!("{}/v1", upstream.url),
ProviderLimits {
active: 1,
execution_timeout: Duration::from_secs(1),
@@ -573,7 +573,7 @@ async fn provider_future_finalizes_on_timeout_and_cancellation_before_headers()
.await;
let context = resolved_request_context_with_mode("provider-secret", "full").await;
let provider = OpenAiProvider::for_test(
- format!("{}/v1/responses", upstream.url),
+ format!("{}/v1", upstream.url),
ProviderLimits {
active: 1,
headers_timeout: Duration::from_millis(100),
@@ -675,11 +675,9 @@ async fn client_compression_preferences_do_not_disable_capture() {
)
.unwrap();
let context = resolved_request_context_with_mode("provider-secret", "full").await;
- let provider = OpenAiProvider::for_test(
- format!("{}/v1/responses", upstream.url),
- ProviderLimits::default(),
- )
- .with_telemetry(telemetry.clone());
+ let provider =
+ OpenAiProvider::for_test(format!("{}/v1", upstream.url), ProviderLimits::default())
+ .with_telemetry(telemetry.clone());
let mut headers = HeaderMap::new();
headers.insert(
header::ACCEPT_ENCODING,
diff --git a/ai-gateway/src/http.rs b/ai-gateway/src/http.rs
index aad0966047..279bb33b0b 100644
--- a/ai-gateway/src/http.rs
+++ b/ai-gateway/src/http.rs
@@ -3,17 +3,17 @@ use std::{error::Error, sync::Arc, time::Duration};
use axum::{
Json, Router,
- body::{Body, to_bytes},
+ body::{Body, Bytes, to_bytes},
extract::{Request, State},
http::{HeaderMap, StatusCode, header},
response::{IntoResponse, Response},
- routing::post,
+ routing::{get, post},
};
use serde::Serialize;
use crate::{
inference::{InferenceService, RequestPreparationError},
- providers::openai::ProviderError,
+ providers::openai::{OpenAiRoute, ProviderError},
resolution::ResolutionError,
server::GatewayLifecycleState,
};
@@ -31,6 +31,11 @@ struct InferenceRouteState {
pub fn router(inference: Option<InferenceService>, lifecycle: GatewayLifecycleState) -> Router {
Router::new()
.route("/openai/v1/responses", post(handle_responses))
+ .route(
+ "/openai/v1/responses/compact",
+ post(handle_responses_compact),
+ )
+ .route("/openai/v1/models", get(handle_models))
.with_state(InferenceRouteState {
inference: inference.map(Arc::new),
lifecycle,
@@ -40,6 +45,28 @@ pub fn router(inference: Option<InferenceService>, lifecycle: GatewayLifecycleSt
async fn handle_responses(
State(state): State<InferenceRouteState>,
request: Request,
+) -> Result<Response, InferenceHttpError> {
+ handle_openai(state, request, OpenAiRoute::Responses).await
+}
+
+async fn handle_responses_compact(
+ State(state): State<InferenceRouteState>,
+ request: Request,
+) -> Result<Response, InferenceHttpError> {
+ handle_openai(state, request, OpenAiRoute::ResponsesCompact).await
+}
+
+async fn handle_models(
+ State(state): State<InferenceRouteState>,
+ request: Request,
+) -> Result<Response, InferenceHttpError> {
+ handle_openai(state, request, OpenAiRoute::Models).await
+}
+
+async fn handle_openai(
+ state: InferenceRouteState,
+ request: Request,
+ route: OpenAiRoute,
) -> Result<Response, InferenceHttpError> {
let inference = state
.inference
@@ -55,21 +82,26 @@ async fn handle_responses(
RequestPreparationError::Provider(error) => InferenceHttpError::Provider(error),
})?;
let (parts, body) = request.into_parts();
- let bytes = tokio::time::timeout(REQUEST_READ_TIMEOUT, to_bytes(body, MAX_REQUEST_BYTES))
- .await
- .map_err(|_| InferenceHttpError::RequestTimeout)?
- .map_err(|error| {
- if error
- .source()
- .is_some_and(<dyn Error + 'static>::is::<http_body_util::LengthLimitError>)
- {
- InferenceHttpError::TooLarge
- } else {
- InferenceHttpError::InvalidBody
- }
- })?;
+ let bytes = match route {
+ OpenAiRoute::Models => Bytes::new(),
+ OpenAiRoute::Responses | OpenAiRoute::ResponsesCompact => {
+ tokio::time::timeout(REQUEST_READ_TIMEOUT, to_bytes(body, MAX_REQUEST_BYTES))
+ .await
+ .map_err(|_| InferenceHttpError::RequestTimeout)?
+ .map_err(|error| {
+ if error
+ .source()
+ .is_some_and(<dyn Error + 'static>::is::<http_body_util::LengthLimitError>)
+ {
+ InferenceHttpError::TooLarge
+ } else {
+ InferenceHttpError::InvalidBody
+ }
+ })?
+ }
+ };
inference
- .forward(permit, context, &parts.headers, bytes)
+ .forward(permit, context, &parts.headers, bytes, route)
.await
.map_err(InferenceHttpError::Provider)
}
diff --git a/ai-gateway/src/http/tests.rs b/ai-gateway/src/http/tests.rs
index ad6aa91780..9a0e9401e0 100644
--- a/ai-gateway/src/http/tests.rs
+++ b/ai-gateway/src/http/tests.rs
@@ -175,6 +175,14 @@ impl Gateway {
fn post(&self) -> reqwest::RequestBuilder {
reqwest::Client::new().post(format!("{}/openai/v1/responses", self.url))
}
+
+ fn compact(&self) -> reqwest::RequestBuilder {
+ reqwest::Client::new().post(format!("{}/openai/v1/responses/compact", self.url))
+ }
+
+ fn models(&self) -> reqwest::RequestBuilder {
+ reqwest::Client::new().get(format!("{}/openai/v1/models", self.url))
+ }
}
impl Drop for Gateway {
@@ -186,10 +194,7 @@ impl Drop for Gateway {
fn inference(web: &FakeServer, provider: &FakeServer) -> InferenceService {
InferenceService::for_test(
web.control_plane(),
- OpenAiProvider::for_test(
- format!("{}/v1/responses", provider.url),
- ProviderLimits::default(),
- ),
+ OpenAiProvider::for_test(format!("{}/v1", provider.url), ProviderLimits::default()),
128,
)
}
@@ -276,6 +281,70 @@ async fn native_json_and_sse_traverse_resolution_and_relay() {
assert_eq!(provider.calls(), 2);
}
+#[tokio::test]
+async fn compact_relays_opaque_json_and_models_proxies_openai_list_without_query() {
+ let web = FakeServer::start(|_| async { resolution_response("provider-secret") }).await;
+ let provider = FakeServer::start(|request| async move {
+ let method = request.method().clone();
+ let uri = request.uri().to_string();
+ let body = to_bytes(request.into_body(), 4096).await.unwrap();
+ match (method.as_str(), uri.as_str()) {
+ ("POST", "/v1/responses/compact") => {
+ assert_eq!(
+ body,
+ r#"{"model":"gpt-4.1","encrypted_content":"opaque-ciphertext"}"#
+ );
+ Response::builder()
+ .header("content-type", "application/json")
+ .body(Body::from(
+ r#"{"id":"comp_1","object":"response.compaction","encrypted_content":"opaque-ciphertext"}"#,
+ ))
+ .unwrap()
+ }
+ ("GET", "/v1/models") => {
+ assert!(body.is_empty());
+ Response::builder()
+ .header("content-type", "application/json")
+ .body(Body::from(
+ r#"{"object":"list","data":[{"id":"gpt-4.1","object":"model"}]}"#,
+ ))
+ .unwrap()
+ }
+ other => panic!("unexpected provider request {other:?}"),
+ }
+ })
+ .await;
+ let gateway = Gateway::start(Some(inference(&web, &provider))).await;
+ let compact = gateway
+ .compact()
+ .bearer_auth("gateway-key")
+ .body(r#"{"model":"gpt-4.1","encrypted_content":"opaque-ciphertext"}"#)
+ .send()
+ .await
+ .unwrap();
+ assert_eq!(compact.status(), StatusCode::OK);
+ assert_eq!(
+ compact.text().await.unwrap(),
+ r#"{"id":"comp_1","object":"response.compaction","encrypted_content":"opaque-ciphertext"}"#
+ );
+ let models = reqwest::Client::new()
+ .get(format!(
+ "{}/openai/v1/models?secret=query-canary",
+ gateway.url
+ ))
+ .bearer_auth("gateway-key")
+ .send()
+ .await
+ .unwrap();
+ assert_eq!(models.status(), StatusCode::OK);
+ assert_eq!(
+ models.text().await.unwrap(),
+ r#"{"object":"list","data":[{"id":"gpt-4.1","object":"model"}]}"#
+ );
+ assert_eq!(web.calls(), 2);
+ assert_eq!(provider.calls(), 2);
+}
+
#[tokio::test]
async fn malformed_credentials_skip_resolution_and_oversized_body_releases_capacity() {
let web = FakeServer::start(|_| async { resolution_response("provider-secret") }).await;
@@ -455,6 +524,10 @@ async fn unconfigured_gateway_is_live_but_not_ready_and_inference_is_unavailable
gateway.post().send().await.unwrap().status(),
StatusCode::SERVICE_UNAVAILABLE
);
+ assert_eq!(
+ gateway.models().send().await.unwrap().status(),
+ StatusCode::SERVICE_UNAVAILABLE
+ );
}
#[tokio::test]
diff --git a/ai-gateway/src/inference.rs b/ai-gateway/src/inference.rs
index b7bc47246e..5c6e7b0f32 100644
--- a/ai-gateway/src/inference.rs
+++ b/ai-gateway/src/inference.rs
@@ -6,7 +6,7 @@ use axum::{
use tokio::sync::Semaphore;
use crate::{
- providers::openai::{OpenAiProvider, ProviderError, RequestPermit},
+ providers::openai::{OpenAiProvider, OpenAiRoute, ProviderError, RequestPermit},
resolution::{
ApiFormat, ControlPlaneClient, ControlPlaneConfig, ResolutionError, ResolvedRequestContext,
},
@@ -81,8 +81,11 @@ impl InferenceService {
context: ResolvedRequestContext,
headers: &HeaderMap,
body: Bytes,
+ route: OpenAiRoute,
) -> Result<Response<Body>, ProviderError> {
- self.provider.forward(permit, context, headers, body).await
+ self.provider
+ .forward_route(permit, context, headers, body, route)
+ .await
}
#[cfg(test)]
diff --git a/ai-gateway/src/observability/http/tests.rs b/ai-gateway/src/observability/http/tests.rs
index 1a84aa9c6f..ff021a974c 100644
--- a/ai-gateway/src/observability/http/tests.rs
+++ b/ai-gateway/src/observability/http/tests.rs
@@ -190,7 +190,8 @@ async fn provider_http_errors_are_traced_without_changing_the_response() {
.unwrap()
})
.await;
- let provider = OpenAiProvider::for_test(upstream.url.clone(), ProviderLimits::default());
+ let provider =
+ OpenAiProvider::for_test(format!("{}/v1", upstream.url), ProviderLimits::default());
let recording = Recording::start();
let response = provider
.forward(
diff --git a/ai-gateway/src/providers/openai.rs b/ai-gateway/src/providers/openai.rs
index 9806f3211d..714997db0a 100644
--- a/ai-gateway/src/providers/openai.rs
+++ b/ai-gateway/src/providers/openai.rs
@@ -1,9 +1,9 @@
-//! Native Responses transport. Provider requests use only resolved credentials.
+//! Native `OpenAI` v1 transport. Provider requests use only resolved credentials.
use std::{sync::Arc, time::Duration};
use axum::{
body::{Body, Bytes},
- http::{HeaderMap, HeaderValue, Response, header},
+ http::{HeaderMap, HeaderValue, Method, Response, header},
};
use reqwest::Client;
use reqwest_middleware::ClientWithMiddleware;
@@ -21,6 +21,38 @@ use crate::{
pub use crate::transport::ProviderError;
+const OPENAI_V1: &str = "https://api.openai.com/v1";
+
+/// Official `OpenAI` paths relayed without translation. Compact is Responses JSON;
+/// models listing is GET with no request query forwarding and no generation ingest.
+#[derive(Clone, Copy)]
+pub(crate) enum OpenAiRoute {
+ Responses,
+ ResponsesCompact,
+ Models,
+}
+
+impl OpenAiRoute {
+ fn method(self) -> Method {
+ match self {
+ Self::Models => Method::GET,
+ Self::Responses | Self::ResponsesCompact => Method::POST,
+ }
+ }
+
+ fn path(self) -> &'static str {
+ match self {
+ Self::Responses => "/responses",
+ Self::ResponsesCompact => "/responses/compact",
+ Self::Models => "/models",
+ }
+ }
+
+ fn captures_generation(self) -> bool {
+ !matches!(self, Self::Models)
+ }
+}
+
pub(crate) struct ProviderLimits {
pub active: usize,
pub execution_timeout: Duration,
@@ -52,8 +84,7 @@ pub struct OpenAiProvider {
capacity: Arc<Semaphore>,
limits: ProviderLimits,
telemetry: Option<crate::telemetry::Telemetry>,
- #[cfg(test)]
- endpoint: String,
+ base_url: String,
}
impl OpenAiProvider {
@@ -90,8 +121,7 @@ impl OpenAiProvider {
capacity: Arc::new(Semaphore::new(limits.active)),
limits,
telemetry: None,
- #[cfg(test)]
- endpoint: "https://api.openai.com/v1/responses".to_owned(),
+ base_url: OPENAI_V1.to_owned(),
})
}
@@ -129,32 +159,47 @@ impl OpenAiProvider {
headers: &HeaderMap,
body: Bytes,
) -> Result<Response<Body>, ProviderError> {
- #[cfg(not(test))]
- let endpoint = "https://api.openai.com/v1/responses";
- #[cfg(test)]
- let endpoint = &self.endpoint;
+ self.forward_route(permit, context, headers, body, OpenAiRoute::Responses)
+ .await
+ }
+ pub(crate) async fn forward_route(
+ &self,
+ permit: RequestPermit,
+ context: ResolvedRequestContext,
+ headers: &HeaderMap,
+ body: Bytes,
+ route: OpenAiRoute,
+ ) -> Result<Response<Body>, ProviderError> {
let mut authorization =
HeaderValue::from_str(&format!("Bearer {}", context.connection().provider_token()))
.map_err(|_| ProviderError::Configuration)?;
authorization.set_sensitive(true);
- let mut capture = ExecutionCapture::openai_responses(&context, headers, &body);
- if let Some(telemetry) = &self.telemetry {
- capture.deliver_to(telemetry.clone(), &context, headers);
+ let mut capture = if route.captures_generation() {
+ let mut capture = ExecutionCapture::openai_responses(&context, headers, &body);
+ if let Some(telemetry) = &self.telemetry {
+ capture.deliver_to(telemetry.clone(), &context, headers);
+ }
+ capture
+ } else {
+ ExecutionCapture::unobserved()
+ };
+ let mut upstream = self
+ .client
+ .request(route.method(), self.request_url(route))
+ .with_extension(DisableOtelPropagation)
+ .headers(transport::request_headers(headers))
+ // Observe plain JSON/SSE while relaying the provider bytes unchanged.
+ .header(header::ACCEPT_ENCODING, "identity")
+ .header(header::AUTHORIZATION, authorization);
+ if route.captures_generation() {
+ upstream = upstream.body(body);
}
let response = tokio::time::timeout_at(
permit
.deadline
.min(Instant::now() + self.limits.headers_timeout),
- self.client
- .post(endpoint)
- .with_extension(DisableOtelPropagation)
- .headers(transport::request_headers(headers))
- // Observe plain JSON/SSE while relaying the provider bytes unchanged.
- .header(header::ACCEPT_ENCODING, "identity")
- .header(header::AUTHORIZATION, authorization)
- .body(body)
- .send(),
+ upstream.send(),
)
.await
.map_err(|_| ProviderError::Timeout)
@@ -187,10 +232,18 @@ impl OpenAiProvider {
Ok(downstream)
}
+ fn request_url(&self, route: OpenAiRoute) -> String {
+ let base = self.base_url.trim_end_matches('/');
+ let mut url = String::with_capacity(base.len() + route.path().len());
+ url.push_str(base);
+ url.push_str(route.path());
+ url
+ }
+
#[cfg(test)]
- pub(crate) fn for_test(endpoint: String, limits: ProviderLimits) -> Self {
+ pub(crate) fn for_test(base_url: String, limits: ProviderLimits) -> Self {
let mut provider = Self::with_limits(limits).unwrap();
- provider.endpoint = endpoint;
+ provider.base_url = base_url;
provider
}
}
diff --git a/ai-gateway/src/providers/openai/tests.rs b/ai-gateway/src/providers/openai/tests.rs
index f49026d650..bb6b10cf2e 100644
--- a/ai-gateway/src/providers/openai/tests.rs
+++ b/ai-gateway/src/providers/openai/tests.rs
@@ -7,7 +7,7 @@ use tokio::sync::Notify;
fn provider(server: &FakeServer, active: usize) -> OpenAiProvider {
OpenAiProvider::for_test(
- format!("{}/v1/responses", server.url),
+ format!("{}/v1", server.url),
ProviderLimits {
active,
..ProviderLimits::default()
@@ -536,6 +536,97 @@ async fn cancelled_and_timed_out_executions_upload_after_provider_context_is_rel
}
}
+#[tokio::test]
+async fn compact_posts_compact_path_and_models_get_skips_ingestion() {
+ use crate::{resolution::ControlPlaneConfig, telemetry::Telemetry};
+ use serde_json::Value;
+
+ const COMPACT: &[u8] =
+ br#"{"model":"gpt-4.1","input":[{"encrypted_content":"opaque-ciphertext"}]}"#;
+ const COMPACT_RESPONSE: &str =
+ r#"{"id":"comp_1","output":[{"encrypted_content":"opaque-ciphertext"}]}"#;
+ const MODELS: &str = r#"{"object":"list","data":[{"id":"gpt-4.1","object":"model"}]}"#;
+
+ let (sent, mut received) = tokio::sync::mpsc::channel(2);
+ let sink = FakeServer::start(move |request| {
+ let sent = sent.clone();
+ async move {
+ let bytes = to_bytes(request.into_body(), 65536).await.unwrap();
+ sent.send(serde_json::from_slice::<Value>(&bytes).unwrap())
+ .await
+ .unwrap();
+ Response::new(Body::from("{}"))
+ }
+ })
+ .await;
+ let telemetry =
+ Telemetry::new(&ControlPlaneConfig::new(&sink.url, "service-key").unwrap()).unwrap();
+ let upstream = FakeServer::start(|request| async move {
+ let uri = request.uri().to_string();
+ let method = request.method().clone();
+ let body = to_bytes(request.into_body(), 4096).await.unwrap();
+ match (method.as_str(), uri.as_str()) {
+ ("POST", "/v1/responses/compact") => {
+ assert_eq!(body, COMPACT);
+ Response::builder()
+ .header("content-type", "application/json")
+ .body(Body::from(COMPACT_RESPONSE))
+ .unwrap()
+ }
+ ("GET", "/v1/models") => {
+ assert!(body.is_empty());
+ Response::builder()
+ .header("content-type", "application/json")
+ .body(Body::from(MODELS))
+ .unwrap()
+ }
+ other => panic!("unexpected provider request {other:?}"),
+ }
+ })
+ .await;
+ let relay = provider(&upstream, 1).with_telemetry(telemetry.clone());
+ let compact = relay
+ .forward_route(
+ relay.try_admit().unwrap(),
+ resolved_request_context("provider-secret").await,
+ &HeaderMap::new(),
+ Bytes::from_static(COMPACT),
+ OpenAiRoute::ResponsesCompact,
+ )
+ .await
+ .unwrap();
+ assert_eq!(
+ to_bytes(compact.into_body(), 4096).await.unwrap(),
+ COMPACT_RESPONSE
+ );
+ let payload = tokio::time::timeout(Duration::from_secs(1), received.recv())
+ .await
+ .unwrap()
+ .unwrap();
+ assert_eq!(
+ payload["resourceSpans"][0]["scopeSpans"][0]["spans"][0]["name"],
+ "openai.responses"
+ );
+ let models = relay
+ .forward_route(
+ relay.try_admit().unwrap(),
+ resolved_request_context("provider-secret").await,
+ &HeaderMap::new(),
+ Bytes::new(),
+ OpenAiRoute::Models,
+ )
+ .await
+ .unwrap();
+ assert_eq!(to_bytes(models.into_body(), 4096).await.unwrap(), MODELS);
+ tokio::time::sleep(Duration::from_millis(50)).await;
+ assert!(received.try_recv().is_err());
+ telemetry
+ .shutdown(Instant::now() + Duration::from_secs(1))
+ .await;
+ assert_eq!(sink.calls(), 1);
+ assert_eq!(upstream.calls(), 2);
+}
+
fn assert_completed_upload(payload: &serde_json::Value, streaming: bool) {
use serde_json::{Value, json};