Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

67.2 KB
Raw Blame
use super::*;
use crate::{
    config::{DEFAULT_MAX_REQUEST_BYTES as MAX_REQUEST_BYTES, ExecutionLimits, RequestLimits},
    providers::{ProviderLimits, ProviderTransport},
    resolution::ApiFormat,
    test_support::{FakeServer, resolution_response, resolution_response_for},
};
use axum::body::to_bytes;
use futures_util::{StreamExt, stream};
use std::{
    convert::Infallible,
    io::Write,
    sync::atomic::{AtomicBool, Ordering},
    time::Duration,
};
use tokio::{net::TcpListener, sync::oneshot, task::JoinHandle};
use tower::ServiceExt;

/// The budget fits one request: any held reservation leaves less than the overhead.
fn single_request_memory() -> RequestLimits {
    RequestLimits {
        max_request_bytes: MAX_REQUEST_BYTES,
        memory_budget_bytes: 2 * MAX_REQUEST_BYTES + 1,
        request_overhead_bytes: MAX_REQUEST_BYTES + 1,
    }
}

struct Gateway {
    url: String,
    shutdown: Option<oneshot::Sender<()>>,
    task: JoinHandle<std::io::Result<()>>,
    app: Router,
}

#[tokio::test]
async fn held_memory_rejects_others_until_its_holder_ends() {
    let entered = Arc::new(tokio::sync::Notify::new());
    let started = entered.clone();
    let web = FakeServer::start(move |request| {
        let started = started.clone();
        async move {
            let authorization = request.headers()[header::AUTHORIZATION].to_str().unwrap();
            if request.uri().path() == "/api/internal/ai-gateway/v1/models"
                || authorization == "Bearer stalled"
            {
                started.notify_one();
                return std::future::pending().await;
            }
            if authorization == "Bearer invalid" {
                return Response::builder()
                    .status(StatusCode::UNAUTHORIZED)
                    .body(Body::empty())
                    .unwrap();
            }
            resolution_response("provider")
        }
    })
    .await;
    let provider = FakeServer::start(|_| async { Response::new(Body::empty()) }).await;
    let gateway = Gateway::start(Some(
        InferenceService::for_test(
            web.control_plane(),
            ProviderTransport::for_test(provider.url.clone(), ProviderLimits::default()),
        )
        .with_request_limits(single_request_memory()),
    ))
    .await;
    let inference = |key: &str, body: Body| {
        Request::post("/openai/v1/responses")
            .header(header::AUTHORIZATION, key)
            .body(body)
            .unwrap()
    };
    let model = || Body::from(r#"{"model":"Production key:test"}"#);
    let stalled_upload = || {
        let entered = entered.clone();
        Body::from_stream(stream::once(async move {
            entered.notify_one();
            std::future::pending::<Result<Bytes, Infallible>>().await
        }))
    };
    let holders = [
        ("resolution", inference("Bearer stalled", model())),
        ("upload", inference("Bearer upload", stalled_upload())),
        (
            "model catalog",
            Request::get("/openai/v1/models")
                .header(header::AUTHORIZATION, "Bearer models")
                .body(Body::empty())
                .unwrap(),
        ),
    ];

    for (calls, (holder, request)) in (1..).zip(holders) {
        let mut held = Box::pin(gateway.app.clone().oneshot(request));
        tokio::time::timeout(Duration::from_secs(2), async {
            tokio::select! {
                () = entered.notified() => {},
                _ = &mut held => panic!("{holder} should remain pending"),
            }
        })
        .await
        .unwrap();
        let body_polled = Arc::new(AtomicBool::new(false));
        let observed = body_polled.clone();
        let body = Body::from_stream(stream::once(async move {
            observed.store(true, Ordering::SeqCst);
            Ok::<_, Infallible>(Bytes::from_static(br#"{"model":"Production key:test"}"#))
        }));
        let busy = gateway
            .app
            .clone()
            .oneshot(inference("Bearer valid", body))
            .await
            .unwrap();
        assert_eq!(busy.status(), StatusCode::SERVICE_UNAVAILABLE, "{holder}");
        assert!(!body_polled.load(Ordering::SeqCst), "{holder}");

        drop(held);
        // A rejected request must release its memory too, or the next call gets 503.
        for (key, status) in [
            ("Bearer invalid", StatusCode::UNAUTHORIZED),
            ("Bearer valid", StatusCode::OK),
        ] {
            let response = gateway
                .app
                .clone()
                .oneshot(inference(key, model()))
                .await
                .unwrap();
            assert_eq!(response.status(), status, "{holder}");
        }
        assert_eq!(provider.calls(), calls, "{holder}");
    }
}

#[tokio::test]
async fn model_catalog_memory_covers_the_catalog_until_it_is_read_or_stalls() {
    let entry = r#"{"id":"Production key:gpt-4.1","provider":"OPENAI","canonicalSlug":"gpt-4.1","displayName":"GPT-4.1","createdAt":"1970-01-01T00:00:10.000Z"}"#;
    let catalog = format!(r#"{{"data":[{}]}}"#, vec![entry; 2000].join(","));
    let catalog_bytes = catalog.len();
    let web = FakeServer::start(move |_| {
        let catalog = catalog.clone();
        async move {
            Response::builder()
                .header("content-type", "application/json")
                .body(Body::from(catalog))
                .unwrap()
        }
    })
    .await;
    let provider = FakeServer::start(|_| async { Response::new(Body::empty()) }).await;
    let overhead = 1024;
    let gateway = Gateway::start(Some(
        InferenceService::for_test(
            web.control_plane(),
            ProviderTransport::for_test(
                provider.url.clone(),
                ProviderLimits {
                    execution: ExecutionLimits {
                        stall_timeout: Duration::from_millis(500),
                        ..ExecutionLimits::default()
                    },
                },
            ),
        )
        .with_request_limits(RequestLimits {
            max_request_bytes: MAX_REQUEST_BYTES,
            memory_budget_bytes: 2 * overhead + 3 * catalog_bytes - 1,
            request_overhead_bytes: overhead,
        }),
    ))
    .await;
    let catalog_request = || {
        gateway.app.clone().oneshot(
            Request::get("/openai/v1/models")
                .header(header::AUTHORIZATION, "Bearer models")
                .body(Body::empty())
                .unwrap(),
        )
    };

    let unread = catalog_request().await.unwrap();
    assert_eq!(unread.status(), StatusCode::OK);
    assert_eq!(
        catalog_request().await.unwrap().status(),
        StatusCode::SERVICE_UNAVAILABLE
    );
    let body = to_bytes(unread.into_body(), usize::MAX).await.unwrap();
    assert_eq!(
        serde_json::from_slice::<serde_json::Value>(&body).unwrap()["data"]
            .as_array()
            .unwrap()
            .len(),
        2000
    );

    let stalled = catalog_request().await.unwrap();
    tokio::time::sleep(Duration::from_millis(700)).await;
    assert_eq!(catalog_request().await.unwrap().status(), StatusCode::OK);
    assert!(to_bytes(stalled.into_body(), usize::MAX).await.is_err());
}

impl Gateway {
    async fn start(inference: Option<InferenceService>) -> Self {
        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
        let url = format!("http://{}", listener.local_addr().unwrap());
        let state = if inference.is_some() {
            GatewayLifecycleState::default()
        } else {
            GatewayLifecycleState::unconfigured()
        };
        let app = crate::server::router(state.clone()).merge(router(inference, state.clone()));
        let (shutdown, signal) = oneshot::channel();
        let task = tokio::spawn(crate::server::serve(
            listener,
            app.clone(),
            state,
            async {
                let _ = signal.await;
            },
            Duration::from_secs(1),
        ));
        let client = reqwest::Client::new();
        for _ in 0..100 {
            if client.get(format!("{url}/health")).send().await.is_ok() {
                return Self {
                    url,
                    shutdown: Some(shutdown),
                    task,
                    app,
                };
            }
            tokio::time::sleep(Duration::from_millis(10)).await;
        }
        panic!("gateway did not start");
    }

    fn post(&self) -> reqwest::RequestBuilder {
        reqwest::Client::new().post(format!("{}/openai/v1/responses", self.url))
    }

    fn chat_completions(&self) -> reqwest::RequestBuilder {
        reqwest::Client::new().post(format!("{}/openai/v1/chat/completions", self.url))
    }

    fn compact(&self) -> reqwest::RequestBuilder {
        reqwest::Client::new().post(format!("{}/openai/v1/responses/compact", self.url))
    }

    fn models(&self) -> reqwest::RequestBuilder {
        reqwest::Client::new().get(format!("{}/openai/v1/models", self.url))
    }

    fn messages(&self) -> reqwest::RequestBuilder {
        reqwest::Client::new().post(format!("{}/anthropic/v1/messages", self.url))
    }
}

/// Returns the response's `langfuse-request-id` after checking it is a canonical `UUIDv7`.
fn request_id(response: &reqwest::Response) -> String {
    let id = response.headers()["langfuse-request-id"]
        .to_str()
        .unwrap()
        .to_owned();
    assert_eq!(id.len(), 36, "{id}");
    for (index, byte) in id.bytes().enumerate() {
        if [8, 13, 18, 23].contains(&index) {
            assert_eq!(byte, b'-', "{id}");
        } else {
            assert!(matches!(byte, b'0'..=b'9' | b'a'..=b'f'), "{id}");
        }
    }
    assert_eq!(id.as_bytes()[14], b'7', "{id}");
    assert!(
        matches!(id.as_bytes()[19], b'8' | b'9' | b'a' | b'b'),
        "{id}"
    );
    id
}

/// A request that never reaches the provider has no generation, and names only
/// its tenant, and only if it was resolved.
fn assert_pre_upstream_headers(headers: &HeaderMap, resolved: bool) {
    assert!(!headers.contains_key("langfuse-trace-id"));
    assert!(!headers.contains_key("langfuse-observation-id"));
    for name in ["langfuse-organization-id", "langfuse-project-id"] {
        assert_eq!(headers.contains_key(name), resolved, "{name}");
    }
    for name in [
        "langfuse-provider",
        "langfuse-provider-connection-id",
        "langfuse-model",
    ] {
        assert!(!headers.contains_key(name), "{name}");
    }
}

#[tokio::test]
#[expect(
    clippy::too_many_lines,
    reason = "the table keeps native error envelope coverage in one place"
)]
async fn every_error_carries_a_fresh_request_id_in_its_header_and_native_body() {
    let web = FakeServer::start(|request| async move {
        if request.headers().get(header::AUTHORIZATION)
            == Some(&header::HeaderValue::from_static("Bearer valid"))
        {
            let body = to_bytes(request.into_body(), 1024).await.unwrap();
            let api_format =
                serde_json::from_slice::<serde_json::Value>(&body).unwrap()["apiFormat"].clone();
            return resolution_response_for(
                serde_json::from_value(api_format).unwrap(),
                "provider-secret",
            );
        }
        Response::builder()
            .status(StatusCode::UNAUTHORIZED)
            .body(Body::empty())
            .unwrap()
    })
    .await;
    let provider = FakeServer::start(|_| async { Response::new(Body::empty()) }).await;
    let unconfigured = Gateway::start(None).await;
    let configured = Gateway::start(Some(inference(&web, &provider))).await;
    let mut seen = std::collections::HashSet::new();
    for (request, status, anthropic, resolved) in [
        (
            unconfigured.post(),
            StatusCode::SERVICE_UNAVAILABLE,
            false,
            false,
        ),
        (
            unconfigured.messages(),
            StatusCode::SERVICE_UNAVAILABLE,
            true,
            false,
        ),
        (configured.post(), StatusCode::UNAUTHORIZED, false, false),
        (
            unconfigured.chat_completions(),
            StatusCode::SERVICE_UNAVAILABLE,
            false,
            false,
        ),
        (
            configured
                .chat_completions()
                .bearer_auth("rejected")
                .body(r#"{"model":"Production key:test"}"#),
            StatusCode::UNAUTHORIZED,
            false,
            false,
        ),
        (
            configured
                .chat_completions()
                .bearer_auth("valid")
                .body(vec![b'x'; MAX_REQUEST_BYTES + 1]),
            StatusCode::PAYLOAD_TOO_LARGE,
            false,
            false,
        ),
        (
            configured
                .post()
                .bearer_auth("rejected")
                .body(r#"{"model":"Production key:test"}"#),
            StatusCode::UNAUTHORIZED,
            false,
            false,
        ),
        (
            configured
                .messages()
                .header("x-api-key", "rejected")
                .body(r#"{"model":"Production key:test"}"#),
            StatusCode::UNAUTHORIZED,
            true,
            false,
        ),
        (
            configured
                .post()
                .bearer_auth("valid")
                .body(vec![b'x'; MAX_REQUEST_BYTES + 1]),
            StatusCode::PAYLOAD_TOO_LARGE,
            false,
            false,
        ),
    ] {
        let response = request
            .header("x-request-id", "client-request")
            .send()
            .await
            .unwrap();
        assert_eq!(response.status(), status);
        let id = request_id(&response);
        assert_pre_upstream_headers(response.headers(), resolved);
        let body: serde_json::Value =
            serde_json::from_slice(&response.bytes().await.unwrap()).unwrap();
        assert_eq!(body["request_id"], id.as_str());
        if anthropic {
            assert_eq!(body["type"], "error");
            assert!(body["error"]["type"].is_string());
        } else {
            assert!(body["error"]["code"].is_string());
        }
        assert!(seen.insert(id), "request IDs must be unique");
    }
    assert_eq!(provider.calls(), 0);
}

#[tokio::test]
async fn resolved_requests_name_their_tenant_and_the_provider_connection_sent_to() {
    let web = FakeServer::start(|request| async move {
        let body = to_bytes(request.into_body(), 1024).await.unwrap();
        let api_format =
            serde_json::from_slice::<serde_json::Value>(&body).unwrap()["apiFormat"].clone();
        resolution_response_for(
            serde_json::from_value(api_format).unwrap(),
            "provider-secret",
        )
    })
    .await;
    let provider = FakeServer::start(|_| async {
        Response::builder()
            .header("content-type", "application/json")
            .body(Body::from("{}"))
            .unwrap()
    })
    .await;
    let gateway = Gateway::start(Some(inference(&web, &provider))).await;
    let client = reqwest::Client::new();
    let anthropic = |path: &str, body: &'static str| {
        client
            .post(format!("{}/anthropic/v1/{path}", gateway.url))
            .header("x-api-key", "gateway-anthropic")
            .header("content-type", "application/json")
            .body(body)
    };
    for (request, provider_type) in [
        (
            anthropic(
                "messages/count_tokens",
                r#"{"messages":[],"model":"Production key:claude-opus-4-1"}"#,
            ),
            "anthropic",
        ),
        (
            anthropic(
                "messages",
                r#"{"model":"Production key:claude-opus-4-1","messages":[]}"#,
            ),
            "anthropic",
        ),
    ] {
        let response = request.send().await.unwrap();
        assert_eq!(response.status(), StatusCode::OK);
        let headers = response.headers();
        let header = |name: &str| headers.get(name).map(|value| value.to_str().unwrap());
        assert_eq!(header("langfuse-organization-id"), Some("org-1"));
        assert_eq!(header("langfuse-project-id"), Some("project-1"));
        assert_eq!(header("langfuse-provider"), Some(provider_type));
        assert_eq!(
            header("langfuse-provider-connection-id"),
            Some("connection-1")
        );
        // The model is an operator-side span and log field only.
        assert_eq!(header("langfuse-model"), None);
    }
    assert_eq!(provider.calls(), 2);
}

impl Drop for Gateway {
    fn drop(&mut self) {
        self.task.abort();
    }
}

fn inference(web: &FakeServer, provider: &FakeServer) -> InferenceService {
    InferenceService::for_test(
        web.control_plane(),
        ProviderTransport::for_test(format!("{}/v1", provider.url), ProviderLimits::default()),
    )
}

#[tokio::test]
async fn native_json_and_sse_traverse_resolution_and_relay() {
    let web = FakeServer::start(|request| async move {
        assert_eq!(request.uri(), "/api/internal/ai-gateway/v1/resolve");
        assert!(
            request
                .headers()
                .contains_key("langfuse-gateway-authorization")
        );
        let key = request.headers()[header::AUTHORIZATION]
            .to_str()
            .unwrap()
            .to_owned();
        let body: serde_json::Value =
            serde_json::from_slice(&to_bytes(request.into_body(), 1024).await.unwrap()).unwrap();
        assert_eq!(body["apiFormat"], "openai.responses");
        assert!(matches!(
            body["model"].as_str(),
            Some("Production key:unchanged" | "Production key:anything")
        ));
        resolution_response(if key == "Bearer gateway-json" {
            "provider-json"
        } else {
            "provider-sse"
        })
    })
    .await;
    let provider = FakeServer::start(|request| async move {
        assert_eq!(request.uri(), "/v1/responses");
        assert!(!request.headers().contains_key("x-langfuse-project-id"));
        assert!(
            !request
                .headers()
                .contains_key("langfuse-gateway-authorization")
        );
        let key = request.headers()[header::AUTHORIZATION]
            .to_str()
            .unwrap()
            .to_owned();
        let body = to_bytes(request.into_body(), 4096).await.unwrap();
        if key == "Bearer provider-json" {
            assert_eq!(body, r#"{ "model":"provider-model", "unknown":true }"#);
            Response::builder()
                .header("content-type", "application/json")
                .body(Body::from("{\"output\":[]}"))
                .unwrap()
        } else {
            assert_eq!(key, "Bearer provider-sse");
            assert_eq!(body, r#"{"model":"provider-model","stream":true}"#);
            Response::builder()
                .header("content-type", "text/event-stream")
                .body(Body::from_stream(stream::iter([
                    Ok::<_, Infallible>("event: arbitrary\ndata: "),
                    Ok("hello\n\n"),
                ])))
                .unwrap()
        }
    })
    .await;
    let gateway = Gateway::start(Some(inference(&web, &provider))).await;
    let json = gateway
        .post()
        .bearer_auth("gateway-json")
        .header("x-langfuse-project-id", "attacker-project")
        .body("{ \"model\":\"Production key:unchanged\", \"unknown\":true }")
        .send();
    let sse = gateway
        .post()
        .bearer_auth("gateway-sse")
        .body(r#"{"model":"Production key:anything","stream":true}"#)
        .send();
    let (json, sse) = tokio::join!(json, sse);
    let json = json.unwrap();
    let sse = sse.unwrap();
    assert_eq!(json.status(), StatusCode::OK);
    assert_eq!(json.text().await.unwrap(), r#"{"output":[]}"#);
    assert_eq!(sse.headers()[header::CONTENT_TYPE], "text/event-stream");
    assert_eq!(
        sse.text().await.unwrap(),
        "event: arbitrary\ndata: hello\n\n"
    );
    assert_eq!(web.calls(), 2);
    assert_eq!(provider.calls(), 2);
}

#[tokio::test]
async fn compact_rewrites_model_and_models_come_from_the_control_plane() {
    let web = FakeServer::start(|request| async move {
        if request.uri().path() == "/api/internal/ai-gateway/v1/models" {
            assert_eq!(request.uri().query(), Some("api_format=openai.responses"));
            return Response::builder()
                .header("content-type", "application/json")
                .body(Body::from(
                    r#"{"data":[{"id":"Production key:gpt-4.1","provider":"OPENAI","canonicalSlug":"gpt-4.1","displayName":"GPT-4.1","createdAt":"1970-01-01T00:00:10.000Z"}]}"#,
                ))
                .unwrap();
        }
        resolution_response("provider-secret")
    })
    .await;
    let provider = FakeServer::start(|request| async move {
        let method = request.method().clone();
        let uri = request.uri().to_string();
        let body = to_bytes(request.into_body(), 4096).await.unwrap();
        match (method.as_str(), uri.as_str()) {
            ("POST", "/v1/responses/compact") => {
                assert_eq!(
                    body,
                    r#"{"model":"provider-model","encrypted_content":"opaque-ciphertext"}"#
                );
                Response::builder()
                    .header("content-type", "application/json")
                    .body(Body::from(
                        r#"{"id":"comp_1","object":"response.compaction","encrypted_content":"opaque-ciphertext"}"#,
                    ))
                    .unwrap()
            }
            other => panic!("unexpected provider request {other:?}"),
        }
    })
    .await;
    let gateway = Gateway::start(Some(inference(&web, &provider))).await;
    let compact = gateway
        .compact()
        .bearer_auth("gateway-key")
        .body(r#"{"model":"Production key:gpt-4.1","encrypted_content":"opaque-ciphertext"}"#)
        .send()
        .await
        .unwrap();
    assert_eq!(compact.status(), StatusCode::OK);
    assert_eq!(
        compact.text().await.unwrap(),
        r#"{"id":"comp_1","object":"response.compaction","encrypted_content":"opaque-ciphertext"}"#
    );
    let models = reqwest::Client::new()
        .get(format!(
            "{}/openai/v1/models?secret=query-canary",
            gateway.url
        ))
        .bearer_auth("gateway-key")
        .send()
        .await
        .unwrap();
    assert_eq!(models.status(), StatusCode::OK);
    assert_eq!(
        models.text().await.unwrap(),
        r#"{"object":"list","data":[{"id":"Production key:gpt-4.1","object":"model","created":10,"owned_by":"openai"}]}"#
    );
    assert_eq!(web.calls(), 2);
    assert_eq!(provider.calls(), 1);
}

#[tokio::test]
async fn model_routes_request_the_matching_control_plane_catalog() {
    let web = FakeServer::start(|request| async move {
        assert_eq!(request.uri().path(), "/api/internal/ai-gateway/v1/models");
        let api_format = request
            .uri()
            .query()
            .unwrap()
            .strip_prefix("api_format=")
            .unwrap();
        let body = match api_format {
            "openai.responses" => {
                r#"{"data":[{"id":"Production key:gpt-4.1","provider":"OPENAI","canonicalSlug":"gpt-4.1","displayName":"GPT-4.1","createdAt":"1970-01-01T00:00:10.000Z"}]}"#
            }
            "anthropic.messages" => {
                r#"{"data":[{"id":"Production key:claude-sonnet-4-5","provider":"ANTHROPIC","canonicalSlug":"claude-sonnet-4-5","displayName":"Claude Sonnet 4.5","createdAt":"2026-01-01T00:00:00.000Z"}]}"#
            }
            other => panic!("unexpected API format {other}"),
        };
        Response::builder()
            .header("content-type", "application/json")
            .body(Body::from(body))
            .unwrap()
    })
    .await;
    let provider = FakeServer::start(|request| async move {
        panic!("model catalog must not reach a provider: {}", request.uri())
    })
    .await;
    let gateway = Gateway::start(Some(inference(&web, &provider))).await;
    for (path, credential, expected) in [
        (
            "openai/v1/models",
            ("authorization", "Bearer gateway-openai"),
            r#"{"object":"list","data":[{"id":"Production key:gpt-4.1","object":"model","created":10,"owned_by":"openai"}]}"#,
        ),
        (
            "anthropic/v1/models",
            ("x-api-key", "gateway-anthropic"),
            r#"{"data":[{"type":"model","id":"Production key:claude-sonnet-4-5","display_name":"Claude Sonnet 4.5","created_at":"2026-01-01T00:00:00.000Z"}],"has_more":false,"first_id":"Production key:claude-sonnet-4-5","last_id":"Production key:claude-sonnet-4-5"}"#,
        ),
    ] {
        let response = reqwest::Client::new()
            .get(format!("{}/{path}", gateway.url))
            .header(credential.0, credential.1)
            .send()
            .await
            .unwrap();
        assert_eq!(response.status(), StatusCode::OK);
        assert_eq!(response.text().await.unwrap(), expected);
    }
    assert_eq!(web.calls(), 2);
    assert_eq!(provider.calls(), 0);
}

#[tokio::test]
async fn malformed_credentials_skip_resolution_and_oversized_body_releases_memory() {
    let web = FakeServer::start(|_| async { resolution_response("provider-secret") }).await;
    let provider = FakeServer::start(|_| async { Response::new(Body::empty()) }).await;
    let service = InferenceService::for_test(
        web.control_plane(),
        ProviderTransport::for_test(provider.url.clone(), ProviderLimits::default()),
    )
    .with_request_limits(single_request_memory());
    let gateway = Gateway::start(Some(service)).await;
    for authorization in [
        None,
        Some("Basic abc"),
        Some("Bearer "),
        Some("Bearer key,other"),
        Some("Bearer key extra"),
    ] {
        let mut request = gateway.post();
        if let Some(value) = authorization {
            request = request.header(header::AUTHORIZATION, value);
        }
        let response = request.send().await.unwrap();
        assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
        let body: serde_json::Value =
            serde_json::from_slice(&response.bytes().await.unwrap()).unwrap();
        assert_eq!(body["error"]["code"], "invalid_api_key");
    }
    let mut headers = HeaderMap::new();
    headers.append(header::AUTHORIZATION, "Bearer one".parse().unwrap());
    headers.append(header::AUTHORIZATION, "Bearer two".parse().unwrap());
    assert_eq!(
        gateway
            .post()
            .headers(headers)
            .send()
            .await
            .unwrap()
            .status(),
        StatusCode::UNAUTHORIZED
    );
    assert_eq!(web.calls(), 0);
    let response = gateway
        .post()
        .bearer_auth("valid-key")
        .body(vec![b'x'; MAX_REQUEST_BYTES + 1])
        .send()
        .await
        .unwrap();
    assert_eq!(response.status(), StatusCode::PAYLOAD_TOO_LARGE);
    assert_eq!(web.calls(), 0);
    assert_eq!(provider.calls(), 0);
    let next = gateway
        .post()
        .bearer_auth("valid-key")
        .body(r#"{"model":"Production key:test"}"#)
        .send()
        .await
        .unwrap();
    assert_eq!(next.status(), StatusCode::OK);
    assert_eq!(web.calls(), 1);
    assert_eq!(provider.calls(), 1);
}

#[tokio::test]
async fn compressed_request_bodies_are_explicitly_rejected() {
    let web = FakeServer::start(|_| async { resolution_response("provider-secret") }).await;
    let provider = FakeServer::start(|_| async { Response::new(Body::empty()) }).await;
    let gateway = Gateway::start(Some(inference(&web, &provider))).await;
    let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
    encoder
        .write_all(br#"{"model":"connection:model","input":"hello"}"#)
        .unwrap();
    for (encoding, body) in [
        ("gzip", encoder.finish().unwrap()),
        (
            "zstd",
            b"encoded bytes are rejected before parsing".to_vec(),
        ),
    ] {
        let response = gateway
            .post()
            .bearer_auth("valid-key")
            .header(header::CONTENT_ENCODING, encoding)
            .header(header::CONTENT_TYPE, "application/json")
            .body(body)
            .send()
            .await
            .unwrap();
        assert_eq!(response.status(), StatusCode::UNSUPPORTED_MEDIA_TYPE);
        let body: serde_json::Value =
            serde_json::from_slice(&response.bytes().await.unwrap()).unwrap();
        assert_eq!(body["error"]["code"], "unsupported_content_encoding");
    }
    assert_eq!(web.calls(), 0);
    assert_eq!(provider.calls(), 0);
}

#[tokio::test]
async fn requests_without_a_model_are_rejected_before_resolution() {
    let web = FakeServer::start(|_| async { resolution_response("provider-secret") }).await;
    let provider = FakeServer::start(|_| async { Response::new(Body::empty()) }).await;
    let gateway = Gateway::start(Some(inference(&web, &provider))).await;

    let response = gateway
        .post()
        .bearer_auth("valid-key")
        .header(header::CONTENT_TYPE, "application/json")
        .body(r#"{"prompt":{"id":"pmpt_123"}}"#)
        .send()
        .await
        .unwrap();

    assert_eq!(response.status(), StatusCode::BAD_REQUEST);
    let body: serde_json::Value = serde_json::from_slice(&response.bytes().await.unwrap()).unwrap();
    assert_eq!(body["error"]["code"], "invalid_body");
    assert_eq!(web.calls(), 0);
    assert_eq!(provider.calls(), 0);
}

#[tokio::test]
async fn resolution_failures_are_sanitized_and_never_call_provider() {
    let web = FakeServer::start(|request| async move {
        let key = request.headers()[header::AUTHORIZATION].to_str().unwrap();
        match key {
            "Bearer malformed" => Response::new(Body::from("secret malformed response")),
            "Bearer invalid-origin" => {
                let body = to_bytes(resolution_response("provider-secret").into_body(), 4096)
                    .await
                    .unwrap();
                let mut body: serde_json::Value = serde_json::from_slice(&body).unwrap();
                body["connection"]["base_url"] = "https://attacker.example/v1".into();
                Response::new(Body::from(body.to_string()))
            }
            _ => {
                let status: u16 = key.strip_prefix("Bearer ").unwrap().parse().unwrap();
                Response::builder()
                    .status(status)
                    .body(Body::from("secret upstream error"))
                    .unwrap()
            }
        }
    })
    .await;
    let provider = FakeServer::start(|_| async { Response::new(Body::empty()) }).await;
    let gateway = Gateway::start(Some(inference(&web, &provider))).await;
    for (key, status) in [
        ("400", 400),
        ("401", 401),
        ("403", 403),
        ("404", 404),
        ("503", 503),
        ("malformed", 502),
        ("invalid-origin", 502),
    ] {
        let response = gateway
            .post()
            .bearer_auth(key)
            .body(r#"{"model":"Production key:test"}"#)
            .send()
            .await
            .unwrap();
        assert_eq!(response.status().as_u16(), status);
        let body = response.text().await.unwrap();
        assert!(!body.contains("secret"));
        assert!(serde_json::from_str::<serde_json::Value>(&body).unwrap()["error"].is_object());
    }
    assert_eq!(provider.calls(), 0);
}

#[tokio::test]
async fn slow_resolution_has_a_bounded_wait() {
    let web = FakeServer::start(|_| async {
        Response::new(Body::from_stream(stream::pending::<
            Result<&'static str, Infallible>,
        >()))
    })
    .await;
    let provider = FakeServer::start(|_| async { Response::new(Body::empty()) }).await;
    let gateway = Gateway::start(Some(inference(&web, &provider))).await;
    let resolution = tokio::time::timeout(
        Duration::from_secs(12),
        gateway
            .post()
            .bearer_auth("slow-resolution")
            .body(r#"{"model":"Production key:test"}"#)
            .send(),
    )
    .await
    .unwrap();
    assert_eq!(resolution.unwrap().status(), StatusCode::GATEWAY_TIMEOUT);
    assert_eq!(web.calls(), 1);
    assert_eq!(provider.calls(), 0);
}

#[tokio::test(start_paused = true)]
async fn a_stalled_request_body_times_out_at_the_read_deadline() {
    let web = FakeServer::start(|_| async { resolution_response("provider") }).await;
    let provider = FakeServer::start(|_| async { Response::new(Body::empty()) }).await;
    let service = inference(&web, &provider);
    // The 16 MiB default keeps the 10 MiB per 30 seconds minimum upload rate.
    assert_eq!(service.request_read_timeout(), Duration::from_secs(49));
    let gateway = Gateway::start(Some(service)).await;
    let stalled = Body::from_stream(stream::pending::<Result<&'static str, Infallible>>());
    let started = tokio::time::Instant::now();
    let request = gateway.app.clone().oneshot(
        Request::post("/openai/v1/responses")
            .header(header::AUTHORIZATION, "Bearer stalled-upload")
            .body(stalled)
            .unwrap(),
    );
    let response = tokio::time::timeout(Duration::from_secs(120), request)
        .await
        .expect("the router must bound the body read")
        .unwrap();
    assert_eq!(response.status(), StatusCode::REQUEST_TIMEOUT);
    assert_eq!(started.elapsed(), Duration::from_secs(49));
    let body = to_bytes(response.into_body(), 4096).await.unwrap();
    assert!(String::from_utf8_lossy(&body).contains("request_timeout"));
    assert_eq!(web.calls(), 0);
}

#[tokio::test]
async fn pre_header_provider_failures_export_the_status_returned_to_the_caller() {
    use crate::{resolution::ControlPlaneConfig, telemetry::Telemetry};
    use serde_json::Value;

    for expected in [StatusCode::GATEWAY_TIMEOUT, StatusCode::BAD_GATEWAY] {
        let (sent, mut received) = tokio::sync::mpsc::channel(1);
        let sink = FakeServer::start(move |request| {
            let sent = sent.clone();
            async move {
                let bytes = to_bytes(request.into_body(), 65536).await.unwrap();
                sent.send(crate::test_support::upload_json(&bytes))
                    .await
                    .unwrap();
                Response::new(Body::from("{}"))
            }
        })
        .await;
        let telemetry =
            Telemetry::for_test(&ControlPlaneConfig::new(&sink.url, "service-key").unwrap());
        let upstream = TcpListener::bind("127.0.0.1:0").await.unwrap();
        let upstream_url = format!("http://{}", upstream.local_addr().unwrap());
        let upstream_task = tokio::spawn(async move {
            let (_connection, _) = upstream.accept().await.unwrap();
            if expected == StatusCode::GATEWAY_TIMEOUT {
                std::future::pending::<()>().await;
            }
        });
        let web = FakeServer::start(|_| async { resolution_response("provider-secret") }).await;
        let provider = ProviderTransport::for_test(
            upstream_url,
            ProviderLimits {
                execution: ExecutionLimits {
                    stall_timeout: Duration::from_millis(100),
                    ..ExecutionLimits::default()
                },
            },
        )
        .with_telemetry(telemetry.clone());
        let gateway = Gateway::start(Some(InferenceService::for_test(
            web.control_plane(),
            provider,
        )))
        .await;
        let response = gateway
            .post()
            .bearer_auth("gateway-key")
            .body(r#"{"model":"requested","input":"hello","stream":true}"#)
            .send()
            .await
            .unwrap();
        assert_eq!(response.status(), expected);
        let id = request_id(&response);
        let header = |name: &str| response.headers()[name].to_str().unwrap().to_owned();
        let (trace_id, observation_id) = (
            header("langfuse-trace-id"),
            header("langfuse-observation-id"),
        );
        let status = response.status();
        let body: Value = serde_json::from_slice(&response.bytes().await.unwrap()).unwrap();
        assert_eq!(body["request_id"], id.as_str());
        let payload = tokio::time::timeout(Duration::from_secs(2), received.recv())
            .await
            .unwrap()
            .unwrap();
        let span = &payload["resourceSpans"][0]["scopeSpans"][0]["spans"][0];
        assert_eq!(span["traceId"], trace_id.as_str());
        assert_eq!(span["spanId"], observation_id.as_str());
        let attrs = span["attributes"].as_array().unwrap();
        let metadata: Value = serde_json::from_str(
            attrs
                .iter()
                .find(|a| a["key"] == "langfuse.observation.metadata")
                .unwrap()["value"]["stringValue"]
                .as_str()
                .unwrap(),
        )
        .unwrap();
        assert_eq!(
            metadata["langfuse.gateway.response.status_code"],
            status.as_u16()
        );
        assert_eq!(metadata["langfuse.gateway.request.id"], id.as_str());
        assert!(
            metadata
                .get("langfuse.gateway.upstream.request.id")
                .is_none()
        );
        telemetry
            .shutdown(tokio::time::Instant::now() + Duration::from_secs(1))
            .await;
        assert_eq!(sink.calls(), 1);
        upstream_task.abort();
    }
}

#[tokio::test]
async fn chat_completions_streams_carry_the_request_and_generation_ids() {
    use crate::{resolution::ControlPlaneConfig, telemetry::Telemetry};
    use serde_json::Value;

    const CONTENT: &str = "data: {\"id\":\"chatcmpl-1\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"hi\"},\"finish_reason\":\"stop\"}],\"usage\":null}\n\n";
    const USAGE: &str = "data: {\"id\":\"chatcmpl-1\",\"choices\":[],\"usage\":{\"prompt_tokens\":2,\"completion_tokens\":1,\"total_tokens\":3}}\n\n";
    const DONE: &str = "data: [DONE]\n\n";
    let (sent, mut received) = tokio::sync::mpsc::channel(1);
    let sink = FakeServer::start(move |request| {
        let sent = sent.clone();
        async move {
            let bytes = to_bytes(request.into_body(), 65536).await.unwrap();
            sent.send(crate::test_support::upload_json(&bytes))
                .await
                .unwrap();
            Response::new(Body::from("{}"))
        }
    })
    .await;
    let telemetry =
        Telemetry::for_test(&ControlPlaneConfig::new(&sink.url, "service-key").unwrap());
    let (resolved, mut resolution_ids) = tokio::sync::mpsc::channel(1);
    let web = chat_completion_resolution_server(resolved).await;
    let provider = FakeServer::start(|request| async move {
        assert_eq!(request.uri(), "/v1/chat/completions");
        Response::builder()
            .header("content-type", "text/event-stream")
            .header("x-request-id", "req-upstream")
            .body(Body::from([CONTENT, USAGE, DONE].concat()))
            .unwrap()
    })
    .await;
    let gateway = Gateway::start(Some(InferenceService::for_test(
        web.control_plane(),
        ProviderTransport::for_test(format!("{}/v1", provider.url), ProviderLimits::default())
            .with_telemetry(telemetry.clone()),
    )))
    .await;
    let response = gateway
        .chat_completions()
        .bearer_auth("gateway-chat")
        .header("x-request-id", "client-request")
        .body(r#"{"model":"gpt-4.1-mini","messages":[],"stream":true}"#)
        .send()
        .await
        .unwrap();
    assert_eq!(response.status(), StatusCode::OK);
    let id = request_id(&response);
    let header = |name: &str| response.headers()[name].to_str().unwrap().to_owned();
    let (trace_id, observation_id) = (
        header("langfuse-trace-id"),
        header("langfuse-observation-id"),
    );
    assert_eq!(header("x-request-id"), "req-upstream");
    assert_eq!(
        response.text().await.unwrap(),
        [CONTENT, USAGE, DONE].concat()
    );
    assert_eq!(resolution_ids.recv().await.unwrap(), id);

    let payload = tokio::time::timeout(Duration::from_secs(2), received.recv())
        .await
        .unwrap()
        .unwrap();
    let span = &payload["resourceSpans"][0]["scopeSpans"][0]["spans"][0];
    assert_eq!(span["name"], "openai.chat-completions");
    assert_eq!(span["traceId"], trace_id.as_str());
    assert_eq!(span["spanId"], observation_id.as_str());
    let metadata: Value = serde_json::from_str(
        span["attributes"]
            .as_array()
            .unwrap()
            .iter()
            .find(|a| a["key"] == "langfuse.observation.metadata")
            .unwrap()["value"]["stringValue"]
            .as_str()
            .unwrap(),
    )
    .unwrap();
    assert_eq!(metadata["langfuse.gateway.request.id"], id.as_str());
    assert_eq!(
        metadata["langfuse.gateway.client.request.id"],
        "client-request"
    );
    assert_eq!(
        metadata["langfuse.gateway.upstream.request.id"],
        "req-upstream"
    );
    telemetry
        .shutdown(tokio::time::Instant::now() + Duration::from_secs(1))
        .await;
}

async fn chat_completion_resolution_server(
    resolved: tokio::sync::mpsc::Sender<String>,
) -> FakeServer {
    FakeServer::start(move |request| {
        let resolved = resolved.clone();
        async move {
            let id = request.headers()["langfuse-gateway-request-id"]
                .to_str()
                .unwrap()
                .to_owned();
            resolved.send(id).await.unwrap();
            resolution_response_for(ApiFormat::OpenAiChatCompletions, "provider-chat")
        }
    })
    .await
}

#[tokio::test]
async fn unconfigured_gateway_is_live_but_not_ready_and_inference_is_unavailable() {
    let gateway = Gateway::start(None).await;
    let client = reqwest::Client::new();
    assert_eq!(
        client
            .get(format!("{}/health", gateway.url))
            .send()
            .await
            .unwrap()
            .status(),
        StatusCode::OK
    );
    assert_eq!(
        client
            .get(format!("{}/ready", gateway.url))
            .send()
            .await
            .unwrap()
            .status(),
        StatusCode::SERVICE_UNAVAILABLE
    );
    assert_eq!(
        gateway.post().send().await.unwrap().status(),
        StatusCode::SERVICE_UNAVAILABLE
    );
    assert_eq!(
        gateway.models().send().await.unwrap().status(),
        StatusCode::SERVICE_UNAVAILABLE
    );
}

#[tokio::test]
async fn request_memory_is_held_through_stream_and_shutdown_drains_it() {
    let (release, released) = tokio::sync::watch::channel(false);
    let web = FakeServer::start(|_| async { resolution_response("provider") }).await;
    let provider = FakeServer::start(move |_| {
        let mut released = released.clone();
        async move {
            let stream = stream::once(async { Ok::<_, Infallible>("data: first\n\n") }).chain(
                stream::once(async move {
                    released.wait_for(|done| *done).await.unwrap();
                    Ok("data: last\n\n")
                }),
            );
            Response::builder()
                .header("content-type", "text/event-stream")
                .body(Body::from_stream(stream))
                .unwrap()
        }
    })
    .await;
    let service = InferenceService::for_test(
        web.control_plane(),
        ProviderTransport::for_test(provider.url.clone(), ProviderLimits::default()),
    )
    .with_request_limits(single_request_memory());
    let mut gateway = Gateway::start(Some(service)).await;
    let mut response = gateway
        .post()
        .bearer_auth("key")
        .body(r#"{"model":"Production key:test"}"#)
        .send()
        .await
        .unwrap();
    // The stream is still open, so the ID was sent with the initial headers.
    request_id(&response);
    assert_eq!(response.chunk().await.unwrap().unwrap(), "data: first\n\n");
    assert_eq!(
        gateway
            .post()
            .bearer_auth("key")
            .body(r#"{"model":"Production key:test"}"#)
            .send()
            .await
            .unwrap()
            .status(),
        StatusCode::SERVICE_UNAVAILABLE
    );
    assert_eq!(web.calls(), 1);
    assert_eq!(provider.calls(), 1);
    gateway.shutdown.take().unwrap().send(()).unwrap();
    assert!(!gateway.task.is_finished());
    release.send(true).unwrap();
    assert_eq!(response.text().await.unwrap(), "data: last\n\n");
    tokio::time::timeout(Duration::from_secs(2), &mut gateway.task)
        .await
        .unwrap()
        .unwrap()
        .unwrap();
}

#[tokio::test]
async fn client_disconnect_releases_memory_for_the_next_request() {
    let web = FakeServer::start(|_| async { resolution_response("provider") }).await;
    let provider = FakeServer::start(|_| async {
        let stream =
            stream::once(async { Ok::<_, Infallible>("data: first\n\n") }).chain(stream::pending());
        Response::builder()
            .header("content-type", "text/event-stream")
            .body(Body::from_stream(stream))
            .unwrap()
    })
    .await;
    let service = InferenceService::for_test(
        web.control_plane(),
        ProviderTransport::for_test(provider.url.clone(), ProviderLimits::default()),
    )
    .with_request_limits(single_request_memory());
    let gateway = Gateway::start(Some(service)).await;
    let mut first = gateway
        .post()
        .bearer_auth("key")
        .body(r#"{"model":"Production key:test"}"#)
        .send()
        .await
        .unwrap();
    assert_eq!(first.chunk().await.unwrap().unwrap(), "data: first\n\n");
    drop(first);
    let mut next = tokio::time::timeout(Duration::from_secs(2), async {
        loop {
            let response = gateway
                .post()
                .bearer_auth("key")
                .body(r#"{"model":"Production key:test"}"#)
                .send()
                .await
                .unwrap();
            if response.status() == StatusCode::OK {
                break response;
            }
            assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
            tokio::time::sleep(Duration::from_millis(10)).await;
        }
    })
    .await
    .unwrap();
    assert_eq!(next.chunk().await.unwrap().unwrap(), "data: first\n\n");
    assert!(web.calls() >= 2);
    assert_eq!(provider.calls(), 2);
}

fn anthropic_gateway_key(request: &Request<Body>) -> Option<String> {
    request
        .headers()
        .get(header::AUTHORIZATION)
        .and_then(|value| value.to_str().ok())
        .map(|value| value.trim_start_matches("Bearer ").to_owned())
}

#[tokio::test]
async fn anthropic_messages_accept_either_credential_header_and_relay_native_sse() {
    let web = FakeServer::start(|request| async move {
        let key = anthropic_gateway_key(&request).unwrap();
        assert_eq!(
            to_bytes(request.into_body(), 1024).await.unwrap(),
            r#"{"apiFormat":"anthropic.messages","model":"Production key:claude-sonnet-4-5","requestBytes":72}"#
        );
        if key == "gateway-anthropic" {
            resolution_response_for(ApiFormat::AnthropicMessages, "provider-anthropic")
        } else {
            Response::builder()
                .status(StatusCode::UNAUTHORIZED)
                .body(Body::empty())
                .unwrap()
        }
    })
    .await;
    let provider = FakeServer::start(|request| async move {
        assert_eq!(request.uri(), "/v1/messages");
        assert_eq!(request.headers()["x-api-key"], "provider-anthropic");
        assert_eq!(request.headers()["anthropic-version"], "2023-06-01");
        assert!(!request.headers().contains_key(header::AUTHORIZATION));
        assert_eq!(
            to_bytes(request.into_body(), 4096).await.unwrap(),
            r#"{"model":"provider-model","stream":true,"messages":[]}"#
        );
        Response::builder()
            .header("content-type", "text/event-stream")
            .header("request-id", "req_upstream")
            .body(Body::from_stream(stream::iter([
                Ok::<_, Infallible>("event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"id\":\"msg_1\",\"usage\":{\"input_tokens\":3,\"output_tokens\":1}}}\n\n"),
                Ok("event: ping\ndata: {\"type\":\"ping\"}\n\n"),
                Ok("event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n"),
            ])))
            .unwrap()
    })
    .await;
    let gateway = Gateway::start(Some(inference(&web, &provider))).await;
    let client = reqwest::Client::new();
    // Claude Code sends x-api-key for ANTHROPIC_API_KEY, Authorization for
    // ANTHROPIC_AUTH_TOKEN, and both when a helper supplies one credential.
    for credential_headers in [
        vec![("x-api-key", "gateway-anthropic")],
        vec![("authorization", "Bearer gateway-anthropic")],
        vec![
            ("x-api-key", "gateway-anthropic"),
            ("authorization", "Bearer gateway-anthropic"),
        ],
    ] {
        let mut request = client
            .post(format!("{}/anthropic/v1/messages?beta=true", gateway.url))
            .header("anthropic-version", "2023-06-01")
            .header("content-type", "application/json")
            .body(r#"{"model":"Production key:claude-sonnet-4-5","stream":true,"messages":[]}"#);
        for (name, value) in credential_headers {
            request = request.header(name, value);
        }
        let response = request.send().await.unwrap();
        assert_eq!(response.status(), StatusCode::OK);
        assert_eq!(response.headers()["request-id"], "req_upstream");
        let body = response.text().await.unwrap();
        assert!(body.starts_with("event: message_start\n"));
        assert!(body.ends_with("event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n"));
    }
    assert_eq!(web.calls(), 3);
    assert_eq!(provider.calls(), 3);
}

type EnvelopeCase = (
    &'static [(&'static str, &'static str)],
    StatusCode,
    &'static str,
    Option<&'static str>,
);

#[tokio::test]
async fn anthropic_credential_conflicts_and_failures_use_the_native_error_envelope() {
    let web = FakeServer::start(|request| async move {
        let status = match anthropic_gateway_key(&request).as_deref() {
            Some("forbidden") => StatusCode::FORBIDDEN,
            Some("unrouted") => StatusCode::NOT_FOUND,
            _ => StatusCode::UNAUTHORIZED,
        };
        Response::builder()
            .status(status)
            .body(Body::from("private-web-body"))
            .unwrap()
    })
    .await;
    let provider = FakeServer::start(|_| async { Response::new(Body::empty()) }).await;
    let gateway = Gateway::start(Some(inference(&web, &provider))).await;
    let client = reqwest::Client::new();
    let cases: [EnvelopeCase; 6] = [
        (
            &[
                ("x-api-key", "real-anthropic-key"),
                ("authorization", "Bearer gateway-key"),
            ],
            StatusCode::UNAUTHORIZED,
            "authentication_error",
            Some("Conflicting gateway credentials in the Authorization and x-api-key headers"),
        ),
        (&[], StatusCode::UNAUTHORIZED, "authentication_error", None),
        (
            &[("x-api-key", "bad key")],
            StatusCode::UNAUTHORIZED,
            "authentication_error",
            None,
        ),
        (
            &[("authorization", "Basic Zm9v")],
            StatusCode::UNAUTHORIZED,
            "authentication_error",
            None,
        ),
        (
            &[("x-api-key", "forbidden")],
            StatusCode::FORBIDDEN,
            "permission_error",
            None,
        ),
        (
            &[("x-api-key", "unrouted")],
            StatusCode::NOT_FOUND,
            "not_found_error",
            None,
        ),
    ];
    for (credential_headers, status, kind, message) in cases {
        let mut request = client
            .post(format!("{}/anthropic/v1/messages", gateway.url))
            .body(r#"{"model":"Production key:test"}"#);
        for (name, value) in credential_headers {
            request = request.header(*name, *value);
        }
        let response = request.send().await.unwrap();
        assert_eq!(response.status(), status, "{credential_headers:?}");
        let body: serde_json::Value =
            serde_json::from_str(&response.text().await.unwrap()).unwrap();
        assert_eq!(body["type"], "error", "{credential_headers:?}");
        assert_eq!(body["error"]["type"], kind, "{credential_headers:?}");
        assert!(body["error"]["message"].is_string());
        if let Some(message) = message {
            assert_eq!(body["error"]["message"], message);
        }
        assert!(!body.to_string().contains("private-web-body"));
    }
    // Conflicting or malformed credentials never reach Web; the two Web rejections do.
    assert_eq!(web.calls(), 2);
    assert_eq!(provider.calls(), 0);
}

#[tokio::test]
async fn model_policy_rejections_use_provider_native_permission_errors() {
    let web = FakeServer::start(|_| async {
        Response::builder()
            .status(StatusCode::FORBIDDEN)
            .header("langfuse-gateway-error-code", "model_not_allowed")
            .body(Body::from("private-web-body"))
            .unwrap()
    })
    .await;
    let provider = FakeServer::start(|_| async { Response::new(Body::empty()) }).await;
    let gateway = Gateway::start(Some(inference(&web, &provider))).await;

    let openai = gateway
        .post()
        .bearer_auth("gateway-key")
        .body(r#"{"model":"Production key:gpt-4.1"}"#)
        .send()
        .await
        .unwrap();
    assert_eq!(openai.status(), StatusCode::FORBIDDEN);
    let openai_body: serde_json::Value =
        serde_json::from_str(&openai.text().await.unwrap()).unwrap();
    assert_eq!(openai_body["error"]["type"], "permission_error");
    assert_eq!(openai_body["error"]["code"], "model_not_allowed");

    let anthropic = reqwest::Client::new()
        .post(format!("{}/anthropic/v1/messages", gateway.url))
        .header("x-api-key", "gateway-key")
        .body(r#"{"model":"Production key:claude-sonnet-4-5"}"#)
        .send()
        .await
        .unwrap();
    assert_eq!(anthropic.status(), StatusCode::FORBIDDEN);
    let anthropic_body: serde_json::Value =
        serde_json::from_str(&anthropic.text().await.unwrap()).unwrap();
    assert_eq!(anthropic_body["error"]["type"], "permission_error");
    assert_eq!(
        anthropic_body["error"]["message"],
        "Model is not allowed for this provider connection"
    );
    assert_eq!(web.calls(), 2);
    assert_eq!(provider.calls(), 0);
}

const RATE_LIMIT_MESSAGE: &str = "Langfuse AI Gateway rate limit exceeded: your organization's Hobby plan allows 100 requests per minute. Retry after 23s or upgrade your plan.";

#[tokio::test]
async fn organization_rate_limits_become_provider_native_429s_with_retry_after() {
    let web = FakeServer::start(|request| async move {
        let body: serde_json::Value =
            serde_json::from_slice(&to_bytes(request.into_body(), 1024).await.unwrap()).unwrap();
        assert_eq!(body["requestBytes"], 34);
        Response::builder()
            .status(StatusCode::TOO_MANY_REQUESTS)
            .header("retry-after", "23")
            .header("langfuse-gateway-error-code", "rate_limit_exceeded")
            .body(Body::from(
                serde_json::json!({
                    "error": RATE_LIMIT_MESSAGE,
                    "limit": "requests",
                    "organizationId": "org-1",
                })
                .to_string(),
            ))
            .unwrap()
    })
    .await;
    let provider = FakeServer::start(|_| async { Response::new(Body::empty()) }).await;
    let gateway = Gateway::start(Some(inference(&web, &provider))).await;

    let openai = gateway
        .post()
        .bearer_auth("gateway-key")
        .body(r#"{"model":"Production key:gpt-4.1"}"#)
        .send()
        .await
        .unwrap();
    assert_eq!(openai.status(), StatusCode::TOO_MANY_REQUESTS);
    assert_eq!(openai.headers()[header::RETRY_AFTER], "23");
    let openai_body: serde_json::Value =
        serde_json::from_str(&openai.text().await.unwrap()).unwrap();
    assert_eq!(openai_body["error"]["message"], RATE_LIMIT_MESSAGE);
    assert_eq!(openai_body["error"]["type"], "requests");
    assert_eq!(openai_body["error"]["code"], "rate_limit_exceeded");

    let anthropic = reqwest::Client::new()
        .post(format!("{}/anthropic/v1/messages", gateway.url))
        .header("x-api-key", "gateway-key")
        .body(r#"{"model":"Production key:gpt-4.1"}"#)
        .send()
        .await
        .unwrap();
    assert_eq!(anthropic.status(), StatusCode::TOO_MANY_REQUESTS);
    assert_eq!(anthropic.headers()[header::RETRY_AFTER], "23");
    let anthropic_body: serde_json::Value =
        serde_json::from_str(&anthropic.text().await.unwrap()).unwrap();
    assert_eq!(anthropic_body["type"], "error");
    assert_eq!(anthropic_body["error"]["type"], "rate_limit_error");
    assert_eq!(anthropic_body["error"]["message"], RATE_LIMIT_MESSAGE);
    assert_eq!(web.calls(), 2);
    assert_eq!(provider.calls(), 0);
}

#[tokio::test]
async fn anthropic_capacity_rejections_use_the_native_envelope() {
    let provider = FakeServer::start(|_| async { Response::new(Body::empty()) }).await;
    let web = FakeServer::start(|_| async {
        resolution_response_for(ApiFormat::AnthropicMessages, "provider-anthropic")
    })
    .await;
    let service = InferenceService::for_test(
        web.control_plane(),
        ProviderTransport::for_test(format!("{}/v1", provider.url), ProviderLimits::default()),
    )
    .with_request_limits(single_request_memory());
    let _held = service.try_reserve_memory("request").unwrap();
    let gateway = Gateway::start(Some(service)).await;
    let response = reqwest::Client::new()
        .post(format!("{}/anthropic/v1/messages", gateway.url))
        .header("x-api-key", "gateway-anthropic")
        .body(r#"{"model":"Production key:test"}"#)
        .send()
        .await
        .unwrap();
    assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
    let body: serde_json::Value = serde_json::from_str(&response.text().await.unwrap()).unwrap();
    assert_eq!(body["error"]["type"], "overloaded_error");
    assert_eq!(provider.calls(), 0);
}

#[tokio::test]
async fn openai_routes_ignore_x_api_key_and_keep_their_envelope() {
    let web = FakeServer::start(|_| async {
        Response::builder()
            .status(StatusCode::UNAUTHORIZED)
            .body(Body::empty())
            .unwrap()
    })
    .await;
    let provider = FakeServer::start(|_| async { Response::new(Body::empty()) }).await;
    let gateway = Gateway::start(Some(inference(&web, &provider))).await;
    for request in [gateway.post(), gateway.chat_completions()] {
        let response = request
            .header("x-api-key", "gateway-key")
            .body("{}")
            .send()
            .await
            .unwrap();
        assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
        let body: serde_json::Value =
            serde_json::from_str(&response.text().await.unwrap()).unwrap();
        assert_eq!(body["error"]["code"], "invalid_api_key");
        assert!(body.get("type").is_none());
    }
    assert_eq!(web.calls(), 0);
}

#[tokio::test]
async fn chat_completions_resolve_their_own_format_and_relay_native_json_and_errors() {
    let web = FakeServer::start(|request| async move {
        let request: serde_json::Value =
            serde_json::from_slice(&to_bytes(request.into_body(), 1024).await.unwrap()).unwrap();
        assert_eq!(request["apiFormat"], "openai.chat-completions");
        let provider_model = request["model"]
            .as_str()
            .and_then(|model| model.split_once(':'))
            .map(|(_, provider_model)| provider_model)
            .unwrap();
        let bytes = to_bytes(
            resolution_response_for(ApiFormat::OpenAiChatCompletions, "provider-chat").into_body(),
            4096,
        )
        .await
        .unwrap();
        let mut response: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
        response["connection"]["provider_model"] = provider_model.into();
        Response::builder()
            .header("content-type", "application/json")
            .body(Body::from(response.to_string()))
            .unwrap()
    })
    .await;
    let provider = FakeServer::start(|request| async move {
        assert_eq!(request.uri(), "/v1/chat/completions");
        assert_eq!(
            request.headers()[header::AUTHORIZATION],
            "Bearer provider-chat"
        );
        let body = to_bytes(request.into_body(), 4096).await.unwrap();
        if body == r#"{"model":"unknown-model","messages":[]}"# {
            return Response::builder()
                .status(StatusCode::NOT_FOUND)
                .header("content-type", "application/json")
                .body(Body::from(r#"{"error":{"code":"model_not_found"}}"#))
                .unwrap();
        }
        assert_eq!(body, r#"{ "model":"gpt-4.1-mini", "messages":[] }"#);
        Response::builder()
            .header("content-type", "application/json")
            .body(Body::from(r#"{"id":"chatcmpl-1","choices":[]}"#))
            .unwrap()
    })
    .await;
    let gateway = Gateway::start(Some(inference(&web, &provider))).await;
    for (body, status, expected) in [
        (
            r#"{ "model":"Production key:gpt-4.1-mini", "messages":[] }"#,
            StatusCode::OK,
            r#"{"id":"chatcmpl-1","choices":[]}"#,
        ),
        (
            r#"{"model":"Production key:unknown-model","messages":[]}"#,
            StatusCode::NOT_FOUND,
            r#"{"error":{"code":"model_not_found"}}"#,
        ),
    ] {
        let response = gateway
            .chat_completions()
            .bearer_auth("gateway-chat")
            .body(body)
            .send()
            .await
            .unwrap();
        assert_eq!(response.status(), status);
        assert_eq!(response.text().await.unwrap(), expected);
    }
    let missing = gateway.chat_completions().body("{}").send().await.unwrap();
    assert_eq!(missing.status(), StatusCode::UNAUTHORIZED);
    let body: serde_json::Value = serde_json::from_str(&missing.text().await.unwrap()).unwrap();
    assert_eq!(body["error"]["type"], "authentication_error");
    assert_eq!(web.calls(), 2);
    assert_eq!(provider.calls(), 2);
}

#[tokio::test]
async fn anthropic_routes_share_the_request_body_limit() {
    let web = FakeServer::start(|_| async {
        resolution_response_for(ApiFormat::AnthropicMessages, "provider-anthropic")
    })
    .await;
    let provider = FakeServer::start(|_| async move { Response::new(Body::empty()) }).await;
    let gateway = Gateway::start(Some(inference(&web, &provider))).await;
    let client = reqwest::Client::new();
    for path in ["messages", "messages/count_tokens"] {
        let send = |size| {
            let prefix = r#"{"model":"Production key:test","padding":""#;
            let suffix = r#""}"#;
            let padding = "x".repeat(size - prefix.len() - suffix.len());
            client
                .post(format!("{}/anthropic/v1/{path}", gateway.url))
                .header("x-api-key", "gateway-anthropic")
                .body(format!("{prefix}{padding}{suffix}"))
                .send()
        };
        assert_eq!(
            send(MAX_REQUEST_BYTES).await.unwrap().status(),
            StatusCode::OK
        );
        let rejected = send(MAX_REQUEST_BYTES + 1).await.unwrap();
        assert_eq!(rejected.status(), StatusCode::PAYLOAD_TOO_LARGE);
        let body: serde_json::Value =
            serde_json::from_str(&rejected.text().await.unwrap()).unwrap();
        assert_eq!(body["error"]["type"], "request_too_large");
    }
    assert_eq!(provider.calls(), 2);
}

#[tokio::test]
async fn body_reservations_follow_the_allocation_on_top_of_the_overhead() {
    let web = FakeServer::start(|_| async { resolution_response("provider") }).await;
    let provider = FakeServer::start(|_| async { Response::new(Body::empty()) }).await;
    let service = inference(&web, &provider).with_request_limits(RequestLimits {
        max_request_bytes: 64,
        memory_budget_bytes: 100,
        request_overhead_bytes: 10,
    });
    let body = |chunks: Vec<&'static [u8]>| {
        Body::from_stream(stream::iter(chunks.into_iter().map(Ok::<_, Infallible>)))
    };

    // A declared length is not reserved up front: a stalled upload holds what it sent.
    let mut declared = HeaderMap::new();
    declared.insert(header::CONTENT_LENGTH, "60".parse().unwrap());
    let mut stalled = Box::pin(read_request_body(
        Body::from_stream(
            body(vec![&[b'x'; 10]])
                .into_data_stream()
                .chain(stream::pending()),
        ),
        &declared,
        64,
        service.try_reserve_memory("request").ok().unwrap(),
    ));
    assert!(
        tokio::time::timeout(Duration::from_millis(50), &mut stalled)
            .await
            .is_err()
    );
    assert_eq!(service.available_memory(), 100 - 10 - 10);
    drop(stalled);
    assert_eq!(service.available_memory(), 100);

    // Growth reserves the buffer's allocation, not just its length.
    let (bytes, held) = read_request_body(
        body(vec![&[b'x'; 32], b"x"]),
        &HeaderMap::new(),
        64,
        service.try_reserve_memory("request").ok().unwrap(),
    )
    .await
    .ok()
    .unwrap();
    assert_eq!(bytes.len(), 33);
    assert_eq!(service.available_memory(), 100 - 10 - 64);
    drop(held);
    assert_eq!(service.available_memory(), 100);
}

#[test]
fn only_a_literal_true_stream_member_marks_a_streamed_call() {
    for (body, streaming) in [
        (r#"{"model":"m","stream":true}"#, true),
        (r#"{"model":"m","stream":false}"#, false),
        (r#"{"model":"m","stream":"true"}"#, false),
        (r#"{"model":"m","stream":null}"#, false),
        (r#"{"model":"m"}"#, false),
    ] {
        let request = request_body_and_model(Bytes::from_static(body.as_bytes()))
            .ok()
            .unwrap();
        assert_eq!(request.streaming, streaming, "{body}");
        assert_eq!(&body.as_bytes()[request.model_range], br#""m""#);
    }
    for body in [
        r#"{"model":"m","model":"n"}"#,
        r#"{"model":"m","stream":true,"stream":true}"#,
        r#"{"stream":true}"#,
    ] {
        assert!(
            request_body_and_model(Bytes::from_static(body.as_bytes())).is_err(),
            "{body}"
        );
    }
}

#[tokio::test]
async fn only_non_streamed_generations_wait_past_the_stall_for_provider_headers() {
    let web = FakeServer::start(|_| async {
        resolution_response_for(ApiFormat::AnthropicMessages, "provider-anthropic")
    })
    .await;
    let provider = FakeServer::start(|_| async {
        tokio::time::sleep(Duration::from_millis(300)).await;
        Response::new(Body::from("{}"))
    })
    .await;
    let service = InferenceService::for_test(
        web.control_plane(),
        ProviderTransport::for_test(
            format!("{}/v1", provider.url),
            ProviderLimits {
                execution: ExecutionLimits {
                    stall_timeout: Duration::from_millis(100),
                    ..ExecutionLimits::default()
                },
            },
        ),
    );
    let gateway = Gateway::start(Some(service)).await;
    let client = reqwest::Client::new();
    for (path, body, status) in [
        (
            "messages",
            r#"{"model":"Production key:test"}"#,
            StatusCode::OK,
        ),
        (
            "messages",
            r#"{"model":"Production key:test","stream":false}"#,
            StatusCode::OK,
        ),
        (
            "messages",
            r#"{"model":"Production key:test","stream":true}"#,
            StatusCode::GATEWAY_TIMEOUT,
        ),
        (
            "messages/count_tokens",
            r#"{"model":"Production key:test"}"#,
            StatusCode::GATEWAY_TIMEOUT,
        ),
    ] {
        let response = client
            .post(format!("{}/anthropic/v1/{path}", gateway.url))
            .header("x-api-key", "gateway-anthropic")
            .body(body)
            .send()
            .await;
        assert_eq!(response.unwrap().status(), status, "{path} {body}");
    }
}