Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

46.1 KB
Raw Blame
use super::*;
use crate::config::ExecutionLimits;
use crate::test_support::{
    FakeServer, ingestion_token, resolved_request_context, resolved_request_context_for,
};
use axum::{
    body::to_bytes,
    http::{HeaderName, StatusCode},
};
use futures_util::{StreamExt, stream};
use std::{convert::Infallible, future::pending, sync::Arc};
use tokio::sync::Notify;

fn provider(server: &FakeServer) -> ProviderTransport {
    ProviderTransport::for_test(format!("{}/v1", server.url), ProviderLimits::default())
}

/// Consumed as generation context; never forwarded upstream.
const CALLER_CONTEXT_HEADERS: [(&str, &str); 9] = [
    (
        "traceparent",
        "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01",
    ),
    ("tracestate", "caller=private"),
    ("baggage", "langfuse_user_id=private"),
    ("langfuse-trace-name", "private-trace"),
    ("langfuse-session-id", "private-session"),
    ("langfuse-user-id", "private-user"),
    ("langfuse-environment", "private-environment"),
    ("langfuse-tags", "private-tag"),
    ("langfuse-metadata", "private:value"),
];

#[test]
fn model_rewrite_changes_only_the_top_level_model_bytes() {
    let original = Bytes::from_static(
        br#"{ "z": {"second":2, "first":1}, "model" : "connection:model", "number": 1.0e2, "escaped":"\u00e9" }"#,
    );
    let expected = Bytes::from_static(
        br#"{ "z": {"second":2, "first":1}, "model" : "provider-model", "number": 1.0e2, "escaped":"\u00e9" }"#,
    );
    let model_range = model_range(&original).unwrap();
    assert_eq!(
        rewrite_model(original, model_range, "provider-model").unwrap(),
        expected
    );
}

#[test]
fn model_rewrite_fails_closed_for_an_invalid_model_range() {
    let missing = Bytes::from_static(br#"{"input":{"model":"nested-model"}}"#);
    assert!(matches!(
        rewrite_model(missing, 100..110, "provider-model"),
        Err(ProviderError::Configuration)
    ));
}

#[tokio::test]
async fn invalid_model_rewrite_never_reaches_the_provider() {
    let upstream = FakeServer::start(|_| async { Response::new(Body::empty()) }).await;
    let relay = provider(&upstream);
    let result = relay
        .forward_route(
            relay.admit(),
            resolved_request_context("provider-secret").await,
            &HeaderMap::new(),
            RequestBody {
                bytes: Bytes::from_static(br#"{"model":"requested"}"#),
                model_range: 100..110,
                streaming: true,
                reservation: None,
            },
            Route::OpenAiResponses,
            &mut RequestCorrelation::default(),
        )
        .await;

    assert!(matches!(result, Err(ProviderError::Configuration)));
    assert_eq!(upstream.calls(), 0);
}

#[test]
fn model_rewrite_accepts_a_shorter_provider_model() {
    let original = Bytes::from_static(br#"{"model":"connection:model","input":"large"}"#);
    let model_range = model_range(&original).unwrap();
    assert_eq!(
        rewrite_model(original, model_range, "m").unwrap(),
        r#"{"model":"m","input":"large"}"#
    );
}

#[tokio::test]
async fn missing_provider_model_fails_resolution() {
    let web = FakeServer::start(|_| async {
        let bytes = to_bytes(
            crate::test_support::resolution_response("provider-secret").into_body(),
            4096,
        )
        .await
        .unwrap();
        let mut body: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
        body["connection"]
            .as_object_mut()
            .unwrap()
            .remove("provider_model");
        Response::new(Body::from(body.to_string()))
    })
    .await;
    let result = web
        .control_plane()
        .resolve(
            "gateway-secret",
            ApiFormat::OpenAiResponses,
            "Production key:gpt-4o",
            0,
            "test-request-id",
        )
        .await;

    assert!(matches!(
        result,
        Err(crate::resolution::ResolutionError::InvalidResponse)
    ));
}

#[tokio::test]
async fn rewrites_the_model_and_isolates_request_and_response_headers() {
    const REQUEST: &[u8] =
        br#"{ "model": "opaque", "future_field": {"model":"opaque","values":[true,3]} }"#;
    const UPSTREAM_REQUEST: &[u8] =
        br#"{ "model": "provider-model", "future_field": {"model":"opaque","values":[true,3]} }"#;
    const RESPONSE: &[u8] = b"\x1f\x8b\x08\x00opaque-encoded-response";
    let upstream = FakeServer::start(|request| async move {
        assert_eq!(request.uri(), "/v1/responses");
        assert_eq!(request.method(), "POST");
        assert_eq!(
            request.headers()[header::AUTHORIZATION],
            "Bearer provider-secret"
        );
        assert_eq!(request.headers()[header::CONTENT_TYPE], "application/json");
        assert_eq!(request.headers()[header::ACCEPT_ENCODING], "identity");
        assert_eq!(
            request.headers()[header::CONTENT_LENGTH].to_str().unwrap(),
            UPSTREAM_REQUEST.len().to_string()
        );
        for name in [
            "cookie",
            "langfuse-gateway-authorization",
            "openai-organization",
            "openai-project",
            "x-api-key",
            "x-forwarded-host",
        ]
        .into_iter()
        .chain(CALLER_CONTEXT_HEADERS.map(|(name, _)| name))
        {
            assert!(!request.headers().contains_key(name), "forwarded {name}");
        }
        assert_eq!(
            to_bytes(request.into_body(), 1024).await.unwrap(),
            UPSTREAM_REQUEST
        );
        Response::builder()
            .status(201)
            .header("content-type", "application/json")
            .header("content-encoding", "gzip")
            .header("x-request-id", "request-123")
            .header("set-cookie", "private-cookie")
            .header("connection", "openai-version")
            .header("openai-version", "private-hop-value")
            .body(Body::from(RESPONSE))
            .unwrap()
    })
    .await;
    let relay = provider(&upstream);
    let mut headers = HeaderMap::new();
    for (name, value) in [
        ("authorization", "Bearer gateway-secret"),
        ("cookie", "session=secret"),
        ("langfuse-gateway-authorization", "service-signature"),
        ("openai-organization", "wrong-org"),
        ("openai-project", "wrong-project"),
        ("x-api-key", "wrong-key"),
        ("x-forwarded-host", "attacker.example"),
        ("content-type", "application/json"),
        ("accept-encoding", "gzip"),
        ("connection", "accept-encoding"),
    ]
    .into_iter()
    .chain(CALLER_CONTEXT_HEADERS)
    {
        headers.insert(
            axum::http::HeaderName::from_static(name),
            HeaderValue::from_static(value),
        );
    }
    let response = relay
        .forward(
            relay.admit(),
            resolved_request_context("provider-secret").await,
            &headers,
            Bytes::from_static(REQUEST),
        )
        .await
        .unwrap();
    assert_eq!(response.status(), StatusCode::CREATED);
    assert_eq!(response.headers()["content-encoding"], "gzip");
    assert_eq!(response.headers()["x-request-id"], "request-123");
    for name in [
        "set-cookie",
        "connection",
        "openai-version",
        "content-length",
        "transfer-encoding",
    ] {
        assert!(!response.headers().contains_key(name));
    }
    assert_eq!(
        to_bytes(response.into_body(), 1024).await.unwrap(),
        RESPONSE
    );
    assert_eq!(upstream.calls(), 1);
}

#[tokio::test]
async fn preserves_provider_failures_without_retries_or_redirects() {
    let target = FakeServer::start(|_| async { Response::new(Body::empty()) }).await;
    for status in [307, 429, 500, 503] {
        let location = target.url.clone();
        let upstream = FakeServer::start(move |_| {
            let location = location.clone();
            async move {
                Response::builder()
                    .status(status)
                    .header("location", location)
                    .header("retry-after", "7")
                    .body(Body::from("native-provider-body"))
                    .unwrap()
            }
        })
        .await;
        let relay = provider(&upstream);
        let response = relay
            .forward(
                relay.admit(),
                resolved_request_context("provider-secret").await,
                &HeaderMap::new(),
                Bytes::from_static(br#"{"model":"requested"}"#),
            )
            .await
            .unwrap();
        assert_eq!(response.status().as_u16(), status);
        assert_eq!(response.headers()["retry-after"], "7");
        assert!(!response.headers().contains_key("location"));
        assert_eq!(
            to_bytes(response.into_body(), 1024).await.unwrap(),
            "native-provider-body"
        );
        assert_eq!(upstream.calls(), 1);
    }
    assert_eq!(target.calls(), 0);
}

#[tokio::test]
async fn streams_sse_incrementally() {
    let release = Arc::new(Notify::new());
    let upstream_release = release.clone();
    let upstream = FakeServer::start(move |_| {
        let release = upstream_release.clone();
        async move {
            let chunks = stream::once(async { Ok::<_, Infallible>("data: first\n\n") }).chain(
                stream::once(async move {
                    release.notified().await;
                    Ok("data: [DONE]\n\n")
                }),
            );
            Response::builder()
                .header("content-type", "text/event-stream")
                .body(Body::from_stream(chunks))
                .unwrap()
        }
    })
    .await;
    let relay = provider(&upstream);
    let response = relay
        .forward(
            relay.admit(),
            resolved_request_context("provider-secret").await,
            &HeaderMap::new(),
            Bytes::from_static(br#"{"model":"requested"}"#),
        )
        .await
        .unwrap();
    assert_eq!(response.headers()["content-type"], "text/event-stream");
    let mut body = response.into_body().into_data_stream();
    assert_eq!(
        tokio::time::timeout(Duration::from_secs(1), body.next())
            .await
            .unwrap()
            .unwrap()
            .unwrap(),
        "data: first\n\n"
    );
    release.notify_one();
    let mut remaining = Vec::new();
    while let Some(chunk) = body.next().await {
        remaining.extend_from_slice(&chunk.unwrap());
    }
    assert_eq!(remaining, b"data: [DONE]\n\n");
}

#[tokio::test]
async fn simultaneous_requests_keep_provider_credentials_and_bodies_isolated() {
    let upstream = FakeServer::start(|request| async move {
        let authorization = request.headers()[header::AUTHORIZATION]
            .to_str()
            .unwrap()
            .to_owned();
        let body = to_bytes(request.into_body(), 1024).await.unwrap();
        Response::new(Body::from(format!(
            "{authorization}:{}",
            String::from_utf8(body.to_vec()).unwrap()
        )))
    })
    .await;
    let relay = provider(&upstream);
    let alice = resolved_request_context("alice-secret").await;
    let bob = resolved_request_context("bob-secret").await;
    let headers = HeaderMap::new();
    let (alice, bob) = tokio::join!(
        relay.forward(
            relay.admit(),
            alice,
            &headers,
            Bytes::from_static(br#"{"model":"requested","owner":"alice"}"#)
        ),
        relay.forward(
            relay.admit(),
            bob,
            &headers,
            Bytes::from_static(br#"{"model":"requested","owner":"bob"}"#)
        ),
    );
    assert_eq!(
        to_bytes(alice.unwrap().into_body(), 1024).await.unwrap(),
        r#"Bearer alice-secret:{"model":"provider-model","owner":"alice"}"#
    );
    assert_eq!(
        to_bytes(bob.unwrap().into_body(), 1024).await.unwrap(),
        r#"Bearer bob-secret:{"model":"provider-model","owner":"bob"}"#
    );
    assert_eq!(upstream.calls(), 2);
}

struct Dropped(Arc<Notify>);
impl Drop for Dropped {
    fn drop(&mut self) {
        self.0.notify_one();
    }
}

async fn stalled_provider(dropped: Arc<Notify>) -> FakeServer {
    FakeServer::start(move |_| {
        let guard = Dropped(dropped.clone());
        async move {
            let tail = stream::once(async move {
                let _guard = guard;
                pending::<Result<&str, Infallible>>().await
            });
            Response::new(Body::from_stream(
                stream::once(async { Ok::<_, Infallible>("first") }).chain(tail),
            ))
        }
    })
    .await
}

#[tokio::test]
async fn dropping_downstream_cancels_upstream() {
    let dropped = Arc::new(Notify::new());
    let upstream = stalled_provider(dropped.clone()).await;
    let relay = provider(&upstream);
    let response = relay
        .forward(
            relay.admit(),
            resolved_request_context("provider-secret").await,
            &HeaderMap::new(),
            Bytes::from_static(br#"{"model":"requested"}"#),
        )
        .await
        .unwrap();
    let mut body = response.into_body().into_data_stream();
    assert_eq!(body.next().await.unwrap().unwrap(), "first");
    drop(body);
    tokio::time::timeout(Duration::from_secs(1), dropped.notified())
        .await
        .expect("upstream body should be dropped on cancellation");
}

#[tokio::test]
async fn deadlines_bound_headers_and_stalled_bodies_without_exposing_transport_details() {
    let upstream = FakeServer::start(|_| async {
        pending::<()>().await;
        Response::new(Body::empty())
    })
    .await;
    let relay = ProviderTransport::for_test(
        upstream.url.clone(),
        ProviderLimits {
            execution: ExecutionLimits {
                stall_timeout: Duration::from_millis(30),
                ..ExecutionLimits::default()
            },
        },
    );
    let response = relay
        .forward(
            relay.admit(),
            resolved_request_context("provider-secret").await,
            &HeaderMap::new(),
            Bytes::from_static(br#"{"model":"requested"}"#),
        )
        .await;
    assert!(matches!(response, Err(ProviderError::Timeout)));

    let dropped = Arc::new(Notify::new());
    let upstream = stalled_provider(dropped.clone()).await;
    let relay = ProviderTransport::for_test(
        upstream.url.clone(),
        ProviderLimits {
            execution: ExecutionLimits {
                max_execution_time: Duration::from_millis(100),
                ..ExecutionLimits::default()
            },
        },
    );
    let context = resolved_request_context("provider-secret").await;
    let response = relay
        .forward(
            relay.admit(),
            context,
            &HeaderMap::new(),
            Bytes::from_static(br#"{"model":"requested"}"#),
        )
        .await
        .unwrap();
    // The deadline must cancel upstream even when downstream does not poll its body.
    tokio::time::timeout(Duration::from_secs(1), dropped.notified())
        .await
        .expect("deadline must run without downstream polling");
    let error = to_bytes(response.into_body(), 1024).await.unwrap_err();
    assert!(!error.to_string().contains(&upstream.url));
    assert!(!error.to_string().contains("provider-secret"));
}

#[tokio::test]
async fn completed_json_and_sse_upload_once_without_waiting_for_ingestion() {
    use crate::{
        resolution::ControlPlaneConfig, telemetry::Telemetry,
        test_support::resolved_request_context_with_mode,
    };

    const JSON: &str = r#"{"id":"resp-1","model":"actual","status":"completed","output":[],"usage":{"input_tokens":2,"output_tokens":1,"total_tokens":3}}"#;
    const SSE: &str = "data: {\"type\":\"response.output_text.delta\",\"delta\":\"hello\"}\n\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp-1\",\"model\":\"actual\",\"status\":\"completed\",\"output\":[],\"usage\":{\"input_tokens\":2,\"output_tokens\":1,\"total_tokens\":3}}}\n\n";
    for (content_type, native, sink_status, close_before_eof) in [
        ("application/json", JSON, 200, false),
        ("text/event-stream", SSE, 503, false),
        ("text/event-stream", SSE, 200, true),
    ] {
        let (sent, mut received) = tokio::sync::mpsc::channel(2);
        let release = Arc::new(Notify::new());
        let sink_release = release.clone();
        let sink = FakeServer::start(move |request| {
            let sent = sent.clone();
            let release = sink_release.clone();
            async move {
                let bytes = to_bytes(request.into_body(), 65536).await.unwrap();
                sent.send(crate::test_support::upload_json(&bytes))
                    .await
                    .unwrap();
                release.notified().await;
                Response::builder()
                    .status(sink_status)
                    .body(Body::from("{}"))
                    .unwrap()
            }
        })
        .await;
        let telemetry =
            Telemetry::for_test(&ControlPlaneConfig::new(&sink.url, "service-key").unwrap());
        let upstream = FakeServer::start(move |_| async move {
            let body = if close_before_eof {
                Body::from_stream(
                    stream::iter([Ok::<_, Infallible>(Bytes::from_static(native.as_bytes()))])
                        .chain(stream::pending()),
                )
            } else {
                Body::from(native)
            };
            Response::builder()
                .header("content-type", content_type)
                .header("x-request-id", "req-1")
                .body(body)
                .unwrap()
        })
        .await;
        let relay = provider(&upstream).with_telemetry(telemetry.clone());
        let headers = HeaderMap::from_iter([
            (
                axum::http::HeaderName::from_static("traceparent"),
                HeaderValue::from_static("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00"),
            ),
            (
                axum::http::HeaderName::from_static("tracestate"),
                HeaderValue::from_static("caller=state"),
            ),
            (
                axum::http::HeaderName::from_static("langfuse-user-id"),
                HeaderValue::from_static("caller-user"),
            ),
        ]);
        let response = relay
            .forward(
                relay.admit(),
                resolved_request_context_with_mode("provider-secret", "full").await,
                &headers,
                Bytes::from_static(br#"{"model":"requested","input":"hello"}"#),
            )
            .await
            .unwrap();
        // Neither the last byte nor the provider permit waits for the sink response.
        let mut body = response.into_body().into_data_stream();
        let mut forwarded = Vec::new();
        while let Some(chunk) = tokio::time::timeout(Duration::from_secs(1), body.next())
            .await
            .unwrap()
        {
            forwarded.extend_from_slice(&chunk.unwrap());
            if close_before_eof && forwarded.len() == native.len() {
                break;
            }
        }
        drop(body);
        assert_eq!(forwarded, native.as_bytes());
        let payload = tokio::time::timeout(Duration::from_secs(1), received.recv())
            .await
            .unwrap()
            .unwrap();
        assert_completed_upload(&payload, content_type == "text/event-stream");
        release.notify_one();
        telemetry
            .shutdown(Instant::now() + Duration::from_secs(1))
            .await;
        assert!(received.try_recv().is_err());
        assert_eq!(sink.calls(), 1);
        assert_eq!(upstream.calls(), 1);
    }
}

#[tokio::test]
async fn ingestion_mode_none_relays_without_uploading() {
    use crate::{
        resolution::ControlPlaneConfig, telemetry::Telemetry,
        test_support::resolved_request_context_with_mode,
    };

    const JSON: &str = r#"{"id":"resp-1","model":"actual","status":"completed","output":[],"usage":{"input_tokens":2,"output_tokens":1,"total_tokens":3}}"#;
    let sink = FakeServer::start(|_| async { Response::new(Body::from("{}")) }).await;
    let telemetry =
        Telemetry::for_test(&ControlPlaneConfig::new(&sink.url, "service-key").unwrap());
    let upstream = FakeServer::start(|_| async {
        Response::builder()
            .header("content-type", "application/json")
            .body(Body::from(JSON))
            .unwrap()
    })
    .await;
    let relay = provider(&upstream).with_telemetry(telemetry.clone());
    let response = relay
        .forward(
            relay.admit(),
            resolved_request_context_with_mode("provider-secret", "none").await,
            &HeaderMap::new(),
            Bytes::from_static(br#"{"model":"requested","input":"prompt-canary"}"#),
        )
        .await
        .unwrap();
    assert_eq!(to_bytes(response.into_body(), 4096).await.unwrap(), JSON);
    telemetry
        .shutdown(Instant::now() + Duration::from_secs(1))
        .await;
    assert_eq!(upstream.calls(), 1);
    assert_eq!(sink.calls(), 0);
}

#[tokio::test]
async fn cancelled_and_timed_out_executions_upload_after_provider_context_is_released() {
    use crate::{resolution::ControlPlaneConfig, telemetry::Telemetry};
    use serde_json::Value;

    for cancelled in [true, false] {
        let (sent, mut received) = tokio::sync::mpsc::channel(2);
        let sink = FakeServer::start(move |request| {
            let sent = sent.clone();
            async move {
                assert_eq!(
                    request.headers()[header::AUTHORIZATION],
                    format!("Bearer {}", ingestion_token("org-1", "project-1")).as_str()
                );
                let bytes = to_bytes(request.into_body(), 65536).await.unwrap();
                sent.send(crate::test_support::upload_json(&bytes))
                    .await
                    .unwrap();
                Response::new(Body::from("{}"))
            }
        })
        .await;
        let telemetry =
            Telemetry::for_test(&ControlPlaneConfig::new(&sink.url, "service-key").unwrap());
        let dropped = Arc::new(Notify::new());
        let upstream = stalled_provider(dropped).await;
        let relay = ProviderTransport::for_test(
            upstream.url.clone(),
            ProviderLimits {
                execution: ExecutionLimits {
                    max_execution_time: Duration::from_millis(150),
                    ..ExecutionLimits::default()
                },
            },
        )
        .with_telemetry(telemetry.clone());
        let context = resolved_request_context("provider-secret").await;
        let response = relay
            .forward(
                relay.admit(),
                context,
                &HeaderMap::new(),
                Bytes::from_static(br#"{"model":"requested","input":"prompt-canary"}"#),
            )
            .await
            .unwrap();
        let mut body = response.into_body().into_data_stream();
        assert_eq!(body.next().await.unwrap().unwrap(), "first");
        assert_eq!(sink.calls(), 0);
        if cancelled {
            drop(body);
        } else {
            assert!(body.next().await.unwrap().is_err());
            drop(body);
        }
        let payload = tokio::time::timeout(Duration::from_secs(1), received.recv())
            .await
            .unwrap()
            .unwrap();
        assert!(!payload.to_string().contains("prompt-canary"));
        let attrs = payload["resourceSpans"][0]["scopeSpans"][0]["spans"][0]["attributes"]
            .as_array()
            .unwrap();
        let metadata: Value = serde_json::from_str(
            attrs
                .iter()
                .find(|a| a["key"] == "langfuse.observation.metadata")
                .unwrap()["value"]["stringValue"]
                .as_str()
                .unwrap(),
        )
        .unwrap();
        assert_eq!(
            attrs
                .iter()
                .find(|a| a["key"] == "langfuse.observation.level")
                .unwrap()["value"]["stringValue"],
            if cancelled { "WARNING" } else { "ERROR" }
        );
        assert_eq!(
            attrs
                .iter()
                .find(|a| a["key"] == "langfuse.observation.model.name")
                .unwrap()["value"]["stringValue"],
            "provider-model"
        );
        assert_eq!(metadata["langfuse.gateway.response.status_code"], 200);
        assert!(metadata.get("relay_outcome").is_none());
        telemetry
            .shutdown(Instant::now() + Duration::from_secs(1))
            .await;
        assert!(received.try_recv().is_err());
        assert_eq!(sink.calls(), 1);
    }
}

#[tokio::test]
async fn compact_posts_compact_path_and_captures_ingestion() {
    use crate::{resolution::ControlPlaneConfig, telemetry::Telemetry};

    const COMPACT: &[u8] =
        br#"{"model":"gpt-4.1","input":[{"encrypted_content":"opaque-ciphertext"}]}"#;
    const UPSTREAM_COMPACT: &[u8] =
        br#"{"model":"provider-model","input":[{"encrypted_content":"opaque-ciphertext"}]}"#;
    const COMPACT_RESPONSE: &str =
        r#"{"id":"comp_1","output":[{"encrypted_content":"opaque-ciphertext"}]}"#;
    let (sent, mut received) = tokio::sync::mpsc::channel(2);
    let sink = FakeServer::start(move |request| {
        let sent = sent.clone();
        async move {
            let bytes = to_bytes(request.into_body(), 65536).await.unwrap();
            sent.send(crate::test_support::upload_json(&bytes))
                .await
                .unwrap();
            Response::new(Body::from("{}"))
        }
    })
    .await;
    let telemetry =
        Telemetry::for_test(&ControlPlaneConfig::new(&sink.url, "service-key").unwrap());
    let upstream = FakeServer::start(|request| async move {
        assert_eq!(request.uri(), "/v1/responses/compact");
        assert_eq!(request.method(), "POST");
        let body = to_bytes(request.into_body(), 4096).await.unwrap();
        assert_eq!(body, UPSTREAM_COMPACT);
        Response::builder()
            .header("content-type", "application/json")
            .body(Body::from(COMPACT_RESPONSE))
            .unwrap()
    })
    .await;
    let relay = provider(&upstream).with_telemetry(telemetry.clone());
    let compact = relay
        .forward_route(
            relay.admit(),
            resolved_request_context("provider-secret").await,
            &HeaderMap::new(),
            RequestBody {
                bytes: Bytes::from_static(COMPACT),
                model_range: model_range(COMPACT).unwrap(),
                streaming: true,
                reservation: None,
            },
            Route::OpenAiResponsesCompact,
            &mut RequestCorrelation::default(),
        )
        .await
        .unwrap();
    assert_eq!(
        to_bytes(compact.into_body(), 4096).await.unwrap(),
        COMPACT_RESPONSE
    );
    let payload = tokio::time::timeout(Duration::from_secs(1), received.recv())
        .await
        .unwrap()
        .unwrap();
    assert_eq!(
        payload["resourceSpans"][0]["scopeSpans"][0]["spans"][0]["name"],
        "openai.responses"
    );
    assert!(received.try_recv().is_err());
    telemetry
        .shutdown(Instant::now() + Duration::from_secs(1))
        .await;
    assert_eq!(sink.calls(), 1);
    assert_eq!(upstream.calls(), 1);
}

fn assert_completed_upload(payload: &serde_json::Value, streaming: bool) {
    use serde_json::{Value, json};

    let generation = &payload["resourceSpans"][0]["scopeSpans"][0]["spans"][0];
    assert_eq!(generation["traceId"], "4bf92f3577b34da6a3ce929d0e0e4736");
    assert_eq!(generation["parentSpanId"], "00f067aa0ba902b7");
    assert_eq!(generation["traceState"], "caller=state");
    assert!(
        generation["attributes"]
            .as_array()
            .unwrap()
            .iter()
            .any(|attribute| {
                attribute["key"] == "user.id" && attribute["value"]["stringValue"] == "caller-user"
            })
    );
    let span = &payload["resourceSpans"][0]["scopeSpans"][0]["spans"][0];
    let attrs = span["attributes"].as_array().unwrap();
    assert!(attrs.iter().any(
        |a| a["key"] == "langfuse.observation.level" && a["value"]["stringValue"] == "DEFAULT"
    ));
    assert!(
        !attrs
            .iter()
            .any(|a| a["key"] == "langfuse.observation.status_message")
    );
    assert!(
        attrs
            .iter()
            .any(|a| a["key"] == "langfuse.observation.model.name"
                && a["value"]["stringValue"] == "actual")
    );
    assert_eq!(
        attrs
            .iter()
            .any(|a| a["key"] == "langfuse.observation.completion_start_time"),
        streaming,
    );
    let metadata: Value = serde_json::from_str(
        attrs
            .iter()
            .find(|a| a["key"] == "langfuse.observation.metadata")
            .unwrap()["value"]["stringValue"]
            .as_str()
            .unwrap(),
    )
    .unwrap();
    assert!(metadata.get("relay_outcome").is_none());
    assert!(metadata.get("native_usage").is_none());
    assert_eq!(metadata["langfuse.gateway.upstream.response.id"], "resp-1");
    assert_eq!(metadata["langfuse.gateway.upstream.request.id"], "req-1");
    assert!(metadata["langfuse.gateway.request.id"].is_string());
    assert!(metadata.get("langfuse.gateway.client.request.id").is_none());
    let usage: Value = serde_json::from_str(
        attrs
            .iter()
            .find(|a| a["key"] == "langfuse.observation.usage_details")
            .unwrap()["value"]["stringValue"]
            .as_str()
            .unwrap(),
    )
    .unwrap();
    assert_eq!(
        usage,
        json!({"input_tokens":2,"output_tokens":1,"total_tokens":3})
    );
    for secret in [
        "provider-secret",
        "private-ingestion-token",
        "gateway-secret",
        "service-key",
    ] {
        assert!(!payload.to_string().contains(secret));
    }
}

fn transport(server: &FakeServer) -> ProviderTransport {
    ProviderTransport::for_test(format!("{}/v1", server.url), ProviderLimits::default())
}

/// Headers the gateway consumes for attribution or that belong to the client
/// hop; none may reach the provider.
const CONSUMED_HEADERS: &[(&str, &str)] = &[
    ("x-claude-code-session-id", "session-1"),
    ("x-claude-code-agent-id", "agent-1"),
    ("cookie", "session=secret"),
    ("langfuse-gateway-authorization", "service-signature"),
    (
        "traceparent",
        "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01",
    ),
    ("baggage", "langfuse_user_id=private"),
    ("langfuse-session-id", "private-session"),
    ("x-forwarded-host", "attacker.example"),
    ("x-stainless-retry-count", "0"),
];

fn headers(pairs: &[(&'static str, &'static str)]) -> HeaderMap {
    let mut headers = HeaderMap::new();
    for (name, value) in pairs {
        headers.append(
            HeaderName::from_static(name),
            HeaderValue::from_static(value),
        );
    }
    headers
}

#[tokio::test]
async fn sends_the_x_api_key_credential_and_forwards_the_open_anthropic_header_family() {
    const REQUEST: &[u8] = br#"{"model":"claude-sonnet-4-5","system":[{"type":"text","text":"attribution","cache_control":{"type":"ephemeral"}}],"future_field":true}"#;
    const UPSTREAM_REQUEST: &[u8] = br#"{"model":"provider-model","system":[{"type":"text","text":"attribution","cache_control":{"type":"ephemeral"}}],"future_field":true}"#;
    let upstream = FakeServer::start(|request| async move {
        assert_eq!(request.uri(), "/v1/messages");
        assert_eq!(request.method(), "POST");
        assert_eq!(request.headers()["x-api-key"], "provider-secret");
        assert!(!request.headers().contains_key(header::AUTHORIZATION));
        assert_eq!(request.headers()["anthropic-version"], "2023-06-01");
        assert_eq!(
            request.headers()["anthropic-beta"],
            "interleaved-thinking-2025-05-14,context-1m-2025-08-07"
        );
        assert_eq!(request.headers()["anthropic-future-capability"], "on");
        assert_eq!(request.headers()[header::ACCEPT_ENCODING], "identity");
        for (name, _) in CONSUMED_HEADERS {
            assert!(!request.headers().contains_key(*name), "forwarded {name}");
        }
        assert_eq!(
            to_bytes(request.into_body(), 1024).await.unwrap(),
            UPSTREAM_REQUEST
        );
        Response::builder()
            .status(StatusCode::OK)
            .header("content-type", "application/json")
            .header("request-id", "req_123")
            .header("x-should-retry", "false")
            .header("anthropic-ratelimit-unified-status", "allowed")
            .header("anthropic-ratelimit-input-tokens-remaining", "1000")
            .header("anthropic-organization-id", "org_456")
            .header("retry-after", "7")
            .header("set-cookie", "private-cookie")
            .header("cf-ray", "edge-private")
            .body(Body::from(r#"{"type":"message","content":[]}"#))
            .unwrap()
    })
    .await;
    let relay = transport(&upstream);
    let context =
        resolved_request_context_for(ApiFormat::AnthropicMessages, "provider-secret", "usage")
            .await;
    let response = relay
        .forward_route(
            relay.admit(),
            context,
            &headers(
                &[
                    &[
                        ("x-api-key", "gateway-secret"),
                        ("authorization", "Bearer gateway-secret"),
                        ("anthropic-version", "2023-06-01"),
                        (
                            "anthropic-beta",
                            "interleaved-thinking-2025-05-14,context-1m-2025-08-07",
                        ),
                        ("anthropic-future-capability", "on"),
                        ("content-type", "application/json"),
                        ("accept-encoding", "gzip"),
                    ],
                    CONSUMED_HEADERS,
                ]
                .concat(),
            ),
            RequestBody {
                bytes: Bytes::from_static(REQUEST),
                model_range: model_range(REQUEST).unwrap(),
                streaming: true,
                reservation: None,
            },
            Route::AnthropicMessages,
            &mut RequestCorrelation::default(),
        )
        .await
        .unwrap();
    assert_eq!(response.status(), StatusCode::OK);
    let response_headers = response.headers().clone();
    for (name, value) in [
        ("request-id", "req_123"),
        ("x-should-retry", "false"),
        ("anthropic-ratelimit-unified-status", "allowed"),
        ("anthropic-ratelimit-input-tokens-remaining", "1000"),
        ("anthropic-organization-id", "org_456"),
        ("retry-after", "7"),
        ("content-type", "application/json"),
    ] {
        assert_eq!(response_headers[name], value, "missing {name}");
    }
    for name in ["set-cookie", "cf-ray"] {
        assert!(!response_headers.contains_key(name), "retained {name}");
    }
    assert_eq!(
        to_bytes(response.into_body(), 1024).await.unwrap(),
        r#"{"type":"message","content":[]}"#
    );
    assert_eq!(upstream.calls(), 1);
}

#[tokio::test]
async fn count_tokens_relays_the_body() {
    let upstream = FakeServer::start(|request| async move {
        assert_eq!(request.uri(), "/v1/messages/count_tokens");
        assert_eq!(request.method(), "POST");
        assert_eq!(
            to_bytes(request.into_body(), 1024).await.unwrap(),
            r#"{"model":"provider-model","messages":[]}"#
        );
        Response::builder()
            .header("content-type", "application/json")
            .body(Body::from(r#"{"input_tokens":42}"#))
            .unwrap()
    })
    .await;
    let relay = transport(&upstream);
    let context =
        resolved_request_context_for(ApiFormat::AnthropicMessages, "provider-secret", "usage")
            .await;
    let response = relay
        .forward_route(
            relay.admit(),
            context,
            &headers(&[("content-type", "application/json")]),
            RequestBody {
                bytes: Bytes::from_static(br#"{"model":"claude-sonnet-4-5","messages":[]}"#),
                model_range: model_range(br#"{"model":"claude-sonnet-4-5","messages":[]}"#)
                    .unwrap(),
                streaming: true,
                reservation: None,
            },
            Route::AnthropicCountTokens,
            &mut RequestCorrelation::default(),
        )
        .await
        .unwrap();
    assert_eq!(response.status(), StatusCode::OK);
    assert_eq!(
        to_bytes(response.into_body(), 1024).await.unwrap(),
        r#"{"input_tokens":42}"#
    );
    assert_eq!(upstream.calls(), 1);
}

const CHAT_CONTENT_CHUNK: &str = "data: {\"id\":\"chatcmpl-1\",\"object\":\"chat.completion.chunk\",\"model\":\"gpt-4.1-mini-2025-04-14\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"Hello\"},\"finish_reason\":\"stop\"}],\"usage\":null}\n\n";
const CHAT_USAGE_CHUNK: &str = "data: {\"id\":\"chatcmpl-1\",\"object\":\"chat.completion.chunk\",\"model\":\"gpt-4.1-mini-2025-04-14\",\"choices\":[],\"usage\":{\"prompt_tokens\":9,\"completion_tokens\":1,\"total_tokens\":10,\"prompt_tokens_details\":{\"cached_tokens\":4},\"completion_tokens_details\":{\"reasoning_tokens\":0}}}\n\n";
const CHAT_DONE: &str = "data: [DONE]\n\n";

/// Generation context headers in the shape SDK users send them.
const CHAT_CONTEXT_HEADERS: [(&str, &str); 8] = [
    (
        "traceparent",
        "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01",
    ),
    ("langfuse-user-id", "chat-user"),
    ("langfuse-session-id", "chat-session"),
    ("langfuse-trace-name", "chat-trace"),
    ("langfuse-environment", "staging"),
    ("langfuse-tags", "tag-a,tag-b"),
    ("langfuse-metadata", "team:search"),
    ("content-type", "application/json"),
];

#[tokio::test]
async fn chat_completions_streams_request_usage_and_relay_the_usage_chunk() {
    use crate::{resolution::ControlPlaneConfig, telemetry::Telemetry, test_support::upload_json};

    const UNREQUESTED: &str = r#"{"model":"gpt-4.1-mini","messages":[{"role":"user","content":"Say hello"}],"stream":true}"#;
    const REQUESTED: &str = r#"{"model":"gpt-4.1-mini","messages":[{"role":"user","content":"Say hello"}],"stream":true,"stream_options":{"include_usage":true}}"#;
    // Callers receive the usage chunk whether or not they asked for it.
    let relayed = [CHAT_CONTENT_CHUNK, CHAT_USAGE_CHUNK, CHAT_DONE].concat();
    for request in [UNREQUESTED, REQUESTED] {
        let (sent, mut received) = tokio::sync::mpsc::channel(2);
        let sink = FakeServer::start(move |request| {
            let sent = sent.clone();
            async move {
                let bytes = to_bytes(request.into_body(), 65536).await.unwrap();
                sent.send(upload_json(&bytes)).await.unwrap();
                Response::new(Body::from("{}"))
            }
        })
        .await;
        let telemetry =
            Telemetry::for_test(&ControlPlaneConfig::new(&sink.url, "service-key").unwrap());
        let upstream = FakeServer::start(move |request| async move {
            assert_eq!(request.uri(), "/v1/chat/completions");
            assert_eq!(
                request.headers()[header::AUTHORIZATION],
                "Bearer provider-secret"
            );
            for (name, _) in CHAT_CONTEXT_HEADERS {
                if name != "content-type" {
                    assert!(!request.headers().contains_key(name), "forwarded {name}");
                }
            }
            assert_eq!(
                to_bytes(request.into_body(), 4096).await.unwrap(),
                r#"{"model":"provider-model","messages":[{"role":"user","content":"Say hello"}],"stream":true,"stream_options":{"include_usage":true}}"#
            );
            // Arrive split mid-event, as a provider stream may.
            let native = [CHAT_CONTENT_CHUNK, CHAT_USAGE_CHUNK, CHAT_DONE].concat();
            let (head, tail) = native.split_at(CHAT_CONTENT_CHUNK.len() + 20);
            Response::builder()
                .header("content-type", "text/event-stream; charset=utf-8")
                .header("x-request-id", "req-chat")
                .body(Body::from_stream(stream::iter([
                    Ok::<_, Infallible>(head.to_owned()),
                    Ok(tail.to_owned()),
                ])))
                .unwrap()
        })
        .await;
        let relay = provider(&upstream).with_telemetry(telemetry.clone());
        let response = relay
            .forward_route(
                relay.admit(),
                resolved_request_context_for(
                    ApiFormat::OpenAiChatCompletions,
                    "provider-secret",
                    "full",
                )
                .await,
                &headers(
                    &[
                        CHAT_CONTEXT_HEADERS.as_slice(),
                        &[("authorization", "Bearer gateway-secret")],
                    ]
                    .concat(),
                ),
                RequestBody {
                    bytes: Bytes::from_static(request.as_bytes()),
                    model_range: model_range(request.as_bytes()).unwrap(),
                    streaming: true,
                    reservation: None,
                },
                Route::OpenAiChatCompletions,
                &mut RequestCorrelation::default(),
            )
            .await
            .unwrap();
        assert_eq!(response.headers()["x-request-id"], "req-chat");
        assert_eq!(
            to_bytes(response.into_body(), 65536).await.unwrap(),
            relayed
        );
        let payload = tokio::time::timeout(Duration::from_secs(5), received.recv())
            .await
            .unwrap()
            .unwrap();
        assert_chat_upload(&payload, request);
        telemetry
            .shutdown(Instant::now() + Duration::from_secs(1))
            .await;
    }
}

#[tokio::test]
async fn chat_completions_rewrite_non_streamed_models() {
    const JSON_REQUEST: &str = r#"{"model":"gpt-4.1-mini", "messages":[]}"#;
    let upstream = FakeServer::start(|request| async move {
        assert_eq!(
            to_bytes(request.into_body(), 4096).await.unwrap(),
            r#"{"model":"provider-model", "messages":[]}"#
        );
        Response::builder()
            .header("content-type", "application/json")
            .body(Body::from(r#"{"id":"chatcmpl-1","choices":[]}"#))
            .unwrap()
    })
    .await;
    let relay = transport(&upstream);
    let response = relay
        .forward_route(
            relay.admit(),
            resolved_request_context_for(
                ApiFormat::OpenAiChatCompletions,
                "provider-secret",
                "usage",
            )
            .await,
            &headers(&[("content-type", "application/json")]),
            RequestBody {
                bytes: Bytes::from_static(JSON_REQUEST.as_bytes()),
                model_range: model_range(JSON_REQUEST.as_bytes()).unwrap(),
                streaming: true,
                reservation: None,
            },
            Route::OpenAiChatCompletions,
            &mut RequestCorrelation::default(),
        )
        .await
        .unwrap();
    assert_eq!(response.status(), StatusCode::OK);
    assert_eq!(
        to_bytes(response.into_body(), 4096).await.unwrap(),
        r#"{"id":"chatcmpl-1","choices":[]}"#
    );
    assert_eq!(upstream.calls(), 1);
}

fn assert_chat_upload(payload: &serde_json::Value, request: &str) {
    use serde_json::{Value, json};

    let span = &payload["resourceSpans"][0]["scopeSpans"][0]["spans"][0];
    assert_eq!(span["name"], "openai.chat-completions");
    assert_eq!(span["traceId"], "4bf92f3577b34da6a3ce929d0e0e4736");
    assert_eq!(span["parentSpanId"], "00f067aa0ba902b7");
    let attribute = |key: &str| {
        let attribute = span["attributes"]
            .as_array()
            .unwrap()
            .iter()
            .find(|attribute| attribute["key"] == key)
            .unwrap_or_else(|| panic!("missing {key}"));
        attribute["value"]["stringValue"].clone()
    };
    let parsed =
        |key: &str| -> Value { serde_json::from_str(attribute(key).as_str().unwrap()).unwrap() };
    for (key, value) in [
        ("user.id", "chat-user"),
        ("session.id", "chat-session"),
        ("langfuse.trace.name", "chat-trace"),
        ("langfuse.environment", "staging"),
        ("langfuse.observation.level", "DEFAULT"),
        ("langfuse.observation.model.name", "gpt-4.1-mini-2025-04-14"),
    ] {
        assert_eq!(attribute(key), value, "{key}");
    }
    assert_eq!(parsed("langfuse.trace.tags"), json!(["tag-a", "tag-b"]));
    assert_eq!(
        parsed("langfuse.observation.usage_details"),
        json!({"prompt_tokens":9,"completion_tokens":1,"total_tokens":10,"prompt_tokens_details":{"cached_tokens":4},"completion_tokens_details":{"reasoning_tokens":0}})
    );
    // The recorded input is the caller's request, not the upstream rewrite.
    assert_eq!(
        parsed("langfuse.observation.input"),
        json!({"messages":[{"role":"user","content":"Say hello"}]})
    );
    assert_eq!(
        parsed("langfuse.observation.model.parameters").get("stream_options"),
        serde_json::from_str::<Value>(request)
            .unwrap()
            .get("stream_options")
    );
    assert_eq!(
        parsed("langfuse.observation.output"),
        json!({"choices":[{"index":0,"finish_reason":"stop","message":{"role":"assistant","content":"Hello"}}]})
    );
    attribute("langfuse.observation.completion_start_time");
    let metadata = parsed("langfuse.observation.metadata");
    assert_eq!(metadata["team"], "search");
    assert_eq!(
        metadata["langfuse.gateway.request.api_format"],
        "openai.chat-completions"
    );
    assert_eq!(
        metadata["langfuse.gateway.upstream.response.id"],
        "chatcmpl-1"
    );
    assert_eq!(metadata["langfuse.gateway.upstream.request.id"], "req-chat");
    for (key, value) in [
        ("langfuse.gateway.model.requested", json!("gpt-4.1-mini")),
        ("langfuse.gateway.model.upstream", json!("provider-model")),
        (
            "langfuse.gateway.model.response",
            json!("gpt-4.1-mini-2025-04-14"),
        ),
        ("langfuse.gateway.connection.name", json!("Production key")),
        ("langfuse.gateway.connection.provider", json!("openai")),
        ("langfuse.gateway.upstream.status_code", json!(200)),
        ("langfuse.gateway.request.body_bytes", json!(request.len())),
    ] {
        assert_eq!(metadata[key], value, "{key}");
    }
    assert!(metadata["langfuse.gateway.response.body_bytes"].as_u64() > Some(0));
    for secret in [
        "provider-secret",
        "gateway-secret",
        "private-ingestion-token",
    ] {
        assert!(!payload.to_string().contains(secret));
    }
}