Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(ai-gateway): accept Anthropic connections in the resolution contract (#17754)

Hassieb Pakzad6e8ef396ede9
diff --git a/ai-gateway/README.md b/ai-gateway/README.md
index 8006c4a59e..9f0c340731 100644
--- a/ai-gateway/README.md
+++ b/ai-gateway/README.md
@@ -568,17 +568,32 @@ async fn example(web_base_url: &str, service_key: &str, gateway_key: &str)
Each call sends `POST <base path>/api/internal/ai-gateway/v1/resolve` with
`Authorization: Bearer <gateway key>`, the Web v1 HMAC header, and exactly
-`{"apiFormat":"openai.responses"}`. The client neither receives nor parses an
-inference request body. Reuse the resolver across requests: its connection pool is
-shared, while credentials and execution contexts stay request-local.
+`{"apiFormat":"<format>"}`, where the format is the one the public route serves
+(`openai.responses` for every `/openai/v1` route). The client neither receives nor
+parses an inference request body. Reuse the resolver across requests: its connection
+pool is shared, while credentials and execution contexts stay request-local.
The whole HTTP exchange has a five-second deadline and a 256 KiB response limit,
including chunked responses. Redirects, automatic retries, ambient proxy settings
and transparent decompression are disabled. Errors expose fixed categories;
-upstream error bodies and transport details are discarded. Successful responses
-must match the strict v1 schema, the official OpenAI Responses connection, and an
-unexpired project ingestion grant. Ingestion tokens remain opaque. Resolution
-caching, provider execution and telemetry are separate slices.
+upstream error bodies and transport details are discarded. A successful response
+must match the strict v1 schema and carry an unexpired project ingestion grant.
+Its connection must also be one of the known official pairings: the `provider`
+serves the requested `api_format`, `base_url` is that provider's official origin,
+and `auth` uses that provider's credential scheme. Any other combination is an
+invalid response, even if Web selected it.
+
+| Provider | API format | Official origin | Credential |
+| --- | --- | --- | --- |
+| `openai` | `openai.responses` | `https://api.openai.com/v1` | `{"type":"Bearer","token":…}` → `Authorization: Bearer` |
+| `anthropic` | `anthropic.messages` | `https://api.anthropic.com/v1` | `{"type":"x-api-key","header":"x-api-key","value":…}` → `x-api-key` |
+
+`ProviderConnection::credential()` exposes the secret in its header position
+(`ProviderCredential::Bearer` or `ProviderCredential::XApiKey`) and has no `Debug`
+output. The Anthropic pairing is accepted by the contract ahead of its routes: no
+`/anthropic/v1` endpoint, transport or capture exists yet, so a resolver caller
+requesting `anthropic.messages` today is only the test suite. Ingestion tokens remain
+opaque. Resolution caching, provider execution and telemetry are separate slices.
### Verification
diff --git a/ai-gateway/src/http.rs b/ai-gateway/src/http.rs
index 3e879faed6..09f8bcf902 100644
--- a/ai-gateway/src/http.rs
+++ b/ai-gateway/src/http.rs
@@ -76,7 +76,7 @@ async fn handle_openai(
.ok_or(InferenceHttpError::Unavailable)?;
let gateway_key = gateway_key(request.headers())?.to_owned();
let (permit, context) = inference
- .resolve_and_admit(&gateway_key)
+ .resolve_and_admit(&gateway_key, route.api_format())
.await
.map_err(|error| match error {
RequestPreparationError::Resolution(error) => InferenceHttpError::Resolution(error),
diff --git a/ai-gateway/src/inference.rs b/ai-gateway/src/inference.rs
index 193f347b40..a724b86bf2 100644
--- a/ai-gateway/src/inference.rs
+++ b/ai-gateway/src/inference.rs
@@ -57,6 +57,7 @@ impl InferenceService {
pub(crate) async fn resolve_and_admit(
&self,
gateway_key: &str,
+ api_format: ApiFormat,
) -> Result<(RequestPermit, ResolvedRequestContext), RequestPreparationError> {
let span = tracing::info_span!(
"resolution",
@@ -64,7 +65,7 @@ impl InferenceService {
gateway.outcome = tracing::field::Empty
);
async {
- let prepared = self.prepare(gateway_key).await;
+ let prepared = self.prepare(gateway_key, api_format).await;
tracing::Span::current().record(
"gateway.outcome",
match &prepared {
@@ -82,6 +83,7 @@ impl InferenceService {
async fn prepare(
&self,
gateway_key: &str,
+ api_format: ApiFormat,
) -> Result<(RequestPermit, ResolvedRequestContext), RequestPreparationError> {
let context = {
let _permit = self.resolution_capacity.try_acquire().map_err(|_| {
@@ -90,7 +92,7 @@ impl InferenceService {
})?;
let _active = crate::observability::Active::new("resolution");
self.control_plane
- .resolve(gateway_key, ApiFormat::OpenAiResponses)
+ .resolve(gateway_key, api_format)
.await
.map_err(RequestPreparationError::Resolution)?
};
diff --git a/ai-gateway/src/providers/openai.rs b/ai-gateway/src/providers/openai.rs
index e268cfca82..8c06584fe5 100644
--- a/ai-gateway/src/providers/openai.rs
+++ b/ai-gateway/src/providers/openai.rs
@@ -15,7 +15,7 @@ use tokio::{
use crate::{
capture::{ExecutionCapture, RelayOutcome},
- resolution::ResolvedRequestContext,
+ resolution::{ApiFormat, ProviderCredential, ResolvedRequestContext},
transport,
};
@@ -51,6 +51,12 @@ impl OpenAiRoute {
fn captures_generation(self) -> bool {
!matches!(self, Self::Models)
}
+
+ pub(crate) fn api_format(self) -> ApiFormat {
+ match self {
+ Self::Responses | Self::ResponsesCompact | Self::Models => ApiFormat::OpenAiResponses,
+ }
+ }
}
pub(crate) struct ProviderLimits {
@@ -171,9 +177,11 @@ impl OpenAiProvider {
body: Bytes,
route: OpenAiRoute,
) -> Result<Response<Body>, ProviderError> {
- let mut authorization =
- HeaderValue::from_str(&format!("Bearer {}", context.connection().provider_token()))
- .map_err(|_| ProviderError::Configuration)?;
+ let ProviderCredential::Bearer(token) = context.connection().credential() else {
+ return Err(ProviderError::Configuration);
+ };
+ let mut authorization = HeaderValue::from_str(&format!("Bearer {token}"))
+ .map_err(|_| ProviderError::Configuration)?;
authorization.set_sensitive(true);
let mut capture = if route.captures_generation() {
let mut capture = ExecutionCapture::for_openai_responses(&context, headers, &body);
diff --git a/ai-gateway/src/resolution/contracts.rs b/ai-gateway/src/resolution/contracts.rs
index 3474b7d616..0a3b744556 100644
--- a/ai-gateway/src/resolution/contracts.rs
+++ b/ai-gateway/src/resolution/contracts.rs
@@ -9,6 +9,40 @@ use std::{collections::BTreeMap, fmt};
pub enum ApiFormat {
#[serde(rename = "openai.responses")]
OpenAiResponses,
+ #[serde(rename = "anthropic.messages")]
+ AnthropicMessages,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum Provider {
+ OpenAi,
+ Anthropic,
+}
+
+impl Provider {
+ pub fn official_origin(self) -> &'static str {
+ match self {
+ Self::OpenAi => "https://api.openai.com/v1",
+ Self::Anthropic => "https://api.anthropic.com/v1",
+ }
+ }
+
+ fn supports(self, api_format: ApiFormat) -> bool {
+ matches!(
+ (self, api_format),
+ (Self::OpenAi, ApiFormat::OpenAiResponses)
+ | (Self::Anthropic, ApiFormat::AnthropicMessages)
+ )
+ }
+
+ fn accepts(self, credential: ProviderCredential<'_>) -> bool {
+ matches!(
+ (self, credential),
+ (Self::OpenAi, ProviderCredential::Bearer(_))
+ | (Self::Anthropic, ProviderCredential::XApiKey(_))
+ )
+ }
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
@@ -26,31 +60,73 @@ pub enum MetadataValue {
Bool(bool),
}
-#[derive(Deserialize)]
-enum Provider {
- #[serde(rename = "openai")]
- OpenAi,
+#[derive(Clone, Copy)]
+pub enum ProviderCredential<'a> {
+ /// `Authorization: Bearer <token>`
+ Bearer(&'a str),
+ /// `x-api-key: <key>`
+ XApiKey(&'a str),
+}
+
+impl<'a> ProviderCredential<'a> {
+ fn secret(self) -> &'a str {
+ match self {
+ Self::Bearer(token) | Self::XApiKey(token) => token,
+ }
+ }
}
#[derive(Deserialize)]
-enum TokenType {
+enum BearerType {
Bearer,
}
+#[derive(Deserialize)]
+enum XApiKeyType {
+ #[serde(rename = "x-api-key")]
+ XApiKey,
+}
+
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
-struct Auth {
+struct BearerAuth {
#[serde(rename = "type", deserialize_with = "string_enum")]
- _token_type: TokenType,
+ _token_type: BearerType,
token: String,
}
+#[derive(Deserialize)]
+#[serde(deny_unknown_fields)]
+struct XApiKeyAuth {
+ #[serde(rename = "type", deserialize_with = "string_enum")]
+ _kind: XApiKeyType,
+ #[serde(rename = "header", deserialize_with = "string_enum")]
+ _header: XApiKeyType,
+ value: String,
+}
+
+#[derive(Deserialize)]
+#[serde(untagged)]
+enum Auth {
+ Bearer(BearerAuth),
+ XApiKey(XApiKeyAuth),
+}
+
+impl Auth {
+ fn credential(&self) -> ProviderCredential<'_> {
+ match self {
+ Self::Bearer(auth) => ProviderCredential::Bearer(&auth.token),
+ Self::XApiKey(auth) => ProviderCredential::XApiKey(&auth.value),
+ }
+ }
+}
+
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ProviderConnection {
id: String,
- #[serde(rename = "provider", deserialize_with = "string_enum")]
- _provider: Provider,
+ #[serde(deserialize_with = "string_enum")]
+ provider: Provider,
#[serde(deserialize_with = "string_enum")]
api_format: ApiFormat,
base_url: String,
@@ -61,14 +137,17 @@ impl ProviderConnection {
pub fn id(&self) -> &str {
&self.id
}
+ pub fn provider(&self) -> Provider {
+ self.provider
+ }
pub fn api_format(&self) -> ApiFormat {
self.api_format
}
pub fn base_url(&self) -> &str {
&self.base_url
}
- pub fn provider_token(&self) -> &str {
- &self.auth.token
+ pub fn credential(&self) -> ProviderCredential<'_> {
+ self.auth.credential()
}
}
@@ -105,7 +184,7 @@ impl RequestAttribution {
pub struct IngestionGrant {
access_token: String,
#[serde(rename = "token_type", deserialize_with = "string_enum")]
- _token_type: TokenType,
+ _token_type: BearerType,
expires_at: u64,
}
@@ -170,10 +249,13 @@ pub(super) fn decode(
serde_json::from_slice(bytes).map_err(|_| ResolutionError::InvalidResponse)?;
let connection = &response.connection;
let attribution = &response.attribution;
+ let provider = connection.provider;
if response.version != 1
|| connection.api_format != expected_format
- || connection.base_url != "https://api.openai.com/v1"
- || !valid_token(&connection.auth.token)
+ || !provider.supports(connection.api_format)
+ || connection.base_url != provider.official_origin()
+ || !provider.accepts(connection.credential())
+ || !valid_token(connection.credential().secret())
|| !valid_token(&response.ingestion.access_token)
|| response.ingestion.expires_at <= now
|| [
diff --git a/ai-gateway/src/resolution/mod.rs b/ai-gateway/src/resolution/mod.rs
index 6897ce4d4b..60a98d271d 100644
--- a/ai-gateway/src/resolution/mod.rs
+++ b/ai-gateway/src/resolution/mod.rs
@@ -3,8 +3,8 @@ mod contracts;
pub(crate) mod signing;
pub use contracts::{
- ApiFormat, IngestionGrant, IngestionMode, MetadataValue, ProviderConnection,
- RequestAttribution, ResolvedRequestContext,
+ ApiFormat, IngestionGrant, IngestionMode, MetadataValue, Provider, ProviderConnection,
+ ProviderCredential, RequestAttribution, ResolvedRequestContext,
};
use reqwest::{
Client, Url,
diff --git a/ai-gateway/src/resolution/tests.rs b/ai-gateway/src/resolution/tests.rs
index ef74457584..96db6264dc 100644
--- a/ai-gateway/src/resolution/tests.rs
+++ b/ai-gateway/src/resolution/tests.rs
@@ -129,10 +129,11 @@ async fn signs_the_exact_key_and_sends_only_the_api_format() {
.await
.unwrap();
assert_eq!(context.connection().id(), "connection-1");
- assert_eq!(
- context.connection().provider_token(),
- "private-provider-token"
- );
+ assert_eq!(context.connection().provider(), Provider::OpenAi);
+ assert!(matches!(
+ context.connection().credential(),
+ ProviderCredential::Bearer("private-provider-token")
+ ));
assert_eq!(context.attribution().project_id(), "project-1");
assert_eq!(
context.attribution().provider_connection_id(),
@@ -319,12 +320,128 @@ async fn concurrent_resolutions_keep_credentials_and_contexts_isolated() {
let alice = alice.unwrap();
let bob = bob.unwrap();
assert_eq!(alice.attribution().project_id(), "project-alice");
- assert_eq!(alice.connection().provider_token(), "token-alice");
+ assert!(matches!(
+ alice.connection().credential(),
+ ProviderCredential::Bearer("token-alice")
+ ));
assert_eq!(bob.attribution().project_id(), "project-bob");
- assert_eq!(bob.connection().provider_token(), "token-bob");
+ assert!(matches!(
+ bob.connection().credential(),
+ ProviderCredential::Bearer("token-bob")
+ ));
assert_eq!(web.calls(), 2);
}
+fn anthropic_success(provider_key: &str) -> Value {
+ let mut body = success("project-1", "unused");
+ body["connection"] = json!({
+ "id": "connection-anthropic",
+ "provider": "anthropic",
+ "api_format": "anthropic.messages",
+ "base_url": "https://api.anthropic.com/v1",
+ "auth": {"type": "x-api-key", "header": "x-api-key", "value": provider_key}
+ });
+ body["attribution"]["provider_connection_id"] = json!("connection-anthropic");
+ body
+}
+
+#[tokio::test]
+async fn resolves_anthropic_messages_with_the_native_credential_scheme() {
+ let web = FakeWeb::start(|request| async move {
+ assert_eq!(
+ to_bytes(request.into_body(), 1024).await.unwrap(),
+ r#"{"apiFormat":"anthropic.messages"}"#
+ );
+ response(anthropic_success("private-anthropic-key"))
+ })
+ .await;
+
+ let context = web
+ .control_plane()
+ .resolve_at(GATEWAY_KEY, ApiFormat::AnthropicMessages, NOW_TIME)
+ .await
+ .unwrap();
+ assert_eq!(context.connection().id(), "connection-anthropic");
+ assert_eq!(context.connection().provider(), Provider::Anthropic);
+ assert_eq!(
+ context.connection().api_format(),
+ ApiFormat::AnthropicMessages
+ );
+ assert_eq!(
+ context.connection().base_url(),
+ "https://api.anthropic.com/v1"
+ );
+ assert!(matches!(
+ context.connection().credential(),
+ ProviderCredential::XApiKey("private-anthropic-key")
+ ));
+ assert!(
+ !format!("{context:?}").contains("private-anthropic-key"),
+ "execution context leaked the provider key"
+ );
+ assert_eq!(web.calls(), 1);
+}
+
+#[tokio::test]
+async fn rejects_connections_whose_provider_format_origin_and_credential_disagree() {
+ // Web returned an Anthropic connection for an OpenAI Responses request, or the
+ // OpenAI connection with Anthropic's credential scheme.
+ assert_invalid_context_for(
+ anthropic_success("provider-key"),
+ ApiFormat::OpenAiResponses,
+ )
+ .await;
+ let mut body = success("project-1", "provider-token");
+ body["connection"]["auth"] =
+ json!({"type": "x-api-key", "header": "x-api-key", "value": "provider-key"});
+ assert_invalid_context_for(body, ApiFormat::OpenAiResponses).await;
+
+ // Every single-field deviation from the official Anthropic pairing.
+ let mutations = [
+ ("/connection/provider", json!("openai")),
+ ("/connection/api_format", json!("openai.responses")),
+ ("/connection/base_url", json!("https://api.openai.com/v1")),
+ ("/connection/base_url", json!("http://api.anthropic.com/v1")),
+ (
+ "/connection/base_url",
+ json!("https://api.anthropic.com.attacker.example/v1"),
+ ),
+ (
+ "/connection/base_url",
+ json!("https://api.anthropic.com/v1/"),
+ ),
+ (
+ "/connection/auth",
+ json!({"type": "Bearer", "token": "provider-key"}),
+ ),
+ (
+ "/connection/auth",
+ json!({"type": "x-api-key", "header": "authorization", "value": "provider-key"}),
+ ),
+ (
+ "/connection/auth",
+ json!({"type": "x-api-key", "value": "provider-key"}),
+ ),
+ (
+ "/connection/auth",
+ json!({"type": "x-api-key", "header": "x-api-key", "value": "provider-key", "token": "extra"}),
+ ),
+ ("/connection/auth/value", json!("")),
+ ("/connection/auth/value", json!("bad\r\nheader")),
+ ];
+ for (pointer, value) in mutations {
+ let mut body = anthropic_success("provider-key");
+ *body.pointer_mut(pointer).unwrap() = value;
+ assert_invalid_context_for(body, ApiFormat::AnthropicMessages).await;
+ }
+ // A valid OpenAI connection never satisfies an Anthropic request either.
+ assert_invalid_context_for(
+ success("project-1", "provider-token"),
+ ApiFormat::AnthropicMessages,
+ )
+ .await;
+}
+
#[tokio::test]
async fn rejects_incompatible_or_incomplete_execution_contexts() {
let mutations = [
@@ -384,6 +501,10 @@ async fn rejects_incompatible_or_incomplete_execution_contexts() {
}
async fn assert_invalid_context(body: Value) {
+ assert_invalid_context_for(body, ApiFormat::OpenAiResponses).await;
+}
+
+async fn assert_invalid_context_for(body: Value, api_format: ApiFormat) {
let web = FakeWeb::start(move |_| {
let body = body.clone();
async move { response(body) }
@@ -391,7 +512,7 @@ async fn assert_invalid_context(body: Value) {
.await;
let result = web
.control_plane()
- .resolve_at(GATEWAY_KEY, ApiFormat::OpenAiResponses, NOW_TIME)
+ .resolve_at(GATEWAY_KEY, api_format, NOW_TIME)
.await;
assert!(matches!(result, Err(ResolutionError::InvalidResponse)));
assert_eq!(web.calls(), 1);