🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
refactor(ai-gateway): SecretString credentials, typed tenant IDs, and thiserror errors (#18920)
Tobias Wochinger35c55be0b31b
diff --git a/ai-gateway/Cargo.lock b/ai-gateway/Cargo.lockindex 72a0fe70e6..ae3ed227a2 100644--- a/ai-gateway/Cargo.lock+++ b/ai-gateway/Cargo.lock@@ -38,9 +38,11 @@ dependencies = ["reqwest","reqwest-middleware","reqwest-tracing",+ "secrecy","serde","serde_json","sha2",+ "thiserror","tokio","tower","tower-http 0.7.1",@@ -1413,6 +1415,16 @@ dependencies = ["windows-sys 0.61.2",]+[[package]]+name = "secrecy"+version = "0.10.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e891af845473308773346dc847b2c23ee78fe442e0472ac50e22a18a93d3ae5a"+dependencies = [+ "serde",+ "zeroize",+]+[[package]]name = "security-framework"version = "3.7.0"diff --git a/ai-gateway/Cargo.toml b/ai-gateway/Cargo.tomlindex 6e3a8bfb15..9237ef0d28 100644--- a/ai-gateway/Cargo.toml+++ b/ai-gateway/Cargo.toml@@ -29,6 +29,8 @@ tracing = "0.1.44"tracing-opentelemetry = { version = "0.34", default-features = false }tracing-subscriber = { version = "0.3.23", default-features = false, features = ["env-filter", "fmt", "json"] }uuid = { version = "1.26.1", default-features = false, features = ["v7", "std"] }+secrecy = { version = "0.10", features = ["serde"] }+thiserror = "2"[dev-dependencies]http-body = "1"diff --git a/ai-gateway/src/capture/mod.rs b/ai-gateway/src/capture/mod.rsindex c05a039ada..0c7e189fdb 100644--- a/ai-gateway/src/capture/mod.rs+++ b/ai-gateway/src/capture/mod.rs@@ -168,8 +168,8 @@ impl ExecutionCapture {completion_start_ms: None,http_status: None,metadata: json!({- "organization_id": attribution.organization_id(),- "project_id": attribution.project_id(),+ "organization_id": attribution.organization_id().as_str(),+ "project_id": attribution.project_id().as_str(),"key_id": attribution.key_id(),"provider_connection_id": attribution.provider_connection_id(),"key_metadata": key_metadata,diff --git a/ai-gateway/src/config.rs b/ai-gateway/src/config.rsindex 5f59f685bf..c801bf4af9 100644--- a/ai-gateway/src/config.rs+++ b/ai-gateway/src/config.rs@@ -1,5 +1,5 @@use crate::resolution::ControlPlaneConfig;-use std::{env, error::Error, fmt, net::SocketAddr, ops::RangeInclusive, time::Duration};+use std::{env, net::SocketAddr, ops::RangeInclusive, time::Duration};use tracing::level_filters::LevelFilter;pub struct GatewayConfig {@@ -64,17 +64,10 @@ pub enum LogFormat {Json,}-#[derive(Debug, PartialEq, Eq)]+#[derive(Debug, PartialEq, Eq, thiserror::Error)]+#[error("{0}")]pub struct GatewayConfigError(&'static str);-impl fmt::Display for GatewayConfigError {- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {- f.write_str(self.0)- }-}--impl Error for GatewayConfigError {}-impl GatewayConfig {/// Read gateway configuration from the process environment, using defaults for absent values.///diff --git a/ai-gateway/src/correlation.rs b/ai-gateway/src/correlation.rsindex 681bb1cbf3..27dc35d864 100644--- a/ai-gateway/src/correlation.rs+++ b/ai-gateway/src/correlation.rs@@ -9,7 +9,7 @@ use axum::{response::Response,};-use crate::resolution::{ProviderConnection, ResolvedRequestContext};+use crate::resolution::{OrganizationId, ProjectId, ProviderConnection, ResolvedRequestContext};pub(crate) const REQUEST_ID_HEADER: HeaderName = HeaderName::from_static("langfuse-request-id");const TRACE_ID_HEADER: HeaderName = HeaderName::from_static("langfuse-trace-id");@@ -32,8 +32,8 @@ pub(crate) struct GenerationIds {#[derive(Clone, Debug)]struct Tenant {- organization_id: String,- project_id: String,+ organization_id: OrganizationId,+ project_id: ProjectId,}/// The provider connection and model of an upstream attempt. With several@@ -121,13 +121,15 @@ impl RequestCorrelation {pub(crate) fn record_resolution(&mut self, context: &ResolvedRequestContext) {let attribution = context.attribution();+ self.server_span.record(+ "langfuse.organization.id",+ attribution.organization_id().as_str(),+ );self.server_span- .record("langfuse.organization.id", attribution.organization_id());- self.server_span- .record("langfuse.project.id", attribution.project_id());+ .record("langfuse.project.id", attribution.project_id().as_str());self.tenant = Some(Tenant {- organization_id: attribution.organization_id().to_owned(),- project_id: attribution.project_id().to_owned(),+ organization_id: attribution.organization_id().clone(),+ project_id: attribution.project_id().clone(),});}diff --git a/ai-gateway/src/observability/mod.rs b/ai-gateway/src/observability/mod.rsindex 45012fa99e..6bae23dde1 100644--- a/ai-gateway/src/observability/mod.rs+++ b/ai-gateway/src/observability/mod.rs@@ -16,7 +16,7 @@ use opentelemetry_sdk::{propagation::TraceContextPropagator,trace::{BatchConfigBuilder, BatchSpanProcessor, Sampler, SdkTracerProvider},};-use std::{env, error::Error, time::Duration};+use std::{env, time::Duration};use tokio::time::Instant;use tracing_subscriber::{EnvFilter, Layer, filter::filter_fn, prelude::*};@@ -25,12 +25,31 @@ pub struct Observability {metrics: Option<SdkMeterProvider>,}+/// Sanitized startup failures; supplied values and exporter errors are never retained.+#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]+pub enum ObservabilityError {+ #[error("observability configuration must be Unicode")]+ NonUnicodeSetting,+ #[error("OTEL_TRACES_SAMPLER_ARG must be a number from 0 to 1")]+ InvalidSamplingRatio,+ #[error("invalid OTEL_EXPORTER_OTLP_ENDPOINT")]+ InvalidOtlpEndpoint,+ #[error("failed to initialize operational trace exporter")]+ TraceExporter,+ #[error("failed to initialize operational metric exporter")]+ MetricExporter,+ #[error("invalid gateway log filter")]+ LogFilter,+ #[error("failed to initialize gateway logging")]+ Logging,+}+/// Initialize process logging and optional operational OTLP exports.////// # Errors-/// Returns a sanitized error for invalid settings, exporter construction, or an+/// Returns an [`ObservabilityError`] for invalid settings, exporter construction, or an/// already installed tracing subscriber.-pub fn init(config: &GatewayConfig) -> Result<Observability, Box<dyn Error>> {+pub fn init(config: &GatewayConfig) -> Result<Observability, ObservabilityError> {let service = setting("OTEL_SERVICE_NAME")?.unwrap_or_else(|| "ai-gateway".into());let resource = Resource::builder().with_service_name(service)@@ -53,16 +72,16 @@ pub fn init(config: &GatewayConfig) -> Result<Observability, Box<dyn Error>> {))));let metrics = if let Some(endpoint) = setting("OTEL_EXPORTER_OTLP_ENDPOINT")? {let endpoint =- reqwest::Url::parse(&endpoint).map_err(|_| "invalid OTEL_EXPORTER_OTLP_ENDPOINT")?;+ reqwest::Url::parse(&endpoint).map_err(|_| ObservabilityError::InvalidOtlpEndpoint)?;if !matches!(endpoint.scheme(), "http" | "https") || endpoint.host_str().is_none() {- return Err("invalid OTEL_EXPORTER_OTLP_ENDPOINT".into());+ return Err(ObservabilityError::InvalidOtlpEndpoint);}let exporter = opentelemetry_otlp::SpanExporter::builder().with_http().with_protocol(Protocol::HttpBinary).with_timeout(Duration::from_secs(3)).build()- .map_err(|_| "failed to initialize operational trace exporter")?;+ .map_err(|_| ObservabilityError::TraceExporter)?;traces = traces.with_span_processor(BatchSpanProcessor::builder(exporter).with_batch_config(@@ -79,7 +98,7 @@ pub fn init(config: &GatewayConfig) -> Result<Observability, Box<dyn Error>> {.with_protocol(Protocol::HttpBinary).with_timeout(Duration::from_secs(3)).build()- .map_err(|_| "failed to initialize operational metric exporter")?;+ .map_err(|_| ObservabilityError::MetricExporter)?;Some(SdkMeterProvider::builder().with_resource(resource)@@ -98,7 +117,8 @@ pub fn init(config: &GatewayConfig) -> Result<Observability, Box<dyn Error>> {if let Some(metrics) = &metrics {global::set_meter_provider(metrics.clone());}- let log_filter = EnvFilter::try_new(format!("warn,ai_gateway={}", config.log_level))?;+ let log_filter = EnvFilter::try_new(format!("warn,ai_gateway={}", config.log_level))+ .map_err(|_| ObservabilityError::LogFilter)?;let logger = match config.log_format {LogFormat::Json => tracing_subscriber::fmt::layer().json()@@ -126,7 +146,7 @@ pub fn init(config: &GatewayConfig) -> Result<Observability, Box<dyn Error>> {.with(spans).with(logs::CorrelationSpans).try_init()- .map_err(|_| "failed to initialize gateway logging")?;+ .map_err(|_| ObservabilityError::Logging)?;Ok(Observability { traces, metrics })}@@ -154,24 +174,22 @@ impl Observability {}}-fn setting(name: &str) -> Result<Option<String>, Box<dyn Error>> {+fn setting(name: &str) -> Result<Option<String>, ObservabilityError> {match env::var(name) {Ok(value) if value.trim().is_empty() => Ok(None),Ok(value) => Ok(Some(value)),Err(env::VarError::NotPresent) => Ok(None),- Err(env::VarError::NotUnicode(_)) => {- Err("observability configuration must be Unicode".into())- }+ Err(env::VarError::NotUnicode(_)) => Err(ObservabilityError::NonUnicodeSetting),}}-fn sampling_ratio(value: Option<&str>) -> Result<f64, Box<dyn Error>> {+fn sampling_ratio(value: Option<&str>) -> Result<f64, ObservabilityError> {value.unwrap_or("1").parse::<f64>().ok().filter(|value| (0.0..=1.0).contains(value))- .ok_or_else(|| "OTEL_TRACES_SAMPLER_ARG must be a number from 0 to 1".into())+ .ok_or(ObservabilityError::InvalidSamplingRatio)}#[cfg(test)]diff --git a/ai-gateway/src/resolution/contracts.rs b/ai-gateway/src/resolution/contracts.rsindex da7424fbb6..d40f677a99 100644--- a/ai-gateway/src/resolution/contracts.rs+++ b/ai-gateway/src/resolution/contracts.rs@@ -1,5 +1,6 @@use super::{ResolutionError, valid_token};use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD};+use secrecy::{ExposeSecret, SecretString};use serde::{Deserialize, Deserializer, Serialize,de::{DeserializeOwned, IntoDeserializer},@@ -83,6 +84,8 @@ pub enum MetadataValue {Bool(bool),}+/// A borrowed, exposed view of the stored provider secret, for building the+/// upstream header. It deliberately implements neither `Debug` nor `Display`.#[derive(Clone, Copy)]pub enum ProviderCredential<'a> {/// `Authorization: Bearer <token>`@@ -115,7 +118,7 @@ enum XApiKeyType {struct BearerAuth {#[serde(rename = "type", deserialize_with = "string_enum")]_token_type: BearerType,- token: String,+ token: SecretString,}#[derive(Deserialize)]@@ -125,7 +128,7 @@ struct XApiKeyAuth {_kind: XApiKeyType,#[serde(rename = "header", deserialize_with = "string_enum")]_header: XApiKeyType,- value: String,+ value: SecretString,}#[derive(Deserialize)]@@ -138,8 +141,8 @@ enum Auth {impl Auth {fn credential(&self) -> ProviderCredential<'_> {match self {- Self::Bearer(auth) => ProviderCredential::Bearer(&auth.token),- Self::XApiKey(auth) => ProviderCredential::XApiKey(&auth.value),+ Self::Bearer(auth) => ProviderCredential::Bearer(auth.token.expose_secret()),+ Self::XApiKey(auth) => ProviderCredential::XApiKey(auth.value.expose_secret()),}}}@@ -185,21 +188,51 @@ impl ProviderConnection {}}+/// A Langfuse organization ID, typed apart from [`ProjectId`] so the tenant pair+/// cannot be swapped.+#[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize)]+#[serde(transparent)]+pub struct OrganizationId(String);++impl OrganizationId {+ pub fn as_str(&self) -> &str {+ &self.0+ }+}++/// A Langfuse project ID; telemetry batches and ingestion grants are per project.+#[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize)]+#[serde(transparent)]+pub struct ProjectId(String);++impl ProjectId {+ pub fn as_str(&self) -> &str {+ &self.0+ }+}++#[cfg(test)]+impl From<&str> for ProjectId {+ fn from(id: &str) -> Self {+ Self(id.to_owned())+ }+}+#[derive(Deserialize)]#[serde(deny_unknown_fields)]pub struct RequestAttribution {- organization_id: String,- project_id: String,+ organization_id: OrganizationId,+ project_id: ProjectId,key_id: String,key_metadata: BTreeMap<String, MetadataValue>,provider_connection_id: String,}impl RequestAttribution {- pub fn organization_id(&self) -> &str {+ pub fn organization_id(&self) -> &OrganizationId {&self.organization_id}- pub fn project_id(&self) -> &str {+ pub fn project_id(&self) -> &ProjectId {&self.project_id}pub fn key_id(&self) -> &str {@@ -216,14 +249,14 @@ impl RequestAttribution {#[derive(Deserialize)]#[serde(deny_unknown_fields)]pub struct IngestionGrant {- access_token: String,+ access_token: SecretString,#[serde(rename = "token_type", deserialize_with = "string_enum")]_token_type: BearerType,expires_at: u64,}impl IngestionGrant {- pub fn access_token(&self) -> &str {+ pub fn access_token(&self) -> &SecretString {&self.access_token}pub fn expires_at(&self) -> u64 {@@ -276,8 +309,8 @@ impl fmt::Debug for ResolvedRequestContext {#[derive(Deserialize)]struct IngestionClaims {- organization_id: String,- project_id: String,+ organization_id: OrganizationId,+ project_id: ProjectId,}fn ingestion_claims_match(token: &str, attribution: &RequestAttribution) -> bool {@@ -316,15 +349,15 @@ pub(super) fn decode(|| connection.base_url != provider.official_origin()|| !provider.accepts(connection.credential())|| !valid_token(connection.credential().secret())- || !valid_token(&response.ingestion.access_token)- || !ingestion_claims_match(&response.ingestion.access_token, attribution)+ || !valid_token(response.ingestion.access_token.expose_secret())+ || !ingestion_claims_match(response.ingestion.access_token.expose_secret(), attribution)|| response.ingestion.expires_at <= now|| [- &connection.id,- &attribution.organization_id,- &attribution.project_id,- &attribution.key_id,- &attribution.provider_connection_id,+ connection.id.as_str(),+ attribution.organization_id.as_str(),+ attribution.project_id.as_str(),+ attribution.key_id.as_str(),+ attribution.provider_connection_id.as_str(),].iter().any(|value| value.trim().is_empty())diff --git a/ai-gateway/src/resolution/mod.rs b/ai-gateway/src/resolution/mod.rsindex a9eab072f7..80963f03c9 100644--- a/ai-gateway/src/resolution/mod.rs+++ b/ai-gateway/src/resolution/mod.rs@@ -3,16 +3,16 @@ mod contracts;pub(crate) mod signing;pub use contracts::{- ApiFormat, IngestionGrant, IngestionMode, MetadataValue, Provider, ProviderConnection,- ProviderCredential, RequestAttribution, ResolvedRequestContext,+ ApiFormat, IngestionGrant, IngestionMode, MetadataValue, OrganizationId, ProjectId, Provider,+ ProviderConnection, ProviderCredential, RequestAttribution, ResolvedRequestContext,};use reqwest::{Client, Method, Url,header::{AUTHORIZATION, CONTENT_TYPE, HeaderValue},};use reqwest_middleware::{ClientWithMiddleware, RequestBuilder};+use secrecy::{ExposeSecret, SecretString};use std::{- fmt,net::IpAddr,time::{Duration, Instant, SystemTime, UNIX_EPOCH},};@@ -36,7 +36,7 @@ struct ControlPlaneLimits {/// Operator-supplied Web base URL and the existing gateway/Web service key.pub struct ControlPlaneConfig {web_url: Url,- service_key: String,+ service_key: SecretString,resolve_limits: ControlPlaneLimits,models_limits: ControlPlaneLimits,}@@ -69,7 +69,7 @@ impl ControlPlaneConfig {}Ok(Self {web_url: url,- service_key: service_key.to_owned(),+ service_key: service_key.into(),resolve_limits: ControlPlaneLimits {timeout: Duration::from_secs(5),max_response_bytes: 256 * 1024,@@ -88,7 +88,7 @@ impl ControlPlaneConfig {url}- pub(crate) fn service_key(&self) -> &str {+ pub(crate) fn service_key(&self) -> &SecretString {&self.service_key}}@@ -218,7 +218,7 @@ impl ControlPlaneClient {.map_err(|_| ResolutionError::InvalidCredential)?;credential.set_sensitive(true);let mut signature = HeaderValue::from_str(&signing::authorization(- &self.config.service_key,+ self.config.service_key.expose_secret(),gateway_key,timestamp,))@@ -294,42 +294,33 @@ fn valid_token(value: &str) -> bool {}/// Sanitized failure categories. Upstream bodies and transport errors are never retained.-#[derive(Debug, Clone, Copy, PartialEq, Eq)]+#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]pub enum ResolutionError {+ #[error("invalid resolver configuration")]Configuration,+ #[error("invalid gateway credential")]InvalidCredential,+ #[error("invalid gateway request")]InvalidRequest,+ #[error("gateway authentication failed")]Authentication,+ #[error("gateway access forbidden")]Forbidden,+ #[error("model is not allowed")]ModelNotAllowed,+ #[error("no eligible connection")]NoRoute,+ #[error("resolution unavailable")]Unavailable,+ #[error("resolution deadline exceeded")]Timeout,+ #[error("resolution transport failed")]Transport,+ #[error("invalid resolution response")]InvalidResponse,+ #[error("resolution response too large")]ResponseTooLarge,}-impl fmt::Display for ResolutionError {- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {- f.write_str(match self {- Self::Configuration => "invalid resolver configuration",- Self::InvalidCredential => "invalid gateway credential",- Self::InvalidRequest => "invalid gateway request",- Self::Authentication => "gateway authentication failed",- Self::Forbidden => "gateway access forbidden",- Self::ModelNotAllowed => "model is not allowed",- Self::NoRoute => "no eligible connection",- Self::Unavailable => "resolution unavailable",- Self::Timeout => "resolution deadline exceeded",- Self::Transport => "resolution transport failed",- Self::InvalidResponse => "invalid resolution response",- Self::ResponseTooLarge => "resolution response too large",- })- }-}--impl std::error::Error for ResolutionError {}-#[cfg(test)]mod tests;diff --git a/ai-gateway/src/resolution/tests.rs b/ai-gateway/src/resolution/tests.rsindex e4ff61615f..3d22a4c227 100644--- a/ai-gateway/src/resolution/tests.rs+++ b/ai-gateway/src/resolution/tests.rs@@ -149,13 +149,13 @@ async fn signs_the_exact_key_and_sends_api_format_and_model() {context.connection().credential(),ProviderCredential::Bearer("private-provider-token")));- assert_eq!(context.attribution().project_id(), "project-1");+ assert_eq!(context.attribution().project_id().as_str(), "project-1");assert_eq!(context.attribution().provider_connection_id(),"connection-1");assert_eq!(- context.ingestion().access_token(),+ context.ingestion().access_token().expose_secret(),ingestion_token("org-1", "project-1"));let debug = format!("{context:?}");@@ -475,12 +475,12 @@ async fn concurrent_resolutions_keep_credentials_and_contexts_isolated() {);let alice = alice.unwrap();let bob = bob.unwrap();- assert_eq!(alice.attribution().project_id(), "project-alice");+ assert_eq!(alice.attribution().project_id().as_str(), "project-alice");assert!(matches!(alice.connection().credential(),ProviderCredential::Bearer("token-alice")));- assert_eq!(bob.attribution().project_id(), "project-bob");+ assert_eq!(bob.attribution().project_id().as_str(), "project-bob");assert!(matches!(bob.connection().credential(),ProviderCredential::Bearer("token-bob")diff --git a/ai-gateway/src/telemetry/batch.rs b/ai-gateway/src/telemetry/batch.rsindex f90bccd676..c0950991a4 100644--- a/ai-gateway/src/telemetry/batch.rs+++ b/ai-gateway/src/telemetry/batch.rs@@ -8,6 +8,7 @@ use serde_json::Value;use tokio::{sync::OwnedSemaphorePermit, time::Instant};use super::Grant;+use crate::resolution::ProjectId;#[derive(Clone, Copy)]pub(super) struct BatchPolicy {@@ -100,7 +101,7 @@ fn due_at(grant: &Grant, opened: Instant, now: Instant, policy: &BatchPolicy) ->pub(super) struct Batches {policy: BatchPolicy,- open: HashMap<String, Batch>,+ open: HashMap<ProjectId, Batch>,}impl Batches {@@ -166,7 +167,7 @@ impl Batches {self.open.values().map(|batch| batch.due).min()}- fn take_project(&mut self, project: &str) -> Option<Flush> {+ fn take_project(&mut self, project: &ProjectId) -> Option<Flush> {self.open.remove(project).map(Batch::into_flush)}}diff --git a/ai-gateway/src/telemetry/batch/tests.rs b/ai-gateway/src/telemetry/batch/tests.rsindex ba7fc29e0e..23bbc747d7 100644--- a/ai-gateway/src/telemetry/batch/tests.rs+++ b/ai-gateway/src/telemetry/batch/tests.rs@@ -4,6 +4,7 @@ use std::{};use opentelemetry::trace::SpanContext;+use secrecy::ExposeSecret;use serde_json::json;use tokio::{sync::Semaphore, time::Instant};@@ -122,7 +123,7 @@ fn a_later_expiring_grant_replaces_the_batch_grant() {assert_eq!(batches.next_due(), Some(now + LINGER));batches.push_record(grant("a", "earlier", 5), item(1), now);let flush = batches.take_all().pop().unwrap();- assert_eq!(flush.grant.access_token, "later");+ assert_eq!(flush.grant.access_token.expose_secret(), "later");}#[test]diff --git a/ai-gateway/src/telemetry/mod.rs b/ai-gateway/src/telemetry/mod.rsindex b6d0f8d5c2..6de1aca06e 100644--- a/ai-gateway/src/telemetry/mod.rs+++ b/ai-gateway/src/telemetry/mod.rs@@ -27,13 +27,14 @@ use tracing_opentelemetry::OpenTelemetrySpanExt;use crate::{capture::{InferenceFacts, InputOmissionReason},- resolution::{ControlPlaneConfig, ResolutionError, ResolvedRequestContext},+ resolution::{ControlPlaneConfig, ProjectId, ResolutionError, ResolvedRequestContext},};use batch::{BatchPolicy, Batches, Pending};use context::GenerationContext;pub(crate) use context::take_agent_client_metadata;use otlp::Uploader;use retry::RetryPolicy;+use secrecy::{ExposeSecret, SecretString};use worker::{Message, Uploads};const MAX_UPLOADS: usize = 32;@@ -47,8 +48,8 @@ pub(crate) const DEFAULT_RETAINED_BYTES: usize = 64 * 1024 * 1024;const MAX_QUEUED_RECORDS: usize = 1024;struct Grant {- project_id: String,- access_token: String,+ project_id: ProjectId,+ access_token: SecretString,expires_at: u64,}@@ -74,8 +75,8 @@ impl DeliveryContext {) -> Self {Self {grant: Grant {- project_id: context.attribution().project_id().to_owned(),- access_token: context.ingestion().access_token().to_owned(),+ project_id: context.attribution().project_id().clone(),+ access_token: context.ingestion().access_token().clone(),expires_at: context.ingestion().expires_at(),},generation: GenerationContext::from_request(headers, client_metadata),@@ -195,7 +196,8 @@ impl Telemetry {.span_context().clone();let mut span = mapping::span(facts, &context.generation);- let credentials = context.grant.access_token.len() + context.grant.project_id.len();+ let credentials = context.grant.access_token.expose_secret().len()+ + context.grant.project_id.as_str().len();let limit = self.0.record_limit.saturating_sub(credentials);let acquire = |bytes: usize| {let permits = u32::try_from(bytes + credentials).ok()?;diff --git a/ai-gateway/src/telemetry/otlp.rs b/ai-gateway/src/telemetry/otlp.rsindex 06af3b430e..fcb624d72c 100644--- a/ai-gateway/src/telemetry/otlp.rs+++ b/ai-gateway/src/telemetry/otlp.rs@@ -10,6 +10,7 @@ use axum::{use flate2::{Compression, write::GzEncoder};use reqwest::{Client, Url};use reqwest_middleware::ClientWithMiddleware;+use secrecy::{ExposeSecret, SecretString};use serde::Serialize;use serde_json::{Value, json};@@ -24,7 +25,7 @@ const UPLOAD_TIMEOUT: Duration = Duration::from_secs(30);pub(super) struct Uploader {client: ClientWithMiddleware,endpoint: Url,- service_key: String,+ service_key: SecretString,}#[derive(Debug, PartialEq, Eq)]@@ -86,7 +87,7 @@ impl Uploader {Ok(Self {client: crate::observability::instrument_client(client, "ingestion"),endpoint: config.endpoint(INGESTION_PATH),- service_key: config.service_key().to_owned(),+ service_key: config.service_key().clone(),})}@@ -98,12 +99,13 @@ impl Uploader {.checked_sub(now).filter(|duration| !duration.is_zero()).ok_or(ExportError::Expired)?;- let mut authorization = HeaderValue::from_str(&format!("Bearer {}", grant.access_token))- .map_err(|_| ExportError::Payload)?;+ let mut authorization =+ HeaderValue::from_str(&format!("Bearer {}", grant.access_token.expose_secret()))+ .map_err(|_| ExportError::Payload)?;authorization.set_sensitive(true);let mut signature = HeaderValue::from_str(&signing::authorization(- &self.service_key,- &grant.access_token,+ self.service_key.expose_secret(),+ grant.access_token.expose_secret(),now.as_secs(),)).map_err(|_| ExportError::Payload)?;diff --git a/ai-gateway/src/telemetry/tests.rs b/ai-gateway/src/telemetry/tests.rsindex f8979d3e52..77f0fd79ea 100644--- a/ai-gateway/src/telemetry/tests.rs+++ b/ai-gateway/src/telemetry/tests.rs@@ -164,7 +164,7 @@ async fn concurrent_projects_keep_their_original_grants_and_attribution() {for project in ["project-a", "project-b"] {let mut grant = grant().await;grant.grant.project_id = project.into();- grant.grant.access_token = format!("token-{project}");+ grant.grant.access_token = format!("token-{project}").into();telemetry.record(grant, facts(project));}telemetry@@ -304,8 +304,8 @@ fn record_bytes(context: &DeliveryContext, facts: InferenceFacts) -> usize {serde_json::to_vec(&mapping::span(facts, &context.generation)).unwrap().len()- + context.grant.access_token.len()- + context.grant.project_id.len()+ + context.grant.access_token.expose_secret().len()+ + context.grant.project_id.as_str().len()}fn exported_spans(payload: &Value) -> usize {@@ -418,7 +418,7 @@ async fn interleaved_projects_never_share_an_upload_or_a_grant() {for project in ["project-a", "project-b"] {let mut context = grant().await;context.grant.project_id = project.into();- context.grant.access_token = format!("{project}-token-{round}");+ context.grant.access_token = format!("{project}-token-{round}").into();context.grant.expires_at += round * 10;telemetry.record(context, facts(project));}diff --git a/ai-gateway/src/transport/mod.rs b/ai-gateway/src/transport/mod.rsindex 754d3167f6..0fbd5cc235 100644--- a/ai-gateway/src/transport/mod.rs+++ b/ai-gateway/src/transport/mod.rs@@ -1,6 +1,5 @@//! Bounded byte relay. One task owns upstream reads; the body owns its lifetime.use std::{- fmt,pin::Pin,sync::{Arc, Mutex,@@ -26,26 +25,18 @@ use tokio::{};/// Sanitized failures; upstream URLs, bodies and credentials are never retained.-#[derive(Debug, Clone, Copy, PartialEq, Eq)]+#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]pub enum ProviderError {+ #[error("invalid provider configuration")]Configuration,+ #[error("gateway execution capacity exhausted")]Busy,+ #[error("provider deadline exceeded")]Timeout,+ #[error("provider transport failed")]Transport,}-impl fmt::Display for ProviderError {- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {- f.write_str(match self {- Self::Configuration => "invalid provider configuration",- Self::Busy => "gateway execution capacity exhausted",- Self::Timeout => "provider deadline exceeded",- Self::Transport => "provider transport failed",- })- }-}-impl std::error::Error for ProviderError {}-// An allowlist prevents gateway credentials, tenant routing overrides and cookies// from crossing the boundary. Headers nominated by `Connection` are never end-to-end.// Prefixes admit a provider's own evolving header family without naming each member.