Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

refactor(ai-gateway): SecretString credentials, typed tenant IDs, and thiserror errors (#18920)

Tobias Wochinger35c55be0b31b
diff --git a/ai-gateway/Cargo.lock b/ai-gateway/Cargo.lock
index 72a0fe70e6..ae3ed227a2 100644
--- a/ai-gateway/Cargo.lock
+++ b/ai-gateway/Cargo.lock
@@ -38,9 +38,11 @@ dependencies = [
"reqwest",
"reqwest-middleware",
"reqwest-tracing",
+ "secrecy",
"serde",
"serde_json",
"sha2",
+ "thiserror",
"tokio",
"tower",
"tower-http 0.7.1",
@@ -1413,6 +1415,16 @@ dependencies = [
"windows-sys 0.61.2",
]
+[[package]]
+name = "secrecy"
+version = "0.10.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e891af845473308773346dc847b2c23ee78fe442e0472ac50e22a18a93d3ae5a"
+dependencies = [
+ "serde",
+ "zeroize",
+]
+
[[package]]
name = "security-framework"
version = "3.7.0"
diff --git a/ai-gateway/Cargo.toml b/ai-gateway/Cargo.toml
index 6e3a8bfb15..9237ef0d28 100644
--- a/ai-gateway/Cargo.toml
+++ b/ai-gateway/Cargo.toml
@@ -29,6 +29,8 @@ tracing = "0.1.44"
tracing-opentelemetry = { version = "0.34", default-features = false }
tracing-subscriber = { version = "0.3.23", default-features = false, features = ["env-filter", "fmt", "json"] }
uuid = { version = "1.26.1", default-features = false, features = ["v7", "std"] }
+secrecy = { version = "0.10", features = ["serde"] }
+thiserror = "2"
[dev-dependencies]
http-body = "1"
diff --git a/ai-gateway/src/capture/mod.rs b/ai-gateway/src/capture/mod.rs
index c05a039ada..0c7e189fdb 100644
--- a/ai-gateway/src/capture/mod.rs
+++ b/ai-gateway/src/capture/mod.rs
@@ -168,8 +168,8 @@ impl ExecutionCapture {
completion_start_ms: None,
http_status: None,
metadata: json!({
- "organization_id": attribution.organization_id(),
- "project_id": attribution.project_id(),
+ "organization_id": attribution.organization_id().as_str(),
+ "project_id": attribution.project_id().as_str(),
"key_id": attribution.key_id(),
"provider_connection_id": attribution.provider_connection_id(),
"key_metadata": key_metadata,
diff --git a/ai-gateway/src/config.rs b/ai-gateway/src/config.rs
index 5f59f685bf..c801bf4af9 100644
--- a/ai-gateway/src/config.rs
+++ b/ai-gateway/src/config.rs
@@ -1,5 +1,5 @@
use crate::resolution::ControlPlaneConfig;
-use std::{env, error::Error, fmt, net::SocketAddr, ops::RangeInclusive, time::Duration};
+use std::{env, net::SocketAddr, ops::RangeInclusive, time::Duration};
use tracing::level_filters::LevelFilter;
pub struct GatewayConfig {
@@ -64,17 +64,10 @@ pub enum LogFormat {
Json,
}
-#[derive(Debug, PartialEq, Eq)]
+#[derive(Debug, PartialEq, Eq, thiserror::Error)]
+#[error("{0}")]
pub struct GatewayConfigError(&'static str);
-impl fmt::Display for GatewayConfigError {
- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- f.write_str(self.0)
- }
-}
-
-impl Error for GatewayConfigError {}
-
impl GatewayConfig {
/// Read gateway configuration from the process environment, using defaults for absent values.
///
diff --git a/ai-gateway/src/correlation.rs b/ai-gateway/src/correlation.rs
index 681bb1cbf3..27dc35d864 100644
--- a/ai-gateway/src/correlation.rs
+++ b/ai-gateway/src/correlation.rs
@@ -9,7 +9,7 @@ use axum::{
response::Response,
};
-use crate::resolution::{ProviderConnection, ResolvedRequestContext};
+use crate::resolution::{OrganizationId, ProjectId, ProviderConnection, ResolvedRequestContext};
pub(crate) const REQUEST_ID_HEADER: HeaderName = HeaderName::from_static("langfuse-request-id");
const TRACE_ID_HEADER: HeaderName = HeaderName::from_static("langfuse-trace-id");
@@ -32,8 +32,8 @@ pub(crate) struct GenerationIds {
#[derive(Clone, Debug)]
struct Tenant {
- organization_id: String,
- project_id: String,
+ organization_id: OrganizationId,
+ project_id: ProjectId,
}
/// The provider connection and model of an upstream attempt. With several
@@ -121,13 +121,15 @@ impl RequestCorrelation {
pub(crate) fn record_resolution(&mut self, context: &ResolvedRequestContext) {
let attribution = context.attribution();
+ self.server_span.record(
+ "langfuse.organization.id",
+ attribution.organization_id().as_str(),
+ );
self.server_span
- .record("langfuse.organization.id", attribution.organization_id());
- self.server_span
- .record("langfuse.project.id", attribution.project_id());
+ .record("langfuse.project.id", attribution.project_id().as_str());
self.tenant = Some(Tenant {
- organization_id: attribution.organization_id().to_owned(),
- project_id: attribution.project_id().to_owned(),
+ organization_id: attribution.organization_id().clone(),
+ project_id: attribution.project_id().clone(),
});
}
diff --git a/ai-gateway/src/observability/mod.rs b/ai-gateway/src/observability/mod.rs
index 45012fa99e..6bae23dde1 100644
--- a/ai-gateway/src/observability/mod.rs
+++ b/ai-gateway/src/observability/mod.rs
@@ -16,7 +16,7 @@ use opentelemetry_sdk::{
propagation::TraceContextPropagator,
trace::{BatchConfigBuilder, BatchSpanProcessor, Sampler, SdkTracerProvider},
};
-use std::{env, error::Error, time::Duration};
+use std::{env, time::Duration};
use tokio::time::Instant;
use tracing_subscriber::{EnvFilter, Layer, filter::filter_fn, prelude::*};
@@ -25,12 +25,31 @@ pub struct Observability {
metrics: Option<SdkMeterProvider>,
}
+/// Sanitized startup failures; supplied values and exporter errors are never retained.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
+pub enum ObservabilityError {
+ #[error("observability configuration must be Unicode")]
+ NonUnicodeSetting,
+ #[error("OTEL_TRACES_SAMPLER_ARG must be a number from 0 to 1")]
+ InvalidSamplingRatio,
+ #[error("invalid OTEL_EXPORTER_OTLP_ENDPOINT")]
+ InvalidOtlpEndpoint,
+ #[error("failed to initialize operational trace exporter")]
+ TraceExporter,
+ #[error("failed to initialize operational metric exporter")]
+ MetricExporter,
+ #[error("invalid gateway log filter")]
+ LogFilter,
+ #[error("failed to initialize gateway logging")]
+ Logging,
+}
+
/// Initialize process logging and optional operational OTLP exports.
///
/// # Errors
-/// Returns a sanitized error for invalid settings, exporter construction, or an
+/// Returns an [`ObservabilityError`] for invalid settings, exporter construction, or an
/// already installed tracing subscriber.
-pub fn init(config: &GatewayConfig) -> Result<Observability, Box<dyn Error>> {
+pub fn init(config: &GatewayConfig) -> Result<Observability, ObservabilityError> {
let service = setting("OTEL_SERVICE_NAME")?.unwrap_or_else(|| "ai-gateway".into());
let resource = Resource::builder()
.with_service_name(service)
@@ -53,16 +72,16 @@ pub fn init(config: &GatewayConfig) -> Result<Observability, Box<dyn Error>> {
))));
let metrics = if let Some(endpoint) = setting("OTEL_EXPORTER_OTLP_ENDPOINT")? {
let endpoint =
- reqwest::Url::parse(&endpoint).map_err(|_| "invalid OTEL_EXPORTER_OTLP_ENDPOINT")?;
+ reqwest::Url::parse(&endpoint).map_err(|_| ObservabilityError::InvalidOtlpEndpoint)?;
if !matches!(endpoint.scheme(), "http" | "https") || endpoint.host_str().is_none() {
- return Err("invalid OTEL_EXPORTER_OTLP_ENDPOINT".into());
+ return Err(ObservabilityError::InvalidOtlpEndpoint);
}
let exporter = opentelemetry_otlp::SpanExporter::builder()
.with_http()
.with_protocol(Protocol::HttpBinary)
.with_timeout(Duration::from_secs(3))
.build()
- .map_err(|_| "failed to initialize operational trace exporter")?;
+ .map_err(|_| ObservabilityError::TraceExporter)?;
traces = traces.with_span_processor(
BatchSpanProcessor::builder(exporter)
.with_batch_config(
@@ -79,7 +98,7 @@ pub fn init(config: &GatewayConfig) -> Result<Observability, Box<dyn Error>> {
.with_protocol(Protocol::HttpBinary)
.with_timeout(Duration::from_secs(3))
.build()
- .map_err(|_| "failed to initialize operational metric exporter")?;
+ .map_err(|_| ObservabilityError::MetricExporter)?;
Some(
SdkMeterProvider::builder()
.with_resource(resource)
@@ -98,7 +117,8 @@ pub fn init(config: &GatewayConfig) -> Result<Observability, Box<dyn Error>> {
if let Some(metrics) = &metrics {
global::set_meter_provider(metrics.clone());
}
- let log_filter = EnvFilter::try_new(format!("warn,ai_gateway={}", config.log_level))?;
+ let log_filter = EnvFilter::try_new(format!("warn,ai_gateway={}", config.log_level))
+ .map_err(|_| ObservabilityError::LogFilter)?;
let logger = match config.log_format {
LogFormat::Json => tracing_subscriber::fmt::layer()
.json()
@@ -126,7 +146,7 @@ pub fn init(config: &GatewayConfig) -> Result<Observability, Box<dyn Error>> {
.with(spans)
.with(logs::CorrelationSpans)
.try_init()
- .map_err(|_| "failed to initialize gateway logging")?;
+ .map_err(|_| ObservabilityError::Logging)?;
Ok(Observability { traces, metrics })
}
@@ -154,24 +174,22 @@ impl Observability {
}
}
-fn setting(name: &str) -> Result<Option<String>, Box<dyn Error>> {
+fn setting(name: &str) -> Result<Option<String>, ObservabilityError> {
match env::var(name) {
Ok(value) if value.trim().is_empty() => Ok(None),
Ok(value) => Ok(Some(value)),
Err(env::VarError::NotPresent) => Ok(None),
- Err(env::VarError::NotUnicode(_)) => {
- Err("observability configuration must be Unicode".into())
- }
+ Err(env::VarError::NotUnicode(_)) => Err(ObservabilityError::NonUnicodeSetting),
}
}
-fn sampling_ratio(value: Option<&str>) -> Result<f64, Box<dyn Error>> {
+fn sampling_ratio(value: Option<&str>) -> Result<f64, ObservabilityError> {
value
.unwrap_or("1")
.parse::<f64>()
.ok()
.filter(|value| (0.0..=1.0).contains(value))
- .ok_or_else(|| "OTEL_TRACES_SAMPLER_ARG must be a number from 0 to 1".into())
+ .ok_or(ObservabilityError::InvalidSamplingRatio)
}
#[cfg(test)]
diff --git a/ai-gateway/src/resolution/contracts.rs b/ai-gateway/src/resolution/contracts.rs
index da7424fbb6..d40f677a99 100644
--- a/ai-gateway/src/resolution/contracts.rs
+++ b/ai-gateway/src/resolution/contracts.rs
@@ -1,5 +1,6 @@
use super::{ResolutionError, valid_token};
use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD};
+use secrecy::{ExposeSecret, SecretString};
use serde::{
Deserialize, Deserializer, Serialize,
de::{DeserializeOwned, IntoDeserializer},
@@ -83,6 +84,8 @@ pub enum MetadataValue {
Bool(bool),
}
+/// A borrowed, exposed view of the stored provider secret, for building the
+/// upstream header. It deliberately implements neither `Debug` nor `Display`.
#[derive(Clone, Copy)]
pub enum ProviderCredential<'a> {
/// `Authorization: Bearer <token>`
@@ -115,7 +118,7 @@ enum XApiKeyType {
struct BearerAuth {
#[serde(rename = "type", deserialize_with = "string_enum")]
_token_type: BearerType,
- token: String,
+ token: SecretString,
}
#[derive(Deserialize)]
@@ -125,7 +128,7 @@ struct XApiKeyAuth {
_kind: XApiKeyType,
#[serde(rename = "header", deserialize_with = "string_enum")]
_header: XApiKeyType,
- value: String,
+ value: SecretString,
}
#[derive(Deserialize)]
@@ -138,8 +141,8 @@ enum Auth {
impl Auth {
fn credential(&self) -> ProviderCredential<'_> {
match self {
- Self::Bearer(auth) => ProviderCredential::Bearer(&auth.token),
- Self::XApiKey(auth) => ProviderCredential::XApiKey(&auth.value),
+ Self::Bearer(auth) => ProviderCredential::Bearer(auth.token.expose_secret()),
+ Self::XApiKey(auth) => ProviderCredential::XApiKey(auth.value.expose_secret()),
}
}
}
@@ -185,21 +188,51 @@ impl ProviderConnection {
}
}
+/// A Langfuse organization ID, typed apart from [`ProjectId`] so the tenant pair
+/// cannot be swapped.
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize)]
+#[serde(transparent)]
+pub struct OrganizationId(String);
+
+impl OrganizationId {
+ pub fn as_str(&self) -> &str {
+ &self.0
+ }
+}
+
+/// A Langfuse project ID; telemetry batches and ingestion grants are per project.
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize)]
+#[serde(transparent)]
+pub struct ProjectId(String);
+
+impl ProjectId {
+ pub fn as_str(&self) -> &str {
+ &self.0
+ }
+}
+
+#[cfg(test)]
+impl From<&str> for ProjectId {
+ fn from(id: &str) -> Self {
+ Self(id.to_owned())
+ }
+}
+
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
pub struct RequestAttribution {
- organization_id: String,
- project_id: String,
+ organization_id: OrganizationId,
+ project_id: ProjectId,
key_id: String,
key_metadata: BTreeMap<String, MetadataValue>,
provider_connection_id: String,
}
impl RequestAttribution {
- pub fn organization_id(&self) -> &str {
+ pub fn organization_id(&self) -> &OrganizationId {
&self.organization_id
}
- pub fn project_id(&self) -> &str {
+ pub fn project_id(&self) -> &ProjectId {
&self.project_id
}
pub fn key_id(&self) -> &str {
@@ -216,14 +249,14 @@ impl RequestAttribution {
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
pub struct IngestionGrant {
- access_token: String,
+ access_token: SecretString,
#[serde(rename = "token_type", deserialize_with = "string_enum")]
_token_type: BearerType,
expires_at: u64,
}
impl IngestionGrant {
- pub fn access_token(&self) -> &str {
+ pub fn access_token(&self) -> &SecretString {
&self.access_token
}
pub fn expires_at(&self) -> u64 {
@@ -276,8 +309,8 @@ impl fmt::Debug for ResolvedRequestContext {
#[derive(Deserialize)]
struct IngestionClaims {
- organization_id: String,
- project_id: String,
+ organization_id: OrganizationId,
+ project_id: ProjectId,
}
fn ingestion_claims_match(token: &str, attribution: &RequestAttribution) -> bool {
@@ -316,15 +349,15 @@ pub(super) fn decode(
|| connection.base_url != provider.official_origin()
|| !provider.accepts(connection.credential())
|| !valid_token(connection.credential().secret())
- || !valid_token(&response.ingestion.access_token)
- || !ingestion_claims_match(&response.ingestion.access_token, attribution)
+ || !valid_token(response.ingestion.access_token.expose_secret())
+ || !ingestion_claims_match(response.ingestion.access_token.expose_secret(), attribution)
|| response.ingestion.expires_at <= now
|| [
- &connection.id,
- &attribution.organization_id,
- &attribution.project_id,
- &attribution.key_id,
- &attribution.provider_connection_id,
+ connection.id.as_str(),
+ attribution.organization_id.as_str(),
+ attribution.project_id.as_str(),
+ attribution.key_id.as_str(),
+ attribution.provider_connection_id.as_str(),
]
.iter()
.any(|value| value.trim().is_empty())
diff --git a/ai-gateway/src/resolution/mod.rs b/ai-gateway/src/resolution/mod.rs
index a9eab072f7..80963f03c9 100644
--- a/ai-gateway/src/resolution/mod.rs
+++ b/ai-gateway/src/resolution/mod.rs
@@ -3,16 +3,16 @@ mod contracts;
pub(crate) mod signing;
pub use contracts::{
- ApiFormat, IngestionGrant, IngestionMode, MetadataValue, Provider, ProviderConnection,
- ProviderCredential, RequestAttribution, ResolvedRequestContext,
+ ApiFormat, IngestionGrant, IngestionMode, MetadataValue, OrganizationId, ProjectId, Provider,
+ ProviderConnection, ProviderCredential, RequestAttribution, ResolvedRequestContext,
};
use reqwest::{
Client, Method, Url,
header::{AUTHORIZATION, CONTENT_TYPE, HeaderValue},
};
use reqwest_middleware::{ClientWithMiddleware, RequestBuilder};
+use secrecy::{ExposeSecret, SecretString};
use std::{
- fmt,
net::IpAddr,
time::{Duration, Instant, SystemTime, UNIX_EPOCH},
};
@@ -36,7 +36,7 @@ struct ControlPlaneLimits {
/// Operator-supplied Web base URL and the existing gateway/Web service key.
pub struct ControlPlaneConfig {
web_url: Url,
- service_key: String,
+ service_key: SecretString,
resolve_limits: ControlPlaneLimits,
models_limits: ControlPlaneLimits,
}
@@ -69,7 +69,7 @@ impl ControlPlaneConfig {
}
Ok(Self {
web_url: url,
- service_key: service_key.to_owned(),
+ service_key: service_key.into(),
resolve_limits: ControlPlaneLimits {
timeout: Duration::from_secs(5),
max_response_bytes: 256 * 1024,
@@ -88,7 +88,7 @@ impl ControlPlaneConfig {
url
}
- pub(crate) fn service_key(&self) -> &str {
+ pub(crate) fn service_key(&self) -> &SecretString {
&self.service_key
}
}
@@ -218,7 +218,7 @@ impl ControlPlaneClient {
.map_err(|_| ResolutionError::InvalidCredential)?;
credential.set_sensitive(true);
let mut signature = HeaderValue::from_str(&signing::authorization(
- &self.config.service_key,
+ self.config.service_key.expose_secret(),
gateway_key,
timestamp,
))
@@ -294,42 +294,33 @@ fn valid_token(value: &str) -> bool {
}
/// Sanitized failure categories. Upstream bodies and transport errors are never retained.
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
pub enum ResolutionError {
+ #[error("invalid resolver configuration")]
Configuration,
+ #[error("invalid gateway credential")]
InvalidCredential,
+ #[error("invalid gateway request")]
InvalidRequest,
+ #[error("gateway authentication failed")]
Authentication,
+ #[error("gateway access forbidden")]
Forbidden,
+ #[error("model is not allowed")]
ModelNotAllowed,
+ #[error("no eligible connection")]
NoRoute,
+ #[error("resolution unavailable")]
Unavailable,
+ #[error("resolution deadline exceeded")]
Timeout,
+ #[error("resolution transport failed")]
Transport,
+ #[error("invalid resolution response")]
InvalidResponse,
+ #[error("resolution response too large")]
ResponseTooLarge,
}
-impl fmt::Display for ResolutionError {
- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- f.write_str(match self {
- Self::Configuration => "invalid resolver configuration",
- Self::InvalidCredential => "invalid gateway credential",
- Self::InvalidRequest => "invalid gateway request",
- Self::Authentication => "gateway authentication failed",
- Self::Forbidden => "gateway access forbidden",
- Self::ModelNotAllowed => "model is not allowed",
- Self::NoRoute => "no eligible connection",
- Self::Unavailable => "resolution unavailable",
- Self::Timeout => "resolution deadline exceeded",
- Self::Transport => "resolution transport failed",
- Self::InvalidResponse => "invalid resolution response",
- Self::ResponseTooLarge => "resolution response too large",
- })
- }
-}
-
-impl std::error::Error for ResolutionError {}
-
#[cfg(test)]
mod tests;
diff --git a/ai-gateway/src/resolution/tests.rs b/ai-gateway/src/resolution/tests.rs
index e4ff61615f..3d22a4c227 100644
--- a/ai-gateway/src/resolution/tests.rs
+++ b/ai-gateway/src/resolution/tests.rs
@@ -149,13 +149,13 @@ async fn signs_the_exact_key_and_sends_api_format_and_model() {
context.connection().credential(),
ProviderCredential::Bearer("private-provider-token")
));
- assert_eq!(context.attribution().project_id(), "project-1");
+ assert_eq!(context.attribution().project_id().as_str(), "project-1");
assert_eq!(
context.attribution().provider_connection_id(),
"connection-1"
);
assert_eq!(
- context.ingestion().access_token(),
+ context.ingestion().access_token().expose_secret(),
ingestion_token("org-1", "project-1")
);
let debug = format!("{context:?}");
@@ -475,12 +475,12 @@ async fn concurrent_resolutions_keep_credentials_and_contexts_isolated() {
);
let alice = alice.unwrap();
let bob = bob.unwrap();
- assert_eq!(alice.attribution().project_id(), "project-alice");
+ assert_eq!(alice.attribution().project_id().as_str(), "project-alice");
assert!(matches!(
alice.connection().credential(),
ProviderCredential::Bearer("token-alice")
));
- assert_eq!(bob.attribution().project_id(), "project-bob");
+ assert_eq!(bob.attribution().project_id().as_str(), "project-bob");
assert!(matches!(
bob.connection().credential(),
ProviderCredential::Bearer("token-bob")
diff --git a/ai-gateway/src/telemetry/batch.rs b/ai-gateway/src/telemetry/batch.rs
index f90bccd676..c0950991a4 100644
--- a/ai-gateway/src/telemetry/batch.rs
+++ b/ai-gateway/src/telemetry/batch.rs
@@ -8,6 +8,7 @@ use serde_json::Value;
use tokio::{sync::OwnedSemaphorePermit, time::Instant};
use super::Grant;
+use crate::resolution::ProjectId;
#[derive(Clone, Copy)]
pub(super) struct BatchPolicy {
@@ -100,7 +101,7 @@ fn due_at(grant: &Grant, opened: Instant, now: Instant, policy: &BatchPolicy) ->
pub(super) struct Batches {
policy: BatchPolicy,
- open: HashMap<String, Batch>,
+ open: HashMap<ProjectId, Batch>,
}
impl Batches {
@@ -166,7 +167,7 @@ impl Batches {
self.open.values().map(|batch| batch.due).min()
}
- fn take_project(&mut self, project: &str) -> Option<Flush> {
+ fn take_project(&mut self, project: &ProjectId) -> Option<Flush> {
self.open.remove(project).map(Batch::into_flush)
}
}
diff --git a/ai-gateway/src/telemetry/batch/tests.rs b/ai-gateway/src/telemetry/batch/tests.rs
index ba7fc29e0e..23bbc747d7 100644
--- a/ai-gateway/src/telemetry/batch/tests.rs
+++ b/ai-gateway/src/telemetry/batch/tests.rs
@@ -4,6 +4,7 @@ use std::{
};
use opentelemetry::trace::SpanContext;
+use secrecy::ExposeSecret;
use serde_json::json;
use tokio::{sync::Semaphore, time::Instant};
@@ -122,7 +123,7 @@ fn a_later_expiring_grant_replaces_the_batch_grant() {
assert_eq!(batches.next_due(), Some(now + LINGER));
batches.push_record(grant("a", "earlier", 5), item(1), now);
let flush = batches.take_all().pop().unwrap();
- assert_eq!(flush.grant.access_token, "later");
+ assert_eq!(flush.grant.access_token.expose_secret(), "later");
}
#[test]
diff --git a/ai-gateway/src/telemetry/mod.rs b/ai-gateway/src/telemetry/mod.rs
index b6d0f8d5c2..6de1aca06e 100644
--- a/ai-gateway/src/telemetry/mod.rs
+++ b/ai-gateway/src/telemetry/mod.rs
@@ -27,13 +27,14 @@ use tracing_opentelemetry::OpenTelemetrySpanExt;
use crate::{
capture::{InferenceFacts, InputOmissionReason},
- resolution::{ControlPlaneConfig, ResolutionError, ResolvedRequestContext},
+ resolution::{ControlPlaneConfig, ProjectId, ResolutionError, ResolvedRequestContext},
};
use batch::{BatchPolicy, Batches, Pending};
use context::GenerationContext;
pub(crate) use context::take_agent_client_metadata;
use otlp::Uploader;
use retry::RetryPolicy;
+use secrecy::{ExposeSecret, SecretString};
use worker::{Message, Uploads};
const MAX_UPLOADS: usize = 32;
@@ -47,8 +48,8 @@ pub(crate) const DEFAULT_RETAINED_BYTES: usize = 64 * 1024 * 1024;
const MAX_QUEUED_RECORDS: usize = 1024;
struct Grant {
- project_id: String,
- access_token: String,
+ project_id: ProjectId,
+ access_token: SecretString,
expires_at: u64,
}
@@ -74,8 +75,8 @@ impl DeliveryContext {
) -> Self {
Self {
grant: Grant {
- project_id: context.attribution().project_id().to_owned(),
- access_token: context.ingestion().access_token().to_owned(),
+ project_id: context.attribution().project_id().clone(),
+ access_token: context.ingestion().access_token().clone(),
expires_at: context.ingestion().expires_at(),
},
generation: GenerationContext::from_request(headers, client_metadata),
@@ -195,7 +196,8 @@ impl Telemetry {
.span_context()
.clone();
let mut span = mapping::span(facts, &context.generation);
- let credentials = context.grant.access_token.len() + context.grant.project_id.len();
+ let credentials = context.grant.access_token.expose_secret().len()
+ + context.grant.project_id.as_str().len();
let limit = self.0.record_limit.saturating_sub(credentials);
let acquire = |bytes: usize| {
let permits = u32::try_from(bytes + credentials).ok()?;
diff --git a/ai-gateway/src/telemetry/otlp.rs b/ai-gateway/src/telemetry/otlp.rs
index 06af3b430e..fcb624d72c 100644
--- a/ai-gateway/src/telemetry/otlp.rs
+++ b/ai-gateway/src/telemetry/otlp.rs
@@ -10,6 +10,7 @@ use axum::{
use flate2::{Compression, write::GzEncoder};
use reqwest::{Client, Url};
use reqwest_middleware::ClientWithMiddleware;
+use secrecy::{ExposeSecret, SecretString};
use serde::Serialize;
use serde_json::{Value, json};
@@ -24,7 +25,7 @@ const UPLOAD_TIMEOUT: Duration = Duration::from_secs(30);
pub(super) struct Uploader {
client: ClientWithMiddleware,
endpoint: Url,
- service_key: String,
+ service_key: SecretString,
}
#[derive(Debug, PartialEq, Eq)]
@@ -86,7 +87,7 @@ impl Uploader {
Ok(Self {
client: crate::observability::instrument_client(client, "ingestion"),
endpoint: config.endpoint(INGESTION_PATH),
- service_key: config.service_key().to_owned(),
+ service_key: config.service_key().clone(),
})
}
@@ -98,12 +99,13 @@ impl Uploader {
.checked_sub(now)
.filter(|duration| !duration.is_zero())
.ok_or(ExportError::Expired)?;
- let mut authorization = HeaderValue::from_str(&format!("Bearer {}", grant.access_token))
- .map_err(|_| ExportError::Payload)?;
+ let mut authorization =
+ HeaderValue::from_str(&format!("Bearer {}", grant.access_token.expose_secret()))
+ .map_err(|_| ExportError::Payload)?;
authorization.set_sensitive(true);
let mut signature = HeaderValue::from_str(&signing::authorization(
- &self.service_key,
- &grant.access_token,
+ self.service_key.expose_secret(),
+ grant.access_token.expose_secret(),
now.as_secs(),
))
.map_err(|_| ExportError::Payload)?;
diff --git a/ai-gateway/src/telemetry/tests.rs b/ai-gateway/src/telemetry/tests.rs
index f8979d3e52..77f0fd79ea 100644
--- a/ai-gateway/src/telemetry/tests.rs
+++ b/ai-gateway/src/telemetry/tests.rs
@@ -164,7 +164,7 @@ async fn concurrent_projects_keep_their_original_grants_and_attribution() {
for project in ["project-a", "project-b"] {
let mut grant = grant().await;
grant.grant.project_id = project.into();
- grant.grant.access_token = format!("token-{project}");
+ grant.grant.access_token = format!("token-{project}").into();
telemetry.record(grant, facts(project));
}
telemetry
@@ -304,8 +304,8 @@ fn record_bytes(context: &DeliveryContext, facts: InferenceFacts) -> usize {
serde_json::to_vec(&mapping::span(facts, &context.generation))
.unwrap()
.len()
- + context.grant.access_token.len()
- + context.grant.project_id.len()
+ + context.grant.access_token.expose_secret().len()
+ + context.grant.project_id.as_str().len()
}
fn exported_spans(payload: &Value) -> usize {
@@ -418,7 +418,7 @@ async fn interleaved_projects_never_share_an_upload_or_a_grant() {
for project in ["project-a", "project-b"] {
let mut context = grant().await;
context.grant.project_id = project.into();
- context.grant.access_token = format!("{project}-token-{round}");
+ context.grant.access_token = format!("{project}-token-{round}").into();
context.grant.expires_at += round * 10;
telemetry.record(context, facts(project));
}
diff --git a/ai-gateway/src/transport/mod.rs b/ai-gateway/src/transport/mod.rs
index 754d3167f6..0fbd5cc235 100644
--- a/ai-gateway/src/transport/mod.rs
+++ b/ai-gateway/src/transport/mod.rs
@@ -1,6 +1,5 @@
//! Bounded byte relay. One task owns upstream reads; the body owns its lifetime.
use std::{
- fmt,
pin::Pin,
sync::{
Arc, Mutex,
@@ -26,26 +25,18 @@ use tokio::{
};
/// Sanitized failures; upstream URLs, bodies and credentials are never retained.
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
pub enum ProviderError {
+ #[error("invalid provider configuration")]
Configuration,
+ #[error("gateway execution capacity exhausted")]
Busy,
+ #[error("provider deadline exceeded")]
Timeout,
+ #[error("provider transport failed")]
Transport,
}
-impl fmt::Display for ProviderError {
- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- f.write_str(match self {
- Self::Configuration => "invalid provider configuration",
- Self::Busy => "gateway execution capacity exhausted",
- Self::Timeout => "provider deadline exceeded",
- Self::Transport => "provider transport failed",
- })
- }
-}
-impl std::error::Error for ProviderError {}
-
// An allowlist prevents gateway credentials, tenant routing overrides and cookies
// from crossing the boundary. Headers nominated by `Connection` are never end-to-end.
// Prefixes admit a provider's own evolving header family without naming each member.