Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(rust-proxy): forward request paths verbatim and pin the rustls provider (#3853)
Tejas Chopra9d98ea59082c
diff --git a/Cargo.lock b/Cargo.lockindex f0c9b955a..87429c2f7 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -182,6 +182,45 @@ dependencies = ["stable_deref_trait",]+[[package]]+name = "asn1-rs"+version = "0.7.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8"+dependencies = [+ "asn1-rs-derive",+ "asn1-rs-impl",+ "displaydoc",+ "nom 7.1.3",+ "num-traits",+ "rusticata-macros",+ "thiserror 2.0.20",+ "time",+]++[[package]]+name = "asn1-rs-derive"+version = "0.6.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 2.0.118",+ "synstructure",+]++[[package]]+name = "asn1-rs-impl"+version = "0.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 2.0.118",+]+[[package]]name = "assert-json-diff"version = "2.0.2"@@ -733,6 +772,12 @@ version = "0.22.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"+[[package]]+name = "base64"+version = "0.23.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"+[[package]]name = "base64-simd"version = "0.8.0"@@ -749,7 +794,7 @@ version = "0.8.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3"dependencies = [- "bit-vec",+ "bit-vec 0.8.0",][[package]]@@ -758,6 +803,15 @@ version = "0.8.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7"+[[package]]+name = "bit-vec"+version = "0.9.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51"+dependencies = [+ "serde",+]+[[package]]name = "bit_field"version = "0.10.3"@@ -1343,6 +1397,20 @@ version = "0.1.4"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b"+[[package]]+name = "der-parser"+version = "10.0.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6"+dependencies = [+ "asn1-rs",+ "displaydoc",+ "nom 7.1.3",+ "num-bigint",+ "num-traits",+ "rusticata-macros",+]+[[package]]name = "deranged"version = "0.5.8"@@ -1962,6 +2030,7 @@ dependencies = ["pin-project-lite","prometheus","proptest",+ "rcgen","reqwest","rustls","rustls-native-certs",@@ -1971,6 +2040,7 @@ dependencies = ["sha2","thiserror 2.0.20","tokio",+ "tokio-rustls","tokio-stream","tokio-tungstenite 0.30.0","tokio-util",@@ -2930,6 +3000,15 @@ dependencies = ["libc",]+[[package]]+name = "oid-registry"+version = "0.8.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7"+dependencies = [+ "asn1-rs",+]+[[package]]name = "once_cell"version = "1.21.4"@@ -3052,6 +3131,16 @@ version = "0.1.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "35fb2e5f958ec131621fdd531e9fc186ed768cbe395337403ae56c17a74c68ec"+[[package]]+name = "pem"+version = "4.0.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d354a98a3d1251555de99e8fdd8afda05573c31b82f59063a7b0a29b5527f120"+dependencies = [+ "base64 0.23.1",+ "serde_core",+]+[[package]]name = "percent-encoding"version = "2.3.2"@@ -3227,7 +3316,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744"dependencies = ["bit-set",- "bit-vec",+ "bit-vec 0.8.0","bitflags","num-traits","rand 0.9.4",@@ -3552,6 +3641,20 @@ dependencies = ["crossbeam-utils",]+[[package]]+name = "rcgen"+version = "0.14.10"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8774e05a7d0de114588e6a28fe7e71694b82614ed569d86d8b389dfbc98b8ad8"+dependencies = [+ "pem",+ "ring",+ "rustls-pki-types",+ "time",+ "x509-parser",+ "yasna",+]+[[package]]name = "redis"version = "1.7.0"@@ -3726,6 +3829,15 @@ dependencies = ["semver",]+[[package]]+name = "rusticata-macros"+version = "4.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632"+dependencies = [+ "nom 7.1.3",+]+[[package]]name = "rustix"version = "1.1.4"@@ -5356,6 +5468,24 @@ version = "0.6.3"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4"+[[package]]+name = "x509-parser"+version = "0.18.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202"+dependencies = [+ "asn1-rs",+ "data-encoding",+ "der-parser",+ "lazy_static",+ "nom 7.1.3",+ "oid-registry",+ "ring",+ "rusticata-macros",+ "thiserror 2.0.20",+ "time",+]+[[package]]name = "xmlparser"version = "0.13.6"@@ -5374,6 +5504,16 @@ version = "0.8.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "7a5a4b21e1a62b67a2970e6831bc091d7b87e119e7f9791aef9702e3bef04448"+[[package]]+name = "yasna"+version = "0.6.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b5f6765e852b9b4dc8e2a76843e4d64d1cea8e79bcde0b6901aea8e7c7f08282"+dependencies = [+ "bit-vec 0.9.1",+ "time",+]+[[package]]name = "yoke"version = "0.8.3"diff --git a/Cargo.toml b/Cargo.tomlindex 3fa9dbbb9..c0effd2fa 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -116,6 +116,15 @@ strip = "symbols"lto = "thin"codegen-units = 1+# The proxy binary turns untrusted network input into byte and token counts;+# a wrapping add there is a silent correctness bug at best, so its release+# build keeps overflow checks (a panic is caught per task by tokio and+# surfaces in the log; a wrong number does not). Scoped to this package:+# `headroom-core` is also the `_core` Python extension's compression hot+# path and is not included until the cost is benchmarked there.+[profile.release.package.headroom-proxy]+overflow-checks = true+# Fast-to-compile profile for CI test wheels. The shipped wheel uses# `release` (lto + codegen-units=1) for runtime/size; CI only needs a working# extension, so trade runtime perf for ~parallel, lto-free compilation. Useddiff --git a/crates/headroom-core/src/lib.rs b/crates/headroom-core/src/lib.rsindex a9417bd49..76d4b4daf 100644--- a/crates/headroom-core/src/lib.rs+++ b/crates/headroom-core/src/lib.rs@@ -1,5 +1,9 @@//! headroom-core: foundation crate for the Rust port of Headroom.+// This crate has never needed `unsafe`; keep it that way. The FFI shim lives+// in `headroom-py`, which is deliberately not under this lint.+#![forbid(unsafe_code)]+pub mod auth_mode;pub mod cache_control;pub mod ccr;diff --git a/crates/headroom-parity/src/lib.rs b/crates/headroom-parity/src/lib.rsindex a3d4c101d..adb5559cc 100644--- a/crates/headroom-parity/src/lib.rs+++ b/crates/headroom-parity/src/lib.rs@@ -6,6 +6,8 @@//! stubs and report `Skipped` — see the `stub_comparator!` block below for what//! each is waiting on.+#![forbid(unsafe_code)]+use anyhow::{bail, Context, Result};use serde::{Deserialize, Serialize};use std::fs;diff --git a/crates/headroom-proxy/Cargo.toml b/crates/headroom-proxy/Cargo.tomlindex 39c1e9f57..f664c4129 100644--- a/crates/headroom-proxy/Cargo.toml+++ b/crates/headroom-proxy/Cargo.toml@@ -107,6 +107,15 @@ webpki-roots = "0.26"[dev-dependencies]tower = { workspace = true, features = ["util"] }wiremock = "0.6"+# `wss://` relay test (tests/integration_ws_tls.rs): a local TLS WebSocket+# echo upstream. `tokio-rustls` is already in the build via reqwest and+# hyper-rustls at this same version, so no new crate enters the tree; the+# `ring` feature matches the provider the proxy installs (src/tls.rs).+tokio-rustls = { version = "0.26", default-features = false, features = ["ring", "tls12"] }+# Throwaway certificate for that upstream, generated at test time so no+# private key is ever committed (gitleaks would rightly refuse one). rcgen+# is the generator rustls' own test-suite uses; dev-only, never shipped.+rcgen = { version = "0.14", default-features = false, features = ["ring", "pem"] }reqwest = { version = "0.12", default-features = false, features = ["stream", "rustls-tls", "http2", "json"] }tokio-tungstenite = { version = "0.30", default-features = false, features = ["connect", "rustls-tls-webpki-roots"] }futures-util = "0.3"diff --git a/crates/headroom-proxy/src/bedrock/invoke.rs b/crates/headroom-proxy/src/bedrock/invoke.rsindex a30e0027a..6f0030f2b 100644--- a/crates/headroom-proxy/src/bedrock/invoke.rs+++ b/crates/headroom-proxy/src/bedrock/invoke.rs@@ -57,6 +57,8 @@ use crate::compression::{use crate::headers::filter_response_headers;use crate::observability::{observe_bedrock_invoke_latency, record_bedrock_invoke};use crate::proxy::AppState;+use crate::upstream_path::append_segments;+use crate::ProxyError;// Phase F PR-F1 + PR-D3: the bedrock auth-mode layer// (`classify_and_attach_auth_mode`) populates `request.extensions()`// with `AuthMode` BEFORE this handler runs. We extract it via@@ -212,7 +214,21 @@ pub async fn handle_invoke(// region-derived default.let upstream_url = match build_bedrock_upstream(&state, &model_id, &uri, action) {Ok(u) => u,- Err(msg) => {+ // The client sent a model id the proxy would have to rewrite+ // (dot segments, control characters): its fault, 400, and the+ // signer never sees the request.+ Err(ProxyError::InvalidPath(msg)) => {+ tracing::warn!(+ event = "bedrock_path_rejected",+ request_id = %request_id,+ model_id = %model_id,+ error = %msg,+ "bedrock invoke: refusing to forward a model id that would rewrite the upstream path"+ );+ return error_response(StatusCode::BAD_REQUEST, "bedrock_path_rejected", &msg);+ }+ Err(e) => {+ let msg = e.to_string();tracing::error!(event = "bedrock_endpoint_invalid",request_id = %request_id,@@ -475,34 +491,30 @@ fn run_anthropic_compression(}/// Build the upstream URL for the Bedrock route. Honours the-/// operator-supplied `bedrock_endpoint` first, falling back to the-/// region-derived default. The path/query portion is taken from the-/// original URI verbatim — Bedrock's path schema (`/model/{id}/{action}`)-/// is identical to the proxy's external path.+/// operator-supplied `bedrock_endpoint` first (including any path prefix+/// on it), falling back to the region-derived default. The path is rebuilt+/// segment by segment — `/model/{model_id}/{action}` — from the decoded+/// route parameter, so a model id containing `/` (an inference-profile+/// ARN) is re-encoded as one `%2F` segment, and a model id that would+/// change the path's shape (`..`, empty, control characters) is rejected+/// with `ProxyError::InvalidPath` (400). See `crate::upstream_path`.fn build_bedrock_upstream(state: &AppState,model_id: &str,uri: &Uri,action: &str,-) -> Result<Url, String> {+) -> Result<Url, ProxyError> {let base = match state.config.bedrock_endpoint.as_ref() {Some(u) => u.clone(),None => {let host =BEDROCK_RUNTIME_HOST_TEMPLATE.replace("{region}", &state.config.bedrock_region);- Url::parse(&format!("https://{host}/"))- .map_err(|e| format!("bedrock derived base URL parse error: {e}"))?+ Url::parse(&format!("https://{host}/")).map_err(|e| {+ ProxyError::InvalidUpstream(format!("bedrock derived base URL parse error: {e}"))+ })?}};- // Compose the path. We trust the captured `model_id` (Axum- // already URL-decoded it) and append `/{action}`.- let path = format!(- "/model/{model_id}/{action}",- model_id = model_id,- action = action,- );- let mut joined = base;- joined.set_path(&path);+ let mut joined = append_segments(&base, ["model", model_id, action])?;if let Some(q) = uri.query() {joined.set_query(Some(q));}diff --git a/crates/headroom-proxy/src/bedrock/invoke_streaming.rs b/crates/headroom-proxy/src/bedrock/invoke_streaming.rsindex 62501003c..64e0f5d2f 100644--- a/crates/headroom-proxy/src/bedrock/invoke_streaming.rs+++ b/crates/headroom-proxy/src/bedrock/invoke_streaming.rs@@ -69,6 +69,8 @@ use crate::observability::{observe_bedrock_invoke_latency, record_bedrock_eventstream_message, record_bedrock_invoke,};use crate::proxy::AppState;+use crate::upstream_path::append_segments;+use crate::ProxyError;// Phase F PR-F1 + PR-D3: pre-classified by `classify_and_attach_auth_mode`// middleware on the bedrock router; we read it back via the// `Extension<AuthMode>` extractor.@@ -187,7 +189,20 @@ pub async fn handle_invoke_streaming(let upstream_url = match build_bedrock_streaming_upstream(&state, &model_id, &uri, action) {Ok(u) => u,- Err(msg) => {+ // Client-caused: a model id the proxy would have to rewrite. 400,+ // and nothing is signed or forwarded.+ Err(ProxyError::InvalidPath(msg)) => {+ tracing::warn!(+ event = "bedrock_path_rejected",+ request_id = %request_id,+ model_id = %model_id,+ error = %msg,+ "bedrock invoke-streaming: refusing to forward a model id that would rewrite the upstream path"+ );+ return error_response(StatusCode::BAD_REQUEST, "bedrock_path_rejected", &msg);+ }+ Err(e) => {+ let msg = e.to_string();tracing::error!(event = "bedrock_endpoint_invalid",request_id = %request_id,@@ -905,28 +920,27 @@ fn run_anthropic_compression(}}+/// Streaming twin of `invoke::build_bedrock_upstream`: same base+/// resolution, same segment-safe path construction (see+/// `crate::upstream_path`), so an ARN model id stays one `%2F` segment+/// and a traversal attempt is rejected with `ProxyError::InvalidPath`.fn build_bedrock_streaming_upstream(state: &AppState,model_id: &str,uri: &Uri,action: &str,-) -> Result<Url, String> {+) -> Result<Url, ProxyError> {let base = match state.config.bedrock_endpoint.as_ref() {Some(u) => u.clone(),None => {let host =BEDROCK_RUNTIME_HOST_TEMPLATE.replace("{region}", &state.config.bedrock_region);- Url::parse(&format!("https://{host}/"))- .map_err(|e| format!("bedrock derived base URL parse error: {e}"))?+ Url::parse(&format!("https://{host}/")).map_err(|e| {+ ProxyError::InvalidUpstream(format!("bedrock derived base URL parse error: {e}"))+ })?}};- let path = format!(- "/model/{model_id}/{action}",- model_id = model_id,- action = action,- );- let mut joined = base;- joined.set_path(&path);+ let mut joined = append_segments(&base, ["model", model_id, action])?;if let Some(q) = uri.query() {joined.set_query(Some(q));}diff --git a/crates/headroom-proxy/src/error.rs b/crates/headroom-proxy/src/error.rsindex 646548a18..61cf8a2bc 100644--- a/crates/headroom-proxy/src/error.rs+++ b/crates/headroom-proxy/src/error.rs@@ -28,6 +28,14 @@ pub enum ProxyError {#[error("request body exceeds configured limit: {0}")]PayloadTooLarge(String),+ /// The request path, or a decoded route parameter, would have to be+ /// rewritten on the way upstream (dot segments, backslashes, or a+ /// segment that would gain or lose a `/` boundary). The proxy forwards+ /// paths verbatim or not at all — see `crate::upstream_path`. RFC 7231+ /// §6.5.1: 400.+ #[error("request path rejected: {0}")]+ InvalidPath(String),+/// Surfaced when `--compression` is enabled but the proxy can't/// build the IntelligentContextManager at startup (e.g. the/// embedded tokenizer asset failed to initialize). Bubbles up to@@ -39,6 +47,12 @@ pub enum ProxyError {CompressionStartup(String),}+impl From<crate::upstream_path::PathError> for ProxyError {+ fn from(e: crate::upstream_path::PathError) -> Self {+ ProxyError::InvalidPath(e.to_string())+ }+}+impl IntoResponse for ProxyError {fn into_response(self) -> Response {let (status, msg) = match &self {@@ -54,6 +68,7 @@ impl IntoResponse for ProxyError {ProxyError::InvalidUpstream(_) => (StatusCode::BAD_GATEWAY, self.to_string()),ProxyError::InvalidHeader(_) => (StatusCode::BAD_REQUEST, self.to_string()),ProxyError::PayloadTooLarge(_) => (StatusCode::PAYLOAD_TOO_LARGE, self.to_string()),+ ProxyError::InvalidPath(_) => (StatusCode::BAD_REQUEST, self.to_string()),ProxyError::WebSocket(_) => (StatusCode::BAD_GATEWAY, self.to_string()),ProxyError::Io(_) => (StatusCode::INTERNAL_SERVER_ERROR, self.to_string()),// CompressionStartup is a startup-time error, not adiff --git a/crates/headroom-proxy/src/lib.rs b/crates/headroom-proxy/src/lib.rsindex a4e29f98a..a835ef069 100644--- a/crates/headroom-proxy/src/lib.rs+++ b/crates/headroom-proxy/src/lib.rs@@ -1,6 +1,8 @@//! headroom-proxy library: transparent reverse proxy in front of the Python//! Headroom proxy. Used by both `main.rs` and the integration tests.+#![forbid(unsafe_code)]+pub mod bedrock;pub mod cache_stabilization;pub mod compression;@@ -14,6 +16,7 @@ pub mod proxy;pub mod responses_items;pub mod sse;pub mod tls;+pub mod upstream_path;pub mod vertex;pub mod websocket;diff --git a/crates/headroom-proxy/src/main.rs b/crates/headroom-proxy/src/main.rsindex f21d0117f..005cd0f90 100644--- a/crates/headroom-proxy/src/main.rs+++ b/crates/headroom-proxy/src/main.rs@@ -4,6 +4,8 @@//! port; this binary forwards every HTTP/SSE/WebSocket request verbatim to//! `--upstream`. See RUST_DEV.md for the operator runbook.+#![forbid(unsafe_code)]+use std::net::SocketAddr;use clap::Parser;diff --git a/crates/headroom-proxy/src/proxy.rs b/crates/headroom-proxy/src/proxy.rsindex 650200f86..725f455fd 100644--- a/crates/headroom-proxy/src/proxy.rs+++ b/crates/headroom-proxy/src/proxy.rs@@ -97,6 +97,12 @@ const DRIFT_DETECTOR_CAPACITY: usize = 1000;impl AppState {pub fn new(config: Config) -> Result<Self, ProxyError> {+ // Two rustls crypto providers are linked into this binary (ring via+ // reqwest, aws-lc-rs via the AWS SDK), so anything that reaches for+ // the process default — reqwest below, tokio-tungstenite, any future+ // dependency — would panic without one. Pin it before the first TLS+ // client is built. Idempotent; see `crate::tls`.+ crate::tls::install_process_crypto_provider();let mut builder = reqwest::Client::builder();// Corporate roots handed over as PEM files, added on top of the// bundled + OS roots (see `crate::tls`).@@ -379,30 +385,21 @@ fn is_websocket_upgrade(headers: &HeaderMap) -> bool {/// Build the upstream URL by joining the configured base with the incoming/// path-and-query. Preserves '?' and the query string verbatim.pub(crate) fn build_upstream_url(base: &url::Url, uri: &Uri) -> Result<url::Url, ProxyError> {- Ok(join_upstream_path(base, uri.path(), uri.query()))+ join_upstream_path(base, uri.path(), uri.query())}/// Shared path-join helper used by HTTP and WebSocket handlers.-/// Appends `path` to `base`, preserving any base path prefix, then sets `query`.-pub(crate) fn join_upstream_path(base: &url::Url, path: &str, query: Option<&str>) -> url::Url {- let mut joined = base.clone();- // Strip trailing slash from base path so "http://x:1/api" + "/v1/foo"- // yields "http://x:1/api/v1/foo" rather than "http://x:1/v1/foo".- let base_path = joined.path().trim_end_matches('/').to_string();- let combined = if path.is_empty() || path == "/" {- if base_path.is_empty() {- "/".to_string()- } else {- base_path- }- } else if base_path.is_empty() {- path.to_string()- } else {- format!("{base_path}{path}")- };- joined.set_path(&combined);- joined.set_query(query);- joined+/// Appends `path` to `base`, preserving any base path prefix, then sets+/// `query`. Rejects (400) any path the URL parser would rewrite — dot+/// segments, backslashes — so the upstream always receives exactly the path+/// the client sent, under exactly the configured prefix. See+/// [`crate::upstream_path`].+pub(crate) fn join_upstream_path(+ base: &url::Url,+ path: &str,+ query: Option<&str>,+) -> Result<url::Url, ProxyError> {+ Ok(crate::upstream_path::join_request_path(base, path, query)?)}/// Forward an HTTP request to the upstream and stream the response back.@@ -1699,4 +1696,14 @@ mod tests {let out = build_upstream_url(&base, &uri).unwrap();assert_eq!(out.as_str(), "http://up:8080/");}++ #[test]+ fn url_build_refuses_to_escape_base_path() {+ // `url::Url::set_path` would resolve this to `/tenant-b/v1/messages`.+ let base: url::Url = "http://gw/tenant-a".parse().unwrap();+ let uri: Uri = "/../tenant-b/v1/messages".parse().unwrap();+ let err = build_upstream_url(&base, &uri).unwrap_err();+ assert!(matches!(err, ProxyError::InvalidPath(_)), "{err:?}");+ assert_eq!(err.into_response().status(), StatusCode::BAD_REQUEST);+ }}diff --git a/crates/headroom-proxy/src/tls.rs b/crates/headroom-proxy/src/tls.rsindex eb23982ec..057b3fc17 100644--- a/crates/headroom-proxy/src/tls.rs+++ b/crates/headroom-proxy/src/tls.rs@@ -35,6 +35,28 @@ pub fn extra_root_certificates() -> Vec<reqwest::Certificate> {.collect()}+/// Install ring as the process-wide rustls [`CryptoProvider`].+///+/// This binary links two providers — ring (reqwest, tokio-tungstenite) and+/// aws-lc-rs (the AWS SDK behind `aws-config`). rustls refuses to guess+/// between them: `ClientConfig::builder()` panics unless a default has been+/// installed, and reqwest reaches for that default when it builds its TLS+/// client. Installing ring explicitly makes the choice deliberate and keeps+/// every TLS client in the process on one provider — the same one+/// [`websocket_tls_config`] names. Idempotent: a second call (tests, embedded+/// use) is a no-op.+///+/// ring rather than aws-lc-rs because it is the provider reqwest already+/// defaults to, needs no C toolchain or cmake at build time, and the AWS SDK+/// carries its own explicit configuration either way.+///+/// [`CryptoProvider`]: rustls::crypto::CryptoProvider+pub fn install_process_crypto_provider() {+ // `Err` means a provider is already installed, which is exactly the+ // state we want.+ let _ = rustls::crypto::ring::default_provider().install_default();+}+/// rustls client config for `wss://` upstreams: webpki + OS + additive roots,/// on an explicit crypto provider. Built once per process.pub fn websocket_tls_config() -> Arc<rustls::ClientConfig> {diff --git a/crates/headroom-proxy/src/upstream_path.rs b/crates/headroom-proxy/src/upstream_path.rsnew file mode 100644index 000000000..17d3e80e2--- /dev/null+++ b/crates/headroom-proxy/src/upstream_path.rs@@ -0,0 +1,299 @@+//! Upstream path construction that never rewrites what the client asked for.+//!+//! A reverse proxy has one invariant on the request line: the path it sends+//! upstream is the operator-configured base prefix followed by exactly the+//! path the client sent. Two things break that invariant silently:+//!+//! * [`url::Url::set_path`] normalises dot segments (`.` and `..`, in any+//! percent-encoded spelling) and folds `\` into `/`. A client could walk+//! out of a tenant prefix (`https://gw/tenant-a` + `/../tenant-b/...`) or,+//! on the provider routes, move a SigV4- or ADC-signed request to a+//! different resource (`/model/../../other/invoke`).+//! * Route parameters arrive percent-decoded from axum. Interpolating them+//! back into a path string re-parses `/` as a segment separator, so a model+//! id that legitimately contains `/` (Bedrock inference-profile ARNs) is+//! split into two segments, and a hostile one gains segments.+//!+//! Every upstream URL in the proxy goes through this module: raw request+//! paths through [`join_request_path`], decoded route parameters through+//! [`append_segments`]. Anything the module would have to rewrite is+//! rejected instead, and the caller answers the client with 400.++use url::Url;++/// Why a path could not be forwarded unchanged.+#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]+pub enum PathError {+ /// A `.` or `..` segment (including `%2e` spellings). The proxy does not+ /// resolve these; the client must send the path it means.+ #[error("path contains a dot segment {0:?}; the proxy does not normalise paths")]+ DotSegment(String),+ /// A backslash, which the URL standard folds into `/` for http(s).+ #[error("path contains a backslash; the proxy does not normalise paths")]+ Backslash,+ /// A decoded route parameter was empty.+ #[error("path segment is empty")]+ EmptySegment,+ /// A decoded route parameter carried a control character.+ #[error("path segment contains a control character")]+ ControlCharacter,+ /// Post-condition failure: the joined path no longer sits under the+ /// configured base path. Guards against future changes to the join+ /// logic; unreachable while [`validate_request_path`] runs first.+ #[error("joined path {joined:?} escapes the configured base path {base:?}")]+ EscapesBase { base: String, joined: String },+ /// The upstream URL is not hierarchical (`mailto:`-style), so it cannot+ /// carry path segments. Not reachable with http(s)/ws(s) upstreams.+ #[error("upstream URL cannot carry a path")]+ CannotBeABase,+}++/// True when a raw (still percent-encoded) segment is a dot segment under+/// the WHATWG URL standard, which treats `%2e` as `.` for this purpose.+fn is_dot_segment(segment: &str) -> bool {+ matches!(+ segment.to_ascii_lowercase().as_str(),+ "." | ".." | "%2e" | ".%2e" | "%2e." | "%2e%2e"+ )+}++/// Reject a raw request path that `Url::set_path` would rewrite.+///+/// `path` is the request-target path exactly as received (percent-encoded,+/// no query). Only structural rewrites are rejected — dot segments and+/// backslashes; ordinary percent-encoding of reserved characters is fine+/// because it does not change which resource the path names.+pub fn validate_request_path(path: &str) -> Result<(), PathError> {+ if path.contains('\\') {+ return Err(PathError::Backslash);+ }+ if let Some(bad) = path.split('/').find(|segment| is_dot_segment(segment)) {+ return Err(PathError::DotSegment(bad.to_string()));+ }+ Ok(())+}++/// Append the client's raw request path (and query) to the configured+/// upstream base, preserving any base path prefix.+///+/// `"http://x:1/api"` + `"/v1/foo"` → `"http://x:1/api/v1/foo"`. The result+/// is guaranteed to start with the base path: the input is validated first+/// and the output is checked afterwards.+pub fn join_request_path(base: &Url, path: &str, query: Option<&str>) -> Result<Url, PathError> {+ validate_request_path(path)?;++ let mut joined = base.clone();+ let base_path = joined.path().trim_end_matches('/').to_string();+ let combined = if path.is_empty() || path == "/" {+ if base_path.is_empty() {+ "/".to_string()+ } else {+ base_path.clone()+ }+ } else if base_path.is_empty() {+ path.to_string()+ } else {+ format!("{base_path}{path}")+ };+ joined.set_path(&combined);+ joined.set_query(query);++ if !base_path.is_empty() {+ let got = joined.path();+ if got != base_path && !got.starts_with(&format!("{base_path}/")) {+ return Err(PathError::EscapesBase {+ base: base_path,+ joined: got.to_string(),+ });+ }+ }+ Ok(joined)+}++/// Reject a decoded route parameter that cannot be a single path segment.+///+/// `/` is allowed: [`append_segments`] percent-encodes it, so a Bedrock+/// inference-profile ARN stays one segment on the wire (`%2F`), exactly as+/// the AWS SDKs send it.+pub fn validate_segment(segment: &str) -> Result<(), PathError> {+ if segment.is_empty() {+ return Err(PathError::EmptySegment);+ }+ if matches!(segment, "." | "..") {+ return Err(PathError::DotSegment(segment.to_string()));+ }+ if segment.chars().any(char::is_control) {+ return Err(PathError::ControlCharacter);+ }+ Ok(())+}++/// Append already-decoded segments to `base` as individual path segments,+/// percent-encoding each one so it stays a single segment on the wire.+///+/// Keeps any path prefix on `base` (`https://gw/prefix/` + `["model", id,+/// "invoke"]` → `/prefix/model/<id>/invoke`), never produces a double slash,+/// and refuses segments that would change the path's shape.+pub fn append_segments<'a, I>(base: &Url, segments: I) -> Result<Url, PathError>+where+ I: IntoIterator<Item = &'a str>,+{+ let mut joined = base.clone();+ {+ let mut path = joined+ .path_segments_mut()+ .map_err(|()| PathError::CannotBeABase)?;+ path.pop_if_empty();+ for segment in segments {+ validate_segment(segment)?;+ path.push(segment);+ }+ }+ Ok(joined)+}++#[cfg(test)]+mod tests {+ use super::*;++ fn base(s: &str) -> Url {+ Url::parse(s).expect("test base url")+ }++ #[test]+ fn join_preserves_base_prefix_and_query() {+ let out =+ join_request_path(&base("http://up:8080/api"), "/v1/messages", Some("a=1")).unwrap();+ assert_eq!(out.as_str(), "http://up:8080/api/v1/messages?a=1");+ }++ #[test]+ fn join_root_forms() {+ assert_eq!(+ join_request_path(&base("http://up:8080/"), "/", None)+ .unwrap()+ .as_str(),+ "http://up:8080/"+ );+ assert_eq!(+ join_request_path(&base("http://up:8080/api/"), "/", None)+ .unwrap()+ .as_str(),+ "http://up:8080/api"+ );+ assert_eq!(+ join_request_path(&base("http://up:8080"), "/v1", None)+ .unwrap()+ .as_str(),+ "http://up:8080/v1"+ );+ }++ #[test]+ fn join_rejects_every_dot_segment_spelling() {+ for path in [+ "/../tenant-b/v1",+ "/v1/./x",+ "/v1/%2e%2e/x",+ "/v1/%2E%2E/x",+ "/v1/.%2e/x",+ "/v1/%2e./x",+ "/v1/%2e/x",+ "/..",+ ] {+ let err = join_request_path(&base("http://gw/tenant-a"), path, None).unwrap_err();+ assert!(matches!(err, PathError::DotSegment(_)), "{path}: {err:?}");+ }+ }++ #[test]+ fn join_rejects_backslash() {+ let err =+ join_request_path(&base("http://gw/tenant-a"), "/..\\tenant-b", None).unwrap_err();+ assert_eq!(err, PathError::Backslash);+ }++ #[test]+ fn join_rejects_dot_segments_even_without_a_base_prefix() {+ // No prefix to escape, but the upstream would still receive a+ // different path than the client sent (`/x` instead of `/a/../x`).+ let err = join_request_path(&base("http://up"), "/a/../x", None).unwrap_err();+ assert!(matches!(err, PathError::DotSegment(_)));+ }++ #[test]+ fn join_keeps_encoded_slashes_as_one_segment() {+ // `..%2F..` is not a dot segment; it stays a literal segment upstream.+ let out = join_request_path(&base("http://gw/tenant-a"), "/v1/..%2F..%2Fx", None).unwrap();+ assert_eq!(out.path(), "/tenant-a/v1/..%2F..%2Fx");+ }++ #[test]+ fn append_encodes_slashes_inside_a_segment() {+ let arn =+ "arn:aws:bedrock:us-east-1:123456789012:inference-profile/us.anthropic.claude-3-5";+ let out = append_segments(+ &base("https://bedrock-runtime.us-east-1.amazonaws.com/"),+ ["model", arn, "invoke"],+ )+ .unwrap();+ assert_eq!(+ out.as_str(),+ "https://bedrock-runtime.us-east-1.amazonaws.com/model/arn:aws:bedrock:us-east-1:123456789012:inference-profile%2Fus.anthropic.claude-3-5/invoke"+ );+ }++ #[test]+ fn append_keeps_base_prefix_without_double_slash() {+ let out = append_segments(&base("https://gw/prefix/"), ["model", "m", "invoke"]).unwrap();+ assert_eq!(out.path(), "/prefix/model/m/invoke");+ let out = append_segments(&base("https://gw"), ["model", "m", "invoke"]).unwrap();+ assert_eq!(out.path(), "/model/m/invoke");+ }++ #[test]+ fn append_rejects_traversal_and_malformed_segments() {+ let b = base("https://gw/");+ // What `..%2F..%2Fother` decodes to: a `/`-containing segment is+ // fine (it is re-encoded and stays one segment); a bare `..` is not.+ assert_eq!(+ append_segments(&b, ["model", "../../other", "invoke"])+ .unwrap()+ .path(),+ "/model/..%2F..%2Fother/invoke"+ );+ assert!(matches!(+ append_segments(&b, ["model", "..", "invoke"]).unwrap_err(),+ PathError::DotSegment(_)+ ));+ assert!(matches!(+ append_segments(&b, ["model", ".", "invoke"]).unwrap_err(),+ PathError::DotSegment(_)+ ));+ assert_eq!(+ append_segments(&b, ["model", "", "invoke"]).unwrap_err(),+ PathError::EmptySegment+ );+ assert_eq!(+ append_segments(&b, ["model", "m\r\nX-Injected: 1", "invoke"]).unwrap_err(),+ PathError::ControlCharacter+ );+ }++ #[test]+ fn append_does_not_alter_ordinary_model_ids() {+ let out = append_segments(+ &base("https://bedrock-runtime.us-west-2.amazonaws.com/"),+ [+ "model",+ "anthropic.claude-3-haiku-20240307-v1:0",+ "converse",+ ],+ )+ .unwrap();+ assert_eq!(+ out.as_str(),+ "https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-3-haiku-20240307-v1:0/converse"+ );+ }+}diff --git a/crates/headroom-proxy/src/vertex/mod.rs b/crates/headroom-proxy/src/vertex/mod.rsindex 115705e0c..2302fce3c 100644--- a/crates/headroom-proxy/src/vertex/mod.rs+++ b/crates/headroom-proxy/src/vertex/mod.rs@@ -120,6 +120,33 @@ pub async fn handle_vertex_predict_dispatch(.map(|s| s.to_string()).unwrap_or_else(|| uuid::Uuid::new_v4().to_string());+ // The upstream URL is built from the raw request path (which+ // `crate::upstream_path` already refuses to normalise), but the+ // decoded parameters are logged, labelled and handed to the+ // envelope logic. Reject the shapes that could only mean a+ // traversal attempt (`..`, empty, control characters) before any of+ // that happens, so a later refactor that interpolates these values+ // cannot reopen the hole.+ for (name, value) in [+ ("project", project.as_str()),+ ("location", location.as_str()),+ ("model_action", model_action.as_str()),+ ] {+ if let Err(e) = crate::upstream_path::validate_segment(value) {+ tracing::warn!(+ event = "vertex_path_rejected",+ request_id = %request_id,+ segment = name,+ error = %e,+ "vertex path parameter would rewrite the upstream path; refusing"+ );+ return Response::builder()+ .status(StatusCode::BAD_REQUEST)+ .body(Body::from(format!("vertex path: {name} rejected: {e}")))+ .expect("static");+ }+ }+let (model_id, verb_str) = match split_model_action(&model_action) {Some(parts) => parts,None => {diff --git a/crates/headroom-proxy/src/websocket.rs b/crates/headroom-proxy/src/websocket.rsindex 65d9b2b8c..9435f268f 100644--- a/crates/headroom-proxy/src/websocket.rs+++ b/crates/headroom-proxy/src/websocket.rs@@ -8,7 +8,8 @@ use std::net::SocketAddr;use axum::body::Body;use axum::extract::ws::{CloseFrame, Message as AxMsg, WebSocket, WebSocketUpgrade};-use axum::http::{HeaderName, HeaderValue, Request, Response, StatusCode};+use axum::http::{HeaderName, HeaderValue, Request, Response};+use axum::response::IntoResponse;use futures_util::{SinkExt, StreamExt};use tokio_tungstenite::tungstenite::client::IntoClientRequest;use tokio_tungstenite::tungstenite::protocol::CloseFrame as TgCloseFrame;@@ -32,11 +33,13 @@ pub async fn ws_handler(.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());// Build the upstream WS URL.+ // A rejected path (dot segments, backslash) is the client's fault and+ // maps to 400; a bad upstream scheme is the operator's and maps to 502.let upstream_url = match build_upstream_ws_url(&state.config.upstream, req.uri()) {Ok(u) => u,Err(e) => {tracing::warn!(error = %e, "failed to build upstream ws url");- return (StatusCode::BAD_GATEWAY, e).into_response_body();+ return e.into_response();}};@@ -84,30 +87,25 @@ pub async fn ws_handler(})}-trait IntoResponseBody {- fn into_response_body(self) -> Response<Body>;-}-impl IntoResponseBody for (StatusCode, String) {- fn into_response_body(self) -> Response<Body> {- Response::builder()- .status(self.0)- .body(Body::from(self.1))- .unwrap()- }-}--fn build_upstream_ws_url(base: &url::Url, req_uri: &http::Uri) -> Result<url::Url, String> {+fn build_upstream_ws_url(+ base: &url::Url,+ req_uri: &http::Uri,+) -> Result<url::Url, crate::ProxyError> {let mut joined = base.clone();let new_scheme = match joined.scheme() {"http" => "ws","https" => "wss","ws" | "wss" => "ws", // already WS; set_scheme is a no-op but keeps it uniform- other => return Err(format!("unsupported upstream scheme: {other}")),+ other => {+ return Err(crate::ProxyError::InvalidUpstream(format!(+ "unsupported upstream scheme: {other}"+ )))+ }};joined.set_scheme(new_scheme)- .map_err(|()| "failed to set ws scheme".to_string())?;- Ok(join_upstream_path(&joined, req_uri.path(), req_uri.query()))+ .map_err(|()| crate::ProxyError::InvalidUpstream("failed to set ws scheme".to_string()))?;+ join_upstream_path(&joined, req_uri.path(), req_uri.query())}async fn run_ws_pump(diff --git a/crates/headroom-proxy/tests/integration_path_traversal.rs b/crates/headroom-proxy/tests/integration_path_traversal.rsnew file mode 100644index 000000000..08f6b5efb--- /dev/null+++ b/crates/headroom-proxy/tests/integration_path_traversal.rs@@ -0,0 +1,272 @@+//! The proxy forwards the client's path verbatim under the configured+//! prefix, or not at all. `url::Url::set_path` resolves `.`/`..` segments+//! (also spelled `%2e`) and folds `\` into `/`, so without a guard a client+//! could:+//!+//! * walk out of a tenant prefix on the generic forwarder+//! (`https://gw/tenant-a` + `/../tenant-b/...`),+//! * move a SigV4-signed Bedrock request to a different resource+//! (`/model/../invoke`, `/model/..%2F..%2Fother/invoke`),+//! * move an ADC-signed Vertex request to another project+//! (`/v1beta1/projects/../../other/...`).+//!+//! HTTP clients normalise dot segments before sending, so the traversal+//! requests here go over a raw TCP socket. See `headroom_proxy::upstream_path`.++mod common;++use std::net::SocketAddr;++use aws_credential_types::Credentials;+use common::{install_static_token_source, start_proxy, start_proxy_with_state};+use tokio::io::{AsyncReadExt, AsyncWriteExt};+use url::Url;+use wiremock::matchers::any;+use wiremock::{Mock, MockServer, ResponseTemplate};++/// Send one HTTP/1.1 request exactly as written and return the status code.+async fn raw_request(+ addr: SocketAddr,+ method: &str,+ target: &str,+ extra_headers: &[(&str, &str)],+ body: &[u8],+) -> u16 {+ let mut stream = tokio::net::TcpStream::connect(addr).await.unwrap();+ let mut req = format!("{method} {target} HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n");+ for (k, v) in extra_headers {+ req.push_str(&format!("{k}: {v}\r\n"));+ }+ if !body.is_empty() {+ req.push_str("content-type: application/json\r\n");+ req.push_str(&format!("content-length: {}\r\n", body.len()));+ }+ req.push_str("\r\n");+ stream.write_all(req.as_bytes()).await.unwrap();+ stream.write_all(body).await.unwrap();+ let mut out = Vec::new();+ stream.read_to_end(&mut out).await.unwrap();+ let head = String::from_utf8_lossy(&out);+ let status_line = head.lines().next().unwrap_or("");+ status_line+ .split_whitespace()+ .nth(1)+ .and_then(|s| s.parse().ok())+ .unwrap_or_else(|| panic!("no status line in response: {head:?}"))+}++async fn upstream_paths(mock: &MockServer) -> Vec<String> {+ mock.received_requests()+ .await+ .unwrap_or_default()+ .iter()+ .map(|r| r.url.path().to_string())+ .collect()+}++async fn accept_all(mock: &MockServer) {+ Mock::given(any())+ .respond_with(ResponseTemplate::new(200).set_body_string(r#"{"id":"msg_x","content":[]}"#))+ .mount(mock)+ .await;+}++// ─── generic forwarder ───────────────────────────────────────────────────++#[tokio::test]+async fn forwarder_keeps_requests_under_the_base_prefix() {+ let mock = MockServer::start().await;+ accept_all(&mock).await;+ let proxy = start_proxy(&format!("{}/tenant-a", mock.uri())).await;++ // Control: an ordinary path lands under the prefix.+ assert_eq!(+ raw_request(proxy.addr, "GET", "/v1/models", &[], b"").await,+ 200+ );+ assert_eq!(+ upstream_paths(&mock).await,+ vec!["/tenant-a/v1/models".to_string()]+ );++ // Every dot-segment spelling and the backslash are refused before any+ // upstream call. `url` would have resolved these to `/tenant-b/...`.+ for target in [+ "/../tenant-b/v1/models",+ "/v1/../../tenant-b/v1/models",+ "/%2e%2e/tenant-b/v1/models",+ "/.%2e/tenant-b/v1/models",+ "/%2E%2E/tenant-b/v1/models",+ "/..\\tenant-b/v1/models",+ ] {+ let status = raw_request(proxy.addr, "GET", target, &[], b"").await;+ assert_eq!(status, 400, "{target} must be rejected");+ }+ assert_eq!(+ upstream_paths(&mock).await.len(),+ 1,+ "no traversal request may reach the upstream"+ );+ proxy.shutdown().await;+}++#[tokio::test]+async fn websocket_upgrade_with_traversal_path_is_rejected_before_upgrade() {+ let mock = MockServer::start().await;+ let proxy = start_proxy(&format!("{}/tenant-a", mock.uri())).await;+ let status = raw_request(+ proxy.addr,+ "GET",+ "/../tenant-b/ws",+ &[+ ("Upgrade", "websocket"),+ ("Connection", "Upgrade"),+ ("Sec-WebSocket-Key", "dGhlIHNhbXBsZSBub25jZQ=="),+ ("Sec-WebSocket-Version", "13"),+ ],+ b"",+ )+ .await;+ assert_eq!(status, 400, "a traversal path must not be upgraded (101)");+ proxy.shutdown().await;+}++// ─── Bedrock (SigV4-signed) ──────────────────────────────────────────────++fn test_credentials() -> Credentials {+ Credentials::new(+ "AKIAEXAMPLEAKIDFORTEST",+ "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",+ None,+ None,+ "test",+ )+}++async fn bedrock_proxy(mock: &MockServer) -> common::ProxyHandle {+ let endpoint: Url = mock.uri().parse().unwrap();+ start_proxy_with_state(+ &mock.uri(),+ |c| c.bedrock_endpoint = Some(endpoint),+ |s| s.with_bedrock_credentials(test_credentials()),+ )+ .await+}++const BODY: &[u8] =+ br#"{"anthropic_version":"bedrock-2023-05-31","max_tokens":8,"messages":[{"role":"user","content":"hi"}]}"#;++#[tokio::test]+async fn bedrock_dot_segment_model_id_is_rejected_unsigned() {+ let mock = MockServer::start().await;+ accept_all(&mock).await;+ let proxy = bedrock_proxy(&mock).await;++ // `/model/../invoke` used to become a signed request for `/invoke`.+ for target in [+ "/model/../invoke",+ "/model/./converse",+ "/model/%2e%2e/invoke",+ ] {+ let status = raw_request(proxy.addr, "POST", target, &[], BODY).await;+ assert_eq!(status, 400, "{target} must be rejected");+ }+ assert!(+ upstream_paths(&mock).await.is_empty(),+ "no signed request may leave for a rejected model id"+ );+ proxy.shutdown().await;+}++#[tokio::test]+async fn bedrock_encoded_slashes_in_model_id_stay_one_segment() {+ let mock = MockServer::start().await;+ accept_all(&mock).await;+ let proxy = bedrock_proxy(&mock).await;++ // The traversal shape: decoded, this model id is `../../guardrail/...`.+ // Before the fix the signed request went to+ // `/guardrail/GID/version/1/apply/invoke`. Now the id is re-encoded and+ // the upstream sees it as one (nonsensical) model segment.+ let resp = reqwest::Client::new()+ .post(format!(+ "{}/model/..%2F..%2Fguardrail%2FGID%2Fversion%2F1%2Fapply/invoke",+ proxy.url()+ ))+ .header("content-type", "application/json")+ .body(BODY)+ .send()+ .await+ .unwrap();+ assert_ne!(resp.status(), 500);+ let paths = upstream_paths(&mock).await;+ assert!(+ !paths.iter().any(|p| p.starts_with("/guardrail/")),+ "traversal reached the upstream: {paths:?}"+ );+ for p in &paths {+ assert!(p.starts_with("/model/") && p.ends_with("/invoke"), "{p}");+ assert!(+ p.contains("%2F"),+ "slashes inside the model id must stay encoded: {p}"+ );+ }++ // The legitimate case with the same shape: an inference-profile ARN+ // carries a `/`, which the AWS SDKs send as `%2F`. It must arrive as one+ // segment, exactly as sent.+ let arn = "arn:aws:bedrock:us-east-1:123456789012:inference-profile%2Fus.anthropic.claude-3-5-haiku-20241022-v1:0";+ let resp = reqwest::Client::new()+ .post(format!("{}/model/{arn}/invoke", proxy.url()))+ .header("content-type", "application/json")+ .body(BODY)+ .send()+ .await+ .unwrap();+ assert_eq!(resp.status(), 200);+ let paths = upstream_paths(&mock).await;+ assert_eq!(+ paths.last().map(String::as_str),+ Some(format!("/model/{arn}/invoke").as_str())+ );+ proxy.shutdown().await;+}++#[tokio::test]+async fn bedrock_streaming_dot_segment_model_id_is_rejected_unsigned() {+ let mock = MockServer::start().await;+ accept_all(&mock).await;+ let proxy = bedrock_proxy(&mock).await;++ for target in [+ "/model/../invoke-with-response-stream",+ "/model/../converse-stream",+ ] {+ let status = raw_request(proxy.addr, "POST", target, &[], BODY).await;+ assert_eq!(status, 400, "{target} must be rejected");+ }+ assert!(upstream_paths(&mock).await.is_empty());+ proxy.shutdown().await;+}++// ─── Vertex (ADC-signed) ─────────────────────────────────────────────────++#[tokio::test]+async fn vertex_dot_segment_project_is_rejected_unsigned() {+ let mock = MockServer::start().await;+ accept_all(&mock).await;+ let proxy = start_proxy_with_state(+ &mock.uri(),+ |_| {},+ |s| install_static_token_source(s, "test-bearer"),+ )+ .await;++ // Would have resolved to `/evil/locations/...` carrying the operator's+ // ADC bearer.+ let target = "/v1beta1/projects/../../evil/locations/us-central1/publishers/anthropic/models/claude-3-5-sonnet@20240620:rawPredict";+ let status = raw_request(proxy.addr, "POST", target, &[], BODY).await;+ assert_eq!(status, 400);+ assert!(upstream_paths(&mock).await.is_empty());+ proxy.shutdown().await;+}diff --git a/crates/headroom-proxy/tests/integration_ws_tls.rs b/crates/headroom-proxy/tests/integration_ws_tls.rsnew file mode 100644index 000000000..1c75ae5ee--- /dev/null+++ b/crates/headroom-proxy/tests/integration_ws_tls.rs@@ -0,0 +1,165 @@+//! `wss://` relay: an `https://` upstream means the WebSocket pump has to+//! complete a TLS handshake. This binary links two rustls crypto providers+//! (ring via reqwest, aws-lc-rs via the AWS SDK) and rustls panics on+//! `ClientConfig::builder()` when no process-level default is installed —+//! which used to kill every `wss://` relay from inside the pump task, on any+//! client's request. The other ws tests all use plaintext `ws://`; this file+//! is the only one that drives the TLS path end to end.+//!+//! The upstream certificate is generated per test process (rcgen) and+//! handed to the proxy through `HEADROOM_CA_BUNDLE`, the same knob an+//! operator uses for a corporate TLS-inspection root.++mod common;++use std::net::SocketAddr;+use std::sync::{Arc, OnceLock};+use std::time::Duration;++use common::start_proxy;+use futures_util::{SinkExt, StreamExt};+use rustls_pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer};+use tokio::sync::oneshot;+use tokio_tungstenite::tungstenite::Message;++struct TestCert {+ cert: CertificateDer<'static>,+ cert_pem: String,+ key_pkcs8: Vec<u8>,+}++fn self_signed_for_loopback() -> TestCert {+ let ck =+ rcgen::generate_simple_self_signed(vec!["localhost".to_string(), "127.0.0.1".to_string()])+ .expect("self-signed test certificate");+ TestCert {+ cert: ck.cert.der().clone(),+ cert_pem: ck.cert.pem(),+ key_pkcs8: ck.signing_key.serialize_der(),+ }+}++/// The one certificate the proxy trusts in this process. The proxy reads+/// `HEADROOM_CA_BUNDLE` once (at `AppState::new` for HTTP, lazily for the+/// first `wss://` connect), so the variable is set here, before any proxy+/// starts, and never changed again.+fn trusted_cert() -> &'static TestCert {+ static TRUSTED: OnceLock<TestCert> = OnceLock::new();+ TRUSTED.get_or_init(|| {+ let cert = self_signed_for_loopback();+ let dir = std::env::temp_dir().join(format!("headroom-ws-tls-{}", std::process::id()));+ std::fs::create_dir_all(&dir).expect("temp dir");+ let bundle = dir.join("upstream-ca.pem");+ std::fs::write(&bundle, &cert.cert_pem).expect("write bundle");+ std::env::set_var("HEADROOM_CA_BUNDLE", &bundle);+ cert+ })+}++/// A TLS WebSocket echo server on 127.0.0.1, presenting `cert`.+async fn tls_echo_upstream(cert: &TestCert) -> (SocketAddr, oneshot::Sender<()>) {+ let provider = Arc::new(rustls::crypto::ring::default_provider());+ let config = rustls::ServerConfig::builder_with_provider(provider)+ .with_safe_default_protocol_versions()+ .expect("ring supports the default TLS versions")+ .with_no_client_auth()+ .with_single_cert(+ vec![cert.cert.clone()],+ PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(cert.key_pkcs8.clone())),+ )+ .expect("server config");+ let acceptor = tokio_rustls::TlsAcceptor::from(Arc::new(config));++ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();+ let addr = listener.local_addr().unwrap();+ let (stop_tx, mut stop_rx) = oneshot::channel();+ tokio::spawn(async move {+ loop {+ tokio::select! {+ _ = &mut stop_rx => break,+ accepted = listener.accept() => {+ let Ok((stream, _)) = accepted else { continue };+ let acceptor = acceptor.clone();+ tokio::spawn(async move {+ let Ok(tls) = acceptor.accept(stream).await else { return };+ let Ok(ws) = tokio_tungstenite::accept_async(tls).await else { return };+ let (mut sink, mut src) = ws.split();+ while let Some(Ok(msg)) = src.next().await {+ match msg {+ Message::Close(cf) => {+ let _ = sink.send(Message::Close(cf)).await;+ break;+ }+ m => {+ if sink.send(m).await.is_err() {+ break;+ }+ }+ }+ }+ });+ }+ }+ }+ });+ (addr, stop_tx)+}++#[tokio::test]+async fn wss_upstream_relays_frames_over_tls() {+ let (upstream, _stop) = tls_echo_upstream(trusted_cert()).await;+ let proxy = start_proxy(&format!("https://127.0.0.1:{}", upstream.port())).await;++ let (mut ws, _) = tokio_tungstenite::connect_async(format!("{}/v1/realtime", proxy.ws_url()))+ .await+ .expect("client handshake with the proxy");++ ws.send(Message::Text("over-tls".into())).await.unwrap();+ let echoed = tokio::time::timeout(Duration::from_secs(10), ws.next())+ .await+ .expect("echo within 10s: the wss:// pump must not die on connect")+ .expect("stream still open")+ .expect("frame, not a transport error");+ match echoed {+ Message::Text(t) => assert_eq!(t.as_str(), "over-tls"),+ other => panic!("expected the echoed text frame, got {other:?}"),+ }++ ws.send(Message::Close(None)).await.ok();+ proxy.shutdown().await;+}++#[tokio::test]+async fn wss_upstream_with_untrusted_cert_fails_closed_and_proxy_survives() {+ // Make sure the trusted bundle is installed first, so this test's+ // untrusted certificate is genuinely untrusted rather than "no bundle".+ let _ = trusted_cert();+ let untrusted = self_signed_for_loopback();+ let (upstream, _stop) = tls_echo_upstream(&untrusted).await;+ let proxy = start_proxy(&format!("https://127.0.0.1:{}", upstream.port())).await;++ let (mut ws, _) = tokio_tungstenite::connect_async(format!("{}/v1/realtime", proxy.ws_url()))+ .await+ .expect("client handshake with the proxy");+ ws.send(Message::Text("should-not-arrive".into()))+ .await+ .ok();++ // The upstream handshake must fail verification and the relay must end+ // cleanly: no echo, no hang, no panic.+ let outcome = tokio::time::timeout(Duration::from_secs(10), ws.next())+ .await+ .expect("relay must end within 10s when the upstream cert is untrusted");+ match outcome {+ Some(Ok(Message::Text(t))) => panic!("frame relayed to an untrusted upstream: {t}"),+ Some(Ok(Message::Binary(_))) => panic!("frame relayed to an untrusted upstream"),+ _ => {}+ }++ // The proxy is still serving: the failure stayed inside the one relay.+ let health = reqwest::get(format!("{}/healthz", proxy.url()))+ .await+ .unwrap();+ assert_eq!(health.status(), 200);+ proxy.shutdown().await;+}diff --git a/crates/headroom-simulators/src/lib.rs b/crates/headroom-simulators/src/lib.rsindex 11fa4d57e..14c516470 100644--- a/crates/headroom-simulators/src/lib.rs+++ b/crates/headroom-simulators/src/lib.rs@@ -3,6 +3,8 @@//! The crate is intentionally standalone: it behaves like a provider upstream//! that Headroom can proxy to, but it never calls a real LLM service.+#![forbid(unsafe_code)]+pub mod application;pub mod config;pub mod domain;