Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(rust-proxy): forward request paths verbatim and pin the rustls provider (#3853)

Tejas Chopra9d98ea59082c
diff --git a/Cargo.lock b/Cargo.lock
index f0c9b955a..87429c2f7 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -182,6 +182,45 @@ dependencies = [
"stable_deref_trait",
]
+[[package]]
+name = "asn1-rs"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8"
+dependencies = [
+ "asn1-rs-derive",
+ "asn1-rs-impl",
+ "displaydoc",
+ "nom 7.1.3",
+ "num-traits",
+ "rusticata-macros",
+ "thiserror 2.0.20",
+ "time",
+]
+
+[[package]]
+name = "asn1-rs-derive"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.118",
+ "synstructure",
+]
+
+[[package]]
+name = "asn1-rs-impl"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.118",
+]
+
[[package]]
name = "assert-json-diff"
version = "2.0.2"
@@ -733,6 +772,12 @@ version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
+[[package]]
+name = "base64"
+version = "0.23.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
+
[[package]]
name = "base64-simd"
version = "0.8.0"
@@ -749,7 +794,7 @@ version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3"
dependencies = [
- "bit-vec",
+ "bit-vec 0.8.0",
]
[[package]]
@@ -758,6 +803,15 @@ version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7"
+[[package]]
+name = "bit-vec"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51"
+dependencies = [
+ "serde",
+]
+
[[package]]
name = "bit_field"
version = "0.10.3"
@@ -1343,6 +1397,20 @@ version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b"
+[[package]]
+name = "der-parser"
+version = "10.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6"
+dependencies = [
+ "asn1-rs",
+ "displaydoc",
+ "nom 7.1.3",
+ "num-bigint",
+ "num-traits",
+ "rusticata-macros",
+]
+
[[package]]
name = "deranged"
version = "0.5.8"
@@ -1962,6 +2030,7 @@ dependencies = [
"pin-project-lite",
"prometheus",
"proptest",
+ "rcgen",
"reqwest",
"rustls",
"rustls-native-certs",
@@ -1971,6 +2040,7 @@ dependencies = [
"sha2",
"thiserror 2.0.20",
"tokio",
+ "tokio-rustls",
"tokio-stream",
"tokio-tungstenite 0.30.0",
"tokio-util",
@@ -2930,6 +3000,15 @@ dependencies = [
"libc",
]
+[[package]]
+name = "oid-registry"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7"
+dependencies = [
+ "asn1-rs",
+]
+
[[package]]
name = "once_cell"
version = "1.21.4"
@@ -3052,6 +3131,16 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35fb2e5f958ec131621fdd531e9fc186ed768cbe395337403ae56c17a74c68ec"
+[[package]]
+name = "pem"
+version = "4.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d354a98a3d1251555de99e8fdd8afda05573c31b82f59063a7b0a29b5527f120"
+dependencies = [
+ "base64 0.23.1",
+ "serde_core",
+]
+
[[package]]
name = "percent-encoding"
version = "2.3.2"
@@ -3227,7 +3316,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744"
dependencies = [
"bit-set",
- "bit-vec",
+ "bit-vec 0.8.0",
"bitflags",
"num-traits",
"rand 0.9.4",
@@ -3552,6 +3641,20 @@ dependencies = [
"crossbeam-utils",
]
+[[package]]
+name = "rcgen"
+version = "0.14.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8774e05a7d0de114588e6a28fe7e71694b82614ed569d86d8b389dfbc98b8ad8"
+dependencies = [
+ "pem",
+ "ring",
+ "rustls-pki-types",
+ "time",
+ "x509-parser",
+ "yasna",
+]
+
[[package]]
name = "redis"
version = "1.7.0"
@@ -3726,6 +3829,15 @@ dependencies = [
"semver",
]
+[[package]]
+name = "rusticata-macros"
+version = "4.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632"
+dependencies = [
+ "nom 7.1.3",
+]
+
[[package]]
name = "rustix"
version = "1.1.4"
@@ -5356,6 +5468,24 @@ version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4"
+[[package]]
+name = "x509-parser"
+version = "0.18.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202"
+dependencies = [
+ "asn1-rs",
+ "data-encoding",
+ "der-parser",
+ "lazy_static",
+ "nom 7.1.3",
+ "oid-registry",
+ "ring",
+ "rusticata-macros",
+ "thiserror 2.0.20",
+ "time",
+]
+
[[package]]
name = "xmlparser"
version = "0.13.6"
@@ -5374,6 +5504,16 @@ version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a5a4b21e1a62b67a2970e6831bc091d7b87e119e7f9791aef9702e3bef04448"
+[[package]]
+name = "yasna"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b5f6765e852b9b4dc8e2a76843e4d64d1cea8e79bcde0b6901aea8e7c7f08282"
+dependencies = [
+ "bit-vec 0.9.1",
+ "time",
+]
+
[[package]]
name = "yoke"
version = "0.8.3"
diff --git a/Cargo.toml b/Cargo.toml
index 3fa9dbbb9..c0effd2fa 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -116,6 +116,15 @@ strip = "symbols"
lto = "thin"
codegen-units = 1
+# The proxy binary turns untrusted network input into byte and token counts;
+# a wrapping add there is a silent correctness bug at best, so its release
+# build keeps overflow checks (a panic is caught per task by tokio and
+# surfaces in the log; a wrong number does not). Scoped to this package:
+# `headroom-core` is also the `_core` Python extension's compression hot
+# path and is not included until the cost is benchmarked there.
+[profile.release.package.headroom-proxy]
+overflow-checks = true
+
# Fast-to-compile profile for CI test wheels. The shipped wheel uses
# `release` (lto + codegen-units=1) for runtime/size; CI only needs a working
# extension, so trade runtime perf for ~parallel, lto-free compilation. Used
diff --git a/crates/headroom-core/src/lib.rs b/crates/headroom-core/src/lib.rs
index a9417bd49..76d4b4daf 100644
--- a/crates/headroom-core/src/lib.rs
+++ b/crates/headroom-core/src/lib.rs
@@ -1,5 +1,9 @@
//! headroom-core: foundation crate for the Rust port of Headroom.
+// This crate has never needed `unsafe`; keep it that way. The FFI shim lives
+// in `headroom-py`, which is deliberately not under this lint.
+#![forbid(unsafe_code)]
+
pub mod auth_mode;
pub mod cache_control;
pub mod ccr;
diff --git a/crates/headroom-parity/src/lib.rs b/crates/headroom-parity/src/lib.rs
index a3d4c101d..adb5559cc 100644
--- a/crates/headroom-parity/src/lib.rs
+++ b/crates/headroom-parity/src/lib.rs
@@ -6,6 +6,8 @@
//! stubs and report `Skipped` — see the `stub_comparator!` block below for what
//! each is waiting on.
+#![forbid(unsafe_code)]
+
use anyhow::{bail, Context, Result};
use serde::{Deserialize, Serialize};
use std::fs;
diff --git a/crates/headroom-proxy/Cargo.toml b/crates/headroom-proxy/Cargo.toml
index 39c1e9f57..f664c4129 100644
--- a/crates/headroom-proxy/Cargo.toml
+++ b/crates/headroom-proxy/Cargo.toml
@@ -107,6 +107,15 @@ webpki-roots = "0.26"
[dev-dependencies]
tower = { workspace = true, features = ["util"] }
wiremock = "0.6"
+# `wss://` relay test (tests/integration_ws_tls.rs): a local TLS WebSocket
+# echo upstream. `tokio-rustls` is already in the build via reqwest and
+# hyper-rustls at this same version, so no new crate enters the tree; the
+# `ring` feature matches the provider the proxy installs (src/tls.rs).
+tokio-rustls = { version = "0.26", default-features = false, features = ["ring", "tls12"] }
+# Throwaway certificate for that upstream, generated at test time so no
+# private key is ever committed (gitleaks would rightly refuse one). rcgen
+# is the generator rustls' own test-suite uses; dev-only, never shipped.
+rcgen = { version = "0.14", default-features = false, features = ["ring", "pem"] }
reqwest = { version = "0.12", default-features = false, features = ["stream", "rustls-tls", "http2", "json"] }
tokio-tungstenite = { version = "0.30", default-features = false, features = ["connect", "rustls-tls-webpki-roots"] }
futures-util = "0.3"
diff --git a/crates/headroom-proxy/src/bedrock/invoke.rs b/crates/headroom-proxy/src/bedrock/invoke.rs
index a30e0027a..6f0030f2b 100644
--- a/crates/headroom-proxy/src/bedrock/invoke.rs
+++ b/crates/headroom-proxy/src/bedrock/invoke.rs
@@ -57,6 +57,8 @@ use crate::compression::{
use crate::headers::filter_response_headers;
use crate::observability::{observe_bedrock_invoke_latency, record_bedrock_invoke};
use crate::proxy::AppState;
+use crate::upstream_path::append_segments;
+use crate::ProxyError;
// Phase F PR-F1 + PR-D3: the bedrock auth-mode layer
// (`classify_and_attach_auth_mode`) populates `request.extensions()`
// with `AuthMode` BEFORE this handler runs. We extract it via
@@ -212,7 +214,21 @@ pub async fn handle_invoke(
// region-derived default.
let upstream_url = match build_bedrock_upstream(&state, &model_id, &uri, action) {
Ok(u) => u,
- Err(msg) => {
+ // The client sent a model id the proxy would have to rewrite
+ // (dot segments, control characters): its fault, 400, and the
+ // signer never sees the request.
+ Err(ProxyError::InvalidPath(msg)) => {
+ tracing::warn!(
+ event = "bedrock_path_rejected",
+ request_id = %request_id,
+ model_id = %model_id,
+ error = %msg,
+ "bedrock invoke: refusing to forward a model id that would rewrite the upstream path"
+ );
+ return error_response(StatusCode::BAD_REQUEST, "bedrock_path_rejected", &msg);
+ }
+ Err(e) => {
+ let msg = e.to_string();
tracing::error!(
event = "bedrock_endpoint_invalid",
request_id = %request_id,
@@ -475,34 +491,30 @@ fn run_anthropic_compression(
}
/// Build the upstream URL for the Bedrock route. Honours the
-/// operator-supplied `bedrock_endpoint` first, falling back to the
-/// region-derived default. The path/query portion is taken from the
-/// original URI verbatim — Bedrock's path schema (`/model/{id}/{action}`)
-/// is identical to the proxy's external path.
+/// operator-supplied `bedrock_endpoint` first (including any path prefix
+/// on it), falling back to the region-derived default. The path is rebuilt
+/// segment by segment — `/model/{model_id}/{action}` — from the decoded
+/// route parameter, so a model id containing `/` (an inference-profile
+/// ARN) is re-encoded as one `%2F` segment, and a model id that would
+/// change the path's shape (`..`, empty, control characters) is rejected
+/// with `ProxyError::InvalidPath` (400). See `crate::upstream_path`.
fn build_bedrock_upstream(
state: &AppState,
model_id: &str,
uri: &Uri,
action: &str,
-) -> Result<Url, String> {
+) -> Result<Url, ProxyError> {
let base = match state.config.bedrock_endpoint.as_ref() {
Some(u) => u.clone(),
None => {
let host =
BEDROCK_RUNTIME_HOST_TEMPLATE.replace("{region}", &state.config.bedrock_region);
- Url::parse(&format!("https://{host}/"))
- .map_err(|e| format!("bedrock derived base URL parse error: {e}"))?
+ Url::parse(&format!("https://{host}/")).map_err(|e| {
+ ProxyError::InvalidUpstream(format!("bedrock derived base URL parse error: {e}"))
+ })?
}
};
- // Compose the path. We trust the captured `model_id` (Axum
- // already URL-decoded it) and append `/{action}`.
- let path = format!(
- "/model/{model_id}/{action}",
- model_id = model_id,
- action = action,
- );
- let mut joined = base;
- joined.set_path(&path);
+ let mut joined = append_segments(&base, ["model", model_id, action])?;
if let Some(q) = uri.query() {
joined.set_query(Some(q));
}
diff --git a/crates/headroom-proxy/src/bedrock/invoke_streaming.rs b/crates/headroom-proxy/src/bedrock/invoke_streaming.rs
index 62501003c..64e0f5d2f 100644
--- a/crates/headroom-proxy/src/bedrock/invoke_streaming.rs
+++ b/crates/headroom-proxy/src/bedrock/invoke_streaming.rs
@@ -69,6 +69,8 @@ use crate::observability::{
observe_bedrock_invoke_latency, record_bedrock_eventstream_message, record_bedrock_invoke,
};
use crate::proxy::AppState;
+use crate::upstream_path::append_segments;
+use crate::ProxyError;
// Phase F PR-F1 + PR-D3: pre-classified by `classify_and_attach_auth_mode`
// middleware on the bedrock router; we read it back via the
// `Extension<AuthMode>` extractor.
@@ -187,7 +189,20 @@ pub async fn handle_invoke_streaming(
let upstream_url = match build_bedrock_streaming_upstream(&state, &model_id, &uri, action) {
Ok(u) => u,
- Err(msg) => {
+ // Client-caused: a model id the proxy would have to rewrite. 400,
+ // and nothing is signed or forwarded.
+ Err(ProxyError::InvalidPath(msg)) => {
+ tracing::warn!(
+ event = "bedrock_path_rejected",
+ request_id = %request_id,
+ model_id = %model_id,
+ error = %msg,
+ "bedrock invoke-streaming: refusing to forward a model id that would rewrite the upstream path"
+ );
+ return error_response(StatusCode::BAD_REQUEST, "bedrock_path_rejected", &msg);
+ }
+ Err(e) => {
+ let msg = e.to_string();
tracing::error!(
event = "bedrock_endpoint_invalid",
request_id = %request_id,
@@ -905,28 +920,27 @@ fn run_anthropic_compression(
}
}
+/// Streaming twin of `invoke::build_bedrock_upstream`: same base
+/// resolution, same segment-safe path construction (see
+/// `crate::upstream_path`), so an ARN model id stays one `%2F` segment
+/// and a traversal attempt is rejected with `ProxyError::InvalidPath`.
fn build_bedrock_streaming_upstream(
state: &AppState,
model_id: &str,
uri: &Uri,
action: &str,
-) -> Result<Url, String> {
+) -> Result<Url, ProxyError> {
let base = match state.config.bedrock_endpoint.as_ref() {
Some(u) => u.clone(),
None => {
let host =
BEDROCK_RUNTIME_HOST_TEMPLATE.replace("{region}", &state.config.bedrock_region);
- Url::parse(&format!("https://{host}/"))
- .map_err(|e| format!("bedrock derived base URL parse error: {e}"))?
+ Url::parse(&format!("https://{host}/")).map_err(|e| {
+ ProxyError::InvalidUpstream(format!("bedrock derived base URL parse error: {e}"))
+ })?
}
};
- let path = format!(
- "/model/{model_id}/{action}",
- model_id = model_id,
- action = action,
- );
- let mut joined = base;
- joined.set_path(&path);
+ let mut joined = append_segments(&base, ["model", model_id, action])?;
if let Some(q) = uri.query() {
joined.set_query(Some(q));
}
diff --git a/crates/headroom-proxy/src/error.rs b/crates/headroom-proxy/src/error.rs
index 646548a18..61cf8a2bc 100644
--- a/crates/headroom-proxy/src/error.rs
+++ b/crates/headroom-proxy/src/error.rs
@@ -28,6 +28,14 @@ pub enum ProxyError {
#[error("request body exceeds configured limit: {0}")]
PayloadTooLarge(String),
+ /// The request path, or a decoded route parameter, would have to be
+ /// rewritten on the way upstream (dot segments, backslashes, or a
+ /// segment that would gain or lose a `/` boundary). The proxy forwards
+ /// paths verbatim or not at all — see `crate::upstream_path`. RFC 7231
+ /// §6.5.1: 400.
+ #[error("request path rejected: {0}")]
+ InvalidPath(String),
+
/// Surfaced when `--compression` is enabled but the proxy can't
/// build the IntelligentContextManager at startup (e.g. the
/// embedded tokenizer asset failed to initialize). Bubbles up to
@@ -39,6 +47,12 @@ pub enum ProxyError {
CompressionStartup(String),
}
+impl From<crate::upstream_path::PathError> for ProxyError {
+ fn from(e: crate::upstream_path::PathError) -> Self {
+ ProxyError::InvalidPath(e.to_string())
+ }
+}
+
impl IntoResponse for ProxyError {
fn into_response(self) -> Response {
let (status, msg) = match &self {
@@ -54,6 +68,7 @@ impl IntoResponse for ProxyError {
ProxyError::InvalidUpstream(_) => (StatusCode::BAD_GATEWAY, self.to_string()),
ProxyError::InvalidHeader(_) => (StatusCode::BAD_REQUEST, self.to_string()),
ProxyError::PayloadTooLarge(_) => (StatusCode::PAYLOAD_TOO_LARGE, self.to_string()),
+ ProxyError::InvalidPath(_) => (StatusCode::BAD_REQUEST, self.to_string()),
ProxyError::WebSocket(_) => (StatusCode::BAD_GATEWAY, self.to_string()),
ProxyError::Io(_) => (StatusCode::INTERNAL_SERVER_ERROR, self.to_string()),
// CompressionStartup is a startup-time error, not a
diff --git a/crates/headroom-proxy/src/lib.rs b/crates/headroom-proxy/src/lib.rs
index a4e29f98a..a835ef069 100644
--- a/crates/headroom-proxy/src/lib.rs
+++ b/crates/headroom-proxy/src/lib.rs
@@ -1,6 +1,8 @@
//! headroom-proxy library: transparent reverse proxy in front of the Python
//! Headroom proxy. Used by both `main.rs` and the integration tests.
+#![forbid(unsafe_code)]
+
pub mod bedrock;
pub mod cache_stabilization;
pub mod compression;
@@ -14,6 +16,7 @@ pub mod proxy;
pub mod responses_items;
pub mod sse;
pub mod tls;
+pub mod upstream_path;
pub mod vertex;
pub mod websocket;
diff --git a/crates/headroom-proxy/src/main.rs b/crates/headroom-proxy/src/main.rs
index f21d0117f..005cd0f90 100644
--- a/crates/headroom-proxy/src/main.rs
+++ b/crates/headroom-proxy/src/main.rs
@@ -4,6 +4,8 @@
//! port; this binary forwards every HTTP/SSE/WebSocket request verbatim to
//! `--upstream`. See RUST_DEV.md for the operator runbook.
+#![forbid(unsafe_code)]
+
use std::net::SocketAddr;
use clap::Parser;
diff --git a/crates/headroom-proxy/src/proxy.rs b/crates/headroom-proxy/src/proxy.rs
index 650200f86..725f455fd 100644
--- a/crates/headroom-proxy/src/proxy.rs
+++ b/crates/headroom-proxy/src/proxy.rs
@@ -97,6 +97,12 @@ const DRIFT_DETECTOR_CAPACITY: usize = 1000;
impl AppState {
pub fn new(config: Config) -> Result<Self, ProxyError> {
+ // Two rustls crypto providers are linked into this binary (ring via
+ // reqwest, aws-lc-rs via the AWS SDK), so anything that reaches for
+ // the process default — reqwest below, tokio-tungstenite, any future
+ // dependency — would panic without one. Pin it before the first TLS
+ // client is built. Idempotent; see `crate::tls`.
+ crate::tls::install_process_crypto_provider();
let mut builder = reqwest::Client::builder();
// Corporate roots handed over as PEM files, added on top of the
// bundled + OS roots (see `crate::tls`).
@@ -379,30 +385,21 @@ fn is_websocket_upgrade(headers: &HeaderMap) -> bool {
/// Build the upstream URL by joining the configured base with the incoming
/// path-and-query. Preserves '?' and the query string verbatim.
pub(crate) fn build_upstream_url(base: &url::Url, uri: &Uri) -> Result<url::Url, ProxyError> {
- Ok(join_upstream_path(base, uri.path(), uri.query()))
+ join_upstream_path(base, uri.path(), uri.query())
}
/// Shared path-join helper used by HTTP and WebSocket handlers.
-/// Appends `path` to `base`, preserving any base path prefix, then sets `query`.
-pub(crate) fn join_upstream_path(base: &url::Url, path: &str, query: Option<&str>) -> url::Url {
- let mut joined = base.clone();
- // Strip trailing slash from base path so "http://x:1/api" + "/v1/foo"
- // yields "http://x:1/api/v1/foo" rather than "http://x:1/v1/foo".
- let base_path = joined.path().trim_end_matches('/').to_string();
- let combined = if path.is_empty() || path == "/" {
- if base_path.is_empty() {
- "/".to_string()
- } else {
- base_path
- }
- } else if base_path.is_empty() {
- path.to_string()
- } else {
- format!("{base_path}{path}")
- };
- joined.set_path(&combined);
- joined.set_query(query);
- joined
+/// Appends `path` to `base`, preserving any base path prefix, then sets
+/// `query`. Rejects (400) any path the URL parser would rewrite — dot
+/// segments, backslashes — so the upstream always receives exactly the path
+/// the client sent, under exactly the configured prefix. See
+/// [`crate::upstream_path`].
+pub(crate) fn join_upstream_path(
+ base: &url::Url,
+ path: &str,
+ query: Option<&str>,
+) -> Result<url::Url, ProxyError> {
+ Ok(crate::upstream_path::join_request_path(base, path, query)?)
}
/// Forward an HTTP request to the upstream and stream the response back.
@@ -1699,4 +1696,14 @@ mod tests {
let out = build_upstream_url(&base, &uri).unwrap();
assert_eq!(out.as_str(), "http://up:8080/");
}
+
+ #[test]
+ fn url_build_refuses_to_escape_base_path() {
+ // `url::Url::set_path` would resolve this to `/tenant-b/v1/messages`.
+ let base: url::Url = "http://gw/tenant-a".parse().unwrap();
+ let uri: Uri = "/../tenant-b/v1/messages".parse().unwrap();
+ let err = build_upstream_url(&base, &uri).unwrap_err();
+ assert!(matches!(err, ProxyError::InvalidPath(_)), "{err:?}");
+ assert_eq!(err.into_response().status(), StatusCode::BAD_REQUEST);
+ }
}
diff --git a/crates/headroom-proxy/src/tls.rs b/crates/headroom-proxy/src/tls.rs
index eb23982ec..057b3fc17 100644
--- a/crates/headroom-proxy/src/tls.rs
+++ b/crates/headroom-proxy/src/tls.rs
@@ -35,6 +35,28 @@ pub fn extra_root_certificates() -> Vec<reqwest::Certificate> {
.collect()
}
+/// Install ring as the process-wide rustls [`CryptoProvider`].
+///
+/// This binary links two providers — ring (reqwest, tokio-tungstenite) and
+/// aws-lc-rs (the AWS SDK behind `aws-config`). rustls refuses to guess
+/// between them: `ClientConfig::builder()` panics unless a default has been
+/// installed, and reqwest reaches for that default when it builds its TLS
+/// client. Installing ring explicitly makes the choice deliberate and keeps
+/// every TLS client in the process on one provider — the same one
+/// [`websocket_tls_config`] names. Idempotent: a second call (tests, embedded
+/// use) is a no-op.
+///
+/// ring rather than aws-lc-rs because it is the provider reqwest already
+/// defaults to, needs no C toolchain or cmake at build time, and the AWS SDK
+/// carries its own explicit configuration either way.
+///
+/// [`CryptoProvider`]: rustls::crypto::CryptoProvider
+pub fn install_process_crypto_provider() {
+ // `Err` means a provider is already installed, which is exactly the
+ // state we want.
+ let _ = rustls::crypto::ring::default_provider().install_default();
+}
+
/// rustls client config for `wss://` upstreams: webpki + OS + additive roots,
/// on an explicit crypto provider. Built once per process.
pub fn websocket_tls_config() -> Arc<rustls::ClientConfig> {
diff --git a/crates/headroom-proxy/src/upstream_path.rs b/crates/headroom-proxy/src/upstream_path.rs
new file mode 100644
index 000000000..17d3e80e2
--- /dev/null
+++ b/crates/headroom-proxy/src/upstream_path.rs
@@ -0,0 +1,299 @@
+//! Upstream path construction that never rewrites what the client asked for.
+//!
+//! A reverse proxy has one invariant on the request line: the path it sends
+//! upstream is the operator-configured base prefix followed by exactly the
+//! path the client sent. Two things break that invariant silently:
+//!
+//! * [`url::Url::set_path`] normalises dot segments (`.` and `..`, in any
+//! percent-encoded spelling) and folds `\` into `/`. A client could walk
+//! out of a tenant prefix (`https://gw/tenant-a` + `/../tenant-b/...`) or,
+//! on the provider routes, move a SigV4- or ADC-signed request to a
+//! different resource (`/model/../../other/invoke`).
+//! * Route parameters arrive percent-decoded from axum. Interpolating them
+//! back into a path string re-parses `/` as a segment separator, so a model
+//! id that legitimately contains `/` (Bedrock inference-profile ARNs) is
+//! split into two segments, and a hostile one gains segments.
+//!
+//! Every upstream URL in the proxy goes through this module: raw request
+//! paths through [`join_request_path`], decoded route parameters through
+//! [`append_segments`]. Anything the module would have to rewrite is
+//! rejected instead, and the caller answers the client with 400.
+
+use url::Url;
+
+/// Why a path could not be forwarded unchanged.
+#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
+pub enum PathError {
+ /// A `.` or `..` segment (including `%2e` spellings). The proxy does not
+ /// resolve these; the client must send the path it means.
+ #[error("path contains a dot segment {0:?}; the proxy does not normalise paths")]
+ DotSegment(String),
+ /// A backslash, which the URL standard folds into `/` for http(s).
+ #[error("path contains a backslash; the proxy does not normalise paths")]
+ Backslash,
+ /// A decoded route parameter was empty.
+ #[error("path segment is empty")]
+ EmptySegment,
+ /// A decoded route parameter carried a control character.
+ #[error("path segment contains a control character")]
+ ControlCharacter,
+ /// Post-condition failure: the joined path no longer sits under the
+ /// configured base path. Guards against future changes to the join
+ /// logic; unreachable while [`validate_request_path`] runs first.
+ #[error("joined path {joined:?} escapes the configured base path {base:?}")]
+ EscapesBase { base: String, joined: String },
+ /// The upstream URL is not hierarchical (`mailto:`-style), so it cannot
+ /// carry path segments. Not reachable with http(s)/ws(s) upstreams.
+ #[error("upstream URL cannot carry a path")]
+ CannotBeABase,
+}
+
+/// True when a raw (still percent-encoded) segment is a dot segment under
+/// the WHATWG URL standard, which treats `%2e` as `.` for this purpose.
+fn is_dot_segment(segment: &str) -> bool {
+ matches!(
+ segment.to_ascii_lowercase().as_str(),
+ "." | ".." | "%2e" | ".%2e" | "%2e." | "%2e%2e"
+ )
+}
+
+/// Reject a raw request path that `Url::set_path` would rewrite.
+///
+/// `path` is the request-target path exactly as received (percent-encoded,
+/// no query). Only structural rewrites are rejected — dot segments and
+/// backslashes; ordinary percent-encoding of reserved characters is fine
+/// because it does not change which resource the path names.
+pub fn validate_request_path(path: &str) -> Result<(), PathError> {
+ if path.contains('\\') {
+ return Err(PathError::Backslash);
+ }
+ if let Some(bad) = path.split('/').find(|segment| is_dot_segment(segment)) {
+ return Err(PathError::DotSegment(bad.to_string()));
+ }
+ Ok(())
+}
+
+/// Append the client's raw request path (and query) to the configured
+/// upstream base, preserving any base path prefix.
+///
+/// `"http://x:1/api"` + `"/v1/foo"` → `"http://x:1/api/v1/foo"`. The result
+/// is guaranteed to start with the base path: the input is validated first
+/// and the output is checked afterwards.
+pub fn join_request_path(base: &Url, path: &str, query: Option<&str>) -> Result<Url, PathError> {
+ validate_request_path(path)?;
+
+ let mut joined = base.clone();
+ let base_path = joined.path().trim_end_matches('/').to_string();
+ let combined = if path.is_empty() || path == "/" {
+ if base_path.is_empty() {
+ "/".to_string()
+ } else {
+ base_path.clone()
+ }
+ } else if base_path.is_empty() {
+ path.to_string()
+ } else {
+ format!("{base_path}{path}")
+ };
+ joined.set_path(&combined);
+ joined.set_query(query);
+
+ if !base_path.is_empty() {
+ let got = joined.path();
+ if got != base_path && !got.starts_with(&format!("{base_path}/")) {
+ return Err(PathError::EscapesBase {
+ base: base_path,
+ joined: got.to_string(),
+ });
+ }
+ }
+ Ok(joined)
+}
+
+/// Reject a decoded route parameter that cannot be a single path segment.
+///
+/// `/` is allowed: [`append_segments`] percent-encodes it, so a Bedrock
+/// inference-profile ARN stays one segment on the wire (`%2F`), exactly as
+/// the AWS SDKs send it.
+pub fn validate_segment(segment: &str) -> Result<(), PathError> {
+ if segment.is_empty() {
+ return Err(PathError::EmptySegment);
+ }
+ if matches!(segment, "." | "..") {
+ return Err(PathError::DotSegment(segment.to_string()));
+ }
+ if segment.chars().any(char::is_control) {
+ return Err(PathError::ControlCharacter);
+ }
+ Ok(())
+}
+
+/// Append already-decoded segments to `base` as individual path segments,
+/// percent-encoding each one so it stays a single segment on the wire.
+///
+/// Keeps any path prefix on `base` (`https://gw/prefix/` + `["model", id,
+/// "invoke"]` → `/prefix/model/<id>/invoke`), never produces a double slash,
+/// and refuses segments that would change the path's shape.
+pub fn append_segments<'a, I>(base: &Url, segments: I) -> Result<Url, PathError>
+where
+ I: IntoIterator<Item = &'a str>,
+{
+ let mut joined = base.clone();
+ {
+ let mut path = joined
+ .path_segments_mut()
+ .map_err(|()| PathError::CannotBeABase)?;
+ path.pop_if_empty();
+ for segment in segments {
+ validate_segment(segment)?;
+ path.push(segment);
+ }
+ }
+ Ok(joined)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn base(s: &str) -> Url {
+ Url::parse(s).expect("test base url")
+ }
+
+ #[test]
+ fn join_preserves_base_prefix_and_query() {
+ let out =
+ join_request_path(&base("http://up:8080/api"), "/v1/messages", Some("a=1")).unwrap();
+ assert_eq!(out.as_str(), "http://up:8080/api/v1/messages?a=1");
+ }
+
+ #[test]
+ fn join_root_forms() {
+ assert_eq!(
+ join_request_path(&base("http://up:8080/"), "/", None)
+ .unwrap()
+ .as_str(),
+ "http://up:8080/"
+ );
+ assert_eq!(
+ join_request_path(&base("http://up:8080/api/"), "/", None)
+ .unwrap()
+ .as_str(),
+ "http://up:8080/api"
+ );
+ assert_eq!(
+ join_request_path(&base("http://up:8080"), "/v1", None)
+ .unwrap()
+ .as_str(),
+ "http://up:8080/v1"
+ );
+ }
+
+ #[test]
+ fn join_rejects_every_dot_segment_spelling() {
+ for path in [
+ "/../tenant-b/v1",
+ "/v1/./x",
+ "/v1/%2e%2e/x",
+ "/v1/%2E%2E/x",
+ "/v1/.%2e/x",
+ "/v1/%2e./x",
+ "/v1/%2e/x",
+ "/..",
+ ] {
+ let err = join_request_path(&base("http://gw/tenant-a"), path, None).unwrap_err();
+ assert!(matches!(err, PathError::DotSegment(_)), "{path}: {err:?}");
+ }
+ }
+
+ #[test]
+ fn join_rejects_backslash() {
+ let err =
+ join_request_path(&base("http://gw/tenant-a"), "/..\\tenant-b", None).unwrap_err();
+ assert_eq!(err, PathError::Backslash);
+ }
+
+ #[test]
+ fn join_rejects_dot_segments_even_without_a_base_prefix() {
+ // No prefix to escape, but the upstream would still receive a
+ // different path than the client sent (`/x` instead of `/a/../x`).
+ let err = join_request_path(&base("http://up"), "/a/../x", None).unwrap_err();
+ assert!(matches!(err, PathError::DotSegment(_)));
+ }
+
+ #[test]
+ fn join_keeps_encoded_slashes_as_one_segment() {
+ // `..%2F..` is not a dot segment; it stays a literal segment upstream.
+ let out = join_request_path(&base("http://gw/tenant-a"), "/v1/..%2F..%2Fx", None).unwrap();
+ assert_eq!(out.path(), "/tenant-a/v1/..%2F..%2Fx");
+ }
+
+ #[test]
+ fn append_encodes_slashes_inside_a_segment() {
+ let arn =
+ "arn:aws:bedrock:us-east-1:123456789012:inference-profile/us.anthropic.claude-3-5";
+ let out = append_segments(
+ &base("https://bedrock-runtime.us-east-1.amazonaws.com/"),
+ ["model", arn, "invoke"],
+ )
+ .unwrap();
+ assert_eq!(
+ out.as_str(),
+ "https://bedrock-runtime.us-east-1.amazonaws.com/model/arn:aws:bedrock:us-east-1:123456789012:inference-profile%2Fus.anthropic.claude-3-5/invoke"
+ );
+ }
+
+ #[test]
+ fn append_keeps_base_prefix_without_double_slash() {
+ let out = append_segments(&base("https://gw/prefix/"), ["model", "m", "invoke"]).unwrap();
+ assert_eq!(out.path(), "/prefix/model/m/invoke");
+ let out = append_segments(&base("https://gw"), ["model", "m", "invoke"]).unwrap();
+ assert_eq!(out.path(), "/model/m/invoke");
+ }
+
+ #[test]
+ fn append_rejects_traversal_and_malformed_segments() {
+ let b = base("https://gw/");
+ // What `..%2F..%2Fother` decodes to: a `/`-containing segment is
+ // fine (it is re-encoded and stays one segment); a bare `..` is not.
+ assert_eq!(
+ append_segments(&b, ["model", "../../other", "invoke"])
+ .unwrap()
+ .path(),
+ "/model/..%2F..%2Fother/invoke"
+ );
+ assert!(matches!(
+ append_segments(&b, ["model", "..", "invoke"]).unwrap_err(),
+ PathError::DotSegment(_)
+ ));
+ assert!(matches!(
+ append_segments(&b, ["model", ".", "invoke"]).unwrap_err(),
+ PathError::DotSegment(_)
+ ));
+ assert_eq!(
+ append_segments(&b, ["model", "", "invoke"]).unwrap_err(),
+ PathError::EmptySegment
+ );
+ assert_eq!(
+ append_segments(&b, ["model", "m\r\nX-Injected: 1", "invoke"]).unwrap_err(),
+ PathError::ControlCharacter
+ );
+ }
+
+ #[test]
+ fn append_does_not_alter_ordinary_model_ids() {
+ let out = append_segments(
+ &base("https://bedrock-runtime.us-west-2.amazonaws.com/"),
+ [
+ "model",
+ "anthropic.claude-3-haiku-20240307-v1:0",
+ "converse",
+ ],
+ )
+ .unwrap();
+ assert_eq!(
+ out.as_str(),
+ "https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-3-haiku-20240307-v1:0/converse"
+ );
+ }
+}
diff --git a/crates/headroom-proxy/src/vertex/mod.rs b/crates/headroom-proxy/src/vertex/mod.rs
index 115705e0c..2302fce3c 100644
--- a/crates/headroom-proxy/src/vertex/mod.rs
+++ b/crates/headroom-proxy/src/vertex/mod.rs
@@ -120,6 +120,33 @@ pub async fn handle_vertex_predict_dispatch(
.map(|s| s.to_string())
.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
+ // The upstream URL is built from the raw request path (which
+ // `crate::upstream_path` already refuses to normalise), but the
+ // decoded parameters are logged, labelled and handed to the
+ // envelope logic. Reject the shapes that could only mean a
+ // traversal attempt (`..`, empty, control characters) before any of
+ // that happens, so a later refactor that interpolates these values
+ // cannot reopen the hole.
+ for (name, value) in [
+ ("project", project.as_str()),
+ ("location", location.as_str()),
+ ("model_action", model_action.as_str()),
+ ] {
+ if let Err(e) = crate::upstream_path::validate_segment(value) {
+ tracing::warn!(
+ event = "vertex_path_rejected",
+ request_id = %request_id,
+ segment = name,
+ error = %e,
+ "vertex path parameter would rewrite the upstream path; refusing"
+ );
+ return Response::builder()
+ .status(StatusCode::BAD_REQUEST)
+ .body(Body::from(format!("vertex path: {name} rejected: {e}")))
+ .expect("static");
+ }
+ }
+
let (model_id, verb_str) = match split_model_action(&model_action) {
Some(parts) => parts,
None => {
diff --git a/crates/headroom-proxy/src/websocket.rs b/crates/headroom-proxy/src/websocket.rs
index 65d9b2b8c..9435f268f 100644
--- a/crates/headroom-proxy/src/websocket.rs
+++ b/crates/headroom-proxy/src/websocket.rs
@@ -8,7 +8,8 @@ use std::net::SocketAddr;
use axum::body::Body;
use axum::extract::ws::{CloseFrame, Message as AxMsg, WebSocket, WebSocketUpgrade};
-use axum::http::{HeaderName, HeaderValue, Request, Response, StatusCode};
+use axum::http::{HeaderName, HeaderValue, Request, Response};
+use axum::response::IntoResponse;
use futures_util::{SinkExt, StreamExt};
use tokio_tungstenite::tungstenite::client::IntoClientRequest;
use tokio_tungstenite::tungstenite::protocol::CloseFrame as TgCloseFrame;
@@ -32,11 +33,13 @@ pub async fn ws_handler(
.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
// Build the upstream WS URL.
+ // A rejected path (dot segments, backslash) is the client's fault and
+ // maps to 400; a bad upstream scheme is the operator's and maps to 502.
let upstream_url = match build_upstream_ws_url(&state.config.upstream, req.uri()) {
Ok(u) => u,
Err(e) => {
tracing::warn!(error = %e, "failed to build upstream ws url");
- return (StatusCode::BAD_GATEWAY, e).into_response_body();
+ return e.into_response();
}
};
@@ -84,30 +87,25 @@ pub async fn ws_handler(
})
}
-trait IntoResponseBody {
- fn into_response_body(self) -> Response<Body>;
-}
-impl IntoResponseBody for (StatusCode, String) {
- fn into_response_body(self) -> Response<Body> {
- Response::builder()
- .status(self.0)
- .body(Body::from(self.1))
- .unwrap()
- }
-}
-
-fn build_upstream_ws_url(base: &url::Url, req_uri: &http::Uri) -> Result<url::Url, String> {
+fn build_upstream_ws_url(
+ base: &url::Url,
+ req_uri: &http::Uri,
+) -> Result<url::Url, crate::ProxyError> {
let mut joined = base.clone();
let new_scheme = match joined.scheme() {
"http" => "ws",
"https" => "wss",
"ws" | "wss" => "ws", // already WS; set_scheme is a no-op but keeps it uniform
- other => return Err(format!("unsupported upstream scheme: {other}")),
+ other => {
+ return Err(crate::ProxyError::InvalidUpstream(format!(
+ "unsupported upstream scheme: {other}"
+ )))
+ }
};
joined
.set_scheme(new_scheme)
- .map_err(|()| "failed to set ws scheme".to_string())?;
- Ok(join_upstream_path(&joined, req_uri.path(), req_uri.query()))
+ .map_err(|()| crate::ProxyError::InvalidUpstream("failed to set ws scheme".to_string()))?;
+ join_upstream_path(&joined, req_uri.path(), req_uri.query())
}
async fn run_ws_pump(
diff --git a/crates/headroom-proxy/tests/integration_path_traversal.rs b/crates/headroom-proxy/tests/integration_path_traversal.rs
new file mode 100644
index 000000000..08f6b5efb
--- /dev/null
+++ b/crates/headroom-proxy/tests/integration_path_traversal.rs
@@ -0,0 +1,272 @@
+//! The proxy forwards the client's path verbatim under the configured
+//! prefix, or not at all. `url::Url::set_path` resolves `.`/`..` segments
+//! (also spelled `%2e`) and folds `\` into `/`, so without a guard a client
+//! could:
+//!
+//! * walk out of a tenant prefix on the generic forwarder
+//! (`https://gw/tenant-a` + `/../tenant-b/...`),
+//! * move a SigV4-signed Bedrock request to a different resource
+//! (`/model/../invoke`, `/model/..%2F..%2Fother/invoke`),
+//! * move an ADC-signed Vertex request to another project
+//! (`/v1beta1/projects/../../other/...`).
+//!
+//! HTTP clients normalise dot segments before sending, so the traversal
+//! requests here go over a raw TCP socket. See `headroom_proxy::upstream_path`.
+
+mod common;
+
+use std::net::SocketAddr;
+
+use aws_credential_types::Credentials;
+use common::{install_static_token_source, start_proxy, start_proxy_with_state};
+use tokio::io::{AsyncReadExt, AsyncWriteExt};
+use url::Url;
+use wiremock::matchers::any;
+use wiremock::{Mock, MockServer, ResponseTemplate};
+
+/// Send one HTTP/1.1 request exactly as written and return the status code.
+async fn raw_request(
+ addr: SocketAddr,
+ method: &str,
+ target: &str,
+ extra_headers: &[(&str, &str)],
+ body: &[u8],
+) -> u16 {
+ let mut stream = tokio::net::TcpStream::connect(addr).await.unwrap();
+ let mut req = format!("{method} {target} HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n");
+ for (k, v) in extra_headers {
+ req.push_str(&format!("{k}: {v}\r\n"));
+ }
+ if !body.is_empty() {
+ req.push_str("content-type: application/json\r\n");
+ req.push_str(&format!("content-length: {}\r\n", body.len()));
+ }
+ req.push_str("\r\n");
+ stream.write_all(req.as_bytes()).await.unwrap();
+ stream.write_all(body).await.unwrap();
+ let mut out = Vec::new();
+ stream.read_to_end(&mut out).await.unwrap();
+ let head = String::from_utf8_lossy(&out);
+ let status_line = head.lines().next().unwrap_or("");
+ status_line
+ .split_whitespace()
+ .nth(1)
+ .and_then(|s| s.parse().ok())
+ .unwrap_or_else(|| panic!("no status line in response: {head:?}"))
+}
+
+async fn upstream_paths(mock: &MockServer) -> Vec<String> {
+ mock.received_requests()
+ .await
+ .unwrap_or_default()
+ .iter()
+ .map(|r| r.url.path().to_string())
+ .collect()
+}
+
+async fn accept_all(mock: &MockServer) {
+ Mock::given(any())
+ .respond_with(ResponseTemplate::new(200).set_body_string(r#"{"id":"msg_x","content":[]}"#))
+ .mount(mock)
+ .await;
+}
+
+// ─── generic forwarder ───────────────────────────────────────────────────
+
+#[tokio::test]
+async fn forwarder_keeps_requests_under_the_base_prefix() {
+ let mock = MockServer::start().await;
+ accept_all(&mock).await;
+ let proxy = start_proxy(&format!("{}/tenant-a", mock.uri())).await;
+
+ // Control: an ordinary path lands under the prefix.
+ assert_eq!(
+ raw_request(proxy.addr, "GET", "/v1/models", &[], b"").await,
+ 200
+ );
+ assert_eq!(
+ upstream_paths(&mock).await,
+ vec!["/tenant-a/v1/models".to_string()]
+ );
+
+ // Every dot-segment spelling and the backslash are refused before any
+ // upstream call. `url` would have resolved these to `/tenant-b/...`.
+ for target in [
+ "/../tenant-b/v1/models",
+ "/v1/../../tenant-b/v1/models",
+ "/%2e%2e/tenant-b/v1/models",
+ "/.%2e/tenant-b/v1/models",
+ "/%2E%2E/tenant-b/v1/models",
+ "/..\\tenant-b/v1/models",
+ ] {
+ let status = raw_request(proxy.addr, "GET", target, &[], b"").await;
+ assert_eq!(status, 400, "{target} must be rejected");
+ }
+ assert_eq!(
+ upstream_paths(&mock).await.len(),
+ 1,
+ "no traversal request may reach the upstream"
+ );
+ proxy.shutdown().await;
+}
+
+#[tokio::test]
+async fn websocket_upgrade_with_traversal_path_is_rejected_before_upgrade() {
+ let mock = MockServer::start().await;
+ let proxy = start_proxy(&format!("{}/tenant-a", mock.uri())).await;
+ let status = raw_request(
+ proxy.addr,
+ "GET",
+ "/../tenant-b/ws",
+ &[
+ ("Upgrade", "websocket"),
+ ("Connection", "Upgrade"),
+ ("Sec-WebSocket-Key", "dGhlIHNhbXBsZSBub25jZQ=="),
+ ("Sec-WebSocket-Version", "13"),
+ ],
+ b"",
+ )
+ .await;
+ assert_eq!(status, 400, "a traversal path must not be upgraded (101)");
+ proxy.shutdown().await;
+}
+
+// ─── Bedrock (SigV4-signed) ──────────────────────────────────────────────
+
+fn test_credentials() -> Credentials {
+ Credentials::new(
+ "AKIAEXAMPLEAKIDFORTEST",
+ "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+ None,
+ None,
+ "test",
+ )
+}
+
+async fn bedrock_proxy(mock: &MockServer) -> common::ProxyHandle {
+ let endpoint: Url = mock.uri().parse().unwrap();
+ start_proxy_with_state(
+ &mock.uri(),
+ |c| c.bedrock_endpoint = Some(endpoint),
+ |s| s.with_bedrock_credentials(test_credentials()),
+ )
+ .await
+}
+
+const BODY: &[u8] =
+ br#"{"anthropic_version":"bedrock-2023-05-31","max_tokens":8,"messages":[{"role":"user","content":"hi"}]}"#;
+
+#[tokio::test]
+async fn bedrock_dot_segment_model_id_is_rejected_unsigned() {
+ let mock = MockServer::start().await;
+ accept_all(&mock).await;
+ let proxy = bedrock_proxy(&mock).await;
+
+ // `/model/../invoke` used to become a signed request for `/invoke`.
+ for target in [
+ "/model/../invoke",
+ "/model/./converse",
+ "/model/%2e%2e/invoke",
+ ] {
+ let status = raw_request(proxy.addr, "POST", target, &[], BODY).await;
+ assert_eq!(status, 400, "{target} must be rejected");
+ }
+ assert!(
+ upstream_paths(&mock).await.is_empty(),
+ "no signed request may leave for a rejected model id"
+ );
+ proxy.shutdown().await;
+}
+
+#[tokio::test]
+async fn bedrock_encoded_slashes_in_model_id_stay_one_segment() {
+ let mock = MockServer::start().await;
+ accept_all(&mock).await;
+ let proxy = bedrock_proxy(&mock).await;
+
+ // The traversal shape: decoded, this model id is `../../guardrail/...`.
+ // Before the fix the signed request went to
+ // `/guardrail/GID/version/1/apply/invoke`. Now the id is re-encoded and
+ // the upstream sees it as one (nonsensical) model segment.
+ let resp = reqwest::Client::new()
+ .post(format!(
+ "{}/model/..%2F..%2Fguardrail%2FGID%2Fversion%2F1%2Fapply/invoke",
+ proxy.url()
+ ))
+ .header("content-type", "application/json")
+ .body(BODY)
+ .send()
+ .await
+ .unwrap();
+ assert_ne!(resp.status(), 500);
+ let paths = upstream_paths(&mock).await;
+ assert!(
+ !paths.iter().any(|p| p.starts_with("/guardrail/")),
+ "traversal reached the upstream: {paths:?}"
+ );
+ for p in &paths {
+ assert!(p.starts_with("/model/") && p.ends_with("/invoke"), "{p}");
+ assert!(
+ p.contains("%2F"),
+ "slashes inside the model id must stay encoded: {p}"
+ );
+ }
+
+ // The legitimate case with the same shape: an inference-profile ARN
+ // carries a `/`, which the AWS SDKs send as `%2F`. It must arrive as one
+ // segment, exactly as sent.
+ let arn = "arn:aws:bedrock:us-east-1:123456789012:inference-profile%2Fus.anthropic.claude-3-5-haiku-20241022-v1:0";
+ let resp = reqwest::Client::new()
+ .post(format!("{}/model/{arn}/invoke", proxy.url()))
+ .header("content-type", "application/json")
+ .body(BODY)
+ .send()
+ .await
+ .unwrap();
+ assert_eq!(resp.status(), 200);
+ let paths = upstream_paths(&mock).await;
+ assert_eq!(
+ paths.last().map(String::as_str),
+ Some(format!("/model/{arn}/invoke").as_str())
+ );
+ proxy.shutdown().await;
+}
+
+#[tokio::test]
+async fn bedrock_streaming_dot_segment_model_id_is_rejected_unsigned() {
+ let mock = MockServer::start().await;
+ accept_all(&mock).await;
+ let proxy = bedrock_proxy(&mock).await;
+
+ for target in [
+ "/model/../invoke-with-response-stream",
+ "/model/../converse-stream",
+ ] {
+ let status = raw_request(proxy.addr, "POST", target, &[], BODY).await;
+ assert_eq!(status, 400, "{target} must be rejected");
+ }
+ assert!(upstream_paths(&mock).await.is_empty());
+ proxy.shutdown().await;
+}
+
+// ─── Vertex (ADC-signed) ─────────────────────────────────────────────────
+
+#[tokio::test]
+async fn vertex_dot_segment_project_is_rejected_unsigned() {
+ let mock = MockServer::start().await;
+ accept_all(&mock).await;
+ let proxy = start_proxy_with_state(
+ &mock.uri(),
+ |_| {},
+ |s| install_static_token_source(s, "test-bearer"),
+ )
+ .await;
+
+ // Would have resolved to `/evil/locations/...` carrying the operator's
+ // ADC bearer.
+ let target = "/v1beta1/projects/../../evil/locations/us-central1/publishers/anthropic/models/claude-3-5-sonnet@20240620:rawPredict";
+ let status = raw_request(proxy.addr, "POST", target, &[], BODY).await;
+ assert_eq!(status, 400);
+ assert!(upstream_paths(&mock).await.is_empty());
+ proxy.shutdown().await;
+}
diff --git a/crates/headroom-proxy/tests/integration_ws_tls.rs b/crates/headroom-proxy/tests/integration_ws_tls.rs
new file mode 100644
index 000000000..1c75ae5ee
--- /dev/null
+++ b/crates/headroom-proxy/tests/integration_ws_tls.rs
@@ -0,0 +1,165 @@
+//! `wss://` relay: an `https://` upstream means the WebSocket pump has to
+//! complete a TLS handshake. This binary links two rustls crypto providers
+//! (ring via reqwest, aws-lc-rs via the AWS SDK) and rustls panics on
+//! `ClientConfig::builder()` when no process-level default is installed —
+//! which used to kill every `wss://` relay from inside the pump task, on any
+//! client's request. The other ws tests all use plaintext `ws://`; this file
+//! is the only one that drives the TLS path end to end.
+//!
+//! The upstream certificate is generated per test process (rcgen) and
+//! handed to the proxy through `HEADROOM_CA_BUNDLE`, the same knob an
+//! operator uses for a corporate TLS-inspection root.
+
+mod common;
+
+use std::net::SocketAddr;
+use std::sync::{Arc, OnceLock};
+use std::time::Duration;
+
+use common::start_proxy;
+use futures_util::{SinkExt, StreamExt};
+use rustls_pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer};
+use tokio::sync::oneshot;
+use tokio_tungstenite::tungstenite::Message;
+
+struct TestCert {
+ cert: CertificateDer<'static>,
+ cert_pem: String,
+ key_pkcs8: Vec<u8>,
+}
+
+fn self_signed_for_loopback() -> TestCert {
+ let ck =
+ rcgen::generate_simple_self_signed(vec!["localhost".to_string(), "127.0.0.1".to_string()])
+ .expect("self-signed test certificate");
+ TestCert {
+ cert: ck.cert.der().clone(),
+ cert_pem: ck.cert.pem(),
+ key_pkcs8: ck.signing_key.serialize_der(),
+ }
+}
+
+/// The one certificate the proxy trusts in this process. The proxy reads
+/// `HEADROOM_CA_BUNDLE` once (at `AppState::new` for HTTP, lazily for the
+/// first `wss://` connect), so the variable is set here, before any proxy
+/// starts, and never changed again.
+fn trusted_cert() -> &'static TestCert {
+ static TRUSTED: OnceLock<TestCert> = OnceLock::new();
+ TRUSTED.get_or_init(|| {
+ let cert = self_signed_for_loopback();
+ let dir = std::env::temp_dir().join(format!("headroom-ws-tls-{}", std::process::id()));
+ std::fs::create_dir_all(&dir).expect("temp dir");
+ let bundle = dir.join("upstream-ca.pem");
+ std::fs::write(&bundle, &cert.cert_pem).expect("write bundle");
+ std::env::set_var("HEADROOM_CA_BUNDLE", &bundle);
+ cert
+ })
+}
+
+/// A TLS WebSocket echo server on 127.0.0.1, presenting `cert`.
+async fn tls_echo_upstream(cert: &TestCert) -> (SocketAddr, oneshot::Sender<()>) {
+ let provider = Arc::new(rustls::crypto::ring::default_provider());
+ let config = rustls::ServerConfig::builder_with_provider(provider)
+ .with_safe_default_protocol_versions()
+ .expect("ring supports the default TLS versions")
+ .with_no_client_auth()
+ .with_single_cert(
+ vec![cert.cert.clone()],
+ PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(cert.key_pkcs8.clone())),
+ )
+ .expect("server config");
+ let acceptor = tokio_rustls::TlsAcceptor::from(Arc::new(config));
+
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let addr = listener.local_addr().unwrap();
+ let (stop_tx, mut stop_rx) = oneshot::channel();
+ tokio::spawn(async move {
+ loop {
+ tokio::select! {
+ _ = &mut stop_rx => break,
+ accepted = listener.accept() => {
+ let Ok((stream, _)) = accepted else { continue };
+ let acceptor = acceptor.clone();
+ tokio::spawn(async move {
+ let Ok(tls) = acceptor.accept(stream).await else { return };
+ let Ok(ws) = tokio_tungstenite::accept_async(tls).await else { return };
+ let (mut sink, mut src) = ws.split();
+ while let Some(Ok(msg)) = src.next().await {
+ match msg {
+ Message::Close(cf) => {
+ let _ = sink.send(Message::Close(cf)).await;
+ break;
+ }
+ m => {
+ if sink.send(m).await.is_err() {
+ break;
+ }
+ }
+ }
+ }
+ });
+ }
+ }
+ }
+ });
+ (addr, stop_tx)
+}
+
+#[tokio::test]
+async fn wss_upstream_relays_frames_over_tls() {
+ let (upstream, _stop) = tls_echo_upstream(trusted_cert()).await;
+ let proxy = start_proxy(&format!("https://127.0.0.1:{}", upstream.port())).await;
+
+ let (mut ws, _) = tokio_tungstenite::connect_async(format!("{}/v1/realtime", proxy.ws_url()))
+ .await
+ .expect("client handshake with the proxy");
+
+ ws.send(Message::Text("over-tls".into())).await.unwrap();
+ let echoed = tokio::time::timeout(Duration::from_secs(10), ws.next())
+ .await
+ .expect("echo within 10s: the wss:// pump must not die on connect")
+ .expect("stream still open")
+ .expect("frame, not a transport error");
+ match echoed {
+ Message::Text(t) => assert_eq!(t.as_str(), "over-tls"),
+ other => panic!("expected the echoed text frame, got {other:?}"),
+ }
+
+ ws.send(Message::Close(None)).await.ok();
+ proxy.shutdown().await;
+}
+
+#[tokio::test]
+async fn wss_upstream_with_untrusted_cert_fails_closed_and_proxy_survives() {
+ // Make sure the trusted bundle is installed first, so this test's
+ // untrusted certificate is genuinely untrusted rather than "no bundle".
+ let _ = trusted_cert();
+ let untrusted = self_signed_for_loopback();
+ let (upstream, _stop) = tls_echo_upstream(&untrusted).await;
+ let proxy = start_proxy(&format!("https://127.0.0.1:{}", upstream.port())).await;
+
+ let (mut ws, _) = tokio_tungstenite::connect_async(format!("{}/v1/realtime", proxy.ws_url()))
+ .await
+ .expect("client handshake with the proxy");
+ ws.send(Message::Text("should-not-arrive".into()))
+ .await
+ .ok();
+
+ // The upstream handshake must fail verification and the relay must end
+ // cleanly: no echo, no hang, no panic.
+ let outcome = tokio::time::timeout(Duration::from_secs(10), ws.next())
+ .await
+ .expect("relay must end within 10s when the upstream cert is untrusted");
+ match outcome {
+ Some(Ok(Message::Text(t))) => panic!("frame relayed to an untrusted upstream: {t}"),
+ Some(Ok(Message::Binary(_))) => panic!("frame relayed to an untrusted upstream"),
+ _ => {}
+ }
+
+ // The proxy is still serving: the failure stayed inside the one relay.
+ let health = reqwest::get(format!("{}/healthz", proxy.url()))
+ .await
+ .unwrap();
+ assert_eq!(health.status(), 200);
+ proxy.shutdown().await;
+}
diff --git a/crates/headroom-simulators/src/lib.rs b/crates/headroom-simulators/src/lib.rs
index 11fa4d57e..14c516470 100644
--- a/crates/headroom-simulators/src/lib.rs
+++ b/crates/headroom-simulators/src/lib.rs
@@ -3,6 +3,8 @@
//! The crate is intentionally standalone: it behaves like a provider upstream
//! that Headroom can proxy to, but it never calls a real LLM service.
+#![forbid(unsafe_code)]
+
pub mod application;
pub mod config;
pub mod domain;