Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix: make Headroom work behind Zscaler and other TLS-inspecting networks (#3831)

Tejas Chopra66258c40a458
diff --git a/Cargo.lock b/Cargo.lock
index 73d4d9b80..f0c9b955a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1963,6 +1963,9 @@ dependencies = [
"prometheus",
"proptest",
"reqwest",
+ "rustls",
+ "rustls-native-certs",
+ "rustls-pki-types",
"serde",
"serde_json",
"sha2",
@@ -1977,6 +1980,7 @@ dependencies = [
"tracing-subscriber",
"url",
"uuid",
+ "webpki-roots 0.26.11",
"wiremock",
]
@@ -3642,6 +3646,7 @@ dependencies = [
"pin-project-lite",
"quinn",
"rustls",
+ "rustls-native-certs",
"rustls-pki-types",
"serde",
"serde_json",
@@ -4451,6 +4456,7 @@ dependencies = [
"futures-util",
"log",
"rustls",
+ "rustls-native-certs",
"rustls-pki-types",
"tokio",
"tokio-rustls",
diff --git a/README.md b/README.md
index 912e426c8..5ae98a97d 100644
--- a/README.md
+++ b/README.md
@@ -525,11 +525,19 @@ published for Windows (`win_amd64`), Linux (`x86_64` / `aarch64`) and macOS
(Apple Silicon and Intel), so those platforms never need a local Rust toolchain —
the Rust-first step above is only for the sdist fallback when no wheel matches.
-Two runtime assets are fetched over TLS. If they are blocked, trust your
-corporate CA through `REQUESTS_CA_BUNDLE` / `SSL_CERT_FILE` / `CURL_CA_BUNDLE`:
+**At runtime** Headroom verifies upstream TLS against the operating system's
+certificate store — where IT installs the Zscaler / Netskope / Palo Alto root —
+plus certifi's public roots, so a managed machine normally needs no setup.
+`headroom doctor --network` shows who signed each upstream certificate, whether
+Headroom trusts it, and whether a gateway block page is in the way. If the root
+is not in the OS store, export it as PEM and set
+`HEADROOM_CA_BUNDLE=/path/to/root.pem` (`NODE_EXTRA_CA_CERTS` also works). Full
+guide and the domain list for IT: [Corporate networks](https://docs.headroomlabs.ai/docs/corporate-networks).
-- **`cdn.pyke.io`** — the ONNX Runtime for the Rust core. Or pre-provide it with `ORT_STRATEGY=system` and `ORT_LIB_LOCATION=/path/to/onnxruntime`.
-- **`huggingface.co`** — the `kompress-base` model. Pre-download it and run with `HF_HUB_OFFLINE=1`, or point `HF_ENDPOINT` at a trusted mirror.
+Two download hosts can still need attention:
+
+- **`cdn.pyke.io`** — the ONNX Runtime for the Rust core, fetched with Mozilla's roots only. Pre-provide it with `ORT_STRATEGY=system` and `ORT_LIB_LOCATION=/path/to/onnxruntime`.
+- **`huggingface.co`** — the `kompress-base` model (Python, OS store) and the Rust core's embedding model (Mozilla's roots only). If the domain is blocked, pre-download and run with `HF_HUB_OFFLINE=1`, or point `HF_ENDPOINT` at a trusted mirror.
Running with compression disabled (pure gateway) needs neither asset.
@@ -552,8 +560,9 @@ export ORT_DYLIB_PATH="$(brew --prefix onnxruntime)/lib/libonnxruntime.dylib"
`ORT_PREFER_DYNAMIC_LINK=1` is required — without it `ORT_STRATEGY=system` still
attempts static linking, which the Homebrew keg does not provide.
-**"Basic Constraints of CA cert not marked critical"** is a different failure. If
-TLS fails with:
+**"Basic Constraints of CA cert not marked critical"** only occurs with
+`HEADROOM_CERT_STORE=bundled` (the OS store verifies these roots fine). If TLS
+fails with:
```
[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed:
@@ -575,10 +584,9 @@ signature, expiry and hostname checks all stay on.
HEADROOM_TLS_STRICT=0 headroom proxy --port 8787
```
-The Rust core's ONNX download uses a separate TLS stack (rustls / OS trust store)
-and is unaffected by `HEADROOM_TLS_STRICT`. On Windows the corporate root must be
-in the **machine** certificate store — browsers already trust it there — or
-pre-provision ONNX Runtime with `ORT_STRATEGY=system` to skip the download.
+The Rust core's downloads use rustls with Mozilla's roots and are unaffected by
+`HEADROOM_TLS_STRICT` and the CA variables; pre-provision ONNX Runtime with
+`ORT_STRATEGY=system` and models via `HF_ENDPOINT` as above.
</details>
diff --git a/crates/headroom-proxy/Cargo.toml b/crates/headroom-proxy/Cargo.toml
index c9f2134ad..39c1e9f57 100644
--- a/crates/headroom-proxy/Cargo.toml
+++ b/crates/headroom-proxy/Cargo.toml
@@ -22,8 +22,11 @@ tower = { workspace = true }
tower-http = { version = "0.7", features = ["trace", "request-id", "util"] }
tracing = { workspace = true }
tracing-subscriber = { version = "0.3", features = ["json", "env-filter", "fmt"] }
-reqwest = { version = "0.12", default-features = false, features = ["stream", "rustls-tls", "http2"] }
-tokio-tungstenite = { version = "0.30", default-features = false, features = ["connect", "rustls-tls-webpki-roots"] }
+# Native roots on top of the bundled webpki set: corporate TLS-inspection
+# roots (Zscaler, Netskope, ...) are installed in the OS store, never in
+# Mozilla's list, so webpki alone rejects every re-signed upstream certificate.
+reqwest = { version = "0.12", default-features = false, features = ["stream", "rustls-tls", "rustls-tls-native-roots", "http2"] }
+tokio-tungstenite = { version = "0.30", default-features = false, features = ["connect", "rustls-tls-webpki-roots", "rustls-tls-native-roots"] }
clap = { workspace = true, features = ["derive", "env"] }
serde = { workspace = true }
serde_json = { workspace = true }
@@ -93,6 +96,13 @@ async-trait = "0.1"
# the core uses for the CCR cache_key, so no additional hash backend
# enters the dep tree.
md-5 = "0.11"
+# Explicit rustls config for WebSocket upstreams (see src/tls.rs). Both ring
+# and aws-lc-rs are compiled into this binary, so rustls cannot pick a default
+# provider and `ClientConfig::builder()` panics; we name ring explicitly.
+rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12", "logging"] }
+rustls-native-certs = "0.8"
+rustls-pki-types = { version = "1", features = ["std"] }
+webpki-roots = "0.26"
[dev-dependencies]
tower = { workspace = true, features = ["util"] }
diff --git a/crates/headroom-proxy/src/lib.rs b/crates/headroom-proxy/src/lib.rs
index 3d86580de..a4e29f98a 100644
--- a/crates/headroom-proxy/src/lib.rs
+++ b/crates/headroom-proxy/src/lib.rs
@@ -13,6 +13,7 @@ pub mod observability;
pub mod proxy;
pub mod responses_items;
pub mod sse;
+pub mod tls;
pub mod vertex;
pub mod websocket;
diff --git a/crates/headroom-proxy/src/proxy.rs b/crates/headroom-proxy/src/proxy.rs
index 0389e14c2..650200f86 100644
--- a/crates/headroom-proxy/src/proxy.rs
+++ b/crates/headroom-proxy/src/proxy.rs
@@ -97,7 +97,13 @@ const DRIFT_DETECTOR_CAPACITY: usize = 1000;
impl AppState {
pub fn new(config: Config) -> Result<Self, ProxyError> {
- let client = reqwest::Client::builder()
+ let mut builder = reqwest::Client::builder();
+ // Corporate roots handed over as PEM files, added on top of the
+ // bundled + OS roots (see `crate::tls`).
+ for cert in crate::tls::extra_root_certificates() {
+ builder = builder.add_root_certificate(cert);
+ }
+ let client = builder
.connect_timeout(config.upstream_connect_timeout)
.timeout(config.upstream_timeout)
// Don't auto-follow redirects: pass them through verbatim.
diff --git a/crates/headroom-proxy/src/tls.rs b/crates/headroom-proxy/src/tls.rs
new file mode 100644
index 000000000..eb23982ec
--- /dev/null
+++ b/crates/headroom-proxy/src/tls.rs
@@ -0,0 +1,183 @@
+//! Upstream TLS trust for the Rust proxy.
+//!
+//! Upstreams are verified against Mozilla's bundled roots plus the operating
+//! system store, which is where IT installs a corporate TLS-inspection root
+//! (Zscaler, Netskope, ...). Deployments without such a store (containers,
+//! CI) hand the root over as a PEM file instead; these env vars add it,
+//! matching the Python proxy's policy:
+//!
+//! * `HEADROOM_CA_BUNDLE` — Headroom's own knob.
+//! * `NODE_EXTRA_CA_CERTS` — what Claude Code users already set.
+//!
+//! Both are additive. `SSL_CERT_FILE` / `SSL_CERT_DIR` are honored by
+//! `rustls-native-certs` itself, as a replacement for the OS store.
+//!
+//! HTTP upstreams (reqwest) get the extras via [`extra_root_certificates`];
+//! WebSocket upstreams get a complete rustls config from
+//! [`websocket_tls_config`].
+
+use std::path::Path;
+use std::sync::{Arc, OnceLock};
+
+use rustls_pki_types::pem::PemObject;
+use rustls_pki_types::CertificateDer;
+
+/// Env vars naming additive PEM bundles, in the order they are loaded.
+pub const ADDITIVE_CA_VARS: [&str; 2] = ["HEADROOM_CA_BUNDLE", "NODE_EXTRA_CA_CERTS"];
+
+/// Root certificates from the additive env vars, for reqwest. Missing or
+/// unparsable files are logged and skipped: a bad path must not take the
+/// proxy down.
+pub fn extra_root_certificates() -> Vec<reqwest::Certificate> {
+ extra_root_ders()
+ .iter()
+ .filter_map(|der| reqwest::Certificate::from_der(der.as_ref()).ok())
+ .collect()
+}
+
+/// rustls client config for `wss://` upstreams: webpki + OS + additive roots,
+/// on an explicit crypto provider. Built once per process.
+pub fn websocket_tls_config() -> Arc<rustls::ClientConfig> {
+ static CONFIG: OnceLock<Arc<rustls::ClientConfig>> = OnceLock::new();
+ CONFIG
+ .get_or_init(|| Arc::new(build_websocket_tls_config(&extra_root_ders())))
+ .clone()
+}
+
+fn build_websocket_tls_config(extra: &[CertificateDer<'static>]) -> rustls::ClientConfig {
+ let mut roots = rustls::RootCertStore::empty();
+ roots.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned());
+ let native = rustls_native_certs::load_native_certs();
+ if !native.errors.is_empty() {
+ tracing::warn!(
+ event = "tls_native_roots_errors",
+ errors = ?native.errors,
+ "some OS root certificates could not be loaded"
+ );
+ }
+ roots.add_parsable_certificates(native.certs);
+ roots.add_parsable_certificates(extra.iter().cloned());
+
+ let provider = Arc::new(rustls::crypto::ring::default_provider());
+ let mut config = rustls::ClientConfig::builder_with_provider(provider)
+ .with_safe_default_protocol_versions()
+ .expect("ring supports the default TLS versions")
+ .with_root_certificates(roots)
+ .with_no_client_auth();
+ // A WebSocket upgrade is an HTTP/1.1 request.
+ config.alpn_protocols = vec![b"http/1.1".to_vec()];
+ config
+}
+
+fn extra_root_ders() -> Vec<CertificateDer<'static>> {
+ extra_root_ders_from(|var| std::env::var(var).ok())
+}
+
+fn extra_root_ders_from(lookup: impl Fn(&str) -> Option<String>) -> Vec<CertificateDer<'static>> {
+ let mut certs = Vec::new();
+ for var in ADDITIVE_CA_VARS {
+ let Some(path) = lookup(var).filter(|p| !p.is_empty()) else {
+ continue;
+ };
+ match load_pem_bundle(Path::new(&path)) {
+ Ok(found) => {
+ tracing::info!(
+ event = "tls_ca_bundle_loaded",
+ env_var = var,
+ path = %path,
+ certificates = found.len(),
+ "loaded extra upstream root certificates"
+ );
+ certs.extend(found);
+ }
+ Err(err) => {
+ tracing::warn!(
+ event = "tls_ca_bundle_skipped",
+ env_var = var,
+ path = %path,
+ error = %err,
+ "could not load extra root certificates; skipping"
+ );
+ }
+ }
+ }
+ certs
+}
+
+fn load_pem_bundle(path: &Path) -> Result<Vec<CertificateDer<'static>>, String> {
+ let certs = CertificateDer::pem_file_iter(path)
+ .map_err(|e| e.to_string())?
+ .collect::<Result<Vec<_>, _>>()
+ .map_err(|e| e.to_string())?;
+ if certs.is_empty() {
+ return Err("no PEM certificates found".to_string());
+ }
+ Ok(certs)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ // Throwaway self-signed CA, only parsed here, never used for a handshake.
+ const TEST_CA: &str = "-----BEGIN CERTIFICATE-----
+MIIBuTCCAV+gAwIBAgIUQJOjUYts91bSLsemwb5EuzdkOnMwCgYIKoZIzj0EAwIw
+MjEVMBMGA1UECgwMWnNjYWxlciBJbmMuMRkwFwYDVQQDDBBIZWFkcm9vbSBUZXN0
+IENBMB4XDTI2MDkyODE0MjM1MloXDTM2MDkyNTE0MjM1MlowMjEVMBMGA1UECgwM
+WnNjYWxlciBJbmMuMRkwFwYDVQQDDBBIZWFkcm9vbSBUZXN0IENBMFkwEwYHKoZI
+zj0CAQYIKoZIzj0DAQcDQgAEKZ0h9e4jj/eJiBVh4eMZ3d+pcugxj/hEhqzoAo9r
++7KL4dGgqrg2GH4IP6sfKNdssKHshDcjz+HWiKVGyCx08KNTMFEwHQYDVR0OBBYE
+FPFmGy3B6aRYxGMB1P/up4Y05PxbMB8GA1UdIwQYMBaAFPFmGy3B6aRYxGMB1P/u
+p4Y05PxbMA8GA1UdEwEB/wQFMAMBAf8wCgYIKoZIzj0EAwIDSAAwRQIgZTHC/ZR0
+SI8i1dYBBaT8a2e/CVF6MmgXObYWEg2YYk8CIQC8e2+jUZDkYV+Ct3y0JQO7OSfp
++Yyjbym7JFsfsp4fVQ==
+-----END CERTIFICATE-----
+";
+
+ fn write_temp(name: &str, contents: &str) -> String {
+ let dir = std::env::temp_dir().join(format!("headroom-tls-{}", std::process::id()));
+ std::fs::create_dir_all(&dir).unwrap();
+ let path = dir.join(name);
+ std::fs::write(&path, contents).unwrap();
+ path.display().to_string()
+ }
+
+ #[test]
+ fn unset_and_empty_vars_add_nothing() {
+ assert!(extra_root_ders_from(|_| None).is_empty());
+ assert!(extra_root_ders_from(|_| Some(String::new())).is_empty());
+ }
+
+ #[test]
+ fn missing_file_is_skipped() {
+ let certs = extra_root_ders_from(|var| {
+ (var == "HEADROOM_CA_BUNDLE").then(|| "/nonexistent/headroom-ca.pem".to_string())
+ });
+ assert!(certs.is_empty());
+ }
+
+ #[test]
+ fn non_pem_file_is_skipped() {
+ let p = write_temp("not-a-cert.pem", "hello");
+ let certs = extra_root_ders_from(|var| (var == "NODE_EXTRA_CA_CERTS").then(|| p.clone()));
+ assert!(certs.is_empty());
+ }
+
+ #[test]
+ fn both_additive_vars_are_loaded() {
+ let p = write_temp("corp-root.pem", TEST_CA);
+ let certs = extra_root_ders_from(|_| Some(p.clone()));
+ assert_eq!(certs.len(), ADDITIVE_CA_VARS.len());
+ assert!(reqwest::Certificate::from_der(certs[0].as_ref()).is_ok());
+ }
+
+ #[test]
+ fn websocket_config_builds_without_a_process_default_provider() {
+ // rustls' own `ClientConfig::builder()` panics in this binary (two
+ // providers compiled in); ours must not, and must trust the extras.
+ let p = write_temp("corp-root-ws.pem", TEST_CA);
+ let extra = extra_root_ders_from(|var| (var == "HEADROOM_CA_BUNDLE").then(|| p.clone()));
+ let config = build_websocket_tls_config(&extra);
+ assert_eq!(config.alpn_protocols, vec![b"http/1.1".to_vec()]);
+ }
+}
diff --git a/crates/headroom-proxy/src/websocket.rs b/crates/headroom-proxy/src/websocket.rs
index 3a1be2e4f..65d9b2b8c 100644
--- a/crates/headroom-proxy/src/websocket.rs
+++ b/crates/headroom-proxy/src/websocket.rs
@@ -151,9 +151,15 @@ async fn run_ws_pump(
}
}
- let (upstream_ws, _resp) = tokio_tungstenite::connect_async(req)
- .await
- .map_err(|e| format!("upstream ws connect: {e}"))?;
+ // Explicit TLS config: corporate roots (OS store + HEADROOM_CA_BUNDLE) and
+ // a named crypto provider. tokio-tungstenite's default path calls
+ // `rustls::ClientConfig::builder()`, which panics in this binary because
+ // two rustls providers are compiled in.
+ let connector = tokio_tungstenite::Connector::Rustls(crate::tls::websocket_tls_config());
+ let (upstream_ws, _resp) =
+ tokio_tungstenite::connect_async_tls_with_config(req, None, false, Some(connector))
+ .await
+ .map_err(|e| format!("upstream ws connect: {e}"))?;
let (mut upstream_sink, mut upstream_stream) = upstream_ws.split();
let (mut client_sink, mut client_stream) = client_ws.split();
diff --git a/docs/content/docs/corporate-networks.mdx b/docs/content/docs/corporate-networks.mdx
new file mode 100644
index 000000000..df314cdf5
--- /dev/null
+++ b/docs/content/docs/corporate-networks.mdx
@@ -0,0 +1,97 @@
+---
+title: Corporate Networks (Zscaler, Netskope, TLS Inspection)
+description: Run Headroom behind Zscaler, Netskope, Palo Alto, Cisco Secure Access and other TLS-inspecting gateways, and what to ask IT for when something is still blocked.
+---
+
+Many companies route laptop traffic through a gateway that **inspects TLS**: it opens each HTTPS connection, checks it, and re-signs it with the company's own root certificate. IT installs that root in the operating system's certificate store, which is why browsers, `curl` and Claude Code keep working.
+
+Headroom verifies upstream certificates against the **same OS certificate store** (macOS Keychain, Windows certificate store, the system bundle on Linux), plus the public roots bundled with Python. On a managed machine this usually needs no configuration.
+
+## Quick check
+
+```bash
+headroom doctor --network
+```
+
+For each endpoint Headroom uses, this prints who signed the certificate the network presented (for example `Zscaler Intermediate Root CA`), whether Headroom trusts it, and whether a gateway block page was returned instead of the service. If anything fails, the last row is a short request you can send to IT as-is.
+
+Add `--network-url https://your-gateway.example.com/v1/models` to test a custom upstream too.
+
+## If a certificate is still not trusted
+
+The error Claude Code (or Codex) shows will read like:
+
+> Headroom could not verify the TLS certificate for api.anthropic.com (unable to get local issuer certificate). The certificate was issued by 'O=Zscaler Inc., CN=Zscaler Intermediate Root CA': Zscaler is inspecting this connection. ...
+
+Pick one fix:
+
+1. **Ask IT to deploy the root to the OS certificate store.** This is the normal setup for Zscaler Client Connector ("Install Zscaler SSL Certificate") and most MDM profiles. Nothing else to change.
+2. **Point Headroom at the root as a file.** Export it (your browser's certificate viewer, or IT) as PEM and set:
+
+ ```bash
+ export HEADROOM_CA_BUNDLE=/path/to/company-root.pem
+ ```
+
+ `HEADROOM_CA_BUNDLE` and `NODE_EXTRA_CA_CERTS` are **added** to the trusted set. If you already set `NODE_EXTRA_CA_CERTS` for Claude Code, Headroom picks it up.
+3. **Ask IT to exempt the provider domains from TLS inspection** (see the list below).
+
+<Callout type="warn" title="Don't disable verification">
+Never turn certificate checking off to get past this. Headroom has no switch for that on purpose.
+</Callout>
+
+## Trust settings
+
+| Env var | Default | Effect |
+|---|---|---|
+| `HEADROOM_CERT_STORE` | `system,bundled` | Where trusted roots come from: `system` = OS certificate store, `bundled` = certifi's public roots. Set `bundled` to go back to the previous behavior (public roots only). Mirrors Claude Code's `CLAUDE_CODE_CERT_STORE`. |
+| `HEADROOM_CA_BUNDLE` | unset | Extra PEM roots, added to the above. |
+| `NODE_EXTRA_CA_CERTS` | unset | Same as `HEADROOM_CA_BUNDLE` (shared with Claude Code). |
+| `SSL_CERT_FILE` / `REQUESTS_CA_BUNDLE` | unset | **Replace** every other source: only this bundle is trusted. Use only if the bundle is complete. |
+| `HEADROOM_TLS_STRICT` | strict | `0` relaxes OpenSSL's RFC 5280 strict checks on the bundled path. Not needed with the OS store; kept for `HEADROOM_CERT_STORE=bundled`. |
+
+The same policy covers the proxy's HTTP and WebSocket upstreams, GitHub Copilot sign-in, and model and tokenizer downloads made from Python. The Rust proxy binary (`headroom-proxy`, used in the Docker image) trusts Mozilla's roots plus the OS store for all upstreams, and adds `HEADROOM_CA_BUNDLE` / `NODE_EXTRA_CA_CERTS` for HTTP upstreams.
+
+One gap remains: the embedding model that the Rust core fetches for relevance scoring still trusts Mozilla's roots only. Behind inspection that download fails and Headroom falls back to BM25 scoring; use `HF_ENDPOINT` (below) or pre-populate the cache to keep embeddings.
+
+In a container there is usually no corporate root in the OS store: mount it and set `HEADROOM_CA_BUNDLE`.
+
+## HTTP proxies
+
+If your company uses an explicit proxy (`HTTPS_PROXY` set, or a PAC file), Headroom's upstream calls go through it. The agent's calls **to** Headroom must not: they target `127.0.0.1`, which the corporate proxy cannot reach. `headroom wrap` adds `127.0.0.1,localhost,::1` to `NO_PROXY` for the processes it launches; if you start the agent yourself, set:
+
+```bash
+export NO_PROXY="$NO_PROXY,127.0.0.1,localhost,::1"
+```
+
+PAC files are not evaluated by Python. Export the proxy the PAC file resolves to as `HTTPS_PROXY`.
+
+## What to ask IT for
+
+Headroom runs on the developer's machine and talks to the same services the AI tools already use, plus two download hosts:
+
+| Domain | Used for | Required |
+|---|---|---|
+| `api.anthropic.com` | Claude / Claude Code | yes, for Anthropic |
+| `api.openai.com`, `chatgpt.com` | OpenAI API, Codex | yes, for OpenAI / Codex |
+| `api.githubcopilot.com`, `api.github.com` | GitHub Copilot | for Copilot |
+| `huggingface.co` and the storage hosts it redirects to (`cdn-lfs.hf.co`, `cas-bridge.xethub.hf.co`) | Compression models (downloaded once) | recommended |
+| `openaipublic.blob.core.windows.net` | Tokenizer vocabulary (downloaded once) | recommended |
+
+Things that help:
+
+- **Deploy the inspection root to the OS store.** This is enough for Headroom; no per-app certificate work is needed.
+- **Do not route these domains to "Caution" or "Isolate".** An API client cannot click through an interstitial page; it sees HTML where it expected JSON.
+- **Streaming:** some gateways buffer server-sent events while inspecting them, so responses arrive in bursts or time out. If users see stalled streams, exempt the provider domains from inspection or DLP buffering.
+- **Please don't exempt `python` as a process** (for example Zscaler's process-based bypass). Headroom does not need it once the root is trusted, and it would exempt every Python program on the machine.
+
+If your company uses Anthropic **tenant restrictions**, those require inspection of `api.anthropic.com` to stay on. That works with Headroom as long as the root is trusted.
+
+## Model downloads when huggingface.co is blocked
+
+Headroom degrades instead of failing: without the compression model it falls back to rule-based compression, and without the tokenizer vocabulary it estimates token counts. To keep ML compression on a locked-down network, point Headroom at an internal Hugging Face mirror (Artifactory, Nexus):
+
+```bash
+export HF_ENDPOINT=https://artifactory.example.com/api/huggingface/huggingface
+```
+
+Or pre-populate `~/.cache/huggingface` from a machine that has access.
diff --git a/docs/content/docs/meta.json b/docs/content/docs/meta.json
index d716df852..dc2e8b970 100644
--- a/docs/content/docs/meta.json
+++ b/docs/content/docs/meta.json
@@ -24,6 +24,7 @@
"failure-learning",
"---Proxy Server---",
"proxy",
+ "corporate-networks",
"local-llm-prefill",
"---Integrations---",
"vercel-ai-sdk",
diff --git a/docs/content/docs/proxy.mdx b/docs/content/docs/proxy.mdx
index eddee0d8d..ca5fa31c9 100644
--- a/docs/content/docs/proxy.mdx
+++ b/docs/content/docs/proxy.mdx
@@ -339,6 +339,8 @@ with a privileged job.
| `--stateless` / `HEADROOM_STATELESS` | `false` | Keep all state in memory; no filesystem writes (disables logs, memory, TOIN). |
| `HEADROOM_STRIP_INTERNAL_HEADERS` | `enabled` | Strip internal `x-headroom-*` headers before forwarding upstream. |
| `HEADROOM_TLS_STRICT` | strict | Set `0` to relax CA-constraint checks behind a corporate TLS-inspection proxy. |
+| `HEADROOM_CERT_STORE` | `system,bundled` | Trusted roots for upstream TLS: OS certificate store and/or certifi. See [Corporate Networks](/docs/corporate-networks). |
+| `HEADROOM_CA_BUNDLE` | unset | Extra PEM roots (e.g. a Zscaler root) added to the trusted set. |
### Performance
diff --git a/headroom/cli/doctor.py b/headroom/cli/doctor.py
index 3be931148..62cdf1e3a 100644
--- a/headroom/cli/doctor.py
+++ b/headroom/cli/doctor.py
@@ -482,6 +482,167 @@ def _codex_block_missing_openai_auth(
return False
+def check_trust_policy(policy: Mapping[str, Any]) -> CheckResult:
+ """Which certificates the proxy trusts for its upstream connections.
+
+ Corporate TLS inspection (Zscaler, Netskope, ...) re-signs every upstream
+ certificate with a company root that IT installs in the OS store. The proxy
+ only accepts it when the OS store is in use or the root is supplied as a
+ bundle, so this line answers "will Headroom trust what my network presents?"
+ """
+ replacement = policy.get("replacement_bundle")
+ additive = policy.get("additive_bundles") or []
+ extras = ", ".join(f"{b['env_var']}={b['path']}" for b in additive)
+ if replacement:
+ return CheckResult(
+ name="tls trust",
+ status=PASS,
+ summary=f"only {replacement['env_var']}={replacement['path']}",
+ hint=(
+ f"{replacement['env_var']} replaces every other trust source; the OS "
+ "certificate store is not consulted. Unset it to trust the OS store."
+ ),
+ )
+ if policy.get("system_store_active"):
+ sources = "OS certificate store" + (
+ " + certifi" if "bundled" in policy.get("cert_store", []) else ""
+ )
+ return CheckResult(
+ name="tls trust",
+ status=PASS,
+ summary=sources + (f" + {extras}" if extras else ""),
+ )
+ reason = (
+ "HEADROOM_CERT_STORE excludes 'system'"
+ if policy.get("system_store_available")
+ else "the truststore package is not installed"
+ )
+ return CheckResult(
+ name="tls trust",
+ status=WARN,
+ summary="certifi bundle only" + (f" + {extras}" if extras else ""),
+ hint=(
+ f"The OS certificate store is not used ({reason}), so a corporate "
+ "TLS-inspection root is only trusted if you pass it via HEADROOM_CA_BUNDLE."
+ ),
+ )
+
+
+def check_proxy_env(environ: Mapping[str, str]) -> CheckResult | None:
+ """Warn when an HTTP(S) proxy would capture the agent's loopback traffic."""
+ from headroom.proxy.tls_diagnostics import loopback_no_proxy_gap
+
+ gap = loopback_no_proxy_gap(environ)
+ if gap is None:
+ return None
+ return CheckResult(
+ name="proxy env",
+ status=WARN,
+ summary=f"{gap} is set but NO_PROXY does not exempt 127.0.0.1/localhost",
+ hint=(
+ "Agents may send their requests to Headroom through the corporate proxy, "
+ "which cannot reach this machine. `headroom wrap` fixes this for the "
+ "processes it launches; for other setups add "
+ "NO_PROXY=127.0.0.1,localhost,::1."
+ ),
+ )
+
+
+# (label, url, required). Required endpoints fail the check; the rest only warn
+# because Headroom degrades (no ML compression, estimated token counts) rather
+# than breaking when they are unreachable.
+NETWORK_ENDPOINTS: tuple[tuple[str, str, bool], ...] = (
+ ("api.anthropic.com", "https://api.anthropic.com/v1/models", True),
+ ("api.openai.com", "https://api.openai.com/v1/models", True),
+ ("api.githubcopilot.com", "https://api.githubcopilot.com/models", False),
+ ("huggingface.co (models)", "https://huggingface.co/api/models?limit=1", False),
+ (
+ "openaipublic (tiktoken)",
+ "https://openaipublic.blob.core.windows.net/encodings/",
+ False,
+ ),
+)
+
+
+def check_network_endpoints(reports: Sequence[Any], required: set[str]) -> list[CheckResult]:
+ """Turn :class:`EndpointReport`s into doctor rows, plus an IT summary row."""
+ results: list[CheckResult] = []
+ vendors: set[str] = set()
+ failing_hosts: list[str] = []
+ for report in reports:
+ vendor = report.chain.inspection_vendor
+ if vendor:
+ vendors.add(vendor)
+ inspected = f"; TLS inspected by {vendor}" if vendor else ""
+ host = urlsplit_host(report.url)
+ if report.ok:
+ results.append(
+ CheckResult(
+ name=f"net {report.name}",
+ status=PASS,
+ summary=(
+ f"reachable, certificate trusted (HTTP {report.status}, "
+ f"{report.elapsed_ms:.0f} ms){inspected}"
+ ),
+ )
+ )
+ continue
+ failing_hosts.append(host)
+ status = FAIL if report.name in required else WARN
+ if report.block_page:
+ summary = f"blocked by a network gateway (HTTP {report.status}){inspected}"
+ elif report.chain.issuer and "verify" in (report.error or "").lower():
+ summary = f"certificate not trusted (issued by {report.chain.issuer})"
+ else:
+ summary = (report.error or "unreachable").split(". ")[0][:160]
+ results.append(
+ CheckResult(
+ name=f"net {report.name}", status=status, summary=summary, hint=report.error
+ )
+ )
+ if failing_hosts:
+ who = " / ".join(sorted(vendors)) or "your TLS-inspection gateway"
+ results.append(
+ CheckResult(
+ name="it request",
+ status=WARN,
+ summary=f"{len(failing_hosts)} endpoint(s) need a network change",
+ hint=(
+ f"Text for your IT team: Headroom (a local AI coding proxy) needs HTTPS "
+ f"access to {', '.join(failing_hosts)}. Either confirm the {who} root "
+ "certificate is deployed to the operating system certificate store on "
+ "developer machines, or exempt these domains from TLS inspection and "
+ "cloud-app blocking. Please do not exempt 'python' as a process; the "
+ "certificate fix is sufficient."
+ ),
+ )
+ )
+ return results
+
+
+def urlsplit_host(url: str) -> str:
+ from urllib.parse import urlsplit
+
+ return urlsplit(url).hostname or url
+
+
+def run_network_probes(extra_urls: Sequence[str]) -> tuple[list[Any], set[str]]:
+ """Probe every endpoint concurrently; returns (reports, required names)."""
+ from concurrent.futures import ThreadPoolExecutor
+
+ from headroom.proxy.tls_diagnostics import probe_endpoint
+
+ targets = [(label, url) for label, url, _ in NETWORK_ENDPOINTS]
+ required = {label for label, _, req in NETWORK_ENDPOINTS if req}
+ for url in extra_urls:
+ label = urlsplit_host(url)
+ targets.append((label, url))
+ required.add(label)
+ with ThreadPoolExecutor(max_workers=len(targets)) as pool:
+ reports = list(pool.map(lambda t: probe_endpoint(*t), targets))
+ return reports, required
+
+
def check_shell_env(environ: Mapping[str, str], port: int) -> CheckResult:
"""Is the *current shell* pointed at the proxy for ad-hoc runs?"""
name = "shell env"
@@ -751,7 +912,23 @@ def _render(checks: list[CheckResult], port: int, installed: str) -> None:
help="Proxy port to check (default: 8787, env: HEADROOM_PORT)",
)
@click.option("--json", "emit_json", is_flag=True, help="Emit JSON instead of formatted output.")
-def doctor(port: int, emit_json: bool) -> None:
+ "--network",
+ is_flag=True,
+ help=(
+ "Also test HTTPS to the provider APIs and model hosts: who signed the "
+ "certificate (Zscaler, Netskope, ...), whether Headroom trusts it, and "
+ "whether a gateway block page is in the way."
+ ),
+)
+ "--network-url",
+ "network_urls",
+ multiple=True,
+ metavar="URL",
+ help="Extra upstream URL to test with --network (repeatable), e.g. a custom gateway.",
+)
+def doctor(port: int, emit_json: bool, network: bool, network_urls: tuple[str, ...]) -> None:
"""Check that the Headroom proxy and client routing are working.
\b
@@ -783,6 +960,21 @@ def doctor(port: int, emit_json: bool) -> None:
check_savings(stats, savings_path()),
check_budget(stats),
]
+ from headroom.proxy.ssl_context import describe_trust_policy
+
+ # Prefer the running proxy's own policy (its env can differ from this
+ # shell's under launchd/systemd); fall back to what this shell would use.
+ proxy_tls = ((health or {}).get("config") or {}).get("tls")
+ trust_row = check_trust_policy(proxy_tls or describe_trust_policy())
+ if not proxy_tls:
+ trust_row.summary += " (this shell; proxy not reporting)"
+ checks.append(trust_row)
+ proxy_env_check = check_proxy_env(os.environ)
+ if proxy_env_check is not None:
+ checks.append(proxy_env_check)
+ if network or network_urls:
+ reports, required = run_network_probes(network_urls)
+ checks.extend(check_network_endpoints(reports, required))
auth_conflict_check = check_claude_auth_conflict(
claude_settings_path(),
project_claude_settings,
diff --git a/headroom/cli/main.py b/headroom/cli/main.py
index e2b8755e6..bd2fd7fce 100644
--- a/headroom/cli/main.py
+++ b/headroom/cli/main.py
@@ -45,6 +45,16 @@ def main(ctx: click.Context) -> None:
except Exception: # noqa: BLE001 — settings load must never break the CLI
pass
+ # Verify TLS through the OS trust store for every client in this process
+ # (after settings.json, which may set HEADROOM_CERT_STORE). Corporate
+ # TLS-inspection roots live there; certifi does not have them.
+ try:
+ from headroom.proxy.ssl_context import ensure_process_trust
+
+ ensure_process_trust()
+ except Exception: # noqa: BLE001 — trust wiring must never break the CLI
+ pass
+
# Fire a rate-limited, opt-out background check for newer releases so other
# surfaces (e.g. the proxy banner) can show an "update available" notice.
# Never blocks, never raises; skipped for `update` (it checks explicitly).
diff --git a/headroom/cli/wrap.py b/headroom/cli/wrap.py
index c9b33ae5f..0388c79f3 100644
--- a/headroom/cli/wrap.py
+++ b/headroom/cli/wrap.py
@@ -33,7 +33,7 @@ import subprocess
import sys
import time
import urllib.parse
-from collections.abc import Callable, Mapping
+from collections.abc import Callable, Mapping, MutableMapping
from contextlib import contextmanager
from functools import wraps
from pathlib import Path
@@ -5194,6 +5194,47 @@ def wrap(ctx: click.Context) -> None:
"""
if _should_purge_context_tools(ctx):
_report_context_tool_purge()
+ # Every wrap path copies os.environ into the launched agent and the proxy,
+ # and probes the proxy over loopback itself, so one fix here covers all.
+ added = _ensure_loopback_no_proxy(os.environ)
+ if added:
+ # stderr: `wrap <tool> --prepare-only` prints machine-read JSON on stdout.
+ click.echo(
+ f" Added 127.0.0.1,localhost,::1 to {'/'.join(added)} so the agent reaches "
+ "Headroom directly instead of through your HTTP proxy.",
+ err=True,
+ )
+
+
+_LOOPBACK_NO_PROXY_HOSTS = ("127.0.0.1", "localhost", "::1")
+
+
+def _ensure_loopback_no_proxy(env: MutableMapping[str, str]) -> list[str]:
+ """Exempt loopback from ``HTTP(S)_PROXY`` in ``env``; return the vars written.
+
+ With a corporate proxy exported and no loopback entry in ``NO_PROXY``, an
+ agent sends its ``http://127.0.0.1:<port>`` requests to the corporate proxy,
+ which cannot reach this machine and answers with a block page or 502.
+ ``NO_PROXY`` and ``no_proxy`` are merged before writing both, because tools
+ disagree on which spelling wins and dropping either side's entries would
+ re-route the user's own internal hosts.
+ """
+ from headroom.proxy.tls_diagnostics import loopback_no_proxy_gap
+
+ if loopback_no_proxy_gap(env) is None:
+ return []
+ merged: list[str] = []
+ for var in ("NO_PROXY", "no_proxy"):
+ for entry in env.get(var, "").split(","):
+ entry = entry.strip()
+ if entry and entry not in merged:
+ merged.append(entry)
+ merged.extend(h for h in _LOOPBACK_NO_PROXY_HOSTS if h not in merged)
+ value = ",".join(merged)
+ written = [var for var in ("NO_PROXY", "no_proxy") if env.get(var) != value]
+ for var in written:
+ env[var] = value
+ return written
@main.group()
diff --git a/headroom/providers/codex/live.py b/headroom/providers/codex/live.py
index 435873c53..7a3414198 100644
--- a/headroom/providers/codex/live.py
+++ b/headroom/providers/codex/live.py
@@ -23,6 +23,7 @@ from headroom.proxy.helpers import (
merge_extra_headers,
sanitize_forwarded_response_headers,
)
+from headroom.proxy.ssl_context import build_websocket_ssl
from headroom.proxy.ws_headers import WS_HOP_BY_HOP_HEADERS
logger = logging.getLogger("headroom.providers.codex.live")
@@ -226,7 +227,7 @@ async def handle_codex_live_websocket(
additional_headers=forwarded_headers,
# websockets types wire protocol tokens as a nominal wrapper.
subprotocols=cast(Any, subprotocols or None),
- ssl=True if upstream_url.startswith("wss://") else None,
+ ssl=build_websocket_ssl() if upstream_url.startswith("wss://") else None,
open_timeout=max(30, getattr(config, "connect_timeout_seconds", 10) * 3),
close_timeout=10,
ping_interval=20,
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.py
index 4e1b39d0f..584bcfafd 100644
--- a/headroom/proxy/handlers/anthropic.py
+++ b/headroom/proxy/handlers/anthropic.py
@@ -5106,6 +5106,19 @@ class AnthropicHandlerMixin:
# Log full error details internally for debugging
logger.error(f"[{request_id}] Request failed: {type(e).__name__}: {e}")
+ # An untrusted TLS-inspection root is the one failure worth
+ # spelling out: it is environmental, never transient, and the
+ # message names only the certificate issuer and the fix.
+ from headroom.proxy.tls_diagnostics import describe_upstream_failure_async
+
+ # Probe the host that actually failed: `url` is the per-request
+ # upstream (Copilot, a custom gateway) once it has been built;
+ # an exception raised before that point never reached a host.
+ _failed_url = locals().get("url")
+ tls_hint = await describe_upstream_failure_async(
+ e, _failed_url if isinstance(_failed_url, str) else self.ANTHROPIC_API_URL
+ )
+
# Return sanitized error message to client (don't expose internal details)
return JSONResponse(
status_code=502,
@@ -5113,7 +5126,8 @@ class AnthropicHandlerMixin:
"type": "error",
"error": {
"type": "api_error",
- "message": "An error occurred while processing your request. Please try again.",
+ "message": tls_hint
+ or "An error occurred while processing your request. Please try again.",
},
},
)
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py
index 21b94d5ad..498734780 100644
--- a/headroom/proxy/handlers/openai.py
+++ b/headroom/proxy/handlers/openai.py
@@ -15,6 +15,7 @@ import logging
import os
import re
import secrets
+import ssl
import threading
import time
import uuid
@@ -7227,10 +7228,15 @@ class OpenAIHandlerMixin:
# --- Connect to upstream OpenAI WebSocket ---
logger.info(f"[{request_id}] WS /v1/responses connecting to {upstream_url}")
- # Use ssl=True to let the websockets library handle SSL natively.
- # Manual ssl.create_default_context() + certifi doesn't load the
- # Windows system cert store, causing HTTP 500 on wss:// connections.
- use_ssl: bool | None = True if upstream_url.startswith("wss://") else None
+ # Same trust policy as the HTTP upstream client: the OS trust store
+ # (Windows machine store, macOS Keychain) plus any configured
+ # corporate bundle. Plain certifi misses both, causing HTTP 500 on
+ # wss:// connections behind TLS inspection.
+ from headroom.proxy.ssl_context import build_websocket_ssl
+
+ use_ssl: ssl.SSLContext | bool | None = (
+ build_websocket_ssl() if upstream_url.startswith("wss://") else None
+ )
ws_connected = False
ws_connect_attempts = max(1, getattr(self.config, "retry_max_attempts", 3))
@@ -9296,6 +9302,11 @@ class OpenAIHandlerMixin:
f"[{request_id}] WS upstream failed ({_ws_detail}), "
f"falling back to HTTP POST streaming"
)
+ # Logs the certificate issuer + fix once per host when the
+ # failure is an untrusted TLS-inspection root.
+ from headroom.proxy.tls_diagnostics import describe_upstream_failure_async
+
+ await describe_upstream_failure_async(ws_err, upstream_url)
(
fb_input_tokens,
fb_output_tokens,
@@ -10967,12 +10978,15 @@ class OpenAIHandlerMixin:
url,
e,
)
+ from headroom.proxy.tls_diagnostics import describe_upstream_failure_async
+
+ tls_hint = await describe_upstream_failure_async(e, url)
return Response(
content=json.dumps(
{
"error": {
"type": "connection_error",
- "message": f"Failed to connect to upstream API: {e}",
+ "message": tls_hint or f"Failed to connect to upstream API: {e}",
}
}
),
@@ -11155,12 +11169,15 @@ class OpenAIHandlerMixin:
url,
e,
)
+ from headroom.proxy.tls_diagnostics import describe_upstream_failure_async
+
+ tls_hint = await describe_upstream_failure_async(e, url)
return Response(
content=json.dumps(
{
"error": {
"type": "connection_error",
- "message": f"Failed to connect to upstream API: {e}",
+ "message": tls_hint or f"Failed to connect to upstream API: {e}",
}
}
),
diff --git a/headroom/proxy/handlers/streaming.py b/headroom/proxy/handlers/streaming.py
index fc6f8af8e..c52885384 100644
--- a/headroom/proxy/handlers/streaming.py
+++ b/headroom/proxy/handlers/streaming.py
@@ -1057,13 +1057,20 @@ class StreamingMixin:
error_msg = str(e) or repr(e)
logger.error(f"[{request_id}] Connection error to upstream API: {error_msg}")
self.metrics.record_upstream_connection_error(provider)
+ # A corporate TLS-inspection root the proxy does not trust shows up
+ # here as CERTIFICATE_VERIFY_FAILED. Name the issuer and the fix so
+ # the agent's error line is actionable instead of a bare 502.
+ from headroom.proxy.tls_diagnostics import describe_upstream_failure_async
+
+ tls_hint = await describe_upstream_failure_async(e, url)
+ client_message = tls_hint or f"Failed to connect to upstream API: {error_msg}"
async def _error_gen():
error_event = {
"type": "error",
"error": {
"type": "connection_error",
- "message": f"Failed to connect to upstream API: {error_msg}",
+ "message": client_message,
},
}
yield f"event: error\ndata: {json.dumps(error_event)}\n\n".encode()
diff --git a/headroom/proxy/server.py b/headroom/proxy/server.py
index a656f73e1..9a0ddf14c 100644
--- a/headroom/proxy/server.py
+++ b/headroom/proxy/server.py
@@ -182,7 +182,11 @@ from headroom.proxy.rate_limiter import TokenBucketRateLimiter # noqa: F401
from headroom.proxy.request_logger import RequestLogger # noqa: F401
from headroom.proxy.savings_tracker import LITELLM_AVAILABLE
from headroom.proxy.semantic_cache import SemanticCache # noqa: F401
-from headroom.proxy.ssl_context import build_httpx_verify
+from headroom.proxy.ssl_context import (
+ build_httpx_verify,
+ describe_trust_policy,
+ ensure_process_trust_if_owned,
+)
from headroom.proxy.tool_schema_savings_policy import tool_schema_saved_from_tags
from headroom.proxy.upstream_pinning import install_upstream_pinning
from headroom.proxy.warmup import WarmupRegistry
@@ -1964,9 +1968,14 @@ class HeadroomProxy(
operation="proxy.startup",
metadata={"port": self.config.port, "host": self.config.host},
)
- # Resolve TLS verification: a custom CA bundle (corporate PKI) if one
- # is configured, else a strict-relaxed default context when
+ # Resolve TLS verification: the OS trust store (where IT installs a
+ # corporate TLS-inspection root) plus certifi by default, else a custom
+ # CA bundle, else a strict-relaxed default context when
# HEADROOM_TLS_STRICT=0, else httpx's default strict verification.
+ # Injection covers the third-party clients that build their own
+ # contexts; it also runs here because uvicorn workers can import the
+ # app without going through the CLI (they inherit the CLI's marker).
+ ensure_process_trust_if_owned()
_verify = build_httpx_verify()
_http2, _client_kwargs = _provider_httpx_client_options(self.config, _verify)
# `install_upstream_pinning` is what makes the SSRF guard's verdict
@@ -3535,6 +3544,9 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
profile_kwargs.get("target_ratio", config.target_ratio),
)
payload["config"] = {
+ # What *this* process trusts upstream; doctor reads it so it
+ # reports the proxy's policy, not the calling shell's.
+ "tls": describe_trust_policy(),
"backend": config.backend,
"optimize": config.optimize,
"cache": config.cache_enabled,
diff --git a/headroom/proxy/ssl_context.py b/headroom/proxy/ssl_context.py
index 9e8396e31..c47f400e8 100644
--- a/headroom/proxy/ssl_context.py
+++ b/headroom/proxy/ssl_context.py
@@ -1,16 +1,27 @@
-"""SSL context builder for the Headroom upstream httpx client.
-
-Respects the standard CA-bundle environment variables used by Python
-(``SSL_CERT_FILE``), requests (``REQUESTS_CA_BUNDLE``), and Node.js /
-Claude Code (``NODE_EXTRA_CA_CERTS``) so that enterprise / corporate
-deployments with custom certificate authorities work without extra
-configuration.
-
-Priority order (first match wins):
+"""TLS trust policy for every outbound connection Headroom makes.
+
+Corporate networks (Zscaler, Netskope, Palo Alto, Cisco Secure Access, ...)
+inspect TLS by re-signing every certificate with a company root. IT installs
+that root in the **operating system** trust store (macOS Keychain, Windows
+certificate store), which is why browsers, curl and Claude Code keep working.
+Python does not read that store: httpx trusts certifi's bundle and the stdlib
+reads a static OpenSSL file, so Headroom used to reject the re-signed chain
+and fail every upstream request with ``CERTIFICATE_VERIFY_FAILED``.
+
+Default policy (``HEADROOM_CERT_STORE=system,bundled``, mirroring Claude
+Code's ``CLAUDE_CODE_CERT_STORE``): verify through the OS trust store via
+``truststore`` *and* trust certifi's public bundle. On macOS and Windows the
+OS verifier does the chain check, so a corporate root that IT pushed to the
+machine just works and OpenSSL's strict-mode quirks never apply.
+
+Sources, in the order they are resolved:
1. ``SSL_CERT_FILE`` — replacement semantics (only these CAs are trusted)
2. ``REQUESTS_CA_BUNDLE`` — replacement semantics
-3. ``NODE_EXTRA_CA_CERTS`` — **additive** semantics (extra roots loaded
- on top of the default/system trust store, matching Node.js behavior)
+3. Otherwise the ``HEADROOM_CERT_STORE`` sources (``system``, ``bundled``),
+ plus the **additive** bundles ``HEADROOM_CA_BUNDLE`` and
+ ``NODE_EXTRA_CA_CERTS`` (extra roots on top, matching Node.js behavior).
+
+``HEADROOM_CERT_STORE=bundled`` restores the pre-0.40 behavior (no OS store).
Strict-mode toggle (``HEADROOM_TLS_STRICT``):
Python 3.13 + OpenSSL 3.x enable ``VERIFY_X509_STRICT`` by default, which
@@ -34,10 +45,37 @@ from __future__ import annotations
import logging
import os
import ssl
+from collections.abc import Callable
from typing import Any, cast
logger = logging.getLogger("headroom.proxy")
+CERT_STORE_ENV = "HEADROOM_CERT_STORE"
+EXTRA_CA_ENV = "HEADROOM_CA_BUNDLE"
+
+_CERT_STORE_SOURCES = frozenset({"system", "bundled"})
+# Spellings people reach for; a typo'd opt-out must not silently re-enable a source.
+_CERT_STORE_ALIASES = {
+ "bundle": "bundled",
+ "certifi": "bundled",
+ "os": "system",
+ "native": "system",
+}
+_DEFAULT_CERT_STORE = frozenset({"system", "bundled"})
+
+# Additive bundles, loaded on top of whatever trust store is in force.
+_ADDITIVE_CA_VARS = (EXTRA_CA_ENV, "NODE_EXTRA_CA_CERTS")
+
+# Set by Headroom's CLI (and inherited by the proxy processes it spawns) so
+# proxy startup only injects process-wide when Headroom owns the process.
+PROCESS_TRUST_ENV = "HEADROOM_PROCESS_TRUST"
+
+_process_trust_injected = False
+
+# Built contexts, keyed by ALPN + every env var that shapes them. Building one
+# parses certifi's ~140 roots, so it must not run per WebSocket connection.
+_system_ctx_cache: dict[tuple[Any, ...], ssl.SSLContext] = {}
+
_REPLACEMENT_CA_VARS = (
"SSL_CERT_FILE",
"REQUESTS_CA_BUNDLE",
@@ -90,16 +128,22 @@ def _relax_x509_strict_for_custom_ca(ctx: ssl.SSLContext, *, path: str) -> ssl.S
return _clear_x509_strict(ctx, reason=f"custom_ca:{path}")
+def _require_tls12(ctx: ssl.SSLContext) -> ssl.SSLContext:
+ """Refuse TLS < 1.2 explicitly (Python's default, stated so it cannot regress)."""
+ ctx.minimum_version = ssl.TLSVersion.TLSv1_2
+ return ctx
+
+
def _replacement_ca_context(path: str) -> ssl.SSLContext:
"""Build a replacement trust-store context from a CA bundle path."""
- ctx = ssl.create_default_context(cafile=path)
+ ctx = _require_tls12(ssl.create_default_context(cafile=path))
ctx.set_alpn_protocols(["h2", "http/1.1"])
return _relax_x509_strict_for_custom_ca(ctx, path=path)
def _additive_ca_context(path: str) -> ssl.SSLContext:
"""Build an additive trust-store context from a CA bundle path."""
- ctx = ssl.create_default_context()
+ ctx = _require_tls12(ssl.create_default_context())
ctx.load_verify_locations(cafile=path)
ctx.set_alpn_protocols(["h2", "http/1.1"])
return _relax_x509_strict_for_custom_ca(ctx, path=path)
@@ -135,20 +179,155 @@ def find_ca_bundle() -> ssl.SSLContext | None:
path,
)
- node_path = os.environ.get("NODE_EXTRA_CA_CERTS")
- if node_path and os.path.isfile(node_path):
- logger.info(
- "event=ssl_ca_bundle_loaded env_var=NODE_EXTRA_CA_CERTS path=%s additive=true",
- node_path,
- )
- return _additive_ca_context(node_path)
- if node_path and not os.path.isfile(node_path):
+ additive = _additive_ca_paths()
+ if additive:
+ ctx = _additive_ca_context(additive[0])
+ for path in additive[1:]:
+ ctx.load_verify_locations(cafile=path)
+ return ctx
+
+ return None
+
+
+def _additive_ca_paths(*, log: bool = True) -> list[str]:
+ """Existing files named by the additive CA vars (logged unless ``log=False``)."""
+ paths: list[str] = []
+ for var in _ADDITIVE_CA_VARS:
+ path = os.environ.get(var)
+ if not path:
+ continue
+ if not log:
+ if os.path.isfile(path):
+ paths.append(path)
+ continue
+ if os.path.isfile(path):
+ logger.info(
+ "event=ssl_ca_bundle_loaded env_var=%s path=%s additive=true",
+ var,
+ path,
+ )
+ paths.append(path)
+ else:
+ logger.warning(
+ "event=ssl_ca_bundle_missing env_var=%s path=%r (skipped)",
+ var,
+ path,
+ )
+ return paths
+
+
+def _replacement_ca_var_set() -> bool:
+ """True when ``SSL_CERT_FILE``/``REQUESTS_CA_BUNDLE`` name an existing file."""
+ return any(
+ (path := os.environ.get(var)) and os.path.isfile(path) for var in _REPLACEMENT_CA_VARS
+ )
+
+
+def cert_store_sources() -> frozenset[str]:
+ """Trust sources selected by ``HEADROOM_CERT_STORE`` (default: system,bundled).
+
+ Unknown tokens are ignored with a warning; a value with no known token falls
+ back to the default rather than trusting nothing.
+ """
+ raw = os.environ.get(CERT_STORE_ENV)
+ if raw is None or not raw.strip():
+ return _DEFAULT_CERT_STORE
+ tokens = {
+ _CERT_STORE_ALIASES.get(t.strip().lower(), t.strip().lower())
+ for t in raw.split(",")
+ if t.strip()
+ }
+ unknown = tokens - _CERT_STORE_SOURCES
+ if unknown:
logger.warning(
- "event=ssl_ca_bundle_missing env_var=NODE_EXTRA_CA_CERTS path=%r (skipped)",
- node_path,
+ "event=ssl_cert_store_unknown env_var=%s ignored=%s",
+ CERT_STORE_ENV,
+ ",".join(sorted(unknown)),
)
+ chosen = frozenset(tokens & _CERT_STORE_SOURCES)
+ return chosen or _DEFAULT_CERT_STORE
- return None
+
+def system_trust_available() -> bool:
+ """True when the ``truststore`` package can verify through the OS store."""
+ try:
+ import truststore # noqa: F401
+ except Exception:
+ return False
+ return True
+
+
+def _system_store_enabled() -> bool:
+ return (
+ "system" in cert_store_sources()
+ and not _replacement_ca_var_set()
+ and system_trust_available()
+ )
+
+
+def _build_system_context(alpn: list[str]) -> ssl.SSLContext:
+ """A context that verifies through the OS trust store. Raises if it cannot.
+
+ ``truststore`` hands chain validation to macOS Security.framework / Windows
+ CryptoAPI (OpenSSL + the system bundle on Linux). Certificates loaded with
+ ``load_verify_locations`` are trusted *in addition* to the OS roots, so
+ certifi (``bundled``) and the additive bundle vars stack on top. Built once
+ per configuration and cached.
+ """
+ key = (
+ tuple(alpn),
+ os.environ.get(CERT_STORE_ENV),
+ *(os.environ.get(var) for var in _ADDITIVE_CA_VARS),
+ )
+ cached = _system_ctx_cache.get(key)
+ if cached is not None:
+ return cached
+ import truststore
+
+ ctx: ssl.SSLContext = _require_tls12(truststore.SSLContext(ssl.PROTOCOL_TLS_CLIENT))
+ # Match create_default_context() on 3.13+: an explicitly trusted
+ # intermediate (a common "Zscaler Intermediate Root CA" export) is a
+ # valid anchor. Only affects the OpenSSL (Linux) backend.
+ partial_chain = getattr(ssl, "VERIFY_X509_PARTIAL_CHAIN", 0)
+ if partial_chain:
+ ctx.verify_flags |= partial_chain
+ if "bundled" in cert_store_sources():
+ try:
+ import certifi
+
+ ctx.load_verify_locations(cafile=certifi.where())
+ except Exception: # certifi is optional; the OS store still applies
+ logger.debug("event=ssl_certifi_unavailable")
+ for path in _additive_ca_paths():
+ ctx.load_verify_locations(cafile=path)
+ ctx.set_alpn_protocols(alpn)
+ _system_ctx_cache[key] = ctx
+ return ctx
+
+
+def describe_trust_policy() -> dict[str, Any]:
+ """What trust configuration is in force, for ``/health`` and ``doctor``."""
+ replacement = next(
+ (
+ {"env_var": var, "path": os.environ[var]}
+ for var in _REPLACEMENT_CA_VARS
+ if os.environ.get(var) and os.path.isfile(os.environ[var])
+ ),
+ None,
+ )
+ return {
+ "cert_store": sorted(cert_store_sources()),
+ "system_store_available": system_trust_available(),
+ "system_store_active": _system_store_enabled(),
+ "replacement_bundle": replacement,
+ "additive_bundles": [
+ {"env_var": var, "path": os.environ[var]}
+ for var in _ADDITIVE_CA_VARS
+ if os.environ.get(var) and os.path.isfile(os.environ[var])
+ ],
+ "tls_strict_disabled": tls_strict_disabled(),
+ "process_injected": _process_trust_injected,
+ }
def _default_strict_relaxed_context() -> ssl.SSLContext:
@@ -159,51 +338,115 @@ def _default_strict_relaxed_context() -> ssl.SSLContext:
tolerate a non-critical ``basicConstraints`` CA. Mirrors what httpx builds
for ``verify=True`` (default context + ALPN), minus the strict flag.
"""
- ctx = ssl.create_default_context()
+ ctx = _require_tls12(ssl.create_default_context())
ctx.set_alpn_protocols(["h2", "http/1.1"])
return _clear_x509_strict(ctx, reason="env_toggle")
-def build_httpx_verify() -> ssl.SSLContext | bool:
- """Return the value for httpx's ``verify=`` parameter.
+def _replacement_ca_path() -> str | None:
+ """The existing file named by ``SSL_CERT_FILE``/``REQUESTS_CA_BUNDLE``, if any."""
+ for var in _REPLACEMENT_CA_VARS:
+ path = os.environ.get(var)
+ if path and os.path.isfile(path):
+ return path
+ return None
+
+
+def _has_configured_trust() -> bool:
+ """True when Headroom's trust settings call for something other than a library default."""
+ return (
+ _system_store_enabled()
+ or _replacement_ca_path() is not None
+ or bool(_additive_ca_paths(log=False))
+ or tls_strict_disabled()
+ )
+
+
+def _build_context(alpn: list[str], fallback: Callable[[], ssl.SSLContext]) -> ssl.SSLContext:
+ """Build the verifying context Headroom's trust configuration calls for.
+
+ Every branch returns a concrete, certificate-verifying ``SSLContext``;
+ ``fallback`` supplies the library-default equivalent. Resolution order:
+
+ 0. The OS trust store (plus certifi and the additive bundle vars) when
+ ``HEADROOM_CERT_STORE`` includes ``system`` — the default — and no
+ replacement bundle var is set.
+ 1. ``SSL_CERT_FILE`` / ``REQUESTS_CA_BUNDLE`` → only that bundle, strict
+ mode relaxed (corporate PKI signal).
+ 2. ``HEADROOM_CA_BUNDLE`` / ``NODE_EXTRA_CA_CERTS`` → default roots plus
+ those, strict mode relaxed.
+ 3. ``HEADROOM_TLS_STRICT=0`` → the default trust store with
+ ``VERIFY_X509_STRICT`` cleared.
+ 4. Otherwise ``fallback()``.
+ """
+ if _system_store_enabled():
+ try:
+ return _build_system_context(alpn)
+ except Exception as exc:
+ # Never let the OS-store path take the proxy down: fall back to the
+ # bundled behavior and say so.
+ logger.warning("event=ssl_system_store_failed error=%r (falling back to bundled)", exc)
+ replacement = _replacement_ca_path()
+ additive = _additive_ca_paths()
+ if replacement is not None:
+ logger.info("event=ssl_ca_bundle_loaded path=%s", replacement)
+ ctx = _replacement_ca_context(replacement)
+ elif additive:
+ ctx = _additive_ca_context(additive[0])
+ for path in additive[1:]:
+ ctx.load_verify_locations(cafile=path)
+ elif tls_strict_disabled():
+ ctx = _default_strict_relaxed_context()
+ else:
+ ctx = fallback()
+ ctx.set_alpn_protocols(alpn)
+ return ctx
- Resolution order:
- 1. A custom CA bundle env var (``SSL_CERT_FILE`` / ``REQUESTS_CA_BUNDLE`` /
- ``NODE_EXTRA_CA_CERTS``) → a context trusting that bundle, with strict
- mode already relaxed (corporate PKI signal).
- 2. No bundle, but ``HEADROOM_TLS_STRICT=0`` → the default trust store with
- ``VERIFY_X509_STRICT`` cleared, so a corporate root that's installed in
- the OS store but trips RFC 5280 strict mode still validates.
- 3. Otherwise → ``True`` (httpx's default strict verification).
+def _bundled_default_context() -> ssl.SSLContext:
+ """Exactly what httpx builds for ``verify=True``: certifi, strict defaults.
- Returning ``True`` rather than a hand-built context in the common case
- keeps httpx's own default behavior (including its certifi fallback) intact.
+ Built here rather than passing ``True`` so every upstream client receives a
+ concrete, always-verifying ``SSLContext``: no code path can hand httpx a
+ value that disables verification.
"""
- ca_ctx = find_ca_bundle()
- if ca_ctx is not None:
- return ca_ctx
- if tls_strict_disabled():
- return _default_strict_relaxed_context()
- return True
+ cert_dir = os.environ.get("SSL_CERT_DIR")
+ if cert_dir:
+ ctx = ssl.create_default_context(capath=cert_dir)
+ else:
+ import certifi
+
+ ctx = ssl.create_default_context(cafile=certifi.where())
+ return _require_tls12(ctx)
+
+
+def _stdlib_default_context() -> ssl.SSLContext:
+ """What ``ssl=True`` / urlopen build: the stdlib default trust store."""
+ return _require_tls12(ssl.create_default_context())
+
+
+def build_httpx_verify() -> ssl.SSLContext:
+ """Return the value for httpx's ``verify=`` parameter: always a verifying context.
+
+ Headroom's configured trust (see :func:`_build_context`) when there is one,
+ else httpx's own default (certifi) built explicitly.
+ """
+ return _build_context(["h2", "http/1.1"], _bundled_default_context)
def build_urlopen_context() -> ssl.SSLContext | None:
"""Return Headroom's configured TLS context for ``urllib.request.urlopen``.
- ``urlopen`` already handles Python's default trust configuration when no
- explicit context is passed. Return only a custom context here so callers
- retain that default while sharing Headroom's corporate CA and strict-mode
- handling when it is configured.
+ ``None`` means "use urlopen's own default" (Python's default trust store),
+ which is what the legacy ``HEADROOM_CERT_STORE=bundled`` path wants when no
+ bundle or strict-mode toggle is configured. urllib.request/http.client only
+ implements HTTP/1.1 framing, so the context only offers ``http/1.1``:
+ offering h2 can make a TLS-inspecting MITM negotiate a protocol it cannot
+ parse.
"""
-
- verify = build_httpx_verify()
- if not isinstance(verify, ssl.SSLContext):
+ if not _has_configured_trust():
return None
- # urllib.request/http.client only implements HTTP/1.1 framing. Offering h2
- # can make a TLS-inspecting MITM negotiate a protocol it cannot parse.
- verify.set_alpn_protocols(["http/1.1"])
- return verify
+ return _build_context(["http/1.1"], _stdlib_default_context)
def apply_global_tls_relaxation() -> bool:
@@ -257,3 +500,62 @@ def apply_global_tls_relaxation() -> bool:
_u3ssl.create_urllib3_context = _relaxed_create_urllib3_context # type: ignore[assignment]
logger.info("event=ssl_x509_strict_disabled reason=urllib3_global_patch")
return True
+
+
+def build_websocket_ssl() -> ssl.SSLContext:
+ """Return the value for ``websockets.connect(ssl=...)`` on a ``wss://`` URL.
+
+ WebSocket upgrades are HTTP/1.1, so the context only offers ``http/1.1``.
+ Without configured trust it is what ``ssl=True`` would build (the stdlib
+ default, which on Windows loads the machine store).
+ """
+ return _build_context(["http/1.1"], _stdlib_default_context)
+
+
+def ensure_process_trust() -> bool:
+ """Make every TLS client in this *application* process use the OS store.
+
+ Marks the process (``HEADROOM_PROCESS_TRUST=1``) so proxy processes the CLI
+ spawns inherit the decision; see :func:`ensure_process_trust_if_owned`.
+
+ Headroom's own upstream clients get an explicit context from the builders
+ above, but third-party code (``huggingface_hub`` model downloads, tiktoken
+ vocab fetches, OTEL exporters, ...) builds its own. ``truststore``'s
+ process-wide injection covers those in one place instead of chasing every
+ call site. It is only ever called from Headroom's entry points (CLI and
+ proxy startup) — never on library import — because injecting into an
+ embedding application's process is not ours to decide.
+
+ Skipped when the operator pinned a replacement bundle (``SSL_CERT_FILE`` /
+ ``REQUESTS_CA_BUNDLE``) or opted out via ``HEADROOM_CERT_STORE``.
+ Idempotent. Returns True when injection is in place.
+ """
+ global _process_trust_injected
+ os.environ[PROCESS_TRUST_ENV] = "1"
+ if _process_trust_injected:
+ return True
+ if not _system_store_enabled():
+ return False
+ try:
+ import truststore
+
+ truststore.inject_into_ssl()
+ except Exception as exc:
+ logger.warning("event=ssl_process_trust_failed error=%r", exc)
+ return False
+ _process_trust_injected = True
+ logger.info("event=ssl_process_trust_injected source=os_trust_store")
+ return True
+
+
+def ensure_process_trust_if_owned() -> bool:
+ """:func:`ensure_process_trust`, but only in a process Headroom's CLI owns.
+
+ Proxy startup calls this: uvicorn workers spawned by ``headroom proxy``
+ inherit ``HEADROOM_PROCESS_TRUST=1`` and get injection, while an application
+ that embeds the proxy app keeps its own process-wide TLS behavior (its
+ Headroom upstream clients still get the OS store via explicit contexts).
+ """
+ if os.environ.get(PROCESS_TRUST_ENV) != "1":
+ return False
+ return ensure_process_trust()
diff --git a/headroom/proxy/tls_diagnostics.py b/headroom/proxy/tls_diagnostics.py
new file mode 100644
index 000000000..0e7d3c303
--- /dev/null
+++ b/headroom/proxy/tls_diagnostics.py
@@ -0,0 +1,529 @@
+"""Explain upstream TLS failures in words a user (and their IT team) can act on.
+
+Behind a corporate TLS-inspection gateway every certificate Headroom sees is
+re-signed by the company's root. When that root is not trusted, the only thing
+that reaches the user today is ``CERTIFICATE_VERIFY_FAILED`` wrapped in a 502.
+This module turns that into: *who* signed the certificate (e.g. "Zscaler
+Intermediate Root CA"), what that means, and the exact fix.
+
+It is used in two places:
+
+* the proxy's upstream error paths, so the agent's error line names the cause
+ (:func:`describe_upstream_failure`), and
+* ``headroom doctor --network`` (:func:`probe_endpoint`).
+
+To learn the issuer, the probe performs a second, *unverified* TLS handshake
+purely to read the presented certificate. Nothing is sent over that
+connection: it is closed straight after the handshake.
+"""
+
+from __future__ import annotations
+
+import base64
+import logging
+import os
+import socket
+import ssl
+import tempfile
+import threading
+import time
+import urllib.request
+from collections.abc import Mapping
+from dataclasses import asdict, dataclass, field
+from typing import Any
+from urllib.parse import urlsplit
+
+logger = logging.getLogger("headroom.proxy")
+
+DOCS_URL = "https://docs.headroomlabs.ai/docs/corporate-networks"
+
+# Issuer substrings (lower-cased) of common TLS-inspection products. Matching is
+# on the issuer's organization / common name, so a customer-uploaded
+# intermediate ("Acme Corp Zscaler SubCA") still resolves to the product.
+_INSPECTION_VENDORS: tuple[tuple[str, str], ...] = (
+ ("zscaler", "Zscaler"),
+ ("netskope", "Netskope"),
+ ("palo alto", "Palo Alto Networks"),
+ ("globalprotect", "Palo Alto Networks"),
+ ("prisma", "Palo Alto Networks"),
+ ("cisco umbrella", "Cisco Umbrella"),
+ ("cisco secure access", "Cisco Secure Access"),
+ ("fortinet", "Fortinet"),
+ ("fortigate", "Fortinet"),
+ ("forcepoint", "Forcepoint"),
+ ("websense", "Forcepoint"),
+ ("blue coat", "Symantec / Broadcom"),
+ ("bluecoat", "Symantec / Broadcom"),
+ ("symantec", "Symantec / Broadcom"),
+ ("cloudflare gateway", "Cloudflare Gateway"),
+ ("cloudflare for teams", "Cloudflare Gateway"),
+ ("iboss", "iboss"),
+ ("check point", "Check Point"),
+ ("checkpoint", "Check Point"),
+ ("sophos", "Sophos"),
+ ("skyhigh", "Skyhigh Security"),
+ ("mcafee web gateway", "Skyhigh Security"),
+ ("menlo security", "Menlo Security"),
+ ("barracuda", "Barracuda"),
+)
+
+# ``_ssl.Certificate.public_bytes`` format flag (Python 3.13+ chain API).
+_ENCODING_DER = getattr(ssl._ssl, "ENCODING_DER", 1) # type: ignore[attr-defined]
+
+_PROBE_TIMEOUT_S = 4.0
+_PROBE_CACHE_TTL_S = 300.0
+_probe_cache: dict[tuple[str, int, bool], tuple[float, ChainInfo]] = {}
+_probe_lock = threading.Lock()
+# One in-flight probe per (host, port): a burst of failing requests waits for
+# the first handshake instead of each opening its own.
+_probe_inflight: dict[tuple[str, int, bool], threading.Lock] = {}
+_logged_hosts: set[str] = set()
+
+
+@dataclass
+class ChainInfo:
+ """What the network presented for ``host`` — no trust decision implied."""
+
+ host: str
+ port: int
+ reachable: bool
+ issuer: str | None = None
+ subject: str | None = None
+ inspection_vendor: str | None = None
+ via_proxy: str | None = None
+ error: str | None = None
+ chain_issuers: list[str] = field(default_factory=list)
+
+ def to_dict(self) -> dict[str, Any]:
+ return asdict(self)
+
+
+# ---------------------------------------------------------------------------
+# Exception inspection
+# ---------------------------------------------------------------------------
+
+
+def find_cert_verify_error(exc: BaseException | None) -> ssl.SSLCertVerificationError | None:
+ """Return the certificate-verification error buried in ``exc``'s chain.
+
+ httpx wraps httpcore, which wraps the ``ssl`` error, and websockets
+ re-raises it directly; walk ``__cause__``/``__context__`` (bounded) and the
+ exception args to find it.
+ """
+ seen: set[int] = set()
+ stack: list[BaseException] = [exc] if exc is not None else []
+ while stack:
+ cur = stack.pop()
+ if id(cur) in seen or len(seen) > 32:
+ continue
+ seen.add(id(cur))
+ if isinstance(cur, ssl.SSLCertVerificationError):
+ return cur
+ for nxt in (cur.__cause__, cur.__context__):
+ if isinstance(nxt, BaseException):
+ stack.append(nxt)
+ for arg in getattr(cur, "args", ()):
+ if isinstance(arg, BaseException):
+ stack.append(arg)
+ return None
+
+
+def _looks_like_cert_failure(exc: BaseException) -> bool:
+ # Some stacks (httpcore on certain paths) stringify the ssl error into a
+ # plain ConnectError without chaining it.
+ text = str(exc)
+ return "CERTIFICATE_VERIFY_FAILED" in text or "certificate verify failed" in text
+
+
+# ---------------------------------------------------------------------------
+# Presented-chain probe
+# ---------------------------------------------------------------------------
+
+
+def identify_inspection_vendor(*names: str | None) -> str | None:
+ """Map certificate issuer / subject strings to a TLS-inspection product."""
+ for name in names:
+ if not name:
+ continue
+ lowered = name.lower()
+ for needle, vendor in _INSPECTION_VENDORS:
+ if needle in lowered:
+ return vendor
+ return None
+
+
+def _name_to_str(name: Any) -> str | None:
+ """Render a decoded ``getpeercert()``-style name tuple as ``O=..., CN=...``."""
+ if not name:
+ return None
+ parts: list[str] = []
+ wanted = {"organizationName": "O", "organizationalUnitName": "OU", "commonName": "CN"}
+ for rdn in name:
+ for key, value in rdn:
+ if key in wanted:
+ parts.append(f"{wanted[key]}={value}")
+ return ", ".join(parts) or None
+
+
+def _decode_der(der: bytes) -> dict[str, Any]:
+ """Decode a DER certificate into ``getpeercert()`` form without verifying it.
+
+ Uses ``cryptography`` when installed, else CPython's own decoder (the one
+ ``ssl`` uses internally), which needs a PEM file on disk.
+ """
+ try:
+ from cryptography import x509
+ from cryptography.x509.oid import NameOID
+
+ cert = x509.load_der_x509_certificate(der)
+
+ def _rdns(n: Any) -> tuple[tuple[tuple[str, str], ...], ...]:
+ mapping = {
+ NameOID.ORGANIZATION_NAME: "organizationName",
+ NameOID.ORGANIZATIONAL_UNIT_NAME: "organizationalUnitName",
+ NameOID.COMMON_NAME: "commonName",
+ }
+ return tuple(((mapping[a.oid], str(a.value)),) for a in n if a.oid in mapping)
+
+ return {"issuer": _rdns(cert.issuer), "subject": _rdns(cert.subject)}
+ except Exception:
+ pass
+ pem = ssl.DER_cert_to_PEM_cert(der)
+ fd, path = tempfile.mkstemp(suffix=".pem")
+ try:
+ with os.fdopen(fd, "w") as fh:
+ fh.write(pem)
+ decoded: dict[str, Any] = ssl._ssl._test_decode_cert(path) # type: ignore[attr-defined]
+ return decoded
+ except Exception:
+ return {}
+ finally:
+ try:
+ os.unlink(path)
+ except OSError:
+ pass
+
+
+def _proxy_for(host: str) -> str | None:
+ """The HTTPS proxy the upstream client would use for ``host``, if any.
+
+ Environment only, like httpx's ``trust_env``: the OS proxy settings that
+ ``urllib.request.getproxies`` also reads on macOS/Windows are not what the
+ proxy's requests use, so probing through them would diagnose another path.
+ """
+ try:
+ if urllib.request.proxy_bypass_environment(host): # type: ignore[attr-defined]
+ return None
+ env = urllib.request.getproxies_environment()
+ except Exception:
+ return None
+ return env.get("https") or env.get("all") or None
+
+
+def _open_tunnel(host: str, port: int, proxy_url: str, timeout: float) -> socket.socket:
+ """Open a TCP connection to ``host:port`` through an HTTP CONNECT proxy."""
+ parts = urlsplit(proxy_url if "://" in proxy_url else f"http://{proxy_url}")
+ if parts.scheme not in ("http", ""):
+ raise OSError(f"unsupported proxy scheme {parts.scheme!r} for the TLS probe")
+ sock = socket.create_connection((parts.hostname or "", parts.port or 8080), timeout=timeout)
+ request = f"CONNECT {host}:{port} HTTP/1.1\r\nHost: {host}:{port}\r\n"
+ if parts.username:
+ token = base64.b64encode(f"{parts.username}:{parts.password or ''}".encode()).decode()
+ request += f"Proxy-Authorization: Basic {token}\r\n"
+ sock.sendall((request + "\r\n").encode())
+ reply = b""
+ while b"\r\n\r\n" not in reply and len(reply) < 8192:
+ chunk = sock.recv(1024)
+ if not chunk:
+ break
+ reply += chunk
+ status_line = reply.split(b"\r\n", 1)[0].decode("latin-1", "replace")
+ if " 200" not in status_line:
+ sock.close()
+ raise OSError(f"proxy refused CONNECT: {status_line or 'no reply'}")
+ return sock
+
+
+def _public_address(host: str, port: int, timeout: float) -> tuple[str, int] | None:
+ """Resolve ``host`` once and return a public address, or None if it is not public.
+
+ The request-path probe can be pointed at a per-request upstream, so it must
+ not become a way to read the certificate of an internal service. Returning
+ the checked address (instead of re-resolving) keeps DNS rebinding out too.
+ """
+ import ipaddress
+
+ # No timeout knob exists for getaddrinfo; the caller already runs off the
+ # event loop, and the process-wide socket default must not be touched here.
+ del timeout
+ infos = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)
+ for _family, _type, _proto, _canon, sockaddr in infos:
+ ip = ipaddress.ip_address(sockaddr[0])
+ if not ip.is_global:
+ return None
+ return (str(infos[0][4][0]), port) if infos else None
+
+
+def probe_presented_chain(
+ host: str,
+ port: int = 443,
+ *,
+ timeout: float = _PROBE_TIMEOUT_S,
+ use_cache: bool = True,
+ allow_private: bool = False,
+) -> ChainInfo:
+ """Handshake with ``host`` *without verification* and report who signed it.
+
+ Goes through the same explicit HTTPS proxy the environment configures, so an
+ explicit-proxy deployment sees the same re-signed chain the proxy would.
+ Results are cached for a few minutes so a burst of failing requests costs a
+ single probe. Hosts resolving to private, loopback or link-local addresses
+ are skipped unless ``allow_private`` (``doctor``, which the user runs on
+ purpose, sets it).
+ """
+ # allow_private is part of the key so a doctor/test probe of a private host
+ # can never be served to the request path.
+ key = (host, port, allow_private)
+ if not use_cache:
+ return _probe_uncached(host, port, timeout=timeout, allow_private=allow_private)
+ with _probe_lock:
+ gate = _probe_inflight.setdefault(key, threading.Lock())
+ with gate:
+ with _probe_lock:
+ cached = _probe_cache.get(key)
+ if cached and time.monotonic() - cached[0] < _PROBE_CACHE_TTL_S:
+ return cached[1]
+ info = _probe_uncached(host, port, timeout=timeout, allow_private=allow_private)
+ with _probe_lock:
+ _probe_cache[key] = (time.monotonic(), info)
+ return info
+
+
+def _probe_uncached(host: str, port: int, *, timeout: float, allow_private: bool) -> ChainInfo:
+ proxy = _proxy_for(host)
+ info = ChainInfo(host=host, port=port, reachable=False, via_proxy=proxy)
+ # Certificate-inspection handshake only: it reads the presented chain and
+ # closes without sending a byte, so it deliberately skips verification (the
+ # chain is what we are diagnosing). It still refuses TLS < 1.2, and it only
+ # connects through the public-address / env-proxy CONNECT guard below.
+ ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+ ctx.minimum_version = ssl.TLSVersion.TLSv1_2
+ ctx.check_hostname = False
+ ctx.verify_mode = ssl.CERT_NONE
+ try:
+ if proxy:
+ raw = _open_tunnel(host, port, proxy, timeout)
+ else:
+ target = (host, port) if allow_private else _public_address(host, port, timeout)
+ if target is None:
+ info.error = "skipped: host resolves to a non-public address"
+ return info
+ raw = socket.create_connection(target, timeout=timeout)
+ with ctx.wrap_socket(raw, server_hostname=host) as tls:
+ info.reachable = True
+ ders: list[bytes] = []
+ get_chain = getattr(tls, "get_unverified_chain", None) # Python 3.13+
+ if get_chain is not None:
+ try:
+ ders = [
+ c if isinstance(c, bytes) else c.public_bytes(_ENCODING_DER)
+ for c in get_chain() or []
+ ]
+ except Exception:
+ ders = []
+ if not ders:
+ leaf = tls.getpeercert(binary_form=True)
+ ders = [leaf] if leaf else []
+ decoded = [_decode_der(d) for d in ders]
+ if decoded:
+ info.subject = _name_to_str(decoded[0].get("subject"))
+ info.issuer = _name_to_str(decoded[0].get("issuer"))
+ info.chain_issuers = [s for d in decoded if (s := _name_to_str(d.get("issuer")))]
+ info.inspection_vendor = identify_inspection_vendor(info.issuer, *info.chain_issuers)
+ except Exception as exc:
+ info.error = f"{type(exc).__name__}: {exc}"
+ return info
+
+
+# ---------------------------------------------------------------------------
+# User-facing explanation
+# ---------------------------------------------------------------------------
+
+
+def _fix_lines(vendor: str | None) -> list[str]:
+ who = f"{vendor} " if vendor else "your network's "
+ return [
+ f"Headroom trusts the operating system's certificate store, where IT normally "
+ f"installs the {who}root certificate. If your browser and Claude Code work but "
+ f"Headroom does not, that root is not in this machine's store.",
+ "Fix (pick one): ask IT to install the root in the OS certificate store; or export "
+ "it to a PEM file and set HEADROOM_CA_BUNDLE=/path/to/root.pem (NODE_EXTRA_CA_CERTS "
+ "also works); or ask IT to exempt the provider domains from TLS inspection.",
+ "Run `headroom doctor --network` for a full report.",
+ ]
+
+
+def explain_cert_failure(host: str, verify_message: str | None, info: ChainInfo | None) -> str:
+ """One paragraph a user can act on (and forward to IT)."""
+ reason = verify_message or "certificate verify failed"
+ head = f"Headroom could not verify the TLS certificate for {host} ({reason})."
+ if info is not None and info.issuer:
+ if info.inspection_vendor:
+ head += (
+ f" The certificate was issued by '{info.issuer}': {info.inspection_vendor} "
+ f"is inspecting this connection."
+ )
+ else:
+ head += f" The certificate was issued by '{info.issuer}'."
+ return " ".join([head, *_fix_lines(info.inspection_vendor if info else None), DOCS_URL])
+
+
+def describe_upstream_failure(
+ exc: BaseException, url: str | None, *, probe: bool = True, allow_private: bool = False
+) -> str | None:
+ """Actionable message for a certificate-verification failure, else ``None``.
+
+ ``None`` means "not a TLS trust problem" and the caller keeps its existing
+ message. Safe to call from a request's error path: the probe is bounded by a
+ short timeout and cached per host. Blocking — async callers should use
+ :func:`describe_upstream_failure_async`.
+ """
+ cert_err = find_cert_verify_error(exc)
+ if cert_err is None and not _looks_like_cert_failure(exc):
+ return None
+ parts = urlsplit(url or "")
+ host = parts.hostname or "the upstream API"
+ port = parts.port or (443 if parts.scheme in ("https", "wss", "") else 80)
+ info = (
+ probe_presented_chain(host, port, allow_private=allow_private)
+ if probe and parts.hostname
+ else None
+ )
+ verify_message = getattr(cert_err, "verify_message", None) if cert_err else None
+ message = explain_cert_failure(host, verify_message, info)
+ if host not in _logged_hosts:
+ _logged_hosts.add(host)
+ logger.error("event=upstream_tls_untrusted host=%s detail=%s", host, message)
+ return message
+
+
+async def describe_upstream_failure_async(
+ exc: BaseException, url: str | None, *, probe: bool = True
+) -> str | None:
+ """:func:`describe_upstream_failure` off the event loop."""
+ import asyncio
+
+ if find_cert_verify_error(exc) is None and not _looks_like_cert_failure(exc):
+ return None
+ return await asyncio.to_thread(describe_upstream_failure, exc, url, probe=probe)
+
+
+# ---------------------------------------------------------------------------
+# Endpoint check for `headroom doctor --network`
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class EndpointReport:
+ """Result of checking one endpoint Headroom needs."""
+
+ name: str
+ url: str
+ ok: bool
+ chain: ChainInfo
+ verified_with: str | None = None
+ error: str | None = None
+ status: int | None = None
+ block_page: bool = False
+ elapsed_ms: float | None = None
+
+ def to_dict(self) -> dict[str, Any]:
+ d = asdict(self)
+ d["chain"] = self.chain.to_dict()
+ return d
+
+
+def _looks_like_block_page(status: int, content_type: str, body: bytes) -> bool:
+ """A TLS-inspection gateway's interstitial rather than the real API."""
+ if "text/html" not in content_type.lower():
+ return False
+ text = body[:8192].decode("utf-8", "replace").lower()
+ if identify_inspection_vendor(text) is not None:
+ return True
+ return status in (403, 407, 451) and ("access denied" in text or "blocked" in text)
+
+
+def probe_endpoint(name: str, url: str, *, timeout: float = 8.0) -> EndpointReport:
+ """Check ``url`` end to end with Headroom's real trust policy.
+
+ Reports the presented chain (and any inspection vendor), whether the chain
+ verifies under the policy Headroom will use, and whether the response is a
+ gateway block page instead of the service.
+ """
+ import httpx
+
+ from headroom.proxy.ssl_context import build_httpx_verify
+
+ parts = urlsplit(url)
+ chain = probe_presented_chain(
+ parts.hostname or "", parts.port or 443, use_cache=False, allow_private=True
+ )
+ started = time.monotonic()
+ try:
+ with httpx.Client(verify=build_httpx_verify(), timeout=timeout) as client:
+ resp = client.get(url, headers={"User-Agent": "headroom-doctor"})
+ elapsed = (time.monotonic() - started) * 1000
+ block = _looks_like_block_page(
+ resp.status_code, resp.headers.get("content-type", ""), resp.content
+ )
+ return EndpointReport(
+ name=name,
+ url=url,
+ ok=not block,
+ chain=chain,
+ verified_with="headroom trust policy",
+ status=resp.status_code,
+ block_page=block,
+ elapsed_ms=round(elapsed, 1),
+ error="gateway returned a block/warning page instead of the service" if block else None,
+ )
+ except Exception as exc:
+ cert_err = find_cert_verify_error(exc)
+ if cert_err is not None or _looks_like_cert_failure(exc):
+ error = explain_cert_failure(
+ parts.hostname or url, getattr(cert_err, "verify_message", None), chain
+ )
+ else:
+ error = f"{type(exc).__name__}: {exc}"
+ return EndpointReport(name=name, url=url, ok=False, chain=chain, error=error)
+
+
+def loopback_no_proxy_gap(environ: Mapping[str, str]) -> str | None:
+ """Name the proxy var that would capture loopback traffic, if NO_PROXY misses it.
+
+ With ``HTTP(S)_PROXY`` set and no loopback entry in ``NO_PROXY``, an agent's
+ requests to ``http://127.0.0.1:8787`` can be sent to the corporate proxy,
+ which cannot reach this laptop.
+ """
+ proxy_var = next(
+ (
+ v
+ for v in (
+ "HTTP_PROXY",
+ "http_proxy",
+ "HTTPS_PROXY",
+ "https_proxy",
+ "ALL_PROXY",
+ "all_proxy",
+ )
+ if environ.get(v)
+ ),
+ None,
+ )
+ if proxy_var is None:
+ return None
+ no_proxy = ",".join(environ.get(v, "") for v in ("NO_PROXY", "no_proxy")).lower()
+ entries = {e.strip() for e in no_proxy.split(",") if e.strip()}
+ if "*" in entries or {"127.0.0.1", "localhost"} <= entries:
+ return None
+ return proxy_var
diff --git a/pyproject.toml b/pyproject.toml
index 0a3e96ba8..f64c8ea68 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -65,6 +65,10 @@ dependencies = [
"pyyaml>=6.0", # omp wrap: parse/merge omp's models.yml registry
"tomli>=2.0.0; python_version < '3.11'", # tomllib backport for helper scripts
"tomlkit>=0.13.0,<1.0", # Loss-minimizing Codex config recovery
+ # Verify upstream TLS through the OS trust store (macOS Keychain, Windows
+ # cert store), where IT installs corporate TLS-inspection roots (Zscaler,
+ # Netskope). Pure Python, no deps; the same library pip uses by default.
+ "truststore>=0.10.0",
]
[project.optional-dependencies]
diff --git a/tests/conftest.py b/tests/conftest.py
index df58299dd..b4f96293a 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -19,6 +19,29 @@ import pytest
from tests._skip_helpers import external_model_skip_reason
[email protected](autouse=True)
+def _undo_process_trust_injection():
+ """Undo ``truststore.inject_into_ssl()`` after any test that triggered it.
+
+ Invoking the CLI (``CliRunner`` on ``main``) or starting the proxy calls
+ ``ensure_process_trust()``, which swaps ``ssl.SSLContext`` process-wide.
+ Left in place it leaks into later tests (server-side test contexts, OpenSSL
+ store stats), so restore the stdlib class after each test.
+ """
+ yield
+ try:
+ from headroom.proxy import ssl_context
+ except Exception:
+ return
+ os.environ.pop(ssl_context.PROCESS_TRUST_ENV, None)
+ ssl_context._system_ctx_cache.clear()
+ if ssl_context._process_trust_injected:
+ import truststore
+
+ truststore.extract_from_ssl()
+ ssl_context._process_trust_injected = False
+
+
# A live `headroom` dev session exports HEADROOM_* into the shell (and the
# Claude wrap adds ANTHROPIC_CUSTOM_HEADERS). Click `envvar=` options pick
# those up inside CliRunner, so assertions would see the developer's proxy
diff --git a/tests/test_copilot_auth.py b/tests/test_copilot_auth.py
index 00581698d..e38932176 100644
--- a/tests/test_copilot_auth.py
+++ b/tests/test_copilot_auth.py
@@ -19,7 +19,7 @@ def test_device_authorization_uses_form_encoded_request(monkeypatch: pytest.Monk
captured = {}
- def fake_urlopen(request, timeout): # noqa: ANN001, ANN202
+ def fake_urlopen(request, timeout, context=None): # noqa: ANN001, ANN202
captured["request"] = request
return io.BytesIO(b'{"device_code":"d","user_code":"u"}')
@@ -37,7 +37,7 @@ def test_device_poll_uses_form_encoded_request(monkeypatch: pytest.MonkeyPatch)
captured = {}
- def fake_urlopen(request, timeout): # noqa: ANN001, ANN202
+ def fake_urlopen(request, timeout, context=None): # noqa: ANN001, ANN202
captured["request"] = request
return io.BytesIO(b'{"access_token":"gho-test"}')
@@ -1601,7 +1601,7 @@ def test_exchange_token_sync_raises_for_http_error(monkeypatch: pytest.MonkeyPat
def close(self) -> None:
return None
- def fake_urlopen(request, timeout: float): # noqa: ANN001, ANN202
+ def fake_urlopen(request, timeout: float, context=None): # noqa: ANN001, ANN202
raise urllib_error.HTTPError(
url=request.full_url,
code=404,
diff --git a/tests/test_corporate_tls.py b/tests/test_corporate_tls.py
new file mode 100644
index 000000000..250165ff3
--- /dev/null
+++ b/tests/test_corporate_tls.py
@@ -0,0 +1,513 @@
+"""Headroom behind a corporate TLS-inspection gateway ("fake Zscaler").
+
+A local HTTPS server presents a certificate signed by a root that looks like
+Zscaler's: ``O=Zscaler Inc.``, and — like the real 2014 Zscaler root —
+``basicConstraints`` present but **not critical**, which Python 3.13's
+``VERIFY_X509_STRICT`` rejects. These tests pin the behavior customers see:
+
+* certifi alone rejects the chain (the original bug);
+* the default trust policy plus the corporate root (``HEADROOM_CA_BUNDLE`` or
+ ``NODE_EXTRA_CA_CERTS``) accepts it, with no strict-mode opt-out;
+* the failure explanation names Zscaler and the fix;
+* process-wide injection respects the operator's replacement bundle;
+* ``wrap``'s loopback ``NO_PROXY`` merge and the new ``doctor`` rows.
+
+The OS-store half of the default policy cannot be exercised without admin
+rights to install a root, so it is covered by the ``truststore`` context type
+and by the additive-bundle path, which goes through the same verifier.
+"""
+
+from __future__ import annotations
+
+import datetime as _dt
+import http.server
+import json
+import re
+import ssl
+import threading
+from collections.abc import Iterator
+from pathlib import Path
+
+import httpx
+import pytest
+
+from headroom.proxy import ssl_context, tls_diagnostics
+from headroom.proxy.ssl_context import (
+ build_httpx_verify,
+ build_urlopen_context,
+ build_websocket_ssl,
+ cert_store_sources,
+ describe_trust_policy,
+ ensure_process_trust,
+)
+
+x509 = pytest.importorskip("cryptography.x509")
+from cryptography.hazmat.primitives import hashes, serialization # noqa: E402
+from cryptography.hazmat.primitives.asymmetric import ec # noqa: E402
+from cryptography.x509.oid import NameOID # noqa: E402
+
+_ENV_VARS = (
+ "SSL_CERT_FILE",
+ "REQUESTS_CA_BUNDLE",
+ "NODE_EXTRA_CA_CERTS",
+ "HEADROOM_CA_BUNDLE",
+ "HEADROOM_CERT_STORE",
+ "HEADROOM_TLS_STRICT",
+ "HTTP_PROXY",
+ "http_proxy",
+ "HTTPS_PROXY",
+ "https_proxy",
+ "ALL_PROXY",
+ "all_proxy",
+ "NO_PROXY",
+ "no_proxy",
+)
+
+
[email protected](autouse=True)
+def _clean_env(monkeypatch: pytest.MonkeyPatch) -> None:
+ for var in _ENV_VARS:
+ monkeypatch.delenv(var, raising=False)
+ tls_diagnostics._probe_cache.clear()
+ tls_diagnostics._logged_hosts.clear()
+
+
+def _name(org: str, cn: str) -> x509.Name:
+ return x509.Name(
+ [
+ x509.NameAttribute(NameOID.ORGANIZATION_NAME, org),
+ x509.NameAttribute(NameOID.COMMON_NAME, cn),
+ ]
+ )
+
+
[email protected](scope="module")
+def fake_zscaler(tmp_path_factory: pytest.TempPathFactory) -> Iterator[dict[str, str | int]]:
+ """HTTPS server on 127.0.0.1 behind a Zscaler-style re-signing root."""
+ d: Path = tmp_path_factory.mktemp("fake-zscaler")
+ now = _dt.datetime.now(_dt.timezone.utc)
+
+ root_key = ec.generate_private_key(ec.SECP256R1())
+ root_name = _name("Zscaler Inc.", "Zscaler Root CA")
+ root = (
+ x509.CertificateBuilder()
+ .subject_name(root_name)
+ .issuer_name(root_name)
+ .public_key(root_key.public_key())
+ .serial_number(x509.random_serial_number())
+ .not_valid_before(now - _dt.timedelta(days=1))
+ .not_valid_after(now + _dt.timedelta(days=30))
+ # Not critical, exactly like the real Zscaler root: the RFC 5280
+ # strict check in Python 3.13 rejects this.
+ .add_extension(x509.BasicConstraints(ca=True, path_length=None), critical=False)
+ .sign(root_key, hashes.SHA256())
+ )
+
+ leaf_key = ec.generate_private_key(ec.SECP256R1())
+ import ipaddress
+
+ leaf = (
+ x509.CertificateBuilder()
+ .subject_name(_name("Zscaler Inc.", "127.0.0.1"))
+ .issuer_name(root_name)
+ .public_key(leaf_key.public_key())
+ .serial_number(x509.random_serial_number())
+ .not_valid_before(now - _dt.timedelta(days=1))
+ .not_valid_after(now + _dt.timedelta(days=30))
+ .add_extension(
+ x509.SubjectAlternativeName(
+ [x509.DNSName("localhost"), x509.IPAddress(ipaddress.ip_address("127.0.0.1"))]
+ ),
+ critical=False,
+ )
+ .add_extension(
+ x509.ExtendedKeyUsage([x509.oid.ExtendedKeyUsageOID.SERVER_AUTH]), critical=False
+ )
+ .sign(root_key, hashes.SHA256())
+ )
+
+ root_pem = d / "zscaler-root.pem"
+ root_pem.write_bytes(root.public_bytes(serialization.Encoding.PEM))
+ leaf_pem = d / "leaf.pem"
+ leaf_pem.write_bytes(
+ leaf.public_bytes(serialization.Encoding.PEM)
+ + leaf_key.private_bytes(
+ serialization.Encoding.PEM,
+ serialization.PrivateFormat.PKCS8,
+ serialization.NoEncryption(),
+ )
+ )
+
+ class Handler(http.server.BaseHTTPRequestHandler):
+ def do_GET(self) -> None: # noqa: N802 - stdlib hook name
+ body = json.dumps({"ok": True}).encode()
+ self.send_response(200)
+ self.send_header("content-type", "application/json")
+ self.send_header("content-length", str(len(body)))
+ self.end_headers()
+ self.wfile.write(body)
+
+ def log_message(self, *args: object) -> None:
+ return
+
+ server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+ server_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+ server_ctx.minimum_version = ssl.TLSVersion.TLSv1_2
+ server_ctx.load_cert_chain(str(leaf_pem))
+ server.socket = server_ctx.wrap_socket(server.socket, server_side=True)
+ thread = threading.Thread(target=server.serve_forever, daemon=True)
+ thread.start()
+ try:
+ yield {
+ "url": f"https://127.0.0.1:{server.server_address[1]}/v1/models",
+ "port": server.server_address[1],
+ "root": str(root_pem),
+ }
+ finally:
+ server.shutdown()
+ server.server_close()
+
+
+class TestTrustPolicy:
+ def test_default_sources_are_system_and_bundled(self) -> None:
+ assert cert_store_sources() == {"system", "bundled"}
+
+ def test_default_verify_is_os_store_context(self) -> None:
+ truststore = pytest.importorskip("truststore")
+ verify = build_httpx_verify()
+ assert isinstance(verify, truststore.SSLContext)
+ assert verify.verify_mode == ssl.CERT_REQUIRED
+
+ def test_bundled_restores_legacy_default(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv("HEADROOM_CERT_STORE", "bundled")
+ truststore = pytest.importorskip("truststore")
+ for ctx in (build_httpx_verify(), build_websocket_ssl()):
+ assert isinstance(ctx, ssl.SSLContext)
+ assert not isinstance(ctx, truststore.SSLContext)
+ assert ctx.verify_mode == ssl.CERT_REQUIRED
+ assert build_urlopen_context() is None
+
+ def test_unknown_tokens_fall_back_to_default(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv("HEADROOM_CERT_STORE", "keychain")
+ assert cert_store_sources() == {"system", "bundled"}
+
+ def test_replacement_bundle_disables_os_store(
+ self, monkeypatch: pytest.MonkeyPatch, fake_zscaler: dict
+ ) -> None:
+ monkeypatch.setenv("SSL_CERT_FILE", fake_zscaler["root"])
+ policy = describe_trust_policy()
+ assert policy["system_store_active"] is False
+ assert policy["replacement_bundle"]["env_var"] == "SSL_CERT_FILE"
+
+ def test_websocket_and_urlopen_use_os_store(self) -> None:
+ truststore = pytest.importorskip("truststore")
+ assert isinstance(build_websocket_ssl(), truststore.SSLContext)
+ assert isinstance(build_urlopen_context(), truststore.SSLContext)
+
+
+class TestFakeZscaler:
+ def test_certifi_alone_rejects_the_chain(
+ self, monkeypatch: pytest.MonkeyPatch, fake_zscaler: dict
+ ) -> None:
+ """The original bug: no corporate root, every upstream call fails."""
+ monkeypatch.setenv("HEADROOM_CERT_STORE", "bundled")
+ with pytest.raises(httpx.ConnectError) as info:
+ httpx.get(fake_zscaler["url"], verify=build_httpx_verify())
+ assert tls_diagnostics.find_cert_verify_error(info.value) is not None
+
+ @pytest.mark.parametrize("var", ["HEADROOM_CA_BUNDLE", "NODE_EXTRA_CA_CERTS"])
+ def test_default_policy_plus_corporate_root_succeeds(
+ self, monkeypatch: pytest.MonkeyPatch, fake_zscaler: dict, var: str
+ ) -> None:
+ """Additive root on top of the OS store, no HEADROOM_TLS_STRICT needed."""
+ pytest.importorskip("truststore")
+ monkeypatch.setenv(var, fake_zscaler["root"])
+ resp = httpx.get(fake_zscaler["url"], verify=build_httpx_verify())
+ assert resp.status_code == 200
+ assert resp.json() == {"ok": True}
+
+ def test_websocket_context_trusts_corporate_root(
+ self, monkeypatch: pytest.MonkeyPatch, fake_zscaler: dict
+ ) -> None:
+ import socket
+
+ monkeypatch.setenv("HEADROOM_CA_BUNDLE", fake_zscaler["root"])
+ ctx = build_websocket_ssl()
+ assert isinstance(ctx, ssl.SSLContext)
+ with socket.create_connection(("127.0.0.1", fake_zscaler["port"]), timeout=5) as raw:
+ with ctx.wrap_socket(raw, server_hostname="127.0.0.1") as tls:
+ assert tls.version() is not None
+
+ def test_probe_identifies_zscaler(self, fake_zscaler: dict) -> None:
+ info = tls_diagnostics.probe_presented_chain(
+ "127.0.0.1", fake_zscaler["port"], allow_private=True
+ )
+ assert info.reachable
+ assert info.issuer is not None and "Zscaler Root CA" in info.issuer
+ assert info.inspection_vendor == "Zscaler"
+
+ def test_failure_message_names_vendor_and_fix(
+ self, monkeypatch: pytest.MonkeyPatch, fake_zscaler: dict
+ ) -> None:
+ monkeypatch.setenv("HEADROOM_CERT_STORE", "bundled")
+ with pytest.raises(httpx.ConnectError) as info:
+ httpx.get(fake_zscaler["url"], verify=build_httpx_verify())
+ message = tls_diagnostics.describe_upstream_failure(
+ info.value, fake_zscaler["url"], allow_private=True
+ )
+ assert message is not None
+ assert "Zscaler" in message
+ assert "HEADROOM_CA_BUNDLE" in message
+ assert "headroom doctor --network" in message
+
+ def test_request_path_probe_skips_private_hosts(self, fake_zscaler: dict) -> None:
+ """A per-request upstream must not reveal an internal service's certificate."""
+ info = tls_diagnostics.probe_presented_chain("127.0.0.1", fake_zscaler["port"])
+ assert not info.reachable
+ assert info.issuer is None
+ assert "non-public" in (info.error or "")
+
+ def test_non_tls_errors_are_left_alone(self) -> None:
+ err = httpx.ConnectError("connection refused")
+ assert tls_diagnostics.describe_upstream_failure(err, "https://x.invalid") is None
+
+ def test_doctor_endpoint_probe_reports_vendor(
+ self, monkeypatch: pytest.MonkeyPatch, fake_zscaler: dict
+ ) -> None:
+ monkeypatch.setenv("HEADROOM_CERT_STORE", "bundled")
+ failing = tls_diagnostics.probe_endpoint("fake", fake_zscaler["url"])
+ assert not failing.ok
+ assert failing.chain.inspection_vendor == "Zscaler"
+
+ monkeypatch.setenv("HEADROOM_CA_BUNDLE", fake_zscaler["root"])
+ monkeypatch.delenv("HEADROOM_CERT_STORE")
+ passing = tls_diagnostics.probe_endpoint("fake", fake_zscaler["url"])
+ assert passing.ok and passing.status == 200
+
+
+class TestProcessTrust:
+ @pytest.fixture(autouse=True)
+ def _reset(self, monkeypatch: pytest.MonkeyPatch) -> Iterator[list[int]]:
+ calls: list[int] = []
+ truststore = pytest.importorskip("truststore")
+ monkeypatch.setattr(truststore, "inject_into_ssl", lambda: calls.append(1))
+ monkeypatch.setattr(ssl_context, "_process_trust_injected", False)
+ yield calls
+
+ def test_injects_once(self, _reset: list[int]) -> None:
+ assert ensure_process_trust() is True
+ assert ensure_process_trust() is True
+ assert _reset == [1]
+
+ def test_respects_replacement_bundle(
+ self, monkeypatch: pytest.MonkeyPatch, fake_zscaler: dict, _reset: list[int]
+ ) -> None:
+ monkeypatch.setenv("REQUESTS_CA_BUNDLE", fake_zscaler["root"])
+ assert ensure_process_trust() is False
+ assert _reset == []
+
+ def test_respects_opt_out(self, monkeypatch: pytest.MonkeyPatch, _reset: list[int]) -> None:
+ monkeypatch.setenv("HEADROOM_CERT_STORE", "bundled")
+ assert ensure_process_trust() is False
+ assert _reset == []
+
+
+class TestLoopbackNoProxy:
+ def test_gap_detected_only_with_proxy(self) -> None:
+ assert tls_diagnostics.loopback_no_proxy_gap({}) is None
+ assert (
+ tls_diagnostics.loopback_no_proxy_gap({"HTTPS_PROXY": "http://proxy:8080"})
+ == "HTTPS_PROXY"
+ )
+ assert (
+ tls_diagnostics.loopback_no_proxy_gap(
+ {"HTTPS_PROXY": "http://p:1", "no_proxy": "localhost,127.0.0.1"}
+ )
+ is None
+ )
+
+ def test_wrap_merges_both_spellings(self) -> None:
+ from headroom.cli.wrap import _ensure_loopback_no_proxy
+
+ env = {"HTTPS_PROXY": "http://p:1", "NO_PROXY": "corp.internal", "no_proxy": ".svc"}
+ written = _ensure_loopback_no_proxy(env)
+ assert set(written) == {"NO_PROXY", "no_proxy"}
+ assert env["NO_PROXY"] == env["no_proxy"] == "corp.internal,.svc,127.0.0.1,localhost,::1"
+ assert _ensure_loopback_no_proxy(env) == []
+
+ def test_wrap_leaves_env_alone_without_proxy(self) -> None:
+ from headroom.cli.wrap import _ensure_loopback_no_proxy
+
+ env = {"NO_PROXY": "corp.internal"}
+ assert _ensure_loopback_no_proxy(env) == []
+ assert env == {"NO_PROXY": "corp.internal"}
+
+
+class TestDoctorRows:
+ def test_trust_policy_rows(self, monkeypatch: pytest.MonkeyPatch, fake_zscaler: dict) -> None:
+ from headroom.cli.doctor import PASS, WARN, check_trust_policy
+
+ pytest.importorskip("truststore")
+ assert check_trust_policy(describe_trust_policy()).status == PASS
+ monkeypatch.setenv("HEADROOM_CERT_STORE", "bundled")
+ assert check_trust_policy(describe_trust_policy()).status == WARN
+ monkeypatch.setenv("SSL_CERT_FILE", fake_zscaler["root"])
+ row = check_trust_policy(describe_trust_policy())
+ assert row.status == PASS and "SSL_CERT_FILE" in row.summary
+
+ def test_network_rows_and_it_request(
+ self, monkeypatch: pytest.MonkeyPatch, fake_zscaler: dict
+ ) -> None:
+ from headroom.cli.doctor import FAIL, check_network_endpoints
+
+ monkeypatch.setenv("HEADROOM_CERT_STORE", "bundled")
+ report = tls_diagnostics.probe_endpoint("api.example", fake_zscaler["url"])
+ rows = check_network_endpoints([report], required={"api.example"})
+ assert rows[0].status == FAIL
+ assert "Zscaler Root CA" in rows[0].summary
+ it_row = rows[-1]
+ assert it_row.name == "it request"
+ assert "Zscaler" in (it_row.hint or "")
+ assert "127.0.0.1" in (it_row.hint or "")
+
+ def test_block_page_detection(self) -> None:
+ html = b"<html><title>Zscaler</title>Website blocked by policy</html>"
+ assert tls_diagnostics._looks_like_block_page(403, "text/html", html)
+ assert not tls_diagnostics._looks_like_block_page(
+ 200, "text/html", b"<html>Hugging Face privacy policy</html>"
+ )
+ assert not tls_diagnostics._looks_like_block_page(403, "application/json", b"{}")
+
+
+class TestStreamingErrorSurface:
+ """The agent's error line names the TLS cause instead of a bare 502."""
+
+ @pytest.mark.asyncio
+ async def test_cert_failure_reaches_client_as_actionable_sse_error(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ from unittest.mock import AsyncMock, MagicMock
+
+ from tests.test_h2_stream_reset_retry import _mock_proxy, _run_stream
+
+ issued_by = tls_diagnostics.ChainInfo(
+ host="api.anthropic.com",
+ port=443,
+ reachable=True,
+ issuer="O=Zscaler Inc., CN=Zscaler Intermediate Root CA",
+ inspection_vendor="Zscaler",
+ )
+ monkeypatch.setattr(tls_diagnostics, "probe_presented_chain", lambda *a, **k: issued_by)
+ cert_err = ssl.SSLCertVerificationError(
+ 1, "certificate verify failed: unable to get local issuer certificate"
+ )
+ cert_err.verify_message = "unable to get local issuer certificate"
+ connect_err = httpx.ConnectError("[SSL: CERTIFICATE_VERIFY_FAILED]")
+ connect_err.__cause__ = cert_err
+
+ proxy = _mock_proxy()
+ proxy.http_client.build_request = MagicMock(return_value=MagicMock())
+ proxy.http_client.send = AsyncMock(side_effect=connect_err)
+
+ result = await _run_stream(proxy)
+ body = b"".join([chunk async for chunk in result.body_iterator]).decode()
+
+ assert result.status_code == 502
+ assert "Zscaler is inspecting this connection" in body
+ assert "HEADROOM_CA_BUNDLE" in body
+ assert re.search(r"certificate for api\.anthropic\.com \(", body)
+
+
+class TestReviewFixes:
+ def test_opt_out_aliases_are_honored(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv("HEADROOM_CERT_STORE", "bundle")
+ assert cert_store_sources() == {"bundled"}
+ monkeypatch.setenv("HEADROOM_CERT_STORE", "os")
+ assert cert_store_sources() == {"system"}
+
+ def test_system_context_is_built_once_per_config(
+ self, monkeypatch: pytest.MonkeyPatch, fake_zscaler: dict
+ ) -> None:
+ pytest.importorskip("truststore")
+ ssl_context._system_ctx_cache.clear()
+ first = build_websocket_ssl()
+ assert build_websocket_ssl() is first
+ monkeypatch.setenv("HEADROOM_CA_BUNDLE", fake_zscaler["root"])
+ assert build_websocket_ssl() is not first
+
+ def test_partial_chain_is_enabled(self) -> None:
+ pytest.importorskip("truststore")
+ flag = getattr(ssl, "VERIFY_X509_PARTIAL_CHAIN", 0)
+ if not flag:
+ pytest.skip("OpenSSL build without partial-chain support")
+ ctx = build_httpx_verify()
+ assert isinstance(ctx, ssl.SSLContext)
+ assert ctx.verify_flags & flag
+
+ def test_proxy_startup_only_injects_in_cli_owned_process(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ truststore = pytest.importorskip("truststore")
+ calls: list[int] = []
+ monkeypatch.setattr(truststore, "inject_into_ssl", lambda: calls.append(1))
+ monkeypatch.setattr(ssl_context, "_process_trust_injected", False)
+ monkeypatch.delenv(ssl_context.PROCESS_TRUST_ENV, raising=False)
+ assert ssl_context.ensure_process_trust_if_owned() is False
+ assert calls == []
+ monkeypatch.setenv(ssl_context.PROCESS_TRUST_ENV, "1")
+ assert ssl_context.ensure_process_trust_if_owned() is True
+ assert calls == [1]
+
+ def test_concurrent_probes_share_one_handshake(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ import time as _time
+ from concurrent.futures import ThreadPoolExecutor
+
+ calls: list[str] = []
+
+ def slow_probe(host: str, port: int, **_: object) -> tls_diagnostics.ChainInfo:
+ calls.append(host)
+ _time.sleep(0.2)
+ return tls_diagnostics.ChainInfo(host=host, port=port, reachable=True)
+
+ monkeypatch.setattr(tls_diagnostics, "_probe_uncached", slow_probe)
+ with ThreadPoolExecutor(8) as pool:
+ list(
+ pool.map(
+ lambda _: tls_diagnostics.probe_presented_chain("api.example.com"), range(8)
+ )
+ )
+ assert calls == ["api.example.com"]
+
+ def test_private_allowed_probe_is_not_served_to_request_path(self, fake_zscaler: dict) -> None:
+ port = fake_zscaler["port"]
+ allowed = tls_diagnostics.probe_presented_chain("127.0.0.1", port, allow_private=True)
+ assert allowed.issuer
+ guarded = tls_diagnostics.probe_presented_chain("127.0.0.1", port)
+ assert guarded.issuer is None
+
+ def test_probe_ignores_os_proxy_settings(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ import urllib.request
+
+ monkeypatch.setattr(urllib.request, "getproxies", lambda: {"https": "http://os-proxy:3128"})
+ assert tls_diagnostics._proxy_for("api.anthropic.com") is None
+ monkeypatch.setenv("HTTPS_PROXY", "http://env-proxy:8080")
+ assert tls_diagnostics._proxy_for("api.anthropic.com") == "http://env-proxy:8080"
+
+ def test_wrap_notice_goes_to_stderr(
+ self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
+ ) -> None:
+ """`wrap <tool> --prepare-only` stdout is machine-read JSON; keep it clean."""
+ import click
+
+ from headroom.cli import wrap as wrap_mod
+
+ monkeypatch.setenv("HTTPS_PROXY", "http://proxy:8080")
+ # Registered so monkeypatch restores them: the callback writes os.environ.
+ monkeypatch.setenv("NO_PROXY", "")
+ monkeypatch.setenv("no_proxy", "")
+ monkeypatch.setattr(wrap_mod, "_should_purge_context_tools", lambda ctx: False)
+ with click.Context(wrap_mod.wrap) as ctx:
+ ctx.invoke(wrap_mod.wrap.callback)
+ out, err = capsys.readouterr()
+ assert out == ""
+ assert "NO_PROXY" in err
diff --git a/tests/test_ssl_context.py b/tests/test_ssl_context.py
index 5d8ee7706..a519c707f 100644
--- a/tests/test_ssl_context.py
+++ b/tests/test_ssl_context.py
@@ -62,14 +62,21 @@ def ca_pem_file(tmp_path):
def _clean_env(monkeypatch):
- """Remove all CA-bundle env vars + the strict toggle for a clean state."""
+ """Remove all CA-bundle env vars + the strict toggle for a clean state.
+
+ Pins ``HEADROOM_CERT_STORE=bundled`` so these tests keep covering the
+ certifi / env-bundle resolution without the OS trust store in front of it;
+ the OS-store default is covered in ``test_corporate_tls.py``.
+ """
for var in (
"SSL_CERT_FILE",
"REQUESTS_CA_BUNDLE",
"NODE_EXTRA_CA_CERTS",
+ "HEADROOM_CA_BUNDLE",
"HEADROOM_TLS_STRICT",
):
monkeypatch.delenv(var, raising=False)
+ monkeypatch.setenv("HEADROOM_CERT_STORE", "bundled")
def _default_x509_ca_count() -> int:
@@ -256,10 +263,20 @@ class TestTlsStrictDisabled:
class TestBuildHttpxVerify:
- def test_default_returns_true(self, monkeypatch):
- """No CA bundle, strict on → httpx's own default verification."""
+ def test_default_is_httpx_equivalent_certifi_context(self, monkeypatch):
+ """No CA bundle, strict on → what httpx builds for verify=True (certifi).
+
+ Always a concrete verifying context, never a boolean.
+ """
+ import certifi
+
_clean_env(monkeypatch)
- assert build_httpx_verify() is True
+ ctx = build_httpx_verify()
+ assert isinstance(ctx, ssl.SSLContext)
+ assert ctx.verify_mode == ssl.CERT_REQUIRED
+ assert ctx.check_hostname is True
+ expected = ssl.create_default_context(cafile=certifi.where())
+ assert ctx.cert_store_stats()["x509_ca"] == expected.cert_store_stats()["x509_ca"]
def test_toggle_off_returns_relaxed_context(self, monkeypatch):
"""No CA bundle, strict OFF → default trust store with strict cleared."""
@@ -306,7 +323,7 @@ class TestBuildUrlopenContext:
assert created_context.alpn_protocols == ["http/1.1"]
def test_default_returns_none(self, monkeypatch):
- """No CA bundle, strict on → build_httpx_verify() is True, nothing to restrict."""
+ """No CA bundle, strict on → no configured context; urlopen keeps its default."""
_clean_env(monkeypatch)
assert build_urlopen_context() is None
diff --git a/uv.lock b/uv.lock
index 43211db42..65f55d8bc 100644
--- a/uv.lock
+++ b/uv.lock
@@ -1847,7 +1847,7 @@ wheels = [
[[package]]
name = "headroom-ai"
-version = "0.38.0"
+version = "0.39.1"
source = { editable = "." }
dependencies = [
{ name = "ast-grep-cli" },
@@ -1860,6 +1860,7 @@ dependencies = [
{ name = "tiktoken" },
{ name = "tomli", marker = "python_full_version < '3.11'" },
{ name = "tomlkit" },
+ { name = "truststore" },
]
[package.optional-dependencies]
@@ -2203,6 +2204,7 @@ requires-dist = [
{ name = "transformers", marker = "extra == 'voice'", specifier = ">=5.5.0,<6.0" },
{ name = "tree-sitter", marker = "extra == 'code'", specifier = ">=0.25.2,<0.27" },
{ name = "tree-sitter-language-pack", marker = "extra == 'code'", specifier = ">=0.10.0,<1.0" },
+ { name = "truststore", specifier = ">=0.10.0" },
{ name = "uvicorn", marker = "extra == 'dev'", specifier = ">=0.23.0,<1.0" },
{ name = "uvicorn", marker = "extra == 'mcp'", specifier = ">=0.23.0,<1.0" },
{ name = "uvicorn", marker = "extra == 'proxy'", specifier = ">=0.23.0,<1.0" },
@@ -7280,6 +7282,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/f0/ac/229b7d4589d2e5937310e72c6d46e89599d16a4a12b479ffa1499fee8eb8/triton-3.7.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:10ba85fa2cca4a2fbdeb36bf1cb082f2c252bda55bf9fccd74f65ec5bc647e68", size = 197824404, upload-time = "2026-06-17T19:53:42.772Z" },
]
+[[package]]
+name = "truststore"
+version = "0.10.4"
+source = { registry = "https://pypi.org/simple/" }
+sdist = { url = "https://files.pythonhosted.org/packages/53/a3/1585216310e344e8102c22482f6060c7a6ea0322b63e026372e6dcefcfd6/truststore-0.10.4.tar.gz", hash = "sha256:9d91bd436463ad5e4ee4aba766628dd6cd7010cf3e2461756b3303710eebc301", size = 26169, upload-time = "2025-08-12T18:49:02.73Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/19/97/56608b2249fe206a67cd573bc93cd9896e1efb9e98bce9c163bcdc704b88/truststore-0.10.4-py3-none-any.whl", hash = "sha256:adaeaecf1cbb5f4de3b1959b42d41f6fab57b2b1666adb59e89cb0b53361d981", size = 18660, upload-time = "2025-08-12T18:49:01.46Z" },
+]
+
[[package]]
name = "typer"
version = "0.21.1"