Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(tls): add HEADROOM_TLS_STRICT=0 toggle for corporate SSL inspection (#1308) (#1341)
Lakshya Sharma52068dd650d0
diff --git a/README.md b/README.mdindex f4c90afb5..f2903aad3 100644--- a/README.md+++ b/README.md@@ -364,6 +364,35 @@ Two runtime assets are fetched over TLS; if they are blocked, trust your corporaRunning with compression disabled (pure gateway) requires neither asset.+#### "Basic Constraints of CA cert not marked critical" (Python 3.13+ strict mode)++A **different** failure from the one above. If TLS fails with:++```+[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed:+Basic Constraints of CA cert not marked critical+```++then the corporate CA *is* found and trusted — adding it to a CA bundle changes nothing.+Python 3.13 + OpenSSL 3.x enable `VERIFY_X509_STRICT` by default, which enforces RFC 5280+§4.2.1.9: a CA cert's `basicConstraints` must be marked *critical*. Inspection roots like+Zscaler set `CA:TRUE` without the critical bit, so the chain is rejected.++Set **`HEADROOM_TLS_STRICT=0`** to clear *only* the strict flag from every TLS context+Headroom controls — the proxy's httpx upstream client **and** the urllib3/`huggingface_hub`+path used for model downloads. Chain validation, signature, expiry, and hostname checks all+stay on; this is strictly narrower than disabling verification.++```bash+HEADROOM_TLS_STRICT=0 headroom proxy --port 8787+```++The Rust core's ONNX download (`cdn.pyke.io`) uses a separate TLS stack (rustls / OS trust+store), unaffected by `HEADROOM_TLS_STRICT`. On Windows the corporate root must be in the+**machine** certificate store (browsers already trust it there); or pre-provision ONNX+Runtime with `ORT_STRATEGY=system` + `ORT_LIB_LOCATION=/path/to/onnxruntime` to skip the+download entirely.+## headroom learn<p align="center">diff --git a/headroom/cli/proxy.py b/headroom/cli/proxy.pyindex 1a8f73aa0..c0d757325 100644--- a/headroom/cli/proxy.py+++ b/headroom/cli/proxy.py@@ -34,6 +34,20 @@ from .main import mainos.environ.setdefault("HF_HUB_DISABLE_IMPLICIT_TOKEN", "1")os.environ.setdefault("HF_HUB_DISABLE_PROGRESS_BARS", "1")+# Corporate TLS-inspection support (issue #1308). When HEADROOM_TLS_STRICT=0,+# strip OpenSSL's RFC 5280 strict CA-constraint check from urllib3's context+# builder *before* huggingface_hub / requests import and cache it — otherwise+# model downloads (huggingface.co) fail with "Basic Constraints of CA cert not+# marked critical" behind Zscaler/Netskope on Python 3.13+. The proxy's own+# httpx upstream client is handled separately in proxy/server.py via+# build_httpx_verify(). No-op unless the toggle is set.+try: # pragma: no cover - exercised via integration, not unit-importable cheaply+ from headroom.proxy.ssl_context import apply_global_tls_relaxation as _apply_tls_relax++ _apply_tls_relax()+except Exception: # never let TLS relaxation wiring break startup+ pass+# Logger-level suppression: httpx HEAD/GET manifest checks + HF advisory msgs.logging.getLogger("httpx").setLevel(logging.WARNING)logging.getLogger("huggingface_hub").setLevel(logging.ERROR)diff --git a/headroom/proxy/server.py b/headroom/proxy/server.pyindex 02086ccd4..07b11a56e 100644--- a/headroom/proxy/server.py+++ b/headroom/proxy/server.py@@ -154,7 +154,7 @@ from headroom.proxy.prometheus_metrics import PrometheusMetrics # noqa: F401from headroom.proxy.rate_limiter import TokenBucketRateLimiter # noqa: F401from headroom.proxy.request_logger import RequestLogger # noqa: F401from headroom.proxy.semantic_cache import SemanticCache # noqa: F401-from headroom.proxy.ssl_context import find_ca_bundle+from headroom.proxy.ssl_context import build_httpx_verifyfrom headroom.proxy.warmup import WarmupRegistryfrom headroom.proxy.ws_session_registry import WebSocketSessionRegistryfrom headroom.subscription.base import get_quota_registry, reset_quota_registry@@ -1227,7 +1227,10 @@ class HeadroomProxy(operation="proxy.startup",metadata={"port": self.config.port, "host": self.config.host},)- _ca_bundle = find_ca_bundle()+ # Resolve TLS verification: a custom CA bundle (corporate PKI) if one+ # is configured, else a strict-relaxed default context when+ # HEADROOM_TLS_STRICT=0, else httpx's default strict verification.+ _verify = build_httpx_verify()_client_kwargs: dict[str, Any] = {"timeout": httpx.Timeout(connect=self.config.connect_timeout_seconds,@@ -1240,7 +1243,7 @@ class HeadroomProxy(max_keepalive_connections=self.config.max_keepalive_connections,keepalive_expiry=self.config.keepalive_expiry,),- "verify": _ca_bundle if _ca_bundle is not None else True,+ "verify": _verify,}self.http_client = httpx.AsyncClient(http2=self.config.http2, **_client_kwargs)# Reuse the primary client when HTTP/2 is already off; otherwise keep adiff --git a/headroom/proxy/ssl_context.py b/headroom/proxy/ssl_context.pyindex 551013b05..7ecba7c31 100644--- a/headroom/proxy/ssl_context.py+++ b/headroom/proxy/ssl_context.py@@ -11,6 +11,22 @@ Priority order (first match wins):2. ``REQUESTS_CA_BUNDLE`` — replacement semantics3. ``NODE_EXTRA_CA_CERTS`` — **additive** semantics (extra roots loadedon top of the default/system trust store, matching Node.js behavior)++Strict-mode toggle (``HEADROOM_TLS_STRICT``):+ Python 3.13 + OpenSSL 3.x enable ``VERIFY_X509_STRICT`` by default, which+ enforces RFC 5280 §4.2.1.9 — a CA cert's ``basicConstraints`` MUST be+ marked critical. Corporate TLS-inspection roots (Zscaler, Netskope, …)+ commonly set ``CA:TRUE`` *without* the critical bit, so the chain is+ rejected with ``Basic Constraints of CA cert not marked critical`` even+ though the root is correctly installed and trusted. A CA bundle env var+ can't fix this — the cert is found, it's the strict check that fails.++ Setting ``HEADROOM_TLS_STRICT=0`` clears *only* ``VERIFY_X509_STRICT`` from+ every TLS context Headroom controls (the httpx upstream client AND the+ urllib3/requests stack used by ``huggingface_hub`` for model downloads).+ Chain validation, signature checks, expiry, and hostname verification all+ stay on — this is strictly narrower than ``verify=False``. Default is+ strict (the flag stays set) to match Python's own default."""from __future__ import annotations@@ -18,6 +34,7 @@ from __future__ import annotationsimport loggingimport osimport ssl+from typing import Any, castlogger = logging.getLogger("headroom.proxy")@@ -26,6 +43,34 @@ _REPLACEMENT_CA_VARS = ("REQUESTS_CA_BUNDLE",)+# Env var that opts out of OpenSSL's RFC 5280 strict CA-constraint checks.+TLS_STRICT_ENV = "HEADROOM_TLS_STRICT"++# Values (case-insensitive) that mean "turn strict mode OFF".+_TLS_STRICT_OFF_VALUES = frozenset({"0", "false", "no", "off"})+++def tls_strict_disabled() -> bool:+ """True when ``HEADROOM_TLS_STRICT`` opts out of OpenSSL strict mode.++ Default (unset / any other value) is strict, matching Python 3.13's own+ default. Only the explicit off-values flip it.+ """+ return os.environ.get(TLS_STRICT_ENV, "").strip().lower() in _TLS_STRICT_OFF_VALUES+++def _clear_x509_strict(ctx: ssl.SSLContext, *, reason: str) -> ssl.SSLContext:+ """Clear only ``VERIFY_X509_STRICT`` from a context, leaving all else on.++ Keeps certificate verification, hostname verification, expiry checks, and+ chain validation enabled — this is far narrower than disabling verify.+ """+ strict_flag = getattr(ssl, "VERIFY_X509_STRICT", 0)+ if strict_flag and ctx.verify_flags & strict_flag:+ ctx.verify_flags &= ~strict_flag+ logger.info("event=ssl_x509_strict_disabled reason=%s", reason)+ return ctx+def _relax_x509_strict_for_custom_ca(ctx: ssl.SSLContext, *, path: str) -> ssl.SSLContext:"""Relax OpenSSL strict-mode checks for an operator-provided CA bundle.@@ -35,12 +80,14 @@ def _relax_x509_strict_for_custom_ca(ctx: ssl.SSLContext, *, path: str) -> ssl.SkeyUsage extension. Clearing only ``VERIFY_X509_STRICT`` keeps certificateverification, hostname verification, expiry checks, and chain validationenabled while making custom CA bundles usable in those environments.++ A custom CA bundle is itself a strong signal of a corporate PKI, so the+ strict flag is relaxed here regardless of ``HEADROOM_TLS_STRICT`` (the+ historical behavior). The env toggle additionally covers the case where+ the corporate root lives in the *default* trust store and no bundle var+ is set — see :func:`build_httpx_verify`."""- strict_flag = getattr(ssl, "VERIFY_X509_STRICT", 0)- if strict_flag and ctx.verify_flags & strict_flag:- ctx.verify_flags &= ~strict_flag- logger.info("event=ssl_x509_strict_disabled_for_custom_ca path=%s", path)- return ctx+ return _clear_x509_strict(ctx, reason=f"custom_ca:{path}")def _replacement_ca_context(path: str) -> ssl.SSLContext:@@ -102,3 +149,93 @@ def find_ca_bundle() -> ssl.SSLContext | None:)return None+++def _default_strict_relaxed_context() -> ssl.SSLContext:+ """Default trust store, but with ``VERIFY_X509_STRICT`` cleared.++ Used when no custom CA bundle is configured (the corporate root lives in+ the OS/default trust store) but ``HEADROOM_TLS_STRICT=0`` asks us to+ tolerate a non-critical ``basicConstraints`` CA. Mirrors what httpx builds+ for ``verify=True`` (default context + ALPN), minus the strict flag.+ """+ ctx = ssl.create_default_context()+ ctx.set_alpn_protocols(["h2", "http/1.1"])+ return _clear_x509_strict(ctx, reason="env_toggle")+++def build_httpx_verify() -> ssl.SSLContext | bool:+ """Return the value for httpx's ``verify=`` parameter.++ Resolution order:++ 1. A custom CA bundle env var (``SSL_CERT_FILE`` / ``REQUESTS_CA_BUNDLE`` /+ ``NODE_EXTRA_CA_CERTS``) → a context trusting that bundle, with strict+ mode already relaxed (corporate PKI signal).+ 2. No bundle, but ``HEADROOM_TLS_STRICT=0`` → the default trust store with+ ``VERIFY_X509_STRICT`` cleared, so a corporate root that's installed in+ the OS store but trips RFC 5280 strict mode still validates.+ 3. Otherwise → ``True`` (httpx's default strict verification).++ Returning ``True`` rather than a hand-built context in the common case+ keeps httpx's own default behavior (including its certifi fallback) intact.+ """+ ca_ctx = find_ca_bundle()+ if ca_ctx is not None:+ return ca_ctx+ if tls_strict_disabled():+ return _default_strict_relaxed_context()+ return True+++def apply_global_tls_relaxation() -> bool:+ """Strip ``VERIFY_X509_STRICT`` from urllib3's context builder when opted in.++ The proxy's upstream httpx client is handled explicitly via+ :func:`build_httpx_verify`, but model downloads go through+ ``huggingface_hub`` → ``requests`` → ``urllib3``, which builds its own+ context via ``urllib3.util.ssl_.create_urllib3_context`` and sets+ ``VERIFY_X509_STRICT`` independently (urllib3 ≥ 2.5). That path never sees+ our httpx context, so a corporate-MITM user hits the same+ ``Basic Constraints ... not marked critical`` rejection on a model cache+ miss.++ When ``HEADROOM_TLS_STRICT=0`` this monkeypatches+ ``create_urllib3_context`` to clear the strict flag from every context it+ returns. The patch is idempotent (guarded by a sentinel attribute) and a+ no-op when urllib3 isn't importable. Returns True if a patch was applied+ (or was already in place), False otherwise.++ Call this as early as possible — before ``huggingface_hub`` / ``requests``+ import and cache their context — i.e. at CLI startup.+ """+ if not tls_strict_disabled():+ return False++ strict_flag = getattr(ssl, "VERIFY_X509_STRICT", 0)+ if not strict_flag:+ return False++ try:+ import urllib3.util.ssl_ as _u3ssl+ except Exception: # pragma: no cover - urllib3 always present in practice+ logger.debug("event=ssl_urllib3_patch_skipped reason=import_failed")+ return False++ if getattr(_u3ssl.create_urllib3_context, "_headroom_strict_relaxed", False):+ return True++ _orig = _u3ssl.create_urllib3_context++ def _relaxed_create_urllib3_context(*args: Any, **kwargs: Any) -> ssl.SSLContext:+ # urllib3's create_urllib3_context signature varies across versions;+ # forward verbatim and cast the (Any-typed) result back to SSLContext.+ ctx = cast(ssl.SSLContext, _orig(*args, **kwargs))+ if ctx.verify_flags & strict_flag:+ ctx.verify_flags &= ~strict_flag+ return ctx++ _relaxed_create_urllib3_context._headroom_strict_relaxed = True # type: ignore[attr-defined]+ _u3ssl.create_urllib3_context = _relaxed_create_urllib3_context # type: ignore[assignment]+ logger.info("event=ssl_x509_strict_disabled reason=urllib3_global_patch")+ return Truediff --git a/tests/test_ssl_context.py b/tests/test_ssl_context.pyindex 80bed9556..dda61f854 100644--- a/tests/test_ssl_context.py+++ b/tests/test_ssl_context.py@@ -18,7 +18,12 @@ import sslimport pytestfrom headroom.proxy import ssl_context-from headroom.proxy.ssl_context import find_ca_bundle+from headroom.proxy.ssl_context import (+ apply_global_tls_relaxation,+ build_httpx_verify,+ find_ca_bundle,+ tls_strict_disabled,+)# Minimal self-signed CA certificate (PEM) used only to verify that# load_verify_locations accepts the file. Generated offline; never used@@ -55,8 +60,13 @@ def ca_pem_file(tmp_path):def _clean_env(monkeypatch):- """Remove all three CA-bundle env vars so tests start from a clean state."""- for var in ("SSL_CERT_FILE", "REQUESTS_CA_BUNDLE", "NODE_EXTRA_CA_CERTS"):+ """Remove all CA-bundle env vars + the strict toggle for a clean state."""+ for var in (+ "SSL_CERT_FILE",+ "REQUESTS_CA_BUNDLE",+ "NODE_EXTRA_CA_CERTS",+ "HEADROOM_TLS_STRICT",+ ):monkeypatch.delenv(var, raising=False)@@ -209,3 +219,87 @@ class TestFindCaBundleNonexistentPaths:ctx = find_ca_bundle()assert isinstance(ctx, ssl.SSLContext)+++# ---------------------------------------------------------------------------+# HEADROOM_TLS_STRICT toggle (issue #1308): corporate TLS-inspection roots+# (Zscaler, Netskope) set CA:TRUE without the critical bit, which Python 3.13+# + OpenSSL 3.x reject under VERIFY_X509_STRICT. A CA bundle can't fix that —+# the cert is found, the strict check fails. The toggle clears only the strict+# flag, on both the httpx upstream path and the urllib3/huggingface path.+# ---------------------------------------------------------------------------+++class TestTlsStrictDisabled:+ @pytest.mark.parametrize("val", ["0", "false", "FALSE", "No", "off", " off "])+ def test_off_values_disable_strict(self, monkeypatch, val):+ _clean_env(monkeypatch)+ monkeypatch.setenv("HEADROOM_TLS_STRICT", val)+ assert tls_strict_disabled() is True++ @pytest.mark.parametrize("val", ["1", "true", "yes", "on", "", "strict", "00"])+ def test_other_values_keep_strict(self, monkeypatch, val):+ _clean_env(monkeypatch)+ monkeypatch.setenv("HEADROOM_TLS_STRICT", val)+ assert tls_strict_disabled() is False++ def test_unset_keeps_strict(self, monkeypatch):+ _clean_env(monkeypatch)+ assert tls_strict_disabled() is False+++class TestBuildHttpxVerify:+ def test_default_returns_true(self, monkeypatch):+ """No CA bundle, strict on → httpx's own default verification."""+ _clean_env(monkeypatch)+ assert build_httpx_verify() is True++ def test_toggle_off_returns_relaxed_context(self, monkeypatch):+ """No CA bundle, strict OFF → default trust store with strict cleared."""+ _clean_env(monkeypatch)+ monkeypatch.setenv("HEADROOM_TLS_STRICT", "0")+ ctx = build_httpx_verify()+ assert isinstance(ctx, ssl.SSLContext)+ strict_flag = getattr(ssl, "VERIFY_X509_STRICT", 0)+ if strict_flag:+ assert ctx.verify_flags & strict_flag == 0+ # Still a real verifying context — NOT verify=False.+ assert ctx.verify_mode == ssl.CERT_REQUIRED+ # Default trust store retained (additive, not a 1-cert replacement).+ assert ctx.cert_store_stats()["x509_ca"] > 1++ def test_custom_ca_takes_precedence_over_toggle(self, monkeypatch, ca_pem_file):+ """A configured CA bundle wins; the result is that bundle's context."""+ _clean_env(monkeypatch)+ monkeypatch.setenv("SSL_CERT_FILE", ca_pem_file)+ monkeypatch.setenv("HEADROOM_TLS_STRICT", "0")+ ctx = build_httpx_verify()+ assert isinstance(ctx, ssl.SSLContext)+ # Replacement bundle → only the single test CA is trusted.+ assert ctx.cert_store_stats()["x509_ca"] == 1+++class TestApplyGlobalTlsRelaxation:+ def test_noop_when_strict_on(self, monkeypatch):+ _clean_env(monkeypatch)+ assert apply_global_tls_relaxation() is False++ def test_patches_urllib3_when_toggle_off(self, monkeypatch):+ _clean_env(monkeypatch)+ monkeypatch.setenv("HEADROOM_TLS_STRICT", "0")+ strict_flag = getattr(ssl, "VERIFY_X509_STRICT", 0)+ if not strict_flag:+ pytest.skip("VERIFY_X509_STRICT unavailable on this OpenSSL build")++ import urllib3.util.ssl_ as u3ssl++ original = u3ssl.create_urllib3_context+ try:+ assert apply_global_tls_relaxation() is True+ ctx = u3ssl.create_urllib3_context()+ assert ctx.verify_flags & strict_flag == 0+ # Idempotent: second call doesn't re-wrap or error.+ assert apply_global_tls_relaxation() is True+ assert getattr(u3ssl.create_urllib3_context, "_headroom_strict_relaxed", False)+ finally:+ u3ssl.create_urllib3_context = original