Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(tls): add HEADROOM_TLS_STRICT=0 toggle for corporate SSL inspection (#1308) (#1341)

Lakshya Sharma52068dd650d0
diff --git a/README.md b/README.md
index f4c90afb5..f2903aad3 100644
--- a/README.md
+++ b/README.md
@@ -364,6 +364,35 @@ Two runtime assets are fetched over TLS; if they are blocked, trust your corpora
Running with compression disabled (pure gateway) requires neither asset.
+#### "Basic Constraints of CA cert not marked critical" (Python 3.13+ strict mode)
+
+A **different** failure from the one above. If TLS fails with:
+
+```
+[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed:
+Basic Constraints of CA cert not marked critical
+```
+
+then the corporate CA *is* found and trusted — adding it to a CA bundle changes nothing.
+Python 3.13 + OpenSSL 3.x enable `VERIFY_X509_STRICT` by default, which enforces RFC 5280
+§4.2.1.9: a CA cert's `basicConstraints` must be marked *critical*. Inspection roots like
+Zscaler set `CA:TRUE` without the critical bit, so the chain is rejected.
+
+Set **`HEADROOM_TLS_STRICT=0`** to clear *only* the strict flag from every TLS context
+Headroom controls — the proxy's httpx upstream client **and** the urllib3/`huggingface_hub`
+path used for model downloads. Chain validation, signature, expiry, and hostname checks all
+stay on; this is strictly narrower than disabling verification.
+
+```bash
+HEADROOM_TLS_STRICT=0 headroom proxy --port 8787
+```
+
+The Rust core's ONNX download (`cdn.pyke.io`) uses a separate TLS stack (rustls / OS trust
+store), unaffected by `HEADROOM_TLS_STRICT`. On Windows the corporate root must be in the
+**machine** certificate store (browsers already trust it there); or pre-provision ONNX
+Runtime with `ORT_STRATEGY=system` + `ORT_LIB_LOCATION=/path/to/onnxruntime` to skip the
+download entirely.
+
## headroom learn
<p align="center">
diff --git a/headroom/cli/proxy.py b/headroom/cli/proxy.py
index 1a8f73aa0..c0d757325 100644
--- a/headroom/cli/proxy.py
+++ b/headroom/cli/proxy.py
@@ -34,6 +34,20 @@ from .main import main
os.environ.setdefault("HF_HUB_DISABLE_IMPLICIT_TOKEN", "1")
os.environ.setdefault("HF_HUB_DISABLE_PROGRESS_BARS", "1")
+# Corporate TLS-inspection support (issue #1308). When HEADROOM_TLS_STRICT=0,
+# strip OpenSSL's RFC 5280 strict CA-constraint check from urllib3's context
+# builder *before* huggingface_hub / requests import and cache it — otherwise
+# model downloads (huggingface.co) fail with "Basic Constraints of CA cert not
+# marked critical" behind Zscaler/Netskope on Python 3.13+. The proxy's own
+# httpx upstream client is handled separately in proxy/server.py via
+# build_httpx_verify(). No-op unless the toggle is set.
+try: # pragma: no cover - exercised via integration, not unit-importable cheaply
+ from headroom.proxy.ssl_context import apply_global_tls_relaxation as _apply_tls_relax
+
+ _apply_tls_relax()
+except Exception: # never let TLS relaxation wiring break startup
+ pass
+
# Logger-level suppression: httpx HEAD/GET manifest checks + HF advisory msgs.
logging.getLogger("httpx").setLevel(logging.WARNING)
logging.getLogger("huggingface_hub").setLevel(logging.ERROR)
diff --git a/headroom/proxy/server.py b/headroom/proxy/server.py
index 02086ccd4..07b11a56e 100644
--- a/headroom/proxy/server.py
+++ b/headroom/proxy/server.py
@@ -154,7 +154,7 @@ from headroom.proxy.prometheus_metrics import PrometheusMetrics # noqa: F401
from headroom.proxy.rate_limiter import TokenBucketRateLimiter # noqa: F401
from headroom.proxy.request_logger import RequestLogger # noqa: F401
from headroom.proxy.semantic_cache import SemanticCache # noqa: F401
-from headroom.proxy.ssl_context import find_ca_bundle
+from headroom.proxy.ssl_context import build_httpx_verify
from headroom.proxy.warmup import WarmupRegistry
from headroom.proxy.ws_session_registry import WebSocketSessionRegistry
from headroom.subscription.base import get_quota_registry, reset_quota_registry
@@ -1227,7 +1227,10 @@ class HeadroomProxy(
operation="proxy.startup",
metadata={"port": self.config.port, "host": self.config.host},
)
- _ca_bundle = find_ca_bundle()
+ # Resolve TLS verification: a custom CA bundle (corporate PKI) if one
+ # is configured, else a strict-relaxed default context when
+ # HEADROOM_TLS_STRICT=0, else httpx's default strict verification.
+ _verify = build_httpx_verify()
_client_kwargs: dict[str, Any] = {
"timeout": httpx.Timeout(
connect=self.config.connect_timeout_seconds,
@@ -1240,7 +1243,7 @@ class HeadroomProxy(
max_keepalive_connections=self.config.max_keepalive_connections,
keepalive_expiry=self.config.keepalive_expiry,
),
- "verify": _ca_bundle if _ca_bundle is not None else True,
+ "verify": _verify,
}
self.http_client = httpx.AsyncClient(http2=self.config.http2, **_client_kwargs)
# Reuse the primary client when HTTP/2 is already off; otherwise keep a
diff --git a/headroom/proxy/ssl_context.py b/headroom/proxy/ssl_context.py
index 551013b05..7ecba7c31 100644
--- a/headroom/proxy/ssl_context.py
+++ b/headroom/proxy/ssl_context.py
@@ -11,6 +11,22 @@ Priority order (first match wins):
2. ``REQUESTS_CA_BUNDLE`` — replacement semantics
3. ``NODE_EXTRA_CA_CERTS`` — **additive** semantics (extra roots loaded
on top of the default/system trust store, matching Node.js behavior)
+
+Strict-mode toggle (``HEADROOM_TLS_STRICT``):
+ Python 3.13 + OpenSSL 3.x enable ``VERIFY_X509_STRICT`` by default, which
+ enforces RFC 5280 §4.2.1.9 — a CA cert's ``basicConstraints`` MUST be
+ marked critical. Corporate TLS-inspection roots (Zscaler, Netskope, …)
+ commonly set ``CA:TRUE`` *without* the critical bit, so the chain is
+ rejected with ``Basic Constraints of CA cert not marked critical`` even
+ though the root is correctly installed and trusted. A CA bundle env var
+ can't fix this — the cert is found, it's the strict check that fails.
+
+ Setting ``HEADROOM_TLS_STRICT=0`` clears *only* ``VERIFY_X509_STRICT`` from
+ every TLS context Headroom controls (the httpx upstream client AND the
+ urllib3/requests stack used by ``huggingface_hub`` for model downloads).
+ Chain validation, signature checks, expiry, and hostname verification all
+ stay on — this is strictly narrower than ``verify=False``. Default is
+ strict (the flag stays set) to match Python's own default.
"""
from __future__ import annotations
@@ -18,6 +34,7 @@ from __future__ import annotations
import logging
import os
import ssl
+from typing import Any, cast
logger = logging.getLogger("headroom.proxy")
@@ -26,6 +43,34 @@ _REPLACEMENT_CA_VARS = (
"REQUESTS_CA_BUNDLE",
)
+# Env var that opts out of OpenSSL's RFC 5280 strict CA-constraint checks.
+TLS_STRICT_ENV = "HEADROOM_TLS_STRICT"
+
+# Values (case-insensitive) that mean "turn strict mode OFF".
+_TLS_STRICT_OFF_VALUES = frozenset({"0", "false", "no", "off"})
+
+
+def tls_strict_disabled() -> bool:
+ """True when ``HEADROOM_TLS_STRICT`` opts out of OpenSSL strict mode.
+
+ Default (unset / any other value) is strict, matching Python 3.13's own
+ default. Only the explicit off-values flip it.
+ """
+ return os.environ.get(TLS_STRICT_ENV, "").strip().lower() in _TLS_STRICT_OFF_VALUES
+
+
+def _clear_x509_strict(ctx: ssl.SSLContext, *, reason: str) -> ssl.SSLContext:
+ """Clear only ``VERIFY_X509_STRICT`` from a context, leaving all else on.
+
+ Keeps certificate verification, hostname verification, expiry checks, and
+ chain validation enabled — this is far narrower than disabling verify.
+ """
+ strict_flag = getattr(ssl, "VERIFY_X509_STRICT", 0)
+ if strict_flag and ctx.verify_flags & strict_flag:
+ ctx.verify_flags &= ~strict_flag
+ logger.info("event=ssl_x509_strict_disabled reason=%s", reason)
+ return ctx
+
def _relax_x509_strict_for_custom_ca(ctx: ssl.SSLContext, *, path: str) -> ssl.SSLContext:
"""Relax OpenSSL strict-mode checks for an operator-provided CA bundle.
@@ -35,12 +80,14 @@ def _relax_x509_strict_for_custom_ca(ctx: ssl.SSLContext, *, path: str) -> ssl.S
keyUsage extension. Clearing only ``VERIFY_X509_STRICT`` keeps certificate
verification, hostname verification, expiry checks, and chain validation
enabled while making custom CA bundles usable in those environments.
+
+ A custom CA bundle is itself a strong signal of a corporate PKI, so the
+ strict flag is relaxed here regardless of ``HEADROOM_TLS_STRICT`` (the
+ historical behavior). The env toggle additionally covers the case where
+ the corporate root lives in the *default* trust store and no bundle var
+ is set — see :func:`build_httpx_verify`.
"""
- strict_flag = getattr(ssl, "VERIFY_X509_STRICT", 0)
- if strict_flag and ctx.verify_flags & strict_flag:
- ctx.verify_flags &= ~strict_flag
- logger.info("event=ssl_x509_strict_disabled_for_custom_ca path=%s", path)
- return ctx
+ return _clear_x509_strict(ctx, reason=f"custom_ca:{path}")
def _replacement_ca_context(path: str) -> ssl.SSLContext:
@@ -102,3 +149,93 @@ def find_ca_bundle() -> ssl.SSLContext | None:
)
return None
+
+
+def _default_strict_relaxed_context() -> ssl.SSLContext:
+ """Default trust store, but with ``VERIFY_X509_STRICT`` cleared.
+
+ Used when no custom CA bundle is configured (the corporate root lives in
+ the OS/default trust store) but ``HEADROOM_TLS_STRICT=0`` asks us to
+ tolerate a non-critical ``basicConstraints`` CA. Mirrors what httpx builds
+ for ``verify=True`` (default context + ALPN), minus the strict flag.
+ """
+ ctx = ssl.create_default_context()
+ ctx.set_alpn_protocols(["h2", "http/1.1"])
+ return _clear_x509_strict(ctx, reason="env_toggle")
+
+
+def build_httpx_verify() -> ssl.SSLContext | bool:
+ """Return the value for httpx's ``verify=`` parameter.
+
+ Resolution order:
+
+ 1. A custom CA bundle env var (``SSL_CERT_FILE`` / ``REQUESTS_CA_BUNDLE`` /
+ ``NODE_EXTRA_CA_CERTS``) → a context trusting that bundle, with strict
+ mode already relaxed (corporate PKI signal).
+ 2. No bundle, but ``HEADROOM_TLS_STRICT=0`` → the default trust store with
+ ``VERIFY_X509_STRICT`` cleared, so a corporate root that's installed in
+ the OS store but trips RFC 5280 strict mode still validates.
+ 3. Otherwise → ``True`` (httpx's default strict verification).
+
+ Returning ``True`` rather than a hand-built context in the common case
+ keeps httpx's own default behavior (including its certifi fallback) intact.
+ """
+ ca_ctx = find_ca_bundle()
+ if ca_ctx is not None:
+ return ca_ctx
+ if tls_strict_disabled():
+ return _default_strict_relaxed_context()
+ return True
+
+
+def apply_global_tls_relaxation() -> bool:
+ """Strip ``VERIFY_X509_STRICT`` from urllib3's context builder when opted in.
+
+ The proxy's upstream httpx client is handled explicitly via
+ :func:`build_httpx_verify`, but model downloads go through
+ ``huggingface_hub`` → ``requests`` → ``urllib3``, which builds its own
+ context via ``urllib3.util.ssl_.create_urllib3_context`` and sets
+ ``VERIFY_X509_STRICT`` independently (urllib3 ≥ 2.5). That path never sees
+ our httpx context, so a corporate-MITM user hits the same
+ ``Basic Constraints ... not marked critical`` rejection on a model cache
+ miss.
+
+ When ``HEADROOM_TLS_STRICT=0`` this monkeypatches
+ ``create_urllib3_context`` to clear the strict flag from every context it
+ returns. The patch is idempotent (guarded by a sentinel attribute) and a
+ no-op when urllib3 isn't importable. Returns True if a patch was applied
+ (or was already in place), False otherwise.
+
+ Call this as early as possible — before ``huggingface_hub`` / ``requests``
+ import and cache their context — i.e. at CLI startup.
+ """
+ if not tls_strict_disabled():
+ return False
+
+ strict_flag = getattr(ssl, "VERIFY_X509_STRICT", 0)
+ if not strict_flag:
+ return False
+
+ try:
+ import urllib3.util.ssl_ as _u3ssl
+ except Exception: # pragma: no cover - urllib3 always present in practice
+ logger.debug("event=ssl_urllib3_patch_skipped reason=import_failed")
+ return False
+
+ if getattr(_u3ssl.create_urllib3_context, "_headroom_strict_relaxed", False):
+ return True
+
+ _orig = _u3ssl.create_urllib3_context
+
+ def _relaxed_create_urllib3_context(*args: Any, **kwargs: Any) -> ssl.SSLContext:
+ # urllib3's create_urllib3_context signature varies across versions;
+ # forward verbatim and cast the (Any-typed) result back to SSLContext.
+ ctx = cast(ssl.SSLContext, _orig(*args, **kwargs))
+ if ctx.verify_flags & strict_flag:
+ ctx.verify_flags &= ~strict_flag
+ return ctx
+
+ _relaxed_create_urllib3_context._headroom_strict_relaxed = True # type: ignore[attr-defined]
+ _u3ssl.create_urllib3_context = _relaxed_create_urllib3_context # type: ignore[assignment]
+ logger.info("event=ssl_x509_strict_disabled reason=urllib3_global_patch")
+ return True
diff --git a/tests/test_ssl_context.py b/tests/test_ssl_context.py
index 80bed9556..dda61f854 100644
--- a/tests/test_ssl_context.py
+++ b/tests/test_ssl_context.py
@@ -18,7 +18,12 @@ import ssl
import pytest
from headroom.proxy import ssl_context
-from headroom.proxy.ssl_context import find_ca_bundle
+from headroom.proxy.ssl_context import (
+ apply_global_tls_relaxation,
+ build_httpx_verify,
+ find_ca_bundle,
+ tls_strict_disabled,
+)
# Minimal self-signed CA certificate (PEM) used only to verify that
# load_verify_locations accepts the file. Generated offline; never used
@@ -55,8 +60,13 @@ def ca_pem_file(tmp_path):
def _clean_env(monkeypatch):
- """Remove all three CA-bundle env vars so tests start from a clean state."""
- for var in ("SSL_CERT_FILE", "REQUESTS_CA_BUNDLE", "NODE_EXTRA_CA_CERTS"):
+ """Remove all CA-bundle env vars + the strict toggle for a clean state."""
+ for var in (
+ "SSL_CERT_FILE",
+ "REQUESTS_CA_BUNDLE",
+ "NODE_EXTRA_CA_CERTS",
+ "HEADROOM_TLS_STRICT",
+ ):
monkeypatch.delenv(var, raising=False)
@@ -209,3 +219,87 @@ class TestFindCaBundleNonexistentPaths:
ctx = find_ca_bundle()
assert isinstance(ctx, ssl.SSLContext)
+
+
+# ---------------------------------------------------------------------------
+# HEADROOM_TLS_STRICT toggle (issue #1308): corporate TLS-inspection roots
+# (Zscaler, Netskope) set CA:TRUE without the critical bit, which Python 3.13
+# + OpenSSL 3.x reject under VERIFY_X509_STRICT. A CA bundle can't fix that —
+# the cert is found, the strict check fails. The toggle clears only the strict
+# flag, on both the httpx upstream path and the urllib3/huggingface path.
+# ---------------------------------------------------------------------------
+
+
+class TestTlsStrictDisabled:
+ @pytest.mark.parametrize("val", ["0", "false", "FALSE", "No", "off", " off "])
+ def test_off_values_disable_strict(self, monkeypatch, val):
+ _clean_env(monkeypatch)
+ monkeypatch.setenv("HEADROOM_TLS_STRICT", val)
+ assert tls_strict_disabled() is True
+
+ @pytest.mark.parametrize("val", ["1", "true", "yes", "on", "", "strict", "00"])
+ def test_other_values_keep_strict(self, monkeypatch, val):
+ _clean_env(monkeypatch)
+ monkeypatch.setenv("HEADROOM_TLS_STRICT", val)
+ assert tls_strict_disabled() is False
+
+ def test_unset_keeps_strict(self, monkeypatch):
+ _clean_env(monkeypatch)
+ assert tls_strict_disabled() is False
+
+
+class TestBuildHttpxVerify:
+ def test_default_returns_true(self, monkeypatch):
+ """No CA bundle, strict on → httpx's own default verification."""
+ _clean_env(monkeypatch)
+ assert build_httpx_verify() is True
+
+ def test_toggle_off_returns_relaxed_context(self, monkeypatch):
+ """No CA bundle, strict OFF → default trust store with strict cleared."""
+ _clean_env(monkeypatch)
+ monkeypatch.setenv("HEADROOM_TLS_STRICT", "0")
+ ctx = build_httpx_verify()
+ assert isinstance(ctx, ssl.SSLContext)
+ strict_flag = getattr(ssl, "VERIFY_X509_STRICT", 0)
+ if strict_flag:
+ assert ctx.verify_flags & strict_flag == 0
+ # Still a real verifying context — NOT verify=False.
+ assert ctx.verify_mode == ssl.CERT_REQUIRED
+ # Default trust store retained (additive, not a 1-cert replacement).
+ assert ctx.cert_store_stats()["x509_ca"] > 1
+
+ def test_custom_ca_takes_precedence_over_toggle(self, monkeypatch, ca_pem_file):
+ """A configured CA bundle wins; the result is that bundle's context."""
+ _clean_env(monkeypatch)
+ monkeypatch.setenv("SSL_CERT_FILE", ca_pem_file)
+ monkeypatch.setenv("HEADROOM_TLS_STRICT", "0")
+ ctx = build_httpx_verify()
+ assert isinstance(ctx, ssl.SSLContext)
+ # Replacement bundle → only the single test CA is trusted.
+ assert ctx.cert_store_stats()["x509_ca"] == 1
+
+
+class TestApplyGlobalTlsRelaxation:
+ def test_noop_when_strict_on(self, monkeypatch):
+ _clean_env(monkeypatch)
+ assert apply_global_tls_relaxation() is False
+
+ def test_patches_urllib3_when_toggle_off(self, monkeypatch):
+ _clean_env(monkeypatch)
+ monkeypatch.setenv("HEADROOM_TLS_STRICT", "0")
+ strict_flag = getattr(ssl, "VERIFY_X509_STRICT", 0)
+ if not strict_flag:
+ pytest.skip("VERIFY_X509_STRICT unavailable on this OpenSSL build")
+
+ import urllib3.util.ssl_ as u3ssl
+
+ original = u3ssl.create_urllib3_context
+ try:
+ assert apply_global_tls_relaxation() is True
+ ctx = u3ssl.create_urllib3_context()
+ assert ctx.verify_flags & strict_flag == 0
+ # Idempotent: second call doesn't re-wrap or error.
+ assert apply_global_tls_relaxation() is True
+ assert getattr(u3ssl.create_urllib3_context, "_headroom_strict_relaxed", False)
+ finally:
+ u3ssl.create_urllib3_context = original