Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep the Hub token off redirect targets and clear a blank HF_ENDPOINT (#11000)

AdamPlatin12304c6cf017d3f
diff --git a/studio/backend/routes/training.py b/studio/backend/routes/training.py
index 0f396ce131e..b6c3e805835 100644
--- a/studio/backend/routes/training.py
+++ b/studio/backend/routes/training.py
@@ -2856,12 +2856,14 @@ def _preflight_gated_base(base_model: str, hf_token: Optional[str]) -> None:
):
return
from utils.hf_endpoint import get_hf_endpoint
+ from utils.utils import auth_safe_open
url = f"{get_hf_endpoint()}/{repo}/resolve/main/model_index.json"
headers = {"Authorization": f"Bearer {hf_token}"} if hf_token else {}
req = urllib.request.Request(url, method = "HEAD", headers = headers)
+ # Not urlopen: a mirror's cross-host 302 would carry this token off-origin.
try:
- urllib.request.urlopen(req, timeout = 5)
+ auth_safe_open(req, timeout = 5)
except urllib.error.HTTPError as e:
if e.code in (401, 403):
raise HTTPException(
diff --git a/studio/backend/tests/test_diffusion_training.py b/studio/backend/tests/test_diffusion_training.py
index 5313b08d63b..62a8589c31a 100644
--- a/studio/backend/tests/test_diffusion_training.py
+++ b/studio/backend/tests/test_diffusion_training.py
@@ -774,7 +774,7 @@ def test_route_start_preflights_the_normalized_fetch_mirror(
assert mirror in req.full_url
return object()
- monkeypatch.setattr(urllib.request, "urlopen", _fake_urlopen)
+ monkeypatch.setattr("utils.utils.auth_safe_open", _fake_urlopen)
r = client.post(
"/api/train/diffusion/start",
json = {**_BODY, "base_model": source, "hf_token": hf_token},
@@ -876,7 +876,7 @@ def test_the_start_preflight_never_heads_the_hub_for_a_local_clone(monkeypatch,
def _explode(*a, **k):
pytest.fail("a local clone must never be probed over the network")
- monkeypatch.setattr(urllib.request, "urlopen", _explode)
+ monkeypatch.setattr("utils.utils.auth_safe_open", _explode)
_preflight_gated_base(local, None)
@@ -1883,7 +1883,7 @@ def test_route_start_still_runs_when_the_install_does_have_the_pipeline(
import sys
import urllib.request
- monkeypatch.setattr(urllib.request, "urlopen", lambda req, timeout = None: object())
+ monkeypatch.setattr("utils.utils.auth_safe_open", lambda req, timeout = None: object())
monkeypatch.setitem(sys.modules, "diffusers", _fake_diffusers("0.39.0", "Krea2Pipeline"))
r = client.post("/api/train/diffusion/start", json = {**_BODY, "base_model": "krea/Krea-2-Raw"})
@@ -2253,7 +2253,7 @@ def test_start_gated_base_without_access_is_400_and_keeps_gpu(client, monkeypatc
def _fake_urlopen(req, timeout = None):
raise urllib.error.HTTPError(req.full_url, 403, "Forbidden", {}, None)
- monkeypatch.setattr(urllib.request, "urlopen", _fake_urlopen)
+ monkeypatch.setattr("utils.utils.auth_safe_open", _fake_urlopen)
r = client.post(
"/api/train/diffusion/start",
json = {**_BODY, "base_model": "black-forest-labs/FLUX.1-dev"},
@@ -2302,7 +2302,7 @@ def test_start_ungated_base_preflight_is_noop(client, monkeypatch):
"core.training.diffusion_train_common.training_precision_preflight_error",
lambda fam, prec: None,
)
- monkeypatch.setattr(urllib.request, "urlopen", lambda req, timeout = None: object())
+ monkeypatch.setattr("utils.utils.auth_safe_open", lambda req, timeout = None: object())
r = client.post(
"/api/train/diffusion/start",
json = {**_BODY, "base_model": "black-forest-labs/FLUX.1-dev"},
diff --git a/studio/backend/tests/test_hf_endpoint.py b/studio/backend/tests/test_hf_endpoint.py
index f3f77216a06..0fcbfb73c97 100644
--- a/studio/backend/tests/test_hf_endpoint.py
+++ b/studio/backend/tests/test_hf_endpoint.py
@@ -351,6 +351,25 @@ def test_the_environment_is_normalised_for_huggingface_hub(monkeypatch):
assert "HF_ENDPOINT" not in os.environ
+def test_a_blank_endpoint_is_cleared_rather_than_left_for_the_library(monkeypatch):
+ """huggingface_hub's os.getenv("HF_ENDPOINT", default) falls back only when the key
+ is ABSENT, and its rstrip("/") does not touch whitespace, so " " reached the
+ library verbatim while Studio served the default. datasets/config.py is the same."""
+ for blank in ("", " ", "\t", "\n", " \t\n "):
+ monkeypatch.setenv("HF_ENDPOINT", blank)
+ hf_endpoint.normalize_hf_endpoint_env()
+ assert "HF_ENDPOINT" not in os.environ, repr(blank)
+
+ # Idempotent, and it does not invent the variable when it was never set.
+ hf_endpoint.normalize_hf_endpoint_env()
+ assert "HF_ENDPOINT" not in os.environ
+
+ # Whitespace AROUND a real endpoint is trimmed, not treated as blank.
+ monkeypatch.setenv("HF_ENDPOINT", " https://hf-mirror.com ")
+ hf_endpoint.normalize_hf_endpoint_env()
+ assert os.environ["HF_ENDPOINT"] == "https://hf-mirror.com"
+
+
def test_a_private_endpoint_reaches_only_a_client_on_a_local_network(monkeypatch):
"""A private address means the VISITOR's network when the visitor is
elsewhere, so it is handed out only to a client that is itself local."""
diff --git a/studio/backend/tests/test_offline_inference_parent.py b/studio/backend/tests/test_offline_inference_parent.py
index ccf3e6c20d9..fb687e5850e 100644
--- a/studio/backend/tests/test_offline_inference_parent.py
+++ b/studio/backend/tests/test_offline_inference_parent.py
@@ -129,7 +129,7 @@ class TestTransformersVersionOfflineShortCircuits:
def boom(*a, **k):
raise AssertionError("urlopen must not be called when offline")
- with patch("urllib.request.urlopen", boom):
+ with patch("utils.utils.auth_safe_open", boom):
assert _check_tokenizer_config_needs_v5(unique) is False
def test_config_550_skips_urllib_when_offline(self, monkeypatch, clean_offline_env, tmp_path):
@@ -139,7 +139,7 @@ class TestTransformersVersionOfflineShortCircuits:
def boom(*a, **k):
raise AssertionError("urlopen must not be called when offline")
- with patch("urllib.request.urlopen", boom):
+ with patch("utils.utils.auth_safe_open", boom):
assert _check_config_needs_550(unique) is False
diff --git a/studio/backend/tests/test_transformers_version.py b/studio/backend/tests/test_transformers_version.py
index 5237bf2cf1f..0038e9da84f 100644
--- a/studio/backend/tests/test_transformers_version.py
+++ b/studio/backend/tests/test_transformers_version.py
@@ -249,7 +249,7 @@ class TestRemoteLoraBase:
def test_fetches_base_from_remote_adapter_config(self, monkeypatch):
monkeypatch.delenv("HF_HUB_OFFLINE", raising = False)
cfg = {"base_model_name_or_path": "nvidia/NVIDIA-Nemotron-3-Nano-4B"}
- with patch("urllib.request.urlopen", return_value = self._resp(cfg)):
+ with patch("utils.utils.auth_safe_open", return_value = self._resp(cfg)):
assert (
_remote_lora_base("someuser/my-nemotron-lora") == "nvidia/NVIDIA-Nemotron-3-Nano-4B"
)
@@ -268,7 +268,7 @@ class TestRemoteLoraBase:
seen["url"] = req.full_url
return self._resp({"base_model_name_or_path": "org/base"})
- with patch("urllib.request.urlopen", side_effect = fake_urlopen):
+ with patch("utils.utils.auth_safe_open", side_effect = fake_urlopen):
assert _remote_lora_base("user/adapter") == "org/base"
assert seen["url"].startswith("https://hf.mirror.internal/user/adapter/resolve/main/")
@@ -290,7 +290,7 @@ class TestRemoteLoraBase:
self._seed_adapter_cache(tmp_path, "user/cached-lora", "nvidia/Nemotron-H-8B")
monkeypatch.setenv("HF_HUB_CACHE", str(tmp_path))
monkeypatch.setenv("HF_HUB_OFFLINE", "1")
- with patch("urllib.request.urlopen") as mock_url:
+ with patch("utils.utils.auth_safe_open") as mock_url:
assert _remote_lora_base("user/cached-lora") == "nvidia/Nemotron-H-8B"
mock_url.assert_not_called() # offline: cache only, no network
@@ -298,20 +298,20 @@ class TestRemoteLoraBase:
self._seed_adapter_cache(tmp_path, "user/cached-lora", "nvidia/Nemotron-H-8B")
monkeypatch.setenv("HF_HUB_CACHE", str(tmp_path))
monkeypatch.delenv("HF_HUB_OFFLINE", raising = False)
- with patch("urllib.request.urlopen", side_effect = OSError("boom")):
+ with patch("utils.utils.auth_safe_open", side_effect = OSError("boom")):
assert _remote_lora_base("user/cached-lora") == "nvidia/Nemotron-H-8B"
def test_offline_uncached_makes_no_request(self, tmp_path: Path, monkeypatch):
monkeypatch.setenv("HF_HUB_CACHE", str(tmp_path))
monkeypatch.setenv("HF_HUB_OFFLINE", "1")
- with patch("urllib.request.urlopen") as mock_url:
+ with patch("utils.utils.auth_safe_open") as mock_url:
assert _remote_lora_base("org/adapter") is None
mock_url.assert_not_called()
def test_non_adapter_repo_returns_none(self, tmp_path: Path, monkeypatch):
monkeypatch.setenv("HF_HUB_CACHE", str(tmp_path))
monkeypatch.delenv("HF_HUB_OFFLINE", raising = False)
- with patch("urllib.request.urlopen", side_effect = OSError("boom")):
+ with patch("utils.utils.auth_safe_open", side_effect = OSError("boom")):
assert _remote_lora_base("org/not-an-adapter") is None
def test_existing_relative_path_not_treated_as_repo(self, monkeypatch):
@@ -319,7 +319,7 @@ class TestRemoteLoraBase:
# a Hub repo: no request, no risk of matching an unrelated remote/cached adapter.
import utils.paths as paths
monkeypatch.setattr(paths, "is_local_path", lambda p: True)
- with patch("urllib.request.urlopen") as mock_url:
+ with patch("utils.utils.auth_safe_open") as mock_url:
assert _remote_lora_base("outputs/run1") is None
mock_url.assert_not_called()
@@ -332,7 +332,7 @@ class TestRemoteLoraBase:
monkeypatch.setenv("HF_HUB_CACHE", str(tmp_path))
monkeypatch.delenv("HF_HUB_OFFLINE", raising = False)
err = urllib.error.HTTPError("url", 404, "Not Found", {}, None)
- with patch("urllib.request.urlopen", side_effect = err):
+ with patch("utils.utils.auth_safe_open", side_effect = err):
assert _remote_lora_base("user/was-a-lora") is None
def test_transient_http_error_falls_back_to_cache(self, tmp_path: Path, monkeypatch):
@@ -342,7 +342,7 @@ class TestRemoteLoraBase:
monkeypatch.setenv("HF_HUB_CACHE", str(tmp_path))
monkeypatch.delenv("HF_HUB_OFFLINE", raising = False)
err = urllib.error.HTTPError("url", 503, "Service Unavailable", {}, None)
- with patch("urllib.request.urlopen", side_effect = err):
+ with patch("utils.utils.auth_safe_open", side_effect = err):
assert _remote_lora_base("user/cached-lora") == "nvidia/Nemotron-H-8B"
@@ -378,7 +378,7 @@ class TestCheckTokenizerConfigNeedsV5:
tc = {"tokenizer_class": "LlamaTokenizerFast"}
(tmp_path / "tokenizer_config.json").write_text(json.dumps(tc))
- with patch("urllib.request.urlopen") as mock_urlopen:
+ with patch("utils.utils.auth_safe_open") as mock_urlopen:
result = _check_tokenizer_config_needs_v5(str(tmp_path))
mock_urlopen.assert_not_called()
assert result is False
@@ -421,7 +421,7 @@ class TestCheckTokenizerConfigNeedsV5:
return _Resp(json.dumps({"tokenizer_class": "TokenizersBackend"}))
raise OSError("HTTP 401")
- monkeypatch.setattr("urllib.request.urlopen", fake_urlopen)
+ monkeypatch.setattr("utils.utils.auth_safe_open", fake_urlopen)
assert _check_tokenizer_config_needs_v5("org/gated") is False # unauth miss
assert _check_tokenizer_config_needs_v5("org/gated", "tok") is True # authed hit
assert seen_auth == [None, "Bearer tok"]
@@ -448,7 +448,7 @@ class TestCheckTokenizerConfigNeedsV5:
seen["url"] = req.full_url
return _Resp()
- monkeypatch.setattr("urllib.request.urlopen", fake_urlopen)
+ monkeypatch.setattr("utils.utils.auth_safe_open", fake_urlopen)
assert _check_tokenizer_config_needs_v5("org/model") is True
assert seen["url"] == (
"https://hf.mirror.internal/org/model/resolve/main/tokenizer_config.json"
@@ -580,7 +580,7 @@ class TestCheckConfigNeeds550:
def test_no_config_json(self, tmp_path: Path):
"""Missing config.json should return False (fail-open)."""
# Patch network call to avoid a real fetch.
- with patch("urllib.request.urlopen") as mock_urlopen:
+ with patch("utils.utils.auth_safe_open") as mock_urlopen:
mock_urlopen.side_effect = Exception("no network")
assert _check_config_needs_550(str(tmp_path)) is False
@@ -599,7 +599,7 @@ class TestCheckConfigNeeds550:
cfg = {"architectures": ["LlamaForCausalLM"]}
(tmp_path / "config.json").write_text(json.dumps(cfg))
- with patch("urllib.request.urlopen") as mock_urlopen:
+ with patch("utils.utils.auth_safe_open") as mock_urlopen:
_check_config_needs_550(str(tmp_path))
mock_urlopen.assert_not_called()
@@ -667,7 +667,7 @@ class TestCheckConfigNeeds510:
def test_no_config_json(self, tmp_path: Path):
"""Missing config.json should return False (fail-open)."""
# Patch network call to avoid real fetch
- with patch("urllib.request.urlopen") as mock_urlopen:
+ with patch("utils.utils.auth_safe_open") as mock_urlopen:
mock_urlopen.side_effect = Exception("no network")
assert _check_config_needs_510(str(tmp_path)) is False
@@ -686,7 +686,7 @@ class TestCheckConfigNeeds510:
cfg = {"architectures": ["LlamaForCausalLM"]}
(tmp_path / "config.json").write_text(json.dumps(cfg))
- with patch("urllib.request.urlopen") as mock_urlopen:
+ with patch("utils.utils.auth_safe_open") as mock_urlopen:
_check_config_needs_510(str(tmp_path))
mock_urlopen.assert_not_called()
@@ -816,7 +816,7 @@ class TestConfigJsonHfCacheFallback:
self._seed_cache(tmp_path, "unsloth/NVIDIA-Nemotron-3-Nano-4B", cfg)
monkeypatch.setenv("HF_HUB_CACHE", str(tmp_path))
monkeypatch.setenv("HF_HUB_OFFLINE", "1")
- with patch("urllib.request.urlopen") as mock_url:
+ with patch("utils.utils.auth_safe_open") as mock_url:
assert _load_config_json("unsloth/NVIDIA-Nemotron-3-Nano-4B") == cfg
mock_url.assert_not_called()
@@ -827,7 +827,7 @@ class TestConfigJsonHfCacheFallback:
monkeypatch.setenv("HF_HUB_CACHE", str(tmp_path))
monkeypatch.delenv("HF_HUB_OFFLINE", raising = False)
monkeypatch.delenv("TRANSFORMERS_OFFLINE", raising = False)
- with patch("urllib.request.urlopen", return_value = _hf_response(fresh)):
+ with patch("utils.utils.auth_safe_open", return_value = _hf_response(fresh)):
assert _load_config_json("org/model") == fresh # network wins, not stale cache
def test_remote_fetch_respects_hf_endpoint(self, monkeypatch):
@@ -840,7 +840,7 @@ class TestConfigJsonHfCacheFallback:
seen["url"] = req.full_url
return _hf_response({"model_type": "llama"})
- with patch("urllib.request.urlopen", side_effect = fake_urlopen):
+ with patch("utils.utils.auth_safe_open", side_effect = fake_urlopen):
assert _load_config_json("org/model") == {"model_type": "llama"}
assert seen["url"] == "https://hf.mirror.internal/org/model/resolve/main/config.json"
@@ -849,13 +849,13 @@ class TestConfigJsonHfCacheFallback:
self._seed_cache(tmp_path, "org/model", cfg)
monkeypatch.setenv("HF_HUB_CACHE", str(tmp_path))
monkeypatch.delenv("HF_HUB_OFFLINE", raising = False)
- with patch("urllib.request.urlopen", side_effect = OSError("boom")):
+ with patch("utils.utils.auth_safe_open", side_effect = OSError("boom")):
assert _load_config_json("org/model") == cfg
def test_offline_uncached_returns_none(self, tmp_path: Path, monkeypatch):
monkeypatch.setenv("HF_HUB_CACHE", str(tmp_path))
monkeypatch.setenv("HF_HUB_OFFLINE", "1")
- with patch("urllib.request.urlopen") as mock_url:
+ with patch("utils.utils.auth_safe_open") as mock_url:
assert _load_config_json("private/unknown") is None
mock_url.assert_not_called()
@@ -886,10 +886,10 @@ class TestConfigJsonHfCacheFallback:
monkeypatch.setenv("HF_HUB_CACHE", str(tmp_path))
monkeypatch.delenv("HF_HUB_OFFLINE", raising = False)
# Network fails -> serve the cached snapshot, but it must not be memoized.
- with patch("urllib.request.urlopen", side_effect = OSError("boom")):
+ with patch("utils.utils.auth_safe_open", side_effect = OSError("boom")):
assert _load_config_json("org/model") == stale
# Connectivity returns: the next call must hit the network for the fresh config.
- with patch("urllib.request.urlopen", return_value = _hf_response(fresh)):
+ with patch("utils.utils.auth_safe_open", return_value = _hf_response(fresh)):
assert _load_config_json("org/model") == fresh
def test_auth_failure_does_not_serve_cache(self, tmp_path: Path, monkeypatch):
@@ -904,7 +904,7 @@ class TestConfigJsonHfCacheFallback:
for code in (401, 403, 404):
_config_json_cache.clear()
err = urllib.error.HTTPError("url", code, "denied", {}, None)
- with patch("urllib.request.urlopen", side_effect = err):
+ with patch("utils.utils.auth_safe_open", side_effect = err):
assert _load_config_json("private/model") is None
def test_server_error_still_falls_back_to_cache(self, tmp_path: Path, monkeypatch):
@@ -916,7 +916,7 @@ class TestConfigJsonHfCacheFallback:
monkeypatch.delenv("HF_HUB_OFFLINE", raising = False)
# A 5xx is transient, not an access decision: keep serving the cache.
err = urllib.error.HTTPError("url", 503, "busy", {}, None)
- with patch("urllib.request.urlopen", side_effect = err):
+ with patch("utils.utils.auth_safe_open", side_effect = err):
assert _load_config_json("org/model") == cfg
@@ -949,11 +949,11 @@ class TestTierCheckTransientRetry:
monkeypatch.setenv("HF_HUB_CACHE", str(tmp_path))
monkeypatch.delenv("HF_HUB_OFFLINE", raising = False)
# Network blip -> serve the cache, but do NOT pin the tier result.
- with patch("urllib.request.urlopen", side_effect = OSError("boom")):
+ with patch("utils.utils.auth_safe_open", side_effect = OSError("boom")):
assert _check_config_needs_510("org/model") is False
assert ("org/model", None) not in _config_needs_510_cache
# Connectivity returns: the next call re-fetches and sees the higher tier.
- with patch("urllib.request.urlopen", return_value = _hf_response(fresh)):
+ with patch("utils.utils.auth_safe_open", return_value = _hf_response(fresh)):
assert _check_config_needs_510("org/model") is True
assert _config_needs_510_cache[("org/model", None)] is True # definitive read memoized
@@ -961,7 +961,7 @@ class TestTierCheckTransientRetry:
fresh = {"architectures": ["Gemma4ForConditionalGeneration"]} # needs 550
monkeypatch.setenv("HF_HUB_CACHE", str(tmp_path))
monkeypatch.delenv("HF_HUB_OFFLINE", raising = False)
- with patch("urllib.request.urlopen", return_value = _hf_response(fresh)) as mock_url:
+ with patch("utils.utils.auth_safe_open", return_value = _hf_response(fresh)) as mock_url:
assert _check_config_needs_550("org/model") is True
assert _check_config_needs_550("org/model") is True
assert mock_url.call_count == 1 # second call served from the tier cache
@@ -1061,7 +1061,7 @@ class TestGetTransformersTier:
json.dumps({"tokenizer_class": "TokenizersBackend"})
)
- with patch("urllib.request.urlopen") as mock_urlopen:
+ with patch("utils.utils.auth_safe_open") as mock_urlopen:
assert get_transformers_tier(str(tmp_path)) == "510"
mock_urlopen.assert_not_called()
@@ -1083,7 +1083,7 @@ class TestGetTransformersTier:
}
).encode()
- with patch("urllib.request.urlopen", return_value = _Response()):
+ with patch("utils.utils.auth_safe_open", return_value = _Response()):
assert get_transformers_tier("unsloth/NVIDIA-Nemotron-3-Nano-4B") == "510"
def test_local_config_json_short_circuits_path_substrings(self, tmp_path: Path):
@@ -1102,7 +1102,7 @@ class TestGetTransformersTier:
json.dumps({"tokenizer_class": "LlamaTokenizerFast"})
)
- with patch("urllib.request.urlopen") as mock_urlopen:
+ with patch("utils.utils.auth_safe_open") as mock_urlopen:
assert get_transformers_tier(str(model_dir)) == "default"
mock_urlopen.assert_not_called()
@@ -1124,7 +1124,7 @@ class TestGetTransformersTier:
}
).encode()
- with patch("urllib.request.urlopen", return_value = _Response()) as mock_urlopen:
+ with patch("utils.utils.auth_safe_open", return_value = _Response()) as mock_urlopen:
assert get_transformers_tier("org/no-fast-substring-model") == "550"
assert mock_urlopen.call_count == 1
@@ -1705,7 +1705,7 @@ class TestLocalCheckpointFilesAppear:
def boom(*a, **k):
raise AssertionError("a local checkpoint must not be fetched from the Hub")
- monkeypatch.setattr("urllib.request.urlopen", boom)
+ monkeypatch.setattr("utils.utils.auth_safe_open", boom)
# Before the file exists: not 5.x, no network, and the miss must not be pinned.
assert _check_tokenizer_config_needs_v5(local) is False
# The file appears with a 5.x-only tokenizer -> the next call must read it.
@@ -1720,7 +1720,7 @@ class TestLocalCheckpointFilesAppear:
def boom(*a, **k):
raise AssertionError("a local checkpoint must not be fetched from the Hub")
- monkeypatch.setattr("urllib.request.urlopen", boom)
+ monkeypatch.setattr("utils.utils.auth_safe_open", boom)
assert _load_config_json(local) is None
(tmp_path / "config.json").write_text(json.dumps({"model_type": "gemma4"}))
assert _load_config_json(local) == {"model_type": "gemma4"}
@@ -3033,7 +3033,7 @@ class TestOfflineCacheNotPoisoned:
def __exit__(self, *a):
return False
- monkeypatch.setattr("urllib.request.urlopen", lambda req, timeout = 10: _Resp())
+ monkeypatch.setattr("utils.utils.auth_safe_open", lambda req, timeout = 10: _Resp())
assert _check_tokenizer_config_needs_v5("org/needs5") is True
def test_offline_config_miss_not_cached(self, monkeypatch):
diff --git a/studio/backend/tests/test_utils.py b/studio/backend/tests/test_utils.py
index 328d52c1a98..8d4c8bee619 100644
--- a/studio/backend/tests/test_utils.py
+++ b/studio/backend/tests/test_utils.py
@@ -709,3 +709,184 @@ class TestFormatErrorMessage:
err = Exception("Something completely unexpected")
msg = format_error_message(err, "any/model")
assert msg == "Something completely unexpected"
+
+
+class TestAuthSafeRedirectHandler:
+ """A Hub token must not leave the origin the operator configured.
+
+ Origin cases run over loopback sockets; scheme cases go through redirect_request
+ directly, since a loopback TLS server would need a cert this suite does not carry.
+ """
+
+ TOKEN = "Bearer hf_FAKE_TOKEN_FOR_TESTS"
+
+ @staticmethod
+ def _serve(plan):
+ """A throwaway loopback server that records the Authorization it was sent."""
+ import http.server
+ import threading
+
+ class _Recorder(http.server.BaseHTTPRequestHandler):
+ protocol_version = "HTTP/1.0"
+ # Defaults to None: a connection that sends nothing wedges serve_forever in
+ # readline(), and shutdown() waits on that loop with no timeout of its own.
+ timeout = 5
+
+ def _handle(self):
+ self.server.seen.append(
+ {"path": self.path, "auth": self.headers.get("Authorization")}
+ )
+ code, location = self.server.plan(self.path)
+ self.send_response(code)
+ if location:
+ self.send_header("Location", location)
+ self.send_header("Content-Length", "0")
+ self.end_headers()
+
+ # do_GET is NOT dead: 3.13 preserves HEAD across a redirect, 3.12 downgrades
+ # it to GET, so HEAD-only answers 501 on every 3.12 runner.
+ do_GET = _handle
+ do_HEAD = _handle
+
+ def log_message(self, *args):
+ pass
+
+ class _Server(http.server.HTTPServer):
+ def server_bind(self):
+ # HTTPServer.server_bind calls socket.getfqdn(), which conftest's network
+ # guard does not patch: a real PTR query on Windows, and it can stall.
+ import socketserver
+
+ socketserver.TCPServer.server_bind(self)
+ self.server_name = "127.0.0.1"
+ self.server_port = self.server_address[1]
+
+ srv = _Server(("127.0.0.1", 0), _Recorder)
+ srv.seen = []
+ srv.plan = plan
+ threading.Thread(target = srv.serve_forever, daemon = True).start()
+ return srv
+
+ @staticmethod
+ def _stop(*servers):
+ """Stop the loop AND close the listening socket, which shutdown() does not."""
+ for srv in servers:
+ srv.shutdown()
+ srv.server_close()
+
+ def _get(self, url):
+ import urllib.request
+ from utils.utils import auth_safe_open
+
+ req = urllib.request.Request(url, method = "HEAD", headers = {"Authorization": self.TOKEN})
+ auth_safe_open(req, timeout = 5).close()
+
+ def test_same_origin_redirect_keeps_the_token(self):
+ srv = self._serve(lambda p: (302, "/final") if p == "/start" else (200, None))
+ try:
+ self._get(f"http://127.0.0.1:{srv.server_port}/start")
+ finally:
+ self._stop(srv)
+ hop2 = [r for r in srv.seen if r["path"] == "/final"]
+ assert hop2 and hop2[0]["auth"] == self.TOKEN
+
+ def test_cross_origin_redirect_drops_the_token(self):
+ """Another port on the same host is another origin, and gets no token."""
+ dest = self._serve(lambda p: (200, None))
+ src = self._serve(lambda p: (302, f"http://127.0.0.1:{dest.server_port}/final"))
+ try:
+ self._get(f"http://127.0.0.1:{src.server_port}/start")
+ finally:
+ self._stop(src, dest)
+ assert src.seen and src.seen[0]["auth"] == self.TOKEN
+ assert dest.seen and dest.seen[0]["auth"] is None
+
+ def test_token_does_not_come_back_on_the_return_hop(self):
+ ports = {}
+ first = self._serve(
+ lambda p: (302, f"http://127.0.0.1:{ports['b']}/via") if p == "/start" else (200, None)
+ )
+ second = self._serve(lambda p: (302, f"http://127.0.0.1:{first.server_port}/back"))
+ ports["b"] = second.server_port
+ try:
+ self._get(f"http://127.0.0.1:{first.server_port}/start")
+ finally:
+ self._stop(first, second)
+ back = [r for r in first.seen if r["path"] == "/back"]
+ assert second.seen and second.seen[0]["auth"] is None
+ assert back and back[0]["auth"] is None
+
+ # --- scheme and host rules, at the handler ---
+
+ def _redirect(
+ self,
+ start,
+ newurl,
+ code = 302,
+ ):
+ import urllib.request
+ from utils.utils import AuthSafeRedirectHandler
+
+ req = urllib.request.Request(start, method = "HEAD", headers = {"Authorization": self.TOKEN})
+ return AuthSafeRedirectHandler().redirect_request(req, None, code, "Found", {}, newurl)
+
+ def test_tls_downgrade_is_not_followed(self):
+ assert self._redirect("https://hub.example/a", "http://hub.example/a") is None
+
+ def test_scheme_change_alone_drops_the_token(self):
+ """Explicit port on both sides, so this cannot pass on http/https's port gap."""
+ new = self._redirect("http://hub.example:8443/a", "https://hub.example:8443/a")
+ assert new is not None
+ assert new.headers.get("Authorization") is None
+
+ def test_explicit_default_port_is_the_same_origin(self):
+ new = self._redirect("https://hub.example/a", "https://hub.example:443/b")
+ assert new is not None
+ assert new.headers.get("Authorization") == self.TOKEN
+
+ def test_lookalike_host_drops_the_token(self):
+ new = self._redirect("https://hub.example/a", "https://hub.example.evil.test/a")
+ assert new is not None
+ assert new.headers.get("Authorization") is None
+
+ def test_the_strip_lands_on_the_redirected_request_not_the_callers(self):
+ """So a caller that reuses its Request still has a token to send."""
+ import urllib.request
+ from utils.utils import AuthSafeRedirectHandler
+
+ req = urllib.request.Request(
+ "https://hub.example/a", method = "HEAD", headers = {"Authorization": self.TOKEN}
+ )
+ new = AuthSafeRedirectHandler().redirect_request(
+ req, None, 302, "Found", {}, "https://other.example/b"
+ )
+ assert new.headers.get("Authorization") is None
+ assert req.headers.get("Authorization") == self.TOKEN
+
+ def test_a_refused_redirect_reaches_the_caller_as_an_http_error(self):
+ """Stubs the refusal rather than driving it, since that needs an https origin.
+
+ Settles the caller contract only: returning None raises HTTPError on the 3xx.
+ """
+ import urllib.error
+ import urllib.request
+
+ class _Refuse(urllib.request.HTTPRedirectHandler):
+ def redirect_request(self, *a, **k):
+ return None
+
+ dest = self._serve(lambda p: (200, None))
+ src = self._serve(lambda p: (302, f"http://127.0.0.1:{dest.server_port}/final"))
+ opener = urllib.request.build_opener(_Refuse())
+ try:
+ with pytest.raises(urllib.error.HTTPError) as excinfo:
+ opener.open(
+ urllib.request.Request(
+ f"http://127.0.0.1:{src.server_port}/start", method = "HEAD"
+ ),
+ timeout = 5,
+ )
+ finally:
+ self._stop(src, dest)
+ assert excinfo.value.code == 302
+ assert dest.seen == []
diff --git a/studio/backend/utils/hf_endpoint.py b/studio/backend/utils/hf_endpoint.py
index 6608dba213a..6272c8d0146 100644
--- a/studio/backend/utils/hf_endpoint.py
+++ b/studio/backend/utils/hf_endpoint.py
@@ -237,7 +237,13 @@ def normalize_hf_endpoint_env() -> None:
Rewriting the variable before huggingface_hub is imported gives the whole
process, and the subprocesses that inherit this environment, one endpoint.
"""
- if not (os.environ.get("HF_ENDPOINT") or "").strip():
+ raw = os.environ.get("HF_ENDPOINT")
+ if raw is None:
+ return
+ if not raw.strip():
+ # Blank is not an endpoint, but the library would read it verbatim;
+ # clear it so Studio and huggingface_hub agree on the default.
+ os.environ.pop("HF_ENDPOINT", None)
return
endpoint = get_hf_endpoint()
if endpoint == _DEFAULT_HF_ENDPOINT:
diff --git a/studio/backend/utils/transformers_version.py b/studio/backend/utils/transformers_version.py
index 23e8125ff88..8a0c2ddd189 100644
--- a/studio/backend/utils/transformers_version.py
+++ b/studio/backend/utils/transformers_version.py
@@ -117,31 +117,40 @@ def _hf_proxy_opener(url: str):
import urllib.request
try:
- from utils.utils import hf_proxy_for_endpoint, hf_proxy_usable_by_urllib
+ from utils.utils import (
+ AuthSafeRedirectHandler,
+ hf_proxy_for_endpoint,
+ hf_proxy_usable_by_urllib,
+ )
proxy = hf_proxy_for_endpoint(url)
scheme = urllib.parse.urlparse(url).scheme or "https"
if proxy:
if not hf_proxy_usable_by_urllib(proxy):
return None
- return urllib.request.build_opener(urllib.request.ProxyHandler({scheme: proxy}))
+ return urllib.request.build_opener(
+ urllib.request.ProxyHandler({scheme: proxy}), AuthSafeRedirectHandler()
+ )
if any(urllib.request.getproxies().get(key) for key in (scheme, "all")):
# The Hub client bypasses the proxy for this host; force a direct opener so
# urllib's coarser NO_PROXY parsing cannot send the request through it anyway.
- return urllib.request.build_opener(urllib.request.ProxyHandler({}))
+ return urllib.request.build_opener(
+ urllib.request.ProxyHandler({}), AuthSafeRedirectHandler()
+ )
except Exception:
pass
return None
def _hf_urlopen(req, timeout: int):
- """``urlopen`` through the same proxy huggingface_hub would use for this request."""
- import urllib.request
+ """``urlopen`` through the same proxy huggingface_hub would use for this request,
+ with redirects that cannot carry the Authorization header off-origin."""
+ from utils.utils import auth_safe_open
opener = _hf_proxy_opener(req.full_url)
if opener is not None:
return opener.open(req, timeout = timeout)
- return urllib.request.urlopen(req, timeout = timeout)
+ return auth_safe_open(req, timeout = timeout)
def hf_endpoint_unreachable(
diff --git a/studio/backend/utils/utils.py b/studio/backend/utils/utils.py
index 2bda305cb02..2f20a8758c4 100644
--- a/studio/backend/utils/utils.py
+++ b/studio/backend/utils/utils.py
@@ -5,6 +5,8 @@
import os
import structlog
+import urllib.parse
+import urllib.request
import threading
from contextvars import ContextVar
import time
@@ -117,6 +119,40 @@ def hf_proxy_configured() -> bool:
return hf_proxy_for_endpoint() is not None
+class AuthSafeRedirectHandler(urllib.request.HTTPRedirectHandler):
+ """Redirect policy for a urllib request carrying a Hub token.
+
+ urllib's default handler copies request headers onto the redirect target, so a
+ mirror's cross-host 302 hands the token to a host the operator never configured.
+ Pass this to ``build_opener``, which then leaves out the default. A refusal
+ surfaces as ``HTTPError`` on the 3xx, which every probe here reads as reachable.
+ """
+
+ @staticmethod
+ def _origin(url):
+ parts = urllib.parse.urlsplit(url)
+ scheme = parts.scheme.lower()
+ port = parts.port or (443 if scheme == "https" else 80)
+ return scheme, (parts.hostname or "").lower(), port
+
+ def redirect_request(self, req, fp, code, msg, headers, newurl):
+ old = self._origin(req.full_url)
+ new = self._origin(newurl)
+ if old[0] == "https" and new[0] == "http":
+ return None # no TLS downgrade, whatever the target is
+ result = super().redirect_request(req, fp, code, msg, headers, newurl)
+ if result is not None and new != old:
+ # The redirected request, not the caller's: urllib copied req.headers into a
+ # fresh dict, so the original stays reusable.
+ result.headers.pop("Authorization", None)
+ return result
+
+
+def auth_safe_open(req, timeout):
+ """The single seam: tests patch THIS, never ``urllib.request.urlopen``."""
+ return urllib.request.build_opener(AuthSafeRedirectHandler()).open(req, timeout = timeout)
+
+
def call_with_deadline(
fn,
timeout_s: float,