π« Toolkit to help you get started with SDD or any other process!
harden: bound HTTP reads and enforce strict redirects (#3140)
Pascal THUET5601830ba38b
diff --git a/src/specify_cli/_download_security.py b/src/specify_cli/_download_security.pynew file mode 100644index 000000000..0d39baca0--- /dev/null+++ b/src/specify_cli/_download_security.py@@ -0,0 +1,89 @@+"""Helpers for bounded HTTP downloads."""++from __future__ import annotations++from typing import NoReturn, TypeVar+from urllib.parse import urlparse+++ErrorT = TypeVar("ErrorT", bound=Exception)++MAX_DOWNLOAD_BYTES = 50 * 1024 * 1024+READ_CHUNK_SIZE = 1024 * 1024++# Tighter ceiling for responses that are read fully into memory and parsed as+# JSON. The 50 MiB MAX_DOWNLOAD_BYTES default is sized for archive/payload+# downloads; JSON metadata responses are far smaller, so capping them close to+# their real size shrinks the memory-DoS surface and keeps the "too large"+# error reachable (rather than only triggering on tens of MiB). Pass it+# explicitly at each JSON call site so the intended bound is pinned there.+# METADATA covers fixed-shape single-object responses (an OAuth token, one+# release's metadata): a few KiB in practice, 1 MiB is already generous.+MAX_JSON_METADATA_BYTES = 1 * 1024 * 1024+_LOOPBACK_HOSTS = frozenset(("localhost", "127.0.0.1", "::1"))+++def is_loopback_url(url: str) -> bool:+ """Return whether *url* targets an explicitly allowed loopback host."""+ return urlparse(url).hostname in _LOOPBACK_HOSTS+++def is_https_or_localhost_http(url: str) -> bool:+ """Return True if *url* is HTTPS, or HTTP limited to loopback hosts.++ Shared scheme-safety predicate used by the auth HTTP redirect handler and+ by the direct URL validations in the CLI download flows, so the rule (and+ any future tightening of it) lives in one place.++ A hostname is always required: a URL without one (e.g. ``https:///x``)+ has no real target and is rejected regardless of scheme.++ The loopback allowance is a deliberate *exact-string* match on+ ``localhost`` / ``127.0.0.1`` / ``::1``, not an IP-range check: other+ loopback addresses (e.g. ``127.0.0.2``) are intentionally not covered.+ ``urlparse`` already lower-cases the hostname, so the comparison is+ case-insensitive.+ """+ parsed = urlparse(url)+ if not parsed.hostname:+ return False+ is_localhost = parsed.hostname in _LOOPBACK_HOSTS+ return parsed.scheme == "https" or (parsed.scheme == "http" and is_localhost)+++def _raise(error_type: type[ErrorT], message: str) -> NoReturn:+ raise error_type(message)+++def read_response_limited(+ response,+ *,+ max_bytes: int = MAX_DOWNLOAD_BYTES,+ error_type: type[ErrorT] = ValueError,+ label: str = "download",+) -> bytes:+ """Read at most *max_bytes* from a response object.++ ``response.read(n)`` is only guaranteed to return *up to* ``n`` bytes and may+ return fewer even when more data is pending (e.g. chunked transfer encoding),+ so a single ``read(max_bytes + 1)`` cannot enforce the bound on its own. Read+ in a loop until EOF or until one byte past the limit has been accumulated.++ *max_bytes* is keyword-only. It defaults to the module-wide+ ``MAX_DOWNLOAD_BYTES`` (50 MiB) ceiling for archive/payload downloads;+ callers with a tighter budget (e.g. small JSON responses) should pass an+ explicit value so the intended bound is pinned at the call site rather than+ tracking changes to the shared default.+ """+ chunks: list[bytes] = []+ total = 0+ limit = max_bytes + 1+ while total < limit:+ chunk = response.read(min(READ_CHUNK_SIZE, limit - total))+ if not chunk:+ break+ chunks.append(chunk)+ total += len(chunk)+ if total > max_bytes:+ _raise(error_type, f"{label} exceeds maximum size of {max_bytes} bytes")+ return b"".join(chunks)diff --git a/src/specify_cli/_github_http.py b/src/specify_cli/_github_http.pyindex aa8223bca..1795ad0ff 100644--- a/src/specify_cli/_github_http.py+++ b/src/specify_cli/_github_http.py@@ -100,6 +100,8 @@ def resolve_github_release_asset_api_url(import jsonimport urllib.error+ from specify_cli._download_security import read_response_limited+parsed = urlparse(download_url)hostname = (parsed.hostname or "").lower()parts = [unquote(part) for part in parsed.path.strip("/").split("/")]@@ -158,10 +160,13 @@ def resolve_github_release_asset_api_url(if redirect_validator is not None:open_kwargs["redirect_validator"] = redirect_validatorwith open_url_fn(release_url, **open_kwargs) as response:- raw_release_data = response.read(max_metadata_bytes + 1)- if len(raw_release_data) > max_metadata_bytes:- raise ValueError("GitHub release metadata exceeds size limit")- release_data = json.loads(raw_release_data)+ release_data = json.loads(+ read_response_limited(+ response,+ max_bytes=max_metadata_bytes,+ label=f"GitHub release metadata {release_url}",+ )+ )except (urllib.error.URLError,json.JSONDecodeError,diff --git a/src/specify_cli/_version.py b/src/specify_cli/_version.pyindex e634a4f28..33dd0983e 100644--- a/src/specify_cli/_version.py+++ b/src/specify_cli/_version.py@@ -4,8 +4,8 @@ Pure helpers for comparing PEP 440 versions and fetching the latest GitHubrelease tag. The ``self_app`` Typer sub-command group is co-located here soall version-related logic lives in one place.-Dependencies: stdlib + packaging + ._console only (no other internal imports-at module level, keeping this layer thin and circular-import-safe).+Dependencies: stdlib + packaging + ._console + ._download_security only+(keeping this layer thin and circular-import-safe)."""from __future__ import annotations@@ -28,6 +28,7 @@ from pathlib import Pathimport typerfrom packaging.version import InvalidVersion, Version+from ._download_security import MAX_JSON_METADATA_BYTES, read_response_limitedfrom ._console import consoleGITHUB_API_LATEST = "https://api.github.com/repos/github/spec-kit/releases/latest"@@ -119,7 +120,13 @@ def _fetch_latest_release_tag() -> tuple[str | None, str | None]:timeout=5,extra_headers={"Accept": "application/vnd.github+json"},) as resp:- payload = json.loads(resp.read().decode("utf-8"))+ payload = json.loads(+ read_response_limited(+ resp,+ max_bytes=MAX_JSON_METADATA_BYTES,+ label="GitHub latest release",+ ).decode("utf-8")+ )tag = payload.get("tag_name")if not isinstance(tag, str) or not tag:raise ValueError("GitHub API response missing valid tag_name")diff --git a/src/specify_cli/authentication/azure_devops.py b/src/specify_cli/authentication/azure_devops.pyindex 0ccdd5afb..c475fa6fd 100644--- a/src/specify_cli/authentication/azure_devops.py+++ b/src/specify_cli/authentication/azure_devops.py@@ -8,6 +8,7 @@ import osimport subprocessfrom typing import TYPE_CHECKING+from .._download_security import MAX_JSON_METADATA_BYTES, read_response_limitedfrom .base import AuthProviderif TYPE_CHECKING:@@ -17,6 +18,10 @@ if TYPE_CHECKING:_ADO_RESOURCE_ID = "499b84ac-1321-427f-aa17-267ca6975798"+class _TokenResponseTooLarge(Exception):+ """Raised when an Azure AD token response exceeds the bounded read limit."""++class AzureDevOpsAuth(AuthProvider):"""Azure DevOps authentication provider.@@ -119,9 +124,38 @@ class AzureDevOpsAuth(AuthProvider):headers={"Content-Type": "application/x-www-form-urlencoded"},)try:- with urllib.request.urlopen(req, timeout=30) as resp: # noqa: S310- payload = _json.loads(resp.read().decode("utf-8"))+ from specify_cli.authentication.http import _StripAuthOnRedirect++ def reject_token_redirect(_old_url: str, new_url: str) -> None:+ # A 307/308 redirect preserves this POST body, including the+ # client_secret. Refuse every redirect so credentials cannot+ # leave the fixed Microsoft token endpoint.+ raise urllib.error.URLError(+ f"Azure AD token request must not be redirected to {new_url}"+ )++ opener = urllib.request.build_opener(+ _StripAuthOnRedirect((), reject_token_redirect)+ )+ with opener.open(req, timeout=30) as resp: # noqa: S310+ payload = _json.loads(+ read_response_limited(+ resp,+ max_bytes=MAX_JSON_METADATA_BYTES,+ error_type=_TokenResponseTooLarge,+ label="Azure DevOps token response",+ ).decode("utf-8")+ )token = payload.get("access_token", "").strip()return token or None- except (urllib.error.URLError, OSError, _json.JSONDecodeError, KeyError):+ except (+ urllib.error.URLError,+ OSError,+ _json.JSONDecodeError,+ UnicodeDecodeError,+ _TokenResponseTooLarge,+ ):+ # Network failure, malformed JSON, or an oversized response β fall+ # through to the next strategy. Unrelated programming errors (other+ # ValueErrors, KeyErrors) intentionally propagate so they surface.return Nonediff --git a/src/specify_cli/authentication/http.py b/src/specify_cli/authentication/http.pyindex 782403cc2..532779c1a 100644--- a/src/specify_cli/authentication/http.py+++ b/src/specify_cli/authentication/http.py@@ -17,6 +17,7 @@ from fnmatch import fnmatchfrom typing import Callablefrom urllib.parse import urlparse+from .._download_security import is_https_or_localhost_http, is_loopback_urlfrom . import get_providerfrom .config import AuthConfigEntry, _default_config_path, find_entries_for_url, load_auth_config@@ -60,8 +61,27 @@ def _hostname_in_hosts(hostname: str, hosts: tuple[str, ...]) -> bool:RedirectValidator = Callable[[str, str], None]+def _validate_strict_redirect(old_url: str, new_url: str) -> None:+ target_is_allowed = is_https_or_localhost_http(new_url)+ remote_to_http_loopback = (+ urlparse(new_url).scheme == "http"+ and not is_loopback_url(old_url)+ )+ if not target_is_allowed or remote_to_http_loopback:+ raise urllib.error.URLError(+ f"unsafe redirect to {new_url}: target must use HTTPS with a hostname, "+ "or stay within localhost over HTTP (127.0.0.1, ::1)"+ )++class _StripAuthOnRedirect(urllib.request.HTTPRedirectHandler):- """Drop ``Authorization`` when a redirect leaves trusted hosts or downgrades."""+ """Redirect handler that guards every redirect it is installed for.++ 1. Run any caller-provided redirect validator.+ 2. Reject redirects that are not HTTPS with a hostname. HTTP loopback is+ allowed only when the previous hop is also loopback.+ 3. Drop ``Authorization`` when a redirect leaves trusted hosts or downgrades.+ """def __init__(self,@@ -82,6 +102,7 @@ class _StripAuthOnRedirect(urllib.request.HTTPRedirectHandler):if self._redirect_validator is not None:self._redirect_validator(req.full_url, newurl)+ _validate_strict_redirect(req.full_url, newurl)original_auth = (req.get_header("Authorization")@@ -155,6 +176,10 @@ def open_url(*extra_headers* (e.g. ``Accept``) are merged into every attempt.*redirect_validator*, when provided, is called with ``(old_url, new_url)``before following each redirect and may raise to reject the redirect.++ Redirect scheme safety: every attempt goes through+ ``_StripAuthOnRedirect``, which rejects redirects to non-HTTPS URLs except+ HTTP between localhost / 127.0.0.1 / ::1 URLs."""entries = find_entries_for_url(url, _load_config())@@ -188,7 +213,7 @@ def open_url(# No entry worked (or none matched) β unauthenticated fallbackreq = _make_req({})- if redirect_validator is not None:- opener = urllib.request.build_opener(_StripAuthOnRedirect((), redirect_validator))- return opener.open(req, timeout=timeout)- return urllib.request.urlopen(req, timeout=timeout) # noqa: S310+ # No auth is attached on this path, so the handler's host list is empty:+ # here it runs redirect validation only, not auth stripping.+ opener = urllib.request.build_opener(_StripAuthOnRedirect((), redirect_validator))+ return opener.open(req, timeout=timeout)diff --git a/tests/http_helpers.py b/tests/http_helpers.pyindex 46e26806b..04696d7c0 100644--- a/tests/http_helpers.py+++ b/tests/http_helpers.py@@ -1,15 +1,46 @@-"""HTTP test helpers shared by version-related CLI tests."""+"""HTTP test helpers shared by CLI tests."""+import ioimport json+import urllib.requestfrom unittest.mock import MagicMock+import pytest+def mock_urlopen_response(payload: dict) -> MagicMock:"""Build a urlopen context-manager mock whose read returns JSON."""body = json.dumps(payload).encode("utf-8")resp = MagicMock()- resp.read.return_value = body+ resp.read.side_effect = io.BytesIO(body).readcm = MagicMock()cm.__enter__.return_value = respcm.__exit__.return_value = Falsereturn cm++[email protected](autouse=True)+def route_opener_open_through_urlopen(monkeypatch):+ """Route build_opener().open through urllib.request.urlopen.++ ``open_url(...)`` fetches via ``build_opener(...).open()``, which bypasses+ ``urllib.request.urlopen`` β and with it the urlopen patches these test+ modules are built on.+ Delegating ``open()`` to urlopen at call time keeps those patches+ effective; the redirect handler's own behavior is covered by+ ``TestRedirectStripping`` in test_authentication.py.++ Import this fixture into a test module to activate it there.+ """++ class _UrlopenDelegatingOpener:+ def open(self, req, data=None, timeout=None):+ if data is None:+ return urllib.request.urlopen(req, timeout=timeout)+ return urllib.request.urlopen(req, data=data, timeout=timeout)++ monkeypatch.setattr(+ urllib.request,+ "build_opener",+ lambda *handlers: _UrlopenDelegatingOpener(),+ )diff --git a/tests/integrations/test_integration_catalog.py b/tests/integrations/test_integration_catalog.pyindex 43eeb6788..2b6c76161 100644--- a/tests/integrations/test_integration_catalog.py+++ b/tests/integrations/test_integration_catalog.py@@ -6,6 +6,8 @@ import osimport pytestimport yaml+from tests.http_helpers import route_opener_open_through_urlopen # noqa: F401+from specify_cli.integrations.catalog import (IntegrationCatalog,IntegrationCatalogEntry,diff --git a/tests/self_upgrade_helpers.py b/tests/self_upgrade_helpers.pyindex c363f57b1..fc0f339f9 100644--- a/tests/self_upgrade_helpers.py+++ b/tests/self_upgrade_helpers.py@@ -18,7 +18,7 @@ from specify_cli._version import (_verify_upgrade,)from tests.conftest import strip_ansi-from tests.http_helpers import mock_urlopen_response+from tests.http_helpers import mock_urlopen_response, route_opener_open_through_urlopen__all__ = ("SENTINEL_GH_TOKEN",@@ -31,6 +31,7 @@ __all__ = ("_verify_upgrade","mock_urlopen_response","requires_posix",+ "route_opener_open_through_urlopen","runner","strip_ansi",)diff --git a/tests/test_authentication.py b/tests/test_authentication.pyindex 84f6d16fb..727a9a37d 100644--- a/tests/test_authentication.py+++ b/tests/test_authentication.py@@ -14,6 +14,7 @@ Covers:from __future__ import annotationsimport base64+import ioimport jsonimport os@@ -524,10 +525,15 @@ class TestAzureDevOpsAuth:tenant_id="tid", client_id="cid", client_secret_env="MY_SECRET",)mock_resp = MagicMock()- mock_resp.read.return_value = b'{"access_token": "ad-acquired-token"}'+ mock_resp.read.side_effect = io.BytesIO(b'{"access_token": "ad-acquired-token"}').readmock_resp.__enter__ = lambda s: smock_resp.__exit__ = MagicMock(return_value=False)- with patch("urllib.request.urlopen", return_value=mock_resp):+ # The token request goes through a strict-redirect opener (so a 307/308+ # cannot forward the client_secret body to a non-HTTPS host), not bare+ # urlopen; patch the opener it builds.+ mock_opener = MagicMock()+ mock_opener.open.return_value = mock_resp+ with patch("urllib.request.build_opener", return_value=mock_opener):assert AzureDevOpsAuth().resolve_token(entry) == "ad-acquired-token"def test_resolve_token_azure_ad_missing_secret_returns_none(self, monkeypatch):@@ -542,14 +548,62 @@ class TestAzureDevOpsAuth:def test_resolve_token_azure_ad_network_error_returns_none(self, monkeypatch):"""azure-ad returns None on network errors."""import urllib.error- from unittest.mock import patch+ from unittest.mock import MagicMock, patch+ monkeypatch.setenv("MY_SECRET", "secret-value")+ entry = AuthConfigEntry(+ hosts=("dev.azure.com",), provider="azure-devops", auth="azure-ad",+ tenant_id="tid", client_id="cid", client_secret_env="MY_SECRET",+ )+ mock_opener = MagicMock()+ mock_opener.open.side_effect = urllib.error.URLError("connection refused")+ with patch("urllib.request.build_opener", return_value=mock_opener):+ assert AzureDevOpsAuth().resolve_token(entry) is None++ def test_resolve_token_azure_ad_rejects_https_redirect(self, monkeypatch):+ """The client-secret POST must never be redirected to another host."""+ import urllib.error+ from unittest.mock import MagicMock, patch+ from urllib.request import Request++ monkeypatch.setenv("MY_SECRET", "secret-value")+ entry = AuthConfigEntry(+ hosts=("dev.azure.com",), provider="azure-devops", auth="azure-ad",+ tenant_id="tid", client_id="cid", client_secret_env="MY_SECRET",+ )+ mock_opener = MagicMock()+ mock_opener.open.side_effect = urllib.error.URLError("stop after setup")++ with patch("urllib.request.build_opener", return_value=mock_opener) as build_opener:+ assert AzureDevOpsAuth().resolve_token(entry) is None++ redirect_handler = build_opener.call_args.args[0]+ request = Request("https://login.microsoftonline.com/tid/oauth2/v2.0/token")+ with pytest.raises(urllib.error.URLError, match="must not be redirected"):+ redirect_handler.redirect_request(+ request,+ io.BytesIO(b""),+ 307,+ "Temporary Redirect",+ {},+ "https://evil.example/token",+ )++ def test_resolve_token_azure_ad_invalid_utf8_returns_none(self, monkeypatch):+ """azure-ad returns None when the token response is not valid UTF-8."""+ from unittest.mock import MagicMock, patchmonkeypatch.setenv("MY_SECRET", "secret-value")entry = AuthConfigEntry(hosts=("dev.azure.com",), provider="azure-devops", auth="azure-ad",tenant_id="tid", client_id="cid", client_secret_env="MY_SECRET",)- with patch("urllib.request.urlopen",- side_effect=urllib.error.URLError("connection refused")):+ mock_resp = MagicMock()+ mock_resp.read.side_effect = io.BytesIO(b"\xff").read+ mock_resp.__enter__ = lambda s: s+ mock_resp.__exit__ = MagicMock(return_value=False)+ mock_opener = MagicMock()+ mock_opener.open.return_value = mock_resp++ with patch("urllib.request.build_opener", return_value=mock_opener):assert AzureDevOpsAuth().resolve_token(entry) is None@@ -615,13 +669,15 @@ class TestAuthenticatedHttp:monkeypatch.setenv("GH_TOKEN", "my-token")self._set_config(monkeypatch, [_github_entry()])captured = {}- def fake_urlopen(req, timeout=None):+ def fake_open(req, timeout=None):captured["req"] = reqresp = MagicMock()resp.__enter__ = lambda s: sresp.__exit__ = MagicMock(return_value=False)return resp- with patch("specify_cli.authentication.http.urllib.request.urlopen", side_effect=fake_urlopen):+ mock_opener = MagicMock()+ mock_opener.open.side_effect = fake_open+ with patch("specify_cli.authentication.http.urllib.request.build_opener", return_value=mock_opener):open_url("https://example.com/file.json")assert captured["req"].get_header("Authorization") is None@@ -630,13 +686,15 @@ class TestAuthenticatedHttp:from specify_cli.authentication.http import open_urlself._set_config(monkeypatch, [])captured = {}- def fake_urlopen(req, timeout=None):+ def fake_open(req, timeout=None):captured["req"] = reqresp = MagicMock()resp.__enter__ = lambda s: sresp.__exit__ = MagicMock(return_value=False)return resp- with patch("specify_cli.authentication.http.urllib.request.urlopen", side_effect=fake_urlopen):+ mock_opener = MagicMock()+ mock_opener.open.side_effect = fake_open+ with patch("specify_cli.authentication.http.urllib.request.build_opener", return_value=mock_opener):open_url("https://github.com/org/repo")assert captured["req"].get_header("Authorization") is None@@ -658,8 +716,7 @@ class TestAuthenticatedHttp:return respmock_opener = MagicMock()mock_opener.open.side_effect = fake_side_effect- with patch("specify_cli.authentication.http.urllib.request.build_opener", return_value=mock_opener), \- patch("specify_cli.authentication.http.urllib.request.urlopen", side_effect=fake_side_effect):+ with patch("specify_cli.authentication.http.urllib.request.build_opener", return_value=mock_opener):open_url("https://github.com/org/repo")assert call_count == 2@@ -700,21 +757,23 @@ class TestAuthenticatedHttpNegative:def test_urlerror_propagates(self, monkeypatch):import urllib.error- from unittest.mock import patch+ from unittest.mock import MagicMock, patchfrom specify_cli.authentication.http import open_urlself._set_config(monkeypatch, [])- with patch("specify_cli.authentication.http.urllib.request.urlopen",- side_effect=urllib.error.URLError("refused")):+ mock_opener = MagicMock()+ mock_opener.open.side_effect = urllib.error.URLError("refused")+ with patch("specify_cli.authentication.http.urllib.request.build_opener", return_value=mock_opener):with pytest.raises(urllib.error.URLError):open_url("https://example.com/file")def test_timeout_propagates(self, monkeypatch):import socket- from unittest.mock import patch+ from unittest.mock import MagicMock, patchfrom specify_cli.authentication.http import open_urlself._set_config(monkeypatch, [])- with patch("specify_cli.authentication.http.urllib.request.urlopen",- side_effect=socket.timeout("timed out")):+ mock_opener = MagicMock()+ mock_opener.open.side_effect = socket.timeout("timed out")+ with patch("specify_cli.authentication.http.urllib.request.build_opener", return_value=mock_opener):with pytest.raises(socket.timeout):open_url("https://example.com/file")@@ -820,17 +879,18 @@ class TestRedirectStripping:assert new_req.headers.get("Authorization") is Noneassert new_req.unredirected_hdrs.get("Authorization") is None- def test_https_to_http_same_host_redirect_strips_auth(self):+ def test_https_to_http_same_host_redirect_rejected(self):from specify_cli.authentication.http import _StripAuthOnRedirectfrom urllib.request import Requestimport io+ import urllib.error+handler = _StripAuthOnRedirect(("github.com",))req = Request("https://github.com/org/repo", headers={"Authorization": "Bearer tok"})- new_req = handler.redirect_request(req, io.BytesIO(b""), 302, "Found", {},- "http://github.com/org/repo")- assert new_req is not None- assert new_req.headers.get("Authorization") is None- assert new_req.unredirected_hdrs.get("Authorization") is None++ with pytest.raises(urllib.error.URLError, match="unsafe redirect"):+ handler.redirect_request(req, io.BytesIO(b""), 302, "Found", {},+ "http://github.com/org/repo")def test_redirect_validator_can_reject_before_following_redirect(self):import urllib.error@@ -888,6 +948,78 @@ class TestRedirectStripping:handler.redirect_request(req, io.BytesIO(b""), 302, "Found", {},"https://[::1/asset")+ def test_redirect_rejects_https_downgrade(self):+ """HTTPS downloads must not follow redirects to non-local HTTP URLs."""+ from specify_cli.authentication.http import _StripAuthOnRedirect+ from urllib.request import Request+ import io+ import urllib.error+ handler = _StripAuthOnRedirect(("example.com",))+ req = Request("https://example.com/archive.zip")+ with pytest.raises(urllib.error.URLError, match="unsafe redirect"):+ handler.redirect_request(req, io.BytesIO(b""), 302, "Found", {},+ "http://evil.example.com/archive.zip")++ def test_redirect_rejects_remote_to_http_loopback(self):+ """A remote response must not redirect a download into loopback."""+ import io+ import urllib.error+ from urllib.request import Request++ from specify_cli.authentication.http import _StripAuthOnRedirect++ handler = _StripAuthOnRedirect(())+ request = Request("https://example.com/archive.zip")++ with pytest.raises(urllib.error.URLError, match="unsafe redirect"):+ handler.redirect_request(+ request,+ io.BytesIO(b""),+ 302,+ "Found",+ {},+ "http://127.0.0.1/internal",+ )++ def test_redirect_allows_loopback_to_http_loopback(self):+ """Local development may continue redirecting between loopback URLs."""+ import io+ from urllib.request import Request++ from specify_cli.authentication.http import _StripAuthOnRedirect++ handler = _StripAuthOnRedirect(())+ request = Request("http://localhost:8000/archive.zip")+ redirected = handler.redirect_request(+ request,+ io.BytesIO(b""),+ 302,+ "Found",+ {},+ "http://127.0.0.1:8001/archive.zip",+ )++ assert redirected is not None++ def test_strict_redirect_error_describes_target_and_allowed_localhost(self):+ from specify_cli.authentication.http import _StripAuthOnRedirect+ from urllib.request import Request+ import io+ import urllib.error++ handler = _StripAuthOnRedirect(("example.com",))+ req = Request("https://example.com/archive.zip")++ with pytest.raises(urllib.error.URLError) as exc_info:+ handler.redirect_request(req, io.BytesIO(b""), 302, "Found", {},+ "http://evil.example.com/archive.zip")++ error_message = str(exc_info.value)+ assert "http://evil.example.com/archive.zip" in error_message+ assert "localhost" in error_message+ assert "127.0.0.1" in error_message+ assert "::1" in error_message+# ---------------------------------------------------------------------------# _fetch_latest_release_tag delegation@@ -907,7 +1039,7 @@ class TestFetchLatestReleaseTagDelegation:captured["request"] = reqbody = _json.dumps({"tag_name": "v9.9.9"}).encode()resp = MagicMock()- resp.read.return_value = body+ resp.read.side_effect = io.BytesIO(body).readcm = MagicMock()cm.__enter__.return_value = respcm.__exit__.return_value = False@@ -927,20 +1059,25 @@ class TestFetchLatestReleaseTagDelegation:assert captured["request"].get_header("Authorization") == "Bearer forwarded-sentinel"def test_no_config_means_no_auth(self, monkeypatch):- from unittest.mock import patch+ from unittest.mock import MagicMock, patchfrom specify_cli._version import _fetch_latest_release_tagself._set_config(monkeypatch, [])captured, side_effect = self._capture_request()- with patch("specify_cli.authentication.http.urllib.request.urlopen", side_effect=side_effect):+ # The unauthenticated path uses the strict redirect opener too.+ mock_opener = MagicMock()+ mock_opener.open.side_effect = side_effect+ with patch("specify_cli.authentication.http.urllib.request.build_opener", return_value=mock_opener):_fetch_latest_release_tag()assert captured["request"].get_header("Authorization") is Nonedef test_accept_header_present(self, monkeypatch):- from unittest.mock import patch+ from unittest.mock import MagicMock, patchfrom specify_cli._version import _fetch_latest_release_tagself._set_config(monkeypatch, [])captured, side_effect = self._capture_request()- with patch("specify_cli.authentication.http.urllib.request.urlopen", side_effect=side_effect):+ mock_opener = MagicMock()+ mock_opener.open.side_effect = side_effect+ with patch("specify_cli.authentication.http.urllib.request.build_opener", return_value=mock_opener):_fetch_latest_release_tag()assert captured["request"].get_header("Accept") == "application/vnd.github+json"diff --git a/tests/test_download_security.py b/tests/test_download_security.pynew file mode 100644index 000000000..24eeedd95--- /dev/null+++ b/tests/test_download_security.py@@ -0,0 +1,68 @@+"""Tests for bounded HTTP download helpers."""++from __future__ import annotations++import pytest++from specify_cli._download_security import (+ is_https_or_localhost_http,+ read_response_limited,+)+++ "url, allowed",+ [+ ("https://example.com/preset.zip", True),+ ("http://localhost:8000/preset.zip", True),+ ("http://127.0.0.1/preset.zip", True),+ ("http://[::1]/preset.zip", True),+ # Non-loopback HTTP is rejected.+ ("http://example.com/preset.zip", False),+ # Loopback allowance is an exact-string match: 127.0.0.2 is not covered.+ ("http://127.0.0.2/preset.zip", False),+ # A hostname is always required, even for HTTPS.+ ("https:///preset.zip", False),+ ("https://", False),+ ],+)+def test_is_https_or_localhost_http(url, allowed):+ assert is_https_or_localhost_http(url) is allowed+++class _Response:+ """Faithful stream stand-in: read() advances a cursor and returns b"" at EOF."""++ def __init__(self, data: bytes, *, chunk: int | None = None):+ self.data = data+ self.pos = 0+ # When set, never return more than *chunk* bytes per call even if more is+ # requested - simulates short reads (e.g. chunked transfer encoding).+ self.chunk = chunk++ def read(self, size: int = -1) -> bytes:+ if size < 0:+ size = len(self.data) - self.pos+ if self.chunk is not None:+ size = min(size, self.chunk)+ out = self.data[self.pos : self.pos + size]+ self.pos += len(out)+ return out+++def test_read_response_limited_rejects_oversized_download():+ with pytest.raises(ValueError, match="exceeds maximum size"):+ read_response_limited(_Response(b"abcde"), max_bytes=4)+++def test_read_response_limited_returns_full_body_within_limit():+ assert read_response_limited(_Response(b"abcde"), max_bytes=10) == b"abcde"+++def test_read_response_limited_enforces_bound_under_short_reads():+ # A server that streams more than max_bytes total while every read() returns+ # fewer bytes than requested (chunked encoding) must still be rejected - a+ # single read(max_bytes + 1) could be fooled, the accumulating loop cannot.+ response = _Response(b"x" * 100, chunk=8)+ with pytest.raises(ValueError, match="exceeds maximum size"):+ read_response_limited(response, max_bytes=16)diff --git a/tests/test_extensions.py b/tests/test_extensions.pyindex 018ed8a3f..00d8da152 100644--- a/tests/test_extensions.py+++ b/tests/test_extensions.py@@ -9,6 +9,7 @@ Tests cover:- Catalog stack (multi-catalog support)"""+import ioimport pytestimport jsonimport os@@ -22,6 +23,7 @@ from datetime import datetime, timezonefrom unittest.mock import MagicMockfrom tests.conftest import strip_ansi+from tests.http_helpers import route_opener_open_through_urlopen # noqa: F401from specify_cli import extensions as _ext_modulefrom specify_cli.extensions import (CatalogEntry,@@ -4978,7 +4980,7 @@ class TestExtensionCatalog:zip_bytes = zip_buf.getvalue()release_response = MagicMock()- release_response.read.return_value = json.dumps(+ release_response.read.side_effect = io.BytesIO(json.dumps({"assets": [{@@ -4987,12 +4989,12 @@ class TestExtensionCatalog:}]}- ).encode()+ ).encode()).readrelease_response.__enter__ = lambda s: srelease_response.__exit__ = MagicMock(return_value=False)asset_response = MagicMock()- asset_response.read.return_value = zip_bytes+ asset_response.read.side_effect = io.BytesIO(zip_bytes).readasset_response.__enter__ = lambda s: sasset_response.__exit__ = MagicMock(return_value=False)@@ -8761,10 +8763,10 @@ def test_extension_wrapper_resolves_ghes_asset_when_host_configured(tmp_path, modef fake_open(url, timeout=None, extra_headers=None):captured.append(url)resp = MagicMock()- resp.read.return_value = json.dumps({+ resp.read.side_effect = io.BytesIO(json.dumps({"assets": [{"name": "ext.zip","url": "https://ghes.example/api/v3/repos/o/r/releases/assets/7"}]- }).encode()+ }).encode()).readyield respmonkeypatch.setattr(catalog, "_open_url", fake_open)diff --git a/tests/test_github_http.py b/tests/test_github_http.pyindex a6a0dd9e2..a02f3906c 100644--- a/tests/test_github_http.py+++ b/tests/test_github_http.py@@ -1,16 +1,20 @@"""Tests for GitHub-authenticated HTTP request helpers."""+import ioimport jsonimport osfrom contextlib import contextmanagerfrom unittest.mock import MagicMock, patch+from urllib.request import Requestimport pytestfrom specify_cli._github_http import (+ GITHUB_HOSTS,build_github_request,resolve_github_release_asset_api_url,)+from specify_cli.authentication.http import _StripAuthOnRedirectclass TestBuildGitHubRequest:@@ -90,7 +94,7 @@ class TestResolveGitHubReleaseAssetApiUrl:@contextmanagerdef fake_open(url, timeout=None, extra_headers=None):resp = MagicMock()- resp.read.return_value = json.dumps(release_json).encode()+ resp.read.side_effect = io.BytesIO(json.dumps(release_json).encode()).readyield respreturn fake_open@@ -198,7 +202,7 @@ class TestResolveGitHubReleaseAssetApiUrl:def capturing_open(url, timeout=None, extra_headers=None):captured_urls.append(url)resp = MagicMock()- resp.read.return_value = json.dumps({"assets": []}).encode()+ resp.read.side_effect = io.BytesIO(json.dumps({"assets": []}).encode()).readyield respresolve_github_release_asset_api_url(@@ -217,7 +221,7 @@ class TestResolveGitHubReleaseAssetApiUrl:def capturing_open(url, timeout=None, extra_headers=None):captured_urls.append(url)resp = MagicMock()- resp.read.return_value = json.dumps({"assets": []}).encode()+ resp.read.side_effect = io.BytesIO(json.dumps({"assets": []}).encode()).readyield respresolve_github_release_asset_api_url(@@ -260,7 +264,7 @@ class TestResolveGitHubReleaseAssetApiUrl:def recording_open(url, timeout=None, extra_headers=None):called.append(url)resp = MagicMock()- resp.read.return_value = b"{}"+ resp.read.side_effect = io.BytesIO(b"{}").readyield respresult = resolve_github_release_asset_api_url(@@ -299,7 +303,7 @@ class TestResolveGitHubReleaseAssetApiUrl:def recording_open(url, timeout=None, extra_headers=None):called.append(url)resp = MagicMock()- resp.read.return_value = b"{}"+ resp.read.side_effect = io.BytesIO(b"{}").readyield respurl = "https://ghes.example/api/v3/repos/o/r/releases/assets/7"@@ -317,7 +321,7 @@ class TestResolveGitHubReleaseAssetApiUrl:def capturing_open(url, timeout=None, extra_headers=None):captured.append(url)resp = MagicMock()- resp.read.return_value = json.dumps({"assets": []}).encode()+ resp.read.side_effect = io.BytesIO(json.dumps({"assets": []}).encode()).readyield respresolve_github_release_asset_api_url(@@ -344,10 +348,10 @@ class TestResolveGitHubReleaseAssetApiUrl:def capturing_open(url, timeout=None, extra_headers=None):captured.append(url)resp = MagicMock()- resp.read.return_value = json.dumps({+ resp.read.side_effect = io.BytesIO(json.dumps({"assets": [{"name": "pack.zip","url": "https://api.github.com/repos/org/repo/releases/assets/99"}]- }).encode()+ }).encode()).readyield respresult = resolve_github_release_asset_api_url(@@ -357,3 +361,43 @@ class TestResolveGitHubReleaseAssetApiUrl:)assert result == "https://api.github.com/repos/org/repo/releases/assets/99"assert captured == ["https://api.github.com/repos/org/repo/releases/tags/v1.0"]+++class TestGitHubRedirectAuth:+ """Tests for GitHub-owned redirect auth handling."""++ def test_multi_hop_github_redirect_preserves_unredirected_auth(self):+ """Auth survives a multi-hop redirect chain within GitHub hosts."""+ handler = _StripAuthOnRedirect(tuple(GITHUB_HOSTS))+ req1 = Request(+ "https://github.com/org/repo",+ headers={"Authorization": "Bearer tok"},+ )++ req2 = handler.redirect_request(+ req1,+ io.BytesIO(b""),+ 302,+ "Found",+ {},+ "https://codeload.github.com/org/repo/zip",+ )+ assert req2 is not None+ auth2 = req2.get_header("Authorization") or req2.unredirected_hdrs.get(+ "Authorization"+ )+ assert auth2 == "Bearer tok"++ req3 = handler.redirect_request(+ req2,+ io.BytesIO(b""),+ 302,+ "Found",+ {},+ "https://raw.githubusercontent.com/org/repo/main/file",+ )+ assert req3 is not None+ auth3 = req3.get_header("Authorization") or req3.unredirected_hdrs.get(+ "Authorization"+ )+ assert auth3 == "Bearer tok"diff --git a/tests/test_presets.py b/tests/test_presets.pyindex aeceaf6a8..26eeb8766 100644--- a/tests/test_presets.py+++ b/tests/test_presets.py@@ -2303,7 +2303,7 @@ class TestPresetCatalog:zip_bytes = zip_buf.getvalue()release_response = MagicMock()- release_response.read.return_value = json.dumps(+ release_response.read.side_effect = io.BytesIO(json.dumps({"assets": [{@@ -2312,12 +2312,12 @@ class TestPresetCatalog:}]}- ).encode()+ ).encode()).readrelease_response.__enter__ = lambda s: srelease_response.__exit__ = MagicMock(return_value=False)asset_response = MagicMock()- asset_response.read.return_value = zip_bytes+ asset_response.read.side_effect = io.BytesIO(zip_bytes).readasset_response.__enter__ = lambda s: sasset_response.__exit__ = MagicMock(return_value=False)@@ -7458,10 +7458,10 @@ def test_preset_wrapper_resolves_ghes_asset_when_host_configured(tmp_path, monkedef fake_open(url, timeout=None, extra_headers=None):captured.append(url)resp = MagicMock()- resp.read.return_value = json.dumps({+ resp.read.side_effect = io.BytesIO(json.dumps({"assets": [{"name": "pack.zip","url": "https://ghes.example/api/v3/repos/o/r/releases/assets/9"}]- }).encode()+ }).encode()).readyield respmonkeypatch.setattr(catalog, "_open_url", fake_open)diff --git a/tests/test_self_upgrade_detection.py b/tests/test_self_upgrade_detection.pyindex ab575e743..73b55ebb7 100644--- a/tests/test_self_upgrade_detection.py+++ b/tests/test_self_upgrade_detection.py@@ -13,6 +13,7 @@ import specify_clifrom specify_cli import appfrom tests.self_upgrade_helpers import (+ route_opener_open_through_urlopen, # noqa: F401 (autouse fixture)_InstallMethod,_assemble_installer_argv,_completed_process,diff --git a/tests/test_self_upgrade_execution.py b/tests/test_self_upgrade_execution.pyindex 6696b4fc7..5c761014b 100644--- a/tests/test_self_upgrade_execution.py+++ b/tests/test_self_upgrade_execution.py@@ -7,6 +7,7 @@ from unittest.mock import patchfrom specify_cli import appfrom tests.self_upgrade_helpers import (+ route_opener_open_through_urlopen, # noqa: F401 (autouse fixture)_completed_process,mock_urlopen_response,requires_posix,diff --git a/tests/test_self_upgrade_verification.py b/tests/test_self_upgrade_verification.pyindex f1a018f06..c4e7eecf1 100644--- a/tests/test_self_upgrade_verification.py+++ b/tests/test_self_upgrade_verification.py@@ -8,6 +8,7 @@ import specify_clifrom specify_cli import appfrom tests.self_upgrade_helpers import (+ route_opener_open_through_urlopen, # noqa: F401 (autouse fixture)SENTINEL_GH_TOKEN,SENTINEL_GITHUB_TOKEN,_InstallMethod,diff --git a/tests/test_upgrade.py b/tests/test_upgrade.pyindex 3ad8c84f6..b6f313428 100644--- a/tests/test_upgrade.py+++ b/tests/test_upgrade.py@@ -17,6 +17,7 @@ import pytestfrom typer.testing import CliRunnerfrom specify_cli import app+from specify_cli._download_security import read_response_limited as _real_read_response_limitedfrom specify_cli._version import (_fetch_latest_release_tag,_get_installed_version,@@ -24,7 +25,10 @@ from specify_cli._version import (_normalize_tag,)from tests.conftest import strip_ansi-from tests.http_helpers import mock_urlopen_response+from tests.http_helpers import (+ mock_urlopen_response,+ route_opener_open_through_urlopen, # noqa: F401 (autouse fixture)+)runner = CliRunner()@@ -235,6 +239,46 @@ class TestFailureCategorization:_fetch_latest_release_tag()+class TestBoundedRead:+ """Regression test for the read_response_limited hardening.++ A future refactor could silently revert `_fetch_latest_release_tag` to+ `resp.read()` (the unbounded form) β this test pins the contract that+ the response body is read through ``read_response_limited`` with a+ bounded ``max_bytes``.+ """++ def test_response_body_is_bounded(self):+ recorded: dict[str, int | str] = {}++ def _spy(response, *, max_bytes: int, label: str, **kwargs):+ # max_bytes and label are keyword-only with no defaults: if the+ # caller forgets to pass either, the call raises TypeError here+ # (instead of recording a misleading None).+ recorded["max_bytes"] = max_bytes+ recorded["label"] = label+ # Forward to the real implementation so the function under test+ # still gets a parseable body.+ return _real_read_response_limited(+ response, max_bytes=max_bytes, label=label, **kwargs+ )++ with patch(+ "specify_cli.authentication.http.urllib.request.urlopen",+ return_value=mock_urlopen_response({"tag_name": "v9.9.9"}),+ ), patch("specify_cli._version.read_response_limited", side_effect=_spy):+ tag, reason = _fetch_latest_release_tag()++ assert tag == "v9.9.9"+ assert reason is None+ # The cap (1 MiB) is a deliberate ceiling for the GitHub release+ # JSON β keep it explicit so a future refactor that drops the+ # `max_bytes=` argument fails this test instead of regressing+ # silently to the default.+ assert recorded["max_bytes"] == 1024 * 1024+ assert recorded["label"] == "GitHub latest release"++_FAILURE_CASES = [("offline or timeout", urllib.error.URLError("down")),(_RATE_LIMITED_REASON, _http_error(403)),diff --git a/tests/test_workflows.py b/tests/test_workflows.pyindex af5584d63..00cf1f2be 100644--- a/tests/test_workflows.py+++ b/tests/test_workflows.py@@ -8658,18 +8658,15 @@ steps:class FakeResponse:def __init__(self, data, url=None):self._data = data+ self._pos = 0self._url = url or "https://api.github.com/repos/org/repo/releases/assets/42"- def read(self, amt=None):- if not hasattr(self, "_pos"):- self._pos = 0- if amt is None:- chunk = self._data[self._pos :]- self._pos = len(self._data)- return chunk- chunk = self._data[self._pos : self._pos + amt]- self._pos += len(chunk)- return chunk+ def read(self, size=-1):+ if size < 0:+ size = len(self._data) - self._pos+ out = self._data[self._pos : self._pos + size]+ self._pos += len(out)+ return outdef geturl(self):return self._url@@ -8729,18 +8726,15 @@ steps:class FakeResponse:def __init__(self, data, url=None):self._data = data+ self._pos = 0self._url = url or "https://api.github.com/repos/org/repo/releases/assets/42"- def read(self, amt=None):- if not hasattr(self, "_pos"):- self._pos = 0- if amt is None:- chunk = self._data[self._pos :]- self._pos = len(self._data)- return chunk- chunk = self._data[self._pos : self._pos + amt]- self._pos += len(chunk)- return chunk+ def read(self, size=-1):+ if size < 0:+ size = len(self._data) - self._pos+ out = self._data[self._pos : self._pos + size]+ self._pos += len(out)+ return outdef geturl(self):return self._url@@ -8780,18 +8774,15 @@ steps:class FakeResponse:def __init__(self, data, url=None):self._data = data+ self._pos = 0self._url = url or "https://api.github.com/repos/org/repo/releases/assets/55"- def read(self, amt=None):- if not hasattr(self, "_pos"):- self._pos = 0- if amt is None:- chunk = self._data[self._pos :]- self._pos = len(self._data)- return chunk- chunk = self._data[self._pos : self._pos + amt]- self._pos += len(chunk)- return chunk+ def read(self, size=-1):+ if size < 0:+ size = len(self._data) - self._pos+ out = self._data[self._pos : self._pos + size]+ self._pos += len(out)+ return outdef geturl(self):return self._url@@ -8873,18 +8864,15 @@ steps:class FakeResponse:def __init__(self, data, url=None):self._data = data+ self._pos = 0self._url = url or "https://ghes.example/api/v3/repos/org/repo/releases/assets/42"- def read(self, amt=None):- if not hasattr(self, "_pos"):- self._pos = 0- if amt is None:- chunk = self._data[self._pos :]- self._pos = len(self._data)- return chunk- chunk = self._data[self._pos : self._pos + amt]- self._pos += len(chunk)- return chunk+ def read(self, size=-1):+ if size < 0:+ size = len(self._data) - self._pos+ out = self._data[self._pos : self._pos + size]+ self._pos += len(out)+ return outdef geturl(self):return self._url@@ -8936,18 +8924,15 @@ steps:class FakeResponse:def __init__(self, data, url=None):self._data = data+ self._pos = 0self._url = url or "https://ghes.example/api/v3/repos/org/repo/releases/assets/55"- def read(self, amt=None):- if not hasattr(self, "_pos"):- self._pos = 0- if amt is None:- chunk = self._data[self._pos :]- self._pos = len(self._data)- return chunk- chunk = self._data[self._pos : self._pos + amt]- self._pos += len(chunk)- return chunk+ def read(self, size=-1):+ if size < 0:+ size = len(self._data) - self._pos+ out = self._data[self._pos : self._pos + size]+ self._pos += len(out)+ return outdef geturl(self):return self._url