💫 Toolkit to help you get started with SDD or any other process!
fix: harden bounded reads and redirect validation (#3671)
Pascal THUET3a7a8758f709
diff --git a/src/specify_cli/_download_security.py b/src/specify_cli/_download_security.pyindex 0d39baca0..b4f6d28ad 100644--- a/src/specify_cli/_download_security.py+++ b/src/specify_cli/_download_security.py@@ -2,14 +2,17 @@from __future__ import annotations+import io+import socket+from ipaddress import IPv4Address, IPv6Address, ip_addressfrom typing import NoReturn, TypeVar-from urllib.parse import urlparse+from urllib.parse import ParseResult, urlparseErrorT = TypeVar("ErrorT", bound=Exception)MAX_DOWNLOAD_BYTES = 50 * 1024 * 1024-READ_CHUNK_SIZE = 1024 * 1024+READ_CHUNK_SIZE = 64 * 1024# Tighter ceiling for responses that are read fully into memory and parsed as# JSON. The 50 MiB MAX_DOWNLOAD_BYTES default is sized for archive/payload@@ -20,35 +23,156 @@ READ_CHUNK_SIZE = 1024 * 1024# METADATA covers fixed-shape single-object responses (an OAuth token, one# release's metadata): a few KiB in practice, 1 MiB is already generous.MAX_JSON_METADATA_BYTES = 1 * 1024 * 1024-_LOOPBACK_HOSTS = frozenset(("localhost", "127.0.0.1", "::1"))+++def _ip_address_without_scope(+ hostname: str,+) -> IPv4Address | IPv6Address | None:+ """Parse a canonical IP literal, validating an optional IPv6 zone ID."""+ if "%" in hostname:+ # Accept only the RFC 6874 ``%25<zone>`` spelling. Other escapes can+ # alter the IPv6 address when urllib unquotes the authority.+ address_text, separator, zone = hostname.partition("%25")+ if (+ not separator+ or ":" not in address_text+ or "%" in address_text+ or "%" in zone+ ):+ return None+ if not zone or any(+ not (character.isascii() and (character.isalnum() or character in "._~-"))+ for character in zone+ ):+ return None+ else:+ address_text = hostname+ try:+ address = ip_address(address_text)+ except ValueError:+ return None+ if "%" in hostname and not isinstance(address, IPv6Address):+ return None+ return address+++def _is_ip_loopback(address: IPv4Address | IPv6Address | None) -> bool:+ if address is None:+ return False+ mapped = getattr(address, "ipv4_mapped", None)+ return address.is_loopback or bool(mapped and mapped.is_loopback)+++def _is_ip_local_redirect_target(+ address: IPv4Address | IPv6Address | None,+) -> bool:+ """Treat loopback and unspecified listener aliases as local targets."""+ if address is None:+ return False+ mapped = getattr(address, "ipv4_mapped", None)+ return _is_ip_loopback(address) or address.is_unspecified or bool(+ mapped and mapped.is_unspecified+ )+++def _parse_url(url: str) -> ParseResult | None:+ """Parse *url*, rejecting missing hosts and malformed ports."""+ try:+ parsed = urlparse(url)+ hostname = parsed.hostname+ # Accessing ``port`` performs urllib's range and syntax validation.+ parsed.port+ except (TypeError, ValueError):+ return None+ if not hostname:+ return None++ if "%" in hostname:+ # urllib unquotes reg-name/IPv4 authorities before connecting. Reject+ # them so encoded dots, characters, ports, or brackets cannot make the+ # validated hostname differ from the effective target. The only safe+ # percent form retained is a validated bracketed IPv6 zone ID.+ if _ip_address_without_scope(hostname) is None:+ return None+ elif ":" not in hostname:+ try:+ hostname.encode("idna")+ except UnicodeError:+ return None+ return parsed+++def _is_definite_loopback_host(hostname: str) -> bool:+ """Recognize only unambiguous hosts that may safely authorize HTTP."""+ if not hostname.isascii():+ return False+ if hostname == "localhost":+ return True+ return _is_ip_loopback(_ip_address_without_scope(hostname))+++def _is_potential_local_target_host(hostname: str) -> bool:+ """Conservatively classify aliases that could reach a local listener."""+ if ":" in hostname:+ return _is_ip_local_redirect_target(_ip_address_without_scope(hostname))+ try:+ host = hostname.encode("idna").decode("ascii").lower().removesuffix(".")+ except UnicodeError:+ return False+ if host == "localhost" or host.endswith(".localhost"):+ return True++ address = _ip_address_without_scope(host)+ if address is None:+ # Historical IPv4 spellings are resolver-dependent. They are never+ # trusted to authorize HTTP, but treating them as potentially local+ # prevents them from bypassing a remote-to-loopback redirect check.+ try:+ address = ip_address(socket.inet_aton(host))+ except OSError:+ return False+ return _is_ip_local_redirect_target(address)def is_loopback_url(url: str) -> bool:- """Return whether *url* targets an explicitly allowed loopback host."""- return urlparse(url).hostname in _LOOPBACK_HOSTS+ """Return whether *url* has an unambiguous loopback host."""+ parsed = _parse_url(url)+ return parsed is not None and _is_definite_loopback_host(parsed.hostname)+++def _is_potential_local_target_url(url: str) -> bool:+ parsed = _parse_url(url)+ return parsed is not None and _is_potential_local_target_host(parsed.hostname)def is_https_or_localhost_http(url: str) -> bool:"""Return True if *url* is HTTPS, or HTTP limited to loopback hosts.Shared scheme-safety predicate used by the auth HTTP redirect handler and- by the direct URL validations in the CLI download flows, so the rule (and- any future tightening of it) lives in one place.+ direct URL validations in CLI download flows.A hostname is always required: a URL without one (e.g. ``https:///x``)has no real target and is rejected regardless of scheme.- The loopback allowance is a deliberate *exact-string* match on- ``localhost`` / ``127.0.0.1`` / ``::1``, not an IP-range check: other- loopback addresses (e.g. ``127.0.0.2``) are intentionally not covered.- ``urlparse`` already lower-cases the hostname, so the comparison is- case-insensitive.+ The HTTP exception is deliberately limited to unambiguous ``localhost``+ and canonical IPv4/IPv6 loopback literals. Ambiguous numeric, Unicode, and+ unspecified-address aliases are classified defensively for redirects but+ never authorize HTTP. No DNS lookup is performed; DNS and hosts-file+ aliases require connection-level rebinding protection outside this helper."""- parsed = urlparse(url)- if not parsed.hostname:+ parsed = _parse_url(url)+ if parsed is None:return False- is_localhost = parsed.hostname in _LOOPBACK_HOSTS- return parsed.scheme == "https" or (parsed.scheme == "http" and is_localhost)+ return parsed.scheme == "https" or (+ parsed.scheme == "http" and _is_definite_loopback_host(parsed.hostname)+ )+++def is_safe_download_redirect(old_url: str, new_url: str) -> bool:+ """Return whether a redirect preserves the shared download URL policy."""+ if not is_https_or_localhost_http(new_url):+ return False+ return not _is_potential_local_target_url(new_url) or is_loopback_url(old_url)def _raise(error_type: type[ErrorT], message: str) -> NoReturn:@@ -75,15 +199,20 @@ def read_response_limited(explicit value so the intended bound is pinned at the call site rather thantracking changes to the shared default."""- chunks: list[bytes] = []+ if isinstance(max_bytes, bool) or not isinstance(max_bytes, int):+ raise TypeError("max_bytes must be an integer")+ if max_bytes < 0:+ raise ValueError("max_bytes must be non-negative")++ output = io.BytesIO()total = 0limit = max_bytes + 1while total < limit:chunk = response.read(min(READ_CHUNK_SIZE, limit - total))if not chunk:break- chunks.append(chunk)total += len(chunk)- if total > max_bytes:- _raise(error_type, f"{label} exceeds maximum size of {max_bytes} bytes")- return b"".join(chunks)+ if total > max_bytes:+ _raise(error_type, f"{label} exceeds maximum size of {max_bytes} bytes")+ output.write(chunk)+ return output.getvalue()diff --git a/src/specify_cli/authentication/azure_devops.py b/src/specify_cli/authentication/azure_devops.pyindex c475fa6fd..907f1b5e2 100644--- a/src/specify_cli/authentication/azure_devops.py+++ b/src/specify_cli/authentication/azure_devops.py@@ -22,6 +22,16 @@ class _TokenResponseTooLarge(Exception):"""Raised when an Azure AD token response exceeds the bounded read limit."""+def _extract_token(payload: object, key: str) -> str | None:+ """Return a normalized token from a JSON object, or None for other shapes."""+ if not isinstance(payload, dict):+ return None+ token = payload.get(key)+ if not isinstance(token, str):+ return None+ return token.strip() or None++class AzureDevOpsAuth(AuthProvider):"""Azure DevOps authentication provider.@@ -79,8 +89,7 @@ class AzureDevOpsAuth(AuthProvider):if result.returncode != 0:return Nonepayload = _json.loads(result.stdout)- token = payload.get("accessToken", "").strip()- return token or None+ return _extract_token(payload, "accessToken")except (OSError,subprocess.TimeoutExpired,@@ -146,8 +155,7 @@ class AzureDevOpsAuth(AuthProvider):label="Azure DevOps token response",).decode("utf-8"))- token = payload.get("access_token", "").strip()- return token or None+ return _extract_token(payload, "access_token")except (urllib.error.URLError,OSError,diff --git a/src/specify_cli/authentication/http.py b/src/specify_cli/authentication/http.pyindex 532779c1a..aa643c908 100644--- a/src/specify_cli/authentication/http.py+++ b/src/specify_cli/authentication/http.py@@ -17,7 +17,7 @@ from fnmatch import fnmatchfrom typing import Callablefrom urllib.parse import urlparse-from .._download_security import is_https_or_localhost_http, is_loopback_url+from .._download_security import is_safe_download_redirectfrom . import get_providerfrom .config import AuthConfigEntry, _default_config_path, find_entries_for_url, load_auth_config@@ -62,15 +62,11 @@ RedirectValidator = Callable[[str, str], None]def _validate_strict_redirect(old_url: str, new_url: str) -> None:- target_is_allowed = is_https_or_localhost_http(new_url)- remote_to_http_loopback = (- urlparse(new_url).scheme == "http"- and not is_loopback_url(old_url)- )- if not target_is_allowed or remote_to_http_loopback:+ if not is_safe_download_redirect(old_url, new_url):raise urllib.error.URLError(f"unsafe redirect to {new_url}: target must use HTTPS with a hostname, "- "or stay within localhost over HTTP (127.0.0.1, ::1)"+ "must not enter a local target from a remote host, and may use HTTP only "+ "within loopback (for example localhost, 127.0.0.1, ::1)")@@ -95,6 +91,8 @@ class _StripAuthOnRedirect(urllib.request.HTTPRedirectHandler):def redirect_request(self, req, fp, code, msg, headers, newurl):try:new_parsed = urlparse(newurl)+ # Force urllib's syntax and range validation before following.+ new_parsed.portexcept ValueError as exc:# Malformed redirect target (e.g. unterminated IPv6 bracket).# Surface as URLError so callers' download error handling applies.@@ -177,9 +175,11 @@ def open_url(*redirect_validator*, when provided, is called with ``(old_url, new_url)``before following each redirect and may raise to reject the redirect.+ Every attempt uses an isolated opener so a process-wide opener installed+ with ``urllib.request.install_opener`` cannot replace the redirect guard.Redirect scheme safety: every attempt goes through``_StripAuthOnRedirect``, which rejects redirects to non-HTTPS URLs except- HTTP between localhost / 127.0.0.1 / ::1 URLs.+ HTTP between loopback URLs, and rejects remote-to-local redirects."""entries = find_entries_for_url(url, _load_config())diff --git a/src/specify_cli/presets/_commands.py b/src/specify_cli/presets/_commands.pyindex ed8b93760..ba2b85b35 100644--- a/src/specify_cli/presets/_commands.py+++ b/src/specify_cli/presets/_commands.py@@ -16,6 +16,10 @@ import yamlfrom rich.markup import escape as _escape_markupfrom .._console import console+from .._download_security import (+ is_https_or_localhost_http,+ is_safe_download_redirect,+)preset_app = typer.Typer(name="preset",@@ -102,38 +106,25 @@ def preset_add(elif from_url:# Validate URL scheme before downloading- from ipaddress import ip_addressfrom urllib.parse import urlparse as _urlparsetry:_parsed = _urlparse(from_url)+ _parsed.portexcept ValueError:console.print(f"[red]Error:[/red] Invalid URL: {_escape_markup(from_url)}")raise typer.Exit(1)- def _is_allowed_download_url(parsed_url):- host = parsed_url.hostname- if not host:- return False- is_loopback = host == "localhost"- if not is_loopback:- try:- is_loopback = ip_address(host).is_loopback- except ValueError:- # Host is not an IP literal (e.g., a regular hostname); treat as non-loopback.- pass- return parsed_url.scheme == "https" or (parsed_url.scheme == "http" and is_loopback)-def _validate_download_redirect(old_url, new_url):- if not _is_allowed_download_url(_urlparse(new_url)):+ if not is_safe_download_redirect(old_url, new_url):import urllib.errorraise urllib.error.URLError(- "redirect target must use HTTPS with a hostname, "- "or HTTP for localhost/loopback"+ "redirect target must use HTTPS without entering a local "+ "target, or stay within loopback over HTTP")- if not _is_allowed_download_url(_parsed):+ if not is_https_or_localhost_http(from_url):console.print("[red]Error:[/red] URL must use HTTPS with a hostname, ""or HTTP for localhost/loopback."@@ -167,7 +158,7 @@ def preset_add(redirect_validator=_validate_download_redirect,) as response:final_url = response.geturl() if hasattr(response, "geturl") else from_url- if not _is_allowed_download_url(_urlparse(final_url)):+ if not is_https_or_localhost_http(final_url):console.print("[red]Error:[/red] Preset URL redirected to a disallowed URL: "f"{final_url}. Redirect targets must use HTTPS with a hostname, "diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.pyindex be8b731d5..459d0e740 100644--- a/src/specify_cli/workflows/_commands.py+++ b/src/specify_cli/workflows/_commands.py@@ -20,6 +20,10 @@ import yamlfrom rich.markup import escape as _escape_markupfrom .._console import console, err_console+from .._download_security import (+ is_https_or_localhost_http,+ is_safe_download_redirect,+)from .._project import _resolve_init_dir_overrideworkflow_app = typer.Typer(@@ -383,27 +387,12 @@ _RESERVED_WORKFLOW_IDS: frozenset[str] = frozenset({"overlays", "runs", "steps"}def _reject_insecure_download_redirect(old_url: str, new_url: str) -> None:"""Reject insecure redirects before they are followed."""import urllib.error- from ipaddress import ip_address- from urllib.parse import urlparse- def _is_loopback_http(url: str) -> bool:- parsed = urlparse(url)- if parsed.scheme != "http":- return False- host = parsed.hostname or ""- if host == "localhost":- return True- try:- return ip_address(host).is_loopback- except ValueError:- return False-- if urlparse(new_url).scheme == "https":- return- if _is_loopback_http(old_url) and _is_loopback_http(new_url):+ if is_safe_download_redirect(old_url, new_url):returnraise urllib.error.URLError(- "redirect target must use HTTPS; loopback HTTP may only redirect from loopback HTTP"+ "redirect target must use HTTPS without entering a local target; "+ "loopback HTTP may only redirect from another loopback URL")@@ -1579,24 +1568,15 @@ def workflow_add(else (source if source.startswith(("http://", "https://")) else None))if download_url is not None:- from ipaddress import ip_addressfrom urllib.parse import urlparsefrom specify_cli.authentication.http import open_url as _open_urltry:- parsed_src = urlparse(download_url)+ urlparse(download_url).portexcept ValueError:console.print(f"[red]Error:[/red] Invalid URL: {_escape_markup(download_url)}")raise typer.Exit(1)- src_host = parsed_src.hostname or ""- src_loopback = src_host == "localhost"- if not src_loopback:- try:- src_loopback = ip_address(src_host).is_loopback- except ValueError:- # Host is not an IP literal (e.g., a DNS name); keep default non-loopback.- pass- if parsed_src.scheme != "https" and not (parsed_src.scheme == "http" and src_loopback):+ if not is_https_or_localhost_http(download_url):console.print("[red]Error:[/red] Only HTTPS URLs are allowed, except HTTP for localhost.")raise typer.Exit(1)@@ -1647,16 +1627,7 @@ def workflow_add(redirect_validator=_reject_insecure_download_redirect,) as resp:final_url = resp.geturl()- final_parsed = urlparse(final_url)- final_host = final_parsed.hostname or ""- final_lb = final_host == "localhost"- if not final_lb:- try:- final_lb = ip_address(final_host).is_loopback- except ValueError:- # Redirect host is not an IP literal; keep loopback as determined above.- pass- if final_parsed.scheme != "https" and not (final_parsed.scheme == "http" and final_lb):+ if not is_https_or_localhost_http(final_url):console.print(f"[red]Error:[/red] URL redirected to non-HTTPS: {_escape_markup(final_url)}")@@ -1788,27 +1759,17 @@ def _install_workflow_from_catalog(raise typer.Exit(1)# Validate URL scheme (HTTPS required, HTTP allowed for localhost only)- from ipaddress import ip_addressfrom urllib.parse import urlparsetry:parsed_url = urlparse(workflow_url)- url_host = parsed_url.hostname or ""+ parsed_url.portexcept ValueError:console.print(f"[red]Error:[/red] Workflow '{safe_wf_id}' has a malformed install URL.")raise typer.Exit(1)- is_loopback = False- if url_host == "localhost":- is_loopback = True- else:- try:- is_loopback = ip_address(url_host).is_loopback- except ValueError:- # Host is not an IP literal (e.g., a regular hostname); treat as non-loopback.- pass- if parsed_url.scheme != "https" and not (parsed_url.scheme == "http" and is_loopback):+ if not is_https_or_localhost_http(workflow_url):console.print(f"[red]Error:[/red] Workflow '{safe_wf_id}' has an invalid install URL. ""Only HTTPS URLs are allowed, except HTTP for localhost/loopback."@@ -1862,16 +1823,7 @@ def _install_workflow_from_catalog() as response:# Validate final URL after redirectsfinal_url = response.geturl()- final_parsed = urlparse(final_url)- final_host = final_parsed.hostname or ""- final_loopback = final_host == "localhost"- if not final_loopback:- try:- final_loopback = ip_address(final_host).is_loopback- except ValueError:- # Host is not an IP literal (e.g., a regular hostname); treat as non-loopback.- pass- if final_parsed.scheme != "https" and not (final_parsed.scheme == "http" and final_loopback):+ if not is_https_or_localhost_http(final_url):_safe_discard_staged_workflow_file(staged_file, workflow_dir, existed_before)console.print(f"[red]Error:[/red] Workflow '{safe_wf_id}' redirected to non-HTTPS URL: {_escape_markup(final_url)}"@@ -2694,28 +2646,17 @@ def workflow_step_add()raise typer.Exit(1)- from urllib.parse import urlparsefrom specify_cli.authentication.http import open_url as _open_urldef _safe_fetch(url: str) -> bytes:- parsed = urlparse(url)- is_localhost = parsed.hostname in ("localhost", "127.0.0.1", "::1")- if parsed.scheme != "https" and not (parsed.scheme == "http" and is_localhost):+ if not is_https_or_localhost_http(url):raise ValueError(f"Refusing to fetch from non-HTTPS URL: {url}")- if not parsed.hostname:- raise ValueError(f"Refusing to fetch from URL with no hostname: {url}")with _open_url(url, timeout=30, redirect_validator=_reject_insecure_download_redirect) as resp:final_url = resp.geturl()- final_parsed = urlparse(final_url)- final_is_localhost = final_parsed.hostname in ("localhost", "127.0.0.1", "::1")- if final_parsed.scheme != "https" and not (- final_parsed.scheme == "http" and final_is_localhost- ):+ if not is_https_or_localhost_http(final_url):raise ValueError(f"Redirect to non-HTTPS URL: {final_url}")- if not final_parsed.hostname:- raise ValueError(f"Redirect to URL with no hostname: {final_url}")return _read_response_within_limit(resp)_validate_step_id_or_exit(step_id)diff --git a/tests/test_authentication.py b/tests/test_authentication.pyindex 727a9a37d..06523c6f2 100644--- a/tests/test_authentication.py+++ b/tests/test_authentication.py@@ -516,6 +516,23 @@ class TestAzureDevOpsAuth:with patch("specify_cli.authentication.azure_devops.subprocess.run", side_effect=boom):assert AzureDevOpsAuth().resolve_token(entry) is None+ @pytest.mark.parametrize("payload", [[], {"accessToken": None}, {"accessToken": 123}])+ def test_resolve_token_azure_cli_unexpected_json_shape_returns_none(+ self, payload+ ):+ from unittest.mock import MagicMock, patch++ entry = AuthConfigEntry(+ hosts=("dev.azure.com",), provider="azure-devops", auth="azure-cli",+ )+ result = MagicMock(returncode=0, stdout=json.dumps(payload))++ with patch(+ "specify_cli.authentication.azure_devops.subprocess.run",+ return_value=result,+ ):+ assert AzureDevOpsAuth().resolve_token(entry) is None+def test_resolve_token_azure_ad_success(self, monkeypatch):"""azure-ad acquires token via OAuth2 client credentials."""from unittest.mock import patch, MagicMock@@ -559,7 +576,13 @@ class TestAzureDevOpsAuth:with patch("urllib.request.build_opener", return_value=mock_opener):assert AzureDevOpsAuth().resolve_token(entry) is None- def test_resolve_token_azure_ad_rejects_https_redirect(self, monkeypatch):+ @pytest.mark.parametrize(+ ("status", "reason"),+ [(307, "Temporary Redirect"), (308, "Permanent Redirect")],+ )+ def test_resolve_token_azure_ad_rejects_https_redirect(+ self, monkeypatch, status, reason+ ):"""The client-secret POST must never be redirected to another host."""import urllib.errorfrom unittest.mock import MagicMock, patch@@ -577,17 +600,72 @@ class TestAzureDevOpsAuth:assert AzureDevOpsAuth().resolve_token(entry) is Noneredirect_handler = build_opener.call_args.args[0]- request = Request("https://login.microsoftonline.com/tid/oauth2/v2.0/token")+ request = Request(+ "https://login.microsoftonline.com/tid/oauth2/v2.0/token",+ data=b"grant_type=client_credentials&client_secret=secret-value",+ headers={"Content-Type": "application/x-www-form-urlencoded"},+ )+ assert request.get_method() == "POST"+ assert b"client_secret=secret-value" in request.datawith pytest.raises(urllib.error.URLError, match="must not be redirected"):redirect_handler.redirect_request(request,io.BytesIO(b""),- 307,- "Temporary Redirect",+ status,+ reason,{},"https://evil.example/token",)+ def test_resolve_token_azure_ad_oversized_response_returns_none(+ self, monkeypatch+ ):+ """Oversized token metadata is rejected before JSON parsing."""+ from unittest.mock import MagicMock, patch++ from specify_cli._download_security import MAX_JSON_METADATA_BYTES++ monkeypatch.setenv("MY_SECRET", "secret-value")+ entry = AuthConfigEntry(+ hosts=("dev.azure.com",), provider="azure-devops", auth="azure-ad",+ tenant_id="tid", client_id="cid", client_secret_env="MY_SECRET",+ )+ mock_resp = MagicMock()+ mock_resp.read.side_effect = io.BytesIO(+ b"x" * (MAX_JSON_METADATA_BYTES + 1)+ ).read+ mock_resp.__enter__ = lambda s: s+ mock_resp.__exit__ = MagicMock(return_value=False)+ mock_opener = MagicMock()+ mock_opener.open.return_value = mock_resp++ with patch("urllib.request.build_opener", return_value=mock_opener), patch(+ "specify_cli.authentication.azure_devops._json.loads",+ side_effect=AssertionError("oversized body must not be parsed"),+ ):+ assert AzureDevOpsAuth().resolve_token(entry) is None++ @pytest.mark.parametrize("payload", [[], {"access_token": None}, {"access_token": 123}])+ def test_resolve_token_azure_ad_unexpected_json_shape_returns_none(+ self, monkeypatch, payload+ ):+ from unittest.mock import MagicMock, patch++ monkeypatch.setenv("MY_SECRET", "secret-value")+ entry = AuthConfigEntry(+ hosts=("dev.azure.com",), provider="azure-devops", auth="azure-ad",+ tenant_id="tid", client_id="cid", client_secret_env="MY_SECRET",+ )+ mock_resp = MagicMock()+ mock_resp.read.side_effect = io.BytesIO(json.dumps(payload).encode()).read+ mock_resp.__enter__ = lambda s: s+ mock_resp.__exit__ = MagicMock(return_value=False)+ mock_opener = MagicMock()+ mock_opener.open.return_value = mock_resp++ with patch("urllib.request.build_opener", return_value=mock_opener):+ assert AzureDevOpsAuth().resolve_token(entry) is None+def test_resolve_token_azure_ad_invalid_utf8_returns_none(self, monkeypatch):"""azure-ad returns None when the token response is not valid UTF-8."""from unittest.mock import MagicMock, patch@@ -960,7 +1038,41 @@ class TestRedirectStripping:handler.redirect_request(req, io.BytesIO(b""), 302, "Found", {},"http://evil.example.com/archive.zip")- def test_redirect_rejects_remote_to_http_loopback(self):+ @pytest.mark.parametrize(+ "target",+ [+ "http://127.0.0.1/internal",+ "https://localhost/internal",+ "https://localhost./internal",+ "https://service.localhost/internal",+ "https://service.localhost./internal",+ "https://127.0.0.2/internal",+ "https://127.1/internal",+ "https://2130706433/internal",+ "https://0x7f000001/internal",+ "https://017700000001/internal",+ "https://0177.0.0.1/internal",+ "https://[::1]/internal",+ "https://[::1%25lo0]/internal",+ "https://[::ffff:127.0.0.1]/internal",+ "https://127%2e0%2e0%2e1/internal",+ "https://%31%32%37.0.0.1/internal",+ "https://127%2E1/internal",+ "https://local%68ost/internal",+ "https://[::ffff:127%2e0.0.1]/internal",+ "https://[::ffff:7f00%3a1]/internal",+ "https://[::ffff%3a127.0.0.1]/internal",+ "https://ℓocalhost/internal",+ "https://localhost/internal",+ "https://127。0。0。1/internal",+ "https://0.0.0.0/internal",+ "https://0/internal",+ "https://00.00.00.00/internal",+ "https://[::]/internal",+ "https://[::ffff:0.0.0.0]/internal",+ ],+ )+ def test_redirect_rejects_remote_to_loopback(self, target):"""A remote response must not redirect a download into loopback."""import ioimport urllib.error@@ -978,10 +1090,27 @@ class TestRedirectStripping:302,"Found",{},- "http://127.0.0.1/internal",+ target,)- def test_redirect_allows_loopback_to_http_loopback(self):+ @pytest.mark.parametrize(+ ("source", "target"),+ [+ (+ "http://localhost:8000/archive.zip",+ "http://127.0.0.1:8001/archive.zip",+ ),+ (+ "http://127.0.0.2:8000/archive.zip",+ "http://127.255.255.254:8001/archive.zip",+ ),+ (+ "https://[0:0:0:0:0:0:0:1]/archive.zip",+ "http://[::1]:8001/archive.zip",+ ),+ ],+ )+ def test_redirect_allows_loopback_to_http_loopback(self, source, target):"""Local development may continue redirecting between loopback URLs."""import iofrom urllib.request import Request@@ -989,18 +1118,66 @@ class TestRedirectStripping:from specify_cli.authentication.http import _StripAuthOnRedirecthandler = _StripAuthOnRedirect(())- request = Request("http://localhost:8000/archive.zip")+ request = Request(source)redirected = handler.redirect_request(request,io.BytesIO(b""),302,"Found",{},- "http://127.0.0.1:8001/archive.zip",+ target,)assert redirected is not None+ def test_multi_hop_remote_to_loopback_chain_is_rejected_at_first_hop(self):+ import io+ import urllib.error+ from urllib.request import Request++ from specify_cli.authentication.http import _StripAuthOnRedirect++ handler = _StripAuthOnRedirect(())+ request = Request("https://example.com/archive.zip")++ with pytest.raises(urllib.error.URLError, match="unsafe redirect"):+ handler.redirect_request(+ request,+ io.BytesIO(b""),+ 302,+ "Found",+ {},+ "https://localhost:4443/hop",+ )++ @pytest.mark.parametrize(+ "target",+ [+ "https://example.com:notaport/archive.zip",+ "https://example.com:+443/archive.zip",+ "https://example.com:65536/archive.zip",+ ],+ )+ def test_malformed_redirect_port_raises_urlerror(self, target):+ import io+ import urllib.error+ from urllib.request import Request++ from specify_cli.authentication.http import _StripAuthOnRedirect++ handler = _StripAuthOnRedirect(())+ request = Request("https://example.com/archive.zip")++ with pytest.raises(urllib.error.URLError, match="malformed redirect URL"):+ handler.redirect_request(+ request,+ io.BytesIO(b""),+ 302,+ "Found",+ {},+ target,+ )+def test_strict_redirect_error_describes_target_and_allowed_localhost(self):from specify_cli.authentication.http import _StripAuthOnRedirectfrom urllib.request import Requestdiff --git a/tests/test_download_security.py b/tests/test_download_security.pyindex 24eeedd95..f5dffe10d 100644--- a/tests/test_download_security.py+++ b/tests/test_download_security.py@@ -2,10 +2,13 @@from __future__ import annotations+import weakref+import pytestfrom specify_cli._download_security import (is_https_or_localhost_http,+ is_loopback_url,read_response_limited,)@@ -16,20 +19,93 @@ from specify_cli._download_security import (("https://example.com/preset.zip", True),("http://localhost:8000/preset.zip", True),("http://127.0.0.1/preset.zip", True),+ ("http://127.0.0.2/preset.zip", True),+ ("http://127.255.255.254/preset.zip", True),("http://[::1]/preset.zip", True),+ ("http://[0:0:0:0:0:0:0:1]/preset.zip", True),+ ("http://[::ffff:127.0.0.2]/preset.zip", True),+ ("http://[::1%25lo0]/preset.zip", True),# Non-loopback HTTP is rejected.("http://example.com/preset.zip", False),- # Loopback allowance is an exact-string match: 127.0.0.2 is not covered.- ("http://127.0.0.2/preset.zip", False),+ ("http://192.0.2.1/preset.zip", False),+ ("http://[fe80::1]/preset.zip", False),+ ("http://[fe80::1%25lo0]/preset.zip", False),+ ("http://0.0.0.0/preset.zip", False),+ ("http://0/preset.zip", False),+ ("http://[::]/preset.zip", False),+ ("http://[::ffff:0.0.0.0]/preset.zip", False),+ # Ambiguous/platform-dependent spellings may never authorize HTTP.+ ("http://127.1/preset.zip", False),+ ("http://2130706433/preset.zip", False),+ ("http://0x7f000001/preset.zip", False),+ ("http://017700000001/preset.zip", False),+ ("http://0177.0.0.1/preset.zip", False),+ ("http://00177.0.0.1/preset.zip", False),+ ("http://localhost./preset.zip", False),+ ("http://ℓocalhost/preset.zip", False),+ ("http://127。0。0。1/preset.zip", False),# A hostname is always required, even for HTTPS.("https:///preset.zip", False),("https://", False),+ # Invalid ports must be rejected before urllib opens the URL.+ ("https://example.com:notaport/preset.zip", False),+ ("https://example.com:+443/preset.zip", False),+ ("https://example.com:65536/preset.zip", False),+ # urllib decodes escapes in the authority before connecting; reject+ # encoded reg-names so validation and connection cannot disagree.+ ("https://127%2e0%2e0%2e1/preset.zip", False),+ ("https://%31%32%37.0.0.1/preset.zip", False),+ ("https://local%68ost/preset.zip", False),+ ("https://example.com%3a443/preset.zip", False),+ ("https://[::1%lo0]/preset.zip", False),+ ("https://[::ffff:127%2e0.0.1]/preset.zip", False),+ ("https://[::ffff:7f00%3a1]/preset.zip", False),+ ("https://[::ffff%3a127.0.0.1]/preset.zip", False),],)def test_is_https_or_localhost_http(url, allowed):assert is_https_or_localhost_http(url) is allowed+ "url",+ [+ "https://localhost/internal",+ "https://127.0.0.2/internal",+ "https://[::1]/internal",+ "https://[::1%25lo0]/internal",+ "https://[::ffff:127.0.0.2]/internal",+ ],+)+def test_is_loopback_url_recognizes_effective_loopback_literals(url):+ assert is_loopback_url(url) is True+++ "url",+ [+ "https://localhost./internal",+ "https://service.localhost/internal",+ "https://service.localhost./internal",+ "https://127.1/internal",+ "https://2130706433/internal",+ "https://0x7f000001/internal",+ "https://017700000001/internal",+ "https://0177.0.0.1/internal",+ "https://ℓocalhost/internal",+ "https://127。0。0。1/internal",+ "https://127%2e0%2e0%2e1/internal",+ "https://0.0.0.0/internal",+ "https://0/internal",+ "https://00.00.00.00/internal",+ "https://[::]/internal",+ "https://[::ffff:0.0.0.0]/internal",+ ],+)+def test_is_loopback_url_does_not_authorize_ambiguous_spellings(url):+ assert is_loopback_url(url) is False++class _Response:"""Faithful stream stand-in: read() advances a cursor and returns b"" at EOF."""@@ -50,6 +126,41 @@ class _Response:return out+class _RecordingResponse(_Response):+ def __init__(self, data: bytes, *, chunk: int | None = None):+ super().__init__(data, chunk=chunk)+ self.requested_sizes: list[int] = []++ def read(self, size: int = -1) -> bytes:+ self.requested_sizes.append(size)+ return super().read(size)+++class _TrackedChunk(bytearray):+ pass+++class _OneByteResponse:+ """Return distinct weak-referenceable chunks to detect retained fragments."""++ def __init__(self, count: int):+ self.remaining = count+ self.refs: list[weakref.ReferenceType[_TrackedChunk]] = []+ self.peak_live = 0++ def read(self, _size: int = -1) -> bytes | _TrackedChunk:+ if self.remaining == 0:+ return b""+ self.remaining -= 1+ chunk = _TrackedChunk(b"x")+ self.refs.append(weakref.ref(chunk))+ self.peak_live = max(+ self.peak_live,+ sum(ref() is not None for ref in self.refs),+ )+ return chunk++def test_read_response_limited_rejects_oversized_download():with pytest.raises(ValueError, match="exceeds maximum size"):read_response_limited(_Response(b"abcde"), max_bytes=4)@@ -66,3 +177,51 @@ def test_read_response_limited_enforces_bound_under_short_reads():response = _Response(b"x" * 100, chunk=8)with pytest.raises(ValueError, match="exceeds maximum size"):read_response_limited(response, max_bytes=16)+++def test_read_response_limited_does_not_retain_short_read_fragments():+ response = _OneByteResponse(64)++ assert read_response_limited(response, max_bytes=64) == b"x" * 64+ assert response.peak_live <= 2+++def test_read_response_limited_caps_underlying_reads_at_64_kib():+ response = _RecordingResponse(b"x" * (64 * 1024 + 1))++ with pytest.raises(ValueError, match="exceeds maximum size"):+ read_response_limited(response, max_bytes=64 * 1024)++ assert max(response.requested_sizes) <= 64 * 1024++[email protected]("value", [None, "1", 1.5, True])+def test_read_response_limited_rejects_non_integer_limits(value):+ with pytest.raises(TypeError, match="integer"):+ read_response_limited(_Response(b""), max_bytes=value)+++def test_read_response_limited_rejects_negative_limit_without_reading():+ response = _RecordingResponse(b"")++ with pytest.raises(ValueError, match="non-negative"):+ read_response_limited(response, max_bytes=-1)++ assert response.requested_sizes == []+++def test_read_response_limited_allows_empty_response_at_zero_limit():+ assert read_response_limited(_Response(b""), max_bytes=0) == b""+++class _CustomLimitError(Exception):+ pass+++def test_read_response_limited_rejects_first_byte_at_zero_limit():+ with pytest.raises(_CustomLimitError, match="exceeds maximum size"):+ read_response_limited(+ _Response(b"x"),+ max_bytes=0,+ error_type=_CustomLimitError,+ )diff --git a/tests/test_self_upgrade_guidance.py b/tests/test_self_upgrade_guidance.pyindex 55d6c2bf7..5fcb6a527 100644--- a/tests/test_self_upgrade_guidance.py+++ b/tests/test_self_upgrade_guidance.py@@ -6,6 +6,7 @@ from specify_cli import appfrom tests.self_upgrade_helpers import (mock_urlopen_response,+ route_opener_open_through_urlopen, # noqa: F401 (autouse fixture)runner,strip_ansi,)diff --git a/tests/test_upgrade.py b/tests/test_upgrade.pyindex b6f313428..edce927e8 100644--- a/tests/test_upgrade.py+++ b/tests/test_upgrade.py@@ -2,11 +2,12 @@Network isolation contract (SC-004 / FR-014): every test that exercises`specify self check` or `_fetch_latest_release_tag()` MUST mock the outbound-urllib path it expects (`urlopen` for unauthenticated requests, `build_opener`-for authenticated requests) so no real outbound call ever reaches api.github.com.-Tests for non-network `self upgrade` behavior should keep that contract explicit-with local mocks. Run this module under `pytest-socket` (if installed) with-`--disable-socket` as an extra safety net.+urllib path so no real call reaches api.github.com. Production always uses an+isolated `build_opener`; this module's autouse fixture routes its `open()` back+through the locally mocked `urlopen`. Tests for non-network `self upgrade`+behavior should keep that contract explicit with local mocks. Run this module+under `pytest-socket` (if installed) with `--disable-socket` as an extra safety+net."""import urllib.errordiff --git a/tests/test_workflows.py b/tests/test_workflows.pyindex b83514ef9..2c26301cc 100644--- a/tests/test_workflows.py+++ b/tests/test_workflows.py@@ -11744,6 +11744,10 @@ steps:_reject_insecure_download_redirect("https://example.com/wf.yml", "http://localhost:8000/wf.yml")+ with pytest.raises(urllib.error.URLError):+ _reject_insecure_download_redirect(+ "https://example.com/wf.yml", "https://127.0.0.2/wf.yml"+ )# Allowed: HTTPS anywhere, or loopback HTTP that stays on loopback HTTP._reject_insecure_download_redirect("https://example.com/wf.yml", "https://cdn.example.com/wf.yml"@@ -11754,6 +11758,9 @@ steps:_reject_insecure_download_redirect("http://127.0.0.1/source.yml", "http://127.0.0.1/wf.yml")+ _reject_insecure_download_redirect(+ "http://127.0.0.2/source.yml", "http://127.255.255.254/wf.yml"+ )def test_add_from_url_passes_redirect_validator(self, project_dir, monkeypatch):from unittest.mock import patch