π« Toolkit to help you get started with SDD or any other process!
fix(authentication): support GHE.com release asset downloads (#4807)
Manfred Riem6244a2ebc404
diff --git a/docs/reference/authentication.md b/docs/reference/authentication.mdindex bf82eff10..9e0ff2144 100644--- a/docs/reference/authentication.md+++ b/docs/reference/authentication.md@@ -98,6 +98,31 @@ and `codeload.` subdomains your catalog/extension URLs use. A`*.ghes.example.com` wildcard matches subdomains but **not** the bare host,so always include the bare host explicitly.+### GitHub Enterprise Cloud with data residency (GHE.com)++GHE.com tenants use separate web and REST API hostnames. List both hosts so+Specify can authenticate the release-metadata lookup and asset download:++```json+{+ "providers": [+ {+ "hosts": ["tenant.ghe.com", "api.tenant.ghe.com"],+ "provider": "github",+ "auth": "bearer",+ "token_env": "GH_ENTERPRISE_TOKEN"+ }+ ]+}+```++For a release URL on `tenant.ghe.com`, Specify resolves metadata and release+assets through `api.tenant.ghe.com`. The returned asset must identify the same+owner and repository and use the exact numeric+`/repos/.../releases/assets/...` endpoint. A wildcard such as+`*.tenant.ghe.com` matches the API hostname but not the bare tenant web+hostname, so list the web hostname explicitly.+### Azure DevOps (`azure-devops`)| Scheme | Header | Use for |diff --git a/src/specify_cli/authentication/github_http.py b/src/specify_cli/authentication/github_http.pyindex 402a147d6..d05c312ab 100644--- a/src/specify_cli/authentication/github_http.py+++ b/src/specify_cli/authentication/github_http.py@@ -25,6 +25,8 @@ GITHUB_HOSTS = frozenset({"api.github.com","codeload.github.com",})+_GHE_COM_SUFFIX = ".ghe.com"+_GHE_COM_API_PREFIX = "api."_MAX_RELEASE_METADATA_BYTES = 5 * 1024 * 1024@@ -41,6 +43,18 @@ def _has_valid_percent_escapes(value: str) -> bool:return True+def _has_valid_asset_url_spelling(value: str) -> bool:+ """Return whether an asset URL uses an unambiguous raw spelling."""+ return (+ not any(+ ord(character) <= 0x20 or ord(character) == 0x7F+ for character in value+ )+ and not any(delimiter in value for delimiter in ("?", "#", ";"))+ and _has_valid_percent_escapes(value)+ )++def build_github_request(url: str) -> urllib.request.Request:"""Build a urllib Request, adding a GitHub auth header when available.@@ -81,6 +95,27 @@ def _host_matches(hostname: str, patterns: tuple[str, ...]) -> bool:return any(p == hostname or fnmatch(hostname, p) for p in patterns)+def _ghe_com_api_hostname(web_hostname: str) -> str | None:+ """Return the paired GHE.com API hostname for a tenant web hostname."""+ if (+ web_hostname == "ghe.com"+ or web_hostname.startswith(_GHE_COM_API_PREFIX)+ or not web_hostname.endswith(_GHE_COM_SUFFIX)+ ):+ return None+ return f"{_GHE_COM_API_PREFIX}{web_hostname}"+++def _ghe_com_web_hostname(api_hostname: str) -> str | None:+ """Return the paired GHE.com web hostname for a tenant API hostname."""+ if not api_hostname.startswith(_GHE_COM_API_PREFIX):+ return None+ web_hostname = api_hostname.removeprefix(_GHE_COM_API_PREFIX)+ if web_hostname == "ghe.com" or not web_hostname.endswith(_GHE_COM_SUFFIX):+ return None+ return web_hostname++def resolve_github_release_asset_api_url(download_url: str,open_url_fn: Callable,@@ -91,18 +126,20 @@ def resolve_github_release_asset_api_url() -> Optional[str]:"""Resolve a GitHub release browser-download URL to its REST API asset URL.- Works for public ``github.com`` and for GitHub Enterprise Server (GHES)- hosts. A host is treated as GHES when it matches one of *github_hosts*- (exact hostname or ``*.suffix``) β supply the hosts the user has trusted- under a ``github`` provider in ``auth.json``. This allowlist is the- security gate: unlisted hosts never receive GHES API treatment, so a- malicious catalog cannot induce an API request to an arbitrary host.+ Works for public ``github.com``, GitHub Enterprise Cloud with data+ residency (GHE.com), and GitHub Enterprise Server (GHES). Enterprise hosts+ must match *github_hosts* (exact hostname or ``*.suffix``), which should be+ the hosts the user trusted under a ``github`` provider in ``auth.json``.+ GHE.com additionally requires both the tenant web hostname and its paired+ ``api.`` hostname to be trusted.- For a public URL the API base is ``https://api.github.com``; for a GHES- host it is ``{scheme}://{host[:port]}/api/v3``. Returns the API asset URL+ Public GitHub uses ``https://api.github.com``; a GHE.com tenant+ ``tenant.ghe.com`` uses ``https://api.tenant.ghe.com``; GHES uses+ ``{scheme}://{host[:port]}/api/v3``. Returns the API asset URL(downloadable with ``Accept: application/octet-stream`` + a token), the- input unchanged if it is already an API asset URL, or ``None`` when the- URL is not a resolvable GitHub release download or the lookup fails.+ input unchanged if it is already a recognized API asset URL, or ``None``+ when the URL is not a resolvable GitHub release download or the lookup+ fails.Args:download_url: The URL to resolve.@@ -110,7 +147,7 @@ def resolve_github_release_asset_api_url(:func:`specify_cli.authentication.http.open_url` used for theauthenticated release-metadata lookup.timeout: Per-request timeout in seconds.- github_hosts: Host patterns to treat as GitHub Enterprise Server.+ github_hosts: Host patterns trusted as GitHub Enterprise deployments.redirect_validator: Optional policy applied to metadata redirects.max_metadata_bytes: Maximum release-metadata response size."""@@ -128,13 +165,23 @@ def resolve_github_release_asset_api_url(try:parsed = urlparse(download_url)hostname = (parsed.hostname or "").lower()+ parsed_port = parsed.portexcept ValueError:return Noneparts = [unquote(part) for part in parsed.path.strip("/").split("/")]+ ghe_com_api_hostname = _ghe_com_api_hostname(hostname)+ is_ghe_com = (+ ghe_com_api_hostname is not None+ and parsed.scheme == "https"+ and parsed_port in (None, 443)+ and _host_matches(hostname, github_hosts)+ and _host_matches(ghe_com_api_hostname, github_hosts)+ )is_ghes = (bool(hostname)and hostname not in GITHUB_HOSTS+ and not hostname.endswith(_GHE_COM_SUFFIX)and _host_matches(hostname, github_hosts))@@ -145,15 +192,46 @@ def resolve_github_release_asset_api_url(and segments[3:5] == ["releases", "assets"])- # Already a REST API asset URL β use it directly. Pure passthrough induces- # no new request: the caller fetches this same URL regardless, so it is- # gated on path shape alone rather than the GHES allowlist. The token stays- # independently gated by auth.json in the download helper, and only the- # resolving path below (which issues a tag-lookup request) needs the- # allowlist as its anti-SSRF gate.+ def _is_exact_raw_asset_path(path: str) -> bool:+ segments = path.split("/")+ return (+ len(segments) == 7+ and segments[:2] == ["", "repos"]+ and bool(segments[2])+ and bool(segments[3])+ and segments[4:6] == ["releases", "assets"]+ and segments[-1].isascii()+ and segments[-1].isdigit()+ )++ # Already a REST API asset URL β use it directly. Existing GitHub.com and+ # GHES passthrough behavior remains path-gated because it induces no new+ # request; the caller would fetch the same URL regardless. GHE.com is new+ # here and uses its stricter tenant-pair trust check below.if hostname == "api.github.com" and _is_asset_path(parts):return download_url- if hostname and parts[:2] == ["api", "v3"] and _is_asset_path(parts[2:]):+ if (+ hostname+ and not hostname.endswith(_GHE_COM_SUFFIX)+ and parts[:2] == ["api", "v3"]+ and _is_asset_path(parts[2:])+ ):+ return download_url+ ghe_com_web_hostname = _ghe_com_web_hostname(hostname)+ if (+ ghe_com_web_hostname is not None+ and _has_valid_asset_url_spelling(download_url)+ and parsed.scheme == "https"+ and parsed_port in (None, 443)+ and parsed.username is None+ and parsed.password is None+ and not parsed.query+ and not parsed.fragment+ and not parsed.params+ and _host_matches(hostname, github_hosts)+ and _host_matches(ghe_com_web_hostname, github_hosts)+ and _is_exact_raw_asset_path(parsed.path)+ ):return download_url# Browser download URLs must be usable HTTP(S) URLs before they can cause@@ -161,27 +239,23 @@ def resolve_github_release_asset_api_url(# retains its existing path-only behavior.if parsed.scheme not in {"http", "https"}:return None- try:- _browser_port = parsed.port- except ValueError:- return None# Determine the REST API base for browser release-download URLs.if hostname == "github.com":api_base = "https://api.github.com"+ expected_asset_prefix = ["", "repos"]+ elif is_ghe_com:+ api_base = f"https://{ghe_com_api_hostname}"+ expected_asset_prefix = ["", "repos"]elif is_ghes:- # ``parsed.port`` raises ValueError on a malformed port (e.g.- # ``host:notaport``); the function's contract is to return None for- # anything it can't resolve, not to raise.- try:- port = parsed.port- except ValueError:- return None# ``urlparse().hostname`` removes IPv6 brackets. Restore them when# constructing an authority so the derived API base remains a URL.authority_host = f"[{hostname}]" if ":" in hostname else hostname- authority = authority_host if port is None else f"{authority_host}:{port}"+ authority = (+ authority_host if parsed_port is None else f"{authority_host}:{parsed_port}"+ )api_base = f"{parsed.scheme}://{authority}/api/v3"+ expected_asset_prefix = ["", "api", "v3", "repos"]else:return None@@ -202,14 +276,7 @@ def resolve_github_release_asset_api_url(# ``urlparse`` tolerates some raw spellings (for example whitespace)# even though the original metadata value is returned to the caller.# Reject those spellings before parsing rather than normalizing them.- if (- any(- ord(character) <= 0x20 or ord(character) == 0x7F- for character in asset_url- )- or any(delimiter in asset_url for delimiter in ("?", "#", ";"))- or not _has_valid_percent_escapes(asset_url)- ):+ if not _has_valid_asset_url_spelling(asset_url):return Falsetry:asset_parsed = urlparse(asset_url)@@ -250,15 +317,10 @@ def resolve_github_release_asset_api_url(return Falseasset_parts = asset_parsed.path.split("/")- owner_index = 2 if api_base == "https://api.github.com" else 4- expected_prefix = (- ["", "repos"]- if api_base == "https://api.github.com"- else ["", "api", "v3", "repos"]- )+ owner_index = len(expected_asset_prefix)return (len(asset_parts) == owner_index + 5- and asset_parts[:owner_index] == expected_prefix+ and asset_parts[:owner_index] == expected_asset_prefixand unquote(asset_parts[owner_index]).casefold() == owner.casefold()and unquote(asset_parts[owner_index + 1]).casefold() == repo.casefold()and asset_parts[owner_index + 2:owner_index + 4] == ["releases", "assets"]diff --git a/src/specify_cli/authentication/http.py b/src/specify_cli/authentication/http.pyindex 32a6ed67c..7dabc752a 100644--- a/src/specify_cli/authentication/http.py+++ b/src/specify_cli/authentication/http.py@@ -154,9 +154,9 @@ def build_request(url: str, extra_headers: dict[str, str] | None = None) -> urlldef github_provider_hosts() -> tuple[str, ...]:"""Return host patterns from every ``github`` provider entry in ``auth.json``.- Used to classify which hosts are GitHub Enterprise Server instances when- resolving release-asset download URLs. Returns an empty tuple when no- ``auth.json`` exists or it contains no ``github`` entries.+ Used to classify trusted GitHub Enterprise Cloud and GitHub Enterprise+ Server hosts when resolving release-asset download URLs. Returns an empty+ tuple when no ``auth.json`` exists or it contains no ``github`` entries."""hosts: list[str] = []for entry in _load_config():diff --git a/tests/specify_cli/authentication/test_github_http.py b/tests/specify_cli/authentication/test_github_http.pyindex ad6440f79..8eae8d0a0 100644--- a/tests/specify_cli/authentication/test_github_http.py+++ b/tests/specify_cli/authentication/test_github_http.py@@ -315,6 +315,151 @@ class TestResolveGitHubReleaseAssetApiUrl:assert len(captured_urls) == 1assert "releases/tags/v1%23beta" in captured_urls[0]+ # --- GHE.com (GitHub Enterprise Cloud with data residency) ---++ def test_resolves_ghecom_browser_url_to_api_url(self):+ """A GHE.com browser URL resolves through its paired API subdomain."""+ asset_url = "https://api.msft.ghe.com/repos/org/repo/releases/assets/42"+ captured = []++ @contextmanager+ def capturing_open(url, timeout=None, extra_headers=None):+ captured.append(url)+ resp = MagicMock()+ resp.read.side_effect = io.BytesIO(+ json.dumps(+ {"assets": [{"name": "bundle.zip", "url": asset_url}]}+ ).encode()+ ).read+ yield resp++ result = resolve_github_release_asset_api_url(+ "https://msft.ghe.com/org/repo/releases/download/v1.0/bundle.zip",+ capturing_open,+ github_hosts=("msft.ghe.com", "api.msft.ghe.com"),+ )++ assert result == asset_url+ assert captured == [+ "https://api.msft.ghe.com/repos/org/repo/releases/tags/v1.0"+ ]++ def test_passthrough_for_trusted_ghecom_api_asset_url(self):+ """A trusted direct GHE.com API asset URL receives asset treatment."""+ url = "https://api.msft.ghe.com/repos/org/repo/releases/assets/42"+ result = resolve_github_release_asset_api_url(+ url,+ lambda *a, **kw: None,+ github_hosts=("msft.ghe.com", "api.msft.ghe.com"),+ )+ assert result == url++ @pytest.mark.parametrize(+ "url",+ [+ "https://api.msft.ghe.com/repos/org/repo/releases/assets/42?",+ "https://api.msft.ghe.com/repos/org/repo/releases/assets/42#",+ "https://api.msft.ghe.com/repos/org/repo/releases/assets/42/",+ "https://api.msft.ghe.com/repos/org/repo/releases/assets/%34%32",+ "https://api.msft.ghe.com/repos/org/repo/releases/assets/42 ",+ "https://api.msft.ghe.com/repos/org/repo/releases/assets/42;",+ "https://api.msft.ghe.com/repos/org/repo/releases/assets/%ZZ",+ ],+ )+ def test_rejects_noncanonical_direct_ghecom_api_asset_url(self, url):+ """Direct GHE.com asset URLs retain strict raw-spelling validation."""+ result = resolve_github_release_asset_api_url(+ url,+ lambda *a, **kw: None,+ github_hosts=("msft.ghe.com", "api.msft.ghe.com"),+ )+ assert result is None++ def test_rejects_direct_ghecom_api_asset_url_without_trusted_pair(self):+ """A direct GHE.com API URL needs both trusted tenant hostnames."""+ url = "https://api.msft.ghe.com/repos/org/repo/releases/assets/42"+ result = resolve_github_release_asset_api_url(+ url,+ lambda *a, **kw: None,+ github_hosts=("api.msft.ghe.com",),+ )+ assert result is None++ @pytest.mark.parametrize(+ "github_hosts",+ [+ (),+ ("msft.ghe.com", "api.msft.ghe.com"),+ ],+ )+ def test_rejects_direct_ghecom_api_asset_url_with_ghes_path(+ self, github_hosts+ ):+ """A GHE.com API host never accepts the legacy GHES /api/v3 path."""+ url = (+ "https://api.msft.ghe.com/api/v3/repos/org/repo/"+ "releases/assets/42"+ )+ result = resolve_github_release_asset_api_url(+ url,+ lambda *a, **kw: None,+ github_hosts=github_hosts,+ )+ assert result is None++ @pytest.mark.parametrize(+ "github_hosts",+ [+ ("msft.ghe.com",),+ ("api.msft.ghe.com",),+ ("other.ghe.com", "api.other.ghe.com"),+ ],+ )+ def test_ghecom_resolution_requires_trusted_web_and_api_hosts(self, github_hosts):+ """GHE.com resolution requires both members of the tenant host pair."""+ called = []++ @contextmanager+ def recording_open(url, timeout=None, extra_headers=None):+ called.append(url)+ resp = MagicMock()+ resp.read.side_effect = io.BytesIO(b"{}").read+ yield resp++ result = resolve_github_release_asset_api_url(+ "https://msft.ghe.com/org/repo/releases/download/v1.0/bundle.zip",+ recording_open,+ github_hosts=github_hosts,+ )++ assert result is None+ assert called == []++ @pytest.mark.parametrize(+ "asset_url",+ [+ "https://api.other.ghe.com/repos/org/repo/releases/assets/42",+ "http://api.msft.ghe.com/repos/org/repo/releases/assets/42",+ "https://msft.ghe.com/api/v3/repos/org/repo/releases/assets/42",+ "https://api.msft.ghe.com/repos/other/repo/releases/assets/42",+ "https://api.msft.ghe.com/repos/org/other/releases/assets/42",+ "https://api.msft.ghe.com/repos/org/repo/releases/assets/not-a-number",+ "https://api.msft.ghe.com/repos/org/repo/releases/assets/42?download=1",+ ],+ )+ def test_rejects_wrong_origin_or_path_for_ghecom_metadata_asset_url(+ self, asset_url+ ):+ """GHE.com metadata must identify the paired tenant and repository."""+ result = resolve_github_release_asset_api_url(+ "https://msft.ghe.com/org/repo/releases/download/v1.0/bundle.zip",+ self._make_open_url_fn(+ {"assets": [{"name": "bundle.zip", "url": asset_url}]}+ ),+ github_hosts=("msft.ghe.com", "api.msft.ghe.com"),+ )+ assert result is None+# --- GHES (GitHub Enterprise Server) ---def test_resolves_ghes_browser_url_to_api_url(self):diff --git a/tests/specify_cli/bundles/test_command_info.py b/tests/specify_cli/bundles/test_command_info.pyindex 6d9f6905a..673e004ea 100644--- a/tests/specify_cli/bundles/test_command_info.py+++ b/tests/specify_cli/bundles/test_command_info.py@@ -578,6 +578,60 @@ def test_bundle_info_resolves_ghes_browser_release_url(project: Path):assert payload["id"] == "demo-bundle"+def test_bundle_info_resolves_ghecom_browser_release_url_zip(project: Path):+ """bundle info resolves a GHE.com release ZIP through its API subdomain."""+ import zipfile++ web_host = "msft.ghe.com"+ api_host = f"api.{web_host}"+ browser_url = f"https://{web_host}/org/repo/releases/download/v2.0/bundle.zip"+ api_asset_url = f"https://{api_host}/repos/org/repo/releases/assets/42"++ archive = io.BytesIO()+ with zipfile.ZipFile(archive, "w") as zf:+ zf.writestr("bundle.yml", yaml.safe_dump(valid_manifest_dict()))+ zip_bytes = archive.getvalue()+ captured = []++ def fake_open_url(url, timeout=None, extra_headers=None, redirect_validator=None):+ captured.append((url, extra_headers))+ if "releases/tags/" in url:+ return FakeBundleResponse(+ json.dumps(+ {"assets": [{"name": "bundle.zip", "url": api_asset_url}]}+ ).encode(),+ url=url,+ )+ return FakeBundleResponse(zip_bytes, url=api_asset_url)++ catalog = project / "catalog.json"+ write_catalog_file(+ catalog,+ {"demo-bundle": catalog_entry_dict("demo-bundle", download_url=browser_url)},+ )+ _make_catalog_config(catalog, project)++ with (+ patch("specify_cli.authentication.http.open_url", side_effect=fake_open_url),+ patch(+ "specify_cli.authentication.http.github_provider_hosts",+ return_value=(web_host, api_host),+ ),+ ):+ result = runner.invoke(app, ["bundle", "info", "demo-bundle", "--json"])++ assert result.exit_code == 0, result.output++ tag_calls = [url for url, _ in captured if "releases/tags/" in url]+ assert tag_calls == [f"https://{api_host}/repos/org/repo/releases/tags/v2.0"]++ asset_calls = [(url, headers) for url, headers in captured if url == api_asset_url]+ assert asset_calls == [(api_asset_url, {"Accept": "application/octet-stream"})]++ payload = json.loads(result.output)+ assert payload["id"] == "demo-bundle"++def test_bundle_download_rejects_oversized_response(project: Path, monkeypatch):"""Bundle download rejects responses exceeding MAX_DOWNLOAD_BYTES."""# Monkeypatch to a small limit so the test is fast and low-memory.