Skip to content

πŸ’« Toolkit to help you get started with SDD or any other process!

fix(authentication): support GHE.com release asset downloads (#4807)

Manfred Riem6244a2ebc404
diff --git a/docs/reference/authentication.md b/docs/reference/authentication.md
index bf82eff10..9e0ff2144 100644
--- a/docs/reference/authentication.md
+++ b/docs/reference/authentication.md
@@ -98,6 +98,31 @@ and `codeload.` subdomains your catalog/extension URLs use. A
`*.ghes.example.com` wildcard matches subdomains but **not** the bare host,
so always include the bare host explicitly.
+### GitHub Enterprise Cloud with data residency (GHE.com)
+
+GHE.com tenants use separate web and REST API hostnames. List both hosts so
+Specify can authenticate the release-metadata lookup and asset download:
+
+```json
+{
+ "providers": [
+ {
+ "hosts": ["tenant.ghe.com", "api.tenant.ghe.com"],
+ "provider": "github",
+ "auth": "bearer",
+ "token_env": "GH_ENTERPRISE_TOKEN"
+ }
+ ]
+}
+```
+
+For a release URL on `tenant.ghe.com`, Specify resolves metadata and release
+assets through `api.tenant.ghe.com`. The returned asset must identify the same
+owner and repository and use the exact numeric
+`/repos/.../releases/assets/...` endpoint. A wildcard such as
+`*.tenant.ghe.com` matches the API hostname but not the bare tenant web
+hostname, so list the web hostname explicitly.
+
### Azure DevOps (`azure-devops`)
| Scheme | Header | Use for |
diff --git a/src/specify_cli/authentication/github_http.py b/src/specify_cli/authentication/github_http.py
index 402a147d6..d05c312ab 100644
--- a/src/specify_cli/authentication/github_http.py
+++ b/src/specify_cli/authentication/github_http.py
@@ -25,6 +25,8 @@ GITHUB_HOSTS = frozenset({
"api.github.com",
"codeload.github.com",
})
+_GHE_COM_SUFFIX = ".ghe.com"
+_GHE_COM_API_PREFIX = "api."
_MAX_RELEASE_METADATA_BYTES = 5 * 1024 * 1024
@@ -41,6 +43,18 @@ def _has_valid_percent_escapes(value: str) -> bool:
return True
+def _has_valid_asset_url_spelling(value: str) -> bool:
+ """Return whether an asset URL uses an unambiguous raw spelling."""
+ return (
+ not any(
+ ord(character) <= 0x20 or ord(character) == 0x7F
+ for character in value
+ )
+ and not any(delimiter in value for delimiter in ("?", "#", ";"))
+ and _has_valid_percent_escapes(value)
+ )
+
+
def build_github_request(url: str) -> urllib.request.Request:
"""Build a urllib Request, adding a GitHub auth header when available.
@@ -81,6 +95,27 @@ def _host_matches(hostname: str, patterns: tuple[str, ...]) -> bool:
return any(p == hostname or fnmatch(hostname, p) for p in patterns)
+def _ghe_com_api_hostname(web_hostname: str) -> str | None:
+ """Return the paired GHE.com API hostname for a tenant web hostname."""
+ if (
+ web_hostname == "ghe.com"
+ or web_hostname.startswith(_GHE_COM_API_PREFIX)
+ or not web_hostname.endswith(_GHE_COM_SUFFIX)
+ ):
+ return None
+ return f"{_GHE_COM_API_PREFIX}{web_hostname}"
+
+
+def _ghe_com_web_hostname(api_hostname: str) -> str | None:
+ """Return the paired GHE.com web hostname for a tenant API hostname."""
+ if not api_hostname.startswith(_GHE_COM_API_PREFIX):
+ return None
+ web_hostname = api_hostname.removeprefix(_GHE_COM_API_PREFIX)
+ if web_hostname == "ghe.com" or not web_hostname.endswith(_GHE_COM_SUFFIX):
+ return None
+ return web_hostname
+
+
def resolve_github_release_asset_api_url(
download_url: str,
open_url_fn: Callable,
@@ -91,18 +126,20 @@ def resolve_github_release_asset_api_url(
) -> Optional[str]:
"""Resolve a GitHub release browser-download URL to its REST API asset URL.
- Works for public ``github.com`` and for GitHub Enterprise Server (GHES)
- hosts. A host is treated as GHES when it matches one of *github_hosts*
- (exact hostname or ``*.suffix``) β€” supply the hosts the user has trusted
- under a ``github`` provider in ``auth.json``. This allowlist is the
- security gate: unlisted hosts never receive GHES API treatment, so a
- malicious catalog cannot induce an API request to an arbitrary host.
+ Works for public ``github.com``, GitHub Enterprise Cloud with data
+ residency (GHE.com), and GitHub Enterprise Server (GHES). Enterprise hosts
+ must match *github_hosts* (exact hostname or ``*.suffix``), which should be
+ the hosts the user trusted under a ``github`` provider in ``auth.json``.
+ GHE.com additionally requires both the tenant web hostname and its paired
+ ``api.`` hostname to be trusted.
- For a public URL the API base is ``https://api.github.com``; for a GHES
- host it is ``{scheme}://{host[:port]}/api/v3``. Returns the API asset URL
+ Public GitHub uses ``https://api.github.com``; a GHE.com tenant
+ ``tenant.ghe.com`` uses ``https://api.tenant.ghe.com``; GHES uses
+ ``{scheme}://{host[:port]}/api/v3``. Returns the API asset URL
(downloadable with ``Accept: application/octet-stream`` + a token), the
- input unchanged if it is already an API asset URL, or ``None`` when the
- URL is not a resolvable GitHub release download or the lookup fails.
+ input unchanged if it is already a recognized API asset URL, or ``None``
+ when the URL is not a resolvable GitHub release download or the lookup
+ fails.
Args:
download_url: The URL to resolve.
@@ -110,7 +147,7 @@ def resolve_github_release_asset_api_url(
:func:`specify_cli.authentication.http.open_url` used for the
authenticated release-metadata lookup.
timeout: Per-request timeout in seconds.
- github_hosts: Host patterns to treat as GitHub Enterprise Server.
+ github_hosts: Host patterns trusted as GitHub Enterprise deployments.
redirect_validator: Optional policy applied to metadata redirects.
max_metadata_bytes: Maximum release-metadata response size.
"""
@@ -128,13 +165,23 @@ def resolve_github_release_asset_api_url(
try:
parsed = urlparse(download_url)
hostname = (parsed.hostname or "").lower()
+ parsed_port = parsed.port
except ValueError:
return None
parts = [unquote(part) for part in parsed.path.strip("/").split("/")]
+ ghe_com_api_hostname = _ghe_com_api_hostname(hostname)
+ is_ghe_com = (
+ ghe_com_api_hostname is not None
+ and parsed.scheme == "https"
+ and parsed_port in (None, 443)
+ and _host_matches(hostname, github_hosts)
+ and _host_matches(ghe_com_api_hostname, github_hosts)
+ )
is_ghes = (
bool(hostname)
and hostname not in GITHUB_HOSTS
+ and not hostname.endswith(_GHE_COM_SUFFIX)
and _host_matches(hostname, github_hosts)
)
@@ -145,15 +192,46 @@ def resolve_github_release_asset_api_url(
and segments[3:5] == ["releases", "assets"]
)
- # Already a REST API asset URL β€” use it directly. Pure passthrough induces
- # no new request: the caller fetches this same URL regardless, so it is
- # gated on path shape alone rather than the GHES allowlist. The token stays
- # independently gated by auth.json in the download helper, and only the
- # resolving path below (which issues a tag-lookup request) needs the
- # allowlist as its anti-SSRF gate.
+ def _is_exact_raw_asset_path(path: str) -> bool:
+ segments = path.split("/")
+ return (
+ len(segments) == 7
+ and segments[:2] == ["", "repos"]
+ and bool(segments[2])
+ and bool(segments[3])
+ and segments[4:6] == ["releases", "assets"]
+ and segments[-1].isascii()
+ and segments[-1].isdigit()
+ )
+
+ # Already a REST API asset URL β€” use it directly. Existing GitHub.com and
+ # GHES passthrough behavior remains path-gated because it induces no new
+ # request; the caller would fetch the same URL regardless. GHE.com is new
+ # here and uses its stricter tenant-pair trust check below.
if hostname == "api.github.com" and _is_asset_path(parts):
return download_url
- if hostname and parts[:2] == ["api", "v3"] and _is_asset_path(parts[2:]):
+ if (
+ hostname
+ and not hostname.endswith(_GHE_COM_SUFFIX)
+ and parts[:2] == ["api", "v3"]
+ and _is_asset_path(parts[2:])
+ ):
+ return download_url
+ ghe_com_web_hostname = _ghe_com_web_hostname(hostname)
+ if (
+ ghe_com_web_hostname is not None
+ and _has_valid_asset_url_spelling(download_url)
+ and parsed.scheme == "https"
+ and parsed_port in (None, 443)
+ and parsed.username is None
+ and parsed.password is None
+ and not parsed.query
+ and not parsed.fragment
+ and not parsed.params
+ and _host_matches(hostname, github_hosts)
+ and _host_matches(ghe_com_web_hostname, github_hosts)
+ and _is_exact_raw_asset_path(parsed.path)
+ ):
return download_url
# Browser download URLs must be usable HTTP(S) URLs before they can cause
@@ -161,27 +239,23 @@ def resolve_github_release_asset_api_url(
# retains its existing path-only behavior.
if parsed.scheme not in {"http", "https"}:
return None
- try:
- _browser_port = parsed.port
- except ValueError:
- return None
# Determine the REST API base for browser release-download URLs.
if hostname == "github.com":
api_base = "https://api.github.com"
+ expected_asset_prefix = ["", "repos"]
+ elif is_ghe_com:
+ api_base = f"https://{ghe_com_api_hostname}"
+ expected_asset_prefix = ["", "repos"]
elif is_ghes:
- # ``parsed.port`` raises ValueError on a malformed port (e.g.
- # ``host:notaport``); the function's contract is to return None for
- # anything it can't resolve, not to raise.
- try:
- port = parsed.port
- except ValueError:
- return None
# ``urlparse().hostname`` removes IPv6 brackets. Restore them when
# constructing an authority so the derived API base remains a URL.
authority_host = f"[{hostname}]" if ":" in hostname else hostname
- authority = authority_host if port is None else f"{authority_host}:{port}"
+ authority = (
+ authority_host if parsed_port is None else f"{authority_host}:{parsed_port}"
+ )
api_base = f"{parsed.scheme}://{authority}/api/v3"
+ expected_asset_prefix = ["", "api", "v3", "repos"]
else:
return None
@@ -202,14 +276,7 @@ def resolve_github_release_asset_api_url(
# ``urlparse`` tolerates some raw spellings (for example whitespace)
# even though the original metadata value is returned to the caller.
# Reject those spellings before parsing rather than normalizing them.
- if (
- any(
- ord(character) <= 0x20 or ord(character) == 0x7F
- for character in asset_url
- )
- or any(delimiter in asset_url for delimiter in ("?", "#", ";"))
- or not _has_valid_percent_escapes(asset_url)
- ):
+ if not _has_valid_asset_url_spelling(asset_url):
return False
try:
asset_parsed = urlparse(asset_url)
@@ -250,15 +317,10 @@ def resolve_github_release_asset_api_url(
return False
asset_parts = asset_parsed.path.split("/")
- owner_index = 2 if api_base == "https://api.github.com" else 4
- expected_prefix = (
- ["", "repos"]
- if api_base == "https://api.github.com"
- else ["", "api", "v3", "repos"]
- )
+ owner_index = len(expected_asset_prefix)
return (
len(asset_parts) == owner_index + 5
- and asset_parts[:owner_index] == expected_prefix
+ and asset_parts[:owner_index] == expected_asset_prefix
and unquote(asset_parts[owner_index]).casefold() == owner.casefold()
and unquote(asset_parts[owner_index + 1]).casefold() == repo.casefold()
and asset_parts[owner_index + 2:owner_index + 4] == ["releases", "assets"]
diff --git a/src/specify_cli/authentication/http.py b/src/specify_cli/authentication/http.py
index 32a6ed67c..7dabc752a 100644
--- a/src/specify_cli/authentication/http.py
+++ b/src/specify_cli/authentication/http.py
@@ -154,9 +154,9 @@ def build_request(url: str, extra_headers: dict[str, str] | None = None) -> urll
def github_provider_hosts() -> tuple[str, ...]:
"""Return host patterns from every ``github`` provider entry in ``auth.json``.
- Used to classify which hosts are GitHub Enterprise Server instances when
- resolving release-asset download URLs. Returns an empty tuple when no
- ``auth.json`` exists or it contains no ``github`` entries.
+ Used to classify trusted GitHub Enterprise Cloud and GitHub Enterprise
+ Server hosts when resolving release-asset download URLs. Returns an empty
+ tuple when no ``auth.json`` exists or it contains no ``github`` entries.
"""
hosts: list[str] = []
for entry in _load_config():
diff --git a/tests/specify_cli/authentication/test_github_http.py b/tests/specify_cli/authentication/test_github_http.py
index ad6440f79..8eae8d0a0 100644
--- a/tests/specify_cli/authentication/test_github_http.py
+++ b/tests/specify_cli/authentication/test_github_http.py
@@ -315,6 +315,151 @@ class TestResolveGitHubReleaseAssetApiUrl:
assert len(captured_urls) == 1
assert "releases/tags/v1%23beta" in captured_urls[0]
+ # --- GHE.com (GitHub Enterprise Cloud with data residency) ---
+
+ def test_resolves_ghecom_browser_url_to_api_url(self):
+ """A GHE.com browser URL resolves through its paired API subdomain."""
+ asset_url = "https://api.msft.ghe.com/repos/org/repo/releases/assets/42"
+ captured = []
+
+ @contextmanager
+ def capturing_open(url, timeout=None, extra_headers=None):
+ captured.append(url)
+ resp = MagicMock()
+ resp.read.side_effect = io.BytesIO(
+ json.dumps(
+ {"assets": [{"name": "bundle.zip", "url": asset_url}]}
+ ).encode()
+ ).read
+ yield resp
+
+ result = resolve_github_release_asset_api_url(
+ "https://msft.ghe.com/org/repo/releases/download/v1.0/bundle.zip",
+ capturing_open,
+ github_hosts=("msft.ghe.com", "api.msft.ghe.com"),
+ )
+
+ assert result == asset_url
+ assert captured == [
+ "https://api.msft.ghe.com/repos/org/repo/releases/tags/v1.0"
+ ]
+
+ def test_passthrough_for_trusted_ghecom_api_asset_url(self):
+ """A trusted direct GHE.com API asset URL receives asset treatment."""
+ url = "https://api.msft.ghe.com/repos/org/repo/releases/assets/42"
+ result = resolve_github_release_asset_api_url(
+ url,
+ lambda *a, **kw: None,
+ github_hosts=("msft.ghe.com", "api.msft.ghe.com"),
+ )
+ assert result == url
+
+ @pytest.mark.parametrize(
+ "url",
+ [
+ "https://api.msft.ghe.com/repos/org/repo/releases/assets/42?",
+ "https://api.msft.ghe.com/repos/org/repo/releases/assets/42#",
+ "https://api.msft.ghe.com/repos/org/repo/releases/assets/42/",
+ "https://api.msft.ghe.com/repos/org/repo/releases/assets/%34%32",
+ "https://api.msft.ghe.com/repos/org/repo/releases/assets/42 ",
+ "https://api.msft.ghe.com/repos/org/repo/releases/assets/42;",
+ "https://api.msft.ghe.com/repos/org/repo/releases/assets/%ZZ",
+ ],
+ )
+ def test_rejects_noncanonical_direct_ghecom_api_asset_url(self, url):
+ """Direct GHE.com asset URLs retain strict raw-spelling validation."""
+ result = resolve_github_release_asset_api_url(
+ url,
+ lambda *a, **kw: None,
+ github_hosts=("msft.ghe.com", "api.msft.ghe.com"),
+ )
+ assert result is None
+
+ def test_rejects_direct_ghecom_api_asset_url_without_trusted_pair(self):
+ """A direct GHE.com API URL needs both trusted tenant hostnames."""
+ url = "https://api.msft.ghe.com/repos/org/repo/releases/assets/42"
+ result = resolve_github_release_asset_api_url(
+ url,
+ lambda *a, **kw: None,
+ github_hosts=("api.msft.ghe.com",),
+ )
+ assert result is None
+
+ @pytest.mark.parametrize(
+ "github_hosts",
+ [
+ (),
+ ("msft.ghe.com", "api.msft.ghe.com"),
+ ],
+ )
+ def test_rejects_direct_ghecom_api_asset_url_with_ghes_path(
+ self, github_hosts
+ ):
+ """A GHE.com API host never accepts the legacy GHES /api/v3 path."""
+ url = (
+ "https://api.msft.ghe.com/api/v3/repos/org/repo/"
+ "releases/assets/42"
+ )
+ result = resolve_github_release_asset_api_url(
+ url,
+ lambda *a, **kw: None,
+ github_hosts=github_hosts,
+ )
+ assert result is None
+
+ @pytest.mark.parametrize(
+ "github_hosts",
+ [
+ ("msft.ghe.com",),
+ ("api.msft.ghe.com",),
+ ("other.ghe.com", "api.other.ghe.com"),
+ ],
+ )
+ def test_ghecom_resolution_requires_trusted_web_and_api_hosts(self, github_hosts):
+ """GHE.com resolution requires both members of the tenant host pair."""
+ called = []
+
+ @contextmanager
+ def recording_open(url, timeout=None, extra_headers=None):
+ called.append(url)
+ resp = MagicMock()
+ resp.read.side_effect = io.BytesIO(b"{}").read
+ yield resp
+
+ result = resolve_github_release_asset_api_url(
+ "https://msft.ghe.com/org/repo/releases/download/v1.0/bundle.zip",
+ recording_open,
+ github_hosts=github_hosts,
+ )
+
+ assert result is None
+ assert called == []
+
+ @pytest.mark.parametrize(
+ "asset_url",
+ [
+ "https://api.other.ghe.com/repos/org/repo/releases/assets/42",
+ "http://api.msft.ghe.com/repos/org/repo/releases/assets/42",
+ "https://msft.ghe.com/api/v3/repos/org/repo/releases/assets/42",
+ "https://api.msft.ghe.com/repos/other/repo/releases/assets/42",
+ "https://api.msft.ghe.com/repos/org/other/releases/assets/42",
+ "https://api.msft.ghe.com/repos/org/repo/releases/assets/not-a-number",
+ "https://api.msft.ghe.com/repos/org/repo/releases/assets/42?download=1",
+ ],
+ )
+ def test_rejects_wrong_origin_or_path_for_ghecom_metadata_asset_url(
+ self, asset_url
+ ):
+ """GHE.com metadata must identify the paired tenant and repository."""
+ result = resolve_github_release_asset_api_url(
+ "https://msft.ghe.com/org/repo/releases/download/v1.0/bundle.zip",
+ self._make_open_url_fn(
+ {"assets": [{"name": "bundle.zip", "url": asset_url}]}
+ ),
+ github_hosts=("msft.ghe.com", "api.msft.ghe.com"),
+ )
+ assert result is None
+
# --- GHES (GitHub Enterprise Server) ---
def test_resolves_ghes_browser_url_to_api_url(self):
diff --git a/tests/specify_cli/bundles/test_command_info.py b/tests/specify_cli/bundles/test_command_info.py
index 6d9f6905a..673e004ea 100644
--- a/tests/specify_cli/bundles/test_command_info.py
+++ b/tests/specify_cli/bundles/test_command_info.py
@@ -578,6 +578,60 @@ def test_bundle_info_resolves_ghes_browser_release_url(project: Path):
assert payload["id"] == "demo-bundle"
+def test_bundle_info_resolves_ghecom_browser_release_url_zip(project: Path):
+ """bundle info resolves a GHE.com release ZIP through its API subdomain."""
+ import zipfile
+
+ web_host = "msft.ghe.com"
+ api_host = f"api.{web_host}"
+ browser_url = f"https://{web_host}/org/repo/releases/download/v2.0/bundle.zip"
+ api_asset_url = f"https://{api_host}/repos/org/repo/releases/assets/42"
+
+ archive = io.BytesIO()
+ with zipfile.ZipFile(archive, "w") as zf:
+ zf.writestr("bundle.yml", yaml.safe_dump(valid_manifest_dict()))
+ zip_bytes = archive.getvalue()
+ captured = []
+
+ def fake_open_url(url, timeout=None, extra_headers=None, redirect_validator=None):
+ captured.append((url, extra_headers))
+ if "releases/tags/" in url:
+ return FakeBundleResponse(
+ json.dumps(
+ {"assets": [{"name": "bundle.zip", "url": api_asset_url}]}
+ ).encode(),
+ url=url,
+ )
+ return FakeBundleResponse(zip_bytes, url=api_asset_url)
+
+ catalog = project / "catalog.json"
+ write_catalog_file(
+ catalog,
+ {"demo-bundle": catalog_entry_dict("demo-bundle", download_url=browser_url)},
+ )
+ _make_catalog_config(catalog, project)
+
+ with (
+ patch("specify_cli.authentication.http.open_url", side_effect=fake_open_url),
+ patch(
+ "specify_cli.authentication.http.github_provider_hosts",
+ return_value=(web_host, api_host),
+ ),
+ ):
+ result = runner.invoke(app, ["bundle", "info", "demo-bundle", "--json"])
+
+ assert result.exit_code == 0, result.output
+
+ tag_calls = [url for url, _ in captured if "releases/tags/" in url]
+ assert tag_calls == [f"https://{api_host}/repos/org/repo/releases/tags/v2.0"]
+
+ asset_calls = [(url, headers) for url, headers in captured if url == api_asset_url]
+ assert asset_calls == [(api_asset_url, {"Accept": "application/octet-stream"})]
+
+ payload = json.loads(result.output)
+ assert payload["id"] == "demo-bundle"
+
+
def test_bundle_download_rejects_oversized_response(project: Path, monkeypatch):
"""Bundle download rejects responses exceeding MAX_DOWNLOAD_BYTES."""
# Monkeypatch to a small limit so the test is fast and low-memory.