π« Toolkit to help you get started with SDD or any other process!
fix: resolve GitHub release asset API URL for private repo bundle downloads (#3136)
lselvar3b30e40aaadc
diff --git a/src/specify_cli/commands/bundle/__init__.py b/src/specify_cli/commands/bundle/__init__.pyindex afae9bcf8..b3c84c6ba 100644--- a/src/specify_cli/commands/bundle/__init__.py+++ b/src/specify_cli/commands/bundle/__init__.py@@ -631,6 +631,14 @@ def catalog_remove(console.print(f"[green]β[/green] Removed catalog source '{removed}'.")+# ZIP magic-byte signatures used to detect .zip payloads from REST API asset+# URLs, which carry no file extension. The three signatures cover all valid+# ZIP variants (PK\x03\x04 = local file header, PK\x05\x06 = empty archive,+# PK\x07\x08 = spanning marker) without the false-positive risk of checking+# only the 2-byte "PK" prefix.+_ZIP_SIGNATURES = (b"PK\x03\x04", b"PK\x05\x06", b"PK\x07\x08")++# ===== internal helpers =====@@ -794,41 +802,110 @@ def _download_remote_manifest(entry_id: str, url: str):"""Fetch a remote bundle artifact over HTTPS and extract its manifest."""import ioimport tempfile+ from pathlib import PurePosixPath+ from urllib.parse import urlparse as _urlparse++ import yaml as _yaml- from ...authentication.http import open_url+ from ...authentication.http import github_provider_hosts, open_url+ from ..._github_http import resolve_github_release_asset_api_url+ from ...bundler.models.manifest import BundleManifestdef _validate_redirect(old_url: str, new_url: str) -> None:_require_https(f"bundle '{entry_id}'", new_url)_require_https(f"bundle '{entry_id}'", url)++ # For private/SSO-protected GitHub repos, browser release download URLs+ # (https://github.com/<owner>/<repo>/releases/download/<tag>/<asset>)+ # redirect to an HTML/SSO page instead of delivering the asset. Resolve+ # such URLs to the GitHub REST API asset URL so the authenticated client+ # can download the actual file.+ extra_headers = None+ effective_url = url+ resolved = resolve_github_release_asset_api_url(+ url, open_url, timeout=30, github_hosts=github_provider_hosts()+ )+ if resolved:+ effective_url = resolved+ _require_https(f"bundle '{entry_id}'", effective_url)+ extra_headers = {"Accept": "application/octet-stream"}++ # Human-readable description of where the bytes came from, reused across+ # all post-download error messages so failures point at the catalog URL+ # (and resolved API URL, if any) instead of an opaque temp path.+ if effective_url != url:+ _source_desc = f"{url} (resolved to {effective_url})"+ else:+ _source_desc = url+try:- with open_url(url, timeout=30, redirect_validator=_validate_redirect) as resp:+ with open_url(+ effective_url,+ timeout=30,+ redirect_validator=_validate_redirect,+ extra_headers=extra_headers,+ ) as resp:_require_https(f"bundle '{entry_id}'", resp.geturl())raw = resp.read()except BundlerError:raiseexcept Exception as exc: # noqa: BLE001- raise BundlerError(f"Failed to download bundle '{entry_id}' from {url}: {exc}") from exc+ # Report the original catalog URL so users know which entry to fix,+ # and include the resolved URL when it differs for easier debugging.+ raise BundlerError(+ f"Failed to download bundle '{entry_id}' from {_source_desc}: {exc}"+ ) from exc# A .zip artifact is written to a temp file and parsed via the local-source# path (which extracts bundle.yml); any other payload is treated as YAML.- if url.lower().endswith(".zip"):- with tempfile.TemporaryDirectory() as tmp:- artifact = Path(tmp) / "bundle.zip"- artifact.write_bytes(raw)- manifest = _local_manifest_source(str(artifact))- if manifest is None:- raise BundlerError(- f"Downloaded artifact for bundle '{entry_id}' is not a valid bundle."- )- return manifest-- import yaml as _yaml-- from ...bundler.models.manifest import BundleManifest+ # Detection uses the path component of the original catalog URL (via+ # PurePosixPath so query strings and fragments are ignored, and URL paths+ # are always treated as POSIX regardless of host OS), falling back to the+ # module-level _ZIP_SIGNATURES magic-byte check for direct REST API asset+ # URLs which carry no file extension.+ _url_ext = PurePosixPath(_urlparse(url).path).suffix.lower()+ try:+ if _url_ext == ".zip" or raw[:4] in _ZIP_SIGNATURES:+ with tempfile.TemporaryDirectory() as tmp:+ artifact = Path(tmp) / "bundle.zip"+ artifact.write_bytes(raw)+ # Wrap ZIP parsing so any failure (BadZipFile, missing+ # bundle.yml, etc.) references the source URL rather than the+ # opaque temporary path, consistent with the download-error+ # handling above.+ try:+ manifest = _local_manifest_source(str(artifact))+ except Exception as exc: # noqa: BLE001+ raise BundlerError(+ f"Downloaded artifact for bundle '{entry_id}' from "+ f"{_source_desc} is not a valid bundle: {exc}"+ ) from exc+ # _local_manifest_source returns None only when the file does+ # not exist; since we just wrote *artifact* that cannot happen+ # here. The explicit guard ensures callers never receive None+ # and silently degrade instead of raising a clear error.+ if manifest is None:+ raise BundlerError(+ f"Downloaded artifact for bundle '{entry_id}' from "+ f"{_source_desc} is not a valid bundle."+ )+ return manifest- data = _yaml.safe_load(io.BytesIO(raw))- return BundleManifest.from_dict(data)+ data = _yaml.safe_load(io.BytesIO(raw))+ return BundleManifest.from_dict(data)+ except BundlerError:+ raise+ except _yaml.YAMLError as exc:+ raise BundlerError(+ f"Downloaded content for bundle '{entry_id}' from {_source_desc} "+ f"is not valid YAML: {exc}"+ ) from exc+ except Exception as exc: # noqa: BLE001+ raise BundlerError(+ f"Failed to parse downloaded bundle '{entry_id}' from "+ f"{_source_desc}: {exc}"+ ) from excdef register(app: typer.Typer) -> None:diff --git a/tests/contract/test_bundle_cli.py b/tests/contract/test_bundle_cli.pyindex 58a26fae9..1705c5945 100644--- a/tests/contract/test_bundle_cli.py+++ b/tests/contract/test_bundle_cli.py@@ -8,6 +8,7 @@ from __future__ import annotationsimport jsonfrom pathlib import Path+from unittest.mock import patchimport pytestimport yaml@@ -404,3 +405,315 @@ def test_install_integration_override_cannot_bypass_clash_guard(project: Path):)assert result.exit_code == 1assert "claude" in result.output and "copilot" in result.output+++# ===== Private GitHub release asset URL resolution =====+++class FakeBundleResponse:+ """Minimal context-manager response stub for open_url fakes."""++ def __init__(self, data: bytes, url: str = "https://api.github.com/repos/org/repo/releases/assets/99"):+ self._data = data+ self._url = url++ def read(self) -> bytes:+ return self._data++ def geturl(self) -> str:+ return self._url++ def __enter__(self):+ return self++ def __exit__(self, *_):+ return False+++def _make_catalog_config(catalog_path: Path, project: Path) -> None:+ """Write a bundle-catalogs.yml pointing at *catalog_path* in *project*."""+ config = {+ "schema_version": "1.0",+ "catalogs": [+ {+ "id": "test",+ "url": str(catalog_path),+ "priority": 1,+ "install_policy": "install-allowed",+ }+ ],+ }+ (project / ".specify" / "bundle-catalogs.yml").write_text(+ yaml.safe_dump(config), encoding="utf-8"+ )+++def test_bundle_info_resolves_github_browser_release_url(project: Path):+ """bundle info resolves a private-repo browser release URL via the GitHub API."""+ browser_url = "https://github.com/org/repo/releases/download/v1.0/bundle.yml"+ api_asset_url = "https://api.github.com/repos/org/repo/releases/assets/99"++ captured = []+ manifest_yaml = yaml.safe_dump(valid_manifest_dict()).encode()++ def fake_open_url(url, timeout=None, extra_headers=None, redirect_validator=None):+ captured.append((url, extra_headers))+ if "releases/tags/" in url:+ # GitHub API release-tags lookup β return asset list+ return FakeBundleResponse(+ json.dumps({+ "assets": [{"name": "bundle.yml", "url": api_asset_url}]+ }).encode(),+ url=url,+ )+ # Actual asset download+ return FakeBundleResponse(manifest_yaml, url=api_asset_url)++ catalog = project / "catalog.json"+ write_catalog_file(+ catalog,+ {"demo-bundle": catalog_entry_dict("demo-bundle", download_url=browser_url)},+ )+ _make_catalog_config(catalog, project)++ with patch("specify_cli.authentication.http.open_url", side_effect=fake_open_url):+ result = runner.invoke(app, ["bundle", "info", "demo-bundle", "--json"])++ assert result.exit_code == 0, result.output++ # The browser release URL must have been resolved via the GitHub tags API+ tag_calls = [url for url, _ in captured if "releases/tags/" in url]+ assert len(tag_calls) == 1, f"Expected exactly one tags API call; got {captured}"+ assert "releases/tags/v1.0" in tag_calls[0]++ # The actual download must use the resolved API asset URL with octet-stream+ asset_calls = [(url, h) for url, h in captured if "releases/assets/" in url]+ assert len(asset_calls) == 1+ assert asset_calls[0][0] == api_asset_url+ assert asset_calls[0][1] == {"Accept": "application/octet-stream"}+++def test_bundle_info_passes_through_api_asset_url(project: Path):+ """bundle info passes a direct GitHub API asset URL through with octet-stream."""+ api_asset_url = "https://api.github.com/repos/org/repo/releases/assets/77"++ captured = []+ manifest_yaml = yaml.safe_dump(valid_manifest_dict()).encode()++ def fake_open_url(url, timeout=None, extra_headers=None, redirect_validator=None):+ captured.append((url, extra_headers))+ return FakeBundleResponse(manifest_yaml, url=api_asset_url)++ catalog = project / "catalog.json"+ write_catalog_file(+ catalog,+ {"demo-bundle": catalog_entry_dict("demo-bundle", download_url=api_asset_url)},+ )+ _make_catalog_config(catalog, project)++ with patch("specify_cli.authentication.http.open_url", side_effect=fake_open_url):+ result = runner.invoke(app, ["bundle", "info", "demo-bundle", "--json"])++ assert result.exit_code == 0, result.output++ # No tags API call β URL was already a REST asset URL+ tag_calls = [url for url, _ in captured if "releases/tags/" in url]+ assert len(tag_calls) == 0++ # Exactly one download call to the asset URL with octet-stream+ asset_calls = [(url, h) for url, h in captured if "releases/assets/" in url]+ assert len(asset_calls) == 1+ assert asset_calls[0][0] == api_asset_url+ assert asset_calls[0][1] == {"Accept": "application/octet-stream"}+++def test_bundle_info_resolves_github_browser_release_url_zip(project: Path):+ """bundle info resolves a browser release URL for a .zip artifact and extracts bundle.yml."""+ import io+ import zipfile++ browser_url = "https://github.com/org/repo/releases/download/v2.0/bundle.zip"+ api_asset_url = "https://api.github.com/repos/org/repo/releases/assets/88"++ # Build a minimal in-memory ZIP containing bundle.yml+ buf = io.BytesIO()+ with zipfile.ZipFile(buf, "w") as zf:+ zf.writestr("bundle.yml", yaml.safe_dump(valid_manifest_dict()))+ zip_bytes = buf.getvalue()++ captured = []++ def fake_open_url(url, timeout=None, extra_headers=None, redirect_validator=None):+ captured.append((url, extra_headers))+ if "releases/tags/" in url:+ return FakeBundleResponse(+ json.dumps({+ "assets": [{"name": "bundle.zip", "url": api_asset_url}]+ }).encode(),+ url=url,+ )+ return FakeBundleResponse(zip_bytes, url=api_asset_url)++ catalog = project / "catalog.json"+ write_catalog_file(+ catalog,+ {"demo-bundle": catalog_entry_dict("demo-bundle", download_url=browser_url)},+ )+ _make_catalog_config(catalog, project)++ with patch("specify_cli.authentication.http.open_url", side_effect=fake_open_url):+ result = runner.invoke(app, ["bundle", "info", "demo-bundle", "--json"])++ assert result.exit_code == 0, result.output++ # tags API lookup must have fired+ tag_calls = [url for url, _ in captured if "releases/tags/" in url]+ assert len(tag_calls) == 1+ assert "releases/tags/v2.0" in tag_calls[0]++ # Asset download uses the resolved API URL with octet-stream+ asset_calls = [(url, h) for url, h in captured if "releases/assets/" in url]+ assert len(asset_calls) == 1+ assert asset_calls[0][0] == api_asset_url+ assert asset_calls[0][1] == {"Accept": "application/octet-stream"}++ # Manifest was successfully parsed from the ZIP+ payload = json.loads(result.output)+ assert payload["id"] == "demo-bundle"+++def test_bundle_info_api_asset_url_zip_detected_by_magic_bytes(project: Path):+ """bundle info correctly handles a direct API asset URL that serves ZIP bytes."""+ import io+ import zipfile++ api_asset_url = "https://api.github.com/repos/org/repo/releases/assets/55"++ # Build a minimal in-memory ZIP containing bundle.yml+ buf = io.BytesIO()+ with zipfile.ZipFile(buf, "w") as zf:+ zf.writestr("bundle.yml", yaml.safe_dump(valid_manifest_dict()))+ zip_bytes = buf.getvalue()++ captured = []++ def fake_open_url(url, timeout=None, extra_headers=None, redirect_validator=None):+ captured.append((url, extra_headers))+ return FakeBundleResponse(zip_bytes, url=api_asset_url)++ catalog = project / "catalog.json"+ write_catalog_file(+ catalog,+ {"demo-bundle": catalog_entry_dict("demo-bundle", download_url=api_asset_url)},+ )+ _make_catalog_config(catalog, project)++ with patch("specify_cli.authentication.http.open_url", side_effect=fake_open_url):+ result = runner.invoke(app, ["bundle", "info", "demo-bundle", "--json"])++ assert result.exit_code == 0, result.output++ # No tags API call β URL was already a REST asset URL+ tag_calls = [url for url, _ in captured if "releases/tags/" in url]+ assert len(tag_calls) == 0++ # Download used octet-stream header+ asset_calls = [(url, h) for url, h in captured if "releases/assets/" in url]+ assert len(asset_calls) == 1+ assert asset_calls[0][1] == {"Accept": "application/octet-stream"}++ # ZIP bytes were detected by magic and bundle.yml extracted correctly+ payload = json.loads(result.output)+ assert payload["id"] == "demo-bundle"+++def test_bundle_info_github_release_url_resolution_failure_falls_back_and_errors(project: Path):+ """When the GitHub tags API lookup finds no matching asset, fall back to the+ original browser URL and surface a meaningful error (not a raw traceback)."""+ browser_url = "https://github.com/org/repo/releases/download/v3.0/bundle.yml"++ captured = []++ def fake_open_url(url, timeout=None, extra_headers=None, redirect_validator=None):+ captured.append((url, extra_headers))+ if "releases/tags/" in url:+ # Tags API responds but the asset list doesn't include our file+ return FakeBundleResponse(+ json.dumps({"assets": []}).encode(),+ url=url,+ )+ # Fallback download: GitHub serves HTML (SSO redirect) instead of YAML+ return FakeBundleResponse(b"<html>SSO login required</html>", url=url)++ catalog = project / "catalog.json"+ write_catalog_file(+ catalog,+ {"demo-bundle": catalog_entry_dict("demo-bundle", download_url=browser_url)},+ )+ _make_catalog_config(catalog, project)++ with patch("specify_cli.authentication.http.open_url", side_effect=fake_open_url):+ result = runner.invoke(app, ["bundle", "info", "demo-bundle", "--json"])++ # Must exit non-zero β the HTML body is not a valid bundle manifest+ assert result.exit_code == 1++ # The tags API lookup must have fired+ tag_calls = [url for url, _ in captured if "releases/tags/" in url]+ assert len(tag_calls) == 1++ # The fallback download should use the original browser URL (no octet-stream)+ fallback_calls = [(url, h) for url, h in captured if url == browser_url]+ assert len(fallback_calls) == 1+ assert fallback_calls[0][1] is None # no Accept header on the original URL++ # Error output must be actionable (not a raw traceback)+ assert "Error:" in result.output+++def test_bundle_info_resolves_ghes_browser_release_url(project: Path):+ """bundle info resolves a GHES private-repo browser release URL via /api/v3."""+ ghes_host = "ghes.example"+ browser_url = f"https://{ghes_host}/org/repo/releases/download/v1.0/bundle.yml"+ api_asset_url = f"https://{ghes_host}/api/v3/repos/org/repo/releases/assets/42"++ captured = []+ manifest_yaml = yaml.safe_dump(valid_manifest_dict()).encode()++ def fake_open_url(url, timeout=None, extra_headers=None, redirect_validator=None):+ captured.append((url, extra_headers))+ if "/api/v3/repos/" in url and "releases/tags/" in url:+ return FakeBundleResponse(+ json.dumps({+ "assets": [{"name": "bundle.yml", "url": api_asset_url}]+ }).encode(),+ url=url,+ )+ return FakeBundleResponse(manifest_yaml, url=api_asset_url)++ catalog = project / "catalog.json"+ write_catalog_file(+ catalog,+ {"demo-bundle": catalog_entry_dict("demo-bundle", download_url=browser_url)},+ )+ _make_catalog_config(catalog, project)++ with patch("specify_cli.authentication.http.open_url", side_effect=fake_open_url), \+ patch("specify_cli.authentication.http.github_provider_hosts", return_value=(ghes_host,)):+ result = runner.invoke(app, ["bundle", "info", "demo-bundle", "--json"])++ assert result.exit_code == 0, result.output++ # The GHES /api/v3 tags lookup must have fired+ tag_calls = [url for url, _ in captured if "releases/tags/" in url]+ assert len(tag_calls) == 1+ assert f"{ghes_host}/api/v3/repos/org/repo/releases/tags/v1.0" in tag_calls[0]++ # Asset download must use the resolved GHES API URL with octet-stream+ asset_calls = [(url, h) for url, h in captured if "releases/assets/" in url]+ assert len(asset_calls) == 1+ assert asset_calls[0][0] == api_asset_url+ assert asset_calls[0][1] == {"Accept": "application/octet-stream"}++ payload = json.loads(result.output)+ assert payload["id"] == "demo-bundle"