Skip to content

πŸ’« Toolkit to help you get started with SDD or any other process!

fix: resolve GitHub release asset API URL for private repo bundle downloads (#3136)

lselvar3b30e40aaadc
diff --git a/src/specify_cli/commands/bundle/__init__.py b/src/specify_cli/commands/bundle/__init__.py
index afae9bcf8..b3c84c6ba 100644
--- a/src/specify_cli/commands/bundle/__init__.py
+++ b/src/specify_cli/commands/bundle/__init__.py
@@ -631,6 +631,14 @@ def catalog_remove(
console.print(f"[green]βœ“[/green] Removed catalog source '{removed}'.")
+# ZIP magic-byte signatures used to detect .zip payloads from REST API asset
+# URLs, which carry no file extension. The three signatures cover all valid
+# ZIP variants (PK\x03\x04 = local file header, PK\x05\x06 = empty archive,
+# PK\x07\x08 = spanning marker) without the false-positive risk of checking
+# only the 2-byte "PK" prefix.
+_ZIP_SIGNATURES = (b"PK\x03\x04", b"PK\x05\x06", b"PK\x07\x08")
+
+
# ===== internal helpers =====
@@ -794,41 +802,110 @@ def _download_remote_manifest(entry_id: str, url: str):
"""Fetch a remote bundle artifact over HTTPS and extract its manifest."""
import io
import tempfile
+ from pathlib import PurePosixPath
+ from urllib.parse import urlparse as _urlparse
+
+ import yaml as _yaml
- from ...authentication.http import open_url
+ from ...authentication.http import github_provider_hosts, open_url
+ from ..._github_http import resolve_github_release_asset_api_url
+ from ...bundler.models.manifest import BundleManifest
def _validate_redirect(old_url: str, new_url: str) -> None:
_require_https(f"bundle '{entry_id}'", new_url)
_require_https(f"bundle '{entry_id}'", url)
+
+ # For private/SSO-protected GitHub repos, browser release download URLs
+ # (https://github.com/<owner>/<repo>/releases/download/<tag>/<asset>)
+ # redirect to an HTML/SSO page instead of delivering the asset. Resolve
+ # such URLs to the GitHub REST API asset URL so the authenticated client
+ # can download the actual file.
+ extra_headers = None
+ effective_url = url
+ resolved = resolve_github_release_asset_api_url(
+ url, open_url, timeout=30, github_hosts=github_provider_hosts()
+ )
+ if resolved:
+ effective_url = resolved
+ _require_https(f"bundle '{entry_id}'", effective_url)
+ extra_headers = {"Accept": "application/octet-stream"}
+
+ # Human-readable description of where the bytes came from, reused across
+ # all post-download error messages so failures point at the catalog URL
+ # (and resolved API URL, if any) instead of an opaque temp path.
+ if effective_url != url:
+ _source_desc = f"{url} (resolved to {effective_url})"
+ else:
+ _source_desc = url
+
try:
- with open_url(url, timeout=30, redirect_validator=_validate_redirect) as resp:
+ with open_url(
+ effective_url,
+ timeout=30,
+ redirect_validator=_validate_redirect,
+ extra_headers=extra_headers,
+ ) as resp:
_require_https(f"bundle '{entry_id}'", resp.geturl())
raw = resp.read()
except BundlerError:
raise
except Exception as exc: # noqa: BLE001
- raise BundlerError(f"Failed to download bundle '{entry_id}' from {url}: {exc}") from exc
+ # Report the original catalog URL so users know which entry to fix,
+ # and include the resolved URL when it differs for easier debugging.
+ raise BundlerError(
+ f"Failed to download bundle '{entry_id}' from {_source_desc}: {exc}"
+ ) from exc
# A .zip artifact is written to a temp file and parsed via the local-source
# path (which extracts bundle.yml); any other payload is treated as YAML.
- if url.lower().endswith(".zip"):
- with tempfile.TemporaryDirectory() as tmp:
- artifact = Path(tmp) / "bundle.zip"
- artifact.write_bytes(raw)
- manifest = _local_manifest_source(str(artifact))
- if manifest is None:
- raise BundlerError(
- f"Downloaded artifact for bundle '{entry_id}' is not a valid bundle."
- )
- return manifest
-
- import yaml as _yaml
-
- from ...bundler.models.manifest import BundleManifest
+ # Detection uses the path component of the original catalog URL (via
+ # PurePosixPath so query strings and fragments are ignored, and URL paths
+ # are always treated as POSIX regardless of host OS), falling back to the
+ # module-level _ZIP_SIGNATURES magic-byte check for direct REST API asset
+ # URLs which carry no file extension.
+ _url_ext = PurePosixPath(_urlparse(url).path).suffix.lower()
+ try:
+ if _url_ext == ".zip" or raw[:4] in _ZIP_SIGNATURES:
+ with tempfile.TemporaryDirectory() as tmp:
+ artifact = Path(tmp) / "bundle.zip"
+ artifact.write_bytes(raw)
+ # Wrap ZIP parsing so any failure (BadZipFile, missing
+ # bundle.yml, etc.) references the source URL rather than the
+ # opaque temporary path, consistent with the download-error
+ # handling above.
+ try:
+ manifest = _local_manifest_source(str(artifact))
+ except Exception as exc: # noqa: BLE001
+ raise BundlerError(
+ f"Downloaded artifact for bundle '{entry_id}' from "
+ f"{_source_desc} is not a valid bundle: {exc}"
+ ) from exc
+ # _local_manifest_source returns None only when the file does
+ # not exist; since we just wrote *artifact* that cannot happen
+ # here. The explicit guard ensures callers never receive None
+ # and silently degrade instead of raising a clear error.
+ if manifest is None:
+ raise BundlerError(
+ f"Downloaded artifact for bundle '{entry_id}' from "
+ f"{_source_desc} is not a valid bundle."
+ )
+ return manifest
- data = _yaml.safe_load(io.BytesIO(raw))
- return BundleManifest.from_dict(data)
+ data = _yaml.safe_load(io.BytesIO(raw))
+ return BundleManifest.from_dict(data)
+ except BundlerError:
+ raise
+ except _yaml.YAMLError as exc:
+ raise BundlerError(
+ f"Downloaded content for bundle '{entry_id}' from {_source_desc} "
+ f"is not valid YAML: {exc}"
+ ) from exc
+ except Exception as exc: # noqa: BLE001
+ raise BundlerError(
+ f"Failed to parse downloaded bundle '{entry_id}' from "
+ f"{_source_desc}: {exc}"
+ ) from exc
def register(app: typer.Typer) -> None:
diff --git a/tests/contract/test_bundle_cli.py b/tests/contract/test_bundle_cli.py
index 58a26fae9..1705c5945 100644
--- a/tests/contract/test_bundle_cli.py
+++ b/tests/contract/test_bundle_cli.py
@@ -8,6 +8,7 @@ from __future__ import annotations
import json
from pathlib import Path
+from unittest.mock import patch
import pytest
import yaml
@@ -404,3 +405,315 @@ def test_install_integration_override_cannot_bypass_clash_guard(project: Path):
)
assert result.exit_code == 1
assert "claude" in result.output and "copilot" in result.output
+
+
+# ===== Private GitHub release asset URL resolution =====
+
+
+class FakeBundleResponse:
+ """Minimal context-manager response stub for open_url fakes."""
+
+ def __init__(self, data: bytes, url: str = "https://api.github.com/repos/org/repo/releases/assets/99"):
+ self._data = data
+ self._url = url
+
+ def read(self) -> bytes:
+ return self._data
+
+ def geturl(self) -> str:
+ return self._url
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *_):
+ return False
+
+
+def _make_catalog_config(catalog_path: Path, project: Path) -> None:
+ """Write a bundle-catalogs.yml pointing at *catalog_path* in *project*."""
+ config = {
+ "schema_version": "1.0",
+ "catalogs": [
+ {
+ "id": "test",
+ "url": str(catalog_path),
+ "priority": 1,
+ "install_policy": "install-allowed",
+ }
+ ],
+ }
+ (project / ".specify" / "bundle-catalogs.yml").write_text(
+ yaml.safe_dump(config), encoding="utf-8"
+ )
+
+
+def test_bundle_info_resolves_github_browser_release_url(project: Path):
+ """bundle info resolves a private-repo browser release URL via the GitHub API."""
+ browser_url = "https://github.com/org/repo/releases/download/v1.0/bundle.yml"
+ api_asset_url = "https://api.github.com/repos/org/repo/releases/assets/99"
+
+ captured = []
+ manifest_yaml = yaml.safe_dump(valid_manifest_dict()).encode()
+
+ def fake_open_url(url, timeout=None, extra_headers=None, redirect_validator=None):
+ captured.append((url, extra_headers))
+ if "releases/tags/" in url:
+ # GitHub API release-tags lookup β€” return asset list
+ return FakeBundleResponse(
+ json.dumps({
+ "assets": [{"name": "bundle.yml", "url": api_asset_url}]
+ }).encode(),
+ url=url,
+ )
+ # Actual asset download
+ return FakeBundleResponse(manifest_yaml, url=api_asset_url)
+
+ catalog = project / "catalog.json"
+ write_catalog_file(
+ catalog,
+ {"demo-bundle": catalog_entry_dict("demo-bundle", download_url=browser_url)},
+ )
+ _make_catalog_config(catalog, project)
+
+ with patch("specify_cli.authentication.http.open_url", side_effect=fake_open_url):
+ result = runner.invoke(app, ["bundle", "info", "demo-bundle", "--json"])
+
+ assert result.exit_code == 0, result.output
+
+ # The browser release URL must have been resolved via the GitHub tags API
+ tag_calls = [url for url, _ in captured if "releases/tags/" in url]
+ assert len(tag_calls) == 1, f"Expected exactly one tags API call; got {captured}"
+ assert "releases/tags/v1.0" in tag_calls[0]
+
+ # The actual download must use the resolved API asset URL with octet-stream
+ asset_calls = [(url, h) for url, h in captured if "releases/assets/" in url]
+ assert len(asset_calls) == 1
+ assert asset_calls[0][0] == api_asset_url
+ assert asset_calls[0][1] == {"Accept": "application/octet-stream"}
+
+
+def test_bundle_info_passes_through_api_asset_url(project: Path):
+ """bundle info passes a direct GitHub API asset URL through with octet-stream."""
+ api_asset_url = "https://api.github.com/repos/org/repo/releases/assets/77"
+
+ captured = []
+ manifest_yaml = yaml.safe_dump(valid_manifest_dict()).encode()
+
+ def fake_open_url(url, timeout=None, extra_headers=None, redirect_validator=None):
+ captured.append((url, extra_headers))
+ return FakeBundleResponse(manifest_yaml, url=api_asset_url)
+
+ catalog = project / "catalog.json"
+ write_catalog_file(
+ catalog,
+ {"demo-bundle": catalog_entry_dict("demo-bundle", download_url=api_asset_url)},
+ )
+ _make_catalog_config(catalog, project)
+
+ with patch("specify_cli.authentication.http.open_url", side_effect=fake_open_url):
+ result = runner.invoke(app, ["bundle", "info", "demo-bundle", "--json"])
+
+ assert result.exit_code == 0, result.output
+
+ # No tags API call β€” URL was already a REST asset URL
+ tag_calls = [url for url, _ in captured if "releases/tags/" in url]
+ assert len(tag_calls) == 0
+
+ # Exactly one download call to the asset URL with octet-stream
+ asset_calls = [(url, h) for url, h in captured if "releases/assets/" in url]
+ assert len(asset_calls) == 1
+ assert asset_calls[0][0] == api_asset_url
+ assert asset_calls[0][1] == {"Accept": "application/octet-stream"}
+
+
+def test_bundle_info_resolves_github_browser_release_url_zip(project: Path):
+ """bundle info resolves a browser release URL for a .zip artifact and extracts bundle.yml."""
+ import io
+ import zipfile
+
+ browser_url = "https://github.com/org/repo/releases/download/v2.0/bundle.zip"
+ api_asset_url = "https://api.github.com/repos/org/repo/releases/assets/88"
+
+ # Build a minimal in-memory ZIP containing bundle.yml
+ buf = io.BytesIO()
+ with zipfile.ZipFile(buf, "w") as zf:
+ zf.writestr("bundle.yml", yaml.safe_dump(valid_manifest_dict()))
+ zip_bytes = buf.getvalue()
+
+ captured = []
+
+ def fake_open_url(url, timeout=None, extra_headers=None, redirect_validator=None):
+ captured.append((url, extra_headers))
+ if "releases/tags/" in url:
+ return FakeBundleResponse(
+ json.dumps({
+ "assets": [{"name": "bundle.zip", "url": api_asset_url}]
+ }).encode(),
+ url=url,
+ )
+ return FakeBundleResponse(zip_bytes, url=api_asset_url)
+
+ catalog = project / "catalog.json"
+ write_catalog_file(
+ catalog,
+ {"demo-bundle": catalog_entry_dict("demo-bundle", download_url=browser_url)},
+ )
+ _make_catalog_config(catalog, project)
+
+ with patch("specify_cli.authentication.http.open_url", side_effect=fake_open_url):
+ result = runner.invoke(app, ["bundle", "info", "demo-bundle", "--json"])
+
+ assert result.exit_code == 0, result.output
+
+ # tags API lookup must have fired
+ tag_calls = [url for url, _ in captured if "releases/tags/" in url]
+ assert len(tag_calls) == 1
+ assert "releases/tags/v2.0" in tag_calls[0]
+
+ # Asset download uses the resolved API URL with octet-stream
+ asset_calls = [(url, h) for url, h in captured if "releases/assets/" in url]
+ assert len(asset_calls) == 1
+ assert asset_calls[0][0] == api_asset_url
+ assert asset_calls[0][1] == {"Accept": "application/octet-stream"}
+
+ # Manifest was successfully parsed from the ZIP
+ payload = json.loads(result.output)
+ assert payload["id"] == "demo-bundle"
+
+
+def test_bundle_info_api_asset_url_zip_detected_by_magic_bytes(project: Path):
+ """bundle info correctly handles a direct API asset URL that serves ZIP bytes."""
+ import io
+ import zipfile
+
+ api_asset_url = "https://api.github.com/repos/org/repo/releases/assets/55"
+
+ # Build a minimal in-memory ZIP containing bundle.yml
+ buf = io.BytesIO()
+ with zipfile.ZipFile(buf, "w") as zf:
+ zf.writestr("bundle.yml", yaml.safe_dump(valid_manifest_dict()))
+ zip_bytes = buf.getvalue()
+
+ captured = []
+
+ def fake_open_url(url, timeout=None, extra_headers=None, redirect_validator=None):
+ captured.append((url, extra_headers))
+ return FakeBundleResponse(zip_bytes, url=api_asset_url)
+
+ catalog = project / "catalog.json"
+ write_catalog_file(
+ catalog,
+ {"demo-bundle": catalog_entry_dict("demo-bundle", download_url=api_asset_url)},
+ )
+ _make_catalog_config(catalog, project)
+
+ with patch("specify_cli.authentication.http.open_url", side_effect=fake_open_url):
+ result = runner.invoke(app, ["bundle", "info", "demo-bundle", "--json"])
+
+ assert result.exit_code == 0, result.output
+
+ # No tags API call β€” URL was already a REST asset URL
+ tag_calls = [url for url, _ in captured if "releases/tags/" in url]
+ assert len(tag_calls) == 0
+
+ # Download used octet-stream header
+ asset_calls = [(url, h) for url, h in captured if "releases/assets/" in url]
+ assert len(asset_calls) == 1
+ assert asset_calls[0][1] == {"Accept": "application/octet-stream"}
+
+ # ZIP bytes were detected by magic and bundle.yml extracted correctly
+ payload = json.loads(result.output)
+ assert payload["id"] == "demo-bundle"
+
+
+def test_bundle_info_github_release_url_resolution_failure_falls_back_and_errors(project: Path):
+ """When the GitHub tags API lookup finds no matching asset, fall back to the
+ original browser URL and surface a meaningful error (not a raw traceback)."""
+ browser_url = "https://github.com/org/repo/releases/download/v3.0/bundle.yml"
+
+ captured = []
+
+ def fake_open_url(url, timeout=None, extra_headers=None, redirect_validator=None):
+ captured.append((url, extra_headers))
+ if "releases/tags/" in url:
+ # Tags API responds but the asset list doesn't include our file
+ return FakeBundleResponse(
+ json.dumps({"assets": []}).encode(),
+ url=url,
+ )
+ # Fallback download: GitHub serves HTML (SSO redirect) instead of YAML
+ return FakeBundleResponse(b"<html>SSO login required</html>", url=url)
+
+ catalog = project / "catalog.json"
+ write_catalog_file(
+ catalog,
+ {"demo-bundle": catalog_entry_dict("demo-bundle", download_url=browser_url)},
+ )
+ _make_catalog_config(catalog, project)
+
+ with patch("specify_cli.authentication.http.open_url", side_effect=fake_open_url):
+ result = runner.invoke(app, ["bundle", "info", "demo-bundle", "--json"])
+
+ # Must exit non-zero β€” the HTML body is not a valid bundle manifest
+ assert result.exit_code == 1
+
+ # The tags API lookup must have fired
+ tag_calls = [url for url, _ in captured if "releases/tags/" in url]
+ assert len(tag_calls) == 1
+
+ # The fallback download should use the original browser URL (no octet-stream)
+ fallback_calls = [(url, h) for url, h in captured if url == browser_url]
+ assert len(fallback_calls) == 1
+ assert fallback_calls[0][1] is None # no Accept header on the original URL
+
+ # Error output must be actionable (not a raw traceback)
+ assert "Error:" in result.output
+
+
+def test_bundle_info_resolves_ghes_browser_release_url(project: Path):
+ """bundle info resolves a GHES private-repo browser release URL via /api/v3."""
+ ghes_host = "ghes.example"
+ browser_url = f"https://{ghes_host}/org/repo/releases/download/v1.0/bundle.yml"
+ api_asset_url = f"https://{ghes_host}/api/v3/repos/org/repo/releases/assets/42"
+
+ captured = []
+ manifest_yaml = yaml.safe_dump(valid_manifest_dict()).encode()
+
+ def fake_open_url(url, timeout=None, extra_headers=None, redirect_validator=None):
+ captured.append((url, extra_headers))
+ if "/api/v3/repos/" in url and "releases/tags/" in url:
+ return FakeBundleResponse(
+ json.dumps({
+ "assets": [{"name": "bundle.yml", "url": api_asset_url}]
+ }).encode(),
+ url=url,
+ )
+ return FakeBundleResponse(manifest_yaml, url=api_asset_url)
+
+ catalog = project / "catalog.json"
+ write_catalog_file(
+ catalog,
+ {"demo-bundle": catalog_entry_dict("demo-bundle", download_url=browser_url)},
+ )
+ _make_catalog_config(catalog, project)
+
+ with patch("specify_cli.authentication.http.open_url", side_effect=fake_open_url), \
+ patch("specify_cli.authentication.http.github_provider_hosts", return_value=(ghes_host,)):
+ result = runner.invoke(app, ["bundle", "info", "demo-bundle", "--json"])
+
+ assert result.exit_code == 0, result.output
+
+ # The GHES /api/v3 tags lookup must have fired
+ tag_calls = [url for url, _ in captured if "releases/tags/" in url]
+ assert len(tag_calls) == 1
+ assert f"{ghes_host}/api/v3/repos/org/repo/releases/tags/v1.0" in tag_calls[0]
+
+ # Asset download must use the resolved GHES API URL with octet-stream
+ asset_calls = [(url, h) for url, h in captured if "releases/assets/" in url]
+ assert len(asset_calls) == 1
+ assert asset_calls[0][0] == api_asset_url
+ assert asset_calls[0][1] == {"Accept": "application/octet-stream"}
+
+ payload = json.loads(result.output)
+ assert payload["id"] == "demo-bundle"