π« Toolkit to help you get started with SDD or any other process!
"""Concrete adapters: catalog fetching and primitive installation.
These wire the bundler's injectable seams to the real environment:
* :func:`make_catalog_fetcher` returns an offline-first fetcher that reads
built-in catalogs and local/pinned file URLs without network, and falls back
to a timeout-bounded HTTP GET only for ``http(s)://`` sources.
* :class:`DefaultPrimitiveInstaller` dispatches component install/remove to the
existing Spec Kit primitive machinery in-process.
"""
from __future__ import annotations
import http.client
import re
import ssl
import urllib.error
import warnings
from pathlib import Path
from typing import TYPE_CHECKING
from urllib.parse import ParseResult, urlparse
from urllib.request import url2pathname
from .._assets import _locate_core_pack, _repo_root
from .._download_security import MAX_JSON_CATALOG_BYTES, read_response_limited
from . import BundlerError
from .catalogs import CatalogSource
from .manifest import ComponentRef
from .yamlio import loads_json
if TYPE_CHECKING:
from .primitives import BundledPatchWarning
COMMUNITY_CATALOG_URL = (
"https://raw.githubusercontent.com/github/spec-kit/main/"
"bundles/catalog.community.json"
)
FIRSTPARTY_CATALOG_URL = (
"https://raw.githubusercontent.com/github/spec-kit/main/"
"bundles/catalog.json"
)
# Built-in catalogs are resolved directly by URL. ``builtin://default`` is the
# repository-shipped first-party bundle catalog; ``builtin://community`` is the
# community catalog. Both are fetched from the repository online and fall back
# to the packaged wheel snapshot offline so discovery works without network.
_BUILTIN_REPOSITORY_URLS: dict[str, str] = {
"builtin://default": FIRSTPARTY_CATALOG_URL,
"builtin://community": COMMUNITY_CATALOG_URL,
}
_BUILTIN_PACKAGED_SNAPSHOTS: dict[str, str] = {
"builtin://default": "catalog.json",
"builtin://community": "catalog.community.json",
}
HTTP_TIMEOUT_SECONDS = 10
_TRANSIENT_HTTP_STATUS_CODES = (408, 429)
class _CatalogUnavailable(BundlerError):
"""A built-in catalog could not be reached (transport/availability failure).
Marks only transient fetch failures β connection/DNS errors, timeouts,
truncated responses, and availability HTTP responses (408, 429, 5xx) β so
the built-in catalog fallback does not swallow content or security
validation failures (malformed JSON, oversized or non-UTF-8 bodies, unsafe
redirects, TLS certificate verification failures, other HTTP 4xx).
"""
# Windows absolute paths like ``C:\catalog.json`` parse with a single-letter
# ``scheme`` under urlparse; treat them as local files rather than URLs.
_WINDOWS_DRIVE_RE = re.compile(r"^[A-Za-z]:[\\/]")
def _is_windows_drive_path(url: str) -> bool:
return bool(_WINDOWS_DRIVE_RE.match(url))
def _file_url_to_path(parsed: ParseResult) -> Path:
"""Convert a ``file://`` URL to a local path.
Uses ``url2pathname`` for percent-decoding and OS-correct separators, and
preserves ``netloc`` so UNC paths (``file://server/share``) and Windows
drive URLs (``file:///C:/x``) resolve correctly instead of dropping host
or producing ``/C:/x``.
"""
netloc = parsed.netloc
if netloc and netloc.lower() != "localhost":
# UNC share: file://server/share/... -> \\server\share\...
return Path(url2pathname(f"//{netloc}{parsed.path}"))
return Path(url2pathname(parsed.path))
def _validate_remote_url(source_id: str, url: str) -> None:
"""Restrict remote catalogs to HTTPS (HTTP only for localhost) with a host.
Mirrors ``specify_cli.catalogs`` URL validation to avoid MITM/downgrade
issues before any network call.
"""
# A malformed authority (e.g. an unclosed IPv6 bracket ``https://[::1``)
# makes urlparse / hostname access raise ValueError. This function's
# contract is to raise BundlerError for a bad URL, so surface that as a
# clean error rather than leaking a raw ValueError to the caller.
try:
parsed = urlparse(url)
hostname = parsed.hostname
# Accessing ``port`` performs urllib's syntax/range validation.
_ = parsed.port
except ValueError:
raise BundlerError(
f"Catalog '{source_id}' URL is malformed: {url}"
) from None
is_localhost = hostname in ("localhost", "127.0.0.1", "::1")
if parsed.scheme != "https" and not (parsed.scheme == "http" and is_localhost):
raise BundlerError(
f"Catalog '{source_id}' URL must use HTTPS (got {parsed.scheme}://). "
"HTTP is only allowed for localhost."
)
# Check hostname, not netloc: netloc is truthy for host-less URLs like
# "https://:8080" or "https://user@...", so requiring netloc would let
# those through even though they carry no host. hostname is None in those
# cases. Mirrors the fix in ``specify_cli.catalogs`` (#3210).
if not hostname:
raise BundlerError(
f"Catalog '{source_id}' URL must be a valid URL with a host: {url}"
)
def _load_packaged_catalog(filename: str) -> dict:
"""Load a packaged bundle catalog snapshot from the wheel or repo root."""
core_pack = _locate_core_pack()
path = (
core_pack / "bundles" / filename
if core_pack is not None
else _repo_root() / "bundles" / filename
)
if not path.is_file():
raise BundlerError(f"Bundled catalog not found: {path}")
return loads_json(path.read_text(encoding="utf-8"), origin=str(path))
def make_catalog_fetcher(*, allow_network: bool = True):
"""Return a fetcher callable suitable for :class:`CatalogStack`.
When *allow_network* is False, ``http(s)://`` sources raise instead of
touching the network (used by offline tests and ``--offline`` flows).
"""
def fetch(source: CatalogSource) -> dict:
url = source.url
try:
parsed = urlparse(url)
# Keep malformed authorities and ports inside the BundlerError
# contract even when a config file was edited by hand.
_ = parsed.port
except ValueError:
raise BundlerError(
f"Catalog {source.id!r} URL is malformed: {url!r}"
) from None
scheme = parsed.scheme.lower()
if source.is_builtin:
repository_url = _BUILTIN_REPOSITORY_URLS.get(url)
if repository_url is None:
raise BundlerError(f"Unknown built-in catalog '{url}'.")
snapshot_name = _BUILTIN_PACKAGED_SNAPSHOTS[url]
if allow_network:
try:
return _http_get_json(source.id, repository_url)
except _CatalogUnavailable as exc:
# Built-in catalogs remain usable when the repository is
# temporarily unavailable; the packaged snapshot is the
# authoritative offline fallback. Only transient fetch
# failures take this path -- content/security validation
# errors propagate so a malformed response is never masked.
warnings.warn(
f"Built-in catalog '{url}' is unavailable ({exc}); "
"using the packaged snapshot.",
UserWarning,
stacklevel=2,
)
try:
return _load_packaged_catalog(snapshot_name)
except BundlerError as snapshot_exc:
raise snapshot_exc from exc
return _load_packaged_catalog(snapshot_name)
if scheme == "file":
path = _file_url_to_path(parsed)
try:
return loads_json(path.read_text(encoding="utf-8"), origin=str(path))
except FileNotFoundError:
raise BundlerError(f"Catalog file not found: {path}") from None
except (OSError, UnicodeError) as exc:
raise BundlerError(f"Could not read {path}: {exc}") from exc
if scheme == "" or _is_windows_drive_path(url):
path = Path(url)
try:
return loads_json(path.read_text(encoding="utf-8"), origin=str(path))
except FileNotFoundError:
raise BundlerError(f"Catalog file not found: {path}") from None
except (OSError, UnicodeError) as exc:
raise BundlerError(f"Could not read {path}: {exc}") from exc
if scheme in ("http", "https"):
if not allow_network:
raise BundlerError(
f"Network access disabled; cannot fetch catalog '{source.id}' "
f"from {url}."
)
_validate_remote_url(source.id, url)
return _http_get_json(source.id, url)
raise BundlerError(f"Unsupported catalog URL scheme: {url}")
return fetch
def _http_get_json(source_id: str, url: str) -> dict:
"""Fetch catalog JSON over HTTP(S) via the shared authenticated client.
Routing through :func:`specify_cli.authentication.http.open_url` gives
``auth.json`` token support and strips the ``Authorization`` header when a
redirect leaves the entry's trusted hosts or downgrades the scheme. We also
reject any redirect that leaves HTTPS (the ``redirect_validator`` runs
*before* each hop) and re-validate the final URL after redirects, so the
HTTPS/host guarantee from ``_validate_remote_url`` is preserved end to end
rather than only on the initial URL.
"""
from ..authentication.github import GitHubCliAuthError
from ..authentication.http import RedirectPolicyError, open_url
def _validate_redirect(_old_url: str, new_url: str) -> None:
_validate_remote_url(source_id, new_url)
try:
with open_url(
url,
timeout=HTTP_TIMEOUT_SECONDS,
redirect_validator=_validate_redirect,
) as response:
final_url = response.geturl()
_validate_remote_url(source_id, final_url)
raw = read_response_limited(
response,
max_bytes=MAX_JSON_CATALOG_BYTES,
error_type=BundlerError,
label=f"bundle catalog '{source_id}'",
).decode("utf-8")
except BundlerError:
# Size limits, redirect/URL validation: content or security failures,
# never transient -- must not be downgraded to availability.
raise
except ssl.SSLCertVerificationError as exc:
# TLS certificate verification is a security check, not a transient
# availability failure: never mask it with the packaged snapshot. Some
# call paths raise this directly (it subclasses OSError).
raise BundlerError(
f"Failed to fetch catalog from {url}: {exc}"
) from exc
except (GitHubCliAuthError, RedirectPolicyError) as exc:
# Authentication and redirect-policy failures are definitive, not
# transient catalog unavailability.
raise BundlerError(
f"Failed to fetch catalog from {url}: {exc}"
) from exc
except urllib.error.HTTPError as exc:
# urllib raises HTTPError for any non-2xx status; only transient
# server-side availability responses (408, 429, 5xx) fall back to the
# snapshot. Any other 4xx (404, 403, ...) is definitive and must
# surface as a hard error.
if exc.code in _TRANSIENT_HTTP_STATUS_CODES or exc.code >= 500:
raise _CatalogUnavailable(
f"Failed to fetch catalog from {url}: HTTP {exc.code} {exc.reason}"
) from exc
raise BundlerError(
f"Failed to fetch catalog from {url}: HTTP {exc.code} {exc.reason}"
) from exc
except urllib.error.URLError as exc:
# urllib wraps a TLS handshake failure as URLError(reason=<ssl error>);
# a certificate-verification failure is a security failure, not a
# transient availability problem, so it must not use the snapshot.
if isinstance(exc.reason, ssl.SSLCertVerificationError):
raise BundlerError(
f"Failed to fetch catalog from {url}: {exc.reason}"
) from exc
raise _CatalogUnavailable(
f"Failed to fetch catalog from {url}: {exc.reason}"
) from exc
except (TimeoutError, OSError) as exc:
# socket.timeout is TimeoutError; OSError covers connection resets and
# other low-level transport failures not wrapped in URLError.
raise _CatalogUnavailable(
f"Failed to fetch catalog from {url}: {exc}"
) from exc
except http.client.IncompleteRead as exc:
# A chunked response truncated mid-read is a transport failure, not a
# definitive or content error, so it should use the packaged snapshot.
raise _CatalogUnavailable(
f"Failed to fetch catalog from {url}: incomplete read ({exc})"
) from exc
except UnicodeDecodeError as exc:
# A non-UTF-8 body is a malformed response, not an availability issue.
raise BundlerError(
f"Failed to fetch catalog from {url}: response was not valid UTF-8 ({exc})"
) from exc
except Exception as exc:
raise BundlerError(f"Failed to fetch catalog from {url}: {exc}") from exc
return loads_json(raw, origin=final_url)
class DefaultPrimitiveInstaller:
"""Dispatch component install/remove to existing primitive machinery.
This adapter is intentionally thin: it owns no install logic of its own,
delegating entirely to the per-primitive managers so the bundler honours
Principle I (no duplicated primitive logic).
*allow_network* mirrors the bundle command's ``--offline`` flag: when False,
component kinds that can only be sourced from a remote catalog refuse rather
than touching the network. Bundled presets/extensions still install offline.
"""
def __init__(self, *, allow_network: bool = True) -> None:
self._allow_network = allow_network
def is_installed(self, project_root: Path, component: ComponentRef) -> bool:
manager = self._manager_for(component, project_root)
return manager.is_installed(component)
def installed_version(
self, project_root: Path, component: ComponentRef
) -> str | None:
manager = self._manager_for(component, project_root)
return manager.installed_version(component)
def bundled_patch_warning(
self, component: ComponentRef, actual: str,
) -> BundledPatchWarning | None:
from .primitives import _bundled_component_version, _bundled_pin_warning
from .versioning import is_newer_patch, same_version
packaged = _bundled_component_version(component)
if (
component.version
and packaged
and same_version(actual, packaged)
and is_newer_patch(actual, component.version)
):
return _bundled_pin_warning(component.kind[:-1].capitalize(), component, actual)
return None
def validate_source(self, project_root: Path, component: ComponentRef) -> None:
if not self._allow_network:
raise BundlerError(
f"Cannot verify catalog source '{component.source}' for "
f"{component.kind[:-1]} '{component.id}' offline; "
"re-run without --offline."
)
from .references import _resolved_in_catalog
result = _resolved_in_catalog(project_root, component)
if result is True:
return
if result is None:
detail = "catalog unreachable"
elif result is False:
detail = "the winning install-allowed catalog has no matching release"
else:
detail = result
raise BundlerError(
f"Cannot verify {component.kind[:-1]} '{component.id}' at "
f"{component.version or 'current'} from catalog "
f"'{component.source}': {detail}."
)
def install(
self, project_root: Path, component: ComponentRef,
) -> BundledPatchWarning | None:
manager = self._manager_for(component, project_root)
return manager.install(component)
def refresh(
self, project_root: Path, component: ComponentRef,
) -> BundledPatchWarning | None:
manager = self._manager_for(component, project_root)
return manager.refresh(component)
def remove(self, project_root: Path, component: ComponentRef) -> None:
manager = self._manager_for(component, project_root)
manager.remove(component)
def _manager_for(self, component: ComponentRef, project_root: Path):
# Lazy import to avoid import cycles and keep startup cheap (Principle IV).
from .primitives import primitive_manager
return primitive_manager(
component.kind, project_root, allow_network=self._allow_network
)