Skip to content

πŸ’« Toolkit to help you get started with SDD or any other process!

16.6 KB
Raw Blame
"""Concrete adapters: catalog fetching and primitive installation.

These wire the bundler's injectable seams to the real environment:

* :func:`make_catalog_fetcher` returns an offline-first fetcher that reads
  built-in catalogs and local/pinned file URLs without network, and falls back
  to a timeout-bounded HTTP GET only for ``http(s)://`` sources.
* :class:`DefaultPrimitiveInstaller` dispatches component install/remove to the
  existing Spec Kit primitive machinery in-process.
"""
from __future__ import annotations

import http.client
import re
import ssl
import urllib.error
import warnings
from pathlib import Path
from typing import TYPE_CHECKING
from urllib.parse import ParseResult, urlparse
from urllib.request import url2pathname

from .._assets import _locate_core_pack, _repo_root
from .._download_security import MAX_JSON_CATALOG_BYTES, read_response_limited
from . import BundlerError
from .catalogs import CatalogSource
from .manifest import ComponentRef
from .yamlio import loads_json

if TYPE_CHECKING:
    from .primitives import BundledPatchWarning

COMMUNITY_CATALOG_URL = (
    "https://raw.githubusercontent.com/github/spec-kit/main/"
    "bundles/catalog.community.json"
)
FIRSTPARTY_CATALOG_URL = (
    "https://raw.githubusercontent.com/github/spec-kit/main/"
    "bundles/catalog.json"
)

# Built-in catalogs are resolved directly by URL. ``builtin://default`` is the
# repository-shipped first-party bundle catalog; ``builtin://community`` is the
# community catalog. Both are fetched from the repository online and fall back
# to the packaged wheel snapshot offline so discovery works without network.
_BUILTIN_REPOSITORY_URLS: dict[str, str] = {
    "builtin://default": FIRSTPARTY_CATALOG_URL,
    "builtin://community": COMMUNITY_CATALOG_URL,
}
_BUILTIN_PACKAGED_SNAPSHOTS: dict[str, str] = {
    "builtin://default": "catalog.json",
    "builtin://community": "catalog.community.json",
}

HTTP_TIMEOUT_SECONDS = 10
_TRANSIENT_HTTP_STATUS_CODES = (408, 429)


class _CatalogUnavailable(BundlerError):
    """A built-in catalog could not be reached (transport/availability failure).

    Marks only transient fetch failures β€” connection/DNS errors, timeouts,
    truncated responses, and availability HTTP responses (408, 429, 5xx) β€” so
    the built-in catalog fallback does not swallow content or security
    validation failures (malformed JSON, oversized or non-UTF-8 bodies, unsafe
    redirects, TLS certificate verification failures, other HTTP 4xx).
    """


# Windows absolute paths like ``C:\catalog.json`` parse with a single-letter
# ``scheme`` under urlparse; treat them as local files rather than URLs.
_WINDOWS_DRIVE_RE = re.compile(r"^[A-Za-z]:[\\/]")


def _is_windows_drive_path(url: str) -> bool:
    return bool(_WINDOWS_DRIVE_RE.match(url))


def _file_url_to_path(parsed: ParseResult) -> Path:
    """Convert a ``file://`` URL to a local path.

    Uses ``url2pathname`` for percent-decoding and OS-correct separators, and
    preserves ``netloc`` so UNC paths (``file://server/share``) and Windows
    drive URLs (``file:///C:/x``) resolve correctly instead of dropping host
    or producing ``/C:/x``.
    """
    netloc = parsed.netloc
    if netloc and netloc.lower() != "localhost":
        # UNC share: file://server/share/... -> \\server\share\...
        return Path(url2pathname(f"//{netloc}{parsed.path}"))
    return Path(url2pathname(parsed.path))


def _validate_remote_url(source_id: str, url: str) -> None:
    """Restrict remote catalogs to HTTPS (HTTP only for localhost) with a host.

    Mirrors ``specify_cli.catalogs`` URL validation to avoid MITM/downgrade
    issues before any network call.
    """
    # A malformed authority (e.g. an unclosed IPv6 bracket ``https://[::1``)
    # makes urlparse / hostname access raise ValueError. This function's
    # contract is to raise BundlerError for a bad URL, so surface that as a
    # clean error rather than leaking a raw ValueError to the caller.
    try:
        parsed = urlparse(url)
        hostname = parsed.hostname
        # Accessing ``port`` performs urllib's syntax/range validation.
        _ = parsed.port
    except ValueError:
        raise BundlerError(
            f"Catalog '{source_id}' URL is malformed: {url}"
        ) from None
    is_localhost = hostname in ("localhost", "127.0.0.1", "::1")
    if parsed.scheme != "https" and not (parsed.scheme == "http" and is_localhost):
        raise BundlerError(
            f"Catalog '{source_id}' URL must use HTTPS (got {parsed.scheme}://). "
            "HTTP is only allowed for localhost."
        )
    # Check hostname, not netloc: netloc is truthy for host-less URLs like
    # "https://:8080" or "https://user@...", so requiring netloc would let
    # those through even though they carry no host. hostname is None in those
    # cases. Mirrors the fix in ``specify_cli.catalogs`` (#3210).
    if not hostname:
        raise BundlerError(
            f"Catalog '{source_id}' URL must be a valid URL with a host: {url}"
        )


def _load_packaged_catalog(filename: str) -> dict:
    """Load a packaged bundle catalog snapshot from the wheel or repo root."""
    core_pack = _locate_core_pack()
    path = (
        core_pack / "bundles" / filename
        if core_pack is not None
        else _repo_root() / "bundles" / filename
    )
    if not path.is_file():
        raise BundlerError(f"Bundled catalog not found: {path}")
    return loads_json(path.read_text(encoding="utf-8"), origin=str(path))


def make_catalog_fetcher(*, allow_network: bool = True):
    """Return a fetcher callable suitable for :class:`CatalogStack`.

    When *allow_network* is False, ``http(s)://`` sources raise instead of
    touching the network (used by offline tests and ``--offline`` flows).
    """

    def fetch(source: CatalogSource) -> dict:
        url = source.url
        try:
            parsed = urlparse(url)
            # Keep malformed authorities and ports inside the BundlerError
            # contract even when a config file was edited by hand.
            _ = parsed.port
        except ValueError:
            raise BundlerError(
                f"Catalog {source.id!r} URL is malformed: {url!r}"
            ) from None
        scheme = parsed.scheme.lower()

        if source.is_builtin:
            repository_url = _BUILTIN_REPOSITORY_URLS.get(url)
            if repository_url is None:
                raise BundlerError(f"Unknown built-in catalog '{url}'.")
            snapshot_name = _BUILTIN_PACKAGED_SNAPSHOTS[url]
            if allow_network:
                try:
                    return _http_get_json(source.id, repository_url)
                except _CatalogUnavailable as exc:
                    # Built-in catalogs remain usable when the repository is
                    # temporarily unavailable; the packaged snapshot is the
                    # authoritative offline fallback. Only transient fetch
                    # failures take this path -- content/security validation
                    # errors propagate so a malformed response is never masked.
                    warnings.warn(
                        f"Built-in catalog '{url}' is unavailable ({exc}); "
                        "using the packaged snapshot.",
                        UserWarning,
                        stacklevel=2,
                    )
                    try:
                        return _load_packaged_catalog(snapshot_name)
                    except BundlerError as snapshot_exc:
                        raise snapshot_exc from exc
            return _load_packaged_catalog(snapshot_name)

        if scheme == "file":
            path = _file_url_to_path(parsed)
            try:
                return loads_json(path.read_text(encoding="utf-8"), origin=str(path))
            except FileNotFoundError:
                raise BundlerError(f"Catalog file not found: {path}") from None
            except (OSError, UnicodeError) as exc:
                raise BundlerError(f"Could not read {path}: {exc}") from exc

        if scheme == "" or _is_windows_drive_path(url):
            path = Path(url)
            try:
                return loads_json(path.read_text(encoding="utf-8"), origin=str(path))
            except FileNotFoundError:
                raise BundlerError(f"Catalog file not found: {path}") from None
            except (OSError, UnicodeError) as exc:
                raise BundlerError(f"Could not read {path}: {exc}") from exc

        if scheme in ("http", "https"):
            if not allow_network:
                raise BundlerError(
                    f"Network access disabled; cannot fetch catalog '{source.id}' "
                    f"from {url}."
                )
            _validate_remote_url(source.id, url)
            return _http_get_json(source.id, url)

        raise BundlerError(f"Unsupported catalog URL scheme: {url}")

    return fetch


def _http_get_json(source_id: str, url: str) -> dict:
    """Fetch catalog JSON over HTTP(S) via the shared authenticated client.

    Routing through :func:`specify_cli.authentication.http.open_url` gives
    ``auth.json`` token support and strips the ``Authorization`` header when a
    redirect leaves the entry's trusted hosts or downgrades the scheme. We also
    reject any redirect that leaves HTTPS (the ``redirect_validator`` runs
    *before* each hop) and re-validate the final URL after redirects, so the
    HTTPS/host guarantee from ``_validate_remote_url`` is preserved end to end
    rather than only on the initial URL.
    """
    from ..authentication.github import GitHubCliAuthError
    from ..authentication.http import RedirectPolicyError, open_url

    def _validate_redirect(_old_url: str, new_url: str) -> None:
        _validate_remote_url(source_id, new_url)

    try:
        with open_url(
            url,
            timeout=HTTP_TIMEOUT_SECONDS,
            redirect_validator=_validate_redirect,
        ) as response:
            final_url = response.geturl()
            _validate_remote_url(source_id, final_url)
            raw = read_response_limited(
                response,
                max_bytes=MAX_JSON_CATALOG_BYTES,
                error_type=BundlerError,
                label=f"bundle catalog '{source_id}'",
            ).decode("utf-8")
    except BundlerError:
        # Size limits, redirect/URL validation: content or security failures,
        # never transient -- must not be downgraded to availability.
        raise
    except ssl.SSLCertVerificationError as exc:
        # TLS certificate verification is a security check, not a transient
        # availability failure: never mask it with the packaged snapshot. Some
        # call paths raise this directly (it subclasses OSError).
        raise BundlerError(
            f"Failed to fetch catalog from {url}: {exc}"
        ) from exc
    except (GitHubCliAuthError, RedirectPolicyError) as exc:
        # Authentication and redirect-policy failures are definitive, not
        # transient catalog unavailability.
        raise BundlerError(
            f"Failed to fetch catalog from {url}: {exc}"
        ) from exc
    except urllib.error.HTTPError as exc:
        # urllib raises HTTPError for any non-2xx status; only transient
        # server-side availability responses (408, 429, 5xx) fall back to the
        # snapshot. Any other 4xx (404, 403, ...) is definitive and must
        # surface as a hard error.
        if exc.code in _TRANSIENT_HTTP_STATUS_CODES or exc.code >= 500:
            raise _CatalogUnavailable(
                f"Failed to fetch catalog from {url}: HTTP {exc.code} {exc.reason}"
            ) from exc
        raise BundlerError(
            f"Failed to fetch catalog from {url}: HTTP {exc.code} {exc.reason}"
        ) from exc
    except urllib.error.URLError as exc:
        # urllib wraps a TLS handshake failure as URLError(reason=<ssl error>);
        # a certificate-verification failure is a security failure, not a
        # transient availability problem, so it must not use the snapshot.
        if isinstance(exc.reason, ssl.SSLCertVerificationError):
            raise BundlerError(
                f"Failed to fetch catalog from {url}: {exc.reason}"
            ) from exc
        raise _CatalogUnavailable(
            f"Failed to fetch catalog from {url}: {exc.reason}"
        ) from exc
    except (TimeoutError, OSError) as exc:
        # socket.timeout is TimeoutError; OSError covers connection resets and
        # other low-level transport failures not wrapped in URLError.
        raise _CatalogUnavailable(
            f"Failed to fetch catalog from {url}: {exc}"
        ) from exc
    except http.client.IncompleteRead as exc:
        # A chunked response truncated mid-read is a transport failure, not a
        # definitive or content error, so it should use the packaged snapshot.
        raise _CatalogUnavailable(
            f"Failed to fetch catalog from {url}: incomplete read ({exc})"
        ) from exc
    except UnicodeDecodeError as exc:
        # A non-UTF-8 body is a malformed response, not an availability issue.
        raise BundlerError(
            f"Failed to fetch catalog from {url}: response was not valid UTF-8 ({exc})"
        ) from exc
    except Exception as exc:
        raise BundlerError(f"Failed to fetch catalog from {url}: {exc}") from exc
    return loads_json(raw, origin=final_url)


class DefaultPrimitiveInstaller:
    """Dispatch component install/remove to existing primitive machinery.

    This adapter is intentionally thin: it owns no install logic of its own,
    delegating entirely to the per-primitive managers so the bundler honours
    Principle I (no duplicated primitive logic).

    *allow_network* mirrors the bundle command's ``--offline`` flag: when False,
    component kinds that can only be sourced from a remote catalog refuse rather
    than touching the network. Bundled presets/extensions still install offline.
    """

    def __init__(self, *, allow_network: bool = True) -> None:
        self._allow_network = allow_network

    def is_installed(self, project_root: Path, component: ComponentRef) -> bool:
        manager = self._manager_for(component, project_root)
        return manager.is_installed(component)

    def installed_version(
        self, project_root: Path, component: ComponentRef
    ) -> str | None:
        manager = self._manager_for(component, project_root)
        return manager.installed_version(component)

    def bundled_patch_warning(
        self, component: ComponentRef, actual: str,
    ) -> BundledPatchWarning | None:
        from .primitives import _bundled_component_version, _bundled_pin_warning
        from .versioning import is_newer_patch, same_version

        packaged = _bundled_component_version(component)
        if (
            component.version
            and packaged
            and same_version(actual, packaged)
            and is_newer_patch(actual, component.version)
        ):
            return _bundled_pin_warning(component.kind[:-1].capitalize(), component, actual)
        return None

    def validate_source(self, project_root: Path, component: ComponentRef) -> None:
        if not self._allow_network:
            raise BundlerError(
                f"Cannot verify catalog source '{component.source}' for "
                f"{component.kind[:-1]} '{component.id}' offline; "
                "re-run without --offline."
            )
        from .references import _resolved_in_catalog

        result = _resolved_in_catalog(project_root, component)
        if result is True:
            return
        if result is None:
            detail = "catalog unreachable"
        elif result is False:
            detail = "the winning install-allowed catalog has no matching release"
        else:
            detail = result
        raise BundlerError(
            f"Cannot verify {component.kind[:-1]} '{component.id}' at "
            f"{component.version or 'current'} from catalog "
            f"'{component.source}': {detail}."
        )

    def install(
        self, project_root: Path, component: ComponentRef,
    ) -> BundledPatchWarning | None:
        manager = self._manager_for(component, project_root)
        return manager.install(component)

    def refresh(
        self, project_root: Path, component: ComponentRef,
    ) -> BundledPatchWarning | None:
        manager = self._manager_for(component, project_root)
        return manager.refresh(component)

    def remove(self, project_root: Path, component: ComponentRef) -> None:
        manager = self._manager_for(component, project_root)
        manager.remove(component)

    def _manager_for(self, component: ComponentRef, project_root: Path):
        # Lazy import to avoid import cycles and keep startup cheap (Principle IV).
        from .primitives import primitive_manager

        return primitive_manager(
            component.kind, project_root, allow_network=self._allow_network
        )