Skip to content

πŸ’« Toolkit to help you get started with SDD or any other process!

23.7 KB
Raw Blame
"""Command handler for ``specify workflow step add``.

The registered handler stays a thin orchestrator: it parses options, validates
them, and dispatches to a per-source private helper (catalog, ``--dev`` local
directory, and ``--from`` archive URL). All three sources converge on
``step/installer.py``'s single validation + staged-commit path.
"""

from __future__ import annotations

import hashlib
import sys
from typing import Annotated

from .. import _commands as cli
from . import _helpers as step_helpers
from . import step_app

_MAX_STEP_CATALOG_RESPONSE_BYTES = 50 * 1024 * 1024


def _cleanup_download_tmp_path(tmp_path: cli.Path | None) -> None:
    """Best-effort unlink of a partially-downloaded step archive temp file.

    A cleanup ``OSError`` must never replace/mask whatever error or interrupt is
    already propagating -- warn about it and keep going.
    """
    if tmp_path is None:
        return
    try:
        tmp_path.unlink(missing_ok=True)
    except OSError as cleanup_exc:
        cli.console.print(
            "[yellow]Warning:[/yellow] Could not remove temporary "
            f"step download file: {cli._escape_markup(str(cleanup_exc))} "
            f"(path: {cli._escape_markup(str(tmp_path))})"
        )


def _print_installed(step_id: str, entry: dict) -> None:
    step_name = entry.get("name") or step_id
    cli.console.print(
        "[green]βœ“[/green] Step type "
        f"'{cli._escape_markup(str(step_name))}' "
        f"({cli._escape_markup(str(step_id))}) installed"
    )
    cli.console.print(
        "  Use [cyan]specify workflow step list[/cyan] to verify the installation."
    )


def _install_from_dev(
    project_root: cli.Path, step_id: str, dev: str, *, force: bool
) -> None:
    """Install a complete step package from a local directory."""
    from . import installer

    dev_path = cli.Path(dev).expanduser()
    if dev_path.is_symlink():
        raise installer.StepInstallError(
            f"Refusing to install from a symlinked source directory: '{dev_path}'"
        )
    if not dev_path.is_dir():
        raise installer.StepInstallError(
            "--dev source must be a directory containing step.yml and "
            f"__init__.py: '{dev_path}'"
        )

    entry = installer.install_step_package(
        project_root, step_id, dev_path, source="local", force=force
    )
    _print_installed(step_id, entry)


def _install_from_url(
    project_root: cli.Path, step_id: str, from_url: str, *, force: bool
) -> None:
    """Install a step package archive from a direct URL."""
    import tempfile
    from urllib.parse import urlparse

    from rich.panel import Panel

    from specify_cli.authentication.github_http import (
        resolve_github_release_asset_api_url as _resolve_gh_asset,
    )
    from specify_cli.authentication.http import (
        github_provider_hosts as _github_provider_hosts,
    )
    from specify_cli.authentication.http import open_url as _open_url

    from . import installer

    try:
        parsed = urlparse(from_url)
        hostname = parsed.hostname
        _ = parsed.port
    except ValueError:
        raise installer.StepInstallError(
            f"Invalid URL: {from_url}"
        ) from None
    if not hostname:
        raise installer.StepInstallError(
            f"Invalid URL: {from_url}"
        )
    if not cli.is_https_or_localhost_http(from_url):
        raise installer.StepInstallError(
            "URL must use HTTPS for security. HTTP is only allowed for "
            "loopback URLs."
        )

    # Reject before the trust prompt and before any network request.
    installer.check_installable(project_root, step_id, force=force)

    # Prompt BEFORE any request (and before any spinner) so the user can see
    # and answer it; a declined prompt issues no request and exits 0.
    cli.console.print()
    cli.console.print(
        Panel(
            "[bold]You are installing a workflow step type directly from an "
            "external URL.\nA step package contains executable Python.[/bold]\n\n"
            f"URL: {cli._escape_markup(from_url)}\n\n"
            "Only install step packages from sources you trust.",
            title="[bold yellow]⚠ Untrusted Source[/bold yellow]",
            border_style="yellow",
            padding=(1, 2),
        )
    )
    cli.console.print()
    if not cli.typer.confirm("Continue with installation?", default=False):
        cli.console.print("Cancelled")
        raise cli.typer.Exit(0)

    download_url = from_url
    extra_headers = None
    tmp_path: cli.Path | None = None
    extract_tmp: tempfile.TemporaryDirectory[str] | None = None
    committed = False
    try:
        resolved_url = _resolve_gh_asset(
            from_url,
            _open_url,
            timeout=30,
            github_hosts=_github_provider_hosts(),
            redirect_validator=cli._reject_insecure_download_redirect,
        )
        if resolved_url:
            download_url = resolved_url
            extra_headers = {"Accept": "application/octet-stream"}

        with _open_url(
            download_url,
            timeout=30,
            extra_headers=extra_headers,
            redirect_validator=cli._reject_insecure_download_redirect,
        ) as resp:
            final_url = resp.geturl()
            if not cli.is_https_or_localhost_http(final_url):
                raise installer.StepInstallError(
                    f"URL redirected to non-HTTPS: {final_url}"
                )
            content_type = (
                resp.getheader("Content-Type")
                if hasattr(resp, "getheader")
                else None
            )
            declarations = [
                ("requested URL", from_url, cli.archive_format_from_name(from_url)),
                ("final URL", final_url, cli.archive_format_from_name(final_url)),
                (
                    "Content-Type",
                    content_type or "",
                    cli.archive_format_from_content_type(content_type),
                ),
            ]
            recognized = [item for item in declarations if item[2] is not None]
            archive_format = recognized[0][2] if recognized else None
            if any(item[2] != archive_format for item in recognized):
                details = ", ".join(
                    f"{label} declares {declared}"
                    for label, _value, declared in recognized
                )
                raise installer.StepInstallError(
                    f"Archive format mismatch: {details}"
                )
            downloaded = cli.read_response_limited(
                resp,
                error_type=ValueError,
                label="step archive download",
            )

        with tempfile.NamedTemporaryFile(
            suffix=cli.archive_suffix(archive_format) if archive_format else ".archive",
            delete=False,
        ) as tmp:
            tmp_path = cli.Path(tmp.name)
            tmp.write(downloaded)

        try:
            extract_tmp = tempfile.TemporaryDirectory(
                prefix="speckit-step-archive-"
            )
        except OSError as exc:
            raise installer.StepInstallError(
                f"Failed to create temporary step archive directory: {exc}"
            ) from exc
        extracted_root = cli.Path(extract_tmp.name)
        try:
            # safe_extract_archive re-detects and confirms the archive bytes.
            cli.safe_extract_archive(
                tmp_path,
                extracted_root,
                source_name=recognized[0][1] if recognized else None,
                content_type=content_type,
            )
            package_root = installer.resolve_package_root(extracted_root)
            entry = installer.install_step_package(
                project_root,
                step_id,
                package_root,
                source="url",
                force=force,
            )
            committed = True
        finally:
            primary_error = sys.exc_info()[1]
            try:
                extract_tmp.cleanup()
            except OSError as cleanup_exc:
                if extract_tmp.name and cli.Path(extract_tmp.name).exists():
                    detail = (
                        f"{cli._escape_markup(str(cleanup_exc))} "
                        f"(path: {cli._escape_markup(extract_tmp.name)})"
                    )
                    cli.console.print(
                        "[yellow]Warning:[/yellow] Could not remove temporary "
                        f"step archive directory: {detail}"
                    )
                if primary_error is None and not committed:
                    raise installer.StepInstallError(
                        "Failed to remove temporary step archive directory: "
                        f"{cleanup_exc}"
                    ) from cleanup_exc
                # Do not raise from cleanup: the primary installation error
                # (if any) is already propagating, and after commit the install
                # has succeeded. The warning above reports the residual path.
    except cli.typer.Exit:
        raise
    except installer.StepInstallError:
        raise
    except Exception as exc:
        raise installer.StepInstallError(
            f"Failed to install step from URL: {exc}"
        ) from exc
    finally:
        _cleanup_download_tmp_path(tmp_path)

    _print_installed(step_id, entry)


def _install_from_catalog(
    project_root: cli.Path, step_id: str, *, force: bool, version: str | None = None,
    selected_info: dict | None = None,
) -> None:
    """Install a step package from the step catalog.

    The catalog fetch (URL/derivation/count preflight) stays a catalog concern;
    the materialized files are then handed to the shared installer.
    """
    import tempfile

    from . import installer
    from .catalog import StepCatalog, StepCatalogError
    from .catalog._versions import validate_checksums

    if selected_info is not None:
        info = selected_info
    else:
        catalog = StepCatalog(project_root)
        try:
            info = (
                catalog.get_step_info(step_id, version=version)
                if version is not None
                else catalog.get_step_info(step_id)
            )
        except StepCatalogError as exc:
            raise installer.StepInstallError(str(exc)) from exc

    if not info:
        raise installer.StepInstallError(
            f"Step type '{step_id}' version '{version}' not found in the winning catalog"
            if version is not None
            else f"Step type '{step_id}' not found in catalog"
        )

    if not info.get("_install_allowed", True):
        cli.console.print(
            "[yellow]Warning:[/yellow] Step type "
            f"'{cli._escape_markup(step_id)}' is from a "
            "discovery-only catalog"
        )
        cli.console.print("Direct installation is not enabled for this catalog source.")
        raise cli.typer.Exit(1)

    try:
        validate_checksums(info, step_id, required=version is not None)
    except StepCatalogError as exc:
        raise installer.StepInstallError(str(exc)) from exc
    checksums = info.get("sha256")

    # Reject built-in collisions and duplicates before any download.
    installer.check_installable(project_root, step_id, force=force)

    declared_step_yml_url = info.get("step_yml_url")
    if declared_step_yml_url is not None and not isinstance(declared_step_yml_url, str):
        raise installer.StepInstallError(
            f"Catalog entry for '{step_id}' has a malformed step.yml URL; "
            "expected a non-empty string"
        )
    step_yml_url = declared_step_yml_url or info.get("url")
    if step_yml_url is None or (
        isinstance(step_yml_url, str) and not step_yml_url.strip()
    ):
        raise installer.StepInstallError(
            f"Catalog entry for '{step_id}' has no URL"
        )
    if not isinstance(step_yml_url, str):
        raise installer.StepInstallError(
            f"Catalog entry for '{step_id}' has a malformed step.yml URL; "
            "expected a non-empty string"
        )

    # Derive __init__.py URL: replace trailing step.yml with __init__.py or use
    # explicit init_url if provided.
    init_url = info.get("init_url")
    if init_url is not None and (not isinstance(init_url, str) or not init_url.strip()):
        raise installer.StepInstallError(
            f"Catalog entry for '{step_id}' has a malformed __init__.py URL; "
            "expected a non-empty string"
        )
    if not init_url:
        if step_yml_url.endswith("step.yml"):
            init_url = step_yml_url[: -len("step.yml")] + "__init__.py"
        else:
            raise installer.StepInstallError(
                f"Cannot derive __init__.py URL from '{step_yml_url}'. "
                "Catalog entry should provide 'init_url' or a 'url' ending in "
                "'step.yml'."
            )

    # Preflight the declared file count before creating a staging directory or
    # issuing any request. The two required files are always part of the
    # package; duplicate declarations for them in extra_files are ignored below
    # and do not count twice.
    extra_files = info.get("extra_files")
    if extra_files is not None and not isinstance(extra_files, dict):
        cli.console.print(
            "[yellow]Warning:[/yellow] Catalog entry 'extra_files' is not a mapping; "
            "additional package files will not be downloaded."
        )
        extra_files = {}

    def _is_required_package_file(rel_path: object) -> bool:
        """Match portable path/case aliases of the two required package files."""
        if not isinstance(rel_path, str):
            return False
        parts = cli.PurePosixPath(rel_path.replace("\\", "/")).parts
        return len(parts) == 1 and parts[0].casefold() in {
            "step.yml",
            "__init__.py",
        }

    declared_extra_count = sum(
        1 for rel_path in (extra_files or {}) if not _is_required_package_file(rel_path)
    )
    package_file_count = 2 + declared_extra_count
    if package_file_count > installer._MAX_STEP_PACKAGE_FILES:
        raise installer.StepInstallError(
            f"Step package declares {package_file_count} files, exceeding the "
            f"{installer._MAX_STEP_PACKAGE_FILES}-entry limit (files and "
            "directories combined)"
        )

    from specify_cli.authentication.http import open_url as _open_url

    def _safe_fetch(url: str) -> bytes:
        if not cli.is_https_or_localhost_http(url):
            raise ValueError(f"Refusing to fetch from non-HTTPS URL: {url}")
        with _open_url(
            url, timeout=30, redirect_validator=cli._reject_insecure_download_redirect
        ) as resp:
            final_url = resp.geturl()
            if not cli.is_https_or_localhost_http(final_url):
                raise ValueError(f"Redirect to non-HTTPS URL: {final_url}")
            return cli.read_response_limited(
                resp,
                max_bytes=_MAX_STEP_CATALOG_RESPONSE_BYTES,
                error_type=ValueError,
                label="step package response",
            )

    def _fetch_checked(url: str, name: str) -> bytes:
        content = _safe_fetch(url)
        if checksums and (
            hashlib.sha256(content).hexdigest().lower() != checksums[name].lower()
        ):
            raise installer.StepInstallError(f"SHA-256 checksum mismatch for '{name}'")
        return content

    try:
        package_tmp = tempfile.TemporaryDirectory(prefix="speckit-step-package-")
    except OSError as exc:
        raise installer.StepInstallError(
            f"Failed to create temporary step package directory: {exc}"
        ) from exc
    package_dir = cli.Path(package_tmp.name)
    committed = False
    try:
        try:
            step_yml_content = _fetch_checked(step_yml_url, "step.yml")
            init_py_content = _fetch_checked(init_url, "__init__.py")
        except Exception as exc:
            raise installer.StepInstallError(
                f"Failed to download step files: {exc}"
            ) from exc

        package_bytes = len(step_yml_content) + len(init_py_content)
        if package_bytes > installer._MAX_STEP_PACKAGE_BYTES:
            raise installer.StepInstallError(
                f"Step package exceeds the "
                f"{installer._MAX_STEP_PACKAGE_BYTES}-byte total size limit"
            )

        try:
            (package_dir / "step.yml").write_bytes(step_yml_content)
            (package_dir / "__init__.py").write_bytes(init_py_content)
        except OSError as exc:
            raise installer.StepInstallError(
                f"Failed to write step files to staging directory: {exc}"
            ) from exc

        # Optionally download additional package files declared in the catalog
        # entry (e.g. helper modules). Each entry in ``extra_files`` is a mapping
        # of relative-path β†’ URL. Paths are validated to stay within the step
        # package directory to prevent path-traversal attacks.
        for rel_path, file_url in (extra_files or {}).items():
            if not isinstance(rel_path, str) or not rel_path.strip():
                raise installer.StepInstallError(
                    "Catalog entry 'extra_files' contains an empty or non-string "
                    "path key"
                )
            if _is_required_package_file(rel_path):
                continue  # already written above
            path_parts = cli.Path(rel_path).parts
            if not path_parts or any(seg in ("", ".", "..") for seg in path_parts):
                raise installer.StepInstallError(
                    f"extra_files path '{rel_path}' is not a valid relative file path"
                )
            if not isinstance(file_url, str) or not file_url.strip():
                raise installer.StepInstallError(
                    f"extra_files entry '{rel_path}' has an empty or non-string URL"
                )
            resolved_base = package_dir.resolve()
            dest = (package_dir / rel_path).resolve()
            try:
                dest.relative_to(resolved_base)
            except ValueError:
                raise installer.StepInstallError(
                    f"extra_files path '{rel_path}' is outside the step package "
                    "directory"
                ) from None
            try:
                file_content = _fetch_checked(file_url, rel_path)
            except Exception as exc:
                raise installer.StepInstallError(
                    f"Failed to download extra file '{rel_path}': {exc}"
                ) from exc
            package_bytes += len(file_content)
            if package_bytes > installer._MAX_STEP_PACKAGE_BYTES:
                raise installer.StepInstallError(
                    f"Step package exceeds the "
                    f"{installer._MAX_STEP_PACKAGE_BYTES}-byte total size limit"
                )
            try:
                dest.parent.mkdir(parents=True, exist_ok=True)
                dest.write_bytes(file_content)
            except OSError as exc:
                raise installer.StepInstallError(
                    f"Failed to write extra file '{rel_path}': {exc}"
                ) from exc

        if version is not None or "releases" in info:
            from packaging.version import InvalidVersion, Version

            step_meta = installer.validate_step_package(package_dir, step_id)
            declared_version = step_meta.get("version")
            try:
                matches = (
                    isinstance(declared_version, str)
                    and Version(declared_version) == Version(info["version"])
                )
            except (InvalidVersion, TypeError, KeyError):
                matches = False
            if not matches:
                raise installer.StepInstallError(
                    f"step.yml version ({declared_version!r}) does not match "
                    f"catalog version ({info.get('version')!r})"
                )

        entry = installer.install_step_package(
            project_root,
            step_id,
            package_dir,
            source="catalog",
            catalog_name=info.get("_catalog_name", ""),
            catalog_metadata=info,
            force=force,
        )
        committed = True
    finally:
        primary_error = sys.exc_info()[1]
        try:
            package_tmp.cleanup()
        except OSError as cleanup_exc:
            if package_tmp.name and cli.Path(package_tmp.name).exists():
                detail = (
                    f"{cli._escape_markup(str(cleanup_exc))} "
                    f"(path: {cli._escape_markup(package_tmp.name)})"
                )
                cli.console.print(
                    "[yellow]Warning:[/yellow] Could not remove temporary "
                    f"step package directory: {detail}"
                )
            if primary_error is None and not committed:
                raise installer.StepInstallError(
                    "Failed to remove temporary step package directory: "
                    f"{cleanup_exc}"
                ) from cleanup_exc
            # Do not raise from cleanup: preserve a primary download/install
            # error, or report a successful install with a warning only.

    _print_installed(step_id, entry)


def _install_preselected_step(
    step_id: str, *, version: str | None, selected_info: dict,
) -> None:
    """Install a bundle-selected release with the normal step ID preflight."""
    project_root = cli._require_specify_project()
    step_helpers._validate_step_id_or_exit(step_id)
    _install_from_catalog(
        project_root, step_id, force=False, version=version,
        selected_info=selected_info,
    )


@step_app.command("add")
def workflow_step_add(
    step_id: str = cli.typer.Argument(..., help="Step type ID"),
    dev: Annotated[str | None, cli.typer.Option("--dev", help="Install from a local step package directory")] = None,
    from_url: Annotated[str | None, cli.typer.Option("--from", help="Install from a .zip/.tar.gz/.tgz archive URL")] = None,
    force: Annotated[bool, cli.typer.Option("--force", help="Replace an existing installation")] = False,
    version: Annotated[
        str | None,
        cli.typer.Option("--version", help="Install an exact catalog release"),
    ] = None,
):
    """Install a custom step type from the catalog, a local directory, or a URL."""
    from . import installer

    project_root = cli._require_specify_project()

    if dev is not None and from_url is not None:
        cli.console.print(
            "[red]Error:[/red] --dev and --from are mutually exclusive"
        )
        raise cli.typer.Exit(1)
    if dev is not None and not dev.strip():
        cli.console.print("[red]Error:[/red] --dev value must not be empty")
        raise cli.typer.Exit(1)
    if from_url is not None and not from_url.strip():
        cli.console.print("[red]Error:[/red] --from value must not be empty")
        raise cli.typer.Exit(1)
    if version is not None and (dev is not None or from_url is not None):
        cli.console.print("[red]Error:[/red] --version requires a catalog step ID")
        raise cli.typer.Exit(1)
    if version is not None and not version.strip():
        cli.console.print("[red]Error:[/red] --version value must not be empty")
        raise cli.typer.Exit(1)

    step_helpers._validate_step_id_or_exit(step_id)

    try:
        if dev is not None:
            _install_from_dev(project_root, step_id, dev, force=force)
        elif from_url is not None:
            _install_from_url(project_root, step_id, from_url, force=force)
        else:
            _install_from_catalog(project_root, step_id, force=force, version=version)
    except installer.StepInstallError as exc:
        notes = getattr(exc, "__notes__", ())
        for note in notes:
            cli.console.print(
                f"[yellow]Warning:[/yellow] {cli._escape_markup(note)}"
            )
        cli.console.print(f"[red]Error:[/red] {cli._escape_markup(str(exc))}")
        raise cli.typer.Exit(1) from exc