Skip to content

💫 Toolkit to help you get started with SDD or any other process!

28.1 KB
Raw Blame
"""External-adapter preparation and rollback shared by public lifecycle commands."""

from __future__ import annotations

import errno
import hashlib
import inspect
import os
import shutil
import stat
import tempfile
from contextlib import ExitStack, contextmanager
from contextvars import ContextVar
from functools import wraps
from pathlib import Path
from typing import Any

import typer
from rich.markup import escape

from .._console import console
from ..integration_state import (
    default_integration_key,
    installed_integration_keys,
    try_read_integration_json,
)
from . import BUILTIN_INTEGRATION_KEYS, INTEGRATION_REGISTRY, installer
from ._file_changes import file_change_observer

_init_directory: ContextVar[tuple[Path, bool, list[Path]] | None] = ContextVar(
    "external_integration_init_directory", default=None
)
_success_messages: ContextVar[list[str] | None] = ContextVar(
    "external_integration_success_messages", default=None
)
_MAX_BACKUP_BYTES = 128 * 1024 * 1024
_MAX_BACKUP_ENTRIES = 4096


def lifecycle_owns_rollback() -> bool:
    """Leave failure cleanup to the active transaction's mutation journal."""
    return file_change_observer.get() is not None


def lifecycle_success(message: str) -> None:
    """Report success only after an external package transaction commits."""
    pending = _success_messages.get()
    if pending is None:
        console.print(message)
    else:
        pending.append(message)


def initial_directory_state(path: Path) -> tuple[bool, list[Path]] | None:
    """Init validation sees the directory before the lifecycle lock created it."""
    state = _init_directory.get()
    if state is not None and state[0] == path:
        return state[1], state[2]
    return None


def _file_identity(path: Path):
    if path.is_symlink():
        return ("link", os.readlink(path))
    try:
        mode = path.stat().st_mode
    except FileNotFoundError:
        return None
    if stat.S_ISDIR(mode):
        return ("directory", tuple(
            (child.name, _file_identity(child)) for child in sorted(path.iterdir())
        ))
    if not stat.S_ISREG(mode):
        raise installer.IntegrationInstallError(f"Unsupported integration output: {path}")
    with path.open("rb") as stream:
        digest = hashlib.file_digest(stream, "sha256").hexdigest()
    return ("file", digest)


def _entry_state(path: Path) -> tuple[int, ...]:
    entry = path.lstat()
    return (
        entry.st_mode, entry.st_dev, entry.st_ino, entry.st_size,
        entry.st_mtime_ns, entry.st_ctime_ns,
    )

def _matches_directory_before_write(current: Any, expected: Any, parents: tuple[str, ...]) -> bool:
    """Allow only new empty directories on the destination's parent chain."""
    if current == expected:
        return True
    if (
        not parents or current is None or expected is None
        or current[0] != "directory" or expected[0] != "directory"
    ):
        return False
    current_entries = dict(current[1])
    expected_entries = dict(expected[1])
    name = parents[0]
    if name in current_entries and _matches_directory_before_write(
        current_entries[name], expected_entries.get(name, ("directory", ())), parents[1:],
    ):
        if name in expected_entries:
            current_entries[name] = expected_entries[name]
        else:
            del current_entries[name]
    return current_entries == expected_entries


def _initial_entries(paths: list[Path]) -> dict[Path, tuple[int, ...]]:
    """Census names and metadata without reading or copying file contents."""
    entries = {}

    def fail_walk(error: OSError) -> None:
        raise error

    for path in paths:
        try:
            entries[path] = _entry_state(path)
        except FileNotFoundError:
            continue
        if stat.S_ISDIR(entries[path][0]):
            for directory, folders, files in os.walk(path, onerror=fail_walk, followlinks=False):
                for name in folders + files:
                    child = Path(directory) / name
                    entries[child] = _entry_state(child)
    return entries


class _FileJournal:
    """Restore observed owned writes, never every file under an output root."""

    def __init__(
        self, root: Path, paths: list[Path], backup: Path, initial: dict[Path, tuple[int, ...]],
        home_scopes: tuple[Path, ...] = (),
    ):
        self.root = root
        self.paths = paths
        self.backup = backup
        self.initial = initial
        self.home_scopes = home_scopes
        self.changes: dict[Path, tuple[Path | None, Any]] = {}
        self.pending: set[Path] = set()
        self.backup_bytes = 0
        self.backup_entries = 0
        self.directory_states: dict[Path, bool] = {}
        for path in paths:
            self._remember_parents(path)

    def _remember_parents(self, path: Path) -> None:
        boundary = self.root if path.is_relative_to(self.root) else Path.home().absolute()
        for parent in path.parents:
            if parent == boundary:
                break
            if parent in self.directory_states:
                continue
            for scope in self.paths:
                if parent == scope or scope in parent.parents:
                    original = self.initial.get(parent)
                    self.directory_states[parent] = original is not None and stat.S_ISDIR(original[0])
                    break
            else:
                self.directory_states[parent] = parent.is_dir()

    def _snapshot(self, source: Path, destination: Path) -> None:
        self.backup_entries += 1
        if self.backup_entries > _MAX_BACKUP_ENTRIES:
            raise installer.IntegrationInstallError("Integration rollback snapshot budget exceeded (entries)")
        mode = source.lstat().st_mode
        if stat.S_ISLNK(mode):
            destination.symlink_to(os.readlink(source))
        elif stat.S_ISDIR(mode):
            destination.mkdir()
            for child in source.iterdir():
                self._snapshot(child, destination / child.name)
            shutil.copystat(source, destination, follow_symlinks=False)
        elif stat.S_ISREG(mode):
            if source.stat().st_size > _MAX_BACKUP_BYTES - self.backup_bytes:
                raise installer.IntegrationInstallError("Integration rollback snapshot budget exceeded (bytes)")
            with source.open("rb") as incoming, destination.open("wb") as outgoing:
                while content := incoming.read(min(65536, _MAX_BACKUP_BYTES - self.backup_bytes + 1)):
                    self.backup_bytes += len(content)
                    if self.backup_bytes > _MAX_BACKUP_BYTES:
                        raise installer.IntegrationInstallError("Integration rollback snapshot budget exceeded (bytes)")
                    outgoing.write(content)
            shutil.copystat(source, destination, follow_symlinks=False)
        else:
            raise installer.IntegrationInstallError(f"Unsupported integration output: {source}")

    def safe_path(self, path: Path, *, allow_leaf_symlink: bool = True) -> Path:
        path = path.absolute()
        boundary = self.root
        if not path.is_relative_to(boundary):
            if not any(path.is_relative_to(scope) for scope in self.home_scopes):
                raise installer.IntegrationInstallError(f"Integration output escapes trusted scopes: {path}")
            boundary = Path.home().absolute()
        relative = path.relative_to(boundary).as_posix()
        installer.safe_project_path(boundary, relative, allow_leaf_symlink=allow_leaf_symlink)
        return path

    def observe(self, path: Path, before: bool, removal: bool = False) -> None:
        path = self.safe_path(path, allow_leaf_symlink=not before or removal)
        for parent in path.parents:
            if parent not in self.changes:
                continue
            if parent in self.pending:
                return
            saved, identity = self.changes[parent]
            if identity is not None and identity[0] == "directory":
                if before:
                    if not _matches_directory_before_write(
                        _file_identity(parent), identity, path.relative_to(parent).parts[:-1],
                    ):
                        raise installer.IntegrationInstallError(
                            f"Integration output changed during the operation; refusing to overwrite {path}"
                        )
                    self.pending.add(path)
                else:
                    self.changes[parent] = (saved, _file_identity(parent))
                    self.pending.discard(path)
                return
        self._remember_parents(path)
        if (
            before and path in self.changes and path not in self.pending
            and _file_identity(path) != self.changes[path][1]
        ):
            raise installer.IntegrationInstallError(
                f"Integration output changed during the operation; refusing to overwrite {path}"
            )
        if path not in self.changes:
            saved = None
            if before:
                if path.exists() or path.is_symlink():
                    saved = self.backup / "writes" / str(len(self.changes))
                    saved.parent.mkdir(parents=True, exist_ok=True)
                    self._snapshot(path, saved)
            else:
                if not any(path == scope or scope in path.parents for scope in self.paths):
                    raise installer.IntegrationInstallError(
                        f"Integration output {path} must use manifest.record_file() "
                        "or the host file-writing helpers outside its declared output root"
                    )
                original = self.initial.get(path)
                if original is not None:
                    if _entry_state(path) == original:
                        return
                    raise installer.IntegrationInstallError(
                        f"Integration output {path} changed without before-write observation; "
                        "cannot restore its original bytes. Use manifest.record_file() "
                        "or IntegrationBase.write_file_and_record() before overwriting existing files."
                    )
            self.changes[path] = (saved, _file_identity(path))
        if not before:
            saved, _ = self.changes[path]
            self.changes[path] = (saved, _file_identity(path))
            self.pending.discard(path)
        else:
            self.pending.add(path)


def _restore_snapshots(root: Path, journal: _FileJournal) -> list[Path]:
    conflicts = []
    for path, (saved, written_identity) in reversed(tuple(journal.changes.items())):
        journal.safe_path(path)
        current_identity = _file_identity(path)
        if path in journal.pending or any(path in pending.parents for pending in journal.pending):
            original_identity = _file_identity(saved) if saved is not None else None
            if current_identity != original_identity:
                conflicts.append(path)
            continue
        if path not in journal.pending and current_identity != written_identity:
            conflicts.append(path)
            continue
        if path.is_dir() and not path.is_symlink():
            shutil.rmtree(path)
        elif path.exists() or path.is_symlink():
            path.unlink()
        if saved is not None:
            path.parent.mkdir(parents=True, exist_ok=True)
            if saved.is_symlink():
                path.symlink_to(os.readlink(saved))
            elif saved.is_dir():
                shutil.copytree(saved, path, symlinks=True)
            else:
                shutil.copy2(saved, path)
        else:
            parent = path.parent
            boundary = root if path.is_relative_to(root) else Path.home().absolute()
            while parent != boundary and not journal.directory_states[parent]:
                try:
                    parent.rmdir()
                except OSError:
                    break
                parent = parent.parent
    return conflicts


@contextmanager
def _transaction(
    root: Path, target: str | None,
    expected_state: dict[str, Any] | None, expected_records: dict[str, dict[str, Any]],
    *, creation_root: Path | None = None,
):
    """Journal operation-owned writes under an inter-process integration lock."""
    from ..shared_infra import _exclusive_project_lock
    from .manifest import IntegrationManifest

    folders = {
        ".specify/integrations", ".specify/templates", ".specify/scripts",
        ".specify/.gitignore", ".specify/presets", ".specify/extensions",
        ".specify/extensions.yml", ".specify/workflows/speckit",
    }
    manifest_leaves: set[str] = set()
    state, error = try_read_integration_json(root)
    if error:
        raise installer.IntegrationInstallError(f"Cannot read integration state: {error.detail}")
    for key in installed_integration_keys(state or {}):
        integration = INTEGRATION_REGISTRY.get(key)
        folder = (integration.config or {}).get("folder") if integration else None
        if folder:
            folders.add(folder.rstrip("/"))
        manifest_path = root / ".specify" / "integrations" / f"{key}.manifest.json"
        if manifest_path.exists():
            manifest = IntegrationManifest.load(key, root)
            for relative in manifest.files:
                installer.safe_project_path(root, relative, allow_leaf_symlink=True)
                manifest_leaves.add(relative)
    for integration in INTEGRATION_REGISTRY.values():
        if type(integration).__module__.startswith(installer._MODULE_PREFIX):
            folders.add(integration.config["folder"].rstrip("/"))
    target_integration = INTEGRATION_REGISTRY.get(target)
    target_folder = (target_integration.config or {}).get("folder") if target_integration else None
    if target_folder:
        folders.add(target_folder.rstrip("/"))
    paths = [
        installer.safe_project_path(
            root, folder, allow_leaf_symlink=folder in manifest_leaves and folder not in folders
        )
        for folder in sorted(folders | manifest_leaves)
    ]
    home_scopes = []
    for key in {*installed_integration_keys(state or {}), target} & BUILTIN_INTEGRATION_KEYS:
        integration = INTEGRATION_REGISTRY[key]
        directory = (integration.registrar_config or {}).get("dir", "")
        if directory.startswith("~/"):
            scope = installer.safe_project_path(Path.home().absolute(), directory[2:])
            home_scopes.append(scope)
    paths.extend(home_scopes)
    paths = [path for path in paths if not any(other != path and other in path.parents for other in paths)]
    root_existed = root.exists()
    lock_root = installer._trust_store(root).parent.parent
    lock_id = hashlib.sha256(os.path.normcase(str(root.resolve())).encode()).hexdigest()
    backup = Path(tempfile.mkdtemp(prefix="speckit-integration-rollback-"))
    preserve_backup = False
    try:
        # Keep the stable project-keyed lock outside an uninitialized target.
        with _exclusive_project_lock(lock_root, f".integration-install-{lock_id}.lock", context="integration"):
            if creation_root is None:
                root.mkdir(parents=True, exist_ok=True)
            else:
                from .._file_policy import prepare_init_root

                prepare_init_root(creation_root)
            current_state, state_error = try_read_integration_json(root)
            if state_error or current_state != expected_state or installer.read_records(root) != expected_records:
                raise installer.IntegrationInstallError(
                    "Integration state changed while preparing the operation; retry with the current project state"
                )
            journal = _FileJournal(root, paths, backup, _initial_entries(paths), tuple(home_scopes))
            token = file_change_observer.set(journal.observe)
            try:
                with installer.event_recovery_transaction():
                    yield
            except BaseException as operation_error:
                try:
                    # Restoration itself must not add writes to the journal.
                    file_change_observer.reset(token)
                    token = None
                    conflicts = _restore_snapshots(root, journal)
                    if conflicts:
                        preserve_backup = True
                        console.print(
                            "[yellow]Warning:[/yellow] Preserved concurrent edits or incomplete writes to "
                            f"{escape(str(conflicts))}. Recovery snapshots retained at {escape(str(backup))}"
                        )
                except (OSError, installer.IntegrationInstallError) as restore_error:
                    preserve_backup = True
                    raise installer.IntegrationInstallError(
                        f"Integration operation failed ({operation_error}); rollback failed "
                        f"({restore_error}). Recovery snapshots retained at {backup}; "
                        f"write snapshot indexes correspond to {[str(path) for path in journal.changes]}"
                    ) from operation_error
                raise
            finally:
                if token is not None:
                    file_change_observer.reset(token)
    finally:
        if not preserve_backup:
            shutil.rmtree(backup)
            if not root_existed:
                try:
                    root.rmdir()
                except FileNotFoundError:
                    pass
                except OSError as exc:
                    if exc.errno not in (errno.ENOTEMPTY, errno.EEXIST):
                        raise


def external_lifecycle(operation: str):
    """Add package preparation to existing handlers without duplicating setup."""
    def decorate(handler):
        signature = inspect.signature(handler)

        @wraps(handler)
        @installer.registry_synchronized
        def invoke(*args, **kwargs):
            values = signature.bind_partial(*args, **kwargs).arguments
            loaded = False
            if operation == "init":
                project_name = values.get("project_name")
                here = values.get("here", False) or project_name == "."
                if (here and project_name not in (None, ".")) or (not here and not project_name):
                    return handler(*args, **kwargs)
                creation_root = Path.cwd() if here else Path(project_name).absolute()
                root = creation_root
                key = values.get("integration")
            else:
                from .. import _require_specify_project

                root = _require_specify_project()
                key = values.get("target" if operation == "switch" else "key")
            recovery_token = None
            recovery_binding = None
            try:
                if operation == "init":
                    from .._file_policy import preflight_init_root

                    root = preflight_init_root(creation_root)
                records = installer.read_records(root)
                state, error = try_read_integration_json(root)
                if error:
                    raise installer.IntegrationInstallError(f"Cannot read integration state: {error.detail}")
                key = key or default_integration_key(state or {})
                try:
                    installer.load_installed_integrations(root)
                except installer.IntegrationInstallError:
                    if operation not in {"upgrade", "uninstall"} or not values.get("force") or key not in records:
                        raise
                    recovery_token = installer.recovery_exclusion.set((root.resolve(), key))
                    installer.load_installed_integrations(root)
                    recovery_binding = installer.recovery_metadata(root, key, records[key])
                    console.print(
                        f"[yellow]Warning:[/yellow] Recovering integration '{escape(key)}' "
                        "from validated ownership metadata without loading its failed implementation."
                    )
                loaded = True
                download = (
                    isinstance(key, str)
                    and key not in BUILTIN_INTEGRATION_KEYS
                    and (
                        (operation == "upgrade" and key in records)
                        or (
                            operation in {"init", "install", "switch"}
                            and key not in INTEGRATION_REGISTRY
                        )
                    )
                )
                if not records and not download:
                    return handler(*args, **kwargs)
                with ExitStack() as stack:
                    messages: list[str] = []
                    message_token = _success_messages.set(messages)
                    stack.callback(_success_messages.reset, message_token)
                    candidate = None
                    if download:
                        package, record = stack.enter_context(
                            installer.catalog_package(
                                root, key, trusted=values.get("trust_integration", False)
                            )
                        )
                        candidate = (package, record)
                        stack.enter_context(installer.prepared_adapter(root, key, package, record))
                        if recovery_token is not None:
                            installer.recovery_exclusion.set(None)
                    if operation == "init":
                        token = _init_directory.set(
                            (creation_root, root.exists(), list(root.iterdir()) if root.is_dir() else [])
                        )
                        stack.callback(_init_directory.reset, token)
                    transaction = (
                        _transaction(root, key, state, records, creation_root=creation_root)
                        if operation == "init"
                        else _transaction(root, key, state, records)
                    )
                    with transaction:
                        if recovery_token is not None and recovery_binding is None:
                            from .manifest import IntegrationManifest

                            path = root / ".specify/integrations" / f"{key}.manifest.json"
                            if path.exists():
                                manifest = IntegrationManifest.load(key, root)
                                IntegrationManifest(key, root, version=manifest.version).save()
                            console.print(
                                "[yellow]Warning:[/yellow] No local recovery ownership record; "
                                "preserving old generated files from cleanup. They may require manual cleanup. "
                                "A trusted replacement can overwrite files at its declared destination."
                            )
                        elif recovery_token is not None and recovery_binding is not None:
                            from .manifest import IntegrationManifest

                            path = root / ".specify/integrations" / f"{key}.manifest.json"
                            if path.exists():
                                manifest = IntegrationManifest.load(key, root)
                                modes = recovery_binding.get("ownership_modes", {})
                                preserved = []
                                for relative in manifest.files:
                                    mode = modes.get(relative, "shared")
                                    manifest.set_ownership(relative, mode)
                                    if mode != "whole":
                                        preserved.append(relative)
                                manifest.save()
                                if preserved:
                                    console.print(
                                        "[yellow]Warning:[/yellow] Preserving shared, partial, or unproven "
                                        f"file ownership: {escape(str(preserved))}. Manual event cleanup may be needed."
                                    )
                        try:
                            result = handler(*args, **kwargs)
                        except typer.Exit as exc:
                            if exc.exit_code != 0 or (candidate and operation == "upgrade"):
                                raise
                            if candidate:
                                exited_state, exited_error = try_read_integration_json(root)
                                if exited_error:
                                    raise installer.IntegrationInstallError(exited_error.detail) from exc
                                if key not in installed_integration_keys(exited_state or {}):
                                    raise
                            result = None
                        new_state, new_error = try_read_integration_json(root)
                        if new_error:
                            raise installer.IntegrationInstallError(
                                f"Cannot read integration state after {operation}: {new_error.detail}"
                            )
                        remaining = installed_integration_keys(new_state or {})
                        if candidate:
                            if key not in remaining:
                                raise installer.IntegrationInstallError(
                                    f"Integration {operation} did not install the requested adapter '{key}'"
                                )
                            # End temporary registration before loading durable code.
                            installer._pending_root = None
                            installer.unload_installed_integrations()
                            installer.persist_package(root, key, *candidate)
                        for removed in records.keys() - set(remaining):
                            # The uninstall command cleans up its own key. A stale
                            # record's contributions must go here, while its adapter
                            # is still loaded.
                            if operation == "uninstall" and removed != key:
                                from ._helpers import (
                                    _unregister_extensions_for_agent,
                                    _unregister_presets_for_agent,
                                )

                                _unregister_extensions_for_agent(
                                    root, removed,
                                    continuing="The adapter was removed, but extension artifacts may need manual cleanup.",
                                )
                                _unregister_presets_for_agent(
                                    root, removed,
                                    continuing="The adapter was removed, but preset artifacts may need manual cleanup.",
                                )
                            installer.remove_package(root, removed)
                        if not candidate:
                            installer.load_installed_integrations(root)
                    for message in messages:
                        console.print(message)
                    return result
            except typer.Exit:
                raise
            except (Exception, SystemExit) as exc:
                console.print(f"[red]Error:[/red] Integration {operation} failed: {escape(str(exc))}")
                raise typer.Exit(1) from exc
            finally:
                # prepared_adapter's context exits after the transaction; restore
                # durable registrations rather than leave the candidate cached.
                if loaded:
                    try:
                        installer.load_installed_integrations(root)
                    except (installer.IntegrationInstallError, OSError) as exc:
                        console.print(
                            f"[red]Error:[/red] Could not reload integration state after {operation}: {escape(str(exc))}"
                        )
                        raise typer.Exit(1) from exc
                    finally:
                        if recovery_token is not None:
                            installer.recovery_exclusion.reset(recovery_token)
                elif recovery_token is not None:
                    installer.recovery_exclusion.reset(recovery_token)

        return invoke
    return decorate