💫 Toolkit to help you get started with SDD or any other process!
"""External-adapter preparation and rollback shared by public lifecycle commands."""
from __future__ import annotations
import errno
import hashlib
import inspect
import os
import shutil
import stat
import tempfile
from contextlib import ExitStack, contextmanager
from contextvars import ContextVar
from functools import wraps
from pathlib import Path
from typing import Any
import typer
from rich.markup import escape
from .._console import console
from ..integration_state import (
default_integration_key,
installed_integration_keys,
try_read_integration_json,
)
from . import BUILTIN_INTEGRATION_KEYS, INTEGRATION_REGISTRY, installer
from ._file_changes import file_change_observer
_init_directory: ContextVar[tuple[Path, bool, list[Path]] | None] = ContextVar(
"external_integration_init_directory", default=None
)
_success_messages: ContextVar[list[str] | None] = ContextVar(
"external_integration_success_messages", default=None
)
_MAX_BACKUP_BYTES = 128 * 1024 * 1024
_MAX_BACKUP_ENTRIES = 4096
def lifecycle_owns_rollback() -> bool:
"""Leave failure cleanup to the active transaction's mutation journal."""
return file_change_observer.get() is not None
def lifecycle_success(message: str) -> None:
"""Report success only after an external package transaction commits."""
pending = _success_messages.get()
if pending is None:
console.print(message)
else:
pending.append(message)
def initial_directory_state(path: Path) -> tuple[bool, list[Path]] | None:
"""Init validation sees the directory before the lifecycle lock created it."""
state = _init_directory.get()
if state is not None and state[0] == path:
return state[1], state[2]
return None
def _file_identity(path: Path):
if path.is_symlink():
return ("link", os.readlink(path))
try:
mode = path.stat().st_mode
except FileNotFoundError:
return None
if stat.S_ISDIR(mode):
return ("directory", tuple(
(child.name, _file_identity(child)) for child in sorted(path.iterdir())
))
if not stat.S_ISREG(mode):
raise installer.IntegrationInstallError(f"Unsupported integration output: {path}")
with path.open("rb") as stream:
digest = hashlib.file_digest(stream, "sha256").hexdigest()
return ("file", digest)
def _entry_state(path: Path) -> tuple[int, ...]:
entry = path.lstat()
return (
entry.st_mode, entry.st_dev, entry.st_ino, entry.st_size,
entry.st_mtime_ns, entry.st_ctime_ns,
)
def _matches_directory_before_write(current: Any, expected: Any, parents: tuple[str, ...]) -> bool:
"""Allow only new empty directories on the destination's parent chain."""
if current == expected:
return True
if (
not parents or current is None or expected is None
or current[0] != "directory" or expected[0] != "directory"
):
return False
current_entries = dict(current[1])
expected_entries = dict(expected[1])
name = parents[0]
if name in current_entries and _matches_directory_before_write(
current_entries[name], expected_entries.get(name, ("directory", ())), parents[1:],
):
if name in expected_entries:
current_entries[name] = expected_entries[name]
else:
del current_entries[name]
return current_entries == expected_entries
def _initial_entries(paths: list[Path]) -> dict[Path, tuple[int, ...]]:
"""Census names and metadata without reading or copying file contents."""
entries = {}
def fail_walk(error: OSError) -> None:
raise error
for path in paths:
try:
entries[path] = _entry_state(path)
except FileNotFoundError:
continue
if stat.S_ISDIR(entries[path][0]):
for directory, folders, files in os.walk(path, onerror=fail_walk, followlinks=False):
for name in folders + files:
child = Path(directory) / name
entries[child] = _entry_state(child)
return entries
class _FileJournal:
"""Restore observed owned writes, never every file under an output root."""
def __init__(
self, root: Path, paths: list[Path], backup: Path, initial: dict[Path, tuple[int, ...]],
home_scopes: tuple[Path, ...] = (),
):
self.root = root
self.paths = paths
self.backup = backup
self.initial = initial
self.home_scopes = home_scopes
self.changes: dict[Path, tuple[Path | None, Any]] = {}
self.pending: set[Path] = set()
self.backup_bytes = 0
self.backup_entries = 0
self.directory_states: dict[Path, bool] = {}
for path in paths:
self._remember_parents(path)
def _remember_parents(self, path: Path) -> None:
boundary = self.root if path.is_relative_to(self.root) else Path.home().absolute()
for parent in path.parents:
if parent == boundary:
break
if parent in self.directory_states:
continue
for scope in self.paths:
if parent == scope or scope in parent.parents:
original = self.initial.get(parent)
self.directory_states[parent] = original is not None and stat.S_ISDIR(original[0])
break
else:
self.directory_states[parent] = parent.is_dir()
def _snapshot(self, source: Path, destination: Path) -> None:
self.backup_entries += 1
if self.backup_entries > _MAX_BACKUP_ENTRIES:
raise installer.IntegrationInstallError("Integration rollback snapshot budget exceeded (entries)")
mode = source.lstat().st_mode
if stat.S_ISLNK(mode):
destination.symlink_to(os.readlink(source))
elif stat.S_ISDIR(mode):
destination.mkdir()
for child in source.iterdir():
self._snapshot(child, destination / child.name)
shutil.copystat(source, destination, follow_symlinks=False)
elif stat.S_ISREG(mode):
if source.stat().st_size > _MAX_BACKUP_BYTES - self.backup_bytes:
raise installer.IntegrationInstallError("Integration rollback snapshot budget exceeded (bytes)")
with source.open("rb") as incoming, destination.open("wb") as outgoing:
while content := incoming.read(min(65536, _MAX_BACKUP_BYTES - self.backup_bytes + 1)):
self.backup_bytes += len(content)
if self.backup_bytes > _MAX_BACKUP_BYTES:
raise installer.IntegrationInstallError("Integration rollback snapshot budget exceeded (bytes)")
outgoing.write(content)
shutil.copystat(source, destination, follow_symlinks=False)
else:
raise installer.IntegrationInstallError(f"Unsupported integration output: {source}")
def safe_path(self, path: Path, *, allow_leaf_symlink: bool = True) -> Path:
path = path.absolute()
boundary = self.root
if not path.is_relative_to(boundary):
if not any(path.is_relative_to(scope) for scope in self.home_scopes):
raise installer.IntegrationInstallError(f"Integration output escapes trusted scopes: {path}")
boundary = Path.home().absolute()
relative = path.relative_to(boundary).as_posix()
installer.safe_project_path(boundary, relative, allow_leaf_symlink=allow_leaf_symlink)
return path
def observe(self, path: Path, before: bool, removal: bool = False) -> None:
path = self.safe_path(path, allow_leaf_symlink=not before or removal)
for parent in path.parents:
if parent not in self.changes:
continue
if parent in self.pending:
return
saved, identity = self.changes[parent]
if identity is not None and identity[0] == "directory":
if before:
if not _matches_directory_before_write(
_file_identity(parent), identity, path.relative_to(parent).parts[:-1],
):
raise installer.IntegrationInstallError(
f"Integration output changed during the operation; refusing to overwrite {path}"
)
self.pending.add(path)
else:
self.changes[parent] = (saved, _file_identity(parent))
self.pending.discard(path)
return
self._remember_parents(path)
if (
before and path in self.changes and path not in self.pending
and _file_identity(path) != self.changes[path][1]
):
raise installer.IntegrationInstallError(
f"Integration output changed during the operation; refusing to overwrite {path}"
)
if path not in self.changes:
saved = None
if before:
if path.exists() or path.is_symlink():
saved = self.backup / "writes" / str(len(self.changes))
saved.parent.mkdir(parents=True, exist_ok=True)
self._snapshot(path, saved)
else:
if not any(path == scope or scope in path.parents for scope in self.paths):
raise installer.IntegrationInstallError(
f"Integration output {path} must use manifest.record_file() "
"or the host file-writing helpers outside its declared output root"
)
original = self.initial.get(path)
if original is not None:
if _entry_state(path) == original:
return
raise installer.IntegrationInstallError(
f"Integration output {path} changed without before-write observation; "
"cannot restore its original bytes. Use manifest.record_file() "
"or IntegrationBase.write_file_and_record() before overwriting existing files."
)
self.changes[path] = (saved, _file_identity(path))
if not before:
saved, _ = self.changes[path]
self.changes[path] = (saved, _file_identity(path))
self.pending.discard(path)
else:
self.pending.add(path)
def _restore_snapshots(root: Path, journal: _FileJournal) -> list[Path]:
conflicts = []
for path, (saved, written_identity) in reversed(tuple(journal.changes.items())):
journal.safe_path(path)
current_identity = _file_identity(path)
if path in journal.pending or any(path in pending.parents for pending in journal.pending):
original_identity = _file_identity(saved) if saved is not None else None
if current_identity != original_identity:
conflicts.append(path)
continue
if path not in journal.pending and current_identity != written_identity:
conflicts.append(path)
continue
if path.is_dir() and not path.is_symlink():
shutil.rmtree(path)
elif path.exists() or path.is_symlink():
path.unlink()
if saved is not None:
path.parent.mkdir(parents=True, exist_ok=True)
if saved.is_symlink():
path.symlink_to(os.readlink(saved))
elif saved.is_dir():
shutil.copytree(saved, path, symlinks=True)
else:
shutil.copy2(saved, path)
else:
parent = path.parent
boundary = root if path.is_relative_to(root) else Path.home().absolute()
while parent != boundary and not journal.directory_states[parent]:
try:
parent.rmdir()
except OSError:
break
parent = parent.parent
return conflicts
@contextmanager
def _transaction(
root: Path, target: str | None,
expected_state: dict[str, Any] | None, expected_records: dict[str, dict[str, Any]],
*, creation_root: Path | None = None,
):
"""Journal operation-owned writes under an inter-process integration lock."""
from ..shared_infra import _exclusive_project_lock
from .manifest import IntegrationManifest
folders = {
".specify/integrations", ".specify/templates", ".specify/scripts",
".specify/.gitignore", ".specify/presets", ".specify/extensions",
".specify/extensions.yml", ".specify/workflows/speckit",
}
manifest_leaves: set[str] = set()
state, error = try_read_integration_json(root)
if error:
raise installer.IntegrationInstallError(f"Cannot read integration state: {error.detail}")
for key in installed_integration_keys(state or {}):
integration = INTEGRATION_REGISTRY.get(key)
folder = (integration.config or {}).get("folder") if integration else None
if folder:
folders.add(folder.rstrip("/"))
manifest_path = root / ".specify" / "integrations" / f"{key}.manifest.json"
if manifest_path.exists():
manifest = IntegrationManifest.load(key, root)
for relative in manifest.files:
installer.safe_project_path(root, relative, allow_leaf_symlink=True)
manifest_leaves.add(relative)
for integration in INTEGRATION_REGISTRY.values():
if type(integration).__module__.startswith(installer._MODULE_PREFIX):
folders.add(integration.config["folder"].rstrip("/"))
target_integration = INTEGRATION_REGISTRY.get(target)
target_folder = (target_integration.config or {}).get("folder") if target_integration else None
if target_folder:
folders.add(target_folder.rstrip("/"))
paths = [
installer.safe_project_path(
root, folder, allow_leaf_symlink=folder in manifest_leaves and folder not in folders
)
for folder in sorted(folders | manifest_leaves)
]
home_scopes = []
for key in {*installed_integration_keys(state or {}), target} & BUILTIN_INTEGRATION_KEYS:
integration = INTEGRATION_REGISTRY[key]
directory = (integration.registrar_config or {}).get("dir", "")
if directory.startswith("~/"):
scope = installer.safe_project_path(Path.home().absolute(), directory[2:])
home_scopes.append(scope)
paths.extend(home_scopes)
paths = [path for path in paths if not any(other != path and other in path.parents for other in paths)]
root_existed = root.exists()
lock_root = installer._trust_store(root).parent.parent
lock_id = hashlib.sha256(os.path.normcase(str(root.resolve())).encode()).hexdigest()
backup = Path(tempfile.mkdtemp(prefix="speckit-integration-rollback-"))
preserve_backup = False
try:
# Keep the stable project-keyed lock outside an uninitialized target.
with _exclusive_project_lock(lock_root, f".integration-install-{lock_id}.lock", context="integration"):
if creation_root is None:
root.mkdir(parents=True, exist_ok=True)
else:
from .._file_policy import prepare_init_root
prepare_init_root(creation_root)
current_state, state_error = try_read_integration_json(root)
if state_error or current_state != expected_state or installer.read_records(root) != expected_records:
raise installer.IntegrationInstallError(
"Integration state changed while preparing the operation; retry with the current project state"
)
journal = _FileJournal(root, paths, backup, _initial_entries(paths), tuple(home_scopes))
token = file_change_observer.set(journal.observe)
try:
with installer.event_recovery_transaction():
yield
except BaseException as operation_error:
try:
# Restoration itself must not add writes to the journal.
file_change_observer.reset(token)
token = None
conflicts = _restore_snapshots(root, journal)
if conflicts:
preserve_backup = True
console.print(
"[yellow]Warning:[/yellow] Preserved concurrent edits or incomplete writes to "
f"{escape(str(conflicts))}. Recovery snapshots retained at {escape(str(backup))}"
)
except (OSError, installer.IntegrationInstallError) as restore_error:
preserve_backup = True
raise installer.IntegrationInstallError(
f"Integration operation failed ({operation_error}); rollback failed "
f"({restore_error}). Recovery snapshots retained at {backup}; "
f"write snapshot indexes correspond to {[str(path) for path in journal.changes]}"
) from operation_error
raise
finally:
if token is not None:
file_change_observer.reset(token)
finally:
if not preserve_backup:
shutil.rmtree(backup)
if not root_existed:
try:
root.rmdir()
except FileNotFoundError:
pass
except OSError as exc:
if exc.errno not in (errno.ENOTEMPTY, errno.EEXIST):
raise
def external_lifecycle(operation: str):
"""Add package preparation to existing handlers without duplicating setup."""
def decorate(handler):
signature = inspect.signature(handler)
@wraps(handler)
@installer.registry_synchronized
def invoke(*args, **kwargs):
values = signature.bind_partial(*args, **kwargs).arguments
loaded = False
if operation == "init":
project_name = values.get("project_name")
here = values.get("here", False) or project_name == "."
if (here and project_name not in (None, ".")) or (not here and not project_name):
return handler(*args, **kwargs)
creation_root = Path.cwd() if here else Path(project_name).absolute()
root = creation_root
key = values.get("integration")
else:
from .. import _require_specify_project
root = _require_specify_project()
key = values.get("target" if operation == "switch" else "key")
recovery_token = None
recovery_binding = None
try:
if operation == "init":
from .._file_policy import preflight_init_root
root = preflight_init_root(creation_root)
records = installer.read_records(root)
state, error = try_read_integration_json(root)
if error:
raise installer.IntegrationInstallError(f"Cannot read integration state: {error.detail}")
key = key or default_integration_key(state or {})
try:
installer.load_installed_integrations(root)
except installer.IntegrationInstallError:
if operation not in {"upgrade", "uninstall"} or not values.get("force") or key not in records:
raise
recovery_token = installer.recovery_exclusion.set((root.resolve(), key))
installer.load_installed_integrations(root)
recovery_binding = installer.recovery_metadata(root, key, records[key])
console.print(
f"[yellow]Warning:[/yellow] Recovering integration '{escape(key)}' "
"from validated ownership metadata without loading its failed implementation."
)
loaded = True
download = (
isinstance(key, str)
and key not in BUILTIN_INTEGRATION_KEYS
and (
(operation == "upgrade" and key in records)
or (
operation in {"init", "install", "switch"}
and key not in INTEGRATION_REGISTRY
)
)
)
if not records and not download:
return handler(*args, **kwargs)
with ExitStack() as stack:
messages: list[str] = []
message_token = _success_messages.set(messages)
stack.callback(_success_messages.reset, message_token)
candidate = None
if download:
package, record = stack.enter_context(
installer.catalog_package(
root, key, trusted=values.get("trust_integration", False)
)
)
candidate = (package, record)
stack.enter_context(installer.prepared_adapter(root, key, package, record))
if recovery_token is not None:
installer.recovery_exclusion.set(None)
if operation == "init":
token = _init_directory.set(
(creation_root, root.exists(), list(root.iterdir()) if root.is_dir() else [])
)
stack.callback(_init_directory.reset, token)
transaction = (
_transaction(root, key, state, records, creation_root=creation_root)
if operation == "init"
else _transaction(root, key, state, records)
)
with transaction:
if recovery_token is not None and recovery_binding is None:
from .manifest import IntegrationManifest
path = root / ".specify/integrations" / f"{key}.manifest.json"
if path.exists():
manifest = IntegrationManifest.load(key, root)
IntegrationManifest(key, root, version=manifest.version).save()
console.print(
"[yellow]Warning:[/yellow] No local recovery ownership record; "
"preserving old generated files from cleanup. They may require manual cleanup. "
"A trusted replacement can overwrite files at its declared destination."
)
elif recovery_token is not None and recovery_binding is not None:
from .manifest import IntegrationManifest
path = root / ".specify/integrations" / f"{key}.manifest.json"
if path.exists():
manifest = IntegrationManifest.load(key, root)
modes = recovery_binding.get("ownership_modes", {})
preserved = []
for relative in manifest.files:
mode = modes.get(relative, "shared")
manifest.set_ownership(relative, mode)
if mode != "whole":
preserved.append(relative)
manifest.save()
if preserved:
console.print(
"[yellow]Warning:[/yellow] Preserving shared, partial, or unproven "
f"file ownership: {escape(str(preserved))}. Manual event cleanup may be needed."
)
try:
result = handler(*args, **kwargs)
except typer.Exit as exc:
if exc.exit_code != 0 or (candidate and operation == "upgrade"):
raise
if candidate:
exited_state, exited_error = try_read_integration_json(root)
if exited_error:
raise installer.IntegrationInstallError(exited_error.detail) from exc
if key not in installed_integration_keys(exited_state or {}):
raise
result = None
new_state, new_error = try_read_integration_json(root)
if new_error:
raise installer.IntegrationInstallError(
f"Cannot read integration state after {operation}: {new_error.detail}"
)
remaining = installed_integration_keys(new_state or {})
if candidate:
if key not in remaining:
raise installer.IntegrationInstallError(
f"Integration {operation} did not install the requested adapter '{key}'"
)
# End temporary registration before loading durable code.
installer._pending_root = None
installer.unload_installed_integrations()
installer.persist_package(root, key, *candidate)
for removed in records.keys() - set(remaining):
# The uninstall command cleans up its own key. A stale
# record's contributions must go here, while its adapter
# is still loaded.
if operation == "uninstall" and removed != key:
from ._helpers import (
_unregister_extensions_for_agent,
_unregister_presets_for_agent,
)
_unregister_extensions_for_agent(
root, removed,
continuing="The adapter was removed, but extension artifacts may need manual cleanup.",
)
_unregister_presets_for_agent(
root, removed,
continuing="The adapter was removed, but preset artifacts may need manual cleanup.",
)
installer.remove_package(root, removed)
if not candidate:
installer.load_installed_integrations(root)
for message in messages:
console.print(message)
return result
except typer.Exit:
raise
except (Exception, SystemExit) as exc:
console.print(f"[red]Error:[/red] Integration {operation} failed: {escape(str(exc))}")
raise typer.Exit(1) from exc
finally:
# prepared_adapter's context exits after the transaction; restore
# durable registrations rather than leave the candidate cached.
if loaded:
try:
installer.load_installed_integrations(root)
except (installer.IntegrationInstallError, OSError) as exc:
console.print(
f"[red]Error:[/red] Could not reload integration state after {operation}: {escape(str(exc))}"
)
raise typer.Exit(1) from exc
finally:
if recovery_token is not None:
installer.recovery_exclusion.reset(recovery_token)
elif recovery_token is not None:
installer.recovery_exclusion.reset(recovery_token)
return invoke
return decorate