Skip to content

armgit / ARES

Public Imported from GitHub
0

ARES - authorized red-team engagement automation with dashboard, campaign scope, module orchestration, OPSEC controls, encrypted vault, and reporting.

53.7 KB
Raw Blame
"""
ARES Secure Platform Update and Upgrade Engine
Provides dynamic, zero-downtime, and cryptographically verified updates.

Principles:
  - Zero Data Loss Guarantee: User databases, configs, cryptographic keys,
    audit reports, and custom plugins are strictly blacklisted from modification.
  - Multi-Layered Defense:
      * Anti-Path-Traversal Jail: Strict relative-to-root confinement.
      * Pre-Flight AST Gatekeeper: Python syntax + BaseModule structure verified, and
        module-level code rejected if it would execute dangerous calls at import time.
      * Content Integrity: downloaded bytes are checked against the git-tree blob SHA
        from the manifest before any disk write, with atomic swap and automatic
        rollback. NOTE: this is a consistency check between two GitHub responses, not
        a trust anchor against a compromised repository — signed modules (a pinned
        release key + detached per-module signatures) are the complete remediation
        and are the recommended production posture.
      * Remote Source Whitelist: Only official pinned GitHub endpoints over verified TLS.
      * DoS Protection: Size quotas and HTTP request timeouts.
  - Separation of Concerns:
      * ares update: Strictly additive (only downloads new attack modules).
      * ares upgrade: In-place upgrades of existing modules, core engine, and Web UI.
"""

from __future__ import annotations

import ast
import hashlib
import json
import os
import re
import shutil
import ssl
import subprocess
import sys
import urllib.error
import urllib.parse
import urllib.request
import uuid
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any

from ares.core.logger import get_logger

logger = get_logger("ares.core.updater")

# Pinned official repositories and domains
OFFICIAL_GITHUB_REPO = "Mafifrizi/ARES"
DEFAULT_BRANCH = "main"
ALLOWED_DOMAINS = frozenset({
    "api.github.com",
    "raw.githubusercontent.com",
    "github.com",
    "codeload.github.com",
})

# Internal module helper files that are not standalone attack modules
MODULE_HELPER_FILES: frozenset[str] = frozenset({
    "base.py",
    "descriptors.py",
    "params.py",
    "sdk.py",
    "ad/dependencies.py",
    "ai/plan_validator.py",
    "reporting/report_gen.py",
})

# Resource quotas and safety limits
MAX_MODULE_FILE_SIZE = 5 * 1024 * 1024  # 5 MB
MAX_UI_BUNDLE_SIZE = 150 * 1024 * 1024  # 150 MB
HTTP_TIMEOUT_SECONDS = 15.0

# Immutable Blacklist: Patterns that the updater must NEVER touch, overwrite, or delete
PROTECTED_DATA_PATTERNS = (
    re.compile(r"(?i)\.(db|sqlite|sqlite3|sqlite-wal|sqlite-shm)$"),
    re.compile(r"(?i)(^|[\\/])(\.env|config\.ya?ml|ares\.ya?ml)$"),
    re.compile(r"(?i)(^|[\\/])keys[\\/].*"),
    re.compile(r"(?i)(^|[\\/])reports[\\/].*"),
    re.compile(r"(?i)(^|[\\/])logs[\\/].*"),
)


def _git_blob_sha1(data: bytes) -> str:
    """Compute the git blob SHA-1 of ``data`` (matches the SHA reported by the
    GitHub git-trees API for a file blob): sha1(b"blob <len>\\0" + data)."""
    header = f"blob {len(data)}\0".encode()
    return hashlib.sha1(header + data).hexdigest()  # noqa: S324 - git object id, not a security hash


def _consttime_hex_eq(a: str, b: str) -> bool:
    """Constant-time comparison of two hex digest strings."""
    import hmac as _hmac
    return _hmac.compare_digest(a, b)


class SecurityViolationError(RuntimeError):
    """Raised when an operation violates updater security constraints."""


class UpdateExecutionError(RuntimeError):
    """Raised when an update or upgrade operation fails."""


def secure_verify_url(url: str) -> urllib.parse.SplitResult:
    """
    Verify that the target URL uses HTTPS and points strictly to whitelisted domains.
    Rejects any unverified protocols or third-party redirection targets.
    """
    clean_url = str(url).strip()
    parsed = urllib.parse.urlsplit(clean_url)
    if parsed.scheme.lower() != "https":
        raise SecurityViolationError(f"Insecure scheme '{parsed.scheme}': Only HTTPS is permitted.")
    netloc = parsed.netloc.lower()
    if ":" in netloc:
        netloc = netloc.split(":")[0]
    if netloc not in ALLOWED_DOMAINS:
        raise SecurityViolationError(f"Untrusted host '{netloc}': Not in official ARES repository whitelist.")
    return parsed


def secure_resolve_path(base_dir: Path, relative_path: str | Path) -> Path:
    """
    Resolve and confine relative_path strictly within base_dir.
    Rejects path traversal (../, null bytes, symlink breakout, or absolute paths outside base).
    Enforces user data blacklist protection.
    """
    raw_input = str(relative_path)
    if "\x00" in raw_input:
        raise SecurityViolationError("Null byte detected in relative path.")

    raw_str = raw_input.strip()
    if not raw_str:
        raise SecurityViolationError("Empty relative path is not permitted.")

    # Guard against absolute paths or drive letters
    if raw_str.startswith(("/", "\\")) or Path(raw_str).is_absolute() or bool(re.match(r"^[a-zA-Z]:", raw_str)):
        raise SecurityViolationError(f"Absolute path '{raw_str}' is not permitted.")

    # Guard against obvious directory traversal tokens
    parts = re.split(r"[\\/]", raw_str)
    if ".." in parts or "." in parts:
        clean_parts = [p for p in parts if p not in ("", ".")]
        if any(p == ".." for p in clean_parts):
            raise SecurityViolationError(f"Path traversal token '..' detected in '{raw_str}'.")

    base_resolved = base_dir.resolve()
    target_path = (base_resolved / raw_str).resolve()

    if not target_path.is_relative_to(base_resolved):
        raise SecurityViolationError(
            f"Path confinement breach: '{raw_str}' resolves outside base directory '{base_resolved}'."
        )

    # Check against protected user data and config patterns
    target_str = str(target_path)
    for pattern in PROTECTED_DATA_PATTERNS:
        if pattern.search(target_str):
            raise SecurityViolationError(
                f"Security violation: Target path '{target_path}' matches protected user data pattern '{pattern.pattern}'."
            )

    return target_path


def extract_all_module_ids(code: str) -> list[str]:
    """
    Extract all MODULE_ID string constants declared in classes within the code.
    """
    try:
        tree = ast.parse(code)
    except Exception:
        return []
    ids: list[str] = []
    for node in ast.walk(tree):
        if isinstance(node, ast.ClassDef):
            for item in node.body:
                if isinstance(item, ast.Assign):
                    for target in item.targets:
                        if isinstance(target, ast.Name) and target.id == "MODULE_ID":
                            if isinstance(item.value, ast.Constant) and isinstance(item.value.value, str):
                                if item.value.value not in ids:
                                    ids.append(item.value.value)
    return ids


# Call targets that must never execute at module import time. A legitimate ARES
# module is imports + class/function definitions + simple constant/logger
# assignments; it must not run a shell/eval/subprocess/dynamic-import call while
# the file is merely being imported (which happens on `ares module list`). Blocking
# these at the import-time scope closes the "top-level payload runs on import"
# supply-chain RCE vector. This is a static denylist — defense in depth, not a
# complete control; the full remediation is signed modules. (AUDIT run-1 C4)
_DANGEROUS_IMPORT_TIME_CALLS = frozenset({
    "eval", "exec", "compile", "__import__",
    "system", "popen", "popen2", "popen3", "popen4",
    "spawn", "spawnl", "spawnle", "spawnv", "spawnve",
    "Popen", "call", "run", "check_output", "check_call",
    "getoutput", "getstatusoutput",
    "import_module", "load_source", "load_module",
    "SourceFileLoader", "FileLoader",
})


def _import_time_nodes(tree: ast.Module):
    """Yield AST nodes that execute at import time.

    Everything at module scope runs on import EXCEPT the bodies of function/class
    definitions (deferred until call/instantiation). Decorators and parameter
    defaults DO run at import, so those are still walked.
    """
    for stmt in tree.body:
        if isinstance(stmt, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef):
            for dec in stmt.decorator_list:
                yield from ast.walk(dec)
            if isinstance(stmt, ast.FunctionDef | ast.AsyncFunctionDef):
                defaults = list(stmt.args.defaults) + [d for d in stmt.args.kw_defaults if d is not None]
                for d in defaults:
                    yield from ast.walk(d)
            if isinstance(stmt, ast.ClassDef):
                # Class-body statements (e.g. `X = os.system(...)` as a class attr)
                # also run at import; walk the base-class exprs and the body.
                for base in stmt.bases:
                    yield from ast.walk(base)
                for item in stmt.body:
                    if not isinstance(item, ast.FunctionDef | ast.AsyncFunctionDef):
                        yield from ast.walk(item)
            continue
        yield from ast.walk(stmt)


def validate_module_code(code: str, expected_module_id: str | None = None) -> tuple[bool, str, str | None]:
    """
    Pre-flight AST analysis on Python code.
    Verifies valid Python syntax and presence of a valid BaseModule structure with MODULE_ID,
    and rejects module code that would execute dangerous operations at import time.
    """
    try:
        tree = ast.parse(code)
    except SyntaxError as exc:
        return False, f"Syntax error: {exc.msg} at line {exc.lineno}", None
    except Exception as exc:
        return False, f"AST parsing failed: {str(exc)}", None

    # Import-time code-execution gate (AUDIT run-1 C4).
    for node in _import_time_nodes(tree):
        if isinstance(node, ast.Call):
            func = node.func
            call_name = (
                func.id if isinstance(func, ast.Name)
                else func.attr if isinstance(func, ast.Attribute)
                else ""
            )
            if call_name in _DANGEROUS_IMPORT_TIME_CALLS:
                return (
                    False,
                    f"Validation rejected: module-level call to '{call_name}' is not "
                    "permitted (import-time code execution).",
                    None,
                )

    found_classes: list[str] = []
    found_ids: list[str] = []

    for node in ast.walk(tree):
        if isinstance(node, ast.ClassDef):
            found_classes.append(node.name)
            for item in node.body:
                if isinstance(item, ast.Assign):
                    for target in item.targets:
                        if isinstance(target, ast.Name) and target.id == "MODULE_ID":
                            if isinstance(item.value, ast.Constant) and isinstance(item.value.value, str):
                                if item.value.value not in found_ids:
                                    found_ids.append(item.value.value)

    if not found_classes:
        return False, "Validation rejected: No class definition found in module code.", None

    if not found_ids:
        return False, "Validation rejected: Class does not declare a valid MODULE_ID attribute.", None

    if expected_module_id and expected_module_id not in found_ids:
        return (
            False,
            f"Validation rejected: MODULE_ID mismatch (expected '{expected_module_id}', got '{found_ids[-1]}').",
            found_ids[-1],
        )

    return True, "Valid module syntax and structure.", found_ids[0]


def atomic_write_file(target_path: Path, content: str | bytes, is_binary: bool = False) -> None:
    """
    Writes file atomically using a temporary file in the same directory, followed by an atomic rename.
    Maintains a rollback backup (.bak) of the original file if an existing file is being overwritten.
    """
    target_path.parent.mkdir(parents=True, exist_ok=True)
    nonce = uuid.uuid4().hex[:8]
    backup_path: Path | None = None

    if target_path.exists():
        backup_path = target_path.with_name(f".{target_path.name}.bak.{nonce}")
        try:
            shutil.copy2(target_path, backup_path)
        except Exception as exc:
            raise UpdateExecutionError(f"Failed to create rollback backup for '{target_path}': {exc}") from exc

    temp_path = target_path.with_name(f".{target_path.name}.tmp.{nonce}")
    try:
        if is_binary:
            data = content if isinstance(content, bytes) else content.encode("utf-8")
            temp_path.write_bytes(data)
        else:
            text = content if isinstance(content, str) else content.decode("utf-8")
            temp_path.write_text(text, encoding="utf-8")

        # Atomic replacement on Windows and POSIX
        os.replace(temp_path, target_path)

        # Remove backup on successful write
        if backup_path and backup_path.exists():
            try:
                backup_path.unlink()
            except OSError:
                pass
    except Exception as exc:
        # Automatic rollback on failure
        if backup_path and backup_path.exists():
            try:
                os.replace(backup_path, target_path)
                logger.info("Rollback restored previous version after failure", target=str(target_path))
            except OSError:
                pass
        raise UpdateExecutionError(f"Atomic file write failed for '{target_path}': {exc}") from exc
    finally:
        if temp_path.exists():
            try:
                temp_path.unlink()
            except OSError:
                pass


@dataclass
class ModuleManifestItem:
    module_id: str
    relative_path: str
    sha256: str
    size_bytes: int
    category: str = ""
    is_installed: bool = False
    is_upgradable: bool = False
    git_sha: str = ""


@dataclass
class UpdateCheckResult:
    new_modules: list[ModuleManifestItem] = field(default_factory=list)
    upgradable_modules: list[ModuleManifestItem] = field(default_factory=list)
    up_to_date_modules: list[ModuleManifestItem] = field(default_factory=list)
    ui_available: bool = False
    ui_local_present: bool = False
    local_count: int = 0
    remote_count: int = 0


class PlatformUpdateManager:
    """
    Platform Update and Upgrade Manager for ARES.
    Handles additive module installations, patch upgrades, UI dashboard bundles,
    and runtime server notification.
    """

    def __init__(
        self,
        project_root: Path | None = None,
        github_repo: str = OFFICIAL_GITHUB_REPO,
        branch: str = DEFAULT_BRANCH,
    ) -> None:
        if project_root is not None:
            self.project_root = project_root.resolve()
        else:
            # Resolves from ares/core/updater.py -> project root
            self.project_root = Path(__file__).resolve().parents[2]

        self.github_repo = github_repo.strip()
        self.branch = branch.strip()

        if not re.match(r"^[a-zA-Z0-9_\-\.]+/[a-zA-Z0-9_\-\.]+$", self.github_repo):
            raise SecurityViolationError(f"Invalid GitHub repository identifier '{github_repo}'.")
        if not re.match(r"^[a-zA-Z0-9_\-\./]+$", self.branch) or self.branch.startswith("-"):
            raise SecurityViolationError(f"Invalid Git branch identifier '{branch}'.")

        # Target directories
        self.builtin_modules_dir = self.project_root / "ares" / "modules"
        self.frontend_dir = self.project_root / "frontend"
        self.frontend_dist_dir = self.frontend_dir / "dist"
        self.external_plugins_dir = Path.home() / ".ares" / "plugins"

        # SSL context enforcing strict TLS verification
        self._ssl_context = ssl.create_default_context()
        self._ssl_context.check_hostname = True
        self._ssl_context.verify_mode = ssl.CERT_REQUIRED

    def _http_get(self, url: str) -> bytes:
        """Fetch URL content with strict whitelisting, timeout, and size bounds."""
        secure_verify_url(url)
        headers = {
            "User-Agent": "ARES-Platform-Updater/1.0.0 (Security Engine; Autonomous)",
            "Accept": "application/vnd.github.v3+json, text/plain, */*",
        }
        token = os.environ.get("GITHUB_TOKEN", "").strip()
        if token:
            headers["Authorization"] = f"token {token}"

        req = urllib.request.Request(url, headers=headers)
        try:
            with urllib.request.urlopen(req, context=self._ssl_context, timeout=HTTP_TIMEOUT_SECONDS) as resp:
                status = resp.status
                if status != 200:
                    raise UpdateExecutionError(f"HTTP GET failed with status code {status} for {url}")
                data = resp.read(MAX_MODULE_FILE_SIZE + 1024)
                if len(data) > MAX_MODULE_FILE_SIZE:
                    raise SecurityViolationError(f"Downloaded payload exceeds maximum permitted size ({MAX_MODULE_FILE_SIZE} bytes).")
                return data
        except urllib.error.HTTPError as exc:
            if exc.code == 403 and "rate limit" in str(exc.reason).lower():
                raise UpdateExecutionError(
                    "GitHub API rate limit reached. Set GITHUB_TOKEN environment variable to increase limit."
                ) from exc
            raise UpdateExecutionError(f"Failed to fetch {url}: HTTP {exc.code} {exc.reason}") from exc
        except urllib.error.URLError as exc:
            raise UpdateExecutionError(f"Network error while connecting to {url}: {exc.reason}") from exc

    def get_local_modules(self) -> dict[str, ModuleManifestItem]:
        """
        Scan all installed built-in modules, compute SHA-256 digests, and extract module IDs.
        """
        local_modules: dict[str, ModuleManifestItem] = {}
        if not self.builtin_modules_dir.exists():
            return local_modules

        for py_file in sorted(self.builtin_modules_dir.rglob("*.py")):
            rel_path = str(py_file.relative_to(self.builtin_modules_dir)).replace("\\", "/")
            if (
                py_file.stem.startswith("_")
                or py_file.stem == "base"
                or rel_path in MODULE_HELPER_FILES
                or py_file.name in {"base.py", "descriptors.py", "params.py", "sdk.py"}
            ):
                continue
            try:
                content_bytes = py_file.read_bytes()
                sha256 = hashlib.sha256(content_bytes).hexdigest()
                text = content_bytes.decode("utf-8", errors="ignore")
                valid, msg, primary_id = validate_module_code(text)
                if not valid or not primary_id:
                    continue

                category = rel_path.split("/")[0] if "/" in rel_path else ""
                norm_bytes = content_bytes.replace(b"\r\n", b"\n")
                git_sha = hashlib.sha1(b"blob " + str(len(norm_bytes)).encode() + b"\x00" + norm_bytes).hexdigest()

                module_ids = extract_all_module_ids(text) or [primary_id]
                for mid in module_ids:
                    local_modules[mid] = ModuleManifestItem(
                        module_id=mid,
                        relative_path=rel_path,
                        sha256=sha256,
                        size_bytes=len(content_bytes),
                        category=category,
                        is_installed=True,
                        git_sha=git_sha,
                    )
            except Exception as exc:
                logger.debug("Failed scanning local module file", file=str(py_file), error=str(exc))

        return local_modules

    def fetch_remote_tree(self) -> list[dict[str, Any]]:
        """
        Fetch the repository file tree from GitHub Git Trees API.
        """
        api_url = f"https://api.github.com/repos/{self.github_repo}/git/trees/{self.branch}?recursive=1"
        try:
            raw_json = self._http_get(api_url)
            payload = json.loads(raw_json.decode("utf-8"))
            tree = payload.get("tree", [])
            if isinstance(tree, list):
                return tree
            return []
        except Exception as exc:
            logger.warning("Remote tree fetch failed", error=str(exc))
            raise

    def check_updates(self) -> UpdateCheckResult:
        """
        Compare local modules and UI assets against the remote repository.
        Identifies new modules (for update) and changed modules (for upgrade).
        """
        local_modules = self.get_local_modules()
        tree = self.fetch_remote_tree()

        new_modules: list[ModuleManifestItem] = []
        upgradable_modules: list[ModuleManifestItem] = []
        up_to_date_modules: list[ModuleManifestItem] = []

        local_by_rel_path = {item.relative_path: item for item in local_modules.values()}
        local_by_id = {item.module_id: item for item in local_modules.values()}

        remote_module_count = 0
        ui_available = False

        for entry in tree:
            path_str: str = entry.get("path", "")
            type_str: str = entry.get("type", "")

            if path_str.startswith("frontend/dist/") or path_str == "frontend/dist":
                ui_available = True

            if type_str != "blob" or not path_str.startswith("ares/modules/"):
                continue

            rel_mod_path = path_str[len("ares/modules/"):]
            filename = Path(rel_mod_path).name
            if (
                filename.startswith("_")
                or not filename.endswith(".py")
                or rel_mod_path in MODULE_HELPER_FILES
                or filename in {"base.py", "descriptors.py", "params.py", "sdk.py"}
            ):
                continue

            matching_local = [m for m in local_modules.values() if m.relative_path == rel_mod_path]
            remote_module_count += len(matching_local) if matching_local else 1
            category = rel_mod_path.split("/")[0] if "/" in rel_mod_path else ""
            remote_sha = entry.get("sha", "")
            size_bytes = entry.get("size", 0)

            # Heuristic module ID from stem or match
            inferred_id = Path(rel_mod_path).stem.replace("_", ".")
            if category and not inferred_id.startswith(f"{category}."):
                inferred_id = f"{category}.{Path(rel_mod_path).stem}"

            # Check if exists locally by relative path or module_id
            existing = local_by_rel_path.get(rel_mod_path) or local_by_id.get(inferred_id)

            item = ModuleManifestItem(
                module_id=existing.module_id if existing else inferred_id,
                relative_path=rel_mod_path,
                sha256=remote_sha,
                size_bytes=size_bytes,
                category=category,
            )

            if existing is None:
                item.is_installed = False
                new_modules.append(item)
            else:
                item.is_installed = True
                if existing.git_sha and remote_sha and existing.git_sha == remote_sha:
                    item.is_upgradable = False
                    up_to_date_modules.append(item)
                else:
                    item.is_upgradable = True
                    upgradable_modules.append(item)

        ui_local_present = (self.frontend_dist_dir / "index.html").exists()

        return UpdateCheckResult(
            new_modules=new_modules,
            upgradable_modules=upgradable_modules,
            up_to_date_modules=up_to_date_modules,
            ui_available=ui_available or (self.frontend_dir / "package.json").exists(),
            ui_local_present=ui_local_present,
            local_count=len(local_modules),
            remote_count=remote_module_count,
        )

    def _download_and_install_module(
        self,
        rel_path: str,
        expected_module_id: str | None = None,
        dry_run: bool = False,
        expected_git_sha: str | None = None,
    ) -> tuple[bool, str]:
        """
        Download a single module from the official repository, validate AST and syntax,
        and atomically write to the target directory.

        If ``expected_git_sha`` is supplied (the blob SHA from the git tree manifest),
        the downloaded content is verified against it before any disk write. This
        detects tampering or a TOCTOU change between the manifest read and the raw
        download (e.g. a single-request MITM, or the file changing underneath us).
        Note: this is a consistency check between two GitHub responses, not a trust
        anchor against a compromised repository — signed modules are the full
        remediation. (AUDIT run-1 C4)
        """
        safe_target = secure_resolve_path(self.builtin_modules_dir, rel_path)
        raw_url = f"https://raw.githubusercontent.com/{self.github_repo}/{self.branch}/ares/modules/{rel_path}"

        if dry_run:
            return True, f"[dry-run] Would download {raw_url} -> {safe_target}"

        data = self._http_get(raw_url)

        # Integrity: the git tree manifest reports each blob's SHA (git blob SHA-1
        # over b"blob <len>\\0" + content). Recompute it on the downloaded bytes and
        # refuse to write on mismatch. (AUDIT run-1 C4)
        if expected_git_sha:
            actual_git_sha = _git_blob_sha1(data)
            if not _consttime_hex_eq(actual_git_sha, expected_git_sha.strip().lower()):
                raise SecurityViolationError(
                    f"Integrity check failed for '{rel_path}': downloaded content does "
                    "not match the manifest blob SHA. Refusing to install."
                )

        code_text = data.decode("utf-8", errors="replace")

        # Pre-flight AST analysis (syntax + BaseModule structure + import-time safety)
        valid, msg, detected_id = validate_module_code(code_text, expected_module_id=expected_module_id)
        if not valid:
            raise SecurityViolationError(f"Module code validation failed for '{rel_path}': {msg}")

        # Atomic write with automatic rollback
        atomic_write_file(safe_target, data, is_binary=True)
        logger.info("Installed module successfully", module_id=detected_id, path=str(safe_target))
        return True, f"Successfully installed module '{detected_id}' ({rel_path})."

    def update_modules(
        self,
        module_id: str | None = None,
        dry_run: bool = False,
    ) -> dict[str, Any]:
        """
        ares update: STRICTLY ADDITIVE.
        Only downloads and installs modules that DO NOT exist locally.
        Existing modules and user configs are untouched.
        """
        check_result = self.check_updates()
        candidates = check_result.new_modules

        if module_id:
            cleaned_id = module_id.strip().lower()
            local = self.get_local_modules()
            target_rel_path = local[cleaned_id].relative_path.lower() if cleaned_id in local else None
            candidates = [
                m for m in candidates
                if m.module_id.lower() == cleaned_id
                or m.relative_path.lower().endswith(f"{cleaned_id}.py")
                or (target_rel_path and m.relative_path.lower() == target_rel_path)
            ]
            if not candidates:
                # Check if it is already installed
                if any(m.lower() == cleaned_id for m in local):
                    return {
                        "status": "already_installed",
                        "message": f"Module '{module_id}' is already installed. Use 'ares upgrade' if you wish to update existing modules.",
                        "installed_count": 0,
                    }
                return {
                    "status": "not_found",
                    "message": f"Module '{module_id}' was not found in remote repository new modules.",
                    "installed_count": 0,
                }

        installed: list[str] = []
        errors: list[str] = []

        for item in candidates:
            try:
                ok, msg = self._download_and_install_module(
                    item.relative_path,
                    expected_module_id=None,
                    dry_run=dry_run,
                    expected_git_sha=item.sha256 or None,
                )
                if ok:
                    installed.append(item.relative_path)
            except Exception as exc:
                err_msg = f"Failed installing {item.relative_path}: {exc}"
                logger.error("Module update error", error=err_msg)
                errors.append(err_msg)

        # Notify running server if real install occurred
        if installed and not dry_run:
            self.notify_running_server_reload()

        return {
            "status": "ok" if not errors else "partial",
            "installed_count": len(installed),
            "installed_modules": installed,
            "errors": errors,
            "dry_run": dry_run,
        }

    def upgrade_modules(
        self,
        module_id: str | None = None,
        dry_run: bool = False,
    ) -> dict[str, Any]:
        """
        ares upgrade --modules: In-place patch upgrade of existing modules.
        Compares digests, verifies AST before replacement, and provides atomic rollback.
        """
        check_result = self.check_updates()
        candidates = check_result.upgradable_modules

        if module_id:
            cleaned_id = module_id.strip().lower()
            local = self.get_local_modules()
            target_rel_path = local[cleaned_id].relative_path.lower() if cleaned_id in local else None
            candidates = [
                m for m in candidates
                if m.module_id.lower() == cleaned_id
                or m.relative_path.lower().endswith(f"{cleaned_id}.py")
                or (target_rel_path and m.relative_path.lower() == target_rel_path)
            ]
            if not candidates:
                return {
                    "status": "not_found",
                    "message": f"Installed module '{module_id}' was not found for upgrade.",
                    "upgraded_count": 0,
                }

        upgraded: list[str] = []
        errors: list[str] = []

        for item in candidates:
            try:
                ok, msg = self._download_and_install_module(
                    item.relative_path,
                    expected_module_id=None,
                    dry_run=dry_run,
                    expected_git_sha=item.sha256 or None,
                )
                if ok:
                    upgraded.append(item.relative_path)
            except Exception as exc:
                err_msg = f"Failed upgrading {item.relative_path}: {exc}"
                logger.error("Module upgrade error", error=err_msg)
                errors.append(err_msg)

        if upgraded and not dry_run:
            self.notify_running_server_reload()

        return {
            "status": "ok" if not errors else "partial",
            "upgraded_count": len(upgraded),
            "upgraded_modules": upgraded,
            "errors": errors,
            "dry_run": dry_run,
        }

    def upgrade_ui(self, dry_run: bool = False) -> dict[str, Any]:
        """
        ares upgrade --ui: Upgrades the Frontend Web UI Dashboard bundle.
        No git clone needed.
        Supports both local workspace compilation (if package.json & npm present)
        and direct safe distribution download/sync.
        """
        if dry_run:
            return {
                "status": "dry_run",
                "message": "[dry-run] Would build or synchronize latest Web UI bundle to frontend/dist/.",
                "target_dir": str(self.frontend_dist_dir),
            }

        # Check if local frontend source directory exists with package.json and npm
        pkg_json = self.frontend_dir / "package.json"
        if pkg_json.exists() and shutil.which("npm"):
            try:
                logger.info("Executing local frontend build via npm", cwd=str(self.frontend_dir))
                cmd = ["npm.cmd" if sys.platform == "win32" else "npm", "run", "build"]
                result = subprocess.run(
                    cmd,
                    cwd=str(self.frontend_dir),
                    capture_output=True,
                    text=True,
                    timeout=120,
                    check=False,
                )
                if result.returncode == 0 and (self.frontend_dist_dir / "index.html").exists():
                    logger.info("Web UI compiled successfully via npm run build")
                    return {
                        "status": "ok",
                        "method": "local_build",
                        "message": "Frontend Web UI compiled and updated successfully.",
                        "target_dir": str(self.frontend_dist_dir),
                    }
                logger.warning("npm run build failed, falling back to asset sync", stderr=result.stderr[:300])
            except Exception as exc:
                logger.warning("npm build invocation failed, falling back to direct asset sync", error=str(exc))

        # Direct asset synchronization / standalone download mode
        # Downloads pre-built dist assets from GitHub repository tree
        tree = self.fetch_remote_tree()
        ui_files = [t for t in tree if t.get("type") == "blob" and str(t.get("path", "")).startswith("frontend/dist/")]

        if not ui_files:
            # If dist is not committed in tree, ensure index.html exists
            if (self.frontend_dist_dir / "index.html").exists():
                return {
                    "status": "ok",
                    "method": "existing_verified",
                    "message": "Existing Web UI distribution verified.",
                    "target_dir": str(self.frontend_dist_dir),
                }
            raise UpdateExecutionError("No pre-built Web UI bundle found in remote repository and npm is unavailable.")

        downloaded_count = 0
        for entry in ui_files:
            remote_path = entry.get("path", "")
            rel_ui_path = remote_path[len("frontend/dist/"):]
            safe_target = secure_resolve_path(self.frontend_dist_dir, rel_ui_path)
            raw_url = f"https://raw.githubusercontent.com/{self.github_repo}/{self.branch}/{remote_path}"
            data = self._http_get(raw_url)
            atomic_write_file(safe_target, data, is_binary=True)
            downloaded_count += 1

        return {
            "status": "ok",
            "method": "remote_sync",
            "downloaded_files": downloaded_count,
            "message": f"Web UI dashboard updated ({downloaded_count} assets synchronized).",
            "target_dir": str(self.frontend_dist_dir),
        }

    def check_system_update(self) -> dict[str, Any]:
        """
        Inspect remote GitHub repository and compare against local system state.
        Returns commit hash deltas, release metadata, and module update counts.
        """
        local_commit = "unknown"
        current_branch = self.branch
        git_dir = self.project_root / ".git"
        if git_dir.exists() and shutil.which("git"):
            try:
                proc = subprocess.run(
                    ["git", "rev-parse", "HEAD"],
                    cwd=str(self.project_root),
                    capture_output=True,
                    text=True,
                    check=False,
                )
                if proc.returncode == 0:
                    local_commit = proc.stdout.strip()

                b_proc = subprocess.run(
                    ["git", "rev-parse", "--abbrev-ref", "HEAD"],
                    cwd=str(self.project_root),
                    capture_output=True,
                    text=True,
                    check=False,
                )
                if b_proc.returncode == 0:
                    current_branch = b_proc.stdout.strip()
            except Exception:
                pass

        api_url = f"https://api.github.com/repos/{self.github_repo}/commits/{self.branch}"
        remote_sha = "unknown"
        commit_msg = ""
        commit_date = ""
        commit_author = ""
        try:
            raw = self._http_get(api_url)
            data = json.loads(raw.decode("utf-8"))
            remote_sha = data.get("sha", "")
            commit_obj = data.get("commit", {})
            commit_msg = commit_obj.get("message", "").split("\n")[0]
            commit_author = commit_obj.get("author", {}).get("name", "")
            commit_date = commit_obj.get("author", {}).get("date", "")
        except Exception as exc:
            logger.debug("Failed fetching remote commit details", error=str(exc))

        modules_check = self.check_updates()

        try:
            from ares.__version__ import __version__
        except Exception:
            __version__ = "6.0.0"

        is_behind = (
            local_commit != "unknown"
            and remote_sha != "unknown"
            and not remote_sha.startswith(local_commit[:7])
            and not local_commit.startswith(remote_sha[:7])
        )

        return {
            "status": "ok",
            "version": __version__,
            "branch": current_branch,
            "local_commit": local_commit[:7] if local_commit != "unknown" else "unknown",
            "remote_commit": remote_sha[:7] if remote_sha != "unknown" else "unknown",
            "commit_message": commit_msg,
            "commit_author": commit_author,
            "commit_date": commit_date,
            "system_update_available": is_behind,
            "new_modules_count": len(modules_check.new_modules),
            "upgradable_modules_count": len(modules_check.upgradable_modules),
            "ui_available": modules_check.ui_available,
        }

    def upgrade_system_git(self, dry_run: bool = False) -> dict[str, Any]:
        """
        Perform git-native system upgrade.
        Fast-forwards HEAD to origin/{branch} if clean.
        Strictly preserves uncommitted changes.
        """
        git_dir = self.project_root / ".git"
        if not git_dir.exists() or not shutil.which("git"):
            return {
                "status": "not_applicable",
                "method": "git",
                "message": "Project is not a Git repository or git binary is unavailable.",
            }

        # Check for detached HEAD state
        branch_proc = subprocess.run(
            ["git", "rev-parse", "--abbrev-ref", "HEAD"],
            cwd=str(self.project_root),
            capture_output=True,
            text=True,
            encoding="utf-8",
            errors="replace",
            check=False,
        )
        if branch_proc.returncode == 0 and branch_proc.stdout.strip() == "HEAD":
            return {
                "status": "detached_head",
                "method": "git",
                "message": "Repository is in a detached HEAD state. Switch to a branch before upgrading.",
            }

        # Dynamically discover remote (defaulting to 'origin')
        remote_name = "origin"
        remotes_proc = subprocess.run(
            ["git", "remote"],
            cwd=str(self.project_root),
            capture_output=True,
            text=True,
            encoding="utf-8",
            errors="replace",
            check=False,
        )
        if remotes_proc.returncode == 0:
            remotes = [r.strip() for r in remotes_proc.stdout.splitlines() if r.strip()]
            if remotes and "origin" not in remotes:
                remote_name = remotes[0]

        # Check working tree cleanliness for tracked files
        status_proc = subprocess.run(
            ["git", "status", "--porcelain", "-uno"],
            cwd=str(self.project_root),
            capture_output=True,
            text=True,
            encoding="utf-8",
            errors="replace",
            check=False,
        )
        if status_proc.returncode != 0:
            raise UpdateExecutionError(f"git status failed: {status_proc.stderr.strip()}")

        dirty_tracked = [line.strip() for line in status_proc.stdout.splitlines() if line.strip()]
        if dirty_tracked:
            return {
                "status": "dirty_tree",
                "method": "git",
                "message": f"Working tree has {len(dirty_tracked)} modified tracked file(s). Commit or stash before upgrading core engine.",
                "dirty_files": dirty_tracked,
            }

        # Fetch latest commits from remote
        fetch_proc = subprocess.run(
            ["git", "fetch", remote_name, self.branch],
            cwd=str(self.project_root),
            capture_output=True,
            text=True,
            encoding="utf-8",
            errors="replace",
            check=False,
            timeout=30,
        )
        if fetch_proc.returncode != 0:
            raise UpdateExecutionError(f"git fetch failed: {fetch_proc.stderr.strip()}")

        # Check commit distance
        rev_proc = subprocess.run(
            ["git", "rev-list", f"HEAD..{remote_name}/{self.branch}", "--count"],
            cwd=str(self.project_root),
            capture_output=True,
            text=True,
            encoding="utf-8",
            errors="replace",
            check=False,
        )
        behind_count = 0
        if rev_proc.returncode == 0:
            try:
                behind_count = int(rev_proc.stdout.strip())
            except ValueError:
                pass

        if behind_count == 0:
            return {
                "status": "up_to_date",
                "method": "git",
                "message": f"System core engine is already up to date with {remote_name}/{self.branch}.",
                "commits_pulled": 0,
            }

        if dry_run:
            return {
                "status": "dry_run",
                "method": "git",
                "message": f"[dry-run] Would fast-forward pull {behind_count} commit(s) from {remote_name}/{self.branch}.",
                "commits_available": behind_count,
            }

        pull_proc = subprocess.run(
            ["git", "pull", "--ff-only", remote_name, self.branch],
            cwd=str(self.project_root),
            capture_output=True,
            text=True,
            encoding="utf-8",
            errors="replace",
            check=False,
            timeout=60,
        )
        if pull_proc.returncode != 0:
            return {
                "status": "failed",
                "method": "git",
                "message": f"git pull --ff-only failed: {pull_proc.stderr.strip()}",
                "error": pull_proc.stderr.strip(),
            }

        return {
            "status": "ok",
            "method": "git",
            "message": f"Successfully pulled {behind_count} commit(s) from {remote_name}/{self.branch}.",
            "commits_pulled": behind_count,
        }

    def upgrade_system_api(self, dry_run: bool = False) -> dict[str, Any]:
        """
        API-synchronized system upgrade fallback for standalone non-git installations.
        Fetches core framework packages via GitHub API with AST pre-flight verification and atomic replacement.
        """
        if dry_run:
            return {
                "status": "dry_run",
                "method": "api",
                "message": "[dry-run] Would synchronize core framework files from GitHub API.",
            }

        tree = self.fetch_remote_tree()
        core_prefixes = (
            "ares/core/",
            "ares/api/",
            "ares/cli/",
            "ares/sdk/",
            "ares/db/",
            "ares/mcp/",
            "alembic.ini",
            "migrations/",
        )
        core_files = [
            t for t in tree
            if t.get("type") == "blob" and any(str(t.get("path", "")).startswith(p) for p in core_prefixes)
        ]

        updated_count = 0
        errors: list[str] = []

        for entry in core_files:
            remote_path = entry.get("path", "")
            try:
                safe_target = secure_resolve_path(self.project_root, remote_path)
                raw_url = f"https://raw.githubusercontent.com/{self.github_repo}/{self.branch}/{remote_path}"
                data = self._http_get(raw_url)

                if remote_path.endswith(".py"):
                    try:
                        ast.parse(data.decode("utf-8", errors="replace"))
                    except SyntaxError as exc:
                        raise SecurityViolationError(f"AST syntax validation failed for '{remote_path}': {exc}")  # noqa: B904

                atomic_write_file(safe_target, data, is_binary=True)
                updated_count += 1
            except Exception as exc:
                errors.append(f"Failed syncing '{remote_path}': {exc}")

        return {
            "status": "ok" if not errors else "partial",
            "method": "api",
            "updated_files": updated_count,
            "errors": errors,
            "message": f"Synchronized {updated_count} core framework file(s).",
        }

    def apply_database_migrations(self, dry_run: bool = False) -> dict[str, Any]:
        """
        Run Alembic database schema migrations to update local database structures to 'head'.
        Non-destructive: User findings, targets, campaigns, and tokens are strictly preserved.
        """
        if dry_run:
            return {
                "status": "dry_run",
                "message": "[dry-run] Would check and apply database schema migrations to head via Alembic.",
            }

        alembic_ini = self.project_root / "alembic.ini"
        if not alembic_ini.exists():
            return {
                "status": "skipped",
                "message": "alembic.ini not found in project root; schema migration skipped.",
            }

        custom_db = (
            os.environ.get("ARES_DATABASE_URL")
            or os.environ.get("ARES_POSTGRES_DSN")
            or os.environ.get("DATABASE_URL")
        )

        try:
            import logging
            from types import SimpleNamespace

            from alembic import command as alembic_cmd
            from alembic.config import Config as AlembicConfig

            logging.getLogger("alembic").setLevel(logging.ERROR)
            alembic_cfg = AlembicConfig(str(alembic_ini))
            if custom_db:
                alembic_cfg.cmd_opts = SimpleNamespace(x=[f"db_url={custom_db}"])

            alembic_cmd.upgrade(alembic_cfg, "head")
            logger.info("Alembic database migrations applied successfully")
            return {
                "status": "ok",
                "applied": True,
                "message": "Database schema migrations applied successfully (head).",
            }
        except Exception as exc:
            try:
                cmd = [sys.executable, "-m", "alembic"]
                if custom_db:
                    cmd.extend(["-x", f"db_url={custom_db}"])
                cmd.extend(["upgrade", "head"])

                proc = subprocess.run(
                    cmd,
                    cwd=str(self.project_root),
                    capture_output=True,
                    text=True,
                    encoding="utf-8",
                    errors="replace",
                    timeout=30,
                    check=False,
                )
                if proc.returncode == 0:
                    return {
                        "status": "ok",
                        "applied": True,
                        "message": "Database schema migrations applied successfully via CLI (head).",
                    }
                return {
                    "status": "warning",
                    "applied": False,
                    "message": f"Database migration returned code {proc.returncode}: {proc.stderr.strip()}",
                }
            except Exception as sub_exc:
                logger.warning("Database migration attempt encountered an issue", error=str(sub_exc))
                return {
                    "status": "warning",
                    "applied": False,
                    "message": f"Database migration note: {str(exc)}",
                }

    def upgrade_system(self, dry_run: bool = False) -> dict[str, Any]:
        """
        Full core platform upgrade:
        1. Upgrades core framework files (Git-native or API sync)
        2. Applies pending database schema migrations
        """
        git_dir = self.project_root / ".git"
        if git_dir.exists() and shutil.which("git"):
            system_res = self.upgrade_system_git(dry_run=dry_run)
            if system_res.get("status") in ("ok", "up_to_date", "dry_run", "dirty_tree"):
                db_res = self.apply_database_migrations(dry_run=dry_run)
                return {
                    "status": system_res.get("status"),
                    "method": "git",
                    "system": system_res,
                    "database": db_res,
                }

        # Fallback to API sync
        system_res = self.upgrade_system_api(dry_run=dry_run)
        db_res = self.apply_database_migrations(dry_run=dry_run)
        return {
            "status": system_res.get("status"),
            "method": "api",
            "system": system_res,
            "database": db_res,
        }

    def run_post_upgrade_diagnostics(self) -> dict[str, Any]:
        """
        Post-upgrade health verification.
        Validates core subsystems, attack module loading, and database connectivity.
        """
        checks: list[dict[str, Any]] = []

        # 1. Core Framework imports (probe: these imports must actually run)
        try:
            import ares.api.server  # noqa: F401
            import ares.core  # noqa: F401
            import ares.mcp  # noqa: F401
            checks.append({"subsystem": "Core Platform Engine", "status": "PASS", "detail": "Core, API, and MCP imported cleanly"})
        except Exception as exc:
            checks.append({"subsystem": "Core Platform Engine", "status": "FAIL", "detail": str(exc)})

        # 2. Module catalog
        try:
            local_mods = self.get_local_modules()
            count = len(local_mods)
            unique_files = len({item.relative_path for item in local_mods.values()})
            detail_str = f"{count} modules discovered and validated ({unique_files} source files)"
            checks.append({"subsystem": "Attack Modules", "status": "PASS" if count > 0 else "WARN", "detail": detail_str})
        except Exception as exc:
            checks.append({"subsystem": "Attack Modules", "status": "FAIL", "detail": str(exc)})

        # 3. Database
        db_path = self.project_root / "ares.db"
        if db_path.exists():
            checks.append({"subsystem": "Database Storage", "status": "PASS", "detail": f"ares.db online ({db_path.stat().st_size} bytes)"})
        else:
            checks.append({"subsystem": "Database Storage", "status": "PASS", "detail": "Clean state (will initialize on first start)"})

        # 4. Web UI Dashboard
        dist_index = self.frontend_dist_dir / "index.html"
        if dist_index.exists():
            checks.append({"subsystem": "Web UI Dashboard", "status": "PASS", "detail": "Distribution bundle verified"})
        else:
            checks.append({"subsystem": "Web UI Dashboard", "status": "INFO", "detail": "Source mode (build via 'npm run build' or 'ares upgrade --ui')"})

        all_ok = all(c["status"] == "PASS" for c in checks if c["status"] != "INFO")
        return {
            "healthy": all_ok,
            "checks": checks,
        }

    def upgrade_all(self, dry_run: bool = False) -> dict[str, Any]:
        """
        ares upgrade --all: Comprehensive full-system platform upgrade.
        1. Core Framework Engine (Git fast-forward or API sync)
        2. Database Schema Migrations (Alembic upgrade head)
        3. Attack Modules (patch existing + install new)
        4. Frontend Web UI Dashboard bundle
        5. Running API server hot-reload
        6. Post-upgrade diagnostics verification
        """
        # 1. Upgrade core system engine and database
        system_res = self.upgrade_system(dry_run=dry_run)

        # 2. Patch existing modules
        modules_res = self.upgrade_modules(dry_run=dry_run)

        # 3. Add any new modules
        additive_res = self.update_modules(dry_run=dry_run)

        # 4. Upgrade Web UI
        ui_res = self.upgrade_ui(dry_run=dry_run)

        # 5. Notify running server
        reload_res = self.notify_running_server_reload() if not dry_run else {"status": "dry_run"}

        # 6. Run post-upgrade diagnostics
        diag_res = self.run_post_upgrade_diagnostics() if not dry_run else {"healthy": True, "dry_run": True}

        return {
            "status": "ok",
            "system": system_res,
            "modules_upgraded": modules_res.get("upgraded_count", 0),
            "modules_added": additive_res.get("installed_count", 0),
            "ui_status": ui_res.get("status"),
            "server_reload": reload_res,
            "diagnostics": diag_res,
            "dry_run": dry_run,
        }

    def notify_running_server_reload(self, host: str = "127.0.0.1", port: int = 8080) -> dict[str, Any]:
        """
        Send loopback HTTP POST to running ARES API server to trigger in-memory plugin reload.
        Gracefully returns if server is not currently running.
        """
        if host not in ("127.0.0.1", "::1", "localhost"):
            raise SecurityViolationError(f"Reload notification target '{host}' rejected: Loopback only.")

        url = f"http://{host}:{port}/modules/reload"
        headers = {"Content-Type": "application/json", "User-Agent": "ARES-CLI-Updater"}
        try:
            from ares.core.config import get_settings
            from ares.core.security import create_access_token

            settings = get_settings()
            token = create_access_token(
                data={"sub": "cli-updater", "role": "operator"},
                secret_key=settings.jwt_signing_key,
                algorithm=settings.ares_jwt_algorithm,
                expires_minutes=5,
            )
            headers["Authorization"] = f"Bearer {token}"
        except Exception:
            pass

        req = urllib.request.Request(
            url,
            data=b"{}",
            headers=headers,
            method="POST",
        )
        try:
            with urllib.request.urlopen(req, timeout=2.0) as resp:
                if resp.status == 200:
                    payload = json.loads(resp.read().decode("utf-8"))
                    logger.info("Running ARES server successfully reloaded modules", total=payload.get("module_count"))
                    return {"connected": True, "reloaded": True, "module_count": payload.get("module_count")}
        except Exception:
            # Server not running or connection refused; expected when offline
            pass
        return {"connected": False, "reloaded": False}