Skip to content

The agent that grows with you

12.1 KB
Raw Blame
"""hermes.process.exit: how Hermes processes (and watchdog-killed turns) end.

A crash or SIGKILL cannot record itself. Each long-lived entrypoint drops a small marker under its
profile's store dir at start and restamps it on the way out (``clean``, ``crash`` + class from the
excepthook, ``watchdog`` from a hard-exit watchdog). The NEXT Hermes start in that profile reports
every marker whose owner is gone; one still ``running`` whose pid fails the canonical start-time
liveness check was killed. Reporting runs on a daemon thread so startup never waits on the Relay
runtime; a claimed-but-unreported marker (the reporter died mid-way) is reclaimed later.
"""

from __future__ import annotations

import atexit
import json
import logging
import os
import shutil
import sys
import threading
import time
from pathlib import Path
from typing import Any

logger = logging.getLogger(__name__)

MARKER_DIRNAME = "process_markers"
_REPORTING = ".reporting"
# Most specific first: ModuleNotFoundError is an ImportError, RecursionError a RuntimeError.
_CRASH_CLASSES = (
    (MemoryError, "memory_error"), (ImportError, "import_error"), (OSError, "os_error"),
    (RuntimeError, "runtime_error"),
)
#: Ceiling on ``flush_before_hard_exit``: a wedged flush costs its pending rows, never the exit.
EXIT_FLUSH_TIMEOUT_S = 3.0
_DRAIN_MARGIN_S = 0.1
_STATE: dict[str, Any] = {}
_STATE_LOCK = threading.Lock()


def crash_class(exc: BaseException | None) -> str:
    return next((label for kind, label in _CRASH_CLASSES if isinstance(exc, kind)), "other")


def markers_dir(home: Path) -> Path:
    return home / "telemetry" / "shared_metrics" / MARKER_DIRNAME


def _write_marker(path: Path, record: dict[str, Any]) -> None:
    tmp = path.with_name(f".{path.name}.tmp")
    tmp.write_text(json.dumps(record), encoding="utf-8")
    os.replace(tmp, path)


def stamp_exit(exit_kind: str, crash: str = "none") -> None:
    """Restamp this process's marker with how it is ending. Never raises; no-op when unmarked."""
    try:
        with _STATE_LOCK:
            record = _STATE.get("record")
            path = _STATE.get("path")
            if record is None or path is None or record.get("state") != "running":
                return
            record = {**record, "state": exit_kind, "crash_class": crash}
            _STATE["record"] = record
            _write_marker(path, record)
    except Exception:
        logger.debug("Process exit marker not stamped", exc_info=True)


def flush_before_hard_exit(timeout_s: float = EXIT_FLUSH_TIMEOUT_S) -> bool:
    """For a graceful stop about to ``os._exit``: run the shared-metrics exit flush its skipped atexit
    hooks would have run (close each live runtime's sessions under its own profile, write queued rows,
    and write every store's deferred rows, the root profile's host-level store included, each into
    the store that deferred it).

    Bounded by ``timeout_s`` on a daemon thread, because the Relay flush waits on every managed call
    still in flight and a wedged one must not hold the exit. Never imports the runtime module:
    collection off (or never started) means nothing to flush and nothing created. Never raises; True
    when the flush finished in time."""
    runtimes = sys.modules.get("hermes_cli.observability.relay_shared_metrics")
    stores = sys.modules.get("hermes_cli.observability.shared_metrics")
    if runtimes is None and stores is None:
        return True
    deadline = time.monotonic() + max(0.0, timeout_s)

    def _flush() -> None:
        if runtimes is not None:
            runtimes.shutdown_runtimes()
        if stores is not None:  # every store that deferred a write, root/Desktop ones included
            stores.drain_deferring_stores(deadline - _DRAIN_MARGIN_S)  # done before the join gives up

    try:
        from agent.memory_provider import spawn_context_thread

        worker = spawn_context_thread(_flush, name="hermes-shared-metrics-exit-flush")
        worker.start()
        worker.join(max(0.0, deadline - time.monotonic()))
    except Exception:
        logger.debug("Shared-metrics exit flush not started", exc_info=True)
        return False
    if worker.is_alive():
        logger.warning("Shared-metrics exit flush still running after %.1fs; exiting without it", timeout_s)
        return False
    return True


def _excepthook(previous: Any) -> Any:
    def hook(exc_type: type[BaseException], exc: BaseException, tb: Any) -> None:
        if not issubclass(exc_type, (KeyboardInterrupt, SystemExit)):
            stamp_exit("crash", crash_class(exc))
        previous(exc_type, exc, tb)

    return hook


def purge_pending_receipts(home: Path) -> None:
    """Collection is off: drop the files kept for a later start to count (parked update receipts,
    installer receipts). Every opt-out answer calls this too, not only an opted-out start, so a
    receipt never survives a "No" to be counted after a later opt-in. Never raises."""
    try:
        from .shared_metrics_install_run import purge_pending_installs
        from .shared_metrics_update import purge_pending_updates

        purge_pending_updates(home)
        purge_pending_installs(home)
    except Exception:
        logger.debug("Pending shared-metrics receipts not purged", exc_info=True)


def begin_process(kind: str) -> None:
    """Mark this process as a running ``kind`` and report dead predecessors. Once per process."""
    try:
        from hermes_constants import get_hermes_home

        from .shared_metrics_desktop import ONBOARDING_LATCH_DIRNAME
        from .shared_metrics_setup import markers_dir as setup_markers_dir
        from .shared_metrics_update import _collection_on

        if _STATE:
            return
        _STATE["kind"] = kind  # watchdog turn rows name the surface even when this home is off
        if not _collection_on():
            home = get_hermes_home()
            purge_pending_receipts(home)  # parked while on / left by the installer, never counted once off
            latches = home / "telemetry" / "shared_metrics" / ONBOARDING_LATCH_DIRNAME  # an opt-out outside Desktop
            for directory in (markers_dir(home), setup_markers_dir(home), latches):  # likewise pending exits/setups
                shutil.rmtree(directory, ignore_errors=True)
            return
        from gateway.status import get_process_start_time

        home = get_hermes_home()
        directory = markers_dir(home)
        directory.mkdir(parents=True, exist_ok=True)
        pid = os.getpid()
        path = directory / f"{kind}-{pid}.json"
        record = {"kind": kind, "pid": pid, "start_time": get_process_start_time(pid), "state": "running"}
        with _STATE_LOCK:
            _write_marker(path, record)
            _STATE.update(path=path, record=record)
        sys.excepthook = _excepthook(sys.excepthook)
        atexit.register(stamp_exit, "clean")
        threading.Thread(
            target=_report_dead_markers, args=(home, path), name="hermes-exit-metrics", daemon=True,
        ).start()
    except Exception:
        logger.debug("Process exit marker not started", exc_info=True)


def current_process_kind() -> str:
    return _STATE.get("kind") or "other"


def _claimer_alive(path: Path) -> bool:
    from gateway.status import runtime_status_pid_is_live

    try:
        claimer = int(path.name.rsplit(".", 2)[-2])
    except (IndexError, ValueError):
        return False
    return runtime_status_pid_is_live({"pid": claimer})


def _claim(path: Path) -> Path | None:
    """Rename a reportable marker so exactly one reporter owns it; None when not reportable."""
    from gateway.status import runtime_status_pid_is_live

    if path.name.endswith(_REPORTING):
        if _claimer_alive(path):
            return None
        record_path = path
    else:
        record_path = None
    try:
        record = json.loads(path.read_text(encoding="utf-8-sig"))
    except (OSError, ValueError):
        return None
    if not isinstance(record, dict):
        return None
    if record_path is None and record.get("state") == "running" and runtime_status_pid_is_live(record):
        return None
    claimed = path.with_name(f"{path.name.split('.json')[0]}.json.{os.getpid()}{_REPORTING}")
    try:
        os.replace(path, claimed)
    except OSError:  # a concurrent reporter won
        return None
    return claimed


def process_exit_fields(record: dict[str, Any]) -> dict[str, str]:
    from .shared_metrics_contract import CRASH_CLASSES, PROCESS_EXIT_KINDS, PROCESS_KINDS

    state = record.get("state")
    exit_kind = "killed" if state == "running" or state not in PROCESS_EXIT_KINDS else state
    crash = record.get("crash_class") if exit_kind == "crash" else "none"
    kind = record.get("kind")
    return {
        "crash_class": crash if crash in CRASH_CLASSES else "other",
        "exit_kind": exit_kind,
        "process_kind": kind if kind in PROCESS_KINDS else "other",
    }


def settle_claim(claimed: Path, original: Path, saved: bool) -> None:
    """Delete a claimed file once its row is saved; otherwise hand it back for the next start."""
    try:
        if saved:
            claimed.unlink(missing_ok=True)
        else:
            os.replace(claimed, original)
    except OSError:  # a leftover claim is reclaimed once this reporter is gone
        logger.debug("Claimed shared-metrics file not settled", exc_info=True)


def _report_dead_markers(home: Path, own: Path) -> None:
    from hermes_constants import reset_hermes_home_override, set_hermes_home_override

    from . import shared_metrics_contract as contract
    from .shared_metrics_events import emit_saved

    token = set_hermes_home_override(home)  # a thread does not inherit the profile binding
    try:
        for path in sorted(markers_dir(home).iterdir()):
            if path == own or path.name.startswith(".") or ".json" not in path.name:
                continue
            claimed = _claim(path)
            if claimed is None:
                continue
            try:
                record = json.loads(claimed.read_text(encoding="utf-8-sig"))
            except (OSError, ValueError):
                record = None
            rows = [(contract.PROCESS_EXIT_MARK, process_exit_fields(record))] if isinstance(record, dict) else []
            settle_claim(claimed, path, emit_saved(rows) == len(rows))
        from .shared_metrics_setup import report_abandoned_setups
        from .shared_metrics_update import report_pending_updates

        report_pending_updates()
        report_abandoned_setups(home)
        # ---- iuf c2 ----
        from .shared_metrics_install_run import report_pending_installs

        report_pending_installs(home)
        # ---- end iuf c2 ----
    except Exception:
        logger.debug("Dead process markers not reported", exc_info=True)
    finally:
        reset_hermes_home_override(token)


def arm_turn(agent: Any) -> None:
    """At turn entry (profile scope bound): the owning home for an off-thread watchdog abort, and
    a fresh once-per-turn latch shared by both turn watchdogs."""
    try:
        from hermes_constants import get_hermes_home

        agent._metrics_turn_home = str(get_hermes_home())
        agent._metrics_watchdog_abort_counted = False
    except Exception:
        logger.debug("Turn exit metrics not armed", exc_info=True)


def record_watchdog_turn_abort(agent: Any) -> None:
    """One exit_kind=watchdog row per turn a watchdog killed, in the turn's own profile. Never raises."""
    try:
        home = getattr(agent, "_metrics_turn_home", None)
        # No armed turn means no known owning profile: never guess the scope.
        if not home or getattr(agent, "_metrics_watchdog_abort_counted", True):
            return
        agent._metrics_watchdog_abort_counted = True
        from hermes_constants import reset_hermes_home_override, set_hermes_home_override

        from . import shared_metrics_contract as contract
        from .shared_metrics_events import _emit

        token = set_hermes_home_override(home)
        try:
            _emit(contract.PROCESS_EXIT_MARK, lambda: {
                "crash_class": "none", "exit_kind": "watchdog", "process_kind": current_process_kind(),
            })
        finally:
            reset_hermes_home_override(token)
    except Exception:
        logger.debug("Watchdog turn abort not recorded", exc_info=True)