The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
"""hermes.process.exit: how Hermes processes (and watchdog-killed turns) end.
A crash or SIGKILL cannot record itself. Each long-lived entrypoint drops a small marker under its
profile's store dir at start and restamps it on the way out (``clean``, ``crash`` + class from the
excepthook, ``watchdog`` from a hard-exit watchdog). The NEXT Hermes start in that profile reports
every marker whose owner is gone; one still ``running`` whose pid fails the canonical start-time
liveness check was killed. Reporting runs on a daemon thread so startup never waits on the Relay
runtime; a claimed-but-unreported marker (the reporter died mid-way) is reclaimed later.
"""
from __future__ import annotations
import atexit
import json
import logging
import os
import shutil
import sys
import threading
import time
from pathlib import Path
from typing import Any
logger = logging.getLogger(__name__)
MARKER_DIRNAME = "process_markers"
_REPORTING = ".reporting"
# Most specific first: ModuleNotFoundError is an ImportError, RecursionError a RuntimeError.
_CRASH_CLASSES = (
(MemoryError, "memory_error"), (ImportError, "import_error"), (OSError, "os_error"),
(RuntimeError, "runtime_error"),
)
#: Ceiling on ``flush_before_hard_exit``: a wedged flush costs its pending rows, never the exit.
EXIT_FLUSH_TIMEOUT_S = 3.0
_DRAIN_MARGIN_S = 0.1
_STATE: dict[str, Any] = {}
_STATE_LOCK = threading.Lock()
def crash_class(exc: BaseException | None) -> str:
return next((label for kind, label in _CRASH_CLASSES if isinstance(exc, kind)), "other")
def markers_dir(home: Path) -> Path:
return home / "telemetry" / "shared_metrics" / MARKER_DIRNAME
def _write_marker(path: Path, record: dict[str, Any]) -> None:
tmp = path.with_name(f".{path.name}.tmp")
tmp.write_text(json.dumps(record), encoding="utf-8")
os.replace(tmp, path)
def stamp_exit(exit_kind: str, crash: str = "none") -> None:
"""Restamp this process's marker with how it is ending. Never raises; no-op when unmarked."""
try:
with _STATE_LOCK:
record = _STATE.get("record")
path = _STATE.get("path")
if record is None or path is None or record.get("state") != "running":
return
record = {**record, "state": exit_kind, "crash_class": crash}
_STATE["record"] = record
_write_marker(path, record)
except Exception:
logger.debug("Process exit marker not stamped", exc_info=True)
def flush_before_hard_exit(timeout_s: float = EXIT_FLUSH_TIMEOUT_S) -> bool:
"""For a graceful stop about to ``os._exit``: run the shared-metrics exit flush its skipped atexit
hooks would have run (close each live runtime's sessions under its own profile, write queued rows,
and write every store's deferred rows, the root profile's host-level store included, each into
the store that deferred it).
Bounded by ``timeout_s`` on a daemon thread, because the Relay flush waits on every managed call
still in flight and a wedged one must not hold the exit. Never imports the runtime module:
collection off (or never started) means nothing to flush and nothing created. Never raises; True
when the flush finished in time."""
runtimes = sys.modules.get("hermes_cli.observability.relay_shared_metrics")
stores = sys.modules.get("hermes_cli.observability.shared_metrics")
if runtimes is None and stores is None:
return True
deadline = time.monotonic() + max(0.0, timeout_s)
def _flush() -> None:
if runtimes is not None:
runtimes.shutdown_runtimes()
if stores is not None: # every store that deferred a write, root/Desktop ones included
stores.drain_deferring_stores(deadline - _DRAIN_MARGIN_S) # done before the join gives up
try:
from agent.memory_provider import spawn_context_thread
worker = spawn_context_thread(_flush, name="hermes-shared-metrics-exit-flush")
worker.start()
worker.join(max(0.0, deadline - time.monotonic()))
except Exception:
logger.debug("Shared-metrics exit flush not started", exc_info=True)
return False
if worker.is_alive():
logger.warning("Shared-metrics exit flush still running after %.1fs; exiting without it", timeout_s)
return False
return True
def _excepthook(previous: Any) -> Any:
def hook(exc_type: type[BaseException], exc: BaseException, tb: Any) -> None:
if not issubclass(exc_type, (KeyboardInterrupt, SystemExit)):
stamp_exit("crash", crash_class(exc))
previous(exc_type, exc, tb)
return hook
def purge_pending_receipts(home: Path) -> None:
"""Collection is off: drop the files kept for a later start to count (parked update receipts,
installer receipts). Every opt-out answer calls this too, not only an opted-out start, so a
receipt never survives a "No" to be counted after a later opt-in. Never raises."""
try:
from .shared_metrics_install_run import purge_pending_installs
from .shared_metrics_update import purge_pending_updates
purge_pending_updates(home)
purge_pending_installs(home)
except Exception:
logger.debug("Pending shared-metrics receipts not purged", exc_info=True)
def begin_process(kind: str) -> None:
"""Mark this process as a running ``kind`` and report dead predecessors. Once per process."""
try:
from hermes_constants import get_hermes_home
from .shared_metrics_desktop import ONBOARDING_LATCH_DIRNAME
from .shared_metrics_setup import markers_dir as setup_markers_dir
from .shared_metrics_update import _collection_on
if _STATE:
return
_STATE["kind"] = kind # watchdog turn rows name the surface even when this home is off
if not _collection_on():
home = get_hermes_home()
purge_pending_receipts(home) # parked while on / left by the installer, never counted once off
latches = home / "telemetry" / "shared_metrics" / ONBOARDING_LATCH_DIRNAME # an opt-out outside Desktop
for directory in (markers_dir(home), setup_markers_dir(home), latches): # likewise pending exits/setups
shutil.rmtree(directory, ignore_errors=True)
return
from gateway.status import get_process_start_time
home = get_hermes_home()
directory = markers_dir(home)
directory.mkdir(parents=True, exist_ok=True)
pid = os.getpid()
path = directory / f"{kind}-{pid}.json"
record = {"kind": kind, "pid": pid, "start_time": get_process_start_time(pid), "state": "running"}
with _STATE_LOCK:
_write_marker(path, record)
_STATE.update(path=path, record=record)
sys.excepthook = _excepthook(sys.excepthook)
atexit.register(stamp_exit, "clean")
threading.Thread(
target=_report_dead_markers, args=(home, path), name="hermes-exit-metrics", daemon=True,
).start()
except Exception:
logger.debug("Process exit marker not started", exc_info=True)
def current_process_kind() -> str:
return _STATE.get("kind") or "other"
def _claimer_alive(path: Path) -> bool:
from gateway.status import runtime_status_pid_is_live
try:
claimer = int(path.name.rsplit(".", 2)[-2])
except (IndexError, ValueError):
return False
return runtime_status_pid_is_live({"pid": claimer})
def _claim(path: Path) -> Path | None:
"""Rename a reportable marker so exactly one reporter owns it; None when not reportable."""
from gateway.status import runtime_status_pid_is_live
if path.name.endswith(_REPORTING):
if _claimer_alive(path):
return None
record_path = path
else:
record_path = None
try:
record = json.loads(path.read_text(encoding="utf-8-sig"))
except (OSError, ValueError):
return None
if not isinstance(record, dict):
return None
if record_path is None and record.get("state") == "running" and runtime_status_pid_is_live(record):
return None
claimed = path.with_name(f"{path.name.split('.json')[0]}.json.{os.getpid()}{_REPORTING}")
try:
os.replace(path, claimed)
except OSError: # a concurrent reporter won
return None
return claimed
def process_exit_fields(record: dict[str, Any]) -> dict[str, str]:
from .shared_metrics_contract import CRASH_CLASSES, PROCESS_EXIT_KINDS, PROCESS_KINDS
state = record.get("state")
exit_kind = "killed" if state == "running" or state not in PROCESS_EXIT_KINDS else state
crash = record.get("crash_class") if exit_kind == "crash" else "none"
kind = record.get("kind")
return {
"crash_class": crash if crash in CRASH_CLASSES else "other",
"exit_kind": exit_kind,
"process_kind": kind if kind in PROCESS_KINDS else "other",
}
def settle_claim(claimed: Path, original: Path, saved: bool) -> None:
"""Delete a claimed file once its row is saved; otherwise hand it back for the next start."""
try:
if saved:
claimed.unlink(missing_ok=True)
else:
os.replace(claimed, original)
except OSError: # a leftover claim is reclaimed once this reporter is gone
logger.debug("Claimed shared-metrics file not settled", exc_info=True)
def _report_dead_markers(home: Path, own: Path) -> None:
from hermes_constants import reset_hermes_home_override, set_hermes_home_override
from . import shared_metrics_contract as contract
from .shared_metrics_events import emit_saved
token = set_hermes_home_override(home) # a thread does not inherit the profile binding
try:
for path in sorted(markers_dir(home).iterdir()):
if path == own or path.name.startswith(".") or ".json" not in path.name:
continue
claimed = _claim(path)
if claimed is None:
continue
try:
record = json.loads(claimed.read_text(encoding="utf-8-sig"))
except (OSError, ValueError):
record = None
rows = [(contract.PROCESS_EXIT_MARK, process_exit_fields(record))] if isinstance(record, dict) else []
settle_claim(claimed, path, emit_saved(rows) == len(rows))
from .shared_metrics_setup import report_abandoned_setups
from .shared_metrics_update import report_pending_updates
report_pending_updates()
report_abandoned_setups(home)
# ---- iuf c2 ----
from .shared_metrics_install_run import report_pending_installs
report_pending_installs(home)
# ---- end iuf c2 ----
except Exception:
logger.debug("Dead process markers not reported", exc_info=True)
finally:
reset_hermes_home_override(token)
def arm_turn(agent: Any) -> None:
"""At turn entry (profile scope bound): the owning home for an off-thread watchdog abort, and
a fresh once-per-turn latch shared by both turn watchdogs."""
try:
from hermes_constants import get_hermes_home
agent._metrics_turn_home = str(get_hermes_home())
agent._metrics_watchdog_abort_counted = False
except Exception:
logger.debug("Turn exit metrics not armed", exc_info=True)
def record_watchdog_turn_abort(agent: Any) -> None:
"""One exit_kind=watchdog row per turn a watchdog killed, in the turn's own profile. Never raises."""
try:
home = getattr(agent, "_metrics_turn_home", None)
# No armed turn means no known owning profile: never guess the scope.
if not home or getattr(agent, "_metrics_watchdog_abort_counted", True):
return
agent._metrics_watchdog_abort_counted = True
from hermes_constants import reset_hermes_home_override, set_hermes_home_override
from . import shared_metrics_contract as contract
from .shared_metrics_events import _emit
token = set_hermes_home_override(home)
try:
_emit(contract.PROCESS_EXIT_MARK, lambda: {
"crash_class": "none", "exit_kind": "watchdog", "process_kind": current_process_kind(),
})
finally:
reset_hermes_home_override(token)
except Exception:
logger.debug("Watchdog turn abort not recorded", exc_info=True)