Skip to content

The agent that grows with you

4.5 KB
Raw Blame
"""Tests for the shared-metrics smoke artifact."""

import json
import subprocess
from pathlib import Path

import pytest

from gateway import status
from hermes_cli.observability import shared_metrics_contract as contract
from hermes_cli.observability.shared_metrics import SharedMetricsStore
from hermes_cli.observability.shared_metrics_update import update_receipt_fields
from scripts import smoke_nemo_relay_shared_metrics as smoke

# The smoke spawns a real chat and is not run by CI; these pin its expectations to the contract, so a
# renamed metric, a changed field set or a new projection rule fails here instead of rotting there.
_EXACT_EXPECTATIONS = {
    **smoke.V5_EXPECTED_DIMENSIONS,
    **smoke.GATEWAY_TURN_EXPECTED_DIMENSIONS,
    **smoke.FAILURE_EXPECTED_DIMENSIONS,
}


@pytest.mark.parametrize(
    "relative_path",
    [
        Path(".venv") / "bin" / "hermes",
        Path(".venv") / "Scripts" / "hermes.exe",
    ],
)
def test_resolve_hermes_executable_from_repository_venv(
    tmp_path,
    monkeypatch,
    relative_path,
):
    executable = tmp_path / relative_path
    executable.parent.mkdir(parents=True)
    executable.touch()
    monkeypatch.setattr(smoke.shutil, "which", lambda _name: None)

    assert smoke._resolve_hermes_executable(tmp_path) == executable


def test_resolve_hermes_executable_reports_missing_binary(tmp_path, monkeypatch):
    monkeypatch.setattr(smoke.shutil, "which", lambda _name: None)

    with pytest.raises(SystemExit, match="or on PATH"):
        smoke._resolve_hermes_executable(tmp_path)


def test_smoke_expects_only_contract_metrics_with_their_current_fields():
    assert smoke.EXPECTED_METRIC_NAMES <= contract.COUNTER_METRICS
    assert set(_EXACT_EXPECTATIONS) <= smoke.EXPECTED_METRIC_NAMES
    for name, dimensions in _EXACT_EXPECTATIONS.items():
        assert frozenset(dimensions) == contract._METRIC_FIELDS[name], name
        assert contract.counter_dimensions_are_valid(name, dimensions), name
    assert set(smoke.EXPECTED_STARTUP_SURFACES) <= contract.STARTUP_SURFACES


@pytest.mark.platforms("posix")  # flock-held gateway lock
@pytest.mark.parametrize("start_time_shift", [0, 10_000], ids=["live-non-gateway", "recycled-pid"])
def test_gateway_stop_spares_a_process_that_is_not_the_homes_live_gateway(tmp_path, start_time_shift):
    """A held lock and a gateway record naming this home do not make the live PID a gateway."""
    import fcntl

    decoy = subprocess.Popen(["sleep", "300"])
    try:
        record = json.dumps({
            "pid": decoy.pid, "kind": "hermes-gateway", "argv": ["hermes", "gateway", "run"],
            "start_time": status._get_process_start_time(decoy.pid) + start_time_shift,
            "hermes_home": str(tmp_path),
        })
        (tmp_path / "gateway.pid").write_text(record)
        with open(tmp_path / "gateway.lock", "w") as lock:
            lock.write(record)
            lock.flush()
            fcntl.flock(lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
            smoke._stop_auto_started_gateway(tmp_path, timeout=5)
        assert decoy.poll() is None
    finally:
        decoy.kill()
        decoy.wait()


def test_gateway_turn_wait_settles_only_on_the_rows_in_the_real_store(tmp_path):
    """The stop does not flush the gateway's rows, so the smoke waits for them in the store itself."""
    database = tmp_path / "metrics.sqlite3"
    assert not smoke._wait_for_gateway_turn_rows(database, timeout=0)
    assert not database.exists()  # polling never creates the store ahead of the gateway

    resource = {"architecture": "x86_64", "hermes_version": "0.0.0", "install_method": "git", "os_family": "linux"}
    store = SharedMetricsStore(database, tmp_path / "outbox")
    for name, dimensions in [
        *smoke.GATEWAY_TURN_EXPECTED_DIMENSIONS.items(),
        ("hermes.gateway.reply_latency", {"first_response_bucket": "lt_2s", "platform": "local"}),
        ("hermes.startup.latency", {"latency_bucket": "lt_500ms", "surface": "cli"}),
    ]:
        store.record_counter(name, dimensions, resource)
    assert not smoke._wait_for_gateway_turn_rows(database, timeout=0)  # gateway_boot still pending

    store.record_counter("hermes.startup.latency", {"latency_bucket": "2s_to_5s", "surface": "gateway_boot"}, resource)
    assert smoke._wait_for_gateway_turn_rows(database, timeout=0)


def test_smoke_update_expectations_match_the_receipt_projection():
    run, stage_rows = update_receipt_fields(smoke.UPDATE_RECEIPT)

    assert run == smoke.FAILURE_EXPECTED_DIMENSIONS["hermes.update.run"]
    assert sorted((row["stage"], row["outcome"]) for row in stage_rows) == smoke.expected_update_stages()