Skip to content

The agent that grows with you

fix(telemetry): relay shared-metrics smoke passes on main again

Teknium142dc8bbbd4a
diff --git a/scripts/smoke_nemo_relay_shared_metrics.py b/scripts/smoke_nemo_relay_shared_metrics.py
index e70260a4bb6..8b5ce96154e 100644
--- a/scripts/smoke_nemo_relay_shared_metrics.py
+++ b/scripts/smoke_nemo_relay_shared_metrics.py
@@ -255,6 +255,37 @@ def _arguments() -> argparse.Namespace:
return parser.parse_args()
+def _stop_auto_started_gateway(home: Path, timeout: float = 60.0) -> None:
+ """Stop the gateway the one-shot chat auto-started for ``home`` and wait for it to exit.
+
+ Left alone it lingers for ``gateway.unmanaged_idle_exit_seconds`` after every smoke run. Stopping
+ it before the package export also settles its asynchronous rows (delivery, first-reply latency),
+ so SQLite and the export see the same counters.
+ """
+ import psutil
+
+ try:
+ record = json.loads((home / "gateway.pid").read_text(encoding="utf-8-sig"))
+ except (OSError, ValueError):
+ return
+ if (
+ not isinstance(record, dict)
+ or record.get("kind") != "hermes-gateway"
+ or Path(str(record.get("hermes_home"))).resolve() != home.resolve()
+ ):
+ return
+ try:
+ gateway = psutil.Process(int(record["pid"]))
+ gateway.terminate()
+ gateway.wait(timeout=timeout)
+ except psutil.NoSuchProcess:
+ return
+ except psutil.TimeoutExpired as exc:
+ raise AssertionError(
+ f"Auto-started gateway {record['pid']} did not exit within {timeout:.0f}s"
+ ) from exc
+
+
def _write_config(home: Path, port: int) -> None:
home.mkdir(parents=True, exist_ok=True)
(home / "config.yaml").write_text(
@@ -276,8 +307,21 @@ telemetry:
# ---- iuf c1 ----
+# The final receipt of an update whose fetch failed before the checkout moved: the plan and snapshot
+# stages carry end marks, the apply stage it died in carries none.
+UPDATE_RECEIPT: dict[str, Any] = {
+ "schema": 1, "update_id": "0123456789abcdef",
+ "started_at": "2026-10-06T10:00:00+00:00", "finished_at": "2026-10-06T10:00:01+00:00",
+ "outcome": "failed", "exit_code": 1, "stop_reason": "sys.exit(1)", "pre_update": {},
+ "stop_class": "fetch_failed",
+ "stages": [
+ {"name": "plan", "outcome": "success", "at": "2026-10-06T10:00:00+00:00"},
+ {"name": "snapshot", "outcome": "skipped", "at": "2026-10-06T10:00:00+00:00"},
+ ],
+ "steps": [], "fleet": [],
+}
# The offline probes in the skill-lifecycle subprocess: one failed URL plugin install and one failed
-# update run (its fetch failed before the checkout moved), identical in SQLite and in the export.
+# update run (UPDATE_RECEIPT), identical in SQLite and in the export.
FAILURE_EXPECTED_DIMENSIONS = {
"hermes.extension.install.count": {
"failure_class": "clone_failed", "kind": "plugin", "name": "custom", "outcome": "failed",
@@ -291,12 +335,24 @@ FAILURE_EXPECTED_DIMENSIONS = {
}
-def _validate_failure_rows(rows: dict[str, list[dict[str, Any]]]) -> None:
- for name, dimensions in FAILURE_EXPECTED_DIMENSIONS.items():
+def expected_update_stages() -> list[tuple[str, str]]:
+ """One row per marked stage of UPDATE_RECEIPT, plus a failed row for the stage the run died in:
+ stage marks are end marks, so the apply stage the fetch failure stopped has none of its own."""
+ marked = [(stage["name"], stage["outcome"]) for stage in UPDATE_RECEIPT["stages"]]
+ died_in = FAILURE_EXPECTED_DIMENSIONS["hermes.update.run"]["failed_stage"]
+ return sorted([*marked, (died_in, "failed")])
+
+
+def _validate_exact_rows(rows: dict[str, list[dict[str, Any]]], expected: dict[str, dict[str, str]]) -> None:
+ for name, dimensions in expected.items():
if [row["dimensions"] for row in rows[name]] != [dimensions] or rows[name][0]["value"] != 1:
raise AssertionError(f"Unexpected {name}: {rows[name]}")
+
+
+def _validate_failure_rows(rows: dict[str, list[dict[str, Any]]]) -> None:
+ _validate_exact_rows(rows, FAILURE_EXPECTED_DIMENSIONS)
stages = sorted((row["dimensions"]["stage"], row["dimensions"]["outcome"]) for row in rows["hermes.update.stage"])
- if stages != [("plan", "success"), ("snapshot", "skipped")]:
+ if stages != expected_update_stages():
raise AssertionError(f"Unexpected update stages: {rows['hermes.update.stage']}")
# ---- end iuf c1 ----
@@ -315,9 +371,7 @@ V5_EXPECTED_DIMENSIONS = {
def _validate_v5_rows(rows: dict[str, list[dict[str, Any]]]) -> None:
- for name, dimensions in V5_EXPECTED_DIMENSIONS.items():
- if [row["dimensions"] for row in rows[name]] != [dimensions] or rows[name][0]["value"] != 1:
- raise AssertionError(f"Unexpected {name}: {rows[name]}")
+ _validate_exact_rows(rows, V5_EXPECTED_DIMENSIONS)
[overhead] = rows["hermes.tool_overhead.count"] # the file toolset's 4 tools, schema tokens estimated
if (
overhead["value"] != 1
@@ -328,6 +382,62 @@ def _validate_v5_rows(rows: dict[str, list[dict[str, Any]]]) -> None:
raise AssertionError(f"Unexpected tool overhead: {overhead}")
+# A one-shot chat is served by the gateway it auto-starts for the home (unified gateway runtime): that
+# gateway reports its own boot latency and the local platform's delivery and first-reply latency for
+# the turn. The turn's single read_file call is one tool batch.
+GATEWAY_TURN_EXPECTED_DIMENSIONS = {
+ "hermes.platform.delivery": {"failure_class": "none", "outcome": "sent", "platform": "local"},
+ "hermes.tool_batch.count": {"connector_share": "none", "model": "custom", "provider": "custom", "size_bucket": "1"},
+}
+# Each process that launched once reports one startup row: the chat client and the gateway it started.
+EXPECTED_STARTUP_SURFACES = ("cli", "gateway_boot")
+
+
+def _validate_gateway_turn_rows(rows: dict[str, list[dict[str, Any]]]) -> None:
+ _validate_exact_rows(rows, GATEWAY_TURN_EXPECTED_DIMENSIONS)
+ replies = rows["hermes.gateway.reply_latency"] # the bucket is whatever the host's load made it
+ if any(row["dimensions"]["platform"] != "local" for row in replies) or sum(row["value"] for row in replies) != 1:
+ raise AssertionError(f"Unexpected gateway reply latency: {replies}")
+ startup = rows["hermes.startup.latency"]
+ if sorted(row["dimensions"]["surface"] for row in startup) != sorted(EXPECTED_STARTUP_SURFACES) or any(
+ row["value"] != 1 for row in startup
+ ):
+ raise AssertionError(f"Unexpected startup latency: {startup}")
+
+
+# The metric names the scenario must produce, exactly, in SQLite and in the export.
+EXPECTED_METRIC_NAMES = frozenset({
+ "hermes.client.active",
+ "hermes.context_peak.count",
+ "hermes.extension.install.count", # iuf c1
+ "hermes.gateway.reply_latency",
+ "hermes.install.milestone",
+ "hermes.install.snapshot",
+ "hermes.model_reply_issue.count",
+ "hermes.model_route.count",
+ "hermes.model_tokens.sum",
+ "hermes.model_tool_quality.count",
+ "hermes.platform.delivery",
+ "hermes.session.count",
+ "hermes.skill.lifecycle.count",
+ "hermes.skill.load.count",
+ "hermes.startup.latency",
+ "hermes.task_cost.count",
+ "hermes.task_run.duration",
+ "hermes.task_run.finished",
+ "hermes.task_run.started",
+ "hermes.tool.usage.count",
+ "hermes.tool_batch.count",
+ "hermes.tool_call.count",
+ "hermes.tool_call.latency",
+ "hermes.tool_enabled_unused.count",
+ "hermes.tool_output_truncation.count",
+ "hermes.tool_overhead.count",
+ "hermes.update.run", # iuf c1
+ "hermes.update.stage", # iuf c1
+})
+
+
def _validate_store(database_path: Path) -> list[dict[str, Any]]:
if not database_path.is_file():
raise AssertionError(f"Metrics database was not created: {database_path}")
@@ -351,33 +461,7 @@ def _validate_store(database_path: Path) -> list[dict[str, Any]]:
by_name: dict[str, list[dict[str, Any]]] = {}
for counter in counters:
by_name.setdefault(counter["name"], []).append(counter)
- if set(by_name) != {
- "hermes.client.active",
- "hermes.context_peak.count",
- "hermes.extension.install.count", # iuf c1
- "hermes.install.milestone",
- "hermes.install.snapshot",
- "hermes.model_reply_issue.count",
- "hermes.model_route.count",
- "hermes.model_tokens.sum",
- "hermes.model_tool_quality.count",
- "hermes.session.count",
- "hermes.skill.lifecycle.count",
- "hermes.skill.load.count",
- "hermes.startup.latency",
- "hermes.task_cost.count",
- "hermes.task_run.duration",
- "hermes.task_run.finished",
- "hermes.task_run.started",
- "hermes.tool.usage.count",
- "hermes.tool_call.count",
- "hermes.tool_call.latency",
- "hermes.tool_enabled_unused.count",
- "hermes.tool_output_truncation.count",
- "hermes.tool_overhead.count",
- "hermes.update.run", # iuf c1
- "hermes.update.stage", # iuf c1
- }:
+ if set(by_name) != EXPECTED_METRIC_NAMES:
raise AssertionError(
f"Unexpected SQLite counters:\n{json.dumps(counters, indent=2)}"
)
@@ -471,9 +555,6 @@ def _validate_store(database_path: Path) -> list[dict[str, Any]]:
"custom", "custom", "no",
):
raise AssertionError(f"Unexpected context peak: {peak}")
- [startup] = by_name["hermes.startup.latency"]
- if startup["dimensions"]["surface"] != "cli":
- raise AssertionError(f"Unexpected startup latency: {startup}")
milestones = {c["dimensions"]["milestone"] for c in by_name["hermes.install.milestone"]}
if not {"first_task_started", "first_task_success", "first_tool_success"} <= milestones:
raise AssertionError(f"Missing install milestones: {sorted(milestones)}")
@@ -527,6 +608,7 @@ def _validate_store(database_path: Path) -> list[dict[str, Any]]:
):
raise AssertionError(f"Unexpected skill load counters: {loads}")
_validate_v5_rows(by_name)
+ _validate_gateway_turn_rows(by_name)
_validate_failure_rows(by_name) # iuf c1
return counters
@@ -579,33 +661,7 @@ def _validate_packages(
for package in packages:
for metric in package.get("metrics", []):
metrics.setdefault(metric["name"], []).append(metric)
- if set(metrics) != {
- "hermes.client.active",
- "hermes.context_peak.count",
- "hermes.extension.install.count", # iuf c1
- "hermes.install.milestone",
- "hermes.install.snapshot",
- "hermes.model_reply_issue.count",
- "hermes.model_route.count",
- "hermes.model_tokens.sum",
- "hermes.model_tool_quality.count",
- "hermes.session.count",
- "hermes.skill.lifecycle.count",
- "hermes.skill.load.count",
- "hermes.startup.latency",
- "hermes.task_cost.count",
- "hermes.task_run.duration",
- "hermes.task_run.finished",
- "hermes.task_run.started",
- "hermes.tool.usage.count",
- "hermes.tool_call.count",
- "hermes.tool_call.latency",
- "hermes.tool_enabled_unused.count",
- "hermes.tool_output_truncation.count",
- "hermes.tool_overhead.count",
- "hermes.update.run", # iuf c1
- "hermes.update.stage", # iuf c1
- }:
+ if set(metrics) != EXPECTED_METRIC_NAMES:
raise AssertionError(
f"Unexpected package metrics:\n{json.dumps(metrics, indent=2)}"
)
@@ -675,6 +731,7 @@ def _validate_packages(
}:
raise AssertionError(f"Unexpected skill load metrics: {loads}")
_validate_v5_rows(metrics)
+ _validate_gateway_turn_rows(metrics)
_validate_failure_rows(metrics) # iuf c1
return package_paths, packages
@@ -770,6 +827,7 @@ def main() -> int:
server.shutdown()
server.server_close()
thread.join(timeout=5)
+ _stop_auto_started_gateway(home)
(root / "hermes.stdout.txt").write_text(result.stdout, encoding="utf-8")
(root / "hermes.stderr.txt").write_text(result.stderr, encoding="utf-8")
@@ -829,13 +887,7 @@ def main() -> int:
"except PluginOperationError:",
" pass",
"from hermes_cli.observability.shared_metrics_update import record_update_receipt",
- "record_update_receipt({'schema': 1, 'update_id': '0123456789abcdef',",
- " 'started_at': '2026-10-06T10:00:00+00:00', 'finished_at': '2026-10-06T10:00:01+00:00',",
- " 'outcome': 'failed', 'exit_code': 1, 'stop_reason': 'sys.exit(1)', 'pre_update': {},",
- " 'stop_class': 'fetch_failed',",
- " 'stages': [{'name': 'plan', 'outcome': 'success', 'at': '2026-10-06T10:00:00+00:00'},",
- " {'name': 'snapshot', 'outcome': 'skipped', 'at': '2026-10-06T10:00:00+00:00'}],",
- " 'steps': [], 'fleet': []})",
+ f"record_update_receipt({UPDATE_RECEIPT!r})",
"runtime = relay_shared_metrics._get_runtime()",
"assert runtime is not None",
"runtime.shutdown()",
diff --git a/tests/scripts/test_smoke_nemo_relay_shared_metrics.py b/tests/scripts/test_smoke_nemo_relay_shared_metrics.py
index 4dbd4c78e8e..7cf6834d7a9 100644
--- a/tests/scripts/test_smoke_nemo_relay_shared_metrics.py
+++ b/tests/scripts/test_smoke_nemo_relay_shared_metrics.py
@@ -4,8 +4,18 @@ from pathlib import Path
import pytest
+from hermes_cli.observability import shared_metrics_contract as contract
+from hermes_cli.observability.shared_metrics_update import update_receipt_fields
from scripts import smoke_nemo_relay_shared_metrics as smoke
+# The smoke spawns a real chat and is not run by CI; these pin its expectations to the contract, so a
+# renamed metric, a changed field set or a new projection rule fails here instead of rotting there.
+_EXACT_EXPECTATIONS = {
+ **smoke.V5_EXPECTED_DIMENSIONS,
+ **smoke.GATEWAY_TURN_EXPECTED_DIMENSIONS,
+ **smoke.FAILURE_EXPECTED_DIMENSIONS,
+}
+
@pytest.mark.parametrize(
"relative_path",
@@ -32,3 +42,19 @@ def test_resolve_hermes_executable_reports_missing_binary(tmp_path, monkeypatch)
with pytest.raises(SystemExit, match="or on PATH"):
smoke._resolve_hermes_executable(tmp_path)
+
+
+def test_smoke_expects_only_contract_metrics_with_their_current_fields():
+ assert smoke.EXPECTED_METRIC_NAMES <= contract.COUNTER_METRICS
+ assert set(_EXACT_EXPECTATIONS) <= smoke.EXPECTED_METRIC_NAMES
+ for name, dimensions in _EXACT_EXPECTATIONS.items():
+ assert frozenset(dimensions) == contract._METRIC_FIELDS[name], name
+ assert contract.counter_dimensions_are_valid(name, dimensions), name
+ assert set(smoke.EXPECTED_STARTUP_SURFACES) <= contract.STARTUP_SURFACES
+
+
+def test_smoke_update_expectations_match_the_receipt_projection():
+ run, stage_rows = update_receipt_fields(smoke.UPDATE_RECEIPT)
+
+ assert run == smoke.FAILURE_EXPECTED_DIMENSIONS["hermes.update.run"]
+ assert sorted((row["stage"], row["outcome"]) for row in stage_rows) == smoke.expected_update_stages()