The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(telemetry): relay shared-metrics smoke passes on main again
Teknium142dc8bbbd4a
diff --git a/scripts/smoke_nemo_relay_shared_metrics.py b/scripts/smoke_nemo_relay_shared_metrics.pyindex e70260a4bb6..8b5ce96154e 100644--- a/scripts/smoke_nemo_relay_shared_metrics.py+++ b/scripts/smoke_nemo_relay_shared_metrics.py@@ -255,6 +255,37 @@ def _arguments() -> argparse.Namespace:return parser.parse_args()+def _stop_auto_started_gateway(home: Path, timeout: float = 60.0) -> None:+ """Stop the gateway the one-shot chat auto-started for ``home`` and wait for it to exit.++ Left alone it lingers for ``gateway.unmanaged_idle_exit_seconds`` after every smoke run. Stopping+ it before the package export also settles its asynchronous rows (delivery, first-reply latency),+ so SQLite and the export see the same counters.+ """+ import psutil++ try:+ record = json.loads((home / "gateway.pid").read_text(encoding="utf-8-sig"))+ except (OSError, ValueError):+ return+ if (+ not isinstance(record, dict)+ or record.get("kind") != "hermes-gateway"+ or Path(str(record.get("hermes_home"))).resolve() != home.resolve()+ ):+ return+ try:+ gateway = psutil.Process(int(record["pid"]))+ gateway.terminate()+ gateway.wait(timeout=timeout)+ except psutil.NoSuchProcess:+ return+ except psutil.TimeoutExpired as exc:+ raise AssertionError(+ f"Auto-started gateway {record['pid']} did not exit within {timeout:.0f}s"+ ) from exc++def _write_config(home: Path, port: int) -> None:home.mkdir(parents=True, exist_ok=True)(home / "config.yaml").write_text(@@ -276,8 +307,21 @@ telemetry:# ---- iuf c1 ----+# The final receipt of an update whose fetch failed before the checkout moved: the plan and snapshot+# stages carry end marks, the apply stage it died in carries none.+UPDATE_RECEIPT: dict[str, Any] = {+ "schema": 1, "update_id": "0123456789abcdef",+ "started_at": "2026-10-06T10:00:00+00:00", "finished_at": "2026-10-06T10:00:01+00:00",+ "outcome": "failed", "exit_code": 1, "stop_reason": "sys.exit(1)", "pre_update": {},+ "stop_class": "fetch_failed",+ "stages": [+ {"name": "plan", "outcome": "success", "at": "2026-10-06T10:00:00+00:00"},+ {"name": "snapshot", "outcome": "skipped", "at": "2026-10-06T10:00:00+00:00"},+ ],+ "steps": [], "fleet": [],+}# The offline probes in the skill-lifecycle subprocess: one failed URL plugin install and one failed-# update run (its fetch failed before the checkout moved), identical in SQLite and in the export.+# update run (UPDATE_RECEIPT), identical in SQLite and in the export.FAILURE_EXPECTED_DIMENSIONS = {"hermes.extension.install.count": {"failure_class": "clone_failed", "kind": "plugin", "name": "custom", "outcome": "failed",@@ -291,12 +335,24 @@ FAILURE_EXPECTED_DIMENSIONS = {}-def _validate_failure_rows(rows: dict[str, list[dict[str, Any]]]) -> None:- for name, dimensions in FAILURE_EXPECTED_DIMENSIONS.items():+def expected_update_stages() -> list[tuple[str, str]]:+ """One row per marked stage of UPDATE_RECEIPT, plus a failed row for the stage the run died in:+ stage marks are end marks, so the apply stage the fetch failure stopped has none of its own."""+ marked = [(stage["name"], stage["outcome"]) for stage in UPDATE_RECEIPT["stages"]]+ died_in = FAILURE_EXPECTED_DIMENSIONS["hermes.update.run"]["failed_stage"]+ return sorted([*marked, (died_in, "failed")])+++def _validate_exact_rows(rows: dict[str, list[dict[str, Any]]], expected: dict[str, dict[str, str]]) -> None:+ for name, dimensions in expected.items():if [row["dimensions"] for row in rows[name]] != [dimensions] or rows[name][0]["value"] != 1:raise AssertionError(f"Unexpected {name}: {rows[name]}")+++def _validate_failure_rows(rows: dict[str, list[dict[str, Any]]]) -> None:+ _validate_exact_rows(rows, FAILURE_EXPECTED_DIMENSIONS)stages = sorted((row["dimensions"]["stage"], row["dimensions"]["outcome"]) for row in rows["hermes.update.stage"])- if stages != [("plan", "success"), ("snapshot", "skipped")]:+ if stages != expected_update_stages():raise AssertionError(f"Unexpected update stages: {rows['hermes.update.stage']}")# ---- end iuf c1 ----@@ -315,9 +371,7 @@ V5_EXPECTED_DIMENSIONS = {def _validate_v5_rows(rows: dict[str, list[dict[str, Any]]]) -> None:- for name, dimensions in V5_EXPECTED_DIMENSIONS.items():- if [row["dimensions"] for row in rows[name]] != [dimensions] or rows[name][0]["value"] != 1:- raise AssertionError(f"Unexpected {name}: {rows[name]}")+ _validate_exact_rows(rows, V5_EXPECTED_DIMENSIONS)[overhead] = rows["hermes.tool_overhead.count"] # the file toolset's 4 tools, schema tokens estimatedif (overhead["value"] != 1@@ -328,6 +382,62 @@ def _validate_v5_rows(rows: dict[str, list[dict[str, Any]]]) -> None:raise AssertionError(f"Unexpected tool overhead: {overhead}")+# A one-shot chat is served by the gateway it auto-starts for the home (unified gateway runtime): that+# gateway reports its own boot latency and the local platform's delivery and first-reply latency for+# the turn. The turn's single read_file call is one tool batch.+GATEWAY_TURN_EXPECTED_DIMENSIONS = {+ "hermes.platform.delivery": {"failure_class": "none", "outcome": "sent", "platform": "local"},+ "hermes.tool_batch.count": {"connector_share": "none", "model": "custom", "provider": "custom", "size_bucket": "1"},+}+# Each process that launched once reports one startup row: the chat client and the gateway it started.+EXPECTED_STARTUP_SURFACES = ("cli", "gateway_boot")+++def _validate_gateway_turn_rows(rows: dict[str, list[dict[str, Any]]]) -> None:+ _validate_exact_rows(rows, GATEWAY_TURN_EXPECTED_DIMENSIONS)+ replies = rows["hermes.gateway.reply_latency"] # the bucket is whatever the host's load made it+ if any(row["dimensions"]["platform"] != "local" for row in replies) or sum(row["value"] for row in replies) != 1:+ raise AssertionError(f"Unexpected gateway reply latency: {replies}")+ startup = rows["hermes.startup.latency"]+ if sorted(row["dimensions"]["surface"] for row in startup) != sorted(EXPECTED_STARTUP_SURFACES) or any(+ row["value"] != 1 for row in startup+ ):+ raise AssertionError(f"Unexpected startup latency: {startup}")+++# The metric names the scenario must produce, exactly, in SQLite and in the export.+EXPECTED_METRIC_NAMES = frozenset({+ "hermes.client.active",+ "hermes.context_peak.count",+ "hermes.extension.install.count", # iuf c1+ "hermes.gateway.reply_latency",+ "hermes.install.milestone",+ "hermes.install.snapshot",+ "hermes.model_reply_issue.count",+ "hermes.model_route.count",+ "hermes.model_tokens.sum",+ "hermes.model_tool_quality.count",+ "hermes.platform.delivery",+ "hermes.session.count",+ "hermes.skill.lifecycle.count",+ "hermes.skill.load.count",+ "hermes.startup.latency",+ "hermes.task_cost.count",+ "hermes.task_run.duration",+ "hermes.task_run.finished",+ "hermes.task_run.started",+ "hermes.tool.usage.count",+ "hermes.tool_batch.count",+ "hermes.tool_call.count",+ "hermes.tool_call.latency",+ "hermes.tool_enabled_unused.count",+ "hermes.tool_output_truncation.count",+ "hermes.tool_overhead.count",+ "hermes.update.run", # iuf c1+ "hermes.update.stage", # iuf c1+})++def _validate_store(database_path: Path) -> list[dict[str, Any]]:if not database_path.is_file():raise AssertionError(f"Metrics database was not created: {database_path}")@@ -351,33 +461,7 @@ def _validate_store(database_path: Path) -> list[dict[str, Any]]:by_name: dict[str, list[dict[str, Any]]] = {}for counter in counters:by_name.setdefault(counter["name"], []).append(counter)- if set(by_name) != {- "hermes.client.active",- "hermes.context_peak.count",- "hermes.extension.install.count", # iuf c1- "hermes.install.milestone",- "hermes.install.snapshot",- "hermes.model_reply_issue.count",- "hermes.model_route.count",- "hermes.model_tokens.sum",- "hermes.model_tool_quality.count",- "hermes.session.count",- "hermes.skill.lifecycle.count",- "hermes.skill.load.count",- "hermes.startup.latency",- "hermes.task_cost.count",- "hermes.task_run.duration",- "hermes.task_run.finished",- "hermes.task_run.started",- "hermes.tool.usage.count",- "hermes.tool_call.count",- "hermes.tool_call.latency",- "hermes.tool_enabled_unused.count",- "hermes.tool_output_truncation.count",- "hermes.tool_overhead.count",- "hermes.update.run", # iuf c1- "hermes.update.stage", # iuf c1- }:+ if set(by_name) != EXPECTED_METRIC_NAMES:raise AssertionError(f"Unexpected SQLite counters:\n{json.dumps(counters, indent=2)}")@@ -471,9 +555,6 @@ def _validate_store(database_path: Path) -> list[dict[str, Any]]:"custom", "custom", "no",):raise AssertionError(f"Unexpected context peak: {peak}")- [startup] = by_name["hermes.startup.latency"]- if startup["dimensions"]["surface"] != "cli":- raise AssertionError(f"Unexpected startup latency: {startup}")milestones = {c["dimensions"]["milestone"] for c in by_name["hermes.install.milestone"]}if not {"first_task_started", "first_task_success", "first_tool_success"} <= milestones:raise AssertionError(f"Missing install milestones: {sorted(milestones)}")@@ -527,6 +608,7 @@ def _validate_store(database_path: Path) -> list[dict[str, Any]]:):raise AssertionError(f"Unexpected skill load counters: {loads}")_validate_v5_rows(by_name)+ _validate_gateway_turn_rows(by_name)_validate_failure_rows(by_name) # iuf c1return counters@@ -579,33 +661,7 @@ def _validate_packages(for package in packages:for metric in package.get("metrics", []):metrics.setdefault(metric["name"], []).append(metric)- if set(metrics) != {- "hermes.client.active",- "hermes.context_peak.count",- "hermes.extension.install.count", # iuf c1- "hermes.install.milestone",- "hermes.install.snapshot",- "hermes.model_reply_issue.count",- "hermes.model_route.count",- "hermes.model_tokens.sum",- "hermes.model_tool_quality.count",- "hermes.session.count",- "hermes.skill.lifecycle.count",- "hermes.skill.load.count",- "hermes.startup.latency",- "hermes.task_cost.count",- "hermes.task_run.duration",- "hermes.task_run.finished",- "hermes.task_run.started",- "hermes.tool.usage.count",- "hermes.tool_call.count",- "hermes.tool_call.latency",- "hermes.tool_enabled_unused.count",- "hermes.tool_output_truncation.count",- "hermes.tool_overhead.count",- "hermes.update.run", # iuf c1- "hermes.update.stage", # iuf c1- }:+ if set(metrics) != EXPECTED_METRIC_NAMES:raise AssertionError(f"Unexpected package metrics:\n{json.dumps(metrics, indent=2)}")@@ -675,6 +731,7 @@ def _validate_packages(}:raise AssertionError(f"Unexpected skill load metrics: {loads}")_validate_v5_rows(metrics)+ _validate_gateway_turn_rows(metrics)_validate_failure_rows(metrics) # iuf c1return package_paths, packages@@ -770,6 +827,7 @@ def main() -> int:server.shutdown()server.server_close()thread.join(timeout=5)+ _stop_auto_started_gateway(home)(root / "hermes.stdout.txt").write_text(result.stdout, encoding="utf-8")(root / "hermes.stderr.txt").write_text(result.stderr, encoding="utf-8")@@ -829,13 +887,7 @@ def main() -> int:"except PluginOperationError:"," pass","from hermes_cli.observability.shared_metrics_update import record_update_receipt",- "record_update_receipt({'schema': 1, 'update_id': '0123456789abcdef',",- " 'started_at': '2026-10-06T10:00:00+00:00', 'finished_at': '2026-10-06T10:00:01+00:00',",- " 'outcome': 'failed', 'exit_code': 1, 'stop_reason': 'sys.exit(1)', 'pre_update': {},",- " 'stop_class': 'fetch_failed',",- " 'stages': [{'name': 'plan', 'outcome': 'success', 'at': '2026-10-06T10:00:00+00:00'},",- " {'name': 'snapshot', 'outcome': 'skipped', 'at': '2026-10-06T10:00:00+00:00'}],",- " 'steps': [], 'fleet': []})",+ f"record_update_receipt({UPDATE_RECEIPT!r})","runtime = relay_shared_metrics._get_runtime()","assert runtime is not None","runtime.shutdown()",diff --git a/tests/scripts/test_smoke_nemo_relay_shared_metrics.py b/tests/scripts/test_smoke_nemo_relay_shared_metrics.pyindex 4dbd4c78e8e..7cf6834d7a9 100644--- a/tests/scripts/test_smoke_nemo_relay_shared_metrics.py+++ b/tests/scripts/test_smoke_nemo_relay_shared_metrics.py@@ -4,8 +4,18 @@ from pathlib import Pathimport pytest+from hermes_cli.observability import shared_metrics_contract as contract+from hermes_cli.observability.shared_metrics_update import update_receipt_fieldsfrom scripts import smoke_nemo_relay_shared_metrics as smoke+# The smoke spawns a real chat and is not run by CI; these pin its expectations to the contract, so a+# renamed metric, a changed field set or a new projection rule fails here instead of rotting there.+_EXACT_EXPECTATIONS = {+ **smoke.V5_EXPECTED_DIMENSIONS,+ **smoke.GATEWAY_TURN_EXPECTED_DIMENSIONS,+ **smoke.FAILURE_EXPECTED_DIMENSIONS,+}+@pytest.mark.parametrize("relative_path",@@ -32,3 +42,19 @@ def test_resolve_hermes_executable_reports_missing_binary(tmp_path, monkeypatch)with pytest.raises(SystemExit, match="or on PATH"):smoke._resolve_hermes_executable(tmp_path)+++def test_smoke_expects_only_contract_metrics_with_their_current_fields():+ assert smoke.EXPECTED_METRIC_NAMES <= contract.COUNTER_METRICS+ assert set(_EXACT_EXPECTATIONS) <= smoke.EXPECTED_METRIC_NAMES+ for name, dimensions in _EXACT_EXPECTATIONS.items():+ assert frozenset(dimensions) == contract._METRIC_FIELDS[name], name+ assert contract.counter_dimensions_are_valid(name, dimensions), name+ assert set(smoke.EXPECTED_STARTUP_SURFACES) <= contract.STARTUP_SURFACES+++def test_smoke_update_expectations_match_the_receipt_projection():+ run, stage_rows = update_receipt_fields(smoke.UPDATE_RECEIPT)++ assert run == smoke.FAILURE_EXPECTED_DIMENSIONS["hermes.update.run"]+ assert sorted((row["stage"], row["outcome"]) for row in stage_rows) == smoke.expected_update_stages()