The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
feat(shared-metrics): count prepared-compaction adoption and outcomes
Teknium4ea24baeb16f
diff --git a/hermes_cli/observability/schemas/hermes.shared_metrics.v4.schema.json b/hermes_cli/observability/schemas/hermes.shared_metrics.v4.schema.jsonindex 612abaca109..efe623be862 100644--- a/hermes_cli/observability/schemas/hermes.shared_metrics.v4.schema.json+++ b/hermes_cli/observability/schemas/hermes.shared_metrics.v4.schema.json@@ -286,6 +286,9 @@},{"$ref": "#/$defs/connector_chain_counter"+ },+ {+ "$ref": "#/$defs/prepared_compaction_counter"}]}@@ -1219,9 +1222,15 @@"gte_90d","unknown"]+ },+ "compression_prepare_ahead": {+ "enum": [+ "off",+ "on"+ ]}},- "description": "The five decision-data fields are absent on snapshots counted before they existed. The six version-lag/channel/hardware fields are absent on snapshots counted before they existed."+ "description": "The five decision-data fields are absent on snapshots counted before they existed. The six version-lag/channel/hardware fields are absent on snapshots counted before they existed. compression_prepare_ahead is absent on snapshots counted before it existed."},"value": {"const": 1@@ -5257,6 +5266,128 @@"minimum": 1}}+ },+ "prepared_compaction_counter": {+ "type": "object",+ "additionalProperties": false,+ "required": [+ "name",+ "type",+ "dimensions",+ "value"+ ],+ "properties": {+ "name": {+ "const": "hermes.prepared_compaction.count"+ },+ "type": {+ "const": "counter"+ },+ "dimensions": {+ "type": "object",+ "additionalProperties": false,+ "required": [+ "event",+ "failure_class",+ "origin",+ "reason",+ "wait_bucket"+ ],+ "properties": {+ "event": {+ "enum": [+ "auto_disabled",+ "compaction_failed",+ "discarded",+ "internal_error",+ "pass_cancelled",+ "pass_completed",+ "pass_failed",+ "pass_started",+ "skipped_busy",+ "used"+ ]+ },+ "failure_class": {+ "enum": [+ "attempt_superseded",+ "aux_model_fallback",+ "commit_fence_cancelled",+ "empty_post_handoff_window",+ "empty_transcript",+ "exception",+ "explicit_interrupt",+ "feasibility_skip",+ "insufficient_messages",+ "lock_contended",+ "no_compressible_window",+ "no_progress",+ "none",+ "other",+ "pool_saturated",+ "rollback",+ "session_split_failed",+ "snapshot_stale",+ "stall_deterministic_fallback",+ "stall_interrupted",+ "summary_auth_failure",+ "summary_empty_content_failure",+ "summary_generation_aborted",+ "summary_generation_failed",+ "summary_network_failure",+ "summary_overload_degraded",+ "summary_overload_failure",+ "summary_truncated_failure",+ "unknown",+ "would_grow"+ ]+ },+ "origin": {+ "enum": [+ "none",+ "tool_batch",+ "turn_end"+ ]+ },+ "reason": {+ "enum": [+ "aux_fallback_main",+ "discarded_while_waiting",+ "fresh_required",+ "head_moved",+ "host_cancelled",+ "inside_tail",+ "none",+ "other_session",+ "pass_failed_while_waiting",+ "provider_context",+ "rewound",+ "running_not_fit",+ "session_boundary",+ "shape_changed",+ "superseded",+ "text_changed",+ "transcript_shorter",+ "wait_timeout"+ ]+ },+ "wait_bucket": {+ "enum": [+ "none",+ "lt_1s",+ "1s_to_5s",+ "5s_to_30s",+ "30s_to_120s",+ "gte_120s"+ ]+ }+ }+ },+ "value": {+ "type": "integer",+ "minimum": 1+ }+ }}}}diff --git a/hermes_cli/observability/shared_metrics_contract.py b/hermes_cli/observability/shared_metrics_contract.pyindex 825ecd49631..3ba33ea41b1 100644--- a/hermes_cli/observability/shared_metrics_contract.py+++ b/hermes_cli/observability/shared_metrics_contract.py@@ -16,6 +16,11 @@ from hermes_cli.platforms import PLATFORMSfrom tools.connectors.gateway.names import CONNECTOR_NAME_PREFIXfrom toolsets import BUILTIN_TOOL_NAMES, BUILTIN_TOOLSET_NAMES+from .shared_metrics_prepared_compaction import (+ PREPARED_COMPACTION_DIMENSIONS, PREPARED_COMPACTION_MARK, PREPARED_COMPACTION_METRIC,+ PREPARED_COMPACTION_SNAPSHOT_DIMENSIONS,+)+SCHEMA_KEY = "hermes.metrics.schema_version"# A random per-emit token (no payload) on rows whose producer waits to learn they were SAVED: facts# recovered from a file that is deleted only then. Events persist on the Relay thread.@@ -902,7 +907,7 @@ _COUNTER_DIMENSION_VALUES: dict[str, dict[str, frozenset[str]]] = {"memory_provider": MEMORY_PROVIDERS, "messaging_platform_count_bucket": SIZE_BUCKETS,"plugin_count_bucket": SIZE_BUCKETS, "profile_count_bucket": SIZE_BUCKETS,"skill_count_bucket": SIZE_BUCKETS, "terminal_backend": TERMINAL_BACKENDS,- **_INSTALL_V4_SNAPSHOT_DIMENSIONS,+ **_INSTALL_V4_SNAPSHOT_DIMENSIONS, **PREPARED_COMPACTION_SNAPSHOT_DIMENSIONS,},SESSION_METRIC: {"active_duration_bucket": SESSION_DURATION_BUCKETS, "entrypoint": TASK_ENTRYPOINTS,@@ -1057,6 +1062,7 @@ _COUNTER_DIMENSION_VALUES: dict[str, dict[str, frozenset[str]]] = {"described_first": YES_NO, "outcome": TOOL_OUTCOMES, "searches_before_bucket": COUNT_BUCKETS,},# ---- end v4 connectors ----+ PREPARED_COMPACTION_METRIC: {**PREPARED_COMPACTION_DIMENSIONS, "failure_class": COMPRESSION_FAILURE_CLASSES},}_MODEL_ROUTE_MAX_LENGTHS = {"model": MODEL_IDENTIFIER_MAX_LENGTH, "provider": PROVIDER_IDENTIFIER_MAX_LENGTH,@@ -1136,10 +1142,11 @@ _LEGACY_METRIC_FIELDS: dict[str, tuple[frozenset[str], ...]] = {SKILL_LOAD_METRIC: (_METRIC_FIELDS[SKILL_LOAD_METRIC] - {"skill_name"},),INSTALL_SNAPSHOT_METRIC: (_METRIC_FIELDS[INSTALL_SNAPSHOT_METRIC] - {"display_language", "install_age_bucket", "main_provider", "messaging_platform_count_bucket",- "terminal_backend", *_INSTALL_V4_SNAPSHOT_DIMENSIONS,+ "terminal_backend", *_INSTALL_V4_SNAPSHOT_DIMENSIONS, *PREPARED_COMPACTION_SNAPSHOT_DIMENSIONS,},# ---- v4 install ----- _METRIC_FIELDS[INSTALL_SNAPSHOT_METRIC] - set(_INSTALL_V4_SNAPSHOT_DIMENSIONS),+ _METRIC_FIELDS[INSTALL_SNAPSHOT_METRIC] - {*_INSTALL_V4_SNAPSHOT_DIMENSIONS, *PREPARED_COMPACTION_SNAPSHOT_DIMENSIONS},+ _METRIC_FIELDS[INSTALL_SNAPSHOT_METRIC] - set(PREPARED_COMPACTION_SNAPSHOT_DIMENSIONS),),}# ---- v5 engagement ----@@ -1207,7 +1214,7 @@ _DECISION_MARK_METRICS = {INSTALL_RUN_MARK: INSTALL_RUN_METRIC,# ---- end iuf c2 ----TOOL_SEARCH_MARK: TOOL_SEARCH_METRIC, TOOL_BATCH_MARK: TOOL_BATCH_METRIC,- CONNECTOR_CHAIN_MARK: CONNECTOR_CHAIN_METRIC,+ CONNECTOR_CHAIN_MARK: CONNECTOR_CHAIN_METRIC, PREPARED_COMPACTION_MARK: PREPARED_COMPACTION_METRIC,}diff --git a/hermes_cli/observability/shared_metrics_prepared_compaction.py b/hermes_cli/observability/shared_metrics_prepared_compaction.pynew file mode 100644index 00000000000..ce02cecc397--- /dev/null+++ b/hermes_cli/observability/shared_metrics_prepared_compaction.py@@ -0,0 +1,87 @@+"""Prepared compaction (``compression.prepare_ahead``): adoption and every way a prepared pass is wasted.++The daily snapshot carries ``compression_prepare_ahead`` (``on``/``off``), read with the agent's own+truthiness, so "active installs with the feature on" needs no join. ``hermes.prepared_compaction.count``+counts each pass and splice outcome from ``agent/prepared_compaction.py`` (and a spliced compaction that+then failed, from ``agent/conversation_compression_telemetry.py``) over closed sets only: the event, the+hook that started the pass, a reason code (never the free-text miss reason), the compression failure+class, and how long a compaction waited on a running pass, bucketed.++The contract imports this module at load time, so it never imports the contract at module level.+"""++from __future__ import annotations++from typing import Any++PREPARED_COMPACTION_MARK = PREPARED_COMPACTION_METRIC = "hermes.prepared_compaction.count"+PREPARED_COMPACTION_EVENTS = frozenset({+ "auto_disabled", "compaction_failed", "discarded", "internal_error", "pass_cancelled", "pass_completed",+ "pass_failed", "pass_started", "skipped_busy", "used",+})+PREPARED_COMPACTION_ORIGINS = frozenset({"none", "tool_batch", "turn_end"})+PREPARED_COMPACTION_REASONS = frozenset({+ "aux_fallback_main", "discarded_while_waiting", "fresh_required", "head_moved", "host_cancelled",+ "inside_tail", "none", "other_session", "pass_failed_while_waiting", "provider_context", "rewound",+ "running_not_fit", "session_boundary", "shape_changed", "superseded", "text_changed",+ "transcript_shorter", "wait_timeout",+})+PREPARED_COMPACTION_WAIT_BUCKETS = frozenset({"none", "lt_1s", "1s_to_5s", "5s_to_30s", "30s_to_120s", "gte_120s"})+_WAIT_THRESHOLDS = ((1, "lt_1s"), (5, "1s_to_5s"), (30, "5s_to_30s"), (120, "30s_to_120s"))+# failure_class (the compression vocabulary) is added by the contract, which owns that set.+PREPARED_COMPACTION_DIMENSIONS = {+ "event": PREPARED_COMPACTION_EVENTS, "origin": PREPARED_COMPACTION_ORIGINS,+ "reason": PREPARED_COMPACTION_REASONS, "wait_bucket": PREPARED_COMPACTION_WAIT_BUCKETS,+}+PREPARED_COMPACTION_SNAPSHOT_DIMENSIONS = {"compression_prepare_ahead": frozenset({"off", "on"})}+++def prepare_ahead_state(config: Any) -> str:+ """``on`` when the profile config turns ``compression.prepare_ahead`` on, read as the agent reads it."""+ from utils import is_truthy_value++ compression = config.get("compression") if isinstance(config, dict) else None+ value = compression.get("prepare_ahead") if isinstance(compression, dict) else None+ return "on" if is_truthy_value(value, default=False) else "off"+++def wait_bucket(seconds: Any) -> str:+ if isinstance(seconds, bool) or not isinstance(seconds, (int, float)) or seconds < 0:+ return "none"+ return next((label for limit, label in _WAIT_THRESHOLDS if seconds < limit), "gte_120s")+++def _closed(value: Any, allowed: frozenset[str], fallback: str) -> str:+ text = str(value or "").strip().lower().replace("-", "_")+ return text if text in allowed else fallback+++def prepared_compaction_fields(+ *, event: Any, origin: Any, reason: Any, failure_class: Any, wait_seconds: Any,+) -> dict[str, str] | None:+ """Closed dimensions for one event; ``None`` for an event outside the set (never recorded)."""+ from .shared_metrics_fields import compression_failure_class++ name = _closed(event, PREPARED_COMPACTION_EVENTS, "")+ if not name:+ return None+ return {+ "event": name,+ "failure_class": "none" if failure_class is None else compression_failure_class("failed", failure_class),+ "origin": _closed(origin, PREPARED_COMPACTION_ORIGINS, "none"),+ "reason": _closed(reason, PREPARED_COMPACTION_REASONS, "none"),+ "wait_bucket": wait_bucket(wait_seconds),+ }+++def record_prepared_compaction(+ event: str, *, origin: str | None = None, reason: str | None = None, failure_class: str | None = None,+ wait_seconds: float | None = None,+) -> None:+ """Count one prepared-compaction event (no-op unless shared metrics are on; never raises)."""+ from .shared_metrics_events import _emit++ _emit(+ PREPARED_COMPACTION_MARK, prepared_compaction_fields, event=event, origin=origin, reason=reason,+ failure_class=failure_class, wait_seconds=wait_seconds,+ )diff --git a/hermes_cli/observability/shared_metrics_snapshot.py b/hermes_cli/observability/shared_metrics_snapshot.pyindex f15dd831cbf..563df11eb7d 100644--- a/hermes_cli/observability/shared_metrics_snapshot.py+++ b/hermes_cli/observability/shared_metrics_snapshot.py@@ -127,6 +127,7 @@ def collect_install_snapshot(config: dict[str, Any]) -> dict[str, str]:with contextlib.suppress(Exception):language = _display_language()from .shared_metrics_install import install_v4_snapshot_fields+ from .shared_metrics_prepared_compaction import prepare_ahead_statereturn {**contract.install_snapshot_fields(memory_provider=_sub(_sub(config, "memory"), "provider"),@@ -140,4 +141,4 @@ def collect_install_snapshot(config: dict[str, Any]) -> dict[str, str]:main_provider=_sub(model, "provider") if isinstance(model, dict) else None,terminal_backend=_sub(_sub(config, "terminal"), "backend"),display_language=language,- ), **install_v4_snapshot_fields(config)}+ ), **install_v4_snapshot_fields(config), "compression_prepare_ahead": prepare_ahead_state(config)}diff --git a/scripts/smoke_nemo_relay_shared_metrics.py b/scripts/smoke_nemo_relay_shared_metrics.pyindex 50cccb49293..0f80c32b07e 100644--- a/scripts/smoke_nemo_relay_shared_metrics.py+++ b/scripts/smoke_nemo_relay_shared_metrics.py@@ -557,7 +557,11 @@ def _validate_store(database_path: Path) -> list[dict[str, Any]]:]:raise AssertionError(f"Unexpected tool usage: {by_name['hermes.tool.usage.count']}")[snapshot] = by_name["hermes.install.snapshot"]- if snapshot["value"] != 1 or snapshot["dimensions"]["memory_provider"] != "builtin":+ if (+ snapshot["value"] != 1 or snapshot["dimensions"]["memory_provider"] != "builtin"+ # The smoke config leaves compression.prepare_ahead at its default.+ or snapshot["dimensions"]["compression_prepare_ahead"] != "off"+ ):raise AssertionError(f"Unexpected install snapshot: {snapshot}")[session] = by_name["hermes.session.count"]if (session["dimensions"]["turn_count_bucket"], session["dimensions"]["last_outcome"]) != (diff --git a/tests/hermes_cli/test_relay_shared_metrics_runtime.py b/tests/hermes_cli/test_relay_shared_metrics_runtime.pyindex 5aa9164c5d4..eb71458f473 100644--- a/tests/hermes_cli/test_relay_shared_metrics_runtime.py+++ b/tests/hermes_cli/test_relay_shared_metrics_runtime.py@@ -2863,7 +2863,7 @@ def test_install_snapshot_is_daily_and_carries_only_bucketed_counts("main_provider": "none", "mcp_server_count_bucket": "1", "memory_provider": "plugin","messaging_platform_count_bucket": "0", "plugin_count_bucket": "3_to_5","profile_count_bucket": snapshot["profile_count_bucket"], "skill_count_bucket": "0",- "terminal_backend": "local", "local_model_provider_used": "no",+ "terminal_backend": "local", "local_model_provider_used": "no", "compression_prepare_ahead": "off",**{k: snapshot[k] for k in ("behind_bucket", "gpu_class", "ram_bucket", "release_channel", "version_age_bucket")},}diff --git a/tests/hermes_cli/test_shared_metrics_prepared_compaction.py b/tests/hermes_cli/test_shared_metrics_prepared_compaction.pynew file mode 100644index 00000000000..f6b23809de4--- /dev/null+++ b/tests/hermes_cli/test_shared_metrics_prepared_compaction.py@@ -0,0 +1,47 @@+"""Opt-in shared metrics for prepared compaction (``compression.prepare_ahead``).++* Adoption: the daily snapshot says whether the profile turned the feature on, read as the agent reads it,+ and snapshots counted before the field existed still package against the v4 schema.+* ``hermes.prepared_compaction.count``: every pass and splice outcome lands as one closed-enum row in the+ profile that owns the compressor, including the rows the background pass thread records, and every+ waste mode (a summary nothing uses, a wait that timed out, a failed pass, a spliced compaction that did+ not commit) says why.+"""+from __future__ import annotations++import json+from pathlib import Path++import pytest++from hermes_cli.observability import shared_metrics_contract as contract+from hermes_cli.observability.shared_metrics import SharedMetricsStore+from hermes_cli.observability.shared_metrics_snapshot import collect_install_snapshot++SCHEMA_PATH = (+ Path(__file__).resolve().parents[2] / "hermes_cli" / "observability" / "schemas"+ / "hermes.shared_metrics.v4.schema.json"+)+RESOURCE = {"architecture": "x86_64", "hermes_version": "test", "install_method": "git", "os_family": "linux"}+++def test_snapshot_reports_prepare_ahead_and_older_snapshots_still_package(tmp_path, monkeypatch):+ jsonschema = pytest.importorskip("jsonschema")+ monkeypatch.setenv("HERMES_HOME", str(tmp_path / "home"))+ states = {+ value: collect_install_snapshot({"compression": {"prepare_ahead": value}})["compression_prepare_ahead"]+ for value in (True, "yes", "on", False, "false", None)+ }+ assert states == {True: "on", "yes": "on", "on": "on", False: "off", "false": "off", None: "off"}+ current = collect_install_snapshot({})+ assert current["compression_prepare_ahead"] == "off", "absent reads off, as the agent's default"++ store = SharedMetricsStore(tmp_path / "metrics.sqlite3", tmp_path / "outbox")+ before_upgrade = {k: v for k, v in current.items() if k != "compression_prepare_ahead"}+ store.record_counter(contract.INSTALL_SNAPSHOT_METRIC, before_upgrade, RESOURCE)+ store.record_counter(contract.INSTALL_SNAPSHOT_METRIC, {**current, "compression_prepare_ahead": "on"}, RESOURCE)+ [package_path] = store.create_and_export_package()+ package = json.loads(package_path.read_text(encoding="utf-8"))+ jsonschema.Draft202012Validator(json.loads(SCHEMA_PATH.read_text(encoding="utf-8"))).validate(package)+ snapshots = [m["dimensions"] for m in package["metrics"] if m["name"] == contract.INSTALL_SNAPSHOT_METRIC]+ assert sorted(d.get("compression_prepare_ahead", "absent") for d in snapshots) == ["absent", "on"]diff --git a/website/docs/developer-guide/relay-shared-metrics.md b/website/docs/developer-guide/relay-shared-metrics.mdindex 7d49802e120..db83ed9cae8 100644--- a/website/docs/developer-guide/relay-shared-metrics.md+++ b/website/docs/developer-guide/relay-shared-metrics.md@@ -453,6 +453,10 @@ offline (no network call, no subprocess):auxiliary task runs on a local or self-hosted server (a local provider id suchas Ollama/LM Studio/llama.cpp, or a loopback/private-network base URL). TheURL itself stays local.+- `compression_prepare_ahead` (`on`/`off`): whether the profile's config turns on+ `compression.prepare_ahead` (prepared compaction), read with the same truthiness+ the agent uses (absent or false reads `off`). Snapshots counted before this field+ existed carry no value for it and still package.### Decision-data metrics@@ -556,6 +560,17 @@ Added inside package schema v4. No connector slug, tool name or query text appea| `hermes.tool_batch.count` | provider, model, size bucket (logical calls; each `tool_call` entry counts once), connector share (`none`, `some`, `all`) | How many calls a model packs into one reply, and how much of it is connector work. One row per model reply that calls tools. || `hermes.connector_chain.count` | searches-before bucket (`tool_search` calls since the previous connector execute), described first (`yes`/`no`: the tool was named in an earlier `tool_describe` of the conversation), outcome (the tool outcome) | Do models search and describe before they execute a connector tool, and does skipping it fail more. One row per executed connector entry. |+#### Prepared compaction++Added inside package schema v4. Answers whether installs that turn on `compression.prepare_ahead`+(see `compression_prepare_ahead` on the snapshot) get a usable summary, or pay for passes nothing uses,+wait on them, or see a spliced compaction fail. No transcript text, window index, summary or miss text+appears in any dimension: the free-text splice-miss reason is reduced to a closed code.++| Metric | Dimensions | Question it answers |+|---|---|---|+| `hermes.prepared_compaction.count` | event (`pass_started`, `pass_completed`, `pass_failed`, `pass_cancelled`, `used`, `discarded`, `compaction_failed`, `auto_disabled`, `skipped_busy`, `internal_error`), origin (`tool_batch`, `turn_end`: the hook that started the pass; `none` on compaction-side events), reason (`none`, `fresh_required`, `provider_context`, `other_session`, `head_moved`, `inside_tail`, `transcript_shorter`, `shape_changed`, `rewound`, `text_changed`, `running_not_fit`, `wait_timeout`, `host_cancelled`, `discarded_while_waiting`, `pass_failed_while_waiting`, `superseded`, `session_boundary`, `aux_fallback_main`), failure class (the `hermes.compression.count` vocabulary; `none` unless the event is a failure), wait bucket (`none`, `lt_1s`, `1s_to_5s`, `5s_to_30s`, `30s_to_120s`, `gte_120s`: how long a compaction waited on a running pass) | Do prepared passes pay off, and if not, why. Pass side (from the background thread): `pass_started`, `pass_completed` (summary published), `pass_failed` (the summariser returned nothing; its class is the one an inline attempt would log, a terminal summary failure first), `pass_cancelled` (the pass saw its cancel event; reason `superseded`, `session_boundary`, `wait_timeout` or `host_cancelled`, set by whoever cancelled it), `skipped_busy` (the process-wide cap of two concurrent passes refused one) and `internal_error` (failure class `exception`). Compaction side: `used` (the summary was spliced; reason `aux_fallback_main` when the pass fell back from the summary model to the main model), `discarded` (a completed or running summary the compaction did not use, with the splice-miss code and any wait; a session boundary dropping one reads `session_boundary`), `compaction_failed` (a compaction that spliced a prepared summary but did not commit, with its attempt class) and `auto_disabled` (the first compaction needing memory-provider context turned passes off for the agent). One pass can count on both sides, e.g. `discarded` + `pass_cancelled` for a running pass that did not fit. |+<!-- ---- v5 harness ---- -->#### Agent-harness accuracy@@ -679,7 +694,8 @@ Package schemas v1, v2 and v3 remain unchanged for existing outbox files. Newpackages use v4, which adds `failure_class` to `hermes.compression.count` and`hermes.memory.op.count` and still accepts their v3 field sets (and, extended in place with additive values only, also after v4 shipped in a stable release,`failure_class` on `hermes.extension.install.count` / `hermes.update.run` plus `registry` on the-former, still accepting their earlier v4 field sets), as v3 accepted the v2+former, still accepting their earlier v4 field sets; `hermes.prepared_compaction.count` and the+snapshot's optional `compression_prepare_ahead` arrived the same way), as v3 accepted the v2field sets of `hermes.model_route.count`, `hermes.tool_call.count` and the taskcounters, so counters recorded before an upgrade drain safely.Vocabularies derived from in-repo registries (tool names, platforms, memory