The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(compression): count each prepared summary once, at the splice
Teknium5bdf1de16ff9
diff --git a/agent/prepared_compaction.py b/agent/prepared_compaction.pyindex da6dc5344ec..3428611ee01 100644--- a/agent/prepared_compaction.py+++ b/agent/prepared_compaction.py@@ -45,7 +45,7 @@ import hashlibimport loggingimport threadingimport time-from dataclasses import dataclass, field+from dataclasses import dataclass, field, replacefrom typing import Any, Callable, Optionallogger = logging.getLogger(__name__)@@ -111,6 +111,7 @@ class PreparedSummary(_Window):body: str # bare body the next compaction extends iterativelycall_state: dict[str, Any] = field(default_factory=dict, compare=False) # _SUMMARY_CALL_STATE after the passtelemetry: dict[str, Any] = field(default_factory=dict, compare=False) # _SUMMARY_CALL_TELEMETRY of the pass+ waited: Optional[float] = field(default=None, compare=False) # seconds the taking compaction waited on the passclass _PendingPass:@@ -232,14 +233,15 @@ class PreparedCompaction:self._retry_after = 0.0 # monotonic; set by a failed pass# Set by the first compaction that needed memory-provider context; such compactions summarise fresh.self._needs_provider_context = False+ self._busy_window: Optional[tuple[str, int, int]] = None # last window the slot cap refused (counted once)def discard(self, reason: str, code: str = "session_boundary") -> None:"""Drop the candidate and cancel a running pass (it can no longer publish); ``code`` is the closedreason opt-in shared metrics count."""with self._lock:entry, pending, self._entry, self._pending = self._entry, self._pending, None, None- if pending is not None:- pending.cancel(code)+ if pending is not None:+ pending.cancel(code) # under the lock: the pass reads its cancel reason after itif entry is not None or pending is not None:_record("discarded", reason=code)dropped = entry or (pending.window if pending is not None else None)@@ -287,21 +289,21 @@ class PreparedCompaction:self._pending = None# A pass that published after the snapshot above planned on the pre-compaction transcript.self._entry = None- if pending is not None:- # Done already, or superseded: stop paying for it.- code = getattr(reason, "code", None)- pending.cancel(code if code in _WAIT_CANCEL_REASONS else "superseded")+ if pending is not None:+ # Done already, or superseded: stop paying for it.+ code = getattr(reason, "code", None)+ pending.cancel(code if code in _WAIT_CANCEL_REASONS else "superseded")if reason is not None or entry is None:if entry is not None or pending is not None:logger.info("prepared compaction: splice miss session=%s reason=%s", session_id, reason)- _record("discarded", reason=reason.code, wait_seconds=waited)+ if reason.code != "discarded_while_waiting": # the discard() that woke the wait counted it+ _record("discarded", reason=reason.code, wait_seconds=waited)return Nonelogger.info("prepared compaction: splice hit session=%s window=%d-%d (fresh cut %d)", session_id, entry.compress_start,entry.compress_end, end,)- _record("used", reason="aux_fallback_main" if entry.telemetry.get("fallback_used") else None, wait_seconds=waited)- return entry+ return replace(entry, waited=waited) # counted as used once adopt() splices itdef _await(self, pending: _PendingPass) -> tuple[Optional[PreparedSummary], Optional[_Miss]]:"""Wait on the compacting thread for *pending*: what it published, or None and why (the host's@@ -391,7 +393,9 @@ class PreparedCompaction:window = _Window(compressor._session_id, start, end, prefix_identities(working, end))# Never queue threads that retain transcripts while every slot is busy.if not _slots.acquire(blocking=False):- _record("skipped_busy", origin=origin)+ if self._busy_window != (window.session_id, start, end): # count refused passes, not tool batches+ self._busy_window = (window.session_id, start, end)+ _record("skipped_busy", origin=origin)return Nonewith self._lock:if self._pending is not None or self._entry is not None:@@ -510,6 +514,8 @@ def adopt(compressor: Any, prepared: PreparedSummary) -> None:if isinstance(telemetry, dict):telemetry.update({key: value for key, value in prepared.telemetry.items() if value is not None})telemetry["prepared_summary"] = True+ _record("used", reason="aux_fallback_main" if prepared.telemetry.get("fallback_used") else None,+ wait_seconds=prepared.waited)def maybe_prepare(agent: Any, messages: list[dict[str, Any]], *, origin: str) -> Optional[threading.Thread]:diff --git a/hermes_cli/observability/schemas/hermes.shared_metrics.v4.schema.json b/hermes_cli/observability/schemas/hermes.shared_metrics.v4.schema.jsonindex 63a8a34a077..07ab53e67aa 100644--- a/hermes_cli/observability/schemas/hermes.shared_metrics.v4.schema.json+++ b/hermes_cli/observability/schemas/hermes.shared_metrics.v4.schema.json@@ -5353,7 +5353,6 @@"enum": ["agent_released","aux_fallback_main",- "discarded_while_waiting","fresh_required","head_moved","host_cancelled",diff --git a/hermes_cli/observability/shared_metrics_prepared_compaction.py b/hermes_cli/observability/shared_metrics_prepared_compaction.pyindex bd563590d08..510d9f53da0 100644--- a/hermes_cli/observability/shared_metrics_prepared_compaction.py+++ b/hermes_cli/observability/shared_metrics_prepared_compaction.py@@ -21,7 +21,7 @@ PREPARED_COMPACTION_EVENTS = frozenset({})PREPARED_COMPACTION_ORIGINS = frozenset({"none", "tool_batch", "turn_end"})PREPARED_COMPACTION_REASONS = frozenset({- "agent_released", "aux_fallback_main", "discarded_while_waiting", "fresh_required", "head_moved", "host_cancelled",+ "agent_released", "aux_fallback_main", "fresh_required", "head_moved", "host_cancelled","inside_tail", "none", "other_session", "pass_failed_while_waiting", "provider_context", "rewound","running_not_fit", "session_boundary", "shape_changed", "superseded", "text_changed","transcript_shorter", "wait_timeout",diff --git a/tests/hermes_cli/test_shared_metrics_prepared_compaction.py b/tests/hermes_cli/test_shared_metrics_prepared_compaction.pyindex 00c30793c4e..e48618f3de3 100644--- a/tests/hermes_cli/test_shared_metrics_prepared_compaction.py+++ b/tests/hermes_cli/test_shared_metrics_prepared_compaction.py@@ -150,6 +150,20 @@ def test_waste_modes_say_why(monkeypatch, marks):attempt_seed={"attempt_id": attempt["attempt_id"], "session_id": "S1", "trigger_source": "auto"},)+ # A session boundary that wakes a waiting compaction is one discard, counted by the boundary.+ woken = _compressor()+ woken.gate = threading.Event()+ before = len(marks)+ running = pc.maybe_prepare(_agent(woken), msgs, origin="tool-batch")+ threading.Timer(0.2, woken.prepared_compaction.discard, args=("session reset",)).start()+ woken.compress(msgs, current_tokens=woken.threshold_tokens + 1)+ woken.gate.set()+ running.join(5.0)+ woken_rows = _events(marks[before:])+ assert [row for row in woken_rows if row[0] == "discarded"] == [+ ("discarded", "none", "session_boundary", "none", "none"),+ ]+# A pass whose summary call fails with a terminal access error.failed = _compressor()monkeypatch.setattr(type(failed), "_generate_summary", module.ContextCompressor._generate_summary)diff --git a/website/docs/developer-guide/relay-shared-metrics.md b/website/docs/developer-guide/relay-shared-metrics.mdindex 2129e088555..41b53a0f216 100644--- a/website/docs/developer-guide/relay-shared-metrics.md+++ b/website/docs/developer-guide/relay-shared-metrics.md@@ -455,7 +455,8 @@ offline (no network call, no subprocess):URL itself stays local.- `compression_prepare_ahead` (`on`/`off`): whether the profile's config turns on`compression.prepare_ahead` (prepared compaction), read with the same truthiness- the agent uses (absent or false reads `off`). Snapshots counted before this field+ the agent uses (absent or false reads `off`), from the profile's own `config.yaml` like the other+ snapshot fields (a managed-scope overlay is not read). Snapshots counted before this fieldexisted carry no value for it and still package.### Decision-data metrics@@ -569,7 +570,7 @@ appears in any dimension: the free-text splice-miss reason is reduced to a close| Metric | Dimensions | Question it answers ||---|---|---|-| `hermes.prepared_compaction.count` | event (`pass_started`, `pass_completed`, `pass_failed`, `pass_cancelled`, `used`, `discarded`, `compaction_failed`, `auto_disabled`, `skipped_busy`, `internal_error`), origin (`tool_batch`, `turn_end`: the hook that started the pass; `none` on compaction-side events), reason (`none`, `fresh_required`, `provider_context`, `other_session`, `head_moved`, `inside_tail`, `transcript_shorter`, `shape_changed`, `rewound`, `text_changed`, `running_not_fit`, `wait_timeout`, `host_cancelled`, `discarded_while_waiting`, `pass_failed_while_waiting`, `superseded`, `session_boundary`, `agent_released`, `aux_fallback_main`), failure class (the `hermes.compression.count` vocabulary; `none` unless the event is a failure), wait bucket (`none`, `lt_1s`, `1s_to_5s`, `5s_to_30s`, `30s_to_120s`, `gte_120s`: how long a compaction waited on a running pass) | Do prepared passes pay off, and if not, why. Pass side (from the background thread): `pass_started`, `pass_completed` (summary published), `pass_failed` (the summariser returned nothing; its class is the one an inline attempt would log, a terminal summary failure first), `pass_cancelled` (the pass saw its cancel event; reason `superseded`, `session_boundary`, `agent_released`, `wait_timeout` or `host_cancelled`, set by whoever cancelled it), `skipped_busy` (the process-wide cap of two concurrent passes refused one) and `internal_error` (failure class `exception`). Compaction side: `used` (the summary was spliced; reason `aux_fallback_main` when the pass fell back from the summary model to the main model), `discarded` (a completed or running summary the compaction did not use, with the splice-miss code and any wait; a session boundary dropping one reads `session_boundary`, releasing the agent (cache eviction, rewind, close, a one-shot or managed-worker turn ending) reads `agent_released`), `compaction_failed` (a compaction that spliced a prepared summary but did not commit, with its attempt class) and `auto_disabled` (the first compaction needing memory-provider context turned passes off for the agent). One pass can count on both sides, e.g. `discarded` + `pass_cancelled` for a running pass that did not fit. |+| `hermes.prepared_compaction.count` | event (`pass_started`, `pass_completed`, `pass_failed`, `pass_cancelled`, `used`, `discarded`, `compaction_failed`, `auto_disabled`, `skipped_busy`, `internal_error`), origin (`tool_batch`, `turn_end`: the hook that started the pass; `none` on compaction-side events), reason (`none`, `fresh_required`, `provider_context`, `other_session`, `head_moved`, `inside_tail`, `transcript_shorter`, `shape_changed`, `rewound`, `text_changed`, `running_not_fit`, `wait_timeout`, `host_cancelled`, `pass_failed_while_waiting`, `superseded`, `session_boundary`, `agent_released`, `aux_fallback_main`), failure class (the `hermes.compression.count` vocabulary; `none` unless the event is a failure), wait bucket (`none`, `lt_1s`, `1s_to_5s`, `5s_to_30s`, `30s_to_120s`, `gte_120s`: how long a compaction waited on a running pass) | Do prepared passes pay off, and if not, why. Pass side (from the background thread): `pass_started`, `pass_completed` (summary published), `pass_failed` (the summariser returned nothing; its class is the one an inline attempt would log, a terminal summary failure first), `pass_cancelled` (the pass saw its cancel event; reason `superseded`, `session_boundary`, `agent_released`, `wait_timeout` or `host_cancelled`, set by whoever cancelled it), `skipped_busy` (the process-wide cap of two concurrent passes refused a pass; counted once per refused window, not per tool batch) and `internal_error` (failure class `exception`). Compaction side: `used` (the summary was spliced into the compacted transcript, counted at the splice itself; reason `aux_fallback_main` when the pass fell back from the summary model to the main model), `discarded` (a completed or running summary the compaction did not use, with the splice-miss code and any wait; a session boundary dropping one reads `session_boundary`, releasing the agent (cache eviction, rewind, close, a one-shot or managed-worker turn ending) reads `agent_released`), `compaction_failed` (a compaction that spliced a prepared summary but did not commit, with its attempt class) and `auto_disabled` (the first compaction needing memory-provider context turned passes off for the agent). One pass can count on both sides, so never add sides together: `discarded` + `pass_cancelled` for a running pass that did not fit or was dropped at a boundary (a boundary that wakes a waiting compaction counts `discarded` once, from the boundary). |<!-- ---- v5 harness ---- -->#### Agent-harness accuracy