Skip to content

The agent that grows with you

fix(compression): count each prepared summary once, at the splice

Teknium5bdf1de16ff9
diff --git a/agent/prepared_compaction.py b/agent/prepared_compaction.py
index da6dc5344ec..3428611ee01 100644
--- a/agent/prepared_compaction.py
+++ b/agent/prepared_compaction.py
@@ -45,7 +45,7 @@ import hashlib
import logging
import threading
import time
-from dataclasses import dataclass, field
+from dataclasses import dataclass, field, replace
from typing import Any, Callable, Optional
logger = logging.getLogger(__name__)
@@ -111,6 +111,7 @@ class PreparedSummary(_Window):
body: str # bare body the next compaction extends iteratively
call_state: dict[str, Any] = field(default_factory=dict, compare=False) # _SUMMARY_CALL_STATE after the pass
telemetry: dict[str, Any] = field(default_factory=dict, compare=False) # _SUMMARY_CALL_TELEMETRY of the pass
+ waited: Optional[float] = field(default=None, compare=False) # seconds the taking compaction waited on the pass
class _PendingPass:
@@ -232,14 +233,15 @@ class PreparedCompaction:
self._retry_after = 0.0 # monotonic; set by a failed pass
# Set by the first compaction that needed memory-provider context; such compactions summarise fresh.
self._needs_provider_context = False
+ self._busy_window: Optional[tuple[str, int, int]] = None # last window the slot cap refused (counted once)
def discard(self, reason: str, code: str = "session_boundary") -> None:
"""Drop the candidate and cancel a running pass (it can no longer publish); ``code`` is the closed
reason opt-in shared metrics count."""
with self._lock:
entry, pending, self._entry, self._pending = self._entry, self._pending, None, None
- if pending is not None:
- pending.cancel(code)
+ if pending is not None:
+ pending.cancel(code) # under the lock: the pass reads its cancel reason after it
if entry is not None or pending is not None:
_record("discarded", reason=code)
dropped = entry or (pending.window if pending is not None else None)
@@ -287,21 +289,21 @@ class PreparedCompaction:
self._pending = None
# A pass that published after the snapshot above planned on the pre-compaction transcript.
self._entry = None
- if pending is not None:
- # Done already, or superseded: stop paying for it.
- code = getattr(reason, "code", None)
- pending.cancel(code if code in _WAIT_CANCEL_REASONS else "superseded")
+ if pending is not None:
+ # Done already, or superseded: stop paying for it.
+ code = getattr(reason, "code", None)
+ pending.cancel(code if code in _WAIT_CANCEL_REASONS else "superseded")
if reason is not None or entry is None:
if entry is not None or pending is not None:
logger.info("prepared compaction: splice miss session=%s reason=%s", session_id, reason)
- _record("discarded", reason=reason.code, wait_seconds=waited)
+ if reason.code != "discarded_while_waiting": # the discard() that woke the wait counted it
+ _record("discarded", reason=reason.code, wait_seconds=waited)
return None
logger.info(
"prepared compaction: splice hit session=%s window=%d-%d (fresh cut %d)", session_id, entry.compress_start,
entry.compress_end, end,
)
- _record("used", reason="aux_fallback_main" if entry.telemetry.get("fallback_used") else None, wait_seconds=waited)
- return entry
+ return replace(entry, waited=waited) # counted as used once adopt() splices it
def _await(self, pending: _PendingPass) -> tuple[Optional[PreparedSummary], Optional[_Miss]]:
"""Wait on the compacting thread for *pending*: what it published, or None and why (the host's
@@ -391,7 +393,9 @@ class PreparedCompaction:
window = _Window(compressor._session_id, start, end, prefix_identities(working, end))
# Never queue threads that retain transcripts while every slot is busy.
if not _slots.acquire(blocking=False):
- _record("skipped_busy", origin=origin)
+ if self._busy_window != (window.session_id, start, end): # count refused passes, not tool batches
+ self._busy_window = (window.session_id, start, end)
+ _record("skipped_busy", origin=origin)
return None
with self._lock:
if self._pending is not None or self._entry is not None:
@@ -510,6 +514,8 @@ def adopt(compressor: Any, prepared: PreparedSummary) -> None:
if isinstance(telemetry, dict):
telemetry.update({key: value for key, value in prepared.telemetry.items() if value is not None})
telemetry["prepared_summary"] = True
+ _record("used", reason="aux_fallback_main" if prepared.telemetry.get("fallback_used") else None,
+ wait_seconds=prepared.waited)
def maybe_prepare(agent: Any, messages: list[dict[str, Any]], *, origin: str) -> Optional[threading.Thread]:
diff --git a/hermes_cli/observability/schemas/hermes.shared_metrics.v4.schema.json b/hermes_cli/observability/schemas/hermes.shared_metrics.v4.schema.json
index 63a8a34a077..07ab53e67aa 100644
--- a/hermes_cli/observability/schemas/hermes.shared_metrics.v4.schema.json
+++ b/hermes_cli/observability/schemas/hermes.shared_metrics.v4.schema.json
@@ -5353,7 +5353,6 @@
"enum": [
"agent_released",
"aux_fallback_main",
- "discarded_while_waiting",
"fresh_required",
"head_moved",
"host_cancelled",
diff --git a/hermes_cli/observability/shared_metrics_prepared_compaction.py b/hermes_cli/observability/shared_metrics_prepared_compaction.py
index bd563590d08..510d9f53da0 100644
--- a/hermes_cli/observability/shared_metrics_prepared_compaction.py
+++ b/hermes_cli/observability/shared_metrics_prepared_compaction.py
@@ -21,7 +21,7 @@ PREPARED_COMPACTION_EVENTS = frozenset({
})
PREPARED_COMPACTION_ORIGINS = frozenset({"none", "tool_batch", "turn_end"})
PREPARED_COMPACTION_REASONS = frozenset({
- "agent_released", "aux_fallback_main", "discarded_while_waiting", "fresh_required", "head_moved", "host_cancelled",
+ "agent_released", "aux_fallback_main", "fresh_required", "head_moved", "host_cancelled",
"inside_tail", "none", "other_session", "pass_failed_while_waiting", "provider_context", "rewound",
"running_not_fit", "session_boundary", "shape_changed", "superseded", "text_changed",
"transcript_shorter", "wait_timeout",
diff --git a/tests/hermes_cli/test_shared_metrics_prepared_compaction.py b/tests/hermes_cli/test_shared_metrics_prepared_compaction.py
index 00c30793c4e..e48618f3de3 100644
--- a/tests/hermes_cli/test_shared_metrics_prepared_compaction.py
+++ b/tests/hermes_cli/test_shared_metrics_prepared_compaction.py
@@ -150,6 +150,20 @@ def test_waste_modes_say_why(monkeypatch, marks):
attempt_seed={"attempt_id": attempt["attempt_id"], "session_id": "S1", "trigger_source": "auto"},
)
+ # A session boundary that wakes a waiting compaction is one discard, counted by the boundary.
+ woken = _compressor()
+ woken.gate = threading.Event()
+ before = len(marks)
+ running = pc.maybe_prepare(_agent(woken), msgs, origin="tool-batch")
+ threading.Timer(0.2, woken.prepared_compaction.discard, args=("session reset",)).start()
+ woken.compress(msgs, current_tokens=woken.threshold_tokens + 1)
+ woken.gate.set()
+ running.join(5.0)
+ woken_rows = _events(marks[before:])
+ assert [row for row in woken_rows if row[0] == "discarded"] == [
+ ("discarded", "none", "session_boundary", "none", "none"),
+ ]
+
# A pass whose summary call fails with a terminal access error.
failed = _compressor()
monkeypatch.setattr(type(failed), "_generate_summary", module.ContextCompressor._generate_summary)
diff --git a/website/docs/developer-guide/relay-shared-metrics.md b/website/docs/developer-guide/relay-shared-metrics.md
index 2129e088555..41b53a0f216 100644
--- a/website/docs/developer-guide/relay-shared-metrics.md
+++ b/website/docs/developer-guide/relay-shared-metrics.md
@@ -455,7 +455,8 @@ offline (no network call, no subprocess):
URL itself stays local.
- `compression_prepare_ahead` (`on`/`off`): whether the profile's config turns on
`compression.prepare_ahead` (prepared compaction), read with the same truthiness
- the agent uses (absent or false reads `off`). Snapshots counted before this field
+ the agent uses (absent or false reads `off`), from the profile's own `config.yaml` like the other
+ snapshot fields (a managed-scope overlay is not read). Snapshots counted before this field
existed carry no value for it and still package.
### Decision-data metrics
@@ -569,7 +570,7 @@ appears in any dimension: the free-text splice-miss reason is reduced to a close
| Metric | Dimensions | Question it answers |
|---|---|---|
-| `hermes.prepared_compaction.count` | event (`pass_started`, `pass_completed`, `pass_failed`, `pass_cancelled`, `used`, `discarded`, `compaction_failed`, `auto_disabled`, `skipped_busy`, `internal_error`), origin (`tool_batch`, `turn_end`: the hook that started the pass; `none` on compaction-side events), reason (`none`, `fresh_required`, `provider_context`, `other_session`, `head_moved`, `inside_tail`, `transcript_shorter`, `shape_changed`, `rewound`, `text_changed`, `running_not_fit`, `wait_timeout`, `host_cancelled`, `discarded_while_waiting`, `pass_failed_while_waiting`, `superseded`, `session_boundary`, `agent_released`, `aux_fallback_main`), failure class (the `hermes.compression.count` vocabulary; `none` unless the event is a failure), wait bucket (`none`, `lt_1s`, `1s_to_5s`, `5s_to_30s`, `30s_to_120s`, `gte_120s`: how long a compaction waited on a running pass) | Do prepared passes pay off, and if not, why. Pass side (from the background thread): `pass_started`, `pass_completed` (summary published), `pass_failed` (the summariser returned nothing; its class is the one an inline attempt would log, a terminal summary failure first), `pass_cancelled` (the pass saw its cancel event; reason `superseded`, `session_boundary`, `agent_released`, `wait_timeout` or `host_cancelled`, set by whoever cancelled it), `skipped_busy` (the process-wide cap of two concurrent passes refused one) and `internal_error` (failure class `exception`). Compaction side: `used` (the summary was spliced; reason `aux_fallback_main` when the pass fell back from the summary model to the main model), `discarded` (a completed or running summary the compaction did not use, with the splice-miss code and any wait; a session boundary dropping one reads `session_boundary`, releasing the agent (cache eviction, rewind, close, a one-shot or managed-worker turn ending) reads `agent_released`), `compaction_failed` (a compaction that spliced a prepared summary but did not commit, with its attempt class) and `auto_disabled` (the first compaction needing memory-provider context turned passes off for the agent). One pass can count on both sides, e.g. `discarded` + `pass_cancelled` for a running pass that did not fit. |
+| `hermes.prepared_compaction.count` | event (`pass_started`, `pass_completed`, `pass_failed`, `pass_cancelled`, `used`, `discarded`, `compaction_failed`, `auto_disabled`, `skipped_busy`, `internal_error`), origin (`tool_batch`, `turn_end`: the hook that started the pass; `none` on compaction-side events), reason (`none`, `fresh_required`, `provider_context`, `other_session`, `head_moved`, `inside_tail`, `transcript_shorter`, `shape_changed`, `rewound`, `text_changed`, `running_not_fit`, `wait_timeout`, `host_cancelled`, `pass_failed_while_waiting`, `superseded`, `session_boundary`, `agent_released`, `aux_fallback_main`), failure class (the `hermes.compression.count` vocabulary; `none` unless the event is a failure), wait bucket (`none`, `lt_1s`, `1s_to_5s`, `5s_to_30s`, `30s_to_120s`, `gte_120s`: how long a compaction waited on a running pass) | Do prepared passes pay off, and if not, why. Pass side (from the background thread): `pass_started`, `pass_completed` (summary published), `pass_failed` (the summariser returned nothing; its class is the one an inline attempt would log, a terminal summary failure first), `pass_cancelled` (the pass saw its cancel event; reason `superseded`, `session_boundary`, `agent_released`, `wait_timeout` or `host_cancelled`, set by whoever cancelled it), `skipped_busy` (the process-wide cap of two concurrent passes refused a pass; counted once per refused window, not per tool batch) and `internal_error` (failure class `exception`). Compaction side: `used` (the summary was spliced into the compacted transcript, counted at the splice itself; reason `aux_fallback_main` when the pass fell back from the summary model to the main model), `discarded` (a completed or running summary the compaction did not use, with the splice-miss code and any wait; a session boundary dropping one reads `session_boundary`, releasing the agent (cache eviction, rewind, close, a one-shot or managed-worker turn ending) reads `agent_released`), `compaction_failed` (a compaction that spliced a prepared summary but did not commit, with its attempt class) and `auto_disabled` (the first compaction needing memory-provider context turned passes off for the agent). One pass can count on both sides, so never add sides together: `discarded` + `pass_cancelled` for a running pass that did not fit or was dropped at a boundary (a boundary that wakes a waiting compaction counts `discarded` once, from the boundary). |
<!-- ---- v5 harness ---- -->
#### Agent-harness accuracy