Skip to content

The agent that grows with you

fix(agent): drop the unscored-run warning once newer evidence is recorded

Teknium8bfa0160f888
diff --git a/agent/verification_evidence.py b/agent/verification_evidence.py
index 83e3de38600..17931f8d3e1 100644
--- a/agent/verification_evidence.py
+++ b/agent/verification_evidence.py
@@ -25,9 +25,10 @@ _MAX_EVENTS_PER_SESSION_ROOT = 100
_MAX_TOTAL_UNREFERENCED_EVENTS = 10_000
_AD_HOC_SCRIPT_NAME_PREFIXES = ("hermes-verify-", "hermes-ad-hoc-")
_VERIFY_SCHEMA_VERSION = 1
-# Latest verify-command run per (session, root) whose own exit status the shell hid. Read by
-# the stop nudge in this same process, so it stays in memory; bounded for long-lived gateways.
-_UNATTRIBUTED_RUNS: dict[tuple[str, str], str] = {}
+# Latest verify-command run per (session, root) whose own exit status the shell hid, with the
+# time it ran. Read by the stop nudge in this same process, so it stays in memory; bounded for
+# long-lived gateways.
+_UNATTRIBUTED_RUNS: dict[tuple[str, str], tuple[str, str]] = {}
_MAX_UNATTRIBUTED_RUNS = 256
_INTERPRETERS = {"python", "python3", "py", "node", "bash", "sh", "ruby", "perl"}
@@ -582,20 +583,27 @@ def record_terminal_result(
pipeline_status=pipeline_status,
)
if evidence is not None:
- _UNATTRIBUTED_RUNS.pop(key, None)
return _insert_evidence(evidence)
verify_commands = list(facts.get("verifyCommands") or [])
if _find_canonical_match(command, verify_commands, int(exit_code), attributed=False):
_UNATTRIBUTED_RUNS.pop(key, None)
- _UNATTRIBUTED_RUNS[key] = command
+ _UNATTRIBUTED_RUNS[key] = (command, _utc_now())
while len(_UNATTRIBUTED_RUNS) > _MAX_UNATTRIBUTED_RUNS:
_UNATTRIBUTED_RUNS.pop(next(iter(_UNATTRIBUTED_RUNS)))
return None
-def last_unattributed_run(*, session_id: str | None, root: str | None) -> Optional[str]:
- """The latest verify-command run in ``root`` whose own exit status could not be read."""
- return _UNATTRIBUTED_RUNS.get((str(session_id or "default"), str(root)))
+def last_unattributed_run(
+ *, session_id: str | None, root: str | None, since: str | None = None,
+) -> Optional[str]:
+ """The latest verify-command run in ``root`` whose own exit status could not be read,
+ unless evidence recorded at ``since`` came after it. ``hermes verify`` usually records
+ from its own process, so this process's memory alone cannot know about that run.
+ """
+ run = _UNATTRIBUTED_RUNS.get((str(session_id or "default"), str(root)))
+ if run is None or (since and since >= run[1]):
+ return None
+ return run[0]
def record_verify_run(
@@ -624,6 +632,7 @@ def _insert_evidence(evidence: VerificationEvidence) -> dict[str, Any]:
"""Insert a classified evidence row and repoint the workspace state."""
created_at = _utc_now()
e = evidence
+ _UNATTRIBUTED_RUNS.pop((e.session_id, e.root), None)
with _DB_LOCK, _transaction() as conn:
cur = conn.execute(
"INSERT INTO verification_events("
diff --git a/agent/verification_stop.py b/agent/verification_stop.py
index 74b4a1c9f40..20af7a65d61 100644
--- a/agent/verification_stop.py
+++ b/agent/verification_stop.py
@@ -181,7 +181,9 @@ def build_verify_on_stop_nudge(
status, facts = snapshot
if str(status.get("status") or "unverified") == "passed":
return None
- unattributed = last_unattributed_run(session_id=status.get("session_id"), root=status.get("root"))
+ unattributed = last_unattributed_run(
+ session_id=status.get("session_id"), root=status.get("root"),
+ since=(status.get("evidence") or {}).get("created_at"))
run_warning = _UNATTRIBUTED_RUN_WARNING.format(
command=elide(unattributed, _MAX_UNATTRIBUTED_COMMAND_CHARS)) if unattributed else ""
verify_commands = [str(cmd).strip() for cmd in (facts.get("verifyCommands") or []) if str(cmd).strip()]
diff --git a/tests/agent/test_verification_stop.py b/tests/agent/test_verification_stop.py
index 82ca2eca1ad..8b4a6b21de7 100644
--- a/tests/agent/test_verification_stop.py
+++ b/tests/agent/test_verification_stop.py
@@ -1,4 +1,5 @@
import json
+import subprocess
import sys
import tempfile
from pathlib import Path
@@ -189,6 +190,24 @@ def test_nudge_explains_only_a_check_run_the_shell_could_not_attribute(tmp_path,
assert "Your last check run (`pnpm test || true`) did not count" in hidden
assert "did not count" not in scored
+def test_nudge_drops_the_unscored_run_warning_once_hermes_verify_records(tmp_path, monkeypatch):
+ """``hermes verify`` records from its own process: its newer failure is the latest check."""
+ monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes"))
+ _node_project(tmp_path)
+ changed = str(tmp_path / "src" / "app.ts")
+ mark_workspace_edited(session_id="s1", cwd=tmp_path, paths=[changed])
+ record_terminal_result(command="pnpm test || true", cwd=tmp_path, session_id="s1", exit_code=0)
+
+ subprocess.run(
+ [sys.executable, "-c", "import sys; from agent.verification_evidence import record_verify_run; "
+ "record_verify_run(root=sys.argv[1], session_id='s1', ok=False)", str(tmp_path)],
+ cwd=Path(__file__).resolve().parents[2], check=True,
+ )
+ nudge = build_verify_on_stop_nudge(session_id="s1", changed_paths=[changed])
+
+ assert nudge is not None
+ assert "hermes verify" in nudge and "did not count" not in nudge
+
# ---------------------------------------------------------------------------
# Fix C: documentation/prose edits carry no verifiable behavior and must never
# trip the nudge, even on an unverified workspace.