The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(agent): drop the unscored-run warning once newer evidence is recorded
Teknium8bfa0160f888
diff --git a/agent/verification_evidence.py b/agent/verification_evidence.pyindex 83e3de38600..17931f8d3e1 100644--- a/agent/verification_evidence.py+++ b/agent/verification_evidence.py@@ -25,9 +25,10 @@ _MAX_EVENTS_PER_SESSION_ROOT = 100_MAX_TOTAL_UNREFERENCED_EVENTS = 10_000_AD_HOC_SCRIPT_NAME_PREFIXES = ("hermes-verify-", "hermes-ad-hoc-")_VERIFY_SCHEMA_VERSION = 1-# Latest verify-command run per (session, root) whose own exit status the shell hid. Read by-# the stop nudge in this same process, so it stays in memory; bounded for long-lived gateways.-_UNATTRIBUTED_RUNS: dict[tuple[str, str], str] = {}+# Latest verify-command run per (session, root) whose own exit status the shell hid, with the+# time it ran. Read by the stop nudge in this same process, so it stays in memory; bounded for+# long-lived gateways.+_UNATTRIBUTED_RUNS: dict[tuple[str, str], tuple[str, str]] = {}_MAX_UNATTRIBUTED_RUNS = 256_INTERPRETERS = {"python", "python3", "py", "node", "bash", "sh", "ruby", "perl"}@@ -582,20 +583,27 @@ def record_terminal_result(pipeline_status=pipeline_status,)if evidence is not None:- _UNATTRIBUTED_RUNS.pop(key, None)return _insert_evidence(evidence)verify_commands = list(facts.get("verifyCommands") or [])if _find_canonical_match(command, verify_commands, int(exit_code), attributed=False):_UNATTRIBUTED_RUNS.pop(key, None)- _UNATTRIBUTED_RUNS[key] = command+ _UNATTRIBUTED_RUNS[key] = (command, _utc_now())while len(_UNATTRIBUTED_RUNS) > _MAX_UNATTRIBUTED_RUNS:_UNATTRIBUTED_RUNS.pop(next(iter(_UNATTRIBUTED_RUNS)))return None-def last_unattributed_run(*, session_id: str | None, root: str | None) -> Optional[str]:- """The latest verify-command run in ``root`` whose own exit status could not be read."""- return _UNATTRIBUTED_RUNS.get((str(session_id or "default"), str(root)))+def last_unattributed_run(+ *, session_id: str | None, root: str | None, since: str | None = None,+) -> Optional[str]:+ """The latest verify-command run in ``root`` whose own exit status could not be read,+ unless evidence recorded at ``since`` came after it. ``hermes verify`` usually records+ from its own process, so this process's memory alone cannot know about that run.+ """+ run = _UNATTRIBUTED_RUNS.get((str(session_id or "default"), str(root)))+ if run is None or (since and since >= run[1]):+ return None+ return run[0]def record_verify_run(@@ -624,6 +632,7 @@ def _insert_evidence(evidence: VerificationEvidence) -> dict[str, Any]:"""Insert a classified evidence row and repoint the workspace state."""created_at = _utc_now()e = evidence+ _UNATTRIBUTED_RUNS.pop((e.session_id, e.root), None)with _DB_LOCK, _transaction() as conn:cur = conn.execute("INSERT INTO verification_events("diff --git a/agent/verification_stop.py b/agent/verification_stop.pyindex 74b4a1c9f40..20af7a65d61 100644--- a/agent/verification_stop.py+++ b/agent/verification_stop.py@@ -181,7 +181,9 @@ def build_verify_on_stop_nudge(status, facts = snapshotif str(status.get("status") or "unverified") == "passed":return None- unattributed = last_unattributed_run(session_id=status.get("session_id"), root=status.get("root"))+ unattributed = last_unattributed_run(+ session_id=status.get("session_id"), root=status.get("root"),+ since=(status.get("evidence") or {}).get("created_at"))run_warning = _UNATTRIBUTED_RUN_WARNING.format(command=elide(unattributed, _MAX_UNATTRIBUTED_COMMAND_CHARS)) if unattributed else ""verify_commands = [str(cmd).strip() for cmd in (facts.get("verifyCommands") or []) if str(cmd).strip()]diff --git a/tests/agent/test_verification_stop.py b/tests/agent/test_verification_stop.pyindex 82ca2eca1ad..8b4a6b21de7 100644--- a/tests/agent/test_verification_stop.py+++ b/tests/agent/test_verification_stop.py@@ -1,4 +1,5 @@import json+import subprocessimport sysimport tempfilefrom pathlib import Path@@ -189,6 +190,24 @@ def test_nudge_explains_only_a_check_run_the_shell_could_not_attribute(tmp_path,assert "Your last check run (`pnpm test || true`) did not count" in hiddenassert "did not count" not in scored+def test_nudge_drops_the_unscored_run_warning_once_hermes_verify_records(tmp_path, monkeypatch):+ """``hermes verify`` records from its own process: its newer failure is the latest check."""+ monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes"))+ _node_project(tmp_path)+ changed = str(tmp_path / "src" / "app.ts")+ mark_workspace_edited(session_id="s1", cwd=tmp_path, paths=[changed])+ record_terminal_result(command="pnpm test || true", cwd=tmp_path, session_id="s1", exit_code=0)++ subprocess.run(+ [sys.executable, "-c", "import sys; from agent.verification_evidence import record_verify_run; "+ "record_verify_run(root=sys.argv[1], session_id='s1', ok=False)", str(tmp_path)],+ cwd=Path(__file__).resolve().parents[2], check=True,+ )+ nudge = build_verify_on_stop_nudge(session_id="s1", changed_paths=[changed])++ assert nudge is not None+ assert "hermes verify" in nudge and "did not count" not in nudge+# ---------------------------------------------------------------------------# Fix C: documentation/prose edits carry no verifiable behavior and must never# trip the nudge, even on an unverified workspace.