The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(agent): show the run-it-directly advice only when a check run went unscored
Tekniumfb844ec7f9c6
diff --git a/agent/verification_evidence.py b/agent/verification_evidence.pyindex 32c12dca6ce..84f39d7e6fa 100644--- a/agent/verification_evidence.py+++ b/agent/verification_evidence.py@@ -25,6 +25,10 @@ _MAX_EVENTS_PER_SESSION_ROOT = 100_MAX_TOTAL_UNREFERENCED_EVENTS = 10_000_AD_HOC_SCRIPT_NAME_PREFIXES = ("hermes-verify-", "hermes-ad-hoc-")_VERIFY_SCHEMA_VERSION = 1+# Latest verify-command run per (session, root) whose own exit status the shell hid. Read by+# the stop nudge in this same process, so it stays in memory; bounded for long-lived gateways.+_UNATTRIBUTED_RUNS: dict[tuple[str, str], str] = {}+_MAX_UNATTRIBUTED_RUNS = 256_INTERPRETERS = {"python", "python3", "py", "node", "bash", "sh", "ruby", "perl"}# Windows spells the same interpreters `python.exe` / `py.exe` (a venv's absolute Scripts path).@@ -332,9 +336,12 @@ def _equivalent_needles(needle: list[str]) -> list[list[str]]:def _find_canonical_match(command: str, canonical_commands: list[str], exit_code: int,- pipeline_status: Optional[list[int]] = None,-) -> Optional[tuple[str, list[str], int]]:- """Return ``(canonical, trailing_args, status)`` for the first detected command."""+ pipeline_status: Optional[list[int]] = None, *, attributed: bool = True,+) -> Optional[tuple[str, list[str], Optional[int]]]:+ """Return ``(canonical, trailing_args, status)`` for the first detected command.++ ``attributed=False`` also matches a run whose own status the shell hid (status ``None``).+ """segments = _split_shell_segments(command)for canonical in canonical_commands:needle = _canonical_tokens(canonical)@@ -343,7 +350,7 @@ def _find_canonical_match(for index, segment in enumerate(segments):candidate_tokens = _normalize_tokens(_strip_command_prefix(segment.tokens))status = _segment_status(segments, index, exit_code, pipeline_status)- if status is None:+ if status is None and attributed:continuefor candidate in _equivalent_needles(needle):candidate = _normalize_tokens(candidate)@@ -515,10 +522,19 @@ def classify_verification_command(facts = _project_facts(cwd)if not facts:return None+ return _classify_in_workspace(+ command, facts, cwd=cwd, session_id=session_id, exit_code=exit_code, output=output,+ pipeline_status=pipeline_status,+ )++def _classify_in_workspace(+ command: str, facts: dict[str, Any], *, cwd: str | Path | None, session_id: str | None,+ exit_code: int, output: str, pipeline_status: Optional[list[int]],+) -> Optional[VerificationEvidence]:verify_commands = list(facts.get("verifyCommands") or [])match = _find_canonical_match(command, verify_commands, int(exit_code), pipeline_status)- if match is not None:+ if match is not None and match[2] is not None:canonical, trailing_args, status = matchkind = _kind_for_command(canonical)scope = "targeted" if any(map(_looks_like_target, trailing_args)) else "full"@@ -539,14 +555,37 @@ def record_terminal_result(*, command: str, cwd: str | Path | None, session_id: str | None, exit_code: int, output: str = "",pipeline_status: Optional[list[int]] = None,) -> Optional[dict[str, Any]]:- """Record a foreground terminal result when it is verification evidence."""- if not _ledger_enabled():+ """Record a foreground terminal result when it is verification evidence.++ A run of a detected verify command whose own status the shell hid (``pytest || true``,+ a pipe on a backend without ``PIPESTATUS``) is remembered so the stop nudge can say why+ it did not count.+ """+ if not _ledger_enabled() or not command or not isinstance(command, str):+ return None+ facts = _project_facts(cwd)+ if not facts:return None- evidence = classify_verification_command(- command, cwd=cwd, session_id=session_id, exit_code=exit_code, output=output,+ key = (str(session_id or "default"), _root_for(facts, cwd))+ evidence = _classify_in_workspace(+ command, facts, cwd=cwd, session_id=session_id, exit_code=exit_code, output=output,pipeline_status=pipeline_status,)- return None if evidence is None else _insert_evidence(evidence)+ if evidence is not None:+ _UNATTRIBUTED_RUNS.pop(key, None)+ return _insert_evidence(evidence)+ verify_commands = list(facts.get("verifyCommands") or [])+ if _find_canonical_match(command, verify_commands, int(exit_code), attributed=False):+ _UNATTRIBUTED_RUNS.pop(key, None)+ _UNATTRIBUTED_RUNS[key] = command+ while len(_UNATTRIBUTED_RUNS) > _MAX_UNATTRIBUTED_RUNS:+ _UNATTRIBUTED_RUNS.pop(next(iter(_UNATTRIBUTED_RUNS)))+ return None+++def last_unattributed_run(*, session_id: str | None, root: str | None) -> Optional[str]:+ """The latest verify-command run in ``root`` whose own exit status could not be read."""+ return _UNATTRIBUTED_RUNS.get((str(session_id or "default"), str(root)))def record_verify_run(diff --git a/agent/verification_stop.py b/agent/verification_stop.pyindex f5f835fc6a4..ca7c69b16e3 100644--- a/agent/verification_stop.py+++ b/agent/verification_stop.py@@ -10,15 +10,18 @@ from pathlib import Pathfrom typing import Any, Iterablefrom agent.compression_marker import elide+from agent.verification_evidence import last_unattributed_run_MAX_CHANGED_PATHS_IN_NUDGE = 8_MAX_STATUS_SUMMARY_CHARS = 1200-_UNPIPED_RUN_WARNING = (- " Run the check directly. If you pipe it, keep the check as the first command in "- "the pipeline so its exit status can be attributed; the terminal already truncates "- "long output and spills the rest to a file."+_MAX_UNATTRIBUTED_COMMAND_CHARS = 200+_UNATTRIBUTED_RUN_WARNING = (+ " Your last check run (`{command}`) did not count: the shell could not tell the "+ "check's own exit status apart from the commands around it. Run the check directly; "+ "if you pipe it, keep the check as the first command in the pipeline. The terminal "+ "already truncates long output and spills the rest to a file.")# Prose/data extensions and extension-less prose filenames (case-insensitive) with@@ -182,6 +185,9 @@ def build_verify_on_stop_nudge(status, facts = snapshotif str(status.get("status") or "unverified") == "passed":return None+ unattributed = last_unattributed_run(session_id=status.get("session_id"), root=status.get("root"))+ run_warning = _UNATTRIBUTED_RUN_WARNING.format(+ command=elide(unattributed, _MAX_UNATTRIBUTED_COMMAND_CHARS)) if unattributed else ""verify_commands = [str(cmd).strip() for cmd in (facts.get("verifyCommands") or []) if str(cmd).strip()]has_recipe = _workspace_has_runnable_recipe(facts.get("root"))@@ -231,7 +237,7 @@ def build_verify_on_stop_nudge("fresh passing verification evidence yet.\n\n"f"Verification status: {_status_detail(status)}\n\n"f"Changed paths:\n{_format_changed_paths(paths)}\n\n"- f"{command_instruction}{_UNPIPED_RUN_WARNING} If verification is not possible, "+ f"{command_instruction}{run_warning} If verification is not possible, ""explain the concrete blocker instead of claiming the work is fully verified."f"{addendum}]")diff --git a/tests/agent/test_verification_stop.py b/tests/agent/test_verification_stop.pyindex 865390e2723..fbd74f57a3b 100644--- a/tests/agent/test_verification_stop.py+++ b/tests/agent/test_verification_stop.py@@ -182,17 +182,28 @@ def test_path_under_a_deleted_directory_does_not_nudge(tmp_path, monkeypatch):assert build_verify_on_stop_nudge(session_id="s1", changed_paths=[gone]) is None-def test_nudge_asks_for_an_attributable_pipeline(tmp_path, monkeypatch):- """A piped run counts only when the check leads the pipeline."""+def test_nudge_explains_only_a_check_run_the_shell_could_not_attribute(tmp_path, monkeypatch):+ """The how-to-run advice names the hidden-status run, and only until a run is scored."""monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes"))_node_project(tmp_path)changed = _edited(tmp_path, "src/app.ts")mark_workspace_edited(session_id="s1", cwd=tmp_path, paths=[changed])- nudge = build_verify_on_stop_nudge(session_id="s1", changed_paths=[changed])-- assert nudge is not None- assert "keep the check as the first command in the pipeline" in nudge+ def nudge() -> str:+ text = build_verify_on_stop_nudge(session_id="s1", changed_paths=[changed])+ assert text is not None+ return text++ no_run = nudge()+ record_terminal_result(command="pnpm test || true", cwd=tmp_path, session_id="s1", exit_code=0)+ hidden = nudge()+ record_terminal_result(command="pnpm test 2>&1 | tail -5", cwd=tmp_path, session_id="s1",+ exit_code=0, pipeline_status=[1, 0])+ scored = nudge()++ assert "did not count" not in no_run+ assert "Your last check run (`pnpm test || true`) did not count" in hidden+ assert "did not count" not in scored# ---------------------------------------------------------------------------# Fix C: documentation/prose edits carry no verifiable behavior and must never