Skip to content

The agent that grows with you

fix(agent): show the run-it-directly advice only when a check run went unscored

Tekniumfb844ec7f9c6
diff --git a/agent/verification_evidence.py b/agent/verification_evidence.py
index 32c12dca6ce..84f39d7e6fa 100644
--- a/agent/verification_evidence.py
+++ b/agent/verification_evidence.py
@@ -25,6 +25,10 @@ _MAX_EVENTS_PER_SESSION_ROOT = 100
_MAX_TOTAL_UNREFERENCED_EVENTS = 10_000
_AD_HOC_SCRIPT_NAME_PREFIXES = ("hermes-verify-", "hermes-ad-hoc-")
_VERIFY_SCHEMA_VERSION = 1
+# Latest verify-command run per (session, root) whose own exit status the shell hid. Read by
+# the stop nudge in this same process, so it stays in memory; bounded for long-lived gateways.
+_UNATTRIBUTED_RUNS: dict[tuple[str, str], str] = {}
+_MAX_UNATTRIBUTED_RUNS = 256
_INTERPRETERS = {"python", "python3", "py", "node", "bash", "sh", "ruby", "perl"}
# Windows spells the same interpreters `python.exe` / `py.exe` (a venv's absolute Scripts path).
@@ -332,9 +336,12 @@ def _equivalent_needles(needle: list[str]) -> list[list[str]]:
def _find_canonical_match(
command: str, canonical_commands: list[str], exit_code: int,
- pipeline_status: Optional[list[int]] = None,
-) -> Optional[tuple[str, list[str], int]]:
- """Return ``(canonical, trailing_args, status)`` for the first detected command."""
+ pipeline_status: Optional[list[int]] = None, *, attributed: bool = True,
+) -> Optional[tuple[str, list[str], Optional[int]]]:
+ """Return ``(canonical, trailing_args, status)`` for the first detected command.
+
+ ``attributed=False`` also matches a run whose own status the shell hid (status ``None``).
+ """
segments = _split_shell_segments(command)
for canonical in canonical_commands:
needle = _canonical_tokens(canonical)
@@ -343,7 +350,7 @@ def _find_canonical_match(
for index, segment in enumerate(segments):
candidate_tokens = _normalize_tokens(_strip_command_prefix(segment.tokens))
status = _segment_status(segments, index, exit_code, pipeline_status)
- if status is None:
+ if status is None and attributed:
continue
for candidate in _equivalent_needles(needle):
candidate = _normalize_tokens(candidate)
@@ -515,10 +522,19 @@ def classify_verification_command(
facts = _project_facts(cwd)
if not facts:
return None
+ return _classify_in_workspace(
+ command, facts, cwd=cwd, session_id=session_id, exit_code=exit_code, output=output,
+ pipeline_status=pipeline_status,
+ )
+
+def _classify_in_workspace(
+ command: str, facts: dict[str, Any], *, cwd: str | Path | None, session_id: str | None,
+ exit_code: int, output: str, pipeline_status: Optional[list[int]],
+) -> Optional[VerificationEvidence]:
verify_commands = list(facts.get("verifyCommands") or [])
match = _find_canonical_match(command, verify_commands, int(exit_code), pipeline_status)
- if match is not None:
+ if match is not None and match[2] is not None:
canonical, trailing_args, status = match
kind = _kind_for_command(canonical)
scope = "targeted" if any(map(_looks_like_target, trailing_args)) else "full"
@@ -539,14 +555,37 @@ def record_terminal_result(
*, command: str, cwd: str | Path | None, session_id: str | None, exit_code: int, output: str = "",
pipeline_status: Optional[list[int]] = None,
) -> Optional[dict[str, Any]]:
- """Record a foreground terminal result when it is verification evidence."""
- if not _ledger_enabled():
+ """Record a foreground terminal result when it is verification evidence.
+
+ A run of a detected verify command whose own status the shell hid (``pytest || true``,
+ a pipe on a backend without ``PIPESTATUS``) is remembered so the stop nudge can say why
+ it did not count.
+ """
+ if not _ledger_enabled() or not command or not isinstance(command, str):
+ return None
+ facts = _project_facts(cwd)
+ if not facts:
return None
- evidence = classify_verification_command(
- command, cwd=cwd, session_id=session_id, exit_code=exit_code, output=output,
+ key = (str(session_id or "default"), _root_for(facts, cwd))
+ evidence = _classify_in_workspace(
+ command, facts, cwd=cwd, session_id=session_id, exit_code=exit_code, output=output,
pipeline_status=pipeline_status,
)
- return None if evidence is None else _insert_evidence(evidence)
+ if evidence is not None:
+ _UNATTRIBUTED_RUNS.pop(key, None)
+ return _insert_evidence(evidence)
+ verify_commands = list(facts.get("verifyCommands") or [])
+ if _find_canonical_match(command, verify_commands, int(exit_code), attributed=False):
+ _UNATTRIBUTED_RUNS.pop(key, None)
+ _UNATTRIBUTED_RUNS[key] = command
+ while len(_UNATTRIBUTED_RUNS) > _MAX_UNATTRIBUTED_RUNS:
+ _UNATTRIBUTED_RUNS.pop(next(iter(_UNATTRIBUTED_RUNS)))
+ return None
+
+
+def last_unattributed_run(*, session_id: str | None, root: str | None) -> Optional[str]:
+ """The latest verify-command run in ``root`` whose own exit status could not be read."""
+ return _UNATTRIBUTED_RUNS.get((str(session_id or "default"), str(root)))
def record_verify_run(
diff --git a/agent/verification_stop.py b/agent/verification_stop.py
index f5f835fc6a4..ca7c69b16e3 100644
--- a/agent/verification_stop.py
+++ b/agent/verification_stop.py
@@ -10,15 +10,18 @@ from pathlib import Path
from typing import Any, Iterable
from agent.compression_marker import elide
+from agent.verification_evidence import last_unattributed_run
_MAX_CHANGED_PATHS_IN_NUDGE = 8
_MAX_STATUS_SUMMARY_CHARS = 1200
-_UNPIPED_RUN_WARNING = (
- " Run the check directly. If you pipe it, keep the check as the first command in "
- "the pipeline so its exit status can be attributed; the terminal already truncates "
- "long output and spills the rest to a file."
+_MAX_UNATTRIBUTED_COMMAND_CHARS = 200
+_UNATTRIBUTED_RUN_WARNING = (
+ " Your last check run (`{command}`) did not count: the shell could not tell the "
+ "check's own exit status apart from the commands around it. Run the check directly; "
+ "if you pipe it, keep the check as the first command in the pipeline. The terminal "
+ "already truncates long output and spills the rest to a file."
)
# Prose/data extensions and extension-less prose filenames (case-insensitive) with
@@ -182,6 +185,9 @@ def build_verify_on_stop_nudge(
status, facts = snapshot
if str(status.get("status") or "unverified") == "passed":
return None
+ unattributed = last_unattributed_run(session_id=status.get("session_id"), root=status.get("root"))
+ run_warning = _UNATTRIBUTED_RUN_WARNING.format(
+ command=elide(unattributed, _MAX_UNATTRIBUTED_COMMAND_CHARS)) if unattributed else ""
verify_commands = [str(cmd).strip() for cmd in (facts.get("verifyCommands") or []) if str(cmd).strip()]
has_recipe = _workspace_has_runnable_recipe(facts.get("root"))
@@ -231,7 +237,7 @@ def build_verify_on_stop_nudge(
"fresh passing verification evidence yet.\n\n"
f"Verification status: {_status_detail(status)}\n\n"
f"Changed paths:\n{_format_changed_paths(paths)}\n\n"
- f"{command_instruction}{_UNPIPED_RUN_WARNING} If verification is not possible, "
+ f"{command_instruction}{run_warning} If verification is not possible, "
"explain the concrete blocker instead of claiming the work is fully verified."
f"{addendum}]"
)
diff --git a/tests/agent/test_verification_stop.py b/tests/agent/test_verification_stop.py
index 865390e2723..fbd74f57a3b 100644
--- a/tests/agent/test_verification_stop.py
+++ b/tests/agent/test_verification_stop.py
@@ -182,17 +182,28 @@ def test_path_under_a_deleted_directory_does_not_nudge(tmp_path, monkeypatch):
assert build_verify_on_stop_nudge(session_id="s1", changed_paths=[gone]) is None
-def test_nudge_asks_for_an_attributable_pipeline(tmp_path, monkeypatch):
- """A piped run counts only when the check leads the pipeline."""
+def test_nudge_explains_only_a_check_run_the_shell_could_not_attribute(tmp_path, monkeypatch):
+ """The how-to-run advice names the hidden-status run, and only until a run is scored."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes"))
_node_project(tmp_path)
changed = _edited(tmp_path, "src/app.ts")
mark_workspace_edited(session_id="s1", cwd=tmp_path, paths=[changed])
- nudge = build_verify_on_stop_nudge(session_id="s1", changed_paths=[changed])
-
- assert nudge is not None
- assert "keep the check as the first command in the pipeline" in nudge
+ def nudge() -> str:
+ text = build_verify_on_stop_nudge(session_id="s1", changed_paths=[changed])
+ assert text is not None
+ return text
+
+ no_run = nudge()
+ record_terminal_result(command="pnpm test || true", cwd=tmp_path, session_id="s1", exit_code=0)
+ hidden = nudge()
+ record_terminal_result(command="pnpm test 2>&1 | tail -5", cwd=tmp_path, session_id="s1",
+ exit_code=0, pipeline_status=[1, 0])
+ scored = nudge()
+
+ assert "did not count" not in no_run
+ assert "Your last check run (`pnpm test || true`) did not count" in hidden
+ assert "did not count" not in scored
# ---------------------------------------------------------------------------
# Fix C: documentation/prose edits carry no verifiable behavior and must never