Skip to content
Python

fix(observability): a scanner that is not installed did not scan (P3B-1)

Devika Verma2213c77bc6d6
diff --git a/coworker/engine.py b/coworker/engine.py
index 557beb35..6df63da6 100644
--- a/coworker/engine.py
+++ b/coworker/engine.py
@@ -40,6 +40,7 @@ from .observability.engine_hooks import ToolRun, TurnRecording, set_tool_run
from .observability.recorder import short_hash as _short_hash
from .observability import stamps as _stamps
from .observability.canonical import short_hash as _args_hash, text_hash as _text_hash
+from .observability.scopes import scanner_not_found as _scanner_not_found
from .observability.scopes import scanner_of as _scanner_of
from .observability.provider_view import tag as _tag_provider, view_for_engine
from .observability.ids import new_ulid
@@ -3087,6 +3088,11 @@ class TurnEngine:
# 1.3 (E17b): the scanner this command runs, by the signals.yaml word rule.
command = arguments.get("command") if isinstance(arguments, dict) else None
out["scanner"] = _scanner_of(command) if isinstance(command, str) else None
+ # A scanner that is not installed did not scan (P3B-1): on Windows that exits 1,
+ # the same code as "found something", so the output tells them apart.
+ if ran and out["scanner"] and isinstance(result, dict) and _scanner_not_found(
+ out["scanner"], result.get("exit_code"), result.get("output") or result.get("error")):
+ out["scanner"] = None
# 1.3 (E17b): the full SHA-256 of the raw result, the evidence card's "output sha256".
out["result_sha256"] = _result_sha256(result) if ran else None
if name == "todo_write" and isinstance(arguments, dict):
diff --git a/coworker/observability/scopes.py b/coworker/observability/scopes.py
index fa281f17..c782bae2 100644
--- a/coworker/observability/scopes.py
+++ b/coworker/observability/scopes.py
@@ -59,6 +59,27 @@ def scanner_of(command: str) -> Optional[dict[str, str]]:
return None
+def scanner_not_found(scanner: Optional[dict], exit_code: Any, output: Any) -> bool:
+ """Whether a shell step that names a scanner only showed the scanner is not installed
+ (signals.yaml `scanners.not_found`, P3B-1): a not-found exit code, or a line of the output
+ that names the scanner's word with a not-found phrase. Then the engine records no scan."""
+ if not scanner:
+ return False
+ rule = _scanners().get("not_found") or {}
+ if exit_code in (rule.get("exit") or []):
+ return True
+ if not isinstance(output, str) or not output:
+ return False
+ sc = (_scanners().get("list") or {}).get(scanner.get("name")) or {}
+ word = str(sc.get("word") or scanner.get("name") or "").lower()
+ words = {word, word.replace("-", "_")} - {""}
+ phrases = [str(p).lower() for p in rule.get("phrases") or []]
+ for line in output.lower().splitlines():
+ if any(w in line for w in words) and any(p in line for p in phrases):
+ return True
+ return False
+
+
def area_word(area: str) -> str:
return (_scanners().get("area_words") or {}).get(area, area)
diff --git a/coworker/observability/signals.yaml b/coworker/observability/signals.yaml
index 3bc417c0..2383e0ff 100644
--- a/coworker/observability/signals.yaml
+++ b/coworker/observability/signals.yaml
@@ -647,6 +647,20 @@ scanners:
area_words: {code: code, secrets: secret, deps: dependency, iac: infrastructure}
lead_ins: [sudo, npx, uvx, pipx, run, exec, env, time]
probe_words: ["--version", version, "-v", "--help"]
+ # A scanner that is not installed did not scan (P3B-1, found in review 2026-11-04; local to
+ # this copy until the shared file takes it, see the Phase 3 build notes). PowerShell exits 1
+ # for a command it does not know, the same code most scanners use for "found something", so
+ # the exit code alone cannot tell them apart. When recording a shell step whose command names
+ # a scanner, the engine records `scanner: null` (no scan ran) when:
+ # - the command exited with one of `exit` (127: sh "command not found"; 9009: cmd.exe), or
+ # - a line of its output names the scanner's word and holds one of `phrases` (case-insensitive).
+ # The checks then read it as a scan that never ran: scan_missing fires, scanner_failed and
+ # the coverage list do not count it.
+ not_found:
+ exit: [127, 9009]
+ # a line must also name the scanner's word (or the word with _ for -, as python -m prints it)
+ phrases: ["is not recognized as", "command not found", ": not found", "commandnotfoundexception",
+ "no module named", "could not determine executable to run"]
list:
semgrep: {word: semgrep, area: code, findings_exit: [1]}
gitleaks: {word: gitleaks, area: secrets, findings_exit: [1]}
diff --git a/tests/test_observability_phase3_engine.py b/tests/test_observability_phase3_engine.py
index 8a0b5fbd..4ca11d42 100644
--- a/tests/test_observability_phase3_engine.py
+++ b/tests/test_observability_phase3_engine.py
@@ -390,3 +390,59 @@ def test_a_turn_on_a_new_event_loop_is_not_read_as_stopped(tmp_path):
models = [s for s in d["steps"] if s["kind"] == "model"]
assert len(models) == 2 and all(s["turn_id"] for s in models)
assert {s["turn_id"] for s in models} == {t["turn_id"] for t in d["turns"]}
+
+
+POWERSHELL_NOT_FOUND = (
+ "semgrep : The term 'semgrep' is not recognized as the name of a cmdlet, function, script file, or operable\n"
+ "program. Check the spelling of the name, or if a path was included, verify that the path is correct and try again.\n"
+ " + CategoryInfo : ObjectNotFound: (semgrep:String) [], CommandNotFoundException\n")
+
+
[email protected]("name,code,output,missing", [
+ ("semgrep", 1, POWERSHELL_NOT_FOUND, True), # Windows PowerShell
+ ("gitleaks", 1, "'gitleaks' is not recognized as an internal or external command,", True), # cmd.exe
+ ("gitleaks", 127, "bash: gitleaks: command not found", True), # sh
+ ("gitleaks", 127, "", True), # the code alone
+ ("trivy", 9009, "", True), # cmd.exe's code
+ ("pip-audit", 1, "/usr/bin/python3: No module named pip_audit", True), # python -m
+ ("semgrep", 1, "Ran 312 rules on 40 files: 25 findings.", False), # found something
+ ("gitleaks", 1, "leaks found: 2", False),
+ ("semgrep", 2, "semgrep: error: invalid configuration", False), # a real failure
+ ("semgrep", 1, "warning: rule 'x' not found in registry", False), # names no scanner word with the phrase
+])
+def test_a_scanner_that_is_not_installed_did_not_scan(name, code, output, missing):
+ assert scopes.scanner_not_found({"name": name, "action": "scan"}, code, output) is missing
+
+
+def test_a_missing_scanner_on_windows_reads_as_a_scan_that_did_not_run(tmp_path):
+ """P3B-1: PowerShell exits 1 for an unknown command, the same as semgrep's "found something".
+ The step records no scanner, so "A promised scan did not run" fires instead of nothing."""
+ registry = ToolRegistry()
+
+ def run_shell(command: str) -> dict:
+ """Run a shell command.
+
+ Args:
+ command: the command
+ """
+ return {"exit_code": 1, "output": POWERSHELL_NOT_FOUND}
+
+ registry.register(run_shell, metadata=_Meta())
+ engine = TurnEngine(provider=_Scripted([_call("c1", "run_shell", {"command": "semgrep scan --config auto"}),
+ AssistantTurn(text="done", usage=TokenUsage(input=10, output=2))]),
+ registry=registry,
+ permissions=PermissionEngine(workspace_root=tmp_path, mode=Mode.BYPASS_APPROVALS),
+ model="openai:model-x", instructions="test", max_iterations=12)
+ engine.audit_context = {"session_id": "sess_p3b1"}
+ rec = Recorder()
+ engine.recorder = rec
+
+ async def go():
+ return [ev async for ev in engine.run("scan", trigger="user")]
+
+ asyncio.run(go())
+ [step] = [s for s in rec.as_dict()["steps"] if s["kind"] == "tool"]
+ assert step["attrs"]["scanner"] is None and step["attrs"]["exit_code"] == 1
+ fired = scopes.coverage_checks({"promised": ["code"], "steps": [
+ {"id": step["step_id"], "scanner": step["attrs"]["scanner"], "exit_code": step["attrs"]["exit_code"]}]})
+ assert fired == {"scan_missing": [{"area": "code"}]}