Skip to content
Python

chore(observability): signals.yaml 0.7 from the shared docs (scanners.not_found)

Devika Verma32cd6f2f10d1
diff --git a/coworker/engine.py b/coworker/engine.py
index 6df63da6..d56bd6c2 100644
--- a/coworker/engine.py
+++ b/coworker/engine.py
@@ -40,8 +40,7 @@ from .observability.engine_hooks import ToolRun, TurnRecording, set_tool_run
from .observability.recorder import short_hash as _short_hash
from .observability import stamps as _stamps
from .observability.canonical import short_hash as _args_hash, text_hash as _text_hash
-from .observability.scopes import scanner_not_found as _scanner_not_found
-from .observability.scopes import scanner_of as _scanner_of
+from .observability.scopes import recorded_scanner as _recorded_scanner
from .observability.provider_view import tag as _tag_provider, view_for_engine
from .observability.ids import new_ulid
@@ -3086,13 +3085,13 @@ class TurnEngine:
)
if name == "run_shell":
# 1.3 (E17b): the scanner this command runs, by the signals.yaml word rule.
+ # A scanner that is not installed did not scan (signals 0.7 `scanners.not_found`,
+ # P3B-1): on Windows that exits 1, the same code as "found something", so its
+ # output and error tell them apart.
command = arguments.get("command") if isinstance(arguments, dict) else None
- out["scanner"] = _scanner_of(command) if isinstance(command, str) else None
- # A scanner that is not installed did not scan (P3B-1): on Windows that exits 1,
- # the same code as "found something", so the output tells them apart.
- if ran and out["scanner"] and isinstance(result, dict) and _scanner_not_found(
- out["scanner"], result.get("exit_code"), result.get("output") or result.get("error")):
- out["scanner"] = None
+ res = result if isinstance(result, dict) else {}
+ out["scanner"] = _recorded_scanner(command, ran=ran and bool(res), exit_code=res.get("exit_code"),
+ output=res.get("output"), error=res.get("error"))
# 1.3 (E17b): the full SHA-256 of the raw result, the evidence card's "output sha256".
out["result_sha256"] = _result_sha256(result) if ran else None
if name == "todo_write" and isinstance(arguments, dict):
diff --git a/coworker/observability/scopes.py b/coworker/observability/scopes.py
index 944222d5..fef352c7 100644
--- a/coworker/observability/scopes.py
+++ b/coworker/observability/scopes.py
@@ -80,6 +80,19 @@ def scanner_not_found(scanner: Optional[dict], exit_code: Any, output: Any) -> b
return False
+def recorded_scanner(command: Any, *, ran: bool = True, exit_code: Any = None, output: Any = None,
+ error: Any = None) -> Optional[dict[str, str]]:
+ """The `scanner` the engine records for a run_shell step (1.3): the word rule
+ (`scanner_of`), then, once the command ran, null when it only showed the scanner is not
+ installed (signals.yaml 0.7 `scanners.not_found`). The output and the error are both read,
+ a line at a time."""
+ found = scanner_of(command) if isinstance(command, str) else None
+ if not found or not ran:
+ return found
+ text = "\n".join(t for t in (output, error) if isinstance(t, str) and t)
+ return None if scanner_not_found(found, exit_code, text) else found
+
+
def area_word(area: str) -> str:
return (_scanners().get("area_words") or {}).get(area, area)
diff --git a/coworker/observability/signals.yaml b/coworker/observability/signals.yaml
index 2383e0ff..5bea3446 100644
--- a/coworker/observability/signals.yaml
+++ b/coworker/observability/signals.yaml
@@ -6,7 +6,7 @@
# the same issues in both. The numbers in a rule ("3 times", "80%") are settings here, not
# code.
#
-# Status: DRAFT 0.6 (2026-11-04). Not frozen. The Phase 2 checks freeze when the labelling
+# Status: DRAFT 0.7 (2026-11-04). Not frozen. The Phase 2 checks freeze when the labelling
# gate is closed (phase-2 TD2-21). The Phase 3 checks (scope series and task) freeze after
# they are run on real team, security and automation sessions (phase-3/validation-report.md).
#
@@ -47,8 +47,12 @@
# golden the scenarios that test it (phase-2/scenarios.md)
# source where the check was asked for
-version: "0.6"
+version: "0.7"
draft: 2026-11-04
+# 0.7: `scanners.not_found` (P3B-1, found while building Phase 3). A scanner that is not
+# installed did not scan: PowerShell exits 1 for a command it does not know, the same code
+# most scanners use for "found something". No check, threshold or text changed; the golden
+# answers are unchanged apart from the version. New vector: golden/phase3/vectors/scanner-not-found.json.
# 0.6: Phase 3 (phase-3/technical-design.md §5, §10 to §12; decisions TD3-15 to TD3-24, ruled
# 2026-11-04). A `scope` on every check (default turn). New checks: wake_capped and
# watchdog_nudged (turn); lead_polling and failed_in_a_row (series); scan_missing,
@@ -632,7 +636,7 @@ checks:
golden: [S3-60, S3-61, S3-61b, S3-62]
source: "across-features.md OB-18"
-# -- 0.6: how a scan is recognised (OB-16, TD3-16). ------------------------------------------
+# -- 0.6: how a scan is recognised (OB-16, TD3-16; not installed: 0.7). -----------------------
# The engine sets a run_shell step's `scanner` from the command, by this rule only:
# 1. split the command into parts at &&, ||, ; and |;
# 2. in each part, drop `lead_ins` and any NAME=value words at the start, and a leading
@@ -647,18 +651,18 @@ scanners:
area_words: {code: code, secrets: secret, deps: dependency, iac: infrastructure}
lead_ins: [sudo, npx, uvx, pipx, run, exec, env, time]
probe_words: ["--version", version, "-v", "--help"]
- # A scanner that is not installed did not scan (P3B-1, found in review 2026-11-04; local to
- # this copy until the shared file takes it, see the Phase 3 build notes). PowerShell exits 1
- # for a command it does not know, the same code most scanners use for "found something", so
- # the exit code alone cannot tell them apart. When recording a shell step whose command names
- # a scanner, the engine records `scanner: null` (no scan ran) when:
+ # 0.7: a scanner that is not installed did not scan (P3B-1). PowerShell exits 1 for a command
+ # it does not know, the same code most scanners use for "found something", so the exit code
+ # alone cannot tell them apart. When recording a run_shell step whose command names a scanner,
+ # the engine records `scanner: null` (no scan ran) when:
# - the command exited with one of `exit` (127: sh "command not found"; 9009: cmd.exe), or
- # - a line of its output names the scanner's word and holds one of `phrases` (case-insensitive).
- # The checks then read it as a scan that never ran: scan_missing fires, scanner_failed and
- # the coverage list do not count it.
+ # - a line of its output (or error) names the scanner's `word`, or the word with _ for -
+ # (as python -m prints it), and holds one of `phrases`; both case-insensitive.
+ # The checks then read it as a scan that never ran: scan_missing fires, and scanner_failed and
+ # the coverage list do not count it. "No such file or directory" is left out on purpose: a real
+ # scan of a missing folder can print it.
not_found:
exit: [127, 9009]
- # a line must also name the scanner's word (or the word with _ for -, as python -m prints it)
phrases: ["is not recognized as", "command not found", ": not found", "commandnotfoundexception",
"no module named", "could not determine executable to run"]
list:
diff --git a/tests/corpora/observability/golden-phase1/README.md b/tests/corpora/observability/golden-phase1/README.md
index 13fc5904..df0f7ba7 100644
--- a/tests/corpora/observability/golden-phase1/README.md
+++ b/tests/corpora/observability/golden-phase1/README.md
@@ -20,7 +20,10 @@ There are **30 scenarios**: S1-01 to S1-13, S1-16, and M-01a, M-01b, M-02 to M-0
**Not here.** Only scenarios that run a turn through the recorder produce events. The standalone scenarios S1-14 (machine and provider stamp), S1-15 (coworker fingerprint) and M-11 (token mapping) test small functions, not the recorder; S1-14 and S1-15 are covered by the stamp rules and M-11 by the usage test vectors in recording-format.md section 5 (and `vectors/usage.json`). S1-17 is a live-run checklist, not a scripted scenario.
-## Format 1.0
+## Format 1.0 (valid under 1.1)
+
+Format 1.1 (2026-11-03) only adds two decision values, `readonly_tier` and `gate_tier`. No scenario here uses them, so every file in this folder is unchanged and valid under 1.1.
+
**Context.** The files here are recording format 1.0. The lab recorder was brought to the format on 2026-10-31 and this folder was made again from it, so:
- the events in every `events.jsonl` are format 1.0 events (section 2 of recording-format.md): each has `v`, `at_ms` and `mono_ms`; content travels in its own `content` part; every `step_ended` carries a copy of the step's identity; `turn_started` carries `first_message_id`;
diff --git a/tests/corpora/observability/golden-phase2/README.md b/tests/corpora/observability/golden-phase2/README.md
index 3ab4f517..5fcb3f75 100644
--- a/tests/corpora/observability/golden-phase2/README.md
+++ b/tests/corpora/observability/golden-phase2/README.md
@@ -1,6 +1,6 @@
# Golden test, Phase 2
-*Written 2026-11-02 with [recording-format.md](../../recording-format.md) 1.2 and [signals.yaml](../../signals.yaml) 0.3; rebuilt 2026-11-03 for signals 0.4 (S2-17d added) and 0.5, and 2026-11-04 for 0.6: only the version and two new turn checks (`wake_capped`, `watchdog_nudged`) under `not_checked` changed, because a 1.2 recording has no `wake`. The recordings are unchanged and valid under 1.3. Made by `phase-2/lab/make_golden2.py`, checked by `phase-2/lab/check_golden2.py`.*
+*Written 2026-11-02 with [recording-format.md](../../recording-format.md) 1.2 and [signals.yaml](../../signals.yaml) 0.3; rebuilt 2026-11-03 for signals 0.4 (S2-17d added) and 0.5, and 2026-11-04 for 0.6 and 0.7 (0.7 changed only the version here): only the version and two new turn checks (`wake_capped`, `watchdog_nudged`) under `not_checked` changed, because a 1.2 recording has no `wake`. The recordings are unchanged and valid under 1.3. Made by `phase-2/lab/make_golden2.py`, checked by `phase-2/lab/check_golden2.py`.*
## What it is
diff --git a/tests/corpora/observability/golden-phase2/S2-01/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-01/expected-issues.json
index fbddc27c..f29d8bb5 100644
--- a/tests/corpora/observability/golden-phase2/S2-01/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-01/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.6",
+ "signals_version": "0.7",
"warn": 1,
"fired": {
"repeated_step": [
diff --git a/tests/corpora/observability/golden-phase2/S2-01c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-01c/expected-issues.json
index bd785c7f..dd674a44 100644
--- a/tests/corpora/observability/golden-phase2/S2-01c/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-01c/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.6",
+ "signals_version": "0.7",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/S2-02/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-02/expected-issues.json
index 4a9daa4c..9ecd4701 100644
--- a/tests/corpora/observability/golden-phase2/S2-02/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-02/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.6",
+ "signals_version": "0.7",
"warn": 1,
"fired": {
"off_plan": [
diff --git a/tests/corpora/observability/golden-phase2/S2-02b/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-02b/expected-issues.json
index d7a09230..0cce55e9 100644
--- a/tests/corpora/observability/golden-phase2/S2-02b/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-02b/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.6",
+ "signals_version": "0.7",
"warn": 1,
"fired": {
"off_plan": [
diff --git a/tests/corpora/observability/golden-phase2/S2-02c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-02c/expected-issues.json
index 8301ff00..aa710338 100644
--- a/tests/corpora/observability/golden-phase2/S2-02c/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-02c/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.6",
+ "signals_version": "0.7",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/S2-02n/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-02n/expected-issues.json
index bd785c7f..dd674a44 100644
--- a/tests/corpora/observability/golden-phase2/S2-02n/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-02n/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.6",
+ "signals_version": "0.7",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/S2-03/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-03/expected-issues.json
index 93f98b6a..28af116c 100644
--- a/tests/corpora/observability/golden-phase2/S2-03/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-03/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.6",
+ "signals_version": "0.7",
"warn": 1,
"fired": {
"refused": [
diff --git a/tests/corpora/observability/golden-phase2/S2-03b/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-03b/expected-issues.json
index bd785c7f..dd674a44 100644
--- a/tests/corpora/observability/golden-phase2/S2-03b/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-03b/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.6",
+ "signals_version": "0.7",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/S2-04/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-04/expected-issues.json
index 88b2ad15..75bcb9d9 100644
--- a/tests/corpora/observability/golden-phase2/S2-04/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-04/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.6",
+ "signals_version": "0.7",
"warn": 0,
"fired": {
"overridden": [
diff --git a/tests/corpora/observability/golden-phase2/S2-04n/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-04n/expected-issues.json
index bd785c7f..dd674a44 100644
--- a/tests/corpora/observability/golden-phase2/S2-04n/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-04n/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.6",
+ "signals_version": "0.7",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/S2-05/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-05/expected-issues.json
index f53a9cef..365dc821 100644
--- a/tests/corpora/observability/golden-phase2/S2-05/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-05/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.6",
+ "signals_version": "0.7",
"warn": 1,
"fired": {
"reviewer_could_not_run": [
diff --git a/tests/corpora/observability/golden-phase2/S2-05c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-05c/expected-issues.json
index 7df4d6eb..c4c73040 100644
--- a/tests/corpora/observability/golden-phase2/S2-05c/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-05c/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.6",
+ "signals_version": "0.7",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/S2-06/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-06/expected-issues.json
index 037ac760..51466191 100644
--- a/tests/corpora/observability/golden-phase2/S2-06/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-06/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.6",
+ "signals_version": "0.7",
"warn": 1,
"fired": {
"reviewer_paused": [
diff --git a/tests/corpora/observability/golden-phase2/S2-07/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-07/expected-issues.json
index f963b41d..dc2f7238 100644
--- a/tests/corpora/observability/golden-phase2/S2-07/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-07/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.6",
+ "signals_version": "0.7",
"warn": 0,
"fired": {
"shadow_disagreed": [
diff --git a/tests/corpora/observability/golden-phase2/S2-08/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-08/expected-issues.json
index eec479b6..900f70d3 100644
--- a/tests/corpora/observability/golden-phase2/S2-08/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-08/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.6",
+ "signals_version": "0.7",
"warn": 1,
"fired": {
"tool_failed": [
diff --git a/tests/corpora/observability/golden-phase2/S2-08b/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-08b/expected-issues.json
index eec479b6..900f70d3 100644
--- a/tests/corpora/observability/golden-phase2/S2-08b/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-08b/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.6",
+ "signals_version": "0.7",
"warn": 1,
"fired": {
"tool_failed": [
diff --git a/tests/corpora/observability/golden-phase2/S2-08c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-08c/expected-issues.json
index f98868c7..fe80fd9a 100644
--- a/tests/corpora/observability/golden-phase2/S2-08c/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-08c/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.6",
+ "signals_version": "0.7",
"warn": 1,
"fired": {
"made_up_tool": [
diff --git a/tests/corpora/observability/golden-phase2/S2-09/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-09/expected-issues.json
index 04593bf5..bb0ff2f5 100644
--- a/tests/corpora/observability/golden-phase2/S2-09/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-09/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.6",
+ "signals_version": "0.7",
"warn": 1,
"fired": {
"model_failed": [
diff --git a/tests/corpora/observability/golden-phase2/S2-09c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-09c/expected-issues.json
index bd785c7f..dd674a44 100644
--- a/tests/corpora/observability/golden-phase2/S2-09c/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-09c/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.6",
+ "signals_version": "0.7",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/S2-10/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-10/expected-issues.json
index 520165d3..3c99a969 100644
--- a/tests/corpora/observability/golden-phase2/S2-10/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-10/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.6",
+ "signals_version": "0.7",
"warn": 1,
"fired": {
"retried": [
diff --git a/tests/corpora/observability/golden-phase2/S2-10b/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-10b/expected-issues.json
index bd785c7f..dd674a44 100644
--- a/tests/corpora/observability/golden-phase2/S2-10b/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-10b/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.6",
+ "signals_version": "0.7",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/S2-11/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-11/expected-issues.json
index ee6c520b..0a393718 100644
--- a/tests/corpora/observability/golden-phase2/S2-11/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-11/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.6",
+ "signals_version": "0.7",
"warn": 1,
"fired": {
"cut_off": [
diff --git a/tests/corpora/observability/golden-phase2/S2-12/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-12/expected-issues.json
index 32899d7f..4f560003 100644
--- a/tests/corpora/observability/golden-phase2/S2-12/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-12/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.6",
+ "signals_version": "0.7",
"warn": 1,
"fired": {
"step_limit": [
diff --git a/tests/corpora/observability/golden-phase2/S2-13/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-13/expected-issues.json
index 0d3b05c6..3b2506dd 100644
--- a/tests/corpora/observability/golden-phase2/S2-13/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-13/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.6",
+ "signals_version": "0.7",
"warn": 2,
"fired": {
"made_up_tool": [
diff --git a/tests/corpora/observability/golden-phase2/S2-14/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-14/expected-issues.json
index 52524575..65896460 100644
--- a/tests/corpora/observability/golden-phase2/S2-14/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-14/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.6",
+ "signals_version": "0.7",
"warn": 1,
"fired": {
"context_full": [
diff --git a/tests/corpora/observability/golden-phase2/S2-14b/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-14b/expected-issues.json
index bd785c7f..dd674a44 100644
--- a/tests/corpora/observability/golden-phase2/S2-14b/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-14b/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.6",
+ "signals_version": "0.7",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/S2-14d/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-14d/expected-issues.json
index b0e54e82..de83cf8e 100644
--- a/tests/corpora/observability/golden-phase2/S2-14d/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-14d/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.6",
+ "signals_version": "0.7",
"warn": 1,
"fired": {
"context_full": [
diff --git a/tests/corpora/observability/golden-phase2/S2-15/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-15/expected-issues.json
index ce6a9e56..3203ffab 100644
--- a/tests/corpora/observability/golden-phase2/S2-15/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-15/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.6",
+ "signals_version": "0.7",
"warn": 1,
"fired": {
"blocked_by_sandbox": [
diff --git a/tests/corpora/observability/golden-phase2/S2-15c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-15c/expected-issues.json
index 6ce69156..e4fe56c0 100644
--- a/tests/corpora/observability/golden-phase2/S2-15c/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-15c/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.6",
+ "signals_version": "0.7",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/S2-16/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-16/expected-issues.json
index 62db8588..22001fb7 100644
--- a/tests/corpora/observability/golden-phase2/S2-16/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-16/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.6",
+ "signals_version": "0.7",
"warn": 1,
"fired": {
"ran_without_sandbox": [
diff --git a/tests/corpora/observability/golden-phase2/S2-16c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-16c/expected-issues.json
index 71013fa9..102987d4 100644
--- a/tests/corpora/observability/golden-phase2/S2-16c/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-16c/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.6",
+ "signals_version": "0.7",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/S2-17/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-17/expected-issues.json
index b331dd11..fddb18bb 100644
--- a/tests/corpora/observability/golden-phase2/S2-17/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-17/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.6",
+ "signals_version": "0.7",
"warn": 1,
"fired": {
"waited_unwatched": [
diff --git a/tests/corpora/observability/golden-phase2/S2-17b/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-17b/expected-issues.json
index bd785c7f..dd674a44 100644
--- a/tests/corpora/observability/golden-phase2/S2-17b/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-17b/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.6",
+ "signals_version": "0.7",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/S2-17c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-17c/expected-issues.json
index bd785c7f..dd674a44 100644
--- a/tests/corpora/observability/golden-phase2/S2-17c/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-17c/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.6",
+ "signals_version": "0.7",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/S2-17d/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-17d/expected-issues.json
index b331dd11..fddb18bb 100644
--- a/tests/corpora/observability/golden-phase2/S2-17d/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-17d/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.6",
+ "signals_version": "0.7",
"warn": 1,
"fired": {
"waited_unwatched": [
diff --git a/tests/corpora/observability/golden-phase2/S2-18/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-18/expected-issues.json
index a16f1dd2..25b02433 100644
--- a/tests/corpora/observability/golden-phase2/S2-18/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-18/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.6",
+ "signals_version": "0.7",
"warn": 1,
"fired": {
"sandbox_failed": [
diff --git a/tests/corpora/observability/golden-phase2/S2-18c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-18c/expected-issues.json
index efbedc3f..3e6d5e3a 100644
--- a/tests/corpora/observability/golden-phase2/S2-18c/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-18c/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.6",
+ "signals_version": "0.7",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/S2-19/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-19/expected-issues.json
index dc219aad..0f1ab73f 100644
--- a/tests/corpora/observability/golden-phase2/S2-19/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-19/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.6",
+ "signals_version": "0.7",
"warn": 0,
"fired": {
"reviewer_allowed_send": [
diff --git a/tests/corpora/observability/golden-phase2/S2-19c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-19c/expected-issues.json
index 7df4d6eb..c4c73040 100644
--- a/tests/corpora/observability/golden-phase2/S2-19c/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-19c/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.6",
+ "signals_version": "0.7",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/index.json b/tests/corpora/observability/golden-phase2/index.json
index 5cb64408..62b537d6 100644
--- a/tests/corpora/observability/golden-phase2/index.json
+++ b/tests/corpora/observability/golden-phase2/index.json
@@ -1,6 +1,6 @@
{
"format": "1.2",
- "signals_version": "0.6",
+ "signals_version": "0.7",
"built": true,
"scenarios": [
{
diff --git a/tests/corpora/observability/golden-phase3/README.md b/tests/corpora/observability/golden-phase3/README.md
index 095d0e7b..a85649ca 100644
--- a/tests/corpora/observability/golden-phase3/README.md
+++ b/tests/corpora/observability/golden-phase3/README.md
@@ -1,6 +1,6 @@
# Golden test, Phase 3
-*Written 2026-11-04 with [recording-format.md](../../recording-format.md) 1.3 and [signals.yaml](../../signals.yaml) 0.6. It is made by `phase-3/lab/make_golden3.py` and checked by `phase-3/lab/check_golden3.py`.*
+*Written 2026-11-04 with [recording-format.md](../../recording-format.md) 1.3 and [signals.yaml](../../signals.yaml) 0.6; rebuilt the same day for 0.7, which adds only the not-installed vector. It is made by `phase-3/lab/make_golden3.py` and checked by `phase-3/lab/check_golden3.py`.*
## What it is
@@ -38,6 +38,7 @@ Every rollup uses the viewer's days in [vectors/days-europe-london.json](vectors
|---|---|
| `vectors/days-europe-london.json` | The 30 local days from 6 Oct 2026 as `[label, start, end)` epoch ms. 25 Oct is 25 hours long |
| `vectors/scanners.json` | Shell commands, and the `scanner` each must give by the rule in `signals.yaml` (`scanners`) |
+| `vectors/scanner-not-found.json` | Ten shell steps after they ran (command, exit code, output), and whether the scanner was not installed, by `scanners.not_found` (0.7, P3B-1). When it was not, the recorded `scanner` is `null`. The answers are written by hand, from the app's own test |
`index.json` lists the 26 scenarios: what each one tests, whether its check should fire or stay quiet, and, for the edge cases, the values worked out by hand in [phase-3/scenarios.md](../../phase-3/scenarios.md).
diff --git a/tests/corpora/observability/golden-phase3/index.json b/tests/corpora/observability/golden-phase3/index.json
index fc57b805..9735ac04 100644
--- a/tests/corpora/observability/golden-phase3/index.json
+++ b/tests/corpora/observability/golden-phase3/index.json
@@ -1,6 +1,6 @@
{
"format": "1.3",
- "signals_version": "0.6",
+ "signals_version": "0.7",
"built": true,
"groupings": [
"all",
diff --git a/tests/corpora/observability/golden-phase3/vectors/scanner-not-found.json b/tests/corpora/observability/golden-phase3/vectors/scanner-not-found.json
new file mode 100644
index 00000000..62826837
--- /dev/null
+++ b/tests/corpora/observability/golden-phase3/vectors/scanner-not-found.json
@@ -0,0 +1,97 @@
+{
+ "rule": "signals.yaml scanners.not_found (0.7): the `scanner` the engine records for a run_shell step after it ran. null when the command named a scanner that was not installed.",
+ "cases": [
+ {
+ "shows": "Windows PowerShell",
+ "command": "semgrep scan --config auto --json",
+ "exit_code": 1,
+ "output": "semgrep : The term 'semgrep' is not recognized as the name of a cmdlet, function, script file, or operable\nprogram. Check the spelling of the name, or if a path was included, verify that the path is correct and try again.\n + CategoryInfo : ObjectNotFound: (semgrep:String) [], CommandNotFoundException\n",
+ "not_installed": true,
+ "scanner": null
+ },
+ {
+ "shows": "cmd.exe",
+ "command": "gitleaks detect --source .",
+ "exit_code": 1,
+ "output": "'gitleaks' is not recognized as an internal or external command,",
+ "not_installed": true,
+ "scanner": null
+ },
+ {
+ "shows": "sh",
+ "command": "gitleaks detect --source .",
+ "exit_code": 127,
+ "output": "bash: gitleaks: command not found",
+ "not_installed": true,
+ "scanner": null
+ },
+ {
+ "shows": "the sh code alone",
+ "command": "gitleaks detect --source .",
+ "exit_code": 127,
+ "output": "",
+ "not_installed": true,
+ "scanner": null
+ },
+ {
+ "shows": "the cmd.exe code alone",
+ "command": "trivy fs .",
+ "exit_code": 9009,
+ "output": "",
+ "not_installed": true,
+ "scanner": null
+ },
+ {
+ "shows": "python -m prints the word with _",
+ "command": "python3 -m pip-audit -f json",
+ "exit_code": 1,
+ "output": "/usr/bin/python3: No module named pip_audit",
+ "not_installed": true,
+ "scanner": null
+ },
+ {
+ "shows": "found something",
+ "command": "semgrep scan --config auto",
+ "exit_code": 1,
+ "output": "Ran 312 rules on 40 files: 25 findings.",
+ "not_installed": false,
+ "scanner": {
+ "name": "semgrep",
+ "action": "scan"
+ }
+ },
+ {
+ "shows": "found something",
+ "command": "gitleaks detect --source .",
+ "exit_code": 1,
+ "output": "leaks found: 2",
+ "not_installed": false,
+ "scanner": {
+ "name": "gitleaks",
+ "action": "scan"
+ }
+ },
+ {
+ "shows": "a real failure",
+ "command": "semgrep scan --config bad.yml",
+ "exit_code": 2,
+ "output": "semgrep: error: invalid configuration",
+ "not_installed": false,
+ "scanner": {
+ "name": "semgrep",
+ "action": "scan"
+ }
+ },
+ {
+ "shows": "a not-found line that names no scanner",
+ "command": "semgrep scan --config auto",
+ "exit_code": 1,
+ "output": "warning: rule 'x' not found in registry",
+ "not_installed": false,
+ "scanner": {
+ "name": "semgrep",
+ "action": "scan"
+ }
+ }
+ ]
+}
diff --git a/tests/corpora/observability/recording-format.schema.json b/tests/corpora/observability/recording-format.schema.json
index 7dd004da..8f8fbe42 100644
--- a/tests/corpora/observability/recording-format.schema.json
+++ b/tests/corpora/observability/recording-format.schema.json
@@ -392,7 +392,7 @@
"team": {"type": "object", "additionalProperties": false, "required": ["team_id"], "description": "1.2. Team tools only: ids, no text.",
"properties": {"team_id": {"type": "string"}, "item_id": {"type": ["string", "null"]}, "worker_session_id": {"type": ["string", "null"]},
"worker_session_ids": {"type": "array", "items": {"type": "string"}, "description": "propose_team only: the sessions it created."}}},
- "scanner": {"type": ["object", "null"], "additionalProperties": false, "required": ["name", "action"], "description": "1.3. run_shell only: the security scanner the command runs, by the fixed word rule of signals.yaml (scanners). null = a shell command that runs no scanner. Absent = not a shell step, or not recorded.",
+ "scanner": {"type": ["object", "null"], "additionalProperties": false, "required": ["name", "action"], "description": "1.3. run_shell only: the security scanner the command runs, by the fixed word rule of signals.yaml (scanners). null = a shell command that runs no scanner, or names one that was not installed (signals.yaml scanners.not_found). Absent = not a shell step, or not recorded.",
"properties": {"name": {"enum": ["semgrep", "gitleaks", "osv-scanner", "npm-audit", "pip-audit", "trivy", "checkov"]}, "action": {"enum": ["scan", "version"]}}},
"result_sha256": {"oneOf": [{"$ref": "#/$defs/sha256"}, {"type": "null"}], "description": "1.3. SHA-256 of the raw result, for evidence (OB-16). null = the call never ran."},
"ended_after_turn_ms": {"$ref": "#/$defs/ended_after_turn_ms"}
diff --git a/tests/test_observability_golden_phase3.py b/tests/test_observability_golden_phase3.py
index 3ae7d52d..2f20536f 100644
--- a/tests/test_observability_golden_phase3.py
+++ b/tests/test_observability_golden_phase3.py
@@ -1,7 +1,7 @@
"""Observability Phase 3 golden test: rollups over many turns, and the series and task checks.
`tests/corpora/observability/golden-phase3/` is a copy of ocw-context
-docs/observability/golden/phase3 (recording format 1.3, signals.yaml 0.6). The Rust harness runs
+docs/observability/golden/phase3 (recording format 1.3, signals.yaml 0.7). The Rust harness runs
the same files, so both engines must give the same totals for the same recordings and the same
issues for the same team, security task or automation.
@@ -266,6 +266,20 @@ def test_scanner_vectors():
assert scopes.scanner_of(case["command"]) == case["scanner"], case["command"]
+def test_scanner_not_found_vectors():
+ """signals 0.7 `scanners.not_found` (P3B-1): the scanner the engine records after the step
+ ran, by the same function the engine calls."""
+ v = _load(GOLDEN / "vectors" / "scanner-not-found.json")
+ assert len(v["cases"]) == 10
+ for case in v["cases"]:
+ got = scopes.recorded_scanner(case["command"], exit_code=case["exit_code"], output=case["output"])
+ assert got == case["scanner"], case["shows"]
+ assert (got is None) == case["not_installed"], case["shows"]
+ # the message may come in the result's error instead of its output
+ assert scopes.recorded_scanner(case["command"], exit_code=case["exit_code"], output="",
+ error=case["output"]) == case["scanner"], case["shows"]
+
+
def test_day_vectors_read_as_the_client_sends_them():
v = _load(GOLDEN / "vectors" / "days-europe-london.json")
days = rollups.parse_days(json.dumps(v["days"]))
diff --git a/tests/test_observability_issues.py b/tests/test_observability_issues.py
index 9e131aee..26e00b4a 100644
--- a/tests/test_observability_issues.py
+++ b/tests/test_observability_issues.py
@@ -1,7 +1,7 @@
"""Observability Phase 2, OB-3: the Issues checks (E8b) and the 1.2 fields they read (E8a).
The scenarios in `tests/corpora/observability/golden-phase2/` are the shared test of the checks:
-a copy of ocw-context docs/observability/golden/phase2 (signals.yaml 0.6, recording format 1.2).
+a copy of ocw-context docs/observability/golden/phase2 (signals.yaml 0.7, recording format 1.2).
Every recording must give exactly its expected issues, and the hash vectors must hash the same,
in this engine and in the Rust harness."""