Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
Run the suite, not a list: 63 test files no workflow reached (#826)
Beenu Aroraf86e9cf6652f
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 2a324999d..a4d3b2e23 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -557,16 +557,20 @@ jobs:SECRET_KEY: ci-dummy-secret-key-at-least-32bytes!DATABASE_URL: postgresql+asyncpg://x:x@localhost/xrun: python -m pytest tests/ -v --tb=short- # This list is hand-maintained, which is the same "a matrix is a list- # somebody has to remember to add to" shape called out for the Go- # modules further down — and it has the same consequence. 62 of the 82- # test files under services/agents/tests are named by no workflow at- # all. `test_playbook_engine_correctness.py` was one of them, and it- # carried a test that spent fourteen seconds doing the precise thing its- # name said must not happen while reporting green, because nothing ever- # ran it to notice. The playbook and per-tenant business-context files- # are added below; the rest of the gap is real and is not closed here.- - name: Run agents unit tests (Pillar-1 eval suites + audit helpers + fusion client)+ # This used to be a hand-maintained list of 26 file paths, which is the+ # same "a matrix is a list somebody has to remember to add to" shape+ # called out for the Go modules further down, and it had the same+ # consequence: 56 of the 82 test files under services/agents/tests were+ # named by no workflow at all. `test_playbook_engine_correctness.py` was+ # one of them, and it carried a test that spent fourteen seconds doing+ # the precise thing its name said must not happen while reporting green,+ # because nothing ever ran it to notice.+ #+ # It is the directory now. Adding a test file to a directory CI already+ # runs is free; adding one beside a list is silently free of signal.+ # `scripts/check_test_discovery.py` holds it that way in both+ # directions.+ - name: Run agents unit tests (whole suite — discovered, not listed)# `test_audit_helpers.py` imports `app.investigator.state` directly# (it stubs out the package __init__ to avoid pulling in LangGraph),# so it only needs stdlib + pydantic — already installed via API_DEPS.@@ -593,34 +597,25 @@ jobs:'langchain-core>=1.2.22,<2' 'langchain-openai>=1,<2' 'langgraph>=1,<2' \jsonschema 'markdown>=3.5,<4'cd services/agents- python3 -m pytest \- tests/test_router_report.py \- tests/test_mitre_accuracy.py \- tests/test_alert_reduction.py \- tests/test_investigation_completeness.py \- tests/test_response_quality.py \- tests/test_audit_helpers.py \- tests/test_fusion_client.py \- tests/test_prompt_sanitizer.py \- tests/test_prompt_envelope.py \- tests/test_privacy_redactor.py \- tests/test_cost_governor.py \- tests/test_bundle_incident_context.py \- tests/test_deep_investigation.py \- tests/adversarial/ \- tests/test_model_router.py \- tests/test_llm_ops.py \- tests/test_fused_alert_worker.py \- tests/test_business_context_hotpath.py \- tests/test_business_context_per_tenant.py \- tests/test_playbook_engine_correctness.py \- tests/test_playbook_action_bridge.py \- tests/test_playbook_bounds.py \- tests/test_playbook_models_bounds.py \- tests/test_playbook_ssrf_guard.py \- tests/test_swarm.py \- tests/test_swarm_vs_single.py \- -v --tb=short+ python3 -m pytest tests/ -v --tb=short -rs++ # The repository's own gate suites — `tests/` and `scripts/tests/`.+ # `tests/` was reached only as nineteen single-file invocations spread+ # across eleven workflows, which is the agents defect again with the+ # list kept somewhere else: five files were named by nothing, and+ # `scripts/tests/` by nothing at all. Three tests in+ # `scripts/tests/test_run_evals_cli.py` had been failing for as long as+ # they existed, pinning a `--suite` contract the runner had stopped+ # honouring.+ #+ # `tests/isolation/` is excluded because isolation.yml owns it: it needs+ # qdrant-client and live stores this job does not have. That is an+ # exclusion of a directory another workflow runs, not a list of files —+ # and check_test_discovery.py fails the build if isolation.yml ever+ # stops reaching it, so the exclusion cannot quietly become a hole.+ - name: Run the repository's own gate suites (tests/ + scripts/tests/)+ run: python3 -m pytest tests/ scripts/tests/ --ignore=tests/isolation -v --tb=short -rs+- name: Toolchain versions must agree# Six go.mod files declared five different Go versions while the Go# matrix installed a sixth, so one module could not be built by the@@ -696,6 +691,28 @@ jobs:python3 scripts/check_gate_coverage.py --listpython3 -m pytest tests/test_gate_coverage_classifier.py -q+ - name: Every test file is reached by its own tree's invocation+ # The gate on the suites. A test file no workflow collects reports+ # nothing and reads exactly like one that passed — 63 were in that+ # state, 56 of them under services/agents, because the invocation was+ # a hand-maintained list of file paths rather than a directory.+ #+ # Structural, and in both directions: the corpus is `git ls-files`, so+ # a service added tomorrow is scanned with no edit here, and an+ # invocation naming a path that no longer exists fails too. It models+ # what pytest actually collects — directory arguments, `--ignore`,+ # `conftest` collect_ignore, and `-m` marker deselection, because a+ # file whose every test is deselected runs nothing however plainly it+ # is named.+ #+ # Self-test first, on the same tree, so a resolver that quietly starts+ # returning "reached" for everything fails here rather than printing OK.+ run: |+ python -m pip install --quiet pyyaml pytest+ python3 scripts/check_test_discovery.py --self-test+ python3 scripts/check_test_discovery.py --list+ python3 -m pytest tests/test_test_discovery_gate.py -q+- name: No check may report OK over a repository with no content# The gate on the gates, the other axis. Reachability says a check can# fail; this says it does, when there is nothing there to be clean.diff --git a/.github/workflows/compose-smoke.yml b/.github/workflows/compose-smoke.ymlindex e17336492..3550479a9 100644--- a/.github/workflows/compose-smoke.yml+++ b/.github/workflows/compose-smoke.yml@@ -16,47 +16,27 @@ name: Compose Smoke# rebuild from source — useful when validating Dockerfile changes that have# not yet been published.+# Deliberately NOT path-filtered at the workflow level.+#+# It used to be. A workflow that does not trigger reports no check, and a+# required check that never reports blocks a pull request forever — so this+# gate could not be required while the filter lived up here, which is the+# single thing standing between "the first-run experience is tested" and "the+# first-run experience is tested and cannot regress".+#+# The condition moved into the job instead. `smoke` runs on every pull+# request, decides for itself whether anything under a build context changed,+# and reports success under the same name either way. The name is what branch+# protection matches on, and there is only one job producing it, so the two+# cases cannot drift apart.on:pull_request:branches: [main, develop]- paths:- # Compose surface itself.- - 'docker-compose.yml'- - 'infra/compose/docker-compose.demo.yml'- - '.env.example'- - '.github/workflows/compose-smoke.yml'- # Any source under a service build context — Dockerfile changes alone- # are not enough, because a stale GHCR image will silently mask source- # bugs introduced under services/<svc>/. The detect step below decides- # whether to rebuild or pull.- - 'services/**'- - 'apps/web/**'- # The web image's build context is the repo root; these are the things- # its Dockerfile actually copies into the build.- - 'packages/**'- - 'pnpm-lock.yaml'- - 'pnpm-workspace.yaml'- - 'turbo.json'- - 'tsconfig.base.json'- - 'package.json'push:branches:- main- 'release/**'- 'hotfix/**'- paths:- - 'docker-compose.yml'- - 'infra/compose/docker-compose.demo.yml'- - '.env.example'- - '.github/workflows/compose-smoke.yml'- - 'services/**'- - 'apps/web/**'- - 'packages/**'- - 'pnpm-lock.yaml'- - 'pnpm-workspace.yaml'- - 'turbo.json'- - 'tsconfig.base.json'- - 'package.json'workflow_dispatch:inputs:force_build:@@ -98,7 +78,87 @@ jobs:# this runner and the only safe option.fetch-depth: 0+ # The workflow-level path filter used to answer this, at the cost of the+ # job not reporting at all when it said no. Now the job answers it and+ # reports either way.+ - name: Is any of this PR's diff under the compose surface?+ id: relevant+ run: |+ set -euo pipefail+ if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then+ echo "manual dispatch: always run"+ echo "run=true" >> "$GITHUB_OUTPUT"+ exit 0+ fi++ # Same set the workflow-level `paths:` filter carried, in one place+ # now rather than duplicated across the two triggers it had.+ paths=(+ 'docker-compose.yml'+ 'infra/compose/docker-compose.demo.yml'+ '.env.example'+ '.github/workflows/compose-smoke.yml'+ 'services/'+ 'apps/web/'+ 'packages/'+ 'pnpm-lock.yaml'+ 'pnpm-workspace.yaml'+ 'turbo.json'+ 'tsconfig.base.json'+ 'package.json'+ )++ base_ref=""+ if [ -n "${{ github.base_ref }}" ]; then+ git fetch --no-tags --depth=50 origin "${{ github.base_ref }}" || true+ base_ref="origin/${{ github.base_ref }}"+ elif [ -n "${{ github.event.before }}" ] && [ "${{ github.event.before }}" != "0000000000000000000000000000000000000000" ]; then+ base_ref="${{ github.event.before }}"+ fi++ # Fail-open, matching the rebuild decision below: with no base to+ # diff against, run the gate rather than skip it. A gate that+ # declines to run looks identical to one that passed.+ if [ -z "${base_ref}" ]; then+ echo "no diff base; running"+ echo "run=true" >> "$GITHUB_OUTPUT"+ exit 0+ fi++ diff_status=0+ changed=$(git diff --name-only "${base_ref}"...HEAD -- "${paths[@]}") || diff_status=$?+ if [ "${diff_status}" -ne 0 ]; then+ echo "git diff vs ${base_ref} failed (exit ${diff_status}); running"+ echo "run=true" >> "$GITHUB_OUTPUT"+ exit 0+ fi++ if [ -n "${changed}" ]; then+ echo "compose surface touched:"+ printf ' %s\n' ${changed}+ echo "run=true" >> "$GITHUB_OUTPUT"+ else+ echo "nothing under the compose surface changed vs ${base_ref}"+ echo "run=false" >> "$GITHUB_OUTPUT"+ fi++ - name: Nothing to smoke — reporting a verdict anyway+ if: steps.relevant.outputs.run != 'true'+ run: |+ {+ echo "## Compose smoke — not exercised"+ echo ""+ echo "This pull request changes nothing under the compose surface"+ echo "(\`docker-compose.yml\`, \`services/\`, \`apps/web/\`, \`packages/\`,"+ echo "the lockfiles, or this workflow), so the stack was not booted."+ echo ""+ echo "The job still reports, under the same check name, because a"+ echo "required check that never reports blocks the pull request"+ echo "forever."+ } >> "$GITHUB_STEP_SUMMARY"+- name: Show runner capacity+ if: steps.relevant.outputs.run == 'true'run: |echo "## Runner snapshot" >> "$GITHUB_STEP_SUMMARY"echo '```' >> "$GITHUB_STEP_SUMMARY"@@ -110,6 +170,7 @@ jobs:echo '```' >> "$GITHUB_STEP_SUMMARY"- name: Seed environment file+ if: steps.relevant.outputs.run == 'true'run: |cp .env.example .env# Override the image tag so smoke runs always pull a known-good@@ -122,10 +183,12 @@ jobs:echo "Resolved AISOC_VERSION=${AISOC_VERSION}"- name: Validate compose file+ if: steps.relevant.outputs.run == 'true'run: docker compose config --quiet- name: Detect build-context changesid: dockerfile_changes+ if: steps.relevant.outputs.run == 'true'run: |# The published GHCR image becomes stale the moment ANY file inside# a service's build context changes — not only the Dockerfile. A PR@@ -155,35 +218,66 @@ jobs:exit 0fi- # Build paths mirror docker-compose.yml `build.context` entries.- # Keep this list in sync when adding/removing built services. The- # web image's context is the repo root but only a subset of the- # tree is COPY'd in — listing those entries explicitly avoids- # false-positive rebuilds on every PR.- paths=(- 'services/api/'- 'services/ingest/'- 'services/enrichment/'- 'services/fusion/'- 'services/agents/'- 'services/osquery-tls/'- 'services/actions/'- 'services/connectors/'- 'services/threatintel/'- 'services/ueba/'- 'services/honeytokens/'- 'services/purple-team/'- 'services/realtime/'- 'services/slack-bot/'- 'apps/web/'- 'packages/'- 'pnpm-lock.yaml'- 'pnpm-workspace.yaml'- 'turbo.json'- 'tsconfig.base.json'- 'package.json'+ # Build paths come from docker-compose.yml, not from a list kept+ # here. They used to be a list, with "keep this in sync when+ # adding/removing built services" written above it — and a service+ # added to compose but not to the list would have had its source+ # changes silently pulled from GHCR instead of built, so the gate+ # would boot a stale image and pass without ever running the code+ # under review. That is the exact shape this check exists to catch,+ # sitting inside the check.+ #+ # The web image's context is the repository root, so it is expanded+ # to the subset its Dockerfile actually copies; taking the root+ # literally would rebuild everything on every PR.+ python3 - <<'PYDETECT' > /tmp/build-contexts.txt+ import pathlib+ import yaml++ doc = yaml.safe_load(pathlib.Path("docker-compose.yml").read_text()) or {}+ services = doc.get("services") or {}+ if not services:+ raise SystemExit("docker-compose.yml declares no services; the parser is wrong, not the tree")++ # What the repo-root build context (the web image) actually copies.+ ROOT_CONTEXT_SUBSET = (+ "apps/web/",+ "packages/",+ "pnpm-lock.yaml",+ "pnpm-workspace.yaml",+ "turbo.json",+ "tsconfig.base.json",+ "package.json",)+ out: list[str] = []+ for spec in services.values():+ build = spec.get("build")+ if not build:+ continue+ context = build if isinstance(build, str) else build.get("context", "")+ context = str(context).strip().lstrip("./")+ if not context or context == ".":+ out.extend(ROOT_CONTEXT_SUBSET)+ else:+ out.append(context.rstrip("/") + "/")+ if not out:+ raise SystemExit("no build contexts found in docker-compose.yml")+ for entry in sorted(set(out)):+ print(entry)+ PYDETECT++ paths=()+ while IFS= read -r entry; do+ [ -n "${entry}" ] && paths+=("${entry}")+ done < /tmp/build-contexts.txt+ if [ "${#paths[@]}" -eq 0 ]; then+ echo "::error::derived no build contexts from docker-compose.yml; refusing to decide"+ exit 1+ fi+ echo "build contexts derived from docker-compose.yml (${#paths[@]}):"+ printf ' %s\n' "${paths[@]}"+# Fail-open: if `git diff` errors (shallow clone, missing merge# base, etc.) we rebuild rather than risk pulling a stale image# that masks the very fix this PR is shipping. The previous@@ -208,7 +302,7 @@ jobs:fi- name: Pre-pull published images- if: steps.dockerfile_changes.outputs.rebuild != 'true'+ if: steps.relevant.outputs.run == 'true' && steps.dockerfile_changes.outputs.rebuild != 'true'run: |# Best-effort pull. We don't fail the build here — services that# don't yet publish will fall through to the local build step@@ -216,15 +310,115 @@ jobs:docker compose pull --ignore-pull-failures --quiet || true- name: Boot stack (pull-by-default)- if: steps.dockerfile_changes.outputs.rebuild != 'true'+ if: steps.relevant.outputs.run == 'true' && steps.dockerfile_changes.outputs.rebuild != 'true'run: docker compose up -d --quiet-pull- name: Boot stack (force rebuild)- if: steps.dockerfile_changes.outputs.rebuild == 'true'+ if: steps.relevant.outputs.run == 'true' && steps.dockerfile_changes.outputs.rebuild == 'true'run: docker compose up -d --build+ # "Green" has meant two very different things here and the log did not+ # distinguish them: a run that built the changed service from this PR's+ # source, and a run that pulled `:main` and never compiled a line of it.+ # A required check that can pass without exercising the changed service+ # is the defect this whole effort is about, so the run now says which it+ # did, per service, and refuses when the two disagree.+ #+ # Read off the images that are actually running, not predicted from the+ # decision above: a pulled image carries a RepoDigest and a locally+ # built one does not, so this reports what happened rather than what was+ # intended.+ - name: Which services were built, and which were pulled+ id: provenance+ if: steps.relevant.outputs.run == 'true'+ run: |+ set -uo pipefail+ python3 - <<'PYPROV' | tee -a "$GITHUB_STEP_SUMMARY"+ import json+ import pathlib+ import subprocess+ import sys++ import yaml++ compose = yaml.safe_load(pathlib.Path("docker-compose.yml").read_text()) or {}+ buildable = {name for name, spec in (compose.get("services") or {}).items() if spec.get("build")}++ listed = subprocess.run(+ ["docker", "compose", "ps", "--format", "json"],+ capture_output=True, text=True, check=False,+ )+ rows = []+ for line in listed.stdout.splitlines():+ line = line.strip()+ if not line:+ continue+ try:+ parsed = json.loads(line)+ except json.JSONDecodeError:+ continue+ rows.extend(parsed if isinstance(parsed, list) else [parsed])+ if not rows:+ print("::error::`docker compose ps` listed no containers; this run exercised nothing")+ sys.exit(1)++ def provenance(image: str) -> str:+ """A pulled image has a RepoDigest; a locally built one does not."""+ out = subprocess.run(+ ["docker", "image", "inspect", image, "--format", "{{len .RepoDigests}}"],+ capture_output=True, text=True, check=False,+ )+ if out.returncode != 0:+ return "unknown"+ return "pulled" if out.stdout.strip() not in {"0", ""} else "built"++ built, pulled, unknown, running = [], [], [], set()+ print("## What this run actually exercised\n")+ print("| service | image | from |")+ print("|---|---|---|")+ for row in sorted(rows, key=lambda r: r.get("Service", "")):+ service = row.get("Service", "?")+ image = row.get("Image", "?")+ if service not in buildable:+ continue+ running.add(service)+ where = provenance(image)+ {"built": built, "pulled": pulled}.get(where, unknown).append(service)+ print(f"| `{service}` | `{image}` | **{where}** |")++ print()+ print(f"**{len(built)} built from this PR's source**: {', '.join(built) or 'none'} ")+ print(f"**{len(pulled)} pulled from the registry**: {', '.join(pulled) or 'none'} ")+ if unknown:+ print(f"**{len(unknown)} could not be determined**: {', '.join(unknown)} ")++ # Neither built nor pulled: not started at all. This smoke test boots+ # the default profile, and the profile-gated services are excluded by+ # design — but "excluded by design" and "silently not covered" read+ # identically in a green tick, and a change to one of these goes+ # through this gate without the service ever starting.+ absent = sorted(buildable - running)+ if absent:+ print()+ print(f"**{len(absent)} buildable service(s) did not start in this profile and were not exercised at all**: "+ f"{', '.join(absent)}. They are gated behind a compose profile and covered by their own publish "+ f"workflows; this run says nothing about them. ")++ rebuild = "${{ steps.dockerfile_changes.outputs.rebuild }}" == "true"+ if rebuild and not built:+ print()+ print("::error::a build-context change was detected and nothing was built from source; "+ "this run booted published images and did not exercise the change under review")+ sys.exit(1)+ if not rebuild and pulled:+ print()+ print("> No build context changed, so the stack was pulled. This run proves the "+ "published images still boot together; it does not exercise this PR's source.")+ PYPROV+- name: Wait for stack to convergeid: converge+ if: steps.relevant.outputs.run == 'true'run: |set -udeadline=$(( $(date +%s) + HEALTH_BUDGET_SECONDS ))@@ -275,7 +469,7 @@ jobs:exit 1- name: Record success summary- if: success()+ if: success() && steps.relevant.outputs.run == 'true'run: |{echo "## Smoke result"@@ -291,7 +485,7 @@ jobs:} >> "$GITHUB_STEP_SUMMARY"- name: Capture forensics on failure- if: failure()+ if: failure() && steps.relevant.outputs.run == 'true'run: |echo "## Compose state"docker compose ps || true@@ -306,5 +500,5 @@ jobs:free -h || true- name: Teardown- if: always()+ if: always() && steps.relevant.outputs.run == 'true'run: docker compose down -v --remove-orphansdiff --git a/.github/workflows/integration.yml b/.github/workflows/integration.ymlindex 775a654d8..513d0a0a4 100644--- a/.github/workflows/integration.yml+++ b/.github/workflows/integration.yml@@ -31,34 +31,23 @@ name: Integration# `:main` — those surfaces are gated by compose-smoke.yml with its own# rebuild-on-change detection.+# Deliberately NOT path-filtered at the workflow level.+#+# It was, and that is what kept `Backup → destroy → restore` from being a+# required check: a workflow that does not trigger reports nothing, and a+# required check that never reports blocks the pull request forever. That gate+# is the only thing proving the documented disaster-recovery path works, and+# `docs/audit/CLAIM_TO_GATE_MATRIX.md` already cites it as the GATED evidence+# for backup encryption — a row marked GATED whose gate is neither required+# nor guaranteed to run makes that file a liability rather than a control.+#+# The condition moved into the jobs. `changes` computes it once; the three+# expensive jobs skip on it, and `backup-restore` always runs and reports a+# verdict under the same name whether or not it had work to do.on:pull_request:- paths:- - 'services/ingest/**'- - 'services/fusion/**'- - 'services/realtime/**'- - 'services/api/migrations/**'- - 'services/api/app/scripts/run_migrations.py'- - 'services/ueba/alembic/**'- - 'docker-compose.yml'- - 'scripts/integration/**'- - 'scripts/backup.sh'- - 'scripts/restore.sh'- - '.github/workflows/integration.yml'push:branches: [main]- paths:- - 'services/ingest/**'- - 'services/fusion/**'- - 'services/realtime/**'- - 'services/api/migrations/**'- - 'services/api/app/scripts/run_migrations.py'- - 'services/ueba/alembic/**'- - 'docker-compose.yml'- - 'scripts/integration/**'- - 'scripts/backup.sh'- - 'scripts/restore.sh'- - '.github/workflows/integration.yml'workflow_dispatch:concurrency:@@ -69,9 +58,94 @@ permissions:contents: readjobs:+ # ── What this pull request touches ─────────────────────────────────────────+ # One diff, read by every job below, so the four of them cannot disagree+ # about what changed. Fail-open throughout: with no base to diff against,+ # every area reads as touched, because a gate that quietly declines to run+ # is indistinguishable from one that passed.+ changes:+ name: What this change touches+ runs-on: ubuntu-latest+ timeout-minutes: 5+ outputs:+ spine: ${{ steps.areas.outputs.spine }}+ migrations: ${{ steps.areas.outputs.migrations }}+ backup: ${{ steps.areas.outputs.backup }}+ upgrade: ${{ steps.areas.outputs.upgrade }}+ steps:+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7+ with:+ fetch-depth: 0++ - name: Resolve the areas touched+ id: areas+ run: |+ set -euo pipefail++ spine_paths='services/ingest/ services/fusion/ services/realtime/ docker-compose.yml scripts/integration/ .github/workflows/integration.yml'+ migration_paths='services/api/migrations/ services/api/app/scripts/run_migrations.py services/ueba/alembic/ services/api/tests/test_mssp_portfolio_isolation.py tests/isolation/test_postgres_rls.py scripts/check_runtime_db_role.py .github/workflows/integration.yml'+ # backup_crypt.py was missing from the old filter, and it is where+ # every byte of the AES-256-GCM implementation lives — so a change+ # to the encryption itself did not run the gate that proves the+ # encryption works.+ backup_paths='scripts/backup.sh scripts/restore.sh scripts/backup_crypt.py .github/workflows/integration.yml'+ upgrade_paths='services/api/migrations/ services/api/app/scripts/run_migrations.py .github/workflows/integration.yml'++ base_ref=""+ if [ -n "${{ github.base_ref }}" ]; then+ git fetch --no-tags --depth=50 origin "${{ github.base_ref }}" || true+ base_ref="origin/${{ github.base_ref }}"+ elif [ -n "${{ github.event.before }}" ] && [ "${{ github.event.before }}" != "0000000000000000000000000000000000000000" ]; then+ base_ref="${{ github.event.before }}"+ fi++ emit_all_true() {+ for area in spine migrations backup upgrade; do+ echo "${area}=true" >> "$GITHUB_OUTPUT"+ done+ }++ if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then+ echo "manual dispatch: running every area"+ emit_all_true+ exit 0+ fi+ if [ -z "${base_ref}" ]; then+ echo "no diff base; running every area"+ emit_all_true+ exit 0+ fi++ diff_status=0+ changed=$(git diff --name-only "${base_ref}"...HEAD) || diff_status=$?+ if [ "${diff_status}" -ne 0 ]; then+ echo "git diff vs ${base_ref} failed (exit ${diff_status}); running every area"+ emit_all_true+ exit 0+ fi+ echo "${changed}" > /tmp/changed.txt+ echo "files changed vs ${base_ref}: $(wc -l < /tmp/changed.txt)"++ decide() {+ local area="$1"; shift+ local hit=false+ for prefix in $@; do+ if grep -q "^${prefix}" /tmp/changed.txt; then hit=true; break; fi+ done+ echo " ${area}: ${hit}"+ echo "${area}=${hit}" >> "$GITHUB_OUTPUT"+ }+ decide spine "${spine_paths}"+ decide migrations "${migration_paths}"+ decide backup "${backup_paths}"+ decide upgrade "${upgrade_paths}"+# ── The spine, end to end ──────────────────────────────────────────────────spine:name: Event spine (real containers)+ # Not a required check, so a job-level skip is fine here: nothing is waiting on it to report.+ needs: changes+ if: needs.changes.outputs.spine == 'true'runs-on: ubuntu-latesttimeout-minutes: 40env:@@ -260,6 +334,8 @@ jobs:# ── Migration correctness ──────────────────────────────────────────────────migrations:name: Migrations (fresh apply, idempotent re-run, alembic round-trip)+ needs: changes+ if: needs.changes.outputs.migrations == 'true'runs-on: ubuntu-latesttimeout-minutes: 15services:@@ -450,9 +526,18 @@ jobs:echo "all four chains applied to one database"# ── Disaster recovery: an untested backup is not a backup ─────────────────+ #+ # Unlike the three jobs above, this one has no job-level `if`. It runs on+ # every pull request and reports under the same check name whether or not it+ # had work to do, because it is meant to be a *required* check and a+ # required check that never reports blocks the pull request forever. The+ # condition is on the steps instead, so the expensive half — a Postgres, an+ # S3 endpoint, a full dump-and-restore — is skipped when the diff touches+ # none of it, and the verdict still arrives.backup-restore:name: Backup → destroy → restore (Postgres via S3)runs-on: ubuntu-latest+ needs: changestimeout-minutes: 15services:postgres:@@ -482,8 +567,22 @@ jobs:BACKUP_ENCRYPTION_KEY: 2b7e151628aed2a6abf7158809cf4f3c762e7160f38b4da56a784d9045190cfesteps:- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7+ - name: Nothing under the disaster-recovery path changed+ if: needs.changes.outputs.backup != 'true'+ run: |+ {+ echo "## Disaster-recovery gate — not exercised"+ echo ""+ echo "This pull request changes none of \`scripts/backup.sh\`,"+ echo "\`scripts/restore.sh\`, \`scripts/backup_crypt.py\` or this workflow,"+ echo "so the seed → backup → destroy → restore cycle was not run."+ echo ""+ echo "The job still reports, under the same check name, because a"+ echo "required check that never reports blocks the pull request forever."+ } >> "$GITHUB_STEP_SUMMARY"- name: Install tooling (awscli + postgres client)+ if: needs.changes.outputs.backup == 'true'run: |pip install --quiet awscli "cryptography>=46,<51"sudo apt-get update -q && sudo apt-get install -y -q postgresql-client@@ -505,6 +604,7 @@ jobs:# independently of CI itself, and it is not subject to Docker Hub's# anonymous pull limits. Pinned to the v1.8.0 multi-arch index digest.- name: Start the S3 endpoint (versitygw)+ if: needs.changes.outputs.backup == 'true'run: |set -euo pipefail# The posix backend chdirs into its root, so the directory has to@@ -521,6 +621,7 @@ jobs:# burn its retries and fail with a connection error that says nothing# about why. Fail here instead, with the container's own logs.- name: Wait for the S3 endpoint+ if: needs.changes.outputs.backup == 'true'run: |set -uo pipefailfor i in $(seq 1 30); do@@ -541,9 +642,11 @@ jobs:exit 1- name: Create bucket+ if: needs.changes.outputs.backup == 'true'run: aws s3 mb "${BACKUP_S3_BUCKET}"- name: Seed data+ if: needs.changes.outputs.backup == 'true'run: |PGPASSWORD=aisoc_ci_secret psql -h localhost -U aisoc -d aisoc <<'SQL'CREATE TABLE dr_probe (id SERIAL PRIMARY KEY, payload TEXT NOT NULL);@@ -555,6 +658,7 @@ jobs:[ "${COUNT}" = "500" ]- name: Backup (scripts/backup.sh)+ if: needs.changes.outputs.backup == 'true'run: ./scripts/backup.sh --component postgres# The restore assertion below proves the dump round-trips. It cannot@@ -562,6 +666,7 @@ jobs:# bucket, which is the property the encryption exists for — and which# architecture.md claimed for months while backup.sh shipped plaintext.- name: Assert the uploaded artifact is encrypted and manifested+ if: needs.changes.outputs.backup == 'true'run: |set -euo pipefailKEY=$(aws s3 ls "${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/postgres/" \@@ -606,6 +711,7 @@ jobs:"- name: Destroy all data+ if: needs.changes.outputs.backup == 'true'run: |PGPASSWORD=aisoc_ci_secret psql -h localhost -U aisoc -d aisoc \-c "DROP SCHEMA public CASCADE; CREATE SCHEMA public;"@@ -615,6 +721,7 @@ jobs:[ "${COUNT}" = "0" ]- name: Restore (scripts/restore.sh) and measure RTO+ if: needs.changes.outputs.backup == 'true'run: |set -euo pipefailSTART=$(date +%s)@@ -624,6 +731,7 @@ jobs:echo "measured restore time: ${RTO}s"- name: Assert full data integrity+ if: needs.changes.outputs.backup == 'true'run: |COUNT=$(PGPASSWORD=aisoc_ci_secret psql -h localhost -U aisoc -d aisoc -tAc "SELECT count(*) FROM dr_probe")echo "rows after restore: ${COUNT}"@@ -633,6 +741,7 @@ jobs:# backup must refuse to restore rather than produce a partial database,# which is the failure that gets discovered during an incident.- name: Assert a tampered backup is refused+ if: needs.changes.outputs.backup == 'true'run: |set -uo pipefailKEY=$(aws s3 ls "${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/postgres/" \@@ -656,6 +765,7 @@ jobs:echo "tampered backup correctly refused"- name: Publish DR summary+ if: needs.changes.outputs.backup == 'true'run: |{echo "## Disaster-recovery gate"@@ -683,6 +793,8 @@ jobs:# on a pre-existing, populated schema (not a fresh apply in disguise).upgrade:name: Upgrade (${{ matrix.from_ref }} → HEAD, data survives)+ needs: changes+ if: needs.changes.outputs.upgrade == 'true'runs-on: ubuntu-latesttimeout-minutes: 15strategy:diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 2d3354795..093a38a46 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -77,6 +77,54 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Added+- **Every test file in the tree is now executed by a workflow, and a gate+ holds it that way in both directions.** 63 were executed by nothing at all:+ 56 of the 82 under `services/agents`, because that invocation was a+ hand-maintained list of 26 file paths; five under the repository's own+ `tests/`, which was reached only as nineteen single-file invocations spread+ across eleven workflows; and both files under `scripts/tests/`, which no+ workflow named. A file nothing runs reports nothing, which is+ indistinguishable from a file that passed.++ `scripts/check_test_discovery.py` derives the corpus from `git ls-files` and+ models reach the way pytest collects — directory arguments, `--ignore`,+ `conftest` `collect_ignore`, and `-m` marker deselection, since a file whose+ every test is deselected runs nothing however plainly it is named. An+ invocation naming a path that is not in the tree fails too. Quarantine is a+ shrink-only list where every entry carries a reason and an entry that has+ become reachable fails as stale, so "not run" can never again be silent.++ Two blind spots in the gate's own parser were found by enumerating what it+ **credited** rather than what it flagged. A step that `cd`s into a service+ and then names `tests/` had its paths resolved against the repository root,+ which simultaneously reported all 82 agents files unreached and credited the+ root `tests/` tree with a package's invocation — the same defect in both+ directions at once. And `-m` was first treated as "a filter, not a+ collection rule", which would have credited a module in full under+ `-m "not integration"` while nothing in it ran.++ The invocations are directories now. `services/agents` runs `tests/`;+ `tests/` and `scripts/tests/` run as directories in `python-test`, with+ `tests/isolation/` left to `isolation.yml`, which owns its live stores.++- **`--suite <name>` in `scripts/run_evals.py` ran every suite.** Its `--help`+ said "runs that suite in isolation", the module docstring said "run a single+ suite by name", and `args.suite` reached nothing but the wording of the+ banner: the report dict called all eleven `_run_*` helpers inline. So+ `--suite mitre_accuracy` took the full runtime, graded eleven gates, and+ printed `PASS — mitre_accuracy green` — a verdict naming one suite and+ decided by eleven, with `--ci` able to fail an operator's single-suite+ bisection on an unrelated regression. The report now carries `suite_filter`+ and only the requested suites are called; the registry is asserted against+ `_SUITE_NAMES` so a name argparse accepts cannot be one no runner answers+ to. A substrate-import failure also exits **3**, which is what this file's+ own "Exit codes" block has always documented — it exited **2**, and 2 means+ "MITRE accuracy regressed against the baseline", so a fresh clone missing a+ dependency reported an accuracy regression, and the one actionable+ instruction (`pip install -e services/agents`) was not in the message.+ All three defects were pinned by tests in `scripts/tests/` that had been+ failing for as long as they existed, in the tree no workflow ran.+- **No gate may report OK over a repository with no content, and it is a CIgate now rather than a probe somebody happened to run once.** Copying`scripts/` into an empty git repository and running every wired check there@@ -825,6 +873,64 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Changed+- **`Backup → destroy → restore` and `docker compose up — full stack` report+ a verdict on every pull request, so both can be required checks.** Both+ workflows were path-filtered at the workflow level, and a workflow that does+ not trigger reports no check at all — so a required check would never+ arrive and the pull request would block forever. That is the only thing that+ had been standing between the disaster-recovery path being tested and it+ being tested and unable to regress; `docs/audit/CLAIM_TO_GATE_MATRIX.md`+ already cited the DR job as the `GATED` evidence for backup encryption,+ which makes a gate that might not run a liability rather than a control.++ The condition moved from the trigger into the jobs. `integration.yml` gained+ a `changes` job that reads the diff once; `spine`, `migrations` and+ `upgrade` skip on it as before, while `backup-restore` always runs and+ guards its expensive steps, so the check name is produced by the same single+ job either way. `compose-smoke.yml`'s `smoke` job does the same inline. The+ DR filter was also missing `scripts/backup_crypt.py` — where every byte of+ the AES-256-GCM implementation lives — so a change to the encryption did not+ run the gate that proves the encryption works.++- **Compose smoke says which services it built and which it pulled.** It boots+ published `:main` images unless a build context changed, so a green run+ frequently never compiled the service under review — a required check that+ can pass without exercising the change is the defect this whole effort is+ about. The run now reports provenance per service, read off the images that+ are actually running rather than predicted from the decision: a pulled image+ carries a RepoDigest and a locally built one does not. It fails when a+ build-context change was detected and nothing was built, and says plainly,+ when nothing changed, that the run proves the published images still boot+ together and does not exercise this pull request's source.++ The build contexts themselves are now derived from `docker-compose.yml`+ rather than listed in the workflow under a "keep this list in sync" comment.+ A service added to compose and forgotten in that list would have had its+ source changes pulled from the registry instead of built — the gate booting+ a stale image and passing, inside the check that exists to catch exactly+ that. The derived set matches the old list exactly today, so nothing about+ today's behaviour changes.++- **A permanent playbook-step failure is no longer retried as if transient.**+ `dispatch_step` raises before any I/O when the run context has no tenant,+ and the engine retried it with exponential backoff: 2s, 4s, then 8s, to+ arrive at the message it already had on the first attempt. The cost is not+ the fourteen seconds — it is that a permanent misconfiguration presents to+ an operator mid-incident as flakiness, so their next move looks like "wait"+ when it is "go and set the variable".++ `BridgeUnavailable` stays the base class every caller catches, and the+ permanent half is now `BridgeMisconfigured`, marked with a new+ `PermanentStepFailure` that any handler can raise and the engine honours.+ The line: **permanent** when the cause is this deployment's configuration or+ a violation of the API's own contract — the enable switch, the service+ token, the missing tenant, a non-retryable 4xx, a JSON body with no+ `executed` field; **transient** when the cause is reachability — a transport+ error, any 5xx, `408`/`425`/`429`, and a body that did not parse as JSON at+ all, which is overwhelmingly an ingress error page rather than the API. A+ failed step records `permanent` and `attempts`, so the run says why it was+ tried once and not four times.+- **The tenant-predicate gate can now tell a guard from a log line, and theratchet shrank from 34 to 32.** It credited any query addressed by a keypassed to a call that also received the caller's tenant — whichdiff --git a/apps/docs/docs/compliance/evidence-pack.md b/apps/docs/docs/compliance/evidence-pack.mdindex c6013eb0b..48e148abf 100644--- a/apps/docs/docs/compliance/evidence-pack.md+++ b/apps/docs/docs/compliance/evidence-pack.md@@ -33,7 +33,7 @@ and says so.| Prompt-injection resistance | 32-payload adversarial corpus with a recall floor and a false-positive ceiling | `services/agents/tests/adversarial/` || Claim-to-gate traceability | Every product claim mapped to the CI job that fails when it stops being true | `docs/audit/CLAIM_TO_GATE_MATRIX.md` |-The last row is the one worth reading first. It is the honest index: 128 rows+The last row is the one worth reading first. It is the honest index: 129 rows`GATED`, 8 `PARTIAL` with the specific gap named, and a ratchet that refusesto let a row sit at `NO GATE`.diff --git a/docs/audit/CLAIM_TO_GATE_MATRIX.md b/docs/audit/CLAIM_TO_GATE_MATRIX.mdindex 7328835f4..fa2c73cb5 100644--- a/docs/audit/CLAIM_TO_GATE_MATRIX.md+++ b/docs/audit/CLAIM_TO_GATE_MATRIX.md@@ -116,7 +116,7 @@ Statuses: `GATED` (a CI job fails when the claim stops being true) · `PARTIAL`| Hallucination, abstention and calibration are measured, not estimated | README (public benchmark) | `ci.yml` agents job (`test_benchmark.py`) - hallucination is the fraction of cited indicators absent from the evidence the agent was given and is graded on abstentions too; accuracy is computed over answered incidents only so abstaining cannot raise it; and the calibration gap exposes an agent more confident when wrong | GATED | - || Benchmark figures declare the provenance of the corpus they came from | README (benchmark transparency) | `ci.yml` agents job - the runner refuses a corpus whose incidents do not declare provenance, the counts travel with the result so a consumer cannot drop them, `--limit` recounts over the slice rather than reporting the whole corpus, and the report prints provenance before any figure | GATED | the shipped corpus is 200 synthetic incidents; public-dataset and real-anonymised entries are supported by the schema but not yet contributed || Analyst disagreement becomes durable organisation memory | README (learning from analysts) | `ci.yml` api job (`test_analyst_feedback.py`) - the reason vocabulary is closed, corroboration counts distinct analysts rather than rows (one person clicking the same button ten times is one opinion), an approved-pentest statement expires within 14 days because one that outlives the pentest is an attacker's best friend, rule defects route to detection engineering rather than becoming a suppression that hides them, and a compiled statement names the process, principal, host and window rather than degrading to 'PowerShell is expected' | GATED | the triage prompt does not yet read `active_statements`; the memory is written and queryable but not yet consumed |-| Backup artifacts are encrypted at rest and integrity-checked | architecture.md (backup & restore) | `integration.yml` backup-restore job: asserts the uploaded object carries the AiSOC AEAD header, has no gzip magic, leaks no schema names, and is listed in a SHA-256 manifest; then flips a bit and asserts restore refuses it | GATED | - |+| Backup artifacts are encrypted at rest and integrity-checked | architecture.md (backup & restore) | `integration.yml` backup-restore job: asserts the uploaded object carries the AiSOC AEAD header, has no gzip magic, leaks no schema names, and is listed in a SHA-256 manifest; then flips a bit and asserts restore refuses it. The job is no longer path-filtered at the workflow level, so it reports a verdict on every pull request and can therefore be a required check — a row marked GATED whose gate is not guaranteed even to run makes this file a liability rather than a control. `scripts/backup_crypt.py`, where every byte of the AES-256-GCM implementation lives, was missing from the old filter, so a change to the encryption itself did not run the gate that proves the encryption works | GATED | - || A human approval of a response action is authorized, not just logged | operations/action-approvals.md | `ci.yml` actions job (`test_approval_identity.py`) — an approval with no resolvable identity is refused, an unmapped ChatOps user cannot approve, a user below the action's blast-radius tier cannot approve, and separation of duties applies to a ChatOps approval; plus `test_interactions.py` and `test_callbacks_approver.py` assert the verified platform identity reaches the client rather than only the audit row | GATED | pending approvals are still an in-process dict, so they are lost on restart and not shared across replicas; nothing pushes a card proactively || A signed email approval link resolves and carries an identity | operations/action-approvals.md | `ci.yml` api job (`test_email_approval_endpoint.py`) — the path the issuer mints is requested rather than inspected in the route table, the recipient is signed into the token, swapping it breaks the signature, a token without one is refused rather than forwarded, and an already-decided action answers 409 | GATED | - || Every LLM call passes a fail-closed input contract before the request | operations/security.md (LLM input contract) | `ci.yml` agents job (`test_llm_contract_no_bypass.py`) — one gate AST-walks `.ainvoke`/`.astream`, a second flags any file in either service that names a completions endpoint and issues its own POST, each with a synthetic-bypass self-test; `ci.yml` api job (`test_llm_safety.py`) asserts per endpoint that it routes through the wrapper **and** no longer POSTs to a provider itself; `sync_vendored_llm_contract.py --check` keeps both services on one classifier | GATED | the contract is a minimum-leak control, not an injection sanitizer — that is `PromptInjectionGuard`, which demotes rather than refuses |@@ -142,12 +142,13 @@ Statuses: `GATED` (a CI job fails when the claim stops being true) · `PARTIAL`| Every `ActionType` resolves a capability contract, so the confidence matrix is never skipped | concepts/live-actions.md (the one alias, and when it goes) | `ci.yml` actions job (`scripts/check_action_contract.py` directions 7 and 8 — every `ActionType` must resolve a contract by name or alias, and every alias must key a real `ActionType`, point at a real contract, not shadow a capability of the same name, and describe a bridge some adapter actually implements) + `test_approval_gate_wiring.py` (`notify_slack` at 10% confidence was approved for auto-execution and now requires an analyst; the alias is asserted unable to *lower* a requirement) | GATED | one alias entry (`notify_slack` -> `notify`), with its retirement condition recorded: it goes when `ActionType` does || Every check this matrix names is actually run by CI | this matrix (each row names a gate) | `ci.yml` (`scripts/check_gate_coverage.py` — resolves the whole graph rather than reading workflow names: workflows, `make` recipes they invoke, the pytest suites they collect and the scripts those reach, so a check wired only through a test still counts. Both directions — a check no workflow reaches, and a workflow step naming a `scripts/` path that does not exist. `--self-test` runs first and asserts the resolver discriminates rather than returning "reached" for everything, and the corpus counts are printed so it cannot report on a tree it never opened) | GATED | the ratchet for a deliberately-unwired check is empty; it may shrink, never grow || What counts as a check is decided by what a script does, not what it is named | `scripts/check_gate_coverage.py` (the inventory it reports on) | `ci.yml` (three structural signals, all read from the tree: a declared verdict flag, an exit status derived from findings the script accumulates, and a workflow job whose output another job branches on. The first two are intrinsic, so an uncalled gate is still inventoried and the `SCRIPT -> WORKFLOW` direction is not vacuous. The classifier is also checked in reverse — a script CI runs with `--check` that the classifier does not inventory fails as `classifier-blind-spot`) + `tests/test_gate_coverage_classifier.py` (10 cases, including a gate planted under a name no convention would reveal, and a generator that aborts on an empty read asserted *not* to be a check) | GATED | - |-| No check reports OK over a repository with no content | every row above (a gate that credits an empty tree proves nothing about a full one) | `ci.yml :: python-test` (`scripts/check_gate_contract.py` — runs all 69 inventoried checks the way a workflow runs each one, inside a git repository holding only `scripts/`, and requires a non-zero exit. Two tree shapes, because they are two questions: **bare** omits the directories a gate judges, **skeleton** creates them empty. The second is the one that happens — a renamed package, a glob that stopped matching — and a gate opening with `if not X.is_dir(): return 2` answers the first for a reason that says nothing about its corpus. Measured before it was added: 64 of 71 checks already refused the skeleton, so the exception list stayed readable rather than doubling. A run that failed for a reason the empty tree did not cause — an argparse usage error, or an import of a module the repository does not supply — is INCONCLUSIVE and fails rather than counting as a refusal, so the gate cannot credit a failure it did not cause. Exceptions are recorded with the disposition they are excused for and checked in both directions: an entry naming nothing, and an entry whose gate has started behaving differently, both fail) + `tests/test_gate_contract.py` (45 cases) | GATED | 5 recorded exceptions, keyed by tree shape; the probe cannot pose the question to a gate whose only input is a path named on the command line, or one that lives inside `scripts/`. One entry exists because its dispositions legitimately differ: `check_logger_kwargs.py` scans every Python file in the repository, so against a skeleton it has a real corpus in `scripts/` and correctly finds nothing wrong |-| Every check resolves its root from git and carries a `--self-test` | `scripts/` (the gates themselves) | `ci.yml :: python-test` (`scripts/check_gate_contract.py` — a check that derives a repository root from `__file__` without consulting git fails as `root-from-file`, and one that declares no `--self-test` fails as `no-self-test`. Both read from the syntax tree: the first spelling matched `repo_root` anywhere in the text and `check_gate_coverage.py` emits it as a JSON key, so a non-compliant gate read as compliant on the strength of a dictionary key in its own output. Both exception lists are empty — all 69 comply — and an entry added to either must carry a reason) | GATED | - |+| No check reports OK over a repository with no content | every row above (a gate that credits an empty tree proves nothing about a full one) | `ci.yml :: python-test` (`scripts/check_gate_contract.py` — runs all 72 inventoried checks the way a workflow runs each one, inside a git repository holding only `scripts/`, and requires a non-zero exit. Two tree shapes, because they are two questions: **bare** omits the directories a gate judges, **skeleton** creates them empty. The second is the one that happens — a renamed package, a glob that stopped matching — and a gate opening with `if not X.is_dir(): return 2` answers the first for a reason that says nothing about its corpus. Measured before it was added: 64 of 71 checks already refused the skeleton, so the exception list stayed readable rather than doubling. A run that failed for a reason the empty tree did not cause — an argparse usage error, or an import of a module the repository does not supply — is INCONCLUSIVE and fails rather than counting as a refusal, so the gate cannot credit a failure it did not cause. Exceptions are recorded with the disposition they are excused for and checked in both directions: an entry naming nothing, and an entry whose gate has started behaving differently, both fail) + `tests/test_gate_contract.py` (45 cases) | GATED | 5 recorded exceptions, keyed by tree shape; the probe cannot pose the question to a gate whose only input is a path named on the command line, or one that lives inside `scripts/`. One entry exists because its dispositions legitimately differ: `check_logger_kwargs.py` scans every Python file in the repository, so against a skeleton it has a real corpus in `scripts/` and correctly finds nothing wrong |+| Every check resolves its root from git and carries a `--self-test` | `scripts/` (the gates themselves) | `ci.yml :: python-test` (`scripts/check_gate_contract.py` — a check that derives a repository root from `__file__` without consulting git fails as `root-from-file`, and one that declares no `--self-test` fails as `no-self-test`. Both read from the syntax tree: the first spelling matched `repo_root` anywhere in the text and `check_gate_coverage.py` emits it as a JSON key, so a non-compliant gate read as compliant on the strength of a dictionary key in its own output. Both exception lists are empty — all 72 comply — and an entry added to either must carry a reason) | GATED | - |+| Every test file in the tree is executed by a workflow | `CONTRIBUTING.md` / the suites themselves | `ci.yml :: python-test` (`scripts/check_test_discovery.py` — the corpus is `git ls-files`, so a service added tomorrow is scanned with no edit, and an untracked install tree is out by construction rather than by a name somebody remembered. Reach is modelled the way pytest collects: directory arguments, `--ignore`, `conftest` `collect_ignore`, and `-m` marker deselection, because a file whose every test is deselected runs nothing however plainly it is named. Both directions — an invocation naming a path that is not in the tree fails too. Quarantine is a shrink-only list where each entry carries a reason and an entry that has become reachable fails as stale. Two blind spots in this parser were found by counting what it **credited**: a step that `cd`s into a service and then names `tests/` had its paths resolved against the repository root, which reported all 82 agents files unreached *and* credited the root tree with a package's invocation; and `-m` was first skipped as "a filter, not a collection rule", which would have credited a file in full under `-m "not integration"`) + `tests/test_test_discovery_gate.py` | GATED | - |## Summary-- GATED: 128+- GATED: 129- PARTIAL: 8- NO GATE: 0 (**every claim is backed by a failing test.** The last NO GATE — the weekly benchmark scoreboard running live against `main` — closed in Phase E1: `scripts/check_scoreboard.py` ties the published scoreboard to a deterministic per-PR live-agent MITRE-accuracy run, while the funded weekly `wet-eval.yml` appends the LLM-tier rows. The remaining PARTIAL rows are honest, named deferrals — each states the specific gap and what closes it — not unproven claims.)diff --git a/scripts/check_gate_coverage.py b/scripts/check_gate_coverage.pyindex 1caa325bc..b8fef0e7c 100644--- a/scripts/check_gate_coverage.py+++ b/scripts/check_gate_coverage.py@@ -420,7 +420,16 @@ def _command_text(step: dict) -> str:_NEEDS_OUTPUT = re.compile(r"needs\.([A-Za-z0-9_-]+)\.outputs")-_SCRIPT_IN_JOB = re.compile(r"scripts/([\w.-]+\.py)")++#: A script the job *runs*, not one it merely mentions. The interpreter prefix+#: is what makes the difference: a job that computes which areas a diff touched+#: lists `scripts/backup_crypt.py` as a path to match, and the previous+#: spelling — the bare path, anywhere in the job's YAML — read that as the job+#: graph asking an encryption utility for a verdict, inventorying it as a gate+#: it is not. Same shape as the matcher that counted eleven services as+#: CI-covered on the strength of a quoted path inside an `echo`: a path is not+#: an invocation.+_SCRIPT_IN_JOB = re.compile(r"(?:python3?(?:\.\d+)?|uv run|poetry run)\s+(?:-m\s+)?[\w./-]*?scripts/([\w.-]+\.py)")def _gating_jobs(doc: dict, workflow: str) -> dict[str, str]:diff --git a/scripts/check_test_discovery.py b/scripts/check_test_discovery.pynew file mode 100644index 000000000..1d635e7d8--- /dev/null+++ b/scripts/check_test_discovery.py@@ -0,0 +1,936 @@+#!/usr/bin/env python3+"""Every test file in a tree must be reached by that tree's own CI invocation.++Why this exists+---------------+``services/agents`` ships 82 test files. Until this gate landed, the workflow+step that runs them named 26 of those files one per line, and the other 56+were executed by nothing at all. Not skipped, not quarantined, not xfailed —+uncollected, which prints nothing and looks exactly like a suite that passed.++``test_playbook_engine_correctness.py`` was one of the 56. It carried a test+that spent fourteen seconds doing the precise thing its own name said must not+happen, and it did so for as long as it existed, because nothing ever ran it+to notice.++A hand-maintained list of files is the same mechanism as the hand-written+``ConnectorType`` union whose ten members named nothing the platform ingests,+and the hand-kept alias vocabulary that let an authenticated route read as+wide open. In each case the list was correct when written and the tree moved.+The fix is the same in all three: stop maintaining the list, derive it.++The question this asks+----------------------+Not "do the tests pass" — that is the suite's job. This asks **which test+files the tree's own invocation can even reach**, and it asks it in both+directions:++``unreached``+ A test file exists in a tree and no invocation for that tree collects it.+ Adding a file to a directory CI already runs is free; adding one beside a+ hand-maintained list is silently free of signal.++``names-nothing``+ An invocation names a path that is not there. A step that lists a deleted+ file either fails loudly on every run or, more often, the list was updated+ and the file was not — either way the step no longer does what it reads+ as doing.++What counts as reached+----------------------+pytest's own collection rules, applied to the arguments CI actually passes:+a directory argument reaches every ``test_*.py`` / ``*_test.py`` beneath it+(minus ``--ignore``), a file argument reaches that file, and+``--ignore``/``--deselect`` subtract. Anything a ``conftest.py`` adds to+``collect_ignore`` is subtracted too, because a file pytest is told to skip+collecting is not reached however it is named.++Quarantine is the one honest way to not run a file, and it has to be visible:+a file listed in ``QUARANTINE`` below is excused from ``unreached`` and must+carry a stated reason. The list is checked in both directions, so an entry+naming a file that no longer exists — or one that has since become reachable+— fails the build rather than sitting there looking like coverage.++Usage+-----+ python3 scripts/check_test_discovery.py # gate+ python3 scripts/check_test_discovery.py --list # per-tree detail+ python3 scripts/check_test_discovery.py --json+ python3 scripts/check_test_discovery.py --self-test # prove it detects drift++Exit codes: 0 clean, 1 findings, 2 the scan itself could not run.+"""++from __future__ import annotations++import argparse+import ast+import json+import re+import shlex+import sys+from collections.abc import Iterable+from dataclasses import asdict, dataclass, field+from pathlib import Path++# `scripts/` is on sys.path when this file is run as a program, but not when a+# test loads it by path with importlib. Its siblings sit beside it either way.+sys.path.insert(0, str(Path(__file__).resolve().parent))++import check_gate_coverage as coverage # noqa: E402+from gate_toolkit import repo_root, self_test_main # noqa: E402++# --------------------------------------------------------------------------+# Recorded quarantine. Shrink-only, checked in both directions.+# --------------------------------------------------------------------------+#: Test files deliberately not run by any workflow, each with the reason. A+#: quarantined file is still *collected* by nothing, which is the state this+#: gate exists to remove — so the entry is what makes it visible rather than+#: silent. An entry naming a file that is not there, or one that has since+#: become reachable, is a finding.+#:+#: Keys are repository-relative POSIX paths.+QUARANTINE: dict[str, str] = {}++#: The corpus is ``git ls-files``, so there is no exclusion list here and+#: nothing to keep in sync. ``node_modules``, ``.venv`` and ``site-packages``+#: are not tracked, so they are out by construction rather than by a name+#: somebody remembered to write down — and a list of ephemeral directory names+#: cannot be checked in both directions anyway, because on a clean checkout+#: none of them exist. The first version of this gate carried that list and+#: failed on its own repository for exactly that reason.+++class DiscoveryError(RuntimeError):+ """The scan could not be set up. Never downgraded to a passing result."""+++# --------------------------------------------------------------------------+# What pytest would collect+# --------------------------------------------------------------------------+#: pytest's default ``python_files``. Read from each tree's own config when it+#: declares one, because a tree that renames the pattern renames what "a test+#: file" means for that tree and a gate using the default would scan for files+#: that cannot exist.+DEFAULT_PYTHON_FILES = ("test_*.py", "*_test.py")++#: `pytest` in command position. Matching the bare word pulls in every+#: `pip install ... pytest` line, and an install line has no path argument, so+#: it reads as a bare collection of the entire tree — an over-broad resolver+#: manufactures the coverage this gate exists to disprove.+_PYTEST_COMMAND = re.compile(+ r"(?:^|[\n;&|]|\bpython3?(?:\.\d+)?\s+-m\s+|\buv\s+run\s+|\bpoetry\s+run\s+)\s*pytest\b",+ re.MULTILINE,+)+_PIP_INSTALL = re.compile(r"\b(?:pip3?|python3?(?:\.\d+)?\s+-m\s+pip)\s+install\b")++#: Options that consume the following token, so a resolver does not mistake+#: that token for a test path. ``-k`` selects within what was collected by+#: test *name*, which this gate does not model — it is a filter a developer+#: types, not a standing property of a workflow, and no workflow here uses it.+#: ``-m`` is different and is captured below.+_OPTS_WITH_VALUE = frozenset({"-k", "-p", "-n", "--rootdir", "--cov", "--cov-config", "--cov-report", "--junitxml", "--tb", "-W"})+++def _segments(text: str) -> list[str]:+ """Individual commands inside a workflow `run:` block."""+ joined = text.replace("\\\n", " ")+ out: list[str] = []+ for line in joined.splitlines():+ for piece in re.split(r"&&|\|\||[;|]", line):+ stripped = piece.strip()+ if stripped:+ out.append(stripped)+ return out+++@dataclass+class Invocation:+ """One pytest command a workflow runs, with its arguments resolved."""++ workflow: str+ working_dir: str+ command: str+ paths: list[str] = field(default_factory=list)+ ignores: list[str] = field(default_factory=list)+ markers: str = ""+ bare: bool = False+++def parse_invocations(surface: coverage.Surface) -> list[Invocation]:+ """Every pytest invocation the workflow surface runs, with its paths.++ Working directories and matrix expansion come from ``Surface`` rather than+ from a second parser here: two readers of the same workflows drift the+ first time either one learns something the other has not.++ A step's ``working-directory`` is only half the answer. The largest+ invocation in this repository — the agents one — is a multi-line ``run:``+ block that ``cd``s into the service and then runs pytest with+ ``tests/``-relative paths, and the first version of this parser resolved+ those against the repository root instead. It therefore reported the whole+ agents suite unreached *and* credited the root ``tests/`` tree with an+ invocation belonging to a package, which is the same defect in both+ directions at once. So ``cd`` is followed, in order, within each block.+ """+ found: list[Invocation] = []+ for workflow, working_dir, text in surface.texts():+ cwd = working_dir+ for segment in _segments(text):+ moved = _cd_target(segment, cwd)+ if moved is not None:+ cwd = moved+ continue+ if _PIP_INSTALL.search(segment):+ continue+ if not _PYTEST_COMMAND.search(segment):+ continue+ try:+ tokens = shlex.split(segment, comments=True)+ except ValueError:+ continue+ args = _args_after_pytest(tokens)+ if args is None:+ continue+ paths, ignores, markers = _split_args(args)+ found.append(+ Invocation(+ workflow=workflow,+ working_dir=cwd,+ command=segment.strip(),+ paths=paths,+ ignores=ignores,+ markers=markers,+ bare=not paths,+ )+ )+ return found+++def _cd_target(segment: str, cwd: str) -> str | None:+ """The directory this segment changes to, or None if it is not a ``cd``.++ Relative to the block's current directory, so ``cd services/agents``+ followed by ``cd ..`` lands back where it started. A ``cd`` whose argument+ is an unexpanded variable returns the current directory unchanged rather+ than a path that cannot exist — guessing would move the resolver+ somewhere arbitrary and every path after it would read as missing.+ """+ try:+ tokens = shlex.split(segment, comments=True)+ except ValueError:+ return None+ if len(tokens) != 2 or tokens[0] != "cd":+ return None+ target = tokens[1]+ if target.startswith("$") or target.startswith("/"):+ return cwd+ parts: list[str] = []+ for part in (Path(cwd or ".") / target).parts:+ if part == "..":+ if parts:+ parts.pop()+ elif part not in {".", ""}:+ parts.append(part)+ return Path(*parts).as_posix() if parts else ""+++def _args_after_pytest(tokens: list[str]) -> list[str] | None:+ """The arguments handed to pytest, or None if this is not a pytest run."""+ index = 0+ while index < len(tokens) and re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*=.*", tokens[index]):+ index += 1+ if index < len(tokens) and tokens[index] == "env":+ index += 1+ while index < len(tokens) and re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*=.*", tokens[index]):+ index += 1+ if index < len(tokens) and tokens[index] in {"uv", "poetry"}:+ if index + 1 >= len(tokens) or tokens[index + 1] != "run":+ return None+ index += 2+ if index < len(tokens) and re.fullmatch(r"python3?(?:\.\d+)?", tokens[index]):+ index += 1+ if index < len(tokens) and tokens[index] == "-m":+ index += 1+ if index >= len(tokens) or tokens[index] != "pytest":+ return None+ return tokens[index + 1 :]+++def _split_args(args: list[str]) -> tuple[list[str], list[str], str]:+ """(positional test paths, --ignore paths, the -m marker expression)."""+ paths: list[str] = []+ ignores: list[str] = []+ markers = ""+ index = 0+ while index < len(args):+ token = args[index]+ if token.startswith("--ignore=") or token.startswith("--ignore-glob="):+ ignores.append(token.split("=", 1)[1])+ elif token in {"--ignore", "--ignore-glob"} and index + 1 < len(args):+ ignores.append(args[index + 1])+ index += 1+ elif token == "-m" and index + 1 < len(args):+ markers = args[index + 1]+ index += 1+ elif token.startswith("-m") and len(token) > 2 and not token.startswith("--"):+ markers = token[2:]+ elif token in _OPTS_WITH_VALUE and index + 1 < len(args):+ index += 1+ elif token.startswith("-"):+ pass+ else:+ paths.append(token)+ index += 1+ return paths, ignores, markers+++# --------------------------------------------------------------------------+# Marker deselection+# --------------------------------------------------------------------------+# A file every one of whose tests is deselected by `-m` runs nothing, however+# plainly the invocation names it. The first version of this gate skipped `-m`+# as "a filter, not a collection rule" and would therefore have credited+# `test_graph_freshness.py` in full under `-m "not integration"` — a gate+# crediting a file nothing runs, which is the defect it exists to remove, one+# level up. No workflow here passes `-m` today; the point is that adding one+# cannot quietly empty a tree.+++def module_markers(path: Path) -> frozenset[str]:+ """Markers a module applies to every test in it, via ``pytestmark``.++ Module level only. A marker on a single test deselects that test and not+ the file, and this gate is about files.+ """+ try:+ tree = ast.parse(path.read_text(encoding="utf-8", errors="replace"))+ except (OSError, SyntaxError):+ return frozenset()+ found: set[str] = set()+ for node in tree.body:+ if not isinstance(node, ast.Assign):+ continue+ if not any(isinstance(t, ast.Name) and t.id == "pytestmark" for t in node.targets):+ continue+ for element in ast.walk(node.value):+ if isinstance(element, ast.Attribute) and isinstance(element.value, ast.Attribute) and element.value.attr == "mark":+ found.add(element.attr)+ return frozenset(found)+++def deselects_module(expression: str, markers: frozenset[str]) -> bool:+ """Whether ``-m expression`` deselects every test in a module carrying ``markers``.++ Evaluated from the expression's syntax tree, not by matching text: `not+ integration` and `integration` differ by three characters and invert the+ answer. An expression shaped in a way this cannot evaluate returns False —+ the gate then reports the file as reached, which is the reading that can+ only ever produce a *missed* finding rather than a false one, and the+ unparsed expression is surfaced so it does not stay unnoticed.+ """+ if not expression.strip():+ return False+ try:+ node = ast.parse(expression, mode="eval").body+ except SyntaxError:+ return False++ def value(sub: ast.AST) -> bool | None:+ if isinstance(sub, ast.Name):+ return sub.id in markers+ if isinstance(sub, ast.UnaryOp) and isinstance(sub.op, ast.Not):+ inner = value(sub.operand)+ return None if inner is None else not inner+ if isinstance(sub, ast.BoolOp):+ parts = [value(v) for v in sub.values]+ if any(p is None for p in parts):+ return None+ return all(parts) if isinstance(sub.op, ast.And) else any(parts)+ return None++ selected = value(node)+ return selected is False+++def _resolve(token: str, base: Path, root: Path) -> Path | None:+ """A path argument resolved against the step's working directory."""+ token = token.split("::")[0].strip("'\"")+ if not token or token.startswith("$"):+ return None+ target = (base / token).resolve()+ try:+ target.relative_to(root)+ except ValueError:+ return None+ return target+++# --------------------------------------------------------------------------+# Test trees+# --------------------------------------------------------------------------+@dataclass+class Tree:+ """A directory of test files owned by one part of the repository."""++ name: str+ tests_dir: str+ files: list[str] = field(default_factory=list)+ reached: set[str] = field(default_factory=set)+ invocations: list[str] = field(default_factory=list)++ @property+ def unreached(self) -> list[str]:+ return sorted(set(self.files) - self.reached)+++def python_files_patterns(tree_root: Path) -> tuple[str, ...]:+ """``python_files`` for this tree, from its own config if it sets one."""+ for name, key in (("pyproject.toml", "[tool.pytest.ini_options]"), ("pytest.ini", "[pytest]"), ("setup.cfg", "[tool:pytest]")):+ config = tree_root / name+ if not config.is_file():+ continue+ text = config.read_text(encoding="utf-8", errors="replace")+ if key not in text:+ continue+ section = text.split(key, 1)[1]+ match = re.search(r"^\s*python_files\s*=\s*(.+)$", section, re.MULTILINE)+ if match:+ raw = match.group(1).strip()+ patterns = tuple(p.strip().strip("\"'") for p in re.split(r"[,\s]+", raw.strip("[]")) if p.strip().strip("\"'"))+ if patterns:+ return patterns+ return DEFAULT_PYTHON_FILES+++#: The pseudo-tree for a test-shaped file that sits in no ``tests/`` directory.+#: Such a file is owned by nothing, so scoping the scan to ``tests/`` alone+#: would make it invisible — the precise shape this gate exists to remove, one+#: level up. It is reached when a workflow executes it directly, which is how+#: ``scripts/integration/spine_test.py`` is run.+LOOSE = "(no tests/ directory)"++_TEST_DIR_NAMES = frozenset({"tests", "test"})+++def tracked_files(root: Path) -> list[str]:+ """Every file git tracks, repository-relative.++ ``git ls-files`` rather than a filesystem walk. Three of the four+ directories a walk has to be told to skip — ``node_modules``, ``.venv``,+ ``site-packages`` — are untracked, so asking git removes the list instead+ of maintaining it, and the answer no longer depends on whether anyone has+ run an install in this checkout.+ """+ import subprocess # noqa: PLC0415 - one call, kept next to its only use++ out = subprocess.run( # noqa: S603 - fixed argv, no shell+ ["git", "ls-files", "-z"], cwd=root, capture_output=True, text=True, check=False+ )+ if out.returncode != 0:+ raise DiscoveryError(f"cannot list tracked files under {root}: {out.stderr.strip()}")+ files = [name for name in out.stdout.split("\0") if name]+ if not files:+ raise DiscoveryError(f"git ls-files returned nothing under {root} — there is no tree here to scan")+ return files+++def _owning_tests_dir(rel: str) -> str | None:+ """The outermost ``tests/`` directory this path sits inside, if any."""+ parts = Path(rel).parts+ for index, part in enumerate(parts[:-1]):+ if part in _TEST_DIR_NAMES:+ return Path(*parts[: index + 1]).as_posix()+ return None+++def discover_trees(root: Path) -> list[Tree]:+ """Every test file git tracks, grouped by the part of the tree that owns it.++ A tree is a ``tests/`` directory plus the service or package it sits in.+ Grouping is derived from the paths, not from a list of services: one added+ tomorrow is scanned with no edit here, which is the whole point.++ Which filename patterns count is read from each owner's own pytest+ configuration when it sets ``python_files``, because a tree that renames+ the pattern renames what "a test file" means for that tree — scanning for+ the default there would look for files that cannot exist and credit the+ tree with nothing to find.+ """+ by_tree: dict[tuple[str, str], list[str]] = {}+ for rel in tracked_files(root):+ tests_dir = _owning_tests_dir(rel)+ owner = Path(tests_dir).parent.as_posix() if tests_dir else Path(rel).parent.as_posix()+ owner = owner if owner not in {"", "."} else "."+ patterns = python_files_patterns(root / owner if owner != "." else root)+ name = Path(rel).name+ if not any(Path(name).match(pattern) for pattern in patterns):+ continue+ key = (owner, tests_dir) if tests_dir else (LOOSE, LOOSE)+ by_tree.setdefault(key, []).append(rel)++ trees = [Tree(name=owner, tests_dir=tests_dir, files=sorted(files)) for (owner, tests_dir), files in sorted(by_tree.items())]+ if not trees:+ raise DiscoveryError(f"found no tracked Python test files under {root} — the patterns are wrong, or the tree is not there")+ return trees+++# --------------------------------------------------------------------------+# conftest collect_ignore+# --------------------------------------------------------------------------+def collect_ignored(tests_dir: Path, root: Path) -> set[str]:+ """Paths a ``conftest.py`` removes from collection, read from its syntax tree.++ A file pytest is told not to collect is not reached, however an invocation+ names it — so a gate that ignored this would credit a file that never runs.+ """+ out: set[str] = set()+ # Both sides resolved, or a symlinked temporary directory (/var -> /private/var+ # on macOS) makes the relative_to below raise on paths that are plainly inside.+ root = root.resolve()+ for conftest in tests_dir.resolve().rglob("conftest.py"):+ try:+ tree = ast.parse(conftest.read_text(encoding="utf-8", errors="replace"))+ except SyntaxError:+ continue+ for node in ast.walk(tree):+ if not isinstance(node, ast.Assign):+ continue+ names = {t.id for t in node.targets if isinstance(t, ast.Name)}+ if not names & {"collect_ignore", "collect_ignore_glob"}:+ continue+ for element in ast.walk(node.value):+ if isinstance(element, ast.Constant) and isinstance(element.value, str):+ for match in conftest.parent.glob(element.value):+ if match.is_file():+ out.add(match.resolve().relative_to(root).as_posix())+ return out+++# --------------------------------------------------------------------------+# The scan+# --------------------------------------------------------------------------+def scan(root: Path) -> tuple[list[Tree], list[Invocation], list[tuple[str, str, str]]]:+ """(trees, invocations, paths named by an invocation that are not in the tree)."""+ surface = coverage.Surface(root)+ invocations = parse_invocations(surface)+ if not invocations:+ raise DiscoveryError(+ f"parsed zero pytest invocations from the workflows under {root} — the parser is wrong, or the tree is not there"+ )++ trees = discover_trees(root)+ by_file: dict[str, Tree] = {f: tree for tree in trees for f in tree.files}+ missing: list[tuple[str, str, str]] = []++ for inv in invocations:+ base = (root / inv.working_dir).resolve() if inv.working_dir else root+ if inv.bare:+ targets = [base]+ else:+ targets = []+ for token in inv.paths:+ resolved = _resolve(token, base, root)+ if resolved is None:+ continue+ if not resolved.exists():+ missing.append((inv.workflow, inv.command, resolved.relative_to(root).as_posix()))+ continue+ targets.append(resolved)++ ignored = {r for token in inv.ignores if (r := _resolve(token, base, root)) is not None}+ label = f"{inv.workflow}: {_abbreviate(inv.command)}"++ for target in targets:+ for rel, tree in by_file.items():+ path = root / rel+ covered = path == target or target in path.parents+ if not covered:+ continue+ if any(path == ig or ig in path.parents for ig in ignored):+ continue+ if inv.markers and deselects_module(inv.markers, module_markers(path)):+ continue+ tree.reached.add(rel)+ if label not in tree.invocations:+ tree.invocations.append(label)++ # A file in no tests/ directory is never collected by a directory argument,+ # so the only way it runs is a workflow executing it outright. That is how+ # scripts/integration/spine_test.py runs, and treating it as unreachable+ # would be this gate reporting a gap it had invented.+ _credit_directly_executed(surface, by_file, root)++ for tree in trees:+ if tree.tests_dir == LOOSE:+ continue+ for rel in collect_ignored(root / tree.tests_dir, root):+ tree.reached.discard(rel)++ return trees, invocations, missing+++def _credit_directly_executed(surface: coverage.Surface, by_file: dict[str, Tree], root: Path) -> None:+ """Mark a loose test file reached when a workflow runs it as a program.++ The path has to resolve, from the step's own directory, to exactly that+ file. Matching a bare filename instead would credit any step that happened+ to mention the name — the shape that counted eleven services as covered on+ the strength of a quoted path inside an ``echo``.+ """+ loose = {rel: tree for rel, tree in by_file.items() if tree.tests_dir == LOOSE}+ if not loose:+ return+ for workflow, working_dir, text in surface.texts():+ cwd = working_dir+ for segment in _segments(text):+ moved = _cd_target(segment, cwd)+ if moved is not None:+ cwd = moved+ continue+ try:+ tokens = shlex.split(segment, comments=True)+ except ValueError:+ continue+ args = _args_after_interpreter(tokens)+ if not args:+ continue+ base = (root / cwd).resolve() if cwd else root+ resolved = _resolve(args[0], base, root)+ if resolved is None:+ continue+ rel = resolved.relative_to(root).as_posix()+ tree = loose.get(rel)+ if tree is None:+ continue+ tree.reached.add(rel)+ label = f"{workflow}: {_abbreviate(segment)}"+ if label not in tree.invocations:+ tree.invocations.append(label)+++def _args_after_interpreter(tokens: list[str]) -> list[str] | None:+ """Arguments to ``python <script>``, or None if this is not that shape."""+ index = 0+ while index < len(tokens) and re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*=.*", tokens[index]):+ index += 1+ if index >= len(tokens) or not re.fullmatch(r"python3?(?:\.\d+)?", tokens[index]):+ return None+ index += 1+ # `python -m pkg` runs a module, and `python -c '...'` runs a string.+ if index >= len(tokens) or tokens[index].startswith("-"):+ return None+ return tokens[index:]+++def _abbreviate(command: str, limit: int = 90) -> str:+ flat = " ".join(command.split())+ return flat if len(flat) <= limit else flat[: limit - 1] + "…"+++# --------------------------------------------------------------------------+# Verdict+# --------------------------------------------------------------------------+def evaluate(+ trees: Iterable[Tree],+ missing: Iterable[tuple[str, str, str]],+ *,+ quarantine: dict[str, str],+) -> list[tuple[str, str]]:+ """Every finding, as (code, detail). Pure, so the self-test can drive it."""+ findings: list[tuple[str, str]] = []+ trees = list(trees)+ all_files = {f for tree in trees for f in tree.files}++ for tree in trees:+ unreached = [f for f in tree.unreached if f not in quarantine]+ if unreached:+ findings.append(+ (+ "unreached",+ f"{tree.name}: {len(unreached)} of {len(tree.files)} test file(s) are collected by no workflow invocation. "+ f"A file nothing runs reports nothing and reads exactly like a file that passed. "+ f"Run the directory rather than a list of files, or quarantine it with a stated reason.\n"+ + "\n".join(f" {f}" for f in unreached),+ )+ )++ for workflow, command, path in missing:+ findings.append(+ (+ "names-nothing",+ f"{workflow} runs `{_abbreviate(command)}` which names {path}, and that path is not in the tree. "+ "A step listing a path that is not there no longer does what it reads as doing.",+ )+ )++ for path, reason in sorted(quarantine.items()):+ if path not in all_files:+ findings.append(("quarantine-names-nothing", f"QUARANTINE lists {path!r}, which is not a test file in the tree"))+ elif not reason.strip():+ findings.append(("quarantine-unexplained", f"QUARANTINE[{path!r}] carries no reason"))+ reached_all = {f for tree in trees for f in tree.reached}+ for path in sorted(set(quarantine) & reached_all):+ findings.append(+ (+ "quarantine-stale",+ f"{path} is quarantined and is now collected by a workflow. Remove the entry — a quarantine list that "+ "names running tests stops describing anything.",+ )+ )++ return findings+++# --------------------------------------------------------------------------+# CLI+# --------------------------------------------------------------------------+def main(argv: list[str] | None = None) -> int:+ parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)+ parser.add_argument("--repo-root", type=Path, default=None, help="the tree to inspect (default: git rev-parse)")+ parser.add_argument("--list", action="store_true", help="print every tree with its coverage")+ parser.add_argument("--json", action="store_true")+ parser.add_argument("--check", action="store_true", help="accepted for symmetry; this script is always a gate")+ parser.add_argument("--self-test", action="store_true", help="prove this gate detects the drift it claims to")+ args = parser.parse_args(argv)++ if args.self_test:+ return self_test()++ root = (args.repo_root or repo_root()).resolve()+ try:+ trees, invocations, missing = scan(root)+ except (DiscoveryError, coverage.GateError, FileNotFoundError) as exc:+ print(f"check_test_discovery: FAILED to run the scan: {exc}", file=sys.stderr)+ return 2++ findings = evaluate(trees, missing, quarantine=QUARANTINE)++ total = sum(len(t.files) for t in trees)+ covered = sum(len(t.reached) for t in trees)++ if args.json:+ print(+ json.dumps(+ {+ "root": str(root),+ "trees": [asdict(t) | {"reached": sorted(t.reached), "unreached": t.unreached} for t in trees],+ "invocations": [asdict(i) for i in invocations],+ "missing_paths": [{"workflow": w, "command": c, "path": p} for w, c, p in missing],+ "quarantine": QUARANTINE,+ "findings": [{"code": c, "detail": d} for c, d in findings],+ },+ indent=2,+ sort_keys=True,+ )+ )+ return 1 if findings else 0++ print(f"repo root {root}")+ print(f"workflows {len(invocations)} pytest invocation(s) across .github/workflows/")+ print(f"test trees {len(trees)} ({total} test file(s), {covered} reached, {total - covered} not)")+ print(f"quarantine {len(QUARANTINE)} recorded, each with a reason")+ print()++ if args.list:+ width = max((len(t.name) for t in trees), default=10)+ for tree in sorted(trees, key=lambda t: t.name):+ gap = len(tree.files) - len(tree.reached)+ mark = "OK " if gap == 0 else "GAP "+ print(f" {mark} {tree.name:{width}} {len(tree.reached):>3}/{len(tree.files):<3} reached")+ for label in tree.invocations:+ print(f" {'':{width}} via {label}")+ if not tree.invocations:+ print(f" {'':{width}} via nothing")+ for path in tree.unreached:+ note = f" [quarantined: {QUARANTINE[path]}]" if path in QUARANTINE else ""+ print(f" {'':{width}} NOT REACHED {path}{note}")+ print()++ if findings:+ print(f"FAIL — {len(findings)} finding(s):")+ for code, detail in findings:+ print(f" [{code}] {detail}")+ return 1+ print(f"OK — every one of {total} test file(s) across {len(trees)} tree(s) is collected by an invocation CI runs,")+ print(f" and every path those invocations name is in the tree. {len(QUARANTINE)} quarantined, each with a reason.")+ return 0+++# --------------------------------------------------------------------------+# Self-test+# --------------------------------------------------------------------------+def self_test() -> int:+ """Inject each defect this gate exists to catch and require it to be caught."""++ def tree(files: list[str], reached: list[str]) -> Tree:+ t = Tree(name="services/x", tests_dir="services/x/tests", files=files)+ t.reached = set(reached)+ return t++ one = ["services/x/tests/test_a.py"]+ clean = [tree(one, one)]++ def run(trees, missing=(), *, quarantine=None):+ return {code for code, _ in evaluate(trees, missing, quarantine=quarantine or {})}++ cases: list[tuple[str, str, set[str]]] = [+ (+ "a test file in a tree that no invocation reaches",+ "unreached",+ run([tree([*one, "services/x/tests/test_b.py"], one)]),+ ),+ (+ "an invocation naming a path that is not in the tree",+ "names-nothing",+ run(clean, [("ci.yml", "pytest tests/test_gone.py", "services/x/tests/test_gone.py")]),+ ),+ (+ "a quarantine entry naming a file that does not exist",+ "quarantine-names-nothing",+ run(clean, quarantine={"services/x/tests/test_ghost.py": "reason"}),+ ),+ (+ "a quarantine entry with no reason given",+ "quarantine-unexplained",+ run([tree([*one, "services/x/tests/test_b.py"], one)], quarantine={"services/x/tests/test_b.py": " "}),+ ),+ (+ "a quarantined file that CI has started running",+ "quarantine-stale",+ run(clean, quarantine={"services/x/tests/test_a.py": "reason"}),+ ),+ ]++ print("check_test_discovery self-test")+ print("baseline: one tree, one file, reached; 0 findings (every case below perturbs exactly that)\n")+ ok = not run(clean)+ print(f" {'PASS' if ok else 'FAIL'} the unperturbed baseline reports nothing")++ for description, expected, codes in cases:+ caught = expected in codes+ ok &= caught+ print(f" {'PASS' if caught else 'FAIL'} {description}")+ print(f" expected [{expected}] got {sorted(codes) or 'nothing'}")++ # The argument reader. Crediting a file an invocation never named is the+ # failure that matters here, so the parser is asserted directly rather+ # than inferred from the findings above.+ reader: list[tuple[str, object, object]] = [+ ("a directory argument is a path", _split_args(["tests/", "-q"]), (["tests/"], [], "")),+ ("--ignore= subtracts", _split_args(["tests/", "--ignore=tests/slow"]), (["tests/"], ["tests/slow"], "")),+ ("--ignore with a space subtracts", _split_args(["tests/", "--ignore", "tests/slow"]), (["tests/"], ["tests/slow"], "")),+ ("-k takes a value that is not a path", _split_args(["tests/", "-k", "not_slow"]), (["tests/"], [], "")),+ ("-m is captured, not discarded", _split_args(["tests/", "-m", "not integration"]), (["tests/"], [], "not integration")),+ ("--cov=app is not a path", _split_args(["tests/", "--cov=app"]), (["tests/"], [], "")),+ ("a node id resolves to its file", _split_args(["tests/test_a.py::test_one"]), (["tests/test_a.py::test_one"], [], "")),+ ]+ for label, got, want in reader:+ correct = got == want+ ok &= correct+ print(f" {'PASS' if correct else 'FAIL'} READER: {label}")+ print(f" expected {want} got {got}")++ chooser: list[tuple[str, object, object]] = [+ ("a plain invocation yields its arguments", _args_after_pytest(shlex.split("pytest tests/ -q")), ["tests/", "-q"]),+ ("python -m pytest reaches the same place", _args_after_pytest(shlex.split("python3 -m pytest tests/")), ["tests/"]),+ ("an env prefix is stepped over", _args_after_pytest(shlex.split("PYTHONPATH=. python -m pytest tests/")), ["tests/"]),+ ("poetry run reaches the same place", _args_after_pytest(shlex.split("poetry run pytest tests/")), ["tests/"]),+ ("a pip install line is not an invocation", _args_after_pytest(shlex.split("pip install pytest pytest-asyncio")), None),+ ("another program is not an invocation", _args_after_pytest(shlex.split("ruff check services/")), None),+ ]+ for label, got, want in chooser:+ correct = got == want+ ok &= correct+ print(f" {'PASS' if correct else 'FAIL'} CHOOSER: {label}")+ print(f" expected {want} got {got}")++ # Marker deselection. `not integration` and `integration` differ by three+ # characters and invert the answer, so both directions are asserted —+ # getting this backwards would credit exactly the files nothing runs.+ integration = frozenset({"integration"})+ marks: list[tuple[str, bool, bool]] = [+ ("`not integration` deselects an integration module", deselects_module("not integration", integration), True),+ ("`not integration` keeps an unmarked module", deselects_module("not integration", frozenset()), False),+ ("`integration` keeps an integration module", deselects_module("integration", integration), False),+ ("`integration` deselects an unmarked module", deselects_module("integration", frozenset()), True),+ ("`not a and not b` deselects a module carrying b", deselects_module("not a and not b", frozenset({"b"})), True),+ ("`not a and not b` keeps a module carrying neither", deselects_module("not a and not b", frozenset({"c"})), False),+ ("an empty expression deselects nothing", deselects_module("", integration), False),+ ("an expression that will not parse deselects nothing", deselects_module("not and or", integration), False),+ ]+ for label, got_mark, want_mark in marks:+ correct = got_mark == want_mark+ ok &= correct+ print(f" {'PASS' if correct else 'FAIL'} MARKERS: {label}")+ print(f" expected {want_mark} got {got_mark}")++ # And the corpus itself, against a real git repository — every count above+ # is only as good as the claim that the files were found in the first+ # place, and this is the half of the gate that decides what "a test file"+ # is. An untracked file must not be counted: the whole reason the corpus+ # comes from git is that an install directory is not this tree's tests.+ import subprocess # noqa: PLC0415 - only the self-test builds a scratch repository+ import tempfile # noqa: PLC0415++ with tempfile.TemporaryDirectory(prefix="aisoc-discovery-") as tmp:+ sandbox = Path(tmp)+ for rel, body in (+ ("services/x/tests/test_a.py", ""),+ ("services/x/tests/b_test.py", ""),+ ("services/x/tests/helper.py", ""),+ ("scripts/integration/spine_test.py", ""),+ ):+ (sandbox / rel).parent.mkdir(parents=True, exist_ok=True)+ (sandbox / rel).write_text(body)+ # Tracked by nothing, so the corpus must not see it.+ (sandbox / "node_modules/pkg/tests").mkdir(parents=True)+ (sandbox / "node_modules/pkg/tests/test_vendor.py").write_text("")+ git = ["git", "-c", "user.name=gate", "-c", "user.email=gate@invalid", "-c", "commit.gpgsign=false"]+ for argv in (["init", "-q", "-b", "main"], ["add", "services", "scripts"], ["commit", "-qm", "scratch"]):+ subprocess.run(git + argv, cwd=sandbox, check=True, capture_output=True) # noqa: S603++ (sandbox / "services/x/tests/test_a.py").write_text("import pytest\npytestmark = pytest.mark.integration\n")+ read_marks = module_markers(sandbox / "services/x/tests/test_a.py")+ correct = read_marks == frozenset({"integration"})+ ok &= correct+ print(f" {'PASS' if correct else 'FAIL'} MARKERS: a module-level pytestmark is read off the syntax tree")+ print(f" expected {{'integration'}} got {set(read_marks) or 'nothing'}")+ (sandbox / "services/x/tests/test_a.py").write_text("")++ walked = discover_trees(sandbox)+ names = sorted(f for t in walked for f in t.files)+ want_names = ["scripts/integration/spine_test.py", "services/x/tests/b_test.py", "services/x/tests/test_a.py"]+ correct = names == want_names+ ok &= correct+ print(f" {'PASS' if correct else 'FAIL'} CORPUS: both default patterns count; a helper and an untracked tree do not")+ print(f" expected {want_names} got {names}")++ loose = sorted(t.name for t in walked if t.tests_dir == LOOSE)+ correct = loose == [LOOSE]+ ok &= correct+ print(f" {'PASS' if correct else 'FAIL'} CORPUS: a test-shaped file in no tests/ directory is its own tree, not invisible")+ print(f" expected [{LOOSE!r}] got {loose}")++ # conftest collect_ignore, which is the one way a named file is still+ # not reached — a gate blind to it would credit a file that never runs.+ (sandbox / "services/x/tests/conftest.py").write_text('collect_ignore = ["test_a.py"]\n')+ ignored = collect_ignored(sandbox / "services/x/tests", sandbox)+ correct = ignored == {"services/x/tests/test_a.py"}+ ok &= correct+ print(f" {'PASS' if correct else 'FAIL'} CORPUS: conftest collect_ignore removes a file from reach")+ print(f" expected {{'services/x/tests/test_a.py'}} got {ignored or 'nothing'}")++ print()+ if not ok:+ print("self-test FAILED: this gate did not catch drift it claims to catch")+ return 1+ print(f"self-test OK: {len(cases)} injected defects and {len(reader) + len(chooser) + len(marks) + 5} component assertions")++ # And the property the meta-gate exists for: this gate must refuse a tree+ # with no content rather than reporting it clean.+ return self_test_main(Path(__file__).name, [])+++if __name__ == "__main__":+ sys.exit(main())diff --git a/scripts/run_evals.py b/scripts/run_evals.pyindex ba7b922ab..13992e98a 100644--- a/scripts/run_evals.py+++ b/scripts/run_evals.py@@ -936,13 +936,20 @@ def main() -> None:# Substrate suites need pydantic / langchain etc. If they're not# installed we can still emit the telemetry block — surface the# original ImportError so operators know what to fix.+ #+ # Exit 3, which is what this file's own "Exit codes" block has always+ # said a substrate-import failure returns. It exited 2, and 2 means+ # "MITRE accuracy regressed against the baseline" — so a fresh clone+ # missing a dependency reported an accuracy regression, and the one+ # actionable instruction (install the deps) was not in the message.msg = (- "Substrate-suite imports failed (likely missing agent dev deps "- f"such as pydantic): {_SUBSTRATE_IMPORT_ERROR!r}. "- "Pass --telemetry-only to emit just the T2.4 token/USD/latency block."+ "ERROR: substrate-suite imports failed (likely missing agent dev deps "+ f"such as pydantic): {_SUBSTRATE_IMPORT_ERROR!r}.\n"+ " Fix: pip install -e services/agents\n"+ " Or: pass --telemetry-only to emit just the T2.4 token/USD/latency block.")print(msg, file=sys.stderr)- sys.exit(2)+ sys.exit(3)keep_records = not args.no_telemetry_recordsper_investigation = _build_per_investigation_block(@@ -959,22 +966,40 @@ def main() -> None:}summary["all_passed"] = True # telemetry-only never gates substrateelse:+ # Only the requested suites are *called*. The previous form built this+ # dict with all eleven calls inline, so `--suite mitre_accuracy` ran+ # every suite, took the full runtime, and then printed+ # "PASS — mitre_accuracy green" — a verdict naming one suite and+ # decided by eleven. `--ci` would fail a single-suite run on an+ # unrelated regression, which is the opposite of what an operator+ # bisecting to one gate is asking for. `args.suite` reached nothing+ # but the banner wording.+ runners = {+ "mitre_accuracy": _run_mitre,+ "alert_reduction": _run_alert_reduction,+ "investigation_completeness": _run_completeness,+ "response_quality": _run_response_quality,+ "hunt_corpus": _run_hunt_corpus,+ "adversary_eval": _run_adversary,+ "confidence_calibration": _run_confidence_calibration,+ "memory_recall": _run_memory_recall,+ "override_accuracy": _run_override_accuracy,+ "playbook_completion_rate": _run_playbook_completion,+ "detection_fp_rate": _run_detection_fp_rate,+ }+ # The registry and the name list are the same set, asserted rather than+ # assumed: argparse accepts a name from `_SUITE_NAMES`, so one missing+ # here would be a KeyError at the moment somebody bisects a regression.+ if set(runners) != set(_SUITE_NAMES):+ raise AssertionError(f"suite registry and _SUITE_NAMES disagree: {set(runners) ^ set(_SUITE_NAMES)}")+ selected = _SUITE_NAMES if args.suite == "all" else (args.suite,)summary = {"generated_at": datetime.now(UTC).isoformat(),"dataset": "synthetic_incidents.json (200 cases, deterministic)",- "suites": {- "mitre_accuracy": _run_mitre(),- "alert_reduction": _run_alert_reduction(),- "investigation_completeness": _run_completeness(),- "response_quality": _run_response_quality(),- "hunt_corpus": _run_hunt_corpus(),- "adversary_eval": _run_adversary(),- "confidence_calibration": _run_confidence_calibration(),- "memory_recall": _run_memory_recall(),- "override_accuracy": _run_override_accuracy(),- "playbook_completion_rate": _run_playbook_completion(),- "detection_fp_rate": _run_detection_fp_rate(),- },+ # Which suites this report describes. Without it a single-suite+ # report is indistinguishable from a full one that lost ten suites.+ "suite_filter": args.suite,+ "suites": {name: runners[name]() for name in selected},"telemetry": _summarise_telemetry(),"per_investigation": per_investigation,}diff --git a/services/agents/app/playbook/action_bridge.py b/services/agents/app/playbook/action_bridge.pyindex 1f45134a9..079d5625e 100644--- a/services/agents/app/playbook/action_bridge.py+++ b/services/agents/app/playbook/action_bridge.py@@ -33,6 +33,14 @@ from typing import Anyimport httpx+from .errors import PermanentStepFailure++#: HTTP statuses in the 4xx range that ask to be retried rather than refusing.+#: Everything else a server says in that range means it understood the request+#: and will not serve it, so a second identical request gets a second identical+#: answer.+_RETRYABLE_CLIENT_STATUSES = frozenset({408, 425, 429})+#: Stdlib logging rather than structlog, matching `engine.py`. The playbook#: package states "zero external dependencies beyond httpx + stdlib" in its#: own docstring and it is load-bearing: `scripts/validate_playbooks.py` and@@ -77,6 +85,24 @@ class BridgeUnavailable(RuntimeError):Raised rather than returned so a caller cannot forget to check: theengine's handler turns it into a failed step, and a failed step halts therun under the default policy.++ Still the base class every caller catches, so nothing downstream had to+ learn a second name. What changed is that the permanent half of it is now+ a subclass the engine can recognise — see ``BridgeMisconfigured``.+ """+++class BridgeMisconfigured(BridgeUnavailable, PermanentStepFailure):+ """Dispatch is impossible for this run, and waiting will not help.++ Three of these are process configuration — the enable switch, the service+ token, the tenant on the run context — and none of them changes between a+ step's first attempt and its fourth. The other two are the API answering+ in a way that breaks its own contract, which it will do again.++ Retrying these burned fourteen seconds per step and, worse, made a+ permanent misconfiguration read as an intermittent network problem during+ an incident."""@@ -99,18 +125,21 @@ async def dispatch_step(integration configured" and "previewed", which are the three answers anauthor most needs."""+ # The three configuration refusals below are read from the process+ # environment and the run context, neither of which changes while a step+ # is sleeping between attempts.if not actions_enabled():- raise BridgeUnavailable(+ raise BridgeMisconfigured(f"playbook action dispatch is disabled (AISOC_PLAYBOOK_ACTIONS_ENABLED=0), so '{capability}' was not attempted")if not tenant_id:# The API refuses a service call with no tenant, and guessing one# here would be a cross-tenant action.- raise BridgeUnavailable(f"'{capability}' has no tenant in the run context, and a response action cannot be run without one")+ raise BridgeMisconfigured(f"'{capability}' has no tenant in the run context, and a response action cannot be run without one")token = _service_token()if not token:- raise BridgeUnavailable(+ raise BridgeMisconfigured(f"'{capability}' was not dispatched: AISOC_AGENTS_SERVICE_TOKEN is unset, so the API's service path is closed")@@ -132,17 +161,33 @@ async def dispatch_step(async with httpx.AsyncClient(timeout=_TIMEOUT_S) as client:response = await client.post(url, json=payload, headers={"X-AiSOC-Service-Token": token})except httpx.HTTPError as exc:+ # Connection refused, DNS failure, read timeout. The request did not+ # arrive; the next one might. This is the case the retry loop is for.raise BridgeUnavailable(f"the action service could not be reached for '{capability}': {exc}") from excif response.status_code >= 400:- raise BridgeUnavailable(f"the API refused '{capability}' with HTTP {response.status_code}")+ # 5xx is the server failing to serve a request it accepted, and 408 /+ # 425 / 429 are a server explicitly asking to be asked again. Every+ # other 4xx means it understood and refused — authentication, an+ # unknown capability, a malformed payload — and refusing again is the+ # only thing a second identical request can achieve.+ detail = f"the API refused '{capability}' with HTTP {response.status_code}"+ if response.status_code >= 500 or response.status_code in _RETRYABLE_CLIENT_STATUSES:+ raise BridgeUnavailable(detail)+ raise BridgeMisconfigured(detail)try:report = response.json()except ValueError as exc:+ # Transient. A JSON endpoint that answers 2xx with a non-JSON body is+ # almost always an intermediary — an ingress or proxy error page+ # served in place of the API — and those clear. Below is the opposite+ # case and is treated as such: a body that parsed as JSON came from+ # something speaking the API's own contract, so getting the contract+ # wrong is a defect, not weather.raise BridgeUnavailable(f"the API returned a non-JSON body for '{capability}'") from excif not isinstance(report, dict) or "executed" not in report:- raise BridgeUnavailable(f"the API returned a report with no 'executed' field for '{capability}'")+ raise BridgeMisconfigured(f"the API returned a report with no 'executed' field for '{capability}'")logger.info(# `executed` is echoed verbatim: if it is False no vendor wasdiff --git a/services/agents/app/playbook/engine.py b/services/agents/app/playbook/engine.pyindex 638ac83b8..85ad35802 100644--- a/services/agents/app/playbook/engine.py+++ b/services/agents/app/playbook/engine.py@@ -25,6 +25,7 @@ import httpxfrom . import action_bridgefrom .bounds import clamp_timeout+from .errors import PermanentStepFailurefrom .models import Playbook, PlaybookStep, StepCondition, StepTypefrom .ssrf_guard import SSRFError, validate_outbound_url@@ -811,12 +812,33 @@ class PlaybookEngine:break # the handler answered; status is set aboveexcept Exception as exc: # noqa: BLE001elapsed = time.perf_counter() - t0- logger.error("Step %s attempt %d failed: %s", step.name, attempt, exc)- if attempt <= step.retry_max:+ # A permanent failure will not become a different+ # failure by being asked again. Sleeping 2s, 4s+ # then 8s before repeating "this run has no+ # tenant" costs an operator fourteen seconds of an+ # incident and, worse, dresses a misconfiguration+ # up as flakiness — so they wait for it to settle+ # instead of going and fixing it.+ permanent = isinstance(exc, PermanentStepFailure)+ logger.error(+ "Step %s attempt %d failed (%s): %s",+ step.name,+ attempt,+ "permanent, not retried" if permanent else "retryable",+ exc,+ )+ if not permanent and attempt <= step.retry_max:await asyncio.sleep(min(2**attempt, 30))else:step_status = StepStatus.FAILED- result = {"error": str(exc), "_elapsed_ms": round(elapsed * 1000)}+ result = {+ "error": str(exc),+ # Says, in the run record, why there was+ # one attempt and not four.+ "permanent": permanent,+ "attempts": attempt,+ "_elapsed_ms": round(elapsed * 1000),+ }breakpr.step_results.append({"step_id": step.id, "name": step.name, "status": step_status, "result": result})diff --git a/services/agents/app/playbook/errors.py b/services/agents/app/playbook/errors.pynew file mode 100644index 000000000..1a6c93e99--- /dev/null+++ b/services/agents/app/playbook/errors.py@@ -0,0 +1,41 @@+"""Failures a second attempt cannot change.++The engine retries a failed step with exponential backoff up to+``step.retry_max``. That is right for a bridge that was momentarily+unreachable and wrong for a run whose context has no tenant: no amount of+waiting adds one, so the run spends fourteen seconds sleeping before failing+with the message it already had on the first attempt.++The cost is not the fourteen seconds. It is that a permanent misconfiguration+presents as flakiness — an operator watching a containment playbook during an+incident sees a step "retrying", concludes the network is unhappy, and waits.+The two cases need to look different because the operator's next move is+different: one is "wait", the other is "go and set the variable".++So the distinction is carried by the exception type rather than inferred from+the message. Any handler can raise something marked with+``PermanentStepFailure`` and the engine will fail the step on the first+attempt; nothing else about the failure path changes, and a handler that does+not care keeps the retry it has today.++The line this package draws+---------------------------+**Permanent** — the cause is this deployment's configuration or a violation of+the API's own contract. A second identical request gets a second identical+answer.++**Transient** — the cause is reachability. The request never arrived, or the+far side failed to serve it. A second attempt may well land.+"""++from __future__ import annotations+++class PermanentStepFailure(Exception):+ """Marker: retrying this step cannot change the outcome.++ Raised on its own or mixed into a handler's own exception type, so the+ engine can ask ``isinstance`` rather than parse a message. It is a real+ exception rather than a bare mixin so a handler with no hierarchy of its+ own can raise it directly.+ """diff --git a/services/agents/tests/test_playbook_action_bridge.py b/services/agents/tests/test_playbook_action_bridge.pyindex f851dbc25..02311ac19 100644--- a/services/agents/tests/test_playbook_action_bridge.py+++ b/services/agents/tests/test_playbook_action_bridge.py@@ -14,11 +14,16 @@ returns a dict is not a step that ran.from __future__ import annotations+import ast+import asyncio+import pathlibfrom typing import Any+import httpximport pytestfrom app.playbook import action_bridgefrom app.playbook.engine import RESPONSE_STEP_TYPES, PlaybookEngine, RunStatus, StepStatus+from app.playbook.errors import PermanentStepFailurefrom app.playbook.models import Playbook, PlaybookStep, StepTypepytestmark = pytest.mark.asyncio@@ -28,6 +33,39 @@ def _playbook(*steps: PlaybookStep) -> Playbook:return Playbook(id="pb-1", name="Containment", steps=list(steps))+def _install_response(+ monkeypatch: pytest.MonkeyPatch,+ *,+ status_code: int = 200,+ json_body: dict[str, Any] | None = None,+ text: str | None = None,+ raises: Exception | None = None,+) -> None:+ """Answer the bridge's one POST without a network.++ Patches the transport rather than ``dispatch_step`` itself, because the+ classification under test lives inside ``dispatch_step`` — stubbing the+ function would test the stub.+ """++ class _Response:+ def __init__(self) -> None:+ self.status_code = status_code+ self.text = text or ""++ def json(self) -> Any:+ if json_body is None:+ raise ValueError("not JSON")+ return json_body++ async def _post(self: Any, *args: Any, **kwargs: Any) -> _Response:+ if raises is not None:+ raise raises+ return _Response()++ monkeypatch.setattr(httpx.AsyncClient, "post", _post)++class _Recorder:"""Stands in for the API hop and records exactly what it was asked."""@@ -250,3 +288,168 @@ class TestDryRunPreview:assert result["would_dispatch"] == "isolate_host"assert result["target"] == "ws-042"assert result["executed"] is False+++class TestAPermanentFailureIsNotRetried:+ """A missing tenant will never arrive by waiting.++ The engine retries a failed step with exponential backoff, which is right+ for a bridge that was momentarily unreachable and wrong for a run whose+ context has no tenant: `dispatch_step` refuses on the first line, before+ any I/O, and the engine then slept 2s, 4s and 8s before failing with the+ message it already had. Fourteen seconds of an incident spent proving+ something that was decided immediately — and, worse, a permanent+ misconfiguration presented to the operator as flakiness, so the next move+ looks like "wait" when it is "go and set the variable".++ These assert the property directly by recording the backoff sleeps, the+ same way `test_a_missing_handler_is_not_retried` does. An earlier test in+ this tree asserted a rounded wall-clock field instead and could not tell+ the two paths apart.+ """++ @staticmethod+ def _record_backoff(monkeypatch: pytest.MonkeyPatch) -> list[float]:+ slept: list[float] = []+ real_sleep = asyncio.sleep++ async def _recording_sleep(delay: float, *args: Any, **kwargs: Any) -> Any:+ slept.append(delay)+ return await real_sleep(0, *args, **kwargs)++ monkeypatch.setattr(asyncio, "sleep", _recording_sleep)+ return slept++ async def test_a_run_with_no_tenant_fails_on_the_first_attempt(self, monkeypatch: pytest.MonkeyPatch) -> None:+ slept = self._record_backoff(monkeypatch)+ pb = _playbook(PlaybookStep(id="s1", name="block", type=StepType.BLOCK_IP, retry_max=3))++ run = await PlaybookEngine().run(pb, trigger_context={})++ result = run.step_results[0]["result"]+ assert run.step_results[0]["status"] == StepStatus.FAILED+ assert result["attempts"] == 1, f"a permanent failure was attempted {result['attempts']} times"+ assert result["permanent"] is True+ assert not slept, f"a permanent failure was retried with {slept} of backoff; retry_max was 3"+ assert "no tenant" in result["error"]++ async def test_an_unreachable_bridge_is_still_retried(self, bridge, monkeypatch: pytest.MonkeyPatch) -> None:+ """The other direction. Classifying everything permanent would remove+ the retry this loop exists for, which is the easy way to make the+ assertion above pass while making the product worse."""+ slept = self._record_backoff(monkeypatch)+ bridge(raises=action_bridge.BridgeUnavailable("the action service could not be reached for 'block_ip': timeout"))+ pb = _playbook(PlaybookStep(id="s1", name="block", type=StepType.BLOCK_IP, retry_max=3))++ run = await PlaybookEngine().run(pb, {"tenant_id": "t-1"})++ result = run.step_results[0]["result"]+ assert run.step_results[0]["status"] == StepStatus.FAILED+ assert result["permanent"] is False+ assert slept == [2, 4, 8], f"a transient failure should back off 2/4/8s, got {slept}"+ assert result["attempts"] == 4+++class TestWhichBridgeFailuresArePermanent:+ """The classification itself, at the point it is decided.++ Retrying a permanent failure wastes an operator's time; calling a+ transient one permanent throws away a recovery that would have worked. So+ both directions are pinned rather than only the one that motivated the+ change.+ """++ async def test_configuration_refusals_are_permanent(self, monkeypatch: pytest.MonkeyPatch) -> None:+ """Read from the process environment and the run context, neither of+ which changes while a step sleeps between attempts."""+ monkeypatch.setenv("AISOC_AGENTS_SERVICE_TOKEN", "tok")++ monkeypatch.setenv("AISOC_PLAYBOOK_ACTIONS_ENABLED", "0")+ with pytest.raises(action_bridge.BridgeMisconfigured, match="disabled"):+ await action_bridge.dispatch_step(capability="block_ip", tenant_id="t-1")++ monkeypatch.setenv("AISOC_PLAYBOOK_ACTIONS_ENABLED", "1")+ with pytest.raises(action_bridge.BridgeMisconfigured, match="no tenant"):+ await action_bridge.dispatch_step(capability="block_ip", tenant_id="")++ monkeypatch.setenv("AISOC_AGENTS_SERVICE_TOKEN", "")+ with pytest.raises(action_bridge.BridgeMisconfigured, match="AISOC_AGENTS_SERVICE_TOKEN"):+ await action_bridge.dispatch_step(capability="block_ip", tenant_id="t-1")++ @pytest.mark.parametrize(+ ("status", "permanent"),+ [+ (400, True),+ (401, True),+ (403, True),+ (404, True),+ (422, True),+ (408, False),+ (425, False),+ (429, False),+ (500, False),+ (502, False),+ (503, False),+ ],+ )+ async def test_a_refusal_is_permanent_and_an_outage_is_not(self, monkeypatch: pytest.MonkeyPatch, status: int, permanent: bool) -> None:+ """4xx means the API understood and declined, so asking again gets the+ same answer. 5xx is a server failing to serve a request it accepted,+ and 408/425/429 are a server explicitly asking to be asked again."""+ monkeypatch.setenv("AISOC_PLAYBOOK_ACTIONS_ENABLED", "1")+ monkeypatch.setenv("AISOC_AGENTS_SERVICE_TOKEN", "tok")+ _install_response(monkeypatch, status_code=status, json_body={"executed": False})++ with pytest.raises(action_bridge.BridgeUnavailable) as caught:+ await action_bridge.dispatch_step(capability="block_ip", tenant_id="t-1")++ assert isinstance(caught.value, PermanentStepFailure) is permanent++ async def test_a_contract_violation_is_permanent_but_an_unparseable_body_is_not(self, monkeypatch: pytest.MonkeyPatch) -> None:+ """A body that parsed as JSON came from something speaking the API's+ own contract, so getting the contract wrong is a defect and not+ weather. A body that did not parse at all is almost always an ingress+ or proxy error page served in place of the API, and those clear."""+ monkeypatch.setenv("AISOC_PLAYBOOK_ACTIONS_ENABLED", "1")+ monkeypatch.setenv("AISOC_AGENTS_SERVICE_TOKEN", "tok")++ _install_response(monkeypatch, status_code=200, json_body={"status": "ok"})+ with pytest.raises(action_bridge.BridgeMisconfigured, match="no 'executed' field"):+ await action_bridge.dispatch_step(capability="block_ip", tenant_id="t-1")++ _install_response(monkeypatch, status_code=200, text="<html>502 Bad Gateway</html>")+ with pytest.raises(action_bridge.BridgeUnavailable) as caught:+ await action_bridge.dispatch_step(capability="block_ip", tenant_id="t-1")+ assert not isinstance(caught.value, PermanentStepFailure)++ async def test_an_unreachable_host_is_transient(self, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv("AISOC_PLAYBOOK_ACTIONS_ENABLED", "1")+ monkeypatch.setenv("AISOC_AGENTS_SERVICE_TOKEN", "tok")+ _install_response(monkeypatch, raises=httpx.ConnectError("connection refused"))++ with pytest.raises(action_bridge.BridgeUnavailable) as caught:+ await action_bridge.dispatch_step(capability="block_ip", tenant_id="t-1")++ assert not isinstance(caught.value, PermanentStepFailure)+++async def test_every_bridge_refusal_is_classified_one_way_or_the_other() -> None:+ """Both directions, structurally: a raise site added tomorrow must choose.++ Reading the raises out of the syntax tree rather than listing them here+ means a sixth refusal cannot default into "retry it" because nobody+ remembered this file. `BridgeUnavailable` is still the base every caller+ catches, so the check is that a raise names one of the two concrete+ decisions and never the abstract base by accident.+ """+ source = pathlib.Path(action_bridge.__file__).read_text(encoding="utf-8")+ raised = {+ node.exc.func.id+ for node in ast.walk(ast.parse(source))+ if isinstance(node, ast.Raise) and isinstance(node.exc, ast.Call) and isinstance(node.exc.func, ast.Name)+ }++ assert raised, "no raise sites found — this test would pass over a file it could not read"+ assert raised <= {"BridgeUnavailable", "BridgeMisconfigured"}, f"unclassified bridge failure: {raised}"+ assert "BridgeMisconfigured" in raised, "the permanent half must still be reachable"+ assert issubclass(action_bridge.BridgeMisconfigured, action_bridge.BridgeUnavailable), "callers catch the base; it must stay the base"diff --git a/tests/test_gate_coverage_classifier.py b/tests/test_gate_coverage_classifier.pyindex bf3a15d07..77b8e4894 100644--- a/tests/test_gate_coverage_classifier.py+++ b/tests/test_gate_coverage_classifier.py@@ -233,3 +233,36 @@ def test_generated_connector_types_is_inventoried(inventory) -> None:def test_self_test_passes() -> None:assert cgc.self_test(REPO_ROOT) == 0+++def test_a_script_named_as_a_diff_path_is_not_a_job_asking_it_for_a_verdict() -> None:+ """`gates-a-workflow` must mean the job *runs* it, not that it says its name.++ `integration.yml`'s `changes` job publishes outputs the other jobs branch+ on, and lists `scripts/backup_crypt.py` among the paths whose modification+ should run the disaster-recovery gate. The matcher looked for the bare+ path anywhere in the job's YAML, so an encryption utility was inventoried+ as a gate on the strength of appearing in a filter — the same shape as the+ matcher that counted eleven services as CI-covered because a path sat+ inside a quoted `echo`.+ """+ job = {+ "outputs": {"backup": "${{ steps.areas.outputs.backup }}"},+ "steps": [{"run": "backup_paths='scripts/backup.sh scripts/backup_crypt.py'\ngrep -q \"$p\" /tmp/changed.txt"}],+ }+ doc = {"jobs": {"changes": job, "dr": {"if": "needs.changes.outputs.backup == 'true'"}}}++ assert cgc._gating_jobs(doc, "integration.yml") == {}+++def test_a_job_that_runs_a_script_and_publishes_an_output_still_counts() -> None:+ """The other direction: `wet_eval_check.py` has no intrinsic signal at all+ and is inventoried only through the job graph, so tightening the matcher+ must not drop it."""+ job = {+ "outputs": {"run": "${{ steps.pre.outputs.run }}"},+ "steps": [{"run": "python3 scripts/wet_eval_check.py --status-out /tmp/preflight.json"}],+ }+ doc = {"jobs": {"preflight": job, "eval": {"if": "needs.preflight.outputs.run == 'true'"}}}++ assert cgc._gating_jobs(doc, "wet-eval.yml") == {"wet_eval_check.py": "wet-eval.yml:preflight"}diff --git a/tests/test_test_discovery_gate.py b/tests/test_test_discovery_gate.pynew file mode 100644index 000000000..acc93442a--- /dev/null+++ b/tests/test_test_discovery_gate.py@@ -0,0 +1,293 @@+"""A suite is only worth the files its invocation reaches.++``scripts/check_test_discovery.py`` compares the test files git tracks against+the files the workflows actually collect. Before it existed, 63 were reached+by nothing — 56 of them under ``services/agents``, whose invocation was a+hand-maintained list of 26 file paths.++These tests pin the parts a reviewer cannot see by reading the gate's output:+what it *credits*. Every defect this gate has had came from crediting+something — the first version resolved ``tests/`` against the repository root+for a step that had ``cd``'d into a service, which simultaneously reported the+whole agents suite unreached and credited the root tree with a package's+invocation; the second skipped ``-m`` as "a filter, not a collection rule",+which would have credited a file in full under ``-m "not integration"`` while+nothing in it ran.+"""++from __future__ import annotations++import shlex+import subprocess+import sys+from pathlib import Path++import pytest++REPO_ROOT = Path(__file__).resolve().parent.parent+sys.path.insert(0, str(REPO_ROOT / "scripts"))++import check_test_discovery as ctd # noqa: E402+++# ── Reading an invocation ────────────────────────────────────────────────────+ ("command", "expected"),+ [+ ("pytest tests/", ["tests/"]),+ ("python -m pytest tests/ -q", ["tests/", "-q"]),+ ("python3 -m pytest tests/ --tb=short", ["tests/", "--tb=short"]),+ ("python3.11 -m pytest tests/", ["tests/"]),+ ("PYTHONPATH=. python -m pytest tests/", ["tests/"]),+ ("PYTHONPATH=src FOO=1 python3 -m pytest tests/", ["tests/"]),+ ("poetry run pytest tests/", ["tests/"]),+ ("uv run pytest tests/", ["tests/"]),+ ],+)+def test_an_invocation_is_recognised_however_it_is_spelled(command, expected):+ assert ctd._args_after_pytest(shlex.split(command)) == expected+++ "command",+ [+ "pip install pytest pytest-asyncio",+ "python3 -m pip install --quiet pytest",+ "ruff check services/",+ "echo pytest tests/",+ "npm test",+ ],+)+def test_something_that_is_not_a_pytest_run_is_not_credited(command):+ """A ``pip install ... pytest`` line has no path argument.++ Read as an invocation it looks like a bare collection of the whole+ repository, which would credit every test file in the tree to a step that+ installs a package. An over-broad resolver manufactures exactly the+ coverage this gate exists to disprove.+ """+ assert ctd._args_after_pytest(shlex.split(command)) is None+++ ("args", "paths", "ignores", "markers"),+ [+ (["tests/"], ["tests/"], [], ""),+ (["tests/", "--ignore=tests/isolation"], ["tests/"], ["tests/isolation"], ""),+ (["tests/", "--ignore", "tests/isolation"], ["tests/"], ["tests/isolation"], ""),+ (["tests/", "-k", "not_slow"], ["tests/"], [], ""),+ (["tests/", "-m", "not integration"], ["tests/"], [], "not integration"),+ (["tests/", "--cov=app", "--cov-config=pyproject.toml"], ["tests/"], [], ""),+ (["tests/a.py", "tests/b.py", "-v"], ["tests/a.py", "tests/b.py"], [], ""),+ ],+)+def test_arguments_are_split_into_paths_ignores_and_markers(args, paths, ignores, markers):+ assert ctd._split_args(args) == (paths, ignores, markers)+++# ── Following `cd` ───────────────────────────────────────────────────────────+def test_a_cd_inside_a_run_block_moves_the_resolver():+ """The agents invocation ``cd``s into the service, then names ``tests/``.++ Resolving that against the repository root reported all 82 agents files+ unreached and credited the root ``tests/`` tree with the package's+ invocation — the same defect in both directions at once.+ """+ assert ctd._cd_target("cd services/agents", "") == "services/agents"+ assert ctd._cd_target("cd packages/aisoc-benchmark", "") == "packages/aisoc-benchmark"+++def test_cd_composes_and_unwinds():+ assert ctd._cd_target("cd tests", "services/agents") == "services/agents/tests"+ assert ctd._cd_target("cd ..", "services/agents") == "services"+++def test_a_cd_to_an_unexpanded_variable_does_not_move_the_resolver():+ """Guessing would send every path after it somewhere arbitrary.++ They would then all read as missing, which is a finding the tree did not+ cause.+ """+ assert ctd._cd_target("cd $GITHUB_WORKSPACE", "services/agents") == "services/agents"++[email protected]("command", ["pytest tests/", "cd", "cdrom --help", "echo cd services"])+def test_something_that_is_not_a_cd_is_not_read_as_one(command):+ assert ctd._cd_target(command, "") is None+++# ── Marker deselection ───────────────────────────────────────────────────────+ ("expression", "markers", "deselected"),+ [+ ("not integration", {"integration"}, True),+ ("not integration", set(), False),+ ("integration", {"integration"}, False),+ ("integration", set(), True),+ ("not a and not b", {"b"}, True),+ ("not a and not b", {"c"}, False),+ ("", {"integration"}, False),+ ],+)+def test_a_marker_expression_is_evaluated_not_matched_as_text(expression, markers, deselected):+ """``not integration`` and ``integration`` differ by three characters.++ They invert the answer, so a gate matching the marker name as a substring+ would credit precisely the files nothing runs.+ """+ assert ctd.deselects_module(expression, frozenset(markers)) is deselected+++def test_an_expression_that_cannot_be_evaluated_credits_the_file():+ """The reading that can only miss a finding, never invent one."""+ assert ctd.deselects_module("not and or", frozenset({"integration"})) is False+++def test_a_module_level_pytestmark_is_read_off_the_syntax_tree(tmp_path):+ module = tmp_path / "test_x.py"+ module.write_text("import pytest\npytestmark = pytest.mark.integration\n")++ assert ctd.module_markers(module) == frozenset({"integration"})+++def test_a_marker_on_a_single_test_does_not_mark_the_module(tmp_path):+ """One deselected test is not a file nothing runs, and this gate is about files."""+ module = tmp_path / "test_x.py"+ module.write_text("import pytest\n\n\[email protected]\ndef test_one():\n pass\n")++ assert ctd.module_markers(module) == frozenset()+++def test_a_list_of_pytestmarks_is_read(tmp_path):+ module = tmp_path / "test_x.py"+ module.write_text("import pytest\npytestmark = [pytest.mark.integration, pytest.mark.slow]\n")++ assert ctd.module_markers(module) == frozenset({"integration", "slow"})+++# ── The corpus ───────────────────────────────────────────────────────────────+def test_the_corpus_is_what_git_tracks(tmp_path):+ """An untracked install directory is not this repository's tests.++ Deriving the corpus from git removes the exclusion list a filesystem walk+ needs — and a list of ephemeral directory names cannot be checked in both+ directions, because on a clean checkout none of them exist. The first+ version of this gate carried one and failed on its own repository.+ """+ for rel in ("services/x/tests/test_a.py", "services/x/tests/helper.py"):+ (tmp_path / rel).parent.mkdir(parents=True, exist_ok=True)+ (tmp_path / rel).write_text("")+ (tmp_path / "node_modules/p/tests").mkdir(parents=True)+ (tmp_path / "node_modules/p/tests/test_vendor.py").write_text("")+ git = ["git", "-c", "user.name=t", "-c", "user.email=t@invalid", "-c", "commit.gpgsign=false"]+ for argv in (["init", "-q", "-b", "main"], ["add", "services"], ["commit", "-qm", "t"]):+ subprocess.run(git + argv, cwd=tmp_path, check=True, capture_output=True) # noqa: S603++ files = [f for tree in ctd.discover_trees(tmp_path) for f in tree.files]++ assert files == ["services/x/tests/test_a.py"]+++def test_a_test_shaped_file_in_no_tests_directory_is_still_in_the_corpus(tmp_path):+ """``scripts/integration/spine_test.py`` is one.++ Scoping the scan to ``tests/`` directories would make it invisible, which+ is the defect this gate removes, one level up. It gets its own pseudo-tree+ and is reached by a workflow executing it directly.+ """+ (tmp_path / "scripts/integration").mkdir(parents=True)+ (tmp_path / "scripts/integration/spine_test.py").write_text("")+ git = ["git", "-c", "user.name=t", "-c", "user.email=t@invalid", "-c", "commit.gpgsign=false"]+ for argv in (["init", "-q", "-b", "main"], ["add", "-A"], ["commit", "-qm", "t"]):+ subprocess.run(git + argv, cwd=tmp_path, check=True, capture_output=True) # noqa: S603++ trees = ctd.discover_trees(tmp_path)++ assert [t.name for t in trees] == [ctd.LOOSE]+ assert trees[0].files == ["scripts/integration/spine_test.py"]+++def test_conftest_collect_ignore_removes_a_file_from_reach(tmp_path):+ """A file pytest is told not to collect is not reached, however it is named."""+ tests = tmp_path / "services/x/tests"+ tests.mkdir(parents=True)+ (tests / "test_a.py").write_text("")+ (tests / "conftest.py").write_text('collect_ignore = ["test_a.py"]\n')++ assert ctd.collect_ignored(tests, tmp_path) == {"services/x/tests/test_a.py"}+++def test_a_tree_that_renames_python_files_is_scanned_for_its_own_pattern(tmp_path):+ """Scanning for the default in a tree that renamed it finds nothing, and+ a tree with nothing to find reads as fully covered."""+ (tmp_path / "pyproject.toml").write_text('[tool.pytest.ini_options]\npython_files = ["check_*.py"]\n')++ assert ctd.python_files_patterns(tmp_path) == ("check_*.py",)+ assert ctd.python_files_patterns(tmp_path / "nothing-here") == ctd.DEFAULT_PYTHON_FILES+++# ── The verdict ──────────────────────────────────────────────────────────────+def _tree(files, reached, name="services/x"):+ tree = ctd.Tree(name=name, tests_dir=f"{name}/tests", files=list(files))+ tree.reached = set(reached)+ return tree+++def _codes(trees, missing=(), *, quarantine=None):+ return {code for code, _detail in ctd.evaluate(trees, missing, quarantine=quarantine or {})}+++def test_a_file_no_invocation_reaches_is_a_finding():+ one = ["services/x/tests/test_a.py"]++ assert "unreached" in _codes([_tree([*one, "services/x/tests/test_b.py"], one)])+++def test_an_invocation_naming_a_file_that_is_gone_is_a_finding():+ """The other direction. A step listing a deleted path no longer does what+ it reads as doing, and this repository has shipped that shape repeatedly."""+ one = ["services/x/tests/test_a.py"]+ missing = [("ci.yml", "pytest tests/test_gone.py", "services/x/tests/test_gone.py")]++ assert "names-nothing" in _codes([_tree(one, one)], missing)+++def test_quarantine_excuses_a_file_and_is_checked_in_both_directions():+ files = ["services/x/tests/test_a.py", "services/x/tests/test_b.py"]+ reached = files[:1]++ assert not _codes([_tree(files, reached)], quarantine={files[1]: "needs a live Neo4j and Kafka"})+ assert "quarantine-unexplained" in _codes([_tree(files, reached)], quarantine={files[1]: " "})+ assert "quarantine-names-nothing" in _codes([_tree(files, reached)], quarantine={"services/x/tests/test_ghost.py": "gone"})+ assert "quarantine-stale" in _codes([_tree(files, reached)], quarantine={files[0]: "but CI runs it now"})+++# ── The real tree ────────────────────────────────────────────────────────────+def test_every_quarantine_entry_carries_a_real_justification():+ for path, reason in ctd.QUARANTINE.items():+ assert len(reason.split()) >= 10, f"{path}: a one-line reason is not a justification"+++def test_the_scan_finds_the_trees_this_repository_actually_has():+ """A parser that silently stopped resolving anything would report a clean+ tree, so the counts themselves are asserted."""+ trees, invocations, _missing = ctd.scan(REPO_ROOT)++ assert len(invocations) > 20, "the workflow surface runs far more than this"+ names = {t.name for t in trees}+ for expected in ("services/agents", "services/api", "services/connectors", "scripts", "."):+ assert expected in names, f"{expected} ships tests and must appear as a tree"+++def test_the_agents_suite_is_reached_as_a_directory_not_as_a_list():+ """The regression this gate was built for. 26 of 82 files, named one per+ line, and adding a 83rd would have been silently free of signal."""+ trees, _invocations, _missing = ctd.scan(REPO_ROOT)+ agents = next(t for t in trees if t.name == "services/agents")++ assert agents.unreached == [], f"unreached agents tests: {agents.unreached}"+ assert len(agents.files) > 80+++def test_the_self_test_catches_every_defect_it_claims_to():+ assert ctd.self_test() == 0