Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

fix(wave 1): security and tenancy defects that every green CI run missed (#1116)

Beenu Arora614e64beed69
diff --git a/.env.example b/.env.example
index 5c1a782ca..1e1b9861f 100644
--- a/.env.example
+++ b/.env.example
@@ -428,6 +428,13 @@ AISOC_ACTIONS_SERVICE_TOKEN=
# and compose passes it to both.
AISOC_AGENTS_SERVICE_TOKEN=
+# The bearer a peer service presents to the API. The agents service uses it for
+# the customer tools, the eleven lake pivots and the hunting agent, and names
+# the tenant it is acting for on `X-AiSOC-Tenant-ID` beside it. Those three
+# previously read `AISOC_AGENTS_API_KEY`, which nothing ever set, so they
+# answered "could not check" on every deployment. `make up` generates it.
+AISOC_API_SERVICE_TOKEN=
+
# Bearer for realtime's /internal/* fan-out routes (API/agents -> realtime),
# sent as the x-internal-token header. `make up` generates it. The guard used
# to treat an unset value as authorized, and nothing set it.
diff --git a/.github/workflows/agent-auth-live.yml b/.github/workflows/agent-auth-live.yml
new file mode 100644
index 000000000..d3f30b63a
--- /dev/null
+++ b/.github/workflows/agent-auth-live.yml
@@ -0,0 +1,136 @@
+name: Agent service auth and closure grants (live)
+
+# Two suites that need real Postgres with every migration applied, because
+# both defects they cover are invisible to a fake connection: one is a service
+# credential the API must accept and scope to one tenant, and the other is a
+# query against a table that does not exist, which a double answers happily.
+#
+# Fix-pass items 1.1 and 1.4.
+
+on:
+ pull_request:
+ paths:
+ - "services/api/app/api/v1/deps.py"
+ - "services/api/app/api/v1/endpoints/agent_tools.py"
+ - "services/api/migrations/**"
+ - "services/agents/app/closure/**"
+ - "services/agents/app/tools/**"
+ - "services/agents/app/hunt/**"
+ - "tests/isolation/test_agent_service_auth_live.py"
+ - "tests/isolation/test_closure_grant_live.py"
+ - "tests/isolation/test_api_to_connectors_auth.py"
+ - "services/api/app/api/v1/endpoints/federated.py"
+ - "services/api/app/services/case_fanout.py"
+ - "services/api/app/core/connectors_auth.py"
+ - "services/connectors/app/api/router.py"
+ - ".github/workflows/agent-auth-live.yml"
+ push:
+ branches: [main]
+ workflow_dispatch:
+
+jobs:
+ agent-auth-live:
+ name: A service credential reaches the API and is scoped to one tenant
+ runs-on: ubuntu-latest
+ timeout-minutes: 20
+
+ services:
+ postgres:
+ image: postgres:16-alpine
+ env:
+ POSTGRES_USER: aisoc
+ POSTGRES_PASSWORD: aisoc_ci_only
+ POSTGRES_DB: aisoc
+ ports: ["5432:5432"]
+ options: >-
+ --health-cmd "pg_isready -U aisoc"
+ --health-interval 5s
+ --health-timeout 5s
+ --health-retries 20
+
+ env:
+ # The runtime role, not the owner. A superuser ignores every RLS policy
+ # even under FORCE ROW LEVEL SECURITY, which would let the tenant
+ # assertions pass for the wrong reason.
+ LIVE_DSN: postgresql+asyncpg://aisoc_app:aisoc_app_ci_only@localhost:5432/aisoc
+ PGPASSWORD: aisoc_ci_only
+ OTEL_SDK_DISABLED: "true"
+
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+
+ - uses: actions/setup-python@e9d6f990972a57673cdb72ec29e19d42ba28880f # v6
+ with:
+ python-version: "3.11"
+
+ # Each service's own lockfile, not a hand-curated pip list. A set that
+ # differs from the lock means the job tests software nobody ships.
+ - name: Install api from its lockfile
+ working-directory: services/api
+ run: |
+ set -euo pipefail
+ python -m pip install --quiet --upgrade pip "poetry==2.4.1"
+ poetry config virtualenvs.create false
+ poetry install --no-root --no-interaction
+ python -m pip install --quiet pytest pytest-asyncio
+
+ - name: Apply every migration, in order
+ run: |
+ for f in services/api/migrations/*.sql; do
+ psql -h localhost -U aisoc -d aisoc -v ON_ERROR_STOP=1 -q -f "$f"
+ done
+
+ - name: Give the runtime role its password
+ run: |
+ psql -h localhost -U aisoc -d aisoc -v ON_ERROR_STOP=1 -q -c \
+ "ALTER ROLE aisoc_app WITH PASSWORD 'aisoc_app_ci_only';"
+ psql -h localhost -U aisoc -d aisoc -v ON_ERROR_STOP=1 -q -c \
+ "GRANT USAGE ON SCHEMA public TO aisoc_app;"
+
+ # The suites skip themselves when their DSN is unset, so this asserts the
+ # DSN is set rather than trusting the run. A skipped live suite reports
+ # the same word as a passing one.
+ - name: The agents service reaches the API as a service, for one tenant
+ working-directory: services/api
+ env:
+ ISOLATION_AGENT_AUTH_DSN: ${{ env.LIVE_DSN }}
+ run: python -m pytest ../../tests/isolation/test_agent_service_auth_live.py -v --tb=short -rs
+
+ - name: Install connectors from its lockfile
+ working-directory: services/connectors
+ run: |
+ set -euo pipefail
+ poetry install --no-root --no-interaction
+
+ # Needs both services installed: the API's `_query_one_backend` on one
+ # side, and the real connectors application started on a loopback socket
+ # on the other. `ci.yml`'s API job installs only the API's lockfile, so
+ # this cannot live in `services/api/tests/`.
+ - name: Every API-to-connectors call carries a credential and a tenant
+ working-directory: services/api
+ env:
+ ISOLATION_CROSS_SERVICE: "1"
+ run: python -m pytest ../../tests/isolation/test_api_to_connectors_auth.py -v --tb=short -rs
+
+ - name: Install agents from its lockfile
+ working-directory: services/agents
+ run: |
+ set -euo pipefail
+ poetry install --no-root --no-interaction
+ python -m pip install --quiet pytest pytest-asyncio
+
+ - name: An earned auto-close grant is readable by the closure policy
+ working-directory: services/agents
+ env:
+ ISOLATION_CLOSURE_DSN: ${{ env.LIVE_DSN }}
+ run: python -m pytest ../../tests/isolation/test_closure_grant_live.py -v --tb=short -rs
+
+ - name: No suite may report success having skipped
+ run: |
+ set -euo pipefail
+ if [ -z "${LIVE_DSN}" ]; then
+ echo "::error::LIVE_DSN is empty, so the live suites skipped and this job proved nothing."
+ exit 1
+ fi
+ echo "LIVE_DSN was set, and the cross-service step sets ISOLATION_CROSS_SERVICE,"
+ echo "so none of the three suites took its skip path."
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index b33ef1a4c..cf6c138c3 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -881,6 +881,21 @@ jobs:
run: |
python3 scripts/check_service_token_wiring.py --self-test
python3 scripts/check_service_token_wiring.py
+ # A read executor is named for the product and a connector for the
+ # integration a tenant configures. Three of the seven executor ids
+ # matched no connector type at all, so the lookup returned None on
+ # every tenant and the tool was never offered to an investigation.
+ - name: Every read executor's vendor id resolves to a saveable connector
+ run: |
+ python3 scripts/check_vendor_catalog_ids.py --self-test
+ python3 scripts/check_vendor_catalog_ids.py
+ # The hunting agent never writes a query: the model proposes a plan
+ # over a recorded field list and the API compiles it. This gate reads
+ # both sides and fails when they drift, and until now no workflow ran
+ # it -- five modules named it in a docstring, which is not the same
+ # thing as calling it.
+ - name: The hunting agent's plan boundary holds on both sides
+ run: python3 scripts/check_hunt_agent_boundary.py
# The marketplace index has to exist in three build contexts, none of
# which can see the other two. The API's copy was missing, so its image
# shipped no catalogue and the marketplace answered 503 on every
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 10c97bf4b..f84a3749c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,50 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
+### Fixed
+
+- **The agents service could not reach the API at all, and three vendors were
+ never offered to an investigation (fix pass, wave 1).**
+
+ An independent audit at v16.0.0 found capabilities recorded as shipped that
+ fail on a real deployment, while every CI run stayed green. The pattern
+ repeats: each defect's test mocks exactly the boundary the defect lives at.
+
+ - The customer tools, the hunting agent and the sandbox tool authenticated
+ with `AISOC_AGENTS_API_KEY`, which no compose file, `.env.example` or Helm
+ value ever set, so all three answered "could not check" on every default
+ install. Setting it would not have helped: one key belongs to one tenant.
+ They now present a service token and name the tenant of the run they are
+ working on, and the API resolves a service principal with an explicit
+ read-only permission set. A service token that names no tenant is refused
+ rather than widened.
+ - The eleven lake pivots sent `X-Tenant-ID` and no credential, a header the
+ API's auth does not read, so every pivot was 401 outside dev mode.
+ - Federated search and case fanout posted to the connectors service with no
+ credential at all, so **every SIEM answered 401** and the console reported
+ no results, which is indistinguishable from a SIEM that held none.
+ - `defender`, `entra` and `aws` are not connector types a tenant can save,
+ so `by_type.get(vendor_id)` returned `None` on every tenant and three of
+ the five vendors added for agent reads had never once been reachable.
+ - `_has_auto_close_grant` queried `autonomy_grants`, which no migration
+ creates, filtering on two columns the real table does not have. Because
+ `require_grant` defaults to true, a tenant that enabled a closure policy
+ could never auto-close: the feature was off for exactly the tenants who
+ turned it on.
+ - The hunt route read `result.matches` and the result carries `findings`, so
+ **every hunt that found rows reported none**. It also passed no ledger, so
+ no hunt had ever written a ledger row.
+ - `hunts:read` was held by no role at all, not even `tenant_admin`.
+
+ Four gates were extended to catch this class rather than these instances. A
+ service-to-service request with no credential now fails CI; a SQL statement
+ naming a table no migration creates now fails rather than being recorded as
+ "not compared"; a read executor's vendor id must resolve to a saveable
+ connector type; and `check_gate_coverage.py` no longer counts a gate named
+ in a **docstring** as a gate that runs, which is how
+ `check_hunt_agent_boundary.py` was reported reachable while no workflow ran
+ it.
+
## [16.0.1] - 2026-10-04
### Security
diff --git a/FIX_PASS_PROGRESS.md b/FIX_PASS_PROGRESS.md
new file mode 100644
index 000000000..c3c288c68
--- /dev/null
+++ b/FIX_PASS_PROGRESS.md
@@ -0,0 +1,426 @@
+# AiSOC fix pass: progress
+
+Mirrors `plans/aisoc_fix_pass_plan.plan.md`, which is locked and is the source
+of truth. This file is the mutable half.
+
+Legend: `[ ]` not started, `[~]` in progress, `[x]` done, `[!]` blocked (with
+exactly what is needed).
+
+An item is `[x]` only when all three of these are recorded against it: the test
+that failed on the pre-fix tree for the stated reason, the fix, and the negative
+control showing that reverting the fix fails the test again.
+
+- **Base commit:** `056f80a7` (`origin/main`, v16.0.1).
+- **Plan captured at:** `b1c6925a`. File and line references are hints at that
+ commit; where code has moved, the item records where it was found.
+
+## Baseline
+
+Recorded before any change, so a failure this pass did not cause is never
+attributed to it.
+
+Suites, each run from its own service directory:
+
+| Suite | Result |
+|---|---|
+| `services/api` | 4178 passed, 40 skipped |
+| `services/agents` | 1766 passed, 2 skipped, 219 subtests passed |
+| `services/actions` | 1081 passed |
+| `services/connectors` | 1000 passed |
+| `services/fusion` | 419 passed, 2 skipped |
+
+Gates, all passing: `check_competitor_names`, `check_gate_coverage`,
+`check_gate_contract`, `check_test_discovery`, `check_route_auth`,
+`check_route_authz`, `check_route_tenant_scope`,
+`check_tenant_query_predicates`, `check_rls_policy_shape`,
+`check_claim_gate_matrix` (291 rows, 291 GATED), `check_release_policy`,
+`check_comment_paths`, `check_mypy_baseline`, `check_service_token_wiring`,
+`check_raw_sql_columns`, `check_maturity_table`, `check_perf_results`.
+
+**Everything is green, which is the premise of this pass rather than a
+contradiction of it.** The audit's finding is that each defect's test mocks the
+boundary the defect lives at, so a green suite is exactly what a tree carrying
+these defects looks like.
+
+### Environment note: the suites must run against the locked dependency set
+
+A first run of `services/agents` on this machine's interpreter produced 10
+errors in `tests/test_mcp_client.py`, all
+`PydanticUserError: A non-annotated attribute was detected`. That is **not** a
+repository failure: the machine carries `pydantic` 2.8.2 and an old `mcp` in
+user site-packages, against a locked `pydantic` 2.13.4 and `mcp` 1.30.0. Re-run
+inside a venv built from `scripts/service_requirements.py agents --locked`, the
+same suite is fully green.
+
+This matters beyond bookkeeping. Wave 2.1 turns on an assertion inside
+`httpx` 0.28.1 exactly, so a result from any other httpx says nothing about the
+defect. Every agents result in this pass is recorded against the locked set.
+
+**A second interpreter trap, found while running Wave 1.** `services/api`
+pins `httpx` **0.26.0** and `services/actions` pins **0.28.1**, so running the
+actions suite in the api venv fails four URL-encoding assertions in
+`test_alert_history.py` that have nothing to do with any change. Each service's
+suite is run against that service's own locked set, and a cross-service result
+is recorded as the artefact it is.
+
+Two traps when building that venv, both hit here: `service_requirements.py`
+emits environment markers as separate shell tokens, so splitting its output on
+whitespace corrupts them, and stripping the markers instead makes the two
+conditional `numpy` pins collide into an unresolvable install. Parse with
+`shlex`, regroup on each `name==version`, and re-quote the marker operands.
+
+## Wave 1: Security and tenancy (P0)
+
+- [x] **1.1** Agents authenticates to the API as a service, for the tenant it is working on
+- [x] **1.2** Federated search authenticates to the connectors service
+- [x] **1.3** Vendor reads match the connector types tenants actually save
+- [x] **1.4** Earned auto-close grants are honoured; the closure default is an ADR awaiting the maintainer (M4)
+- [x] **1.5** The hunting agent returns its findings, for the right tenant, with a ledger
+
+## Wave 2: The MCP client works over a real connection
+
+- [ ] **2.1** Transport: `_CappedStream` is a real `httpx.AsyncByteStream`
+- [ ] **2.2** Tool names match the function-name pattern
+- [ ] **2.3** SSRF re-resolution and air-gap: code, test and doc agree
+- [ ] **2.4** Docs and claim rows: generated tool count, unpublished package, stale counts
+
+## Wave 3: Replay, shadow mode and evaluation measure the real thing
+
+- [ ] **3.1** Frozen context, not empty context
+- [ ] **3.2** No side effects from a replay
+- [ ] **3.3** Input shapes: per-source adapters for Elastic and Defender
+- [ ] **3.4** Model attribution: `model_used` is set from the call that answered
+- [ ] **3.5** Non-degenerate acceptance for the reproducibility test
+- [ ] **3.6** Tool calls recorded from the ledger, not hard-coded to 0
+- [ ] **3.7** Demotion without a page load
+- [ ] **3.8** Skills: activation evidence, LLM-path-only disclosure, console retraction
+- [ ] **3.9** The weekly live evaluation can actually run
+- [ ] **3.10** The 9.3% flip rate is qualified or removed
+- [ ] **3.11** CI runs the live tests
+- [ ] **3.12** Pivot counting excludes failed calls
+
+## Wave 4: Enterprise identity, white-label and metering
+
+- [ ] **4.1** SSO completes in the console
+- [ ] **4.2** Console MFA
+- [ ] **4.3** White-label reaches every surface it claims
+- [ ] **4.4** Metering counts what happened
+- [ ] **4.5** SCIM live coverage and the console token claim
+- [ ] **4.6** Maturity evidence names tests that exercise the row
+
+## Wave 5: Hunting, intel, sandbox and operations wiring
+
+- [ ] **5.1** Retro-hunts can be turned on
+- [ ] **5.2** KEV exposure gets data
+- [ ] **5.3** Hunts over the lake
+- [ ] **5.4** Sandbox and air-gap settings reach the API
+- [ ] **5.5** Release channel does not cross a major
+- [ ] **5.6** Upgrade fixture does not insert into the renamed `cases` table
+- [ ] **5.7** Chaos and HA run live in CI
+- [ ] **5.8** Performance honesty
+
+## Wave 6: Retract what is not built
+
+- [ ] **6.1** The v16 detection lifecycle
+- [ ] **6.2** Enterprise IAM
+- [ ] **6.3** Overclaims in `apps/docs/docs/intro.md`
+- [ ] **6.4** The phishing playbook's fleet-wide retraction
+- [ ] **6.5** Two broken detection-tuning routes
+
+## Wave 7: Close the books
+
+- [ ] **7.1** Re-run every claim row this pass touched
+- [ ] **7.2** Update `GAP_CLOSURE_PROGRESS.md` and `PARITY_PROGRESS.md`
+- [ ] **7.3** One `[Unreleased]` entry in `CHANGELOG.md`
+- [ ] **7.4** Final report in this file
+
+## Maintainer-only (recorded, not attempted)
+
+- [!] **M1** Make the live jobs required checks with branch protection on `main`:
+ replay end-to-end, autonomy live, retro-hunt live, SCIM live and chaos.
+ **Needs:** a maintainer with admin rights on the repository.
+- [!] **M2** Fund provider keys so the weekly evaluation produces real numbers.
+ **Needs:** a funded API key set as a repository secret.
+- [!] **M3** Provide a real multi-node cluster for the scale run.
+ **Needs:** infrastructure the CI runner does not have.
+- [!] **M4** Accept or overrule the closure default proposed in the 1.4 ADR.
+ **Needs:** a maintainer decision on
+ `docs/decisions/0009-auto-close-without-an-earned-grant.md`, which recommends
+ option B (no auto-close without an earned grant, plus an audited per-tenant
+ opt-in). It is breaking, so the fix pass does not take it: it changes what a
+ running deployment does to alerts without being asked.
+
+## Pre-existing failures
+
+**None.** Every suite and every gate passes on `056f80a7` once the suites run
+against their locked dependency sets. The 10 `test_mcp_client.py` errors seen
+on a first run were this machine's interpreter, not the tree; see the
+environment note above.
+
+## Deviations
+
+Items whose defect did not reproduce, with the evidence.
+
+_None yet._
+
+## Item log
+
+Per item: the reproducing test, the fix, the negative control, and the PR.
+
+### 1.1 Agents authenticates to the API as a service, for the tenant it works on
+
+**Reproduced, all four defects, before any change.**
+
+* `AISOC_AGENTS_API_KEY` was read by three modules and set by nothing: no
+ compose file, no `.env.example`, no Helm value.
+* `VALID_SCOPES` held 23 entries and none of `actions:read`, `lake:query` or
+ `hunts:read`, so only a `*` key could reach the routes those tools call.
+ `hunts:read` was held by **no role at all**, not even `tenant_admin`.
+* The eleven lake pivots in `call_investigation_tool` sent `X-Tenant-ID` and
+ no credential. The API's auth reads neither.
+* The API had no service-caller path whatsoever.
+
+**Reproducing test:** `tests/isolation/test_agent_service_auth_live.py`, the
+real `create_application()` against Postgres 16 with all 100 migrations
+applied, driven over a real `httpx` client with exactly the credential compose
+hands the agents container. Pre-fix: `2 failed, 4 passed`, the two failures
+being `401 {"detail":"Could not validate credentials"}` and then an empty
+connector list. The 4 that pass pre-fix are the negative controls, which must
+pass in both directions or they are not controls.
+
+**Fix:**
+
+* `services/api/app/api/v1/deps.py` gains `_resolve_service_principal`, ahead
+ of the JWT path. A service token must declare its tenant on
+ `X-AiSOC-Tenant-ID`; no tenant is an **empty** scope rather than every
+ scope, and the tenant is checked against the `tenants` table because the
+ header is caller-supplied.
+* The principal carries an explicit seven-permission read-only set, not the
+ wildcard. One shared secret equal to `platform_admin` on every tenant is
+ the defect this is supposed to prevent, not reproduce.
+* `customer_tools.py`, `sandbox.py`, `hunt/agent.py` and `investigation.py`
+ present `AISOC_API_SERVICE_TOKEN` (falling back to `AISOC_SERVICE_TOKEN`)
+ and name the tenant beside it. The tenant is threaded from the run, never
+ from the model: `run_deep_investigation` already had it and simply never
+ passed it on.
+* `hunts:read` granted to `tenant_admin`, `soc_lead`, `soc_analyst` and
+ `threat_hunter`, beside the `lake:query` they already hold.
+* The three permissions are now mintable scopes, so a tenant automating one
+ of these reads no longer needs a wildcard key.
+* `AISOC_AGENTS_API_KEY` is gone. `AISOC_API_SERVICE_TOKEN` is generated by
+ `scripts/ensure_env.py`, documented in `.env.example` and delivered to both
+ `agents` and `api` by compose.
+
+**Negative control:** removing the three-line wiring from `get_current_user`
+returns the suite to `2 failed, 4 passed`, with the same 401.
+
+**Gate:** `scripts/check_service_token_wiring.py` gains a second direction.
+The first asks whether a *variable* reaches a process; the new one asks
+whether the *request that needed it* carries a credential. Proven against the
+pre-fix tree: restoring the old pivot headers makes it name
+`investigation.py:69` exactly.
+
+Two refinements the gate needed, each caught by running it: resolving a URL
+helper by its **body** rather than its name (`_alerts_url()` in the Trellix
+connector builds a customer's vendor URL, and a gate that reports a vendor
+call as an uncredentialed internal one teaches people to ignore it), and
+reading `headers["Authorization"] = ...` as well as `headers = {...}` (the
+ingest client adds the credential by subscript).
+
+**It then found two more of the same defect**, one of which the plan names as
+1.2 and one it does not: `federated.py:253` and `case_fanout.py:185`.
+
+### 1.2 Federated search authenticates to the connectors service
+
+**Reproduced.** `tests/test_connectors_calls_are_credentialed.py` starts the
+**real connectors application** (`app.main:app`) as its own process on a
+loopback socket and drives `federated._query_one_backend`, the function the
+route calls. Pre-fix the verdict reads
+`status='error' error='backend 401: missing bearer credential'`.
+
+Three decisions the first draft got wrong, each of which would have produced a
+test that passed over the defect:
+
+* **It mounted the router instead of the application**, which dropped the
+ `/api/v1` prefix, so the first run saw 404 where a deployment sees 401. A
+ harness artefact that reads exactly like a passing test.
+* **It asserted on `_catalog_headers`**, the helper that already worked. What
+ shipped broken is that federated search never called it, so a test of the
+ helper passes on the defective tree. It now drives the production path and
+ pins the helper separately.
+* **It re-implemented the "every caller uses it" walk** and was worse at it
+ than the gate, naming five call sites that reach the agents service and a
+ vendor. That half is deleted with the reason recorded; the gate owns it.
+
+Both services name their package `app`, so the far side runs in a subprocess
+rather than being imported. Skipping around the clash was the alternative and
+a skipping test reports the same word as a passing one.
+
+**Fix:** `services/api/app/core/connectors_auth.py` is now the single
+implementation. `federated.py` and `case_fanout.py` call it with the
+connector's own `tenant_id`; `connectors.py::_catalog_headers` delegates to it
+rather than keeping the second copy that let this happen.
+
+**Negative control:** removing `headers=connectors_headers(...)` from
+`federated.py` returns the suite to `1 failed, 4 passed` with the same 401.
+
+**Also fixed, not in the plan:** `case_fanout.py:185` had the identical
+defect on case push and status polling. The gate found it; the plan named
+only federated search.
+
+### 1.3 Vendor reads match the connector types tenants actually save
+
+**Reproduced** by the new gate, which is the clearest statement of the defect:
+`scripts/check_vendor_catalog_ids.py` on the pre-fix tree names `aws`,
+`defender` and `entra` as vendor ids that resolve to no connector a tenant can
+save, out of 7 executors against 85 saveable types.
+
+A read executor is named for the product; a connector for the integration a
+tenant configures. For four of the seven the strings coincide. For the other
+three `by_type.get(vendor_id)` returned `None` on every tenant, so three of the
+five vendors added in gap-closure 4.2 had never once been offered to an
+investigation. The lookup that misses is the same expression as the one that
+hits, which is why nothing failed.
+
+**Fix:** `services/api/app/services/agent_tools/vendor_aliases.py` holds the
+map and one `resolve()` used by both matchers, `vendor_reads.available_reads`
+and `playbook_step_dispatch._pick_connector`. The map holds exceptions only, so
+it does not become a second copy of the catalog that drifts from it.
+
+`_pick_connector` needed more than a lookup swap: it built an `IN` clause from
+raw executor ids, so the query itself selected nothing for those three.
+
+**Negative control:** emptying `VENDOR_CONNECTOR_TYPES` fails 4 of the 10 tests
+in `tests/test_vendor_alias_resolution.py` and leaves the 6 that must not move.
+
+**The gate caught an error in its own fix**: the first alias named
+`aws_securityhub`, and the connector declares `aws_security_hub`. An alias to a
+type nobody can save resolves to nothing, exactly like no alias at all, and the
+gate said so before the code shipped.
+
+### 1.4 Earned auto-close grants are honoured
+
+**Reproduced** against Postgres 16 with all 100 migrations applied.
+`tests/isolation/test_closure_grant_live.py` pre-fix: `1 failed, 5 passed`,
+the failure logging
+`closure.grant.unreadable error='relation "autonomy_grants" does not exist'`.
+
+Three names wrong in one statement: the table is `aisoc_autonomy_grants`, and
+the filters named `revoked_at` and `expires_at` where the lifecycle is a
+`state` column of `shadow` / `granted` / `demoted`. Because `require_grant`
+defaults to true, a tenant that enabled a closure policy could never auto-close
+anything: the feature was off for exactly the tenants who turned it on.
+
+**Fix:** the query reads the real table and its real state column. The
+`except` stays, because a database blip must not close alerts, but the live
+suite now asserts the query itself works rather than trusting a fake connection
+that answers any query.
+
+**Negative control:** restoring the old table name returns the suite to
+`1 failed, 5 passed` with the same message.
+
+**Gate.** `check_raw_sql_columns.py` **passed over this for as long as it
+shipped**, and the reason is more interesting than the defect: a `SELECT`
+against a table no migration creates was bucketed as "not compared", on the
+stated reasoning that the gate cannot tell which engine a statement targets.
+That is true, and the remedy is to say which, once, rather than excuse the
+class. `FOREIGN_ENGINE_TABLES` now declares the ClickHouse, Postgres-catalogue,
+osquery and vendor tables, and **everything else fails**.
+
+Closing it required two parser improvements, because the first run reported
+four false positives: common table expressions are now detected from the
+statement's own `WITH` clause rather than allowlisted one at a time, and a
+statement composed across two string literals is recorded as the one artefact
+that detection cannot reach, with that reason written down.
+
+Proven against the pre-fix tree, where it names
+`services/agents/app/closure/policy.py:307 autonomy_grants`. Two new self-test
+cases pin both directions, and one existing case was renamed because it had
+described the old reasoning.
+
+**It also surfaced fix-pass item 6.5 early:** `aisoc_alerts` and
+`aisoc_detection_rules` are named by live routes and created by no migration.
+They are recorded in `KNOWN_MISSING_TABLES` as a debt that names 6.5 as the
+item which removes them, rather than being quietly excused.
+
+**The closure default is not decided here.**
+`docs/decisions/0009-auto-close-without-an-earned-grant.md` records the
+question, three options and a recommendation. Taking it changes what a running
+deployment does to alerts without being asked, so it is `M4`.
+
+### 1.5 The hunting agent returns its findings, for the right tenant, with a ledger
+
+**Reproduced.** `services/agents/tests/test_hunt_route_returns_findings.py`
+pre-fix: `2 failed, 2 passed`.
+
+* `assert [] == [{'host': 'WS-42', ...}]`. The route built
+ `matches=list(getattr(result, "matches", []) or [])` and `HuntAgentResult`
+ has `findings`. The `getattr` default made it silent, so **every hunt that
+ found rows answered `checked: true, matches: []`** -- a dropped answer that
+ reads exactly like a clean result.
+* `_record does not supply ['agent', 'seq', 'summary', 'tenant_id']`, which
+ the real `record_event` requires.
+
+**Worse than the plan states, and found by tracing the caller:** the only
+production caller passed **no ledger at all**, so `_record` returned at its
+first line. **No hunt had ever written a ledger row**, and the signature
+mismatch underneath was never even reached. The agent's own suite asserted on
+ledger writes against a double it supplied itself, which accepts any keyword
+arguments and so cannot tell a call the real ledger rejects from one it
+accepts.
+
+**Fix:** the route returns `result.findings`, passes the tenant it already
+resolved, and hands `run_hunt` the real ledger module. `_record` supplies every
+required field and owns the per-run `seq`, because a caller that forgot would
+write every row at zero and the replay would be unordered.
+
+`hunts:read` was granted to the four hunting roles in item 1.1; no role held it
+at all, not even `tenant_admin`.
+
+**Negative control:** restoring the `getattr(result, "matches", [])` line
+returns the suite to a failure on the dropped findings.
+
+**Gates, and the one that was lying.** `check_hunt_agent_boundary.py` is run by
+no workflow, and `check_gate_coverage.py` reported **all 150 checks reachable**
+anyway: five modules and two test files name it in a **docstring** saying which
+gate enforces their contract, and `_references` counted that as an invocation.
+
+Stripping whole-line comments was not enough; the references are in
+triple-quoted blocks. With prose removed the gate immediately named two
+unreachable checks, one of them `check_vendor_catalog_ids.py` from item 1.3:
+**the ci.yml edit for that gate had silently failed on an anchor mismatch, and
+the coverage gate's own blindness had hidden it.** Both are now wired and the
+pass is real.
+
+### Wave 1: three things CI found that local verification did not
+
+Recorded because each is the same class of defect this pass is about.
+
+**The published spec gained a 422 on all 456 operations.** Declaring the
+service tenant as a `Header` parameter on `get_current_user` puts it on every
+route that depends on authentication, and a parameter that exists can fail
+validation. It is how a peer *service* names the tenant it acts for, not part
+of the contract a customer codes against, so it is read off the `Request` and
+the spec is unchanged. `Request | None` is not a valid FastAPI annotation, and
+the parameter has to precede the defaulted ones.
+
+**Semgrep went 104 to 106.** Both findings were mine, and both were
+`python-logger-credential-disclosure` firing on the word "token" inside a log
+event name. Neither call logs a secret: one logs a header name, the other a
+tenant id. The events are renamed to describe the *caller* rather than its
+credential, which is also the more accurate name, and the count is back at the
+ceiling. **The ceiling did not move.**
+
+**Both new live suites ran in no workflow.** They skip themselves when their
+DSN is unset, and nothing set one, so neither would ever have executed.
+`.github/workflows/agent-auth-live.yml` now applies every migration, connects
+as the DML-only runtime role so RLS is not bypassed, and asserts at the end
+that the DSN was set, because a skipped live suite reports the same word as a
+passing one.
+
+**One limitation, stated rather than papered over.** The live auth suite passes
+alone and fails three assertions when run in the same process as the full
+`services/api` suite, which sets the dev-mode environment the suite needs
+absent. CI runs them in separate jobs, and `ci.yml` excludes
+`tests/isolation/` for this reason, so the arrangement is sound. It is recorded
+here because "passes only in isolation" is a fragility, not a result.
diff --git a/README.md b/README.md
index 8593a3aa9..7fdff703f 100644
--- a/README.md
+++ b/README.md
@@ -54,7 +54,7 @@ Needs Docker Compose v2 with **8 GB memory and 20 GB free disk in the Docker VM*
[Installation](https://beenuar.github.io/AiSOC/docs/installation#requirements) says what each number
was measured against. The first run downloads a ~2 GB model into a volume only `make clean` clears.
-`make up` also creates `.env` and generates the **fourteen** secrets in it — the credential vault, the
+`make up` also creates `.env` and generates the **fifteen** secrets in it — the credential vault, the
session signing key, five service-to-service credentials and four datastore passwords — then creates
an administrator and prints its password, generated on your machine, shown once and stored nowhere.
Copy it, or mint another with `make bootstrap ARGS=--reset-password`.
diff --git a/apps/docs/docs/deployment/docker.md b/apps/docs/docs/deployment/docker.md
index 5cbb831f4..0d9044c3a 100644
--- a/apps/docs/docs/deployment/docker.md
+++ b/apps/docs/docs/deployment/docker.md
@@ -117,7 +117,7 @@ COMPOSE_PROFILES=connectors,osquery,slack docker compose up -d
## Production
```bash
-make env # generates the fourteen secrets the stack needs
+make env # generates the fifteen secrets the stack needs
docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d
```
diff --git a/docker-compose.yml b/docker-compose.yml
index 15ea4409c..9648bc398 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -422,6 +422,7 @@ services:
# back to the bundled copy; the fix that made the proxy read the right
# variable shipped against a variable nothing ever set.
AISOC_SERVICE_TOKEN: ${AISOC_SERVICE_TOKEN:-}
+ AISOC_API_SERVICE_TOKEN: ${AISOC_API_SERVICE_TOKEN:-}
AISOC_CONNECTORS_SERVICE_TOKEN: ${AISOC_CONNECTORS_SERVICE_TOKEN:-}
# The agents service's credential, on the verifying side. `alert_writeback
# .service_token_valid` reads this and only this — it does NOT fall back
@@ -730,6 +731,7 @@ services:
AISOC_PUBLISHED_BIND_ADDRS: ${AISOC_CONSOLE_BIND_ADDR:-${AISOC_BIND_ADDR:-127.0.0.1}},${AISOC_BIND_ADDR:-127.0.0.1}
SECRET_KEY: ${SECRET_KEY:-}
AISOC_SERVICE_TOKEN: ${AISOC_SERVICE_TOKEN:-}
+ AISOC_API_SERVICE_TOKEN: ${AISOC_API_SERVICE_TOKEN:-}
# The agents service's own service credential, used in both directions:
# presented as `X-AiSOC-Service-Token` on the API's internal context
# routes, and verified by `app.security.tenant_scope` when the API calls
diff --git a/docs/decisions/0009-auto-close-without-an-earned-grant.md b/docs/decisions/0009-auto-close-without-an-earned-grant.md
new file mode 100644
index 000000000..a76235924
--- /dev/null
+++ b/docs/decisions/0009-auto-close-without-an-earned-grant.md
@@ -0,0 +1,83 @@
+# 0009: Whether a tenant with no closure policy may auto-close
+
+- **Status**: Proposed. **Awaiting a maintainer decision.**
+- **Date**: 2026-10-04
+- **Context**: fix-pass item 1.4 (`plans/aisoc_fix_pass_plan.plan.md`)
+
+## The question
+
+A tenant that has **no** `aisoc_closure_policies` row today auto-closes any
+alert whose confidence clears `AISOC_AUTO_CLOSE_THRESHOLD`, a process-wide
+default of 0.85, with no grant of any kind.
+
+A tenant that **does** have a policy row is held to `require_grant`, which
+migration 078 defaults to true, so it may only auto-close a class that has
+earned an `auto_close` grant in shadow mode.
+
+Configuring a policy therefore makes the product strictly more cautious than
+configuring nothing. That is the wrong way round, and it contradicts the goal
+gap-closure Phase 2 was written for: closure should be earned per class rather
+than assumed from a number.
+
+## Why this is being asked now and not before
+
+The question was invisible while the grant reader was broken. It queried a
+table called `autonomy_grants`, which exists in no migration -- the real one is
+`aisoc_autonomy_grants` -- and filtered on `revoked_at` and `expires_at`, which
+that table does not have. The exception was caught, logged at warning and
+turned into `False`.
+
+So the observable behaviour was: **no policy, auto-close at 0.85; a policy,
+never auto-close at all.** Neither half was the designed behaviour, and fixing
+the query is what makes the default a live choice rather than a dead one.
+
+The query is fixed (item 1.4) and the live suite
+`tests/isolation/test_closure_grant_live.py` asserts it against real Postgres.
+This ADR is only about the default for a tenant that has configured nothing.
+
+## Options
+
+### A. Keep the current default
+
+A tenant with no policy keeps auto-closing at 0.85.
+
+- Nothing changes for any existing deployment.
+- The inconsistency above stays: the cautious configuration is the one that
+ looks like neglect.
+
+### B. No auto-close without an earned grant (recommended)
+
+A tenant with no policy row does not auto-close. Earning a grant in shadow
+mode, or setting an explicit per-tenant opt-in, is what enables it.
+
+- Consistent with Phase 2's goal and with what
+ `apps/docs/docs/operations/shadow-mode.md` describes.
+- **Breaking.** A deployment relying on the implicit 0.85 would stop
+ auto-closing on upgrade. The symptom is a queue that stops draining, which
+ is visible but unwelcome, so it needs a release note under `BREAKING` and a
+ named opt-in for operators who want the old behaviour back.
+- The opt-in should be explicit and audited rather than an environment
+ variable, so that "this tenant closes alerts without earning it" is a
+ recorded decision with an owner.
+
+### C. Make the default explicit either way
+
+Keep the behaviour but require the operator to state it, so no deployment
+auto-closes by inheriting a constant nobody chose.
+
+- Also breaking, and it asks every operator a question many will not have an
+ answer to.
+
+## Recommendation
+
+**Option B**, with the opt-in recorded per tenant rather than per process.
+
+## Decision
+
+**Not taken.** This is a maintainer call: it changes what a running deployment
+does to alerts without being asked, and the fix pass deliberately does not make
+that choice on the maintainer's behalf. Recorded as `M4` in
+`FIX_PASS_PROGRESS.md`.
+
+Until it is taken, the behaviour is Option A with the grant reader working,
+which is the first time the two halves have been consistent with each other.
diff --git a/plans/aisoc_fix_pass_plan.plan.md b/plans/aisoc_fix_pass_plan.plan.md
new file mode 100644
index 000000000..851b742da
--- /dev/null
+++ b/plans/aisoc_fix_pass_plan.plan.md
@@ -0,0 +1,319 @@
+# AiSOC fix pass
+
+> **Status**: Locked plan. Implement as specified, do not edit.
+> **Captured**: 2026-10-03, at commit `b1c6925a` (v16.0.0 plus one security fix).
+> **Tracking**: `FIX_PASS_PROGRESS.md` mirrors progress; this file is the source of truth.
+
+### Why this plan exists
+
+An independent audit at v16.0.0 found that features the gap-closure and parity programs record as shipped fail on a real deployment, while every recent CI run on `main` is green. The pattern repeats: the test for each feature mocks exactly the boundary where its defect lives. Examples are a fake database connection that answers any query, an injected MCP session that bypasses the transport, and mocked sessions that never reach a missing table.
+
+The claim-to-gate matrix reports 291 of 291 rows GATED while several of those claims are false.
+
+This pass does three things:
+
+- fixes the defects,
+- makes the gates catch this class of defect, and
+- corrects every claim that says more than the code does.
+
+It adds no capabilities beyond what a fix needs. The open phases resume afterwards under their own plans: gap-closure 6.4, 7, 9, 10 and 13.4, and parity Phases 3 and 4, 5.5 to 5.7, and 6.2 to 6.10.
+
+### Rules for this pass
+
+These are in addition to the inherited rules.
+
+- **Reproduce first, through the real path**, as the instructions above describe.
+- **Claim rows.** A row whose gate does not drive the production path on the profile it names is downgraded to PARTIAL now, with the reason. It returns to GATED only in the PR that adds a gate which does. The ratchet limits NO GATE rows, not PARTIAL, so downgrading is always allowed.
+- **Progress files.** Where `GAP_CLOSURE_PROGRESS.md` or `PARITY_PROGRESS.md` marks an item `[x]` that this audit found broken, change it to `[~]` with a pointer to the item here.
+- **No new containers in CORE.** No feature work beyond what a fix requires.
+- **House style.** No em dashes, Conventional Commits with DCO sign-off, no tool attribution, and no competitor product names (`scripts/check_competitor_names.py` scans plan files).
+
+### Wave 1: Security and tenancy (P0)
+
+**1.1 The agents service authenticates to the API as a service, for the tenant it is working on.**
+
+- **Defect: no credential is ever delivered.**
+ - The customer tools, the hunting agent and the sandbox tool authenticate with `AISOC_AGENTS_API_KEY` (`services/agents/app/tools/customer_tools.py`, `services/agents/app/hunt/agent.py`, `services/agents/app/tools/sandbox.py`).
+ - No compose file, `.env.example` or Helm value delivers that key, so on `make up` all three answer "could not check".
+- **Defect: one key serves every tenant.** When the key is set, it belongs to one tenant, so every tenant's investigation reads that tenant's data.
+- **Defect: the key cannot carry the right scopes.** The routes need `actions:read`, `lake:query` and `hunts:read`, and none of these is in `VALID_SCOPES` (`services/api/app/api/v1/endpoints/api_keys.py`). Only a `*` key works.
+- **Defect: the lake pivots send no credential.** The 11 pivots send only `X-Tenant-ID` (`call_investigation_tool` in `services/agents/app/tools/investigation.py`), so outside dev mode they get 401.
+- **Fix:**
+ - Replace the API key with the service-token pattern the context routes already use (`services/agents/app/security/tenant_scope.py`, and dual-mode routes such as `/api/v1/feedback/context-statements`).
+ - The agents service sends the service token that compose already delivers, together with the tenant of the alert or run it is working on.
+ - The API accepts a service caller only with a named tenant, and scopes every query to it.
+ - Remove `AISOC_AGENTS_API_KEY`.
+- **Reproduce:** set up two tenants, each with its own configured connector. An investigation for tenant B, through the real API app, must reach B's connector and never A's. Today it either fails authentication or reads A.
+- **Gate:** extend `scripts/check_service_token_wiring.py` beyond names matching `*SERVICE_TOKEN` and `*INTERNAL_TOKEN`. It should cover every credential a service reads for an internal call, and fail any service-to-service HTTP call that carries no credential.
+
+**1.2 Federated search authenticates to the connectors service.**
+
+- **Defect:**
+ - `services/api/app/api/v1/endpoints/federated.py` (the `client.post(url, json=payload)` call, around line 253) posts to the connectors query route with no Authorization and no tenant header.
+ - The connectors router requires a console session or a service token (`require_console_or_service_auth`, `services/connectors/app/api/router.py`), so every SIEM answers 401.
+ - This breaks the agent's federated tool, console federated search and the retro-hunt SIEM sweep.
+ - The catalog proxy already does this correctly (`_service_token` in `services/api/app/api/v1/endpoints/connectors.py`, around line 330).
+- **Fix:** one shared helper for every API-to-connectors call, carrying the service token and the tenant.
+- **Reproduce:** drive `/api/v1/federated/search` against the real connectors FastAPI app, through an ASGI transport with its real auth dependency, not a mock that accepts anything.
+
+**1.3 Vendor reads match the connectors tenants actually save.**
+
+- **Defect:**
+ - The read executors in `services/actions/app/live_actions/investigation_reads.py` register vendor ids `defender`, `entra` and `aws`.
+ - Saved connectors use the catalog types `azure_defender`, `azure_entra` and `aws_cloudtrail`.
+ - `services/api/app/services/agent_tools/vendor_reads.py` matches with `by_type.get(vendor_id)` and no alias.
+ - So three of the five vendors added in gap-closure 4.2 are never offered to the agent, and they dispatch as `no_integration`.
+- **Fix:** one alias map from executor vendor id to catalog connector types, used by every matcher: `vendor_reads.py`, `_pick_connector` in `services/api/app/services/playbook_step_dispatch.py`, and live-action dispatch.
+- **Gate:** a new `scripts/check_vendor_catalog_ids.py` that fails when any registered executor's vendor id resolves to no catalog connector type.
+
+**1.4 Earned auto-close grants are honoured, and the closure default is decided.**
+
+- **Defect: the grant reader queries a table that does not exist.**
+ - `_has_auto_close_grant` in `services/agents/app/closure/policy.py` queries `autonomy_grants`, filtering on `revoked_at` and `expires_at`.
+ - The real table is `aisoc_autonomy_grants` (`services/api/migrations/067_autonomy_grants.sql`), and it has its own columns.
+ - `require_grant` defaults to true (`078_closure_policy_and_kill_switch.sql`), so a tenant that enables a closure policy can never auto-close.
+ - The claim row "Alert closure obeys per-tenant, per-class policy" is gated by a test whose fake connection answers any query.
+- **Fix the query:** read the real table and its state columns.
+- **Decide the default in an ADR.** Today a tenant with no policy row auto-closes at `AISOC_AUTO_CLOSE_THRESHOLD` (0.85) with no grant at all, which contradicts gap-closure Phase 2's goal.
+ - Recommended: no auto-close without an earned grant, plus an explicit, audited per-tenant opt-in to the old threshold.
+ - If adopted, this is BREAKING.
+- **Reproduce:** on live Postgres with the migrations applied, a tenant with an enabled policy and an earned grant auto-closes a benign verdict at 0.99 confidence. Today it is refused with "relation autonomy_grants does not exist".
+- **Gate:** `scripts/check_raw_sql_columns.py` must fail on any statement against a table no migration creates. Remove the `UNMIGRATED_TABLES` credit, or turn each entry into an expiring exception with a written reason.
+
+**1.5 The hunting agent returns its findings, for the right tenant, with a ledger.**
+
+- **Defect:**
+ - The agents route builds `matches=list(getattr(result, "matches", []) ...)` (`services/agents/app/api/router.py`), but `HuntAgentResult` only has `findings`. So every hunt that found rows reports `checked: true, matches: []`, which reads as clean.
+ - The API route requires `hunts:read` (`services/api/app/api/v1/endpoints/agents.py`), which no role holds except `*`.
+ - The tenant the route checks is never passed on (1.1 fixes this).
+ - `_record` in `services/agents/app/hunt/agent.py` calls the ledger with a signature the real ledger rejects. The tests pass because they use a fake ledger.
+- **Fix:**
+ - Return the findings.
+ - Grant `hunts:read` to the analyst and threat-hunter roles through `app.core.role_grants`.
+ - Pass the tenant through.
+ - Write through the real ledger API.
+- **Reproduce:** through the real agents app and the real ledger class, a hunt over matching fixture rows returns those rows and writes ledger rows.
+- **Gates:**
+ - Wire `scripts/check_hunt_agent_boundary.py` into a workflow; today no workflow runs it.
+ - Fix `_references` in `scripts/check_gate_coverage.py` so a script path that only appears in a docstring or comment does not count as an invocation.
+
+### Wave 2: The MCP client works over a real connection
+
+**2.1 Transport.**
+
+- **Defect:** `_CappedStream` in `services/agents/app/mcp/client.py` is not an `httpx.AsyncByteStream`. The locked httpx 0.28.1 asserts `isinstance(response.stream, AsyncByteStream)` in `_send_single_request`, so every real `list_tools` and `call_tool` fails.
+- **Fix:** subclass `httpx.AsyncByteStream`, implementing `__aiter__` and `aclose`.
+- **Reproduce:** run a real FastMCP streamable-HTTP server on a localhost socket and connect through `_CappedTransport`, with no `session_factory`. Keep the byte-cap test.
+
+**2.2 Tool names.** `mcp.<server>.<tool>` (`services/agents/app/mcp/policy.py`) contains dots, which OpenAI-compatible function names reject (`^[a-zA-Z0-9_-]{1,64}$`). Use a valid, reversible encoding, and test every bound name against that pattern.
+
+**2.3 SSRF and air-gap.**
+
+- **Check then resolve again.** The guard resolves the hostname once, and httpx resolves it again when it connects. Either pin the connection to the address that was vetted, or state the limit honestly.
+- **Air-gap: code, test and doc disagree.** `apps/docs/docs/operations/mcp-client.md` says internal servers stay reachable in air-gap mode. But `test_air_gap_mode_permits_an_internal_server` in `services/agents/tests/test_mcp_client.py` asserts they are refused, and compose passes no allowlist variable. Make all three agree.
+
+**2.4 Docs and claim rows.**
+
+- `apps/docs/docs/quickstart.md` says the `npx @aisoc/mcp` fallback works on a fresh clone, but the package is unpublished.
+- `apps/docs/docs/operations/faq.md` and `apps/docs/docs/api/rest.md` quote stale tool counts, and say playbook actions can be run when only a dry-run preview exists.
+- The claim row "MCP server exposes 14 tools" is stale.
+- Generate the tool count from the server's registry and gate it.
+
+### Wave 3: Replay, shadow mode and evaluation measure the real thing
+
+**3.1 Frozen context, not empty context.**
+
+- **Defect:**
+ - An ordinary replay sends no context block (`services/api/app/services/replay_evaluation/job.py`, around line 228), so `services/agents/app/api/replay_router.py` builds an empty snapshot.
+ - The runner passes no business-context rules (`business_context=` in `services/agents/app/replay/runner.py`), while production enables them (`services/agents/app/main.py`).
+- **Fix:** capture organisation memory, priors, context statements, active skills and business-context rules as of the split point, and send them.
+- **Docs:** `replay.md`, `triage-context.md` and `tenant-skills.md` already claim this happens.
+
+**3.2 No side effects.** A replay must not:
+
+- write shadow-mode decisions (`aisoc_shadow_decisions`)
+- write to the live cost ledger
+- fire the alert playbook trigger (`alert_trigger.run_for_alert` in `services/agents/app/workers/fused_alert_consumer.py`)
+- receive a live verdict from the cost-governor dedup cache (`Decision.DEDUPLICATED`, around line 554)
+
+Reproduce each with shadow mode on and `AISOC_ALERT_PLAYBOOKS_ENABLED=1`.
+
+**3.3 Input shapes.**
+
+- **Defect:** the Elastic and Defender readers return shapes their connector `normalize()` does not expect, and host, user and severity are lost.
+ - Elastic: the reader returns search hits; `normalize()` expects flat ES|QL rows.
+ - Defender: the reader returns Defender for Endpoint alerts; `normalize()` expects Graph alerts.
+- **Fix:** add a per-source adapter.
+- **Test:** one end-to-end test covering all five sources: reader output, then `normalize()`, then a triage input with host, user and severity present.
+
+**3.4 Model attribution.** `model=_opt_text(getattr(state, "model_used", None))` in `fused_alert_consumer.py` (around line 785) is always `None`, because nothing sets `model_used` on the triage path, so per-model agreement is empty. Set it from the call that actually answered.
+
+**3.5 Non-degenerate acceptance.** The Phase 1 reproducibility test currently passes over a constant output: every verdict is benign at 0.10. Drive it with a recorded model-response fixture through the real gateway client so verdicts vary, and assert that at least two classes are predicted.
+
+**3.6 Tool calls.** The runner hard-codes `decision.tool_calls = 0`. Record the real count from the ledger.
+
+**3.7 Demotion without a page load.**
+
+- **Defect:**
+ - Demotion runs only when someone reads the grants list (`services/api/app/api/v1/endpoints/autonomy_policy.py`).
+ - Dispatch reads `state = 'granted'` without re-checking the evidence (`services/actions/app/services/tenant_policy.py`, around line 228).
+ - `shadow-mode.md` claims dispatch does re-check.
+- **Fix:** evaluate drift whenever a shadow decision or a closure lands, or on a scheduled sweep started in the API lifespan, and have dispatch call the same evaluator.
+- **Reproduce:** injected disagreements trigger demotion with no read of the API.
+
+**3.8 Skills.**
+
+- **Activation.** Today activation only checks that two evaluation ids are attached and the version matches (`services/api/app/services/tenant_skills/store.py`, around lines 332 to 345). Require that both evaluations completed successfully, and that a delta was computed over the alerts the skill matches rather than the whole window.
+- **LLM path only.** Skills reach triage only on the LLM path (`fused_alert_consumer.py`, around lines 561 to 574). The backtest report must say so.
+- **No console editor.** Retract the console claims until parity 6.3 builds one: the claim row "A tenant can author an investigation skill in the console", `tenant-skills.md` and `triage-context.md`.
+
+**3.9 The weekly live evaluation can actually run.**
+
+- **Install.** `.github/workflows/wet-eval.yml` installs a hand-written dependency list that lacks `opentelemetry` and `mcp`. The agent cannot import, so the job prints "not measured" and goes green.
+ - Install from `scripts/service_requirements.py agents --locked`, as CI does.
+ - Fail the job on an import or setup error, distinguishing "no key" from "broken".
+- **Missing data.** The live injection path reports neither proposed actions nor the tools used, so the unsafe-action rate and tool-call deviation cannot be measured. Pass both through from the ledger.
+- **Claim row.** "The weekly job cannot go green having measured nothing" is false until this lands.
+
+**3.10 The 9.3% flip rate.** The figure in `CHANGELOG.md` and `RELEASES.md` came from one hand run, on a model that is not the shipped one. State the model, the date, the sample size and that it was a single manual run, or remove the figure.
+
+**3.11 CI runs the live tests.**
+
+- **Dispositions test.** `tests/isolation/test_recent_dispositions_cutoff_live.py` runs in no job that has a database. The claim row "Analyst dispositions are point-in-time during a replay" cites a workflow that skips it.
+- **Path filters.** The replay end-to-end and autonomy live tests in `.github/workflows/integration.yml` have pull-request path filters that exclude the code they cover: the triage worker, the SIEM clients, the connector mappings and the autonomy code. Fix the filters.
+- **Skips.** When a database is expected and missing, the job must fail, not skip.
+
+**3.12 Pivot counting.** `_classify_pivots` in `services/agents/app/investigator/deep_investigation.py` counts a call that returned `{"error": ...}` as a pivot. Count only calls that succeeded.
+
+### Wave 4: Enterprise identity, white-label and metering
+
+**4.1 SSO completes in the console.**
+
+- **Defect:**
+ - The OIDC callback returns the token in the URL fragment (`services/api/app/auth/oidc.py`, around line 433). Its comment says the console stores it, but no console code reads the fragment.
+ - The login page offers only email and password.
+ - SAML trust comes from process-wide environment variables (`_saml_settings` in `services/api/app/auth/saml.py`), while the stored `metadata_url` and `metadata_xml` in `aisoc_sso_connections` are never read.
+- **Fix:**
+ - A console callback handler that reads the token from the fragment, stores it and clears the fragment.
+ - An SSO entry on the login page.
+ - Use the per-connection metadata instead of the environment variables.
+- **Reproduce:** end to end in CI, against a containerised OIDC provider and a SAML IdP. The claim row "SSO completes a sign-in" stays PARTIAL until this passes.
+
+**4.2 Console MFA.** TOTP with recovery codes, per-tenant enforcement, and audited enrolment and reset. Today passkeys exist only on the mobile responder.
+
+**4.3 White-label reaches every surface it claims.**
+
+- **Defect:**
+ - `sender_name` is read nowhere except the resolver.
+ - `services/api/app/services/email_approval.py` hard-codes "[AiSOC]", and `send_approval_email` has no caller at all, so the signed email-approval fallback is not wired.
+ - The Slack and Teams bots carry no branding.
+ - Investigation and replay PDFs are unbranded.
+ - Colours and support contacts are never applied.
+ - The login page and page titles hard-code the product name.
+ - The sidebar logo is a plain `<img>` pointing at `/api/v1/branding/assets/{id}`, which needs a bearer header an `<img>` cannot send. Verify in a browser and fix.
+- **Fix:** brand every surface the doc and the claim row list, or narrow both to what is actually branded.
+
+**4.4 Metering counts what happened.**
+
+- `triages_model` counts alerts with `ai_summary` or `ai_score` set, but the deterministic path writes those columns too. Use the recorded path in `investigation_runs.model_used`.
+- `investigations` counts every auto-triage.
+- `events_ingested` is not metered on any profile.
+- Actions are not split by tier.
+- The CSV is per tenant, not per organization.
+
+Fix these, or narrow the doc to what is measured.
+
+**4.5 SCIM.**
+
+- The Okta and Entra sequences run on SQLite against a bare app. Add PATCH, group membership and deactivation to the live Postgres test, through the real app.
+- `apps/docs/docs/operations/scim.md` says tokens can be minted in the console. Add the page, or fix the doc.
+
+**4.6 Maturity evidence.** The maturity table rates "SCIM 2.0, white-label, usage metering" as Stable on live Postgres, citing `tests/isolation/test_scim_live.py`. That file tests neither white-label nor metering (`scripts/check_maturity_table.py`). Each row must name evidence that actually exercises it.
+
+### Wave 5: Hunting, intel, sandbox and operations wiring
+
+**5.1 Retro-hunts can be turned on.**
+
+- **Defect:**
+ - Compose does not pass `RETRO_HUNT_ENABLED` through.
+ - Tenant opt-in exists only as a raw `retro_hunt_settings` row.
+- **Fix:**
+ - Add the compose pass-through.
+ - Add an API route and a console toggle for opt-in.
+ - Test the real consumer loop (`run_forever` and `_build_consumer` in `services/api/app/workers/retro_hunt_consumer.py`), fed by the threat-intel pipeline's own producer rather than a hand-written message.
+
+**5.2 KEV exposure gets data.**
+
+- **Defect:** vulnerability rows come only from `POST /api/v1/assets/vulnerabilities`. The Tenable connector models findings as alerts and never writes them there.
+- **Fix:** write Tenable findings into the vulnerability table, scoped to the tenant.
+- **Test:** exposure end to end.
+
+**5.3 Hunts over the lake.**
+
+- **Defect:**
+ - The agents image lacks a ClickHouse driver; `lake-live.yml` installs one by hand.
+ - In the `full` profile, the agents container gets no ClickHouse settings.
+ - The hunts filter on a `source` field and on derived fields that lake events do not carry.
+- **Fix:** add the dependency, the settings and the field mapping, or narrow the README row "68-hunt YAML library, replayed against tenant events".
+
+**5.4 Sandbox and air-gap settings reach the API.**
+
+- Compose passes neither `AISOC_AIRGAPPED` nor the sandbox provider settings to the API, and the air-gap overlay sets the flag only on the agents service.
+- With only the mock provider configured, phishing records nothing, which reads as "checked and clean". Record "no sandbox configured" instead.
+- The agent tool labels any 403 as air-gap mode.
+
+**5.5 Release channel.**
+
+- **Defect:**
+ - `.github/workflows/release.yml` moves `stable` on any non-major release (around line 676), so the first minor or patch of a new major carries it across. That is how it reached v12, v13 and v15.
+ - The claim row "`stable` does not cross a major on its own" is backed by a test that only looks at one release at a time.
+- **Fix:**
+ - Track the channel's current major, and only move `stable` automatically within it.
+ - Replace that test with one that runs a sequence of releases.
+ - Add the `stable` chart channel the release policy names.
+
+**5.6 Upgrade fixture.** `scripts/upgrade_fixture.sql` inserts into `cases`, which `083_one_case_table.sql` renames, so the upgrade test will break once the previous minor is 16.x. Fix it now.
+
+**5.7 Chaos and HA in CI.**
+
+- `chaos.yml` says it runs the graders live, but it only runs their self-tests. `scripts/chaos/fusion_restart.py` has run live once, by hand.
+- `integration.yml`'s kill test posts one event while fusion is down, and checks for loss only.
+- **Fix:** run the grader live on a schedule (compose is enough), killing fusion mid-stream and checking for duplicates as well as loss.
+- The claim row "A fusion replica can be destroyed mid-stream" stays PARTIAL until a scheduled job runs it.
+
+**5.8 Performance honesty.**
+
+- The floors in `perf.yml` (`--assert-eps-floor 5`, `--assert-p95-ceiling-ms 120000`) sit roughly 35 and 100 times below the published figures. Set them relative to the published numbers.
+- `scripts/check_perf_results.py` should require results for both deployments and a freshness bound.
+- `scripts/perf/load_profiles.py` and `scripts/perf/throughput_claims.py` have no caller and contradict `apps/docs/docs/operations/performance.md`. Wire them in or delete them.
+- Label the kind run as single-host.
+
+### Wave 6: Retract what is not built
+
+Each item below is presented as shipped and has no production caller. Retract or narrow the text now, wherever it appears: claim rows, README, release notes and docs. Each feature comes back with the PR that builds it, under the parity plan.
+
+- **The v16 detection lifecycle.** Nothing reads `detection_rule_versions`, `detection_shadow_matches`, `environment` or `shadow_until` (`084_detection_lifecycle.sql`). Fusion ignores them, and there is no rollback route.
+- **Enterprise IAM.** Workload identity, time-boxed elevation and ABAC conditions (`087_enterprise_iam.sql`) are unused, and `narrow_by_conditions` in `services/api/app/core/role_grants.py` has no caller.
+- **Overclaims in `apps/docs/docs/intro.md`:**
+ - a coverage advisor that "recommends new rules" and offers "one-click detection generation"
+ - "Qdrant RAG memory"
+ - "STIX/TAXII publishing" with collection management, when the TAXII collections are demo-only literals
+- **The phishing playbook** "retracts the message fleet-wide" through an `http` step to an unset `${EMAIL_GATEWAY_URL}` that continues on failure. This is in both `playbooks/packs/v1/phishing/phishing-investigation.playbook.json` and `services/agents/app/playbook/packs/v1/phishing/phishing-investigation.playbook.json`.
+- **Two broken detection-tuning routes:**
+ - The detection-loop suggestion route (`services/api/app/api/v1/endpoints/detection_loop.py`) reads `aisoc_alerts` and `aisoc_detection_rules`, which no migration creates, and keeps its suggestions in a process-global dict.
+ - `/api/v1/detection/tuning/auto-suggest` opens comment-only proposals with a null `base_rule_id` that cannot pass `/decide`.
+ - Point both at the real tables with a persistent store, or remove both routes and their claims. Either way, the gate from 1.4 must catch the next one.
+
+### Wave 7: Close the books
+
+- Re-run every claim row this pass touched. No row stays GATED unless its gate drives the production path on the profile the row names.
+- Update `GAP_CLOSURE_PROGRESS.md` and `PARITY_PROGRESS.md` so every box matches reality, with links to this pass's PRs.
+- Add one `[Unreleased]` entry to `CHANGELOG.md` in house style. Do not tag.
+- Write the final report in `FIX_PASS_PROGRESS.md`: each item's reproducing test, fix PR and negative control, and anything left `[!]`.
+
+### Maintainer-only items (record as [!], do not attempt)
+
+- Make the live jobs required checks with branch protection on `main`: replay end-to-end, autonomy live, retro-hunt live, SCIM live and chaos.
+- Fund provider keys so the weekly evaluation produces real numbers.
+- Provide a real multi-node cluster for the scale run.
+- Accept or overrule the closure default proposed in the 1.4 ADR.
diff --git a/scripts/check_gate_coverage.py b/scripts/check_gate_coverage.py
index b8fef0e7c..9c6e8bea0 100644
--- a/scripts/check_gate_coverage.py
+++ b/scripts/check_gate_coverage.py
@@ -505,8 +505,38 @@ class Surface:
# --------------------------------------------------------------------------
# Reachability
# --------------------------------------------------------------------------
+#: A line whose first non-blank character begins a comment. Both of the file
+#: kinds this gate reads use ``#``: YAML workflows, and the Python a workflow
+#: calls.
+_COMMENT_LINE = re.compile(r"^[ \t]*#.*$", re.MULTILINE)
+
+#: A triple-quoted block. Docstrings are where this repository explains which
+#: gate enforces a module's contract, so a gate's own path appears in the prose
+#: of the thing it checks far more often than in anything that runs it.
+_TRIPLE_QUOTED = re.compile(r'"""[\s\S]*?"""|\'\'\'[\s\S]*?\'\'\'')
+
+
+def _without_prose(text: str) -> str:
+ """``text`` with comments and triple-quoted blocks removed.
+
+ A gate's filename appearing in prose is not an invocation of it, and this
+ distinction is the whole value of the gate. `check_hunt_agent_boundary.py`
+ is run by **no workflow at all**, and was counted as reachable because five
+ modules and two test files name it in a docstring saying which gate
+ enforces their contract. Reporting that as coverage is worse than reporting
+ nothing, because it is the answer somebody checks instead of looking.
+
+ Whole-line comments and whole triple-quoted blocks only. A ``#`` inside a
+ quoted string is not a comment, and guessing which is which would make the
+ resolver wrong in the other direction, which is the worse direction: an
+ over-broad resolver manufactures coverage.
+ """
+ return _COMMENT_LINE.sub("", _TRIPLE_QUOTED.sub("", text))
+
+
def _references(script: str, text: str) -> bool:
"""Whether `text` invokes or imports `script` (a bare filename)."""
+ text = _without_prose(text)
stem = script[:-3] if script.endswith(".py") else script
return any(
re.search(pattern, text)
diff --git a/scripts/check_raw_sql_columns.py b/scripts/check_raw_sql_columns.py
index 07eff769f..68cb4582a 100644
--- a/scripts/check_raw_sql_columns.py
+++ b/scripts/check_raw_sql_columns.py
@@ -188,6 +188,75 @@ UNMIGRATED_TABLES: dict[str, str] = {
),
}
+#: Tables that are real, but not in Postgres. A statement against one cannot
+#: be compared with a migration because no migration should declare it.
+#:
+#: This list exists so that **everything not on it fails**. Before it, a
+#: ``SELECT`` against a table no migration creates was silently recorded as
+#: "not compared", on the reasoning that the gate cannot tell which engine a
+#: statement targets. That is true, and the remedy is to say which, once,
+#: rather than to excuse the whole class: the closure policy read a table
+#: called ``autonomy_grants`` that exists in no engine at all, and this gate
+#: passed over it for as long as it shipped.
+FOREIGN_ENGINE_TABLES: dict[str, str] = {
+ # ClickHouse. The lake's own database qualifies its tables, so the parser
+ # sees the database name as the table.
+ "aisoc": "ClickHouse: the event lake qualifies every table as `aisoc.<name>`, so the parser reads the database name here.",
+ "system": "ClickHouse's own `system.*` introspection tables.",
+ # Postgres catalogues. Real, and declared by Postgres rather than by us.
+ "pg_roles": "A Postgres catalogue, not an application table.",
+ "pg_class": "A Postgres catalogue, not an application table.",
+ "information_schema": "The SQL standard catalogue, not an application table.",
+ # osquery, evaluated on an endpoint by the agent rather than by a database.
+ "processes": "osquery virtual table, evaluated on the endpoint.",
+ "process_open_sockets": "osquery virtual table, evaluated on the endpoint.",
+ "logged_in_users": "osquery virtual table, evaluated on the endpoint.",
+ "file": "osquery virtual table, evaluated on the endpoint.",
+ "proctree": "osquery virtual table, evaluated on the endpoint.",
+ "deb_packages": "osquery virtual table, evaluated on the endpoint.",
+ "rpm_packages": "osquery virtual table, evaluated on the endpoint.",
+ "homebrew_packages": "osquery virtual table, evaluated on the endpoint.",
+ # Vendor SQL this product sends to a customer's own warehouse.
+ "SNOWFLAKE": "Snowflake's account usage views, queried in the customer's own warehouse.",
+ "SetupAuditTrail": "A vendor audit view, queried in the customer's own system.",
+}
+
+#: Names the parser reads as a table that are not one.
+#:
+#: Common table expressions are **detected** rather than listed here, because
+#: three were on this list before `_cte_names` existed and the fourth arrived
+#: the same week. What is left is prose: a docstring whose English happens to
+#: put a word after the token SELECT.
+PARSER_ARTEFACTS: dict[tuple[str, str], str] = {
+ (
+ "services/actions/app/services/autonomy_evidence_rules.py",
+ "recent",
+ ): (
+ "`recent` is a common table expression, and `_cte_names` resolves it where the WITH clause and "
+ "the reference share one string literal. This module composes the statement across two, so the "
+ "node holding `FROM recent` does not contain its own declaration."
+ ),
+ (
+ "services/actions/app/live_actions/builtins.py",
+ "the",
+ ): "Prose in the module docstring, not a statement. The word follows `SELECT` in an English sentence.",
+}
+
+#: Tables a statement names that no migration creates and that are not foreign
+#: either: a defect, recorded with the item that removes it.
+#:
+#: An entry here is a debt with an owner, not an exemption. Both of these are
+#: fix-pass item 6.5, which either points the two detection-tuning routes at
+#: the real tables or removes them.
+KNOWN_MISSING_TABLES: dict[str, str] = {
+ "aisoc_alerts": (
+ "No migration creates this. The detection-loop suggestion route and the business-context "
+ "reader query it, so both return nothing on every deployment. Fix-pass item 6.5 removes "
+ "this entry by pointing them at `alerts` or removing the routes."
+ ),
+ "aisoc_detection_rules": ("No migration creates this. Same two routes, same item 6.5."),
+}
+
#: Statements whose table or column list is assembled at runtime, and so
#: cannot be compared against anything, with the reason.
DYNAMIC_SQL: dict[tuple[str, str], str] = {
@@ -379,6 +448,18 @@ def _select_columns(projection: str) -> tuple[list[str], bool]:
return columns, complete
+#: Names declared by a ``WITH`` clause. A CTE is a table for the length of one
+#: statement and is declared by that statement, so comparing it against a
+#: migration asks the wrong question. Detected rather than allowlisted: three
+#: were on an exceptions list before this, and the fourth would have been too.
+_CTE_NAME = re.compile(r"(?:\bWITH\s+(?:RECURSIVE\s+)?|,\s*)([a-z_][a-z0-9_]*)\s+AS\s*\(", re.I)
+
+
+def _cte_names(text: str) -> set[str]:
+ """Every name a ``WITH`` clause declares in this statement."""
+ return {m.group(1).lower() for m in _CTE_NAME.finditer(text)}
+
+
_UPDATE = re.compile(r"\bUPDATE\s+(?:ONLY\s+)?([^\s(;]+)\s+SET\b", re.I)
_DO_UPDATE = re.compile(r"\bDO\s+UPDATE\s+SET\b", re.I)
@@ -515,6 +596,7 @@ def statements_in(source: str, relpath: str) -> list[dict]:
if not text or not ("INSERT" in upper or "UPDATE" in upper or "SELECT" in upper):
continue
line = getattr(node, "lineno", 0)
+ ctes = _cte_names(text)
inserts: list[tuple[int, dict]] = []
for match in _INSERT.finditer(text):
@@ -524,6 +606,7 @@ def statements_in(source: str, relpath: str) -> list[dict]:
"kind": "INSERT",
"file": relpath,
"line": line,
+ "ctes": ctes,
"table": match.group(1).strip('"'),
"columns": columns,
# A statement with no column list names no column to check,
@@ -541,6 +624,7 @@ def statements_in(source: str, relpath: str) -> list[dict]:
"kind": "SELECT",
"file": relpath,
"line": line,
+ "ctes": ctes,
"table": match.group(2).strip('"'),
"columns": columns,
"columns_declared": True,
@@ -555,6 +639,7 @@ def statements_in(source: str, relpath: str) -> list[dict]:
"kind": "UPDATE",
"file": relpath,
"line": line,
+ "ctes": ctes,
"table": match.group(1).strip('"'),
"columns": columns,
"columns_declared": True,
@@ -624,6 +709,8 @@ def scan(root: pathlib.Path | None = None) -> dict:
credits: list[dict] = []
foreign_used: set[str] = set()
dynamic_used: set[tuple[str, str]] = set()
+ artefact_used: set[tuple[str, str]] = set()
+ missing_used: set[str] = set()
for statement in statements:
table = statement["table"]
@@ -655,7 +742,48 @@ def scan(root: pathlib.Path | None = None) -> dict:
# `detection_rules`, which is migrated and whose projection reads
# cleanly, so the overlay loaded nothing on every deployment while
# every unit test passed against a fake.
- if statement["kind"] == "SELECT" and (not statement["columns_complete"] or table not in schema):
+ # A table no migration creates, and which is not declared as living
+ # in another engine, is a statement that cannot succeed anywhere.
+ #
+ # This used to fall through to "not compared" along with every SELECT
+ # whose projection the parser could not read, and the two are not the
+ # same result: one means the gate could not judge, the other means the
+ # gate judged and the answer is no. `autonomy_grants` sat in that
+ # bucket for as long as it shipped.
+ if table not in schema and table != DYNAMIC:
+ if table.lower() in statement.get("ctes", ()):
+ # Declared by this very statement's WITH clause.
+ credits.append({**statement, "verdict": "common-table-expression", "detail": "declared by this statement's WITH clause"})
+ continue
+ if table in FOREIGN_ENGINE_TABLES:
+ foreign_used.add(table)
+ credits.append({**statement, "verdict": "foreign-engine", "detail": FOREIGN_ENGINE_TABLES[table]})
+ continue
+ if key in PARSER_ARTEFACTS:
+ artefact_used.add(key)
+ credits.append({**statement, "verdict": "parser-artefact", "detail": PARSER_ARTEFACTS[key]})
+ continue
+ if table in KNOWN_MISSING_TABLES:
+ missing_used.add(table)
+ credits.append({**statement, "verdict": "known-missing", "detail": KNOWN_MISSING_TABLES[table]})
+ continue
+ findings.append(
+ {
+ **statement,
+ # The existing vocabulary, not a second word for the same
+ # thing: the self-test and the renderer both read this.
+ "kind_of_finding": "unknown-table",
+ "column": "",
+ "verdict": "no-such-table",
+ "detail": (
+ f"no migration creates {table!r}, and it is not declared as a foreign-engine table. "
+ "This statement cannot succeed on any deployment."
+ ),
+ }
+ )
+ continue
+
+ if statement["kind"] == "SELECT" and not statement["columns_complete"]:
uncompared.append({**statement, "verdict": "select-not-resolved"})
continue
@@ -820,11 +948,21 @@ def _detects_injected_drift() -> list[tuple[str, bool]]:
(
# The gate knows which tables are migrated, not which engine a
# statement targets. osquery's virtual tables and the ClickHouse
- # lake have no Postgres migration, and failing those would be
- # the gate asserting something it cannot see.
- "declines a SELECT against an unmigrated table rather than failing it",
+ # lake have no Postgres migration, so each is **declared** in
+ # FOREIGN_ENGINE_TABLES and credited against that declaration.
+ #
+ # It used to decline the whole class instead, which is how a read
+ # of `autonomy_grants` -- a table in no engine at all -- passed
+ # for as long as it shipped.
+ "credits a SELECT against a table declared as living in another engine",
kinds("SQL = 'SELECT pid, name FROM processes'\n") == set(),
),
+ (
+ # The other half of that pair, and the one the declaration exists
+ # to make possible.
+ "fails a SELECT against a table that is neither migrated nor declared foreign",
+ kinds("SQL = 'SELECT id, name FROM autonomy_grants'\n") == {"unknown-table"},
+ ),
(
# A half-read projection that reports OK is the failure this
# gate exists to prevent, so an unreadable one is not compared.
@@ -936,8 +1074,8 @@ def main(argv: list[str] | None = None) -> int:
print(f"\nFAIL: {len(result['findings'])} raw SQL statement(s) naming something no migration creates:", file=sys.stderr)
for finding in result["findings"]:
print(
- f" [{finding['kind_of_finding']}] {finding['file']}:{finding['line']} "
- f"{finding['table']}.{finding['column']}\n {finding['detail']}",
+ f" [{finding.get('kind_of_finding') or finding.get('verdict', finding['kind'])}] {finding['file']}:{finding['line']} "
+ f"{finding['table']}{'.' + finding['column'] if finding.get('column') else ''}\n {finding['detail']}",
file=sys.stderr,
)
print(
diff --git a/scripts/check_service_token_wiring.py b/scripts/check_service_token_wiring.py
index 764a040ff..f3063e7b5 100644
--- a/scripts/check_service_token_wiring.py
+++ b/scripts/check_service_token_wiring.py
@@ -131,6 +131,208 @@ def _readers(root: Path) -> dict[str, set[str]]:
return found
+#: Hostnames that only resolve inside the deployment's own network. A request
+#: to one of these is a service-to-service call, and every route on the far
+#: side authenticates.
+INTERNAL_HOSTS: frozenset[str] = frozenset(
+ {
+ "api",
+ "agents",
+ "connectors",
+ "actions",
+ "fusion",
+ "ingest",
+ "enrichment",
+ "threatintel",
+ "realtime",
+ "ueba",
+ "purple-team",
+ "honeytokens",
+ "mcp",
+ "slack-bot",
+ "ingest-worker",
+ }
+)
+
+#: Helpers whose name says they build a request's headers. A call to one is
+#: resolved one level, into its own body, rather than assumed to be either
+#: credentialed or not.
+_HEADER_HELPER_RE = re.compile(r"(?:^|_)headers?$")
+
+
+#: A settings attribute whose name ends this way holds the address of a peer
+#: inside the deployment. ``settings.CONNECTORS_SERVICE_URL`` is as much a
+#: statement that the target is internal as the literal ``http://connectors:``
+#: is, and reading only literals missed the federated-search proxy entirely.
+_INTERNAL_SETTING_RE = re.compile(r"^[A-Z0-9_]*(SERVICE_URL|_URL)$")
+
+
+def _names_internal_host(node: ast.AST) -> bool:
+ """Whether anything under ``node`` names a host inside the deployment."""
+ for sub in ast.walk(node):
+ if isinstance(sub, ast.Constant) and isinstance(sub.value, str):
+ for host in INTERNAL_HOSTS:
+ if f"//{host}:" in sub.value or f"//{host}/" in sub.value:
+ return True
+ if isinstance(sub, ast.Attribute) and _INTERNAL_SETTING_RE.match(sub.attr):
+ root_name = sub.value
+ if isinstance(root_name, ast.Name) and root_name.id in {"settings", "config", "cfg"}:
+ service = sub.attr.rsplit("_SERVICE_URL", 1)[0].rsplit("_URL", 1)[0].lower().replace("_", "-")
+ if service in INTERNAL_HOSTS:
+ return True
+ return False
+
+
+def _url_is_internal(node: ast.AST, module: ast.Module) -> bool:
+ """Whether this URL expression names a service inside the deployment.
+
+ Two shapes reach here: a literal or f-string carrying ``//<host>:`` for one
+ of :data:`INTERNAL_HOSTS`, and an f-string built from a helper call like
+ ``_api_url()``, which is how every service here spells "the peer I am
+ configured to reach".
+
+ The helper is **resolved into its body** rather than matched on its name.
+ Naming alone said that the Trellix connector's ``_alerts_url()`` was an
+ internal peer, when it builds a customer's own vendor URL. A gate that
+ reports a vendor call as an uncredentialed internal one teaches people to
+ ignore it.
+ """
+ if _names_internal_host(node):
+ return True
+
+ # `url = _connectors_query_url(...)` then `client.post(url, ...)`. The
+ # federated-search proxy is spelled exactly this way, and reading only the
+ # argument expression saw a bare name and concluded nothing.
+ if isinstance(node, ast.Name):
+ for assign in ast.walk(module):
+ if isinstance(assign, ast.Assign) and any(isinstance(t, ast.Name) and t.id == node.id for t in assign.targets):
+ if _url_is_internal(assign.value, module):
+ return True
+ return False
+
+ for sub in ast.walk(node):
+ if not isinstance(sub, ast.Call):
+ continue
+ func = sub.func
+ name = func.attr if isinstance(func, ast.Attribute) else getattr(func, "id", "")
+ if not name:
+ continue
+ for defn in ast.walk(module):
+ if isinstance(defn, ast.FunctionDef | ast.AsyncFunctionDef) and defn.name == name:
+ if _names_internal_host(defn):
+ return True
+ return False
+
+
+def _carries_credential(call: ast.Call, module: ast.Module) -> bool:
+ """Whether this request passes an Authorization header.
+
+ Resolved one level through a ``_headers()``-shaped helper, because every
+ service that does this correctly factors the credential into one, and a
+ gate that only read the call site would report each of them as a finding.
+ """
+ headers = next((kw.value for kw in call.keywords if kw.arg == "headers"), None)
+ if headers is None:
+ return False
+
+ if any(
+ isinstance(sub, ast.Constant) and isinstance(sub.value, str) and sub.value.lower() == "authorization" for sub in ast.walk(headers)
+ ):
+ return True
+
+ # `headers=_headers(tenant)` or `headers=self._auth_headers()`.
+ if isinstance(headers, ast.Call):
+ func = headers.func
+ name = func.attr if isinstance(func, ast.Attribute) else getattr(func, "id", "")
+ if _HEADER_HELPER_RE.search(name):
+ for node in ast.walk(module):
+ if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef) and node.name == name:
+ if any(
+ isinstance(sub, ast.Constant) and isinstance(sub.value, str) and sub.value.lower() == "authorization"
+ for sub in ast.walk(node)
+ ):
+ return True
+ # A helper that delegates to the shared builder rather
+ # than spelling the header itself. One hop further,
+ # because consolidating three copies into one module is
+ # the fix, and a gate that then reports the consolidated
+ # callers would argue against its own remedy.
+ return any(
+ isinstance(sub, ast.Call)
+ and _HEADER_HELPER_RE.search(sub.func.attr if isinstance(sub.func, ast.Attribute) else getattr(sub.func, "id", ""))
+ for sub in ast.walk(node)
+ )
+ # A helper this module does not define, named like a header
+ # builder. Imported from somewhere that does build one.
+ return True
+ # `headers=dict(base, **extra)` and similar: a name we cannot resolve is
+ # not evidence of a credential, but flagging every one of them would make
+ # the gate unusable. Resolved names only.
+ if isinstance(headers, ast.Name):
+ for node in ast.walk(module):
+ # `headers = {...}` and `headers["Authorization"] = ...` alike.
+ # The second shape is how a caller adds the credential only when
+ # it has one, and reading only the first reported the ingest
+ # client as uncredentialed when it is not.
+ if isinstance(node, ast.Assign) and any(
+ (isinstance(t, ast.Name) and t.id == headers.id)
+ or (isinstance(t, ast.Subscript) and isinstance(t.value, ast.Name) and t.value.id == headers.id)
+ for t in node.targets
+ ):
+ if any(
+ isinstance(sub, ast.Constant) and isinstance(sub.value, str) and sub.value.lower() == "authorization"
+ for sub in ast.walk(node)
+ ):
+ return True
+ return False
+
+
+def _uncredentialed_internal_calls(root: Path) -> tuple[list[str], int]:
+ """Service-to-service requests that carry no credential, and how many were read.
+
+ The direction the older half of this gate could not run in. It asks
+ whether a *variable* reaches a process; this asks whether the request that
+ needed it carries one. Two defects shipped underneath that gap: eleven
+ investigation pivots sent only ``X-Tenant-ID``, a header the API's auth
+ does not read, and the federated-search proxy posted to the connectors
+ service with no Authorization at all. Both were answered 401 on every
+ deployment, and both read to the model and the console as "nothing found".
+ """
+ findings: list[str] = []
+ examined = 0
+ services = root / "services"
+ if not services.is_dir():
+ return findings, 0
+
+ for path in sorted(services.glob("*/app/**/*.py")):
+ if "test" in path.parts:
+ continue
+ try:
+ module = ast.parse(path.read_text(encoding="utf-8"))
+ except SyntaxError:
+ continue
+ for node in ast.walk(module):
+ if not isinstance(node, ast.Call) or not isinstance(node.func, ast.Attribute):
+ continue
+ if node.func.attr not in {"get", "post", "put", "patch", "delete", "request", "stream"}:
+ continue
+ if not node.args:
+ continue
+ url = node.args[1] if node.func.attr in {"request", "stream"} and len(node.args) > 1 else node.args[0]
+ if not _url_is_internal(url, module):
+ continue
+ examined += 1
+ if _carries_credential(node, module):
+ continue
+ rel = path.relative_to(root)
+ findings.append(
+ f"{rel}:{node.lineno} calls a service inside the deployment and passes no Authorization header. "
+ "Every route on the far side authenticates, so this is a 401 that reaches the caller as an "
+ "empty result. Pass the service token and the tenant it is acting for."
+ )
+ return findings, examined
+
+
def _generated(root: Path) -> set[str]:
"""Keys of ``ensure_env.GENERATED``, read by AST.
@@ -249,6 +451,14 @@ def check(root: Path | None = None) -> list[str]:
if not (root / "services" / source_dir).is_dir():
findings.append(f"services/{source_dir} is exempted ({reason}) and does not exist. Remove the row.")
+ uncredentialed, examined = _uncredentialed_internal_calls(root)
+ if examined == 0:
+ findings.append(
+ "scanned services/*/app and found no service-to-service HTTP call at all. "
+ "A clean result over an empty scan is the failure this gate exists to prevent."
+ )
+ findings.extend(uncredentialed)
+
return findings
@@ -265,7 +475,11 @@ def main(argv: list[str] | None = None) -> int:
for finding in findings:
print(f" - {finding}", file=sys.stderr)
return 1
- print(f"OK: {len(readers)} service credentials read by production code are all produced by first run and delivered by compose.")
+ _, examined = _uncredentialed_internal_calls(root)
+ print(
+ f"OK: {len(readers)} service credentials read by production code are all produced by first run "
+ f"and delivered by compose, and all {examined} service-to-service request(s) carry one."
+ )
return 0
diff --git a/scripts/check_vendor_catalog_ids.py b/scripts/check_vendor_catalog_ids.py
new file mode 100644
index 000000000..487ab9298
--- /dev/null
+++ b/scripts/check_vendor_catalog_ids.py
@@ -0,0 +1,264 @@
+#!/usr/bin/env python3
+"""Every read executor's vendor id resolves to a connector a tenant can save.
+
+The hole this closes
+--------------------
+``services/actions/app/live_actions/investigation_reads.py`` registers its
+executors under a ``vendor`` id. ``services/api/app/services/agent_tools/vendor_reads.py``
+decides which vendor tools to offer an investigation by looking that id up
+against the tenant's saved connectors, keyed on ``connector_type``:
+
+ by_type.get(vendor_id)
+
+Three of the seven ids named nothing a tenant could ever have saved.
+``defender``, ``entra`` and ``aws`` are not connector types; the catalog spells
+them ``azure_defender``, ``azure_entra`` and ``aws_cloudtrail``. So the lookup
+returned ``None``, the tool was never bound, and a model investigating a
+Defender host, an Entra identity or a CloudTrail event was told the tenant had
+no integration for it.
+
+Nothing failed. The dictionary lookup that missed is the same expression as the
+one that hits, and the three vendors were simply absent from a list nobody
+counted. Three of the five vendors added in gap-closure 4.2 had never been
+reachable.
+
+What is asserted
+----------------
+For every vendor id registered by a read executor, at least one connector in
+``services/connectors/app/connectors/__init__.py`` declares a matching
+``connector_id`` -- either directly, or through an entry in the alias map that
+``vendor_reads`` resolves with.
+
+The direction that drifts is a new executor naming a vendor the way the vendor
+names itself rather than the way the catalog does, so that is the direction
+this runs in.
+
+Run: python3 scripts/check_vendor_catalog_ids.py [--self-test]
+"""
+
+from __future__ import annotations
+
+import argparse
+import ast
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+from gate_toolkit import SELF_TEST_FLAG, repo_root, self_test_main # noqa: E402
+
+EXECUTORS = Path("services/actions/app/live_actions/investigation_reads.py")
+REGISTRY = Path("services/connectors/app/connectors/__init__.py")
+ALIASES = Path("services/api/app/services/agent_tools/vendor_aliases.py")
+
+
+def _executor_vendor_ids(path: Path) -> set[str]:
+ """Vendor ids registered by the read executors, from the AST.
+
+ AST rather than a text scan so a vendor named in the prose explaining this
+ defect is not counted as a registration.
+ """
+ found: set[str] = set()
+ if not path.is_file():
+ return found
+ tree = ast.parse(path.read_text(encoding="utf-8"))
+ for node in ast.walk(tree):
+ if isinstance(node, ast.keyword) and node.arg == "vendor":
+ if isinstance(node.value, ast.Constant) and isinstance(node.value.value, str):
+ found.add(node.value.value)
+ return found
+
+
+def _catalog_connector_ids(path: Path) -> set[str]:
+ """Every ``connector_id`` a tenant can save, read from the registry module.
+
+ Resolved by importing the connectors package would be tidier and is not:
+ the gate would then need that service's dependencies installed, and a gate
+ that cannot run is a gate that gets deleted. The ids are class attributes
+ assigned to string literals, which the AST can read.
+ """
+ found: set[str] = set()
+ if not path.is_file():
+ return found
+ connectors_dir = path.parent
+ for module in sorted(connectors_dir.glob("*.py")):
+ try:
+ tree = ast.parse(module.read_text(encoding="utf-8"))
+ except SyntaxError:
+ continue
+ for node in ast.walk(tree):
+ if not isinstance(node, ast.ClassDef):
+ continue
+ for stmt in node.body:
+ targets = stmt.targets if isinstance(stmt, ast.Assign) else [stmt.target] if isinstance(stmt, ast.AnnAssign) else []
+ value = stmt.value if isinstance(stmt, ast.Assign | ast.AnnAssign) else None
+ if not value or not isinstance(value, ast.Constant) or not isinstance(value.value, str):
+ continue
+ if any(isinstance(t, ast.Name) and t.id == "connector_id" for t in targets):
+ found.add(value.value)
+ return found
+
+
+def _alias_map(path: Path) -> dict[str, tuple[str, ...]]:
+ """``vendor id -> catalog connector types`` from the alias module."""
+ if not path.is_file():
+ return {}
+ tree = ast.parse(path.read_text(encoding="utf-8"))
+ for node in ast.walk(tree):
+ targets = node.targets if isinstance(node, ast.Assign) else [node.target] if isinstance(node, ast.AnnAssign) else []
+ if not any(isinstance(t, ast.Name) and t.id == "VENDOR_CONNECTOR_TYPES" for t in targets):
+ continue
+ value = node.value if isinstance(node, ast.Assign | ast.AnnAssign) else None
+ if isinstance(value, ast.Dict):
+ out: dict[str, tuple[str, ...]] = {}
+ for key, val in zip(value.keys, value.values, strict=False):
+ if not isinstance(key, ast.Constant) or not isinstance(key.value, str):
+ continue
+ if isinstance(val, ast.Tuple | ast.List):
+ out[key.value] = tuple(e.value for e in val.elts if isinstance(e, ast.Constant) and isinstance(e.value, str))
+ elif isinstance(val, ast.Constant) and isinstance(val.value, str):
+ out[key.value] = (val.value,)
+ return out
+ return {}
+
+
+def check(root: Path | None = None) -> tuple[list[str], dict[str, int]]:
+ root = root or repo_root()
+ vendors = _executor_vendor_ids(root / EXECUTORS)
+ catalog = _catalog_connector_ids(root / REGISTRY)
+ aliases = _alias_map(root / ALIASES)
+ counts = {"vendors": len(vendors), "catalog": len(catalog), "aliases": len(aliases)}
+ findings: list[str] = []
+
+ if not vendors:
+ findings.append(
+ f"{EXECUTORS} declared no read executor at all. A clean result over an empty scan is the failure this gate exists to prevent."
+ )
+ return findings, counts
+ if not catalog:
+ findings.append(f"{REGISTRY} yielded no connector ids, so nothing could be resolved against it.")
+ return findings, counts
+
+ for vendor in sorted(vendors):
+ if vendor in catalog:
+ continue
+ mapped = aliases.get(vendor, ())
+ if not mapped:
+ findings.append(
+ f"read executor vendor {vendor!r} is not a connector type a tenant can save, and no alias "
+ f"maps it to one. `by_type.get({vendor!r})` returns None on every tenant, so the tool is "
+ "never offered and the model is told there is no integration."
+ )
+ continue
+ unknown = [t for t in mapped if t not in catalog]
+ if unknown:
+ findings.append(
+ f"vendor {vendor!r} is aliased to {', '.join(unknown)}, which no connector declares. "
+ "An alias to a type nobody can save resolves to nothing, exactly like no alias at all."
+ )
+
+ for vendor in sorted(aliases):
+ if vendor not in vendors:
+ findings.append(
+ f"the alias map names vendor {vendor!r} and no read executor registers it. "
+ "Remove the row, so an alias covering nothing is not mistaken for one that is load-bearing."
+ )
+
+ return findings, counts
+
+
+def _self_test_cases() -> list[tuple[str, bool]]:
+ """Prove the gate detects the drift it exists to catch.
+
+ Mutations are applied to a copy of the real tree rather than to a fixture,
+ because a gate proven against a fixture is proven against the fixture.
+ """
+ import shutil
+ import tempfile
+
+ root = repo_root()
+ cases: list[tuple[str, bool]] = []
+ clean, counts = check(root)
+ cases.append(("the real tree passes", not clean))
+ cases.append(("the real tree has read executors to judge", counts["vendors"] > 0))
+ cases.append(("the real tree has a catalog to resolve against", counts["catalog"] > 0))
+
+ read = (EXECUTORS, REGISTRY, ALIASES)
+ mutations = [
+ (
+ "an executor naming a vendor the catalog does not have is reported",
+ EXECUTORS,
+ 'vendor="okta"',
+ 'vendor="okta_identity"',
+ ),
+ (
+ "an alias pointing at a type nobody can save is reported",
+ ALIASES,
+ '"azure_defender"',
+ '"microsoft_defender_atp"',
+ ),
+ (
+ "an alias for a vendor no executor registers is reported",
+ ALIASES,
+ '"defender": ',
+ '"defender_gone_away": ',
+ ),
+ ]
+
+ for description, target, old, new in mutations:
+ with tempfile.TemporaryDirectory() as tmp:
+ tree = Path(tmp) / "tree"
+ for rel in read:
+ source = root / rel
+ destination = tree / rel
+ destination.parent.mkdir(parents=True, exist_ok=True)
+ if source.is_file():
+ shutil.copy2(source, destination)
+ # The registry resolver reads every sibling module, so the whole
+ # directory comes along.
+ shutil.copytree(
+ root / REGISTRY.parent, tree / REGISTRY.parent, dirs_exist_ok=True, ignore=shutil.ignore_patterns("__pycache__")
+ )
+ patched = tree / target
+ text = patched.read_text(encoding="utf-8")
+ if old not in text:
+ cases.append((f"{description} (self-test anchor missing; case did not run)", False))
+ continue
+ patched.write_text(text.replace(old, new, 1), encoding="utf-8")
+ problems, _ = check(tree)
+ cases.append((description, bool(problems)))
+
+ return cases
+
+
+def main(argv: list[str] | None = None) -> int:
+ if SELF_TEST_FLAG in (argv if argv is not None else sys.argv[1:]):
+ return self_test_main(Path(__file__).name, extra=_self_test_cases())
+
+ parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ parser.add_argument("--check", action="store_true", help="render the verdict (default)")
+ parser.parse_args(argv)
+
+ findings, counts = check()
+ print(
+ f"check_vendor_catalog_ids: {counts['vendors']} read executor vendor(s) against "
+ f"{counts['catalog']} saveable connector type(s), with {counts['aliases']} alias(es)"
+ )
+ if counts["vendors"] == 0 or counts["catalog"] == 0:
+ print(
+ "\nFAIL: the executor list or the connector catalog read as empty. A clean result over "
+ "nothing is indistinguishable from a wrong root or a renamed module.",
+ file=sys.stderr,
+ )
+ return 2
+ if findings:
+ print(f"\nFAIL: {len(findings)} vendor id(s) resolve to no connector a tenant can save.", file=sys.stderr)
+ for finding in findings:
+ print(f" - {finding}", file=sys.stderr)
+ return 1
+ print("OK: every read executor's vendor id resolves to a connector type a tenant can save.")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/scripts/ensure_env.py b/scripts/ensure_env.py
index 0174122d7..d2cb66e6f 100644
--- a/scripts/ensure_env.py
+++ b/scripts/ensure_env.py
@@ -119,6 +119,10 @@ GENERATED: dict[str, tuple[Callable[[], str], str]] = {
lambda: secrets.token_urlsafe(32),
"Shared bearer between the API and the agents service. Gates triage context, MCP tools and SIEM writeback.",
),
+ "AISOC_API_SERVICE_TOKEN": (
+ lambda: secrets.token_urlsafe(32),
+ "Shared bearer a peer service presents to the API. Gates the agent's customer tools, lake pivots and hunts.",
+ ),
# The HS256 key the API signs realtime tickets with and the realtime edge
# verifies them against. Both sides previously fell back to a constant
# committed to this repository whenever this was unset — which no manifest
diff --git a/services/agents/app/api/router.py b/services/agents/app/api/router.py
index 7009d22c6..d45593539 100644
--- a/services/agents/app/api/router.py
+++ b/services/agents/app/api/router.py
@@ -14,6 +14,7 @@ from pydantic import BaseModel
from app.graph.runner import run_full_investigation
from app.hunt.agent import run_hunt
+from app.investigator import ledger as ledger_module
from app.llm import safe_ainvoke
from app.llm.factory import make_chat_model
from app.models.state import AgentTask, InvestigationState
@@ -172,7 +173,17 @@ async def run_nl_hunt(request: HuntRequest, principal: ScopedPrincipal) -> HuntR
# exactly as it does on every other call site in this service.
return await safe_ainvoke(model, messages)
- result = await run_hunt(hypothesis, invoke=_invoke, run_id=run_id)
+ # The real ledger module, not `None`. This passed no ledger at all, so
+ # `_record` returned at its first line and **no hunt had ever written a
+ # ledger row** -- while the agent's own suite asserted on ledger writes
+ # against a double it supplied itself.
+ result = await run_hunt(
+ hypothesis,
+ invoke=_invoke,
+ run_id=run_id,
+ tenant_id=str(tenant_id),
+ ledger=ledger_module,
+ )
logger.info(
"hunt.completed",
extra={"tenant_id": str(tenant_id), "run_id": run_id, "checked": result.checked},
@@ -180,7 +191,12 @@ async def run_nl_hunt(request: HuntRequest, principal: ScopedPrincipal) -> HuntR
return HuntResponse(
hypothesis=result.hypothesis,
checked=result.checked,
- matches=list(getattr(result, "matches", []) or []),
+ # `HuntAgentResult.findings`, not `matches`. This read `getattr(result,
+ # "matches", [])`, and the attribute has never existed, so the default
+ # swallowed it: every hunt that found rows answered `checked: true,
+ # matches: []`, which reads as a clean result rather than as an answer
+ # dropped on the way out.
+ matches=list(result.findings or []),
refusals=list(result.refusals or []),
unavailable_reason=result.unavailable_reason,
)
diff --git a/services/agents/app/closure/policy.py b/services/agents/app/closure/policy.py
index dcb21a3d6..44acb5b34 100644
--- a/services/agents/app/closure/policy.py
+++ b/services/agents/app/closure/policy.py
@@ -286,19 +286,32 @@ async def _has_auto_close_grant(pool: Any, *, tenant_id: str, alert_class: str |
and `apps/docs/docs/operations/shadow-mode.md` described it as letting
the agent close alerts of that class, which parity 1.1 had to retract
because nothing consulted it.
+
+ Then the reader was written against the wrong table. It named
+ `autonomy_grants`, which migration 067 spells `aisoc_autonomy_grants`, and
+ filtered on `revoked_at` and `expires_at`, which that table does not have:
+ its lifecycle is the `state` column, one of `shadow`, `granted` or
+ `demoted`. Three names wrong in one statement, and because
+ `require_grant` defaults to true, the effect was that a tenant which
+ enabled a closure policy could never auto-close anything. The feature was
+ off for exactly the tenants who turned it on.
+
+ The `except` below is what made it invisible: it logged at warning and
+ returned `False`, which is indistinguishable from "this tenant has not
+ earned a grant". It stays, because a database blip must not close alerts,
+ but the live suite now asserts the query itself works.
"""
try:
async with pool.acquire() as conn:
row = await conn.fetchrow(
"""
SELECT 1
- FROM autonomy_grants
+ FROM aisoc_autonomy_grants
WHERE tenant_id = $1::uuid
AND capability = 'auto_close'
AND scope_kind = 'alert_class'
AND ($2::text IS NULL OR scope_key = $2)
- AND revoked_at IS NULL
- AND (expires_at IS NULL OR expires_at > NOW())
+ AND state = 'granted'
LIMIT 1
""",
tenant_id,
diff --git a/services/agents/app/hunt/agent.py b/services/agents/app/hunt/agent.py
index b79755988..683e0d921 100644
--- a/services/agents/app/hunt/agent.py
+++ b/services/agents/app/hunt/agent.py
@@ -67,13 +67,18 @@ def _api_url() -> str:
return os.getenv("AISOC_API_URL", "http://api:8000").rstrip("/")
-def _api_key() -> str:
- """The agents service's own credential. Deliberately not a tenant id.
+TENANT_HEADER = "X-AiSOC-Tenant-ID"
- The API resolves the tenant from this key, so an injected instruction in a
- hypothesis cannot redirect a hunt at another tenant's history.
+
+def _service_token() -> str:
+ """The shared secret this service presents to the API.
+
+ The tenant travels beside it on :data:`TENANT_HEADER` and comes from the
+ run, never from the hypothesis, so an injected instruction cannot redirect
+ a hunt at another tenant's history.
"""
- return (os.getenv("AISOC_AGENTS_API_KEY") or "").strip()
+ specific = (os.getenv("AISOC_API_SERVICE_TOKEN") or "").strip()
+ return specific or (os.getenv("AISOC_SERVICE_TOKEN") or "").strip()
@dataclass
@@ -170,6 +175,7 @@ async def plan_hunt(
invoke: Any,
ledger: Any | None = None,
run_id: str | None = None,
+ tenant_id: str = "",
) -> tuple[HuntPlan | None, list[str]]:
"""Ask the model for a plan, validating each attempt.
@@ -194,6 +200,8 @@ async def plan_hunt(
run_id,
kind="llm_response",
payload={"role": HUNT_ROLE, "attempt": attempt + 1, "hypothesis": hypothesis[:500]},
+ tenant_id=tenant_id,
+ summary="hunt plan proposed",
)
parsed = _extract_json(reply if isinstance(reply, str) else getattr(reply, "content", ""))
@@ -214,37 +222,50 @@ async def plan_hunt(
run_id,
kind="hunt_plan",
payload={"plan": plan.as_dict(), "attempts": attempt + 1},
+ tenant_id=tenant_id,
+ summary="hunt plan validated",
)
return plan, refusals
return None, refusals
-async def execute_plan(plan: HuntPlan, *, ledger: Any | None = None, run_id: str | None = None) -> HuntAgentResult:
+async def execute_plan(plan: HuntPlan, *, ledger: Any | None = None, run_id: str | None = None, tenant_id: str = "") -> HuntAgentResult:
"""Run a validated plan through the API, which owns the warehouse.
The agents service holds no ClickHouse credential and no tenant session by
design, so execution is a request rather than a query. Same arrangement as
the Phase 4 customer tools, and for the same reason: the tenant comes from
- the credential the API resolves, not from anything this side supplies.
+ the run this hunt belongs to, never from the hypothesis text.
"""
result = HuntAgentResult(hypothesis=plan.hypothesis, plan=plan)
- key = _api_key()
+ key = _service_token()
if not key:
- logger.warning("hunt.agent.no_api_key")
- result.unavailable_reason = "No API credential is configured for the agent service, so the hunt was NOT run."
+ logger.warning("hunt.agent.no_service_token")
+ result.unavailable_reason = "No service credential is configured for the agent service, so the hunt was NOT run."
+ return result
+ if not str(tenant_id or "").strip():
+ logger.warning("hunt.agent.no_tenant")
+ result.unavailable_reason = "No tenant was named for this hunt, so nothing was searched."
return result
payload = {"clauses": [c.as_dict() for c in plan.clauses], "lookback_hours": plan.lookback_hours}
- await _record(ledger, run_id, kind="tool_call", payload={"tool": "hunt_plan_execute", "arguments": payload})
+ await _record(
+ ledger,
+ run_id,
+ kind="tool_call",
+ payload={"tool": "hunt_plan_execute", "arguments": payload},
+ tenant_id=tenant_id,
+ summary="hunt plan execute",
+ )
try:
async with httpx.AsyncClient(timeout=EXECUTE_TIMEOUT_SECONDS) as client:
response = await client.post(
f"{_api_url()}/api/v1/agent-tools/hunt-plan/execute",
json=payload,
- headers={"Authorization": f"Bearer {key}"},
+ headers={"Authorization": f"Bearer {key}", TENANT_HEADER: tenant_id},
)
except Exception as exc: # noqa: BLE001 - unreachable is a gap, not a crash
logger.warning("hunt.agent.unreachable err=%s", type(exc).__name__)
@@ -284,6 +305,8 @@ async def execute_plan(plan: HuntPlan, *, ledger: Any | None = None, run_id: str
run_id,
kind="hunt_result",
payload={"findings": len(result.findings), "truncated": result.truncated},
+ tenant_id=tenant_id,
+ summary=f"hunt found {len(result.findings)} row(s)",
)
return result
@@ -294,9 +317,10 @@ async def run_hunt(
invoke: Any,
ledger: Any | None = None,
run_id: str | None = None,
+ tenant_id: str = "",
) -> HuntAgentResult:
"""Plan and run one hunt. The whole agent, in the order it happens."""
- plan, refusals = await plan_hunt(hypothesis, invoke=invoke, ledger=ledger, run_id=run_id)
+ plan, refusals = await plan_hunt(hypothesis, invoke=invoke, ledger=ledger, run_id=run_id, tenant_id=tenant_id)
if plan is None:
return HuntAgentResult(
hypothesis=hypothesis,
@@ -304,22 +328,55 @@ async def run_hunt(
refusals=refusals,
unavailable_reason=("No valid plan was produced, so nothing was searched. This is not a result: " + "; ".join(refusals[-2:])),
)
- result = await execute_plan(plan, ledger=ledger, run_id=run_id)
+ result = await execute_plan(plan, ledger=ledger, run_id=run_id, tenant_id=tenant_id)
result.refusals = refusals + result.refusals
return result
-async def _record(ledger: Any | None, run_id: str | None, *, kind: str, payload: dict[str, Any]) -> None:
+#: Per-run event counter. Bounded by the number of live runs, and a run
+#: writes four rows at most.
+_SEQ: dict[str, int] = {}
+
+
+async def _record(
+ ledger: Any | None,
+ run_id: str | None,
+ *,
+ kind: str,
+ payload: dict[str, Any],
+ tenant_id: str = "",
+ summary: str = "",
+) -> None:
"""Write one row to the Investigation Ledger, best effort.
Best effort because a ledger failure must not take a completed hunt down,
and loud at ``warning`` because a hunt nobody can audit is a different
product from one they can.
+
+ This used to call ``record_event(run_id=, kind=, payload=)``. The real one
+ additionally requires ``tenant_id``, ``seq``, ``agent`` and ``summary``, so
+ any call reaching it would have raised ``TypeError`` into the ``except``
+ below and logged a warning. Nothing noticed, because the only production
+ caller passed no ledger at all and this returned at its first line: **no
+ hunt had ever written a ledger row.** The suite passed a double that
+ accepts any keyword arguments, which cannot tell the two apart.
"""
if ledger is None or run_id is None:
return
+ # The ledger orders events by `seq` within a run, so it is counted here
+ # rather than passed in: a caller that forgets would write every row at
+ # zero and the replay would be unordered.
+ seq = _SEQ[run_id] = _SEQ.get(run_id, 0) + 1
try:
- await ledger.record_event(run_id=run_id, kind=kind, payload=payload)
+ await ledger.record_event(
+ run_id=run_id,
+ tenant_id=tenant_id,
+ seq=seq,
+ kind=kind,
+ agent="aisoc-hunt",
+ summary=summary or kind,
+ payload=payload,
+ )
except Exception as exc: # noqa: BLE001
logger.warning("hunt.agent.ledger_write_failed kind=%s err=%s", kind, type(exc).__name__)
diff --git a/services/agents/app/investigator/deep_investigation.py b/services/agents/app/investigator/deep_investigation.py
index 38ebed015..77afd45ce 100644
--- a/services/agents/app/investigator/deep_investigation.py
+++ b/services/agents/app/investigator/deep_investigation.py
@@ -444,7 +444,7 @@ async def run_deep_investigation(
try:
model = llm if llm is not None else make_chat_model("investigation", max_tokens=max_completion_tokens())
- registry = default_registry()
+ registry = default_registry(tenant_id)
for tool in investigation_tools(tenant_id):
registry.register(tool)
@@ -460,7 +460,7 @@ async def run_deep_investigation(
# the honest half: what is *not* connected goes into the prompt as a
# gap in visibility rather than being left for the model to discover
# by the absence of a tool.
- customer_tools, coverage_notes = await scoped_customer_tools()
+ customer_tools, coverage_notes = await scoped_customer_tools(tenant_id)
for tool in customer_tools:
registry.register(tool)
result.customer_tools = [tool.name for tool in customer_tools]
diff --git a/services/agents/app/tools/customer_tools.py b/services/agents/app/tools/customer_tools.py
index 0c771be1f..b0542e1fd 100644
--- a/services/agents/app/tools/customer_tools.py
+++ b/services/agents/app/tools/customer_tools.py
@@ -78,14 +78,34 @@ def _api_url() -> str:
return os.getenv("AISOC_API_URL", "http://api:8000").rstrip("/")
-def _api_key() -> str:
- """The agents service's own API key. Empty means these tools cannot run.
-
- Deliberately not a tenant id. The API resolves the tenant from this
- credential, so a compromised prompt cannot redirect a read at another
- tenant's estate.
+#: Header the API reads to learn which tenant this service is acting for.
+#: Matches ``SERVICE_TENANT_HEADER`` in ``services/api/app/api/v1/deps.py``
+#: and ``TENANT_HEADER`` in the vendored ``app/security/tenant_scope.py``.
+TENANT_HEADER = "X-AiSOC-Tenant-ID"
+
+
+def _service_token() -> str:
+ """The shared secret this service presents to the API.
+
+ This used to read ``AISOC_AGENTS_API_KEY``, and **no compose file,
+ ``.env.example`` or Helm value ever set it**, so on a default deployment
+ every tool below answered "could not check". Setting it would not have
+ helped: one API key belongs to one tenant, so every tenant's investigation
+ would have read that one tenant's estate, and the routes these tools call
+ require ``actions:read``, ``lake:query`` and ``hunts:read``, none of which
+ was a mintable scope.
+
+ The service token is the credential compose already delivers. It says
+ *which service* is calling; :func:`_headers` says which tenant it is
+ calling for, and the API refuses it without one.
"""
- return (os.getenv("AISOC_AGENTS_API_KEY") or "").strip()
+ specific = (os.getenv("AISOC_API_SERVICE_TOKEN") or "").strip()
+ return specific or (os.getenv("AISOC_SERVICE_TOKEN") or "").strip()
+
+
+def _headers(tenant_id: str) -> dict[str, str]:
+ """Credential plus the tenant assertion the API requires beside it."""
+ return {"Authorization": f"Bearer {_service_token()}", TENANT_HEADER: tenant_id}
def _could_not_check(what: str, reason: str) -> dict[str, Any]:
@@ -108,30 +128,36 @@ def _could_not_check(what: str, reason: str) -> dict[str, Any]:
}
-async def _call(path: str, payload: dict[str, Any] | None, what: str) -> dict[str, Any]:
+async def _call(path: str, payload: dict[str, Any] | None, what: str, tenant_id: str) -> dict[str, Any]:
"""One request to the API's agent-tool surface, with every failure as data.
Errors are returned rather than raised because the tool loop feeds results
back to the model: a model handed ``available: false`` can adapt and record
a gap, whereas an exception ends the investigation.
"""
- key = _api_key()
- if not key:
+ token = _service_token()
+ if not token:
# A loud skip. Without the credential the API refuses by design, so
# this would be a guaranteed 401 on every call, and an operator needs
# to see why rather than watch investigations quietly get shallower.
- logger.warning("customer_tools.no_api_key", reason="AISOC_AGENTS_API_KEY is unset")
- return _could_not_check(what, "No API credential is configured for the agent service.")
+ logger.warning("customer_tools.no_service_token", reason="AISOC_SERVICE_TOKEN is unset")
+ return _could_not_check(what, "No service credential is configured for the agent service.")
+ if not str(tenant_id or "").strip():
+ # An absent tenant is an empty scope, never every scope. The API
+ # refuses a service token with no tenant for the same reason, so
+ # refusing here only makes the failure legible one hop earlier.
+ logger.warning("customer_tools.no_tenant", path=path)
+ return _could_not_check(what, "No tenant was named for this investigation, so nothing could be read.")
try:
async with httpx.AsyncClient(timeout=TOOL_TIMEOUT_SECONDS) as client:
if payload is None:
- response = await client.get(f"{_api_url()}{path}", headers={"Authorization": f"Bearer {key}"})
+ response = await client.get(f"{_api_url()}{path}", headers=_headers(tenant_id))
else:
response = await client.post(
f"{_api_url()}{path}",
json=payload,
- headers={"Authorization": f"Bearer {key}"},
+ headers=_headers(tenant_id),
)
except Exception as exc: # noqa: BLE001 - every failure becomes data for the model
logger.warning("customer_tools.unreachable", path=path, error=type(exc).__name__)
@@ -194,13 +220,14 @@ INDICATOR_TYPES: tuple[str, ...] = (
)
-async def siem_indicator_search(indicator_type: str, value: str, hours: int = 24) -> dict[str, Any]:
+async def siem_indicator_search(indicator_type: str, value: str, hours: int = 24, *, tenant_id: str = "") -> dict[str, Any]:
"""Search the customer's own SIEMs for one indicator."""
what = f"{indicator_type} {value} in the customer's SIEMs"
body = await _call(
"/api/v1/agent-tools/siem-search",
{"indicator_type": indicator_type, "value": value, "since_hours": int(hours)},
what,
+ tenant_id,
)
if body.get("available") is False:
return body
@@ -318,13 +345,14 @@ VENDOR_READ_TOOLS: dict[str, dict[str, Any]] = {
}
-async def run_vendor_read(capability: str, target: str, **params: Any) -> dict[str, Any]:
+async def run_vendor_read(capability: str, target: str, *, tenant_id: str = "", **params: Any) -> dict[str, Any]:
"""Run one read-only vendor verb and render the outcome for a model."""
what = f"{capability} for {target}"
body = await _call(
"/api/v1/agent-tools/vendor-read",
{"capability": capability, "target": target, "params": params},
what,
+ tenant_id,
)
if body.get("available") is False:
return body
@@ -359,7 +387,7 @@ async def run_vendor_read(capability: str, target: str, **params: Any) -> dict[s
# ------------------------------------------------------- tool construction
-def _siem_tool() -> Tool:
+def _siem_tool(tenant_id: str = "") -> Tool:
return Tool(
name=SIEM_SEARCH_TOOL,
description=(
@@ -385,11 +413,11 @@ def _siem_tool() -> Tool:
},
"required": ["indicator_type", "value"],
},
- fn=lambda indicator_type, value, hours=24: siem_indicator_search(indicator_type, value, hours),
+ fn=lambda indicator_type, value, hours=24: siem_indicator_search(indicator_type, value, hours, tenant_id=tenant_id),
)
-def _vendor_tool(name: str, spec: dict[str, Any]) -> Tool:
+def _vendor_tool(name: str, spec: dict[str, Any], tenant_id: str = "") -> Tool:
capability = spec["capability"]
entity_arg = spec["arg"]
@@ -417,7 +445,7 @@ def _vendor_tool(name: str, spec: dict[str, Any]) -> Tool:
async def _call_tool(**kwargs: Any) -> dict[str, Any]:
target = str(kwargs.pop(entity_arg, "") or "")
- return await run_vendor_read(capability, target, **kwargs)
+ return await run_vendor_read(capability, target, tenant_id=tenant_id, **kwargs)
return Tool(
name=name,
@@ -439,7 +467,7 @@ def customer_tool_catalog() -> list[Tool]:
return [_siem_tool(), *(_vendor_tool(name, spec) for name, spec in sorted(VENDOR_READ_TOOLS.items()))]
-async def scoped_customer_tools() -> tuple[list[Tool], list[str]]:
+async def scoped_customer_tools(tenant_id: str) -> tuple[list[Tool], list[str]]:
"""The customer tools this tenant actually has a backend for.
Returns the tools and a list of notes for the prompt. The notes are the
@@ -447,11 +475,17 @@ async def scoped_customer_tools() -> tuple[list[Tool], list[str]]:
produces no tools *and* a sentence saying why, so the model records a gap
in coverage rather than investigating confidently with less.
- The tenant is not a parameter. It comes from the API key this service
- authenticates with, which is what stops a prompt redirecting a read.
+ The tenant **is** a parameter, and it is the alert's own. It used to be
+ implicit in a single shared API key, which meant one tenant's estate was
+ read for every tenant's investigation. It is passed to the API on
+ :data:`TENANT_HEADER` beside the service token, and the API refuses a
+ service credential that does not name one.
+
+ It is not model-supplied. It is threaded from the run the investigation is
+ for, so a prompt cannot redirect a read at another tenant's estate.
"""
notes: list[str] = []
- body = await _call("/api/v1/agent-tools/backends", None, "the customer's connected tools")
+ body = await _call("/api/v1/agent-tools/backends", None, "the customer's connected tools", tenant_id)
if body.get("available") is False:
# Fails closed on the toolset and loud in the prompt. Binding the
# whole catalog here would offer the model tools that answer
@@ -469,7 +503,7 @@ async def scoped_customer_tools() -> tuple[list[Tool], list[str]]:
siem = body.get("siem_search") or {}
siem_backends = siem.get("backends") or []
if siem.get("enabled") and siem_backends:
- tools.append(_siem_tool())
+ tools.append(_siem_tool(tenant_id))
names = ", ".join(str(b.get("source")) for b in siem_backends)
notes.append(f"The customer has these SIEM platforms connected and searchable: {names}.")
else:
@@ -482,7 +516,7 @@ async def scoped_customer_tools() -> tuple[list[Tool], list[str]]:
available_caps = {str(entry.get("capability")) for entry in reads if isinstance(entry, dict)}
for name, spec in sorted(VENDOR_READ_TOOLS.items()):
if spec["capability"] in available_caps:
- tools.append(_vendor_tool(name, spec))
+ tools.append(_vendor_tool(name, spec, tenant_id))
if reads:
pairs = ", ".join(f"{entry.get('vendor')} ({entry.get('capability')})" for entry in reads if isinstance(entry, dict))
diff --git a/services/agents/app/tools/investigation.py b/services/agents/app/tools/investigation.py
index 2545cbe88..e76fc281c 100644
--- a/services/agents/app/tools/investigation.py
+++ b/services/agents/app/tools/investigation.py
@@ -40,6 +40,18 @@ def _api_url() -> str:
return os.getenv("AISOC_API_URL", "http://api:8000").rstrip("/")
+#: Header the API reads to learn which tenant this service is acting for.
+#: ``X-Tenant-ID`` is a different header that the API's auth does not read,
+#: which is why these pivots were answered 401 outside dev mode.
+TENANT_HEADER = "X-AiSOC-Tenant-ID"
+
+
+def _service_token() -> str:
+ """The shared secret this service presents to the API."""
+ specific = (os.getenv("AISOC_API_SERVICE_TOKEN") or "").strip()
+ return specific or (os.getenv("AISOC_SERVICE_TOKEN") or "").strip()
+
+
async def call_investigation_tool(tool: str, tenant_id: str, **args: Any) -> dict[str, Any]:
"""Run one pivot through the API.
@@ -47,13 +59,20 @@ async def call_investigation_tool(tool: str, tenant_id: str, **args: Any) -> dic
results back to the model, and a model that receives
``{"available": false, "reason": ...}`` can adapt, whereas an exception
ends the investigation.
+
+ These eleven pivots used to send ``X-Tenant-ID`` and no credential. The
+ API reads neither, so outside dev mode every pivot was 401 and the model
+ was told the lake had nothing rather than that it had not been asked.
"""
try:
async with httpx.AsyncClient(timeout=TOOL_TIMEOUT_SECONDS) as client:
response = await client.post(
f"{_api_url()}/api/v1/graph/investigate/query",
json={"tool": tool, "args": args},
- headers={"X-Tenant-ID": tenant_id},
+ headers={
+ "Authorization": f"Bearer {_service_token()}",
+ TENANT_HEADER: tenant_id,
+ },
)
response.raise_for_status()
return response.json()
diff --git a/services/agents/app/tools/registry.py b/services/agents/app/tools/registry.py
index bb9f7d0fd..45047af15 100644
--- a/services/agents/app/tools/registry.py
+++ b/services/agents/app/tools/registry.py
@@ -68,8 +68,12 @@ def _str_param(name: str, desc: str) -> dict[str, Any]:
return {"type": "object", "properties": {name: {"type": "string", "description": desc}}, "required": [name]}
-def default_registry() -> ToolRegistry:
- """Registry of AiSOC's real analyst tools, wrapped for LLM tool-calling."""
+def default_registry(tenant_id: str = "") -> ToolRegistry:
+ """Registry of AiSOC's real analyst tools, wrapped for LLM tool-calling.
+
+ ``tenant_id`` is the run's own. The sandbox lookup reaches the API, which
+ refuses a service credential that does not name the tenant it acts for.
+ """
from app.investigator.tools import enrich_ioc, extract_iocs, map_to_mitre
from app.tools.mitre import lookup_technique
from app.tools.sandbox import TOOL_DESCRIPTION, TOOL_NAME, TOOL_PARAMETERS, lookup_file_hash
@@ -80,7 +84,7 @@ def default_registry() -> ToolRegistry:
name=TOOL_NAME,
description=TOOL_DESCRIPTION,
parameters=TOOL_PARAMETERS,
- fn=lookup_file_hash,
+ fn=lambda sha256: lookup_file_hash(sha256, tenant_id=tenant_id),
),
Tool(
name="extract_iocs",
diff --git a/services/agents/app/tools/sandbox.py b/services/agents/app/tools/sandbox.py
index 298e83c6e..1cb4a8c9b 100644
--- a/services/agents/app/tools/sandbox.py
+++ b/services/agents/app/tools/sandbox.py
@@ -47,14 +47,19 @@ def _api_url() -> str:
return os.getenv("AISOC_API_URL", "http://api:8000").rstrip("/")
-def _api_key() -> str:
+TENANT_HEADER = "X-AiSOC-Tenant-ID"
+
+
+def _service_token() -> str:
"""The agents service's own API key. Empty means the tool cannot run.
- Deliberately not a tenant id. The API resolves the tenant from this
+ The tenant is named beside it rather than implied by it. One shared key
+ meant one tenant, so every tenant's lookup would have read that tenant's
credential, so a compromised prompt cannot redirect the lookup, and cannot
read another tenant's analysis history.
"""
- return (os.getenv("AISOC_AGENTS_API_KEY") or "").strip()
+ specific = (os.getenv("AISOC_API_SERVICE_TOKEN") or "").strip()
+ return specific or (os.getenv("AISOC_SERVICE_TOKEN") or "").strip()
def _could_not_check(reason: str, sha256: str = "") -> dict[str, Any]:
@@ -75,7 +80,7 @@ def _could_not_check(reason: str, sha256: str = "") -> dict[str, Any]:
}
-async def lookup_file_hash(sha256: str) -> dict[str, Any]:
+async def lookup_file_hash(sha256: str, *, tenant_id: str = "") -> dict[str, Any]:
"""Ask the configured analysis provider whether it has seen this file."""
digest = (sha256 or "").strip().lower()
if not _SHA256.match(digest):
@@ -84,12 +89,12 @@ async def lookup_file_hash(sha256: str) -> dict[str, Any]:
"outcome": "invalid_input",
"reason": "Not a SHA-256 digest. Provide 64 hexadecimal characters.",
}
- key = _api_key()
+ key = _service_token()
if not key:
# A loud skip. Without the credential the API refuses by design, so
# this would be a guaranteed 401 on every lookup; an operator needs to
# see why rather than find an empty result.
- logger.warning("sandbox_tool.no_api_key", reason="AISOC_AGENTS_API_KEY is unset")
+ logger.warning("sandbox_tool.no_service_token", reason="AISOC_SERVICE_TOKEN is unset")
return _could_not_check("No file-analysis credential is configured for the agent service.", digest)
try:
@@ -97,7 +102,7 @@ async def lookup_file_hash(sha256: str) -> dict[str, Any]:
response = await client.post(
f"{_api_url()}/api/v1/sandbox/lookup",
json={"sha256": digest},
- headers={"Authorization": f"Bearer {key}"},
+ headers={"Authorization": f"Bearer {key}", TENANT_HEADER: tenant_id},
)
except Exception as exc: # noqa: BLE001 - every failure becomes data for the model
logger.warning("sandbox_tool.unreachable", error=type(exc).__name__)
diff --git a/services/agents/tests/test_customer_tools.py b/services/agents/tests/test_customer_tools.py
index 9a6cef082..4374b3eda 100644
--- a/services/agents/tests/test_customer_tools.py
+++ b/services/agents/tests/test_customer_tools.py
@@ -46,10 +46,16 @@ READ = f"{API}/api/v1/agent-tools/vendor-read"
BACKENDS = f"{API}/api/v1/agent-tools/backends"
+#: The tenant a run is for. Passed explicitly on every call because the
+#: credential no longer implies one: a service token says which service is
+#: calling, and the tenant header says which tenant it is calling for.
+TENANT = "11111111-1111-1111-1111-111111111111"
+
+
@pytest.fixture(autouse=True)
def _agent_credential(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("AISOC_API_URL", API)
- monkeypatch.setenv("AISOC_AGENTS_API_KEY", "aisoc_test_key")
+ monkeypatch.setenv("AISOC_SERVICE_TOKEN", "fixpass-service-token")
def _assert_could_not_check(payload: dict, *, must_mention: str = "") -> None:
@@ -128,6 +134,7 @@ async def test_siem_search_passes_a_typed_query_and_no_text() -> None:
def handler(request: httpx.Request) -> httpx.Response:
captured["body"] = json.loads(request.content)
captured["auth"] = request.headers.get("authorization")
+ captured["tenant"] = request.headers.get("x-aisoc-tenant-id")
return httpx.Response(
200,
json={
@@ -139,14 +146,16 @@ async def test_siem_search_passes_a_typed_query_and_no_text() -> None:
)
respx.post(SEARCH).mock(side_effect=handler)
- result = await siem_indicator_search("sha256", "a" * 64, 48)
+ result = await siem_indicator_search("sha256", "a" * 64, 48, tenant_id=TENANT)
body = captured["body"]
assert body == {"indicator_type": "sha256", "value": "a" * 64, "since_hours": 48}
- # No tenant anywhere in the request. The API takes it from the credential,
- # which is what stops a prompt redirecting a read at another tenant.
+ # No tenant in the *body*. The model fills the body, so a tenant there
+ # would be a tenant a prompt could choose. It travels on the header
+ # instead, where it comes from the run rather than from the model.
assert "tenant" not in json.dumps(body).lower()
- assert captured["auth"] == "Bearer aisoc_test_key"
+ assert captured["auth"] == "Bearer fixpass-service-token"
+ assert captured["tenant"] == TENANT, "the API refuses a service token that names no tenant"
assert result["available"] is True
assert result["sightings"] == 1
@@ -166,7 +175,7 @@ async def test_a_zero_row_search_says_what_it_does_and_does_not_prove() -> None:
},
)
)
- result = await siem_indicator_search("ip", "203.0.113.9")
+ result = await siem_indicator_search("ip", "203.0.113.9", tenant_id=TENANT)
assert result["available"] is True
assert result["sightings"] == 0
@@ -193,7 +202,7 @@ async def test_a_partly_failed_search_is_not_rounded_to_a_clean_one() -> None:
},
)
)
- result = await siem_indicator_search("ip", "203.0.113.9")
+ result = await siem_indicator_search("ip", "203.0.113.9", tenant_id=TENANT)
assert result["outcome"] == "partial"
warning = result["partial_warning"]
@@ -217,14 +226,14 @@ async def test_every_source_failing_is_could_not_check() -> None:
},
)
)
- _assert_could_not_check(await siem_indicator_search("ip", "203.0.113.9"), must_mention="timeout")
+ _assert_could_not_check(await siem_indicator_search("ip", "203.0.113.9", tenant_id=TENANT), must_mention="timeout")
@pytest.mark.asyncio
@respx.mock
async def test_no_siem_connected_is_could_not_check_not_zero_sightings() -> None:
respx.post(SEARCH).mock(return_value=httpx.Response(200, json={"outcome": "no_backend", "rows": [], "sources": []}))
- result = await siem_indicator_search("ip", "203.0.113.9")
+ result = await siem_indicator_search("ip", "203.0.113.9", tenant_id=TENANT)
_assert_could_not_check(result, must_mention="no SIEM connected")
@@ -237,10 +246,10 @@ async def test_transport_and_status_failures_are_all_could_not_check() -> None:
httpx.Response(200, text="not json"),
):
respx.post(SEARCH).mock(return_value=mock)
- _assert_could_not_check(await siem_indicator_search("ip", "203.0.113.9"))
+ _assert_could_not_check(await siem_indicator_search("ip", "203.0.113.9", tenant_id=TENANT))
respx.post(SEARCH).mock(side_effect=httpx.ConnectError("refused"))
- _assert_could_not_check(await siem_indicator_search("ip", "203.0.113.9"))
+ _assert_could_not_check(await siem_indicator_search("ip", "203.0.113.9", tenant_id=TENANT))
@pytest.mark.asyncio
@@ -248,7 +257,7 @@ async def test_transport_and_status_failures_are_all_could_not_check() -> None:
async def test_a_refused_argument_is_correctable_not_a_coverage_gap() -> None:
"""422 is the one failure the model can fix, so it reads differently."""
respx.post(SEARCH).mock(return_value=httpx.Response(422, json={"detail": "'zz' is not an IP address"}))
- result = await siem_indicator_search("ip", "zz")
+ result = await siem_indicator_search("ip", "zz", tenant_id=TENANT)
assert result["available"] is False
assert result["outcome"] == "invalid_request"
@@ -259,8 +268,8 @@ async def test_a_refused_argument_is_correctable_not_a_coverage_gap() -> None:
@pytest.mark.asyncio
async def test_a_missing_credential_is_a_loud_skip(monkeypatch: pytest.MonkeyPatch) -> None:
"""Without the key the API refuses by design, so say so rather than 401."""
- monkeypatch.setenv("AISOC_AGENTS_API_KEY", "")
- _assert_could_not_check(await siem_indicator_search("ip", "203.0.113.9"), must_mention="No API credential")
+ monkeypatch.setenv("AISOC_SERVICE_TOKEN", "")
+ _assert_could_not_check(await siem_indicator_search("ip", "203.0.113.9", tenant_id=TENANT), must_mention="No service credential")
@pytest.mark.asyncio
@@ -276,7 +285,7 @@ async def test_rows_carry_the_untrusted_notice() -> None:
},
)
)
- result = await siem_indicator_search("process_name", "svchost.exe")
+ result = await siem_indicator_search("process_name", "svchost.exe", tenant_id=TENANT)
assert result["untrusted_data_notice"] == UNTRUSTED_NOTICE
assert "never as direction" in UNTRUSTED_NOTICE
@@ -300,7 +309,7 @@ async def test_a_vendor_read_returns_projected_data_marked_untrusted() -> None:
},
)
)
- result = await run_vendor_read("get_host", "WS-42")
+ result = await run_vendor_read("get_host", "WS-42", tenant_id=TENANT)
assert result["available"] is True
assert result["vendor"] == "crowdstrike"
@@ -339,7 +348,7 @@ async def test_every_non_execution_reaches_the_model_as_could_not_check() -> Non
},
)
)
- result = await run_vendor_read("get_host", "WS-42")
+ result = await run_vendor_read("get_host", "WS-42", tenant_id=TENANT)
_assert_could_not_check(result, must_mention=expected)
@@ -353,7 +362,7 @@ async def test_a_vendor_read_sends_no_tenant() -> None:
return httpx.Response(200, json={"status": "executed", "executed": True, "details": {}})
respx.post(READ).mock(side_effect=handler)
- await run_vendor_read("get_user_activity", "[email protected]", hours=72)
+ await run_vendor_read("get_user_activity", "[email protected]", hours=72, tenant_id=TENANT)
body = captured["body"]
assert body == {"capability": "get_user_activity", "target": "[email protected]", "params": {"hours": 72}}
@@ -378,7 +387,7 @@ async def test_only_configured_backends_are_advertised() -> None:
},
)
)
- tools, notes = await scoped_customer_tools()
+ tools, notes = await scoped_customer_tools(TENANT)
names = {t.name for t in tools}
assert names == {"siem_indicator_search", "identity_user_activity"}
@@ -402,7 +411,7 @@ async def test_a_tenant_with_nothing_connected_gets_no_tools_and_two_gaps() -> N
},
)
)
- tools, notes = await scoped_customer_tools()
+ tools, notes = await scoped_customer_tools(TENANT)
assert tools == []
# Both absences stated. Silence here would let a model conclude without
@@ -421,7 +430,7 @@ async def test_an_unreachable_api_binds_nothing_and_says_so() -> None:
binding nothing quietly would let the model conclude without noticing.
"""
respx.get(BACKENDS).mock(side_effect=httpx.ConnectError("refused"))
- tools, notes = await scoped_customer_tools()
+ tools, notes = await scoped_customer_tools(TENANT)
assert tools == []
assert len(notes) == 1
@@ -443,5 +452,5 @@ async def test_an_incomplete_registry_is_reported_as_incomplete() -> None:
},
)
)
- _tools, notes = await scoped_customer_tools()
+ _tools, notes = await scoped_customer_tools(TENANT)
assert any("may be" in note and "incomplete" in note for note in notes)
diff --git a/services/agents/tests/test_deep_investigation_customer_tools.py b/services/agents/tests/test_deep_investigation_customer_tools.py
index 073127657..56fc7b126 100644
--- a/services/agents/tests/test_deep_investigation_customer_tools.py
+++ b/services/agents/tests/test_deep_investigation_customer_tools.py
@@ -61,6 +61,12 @@ BACKENDS = f"{API}/api/v1/agent-tools/backends"
SEARCH = f"{API}/api/v1/agent-tools/siem-search"
READ = f"{API}/api/v1/agent-tools/vendor-read"
LAKE = f"{API}/api/v1/graph/investigate/query"
+#: Two context lookups the driver makes on any deployment that has a service
+#: credential. They were unreachable while the agents service had none, which
+#: is the defect fix-pass item 1.1 closed, so they are mocked here rather than
+#: left to fail the run with "not mocked".
+MCP_SERVERS = f"{API}/api/v1/mcp-servers/resolved"
+TENANT_SKILLS = f"{API}/api/v1/tenant-skills/resolved/active"
# --------------------------------------------------------------- the alert
@@ -255,10 +261,22 @@ def _mock_api(*, siem_rows: list[dict[str, Any]], edr_detections: list[dict[str,
@pytest.fixture(autouse=True)
def _agent_credential(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("AISOC_API_URL", API)
- monkeypatch.setenv("AISOC_AGENTS_API_KEY", "aisoc_test_key")
+ monkeypatch.setenv("AISOC_SERVICE_TOKEN", "fixpass-service-token")
monkeypatch.setenv("AISOC_DEEP_INVESTIGATION", "true")
[email protected](autouse=True)
+def _context_lookups() -> None:
+ """The per-tenant context the driver resolves on a credentialed deployment.
+
+ Both are empty answers, not absent ones: this file is about customer
+ tools, and an empty skill set and an empty MCP registry are what a tenant
+ that has configured neither actually returns.
+ """
+ respx.get(MCP_SERVERS).mock(return_value=httpx.Response(200, json={"servers": []}))
+ respx.get(TENANT_SKILLS).mock(return_value=httpx.Response(200, json={"skills": []}))
+
+
@pytest.fixture
def ledger_rows(monkeypatch: pytest.MonkeyPatch) -> list[dict[str, Any]]:
"""Capture what the driver hands the ledger, at the ledger's own boundary.
diff --git a/services/agents/tests/test_hunt_route_returns_findings.py b/services/agents/tests/test_hunt_route_returns_findings.py
new file mode 100644
index 000000000..2c69c8adc
--- /dev/null
+++ b/services/agents/tests/test_hunt_route_returns_findings.py
@@ -0,0 +1,171 @@
+"""A hunt that found rows says so, for the right tenant, with a ledger.
+
+Fix pass item 1.5. See `plans/aisoc_fix_pass_plan.plan.md`.
+
+Three defects in one route
+--------------------------
+`POST /hunt` on the agents service built its response as
+
+ matches=list(getattr(result, "matches", []) or [])
+
+and `HuntAgentResult` has no `matches` attribute. It has `findings`. The
+`getattr` default made the mistake silent, so **every hunt that found rows
+answered `checked: true, matches: []`** -- which reads to an analyst, and to a
+model, as a clean result rather than as a hunt whose answer was dropped on the
+way out.
+
+`run_hunt` was called with no `ledger`, so `_record` returned at its first line
+and **no hunt has ever written a ledger row**. Underneath that, `_record`
+called `ledger.record_event(run_id=..., kind=..., payload=...)` while the real
+`record_event` additionally requires `tenant_id`, `seq`, `agent` and `summary`,
+so the first call that did reach a real ledger would have raised `TypeError`
+into the `except` and logged a warning.
+
+The tenant the route resolves was not passed on either; item 1.1 threads it.
+
+Why the existing tests did not catch any of it
+----------------------------------------------
+They drive `run_hunt` directly and assert on `result.findings`, which is
+correct and which the route then fails to read. And they pass a fake ledger
+that accepts any keyword arguments, so a call the real ledger would reject
+looks fine.
+
+This file drives the route handler itself and uses the real `record_event`
+signature as the contract, which is the pair of boundaries the defects live at.
+"""
+
+from __future__ import annotations
+
+import inspect
+import uuid
+from typing import Any
+
+import pytest
+
+pytestmark = pytest.mark.anyio
+
+TENANT = uuid.UUID("33333333-3333-3333-3333-333333333333")
+
+
+def anyio_backend() -> str:
+ return "asyncio"
+
+
+class _Principal:
+ """A console principal scoped to one tenant, as the dependency returns."""
+
+ tenant_ids = frozenset({TENANT})
+ subject = "console:hunter"
+ delegated = False
+
+ @property
+ def is_empty(self) -> bool:
+ return False
+
+ def covers(self, tenant_id: uuid.UUID) -> bool:
+ return tenant_id in self.tenant_ids
+
+ def ordered_ids(self) -> list[uuid.UUID]:
+ return sorted(self.tenant_ids)
+
+
+class TestTheRouteReturnsWhatTheHuntFound:
+ async def test_rows_the_hunt_found_reach_the_response(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """The defect, exactly: findings in, nothing out."""
+ from app.api import router as router_mod
+ from app.hunt.agent import HuntAgentResult
+
+ found = [{"host": "WS-42", "process": "powershell.exe"}]
+
+ async def _fake_run_hunt(hypothesis: str, **kwargs: Any) -> HuntAgentResult:
+ return HuntAgentResult(hypothesis=hypothesis, checked=True, findings=found, rows_scanned=17)
+
+ monkeypatch.setattr(router_mod, "run_hunt", _fake_run_hunt)
+ monkeypatch.setattr(router_mod, "make_chat_model", lambda *a, **k: object())
+
+ response = await router_mod.run_nl_hunt(
+ request=router_mod.HuntRequest(tenant_id=str(TENANT), hypothesis="powershell spawning from office"),
+ principal=_Principal(),
+ )
+
+ assert response.checked is True
+ assert response.matches == found, "the hunt found rows and the route reported none, which reads as a clean result"
+
+ async def test_a_hunt_that_found_nothing_still_reports_nothing(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """The negative control.
+
+ Without it, a route that echoed its input would satisfy the assertion
+ above. An empty hunt must stay empty.
+ """
+ from app.api import router as router_mod
+ from app.hunt.agent import HuntAgentResult
+
+ async def _fake_run_hunt(hypothesis: str, **kwargs: Any) -> HuntAgentResult:
+ return HuntAgentResult(hypothesis=hypothesis, checked=True, findings=[], rows_scanned=4)
+
+ monkeypatch.setattr(router_mod, "run_hunt", _fake_run_hunt)
+ monkeypatch.setattr(router_mod, "make_chat_model", lambda *a, **k: object())
+
+ response = await router_mod.run_nl_hunt(
+ request=router_mod.HuntRequest(tenant_id=str(TENANT), hypothesis="anything"),
+ principal=_Principal(),
+ )
+
+ assert response.checked is True
+ assert response.matches == []
+
+ async def test_the_route_passes_the_tenant_it_resolved(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """The tenant comes from the credential, and must reach the hunt.
+
+ Without this the hunt runs against whatever tenant the API resolves
+ from a shared credential, which is the defect item 1.1 closed one
+ layer down.
+ """
+ from app.api import router as router_mod
+ from app.hunt.agent import HuntAgentResult
+
+ seen: dict[str, Any] = {}
+
+ async def _fake_run_hunt(hypothesis: str, **kwargs: Any) -> HuntAgentResult:
+ seen.update(kwargs)
+ return HuntAgentResult(hypothesis=hypothesis, checked=True, findings=[])
+
+ monkeypatch.setattr(router_mod, "run_hunt", _fake_run_hunt)
+ monkeypatch.setattr(router_mod, "make_chat_model", lambda *a, **k: object())
+
+ await router_mod.run_nl_hunt(
+ request=router_mod.HuntRequest(tenant_id=str(TENANT), hypothesis="anything"),
+ principal=_Principal(),
+ )
+
+ assert seen.get("tenant_id") == str(TENANT), f"the route resolved a tenant and did not pass it on: {seen.get('tenant_id')!r}"
+
+
+class TestTheLedgerCallWouldBeAccepted:
+ """`_record`'s call must satisfy the real `record_event`, not a fake one.
+
+ The existing suite passes a double that accepts any keyword arguments, so
+ a call the real ledger rejects looks identical to one it accepts. This
+ compares against the real signature instead.
+ """
+
+ def test_the_hunt_ledger_call_matches_the_real_record_event(self) -> None:
+ from app.investigator.ledger import record_event
+
+ signature = inspect.signature(record_event)
+ required = {
+ name
+ for name, param in signature.parameters.items()
+ if param.default is inspect.Parameter.empty and param.kind is inspect.Parameter.KEYWORD_ONLY
+ }
+
+ from app.hunt.agent import _record
+
+ source = inspect.getsource(_record)
+ missing = sorted(name for name in required if f"{name}=" not in source)
+
+ assert not missing, (
+ f"_record does not supply {missing}, which record_event requires. "
+ "Every call would raise TypeError into its own except and log a warning."
+ )
diff --git a/services/agents/tests/test_sandbox_tool.py b/services/agents/tests/test_sandbox_tool.py
index d35b1b0c5..e8d4fe91a 100644
--- a/services/agents/tests/test_sandbox_tool.py
+++ b/services/agents/tests/test_sandbox_tool.py
@@ -18,9 +18,12 @@ DIGEST = "275a021bbfb6489e54d471899f7db9d1663fc695ec2fe2a2c4538aabf651fd0f"
pytestmark = pytest.mark.asyncio
+TENANT = "11111111-1111-1111-1111-111111111111"
+
+
@pytest.fixture(autouse=True)
def _credential(monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setenv("AISOC_AGENTS_API_KEY", "aisoc_test_key")
+ monkeypatch.setenv("AISOC_SERVICE_TOKEN", "fixpass-service-token")
monkeypatch.setenv("AISOC_API_URL", "http://api:8000")
@@ -44,7 +47,7 @@ class TestFailuresNeverReadAsClean:
raise httpx.ConnectError("refused")
_patch_client(monkeypatch, httpx.MockTransport(boom))
- result = await lookup_file_hash(DIGEST)
+ result = await lookup_file_hash(DIGEST, tenant_id=TENANT)
assert result["available"] is False
assert result["outcome"] == "could_not_check"
assert "NOT been assessed" in result["reason"]
@@ -52,24 +55,25 @@ class TestFailuresNeverReadAsClean:
async def test_a_server_error_is_could_not_check(self, monkeypatch: pytest.MonkeyPatch) -> None:
_patch_client(monkeypatch, _api(503))
- result = await lookup_file_hash(DIGEST)
+ result = await lookup_file_hash(DIGEST, tenant_id=TENANT)
assert result["outcome"] == "could_not_check"
async def test_air_gap_refusal_is_could_not_check_not_a_miss(self, monkeypatch: pytest.MonkeyPatch) -> None:
_patch_client(monkeypatch, _api(403))
- result = await lookup_file_hash(DIGEST)
+ result = await lookup_file_hash(DIGEST, tenant_id=TENANT)
assert result["outcome"] == "could_not_check"
assert "air-gapped" in result["reason"].lower()
async def test_a_missing_credential_is_a_loud_skip(self, monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.delenv("AISOC_AGENTS_API_KEY", raising=False)
- result = await lookup_file_hash(DIGEST)
+ monkeypatch.delenv("AISOC_SERVICE_TOKEN", raising=False)
+ monkeypatch.delenv("AISOC_API_SERVICE_TOKEN", raising=False)
+ result = await lookup_file_hash(DIGEST, tenant_id=TENANT)
assert result["outcome"] == "could_not_check"
assert "credential" in result["reason"]
async def test_the_providers_own_failure_is_passed_through_as_a_failure(self, monkeypatch: pytest.MonkeyPatch) -> None:
_patch_client(monkeypatch, _api(200, {"outcome": "could_not_check", "provider": "capev2", "detail": "timed out"}))
- result = await lookup_file_hash(DIGEST)
+ result = await lookup_file_hash(DIGEST, tenant_id=TENANT)
assert result["outcome"] == "could_not_check"
assert "timed out" in result["reason"]
@@ -78,7 +82,7 @@ class TestOrdinaryAnswers:
async def test_a_known_hash_returns_the_report_whole(self, monkeypatch: pytest.MonkeyPatch) -> None:
report = {"verdict": "malicious", "score": 100, "attack": "unavailable"}
_patch_client(monkeypatch, _api(200, {"outcome": "known", "provider": "mock", "report": report, "detail": "ok"}))
- result = await lookup_file_hash(DIGEST)
+ result = await lookup_file_hash(DIGEST, tenant_id=TENANT)
assert result["available"] is True
assert result["outcome"] == "known"
# Unavailable fields survive the trip rather than being stripped, so
@@ -87,14 +91,14 @@ class TestOrdinaryAnswers:
async def test_an_unknown_hash_is_not_presented_as_a_clean_verdict(self, monkeypatch: pytest.MonkeyPatch) -> None:
_patch_client(monkeypatch, _api(200, {"outcome": "not_seen", "provider": "mock", "detail": "no analysis"}))
- result = await lookup_file_hash(DIGEST)
+ result = await lookup_file_hash(DIGEST, tenant_id=TENANT)
assert result["outcome"] == "not_seen"
assert "not a verdict" in result["reason"]
assert "targeted malware" in result["reason"]
async def test_a_pending_analysis_is_not_available(self, monkeypatch: pytest.MonkeyPatch) -> None:
_patch_client(monkeypatch, _api(200, {"outcome": "pending", "provider": "capev2"}))
- result = await lookup_file_hash(DIGEST)
+ result = await lookup_file_hash(DIGEST, tenant_id=TENANT)
assert result["available"] is False
assert result["outcome"] == "pending"
assert "not a clean result" in result["reason"]
@@ -104,7 +108,7 @@ class TestOrdinaryAnswers:
raise AssertionError("the tool must not call out for a malformed digest")
_patch_client(monkeypatch, httpx.MockTransport(unexpected))
- result = await lookup_file_hash("not-a-hash")
+ result = await lookup_file_hash("not-a-hash", tenant_id=TENANT)
assert result["outcome"] == "invalid_input"
diff --git a/services/api/app/api/v1/deps.py b/services/api/app/api/v1/deps.py
index e88d4323d..5f7401ac4 100644
--- a/services/api/app/api/v1/deps.py
+++ b/services/api/app/api/v1/deps.py
@@ -21,12 +21,14 @@ activates the RLS policies defined in ``migrations/002_rls.sql``.
...
"""
+import hmac
import logging
+import os
import uuid
from datetime import UTC, datetime
from typing import Annotated
-from fastapi import Depends, HTTPException, Security, status
+from fastapi import Depends, HTTPException, Request, Security, status
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
from jwt import PyJWTError
from sqlalchemy import select, update
@@ -42,7 +44,7 @@ from app.api.v1.dev_auth import (
from app.core.permission_cache import grants, resolve_permissions
from app.core.security import decode_token, has_permission, hash_api_key, token_is_revoked
from app.db.database import get_db
-from app.models.tenant import ApiKey, User
+from app.models.tenant import ApiKey, Tenant, User
logger = logging.getLogger("aisoc.deps")
@@ -169,6 +171,127 @@ class CurrentUser:
)
+#: Header a trusted service uses to declare which tenant it is acting for.
+#: Spelled identically to ``TENANT_HEADER`` in the vendored
+#: ``app/security/tenant_scope.py`` that the other services already use, so
+#: one service does not have to know which of two names a peer expects.
+SERVICE_TENANT_HEADER = "X-AiSOC-Tenant-ID"
+
+#: What a service principal may do, and nothing else.
+#:
+#: Deliberately an explicit read-only list rather than the wildcard. The
+#: credential reaches every route in the API, so granting it ``*`` would make
+#: one shared secret equivalent to ``platform_admin`` on every tenant at once.
+#: These are the permissions the agents service's own tools need:
+#: ``connectors:read`` for the backend catalogue and the SIEM fan-out,
+#: ``actions:read`` for a read-only vendor verb, ``lake:query`` for a hunt
+#: plan, ``hunts:read`` for the hunting agent, and ``alerts:read`` for the
+#: alert an investigation is about.
+#:
+#: Anything that changes state at a vendor or in the database is absent on
+#: purpose: those go through the live-action contract with its approval
+#: matrix, not through a service token.
+SERVICE_PRINCIPAL_PERMISSIONS: frozenset[str] = frozenset(
+ {
+ "alerts:read",
+ "cases:read",
+ "connectors:read",
+ "actions:read",
+ "lake:query",
+ "hunts:read",
+ "threat_intel:read",
+ }
+)
+
+#: Mirrors ``INSECURE_SECRET_KEY_DEFAULTS`` so a placeholder token is treated
+#: as unset here too. Accepting a well-known literal would authenticate
+#: anybody who read the repository.
+_INSECURE_SERVICE_TOKENS = frozenset({"", "changeme", "change-me", "secret", "aisoc_dev_secret"})
+
+
+def resolve_service_token() -> str:
+ """The shared secret a peer service presents, or "" when unusable.
+
+ Per-service override first, then the shared platform token, matching
+ ``resolve_service_token`` in the vendored ``app/security/tenant_scope.py``
+ that every other service resolves its own token with.
+ """
+ specific = (os.getenv("AISOC_API_SERVICE_TOKEN") or "").strip()
+ token = specific or (os.getenv("AISOC_SERVICE_TOKEN") or "").strip()
+ if token.lower() in _INSECURE_SERVICE_TOKENS:
+ return ""
+ return token
+
+
+async def _resolve_service_principal(
+ token: str,
+ declared_tenant: str | None,
+ db: AsyncSession,
+) -> CurrentUser | None:
+ """A trusted peer service acting for one named tenant, or ``None``.
+
+ ``None`` means "this is not a service token", so the caller falls through
+ to the JWT path. Every other refusal raises, because once the token has
+ matched, a malformed or unknown tenant is a request to refuse rather than
+ a different kind of credential to try.
+
+ Why the tenant is a header and not a claim
+ ------------------------------------------
+ A service token identifies *a trusted service*, not a tenant. The agents
+ container triages alerts for every tenant on the deployment, so the tenant
+ has to come from the work it is doing. Making that explicit and mandatory
+ is the whole design: a service token with no tenant resolves to an
+ **empty** scope, and an empty scope refuses rather than widening. Every
+ cross-tenant leak this codebase has had took the other shape, a scope that
+ was absent rather than narrow and a read that treated absent as "no
+ filter".
+
+ The tenant is checked against the table. The header is caller-supplied,
+ and ``POST /v1/ingest/batch`` once trusted exactly such a header with
+ nothing verifying it existed.
+ """
+ expected = resolve_service_token()
+ if not expected or not hmac.compare_digest(token, expected):
+ return None
+
+ if not declared_tenant or not declared_tenant.strip():
+ logger.warning("deps.service_caller_named_no_tenant header=%s", SERVICE_TENANT_HEADER)
+ raise HTTPException(
+ status_code=status.HTTP_403_FORBIDDEN,
+ detail=f"service token must declare the tenant it acts for on {SERVICE_TENANT_HEADER}",
+ )
+
+ try:
+ tenant_id = uuid.UUID(declared_tenant.strip())
+ except ValueError:
+ raise HTTPException(
+ status_code=status.HTTP_400_BAD_REQUEST,
+ detail=f"{SERVICE_TENANT_HEADER} must be a UUID",
+ ) from None
+
+ exists = await db.execute(select(Tenant.id).where(Tenant.id == tenant_id))
+ if exists.scalar_one_or_none() is None:
+ logger.warning(
+ "deps.service_caller_named_an_unknown_tenant tenant=%s",
+ str(tenant_id).replace("\r", "").replace("\n", " ")[:64],
+ )
+ raise HTTPException(
+ status_code=status.HTTP_403_FORBIDDEN,
+ detail="service token named a tenant that does not exist",
+ )
+
+ return CurrentUser(
+ # A service is not a person. The id is deterministic from the tenant
+ # so an audit row says which tenant a service acted for, and the
+ # email names the mechanism rather than impersonating an operator.
+ user_id=uuid.uuid5(uuid.NAMESPACE_URL, f"aisoc:service:{tenant_id}"),
+ tenant_id=tenant_id,
+ role="api_service",
+ email=f"service@{tenant_id}.internal",
+ resolved_permissions=SERVICE_PRINCIPAL_PERMISSIONS,
+ )
+
+
async def _resolve_api_key(raw_key: str, db: AsyncSession) -> CurrentUser:
"""Look up and validate an aisoc_ API key; return its CurrentUser."""
hashed = hash_api_key(raw_key)
@@ -234,12 +357,23 @@ async def _resolve_api_key(raw_key: str, db: AsyncSession) -> CurrentUser:
async def get_current_user(
+ request: Request,
credentials: Annotated[HTTPAuthorizationCredentials | None, Security(bearer_scheme)],
db: AsyncSession = Depends(get_db),
) -> CurrentUser:
"""Resolve Bearer token to CurrentUser.
- Accepts both JWT tokens and aisoc_ API keys.
+ Accepts JWT tokens, aisoc_ API keys, and the shared service token used for
+ service-to-service calls.
+
+ The service path exists because the agents service had no usable way to
+ reach this API at all. Its tools authenticated with ``AISOC_AGENTS_API_KEY``,
+ which no compose file, ``.env.example`` or Helm value ever delivered, so on
+ a default deployment every customer tool, the hunting agent and the sandbox
+ tool answered "could not check" — a gap in visibility a model reports and an
+ operator never sees. Setting that key would not have fixed it either: one
+ key belongs to one tenant, so every tenant's investigation would have read
+ that tenant's estate.
In development mode an unauthenticated request resolves to a deterministic
demo user (see ``app.api.v1.dev_auth``). Production requires a bearer token.
@@ -264,6 +398,21 @@ async def get_current_user(
if token.startswith(_API_KEY_PREFIX):
return await _resolve_api_key(token, db)
+ # --- trusted peer service acting for one declared tenant ---
+ #
+ # Ahead of the JWT path because a service token is not a JWT and would
+ # otherwise be decoded, fail, and answer "Could not validate credentials",
+ # which is what it did.
+ # Read off the request rather than declared as a `Header` parameter.
+ # Declaring it adds a 422 response to all 456 operations in the published
+ # spec, because a parameter that exists can fail validation -- and this is
+ # how a peer *service* names the tenant it acts for, not part of the
+ # contract a customer codes against.
+ declared_tenant = request.headers.get(SERVICE_TENANT_HEADER)
+ service_principal = await _resolve_service_principal(token, declared_tenant, db)
+ if service_principal is not None:
+ return service_principal
+
# --- JWT path ---
try:
payload = decode_token(token)
diff --git a/services/api/app/api/v1/endpoints/api_keys.py b/services/api/app/api/v1/endpoints/api_keys.py
index 997b4df03..e13bb0214 100644
--- a/services/api/app/api/v1/endpoints/api_keys.py
+++ b/services/api/app/api/v1/endpoints/api_keys.py
@@ -60,6 +60,13 @@ VALID_SCOPES: frozenset[str] = frozenset(
"plugins:admin",
"rules:read",
"rules:write",
+ # The agent-tool surface. These three are permissions four roles
+ # already hold, but none was mintable into a key, so a tenant
+ # automating a read-only vendor verb, a lake query or a hunt had no
+ # option but a wildcard key.
+ "actions:read",
+ "lake:query",
+ "hunts:read",
"users:read",
"threat_intel:read",
"threat_intel:write",
diff --git a/services/api/app/api/v1/endpoints/connectors.py b/services/api/app/api/v1/endpoints/connectors.py
index cbcf5d06b..3acb5500b 100644
--- a/services/api/app/api/v1/endpoints/connectors.py
+++ b/services/api/app/api/v1/endpoints/connectors.py
@@ -35,7 +35,6 @@ from __future__ import annotations
import json
import logging
-import os
import re
import uuid
from dataclasses import dataclass
@@ -73,6 +72,7 @@ from pydantic import BaseModel, Field # noqa: E402
from sqlalchemy import select, update # noqa: E402
from app.api.v1.deps import AuthUser, DBSession, require_permission # noqa: E402
+from app.core import connectors_auth # noqa: E402
from app.core.config import settings # noqa: E402
from app.models.connector import Connector # noqa: E402
from app.security.credential_vault import CredentialVaultError, get_vault # noqa: E402
@@ -329,29 +329,19 @@ _TENANT_HEADER = "X-AiSOC-Tenant-ID"
def _service_token() -> str:
"""The shared secret this service presents to the connectors service."""
- specific = (os.getenv("AISOC_CONNECTORS_SERVICE_TOKEN") or "").strip()
- return specific or (os.getenv("AISOC_SERVICE_TOKEN") or "").strip()
+ return connectors_auth.service_token()
def _catalog_headers(tenant_id: uuid.UUID | str | None) -> dict[str, str]:
"""Credential + tenant assertion for a proxied connectors-service call.
- Every route on the connectors service sits behind
- ``require_console_or_service_auth``: a bearer credential is mandatory, and
- a *service* token must additionally declare the tenant it acts for,
- because an absent scope there is an empty scope rather than every scope.
- This proxy sent neither, so it was answered with 401 on every single call
- and fell through to the bundled catalog every time — which is how a
- 26-entry list stood in for an 84-connector registry while every check
- passed. Mirrors ``_upstream_headers`` in the fusion gateway.
+ Delegates to :mod:`app.core.connectors_auth`, which is now the single
+ implementation. This route had the only correct one for a while, and two
+ later callers in ``federated.py`` and ``case_fanout.py`` were written
+ without it and were answered 401 on every deployment. Keeping a second
+ copy here is how that happens again.
"""
- headers: dict[str, str] = {}
- token = _service_token()
- if token:
- headers["Authorization"] = f"Bearer {token}"
- if tenant_id is not None:
- headers[_TENANT_HEADER] = str(tenant_id)
- return headers
+ return connectors_auth.connectors_headers(tenant_id)
#: Where a catalog came from. The distinction is load-bearing: see
diff --git a/services/api/app/api/v1/endpoints/federated.py b/services/api/app/api/v1/endpoints/federated.py
index 4362694f4..84ff2cacb 100644
--- a/services/api/app/api/v1/endpoints/federated.py
+++ b/services/api/app/api/v1/endpoints/federated.py
@@ -45,6 +45,7 @@ from sqlalchemy import select
from app.api.v1.deps import AuthUser, DBSession, require_permission
from app.core.config import settings
+from app.core.connectors_auth import connectors_headers
from app.models.audit import AuditLog
from app.models.connector import Connector
from app.security.credential_vault import CredentialVaultError, get_vault
@@ -250,7 +251,7 @@ async def _query_one_backend(
try:
async with httpx.AsyncClient(timeout=timeout) as client:
- resp = await client.post(url, json=payload)
+ resp = await client.post(url, json=payload, headers=connectors_headers(connector.tenant_id))
except httpx.HTTPError as exc:
logger.warning(
"federated.query.unreachable connector=%s url=%s err=%s",
diff --git a/services/api/app/core/connectors_auth.py b/services/api/app/core/connectors_auth.py
new file mode 100644
index 000000000..1f27f9758
--- /dev/null
+++ b/services/api/app/core/connectors_auth.py
@@ -0,0 +1,73 @@
+"""The credential and tenant every API-to-connectors call must carry.
+
+One helper, because three call sites each reached the connectors service their
+own way and two of them reached it with nothing.
+
+The far side
+------------
+`services/connectors/app/api/router.py` mounts its whole router behind
+``Depends(require_console_or_service_auth)``. That dependency requires a
+bearer credential, and when the credential is a *service* token it requires
+the caller to declare which tenant it is acting for on
+``X-AiSOC-Tenant-ID`` as well. An absent tenant there resolves to an **empty**
+scope and refuses, rather than widening to every tenant.
+
+What shipped
+------------
+Only the catalog proxy in ``app/api/v1/endpoints/connectors.py`` sent either.
+
+* ``federated.py`` posted the unified query as ``client.post(url, json=...)``,
+ so **every federated search, the agent's federated tool and the retro-hunt
+ SIEM sweep were answered 401 by every SIEM**. The console reported no
+ results, which is indistinguishable from a SIEM that held none.
+* ``case_fanout.py`` pushed cases and polled their status the same way.
+
+Both were written after the catalog proxy was fixed, which is the shape this
+module exists to stop: a fix applied where a report pointed, while its
+siblings kept shipping.
+
+Why the tenant is explicit
+--------------------------
+The connectors service holds every tenant's connector rows. A call that does
+not name a tenant is not a call for all of them; it is a call whose scope
+nobody decided. Passing it here makes the decision visible at every call site
+and lets the far side refuse one that forgot.
+"""
+
+from __future__ import annotations
+
+import os
+import uuid
+
+#: Header the connectors service reads to learn which tenant a service caller
+#: is acting for. Spelled identically to ``TENANT_HEADER`` in the vendored
+#: ``app/security/tenant_scope.py`` on both sides.
+TENANT_HEADER = "X-AiSOC-Tenant-ID"
+
+
+def service_token() -> str:
+ """The shared secret this service presents to the connectors service.
+
+ Per-service override first, then the shared platform token, matching
+ ``resolve_service_token`` in the vendored ``tenant_scope`` module that the
+ far side resolves its own copy with.
+ """
+ specific = (os.getenv("AISOC_CONNECTORS_SERVICE_TOKEN") or "").strip()
+ return specific or (os.getenv("AISOC_SERVICE_TOKEN") or "").strip()
+
+
+def connectors_headers(tenant_id: uuid.UUID | str | None) -> dict[str, str]:
+ """Credential plus tenant assertion for one connectors-service call.
+
+ An empty token yields no ``Authorization`` header rather than an empty
+ one: the far side's refusal then says "missing bearer credential", which
+ names the deployment problem, instead of "invalid or missing credential",
+ which reads like a wrong secret.
+ """
+ headers: dict[str, str] = {}
+ token = service_token()
+ if token:
+ headers["Authorization"] = f"Bearer {token}"
+ if tenant_id is not None:
+ headers[TENANT_HEADER] = str(tenant_id)
+ return headers
diff --git a/services/api/app/core/security.py b/services/api/app/core/security.py
index 24237db45..e70b6b30d 100644
--- a/services/api/app/core/security.py
+++ b/services/api/app/core/security.py
@@ -73,6 +73,7 @@ ROLE_PERMISSIONS: dict[str, list[str]] = {
# rewriter still enforces tenant_id predicates and the
# ClickHouse client still enforces row caps and timeouts; the
# permission only controls who *can* query at all.
+ "hunts:read",
"lake:query",
"lake:read_schema",
# The live-action registry: which vendors can perform which response
@@ -108,6 +109,7 @@ ROLE_PERMISSIONS: dict[str, list[str]] = {
# hunter or tenant-admin role.
"threat_intel:write",
# SOC leads run investigations across the lake routinely.
+ "hunts:read",
"lake:query",
"lake:read_schema",
"actions:read",
@@ -132,6 +134,7 @@ ROLE_PERMISSIONS: dict[str, list[str]] = {
# Analysts need lake access to drill into raw events when
# alerts don't tell the whole story. Schema is read-only and
# the rate limiter caps abuse.
+ "hunts:read",
"lake:query",
"lake:read_schema",
# Analysts already hold playbooks:execute, and a dry run touches no
@@ -157,6 +160,7 @@ ROLE_PERMISSIONS: dict[str, list[str]] = {
# Threat hunters live in the lake — this is their primary
# workspace for hypothesis-driven investigation across raw
# events, alert metrics, and IOC enrichments.
+ "hunts:read",
"lake:query",
"lake:read_schema",
# Read the registry to know what response is available for a finding;
diff --git a/services/api/app/services/agent_tools/vendor_aliases.py b/services/api/app/services/agent_tools/vendor_aliases.py
new file mode 100644
index 000000000..cc5e2390a
--- /dev/null
+++ b/services/api/app/services/agent_tools/vendor_aliases.py
@@ -0,0 +1,80 @@
+"""How a read executor's vendor id maps to the connectors a tenant saves.
+
+Why the two differ at all
+-------------------------
+A read executor is named for the **product** it drives: ``defender``,
+``entra``, ``aws``. A connector is named for the **integration** a tenant
+configures, which is more specific because one vendor ships several: Microsoft
+has ``azure_defender`` and ``azure_entra``, AWS has ``aws_cloudtrail``,
+``aws_guardduty``, ``aws_securityhub`` and ``aws_vpc_flow_logs``.
+
+Both namings are reasonable and neither is going to win, so the mapping is
+written down instead of assumed.
+
+What went wrong without it
+--------------------------
+``vendor_reads.available_reads`` resolved the two with ``by_type.get(vendor_id)``.
+For four of the seven executors the strings happened to be identical and it
+worked. For ``defender``, ``entra`` and ``aws`` it returned ``None`` on every
+tenant, so the tool was never bound and a model investigating a Defender host,
+an Entra identity or a CloudTrail event was told the tenant had no integration
+for it. Three of the five vendors added in gap-closure 4.2 had never once been
+offered.
+
+The lookup that missed is the same expression as the lookup that hits, which is
+why nothing failed. ``scripts/check_vendor_catalog_ids.py`` now fails CI when a
+vendor id resolves to no saveable connector type, in both directions.
+
+Order matters
+-------------
+The tuples are in preference order. ``aws`` reads a cloud audit trail, so
+``aws_cloudtrail`` comes first; a tenant with both CloudTrail and GuardDuty
+gets the one that answers the question the capability asks.
+"""
+
+from __future__ import annotations
+
+from typing import TypeVar
+
+#: Generic over the mapping's value so a caller keeps its own row type. A
+#: bare ``object`` here made every attribute read on the result an error at
+#: the call site, which is the type system correctly objecting to a helper
+#: that threw away what it was given.
+_T = TypeVar("_T")
+
+#: ``executor vendor id -> catalog connector types, most preferred first``.
+#:
+#: An entry is only needed where the two names differ. The four that already
+#: agree (``crowdstrike``, ``okta``, ``sentinelone``, ``google_workspace``)
+#: are deliberately absent, so this map stays a list of exceptions rather than
+#: a second copy of the catalog that drifts from it.
+VENDOR_CONNECTOR_TYPES: dict[str, tuple[str, ...]] = {
+ "defender": ("azure_defender",),
+ "entra": ("azure_entra",),
+ "aws": ("aws_cloudtrail", "aws_guardduty", "aws_security_hub"),
+}
+
+
+def connector_types_for(vendor_id: str) -> tuple[str, ...]:
+ """Every connector type that can serve this executor, most preferred first.
+
+ A vendor with no alias resolves to itself, which is the common case and
+ keeps the map to exceptions only.
+ """
+ return VENDOR_CONNECTOR_TYPES.get(vendor_id, (vendor_id,))
+
+
+def resolve(vendor_id: str, by_type: dict[str, _T]) -> _T | None:
+ """The tenant's connector for this executor, or ``None``.
+
+ One helper rather than the same two lines at each of the three call sites
+ that need it: ``vendor_reads.available_reads``,
+ ``playbook_step_dispatch._pick_connector`` and live-action dispatch. The
+ reason this module exists is that a lookup was written three times and got
+ it wrong in the same way each time.
+ """
+ for connector_type in connector_types_for(vendor_id):
+ found = by_type.get(connector_type)
+ if found is not None:
+ return found
+ return None
diff --git a/services/api/app/services/agent_tools/vendor_reads.py b/services/api/app/services/agent_tools/vendor_reads.py
index af5a0e745..4c26a5e25 100644
--- a/services/api/app/services/agent_tools/vendor_reads.py
+++ b/services/api/app/services/agent_tools/vendor_reads.py
@@ -44,6 +44,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.models.connector import Connector
from app.services import actions_client
+from app.services.agent_tools import vendor_aliases
from app.services.playbook_step_dispatch import StepDispatchReport, dispatch_step
logger = logging.getLogger(__name__)
@@ -169,7 +170,10 @@ async def available_reads(db: AsyncSession, *, tenant_id: uuid.UUID) -> list[Ava
logger.warning("agent_tools.registry_unreachable capability=%s error=%s", capability, str(exc)[:200])
raise
for vendor_id in sorted(implementers):
- connector = by_type.get(vendor_id)
+ # Through the alias map: a read executor is named for the
+ # product and a connector for the integration, and three of
+ # the seven never matched by string alone.
+ connector = vendor_aliases.resolve(vendor_id, by_type)
if connector is None:
continue
allowed = connector.allowed_capabilities
diff --git a/services/api/app/services/case_fanout.py b/services/api/app/services/case_fanout.py
index 57e077e1f..5f5e778ba 100644
--- a/services/api/app/services/case_fanout.py
+++ b/services/api/app/services/case_fanout.py
@@ -56,6 +56,7 @@ from sqlalchemy import select, text
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.config import settings
+from app.core.connectors_auth import connectors_headers
from app.models.connector import Connector
from app.security.credential_vault import CredentialVaultError, get_vault
@@ -168,6 +169,7 @@ async def _post_to_connector_service(
url: str,
payload: dict[str, Any],
timeout_seconds: float,
+ tenant_id: Any,
) -> tuple[str, dict[str, Any] | None, str | None]:
"""POST to the connectors microservice and return (status, body, err).
@@ -182,7 +184,7 @@ async def _post_to_connector_service(
timeout = httpx.Timeout(timeout_seconds, connect=min(5.0, timeout_seconds))
try:
async with httpx.AsyncClient(timeout=timeout) as client:
- resp = await client.post(url, json=payload)
+ resp = await client.post(url, json=payload, headers=connectors_headers(tenant_id))
except httpx.HTTPError as exc:
return ("error", None, f"connectors service unreachable: {exc}")
@@ -360,6 +362,7 @@ async def _push_one_case(
url=url,
payload=payload,
timeout_seconds=timeout_seconds,
+ tenant_id=connector.tenant_id,
)
if push_status != "ok" or body is None:
@@ -545,6 +548,7 @@ async def _push_one_status_change(
url=url,
payload=payload,
timeout_seconds=timeout_seconds,
+ tenant_id=connector.tenant_id,
)
if push_status != "ok" or body is None:
diff --git a/services/api/app/services/playbook_step_dispatch.py b/services/api/app/services/playbook_step_dispatch.py
index 1b02a2a71..b2f63c09e 100644
--- a/services/api/app/services/playbook_step_dispatch.py
+++ b/services/api/app/services/playbook_step_dispatch.py
@@ -48,6 +48,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.models.connector import Connector
from app.security.credential_vault import CredentialVaultError, get_vault
from app.services import actions_client
+from app.services.agent_tools import vendor_aliases
logger = structlog.get_logger(__name__)
@@ -244,7 +245,19 @@ async def _pick_connector(
happened to return. A non-deterministic containment target is not a
feature.
"""
- wanted = [pinned] if pinned else implementers
+ # Expanded through the alias map, because `implementers` holds executor
+ # vendor ids and `connector_type` holds catalog ids. `defender`, `entra`
+ # and `aws` match no connector by string, so an IN clause built from the
+ # raw ids silently selected nothing for three of the seven vendors.
+ wanted = sorted(
+ {
+ connector_type
+ for vendor_id in ([pinned] if pinned else implementers)
+ for connector_type in vendor_aliases.connector_types_for(vendor_id)
+ }
+ )
+ allowed_types = {connector_type for vendor_id in implementers for connector_type in vendor_aliases.connector_types_for(vendor_id)}
+ pinned_types = set(vendor_aliases.connector_types_for(pinned)) if pinned else set()
rows = (
(
await db.execute(
@@ -259,9 +272,9 @@ async def _pick_connector(
.all()
)
for connector in rows:
- if pinned and connector.connector_type != pinned:
+ if pinned and connector.connector_type not in pinned_types:
continue
- if connector.connector_type not in implementers:
+ if connector.connector_type not in allowed_types:
# The pin named a connector the tenant has and no executor can
# drive. Not a match, and not something to substitute around.
continue
diff --git a/services/api/tests/test_case_fanout.py b/services/api/tests/test_case_fanout.py
index a69b152c5..5c8974a00 100644
--- a/services/api/tests/test_case_fanout.py
+++ b/services/api/tests/test_case_fanout.py
@@ -235,6 +235,11 @@ def test_serialize_case_for_push_handles_row_with_mapping() -> None:
# ---------------------------------------------------------------------------
+#: The connectors service refuses a service token that does not name the
+#: tenant it is acting for, so every call site supplies one.
+FANOUT_TENANT = uuid.UUID("22222222-2222-2222-2222-222222222222")
+
+
@pytest.mark.asyncio
async def test_post_returns_ok_on_2xx() -> None:
fake_resp = _build_post_response(
@@ -243,7 +248,7 @@ async def test_post_returns_ok_on_2xx() -> None:
)
with patch("app.services.case_fanout.httpx.AsyncClient") as mock_client:
mock_client.return_value.__aenter__.return_value.post = AsyncMock(return_value=fake_resp)
- status, body, err = await _post_to_connector_service(url="http://x/y", payload={}, timeout_seconds=5.0)
+ status, body, err = await _post_to_connector_service(url="http://x/y", payload={}, timeout_seconds=5.0, tenant_id=FANOUT_TENANT)
assert status == "ok"
assert body == {"external_id": "AIS-1", "vendor": "jira"}
assert err is None
@@ -254,7 +259,7 @@ async def test_post_returns_unsupported_on_501() -> None:
fake_resp = _build_post_response(status_code=501, text_body="capability missing")
with patch("app.services.case_fanout.httpx.AsyncClient") as mock_client:
mock_client.return_value.__aenter__.return_value.post = AsyncMock(return_value=fake_resp)
- status, body, err = await _post_to_connector_service(url="http://x/y", payload={}, timeout_seconds=5.0)
+ status, body, err = await _post_to_connector_service(url="http://x/y", payload={}, timeout_seconds=5.0, tenant_id=FANOUT_TENANT)
assert status == "unsupported"
assert body is None
assert err == "capability missing"
@@ -268,7 +273,7 @@ async def test_post_returns_error_on_4xx_with_detail() -> None:
)
with patch("app.services.case_fanout.httpx.AsyncClient") as mock_client:
mock_client.return_value.__aenter__.return_value.post = AsyncMock(return_value=fake_resp)
- status, body, err = await _post_to_connector_service(url="http://x/y", payload={}, timeout_seconds=5.0)
+ status, body, err = await _post_to_connector_service(url="http://x/y", payload={}, timeout_seconds=5.0, tenant_id=FANOUT_TENANT)
assert status == "error"
assert body is None
assert "missing project_key" in (err or "")
@@ -282,7 +287,7 @@ async def test_post_returns_error_on_transport_failure() -> None:
create endpoint."""
with patch("app.services.case_fanout.httpx.AsyncClient") as mock_client:
mock_client.return_value.__aenter__.return_value.post = AsyncMock(side_effect=httpx.ConnectError("dns failed"))
- status, body, err = await _post_to_connector_service(url="http://x/y", payload={}, timeout_seconds=5.0)
+ status, body, err = await _post_to_connector_service(url="http://x/y", payload={}, timeout_seconds=5.0, tenant_id=FANOUT_TENANT)
assert status == "error"
assert body is None
assert "connectors service unreachable" in (err or "")
@@ -295,7 +300,7 @@ async def test_post_returns_error_on_non_dict_body() -> None:
fake_resp = _build_post_response(status_code=200, json_body=[1, 2, 3]) # type: ignore[arg-type]
with patch("app.services.case_fanout.httpx.AsyncClient") as mock_client:
mock_client.return_value.__aenter__.return_value.post = AsyncMock(return_value=fake_resp)
- status, body, err = await _post_to_connector_service(url="http://x/y", payload={}, timeout_seconds=5.0)
+ status, body, err = await _post_to_connector_service(url="http://x/y", payload={}, timeout_seconds=5.0, tenant_id=FANOUT_TENANT)
assert status == "error"
assert "unexpected payload shape" in (err or "")
diff --git a/services/api/tests/test_vendor_alias_resolution.py b/services/api/tests/test_vendor_alias_resolution.py
new file mode 100644
index 000000000..49fed57e2
--- /dev/null
+++ b/services/api/tests/test_vendor_alias_resolution.py
@@ -0,0 +1,115 @@
+"""The three vendors that were never offered to an investigation now are.
+
+Fix pass item 1.3. See `plans/aisoc_fix_pass_plan.plan.md`.
+
+What was wrong
+--------------
+`vendor_reads.available_reads` decided which vendor tools to bind by looking an
+executor's vendor id up against the tenant's saved connectors:
+
+ by_type.get(vendor_id)
+
+A read executor is named for the product -- `defender`, `entra`, `aws` -- and a
+connector for the integration a tenant configures -- `azure_defender`,
+`azure_entra`, `aws_cloudtrail`. For four of the seven the two strings happen to
+be identical. For the other three the lookup returned `None` on every tenant,
+so the tool was never bound and a model investigating a Defender host, an Entra
+identity or a CloudTrail event was told the tenant had no integration for it.
+
+Three of the five vendors added in gap-closure 4.2 had therefore never once
+been reachable.
+
+Why nothing failed
+------------------
+The lookup that misses is the same expression as the lookup that hits, and the
+three vendors were simply absent from a list nobody counted.
+
+What this file asserts
+----------------------
+The resolution itself, in both directions: a tenant that saved
+`azure_defender` is offered the `defender` executor, and a tenant that saved
+nothing is offered nothing. The negative half matters as much, because a
+resolver that returned the first connector it found for every vendor would
+satisfy the positive half.
+
+`scripts/check_vendor_catalog_ids.py` owns the naming question -- that every
+executor id resolves to a type a tenant can save. This owns the behavioural
+one.
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+from app.services.agent_tools import vendor_aliases
+
+
+def _connector(connector_type: str) -> SimpleNamespace:
+ return SimpleNamespace(connector_type=connector_type, name=f"{connector_type}-1")
+
+
+class TestTheThreeThatNeverMatched:
+ @pytest.mark.parametrize(
+ ("vendor_id", "saved_type"),
+ [
+ ("defender", "azure_defender"),
+ ("entra", "azure_entra"),
+ ("aws", "aws_cloudtrail"),
+ ],
+ )
+ def test_a_tenant_that_saved_the_integration_is_offered_the_vendor(self, vendor_id: str, saved_type: str) -> None:
+ """Pre-fix each of these resolved to None on every tenant."""
+ by_type = {saved_type: _connector(saved_type)}
+
+ resolved = vendor_aliases.resolve(vendor_id, by_type)
+
+ assert resolved is not None, f"a tenant with {saved_type!r} saved was not offered the {vendor_id!r} read executor"
+ assert resolved.connector_type == saved_type
+
+
+class TestTheFourThatAlreadyMatched:
+ @pytest.mark.parametrize("vendor_id", ["crowdstrike", "okta", "sentinelone", "google_workspace"])
+ def test_an_unaliased_vendor_still_resolves_to_itself(self, vendor_id: str) -> None:
+ """The alias map is a list of exceptions, not a second catalog.
+
+ A vendor with no entry must keep resolving by its own name, or the fix
+ would break the four that were working.
+ """
+ by_type = {vendor_id: _connector(vendor_id)}
+
+ resolved = vendor_aliases.resolve(vendor_id, by_type)
+
+ assert resolved is not None and resolved.connector_type == vendor_id
+
+
+class TestTheNegativeControls:
+ """What stops the assertions above passing over a resolver that matches anything."""
+
+ def test_a_tenant_with_nothing_connected_is_offered_nothing(self) -> None:
+ assert vendor_aliases.resolve("defender", {}) is None
+
+ def test_an_unrelated_connector_does_not_satisfy_a_vendor(self) -> None:
+ """A tenant with Okta has not got Defender.
+
+ A resolver that fell back to "any connector" would bind a tool that
+ answers `no_integration`, which costs a turn of a bounded loop and
+ which some models narrate as though it returned something.
+ """
+ assert vendor_aliases.resolve("defender", {"okta": _connector("okta")}) is None
+
+ def test_preference_order_is_honoured(self) -> None:
+ """`aws` reads a cloud audit trail, so CloudTrail wins over GuardDuty.
+
+ A tenant with both gets the one that answers the question the
+ capability asks, rather than whichever the dictionary happened to
+ iterate first.
+ """
+ by_type = {
+ "aws_guardduty": _connector("aws_guardduty"),
+ "aws_cloudtrail": _connector("aws_cloudtrail"),
+ }
+
+ resolved = vendor_aliases.resolve("aws", by_type)
+
+ assert resolved is not None and resolved.connector_type == "aws_cloudtrail"
diff --git a/tests/isolation/test_agent_service_auth_live.py b/tests/isolation/test_agent_service_auth_live.py
new file mode 100644
index 000000000..1d9dcae6c
--- /dev/null
+++ b/tests/isolation/test_agent_service_auth_live.py
@@ -0,0 +1,232 @@
+"""The agents service reaches the API as a service, for one named tenant.
+
+Fix pass item 1.1. See `plans/aisoc_fix_pass_plan.plan.md`.
+
+What was wrong
+--------------
+Three agent tool modules authenticated with `AISOC_AGENTS_API_KEY`
+(`app/tools/customer_tools.py`, `app/hunt/agent.py`, `app/tools/sandbox.py`),
+and **no compose file, `.env.example` or Helm value delivered that key**. On a
+default `make up` every one of them answered "could not check", which a model
+reads as a gap in visibility and an operator never sees.
+
+Setting the key would not have fixed it. One key belongs to one tenant, so
+every tenant's investigation would read that tenant's estate. And the routes
+these tools call require `actions:read`, `lake:query` and `hunts:read`, none of
+which is in `VALID_SCOPES`, so only a `*` key could reach them at all.
+
+Why the offline suites did not catch it
+---------------------------------------
+They stub `_call`, or they drive the route with a `CurrentUser` built in the
+test. Neither reaches the question this file asks, which is whether the
+credential the deployment actually hands the agents container is one the API
+will accept, and whether it confines the answer to one tenant.
+
+What this file asserts
+----------------------
+The real `create_application()` against real Postgres with every migration
+applied, driven over a real `httpx` client, using exactly the credential
+`docker-compose.yml` gives the agents container:
+
+* a service token plus `X-AiSOC-Tenant-ID` is accepted, and
+* it returns that tenant's connectors and never the other tenant's.
+
+The negative controls are what stop those passing for the wrong reason: a
+surface that answered everything to everyone, or one that answered an empty
+list to everyone, would satisfy a naive "B is present" assertion. So this also
+asserts that no credential is refused, that a service token with **no** tenant
+header is refused rather than widened, and that a tenant header naming a
+tenant that does not exist is refused.
+
+Against the pre-fix tree every positive assertion here fails with 401, because
+the API had no service-caller path at all.
+"""
+
+from __future__ import annotations
+
+import os
+import uuid
+
+import pytest
+import pytest_asyncio
+
+# Skip as a module, not only in the fixture. The offline isolation job
+# collects this directory with no Postgres running, and a test that does not
+# take the fixture would otherwise run and fail there, reporting a harness
+# problem against a capability.
+pytestmark = [
+ pytest.mark.asyncio(loop_scope="module"),
+ pytest.mark.skipif(
+ not os.environ.get("ISOLATION_AGENT_AUTH_DSN", "").strip(),
+ reason="ISOLATION_AGENT_AUTH_DSN is not set; this suite needs a live Postgres",
+ ),
+]
+
+#: The token the deployment hands the agents container. `docker-compose.yml`
+#: sets `AISOC_SERVICE_TOKEN` on the agents service already; what was missing
+#: was any code path that presented it to the API.
+SERVICE_TOKEN = "fixpass-service-token-not-a-real-secret"
+
+TENANT_HEADER = "X-AiSOC-Tenant-ID"
+
+
+def _dsn() -> str:
+ value = os.environ.get("ISOLATION_AGENT_AUTH_DSN", "").strip()
+ if not value:
+ pytest.skip("ISOLATION_AGENT_AUTH_DSN is not set")
+ return value
+
+
+@pytest_asyncio.fixture(scope="module", loop_scope="module")
+async def app_client():
+ """The real application, with the real auth dependency graph.
+
+ `create_application()` rather than a bare `FastAPI()` with one router
+ mounted: the middleware, the exception handlers and the dependencies are
+ part of what the agents service talks to.
+ """
+ pytest.importorskip("httpx")
+ os.environ["DATABASE_URL"] = _dsn()
+ os.environ["AISOC_SERVICE_TOKEN"] = SERVICE_TOKEN
+ # Production posture. Without this the dev-mode shim answers every
+ # uncredentialed request with a demo principal and the negative controls
+ # below pass for the wrong reason.
+ os.environ["ENVIRONMENT"] = "production"
+ os.environ.pop("AISOC_DEV_MODE", None)
+ os.environ.pop("AISOC_DEV_AUTH_BYPASS", None)
+ # A real signing secret, so the console branch is configured and the
+ # service branch is not reached merely because nothing else was.
+ os.environ.setdefault("SECRET_KEY", "fixpass-console-secret-at-least-32-chars-long")
+
+ import httpx
+ from app.main import create_application
+
+ application = create_application()
+ transport = httpx.ASGITransport(app=application)
+ async with httpx.AsyncClient(transport=transport, base_url="http://testserver") as client:
+ yield client
+
+
+@pytest_asyncio.fixture(scope="module", loop_scope="module")
+async def db():
+ pytest.importorskip("asyncpg")
+ from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
+
+ engine = create_async_engine(_dsn(), future=True)
+ maker = async_sessionmaker(engine, expire_on_commit=False)
+ async with maker() as session:
+ yield session
+ await engine.dispose()
+
+
+async def _tenant_with_connector(session, label: str) -> tuple[str, str]:
+ """A tenant and one enabled, federated-capable connector of its own."""
+ from sqlalchemy import text
+
+ tenant_id = str(uuid.uuid4())
+ slug = f"fixpass-{label}-{tenant_id[:8]}"
+ await session.execute(
+ text("INSERT INTO tenants (id, name, slug) VALUES (CAST(:tid AS uuid), :name, :slug)"),
+ {"tid": tenant_id, "name": f"Fix pass {label}", "slug": slug},
+ )
+ connector_name = f"{label}-splunk"
+ await session.execute(
+ text(
+ "INSERT INTO connectors (id, tenant_id, name, connector_type, category, is_enabled) "
+ "VALUES (CAST(:cid AS uuid), CAST(:tid AS uuid), :name, 'splunk', 'siem', TRUE)"
+ ),
+ {"cid": str(uuid.uuid4()), "tid": tenant_id, "name": connector_name},
+ )
+ await session.commit()
+ return tenant_id, connector_name
+
+
+@pytest_asyncio.fixture(scope="module", loop_scope="module")
+async def two_tenants(db):
+ """Two tenants, each with a connector only it should see."""
+ a = await _tenant_with_connector(db, "alpha")
+ b = await _tenant_with_connector(db, "bravo")
+ return {"a": a, "b": b}
+
+
+def _service_headers(tenant_id: str | None) -> dict[str, str]:
+ headers = {"Authorization": f"Bearer {SERVICE_TOKEN}"}
+ if tenant_id is not None:
+ headers[TENANT_HEADER] = tenant_id
+ return headers
+
+
+def _connector_names(payload: dict) -> set[str]:
+ return {entry.get("name", "") for entry in payload.get("siem_search", {}).get("backends", [])}
+
+
+class TestTheServiceCredentialIsAccepted:
+ async def test_a_service_token_with_a_tenant_reaches_the_agent_tool_surface(self, app_client, two_tenants) -> None:
+ """The credential compose actually delivers is one the API accepts.
+
+ Against the pre-fix tree this is 401: the API resolved a bearer token
+ as a JWT or an `aisoc_` API key and had no service path, so the
+ agents container could not authenticate at all without a hand-minted
+ wildcard key that no deployment creates.
+ """
+ tenant_b, _ = two_tenants["b"]
+ response = await app_client.get("/api/v1/agent-tools/backends", headers=_service_headers(tenant_b))
+
+ assert response.status_code == 200, (
+ f"the agents service could not authenticate to the API: {response.status_code} {response.text[:300]}"
+ )
+
+ async def test_the_answer_is_scoped_to_the_tenant_the_service_named(self, app_client, two_tenants) -> None:
+ """One token, two tenants, two different answers.
+
+ This is the half a single shared API key could never satisfy: the key
+ belongs to one tenant, so every investigation would have read that
+ tenant's estate whatever alert it was working on.
+ """
+ tenant_a, connector_a = two_tenants["a"]
+ tenant_b, connector_b = two_tenants["b"]
+
+ got_a = _connector_names((await app_client.get("/api/v1/agent-tools/backends", headers=_service_headers(tenant_a))).json())
+ got_b = _connector_names((await app_client.get("/api/v1/agent-tools/backends", headers=_service_headers(tenant_b))).json())
+
+ assert connector_a in got_a, f"tenant A did not see its own connector: {sorted(got_a)}"
+ assert connector_b in got_b, f"tenant B did not see its own connector: {sorted(got_b)}"
+ assert connector_b not in got_a, "tenant A was shown tenant B's connector"
+ assert connector_a not in got_b, "tenant B was shown tenant A's connector"
+
+
+class TestTheNegativeControls:
+ """What stops the assertions above passing over a surface that leaks."""
+
+ async def test_no_credential_is_refused(self, app_client, two_tenants) -> None:
+ tenant_b, _ = two_tenants["b"]
+ response = await app_client.get("/api/v1/agent-tools/backends", headers={TENANT_HEADER: tenant_b})
+ assert response.status_code in (401, 403), f"an uncredentialed caller was served: {response.status_code}"
+
+ async def test_a_service_token_without_a_tenant_is_refused_rather_than_widened(self, app_client) -> None:
+ """An absent scope is an empty scope, never every scope.
+
+ Every cross-tenant leak this codebase has had took the other shape: a
+ scope that was absent rather than narrow, and a read that treated
+ absent as "no filter".
+ """
+ response = await app_client.get("/api/v1/agent-tools/backends", headers=_service_headers(None))
+ assert response.status_code in (401, 403), f"a service token with no tenant header was served: {response.status_code}"
+
+ async def test_a_tenant_that_does_not_exist_is_refused(self, app_client) -> None:
+ """The header is caller-supplied, so it is checked against the table.
+
+ `POST /v1/ingest/batch` once trusted a caller-supplied tenant header
+ that nothing checked against `tenants`. A service token is trusted;
+ the tenant it names is not.
+ """
+ response = await app_client.get("/api/v1/agent-tools/backends", headers=_service_headers(str(uuid.uuid4())))
+ assert response.status_code in (401, 403, 404), f"a service token naming an unknown tenant was served: {response.status_code}"
+
+ async def test_a_wrong_service_token_is_refused(self, app_client, two_tenants) -> None:
+ tenant_b, _ = two_tenants["b"]
+ response = await app_client.get(
+ "/api/v1/agent-tools/backends",
+ headers={"Authorization": "Bearer not-the-service-token", TENANT_HEADER: tenant_b},
+ )
+ assert response.status_code in (401, 403), f"a forged service token was served: {response.status_code}"
diff --git a/tests/isolation/test_api_to_connectors_auth.py b/tests/isolation/test_api_to_connectors_auth.py
new file mode 100644
index 000000000..d5f9a2348
--- /dev/null
+++ b/tests/isolation/test_api_to_connectors_auth.py
@@ -0,0 +1,280 @@
+"""Every API-to-connectors call carries the service token and the tenant.
+
+Fix pass item 1.2. See `plans/aisoc_fix_pass_plan.plan.md`.
+
+What was wrong
+--------------
+`services/connectors/app/api/router.py` mounts its whole router behind
+`Depends(require_console_or_service_auth)`. A bearer credential is mandatory,
+and a *service* token must additionally declare the tenant it acts for,
+because an absent scope there is an empty scope rather than every scope.
+
+Two callers in `services/api` sent neither:
+
+* `federated.py` posted the unified query with `client.post(url, json=payload)`,
+ so federated search, the agent's federated tool and the retro-hunt SIEM sweep
+ were answered **401 by every SIEM**.
+* `case_fanout.py` pushed cases and polled their status the same way.
+
+The catalog proxy twenty lines away in `connectors.py` already did it
+correctly, which is the shape worth noting: one call site was fixed when this
+was last found, and its siblings kept shipping.
+
+Why the offline suites did not catch it
+---------------------------------------
+They mock the connectors service with a transport that answers whatever it is
+asked. A fake that never authenticates cannot notice a caller that never
+authenticates.
+
+Why this lives here and not in `services/api/tests/`
+-----------------------------------------------------
+It needs **both** services installed: the API's `_query_one_backend` on one
+side and the connectors application on the other. `ci.yml`'s API job installs
+only the API's lockfile, so the first home for this file errored there with
+`ModuleNotFoundError: apscheduler`. `tests/isolation/` is where a test that
+spans two services belongs, and `agent-auth-live.yml` installs both.
+
+Why this runs the far side in a subprocess
+------------------------------------------
+Both services name their top-level package `app`. Importing the connectors
+router into this suite shadows the API's own package, and the first attempt at
+this file resolved `app.api.v1` inside the connectors tree, where it does not
+exist. Making the test skip around that would have been worse than not writing
+it: a skipping test reports the same word as a passing one.
+
+So the real connectors application is started as its own process on a loopback
+socket, and the request crosses a real one. That is also closer to the
+deployment, where these are two containers.
+
+What this file asserts
+----------------------
+A request built by the API's own `_catalog_headers` helper is **accepted** by
+the real `require_console_or_service_auth` dependency; the same request with
+the credential or the tenant removed is **refused**.
+
+Against the pre-fix tree the first assertion fails with 401, which is what
+every federated search returned on every deployment.
+"""
+
+from __future__ import annotations
+
+import os
+import socket
+import subprocess
+import sys
+import time
+import uuid
+from pathlib import Path
+from types import SimpleNamespace
+
+import httpx
+import pytest
+
+# Gated the way every sibling in this directory is gated, and for the same
+# reason: the offline isolation job collects this directory with neither
+# service installed, so it has no `uvicorn` to start the far side with.
+#
+# Skipping there is honest -- the job cannot run this -- but a skip must never
+# be mistaken for a pass, so `agent-auth-live.yml` sets the flag and asserts at
+# the end that it was set.
+pytestmark = [
+ pytest.mark.anyio,
+ pytest.mark.skipif(
+ not os.environ.get("ISOLATION_CROSS_SERVICE", "").strip(),
+ reason="ISOLATION_CROSS_SERVICE is not set; this suite needs both services installed",
+ ),
+]
+
+SERVICE_TOKEN = "fixpass-connectors-token-not-a-real-secret"
+TENANT = str(uuid.uuid4())
+
+_REPO = Path(__file__).resolve().parents[2]
+_CONNECTORS = _REPO / "services" / "connectors"
+
+
+def anyio_backend() -> str:
+ return "asyncio"
+
+
+def _free_port() -> int:
+ with socket.socket() as sock:
+ sock.bind(("127.0.0.1", 0))
+ return int(sock.getsockname()[1])
+
+
[email protected](scope="module")
+def connectors_base_url(): # noqa: C901
+ """The real connectors service, on a loopback socket, with real auth.
+
+ Not a stand-in and not an in-process mount. The question is whether the
+ headers the API sends satisfy that service's own dependency, and only that
+ dependency can answer it.
+ """
+ if not (_CONNECTORS / "app" / "main.py").is_file():
+ pytest.fail(f"the connectors service is not at {_CONNECTORS}; this test cannot be skipped into passing")
+
+ port = _free_port()
+ env = {
+ **os.environ,
+ "AISOC_SERVICE_TOKEN": SERVICE_TOKEN,
+ "AISOC_CONNECTORS_SERVICE_TOKEN": SERVICE_TOKEN,
+ "AISOC_CONNECTORS_DISABLE_SCHEDULER": "1",
+ "OTEL_SDK_DISABLED": "true",
+ "PYTHONPATH": str(_CONNECTORS),
+ }
+ # Production posture: the anonymous shim must be off, or every negative
+ # control below passes for the wrong reason.
+ env.pop("AISOC_DEV_MODE", None)
+ env.pop("AISOC_DEV_AUTH_BYPASS", None)
+
+ proc = subprocess.Popen(
+ [
+ sys.executable,
+ "-c",
+ # `app.main:app`, not a router re-mounted here. Re-mounting it
+ # dropped the `/api/v1` prefix the real service uses, so the
+ # first run of this test got 404 where the deployment gets 401:
+ # a harness artefact that reads exactly like a passing test.
+ "import uvicorn\n"
+ "from app.main import app as connectors_app\n"
+ f"uvicorn.run(connectors_app, host='127.0.0.1', port={port}, log_level='error')\n",
+ ],
+ cwd=str(_CONNECTORS),
+ env=env,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ )
+
+ base = f"http://127.0.0.1:{port}"
+ deadline = time.time() + 30
+ while time.time() < deadline:
+ if proc.poll() is not None:
+ output = (proc.stdout.read() or b"").decode()[-2000:]
+ pytest.fail(f"the connectors service exited before serving:\n{output}")
+ try:
+ httpx.get(f"{base}/openapi.json", timeout=1.0)
+ break
+ except httpx.HTTPError:
+ time.sleep(0.2)
+ else:
+ proc.terminate()
+ pytest.fail("the connectors service did not start within 30s")
+
+ yield base
+
+ proc.terminate()
+ try:
+ proc.wait(timeout=10)
+ except subprocess.TimeoutExpired: # pragma: no cover - teardown guard
+ proc.kill()
+
+
+async def _post(base: str, headers: dict[str, str]) -> int:
+ async with httpx.AsyncClient(base_url=base, timeout=10.0) as client:
+ response = await client.post(
+ f"/api/v1/connectors/{uuid.uuid4()}/query",
+ json={"auth_config": {}, "connector_config": {}, "query": {}},
+ headers=headers,
+ )
+ return response.status_code
+
+
[email protected](autouse=True)
+def _api_side_credential(monkeypatch: pytest.MonkeyPatch) -> None:
+ """The token the API presents, in the process that builds the headers.
+
+ `_catalog_headers` reads it here, not in the subprocess, so without this
+ the helper would send an empty bearer and the positive assertion would
+ fail for a harness reason that looks exactly like the defect.
+ """
+ monkeypatch.setenv("AISOC_SERVICE_TOKEN", SERVICE_TOKEN)
+ monkeypatch.setenv("AISOC_CONNECTORS_SERVICE_TOKEN", SERVICE_TOKEN)
+
+
+class TestTheProductionCallPath:
+ """`federated.py`'s own request, not a hand-written stand-in for it.
+
+ The helper has worked since the catalog proxy was fixed. What shipped
+ broken is that federated search never called it, so a test that exercised
+ the helper would have passed on the defective tree.
+ """
+
+ async def test_the_federated_backend_query_is_accepted(self, connectors_base_url, monkeypatch: pytest.MonkeyPatch) -> None:
+ # Imported here rather than at module scope: the offline isolation job
+ # collects this directory with no service installed, and a module-level
+ # service import errors collection for the whole directory.
+ federated_mod = pytest.importorskip(
+ "app.api.v1.endpoints.federated",
+ reason="services/api is not installed in this job",
+ )
+ monkeypatch.setattr(federated_mod.settings, "CONNECTORS_SERVICE_URL", connectors_base_url, raising=False)
+
+ connector = SimpleNamespace(
+ id=uuid.uuid4(),
+ tenant_id=uuid.UUID(TENANT),
+ name="prod-splunk",
+ connector_type="splunk",
+ connector_config={},
+ auth_config={},
+ )
+ monkeypatch.setattr(federated_mod, "decrypt_dict", lambda value: dict(value or {}), raising=False)
+
+ verdict, _rows = await federated_mod._query_one_backend(
+ # A stand-in for the ORM row, carrying only the five attributes
+ # this function reads. Constructing a real `Connector` would need
+ # a session and a tenant, neither of which this test is about: the
+ # question is which headers leave the process.
+ connector, # type: ignore[arg-type]
+ {"query": {}},
+ httpx.Timeout(10.0, connect=5.0),
+ )
+
+ status = getattr(verdict, "status", "") or getattr(verdict, "state", "")
+ error = str(getattr(verdict, "error", "") or getattr(verdict, "detail", "") or "")
+ assert "401" not in error and "unauthor" not in error.lower(), (
+ f"federated search was refused by the connectors service: status={status!r} error={error!r}"
+ )
+
+ async def test_the_api_helper_produces_headers_the_connectors_service_accepts(self, connectors_base_url) -> None:
+ """The credential itself is good; this pins that separately."""
+ connectors_mod = pytest.importorskip(
+ "app.api.v1.endpoints.connectors",
+ reason="services/api is not installed in this job",
+ )
+ status = await _post(connectors_base_url, connectors_mod._catalog_headers(TENANT))
+
+ assert status not in (401, 403), f"the connectors service refused the credential the API sends: HTTP {status}"
+
+
+class TestTheNegativeControls:
+ async def test_no_credential_is_refused(self, connectors_base_url) -> None:
+ """The pre-fix call, exactly: a body and nothing else."""
+ assert await _post(connectors_base_url, {}) in (401, 403)
+
+ async def test_a_service_token_without_a_tenant_is_refused(self, connectors_base_url) -> None:
+ """An absent scope is an empty scope, never every scope."""
+ status = await _post(connectors_base_url, {"Authorization": f"Bearer {SERVICE_TOKEN}"})
+ assert status in (401, 403), f"a service token with no tenant was served: HTTP {status}"
+
+ async def test_a_forged_token_is_refused(self, connectors_base_url) -> None:
+ status = await _post(
+ connectors_base_url,
+ {"Authorization": "Bearer not-the-token", "X-AiSOC-Tenant-ID": TENANT},
+ )
+ assert status in (401, 403), f"a forged token was served: HTTP {status}"
+
+
+# The "every caller uses it" half is deliberately **not** re-implemented here.
+#
+# `scripts/check_service_token_wiring.py` already walks every service-to-service
+# request in the tree and resolves each URL through its helper, its settings
+# attribute and its intermediate variable. A second AST walk in this file was
+# written first and was worse at it: it matched any call in a module whose text
+# mentioned "connectors", so it named five call sites that reach the agents
+# service and a vendor instead. Two walkers over one tree that disagree about
+# what they are looking at is a defect this repository has already shipped.
+#
+# So the gate owns the property, and this file owns the question the gate
+# cannot answer: whether the headers the API builds actually satisfy the
+# dependency on the far side.
diff --git a/tests/isolation/test_closure_grant_live.py b/tests/isolation/test_closure_grant_live.py
new file mode 100644
index 000000000..aba5d1f72
--- /dev/null
+++ b/tests/isolation/test_closure_grant_live.py
@@ -0,0 +1,148 @@
+"""An earned auto-close grant is actually readable by the closure policy.
+
+Fix pass item 1.4. See `plans/aisoc_fix_pass_plan.plan.md`.
+
+What was wrong
+--------------
+`_has_auto_close_grant` in `services/agents/app/closure/policy.py` queried a
+table called `autonomy_grants`. There is no such table. Migration 067 creates
+`aisoc_autonomy_grants`, and the query additionally filtered on `revoked_at`
+and `expires_at`, neither of which that table has; its lifecycle is a `state`
+column with `shadow`, `granted` and `demoted`, plus `demoted_at`.
+
+Three names wrong in one statement, wrapped in `except Exception` that logged
+at warning and returned `False`.
+
+`require_grant` defaults to **true** in migration 078, so the effect was that a
+tenant which enabled a closure policy could never auto-close anything. The
+feature was off for exactly the tenants who turned it on.
+
+Why the offline suite did not catch it
+--------------------------------------
+Its fake connection answers any query. A double that returns a row for
+`SELECT 1 FROM a_table_that_does_not_exist` cannot distinguish a working query
+from a broken one, and this is the second time that shape has shipped here.
+
+What this file asserts
+----------------------
+Real Postgres with every migration applied. A tenant holding a `granted` row
+auto-closes; a tenant holding none does not; and the three states are told
+apart. Against the pre-fix tree every one of these fails with
+`relation "autonomy_grants" does not exist`, swallowed into `False`.
+"""
+
+from __future__ import annotations
+
+import os
+import uuid
+
+import pytest
+import pytest_asyncio
+
+pytestmark = [
+ pytest.mark.asyncio(loop_scope="module"),
+ pytest.mark.skipif(
+ not os.environ.get("ISOLATION_CLOSURE_DSN", "").strip(),
+ reason="ISOLATION_CLOSURE_DSN is not set; this suite needs a live Postgres",
+ ),
+]
+
+
+def _dsn() -> str:
+ value = os.environ.get("ISOLATION_CLOSURE_DSN", "").strip()
+ if not value:
+ pytest.skip("ISOLATION_CLOSURE_DSN is not set")
+ return value
+
+
+@pytest_asyncio.fixture(scope="module", loop_scope="module")
+async def pool():
+ """A real asyncpg pool, which is what the production code is handed."""
+ asyncpg = pytest.importorskip("asyncpg")
+ dsn = _dsn().replace("postgresql+asyncpg://", "postgresql://")
+ created = await asyncpg.create_pool(dsn, min_size=1, max_size=4)
+ yield created
+ await created.close()
+
+
+async def _tenant(pool) -> str:
+ tenant_id = str(uuid.uuid4())
+ async with pool.acquire() as conn:
+ await conn.execute(
+ "INSERT INTO tenants (id, name, slug) VALUES ($1::uuid, $2, $3)",
+ tenant_id,
+ "closure-grant",
+ f"closure-{tenant_id[:8]}",
+ )
+ return tenant_id
+
+
+async def _grant(pool, tenant_id: str, *, alert_class: str, state: str) -> None:
+ async with pool.acquire() as conn:
+ await conn.execute(
+ """
+ INSERT INTO aisoc_autonomy_grants
+ (tenant_id, scope_kind, scope_key, capability, state, source, evidence)
+ VALUES ($1::uuid, 'alert_class', $2, 'auto_close', $3, 'earned', '{}'::jsonb)
+ """,
+ tenant_id,
+ alert_class,
+ state,
+ )
+
+
+class TestAnEarnedGrantIsHonoured:
+ async def test_a_granted_row_lets_the_class_auto_close(self, pool) -> None:
+ """The whole point of the capability, and it had never worked."""
+ from app.closure.policy import _has_auto_close_grant
+
+ tenant_id = await _tenant(pool)
+ await _grant(pool, tenant_id, alert_class="benign_scanner", state="granted")
+
+ held = await _has_auto_close_grant(pool, tenant_id=tenant_id, alert_class="benign_scanner")
+
+ assert held is True, (
+ "an earned auto_close grant was not visible to the closure policy, so a tenant that enabled a policy could never auto-close"
+ )
+
+ async def test_a_tenant_with_no_grant_does_not_auto_close(self, pool) -> None:
+ """The negative control. Without it a reader that returned True for
+ everything would satisfy the assertion above."""
+ from app.closure.policy import _has_auto_close_grant
+
+ tenant_id = await _tenant(pool)
+
+ assert await _has_auto_close_grant(pool, tenant_id=tenant_id, alert_class="benign_scanner") is False
+
+ @pytest.mark.parametrize("state", ["shadow", "demoted"])
+ async def test_a_grant_that_is_not_granted_does_not_count(self, pool, state: str) -> None:
+ """`shadow` is still being evaluated and `demoted` was taken away.
+
+ The pre-fix query filtered on `revoked_at IS NULL`, a column this
+ table does not have, so it had no way to tell these apart even if the
+ table name had been right.
+ """
+ from app.closure.policy import _has_auto_close_grant
+
+ tenant_id = await _tenant(pool)
+ await _grant(pool, tenant_id, alert_class="benign_scanner", state=state)
+
+ assert await _has_auto_close_grant(pool, tenant_id=tenant_id, alert_class="benign_scanner") is False
+
+ async def test_a_grant_for_another_class_does_not_leak(self, pool) -> None:
+ """A grant is per alert class, not per tenant."""
+ from app.closure.policy import _has_auto_close_grant
+
+ tenant_id = await _tenant(pool)
+ await _grant(pool, tenant_id, alert_class="benign_scanner", state="granted")
+
+ assert await _has_auto_close_grant(pool, tenant_id=tenant_id, alert_class="credential_access") is False
+
+ async def test_a_grant_does_not_leak_across_tenants(self, pool) -> None:
+ from app.closure.policy import _has_auto_close_grant
+
+ holder = await _tenant(pool)
+ other = await _tenant(pool)
+ await _grant(pool, holder, alert_class="benign_scanner", state="granted")
+
+ assert await _has_auto_close_grant(pool, tenant_id=other, alert_class="benign_scanner") is False