Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

fix(ci): read ClickHouse's password from .env instead of the literal it used to be (#985)

Beenu Arora06c4fb33e382
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
index 7060b31e1..305486636 100644
--- a/.github/workflows/integration.yml
+++ b/.github/workflows/integration.yml
@@ -282,10 +282,28 @@ jobs:
- name: Assert events landed in the ClickHouse lake
run: |
set -euo pipefail
+ # Read the password out of `.env` rather than hard-coding it. The
+ # literal `clickhouse_dev_secret` was right for as long as
+ # `.env.example` shipped it; `ensure_env.py` now generates a real
+ # one, so the client authenticated against a server that had moved
+ # on. `2>/dev/null || echo 0` then turned "wrong password" into a
+ # row count of zero, and the step reported that the lake was never
+ # populated — naming a subsystem that was working.
+ CH_PASS="$(sed -n 's/^CLICKHOUSE_PASSWORD=//p' .env | tail -1)"
+ [ -n "${CH_PASS}" ] || { echo "::error::CLICKHOUSE_PASSWORD is not set in .env"; exit 1; }
+ echo "::add-mask::${CH_PASS}"
+
+ # Prove the client can authenticate before interpreting a count.
+ # A failed query and an empty table must not produce the same number.
+ docker compose exec -T clickhouse clickhouse-client \
+ --user aisoc --password "${CH_PASS}" --query 'SELECT 1' >/dev/null \
+ || { echo "::error::could not authenticate to ClickHouse as aisoc — the lake assertion below could not have run"; \
+ docker compose logs --tail=40 --no-color clickhouse || true; exit 1; }
+
ok=""
for i in $(seq 1 30); do
N="$(docker compose exec -T clickhouse clickhouse-client \
- --user aisoc --password clickhouse_dev_secret \
+ --user aisoc --password "${CH_PASS}" \
--query 'SELECT count() FROM aisoc.raw_events' 2>/dev/null || echo 0)"
echo "lake row count (attempt ${i}): ${N}"
if [ "${N:-0}" -gt 0 ]; then ok=1; break; fi
diff --git a/tests/test_ci_reads_generated_secrets.py b/tests/test_ci_reads_generated_secrets.py
new file mode 100644
index 000000000..26c9f1f28
--- /dev/null
+++ b/tests/test_ci_reads_generated_secrets.py
@@ -0,0 +1,106 @@
+"""A CI step must not hard-code a value first run now generates.
+
+What happened
+-------------
+`.env.example` shipped `CLICKHOUSE_PASSWORD=clickhouse_dev_secret` and the
+integration job's lake assertion connected with that literal. When the
+datastore passwords moved into `scripts/ensure_env.py::GENERATED` — so a
+production guard could not be satisfied by a published credential — the job
+kept using the literal while the server it was talking to had a freshly
+generated one.
+
+The failure named the wrong subsystem. The query was wrapped in
+`2>/dev/null || echo 0`, so "wrong password" and "empty table" produced the
+same number, and the step reported:
+
+ ::error::ClickHouse lake was never populated from the raw-events stream
+
+against a fusion worker that had written to it correctly.
+
+What is asserted
+----------------
+Any workflow that runs `scripts/ensure_env.py` is in a world where every
+`GENERATED` value is random. Such a workflow must not also contain the
+literal that variable used to ship as. The literals are read from git
+history's `.env.example`? No — they are read from the compose defaults,
+which is where they still live, so this stays true as those move.
+
+Unaffected workflows are not scanned: `upgrade-test.yml` provisions its own
+Postgres with its own password and never runs the generator, and a value it
+chooses for itself is not a stale copy of anything.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import pathlib
+import re
+import sys
+
+import pytest
+
+REPO = pathlib.Path(__file__).resolve().parents[1]
+WORKFLOWS = REPO / ".github" / "workflows"
+SCRIPTS = REPO / "scripts"
+
+sys.path.insert(0, str(SCRIPTS))
+
+
+def _load(name: str):
+ spec = importlib.util.spec_from_file_location(name, SCRIPTS / f"{name}.py")
+ assert spec and spec.loader
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+ensure_env = _load("ensure_env")
+check_published_secrets = _load("check_published_secrets")
+
+#: `${VAR:-literal}` in `docker-compose.yml`, which is where the value a
+#: variable used to ship as still lives.
+_DEFAULT_RE = re.compile(r"\$\{([A-Z0-9_]+):-([^}]+)\}")
+
+
+def _former_literals() -> dict[str, str]:
+ """`variable -> the compose default`, for the variables first run generates."""
+ text = (REPO / "docker-compose.yml").read_text(encoding="utf-8")
+ defaults = dict(_DEFAULT_RE.findall(text))
+ return {name: value for name, value in defaults.items() if name in ensure_env.GENERATED and value.strip()}
+
+
+def _workflows_running_the_generator() -> list[pathlib.Path]:
+ return [p for p in sorted(WORKFLOWS.glob("*.yml")) if "ensure_env.py" in p.read_text(encoding="utf-8")]
+
+
+def test_the_inputs_are_not_empty() -> None:
+ """Both halves. With no generated secret carrying a compose default, or
+ no workflow running the generator, the check below passes over nothing."""
+ assert _former_literals(), "no generated secret has a compose default — the harvest is wrong"
+ assert _workflows_running_the_generator(), "no workflow runs ensure_env.py — the scan would cover nothing"
+
+
[email protected]("workflow", _workflows_running_the_generator(), ids=lambda p: p.name)
+def test_no_step_uses_a_literal_the_generator_replaces(workflow: pathlib.Path) -> None:
+ text = workflow.read_text(encoding="utf-8")
+ # Comments explain the defect and are allowed to name the literal; a
+ # command that uses one is the thing being caught.
+ live = "\n".join(line for line in text.splitlines() if not line.lstrip().startswith("#"))
+
+ found = [f" {name}'s former default {literal!r}" for name, literal in sorted(_former_literals().items()) if literal in live]
+ assert not found, (
+ f"{workflow.name} runs scripts/ensure_env.py, so these are random at run time, "
+ "and it also contains the literal they used to ship as:\n"
+ + "\n".join(found)
+ + "\n\nRead the value out of `.env` instead — `sed -n 's/^NAME=//p' .env | tail -1`."
+ )
+
+
+def test_the_checker_agrees_these_are_published_values() -> None:
+ """Cross-check against the other direction: every literal this test
+ forbids in a generator-running workflow is one `check_published_secrets`
+ refuses in a production environment. If the two lists diverged, one of
+ them would be describing a value that no longer exists."""
+ published = check_published_secrets.published_literals(REPO)
+ unknown = sorted(v for v in _former_literals().values() if v not in published)
+ assert not unknown, f"these compose defaults are not harvested as published values: {unknown}"