Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

8.4 KB
Raw Blame
#!/usr/bin/env python3
"""
Wet-eval preflight gate (T5.5).

This script is the first step of `.github/workflows/wet-eval.yml`. It exists
so the weekly wet-eval workflow can be a no-op on:

  * forks where ``WET_EVAL_OPENAI_KEY`` was never configured;
  * a freshly-cloned dev shell where someone runs the workflow manually;
  * any environment where the bench-bot token / API key haven't been
    rotated in yet.

Without this preflight, the workflow would shell into ``run_evals.py --wet``
which would in turn try to import the live agent stack and crash with a
stack trace instead of a clear "missing secret, skipping" message.

It is intentionally **stdlib-only** so it runs on a bare Python install
without ``pip install`` having been done first.

Behaviour
---------
* Inspects the environment for ``WET_EVAL_OPENAI_KEY`` (the key used to
  call the LLM provider) and ``AISOC_BENCH_BOT_TOKEN`` (the fine-grained
  GH PAT that opens the weekly PR).
* Exit code is **always 0** so the workflow can branch on the JSON
  status file rather than a non-zero exit. We never want a missing
  secret to look like a CI failure on a fork.
* Writes a small JSON status file (path passed via ``--status-out``)
  the workflow consumes:
      {
        "should_run": true | false,
        "missing_secrets": ["WET_EVAL_OPENAI_KEY", ...],
        "reason": "...",
        "checked_at": "2026-05-14T07:00:00Z"
      }
* Prints a clear human-readable line to stdout so workflow logs make
  sense without opening the JSON.

Usage
-----
::

    # CI dry-run (workflow uses this):
    python scripts/wet_eval_check.py --dry-run --status-out /tmp/wet-eval-check.json

    # Real preflight:
    python scripts/wet_eval_check.py --status-out /tmp/wet-eval-check.json
"""

from __future__ import annotations

import argparse
import json
import os
import sys
import tempfile
from datetime import UTC, datetime
from pathlib import Path

# `scripts/` is on sys.path when this file is run as a program, but not when a
# test loads it by path with importlib. gate_toolkit sits beside it either way.
sys.path.insert(0, str(Path(__file__).resolve().parent))

from gate_toolkit import SELF_TEST_FLAG

# Secrets the workflow needs. Names match what's documented in
# ``apps/docs/docs/operations/secrets.md`` so docs and code stay in step.
_REQUIRED_SECRETS_FOR_LIVE_RUN = (
    "WET_EVAL_OPENAI_KEY",  # LLM provider key used by the live agent.
    "AISOC_BENCH_BOT_TOKEN",  # Fine-grained PAT used to open the weekly PR.
)


def _missing_secrets() -> list[str]:
    """Return the names of secrets that aren't set in the environment."""
    return [name for name in _REQUIRED_SECRETS_FOR_LIVE_RUN if not os.environ.get(name)]


def main(argv: list[str] | None = None) -> int:
    parser = argparse.ArgumentParser(
        description=(
            "Preflight gate for the weekly wet-eval CI job. Exits 0 always; "
            "the workflow consumes the JSON status file to decide whether "
            "to dispatch the live run."
        ),
    )
    parser.add_argument(
        "--dry-run",
        action="store_true",
        help=(
            "Treat the run as a CI smoke test even when secrets are present. "
            "Used by the test suite and the workflow's PR-time validation "
            "step so we never accidentally call the live LLM provider on a "
            "non-cron trigger."
        ),
    )
    parser.add_argument(
        "--status-out",
        type=Path,
        default=None,
        help=("Write a JSON status file to this path so the workflow can branch on ``should_run``. If omitted, only stdout is used."),
    )
    args = parser.parse_args(argv)

    missing = _missing_secrets()
    has_all = not missing
    is_live = has_all and not args.dry_run

    if args.dry_run:
        reason = "dry-run mode: status reported as no-op. The workflow will not dispatch the live wet eval."
    elif missing:
        reason = (
            "Missing required secret(s): " + ", ".join(missing) + ". This is expected on forks and first-run CI; configure them "
            "in the repo settings to enable the weekly wet eval. See "
            "`apps/docs/docs/operations/secrets.md`."
        )
    else:
        reason = "All required secrets are present. Proceeding to the live wet-eval run."

    status = {
        "should_run": bool(is_live),
        "dry_run": bool(args.dry_run),
        "missing_secrets": missing,
        "reason": reason,
        # Each secret keyed by name so the workflow can render a checklist
        # without revealing values. Booleans only — never the secret itself.
        "secrets_present": {name: bool(os.environ.get(name)) for name in _REQUIRED_SECRETS_FOR_LIVE_RUN},
        "checked_at": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"),
    }

    if args.status_out is not None:
        args.status_out.parent.mkdir(parents=True, exist_ok=True)
        args.status_out.write_text(json.dumps(status, indent=2))

    if status["should_run"]:
        print("[wet-eval-check] OK — secrets present, live wet eval will run.")
    elif args.dry_run:
        print("[wet-eval-check] dry-run — no live API calls will be attempted.")
    else:
        # NOTE: We intentionally do **not** echo the raw ``reason`` string here,
        # nor the names of missing secrets, even though the names themselves
        # are not secret values. CodeQL (rule ``py/clear-text-logging-sensitive-data``)
        # treats any variable whose name contains ``secret``/``token``/``key`` as
        # tainted; piping ``reason`` (which is built from secret env-var names)
        # into ``print`` is flagged as clear-text logging of sensitive data.
        # The JSON status file already lists ``missing_secrets`` for the
        # workflow to consume, so the human log only needs a count.
        missing_count = len(status["missing_secrets"])
        print(
            "[wet-eval-check] SKIP — "
            f"{missing_count} required secret(s) not configured. "
            "Workflow will exit cleanly. See status JSON for the checklist."
        )

    # Always exit 0 — missing secrets is *expected* on forks. The workflow
    # branches on ``should_run`` from the JSON status, not on the exit code.
    return 0


def _self_test() -> int:
    """Prove the verdict this preflight actually renders.

    The shared empty-tree self-test does not apply here: this script reads two
    environment variables and no repository content, and exits 0 by design so
    a fork with no secrets does not look like a broken build. Its verdict is
    ``should_run`` in the status file, and that is what has to be checked —
    otherwise a preflight that answered "yes, dispatch" unconditionally would
    look identical from the outside, and the weekly workflow would proceed to
    call a provider with no key.
    """
    cases = [
        ("no secrets configured: the workflow must not dispatch", {}, False, False),
        ("both secrets present: the workflow dispatches", dict.fromkeys(_REQUIRED_SECRETS_FOR_LIVE_RUN, "x"), False, True),
        ("one secret missing: still no dispatch", {_REQUIRED_SECRETS_FOR_LIVE_RUN[0]: "x"}, False, False),
        ("dry-run overrides present secrets", dict.fromkeys(_REQUIRED_SECRETS_FOR_LIVE_RUN, "x"), True, False),
    ]
    ok = True
    saved = {name: os.environ.get(name) for name in _REQUIRED_SECRETS_FOR_LIVE_RUN}
    with tempfile.TemporaryDirectory(prefix="wet_eval_check_") as tmp:
        out = Path(tmp) / "status.json"
        for description, environment, dry_run, expected in cases:
            for name in _REQUIRED_SECRETS_FOR_LIVE_RUN:
                os.environ.pop(name, None)
            os.environ.update(environment)
            argv = ["--status-out", str(out)] + (["--dry-run"] if dry_run else [])
            status = main(argv)
            should_run = json.loads(out.read_text())["should_run"]
            passed = status == 0 and should_run is expected
            ok &= passed
            print(f"  {'PASS' if passed else 'FAIL'}  {description}")
            print(f"        exit {status}, should_run={should_run} (want {expected})")
    for name, value in saved.items():
        if value is None:
            os.environ.pop(name, None)
        else:
            os.environ[name] = value

    print()
    if not ok:
        print("wet_eval_check.py: self-test FAILED")
        return 1
    print("wet_eval_check.py: self-test OK")
    return 0


if __name__ == "__main__":
    if SELF_TEST_FLAG in sys.argv[1:]:
        sys.exit(_self_test())
    sys.exit(main())