Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

10.3 KB
Raw Blame
"""Tests for the P0.3 hardening: insecure-default warnings + /metrics auth.

The audit finding was that the API silently shipped:

* a placeholder ``SECRET_KEY``
* an unauthenticated ``/metrics`` endpoint (Prometheus text exposed to
  anyone who could reach the pod)
* ``PLUGIN_TRUST_MODE=disabled`` with no operator-visible warning

These tests pin the corrective behavior so future refactors cannot
regress it without a failing test.
"""

from __future__ import annotations

import pytest
from app.core.config import (
    INSECURE_SECRET_KEY_DEFAULTS,
    InsecureProductionDefaultsError,
    Settings,
    enforce_secure_defaults,
    warn_if_insecure_defaults,
)
from fastapi import FastAPI
from fastapi.testclient import TestClient


def _make_settings(**overrides) -> Settings:
    """Construct a Settings instance without reading the host .env file."""
    base: dict = {
        "ENVIRONMENT": "production",
        "SECRET_KEY": "a" * 64,
        "METRICS_TOKEN": "long-random-token",
        "PLUGIN_TRUST_MODE": "strict",
        # Pre-generated Fernet key. Real deployments mount this via Fly secret.
        "AISOC_CREDENTIAL_KEY": "Z7t9fM3pZ8U_bN5oVxQYr1w2kLsHd4aJiTvE6cNxYpA=",
        # SAML/OIDC session signing secret. Empty/placeholder values now warn
        # in prod (issue #130, finding #10), so the "clean prod" baseline
        # has to set a real value.
        "JWT_SECRET": "b" * 64,
        # Bearer token the agents service uses to push events into realtime.
        # Empty in prod means the channel is unauthenticated, which warns.
        "REALTIME_INTERNAL_TOKEN": "c" * 64,
        # Shared HS256 secret the API uses to mint WS/SSE tickets and the
        # realtime service uses to verify them (issue #239). Empty/placeholder
        # values fail closed (503) in prod, so the "clean prod" baseline must
        # set a real value.
        "AISOC_REALTIME_JWT_SECRET": "d" * 64,
        # An explicit sslmode, because an unset one means libpq's `prefer`:
        # TLS when the server offers it, cleartext when it does not, and no
        # way to tell which happened. A clean production baseline has to make
        # the choice rather than inherit a mode that tolerates a downgrade.
        "DATABASE_URL": "postgresql+asyncpg://aisoc:pw@db:5432/aisoc?sslmode=require",
    }
    base.update(overrides)
    # ``_env_file=None`` skips .env discovery so test runs are deterministic
    # regardless of the developer's local config.
    return Settings(_env_file=None, **base)


# ─── enforce_secure_defaults (Phase 2 hard-fail) ───────────────────────────


def test_enforce_raises_in_prod_with_insecure_default():
    bad = next(iter(INSECURE_SECRET_KEY_DEFAULTS))
    s = _make_settings(ENVIRONMENT="production", SECRET_KEY=bad)
    with pytest.raises(InsecureProductionDefaultsError):
        enforce_secure_defaults(s)


def test_enforce_clean_prod_does_not_raise():
    s = _make_settings()  # clean production baseline
    assert enforce_secure_defaults(s) == []


def test_enforce_non_prod_insecure_does_not_raise():
    bad = next(iter(INSECURE_SECRET_KEY_DEFAULTS))
    s = _make_settings(ENVIRONMENT="staging", SECRET_KEY=bad)
    # Non-production: warns (non-empty), but never hard-fails the boot.
    msgs = enforce_secure_defaults(s)
    assert msgs  # the warning still fired


# ─── warn_if_insecure_defaults ─────────────────────────────────────────────


def test_secret_key_placeholder_in_prod_warns():
    bad = next(iter(INSECURE_SECRET_KEY_DEFAULTS))
    s = _make_settings(SECRET_KEY=bad)
    msgs = warn_if_insecure_defaults(s)
    assert any("SECRET_KEY" in m for m in msgs)


def test_secret_key_placeholder_in_dev_still_warns():
    bad = next(iter(INSECURE_SECRET_KEY_DEFAULTS))
    s = _make_settings(ENVIRONMENT="development", SECRET_KEY=bad)
    msgs = warn_if_insecure_defaults(s)
    # SECRET_KEY check is environment-independent: we always want to nag.
    assert any("SECRET_KEY" in m for m in msgs)


def test_empty_metrics_token_in_prod_warns():
    s = _make_settings(METRICS_TOKEN="")
    msgs = warn_if_insecure_defaults(s)
    assert any("METRICS_TOKEN" in m for m in msgs)


def test_empty_metrics_token_in_dev_does_not_warn():
    s = _make_settings(ENVIRONMENT="development", METRICS_TOKEN="")
    msgs = warn_if_insecure_defaults(s)
    assert not any("METRICS_TOKEN" in m for m in msgs)


def test_plugin_trust_disabled_in_prod_warns():
    s = _make_settings(PLUGIN_TRUST_MODE="disabled")
    msgs = warn_if_insecure_defaults(s)
    assert any("PLUGIN_TRUST_MODE" in m for m in msgs)


def test_plugin_trust_disabled_in_dev_does_not_warn():
    s = _make_settings(ENVIRONMENT="development", PLUGIN_TRUST_MODE="disabled")
    msgs = warn_if_insecure_defaults(s)
    assert not any("PLUGIN_TRUST_MODE" in m for m in msgs)


def test_clean_prod_settings_emit_no_warnings():
    s = _make_settings()
    msgs = warn_if_insecure_defaults(s)
    assert msgs == []


def test_empty_credential_key_in_prod_warns():
    """Operators must set AISOC_CREDENTIAL_KEY before connector secrets touch Postgres."""
    s = _make_settings(AISOC_CREDENTIAL_KEY="")
    msgs = warn_if_insecure_defaults(s)
    assert any("AISOC_CREDENTIAL_KEY" in m for m in msgs)


def test_empty_credential_key_in_dev_does_not_warn():
    """Dev auto-generates an ephemeral key, so silence here is intentional."""
    s = _make_settings(ENVIRONMENT="development", AISOC_CREDENTIAL_KEY="")
    msgs = warn_if_insecure_defaults(s)
    assert not any("AISOC_CREDENTIAL_KEY" in m for m in msgs)


def test_empty_jwt_secret_in_prod_warns():
    """SAML/OIDC session tokens must be signed with a real secret in prod."""
    s = _make_settings(JWT_SECRET="")
    msgs = warn_if_insecure_defaults(s)
    assert any("JWT_SECRET" in m for m in msgs)


def test_placeholder_jwt_secret_in_prod_warns():
    """The "changeme" sentinel is treated as empty — same warning fires."""
    s = _make_settings(JWT_SECRET="changeme-insecure-default")
    msgs = warn_if_insecure_defaults(s)
    assert any("JWT_SECRET" in m for m in msgs)


def test_empty_jwt_secret_in_dev_does_not_warn():
    """Dev environments are noisy; we don't nag there."""
    s = _make_settings(ENVIRONMENT="development", JWT_SECRET="")
    msgs = warn_if_insecure_defaults(s)
    assert not any("JWT_SECRET" in m for m in msgs)


def test_empty_realtime_token_in_prod_warns():
    """Agents → realtime channel is unauthenticated when the token is unset."""
    s = _make_settings(REALTIME_INTERNAL_TOKEN="")
    msgs = warn_if_insecure_defaults(s)
    assert any("REALTIME_INTERNAL_TOKEN" in m for m in msgs)


def test_empty_realtime_token_in_dev_does_not_warn():
    s = _make_settings(ENVIRONMENT="development", REALTIME_INTERNAL_TOKEN="")
    msgs = warn_if_insecure_defaults(s)
    assert not any("REALTIME_INTERNAL_TOKEN" in m for m in msgs)


def test_empty_realtime_jwt_secret_in_prod_warns():
    """WS/SSE ticket signing must use a real shared secret in prod (#239)."""
    s = _make_settings(AISOC_REALTIME_JWT_SECRET="")
    msgs = warn_if_insecure_defaults(s)
    assert any("AISOC_REALTIME_JWT_SECRET" in m for m in msgs)


def test_empty_realtime_jwt_secret_in_dev_does_not_warn():
    """Dev falls back to a hardcoded ticket secret, so silence is intentional."""
    s = _make_settings(ENVIRONMENT="development", AISOC_REALTIME_JWT_SECRET="")
    msgs = warn_if_insecure_defaults(s)
    assert not any("AISOC_REALTIME_JWT_SECRET" in m for m in msgs)


# ─── /metrics auth gate ────────────────────────────────────────────────────


@pytest.fixture
def metrics_app(monkeypatch):
    """A FastAPI app with just the /metrics endpoint, isolated from real settings."""
    import app.main as main_module

    # Bypass the lifespan (DB, Neo4j, plugin discovery) by mounting the
    # endpoint directly on a fresh app — we are only testing the auth gate.
    test_app = FastAPI()
    test_app.add_api_route("/metrics", main_module.metrics, methods=["GET"])
    return test_app, main_module


def test_metrics_requires_token_when_configured(metrics_app, monkeypatch):
    test_app, main_module = metrics_app
    monkeypatch.setattr(main_module.settings, "METRICS_TOKEN", "s3cret")
    monkeypatch.setattr(main_module.settings, "ENVIRONMENT", "production")

    client = TestClient(test_app)
    assert client.get("/metrics").status_code == 401
    assert client.get("/metrics", headers={"Authorization": "Bearer wrong"}).status_code == 401
    ok = client.get("/metrics", headers={"Authorization": "Bearer s3cret"})
    assert ok.status_code == 200
    assert ok.headers["content-type"].startswith("text/plain")


def test_metrics_open_in_dev_when_token_unset(metrics_app, monkeypatch):
    test_app, main_module = metrics_app
    monkeypatch.setattr(main_module.settings, "METRICS_TOKEN", "")
    monkeypatch.setattr(main_module.settings, "ENVIRONMENT", "development")

    client = TestClient(test_app)
    assert client.get("/metrics").status_code == 200


def test_metrics_refused_in_prod_when_token_unset(metrics_app, monkeypatch):
    test_app, main_module = metrics_app
    monkeypatch.setattr(main_module.settings, "METRICS_TOKEN", "")
    monkeypatch.setattr(main_module.settings, "ENVIRONMENT", "production")

    client = TestClient(test_app)
    resp = client.get("/metrics")
    assert resp.status_code == 401
    assert "METRICS_TOKEN" in resp.text


def test_metrics_token_compare_is_constant_time(metrics_app, monkeypatch):
    """Sanity check: we wired up hmac.compare_digest, not ==.

    We can't observe wall-clock timing reliably in CI, so we just assert
    that the exact-match path returns 200 and a near-match returns 401.
    The implementation comment in main.py documents the constant-time
    intent; this test fails loudly if someone replaces it with ``==``.
    """
    test_app, main_module = metrics_app
    monkeypatch.setattr(main_module.settings, "METRICS_TOKEN", "abc123")
    monkeypatch.setattr(main_module.settings, "ENVIRONMENT", "production")

    client = TestClient(test_app)
    assert client.get("/metrics", headers={"Authorization": "Bearer abc123"}).status_code == 200
    assert client.get("/metrics", headers={"Authorization": "Bearer abc124"}).status_code == 401