Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

5.3 KB
Raw Blame
"""The security release gate must execute every real-proxy login scenario."""

# allow: no-sut-import — executes the security workflow's report and exit-status checks

import os
from pathlib import Path
import subprocess
import sys
from xml.etree import ElementTree

import pytest
import yaml


WORKFLOWS = Path(__file__).resolve().parents[2] / ".github" / "workflows"


@pytest.fixture(scope="module")
def security_workflow():
    return yaml.safe_load((WORKFLOWS / "security-tests.yml").read_text())


@pytest.fixture(scope="module")
def security_job(security_workflow):
    return security_workflow["jobs"]["proxy-login-tests"]


def _step(job, step_id):
    return next(step for step in job["steps"] if step.get("id") == step_id)


def test_security_gate_requires_real_proxy_tests(security_job):
    release = yaml.safe_load((WORKFLOWS / "release-gate.yml").read_text())
    jobs = release["jobs"]
    assert (
        jobs["security-tests"]["uses"]
        == "$/.github/workflows/security-tests.yml"
    )
    assert "security-tests" in jobs["release-gate-summary"]["needs"]
    assert not security_job.get("continue-on-error", False)
    run = _step(security_job, "proxy-login-tests")
    assert "if" not in run
    assert not run.get("continue-on-error", False)
    assert run["shell"] == "bash"
    assert run["env"]["LDR_TESTING_WITH_MOCKS"] == "false"
    assert "tests/integration/test_login_proxy_rate_limit.py" in run["run"]
    assert "-n 0" in run["run"]
    validator = _step(security_job, "validate-proxy-login")
    assert validator["if"] == "always()"
    assert not validator.get("continue-on-error", False)


def test_proxy_tests_have_their_own_job_and_timeout(security_workflow):
    # The security-tests job already uses ~22 of its 25 minutes on main, so
    # the proxy matrix must not share that job's timeout budget. Both jobs
    # sit in the reusable workflow release-gate.yml calls, so a failure or
    # timeout in either one fails the release gate's security-tests job.
    jobs = security_workflow["jobs"]
    suite = jobs["security-tests"]
    proxy = jobs["proxy-login-tests"]
    suite_ids = {step.get("id") for step in suite["steps"]}
    assert not suite_ids & {"proxy-login-tests", "validate-proxy-login"}
    assert "test_login_proxy_rate_limit" not in yaml.safe_dump(suite)
    assert isinstance(proxy["timeout-minutes"], int)
    assert 0 < proxy["timeout-minutes"] <= 30
    # Independent of the suite job: no `needs` (it would wait out the suite
    # and be skipped when the suite fails) and no job-level condition.
    assert "needs" not in proxy
    assert "if" not in proxy
    assert not proxy.get("continue-on-error", False)
    installs = " ".join(
        step.get("run", "")
        for step in proxy["steps"]
        if "apt-get" in step.get("run", "")
    )
    assert "nginx" in installs and "libsqlcipher-dev" in installs


def _report():
    root = ElementTree.Element("testsuites")
    suite = ElementTree.SubElement(root, "testsuite")
    for trust in ("trust-off", "trust-on"):
        for attack in ("control", "forged"):
            for mode in ("overwrite", "append", "separate-lines", "real-ip"):
                ElementTree.SubElement(
                    suite,
                    "testcase",
                    name=f"test_password_spray_hits_ip_limit[{trust}-{attack}-{mode}]",
                )
        ElementTree.SubElement(
            suite,
            "testcase",
            name=f"test_account_lockout_remains_separate[{trust}]",
        )
    return root, suite


def _run(script, tmp_path, path):
    return subprocess.run(
        ["bash", "-e", "-o", "pipefail", "-c", script],
        cwd=tmp_path,
        env={**os.environ, "PATH": f"{path}{os.pathsep}{os.environ['PATH']}"},
        capture_output=True,
        text=True,
        timeout=10,
        check=False,
    )


@pytest.mark.parametrize(
    "fault",
    [
        "none",
        "missing",
        "empty",
        "malformed",
        "partial",
        "duplicate",
        "skipped",
        "failure",
        "error",
    ],
)
def test_report_checker_rejects_incomplete_or_failed_runs(
    security_job, tmp_path, fault
):
    root, suite = _report()
    report = tmp_path / "test-results" / "proxy-login.xml"
    report.parent.mkdir()
    if fault == "empty":
        suite.clear()
    elif fault == "partial":
        suite.remove(suite[0])
    elif fault == "duplicate":
        suite.append(suite[0])
    elif fault in {"skipped", "failure", "error"}:
        ElementTree.SubElement(suite[0], fault)
    if fault == "malformed":
        report.write_text("<testsuites>")
    elif fault != "missing":
        ElementTree.ElementTree(root).write(report)
    result = _run(
        _step(security_job, "validate-proxy-login")["run"],
        tmp_path,
        Path(sys.executable).parent,
    )
    assert (result.returncode == 0) is (fault == "none"), (
        result.stdout + result.stderr
    )


@pytest.mark.parametrize(
    "status", [0, 1, 5], ids=["pass", "failure", "no-tests"]
)
def test_logging_pipeline_preserves_pytest_exit_status(
    security_job, tmp_path, status
):
    binary = tmp_path / "pdm"
    binary.write_text(f"#!/bin/sh\nprintf 'pytest output\\n'\nexit {status}\n")
    binary.chmod(0o755)
    result = _run(
        _step(security_job, "proxy-login-tests")["run"], tmp_path, tmp_path
    )
    assert result.returncode == status, result.stdout + result.stderr