Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

17.4 KB
Raw Blame
"""Exact upload receipts, complete native reports and release-only wiring."""

# allow: no-sut-import — standalone CI helpers do not load application code.

import importlib.util
import json
from pathlib import Path
from types import SimpleNamespace
from urllib.parse import parse_qs, urlsplit

import pytest
import yaml


ROOT = Path(__file__).resolve().parents[2]


def load(name):
    spec = importlib.util.spec_from_file_location(
        name, ROOT / ".github/scripts" / f"{name}.py"
    )
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    return module


native = load("check_release_report_age")
reports = load("release_native_reports")
UPLOAD = "11111111-1111-4111-8111-111111111111"
SHA = "a" * 40


def receipt(**changes):
    return {
        "sarif_id": UPLOAD,
        "commit_sha": SHA,
        "ref": "refs/heads/main",
        "analysis_key": ".github/workflows/release.yml:sample",
        "category": "sample-release",
        "tool": {"name": "sample-scanner"},
        "error": "",
        "results_count": 0,
        **changes,
    }


def verify(
    fetch,
    *,
    upload=UPLOAD,
    category="sample-release",
    tool="sample-scanner",
    sleep=lambda _: None,
):
    return native.verify_upload(
        upload,
        category,
        reports.sarif(tool, []),
        "example/project",
        SHA,
        "refs/heads/main",
        "sample",
        "example/project/.github/workflows/release.yml@refs/heads/main",
        fetch=fetch,
        sleep=sleep,
    )


def test_receipt_is_queried_by_exact_uuid_and_scope():
    calls = []

    def fetch(endpoint):
        calls.append(endpoint)
        if endpoint.endswith("/" + UPLOAD):
            return {"processing_status": "complete"}
        assert parse_qs(urlsplit(endpoint).query) == {
            "sarif_id": [UPLOAD],
            "per_page": ["100"],
        }
        return [receipt()]

    verify(fetch)
    assert len(calls) == 2


@pytest.mark.parametrize(
    "change",
    [
        {"sarif_id": "22222222-2222-4222-8222-222222222222"},
        {"commit_sha": "b" * 40},
        {"ref": "refs/heads/other"},
        {"analysis_key": "other:sample"},
        {"category": "other"},
        {"tool": {"name": "other"}},
        {"error": "Unknown Error"},
        {"error": None},
        {"results_count": True},
        {"results_count": -1},
    ],
)
def test_clean_looking_report_cannot_use_another_or_failed_analysis(change):
    def fetch(endpoint):
        return (
            {"processing_status": "complete"}
            if "/sarifs/" in endpoint
            else [receipt(**change)]
        )

    with pytest.raises(ValueError):
        verify(fetch)


@pytest.mark.parametrize(
    "status",
    [
        None,
        {},
        {"processing_status": "failed"},
        {"processing_status": "complete", "errors": ["synthetic"]},
    ],
)
def test_missing_or_failed_processing_is_blocking_even_for_empty_reports(
    status,
):
    with pytest.raises(ValueError):
        verify(lambda _: status)


def test_pending_processing_is_bounded():
    waits = []
    with pytest.raises(ValueError, match="complete"):
        verify(lambda _: {"processing_status": "pending"}, sleep=waits.append)
    assert waits == [5] * 18


def test_nonempty_report_requires_processed_findings():
    with pytest.raises(ValueError, match="absent"):
        native.verify_upload(
            UPLOAD,
            "sample-release",
            reports.sarif(
                "sample-scanner", [("SAMPLE_RULE", "high", "sample.py")]
            ),
            "example/project",
            SHA,
            "refs/heads/main",
            "sample",
            "example/project/.github/workflows/release.yml@refs/heads/main",
            fetch=lambda endpoint: (
                {"processing_status": "complete"}
                if "/sarifs/" in endpoint
                else [receipt()]
            ),
        )


def test_only_osv_can_use_the_existing_implicit_category():
    with pytest.raises(ValueError, match="OSV"):
        verify(
            lambda _: pytest.fail(
                "Invalid category must fail before API access"
            ),
            category="auto",
        )

    row = receipt(
        category=".github/workflows/release.yml:osv-scan",
        tool={"name": "osv-scanner"},
    )
    verify(
        lambda endpoint: (
            {"processing_status": "complete"}
            if "/sarifs/" in endpoint
            else [row]
        ),
        category="auto",
        tool="osv-scanner",
    )


def npm_data():
    return {
        "auditReportVersion": 2,
        "vulnerabilities": {},
        "metadata": {
            "vulnerabilities": {
                "info": 0,
                "low": 0,
                "moderate": 0,
                "high": 0,
                "critical": 0,
                "total": 0,
            }
        },
    }


def test_npm_runner_requires_a_target_and_its_lockfile(tmp_path):
    with pytest.raises(ValueError, match="no target"):
        reports.audit_npm(tmp_path)
    (tmp_path / "package.json").write_text("{}", encoding="utf-8")
    with pytest.raises(ValueError, match="lockfile"):
        reports.audit_npm(tmp_path)


@pytest.mark.parametrize("status,allowed", [(0, True), (1, True), (2, False)])
def test_npm_runner_preserves_operational_failures(tmp_path, status, allowed):
    for file in ("package.json", "package-lock.json"):
        (tmp_path / file).write_text("{}", encoding="utf-8")

    def run(argv, **kwargs):
        assert argv == ["npm", "audit", "--audit-level=moderate", "--json"]
        assert kwargs["cwd"] == tmp_path
        return SimpleNamespace(returncode=status, stdout=json.dumps(npm_data()))

    if allowed:
        assert reports.audit_npm(tmp_path, run)["runs"][0]["results"] == []
    else:
        with pytest.raises(ValueError, match="execute"):
            reports.audit_npm(tmp_path, run)


def test_npm_completion_counts_cannot_hide_partial_records():
    data = npm_data()
    data["metadata"]["vulnerabilities"]["high"] = 1
    with pytest.raises(ValueError, match="counts"):
        reports.npm_findings(data, "package-lock.json")
    data = npm_data()
    data["vulnerabilities"]["synthetic"] = {"severity": "medium", "via": []}
    with pytest.raises(ValueError):
        reports.npm_findings(data, "package-lock.json")


@pytest.mark.parametrize(
    "data",
    [
        [],
        {},
        {"summary": {"parsing_errors": False}, "results": {}},
        {"summary": {"parsing_errors": 0}, "results": []},
    ],
)
def test_checkov_empty_or_malformed_completion_is_fatal(data):
    with pytest.raises(ValueError):
        reports.validate_checkov(data)


def test_checkov_accepts_its_native_complete_empty_summary_without_hiding_errors():
    data = {
        "passed": 0,
        "failed": 0,
        "skipped": 0,
        "parsing_errors": 0,
        "resource_count": 0,
        "checkov_version": "3.2.499",
    }
    reports.validate_checkov(data)
    for field in ("passed", "failed", "parsing_errors", "resource_count"):
        malformed = {**data, field: 1}
        with pytest.raises(ValueError):
            reports.validate_checkov(malformed)


def workflow(name):
    return yaml.safe_load(
        (ROOT / ".github/workflows" / f"{name}.yml").read_text(encoding="utf-8")
    )


def test_zap_preserves_configured_and_default_advisories(tmp_path):
    path = tmp_path / "rules.tsv"
    path.write_text(
        "10001\tFAIL\tSynthetic fail\n10002\tWARN\tSynthetic warning\n10003\tIGNORE\tSynthetic ignored\n",
        encoding="utf-8",
    )
    payload = {
        "@version": "synthetic",
        "site": [
            {
                "alerts": [
                    {"pluginid": str(rule), "riskcode": "2"}
                    for rule in range(10001, 10005)
                ]
            }
        ],
    }
    report = reports.zap_report(payload, path)
    policy = native.policy_module()
    assert [
        policy.blocking_severity(finding)
        for finding in native.report_findings(report, tmp_path)
    ] == [True, False, False, False]
    rules = report["runs"][0]["tool"]["driver"]["rules"]
    assert [rule["properties"]["native-risk"] for rule in rules] == [
        "medium"
    ] * 4
    assert [rule["properties"]["configured-action"] for rule in rules] == [
        "FAIL",
        "WARN",
        "IGNORE",
        "WARN",
    ]


@pytest.mark.parametrize(
    "riskcode,severity", [("0", "medium"), ("1", "medium"), ("3", "high")]
)
def test_zap_configured_fail_rules_block_at_any_native_risk(
    tmp_path, riskcode, severity
):
    """Low/Informational FAIL rules (e.g. 10021, 10035, 10063) must not
    become permanently advisory notes; they enforce the age policy."""
    path = tmp_path / "rules.tsv"
    path.write_text("10021\tFAIL\tSynthetic header rule\n", encoding="utf-8")
    payload = {
        "@version": "synthetic",
        "site": [{"alerts": [{"pluginid": "10021", "riskcode": riskcode}]}],
    }
    report = reports.zap_report(payload, path)
    findings = native.report_findings(report, tmp_path)
    assert len(findings) == 1
    assert findings[0]["rule"]["security_severity_level"] == severity
    assert findings[0]["rule"]["severity"] == "warning"
    assert native.policy_module().blocking_severity(findings[0])
    rule = report["runs"][0]["tool"]["driver"]["rules"][0]
    assert (
        rule["properties"]["native-risk"]
        == (("info", "low", "medium", "high")[int(riskcode)])
    )


@pytest.mark.parametrize("action", ["FAIL", "WARN", "IGNORE"])
@pytest.mark.parametrize(
    "risks", [("1", "0"), ("0", "1"), ("3", "1"), ("1", "3")]
)
def test_zap_aggregates_variants_and_sites_without_lowering_native_risk(
    tmp_path, action, risks
):
    path = tmp_path / "rules.tsv"
    path.write_text(f"100000\t{action}\tSynthetic rule\n", encoding="utf-8")
    sites = [
        {
            "alerts": [
                {
                    "pluginid": "100000",
                    "alertRef": f"100000-{risk}",
                    "riskcode": risk,
                }
            ]
        }
        for risk in risks
    ]
    report = reports.zap_report({"@version": "synthetic", "site": sites}, path)
    assert report == reports.zap_report(
        {"@version": "synthetic", "site": list(reversed(sites))}, path
    )
    run = report["runs"][0]
    assert len(run["results"]) == len(run["tool"]["driver"]["rules"]) == 1
    rule = run["tool"]["driver"]["rules"][0]
    highest = "high" if "3" in risks else "low"
    assert rule["id"] == "ZAP-100000"
    assert rule["properties"]["native-risk"] == highest
    assert rule["properties"]["configured-action"] == action
    assert rule["properties"]["native-alert-refs"] == sorted(
        f"100000-{risk}" for risk in risks
    )
    assert highest in rule["shortDescription"]["text"]
    assert run["results"][0]["locations"][0]["physicalLocation"] == {
        "artifactLocation": {"uri": ".zap/rules.tsv"},
        "region": {"startLine": 1},
    }
    findings = native.report_findings(report, tmp_path)
    assert native.policy_module().blocking_severity(findings[0]) is (
        action == "FAIL"
    )
    expected_score = (
        ("8.0" if highest == "high" else "5.0") if action == "FAIL" else "0.0"
    )
    assert rule["properties"]["security-severity"] == expected_score


def test_zap_variant_metadata_does_not_copy_native_diagnostics(tmp_path):
    path = tmp_path / "rules.tsv"
    path.write_text("10055\tWARN\tSynthetic rule\n", encoding="utf-8")
    marker = "synthetic-private-response"
    finding = {
        "pluginid": "10055",
        "alertRef": "10055-1",
        "riskcode": "2",
        "alert": marker,
        "desc": marker,
        "instances": [{"uri": marker, "evidence": marker}],
    }
    report = reports.zap_report(
        {"@version": "synthetic", "site": [{"alerts": [finding, finding]}]},
        path,
    )
    assert marker not in json.dumps(report)
    rule = report["runs"][0]["tool"]["driver"]["rules"][0]
    assert rule["properties"]["native-alert-refs"] == ["10055-1"]
    assert len(report["runs"][0]["results"]) == 1


@pytest.mark.parametrize("reference", [None, 12, "99999-1", "10055-text"])
def test_zap_rejects_unstructured_variant_references(tmp_path, reference):
    path = tmp_path / "rules.tsv"
    path.write_text("10055\tWARN\tSynthetic rule\n", encoding="utf-8")
    with pytest.raises(ValueError, match="^Invalid ZAP alert reference$"):
        reports.zap_report(
            {
                "@version": "synthetic",
                "site": [
                    {
                        "alerts": [
                            {
                                "pluginid": "10055",
                                "alertRef": reference,
                                "riskcode": "2",
                            }
                        ]
                    }
                ],
            },
            path,
        )


@pytest.mark.parametrize(
    "configuration", ["10001\tOTHER\n", "10001\tFAIL\n10001\tIGNORE\n"]
)
def test_zap_invalid_rule_configuration_does_not_grant_a_pass(
    tmp_path, configuration
):
    path = tmp_path / "rules.tsv"
    path.write_text(configuration, encoding="utf-8")
    with pytest.raises(ValueError):
        reports.zap_report(
            {"@version": "synthetic", "site": [{"alerts": []}]}, path
        )


def test_release_native_jobs_and_installed_audits_precede_the_final_decision():
    gate = workflow("release-gate")["jobs"]
    native_jobs = {
        "osv-scan",
        "npm-audit",
        "retirejs-scan",
        "hadolint-scan",
        "checkov-scan",
        "owasp-zap-scan",
    }
    for name in native_jobs:
        assert gate[name]["with"]["release_policy"] is True
    alert_job = gate["check-code-scanning-alerts"]
    assert native_jobs | {"pip-install-check", "package-install-matrix"} <= set(
        alert_job["needs"]
    )
    summary = gate["release-gate-summary"]
    assert {
        "security-tests",
        "gitleaks-scan",
        "check-code-scanning-alerts",
    } <= set(summary["needs"])
    step = next(s for s in alert_job["steps"] if s.get("id") == "alert-check")
    assert len(json.loads(step["env"]["NATIVE_DEADLINES"])) == 7
    assert "--native-deadlines" in step["run"]
    for name in ("pip-install-check", "package-install-matrix"):
        audits = [
            s["run"]
            for s in gate[name]["steps"]
            if "audit_installed_dependencies.py" in s.get("run", "")
        ]
        assert audits and all(
            "--release-policy" in command for command in audits
        )
        assert any(
            "upload-sarif@" in s.get("uses", "") for s in gate[name]["steps"]
        )


def test_promotion_finding_decision_precedes_tag_publication():
    steps = workflow("docker-publish")["jobs"]["promote"]["steps"]
    position = {s.get("id"): i for i, s in enumerate(steps) if "id" in s}
    publish = next(
        i
        for i, s in enumerate(steps)
        if "docker buildx imagetools create" in s.get("run", "")
    )
    assert (
        position["native-upload"]
        < position["native-policy"]
        < position["native-deadline"]
        < publish
    )
    # Immediate critical and secret checks must also precede tag publication.
    immediate = [
        i
        for i, s in enumerate(steps)
        if "aquasecurity/trivy-action@" in s.get("uses", "")
        and str((s.get("with") or {}).get("exit-code")) == "1"
    ]
    secret = [
        i
        for i in immediate
        if (steps[i].get("with") or {}).get("scanners") == "secret"
    ]
    assert len(immediate) == 2 and len(secret) == 1
    assert max(immediate) < publish
    promotion = workflow("docker-publish")
    assert (
        promotion[True]["workflow_call"]["outputs"]["release_deadline"]["value"]
        == "${{ jobs.promote.outputs.release_deadline }}"
    )
    assert (
        promotion["jobs"]["promote"]["outputs"]["release_deadline"]
        == "${{ steps.native-deadline.outputs.release_deadline }}"
    )


@pytest.mark.parametrize(
    "name,job",
    [
        ("prerelease-docker", "security-scan"),
        ("docker-publish", "promote"),
        ("container-security", "trivy-scan"),
    ],
)
def test_image_secrets_block_immediately_instead_of_being_aged(name, job):
    """Trivy's default image scanners include `secret`; its HIGH rules
    (private-key, pypi-upload-token) must not inherit the 14-day window."""
    checks = [
        step
        for step in workflow(name)["jobs"][job]["steps"]
        if "aquasecurity/trivy-action@" in step.get("uses", "")
        and step.get("with", {}).get("scanners") == "secret"
    ]
    assert len(checks) == 1
    step = checks[0]
    assert "if" not in step and "continue-on-error" not in step
    assert str(step["with"]["exit-code"]) == "1"
    assert {"CRITICAL", "HIGH", "MEDIUM"} <= set(
        step["with"]["severity"].split(",")
    )


def test_release_policy_passthroughs_always_send_a_boolean():
    """Without workflow_call inputs (pull_request, merge_group, schedule,
    dispatch) `inputs.release_policy` is empty; GitHub rejects '' for a
    typed boolean input and the caller run reports no jobs at all."""
    checked = 0
    for path in sorted((ROOT / ".github/workflows").glob("*.yml")):
        for job in (workflow(path.stem).get("jobs") or {}).values():
            value = (job.get("with") or {}).get("release_policy")
            if value is None or not job.get("uses"):
                continue
            checked += 1
            assert value is True or value == (
                "${{ inputs.release_policy == true }}"
            ), f"{path.name}: release_policy passthrough {value!r}"
    assert checked >= 9