~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Exact upload receipts, complete native reports and release-only wiring."""
# allow: no-sut-import — standalone CI helpers do not load application code.
import importlib.util
import json
from pathlib import Path
from types import SimpleNamespace
from urllib.parse import parse_qs, urlsplit
import pytest
import yaml
ROOT = Path(__file__).resolve().parents[2]
def load(name):
spec = importlib.util.spec_from_file_location(
name, ROOT / ".github/scripts" / f"{name}.py"
)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
native = load("check_release_report_age")
reports = load("release_native_reports")
UPLOAD = "11111111-1111-4111-8111-111111111111"
SHA = "a" * 40
def receipt(**changes):
return {
"sarif_id": UPLOAD,
"commit_sha": SHA,
"ref": "refs/heads/main",
"analysis_key": ".github/workflows/release.yml:sample",
"category": "sample-release",
"tool": {"name": "sample-scanner"},
"error": "",
"results_count": 0,
**changes,
}
def verify(
fetch,
*,
upload=UPLOAD,
category="sample-release",
tool="sample-scanner",
sleep=lambda _: None,
):
return native.verify_upload(
upload,
category,
reports.sarif(tool, []),
"example/project",
SHA,
"refs/heads/main",
"sample",
"example/project/.github/workflows/release.yml@refs/heads/main",
fetch=fetch,
sleep=sleep,
)
def test_receipt_is_queried_by_exact_uuid_and_scope():
calls = []
def fetch(endpoint):
calls.append(endpoint)
if endpoint.endswith("/" + UPLOAD):
return {"processing_status": "complete"}
assert parse_qs(urlsplit(endpoint).query) == {
"sarif_id": [UPLOAD],
"per_page": ["100"],
}
return [receipt()]
verify(fetch)
assert len(calls) == 2
@pytest.mark.parametrize(
"change",
[
{"sarif_id": "22222222-2222-4222-8222-222222222222"},
{"commit_sha": "b" * 40},
{"ref": "refs/heads/other"},
{"analysis_key": "other:sample"},
{"category": "other"},
{"tool": {"name": "other"}},
{"error": "Unknown Error"},
{"error": None},
{"results_count": True},
{"results_count": -1},
],
)
def test_clean_looking_report_cannot_use_another_or_failed_analysis(change):
def fetch(endpoint):
return (
{"processing_status": "complete"}
if "/sarifs/" in endpoint
else [receipt(**change)]
)
with pytest.raises(ValueError):
verify(fetch)
@pytest.mark.parametrize(
"status",
[
None,
{},
{"processing_status": "failed"},
{"processing_status": "complete", "errors": ["synthetic"]},
],
)
def test_missing_or_failed_processing_is_blocking_even_for_empty_reports(
status,
):
with pytest.raises(ValueError):
verify(lambda _: status)
def test_pending_processing_is_bounded():
waits = []
with pytest.raises(ValueError, match="complete"):
verify(lambda _: {"processing_status": "pending"}, sleep=waits.append)
assert waits == [5] * 18
def test_nonempty_report_requires_processed_findings():
with pytest.raises(ValueError, match="absent"):
native.verify_upload(
UPLOAD,
"sample-release",
reports.sarif(
"sample-scanner", [("SAMPLE_RULE", "high", "sample.py")]
),
"example/project",
SHA,
"refs/heads/main",
"sample",
"example/project/.github/workflows/release.yml@refs/heads/main",
fetch=lambda endpoint: (
{"processing_status": "complete"}
if "/sarifs/" in endpoint
else [receipt()]
),
)
def test_only_osv_can_use_the_existing_implicit_category():
with pytest.raises(ValueError, match="OSV"):
verify(
lambda _: pytest.fail(
"Invalid category must fail before API access"
),
category="auto",
)
row = receipt(
category=".github/workflows/release.yml:osv-scan",
tool={"name": "osv-scanner"},
)
verify(
lambda endpoint: (
{"processing_status": "complete"}
if "/sarifs/" in endpoint
else [row]
),
category="auto",
tool="osv-scanner",
)
def npm_data():
return {
"auditReportVersion": 2,
"vulnerabilities": {},
"metadata": {
"vulnerabilities": {
"info": 0,
"low": 0,
"moderate": 0,
"high": 0,
"critical": 0,
"total": 0,
}
},
}
def test_npm_runner_requires_a_target_and_its_lockfile(tmp_path):
with pytest.raises(ValueError, match="no target"):
reports.audit_npm(tmp_path)
(tmp_path / "package.json").write_text("{}", encoding="utf-8")
with pytest.raises(ValueError, match="lockfile"):
reports.audit_npm(tmp_path)
@pytest.mark.parametrize("status,allowed", [(0, True), (1, True), (2, False)])
def test_npm_runner_preserves_operational_failures(tmp_path, status, allowed):
for file in ("package.json", "package-lock.json"):
(tmp_path / file).write_text("{}", encoding="utf-8")
def run(argv, **kwargs):
assert argv == ["npm", "audit", "--audit-level=moderate", "--json"]
assert kwargs["cwd"] == tmp_path
return SimpleNamespace(returncode=status, stdout=json.dumps(npm_data()))
if allowed:
assert reports.audit_npm(tmp_path, run)["runs"][0]["results"] == []
else:
with pytest.raises(ValueError, match="execute"):
reports.audit_npm(tmp_path, run)
def test_npm_completion_counts_cannot_hide_partial_records():
data = npm_data()
data["metadata"]["vulnerabilities"]["high"] = 1
with pytest.raises(ValueError, match="counts"):
reports.npm_findings(data, "package-lock.json")
data = npm_data()
data["vulnerabilities"]["synthetic"] = {"severity": "medium", "via": []}
with pytest.raises(ValueError):
reports.npm_findings(data, "package-lock.json")
@pytest.mark.parametrize(
"data",
[
[],
{},
{"summary": {"parsing_errors": False}, "results": {}},
{"summary": {"parsing_errors": 0}, "results": []},
],
)
def test_checkov_empty_or_malformed_completion_is_fatal(data):
with pytest.raises(ValueError):
reports.validate_checkov(data)
def test_checkov_accepts_its_native_complete_empty_summary_without_hiding_errors():
data = {
"passed": 0,
"failed": 0,
"skipped": 0,
"parsing_errors": 0,
"resource_count": 0,
"checkov_version": "3.2.499",
}
reports.validate_checkov(data)
for field in ("passed", "failed", "parsing_errors", "resource_count"):
malformed = {**data, field: 1}
with pytest.raises(ValueError):
reports.validate_checkov(malformed)
def workflow(name):
return yaml.safe_load(
(ROOT / ".github/workflows" / f"{name}.yml").read_text(encoding="utf-8")
)
def test_zap_preserves_configured_and_default_advisories(tmp_path):
path = tmp_path / "rules.tsv"
path.write_text(
"10001\tFAIL\tSynthetic fail\n10002\tWARN\tSynthetic warning\n10003\tIGNORE\tSynthetic ignored\n",
encoding="utf-8",
)
payload = {
"@version": "synthetic",
"site": [
{
"alerts": [
{"pluginid": str(rule), "riskcode": "2"}
for rule in range(10001, 10005)
]
}
],
}
report = reports.zap_report(payload, path)
policy = native.policy_module()
assert [
policy.blocking_severity(finding)
for finding in native.report_findings(report, tmp_path)
] == [True, False, False, False]
rules = report["runs"][0]["tool"]["driver"]["rules"]
assert [rule["properties"]["native-risk"] for rule in rules] == [
"medium"
] * 4
assert [rule["properties"]["configured-action"] for rule in rules] == [
"FAIL",
"WARN",
"IGNORE",
"WARN",
]
@pytest.mark.parametrize(
"riskcode,severity", [("0", "medium"), ("1", "medium"), ("3", "high")]
)
def test_zap_configured_fail_rules_block_at_any_native_risk(
tmp_path, riskcode, severity
):
"""Low/Informational FAIL rules (e.g. 10021, 10035, 10063) must not
become permanently advisory notes; they enforce the age policy."""
path = tmp_path / "rules.tsv"
path.write_text("10021\tFAIL\tSynthetic header rule\n", encoding="utf-8")
payload = {
"@version": "synthetic",
"site": [{"alerts": [{"pluginid": "10021", "riskcode": riskcode}]}],
}
report = reports.zap_report(payload, path)
findings = native.report_findings(report, tmp_path)
assert len(findings) == 1
assert findings[0]["rule"]["security_severity_level"] == severity
assert findings[0]["rule"]["severity"] == "warning"
assert native.policy_module().blocking_severity(findings[0])
rule = report["runs"][0]["tool"]["driver"]["rules"][0]
assert (
rule["properties"]["native-risk"]
== (("info", "low", "medium", "high")[int(riskcode)])
)
@pytest.mark.parametrize("action", ["FAIL", "WARN", "IGNORE"])
@pytest.mark.parametrize(
"risks", [("1", "0"), ("0", "1"), ("3", "1"), ("1", "3")]
)
def test_zap_aggregates_variants_and_sites_without_lowering_native_risk(
tmp_path, action, risks
):
path = tmp_path / "rules.tsv"
path.write_text(f"100000\t{action}\tSynthetic rule\n", encoding="utf-8")
sites = [
{
"alerts": [
{
"pluginid": "100000",
"alertRef": f"100000-{risk}",
"riskcode": risk,
}
]
}
for risk in risks
]
report = reports.zap_report({"@version": "synthetic", "site": sites}, path)
assert report == reports.zap_report(
{"@version": "synthetic", "site": list(reversed(sites))}, path
)
run = report["runs"][0]
assert len(run["results"]) == len(run["tool"]["driver"]["rules"]) == 1
rule = run["tool"]["driver"]["rules"][0]
highest = "high" if "3" in risks else "low"
assert rule["id"] == "ZAP-100000"
assert rule["properties"]["native-risk"] == highest
assert rule["properties"]["configured-action"] == action
assert rule["properties"]["native-alert-refs"] == sorted(
f"100000-{risk}" for risk in risks
)
assert highest in rule["shortDescription"]["text"]
assert run["results"][0]["locations"][0]["physicalLocation"] == {
"artifactLocation": {"uri": ".zap/rules.tsv"},
"region": {"startLine": 1},
}
findings = native.report_findings(report, tmp_path)
assert native.policy_module().blocking_severity(findings[0]) is (
action == "FAIL"
)
expected_score = (
("8.0" if highest == "high" else "5.0") if action == "FAIL" else "0.0"
)
assert rule["properties"]["security-severity"] == expected_score
def test_zap_variant_metadata_does_not_copy_native_diagnostics(tmp_path):
path = tmp_path / "rules.tsv"
path.write_text("10055\tWARN\tSynthetic rule\n", encoding="utf-8")
marker = "synthetic-private-response"
finding = {
"pluginid": "10055",
"alertRef": "10055-1",
"riskcode": "2",
"alert": marker,
"desc": marker,
"instances": [{"uri": marker, "evidence": marker}],
}
report = reports.zap_report(
{"@version": "synthetic", "site": [{"alerts": [finding, finding]}]},
path,
)
assert marker not in json.dumps(report)
rule = report["runs"][0]["tool"]["driver"]["rules"][0]
assert rule["properties"]["native-alert-refs"] == ["10055-1"]
assert len(report["runs"][0]["results"]) == 1
@pytest.mark.parametrize("reference", [None, 12, "99999-1", "10055-text"])
def test_zap_rejects_unstructured_variant_references(tmp_path, reference):
path = tmp_path / "rules.tsv"
path.write_text("10055\tWARN\tSynthetic rule\n", encoding="utf-8")
with pytest.raises(ValueError, match="^Invalid ZAP alert reference$"):
reports.zap_report(
{
"@version": "synthetic",
"site": [
{
"alerts": [
{
"pluginid": "10055",
"alertRef": reference,
"riskcode": "2",
}
]
}
],
},
path,
)
@pytest.mark.parametrize(
"configuration", ["10001\tOTHER\n", "10001\tFAIL\n10001\tIGNORE\n"]
)
def test_zap_invalid_rule_configuration_does_not_grant_a_pass(
tmp_path, configuration
):
path = tmp_path / "rules.tsv"
path.write_text(configuration, encoding="utf-8")
with pytest.raises(ValueError):
reports.zap_report(
{"@version": "synthetic", "site": [{"alerts": []}]}, path
)
def test_release_native_jobs_and_installed_audits_precede_the_final_decision():
gate = workflow("release-gate")["jobs"]
native_jobs = {
"osv-scan",
"npm-audit",
"retirejs-scan",
"hadolint-scan",
"checkov-scan",
"owasp-zap-scan",
}
for name in native_jobs:
assert gate[name]["with"]["release_policy"] is True
alert_job = gate["check-code-scanning-alerts"]
assert native_jobs | {"pip-install-check", "package-install-matrix"} <= set(
alert_job["needs"]
)
summary = gate["release-gate-summary"]
assert {
"security-tests",
"gitleaks-scan",
"check-code-scanning-alerts",
} <= set(summary["needs"])
step = next(s for s in alert_job["steps"] if s.get("id") == "alert-check")
assert len(json.loads(step["env"]["NATIVE_DEADLINES"])) == 7
assert "--native-deadlines" in step["run"]
for name in ("pip-install-check", "package-install-matrix"):
audits = [
s["run"]
for s in gate[name]["steps"]
if "audit_installed_dependencies.py" in s.get("run", "")
]
assert audits and all(
"--release-policy" in command for command in audits
)
assert any(
"upload-sarif@" in s.get("uses", "") for s in gate[name]["steps"]
)
def test_promotion_finding_decision_precedes_tag_publication():
steps = workflow("docker-publish")["jobs"]["promote"]["steps"]
position = {s.get("id"): i for i, s in enumerate(steps) if "id" in s}
publish = next(
i
for i, s in enumerate(steps)
if "docker buildx imagetools create" in s.get("run", "")
)
assert (
position["native-upload"]
< position["native-policy"]
< position["native-deadline"]
< publish
)
# Immediate critical and secret checks must also precede tag publication.
immediate = [
i
for i, s in enumerate(steps)
if "aquasecurity/trivy-action@" in s.get("uses", "")
and str((s.get("with") or {}).get("exit-code")) == "1"
]
secret = [
i
for i in immediate
if (steps[i].get("with") or {}).get("scanners") == "secret"
]
assert len(immediate) == 2 and len(secret) == 1
assert max(immediate) < publish
promotion = workflow("docker-publish")
assert (
promotion[True]["workflow_call"]["outputs"]["release_deadline"]["value"]
== "${{ jobs.promote.outputs.release_deadline }}"
)
assert (
promotion["jobs"]["promote"]["outputs"]["release_deadline"]
== "${{ steps.native-deadline.outputs.release_deadline }}"
)
@pytest.mark.parametrize(
"name,job",
[
("prerelease-docker", "security-scan"),
("docker-publish", "promote"),
("container-security", "trivy-scan"),
],
)
def test_image_secrets_block_immediately_instead_of_being_aged(name, job):
"""Trivy's default image scanners include `secret`; its HIGH rules
(private-key, pypi-upload-token) must not inherit the 14-day window."""
checks = [
step
for step in workflow(name)["jobs"][job]["steps"]
if "aquasecurity/trivy-action@" in step.get("uses", "")
and step.get("with", {}).get("scanners") == "secret"
]
assert len(checks) == 1
step = checks[0]
assert "if" not in step and "continue-on-error" not in step
assert str(step["with"]["exit-code"]) == "1"
assert {"CRITICAL", "HIGH", "MEDIUM"} <= set(
step["with"]["severity"].split(",")
)
def test_release_policy_passthroughs_always_send_a_boolean():
"""Without workflow_call inputs (pull_request, merge_group, schedule,
dispatch) `inputs.release_policy` is empty; GitHub rejects '' for a
typed boolean input and the caller run reports no jobs at all."""
checked = 0
for path in sorted((ROOT / ".github/workflows").glob("*.yml")):
for job in (workflow(path.stem).get("jobs") or {}).values():
value = (job.get("with") or {}).get("release_policy")
if value is None or not job.get("uses"):
continue
checked += 1
assert value is True or value == (
"${{ inputs.release_policy == true }}"
), f"{path.name}: release_policy passthrough {value!r}"
assert checked >= 9