~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Regression tests for the coverage data shown on pull requests."""
# allow: no-sut-import — this guardian test exercises scripts/ci production code
import json
from pathlib import Path
import pytest
from scripts.ci import coverage_summary
from scripts.ci.coverage_summary import build_summary
def _valid_coverage_xml(
*,
line_rate: str = "0.9",
branch_rate: str = "0.8",
lines_covered: str = "9",
lines_valid: str = "10",
branches_covered: str = "8",
branches_valid: str = "10",
contents: str = "",
) -> str:
return (
f'<coverage line-rate="{line_rate}" branch-rate="{branch_rate}" '
f'lines-covered="{lines_covered}" lines-valid="{lines_valid}" '
f'branches-covered="{branches_covered}" '
f'branches-valid="{branches_valid}">'
f"{contents}</coverage>"
)
def test_valid_coverage_xml_is_summarized(tmp_path: Path) -> None:
coverage_path = tmp_path / "coverage.xml"
coverage_path.write_text(
"""<?xml version="1.0" ?>
<coverage line-rate="0.9" branch-rate="0.8"
lines-covered="9" lines-valid="10"
branches-covered="8" branches-valid="10">
<packages><package><classes>
<class filename="src/local_deep_research/low.py" line-rate="0.4" />
<class filename="src/local_deep_research/high.py" line-rate="0.75" />
</classes></package></packages>
</coverage>
""",
encoding="utf-8",
)
summary = build_summary(coverage_path)
assert summary == {
"generated": True,
"valid": True,
"lineRate": 0.9,
"branchRate": 0.8,
"linesCovered": 9,
"linesTotal": 10,
"filesAnalyzed": 2,
"lowestCoverageFiles": [{"filename": "low.py", "rate": 40.0}],
}
@pytest.mark.parametrize(
"xml",
[
'<coverage line-rate="0.9" branch-rate="0.8" '
'lines-covered="9" lines-valid="10" '
'branches-covered="8" branches-valid="10">',
'<coverage line-rate="NaN" branch-rate="0.8" '
'lines-covered="9" lines-valid="10" '
'branches-covered="8" branches-valid="10" />',
'<coverage line-rate="1.1" branch-rate="0.8" '
'lines-covered="9" lines-valid="10" '
'branches-covered="8" branches-valid="10" />',
'<coverage line-rate="0.9" branch-rate="0.8" '
'lines-covered="11" lines-valid="10" '
'branches-covered="8" branches-valid="10" />',
'<coverage line-rate="0.9" branch-rate="0.8" '
'lines-covered="9.5" lines-valid="10" '
'branches-covered="8" branches-valid="10" />',
'<coverage line-rate="0.9" branch-rate="0.8" '
'lines-covered="9" lines-valid="10" '
'branches-covered="11" branches-valid="10" />',
'<coverage line-rate="0.9" branch-rate="0.8" '
'lines-covered="9" lines-valid="10" />',
],
)
def test_malformed_or_impossible_coverage_is_rejected(
tmp_path: Path, xml: str
) -> None:
coverage_path = tmp_path / "coverage.xml"
coverage_path.write_text(xml, encoding="utf-8")
summary = build_summary(coverage_path)
assert summary["generated"] is True
assert summary["valid"] is False
assert isinstance(summary["error"], str)
def test_missing_coverage_is_distinct_from_malformed(tmp_path: Path) -> None:
summary = build_summary(tmp_path / "missing.xml")
assert summary == {"generated": False, "valid": False}
def test_coverage_filename_is_safe_for_bot_markdown(tmp_path: Path) -> None:
coverage_path = tmp_path / "coverage.xml"
coverage_path.write_text(
'<coverage line-rate="0.4" branch-rate="0.3" '
'lines-covered="4" lines-valid="10" '
'branches-covered="3" branches-valid="10">'
'<class filename="src/x` @org/team</code>.py" '
'line-rate="0.4" />'
"</coverage>",
encoding="utf-8",
)
summary = build_summary(coverage_path)
filename = summary["lowestCoverageFiles"][0]["filename"]
assert "\n" not in filename
assert "`" not in filename
assert "@" not in filename
assert "`" in filename
assert "@org/team" in filename
assert "</code>" in filename
def test_coverage_py_rounded_rates_agree_with_counts(tmp_path: Path) -> None:
coverage_path = tmp_path / "coverage.xml"
coverage_path.write_text(
_valid_coverage_xml(
line_rate="0.9257",
branch_rate="0.8805",
lines_covered="50221",
lines_valid="54249",
branches_covered="14271",
branches_valid="16208",
),
encoding="utf-8",
)
summary = build_summary(coverage_path)
assert summary["valid"] is True
@pytest.mark.parametrize(
("line_rate", "branch_rate"),
[("0", "1"), ("1", "0"), ("1", "1"), ("0", "0")],
)
def test_empty_coverage_accepts_common_zero_denominator_rates(
tmp_path: Path, line_rate: str, branch_rate: str
) -> None:
coverage_path = tmp_path / "coverage.xml"
coverage_path.write_text(
_valid_coverage_xml(
line_rate=line_rate,
branch_rate=branch_rate,
lines_covered="0",
lines_valid="0",
branches_covered="0",
branches_valid="0",
),
encoding="utf-8",
)
assert build_summary(coverage_path)["valid"] is True
@pytest.mark.parametrize(
("line_rate", "branch_rate"),
[("0.99", "0.88"), ("0.01", "0.99")],
)
def test_declared_rates_must_agree_with_counts(
tmp_path: Path, line_rate: str, branch_rate: str
) -> None:
coverage_path = tmp_path / "coverage.xml"
coverage_path.write_text(
_valid_coverage_xml(
line_rate=line_rate,
branch_rate=branch_rate,
lines_covered="1",
lines_valid="100",
branches_covered="88",
branches_valid="100",
),
encoding="utf-8",
)
summary = build_summary(coverage_path)
assert summary["valid"] is False
assert "disagrees with its counts" in summary["error"]
def test_counts_must_fit_in_a_javascript_safe_integer(tmp_path: Path) -> None:
coverage_path = tmp_path / "coverage.xml"
coverage_path.write_text(
_valid_coverage_xml(
line_rate="1",
lines_covered=str(2**53),
lines_valid=str(2**53),
),
encoding="utf-8",
)
summary = build_summary(coverage_path)
assert summary["valid"] is False
assert "JavaScript safe integer" in summary["error"]
def test_javascript_safe_integer_boundary_is_accepted(tmp_path: Path) -> None:
coverage_path = tmp_path / "coverage.xml"
coverage_path.write_text(
_valid_coverage_xml(
line_rate="1",
lines_covered=str(2**53 - 1),
lines_valid=str(2**53 - 1),
),
encoding="utf-8",
)
assert build_summary(coverage_path)["valid"] is True
def test_unknown_declared_encoding_is_returned_as_json(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
coverage_path = tmp_path / "coverage.xml"
coverage_path.write_bytes(
b'<?xml version="1.0" encoding="not-a-real-codec"?>'
b'<coverage line-rate="1" branch-rate="1" '
b'lines-covered="1" lines-valid="1" '
b'branches-covered="0" branches-valid="0" />'
)
output_path = tmp_path / "summary.json"
monkeypatch.setattr(
coverage_summary.sys,
"argv",
["coverage_summary.py", str(coverage_path), str(output_path)],
)
assert coverage_summary.main() == 1
summary = json.loads(output_path.read_text(encoding="utf-8"))
assert summary["generated"] is True
assert summary["valid"] is False
assert summary["error"].startswith("LookupError:")
def test_doctype_entities_are_rejected(tmp_path: Path) -> None:
coverage_path = tmp_path / "coverage.xml"
coverage_path.write_text(
"""<?xml version="1.0"?>
<!DOCTYPE coverage [<!ENTITY forged "1">]>
<coverage line-rate="&forged;" branch-rate="1"
lines-covered="1" lines-valid="1"
branches-covered="0" branches-valid="0" />
""",
encoding="utf-8",
)
summary = build_summary(coverage_path)
assert summary["generated"] is True
assert summary["valid"] is False
assert "Forbidden" in summary["error"]
def test_error_diagnostic_is_bounded_and_single_line(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
capsys: pytest.CaptureFixture[str],
) -> None:
coverage_path = tmp_path / "coverage.xml"
coverage_path.write_text("<coverage />", encoding="utf-8")
output_path = tmp_path / "summary.json"
def reject_report(_coverage_path: Path) -> None:
raise ValueError(f"first line\nsecond\tline {'x' * 2_000}")
monkeypatch.setattr(coverage_summary.ElementTree, "parse", reject_report)
monkeypatch.setattr(
coverage_summary.sys,
"argv",
["coverage_summary.py", str(coverage_path), str(output_path)],
)
assert coverage_summary.main() == 1
summary = json.loads(output_path.read_text(encoding="utf-8"))
error = summary["error"]
stderr = capsys.readouterr().err.removesuffix("\n")
assert isinstance(error, str)
assert len(error) <= 1024
assert "\n" not in error
assert "\r" not in error
assert "\t" not in error
assert error.endswith("…")
assert len(stderr) <= 1024
assert "\n" not in stderr
def test_cli_returns_zero_and_emits_json_for_valid_report(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
coverage_path = tmp_path / "coverage.xml"
coverage_path.write_text(_valid_coverage_xml(), encoding="utf-8")
output_path = tmp_path / "summary.json"
monkeypatch.setattr(
coverage_summary.sys,
"argv",
["coverage_summary.py", str(coverage_path), str(output_path)],
)
assert coverage_summary.main() == 0
assert json.loads(output_path.read_text(encoding="utf-8"))["valid"] is True
@pytest.mark.parametrize("malformed", [False, True])
def test_cli_returns_one_and_emits_json_for_unusable_report(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
malformed: bool,
) -> None:
coverage_path = tmp_path / "coverage.xml"
if malformed:
coverage_path.write_text("<coverage", encoding="utf-8")
output_path = tmp_path / "summary.json"
monkeypatch.setattr(
coverage_summary.sys,
"argv",
["coverage_summary.py", str(coverage_path), str(output_path)],
)
assert coverage_summary.main() == 1
summary = json.loads(output_path.read_text(encoding="utf-8"))
assert summary["generated"] is malformed
assert summary["valid"] is False
def test_cli_usage_error_returns_two(
monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
) -> None:
monkeypatch.setattr(coverage_summary.sys, "argv", ["coverage_summary.py"])
assert coverage_summary.main() == 2
assert capsys.readouterr().err.startswith("usage:")