Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

10.7 KB
Raw Blame
"""Regression tests for the coverage data shown on pull requests."""

# allow: no-sut-import — this guardian test exercises scripts/ci production code

import json
from pathlib import Path

import pytest

from scripts.ci import coverage_summary
from scripts.ci.coverage_summary import build_summary


def _valid_coverage_xml(
    *,
    line_rate: str = "0.9",
    branch_rate: str = "0.8",
    lines_covered: str = "9",
    lines_valid: str = "10",
    branches_covered: str = "8",
    branches_valid: str = "10",
    contents: str = "",
) -> str:
    return (
        f'<coverage line-rate="{line_rate}" branch-rate="{branch_rate}" '
        f'lines-covered="{lines_covered}" lines-valid="{lines_valid}" '
        f'branches-covered="{branches_covered}" '
        f'branches-valid="{branches_valid}">'
        f"{contents}</coverage>"
    )


def test_valid_coverage_xml_is_summarized(tmp_path: Path) -> None:
    coverage_path = tmp_path / "coverage.xml"
    coverage_path.write_text(
        """<?xml version="1.0" ?>
<coverage line-rate="0.9" branch-rate="0.8"
          lines-covered="9" lines-valid="10"
          branches-covered="8" branches-valid="10">
  <packages><package><classes>
    <class filename="src/local_deep_research/low.py" line-rate="0.4" />
    <class filename="src/local_deep_research/high.py" line-rate="0.75" />
  </classes></package></packages>
</coverage>
""",
        encoding="utf-8",
    )

    summary = build_summary(coverage_path)

    assert summary == {
        "generated": True,
        "valid": True,
        "lineRate": 0.9,
        "branchRate": 0.8,
        "linesCovered": 9,
        "linesTotal": 10,
        "filesAnalyzed": 2,
        "lowestCoverageFiles": [{"filename": "low.py", "rate": 40.0}],
    }


@pytest.mark.parametrize(
    "xml",
    [
        '<coverage line-rate="0.9" branch-rate="0.8" '
        'lines-covered="9" lines-valid="10" '
        'branches-covered="8" branches-valid="10">',
        '<coverage line-rate="NaN" branch-rate="0.8" '
        'lines-covered="9" lines-valid="10" '
        'branches-covered="8" branches-valid="10" />',
        '<coverage line-rate="1.1" branch-rate="0.8" '
        'lines-covered="9" lines-valid="10" '
        'branches-covered="8" branches-valid="10" />',
        '<coverage line-rate="0.9" branch-rate="0.8" '
        'lines-covered="11" lines-valid="10" '
        'branches-covered="8" branches-valid="10" />',
        '<coverage line-rate="0.9" branch-rate="0.8" '
        'lines-covered="9.5" lines-valid="10" '
        'branches-covered="8" branches-valid="10" />',
        '<coverage line-rate="0.9" branch-rate="0.8" '
        'lines-covered="9" lines-valid="10" '
        'branches-covered="11" branches-valid="10" />',
        '<coverage line-rate="0.9" branch-rate="0.8" '
        'lines-covered="9" lines-valid="10" />',
    ],
)
def test_malformed_or_impossible_coverage_is_rejected(
    tmp_path: Path, xml: str
) -> None:
    coverage_path = tmp_path / "coverage.xml"
    coverage_path.write_text(xml, encoding="utf-8")

    summary = build_summary(coverage_path)

    assert summary["generated"] is True
    assert summary["valid"] is False
    assert isinstance(summary["error"], str)


def test_missing_coverage_is_distinct_from_malformed(tmp_path: Path) -> None:
    summary = build_summary(tmp_path / "missing.xml")

    assert summary == {"generated": False, "valid": False}


def test_coverage_filename_is_safe_for_bot_markdown(tmp_path: Path) -> None:
    coverage_path = tmp_path / "coverage.xml"
    coverage_path.write_text(
        '<coverage line-rate="0.4" branch-rate="0.3" '
        'lines-covered="4" lines-valid="10" '
        'branches-covered="3" branches-valid="10">'
        '<class filename="src/x`&#10;@org/team&lt;/code&gt;.py" '
        'line-rate="0.4" />'
        "</coverage>",
        encoding="utf-8",
    )

    summary = build_summary(coverage_path)

    filename = summary["lowestCoverageFiles"][0]["filename"]
    assert "\n" not in filename
    assert "`" not in filename
    assert "@" not in filename
    assert "&#96;" in filename
    assert "&#64;org/team" in filename
    assert "&lt;/code&gt;" in filename


def test_coverage_py_rounded_rates_agree_with_counts(tmp_path: Path) -> None:
    coverage_path = tmp_path / "coverage.xml"
    coverage_path.write_text(
        _valid_coverage_xml(
            line_rate="0.9257",
            branch_rate="0.8805",
            lines_covered="50221",
            lines_valid="54249",
            branches_covered="14271",
            branches_valid="16208",
        ),
        encoding="utf-8",
    )

    summary = build_summary(coverage_path)

    assert summary["valid"] is True


@pytest.mark.parametrize(
    ("line_rate", "branch_rate"),
    [("0", "1"), ("1", "0"), ("1", "1"), ("0", "0")],
)
def test_empty_coverage_accepts_common_zero_denominator_rates(
    tmp_path: Path, line_rate: str, branch_rate: str
) -> None:
    coverage_path = tmp_path / "coverage.xml"
    coverage_path.write_text(
        _valid_coverage_xml(
            line_rate=line_rate,
            branch_rate=branch_rate,
            lines_covered="0",
            lines_valid="0",
            branches_covered="0",
            branches_valid="0",
        ),
        encoding="utf-8",
    )

    assert build_summary(coverage_path)["valid"] is True


@pytest.mark.parametrize(
    ("line_rate", "branch_rate"),
    [("0.99", "0.88"), ("0.01", "0.99")],
)
def test_declared_rates_must_agree_with_counts(
    tmp_path: Path, line_rate: str, branch_rate: str
) -> None:
    coverage_path = tmp_path / "coverage.xml"
    coverage_path.write_text(
        _valid_coverage_xml(
            line_rate=line_rate,
            branch_rate=branch_rate,
            lines_covered="1",
            lines_valid="100",
            branches_covered="88",
            branches_valid="100",
        ),
        encoding="utf-8",
    )

    summary = build_summary(coverage_path)

    assert summary["valid"] is False
    assert "disagrees with its counts" in summary["error"]


def test_counts_must_fit_in_a_javascript_safe_integer(tmp_path: Path) -> None:
    coverage_path = tmp_path / "coverage.xml"
    coverage_path.write_text(
        _valid_coverage_xml(
            line_rate="1",
            lines_covered=str(2**53),
            lines_valid=str(2**53),
        ),
        encoding="utf-8",
    )

    summary = build_summary(coverage_path)

    assert summary["valid"] is False
    assert "JavaScript safe integer" in summary["error"]


def test_javascript_safe_integer_boundary_is_accepted(tmp_path: Path) -> None:
    coverage_path = tmp_path / "coverage.xml"
    coverage_path.write_text(
        _valid_coverage_xml(
            line_rate="1",
            lines_covered=str(2**53 - 1),
            lines_valid=str(2**53 - 1),
        ),
        encoding="utf-8",
    )

    assert build_summary(coverage_path)["valid"] is True


def test_unknown_declared_encoding_is_returned_as_json(
    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
    coverage_path = tmp_path / "coverage.xml"
    coverage_path.write_bytes(
        b'<?xml version="1.0" encoding="not-a-real-codec"?>'
        b'<coverage line-rate="1" branch-rate="1" '
        b'lines-covered="1" lines-valid="1" '
        b'branches-covered="0" branches-valid="0" />'
    )
    output_path = tmp_path / "summary.json"
    monkeypatch.setattr(
        coverage_summary.sys,
        "argv",
        ["coverage_summary.py", str(coverage_path), str(output_path)],
    )

    assert coverage_summary.main() == 1
    summary = json.loads(output_path.read_text(encoding="utf-8"))
    assert summary["generated"] is True
    assert summary["valid"] is False
    assert summary["error"].startswith("LookupError:")


def test_doctype_entities_are_rejected(tmp_path: Path) -> None:
    coverage_path = tmp_path / "coverage.xml"
    coverage_path.write_text(
        """<?xml version="1.0"?>
<!DOCTYPE coverage [<!ENTITY forged "1">]>
<coverage line-rate="&forged;" branch-rate="1"
          lines-covered="1" lines-valid="1"
          branches-covered="0" branches-valid="0" />
""",
        encoding="utf-8",
    )

    summary = build_summary(coverage_path)

    assert summary["generated"] is True
    assert summary["valid"] is False
    assert "Forbidden" in summary["error"]


def test_error_diagnostic_is_bounded_and_single_line(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
    capsys: pytest.CaptureFixture[str],
) -> None:
    coverage_path = tmp_path / "coverage.xml"
    coverage_path.write_text("<coverage />", encoding="utf-8")
    output_path = tmp_path / "summary.json"

    def reject_report(_coverage_path: Path) -> None:
        raise ValueError(f"first line\nsecond\tline {'x' * 2_000}")

    monkeypatch.setattr(coverage_summary.ElementTree, "parse", reject_report)
    monkeypatch.setattr(
        coverage_summary.sys,
        "argv",
        ["coverage_summary.py", str(coverage_path), str(output_path)],
    )

    assert coverage_summary.main() == 1
    summary = json.loads(output_path.read_text(encoding="utf-8"))
    error = summary["error"]
    stderr = capsys.readouterr().err.removesuffix("\n")

    assert isinstance(error, str)
    assert len(error) <= 1024
    assert "\n" not in error
    assert "\r" not in error
    assert "\t" not in error
    assert error.endswith("…")
    assert len(stderr) <= 1024
    assert "\n" not in stderr


def test_cli_returns_zero_and_emits_json_for_valid_report(
    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
    coverage_path = tmp_path / "coverage.xml"
    coverage_path.write_text(_valid_coverage_xml(), encoding="utf-8")
    output_path = tmp_path / "summary.json"
    monkeypatch.setattr(
        coverage_summary.sys,
        "argv",
        ["coverage_summary.py", str(coverage_path), str(output_path)],
    )

    assert coverage_summary.main() == 0
    assert json.loads(output_path.read_text(encoding="utf-8"))["valid"] is True


@pytest.mark.parametrize("malformed", [False, True])
def test_cli_returns_one_and_emits_json_for_unusable_report(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
    malformed: bool,
) -> None:
    coverage_path = tmp_path / "coverage.xml"
    if malformed:
        coverage_path.write_text("<coverage", encoding="utf-8")
    output_path = tmp_path / "summary.json"
    monkeypatch.setattr(
        coverage_summary.sys,
        "argv",
        ["coverage_summary.py", str(coverage_path), str(output_path)],
    )

    assert coverage_summary.main() == 1
    summary = json.loads(output_path.read_text(encoding="utf-8"))
    assert summary["generated"] is malformed
    assert summary["valid"] is False


def test_cli_usage_error_returns_two(
    monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
) -> None:
    monkeypatch.setattr(coverage_summary.sys, "argv", ["coverage_summary.py"])

    assert coverage_summary.main() == 2
    assert capsys.readouterr().err.startswith("usage:")