Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

5.9 KB
Raw Blame
#!/usr/bin/env python3

import json
import tempfile
from collections import Counter
from pathlib import Path

import unittest

from parameterized import parameterized

from bin.semgrep_ratchet import compare, error_paths, main, parse_baseline, render_baseline


def semgrep_output(*paths: str, errors: list[dict] | None = None) -> dict:
    return {
        "results": [{"path": path, "check_id": "rule", "start": {"line": 1}} for path in paths],
        "errors": errors or [],
    }


TIMEOUT = {"type": "Timeout", "path": "a.py"}


class TestSemgrepRatchet(unittest.TestCase):
    @parameterized.expand(
        [
            ("matches", Counter({"a.py": 2}), {"a.py": 2}, {}, {}),
            ("no_findings_no_baseline", Counter(), {}, {}, {}),
            ("new_file", Counter({"a.py": 1, "b.py": 1}), {"a.py": 1}, {"b.py": (1, 0)}, {}),
            ("more_in_a_listed_file", Counter({"a.py": 3}), {"a.py": 2}, {"a.py": (3, 2)}, {}),
            ("fewer_in_a_listed_file", Counter({"a.py": 1}), {"a.py": 2}, {}, {"a.py": (1, 2)}),
            ("file_fully_migrated", Counter(), {"a.py": 2}, {}, {"a.py": (0, 2)}),
            (
                "moved_file",
                Counter({"new/a.py": 2}),
                {"old/a.py": 2},
                {"new/a.py": (2, 0)},
                {"old/a.py": (0, 2)},
            ),
        ]
    )
    def test_compare(
        self,
        _name: str,
        findings: Counter[str],
        baseline: dict[str, int],
        expected_over: dict[str, tuple[int, int]],
        expected_stale: dict[str, tuple[int, int]],
    ) -> None:
        result = compare(findings, baseline)

        assert (result.over, result.stale, result.unchecked) == (expected_over, expected_stale, {})

    @parameterized.expand(
        [
            ("fewer", Counter({"a.py": 1}), {"a.py": 2}, {}, {}, {"a.py": (1, 2)}),
            ("none", Counter(), {"a.py": 2}, {}, {}, {"a.py": (0, 2)}),
            ("more_still_fails", Counter({"a.py": 3}), {"a.py": 2}, {"a.py": (3, 2)}, {}, {}),
            ("other_file_still_stale", Counter({"a.py": 2}), {"a.py": 2, "b.py": 2}, {}, {"b.py": (0, 2)}, {}),
        ]
    )
    def test_compare_with_an_error_in_a_file(
        self,
        _name: str,
        findings: Counter[str],
        baseline: dict[str, int],
        expected_over: dict[str, tuple[int, int]],
        expected_stale: dict[str, tuple[int, int]],
        expected_unchecked: dict[str, tuple[int, int]],
    ) -> None:
        result = compare(findings, baseline, incomplete=frozenset({"a.py"}))

        assert (result.over, result.stale, result.unchecked) == (expected_over, expected_stale, expected_unchecked)

    def test_error_paths_reads_the_path_and_the_spans(self) -> None:
        output = semgrep_output(
            errors=[TIMEOUT, {"type": "Syntax error", "spans": [{"file": "b.py"}, {"file": "c.py"}]}, {"type": "x"}]
        )

        assert error_paths(output) == frozenset({"a.py", "b.py", "c.py"})

    @parameterized.expand(
        [
            ("no_count", "a.py\n"),
            ("zero_count", "a.py 0\n"),
            ("not_a_number", "a.py two\n"),
            ("repeated_path", "a.py 1\na.py 2\n"),
        ]
    )
    def test_parse_baseline_rejects_bad_lines(self, _name: str, text: str) -> None:
        with self.assertRaises(ValueError):
            parse_baseline(text)

    def test_render_keeps_the_header_and_round_trips(self) -> None:
        text = render_baseline("# header\n#\nold.py 9\n", Counter({"b.py": 1, "a b/c.py": 2}))

        assert text == "# header\n#\na b/c.py 2\nb.py 1\n"
        assert parse_baseline(text) == {"a b/c.py": 2, "b.py": 1}

    @parameterized.expand(
        [
            ("matches", ("a.py", "a.py"), "a.py 2\n", 0),
            ("new_finding", ("a.py", "a.py", "b.py"), "a.py 2\n", 1),
            ("stale_entry", ("a.py",), "a.py 2\n", 1),
            ("stale_entry_with_an_error", ("a.py",), "a.py 2\n", 0, [TIMEOUT]),
            ("new_finding_with_an_error", ("a.py", "a.py", "a.py"), "a.py 2\n", 1, [TIMEOUT]),
        ]
    )
    def test_main_exit_code(
        self, _name: str, paths: tuple[str, ...], baseline: str, expected: int, errors: list[dict] | None = None
    ) -> None:
        with tempfile.TemporaryDirectory() as directory:
            findings_file = Path(directory) / "findings.json"
            baseline_file = Path(directory) / "baseline.txt"
            findings_file.write_text(json.dumps(semgrep_output(*paths, errors=errors)))
            baseline_file.write_text(baseline)

            exit_code = main(["--findings", str(findings_file), "--baseline", str(baseline_file)])

        assert exit_code == expected

    def test_main_update_writes_a_baseline_that_then_passes(self) -> None:
        with tempfile.TemporaryDirectory() as directory:
            findings_file = Path(directory) / "findings.json"
            baseline_file = Path(directory) / "baseline.txt"
            findings_file.write_text(json.dumps(semgrep_output("a.py", "b.py", "a.py")))
            baseline_file.write_text("# header\nold.py 3\n")
            args = ["--findings", str(findings_file), "--baseline", str(baseline_file)]

            update_exit_code = main([*args, "--update"])
            check_exit_code = main(args)
            written = baseline_file.read_text()

        assert (update_exit_code, check_exit_code, written) == (0, 0, "# header\na.py 2\nb.py 1\n")

    def test_main_update_keeps_the_count_of_a_file_with_an_error(self) -> None:
        with tempfile.TemporaryDirectory() as directory:
            findings_file = Path(directory) / "findings.json"
            baseline_file = Path(directory) / "baseline.txt"
            findings_file.write_text(json.dumps(semgrep_output("b.py", errors=[TIMEOUT])))
            baseline_file.write_text("a.py 3\nb.py 2\n")

            main(["--findings", str(findings_file), "--baseline", str(baseline_file), "--update"])
            written = baseline_file.read_text()

        assert written == "a.py 3\nb.py 1\n"