Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

19.7 KB
Raw Blame
"""Preserve native CodeQL evidence while fitting optional location metadata.

# allow: no-sut-import — exercises the actual CI report adapter directly.
"""

import copy
import importlib.util
import json
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path


ROOT = Path(__file__).resolve().parents[3]
SCRIPT = ROOT / ".github/scripts/prepare_codeql_sarif.py"
SPEC = importlib.util.spec_from_file_location("prepare_codeql", SCRIPT)
PREPARE = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(PREPARE)
SOURCE_BASE = "https://example.test/example/project/blob/" + "a" * 40


def sample_report(count=283):
    def link(label, identifier):
        return f"[{label}]({min(identifier, count)})" if count else label

    return {
        "version": "2.1.0",
        "runs": [
            {
                "tool": {
                    "driver": {"name": "CodeQL", "rules": [{"id": "sample"}]}
                },
                "invocations": [{"executionSuccessful": True}],
                "results": [
                    {
                        "ruleId": "sample",
                        "level": "warning",
                        "message": {
                            "text": f"Synthetic {link('first', 280)} and {link('second', 279)}.",
                            "markdown": f"Synthetic {link('third', 281)}.",
                            "arguments": [link("argument", 282)],
                        },
                        "locations": [
                            {
                                "physicalLocation": {
                                    "artifactLocation": {
                                        "uri": "src/example.py"
                                    },
                                    "region": {"startLine": 1},
                                }
                            }
                        ],
                        "partialFingerprints": {
                            "primaryLocationLineHash": "sample:1"
                        },
                        "correlationGuid": "00000000-0000-4000-8000-000000000001",
                        "properties": {"sample": "metadata"},
                        "codeFlows": [
                            {
                                "threadFlows": [
                                    {
                                        "locations": [
                                            {
                                                "location": {
                                                    "message": {
                                                        "text": f"Flow {link('link', 283)}."
                                                    },
                                                    "physicalLocation": {
                                                        "artifactLocation": {
                                                            "uri": "src/example.py"
                                                        },
                                                        "region": {
                                                            "startLine": 1
                                                        },
                                                    },
                                                }
                                            }
                                        ]
                                    }
                                ]
                            }
                        ],
                        "relatedLocations": [
                            {
                                "id": index,
                                "physicalLocation": {
                                    "artifactLocation": {
                                        "uri": "src/example.py"
                                    },
                                    "region": {"startLine": index},
                                },
                            }
                            for index in range(1, count + 1)
                        ],
                    }
                ],
            }
        ],
    }


class CodeQLSarifTests(unittest.TestCase):
    def test_escaped_link_syntax_keeps_native_evidence(self):
        report = sample_report()
        report["runs"][0]["results"][0]["message"] = {
            "text": r"Literal source array\[0\](280)."
        }
        self.assertEqual(
            PREPARE.prepare_report(report, source_base_url=SOURCE_BASE), report
        )

    def test_linked_overflow_uses_source_permalinks_without_new_findings(self):
        report = sample_report()
        result = report["runs"][0]["results"][0]
        result["message"] = {
            "text": " ".join(
                f"[source {index}]({index})" for index in range(1, 284)
            ),
            "markdown": "See [source](280).",
        }
        original = copy.deepcopy(report)
        prepared = PREPARE.prepare_report(report, source_base_url=SOURCE_BASE)
        actual = prepared["runs"][0]["results"][0]
        self.assertEqual(len(actual["relatedLocations"]), 100)
        expected_text = " ".join(
            f"[source {index}]({index})"
            if index <= 100
            else f"[source {index}]({SOURCE_BASE}/src/example.py#L{index})"
            for index in range(1, 284)
        )
        self.assertEqual(actual["message"]["text"], expected_text)
        self.assertEqual(
            actual["message"]["markdown"],
            f"See [source]({SOURCE_BASE}/src/example.py#L280).",
        )
        self.assertEqual(len(prepared["runs"][0]["results"]), 1)
        for field in (
            "ruleId",
            "locations",
            "partialFingerprints",
            "correlationGuid",
        ):
            self.assertEqual(actual[field], result[field])
        flow_location = actual["codeFlows"][0]["threadFlows"][0]["locations"][
            0
        ]["location"]
        self.assertEqual(
            flow_location["physicalLocation"],
            result["codeFlows"][0]["threadFlows"][0]["locations"][0][
                "location"
            ]["physicalLocation"],
        )
        self.assertEqual(
            flow_location["message"]["text"],
            f"Flow [link]({SOURCE_BASE}/src/example.py#L283).",
        )
        self.assertEqual(report, original)

    def test_source_permalinks_escape_paths_and_preserve_line_ranges(self):
        location = {
            "physicalLocation": {
                "artifactLocation": {
                    "uri": "src/example%20%28%23%29.py",
                    "uriBaseId": "%SRCROOT%",
                },
                "region": {"startLine": 10, "endLine": 12},
            }
        }
        self.assertEqual(
            PREPARE._source_link(location, SOURCE_BASE),
            f"{SOURCE_BASE}/src/example%20%28%23%29.py#L10-L12",
        )

    def test_unresolvable_overflow_source_locations_keep_the_native_report(
        self,
    ):
        mutations = (
            lambda loc: loc["physicalLocation"]["artifactLocation"].update(
                uri="../outside.py"
            ),
            lambda loc: loc["physicalLocation"]["artifactLocation"].update(
                uri="%2e%2e/outside.py"
            ),
            lambda loc: loc["physicalLocation"]["artifactLocation"].update(
                uri="file:///outside.py"
            ),
            lambda loc: loc["physicalLocation"]["artifactLocation"].update(
                uriBaseId="external"
            ),
            lambda loc: loc["physicalLocation"].update(region={}),
        )
        for mutate in mutations:
            with self.subTest(mutation=mutate):
                report = sample_report(150)
                result = report["runs"][0]["results"][0]
                result["message"] = {
                    "text": " ".join(
                        f"[source]({index})" for index in range(1, 151)
                    )
                }
                mutate(result["relatedLocations"][149])
                self.assertEqual(
                    PREPARE.prepare_report(report, source_base_url=SOURCE_BASE),
                    report,
                )

    def test_indirect_overflow_links_are_preserved_without_rewriting_shared_templates(
        self,
    ):
        report = sample_report(150)
        run = report["runs"][0]
        run["tool"]["driver"]["globalMessageStrings"] = {
            "shared": {
                "text": " ".join(
                    f"[source]({index})" for index in range(1, 151)
                )
            }
        }
        run["results"][0]["message"] = {"id": "shared"}
        self.assertEqual(
            PREPARE.prepare_report(report, source_base_url=SOURCE_BASE), report
        )

    def test_pr_and_diff_informed_reports_do_not_convert_overflow_links(self):
        report = sample_report(150)
        report["runs"][0]["results"][0]["message"] = {
            "text": " ".join(f"[source]({index})" for index in range(1, 151))
        }
        self.assertEqual(
            PREPARE.prepare_report(
                report, preserve_locations=True, source_base_url=SOURCE_BASE
            ),
            report,
        )
        report["runs"][0]["properties"] = {"incrementalMode": "diff-informed"}
        self.assertEqual(
            PREPARE.prepare_report(report, source_base_url=SOURCE_BASE), report
        )

    def test_source_base_must_be_a_commit_pinned_https_repository_url(self):
        invalid = (
            "https://example.test/example/project/blob/main",
            "http://example.test/example/project/blob/" + "a" * 40,
            SOURCE_BASE + "?query=example",
            SOURCE_BASE + "#fragment",
            SOURCE_BASE.replace("example.test", "[email protected]"),
        )
        for source_base in invalid:
            with self.subTest(source_base=source_base):
                with self.assertRaises(ValueError):
                    PREPARE.prepare_report(
                        sample_report(), source_base_url=source_base
                    )

    def test_pr_preparation_preserves_all_locations_for_diff_filtering(self):
        report = sample_report()
        self.assertEqual(
            PREPARE.prepare_report(report, preserve_locations=True), report
        )

    def test_diff_informed_reports_keep_all_locations_without_a_caller_flag(
        self,
    ):
        report = sample_report()
        report["runs"][0]["properties"] = {"incrementalMode": "diff-informed"}
        self.assertEqual(PREPARE.prepare_report(report), report)

    def test_literal_braces_without_message_arguments_do_not_hide_links(self):
        report = sample_report()
        report["runs"][0]["results"][0]["message"] = {
            "text": "Literal {value} and [source](280)."
        }
        prepared = PREPARE.prepare_report(report)
        locations = prepared["runs"][0]["results"][0]["relatedLocations"]
        self.assertEqual(len(locations), 100)
        self.assertTrue(
            {280, 283}.issubset({location["id"] for location in locations})
        )

    def test_native_message_ids_and_formatted_arguments_keep_their_links(self):
        report = sample_report()
        run = report["runs"][0]
        run["tool"]["driver"]["globalMessageStrings"] = {
            "primary": {"text": "Synthetic [first]({0}) and {1}."},
            "flow": {"text": "Flow {0}."},
            "source": {"text": "Source location."},
        }
        result = run["results"][0]
        result["message"] = {
            "id": "primary",
            "arguments": ["280", "[second](279)"],
        }
        result["codeFlows"][0]["threadFlows"][0]["locations"][0]["location"][
            "message"
        ] = {"id": "flow", "arguments": ["[link](283)"]}
        for location in result["relatedLocations"]:
            location["message"] = {"id": "source"}
        original = copy.deepcopy(report)
        prepared = PREPARE.prepare_report(report)
        actual = prepared["runs"][0]["results"][0]
        self.assertEqual(len(actual["relatedLocations"]), 100)
        self.assertTrue(
            {279, 280, 283}.issubset(
                {loc["id"] for loc in actual["relatedLocations"]}
            )
        )
        self.assertEqual(actual["message"], result["message"])
        self.assertEqual(actual["codeFlows"], result["codeFlows"])
        self.assertEqual(
            actual["partialFingerprints"], result["partialFingerprints"]
        )
        self.assertEqual(report, original)

    def test_ambiguous_rule_message_ids_conservatively_keep_all_referenced_locations(
        self,
    ):
        report = sample_report()
        run = report["runs"][0]
        run["tool"]["driver"]["rules"] = [
            {"id": "one", "messageStrings": {"shared": {"text": "[one](200)"}}},
            {"id": "two", "messageStrings": {"shared": {"text": "[two](201)"}}},
        ]
        run["results"][0]["message"] = {"id": "shared"}
        prepared = PREPARE.prepare_report(report)
        locations = prepared["runs"][0]["results"][0]["relatedLocations"]
        self.assertEqual(len(locations), 100)
        self.assertTrue(
            {200, 201, 283}.issubset({location["id"] for location in locations})
        )

    def test_overflow_keeps_links_identities_findings_and_flow_data(self):
        report = sample_report()
        original = copy.deepcopy(report)
        prepared = PREPARE.prepare_report(report)
        result = prepared["runs"][0]["results"][0]
        retained = {location["id"] for location in result["relatedLocations"]}
        self.assertEqual(retained, set(range(1, 96)) | set(range(279, 284)))
        expected = copy.deepcopy(original)
        expected["runs"][0]["results"][0]["relatedLocations"] = [
            location
            for location in original["runs"][0]["results"][0][
                "relatedLocations"
            ]
            if location["id"] in retained
        ]
        self.assertEqual(prepared, expected)
        self.assertEqual(report, original)

    def test_valid_reports_below_the_limit_are_unchanged(self):
        for count in (0, 20, 100):
            with self.subTest(count=count):
                report = sample_report(count)
                report["runs"][0]["results"][0]["message"] = {
                    "text": "Synthetic result."
                }
                self.assertEqual(PREPARE.prepare_report(report), report)

    def test_more_than_one_hundred_referenced_locations_are_not_removed(self):
        report = sample_report(150)
        report["runs"][0]["results"][0]["message"] = {
            "text": " ".join(f"[location]({index})" for index in range(1, 102))
        }
        self.assertEqual(PREPARE.prepare_report(report), report)

    def test_indirect_messages_relationships_and_markdown_references_are_preserved(
        self,
    ):
        mutations = (
            lambda result: result.update(message={"id": "sample-message"}),
            lambda result: result["relatedLocations"][0].update(
                relationships=[{"target": 200, "kinds": ["relevant"]}]
            ),
            lambda result: result["message"].update(
                markdown="[link][ref]\n[ref]: 200"
            ),
        )
        for mutate in mutations:
            with self.subTest(mutation=mutate):
                report = sample_report()
                mutate(report["runs"][0]["results"][0])
                self.assertEqual(PREPARE.prepare_report(report), report)

    def test_multiple_runs_and_results_are_preserved(self):
        report = sample_report()
        report["runs"].append(copy.deepcopy(report["runs"][0]))
        report["runs"][1]["results"].append(
            {"ruleId": "other", "message": {"text": "Other."}}
        )
        prepared = PREPARE.prepare_report(report)
        self.assertEqual(len(prepared["runs"]), 2)
        self.assertEqual(
            [len(run["results"]) for run in prepared["runs"]], [1, 2]
        )
        self.assertEqual(
            prepared["runs"][1]["results"][1], report["runs"][1]["results"][1]
        )

    def test_invalid_or_unsuccessful_native_reports_are_rejected(self):
        mutations = (
            lambda r: r.update(version="1.0.0"),
            lambda r: r.update(runs=[]),
            lambda r: r["runs"][0].update(tool="invalid"),
            lambda r: r["runs"][0]["tool"]["driver"].update(name="other"),
            lambda r: r["runs"][0].pop("results"),
            lambda r: r["runs"][0].update(results=None),
            lambda r: r["runs"][0].update(results=[None]),
            lambda r: r["runs"][0].update(
                invocations=[{"executionSuccessful": False}]
            ),
            lambda r: r["runs"][0].update(invocations=[{}]),
            lambda r: r["runs"][0]["results"][0].update(
                relatedLocations=[None] * 101
            ),
            lambda r: r["runs"][0]["results"][0]["relatedLocations"][1].update(
                id=1
            ),
            lambda r: r["runs"][0]["results"][0]["relatedLocations"][1].update(
                id=True
            ),
        )
        for mutate in mutations:
            with self.subTest(mutation=mutate):
                report = sample_report()
                mutate(report)
                with self.assertRaises(ValueError):
                    PREPARE.prepare_report(report)

    def test_preparation_preserves_native_bytes_and_rejects_in_place_writes(
        self,
    ):
        with tempfile.TemporaryDirectory() as temporary:
            source = Path(temporary) / "native.sarif"
            destination = Path(temporary) / "upload.sarif"
            raw = json.dumps(sample_report())
            source.write_text(raw, encoding="utf-8")
            PREPARE.prepare(source, destination)
            self.assertEqual(source.read_text(encoding="utf-8"), raw)
            self.assertEqual(
                len(
                    json.loads(destination.read_text(encoding="utf-8"))["runs"][
                        0
                    ]["results"][0]["relatedLocations"]
                ),
                100,
            )
            with self.assertRaises(ValueError):
                PREPARE.prepare(source, source)

    def test_cli_handles_the_native_output_directory(self):
        with tempfile.TemporaryDirectory() as temporary:
            root = Path(temporary)
            source, destination = root / "native", root / "upload"
            source.mkdir()
            raw = json.dumps(sample_report())
            (source / "python.sarif").write_text(raw, encoding="utf-8")
            result = subprocess.run(
                [sys.executable, str(SCRIPT), str(source), str(destination)],
                capture_output=True,
                text=True,
                check=False,
            )
            self.assertEqual(result.returncode, 0, result.stderr)
            self.assertEqual(
                (source / "python.sarif").read_text(encoding="utf-8"), raw
            )
            prepared = json.loads(
                (destination / "python.sarif").read_text(encoding="utf-8")
            )
            self.assertEqual(
                len(prepared["runs"][0]["results"][0]["relatedLocations"]), 100
            )

    def test_nonstandard_json_and_missing_input_never_create_an_upload(self):
        with tempfile.TemporaryDirectory() as temporary:
            source, destination = (
                Path(temporary) / "native.sarif",
                Path(temporary) / "upload.sarif",
            )
            with self.assertRaises(FileNotFoundError):
                PREPARE.prepare(source, destination)
            for raw in (
                "{truncated",
                json.dumps(sample_report()).replace(
                    '"version": "2.1.0"', '"version": NaN'
                ),
            ):
                with self.subTest(raw=raw[:20]):
                    source.write_text(raw, encoding="utf-8")
                    with self.assertRaises(ValueError):
                        PREPARE.prepare(source, destination)
                    self.assertFalse(destination.exists())


if __name__ == "__main__":
    unittest.main()