~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Preserve native CodeQL evidence while fitting optional location metadata.
# allow: no-sut-import — exercises the actual CI report adapter directly.
"""
import copy
import importlib.util
import json
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
ROOT = Path(__file__).resolve().parents[3]
SCRIPT = ROOT / ".github/scripts/prepare_codeql_sarif.py"
SPEC = importlib.util.spec_from_file_location("prepare_codeql", SCRIPT)
PREPARE = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(PREPARE)
SOURCE_BASE = "https://example.test/example/project/blob/" + "a" * 40
def sample_report(count=283):
def link(label, identifier):
return f"[{label}]({min(identifier, count)})" if count else label
return {
"version": "2.1.0",
"runs": [
{
"tool": {
"driver": {"name": "CodeQL", "rules": [{"id": "sample"}]}
},
"invocations": [{"executionSuccessful": True}],
"results": [
{
"ruleId": "sample",
"level": "warning",
"message": {
"text": f"Synthetic {link('first', 280)} and {link('second', 279)}.",
"markdown": f"Synthetic {link('third', 281)}.",
"arguments": [link("argument", 282)],
},
"locations": [
{
"physicalLocation": {
"artifactLocation": {
"uri": "src/example.py"
},
"region": {"startLine": 1},
}
}
],
"partialFingerprints": {
"primaryLocationLineHash": "sample:1"
},
"correlationGuid": "00000000-0000-4000-8000-000000000001",
"properties": {"sample": "metadata"},
"codeFlows": [
{
"threadFlows": [
{
"locations": [
{
"location": {
"message": {
"text": f"Flow {link('link', 283)}."
},
"physicalLocation": {
"artifactLocation": {
"uri": "src/example.py"
},
"region": {
"startLine": 1
},
},
}
}
]
}
]
}
],
"relatedLocations": [
{
"id": index,
"physicalLocation": {
"artifactLocation": {
"uri": "src/example.py"
},
"region": {"startLine": index},
},
}
for index in range(1, count + 1)
],
}
],
}
],
}
class CodeQLSarifTests(unittest.TestCase):
def test_escaped_link_syntax_keeps_native_evidence(self):
report = sample_report()
report["runs"][0]["results"][0]["message"] = {
"text": r"Literal source array\[0\](280)."
}
self.assertEqual(
PREPARE.prepare_report(report, source_base_url=SOURCE_BASE), report
)
def test_linked_overflow_uses_source_permalinks_without_new_findings(self):
report = sample_report()
result = report["runs"][0]["results"][0]
result["message"] = {
"text": " ".join(
f"[source {index}]({index})" for index in range(1, 284)
),
"markdown": "See [source](280).",
}
original = copy.deepcopy(report)
prepared = PREPARE.prepare_report(report, source_base_url=SOURCE_BASE)
actual = prepared["runs"][0]["results"][0]
self.assertEqual(len(actual["relatedLocations"]), 100)
expected_text = " ".join(
f"[source {index}]({index})"
if index <= 100
else f"[source {index}]({SOURCE_BASE}/src/example.py#L{index})"
for index in range(1, 284)
)
self.assertEqual(actual["message"]["text"], expected_text)
self.assertEqual(
actual["message"]["markdown"],
f"See [source]({SOURCE_BASE}/src/example.py#L280).",
)
self.assertEqual(len(prepared["runs"][0]["results"]), 1)
for field in (
"ruleId",
"locations",
"partialFingerprints",
"correlationGuid",
):
self.assertEqual(actual[field], result[field])
flow_location = actual["codeFlows"][0]["threadFlows"][0]["locations"][
0
]["location"]
self.assertEqual(
flow_location["physicalLocation"],
result["codeFlows"][0]["threadFlows"][0]["locations"][0][
"location"
]["physicalLocation"],
)
self.assertEqual(
flow_location["message"]["text"],
f"Flow [link]({SOURCE_BASE}/src/example.py#L283).",
)
self.assertEqual(report, original)
def test_source_permalinks_escape_paths_and_preserve_line_ranges(self):
location = {
"physicalLocation": {
"artifactLocation": {
"uri": "src/example%20%28%23%29.py",
"uriBaseId": "%SRCROOT%",
},
"region": {"startLine": 10, "endLine": 12},
}
}
self.assertEqual(
PREPARE._source_link(location, SOURCE_BASE),
f"{SOURCE_BASE}/src/example%20%28%23%29.py#L10-L12",
)
def test_unresolvable_overflow_source_locations_keep_the_native_report(
self,
):
mutations = (
lambda loc: loc["physicalLocation"]["artifactLocation"].update(
uri="../outside.py"
),
lambda loc: loc["physicalLocation"]["artifactLocation"].update(
uri="%2e%2e/outside.py"
),
lambda loc: loc["physicalLocation"]["artifactLocation"].update(
uri="file:///outside.py"
),
lambda loc: loc["physicalLocation"]["artifactLocation"].update(
uriBaseId="external"
),
lambda loc: loc["physicalLocation"].update(region={}),
)
for mutate in mutations:
with self.subTest(mutation=mutate):
report = sample_report(150)
result = report["runs"][0]["results"][0]
result["message"] = {
"text": " ".join(
f"[source]({index})" for index in range(1, 151)
)
}
mutate(result["relatedLocations"][149])
self.assertEqual(
PREPARE.prepare_report(report, source_base_url=SOURCE_BASE),
report,
)
def test_indirect_overflow_links_are_preserved_without_rewriting_shared_templates(
self,
):
report = sample_report(150)
run = report["runs"][0]
run["tool"]["driver"]["globalMessageStrings"] = {
"shared": {
"text": " ".join(
f"[source]({index})" for index in range(1, 151)
)
}
}
run["results"][0]["message"] = {"id": "shared"}
self.assertEqual(
PREPARE.prepare_report(report, source_base_url=SOURCE_BASE), report
)
def test_pr_and_diff_informed_reports_do_not_convert_overflow_links(self):
report = sample_report(150)
report["runs"][0]["results"][0]["message"] = {
"text": " ".join(f"[source]({index})" for index in range(1, 151))
}
self.assertEqual(
PREPARE.prepare_report(
report, preserve_locations=True, source_base_url=SOURCE_BASE
),
report,
)
report["runs"][0]["properties"] = {"incrementalMode": "diff-informed"}
self.assertEqual(
PREPARE.prepare_report(report, source_base_url=SOURCE_BASE), report
)
def test_source_base_must_be_a_commit_pinned_https_repository_url(self):
invalid = (
"https://example.test/example/project/blob/main",
"http://example.test/example/project/blob/" + "a" * 40,
SOURCE_BASE + "?query=example",
SOURCE_BASE + "#fragment",
SOURCE_BASE.replace("example.test", "[email protected]"),
)
for source_base in invalid:
with self.subTest(source_base=source_base):
with self.assertRaises(ValueError):
PREPARE.prepare_report(
sample_report(), source_base_url=source_base
)
def test_pr_preparation_preserves_all_locations_for_diff_filtering(self):
report = sample_report()
self.assertEqual(
PREPARE.prepare_report(report, preserve_locations=True), report
)
def test_diff_informed_reports_keep_all_locations_without_a_caller_flag(
self,
):
report = sample_report()
report["runs"][0]["properties"] = {"incrementalMode": "diff-informed"}
self.assertEqual(PREPARE.prepare_report(report), report)
def test_literal_braces_without_message_arguments_do_not_hide_links(self):
report = sample_report()
report["runs"][0]["results"][0]["message"] = {
"text": "Literal {value} and [source](280)."
}
prepared = PREPARE.prepare_report(report)
locations = prepared["runs"][0]["results"][0]["relatedLocations"]
self.assertEqual(len(locations), 100)
self.assertTrue(
{280, 283}.issubset({location["id"] for location in locations})
)
def test_native_message_ids_and_formatted_arguments_keep_their_links(self):
report = sample_report()
run = report["runs"][0]
run["tool"]["driver"]["globalMessageStrings"] = {
"primary": {"text": "Synthetic [first]({0}) and {1}."},
"flow": {"text": "Flow {0}."},
"source": {"text": "Source location."},
}
result = run["results"][0]
result["message"] = {
"id": "primary",
"arguments": ["280", "[second](279)"],
}
result["codeFlows"][0]["threadFlows"][0]["locations"][0]["location"][
"message"
] = {"id": "flow", "arguments": ["[link](283)"]}
for location in result["relatedLocations"]:
location["message"] = {"id": "source"}
original = copy.deepcopy(report)
prepared = PREPARE.prepare_report(report)
actual = prepared["runs"][0]["results"][0]
self.assertEqual(len(actual["relatedLocations"]), 100)
self.assertTrue(
{279, 280, 283}.issubset(
{loc["id"] for loc in actual["relatedLocations"]}
)
)
self.assertEqual(actual["message"], result["message"])
self.assertEqual(actual["codeFlows"], result["codeFlows"])
self.assertEqual(
actual["partialFingerprints"], result["partialFingerprints"]
)
self.assertEqual(report, original)
def test_ambiguous_rule_message_ids_conservatively_keep_all_referenced_locations(
self,
):
report = sample_report()
run = report["runs"][0]
run["tool"]["driver"]["rules"] = [
{"id": "one", "messageStrings": {"shared": {"text": "[one](200)"}}},
{"id": "two", "messageStrings": {"shared": {"text": "[two](201)"}}},
]
run["results"][0]["message"] = {"id": "shared"}
prepared = PREPARE.prepare_report(report)
locations = prepared["runs"][0]["results"][0]["relatedLocations"]
self.assertEqual(len(locations), 100)
self.assertTrue(
{200, 201, 283}.issubset({location["id"] for location in locations})
)
def test_overflow_keeps_links_identities_findings_and_flow_data(self):
report = sample_report()
original = copy.deepcopy(report)
prepared = PREPARE.prepare_report(report)
result = prepared["runs"][0]["results"][0]
retained = {location["id"] for location in result["relatedLocations"]}
self.assertEqual(retained, set(range(1, 96)) | set(range(279, 284)))
expected = copy.deepcopy(original)
expected["runs"][0]["results"][0]["relatedLocations"] = [
location
for location in original["runs"][0]["results"][0][
"relatedLocations"
]
if location["id"] in retained
]
self.assertEqual(prepared, expected)
self.assertEqual(report, original)
def test_valid_reports_below_the_limit_are_unchanged(self):
for count in (0, 20, 100):
with self.subTest(count=count):
report = sample_report(count)
report["runs"][0]["results"][0]["message"] = {
"text": "Synthetic result."
}
self.assertEqual(PREPARE.prepare_report(report), report)
def test_more_than_one_hundred_referenced_locations_are_not_removed(self):
report = sample_report(150)
report["runs"][0]["results"][0]["message"] = {
"text": " ".join(f"[location]({index})" for index in range(1, 102))
}
self.assertEqual(PREPARE.prepare_report(report), report)
def test_indirect_messages_relationships_and_markdown_references_are_preserved(
self,
):
mutations = (
lambda result: result.update(message={"id": "sample-message"}),
lambda result: result["relatedLocations"][0].update(
relationships=[{"target": 200, "kinds": ["relevant"]}]
),
lambda result: result["message"].update(
markdown="[link][ref]\n[ref]: 200"
),
)
for mutate in mutations:
with self.subTest(mutation=mutate):
report = sample_report()
mutate(report["runs"][0]["results"][0])
self.assertEqual(PREPARE.prepare_report(report), report)
def test_multiple_runs_and_results_are_preserved(self):
report = sample_report()
report["runs"].append(copy.deepcopy(report["runs"][0]))
report["runs"][1]["results"].append(
{"ruleId": "other", "message": {"text": "Other."}}
)
prepared = PREPARE.prepare_report(report)
self.assertEqual(len(prepared["runs"]), 2)
self.assertEqual(
[len(run["results"]) for run in prepared["runs"]], [1, 2]
)
self.assertEqual(
prepared["runs"][1]["results"][1], report["runs"][1]["results"][1]
)
def test_invalid_or_unsuccessful_native_reports_are_rejected(self):
mutations = (
lambda r: r.update(version="1.0.0"),
lambda r: r.update(runs=[]),
lambda r: r["runs"][0].update(tool="invalid"),
lambda r: r["runs"][0]["tool"]["driver"].update(name="other"),
lambda r: r["runs"][0].pop("results"),
lambda r: r["runs"][0].update(results=None),
lambda r: r["runs"][0].update(results=[None]),
lambda r: r["runs"][0].update(
invocations=[{"executionSuccessful": False}]
),
lambda r: r["runs"][0].update(invocations=[{}]),
lambda r: r["runs"][0]["results"][0].update(
relatedLocations=[None] * 101
),
lambda r: r["runs"][0]["results"][0]["relatedLocations"][1].update(
id=1
),
lambda r: r["runs"][0]["results"][0]["relatedLocations"][1].update(
id=True
),
)
for mutate in mutations:
with self.subTest(mutation=mutate):
report = sample_report()
mutate(report)
with self.assertRaises(ValueError):
PREPARE.prepare_report(report)
def test_preparation_preserves_native_bytes_and_rejects_in_place_writes(
self,
):
with tempfile.TemporaryDirectory() as temporary:
source = Path(temporary) / "native.sarif"
destination = Path(temporary) / "upload.sarif"
raw = json.dumps(sample_report())
source.write_text(raw, encoding="utf-8")
PREPARE.prepare(source, destination)
self.assertEqual(source.read_text(encoding="utf-8"), raw)
self.assertEqual(
len(
json.loads(destination.read_text(encoding="utf-8"))["runs"][
0
]["results"][0]["relatedLocations"]
),
100,
)
with self.assertRaises(ValueError):
PREPARE.prepare(source, source)
def test_cli_handles_the_native_output_directory(self):
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
source, destination = root / "native", root / "upload"
source.mkdir()
raw = json.dumps(sample_report())
(source / "python.sarif").write_text(raw, encoding="utf-8")
result = subprocess.run(
[sys.executable, str(SCRIPT), str(source), str(destination)],
capture_output=True,
text=True,
check=False,
)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(
(source / "python.sarif").read_text(encoding="utf-8"), raw
)
prepared = json.loads(
(destination / "python.sarif").read_text(encoding="utf-8")
)
self.assertEqual(
len(prepared["runs"][0]["results"][0]["relatedLocations"]), 100
)
def test_nonstandard_json_and_missing_input_never_create_an_upload(self):
with tempfile.TemporaryDirectory() as temporary:
source, destination = (
Path(temporary) / "native.sarif",
Path(temporary) / "upload.sarif",
)
with self.assertRaises(FileNotFoundError):
PREPARE.prepare(source, destination)
for raw in (
"{truncated",
json.dumps(sample_report()).replace(
'"version": "2.1.0"', '"version": NaN'
),
):
with self.subTest(raw=raw[:20]):
source.write_text(raw, encoding="utf-8")
with self.assertRaises(ValueError):
PREPARE.prepare(source, destination)
self.assertFalse(destination.exists())
if __name__ == "__main__":
unittest.main()