~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Contracts for the Semgrep scan's report integrity.
A partial or filtered Semgrep report uploaded to code scanning silently
resolves existing alerts. These tests pin the known-good workflow wiring
that prevents that: the workflow has only the pinned top-level keys and the
single ``semgrep-scan`` job, the job runs exactly the pinned steps in order
with pinned action inputs and no job container or services, one
``semgrep scan`` with a fixed command (every configured ruleset, every
severity, ``src/``, no baseline or exclusions) writes the reports, no
``.semgrepignore`` exists at the root or under ``src/``, no tracked file
under ``src/`` matches Semgrep's built-in default ignores, no tracked entry
under ``src/`` is a symlink or a submodule, and none is above
Semgrep's default 1,000,000-byte ``--max-target-bytes`` cap (Semgrep
silently skips a symlink or an oversized file the same way it skips a
default-ignored path), scanner errors fail the job, the raw reports are
validated before deriving a GitHub upload copy that honors source suppressions,
a failed step is never followed
by an upload, and a pull request touching ``src/`` runs the scan (so a file
Semgrep cannot parse fails that PR rather than the next release).
Scope: these are regression tests. They guard against accidental edits of
the kinds seen so far (masking the exit status with ``|| true``, filtering
by severity, narrowing the scanned rules or files) by matching the pinned
text exactly rather than looking for forbidden patterns. A deliberately
adversarial edit to the workflow is outside what static tests can
guarantee; that is covered by the required CODEOWNERS review of
``.github/workflows/semgrep.yml`` and ``.semgrepignore``.
"""
import fnmatch
import json
import re
import shlex
import subprocess
from pathlib import Path
import pytest
import yaml
from packaging.requirements import Requirement
from packaging.utils import canonicalize_name
from packaging.version import Version
REPO_ROOT = Path(__file__).resolve().parents[2]
WORKFLOW = REPO_ROOT / ".github" / "workflows" / "semgrep.yml"
VALIDATOR = ".github/scripts/check_semgrep_report.py"
JSON_REPORT = "semgrep-results.json"
SARIF_REPORT = "semgrep-results.sarif"
GITHUB_SARIF_REPORT = "semgrep-results.github.sarif"
VALIDATOR_RUN = (
f"python {VALIDATOR} {JSON_REPORT} {SARIF_REPORT}"
f" --github-sarif-output {GITHUB_SARIF_REPORT}"
)
VERIFY_RUN = (
"python -m unittest discover -s tests/ci/standalone"
" -p 'test_semgrep*.py' -v"
)
SQL_CONTROL_RUN = (
"semgrep --test --config .semgrep/rules/ldr-security.yaml"
" .semgrep/tests/ldr-security.py"
)
INSTALL_RUN = (
"python -m pip install --require-hashes --only-binary=:all:"
" -r requirements/semgrep.txt"
)
# The complete scan argv. Changing the worker count can destabilize taint
# findings; options such as ``--exclude``, ``--exclude-rule``, ``--include``,
# ``--baseline-commit``, ``--max-target-bytes`` or a dropped ``--config``
# narrow the report, which the validator cannot detect.
SCAN_ARGV = [
"semgrep",
"scan",
"--config=p/security-audit",
"--config=p/secrets",
"--config=.semgrep/rules/",
"--jobs=1",
"--strict",
"--metrics=off",
f"--json-output={JSON_REPORT}",
f"--sarif-output={SARIF_REPORT}",
"src/",
]
# The scan step's exact ``run`` text. It is compared verbatim, as the other
# ``run`` steps are, because a tokenizer can disagree with bash (e.g. a
# ``#\\`` line is one comment to bash but hides the next line from shlex).
SCAN_RUN = (
f"{SCAN_ARGV[0]} {SCAN_ARGV[1]} \\\n"
+ "".join(f" {arg} \\\n" for arg in SCAN_ARGV[2:-1])
+ f" {SCAN_ARGV[-1]}\n"
)
# The job's steps, in order. Actions are matched by name (the pinned SHA
# after ``@`` may change); ``run`` steps by their exact command.
ACTION_STEPS = {
"harden-runner": "step-security/harden-runner",
"checkout": "actions/checkout",
"setup-python": "actions/setup-python",
"upload-sarif": "github/codeql-action/upload-sarif",
"upload-artifact": "actions/upload-artifact",
}
STEP_ORDER = [
"harden-runner",
"checkout",
"setup-python",
"install",
"verify",
"test-sql-rules",
"scan",
"validate",
"upload-sarif",
"upload-artifact",
]
UPLOAD_WITH = {
"sarif_file": GITHUB_SARIF_REPORT,
"category": "semgrep-security",
"wait-for-processing": True,
}
# Every action step's complete ``with``. For checkout, a ``sparse-checkout``,
# ``ref``, ``repository`` or ``path`` would scan a narrower or different tree
# and upload it as this commit's analysis.
ACTION_WITH = {
"harden-runner": {"egress-policy": "audit"},
"checkout": {"persist-credentials": False},
"setup-python": {"python-version": "3.12"},
"upload-sarif": UPLOAD_WITH,
"upload-artifact": {
"name": "semgrep-scan-results",
"path": f"{JSON_REPORT}\n{SARIF_REPORT}\n{GITHUB_SARIF_REPORT}\n",
"retention-days": 7,
},
}
# Job-level keys. ``container`` or ``services`` would run the steps in, or
# next to, an image the contract does not see; ``defaults``, ``env``, ``if``
# and ``continue-on-error`` change how or whether the steps run.
JOB_KEYS = {"runs-on", "timeout-minutes", "permissions", "steps"}
# Workflow-level keys. PyYAML (YAML 1.1) loads the ``on`` key as ``True``.
# ``defaults``, ``env`` or ``concurrency`` would change how, or whether, the
# job runs.
WORKFLOW_KEYS = {"name", True, "permissions", "jobs"}
# Semgrep 1.181.0's built-in ignore patterns, which apply when no
# ``.semgrepignore`` exists (as test_no_semgrepignore_in_the_scanned_tree
# requires). Files or directories with these names, at any depth:
DEFAULT_IGNORED_NAMES = {".git", ".svn", ".hg", "_darcs", "CVS"}
# directories with these names, at any depth:
DEFAULT_IGNORED_DIRS = {
"build",
"vendor",
"dist",
".env",
".tox",
"node_modules",
".npm",
".yarn",
".venv",
"_opam",
"_build",
"_cargo",
"test",
"tests",
"testsuite",
}
# and files matching these patterns.
DEFAULT_IGNORED_FILES = ("*.min.js", "*_test.go")
# Semgrep 1.181.0's default ``--max-target-bytes``: a tracked file above
# this size is silently skipped, the same as a default-ignored path.
MAX_TARGET_BYTES = 1_000_000
# ``run`` steps may carry nothing that changes how their command runs:
# no ``shell``, ``env`` (Semgrep reads e.g. SEMGREP_BASELINE_COMMIT and
# SEMGREP_RULES), ``working-directory``, ``if`` or ``continue-on-error``.
RUN_STEP_KEYS = {"name", "id", "run"}
ACTION_STEP_KEYS = {"name", "id", "uses", "with", "if"}
SEMGREP_EXECUTABLES = {"semgrep", "pysemgrep", "osemgrep", "semgrep-core"}
STATUS_FUNCTIONS = ("success(", "always(", "failure(", "cancelled(")
def _workflow() -> dict:
return yaml.safe_load(WORKFLOW.read_text(encoding="utf-8"))
def _job() -> dict:
return _workflow()["jobs"]["semgrep-scan"]
def _steps() -> list[dict]:
return _job()["steps"]
def _run_tokens(step: dict) -> list[str]:
"""Shell tokens of a ``run``; operators become their own tokens.
Only used to detect Semgrep invocations, never to accept a command.
``#`` is kept as a word character so no text is hidden as a comment,
and backslash-newline is removed as bash removes it.
"""
run = str(step.get("run", "")).replace("\\\n", "")
lexer = shlex.shlex(run, posix=True, punctuation_chars=True)
lexer.whitespace_split = True
lexer.commenters = ""
return list(lexer)
def _invokes_semgrep(step: dict) -> bool:
"""Whether the step runs the Semgrep scanner (not just installs it)."""
if "semgrep" in str(step.get("uses", "")).lower():
return True
tokens = _run_tokens(step)
for i, token in enumerate(tokens):
if token.rsplit("/", 1)[-1] in SEMGREP_EXECUTABLES:
return True
if token == "-m" and tokens[i + 1 : i + 2] == ["semgrep"]:
return True
return False
def _scan_step(steps: list[dict]) -> dict:
invocations = [step for step in steps if _invokes_semgrep(step)]
# Only this exact rule-test command may accompany the production scan.
# It has no report output flags; altered or additional invocations fail.
controls = [
step
for step in invocations
if str(step.get("run", "")).strip() == SQL_CONTROL_RUN
]
assert len(controls) == 1, [step.get("name") for step in controls]
scans = [step for step in invocations if step not in controls]
assert len(scans) == 1, [step.get("name") for step in scans]
return scans[0]
def _scan_tokens(steps: list[dict]) -> list[str]:
return _run_tokens(_scan_step(steps))
def _step_kind(step: dict) -> str:
uses = step.get("uses")
if uses is not None:
action = str(uses).split("@", 1)[0]
for kind, name in ACTION_STEPS.items():
if action == name:
return kind
return f"unknown action {uses!r}"
run = str(step.get("run", "")).strip()
if run == INSTALL_RUN:
return "install"
if run == VERIFY_RUN:
return "verify"
if run == SQL_CONTROL_RUN:
return "test-sql-rules"
if run == VALIDATOR_RUN:
return "validate"
if step.get("run") == SCAN_RUN:
return "scan"
return f"unknown run {run!r}"
def _index(steps: list[dict], kind: str) -> int:
kinds = [_step_kind(step) for step in steps]
assert kinds.count(kind) == 1, kinds
return kinds.index(kind)
def _is_upload_sarif(step: dict) -> bool:
return _step_kind(step) == "upload-sarif"
def _upload_condition(steps: list[dict]) -> str:
condition = " ".join(
str(steps[_index(steps, "upload-sarif")].get("if", "")).split()
)
if condition.startswith("${{") and condition.endswith("}}"):
condition = condition[3:-2].strip()
return condition
# --- A minimal GitHub Actions expression evaluator ------------------------
# Supports the subset the upload condition needs: context paths, string
# literals, the ``true``/``false``/``null`` literals, ==, !=, !, &&, || and
# parentheses, with GitHub's semantics for each. Anything else (notably
# function calls and number literals) is rejected, and so is a path whose
# first segment is not a context of the evaluated run (GitHub rejects an
# unrecognized named-value too), so the condition cannot silently grow a
# status function, a literal or other logic the truth-table test does not
# model. Lookups are strict as well: a property the truth-table context does
# not model raises instead of evaluating to null, because on GitHub such a
# field (``github.actor``, ``github.run_id``, ``pull_request.number``) is
# usually set and truthy, and a null stand-in would let an appended
# ``|| github.actor`` pass the table while making fork PRs upload. Only
# dereferencing a modelled null (a deleted fork's ``head.repo``) yields
# null, as on GitHub. && and || short-circuit as on GitHub: an operand they
# skip is parsed but its properties are not looked up, so a push run need
# not model ``github.event.pull_request``.
_EXPR_TOKEN = re.compile(
r"\s*(?:(?P<op>\|\||&&|==|!=|!|\(|\))"
r"|(?P<str>'(?:[^']|'')*')"
r"|(?P<bool>(?:true|false)(?![A-Za-z0-9_.-]))"
r"|(?P<null>null(?![A-Za-z0-9_.-]))"
r"|(?P<path>[A-Za-z_][A-Za-z0-9_-]*(?:\.[A-Za-z_][A-Za-z0-9_-]*)*))"
)
# The number formats GitHub accepts when it coerces a string to a number.
_NUMBER = re.compile(r"[+-]?(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?")
def _lex(expression: str) -> list[tuple[str, str]]:
tokens, pos = [], 0
expression = expression.strip()
while pos < len(expression):
match = _EXPR_TOKEN.match(expression, pos)
if not match or match.end() == pos:
raise ValueError(f"unsupported expression at {expression[pos:]!r}")
kind = match.lastgroup
tokens.append((kind, match.group(kind)))
pos = match.end()
return tokens
def _lookup(context: dict, path: str):
root, *rest = path.split(".")
if root not in context:
raise ValueError(f"unrecognized named-value: {root!r}")
value = context[root]
for depth, part in enumerate(rest, start=1):
if value is None:
return None # GitHub: a property of null is null
if not isinstance(value, dict) or part not in value:
prefix = ".".join([root, *rest[:depth]])
raise ValueError(f"unmodelled context property: {prefix!r}")
value = value[part]
return value
def _to_number(value) -> float:
# GitHub's coercion when == or != compares values of different types.
if value is None:
return 0.0
if isinstance(value, bool):
return 1.0 if value else 0.0
if isinstance(value, (int, float)):
return float(value)
if isinstance(value, str):
text = value.strip()
if not text:
return 0.0
return float(text) if _NUMBER.fullmatch(text) else float("nan")
return float("nan") # objects and arrays
def _truthy(value) -> bool:
# GitHub's falsy values: false, 0, -0, '', null and NaN.
if value is None or value is False or value == "":
return False
if isinstance(value, (int, float)) and not isinstance(value, bool):
return value == value and value != 0
return True
def _equal(left, right) -> bool:
# Same type: strings compare case-insensitively, objects by identity.
# Different types: both are coerced to numbers (NaN equals nothing), so
# e.g. null == false and null == '' are true, true == 'true' is false.
if isinstance(left, str) and isinstance(right, str):
return left.lower() == right.lower()
if isinstance(left, bool) and isinstance(right, bool):
return left is right
if left is None and right is None:
return True
if isinstance(left, (dict, list)) or isinstance(right, (dict, list)):
return left is right
return _to_number(left) == _to_number(right)
def _evaluate(expression: str, context: dict):
tokens = _lex(expression)
pos = 0
# Depth of operands that && / || short-circuit past. GitHub does not
# evaluate them, so their paths are parsed but not looked up.
skipping = 0
def peek():
return tokens[pos] if pos < len(tokens) else (None, None)
def take(value=None):
nonlocal pos
token = peek()
if token[0] is None or (value is not None and token[1] != value):
raise ValueError(f"expected {value!r}, got {token[1]!r}")
pos += 1
return token
def primary():
kind, value = peek()
if value == "(":
take("(")
result = disjunction()
take(")")
return result
if value == "!":
take("!")
return not _truthy(primary())
take()
if kind == "str":
return value[1:-1].replace("''", "'")
if kind == "bool":
return value == "true"
if kind == "null":
return None
if kind == "path":
if peek()[1] == "(":
raise ValueError(f"function calls are unsupported: {value}")
if skipping: # GitHub still rejects an unknown named-value
_lookup(context, value.split(".")[0])
return None
return _lookup(context, value)
raise ValueError(f"unexpected token {value!r}")
def operand(parse, evaluated: bool):
nonlocal skipping
skipping += not evaluated
try:
return parse()
finally:
skipping -= not evaluated
def comparison():
left = primary()
while peek()[1] in ("==", "!="):
op = take()[1]
right = primary()
left = _equal(left, right) == (op == "==")
return left
def conjunction():
left = comparison()
while peek()[1] == "&&":
take("&&")
right = operand(comparison, _truthy(left))
left = right if _truthy(left) else left
return left
def disjunction():
left = conjunction()
while peek()[1] == "||":
take("||")
right = operand(conjunction, not _truthy(left))
left = left if _truthy(left) else right
return left
result = disjunction()
if pos != len(tokens):
raise ValueError(f"trailing tokens: {tokens[pos:]}")
return result
REPOSITORY = "LearningCircuit/local-deep-research"
FORK = "someone/local-deep-research"
def _pr_context(head_repo) -> dict:
"""A ``pull_request`` run; every field the pinned condition reads is set.
``_lookup`` raises on any other property, so the table stays honest only
if each context models what the condition references, with the value
GitHub would supply for that case.
"""
return {
"github": {
"event_name": "pull_request",
"repository": REPOSITORY,
"event": {"pull_request": {"head": {"repo": head_repo}}},
}
}
def _head_repo(full_name: str) -> dict:
# ``fork`` is GitHub's flag for a head repository other than the base.
return {
"full_name": full_name,
"fork": full_name.lower() != REPOSITORY.lower(),
}
# (description, context, upload expected)
UPLOAD_CASES = [
("same-repo PR", _pr_context(_head_repo(REPOSITORY)), True),
(
"same-repo PR, different case",
_pr_context(_head_repo(REPOSITORY.lower())),
True,
),
("fork PR", _pr_context(_head_repo(FORK)), False),
("deleted-fork PR (head.repo null)", _pr_context(None), False),
] + [
(
f"{event} run",
{
"github": {
"event_name": event,
"repository": REPOSITORY,
"event": {},
}
},
True,
)
# Under workflow_call, event_name is the caller's event.
for event in ("workflow_call", "schedule", "workflow_dispatch", "push")
]
def _locked_versions(lock_text: str) -> dict[str, list[str]]:
"""Map each ``name==version`` line in a pip-compile lock to its versions."""
locked: dict[str, list[str]] = {}
for match in re.finditer(
r"(?m)^([A-Za-z0-9][A-Za-z0-9._-]*)==([^\s;\\]+)", lock_text
):
locked.setdefault(canonicalize_name(match[1]), []).append(match[2])
return locked
def test_scanner_lock_matches_its_input_roots():
"""``semgrep.txt`` locks exactly the Semgrep that ``semgrep.in`` pins.
The input has exactly two roots: Semgrep pinned with ``==`` (so the job
installs one known release) and PyJWT with a ``>=`` security floor. The
hash lock must lock that same Semgrep release once and a PyJWT release
that satisfies the floor, so a hand edit of one file, or a regeneration
that went wrong, fails. Versions are read from the files rather than
repeated here, so a Dependabot bump that updates both files passes.
This does not prove a new release keeps the built-in ignores and
``--max-target-bytes`` default modelled in ``DEFAULT_IGNORED_*`` and
``MAX_TARGET_BYTES``. On a bump the scan job still runs the custom rule
tests (``semgrep --test``), the installed-scanner integration tests in
``tests/ci/standalone/test_semgrep*.py``, and the report validator,
which fails on scanner errors, skipped rules or an empty scanned-file
list. None of those notices a newly default-ignored ``src/`` path, so
re-checking that model remains part of reviewing a Semgrep update.
The model was reviewed for the currently locked 1.181.0 release on
2026-10-22. Across v1.179.0/v1.180.0/v1.181.0,
``default.semgrepignore``, ``Find_targets.ml``, ``Skip_target.ml``,
``Semgrepignore.ml``, ``target_manager.py``, ``libs/paths/Project.ml``
and the default-ignore snapshot were unchanged; the target cap remains
1,000,000 bytes with the same ``size > max_bytes`` boundary. That review
also covered the changed gitignore/glob/PCRE2 wrappers, language
guessing, Git wrapper and ``Nosemgrep.ml`` recognizer before accepting
the unchanged model. Future release reviews must assess those paths as
well as the default patterns; matching the two lock inputs alone does
not establish that behavior.
"""
tools = REPO_ROOT / "requirements"
_assert_scanner_lock_matches(
(tools / "semgrep.in").read_text(), (tools / "semgrep.txt").read_text()
)
def _assert_scanner_lock_matches(inputs: str, lock: str) -> None:
roots = [
Requirement(line.strip())
for line in inputs.splitlines()
if line.strip() and not line.lstrip().startswith("#")
]
assert [canonicalize_name(root.name) for root in roots] == [
"semgrep",
"pyjwt",
]
semgrep, pyjwt = roots
for root in roots:
assert not root.url and not root.extras and root.marker is None, root
(pin,) = semgrep.specifier
assert pin.operator == "==" and "*" not in pin.version, semgrep
(floor,) = pyjwt.specifier
assert floor.operator == ">=", pyjwt
assert Version(floor.version) >= Version("2.15.1"), pyjwt
locked = _locked_versions(lock)
assert locked.get("semgrep") == [pin.version], locked.get("semgrep")
assert len(locked.get("pyjwt", [])) == 1, locked.get("pyjwt")
assert pyjwt.specifier.contains(locked["pyjwt"][0]), locked["pyjwt"]
_GOOD_INPUTS = "# header\nsemgrep==9.0.0\n# note\npyjwt>=2.15.1\n"
_GOOD_LOCK = (
"pyjwt==2.16.0 \\\n --hash=sha256:00\n"
"semgrep==9.0.0 \\\n --hash=sha256:11\n"
)
def test_scanner_lock_check_accepts_a_consistent_synthetic_pair():
_assert_scanner_lock_matches(_GOOD_INPUTS, _GOOD_LOCK)
def test_scanner_lock_check_accepts_a_raised_pyjwt_floor():
_assert_scanner_lock_matches(
_GOOD_INPUTS.replace("2.15.1", "2.17.0"),
_GOOD_LOCK.replace("2.16.0", "2.17.1"),
)
@pytest.mark.parametrize(
("inputs", "lock"),
[
pytest.param(
_GOOD_INPUTS,
_GOOD_LOCK.replace("9.0.0", "9.0.1"),
id="lock-mismatch",
),
pytest.param(
_GOOD_INPUTS.replace("9.0.0", "9.0.1"),
_GOOD_LOCK,
id="input-mismatch",
),
pytest.param(
_GOOD_INPUTS,
_GOOD_LOCK + "semgrep==9.0.0\n",
id="duplicate-semgrep",
),
pytest.param(
_GOOD_INPUTS, _GOOD_LOCK + "pyjwt==2.2.0\n", id="duplicate-pyjwt"
),
pytest.param(
_GOOD_INPUTS,
_GOOD_LOCK.replace("semgrep==", "semgrep>="),
id="semgrep-unlocked",
),
pytest.param(
_GOOD_INPUTS,
_GOOD_LOCK.replace("2.16.0", "2.15.0"),
id="pyjwt-below-floor",
),
pytest.param(
_GOOD_INPUTS.replace("2.15.1", "2.15.0"),
_GOOD_LOCK,
id="pyjwt-lower-input-floor",
),
pytest.param(
_GOOD_INPUTS.replace("2.15.1", "2.15.0"),
_GOOD_LOCK.replace("2.16.0", "2.15.0"),
id="pyjwt-lower-floor-and-lock",
),
pytest.param(
_GOOD_INPUTS,
_GOOD_LOCK.replace("pyjwt==2.16.0 \\\n --hash=sha256:00\n", ""),
id="pyjwt-missing",
),
pytest.param(
_GOOD_INPUTS.replace(
"semgrep==9.0.0", 'semgrep==9.0.0; sys_platform == "linux"'
),
_GOOD_LOCK,
id="marker",
),
pytest.param(
_GOOD_INPUTS.replace("semgrep==9.0.0", "semgrep[extra]==9.0.0"),
_GOOD_LOCK,
id="extras",
),
pytest.param(
_GOOD_INPUTS.replace("semgrep==", "semgrep>="),
_GOOD_LOCK,
id="semgrep-range",
),
pytest.param(
_GOOD_INPUTS.replace("9.0.0", "9.0.*"),
_GOOD_LOCK,
id="semgrep-wildcard",
),
pytest.param(
_GOOD_INPUTS.replace("pyjwt>=", "pyjwt=="),
_GOOD_LOCK,
id="pyjwt-pinned",
),
pytest.param(
_GOOD_INPUTS + "requests==2.0\n", _GOOD_LOCK, id="extra-root"
),
pytest.param(
_GOOD_INPUTS.replace("# note\npyjwt>=2.15.1\n", ""),
_GOOD_LOCK,
id="missing-root",
),
],
)
def test_scanner_lock_check_rejects_inconsistent_pairs(inputs, lock):
with pytest.raises(AssertionError):
_assert_scanner_lock_matches(inputs, lock)
def test_scan_has_no_severity_filter():
"""``--severity`` selects exact levels; any use drops the other levels."""
assert not [t for t in _scan_tokens(_steps()) if "--severity" in t]
def test_exactly_one_step_runs_the_production_semgrep_scan():
"""A second scan (e.g. INFO-only ``|| true``) could overwrite the reports."""
steps = _steps()
assert _scan_step(steps) is steps[_index(steps, "scan")]
def test_scan_command_is_pinned():
"""The scan ``run`` is exactly the pinned text, compared verbatim.
So an accidental ``|| true``, ``set +e``, extra option, dropped ruleset,
comment or second command in the scan step fails.
"""
steps = _steps()
assert _scan_step(steps)["run"] == SCAN_RUN
assert _scan_tokens(steps) == SCAN_ARGV
def test_action_inputs_are_pinned():
"""Each action step's ``with`` is exactly the expected inputs."""
steps = _steps()
for kind, expected in ACTION_WITH.items():
assert steps[_index(steps, kind)].get("with") == expected, kind
def test_workflow_has_only_the_semgrep_scan_job():
"""A sibling job could run a narrowed or exit-masked scan.
Uploaded to the same ``semgrep-security`` category and ref, its report
would replace the full scan's analysis and resolve open alerts, and the
release gate calls this whole workflow.
"""
workflow = _workflow()
assert set(workflow) == WORKFLOW_KEYS, set(workflow) ^ WORKFLOW_KEYS
assert set(workflow["jobs"]) == {"semgrep-scan"}, set(workflow["jobs"])
def _default_ignored(path: str) -> bool:
*dirs, name = path.split("/")
return (
any(part in DEFAULT_IGNORED_NAMES for part in (*dirs, name))
or any(part in DEFAULT_IGNORED_DIRS for part in dirs)
or any(
fnmatch.fnmatchcase(part, pat)
for part in (*dirs, name)
for pat in DEFAULT_IGNORED_FILES
)
)
def test_no_default_ignored_paths_under_src():
"""No tracked file under ``src/`` matches Semgrep's built-in ignores.
With no ``.semgrepignore``, Semgrep 1.181.0 silently skips e.g. any
``tests/``, ``build/``, ``dist/`` or ``vendor/`` directory,
``node_modules/`` and ``*.min.js`` files at any depth, and the narrowed
report still validates. None is tracked under ``src/`` today; adding one
must move it or deliberately change this test and how the scan covers
it. Tracked files are what the scan job's fresh checkout contains; local
build output (e.g. the gitignored ``web/static/dist/``) is not.
"""
listed = subprocess.run(
["git", "ls-files", "-z", "--", "src"],
cwd=REPO_ROOT,
capture_output=True,
check=False,
)
assert listed.returncode == 0, listed.stderr
tracked = [path for path in listed.stdout.decode().split("\0") if path]
assert tracked, "git ls-files listed nothing under src/"
found = [path for path in tracked if _default_ignored(path)]
assert not found, found
def test_no_oversized_or_special_entries_under_src():
"""No tracked ``src/`` entry is a symlink, a submodule, or too big.
With no ``--max-target-bytes`` override, Semgrep 1.181.0 silently skips
any tracked file above its default 1,000,000-byte cap (exit 0, no
error, no skipped-rule entry) the same way it silently skips a
default-ignored path, and the narrowed report still validates. It also
silently skips a tracked symlink. A submodule's gitlink entry is not a
blob Semgrep could scan at all. ``git ls-files -s`` reports a symlink as
mode ``120000`` and a submodule as mode ``160000``; every other tracked
entry is a regular (mode ``100644``/``100755``) blob whose size is
checked with ``git cat-file -s``. None is tracked under ``src/`` today
(the largest tracked file is well under the cap); adding one that trips
this must shrink it, exclude it from ``src/``, or deliberately change
this test and how the scan covers it.
"""
listed = subprocess.run(
["git", "ls-files", "-s", "-z", "--", "src"],
cwd=REPO_ROOT,
capture_output=True,
check=False,
)
assert listed.returncode == 0, listed.stderr
entries = [entry for entry in listed.stdout.decode().split("\0") if entry]
assert entries, "git ls-files listed nothing under src/"
symlinks = []
submodules = []
oversized = []
for entry in entries:
meta, path = entry.split("\t", 1)
mode, sha, _stage = meta.split(" ")
if mode == "120000":
symlinks.append(path)
continue
if mode == "160000":
submodules.append(path)
continue
sized = subprocess.run(
["git", "cat-file", "-s", sha],
cwd=REPO_ROOT,
capture_output=True,
check=False,
)
assert sized.returncode == 0, sized.stderr
if int(sized.stdout) > MAX_TARGET_BYTES:
oversized.append(path)
assert not symlinks, symlinks
assert not submodules, submodules
assert not oversized, oversized
def test_job_has_only_the_pinned_keys():
"""No container, services, defaults, env, ``if`` or continue-on-error."""
job = _job()
assert set(job) == JOB_KEYS, set(job) ^ JOB_KEYS
assert job["runs-on"] == "ubuntu-latest"
def test_no_semgrepignore_in_the_scanned_tree():
"""No ``.semgrepignore`` at the repo root or anywhere under ``src/``.
Semgrep 1.181.0 honours a ``.semgrepignore`` at the root and in any
scanned subdirectory, and silently drops the matching files, so the
narrowed report still validates. None exists; false positives are
suppressed per line with ``nosemgrep``, which stays next to the code it
suppresses. Adding an ignore file must change this test too.
"""
found = [
*REPO_ROOT.glob(".semgrepignore"),
*(REPO_ROOT / "src").rglob(".semgrepignore"),
]
assert not found, found
def test_job_runs_only_the_pinned_steps():
"""Any other step could rewrite, filter or replace the reports."""
assert [_step_kind(step) for step in _steps()] == STEP_ORDER
def test_only_the_scan_writes_the_reports():
"""The validator preserves raw reports and creates a separate upload copy."""
steps = _steps()
referencing = [
_step_kind(step)
for step in steps
if any(
name in str(step.get("run", "")) + json.dumps(step.get("with"))
for name in (
"semgrep-results",
JSON_REPORT,
SARIF_REPORT,
GITHUB_SARIF_REPORT,
)
)
]
assert referencing == [
"scan",
"validate",
"upload-sarif",
"upload-artifact",
]
def test_no_step_or_default_can_mask_failure():
"""No setting may mask a step's exit status or alter how it runs."""
workflow = _workflow()
job = _job()
# A default shell such as ``bash -c '{0} || true'`` masks every run step;
# a default working directory or env changes what the scan covers.
for scope in (workflow, job):
assert "defaults" not in scope
assert "env" not in scope
assert job.get("continue-on-error") in (None, False)
for step in _steps():
allowed = RUN_STEP_KEYS if "run" in step else ACTION_STEP_KEYS
assert set(step) <= allowed, (step.get("name"), set(step) - allowed)
assert step.get("continue-on-error") in (None, False)
for kind in (
"harden-runner",
"checkout",
"setup-python",
"test-sql-rules",
"scan",
):
assert "if" not in _steps()[_index(_steps(), kind)]
def test_reports_are_validated_immediately_before_upload():
steps = _steps()
scan = _index(steps, "scan")
validate = _index(steps, "validate")
upload = _index(steps, "upload-sarif")
assert scan < validate
# Nothing may touch the SARIF between validation and upload.
assert upload == validate + 1
# Exactly the validator command: nothing may suppress its exit status.
assert steps[validate]["run"].strip() == VALIDATOR_RUN
def test_upload_is_the_validated_sarif():
steps = _steps()
upload_with = steps[_index(steps, "upload-sarif")]["with"]
# wait-for-processing surfaces processing errors; no ref/sha/path
# overrides may redirect the upload.
assert upload_with == UPLOAD_WITH
validated = shlex.split(steps[_index(steps, "validate")]["run"])
# Upload only the validator's dedicated output; retain the native input.
assert validated[2:4] == [JSON_REPORT, SARIF_REPORT]
assert validated[-2:] == ["--github-sarif-output", GITHUB_SARIF_REPORT]
assert upload_with["sarif_file"] == GITHUB_SARIF_REPORT
assert GITHUB_SARIF_REPORT not in (JSON_REPORT, SARIF_REPORT)
assert f"--sarif-output={SARIF_REPORT}" in _scan_tokens(steps)
def test_upload_runs_only_after_success():
"""No status function at all, so GitHub's implicit ``success()`` applies.
Any explicit status function (even ``success() || ...``) replaces the
implicit one and can upload the report of a failed scan or validation.
"""
condition = "".join(_upload_condition(_steps()).split()).lower()
for status in STATUS_FUNCTIONS:
assert status not in condition
def test_fork_pull_requests_do_not_upload():
"""Fork PR tokens are read-only; every other run must upload."""
condition = _upload_condition(_steps())
for description, context, expected in UPLOAD_CASES:
assert _truthy(_evaluate(condition, context)) is expected, description
def test_expression_evaluator_models_literals():
"""Literals evaluate as GitHub evaluates them, not as context paths.
Otherwise an appended ``|| true`` would read an absent context, evaluate
to null and pass the truth table while making every run upload. For the
same reason a context property the table does not model raises.
"""
context = _pr_context(None)
assert _evaluate("true", context) is True
assert _evaluate("false", context) is False
assert _evaluate("null", context) is None
assert _evaluate("false || true", context) is True
assert _evaluate("true && false", context) is False
assert _evaluate("!false", context) is True
assert _evaluate("null == false", context) is True
assert _evaluate("null == ''", context) is True
assert _evaluate("true == 'true'", context) is False
assert _evaluate("github.event.pull_request.head.repo == null", context)
assert (
_evaluate("github.event.pull_request.head.repo.fork", context) is None
)
assert _evaluate("true || github.actor", context) is True
assert _evaluate("false && github.actor", context) is False
for unsupported in (
"True",
"success()",
"1",
"true.x",
"nothing.here",
# Real but unmodelled fields must fail loud, not read as null.
"github.actor",
"github.event.pull_request.number",
"github.repository.owner",
"false && nothing.here",
):
try:
_evaluate(unsupported, context)
except ValueError:
continue
raise AssertionError(f"accepted {unsupported!r}")
def test_pull_requests_touching_the_scanned_tree_run_the_scan():
"""``--strict`` fails the scan on a file Semgrep cannot parse.
Without ``src/**`` in the ``pull_request`` paths, a PR adding such a
file merges green and the failure first appears in the release gate.
"""
triggers = _workflow()[True]
assert "src/**" in triggers["pull_request"]["paths"]
assert "paths-ignore" not in triggers["pull_request"]
def _assert_twine_is_isolated(workflow):
"""Pin the actual install and check commands, including extra installs."""
commands = []
for job in workflow.get("jobs", {}).values():
for step in job.get("steps", []):
run = step.get("run", "").replace("\\\n", "")
for line in run.splitlines():
if "twine" not in line:
continue
tokens = shlex.split(line, comments=True)
if tokens and tokens != [
"echo",
"=== twine check (PyPI metadata + long_description rendering) ===",
]:
commands.append(tokens)
assert commands == [
["python", "-m", "venv", "/tmp/twine-verify-env"],
[
"/tmp/twine-verify-env/bin/python",
"-m",
"pip",
"install",
"--quiet",
"--require-hashes",
"--only-binary=:all:",
"-r",
"requirements/twine.txt",
],
["/tmp/twine-verify-env/bin/twine", "check", "--strict", "dist/*"],
], commands
def _release_gate_workflow():
return yaml.safe_load(
(REPO_ROOT / ".github/workflows/release-gate.yml").read_text(
encoding="utf-8"
)
)
def test_release_gate_twine_install_is_hash_pinned():
_assert_twine_is_isolated(_release_gate_workflow())
@pytest.mark.parametrize(
("before", "after"),
[
("--require-hashes", ""),
("--only-binary=:all:", ""),
("requirements/twine.txt", "twine"),
("/tmp/twine-verify-env/bin/python", "python"),
("/tmp/twine-verify-env/bin/twine", "twine"),
("/tmp/twine-verify-env/bin/twine", "python -m twine"),
(" check --strict dist/*", " check dist/*"),
(" check --strict dist/*", " check --strict dist/* || true"),
],
)
def test_twine_contract_rejects_weakened_commands(before, after):
workflow = _release_gate_workflow()
for job in workflow["jobs"].values():
for step in job.get("steps", []):
if "twine-verify-env" in step.get("run", ""):
assert before in step["run"]
step["run"] = step["run"].replace(before, after)
with pytest.raises(AssertionError):
_assert_twine_is_isolated(workflow)
@pytest.mark.parametrize(
"command",
[
"pip install twine",
"pip install --quiet twine",
"python -m pip install twine==6.2.0",
"twine check --strict dist/*",
"python -m twine check --strict dist/*",
],
)
@pytest.mark.parametrize("separate_step", [False, True])
def test_twine_contract_rejects_additional_application_commands(
command, separate_step
):
workflow = _release_gate_workflow()
if separate_step:
workflow["jobs"]["extra"] = {"steps": [{"run": command}]}
else:
for job in workflow["jobs"].values():
for step in job.get("steps", []):
if "twine-verify-env" in step.get("run", ""):
step["run"] += "\n" + command + "\n"
with pytest.raises(AssertionError):
_assert_twine_is_isolated(workflow)