Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(ci): require fresh release scan evidence before alert checks (#6646)

LearningCircuit577f30f25692
diff --git a/.file-whitelist.txt b/.file-whitelist.txt
index 3a1874f81e..a0d4478dd4 100644
--- a/.file-whitelist.txt
+++ b/.file-whitelist.txt
@@ -29,6 +29,7 @@
^package-lock\.json$
^lighthouserc\.json$
^\.github/ai-review-label-policy\.json$
+^\.github/release-scan-policy\.json$
^tests/(.*/)?package\.json$
^tests/(.*/)?package-lock\.json$
^tests/settings/golden_master_settings\.json$
diff --git a/.github/release-scan-policy.json b/.github/release-scan-policy.json
new file mode 100644
index 0000000000..2500f2cdb7
--- /dev/null
+++ b/.github/release-scan-policy.json
@@ -0,0 +1,19 @@
+{
+ "/language:python": "analyze",
+ "/language:javascript-typescript": "analyze",
+ "semgrep-security": "semgrep-scan",
+ "bandit": "security-tests",
+ "bearer-security": "bearer-scan",
+ "devskim": "devskim-scan",
+ "container-security": "trivy-scan",
+ "grype-filesystem": "grype-filesystem",
+ "grype-container": "grype-container",
+ "dockle": "dockle",
+ "hadolint": "hadolint",
+ "checkov-dockerfile": "checkov",
+ "checkov-compose": "checkov",
+ "checkov-workflows": "checkov",
+ "retirejs": "retirejs",
+ "zizmor": "zizmor-scan",
+ "gitleaks-release": "gitleaks-scan"
+}
diff --git a/.github/scripts/check_release_scan_evidence.py b/.github/scripts/check_release_scan_evidence.py
new file mode 100644
index 0000000000..63684a6dc4
--- /dev/null
+++ b/.github/scripts/check_release_scan_evidence.py
@@ -0,0 +1,247 @@
+"""Require fresh release scan analyses before evaluating open alerts."""
+
+import json
+import os
+import re
+import subprocess
+import time
+from datetime import datetime
+from pathlib import Path
+from urllib.parse import urlencode
+
+# gh redacts the token from its own error output, but be defensive: never let
+# anything resembling a GitHub token reach a public log via a wrapped
+# CalledProcessError's stderr.
+_TOKEN_PATTERN = re.compile(
+ r"(?:ghp|gho|ghs|ghr|ghu|github_pat)_[A-Za-z0-9_]{10,}"
+)
+
+
+def timestamp(value):
+ parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
+ if parsed.tzinfo is None:
+ raise ValueError("API timestamp has no timezone")
+ return parsed
+
+
+def api(endpoint):
+ result = subprocess.run(
+ [
+ "gh",
+ "api",
+ "--method",
+ "GET",
+ "-H",
+ "Accept: application/vnd.github+json",
+ "-H",
+ "X-GitHub-Api-Version: 2022-11-28",
+ endpoint,
+ ],
+ check=True,
+ capture_output=True,
+ text=True,
+ timeout=60,
+ )
+ return json.loads(result.stdout)
+
+
+def pages(endpoint, params, fetch=api):
+ for page in range(1, 1001):
+ data = fetch(
+ endpoint
+ + "?"
+ + urlencode({**params, "per_page": 100, "page": page})
+ )
+ if not isinstance(data, list) or any(
+ not isinstance(row, dict) for row in data
+ ):
+ raise ValueError("Expected an array of API records")
+ yield data
+ if len(data) < 100:
+ return
+ raise ValueError("API pagination exceeded the safety limit")
+
+
+def recent_analyses(endpoint, ref, started, fetch=api):
+ records = []
+ for page in pages(
+ endpoint, {"ref": ref, "sort": "created", "direction": "desc"}, fetch
+ ):
+ records.extend(
+ row for row in page if timestamp(row["created_at"]) >= started
+ )
+ if any(timestamp(row["created_at"]) < started for row in page):
+ break
+ return records
+
+
+def check_analyses(records, expected, sha, ref, workflow, started):
+ """Return missing categories; reject failed/malformed matching analyses."""
+ if not isinstance(expected, dict) or not expected:
+ raise ValueError("Release scan policy is empty or malformed")
+ found = {}
+ for row in records:
+ category = row.get("category")
+ if category not in expected:
+ continue
+ if (
+ row.get("ref") != ref
+ or row.get("analysis_key") != f"{workflow}:{expected[category]}"
+ or timestamp(row["created_at"]) < started
+ ):
+ continue
+ previous = found.get(category)
+ if previous is None or timestamp(row["created_at"]) > timestamp(
+ previous["created_at"]
+ ):
+ found[category] = row
+ for category, row in found.items():
+ if row.get("commit_sha") != sha:
+ raise ValueError(
+ f"{category}: latest analysis belongs to another commit"
+ )
+ if row.get("error") != "":
+ raise ValueError(
+ f"{category}: failed analysis or missing error status"
+ )
+ # Zero findings are legitimate. Missing result counts are not evidence.
+ if (
+ type(row.get("results_count")) is not int
+ or row["results_count"] < 0
+ ):
+ raise ValueError(f"{category}: invalid analysis result count")
+ return sorted(set(expected) - found.keys())
+
+
+def _alert_label(alert):
+ number = alert.get("number")
+ tool = (alert.get("tool") or {}).get("name") or "unknown tool"
+ return f"alert #{number} ({tool})"
+
+
+def blocking_alert(alert):
+ if alert.get("state") != "open":
+ raise ValueError(
+ f"Open-alert query returned an unexpected alert state for "
+ f"{_alert_label(alert)}: {alert.get('state')!r}"
+ )
+ rule = alert["rule"]
+ security = rule.get("security_severity_level")
+ if security in {"critical", "high", "medium"}:
+ return True
+ if security == "low":
+ return False
+ if security is not None:
+ raise ValueError(
+ f"Unknown security severity {security!r} in {_alert_label(alert)}"
+ )
+ # Native SARIF can provide a level without a numerical security score.
+ level = rule.get("severity")
+ if level not in {"error", "warning", "note", "none"}:
+ raise ValueError(
+ f"No recognized severity {level!r} in {_alert_label(alert)}"
+ )
+ return level in {"error", "warning"}
+
+
+def evaluate(fetch=api, sleep=time.sleep, monotonic=time.monotonic):
+ repo = os.environ["GITHUB_REPOSITORY"]
+ sha, ref = os.environ["GITHUB_SHA"], os.environ["GITHUB_REF"]
+ root = f"repos/{repo}"
+ run = fetch(f"{root}/actions/runs/{os.environ['GITHUB_RUN_ID']}")
+ if run["head_sha"] != sha:
+ raise ValueError("Workflow run does not match the release commit")
+ # created_at retains successful scans from earlier attempts of a rerun.
+ # run_started_at would incorrectly reject those when rerunning failed jobs.
+ started = timestamp(run["created_at"])
+ workflow = run["path"].split("@", 1)[0]
+ policy_path = (
+ Path(__file__).resolve().parents[1] / "release-scan-policy.json"
+ )
+ expected = json.loads(policy_path.read_text(encoding="utf-8"))
+ deadline = monotonic() + 300
+ while True:
+ analyses = recent_analyses(
+ f"{root}/code-scanning/analyses", ref, started, fetch
+ )
+ missing = check_analyses(
+ analyses, expected, sha, ref, workflow, started
+ )
+ if not missing:
+ break
+ if monotonic() >= deadline:
+ raise ValueError(
+ "No fresh successful analysis for: " + ", ".join(missing)
+ )
+ print("Waiting for analyses: " + ", ".join(missing), flush=True)
+ sleep(15)
+ # ref scopes alerts to the release branch/tag instead of the default branch.
+ alerts = [
+ row
+ for page in pages(
+ f"{root}/code-scanning/alerts", {"state": "open", "ref": ref}, fetch
+ )
+ for row in page
+ ]
+ blocking = [row for row in alerts if blocking_alert(row)]
+ with open(
+ os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8"
+ ) as summary:
+ summary.write("## Release scan evidence\n\n")
+ summary.write(
+ f"Validated {len(expected)} categories for `{sha}` on `{ref}` since {run['created_at']}.\n\n"
+ )
+ summary.write(
+ f"Open blocking alerts: **{len(blocking)}** (critical/high/medium or unscored error/warning).\n"
+ )
+ # Do not copy findings or source excerpts into publicly visible logs.
+ for alert in blocking:
+ number = int(alert["number"])
+ summary.write(
+ f"- [Alert #{number}](https://github.com/{repo}/security/code-scanning/{number})\n"
+ )
+ if blocking:
+ first_n = 5
+ named = ", ".join(_alert_label(alert) for alert in blocking[:first_n])
+ if len(blocking) > first_n:
+ named += f", and {len(blocking) - first_n} more"
+ raise ValueError(
+ f"{len(blocking)} blocking code-scanning alerts remain open: "
+ + named
+ )
+ print(
+ f"Release scan policy passed for {sha}: {len(expected)} fresh categories"
+ )
+
+
+def main():
+ try:
+ evaluate()
+ except (
+ OSError,
+ ValueError,
+ KeyError,
+ TypeError,
+ AttributeError,
+ subprocess.SubprocessError,
+ ) as exc:
+ # CalledProcessError's default str() omits the captured API response
+ # body, so surface gh's own stderr too (truncated: it can be a large
+ # JSON/HTML error page). gh already redacts the token from its
+ # output; _TOKEN_PATTERN is a defensive second layer so a token can
+ # never reach a public workflow log through this path.
+ detail = str(exc)
+ stderr = getattr(exc, "stderr", None)
+ if stderr:
+ if isinstance(stderr, bytes):
+ stderr = stderr.decode("utf-8", "replace")
+ snippet = _TOKEN_PATTERN.sub("[REDACTED]", stderr.strip())[:500]
+ if snippet:
+ detail = f"{detail} | gh stderr: {snippet}"
+ print(f"::error::Release scan evidence failed: {detail}")
+ return 1
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/.github/scripts/retire_json_to_sarif.py b/.github/scripts/retire_json_to_sarif.py
new file mode 100644
index 0000000000..95eb516dbd
--- /dev/null
+++ b/.github/scripts/retire_json_to_sarif.py
@@ -0,0 +1,373 @@
+#!/usr/bin/env python3
+"""Convert a retire.js 5.x JSON report into SARIF 2.1.0 for code scanning.
+
+retire.js 5.x has no SARIF output format (its formats are text, json,
+jsonsimple, depcheck and the CycloneDX variants), so retirejs.yml asks for
+``--outputformat json`` and uploads this conversion under the "retirejs"
+category that the release scan evidence gate requires.
+
+Severity mapping, consistent with check_release_scan_evidence.py, which
+blocks a release on critical/high/medium security severities and ignores low:
+
+ retire severity SARIF level security-severity (GitHub band)
+ critical error 9.5 (critical)
+ high error 8.0 (high)
+ medium warning 5.5 (medium)
+ low note 2.0 (low)
+ none note (omitted)
+
+retire.js's repository schema allows "none" (lib/types.js severityLevels).
+Such an advisory is still uploaded, as a note, so that it stays visible and
+so that a report retire.js flagged with its findings status (a --severity of
+none counts it) still has results, but it carries no security-severity:
+GitHub's bands start at 0.1, and an unscored note is exactly what the
+release gate's SARIF-level fallback treats as non-blocking. Rejecting it
+instead would fail the whole report and block the release.
+
+A vulnerability whose severity is missing or null is treated as critical,
+as retire.js itself does when deciding its exit status
+(``v.severity ?? 'critical'`` covers both).
+
+The conversion fails closed: any report that does not have the shape
+retire.js 5.x writes raises, the script exits nonzero and no SARIF file is
+written, so the workflow uploads nothing instead of an empty analysis that
+would mark every open retirejs alert as fixed.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import sys
+from pathlib import Path, PurePosixPath
+
+INFORMATION_URI = "https://github.com/RetireJS/retire.js"
+
+# retire severity -> (SARIF level, GitHub security-severity score, rank).
+# The ranks are retire.js 5.2.4's own severityLevels (lib/types.js). A None
+# score means the security-severity property is omitted.
+SEVERITIES = {
+ "critical": ("error", "9.5", 4),
+ "high": ("error", "8.0", 3),
+ "medium": ("warning", "5.5", 2),
+ "low": ("note", "2.0", 1),
+ "none": ("note", None, 0),
+}
+# retire.js 5.2.4 (lib/cli.js): `severityLevels[v.severity ?? 'critical']`,
+# so a missing and a null severity are both critical.
+MISSING_SEVERITY = "critical"
+
+
+class ReportError(ValueError):
+ """The retire.js report does not have the expected shape."""
+
+
+def _workflow_command_data(text):
+ """Escape text for a GitHub Actions ::error::/::warning:: command."""
+ return (
+ str(text).replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A")
+ )
+
+
+def _require(condition, message):
+ if not condition:
+ raise ReportError(message)
+
+
+def _string_list(value, what):
+ _require(
+ isinstance(value, list) and all(isinstance(v, str) for v in value),
+ f"{what} must be a list of strings",
+ )
+ return value
+
+
+def _relative_uri(file_name, root):
+ _require(
+ isinstance(file_name, str) and file_name.strip() != "",
+ "data[].file must be a non-empty string",
+ )
+ # Both sides are resolved, so ".." segments and symlinks cannot place a
+ # location outside the checkout; a file that really lives outside it
+ # (for example through a symlink) is rejected rather than reported
+ # under a path code scanning cannot map.
+ root = Path(root).resolve()
+ path = (root / file_name).resolve()
+ try:
+ relative = path.relative_to(root)
+ except ValueError:
+ raise ReportError(
+ f"scanned file {file_name!r} is outside the repository root"
+ ) from None
+ _require(relative.parts, f"scanned file {file_name!r} is the root")
+ return PurePosixPath(*relative.parts).as_posix()
+
+
+def _identifier(vulnerability, component):
+ """A stable rule id for one vulnerability, preferring GHSA over CVE."""
+ identifiers = vulnerability.get("identifiers", {})
+ _require(
+ isinstance(identifiers, dict), "vulnerability identifiers must be a map"
+ )
+ ghsa = identifiers.get("githubID")
+ if ghsa is not None:
+ _require(isinstance(ghsa, str) and ghsa, "githubID must be a string")
+ return ghsa
+ cves = identifiers.get("CVE")
+ if cves is not None:
+ _string_list(cves, "CVE")
+ if cves:
+ return cves[0]
+ retid = identifiers.get("retid")
+ if retid is not None:
+ _require(isinstance(retid, str) and retid, "retid must be a string")
+ return f"RETIRE-{retid}"
+ for key in sorted(identifiers):
+ if key == "summary":
+ continue
+ value = identifiers[key]
+ if isinstance(value, list):
+ value = value[0] if value else None
+ if isinstance(value, str) and value:
+ return f"{key}-{value}"
+ # No identifier at all: derive one from what describes the advisory.
+ digest = hashlib.sha256(
+ json.dumps(
+ [component, identifiers.get("summary"), vulnerability.get("below")],
+ sort_keys=True,
+ ).encode("utf-8")
+ ).hexdigest()[:16]
+ return f"RETIRE-{component}-{digest}"
+
+
+def _severity(vulnerability):
+ severity = vulnerability.get("severity")
+ if severity is None:
+ severity = MISSING_SEVERITY
+ _require(
+ severity in SEVERITIES,
+ f"unknown retire.js severity {severity!r}",
+ )
+ return severity
+
+
+def _message(component, version, severity, vulnerability):
+ identifiers = vulnerability.get("identifiers", {})
+ summary = identifiers.get("summary")
+ ids = []
+ for key in ("githubID", "CVE", "retid", "issue", "bug", "PR"):
+ value = identifiers.get(key)
+ if isinstance(value, list):
+ ids.extend(str(v) for v in value)
+ elif value is not None:
+ ids.append(f"retid {value}" if key == "retid" else str(value))
+ if severity == "none":
+ text = (
+ f"{component} {version} has a known vulnerability that "
+ "retire.js rates severity none"
+ )
+ else:
+ text = f"{component} {version} has a known {severity} vulnerability"
+ if isinstance(summary, str) and summary.strip():
+ text += f": {summary.strip()}"
+ if ids:
+ text += f" ({', '.join(ids)})"
+ below = vulnerability.get("below")
+ if isinstance(below, str) and below:
+ text += f". Affected versions are below {below}"
+ return text + "."
+
+
+def convert(report, root):
+ """Return a SARIF 2.1.0 document for a parsed retire.js JSON report."""
+ _require(isinstance(report, dict), "report must be a JSON object")
+ tool_version = report.get("version")
+ _require(
+ isinstance(tool_version, str) and tool_version,
+ "report version must be a non-empty string",
+ )
+ data = report.get("data")
+ _require(isinstance(data, list), "report data must be a list")
+ _string_list(report.get("errors", []), "report errors")
+
+ rules = {}
+ results = []
+ # (rule id, uri, component, version) -> index of its result in results
+ seen = {}
+ for entry in data:
+ _require(isinstance(entry, dict), "data[] entries must be objects")
+ uri = _relative_uri(entry.get("file"), root)
+ components = entry.get("results")
+ _require(isinstance(components, list), "data[].results must be a list")
+ for found in components:
+ _require(isinstance(found, dict), "results[] must be objects")
+ component = found.get("component")
+ version = found.get("version")
+ _require(
+ isinstance(component, str) and component,
+ "results[].component must be a non-empty string",
+ )
+ _require(
+ isinstance(version, str), "results[].version must be a string"
+ )
+ vulnerabilities = found.get("vulnerabilities", [])
+ _require(
+ isinstance(vulnerabilities, list),
+ "results[].vulnerabilities must be a list",
+ )
+ for vulnerability in vulnerabilities:
+ _require(
+ isinstance(vulnerability, dict),
+ "vulnerabilities[] must be objects",
+ )
+ info = _string_list(
+ vulnerability.get("info", []), "vulnerability info"
+ )
+ severity = _severity(vulnerability)
+ rule_id = _identifier(vulnerability, component)
+ level, score, rank = SEVERITIES[severity]
+ rule = rules.get(rule_id)
+ if rule is None or rank > rule["rank"]:
+ rules[rule_id] = {
+ "rank": rank,
+ "severity": severity,
+ "component": component,
+ "summary": vulnerability.get("identifiers", {}).get(
+ "summary"
+ ),
+ "info": info,
+ }
+ properties = {"component": component, "version": version}
+ if score is not None:
+ properties = {"security-severity": score, **properties}
+ result = {
+ "ruleId": rule_id,
+ "level": level,
+ "message": {
+ "text": _message(
+ component, version, severity, vulnerability
+ )
+ },
+ "locations": [
+ {
+ "physicalLocation": {
+ "artifactLocation": {
+ "uri": uri,
+ "uriBaseId": "%SRCROOT%",
+ },
+ "region": {"startLine": 1},
+ }
+ }
+ ],
+ "properties": properties,
+ }
+ # One result per rule, file and component version, carrying
+ # the highest severity reported for it (with its level and
+ # message), whatever order retire.js listed them in.
+ key = (rule_id, uri, component, version)
+ previous = seen.get(key)
+ if previous is None:
+ seen[key] = (len(results), rank)
+ results.append(result)
+ elif rank > previous[1]:
+ seen[key] = (previous[0], rank)
+ results[previous[0]] = result
+
+ sarif_rules = []
+ for rule_id in sorted(rules):
+ rule = rules[rule_id]
+ level, score, _ = SEVERITIES[rule["severity"]]
+ summary = rule["summary"]
+ if not (isinstance(summary, str) and summary.strip()):
+ summary = f"Known vulnerability in {rule['component']}"
+ sarif_rule = {
+ "id": rule_id,
+ "name": rule_id,
+ "shortDescription": {"text": summary.strip()},
+ "fullDescription": {
+ "text": f"{rule['component']}: {summary.strip()}"
+ },
+ "defaultConfiguration": {"level": level},
+ "properties": {
+ "tags": ["security", "vulnerability", "retire.js"],
+ },
+ }
+ if score is not None:
+ sarif_rule["properties"]["security-severity"] = score
+ if rule["info"]:
+ sarif_rule["helpUri"] = rule["info"][0]
+ sarif_rule["help"] = {"text": "\n".join(rule["info"])}
+ sarif_rules.append(sarif_rule)
+
+ return {
+ "$schema": "https://json.schemastore.org/sarif-2.1.0.json",
+ "version": "2.1.0",
+ "runs": [
+ {
+ "tool": {
+ "driver": {
+ "name": "retire.js",
+ "informationUri": INFORMATION_URI,
+ "version": tool_version,
+ "rules": sarif_rules,
+ }
+ },
+ "results": results,
+ }
+ ],
+ }
+
+
+def main(argv=None):
+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+ parser.add_argument("report", help="retire.js --outputformat json file")
+ parser.add_argument("output", help="SARIF file to write")
+ parser.add_argument(
+ "--root",
+ default=".",
+ help="repository root that result paths are made relative to",
+ )
+ parser.add_argument(
+ "--require-findings",
+ action="store_true",
+ help="fail unless the report contains at least one vulnerability "
+ "(retire.js exited with its findings status)",
+ )
+ args = parser.parse_args(argv)
+
+ output = Path(args.output)
+ try:
+ report = json.loads(Path(args.report).read_text(encoding="utf-8"))
+ sarif = convert(report, args.root)
+ if args.require_findings and not sarif["runs"][0]["results"]:
+ raise ReportError(
+ "retire.js reported findings but the report has none"
+ )
+ except (OSError, ValueError) as error:
+ # json.JSONDecodeError and ReportError are both ValueErrors.
+ print(
+ "::error::retire.js report rejected: "
+ + _workflow_command_data(error),
+ file=sys.stderr,
+ )
+ output.unlink(missing_ok=True)
+ return 1
+
+ for message in report.get("errors", []):
+ print(
+ "::warning::retire.js: " + _workflow_command_data(message),
+ file=sys.stderr,
+ )
+ partial = output.with_name(output.name + ".partial")
+ partial.write_text(json.dumps(sarif, indent=2) + "\n", encoding="utf-8")
+ partial.replace(output)
+ print(
+ f"Wrote {output} with {len(sarif['runs'][0]['results'])} result(s) "
+ f"for {len(sarif['runs'][0]['tool']['driver']['rules'])} rule(s)"
+ )
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/.github/workflows/ci-policy-tests.yml b/.github/workflows/ci-policy-tests.yml
new file mode 100644
index 0000000000..0e1e5e5601
--- /dev/null
+++ b/.github/workflows/ci-policy-tests.yml
@@ -0,0 +1,34 @@
+name: CI Policy Tests
+
+on:
+ pull_request:
+ paths:
+ - ".github/workflows/**"
+ - ".github/scripts/check_release_scan_evidence.py"
+ - ".github/scripts/retire_json_to_sarif.py"
+ - ".github/release-scan-policy.json"
+ - "tests/ci/standalone/test_release_scan_*.py"
+ workflow_dispatch:
+
+permissions:
+ contents: read
+
+jobs:
+ policy-tests:
+ runs-on: ubuntu-latest
+ timeout-minutes: 5
+ steps:
+ - name: Harden runner
+ uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1
+ with:
+ egress-policy: audit
+ - name: Checkout
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+ - name: Set up Python
+ uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+ with:
+ python-version: '3.12'
+ - name: Test CI policies without loading the application
+ run: python -m unittest discover -s tests/ci/standalone -p 'test_release_scan*.py' -v
diff --git a/.github/workflows/gitleaks-main.yml b/.github/workflows/gitleaks-main.yml
index 06ea6b7729..11fc93da5c 100644
--- a/.github/workflows/gitleaks-main.yml
+++ b/.github/workflows/gitleaks-main.yml
@@ -1,8 +1,8 @@
name: Gitleaks Main Branch Scan
# This workflow is specifically for the security release gate.
-# It only scans the main branch history to avoid false positives
-# from feature branches with intentional test data.
+# It scans history reachable from the release commit, excluding unrelated
+# feature branches and commits added to main after the release started.
on:
workflow_call: # Called by release-gate.yml
@@ -30,17 +30,17 @@ jobs:
with:
persist-credentials: false
fetch-depth: 0
- ref: main # Always checkout main branch
+ ref: ${{ github.sha }} # Scan the commit being released
- name: Install Gitleaks
uses: $/.github/actions/install-gitleaks
- - name: Run Gitleaks on main branch only
+ - name: Run Gitleaks on release commit history
run: |
gitleaks detect \
--config=.gitleaks.toml \
--gitleaks-ignore-path=.gitleaksignore \
- --log-opts="origin/main" \
+ --log-opts="HEAD" \
--report-format=sarif \
--report-path=gitleaks-results.sarif \
--redact \
@@ -52,3 +52,5 @@ jobs:
uses: github/codeql-action/upload-sarif@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0
with:
sarif_file: gitleaks-results.sarif
+ category: gitleaks-release
+ wait-for-processing: true
diff --git a/.github/workflows/release-gate.yml b/.github/workflows/release-gate.yml
index b0253a496e..e5ebd74310 100644
--- a/.github/workflows/release-gate.yml
+++ b/.github/workflows/release-gate.yml
@@ -181,7 +181,9 @@ jobs:
actions: read
# ============================================
- # Secret detection - scans only main branch history
+ # Secret detection - scans history reachable from the release commit
+ # (checked out at github.sha; excludes unrelated branches and commits
+ # added to main after the release started — see gitleaks-main.yml)
# ============================================
gitleaks-scan:
uses: $/.github/workflows/gitleaks-main.yml
@@ -750,7 +752,9 @@ jobs:
# ============================================
# Check for open code scanning alerts
# This runs after all SARIF uploads complete and fails if
- # high/critical/medium severity alerts exist from ANY tool
+ # critical/high/medium severity alerts exist from ANY tool, or if a
+ # native SARIF alert has no numerical security score but reports an
+ # unscored error/warning level
# ============================================
check-code-scanning-alerts:
name: Check Code Scanning Alerts
@@ -777,7 +781,10 @@ jobs:
- checkov-scan
- zizmor-scan
- gitleaks-scan
+ timeout-minutes: 10
permissions:
+ contents: read
+ actions: read
security-events: read
steps:
- name: Harden the runner (Audit all outbound calls)
@@ -785,83 +792,15 @@ jobs:
with:
egress-policy: audit
- - name: Wait for SARIF indexing
- run: |
- echo "Waiting 120 seconds for GitHub to index SARIF uploads..."
- sleep 120
+ - name: Checkout release policy
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
- - name: Check for open security alerts
+ - name: Validate fresh analyses and open alerts
env:
GH_TOKEN: ${{ github.token }}
- run: |
- echo "Checking for open code scanning alerts..."
-
- # Query all open code scanning alerts with pagination
- ALERTS="[]"
- PAGE=1
- while true; do
- RESPONSE=$(gh api \
- -H "Accept: application/vnd.github+json" \
- -H "X-GitHub-Api-Version: 2022-11-28" \
- "/repos/${{ github.repository }}/code-scanning/alerts?state=open&per_page=100&page=${PAGE}" \
- 2>&1) || {
- echo "::error::Failed to query code scanning API (page ${PAGE}): ${RESPONSE}"
- exit 1
- }
-
- # Validate response is a JSON array
- if ! echo "$RESPONSE" | jq -e 'type == "array"' > /dev/null 2>&1; then
- echo "::error::Unexpected API response (not a JSON array): ${RESPONSE}"
- exit 1
- fi
-
- COUNT=$(echo "$RESPONSE" | jq 'length')
- if [ "$COUNT" -eq 0 ]; then
- break
- fi
-
- # Merge page results into ALERTS
- ALERTS=$(echo "$ALERTS" "$RESPONSE" | jq -s '.[0] + .[1]')
- PAGE=$((PAGE + 1))
- done
-
- TOTAL_ALERTS=$(echo "$ALERTS" | jq 'length')
- echo "Fetched ${TOTAL_ALERTS} open alert(s) across $((PAGE - 1)) page(s)."
-
- # Count alerts by severity
- CRITICAL=$(echo "$ALERTS" | jq '[.[] | select(.rule.security_severity_level == "critical")] | length')
- HIGH=$(echo "$ALERTS" | jq '[.[] | select(.rule.security_severity_level == "high")] | length')
- MEDIUM=$(echo "$ALERTS" | jq '[.[] | select(.rule.security_severity_level == "medium")] | length')
-
- {
- echo ""
- echo "## Code Scanning Alert Summary"
- echo ""
- echo "| Severity | Count |"
- echo "|----------|-------|"
- echo "| Critical | $CRITICAL |"
- echo "| High | $HIGH |"
- echo "| Medium | $MEDIUM |"
- echo ""
- } >> "$GITHUB_STEP_SUMMARY"
-
- TOTAL=$((CRITICAL + HIGH + MEDIUM))
-
- if [ "$TOTAL" -gt 0 ]; then
- BLOCKING_FILTER='select(.rule.security_severity_level == "critical" or .rule.security_severity_level == "high" or .rule.security_severity_level == "medium")'
- {
- echo "❌ **Found $TOTAL blocking alerts (Critical: $CRITICAL, High: $HIGH, Medium: $MEDIUM)**"
- echo ""
- echo "### Blocking Alerts:"
- echo "$ALERTS" | jq -r ".[] | ${BLOCKING_FILTER} | \"- [\\(.rule.security_severity_level | ascii_upcase)] \\(.rule.description) ([#\\(.number)](\\(.html_url)))\""
- } >> "$GITHUB_STEP_SUMMARY"
- echo ""
- echo "::error::Security gate failed: Found $TOTAL open code scanning alerts (Critical: $CRITICAL, High: $HIGH, Medium: $MEDIUM)"
- exit 1
- else
- echo "✅ **No critical, high, or medium severity alerts found**" >> "$GITHUB_STEP_SUMMARY"
- echo "No blocking code scanning alerts found."
- fi
+ run: python3 .github/scripts/check_release_scan_evidence.py
# ============================================
# Summary job that reports overall status
@@ -925,7 +864,29 @@ jobs:
with:
egress-policy: audit
+ # Every declared dependency must pass, including newly added jobs that
+ # have not yet acquired a dedicated section in the human-readable summary.
+ - name: Require every gate dependency to succeed
+ env:
+ GATE_RESULTS: ${{ toJSON(needs) }}
+ run: |
+ python3 - <<'PYCODE'
+ import json
+ import os
+ results = json.loads(os.environ["GATE_RESULTS"])
+ if not isinstance(results, dict) or not results:
+ raise SystemExit("::error::Release gate has no dependency results")
+ failed = [name for name, job in results.items() if job.get("result") != "success"]
+ if failed:
+ with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as summary:
+ summary.write("Release blocked by: " + ", ".join(failed) + "\n")
+ raise SystemExit("::error::Unsuccessful gate jobs: " + ", ".join(failed))
+ PYCODE
+
- name: Check security scan results
+ if: always()
+ env:
+ GATE_RESULTS: ${{ toJSON(needs) }}
run: |
# Redirect all output to GITHUB_STEP_SUMMARY (fixes SC2129)
exec >> "$GITHUB_STEP_SUMMARY"
@@ -976,6 +937,13 @@ jobs:
check_result "${{ needs.cpu-baseline-smoke.result }}" || true
check_result "${{ needs.check-code-scanning-alerts.result }}" || true
+ # Count the complete dependency map, including any job that has no
+ # dedicated prose section below yet.
+ PASS_COUNT=$(jq '[.[] | select(.result == "success")] | length' <<< "$GATE_RESULTS")
+ FAIL_COUNT=$(jq '[.[] | select(.result != "success")] | length' <<< "$GATE_RESULTS")
+ if [ "$FAIL_COUNT" -gt 0 ]; then
+ FAILED="true"
+ fi
TOTAL=$((PASS_COUNT + FAIL_COUNT))
# ============================================
@@ -999,8 +967,7 @@ jobs:
echo "## Detailed Results"
echo ""
- # Reset for detailed output
- FAILED=""
+ # Preserve the aggregate failure status while rendering details.
# ============================================
# Existing scans
diff --git a/.github/workflows/retirejs.yml b/.github/workflows/retirejs.yml
index 622c460a48..85d393b107 100644
--- a/.github/workflows/retirejs.yml
+++ b/.github/workflows/retirejs.yml
@@ -37,32 +37,90 @@ jobs:
with:
node-version: '24'
+ # retire.js 5.x scans JavaScript files only (it rejects --node and has
+ # no --js option), so install the runtime dependencies the Vite build
+ # bundles into the shipped web UI. The minified bundle itself carries
+ # no library signatures retire.js can match; the files in
+ # node_modules do. --omit=dev keeps build and test tooling out of the
+ # scan, and --ignore-scripts runs no dependency install scripts.
+ - name: Install shipped JavaScript dependencies
+ run: npm ci --omit=dev --ignore-scripts
+
- name: Run Retire.js scan
id: retire-scan
run: |
echo "=== Running Retire.js scan ==="
- # Use npx with specific version for reproducible builds
- # Scan JavaScript files and node_modules for known vulnerabilities
- # Only flag medium severity and above (exit code 13 when issues found)
- # Output both SARIF (for GitHub Security) and human-readable format
- npx [email protected] --js --node --severity medium --outputformat sarif --outputpath retire-results.sarif 2>&1 | tee retire-output.txt
- RETIRE_EXIT_CODE=$?
-
- # Check if vulnerabilities were found (exit code 13 indicates issues)
- if [ $RETIRE_EXIT_CODE -eq 13 ]; then
- echo "RETIRE_FOUND_ISSUES=true" >> "$GITHUB_ENV"
- fi
+ # Use npx with a pinned version for reproducible builds. Scan the
+ # checkout: the web UI's own static JavaScript plus the runtime
+ # dependencies installed above. retire.js 5.x has no SARIF output,
+ # so write its JSON report and convert it for code scanning.
+ #
+ # retire.js 5.2.4 exit codes (lib/cli.js): 0 = the scan completed
+ # with nothing at or above --severity; 13 (--exitwith) = the scan
+ # completed with findings at or above --severity; anything else
+ # (1 for invalid arguments, a failed vulnerability repository
+ # download, an output file that cannot be opened or an uncaught
+ # exception) = no scan result. retire.js's exit 9 for an
+ # unopenable output file is unreachable in 5.2.4: fs.openSync
+ # throws before its check, so that case exits 1 too. npx exits
+ # nonzero when it cannot fetch the package at all. --severity only
+ # sets the exit status: the JSON report lists every vulnerable
+ # library, including low severity ones, which the converter
+ # uploads as notes.
+ #
+ # Capture retire's own exit status. Redirecting instead of piping
+ # through tee keeps it independent of pipefail: with a pipe, the
+ # default `bash -e` shell reports tee's status and every scanner
+ # failure looks like a clean scan.
+ rm -f retire-results.json retire-results.sarif retire-output.txt
+ RETIRE_EXIT_CODE=0
+ npx [email protected] --path . --severity medium --exitwith 13 --outputformat json --outputpath retire-results.json > retire-output.txt 2>&1 || RETIRE_EXIT_CODE=$?
- # Display human-readable summary
+ # Display retire's console output (the findings are in the report).
+ # It echoes text from scanned files, so suspend workflow command
+ # processing around it with a random, unguessable resume token.
echo ""
echo "=== Scan Summary ==="
+ STOP_TOKEN="$(openssl rand -hex 16)"
+ echo "::stop-commands::${STOP_TOKEN}"
cat retire-output.txt
+ echo "::${STOP_TOKEN}::"
+
+ case "$RETIRE_EXIT_CODE" in
+ 0)
+ # A completed scan with nothing at or above --severity. Its
+ # report may still list low severity findings, or none.
+ CONVERT_ARGS=()
+ ;;
+ 13)
+ # Findings at or above --severity: upload them and let
+ # "Check scan results" fail the job. The report must contain
+ # at least one vulnerability.
+ echo "RETIRE_FOUND_ISSUES=true" >> "$GITHUB_ENV"
+ CONVERT_ARGS=(--require-findings)
+ ;;
+ *)
+ # A failed scan is not evidence: never convert or upload a
+ # report for it, since an empty upload would mark every open
+ # retirejs alert as fixed.
+ echo "::error::retire.js failed with exit code ${RETIRE_EXIT_CODE}; no scan result"
+ exit 1
+ ;;
+ esac
- # Check if SARIF file was created (for use in upload step)
- if [ -f "retire-results.sarif" ]; then
- echo "sarif_exists=true" >> "$GITHUB_OUTPUT"
+ # retire.js always writes its JSON report for a completed scan, so
+ # a missing or empty one means the scan did not complete.
+ if [ ! -s "retire-results.json" ]; then
+ echo "::error::retire.js exited ${RETIRE_EXIT_CODE} but wrote no JSON report"
+ exit 1
fi
+ # Fails (and writes no SARIF) on any report that does not have
+ # retire.js 5.x's JSON shape.
+ python3 .github/scripts/retire_json_to_sarif.py --root "$PWD" "${CONVERT_ARGS[@]}" retire-results.json retire-results.sarif
+
+ echo "sarif_exists=true" >> "$GITHUB_OUTPUT"
+
- name: Upload SARIF results
uses: github/codeql-action/upload-sarif@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0
if: always() && steps.retire-scan.outputs.sarif_exists == 'true'
@@ -76,7 +134,7 @@ jobs:
echo "::error::Retire.js found medium or higher severity vulnerabilities"
echo "Review the Security tab for details"
echo "Consider updating or replacing affected libraries"
- # Only fail for PR/push events, not scheduled scans
+ # Fail manual and release-gate runs, not scheduled scans
if [[ "${{ github.event_name }}" != "schedule" ]]; then
exit 1
fi
diff --git a/SECURITY.md b/SECURITY.md
index 23fe32cec4..2a0ad9d6f1 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -283,7 +283,7 @@ We maintain comprehensive automated security scanning across the entire developm
|------|---------|-----------|
| **OSV-Scanner** | Open Source Vulnerability database | Every PR & push |
| **npm audit** | JavaScript dependency vulnerabilities | Every PR & push |
-| **RetireJS** | Known vulnerable JS libraries | Every PR & push |
+| **RetireJS** | Known vulnerable JS libraries | Weekly, manual & release gate |
| **SBOM Generation** | Software Bill of Materials (Syft) | Weekly & releases |
| **License Scanning** | License compliance checking | Every PR |
diff --git a/docs/CI_CD_INFRASTRUCTURE.md b/docs/CI_CD_INFRASTRUCTURE.md
index 26346364ce..f3fd61650d 100644
--- a/docs/CI_CD_INFRASTRUCTURE.md
+++ b/docs/CI_CD_INFRASTRUCTURE.md
@@ -102,7 +102,7 @@ pre-commit install-hooks
| `container-security.yml` | PR, push | Container vulnerability scanning |
| `hadolint.yml` | PR, push | Dockerfile linting |
| `owasp-zap-scan.yml` | Schedule | OWASP ZAP dynamic scanning |
-| `retirejs.yml` | PR, push | JavaScript vulnerability scanning |
+| `retirejs.yml` | Schedule, manual, `workflow_call` from release-gate.yml | JavaScript vulnerability scanning |
| `zizmor-security.yml` | PR, push | Additional security checks |
| `ossf-scorecard.yml` | Schedule | OpenSSF Scorecard |
| `security-headers-validation.yml` | PR, push | HTTP security headers |
diff --git a/docs/ci/workflow-status.md b/docs/ci/workflow-status.md
index 87daf26c61..3e06d9389e 100644
--- a/docs/ci/workflow-status.md
+++ b/docs/ci/workflow-status.md
@@ -120,6 +120,7 @@
| `advanced-search-reminder.yml` | last 30 days | PR | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/advanced-search-reminder.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/advanced-search-reminder.yml) |
| `ai-code-reviewer.yml` | last 30 days | PR-target | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/ai-code-reviewer.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/ai-code-reviewer.yml) |
| `check-workflow-status.yml` | last 30 days | PR, manual | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/check-workflow-status.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/check-workflow-status.yml) |
+| `ci-policy-tests.yml` | last 30 days | PR, manual | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/ci-policy-tests.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/ci-policy-tests.yml) |
| `danger-zone-alert.yml` | last 30 days | PR | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/danger-zone-alert.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/danger-zone-alert.yml) |
| `dependency-review.yml` | last 30 days | PR, manual | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/dependency-review.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/dependency-review.yml) |
| `e2e-research-test.yml` | last 30 days | PR | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/e2e-research-test.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/e2e-research-test.yml) |
diff --git a/tests/ci/standalone/test_release_scan_evidence.py b/tests/ci/standalone/test_release_scan_evidence.py
new file mode 100644
index 0000000000..8b8350eca8
--- /dev/null
+++ b/tests/ci/standalone/test_release_scan_evidence.py
@@ -0,0 +1,1051 @@
+"""Standalone release-policy tests using synthetic API responses."""
+
+# allow: no-sut-import — imports the standalone CI script directly.
+
+import importlib.util
+import io
+import json
+import os
+import re
+import shutil
+import subprocess
+import tempfile
+import unittest
+from contextlib import redirect_stdout
+from pathlib import Path
+from unittest.mock import patch
+
+ROOT = Path(__file__).resolve().parents[3]
+SPEC = importlib.util.spec_from_file_location(
+ "release_scan", ROOT / ".github/scripts/check_release_scan_evidence.py"
+)
+POLICY = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(POLICY)
+
+SHA = "a" * 40
+REF = "refs/tags/v1.0.0"
+WORKFLOW = ".github/workflows/release.yml"
+START = "2026-01-01T00:00:00Z"
+
+
+def analysis(**changes):
+ return {
+ "category": "semgrep-security",
+ "commit_sha": SHA,
+ "ref": REF,
+ "analysis_key": f"{WORKFLOW}:semgrep-scan",
+ "created_at": START,
+ "error": "",
+ "results_count": 0,
+ **changes,
+ }
+
+
+def alert(**changes):
+ return {
+ "number": 1,
+ "state": "open",
+ "tool": {"name": "CodeQL"},
+ "rule": {"security_severity_level": "critical", "severity": None},
+ **changes,
+ }
+
+
+# --- R3: tie release-scan-policy.json to what release-gate.yml actually
+# uploads. This is a plain-text (no PyYAML) walk, deliberately: the standalone
+# suite runs in ci-policy-tests.yml on a bare `actions/setup-python` with no
+# dependency install step, so a `yaml` import here would break that job.
+_JOB_ID_RE = re.compile(r"^ ([A-Za-z0-9_-]+):\s*$")
+_LOCAL_USES_RE = re.compile(r"^\s+uses:\s*(\$/\S+|\./\S+)\s*$")
+_UPLOAD_MARKERS = ("upload-sarif", "codeql-action/analyze")
+_CATEGORY_RE = re.compile(r"^\s*category:\s*(.+?)\s*(#.*)?$")
+
+# Remote reusable "uploaders" whose SARIF category cannot be statically
+# extracted from this repo (a third-party composite/Docker action that
+# uploads internally, not a local `$/` reusable workflow with its own
+# `upload-sarif`/`codeql-action/analyze` step to grep). Each entry must
+# name why it's safe to trust rather than verify by parsing.
+_REMOTE_SARIF_UPLOADERS = {
+ # zizmorcore/zizmor-action (zizmor-security.yml) uploads its own SARIF
+ # internally under a fixed "zizmor" category — confirmed against the
+ # live Security tab, not discoverable by grepping this repo.
+ "zizmor-scan": "zizmor",
+}
+
+# Remote reusable workflows (`owner/repo/.github/workflows/x.yml@sha`) that a
+# gate-called local workflow delegates a whole job to. Their upload-sarif step
+# lives in the other repository, so its category is recorded here instead of
+# parsed. "{workflow}" stands for the top-level workflow file of the run.
+_REMOTE_REUSABLE_SARIF_UPLOADERS = {
+ # osv-scanner.yml -> google/osv-scanner-action's reusable workflow, whose
+ # "Upload to code-scanning" step (v2.5.1) passes no `category:`. Code
+ # scanning therefore files it under the default "<top-level workflow>:
+ # <job id>" category, e.g. ".github/workflows/release-gate.yml:osv-scan"
+ # for the scheduled release-gate run (confirmed against live analyses).
+ "google/osv-scanner-action/.github/workflows/osv-scanner-reusable.yml": (
+ "{workflow}:osv-scan"
+ ),
+}
+
+# Categories the gate uploads but release-scan-policy.json deliberately does
+# not require, each with the reason.
+_UPLOADED_BUT_NOT_IN_POLICY = {
+ # The category embeds the calling workflow's path (release.yml for a tag
+ # release, release-gate.yml for the scheduled run), which the policy's
+ # fixed category names cannot express. OSV is still gated: the
+ # osv-scan job fails on any vulnerability (fail-on-vuln defaults to
+ # true) or scanner error, release-gate-summary requires every
+ # dependency to succeed, and its open alerts on the release ref are
+ # counted by the alert query like any other tool's.
+ "{workflow}:osv-scan": "path-dependent default category",
+}
+
+_REMOTE_REUSABLE_RE = re.compile(
+ r"^\s+uses:\s*[\"']?([\w.-]+/[\w.-]+/\.github/workflows/[^@\"'\s]+)@"
+)
+
+
+def _local_reusable_workflow_jobs(gate_text):
+ """release-gate.yml job id -> the `$/...`/`./...` workflow file it calls."""
+ jobs = {}
+ current = None
+ for line in gate_text.splitlines():
+ job_match = _JOB_ID_RE.match(line)
+ if job_match:
+ current = job_match.group(1)
+ continue
+ if line and not line.startswith(" "):
+ current = None
+ continue
+ uses_match = _LOCAL_USES_RE.match(line)
+ if uses_match and current:
+ jobs[current] = uses_match.group(1)[2:] # strip "$/" / "./"
+ return jobs
+
+
+def _sarif_categories_in(text, source="workflow"):
+ """Every `category:` value that follows an upload-sarif/analyze step.
+
+ An upload step with no `category:` in reach is reported as a
+ "<uncategorized ...>" placeholder rather than skipped, and so is a remote
+ reusable workflow not listed in _REMOTE_REUSABLE_SARIF_UPLOADERS: either
+ can upload SARIF this walk cannot name, so neither may silently vanish
+ from the parity check.
+ """
+ lines = text.splitlines()
+ found = []
+ for i, line in enumerate(lines):
+ remote = _REMOTE_REUSABLE_RE.match(line)
+ if remote:
+ found.append(
+ _REMOTE_REUSABLE_SARIF_UPLOADERS.get(
+ remote.group(1),
+ f"<unrecognised remote reusable workflow "
+ f"{remote.group(1)} in {source}>",
+ )
+ )
+ continue
+ if line.lstrip().startswith("#") or not any(
+ marker in line for marker in _UPLOAD_MARKERS
+ ):
+ continue
+ for follow in lines[i + 1 : i + 8]:
+ match = _CATEGORY_RE.match(follow)
+ if match:
+ found.append(match.group(1).strip().strip("\"'"))
+ break
+ if re.match(r"^\s*-\s*(name|uses):", follow):
+ break # next step started with no category on this one
+ else:
+ match = None
+ if not match:
+ found.append(f"<uncategorized upload in {source} line {i + 1}>")
+ return found
+
+
+def _expand_matrix_language(category, workflow_text):
+ if "matrix.language" not in category:
+ return [category]
+ languages = re.findall(
+ r"^\s*-\s*language:\s*([\w.-]+)", workflow_text, re.MULTILINE
+ )
+ return [
+ category.replace("${{matrix.language}}", lang).replace(
+ "${{ matrix.language }}", lang
+ )
+ for lang in languages
+ ] or [category]
+
+
+def _release_gate_sarif_categories():
+ """Every SARIF category reachable from release-gate.yml, discovered the
+ same way the policy's author verified it by hand: walk each job's local
+ reusable workflow and read off its upload-sarif/analyze category, plus
+ the recorded category of any remote reusable workflow it calls.
+ Uploads the walk cannot name come back as "<...>" placeholders.
+ """
+ gate_text = (ROOT / ".github/workflows/release-gate.yml").read_text(
+ encoding="utf-8"
+ )
+ categories = set()
+ for job_id, rel_path in _local_reusable_workflow_jobs(gate_text).items():
+ workflow_text = (ROOT / rel_path).read_text(encoding="utf-8")
+ expanded = set()
+ for raw_category in _sarif_categories_in(workflow_text, rel_path):
+ expanded.update(
+ _expand_matrix_language(raw_category, workflow_text)
+ )
+ if not expanded and job_id in _REMOTE_SARIF_UPLOADERS:
+ expanded = {_REMOTE_SARIF_UPLOADERS[job_id]}
+ categories.update(expanded)
+ return categories
+
+
+class EvidenceTests(unittest.TestCase):
+ def check(self, records):
+ return POLICY.check_analyses(
+ records,
+ {"semgrep-security": "semgrep-scan"},
+ SHA,
+ REF,
+ WORKFLOW,
+ POLICY.timestamp(START),
+ )
+
+ def test_clean_current_analysis_is_valid(self):
+ self.assertEqual(self.check([analysis()]), [])
+
+ def test_wrong_commit_ref_workflow_and_stale_results_are_not_evidence(self):
+ for changes in (
+ {"ref": "refs/heads/main"},
+ {"analysis_key": ".github/workflows/semgrep.yml:semgrep-scan"},
+ {"created_at": "2025-12-31T23:59:59Z"},
+ {"category": "unrelated"},
+ ):
+ with self.subTest(changes=changes):
+ self.assertEqual(
+ self.check([analysis(**changes)]), ["semgrep-security"]
+ )
+
+ def test_superseding_commit_cannot_supply_release_alert_state(self):
+ with self.assertRaisesRegex(ValueError, "another commit"):
+ self.check(
+ [
+ analysis(),
+ analysis(
+ commit_sha="b" * 40, created_at="2026-01-01T01:00:00Z"
+ ),
+ ]
+ )
+
+ def test_scan_errors_and_missing_counts_fail(self):
+ for changes in (
+ {"error": "scan failed"},
+ {"error": None},
+ {"results_count": None},
+ {"results_count": -1},
+ ):
+ with self.subTest(changes=changes), self.assertRaises(ValueError):
+ self.check([analysis(**changes)])
+
+ def test_latest_analysis_controls_rerun_result(self):
+ failed = analysis(error="failed")
+ passed = analysis(created_at="2026-01-01T01:00:00Z")
+ self.assertEqual(self.check([failed, passed]), [])
+ with self.assertRaises(ValueError):
+ self.check(
+ [
+ passed,
+ analysis(created_at="2026-01-01T02:00:00Z", error="failed"),
+ ]
+ )
+
+ def test_each_category_is_required(self):
+ self.assertEqual(self.check([]), ["semgrep-security"])
+ with self.assertRaises(ValueError):
+ POLICY.check_analyses(
+ [], {}, SHA, REF, WORKFLOW, POLICY.timestamp(START)
+ )
+
+ def test_alert_severity_and_unscored_fallback(self):
+ for security, level, expected in (
+ ("critical", "note", True),
+ ("high", "error", True),
+ ("medium", "warning", True),
+ ("low", "error", False),
+ (None, "error", True),
+ (None, "warning", True),
+ (None, "note", False),
+ (None, "none", False),
+ ):
+ with self.subTest(security=security, level=level):
+ self.assertEqual(
+ POLICY.blocking_alert(
+ {
+ "state": "open",
+ "rule": {
+ "security_severity_level": security,
+ "severity": level,
+ },
+ }
+ ),
+ expected,
+ )
+ for rule in ({}, {"security_severity_level": "unknown"}):
+ with self.assertRaises(ValueError):
+ POLICY.blocking_alert({"state": "open", "rule": rule})
+
+ def test_pagination_and_malformed_api_response(self):
+ responses = iter([[analysis()] * 100, [analysis()]])
+ self.assertEqual(
+ sum(
+ len(page)
+ for page in POLICY.pages(
+ "endpoint", {}, lambda _: next(responses)
+ )
+ ),
+ 101,
+ )
+ with self.assertRaises(ValueError):
+ list(
+ POLICY.pages("endpoint", {}, lambda _: {"message": "forbidden"})
+ )
+
+ def test_recent_analysis_pagination_stops_before_old_runs(self):
+ calls = []
+
+ def fetch(endpoint):
+ calls.append(endpoint)
+ return [analysis(created_at="2025-12-31T00:00:00Z")] * 100
+
+ self.assertEqual(
+ POLICY.recent_analyses(
+ "endpoint", REF, POLICY.timestamp(START), fetch
+ ),
+ [],
+ )
+ self.assertEqual(len(calls), 1)
+
+ def test_end_to_end_polling_and_ref_scoped_alerts(self):
+ expected = json.loads(
+ (ROOT / ".github/release-scan-policy.json").read_text(
+ encoding="utf-8"
+ )
+ )
+ complete = [
+ analysis(category=category, analysis_key=f"{WORKFLOW}:{job}")
+ for category, job in expected.items()
+ ]
+ calls, sleeps = [], []
+ analyses = iter([[], complete])
+
+ def fetch(endpoint):
+ calls.append(endpoint)
+ if "/actions/runs/" in endpoint:
+ return {"head_sha": SHA, "created_at": START, "path": WORKFLOW}
+ if "/analyses?" in endpoint:
+ return next(analyses)
+ self.assertIn("state=open", endpoint)
+ self.assertIn("ref=refs%2Ftags%2Fv1.0.0", endpoint)
+ return []
+
+ with (
+ tempfile.TemporaryDirectory() as directory,
+ patch.dict(
+ os.environ,
+ {
+ "GITHUB_REPOSITORY": "owner/repo",
+ "GITHUB_SHA": SHA,
+ "GITHUB_REF": REF,
+ "GITHUB_RUN_ID": "1",
+ "GITHUB_STEP_SUMMARY": str(Path(directory) / "summary"),
+ },
+ ),
+ ):
+ POLICY.evaluate(fetch, sleeps.append, lambda: 0)
+ self.assertEqual(sleeps, [15])
+ self.assertIn(
+ "Open blocking alerts: **0**",
+ Path(os.environ["GITHUB_STEP_SUMMARY"]).read_text(
+ encoding="utf-8"
+ ),
+ )
+ ticks = iter([0, 301])
+ with self.assertRaisesRegex(
+ ValueError, "No fresh successful analysis"
+ ):
+ POLICY.evaluate(
+ lambda endpoint: (
+ {"head_sha": SHA, "created_at": START, "path": WORKFLOW}
+ if "/actions/runs/" in endpoint
+ else []
+ ),
+ sleeps.append,
+ lambda: next(ticks),
+ )
+
+ @staticmethod
+ def _run_env(directory):
+ return {
+ "GITHUB_REPOSITORY": "owner/repo",
+ "GITHUB_SHA": SHA,
+ "GITHUB_REF": REF,
+ "GITHUB_RUN_ID": "1",
+ "GITHUB_STEP_SUMMARY": str(Path(directory) / "summary"),
+ }
+
+ def test_evaluate_raises_and_names_blocking_alerts_but_allows_low_severity(
+ self,
+ ):
+ """B1: one scored-critical and one unscored-error alert must block;
+ a low-severity control must not. Kills both `if blocking:` ->
+ `if False:` and `blocking = []` mutants, which would swallow the
+ blocking alerts and let evaluate() return normally.
+ """
+ expected = json.loads(
+ (ROOT / ".github/release-scan-policy.json").read_text(
+ encoding="utf-8"
+ )
+ )
+ complete = [
+ analysis(category=category, analysis_key=f"{WORKFLOW}:{job}")
+ for category, job in expected.items()
+ ]
+ alerts = [
+ alert(
+ number=101,
+ tool={"name": "CodeQL"},
+ rule={"security_severity_level": "critical", "severity": None},
+ ),
+ alert(
+ number=102,
+ tool={"name": "Bandit"},
+ rule={"security_severity_level": None, "severity": "error"},
+ ),
+ # Non-blocking control: open, low-scored severity.
+ alert(
+ number=103,
+ tool={"name": "Semgrep"},
+ rule={"security_severity_level": "low", "severity": "error"},
+ ),
+ ]
+
+ def fetch(endpoint):
+ if "/actions/runs/" in endpoint:
+ return {"head_sha": SHA, "created_at": START, "path": WORKFLOW}
+ if "/analyses?" in endpoint:
+ return complete
+ return alerts
+
+ # Bounded ticks (not a constant monotonic): if a category ever fails
+ # to match, this fails the polling deadline instead of looping
+ # forever under a no-op sleep.
+ ticks = iter([0, 301])
+ with (
+ tempfile.TemporaryDirectory() as directory,
+ patch.dict(os.environ, self._run_env(directory)),
+ ):
+ with self.assertRaisesRegex(
+ ValueError,
+ r"2 blocking code-scanning alerts remain open: "
+ r"alert #101 \(CodeQL\), alert #102 \(Bandit\)",
+ ) as caught:
+ POLICY.evaluate(
+ fetch, lambda seconds: None, lambda: next(ticks)
+ )
+ self.assertNotIn("#103", str(caught.exception))
+ summary = Path(os.environ["GITHUB_STEP_SUMMARY"]).read_text(
+ encoding="utf-8"
+ )
+ self.assertIn("Open blocking alerts: **2**", summary)
+
+ def test_main_fails_for_every_evaluate_exception_class(self):
+ """B2: main() must report and return nonzero for every exception
+ class evaluate() can raise. Kills the `return 1` -> `return 0`
+ mutant, which would report an error yet exit success.
+ """
+ cases = (
+ ValueError("release scan policy violated"),
+ subprocess.CalledProcessError(
+ 1, ["gh", "api", "x"], output="", stderr="not found"
+ ),
+ json.JSONDecodeError("Expecting value", "", 0),
+ subprocess.TimeoutExpired(cmd="gh api x", timeout=60),
+ )
+ for exc in cases:
+ with self.subTest(exc=type(exc).__name__):
+ buffer = io.StringIO()
+ with (
+ patch.object(POLICY, "evaluate", side_effect=exc),
+ redirect_stdout(buffer),
+ ):
+ self.assertEqual(POLICY.main(), 1)
+ self.assertIn(
+ "::error::Release scan evidence failed:", buffer.getvalue()
+ )
+
+ def test_main_returns_zero_on_success(self):
+ with patch.object(POLICY, "evaluate", return_value=None):
+ self.assertEqual(POLICY.main(), 0)
+
+ def test_main_reports_truncated_redacted_gh_stderr_on_called_process_error(
+ self,
+ ):
+ """R5: gh's stderr must reach the failure log (truncated, and with
+ any token-shaped substring redacted).
+ """
+ token = "ghp_" + "a" * 40
+ stderr = f"error: {token} invalid, and " + ("z" * 600)
+ exc = subprocess.CalledProcessError(
+ 1, ["gh", "api", "x"], output="", stderr=stderr
+ )
+ buffer = io.StringIO()
+ with (
+ patch.object(POLICY, "evaluate", side_effect=exc),
+ redirect_stdout(buffer),
+ ):
+ self.assertEqual(POLICY.main(), 1)
+ output = buffer.getvalue()
+ self.assertIn("gh stderr:", output)
+ self.assertNotIn(token, output)
+ self.assertIn("[REDACTED]", output)
+ self.assertNotIn("z" * 600, output)
+
+ def test_evaluate_rejects_a_run_for_a_different_commit(self):
+ """R1: head_sha != GITHUB_SHA must be rejected."""
+
+ def fetch(endpoint):
+ if "/actions/runs/" in endpoint:
+ return {
+ "head_sha": "b" * 40,
+ "created_at": START,
+ "path": WORKFLOW,
+ }
+ return []
+
+ # Bounded ticks: evaluate() must raise before ever touching the
+ # polling loop, but a non-advancing monotonic would hang forever
+ # under a no-op sleep if that check ever regressed.
+ ticks = iter([0, 301])
+ with (
+ tempfile.TemporaryDirectory() as directory,
+ patch.dict(os.environ, self._run_env(directory)),
+ ):
+ with self.assertRaisesRegex(
+ ValueError, "does not match the release commit"
+ ):
+ POLICY.evaluate(
+ fetch, lambda seconds: None, lambda: next(ticks)
+ )
+
+ def test_blocking_alert_rejects_a_non_open_state(self):
+ """R1: blocking_alert must reject any state other than 'open'."""
+ with self.assertRaisesRegex(ValueError, "unexpected alert state"):
+ POLICY.blocking_alert(alert(state="dismissed"))
+
+ def test_timestamp_rejects_a_naive_value(self):
+ """R1: API timestamps without a UTC offset must be rejected rather
+ than silently compared as naive datetimes.
+ """
+ with self.assertRaisesRegex(ValueError, "no timezone"):
+ POLICY.timestamp("2026-01-01T00:00:00")
+
+ def test_analysis_from_a_first_attempt_survives_a_later_rerun(self):
+ """R1: `started` must come from the run's created_at (the original
+ run, stable across reruns), not run_started_at (the current
+ attempt). A first-attempt analysis created well before a later
+ rerun's run_started_at must still be accepted. If evaluate() used
+ run_started_at instead, this analysis would look stale and the
+ category would come back missing, which times out immediately below
+ (monotonic ticks straight past the deadline on the first check).
+ """
+ expected = json.loads(
+ (ROOT / ".github/release-scan-policy.json").read_text(
+ encoding="utf-8"
+ )
+ )
+ original_created = "2026-01-01T00:00:00Z"
+ analysis_created = "2026-01-01T00:30:00Z"
+ rerun_started = "2026-01-01T05:00:00Z"
+ complete = [
+ analysis(
+ category=category,
+ analysis_key=f"{WORKFLOW}:{job}",
+ created_at=analysis_created,
+ )
+ for category, job in expected.items()
+ ]
+
+ def fetch(endpoint):
+ if "/actions/runs/" in endpoint:
+ return {
+ "head_sha": SHA,
+ "created_at": original_created,
+ "run_started_at": rerun_started,
+ "path": WORKFLOW,
+ }
+ if "/analyses?" in endpoint:
+ return complete
+ return []
+
+ ticks = iter([0, 301])
+ with (
+ tempfile.TemporaryDirectory() as directory,
+ patch.dict(os.environ, self._run_env(directory)),
+ ):
+ # Must not raise: the analysis is accepted on the first pass.
+ POLICY.evaluate(fetch, lambda seconds: None, lambda: next(ticks))
+
+ def test_blocking_alert_messages_name_the_alert_and_tool(self):
+ """R5: an unrecognised/null severity must name the offending alert
+ number and tool, not just the category of failure.
+ """
+ with self.assertRaisesRegex(ValueError, r"alert #77 \(Bandit\)"):
+ POLICY.blocking_alert(
+ alert(
+ number=77,
+ tool={"name": "Bandit"},
+ rule={"security_severity_level": "unscored"},
+ )
+ )
+ with self.assertRaisesRegex(ValueError, r"alert #88 \(Semgrep\)"):
+ POLICY.blocking_alert(
+ alert(
+ number=88,
+ tool={"name": "Semgrep"},
+ rule={"security_severity_level": None, "severity": "huge"},
+ )
+ )
+
+ def test_policy_categories_match_every_sarif_upload_reachable_from_gate(
+ self,
+ ):
+ """R3: every release-scan-policy.json category must correspond to an
+ actual upload-sarif/analyze category reachable from
+ release-gate.yml, and vice versa. Discovery walks each gate job's
+ local reusable workflow; remote uploaders (a third-party action, or a
+ remote reusable workflow such as OSV's) resolve only through the
+ explicit allowlists above, and anything else it cannot name fails
+ as a placeholder. Uploads deliberately left out of the policy must
+ be listed in _UPLOADED_BUT_NOT_IN_POLICY. A category dropped from
+ either side fails this test.
+ """
+ discovered = _release_gate_sarif_categories()
+ policy = json.loads(
+ (ROOT / ".github/release-scan-policy.json").read_text(
+ encoding="utf-8"
+ )
+ )
+ self.assertTrue(discovered, "SARIF category discovery found nothing")
+ self.assertFalse(set(policy) & set(_UPLOADED_BUT_NOT_IN_POLICY))
+ self.assertEqual(
+ discovered,
+ set(policy) | set(_UPLOADED_BUT_NOT_IN_POLICY),
+ "release-scan-policy.json categories (plus the documented "
+ "exclusions) must exactly match the SARIF categories reachable "
+ "from release-gate.yml",
+ )
+
+ def test_sarif_discovery_never_skips_an_unnamed_upload(self):
+ """R3: an upload with no `category:` and an unlisted remote
+ reusable workflow must surface as placeholders, not disappear.
+ """
+ text = "\n".join(
+ (
+ "jobs:",
+ " a:",
+ " steps:",
+ " # upload-sarif is mentioned only in this comment",
+ " - uses: github/codeql-action/upload-sarif@abc",
+ " with:",
+ " sarif_file: x.sarif",
+ " - name: next",
+ " uses: github/codeql-action/upload-sarif@abc",
+ " with:",
+ " category: named",
+ " b:",
+ " uses: other/repo/.github/workflows/scan.yml@abc",
+ " c:",
+ ' uses: "google/osv-scanner-action/.github/workflows/'
+ 'osv-scanner-reusable.yml@abc" # v2',
+ )
+ )
+ self.assertEqual(
+ _sarif_categories_in(text, "w.yml"),
+ [
+ "<uncategorized upload in w.yml line 5>",
+ "named",
+ "<unrecognised remote reusable workflow "
+ "other/repo/.github/workflows/scan.yml in w.yml>",
+ "{workflow}:osv-scan",
+ ],
+ )
+
+
+def _step_run_script(workflow_text, step_id):
+ """The `run: |` block of the step with `id: <step_id>`, dedented.
+
+ Plain-text extraction (no PyYAML, see the R3 note above): find the step,
+ then take every following line indented deeper than its `run:` key.
+ """
+ lines = workflow_text.splitlines()
+ step_start = next(
+ i
+ for i, line in enumerate(lines)
+ if re.match(rf"^\s+id:\s*{re.escape(step_id)}\s*$", line)
+ )
+ run_index = next(
+ i
+ for i in range(step_start + 1, len(lines))
+ if re.match(r"^\s+run:\s*\|\s*$", lines[i])
+ )
+ key_indent = len(lines[run_index]) - len(lines[run_index].lstrip())
+ body = []
+ for line in lines[run_index + 1 :]:
+ if line.strip() and len(line) - len(line.lstrip()) <= key_indent:
+ break
+ body.append(line)
+ indent = min(len(ln) - len(ln.lstrip()) for ln in body if ln.strip())
+ return "\n".join(ln[indent:] for ln in body).rstrip() + "\n"
+
+
+_FAKE_NPX = """#!/bin/sh
+printf '%s\\n' "$@" > "$FAKE_ARGS"
+out=""
+while [ "$#" -gt 0 ]; do
+ if [ "$1" = "--outputpath" ]; then out="$2"; fi
+ shift
+done
+echo "fake retire.js output"
+echo "::error::from a scanned file"
+if [ -n "$FAKE_REPORT" ]; then
+ printf '%s' "$FAKE_REPORT" | sed "s|@ROOT@|$PWD|g" > "$out"
+fi
+exit "$FAKE_EXIT"
+"""
+
+_RETIRE_CONVERTER = ".github/scripts/retire_json_to_sarif.py"
+
+# retire.js 5.2.4 JSON report shapes (see test_release_scan_retire_sarif.py
+# for a real report). @ROOT@ is replaced with the step's working directory,
+# as retire.js reports absolute paths.
+_EMPTY_REPORT = json.dumps(
+ {"version": "5.2.4", "data": [], "messages": [], "errors": []}
+)
+
+
+def _retire_report(*severities):
+ return json.dumps(
+ {
+ "version": "5.2.4",
+ "data": [
+ {
+ "file": "@ROOT@/node_modules/jquery/dist/jquery.js",
+ "results": [
+ {
+ "version": "3.4.1",
+ "component": "jquery",
+ "vulnerabilities": [
+ {
+ "severity": severity,
+ "identifiers": {
+ "summary": "an advisory",
+ "githubID": f"GHSA-{index}",
+ },
+ }
+ for index, severity in enumerate(severities)
+ ],
+ }
+ ],
+ }
+ ],
+ "messages": [],
+ "errors": [],
+ }
+ )
+
+
+_FINDINGS_REPORT = _retire_report("medium", "high")
+_LOW_ONLY_REPORT = _retire_report("low")
+
+# Every option retire.js 5.2.4 accepts (lib/cli.js, the commander
+# definitions), and the values its --outputformat and --severity accept.
+# 5.x rejects --node/-n outright and has no --js option or sarif format.
+# Re-verify these against the new cli.js before bumping the pinned version.
+RETIRE_5_2_4_OPTIONS = {
+ "-V": False,
+ "--version": False,
+ "-h": False,
+ "--help": False,
+ "-v": False,
+ "--verbose": False,
+ "-c": False,
+ "--nocache": False,
+ "--jspath": True,
+ "--path": True,
+ "--jsrepo": True,
+ "--cachedir": True,
+ "--proxy": True,
+ "--outputformat": True,
+ "--outputpath": True,
+ "--ignore": True,
+ "--ignorefile": True,
+ "--severity": True,
+ "--exitwith": True,
+ "--colors": False,
+ "--insecure": False,
+ "--ext": True,
+ "--cacert": True,
+ "--includeOsv": False,
+ "--deep": False,
+}
+RETIRE_5_2_4_OUTPUT_FORMATS = {
+ "text",
+ "json",
+ "jsonsimple",
+ "depcheck",
+ "cyclonedx",
+ "cyclonedxJSON",
+ "cyclonedxJSON1_6",
+}
+RETIRE_5_2_4_SEVERITIES = {"none", "low", "medium", "high", "critical"}
+
+
[email protected](shutil.which("bash"), "needs bash")
+class RetireScanStepTests(unittest.TestCase):
+ """R4: retirejs.yml's scan step must report retire.js's own exit status.
+
+ The step script is run exactly as GitHub Actions runs a step with no
+ `shell:` key (`bash -e`, no pipefail) and as `shell: bash` would
+ (`bash -eo pipefail`), with a fake `npx` standing in for retire.js and
+ the real JSON-to-SARIF converter. An empty "retirejs" SARIF upload is
+ fresh evidence to the release gate and marks every open retirejs alert
+ as fixed, so it may only follow a scan that completed cleanly (exit 0)
+ with a well-formed report; findings (exit 13) are converted, uploaded
+ and flag the job; any other status, a missing report or a malformed
+ report fails the step with nothing to upload.
+ """
+
+ SHELLS = (
+ ("bash", "-e"),
+ ("bash", "--noprofile", "--norc", "-eo", "pipefail"),
+ )
+
+ @classmethod
+ def setUpClass(cls):
+ cls.script = _step_run_script(
+ (ROOT / ".github/workflows/retirejs.yml").read_text(
+ encoding="utf-8"
+ ),
+ "retire-scan",
+ )
+
+ def run_step(self, shell, exit_code, report=""):
+ with tempfile.TemporaryDirectory() as directory:
+ work = Path(directory)
+ bin_dir = work / "bin"
+ bin_dir.mkdir()
+ npx = bin_dir / "npx"
+ npx.write_text(_FAKE_NPX, encoding="utf-8")
+ npx.chmod(0o755)
+ converter = work / _RETIRE_CONVERTER
+ converter.parent.mkdir(parents=True)
+ shutil.copyfile(ROOT / _RETIRE_CONVERTER, converter)
+ script = work / "step.sh"
+ script.write_text(self.script, encoding="utf-8")
+ env_file, output_file = work / "env", work / "output"
+ args_file = work / "npx-args"
+ env_file.touch()
+ output_file.touch()
+ proc = subprocess.run(
+ [*shell, str(script)],
+ cwd=work,
+ env={
+ **os.environ,
+ "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}",
+ "GITHUB_ENV": str(env_file),
+ "GITHUB_OUTPUT": str(output_file),
+ "FAKE_EXIT": str(exit_code),
+ "FAKE_REPORT": report,
+ "FAKE_ARGS": str(args_file),
+ },
+ capture_output=True,
+ text=True,
+ timeout=60,
+ )
+ sarif = work / "retire-results.sarif"
+ self.npx_args = args_file.read_text(encoding="utf-8").split("\n")
+ self.step_log = proc.stdout + proc.stderr
+ return (
+ proc.returncode,
+ env_file.read_text(encoding="utf-8"),
+ output_file.read_text(encoding="utf-8"),
+ json.loads(sarif.read_text(encoding="utf-8"))
+ if sarif.exists()
+ else None,
+ )
+
+ def test_clean_scan_uploads_an_empty_sarif(self):
+ for shell in self.SHELLS:
+ with self.subTest(shell=shell):
+ code, env, output, sarif = self.run_step(
+ shell, 0, _EMPTY_REPORT
+ )
+ self.assertEqual(code, 0)
+ self.assertNotIn("RETIRE_FOUND_ISSUES", env)
+ self.assertIn("sarif_exists=true", output)
+ self.assertEqual(sarif["version"], "2.1.0")
+ run = sarif["runs"][0]
+ self.assertEqual(run["tool"]["driver"]["name"], "retire.js")
+ self.assertEqual(run["results"], [])
+
+ def test_clean_scan_still_uploads_low_severity_findings(self):
+ for shell in self.SHELLS:
+ with self.subTest(shell=shell):
+ code, env, output, sarif = self.run_step(
+ shell, 0, _LOW_ONLY_REPORT
+ )
+ self.assertEqual(code, 0)
+ self.assertNotIn("RETIRE_FOUND_ISSUES", env)
+ self.assertIn("sarif_exists=true", output)
+ results = sarif["runs"][0]["results"]
+ self.assertEqual([r["level"] for r in results], ["note"])
+
+ def test_findings_are_converted_uploaded_and_flag_the_job(self):
+ for shell in self.SHELLS:
+ with self.subTest(shell=shell):
+ code, env, output, sarif = self.run_step(
+ shell, 13, _FINDINGS_REPORT
+ )
+ self.assertEqual(code, 0)
+ self.assertIn("RETIRE_FOUND_ISSUES=true", env)
+ self.assertIn("sarif_exists=true", output)
+ results = sarif["runs"][0]["results"]
+ self.assertEqual(
+ sorted(r["level"] for r in results), ["error", "warning"]
+ )
+ self.assertEqual(
+ {
+ r["locations"][0]["physicalLocation"][
+ "artifactLocation"
+ ]["uri"]
+ for r in results
+ },
+ {"node_modules/jquery/dist/jquery.js"},
+ )
+
+ def test_missing_empty_or_malformed_reports_fail_the_step(self):
+ cases = (
+ (0, ""),
+ (0, "{not json"),
+ (0, '{"version": "5.2.4", "data": {}}'),
+ (13, ""),
+ (13, "{not json"),
+ # Exit 13 promises findings; an empty report contradicts it.
+ (13, _EMPTY_REPORT),
+ )
+ for shell in self.SHELLS:
+ for exit_code, report in cases:
+ with self.subTest(
+ shell=shell, exit_code=exit_code, report=report
+ ):
+ code, _, output, sarif = self.run_step(
+ shell, exit_code, report
+ )
+ self.assertNotEqual(code, 0)
+ self.assertNotIn("sarif_exists", output)
+ self.assertIsNone(sarif)
+ if not report:
+ # The case statement, not the converter, rejects a
+ # scan that left no report behind.
+ self.assertIn("wrote no JSON report", self.step_log)
+
+ def test_scanner_failure_fails_the_step_and_uploads_nothing(self):
+ for shell in self.SHELLS:
+ for exit_code, report in (
+ (1, ""),
+ (9, ""),
+ (1, _EMPTY_REPORT),
+ (1, _FINDINGS_REPORT),
+ ):
+ with self.subTest(
+ shell=shell, exit_code=exit_code, report=report
+ ):
+ code, env, output, sarif = self.run_step(
+ shell, exit_code, report
+ )
+ self.assertNotEqual(code, 0)
+ self.assertNotIn("sarif_exists", output)
+ self.assertNotIn("RETIRE_FOUND_ISSUES", env)
+ # Nothing may be converted for a failed scan.
+ self.assertIsNone(sarif)
+
+ def test_console_output_is_shown_with_workflow_commands_stopped(self):
+ """retire.js echoes scanned file text; it must not run as commands."""
+ for exit_code, report in ((0, _EMPTY_REPORT), (1, "")):
+ with self.subTest(exit_code=exit_code):
+ self.run_step(self.SHELLS[0], exit_code, report)
+ lines = self.step_log.splitlines()
+ stops = [
+ i
+ for i, line in enumerate(lines)
+ if line.startswith("::stop-commands::")
+ ]
+ self.assertEqual(len(stops), 1)
+ token = lines[stops[0]].removeprefix("::stop-commands::")
+ self.assertRegex(token, r"^[0-9a-f]{32}$")
+ resume = lines.index(f"::{token}::")
+ output = (
+ lines.index("fake retire.js output"),
+ lines.index("::error::from a scanned file"),
+ )
+ for index in output:
+ self.assertLess(stops[0], index)
+ self.assertLess(index, resume)
+ # The token appears only in the stop and resume lines.
+ self.assertEqual(self.step_log.count(token), 2)
+ tokens = set()
+ for _ in range(2):
+ self.run_step(self.SHELLS[0], 0, _EMPTY_REPORT)
+ tokens.add(
+ re.search(r"^::stop-commands::(\S+)$", self.step_log, re.M)[1]
+ )
+ self.assertEqual(len(tokens), 2)
+ # The wrapped cat is the step's only display of retire's output.
+ self.assertEqual(len(re.findall(r"\bcat\b", self.script)), 1)
+ self.assertIn(
+ 'echo "::stop-commands::${STOP_TOKEN}"\n'
+ "cat retire-output.txt\n"
+ 'echo "::${STOP_TOKEN}::"\n',
+ self.script,
+ )
+
+ def test_invocation_uses_only_options_retire_5_2_4_accepts(self):
+ self.run_step(self.SHELLS[0], 0, _EMPTY_REPORT)
+ args = [a for a in self.npx_args if a]
+ self.assertEqual(args[0], "[email protected]")
+ options = {}
+ index = 1
+ while index < len(args):
+ option = args[index]
+ self.assertIn(option, RETIRE_5_2_4_OPTIONS)
+ if RETIRE_5_2_4_OPTIONS[option]:
+ index += 1
+ options[option] = args[index]
+ else:
+ options[option] = True
+ index += 1
+ self.assertIn(options["--outputformat"], RETIRE_5_2_4_OUTPUT_FORMATS)
+ self.assertEqual(options["--outputformat"], "json")
+ self.assertEqual(options["--outputpath"], "retire-results.json")
+ self.assertIn(options["--severity"], RETIRE_5_2_4_SEVERITIES)
+ self.assertEqual(options.get("--exitwith", "13"), "13")
+ # Every retire invocation in the workflow is the one exercised here.
+ workflow = (ROOT / ".github/workflows/retirejs.yml").read_text(
+ encoding="utf-8"
+ )
+ self.assertEqual(len(re.findall(r"\bnpx\s+retire", workflow)), 1)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/ci/standalone/test_release_scan_retire_sarif.py b/tests/ci/standalone/test_release_scan_retire_sarif.py
new file mode 100644
index 0000000000..24d3b42d1e
--- /dev/null
+++ b/tests/ci/standalone/test_release_scan_retire_sarif.py
@@ -0,0 +1,557 @@
+"""Standalone tests for the retire.js JSON to SARIF converter."""
+
+# allow: no-sut-import — imports the standalone CI script directly.
+
+import copy
+import importlib.util
+import io
+import json
+import tempfile
+import unittest
+from contextlib import redirect_stderr, redirect_stdout
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[3]
+
+
+def _load(name, relative):
+ spec = importlib.util.spec_from_file_location(name, ROOT / relative)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+CONVERTER = _load("retire_sarif", ".github/scripts/retire_json_to_sarif.py")
+POLICY = _load(
+ "release_scan_for_retire",
+ ".github/scripts/check_release_scan_evidence.py",
+)
+
+RUNNER_ROOT = "/home/runner/work/local-deep-research/local-deep-research"
+
+# A real `npx [email protected] --severity medium --outputformat json` report
+# (exit status 13) for DOMPurify 2.0.0 and jQuery 3.4.1, trimmed to two
+# files and four vulnerabilities, with the scratch directory replaced by a
+# runner checkout path.
+REAL_REPORT = {
+ "version": "5.2.4",
+ "start": "2026-09-26T12:14:07.084Z",
+ "data": [
+ {
+ "file": f"{RUNNER_ROOT}/node_modules/dompurify/dist/purify.js",
+ "results": [
+ {
+ "version": "2.0.0",
+ "component": "DOMPurify",
+ "npmname": "dompurify",
+ "detection": "filecontent",
+ "vulnerabilities": [
+ {
+ "info": [
+ "https://github.com/cure53/DOMPurify/releases"
+ ],
+ "below": "2.0.3",
+ "severity": "medium",
+ "identifiers": {
+ "summary": "Fixed an mXSS-based bypass "
+ "caused by nested forms inside MathML",
+ "githubID": "GHSA-chqj-j4fh-rw7m",
+ "CVE": ["CVE-2019-16728"],
+ },
+ "cwe": ["CWE-79"],
+ },
+ {
+ "info": [
+ "https://github.com/cure53/DOMPurify/releases"
+ ],
+ "below": "2.0.7",
+ "severity": "high",
+ "identifiers": {
+ "summary": "possible to bypass the package "
+ "sanitization through Mutation XSS",
+ "githubID": "GHSA-mjjq-c88q-qhr6",
+ },
+ "cwe": ["CWE-79"],
+ },
+ {
+ "info": [
+ "https://github.com/cure53/DOMPurify/releases"
+ ],
+ "below": "2.0.16",
+ "severity": "low",
+ "identifiers": {
+ "summary": "Fixed an mXSS-based bypass "
+ "caused by nested forms inside MathML",
+ "retid": "28",
+ },
+ "cwe": ["CWE-79"],
+ },
+ ],
+ "licenses": ["MPL-2.0 OR Apache-2.0"],
+ }
+ ],
+ },
+ {
+ "file": f"{RUNNER_ROOT}/node_modules/jquery/dist/jquery.min.js",
+ "results": [
+ {
+ "version": "3.4.1",
+ "component": "jquery",
+ "npmname": "jquery",
+ "detection": "filecontent",
+ "vulnerabilities": [
+ {
+ "info": [
+ "https://blog.jquery.com/2020/04/10/"
+ "jquery-3-5-0-released/"
+ ],
+ "below": "3.5.0",
+ "atOrAbove": "1.0.3",
+ "severity": "medium",
+ "identifiers": {
+ "summary": "passing HTML containing <option> "
+ "elements from untrusted sources - even after "
+ "sanitizing it - to one of jQuery's DOM "
+ "manipulation methods (i.e. .html(), "
+ ".append(), and others) may execute "
+ "untrusted code.",
+ "CVE": ["CVE-2020-11023"],
+ "issue": "4647",
+ "githubID": "GHSA-jpcq-cgw6-v4j6",
+ },
+ "cwe": ["CWE-79"],
+ }
+ ],
+ "licenses": ["MIT"],
+ }
+ ],
+ },
+ ],
+ "messages": [],
+ "errors": [],
+ "time": 2.244,
+}
+
+# A real clean report (exit status 0) for this repository's checkout.
+EMPTY_REPORT = {
+ "version": "5.2.4",
+ "start": "2026-09-26T12:12:41.179Z",
+ "data": [],
+ "messages": [],
+ "errors": [],
+ "time": 2.22,
+}
+
+
+def _vulnerability(severity=None, **identifiers):
+ vulnerability = {
+ "info": ["https://example.invalid/advisory"],
+ "below": "2.0.0",
+ "identifiers": {"summary": "an advisory", **identifiers},
+ }
+ if severity is not None:
+ vulnerability["severity"] = severity
+ return vulnerability
+
+
+def _report(*vulnerabilities, file="node_modules/lib/dist/lib.js"):
+ return {
+ "version": "5.2.4",
+ "data": [
+ {
+ "file": file,
+ "results": [
+ {
+ "version": "1.0.0",
+ "component": "lib",
+ "vulnerabilities": list(vulnerabilities),
+ }
+ ],
+ }
+ ],
+ "errors": [],
+ }
+
+
+def _results_by_rule(sarif):
+ return {r["ruleId"]: r for r in sarif["runs"][0]["results"]}
+
+
+def _rules(sarif):
+ return {r["id"]: r for r in sarif["runs"][0]["tool"]["driver"]["rules"]}
+
+
+class ConvertTests(unittest.TestCase):
+ def test_real_report_becomes_sarif_with_relative_locations(self):
+ sarif = CONVERTER.convert(REAL_REPORT, RUNNER_ROOT)
+ self.assertEqual(sarif["version"], "2.1.0")
+ run = sarif["runs"][0]
+ self.assertEqual(run["tool"]["driver"]["name"], "retire.js")
+ self.assertEqual(run["tool"]["driver"]["version"], "5.2.4")
+ results = _results_by_rule(sarif)
+ self.assertEqual(
+ sorted(results),
+ [
+ "GHSA-chqj-j4fh-rw7m",
+ "GHSA-jpcq-cgw6-v4j6",
+ "GHSA-mjjq-c88q-qhr6",
+ "RETIRE-28",
+ ],
+ )
+ self.assertEqual(sorted(_rules(sarif)), sorted(results))
+ self.assertEqual(
+ {
+ rule: (r["level"], r["properties"]["security-severity"])
+ for rule, r in results.items()
+ },
+ {
+ "GHSA-chqj-j4fh-rw7m": ("warning", "5.5"),
+ "GHSA-jpcq-cgw6-v4j6": ("warning", "5.5"),
+ "GHSA-mjjq-c88q-qhr6": ("error", "8.0"),
+ "RETIRE-28": ("note", "2.0"),
+ },
+ )
+ location = results["GHSA-jpcq-cgw6-v4j6"]["locations"][0][
+ "physicalLocation"
+ ]["artifactLocation"]
+ self.assertEqual(
+ location["uri"], "node_modules/jquery/dist/jquery.min.js"
+ )
+ self.assertEqual(location["uriBaseId"], "%SRCROOT%")
+ self.assertIn(
+ "CVE-2020-11023", results["GHSA-jpcq-cgw6-v4j6"]["message"]["text"]
+ )
+ # No absolute runner path may reach code scanning.
+ self.assertNotIn(RUNNER_ROOT, json.dumps(sarif))
+ self.assertNotIn('"/', json.dumps(run["results"]))
+
+ def test_empty_report_is_a_valid_sarif_with_no_results(self):
+ sarif = CONVERTER.convert(EMPTY_REPORT, RUNNER_ROOT)
+ self.assertEqual(sarif["version"], "2.1.0")
+ self.assertEqual(len(sarif["runs"]), 1)
+ self.assertEqual(sarif["runs"][0]["results"], [])
+ self.assertEqual(sarif["runs"][0]["tool"]["driver"]["rules"], [])
+ self.assertEqual(
+ sarif["runs"][0]["tool"]["driver"]["name"], "retire.js"
+ )
+
+ def test_severity_mapping(self):
+ expected = {
+ "critical": ("error", "9.5"),
+ "high": ("error", "8.0"),
+ "medium": ("warning", "5.5"),
+ "low": ("note", "2.0"),
+ # retire.js's schema allows "none": an unscored note.
+ "none": ("note", None),
+ # retire.js treats a missing severity as critical.
+ None: ("error", "9.5"),
+ }
+ for severity, (level, score) in expected.items():
+ with self.subTest(severity=severity):
+ sarif = CONVERTER.convert(
+ _report(_vulnerability(severity, githubID="GHSA-1")), "/r"
+ )
+ result = sarif["runs"][0]["results"][0]
+ rule = sarif["runs"][0]["tool"]["driver"]["rules"][0]
+ self.assertEqual(result["level"], level)
+ self.assertEqual(
+ result["properties"].get("security-severity"), score
+ )
+ self.assertEqual(rule["defaultConfiguration"]["level"], level)
+ self.assertEqual(
+ rule["properties"].get("security-severity"), score
+ )
+
+ def test_null_severity_is_critical_like_retire(self):
+ # retire.js 5.2.4 uses `v.severity ?? 'critical'`, so an explicit
+ # JSON null is critical too, not an unknown severity.
+ vulnerability = _vulnerability(githubID="GHSA-1")
+ vulnerability["severity"] = None
+ sarif = CONVERTER.convert(_report(vulnerability), "/r")
+ result = sarif["runs"][0]["results"][0]
+ self.assertEqual(result["level"], "error")
+ self.assertEqual(result["properties"]["security-severity"], "9.5")
+
+ def test_a_none_severity_advisory_does_not_reject_the_report(self):
+ report = copy.deepcopy(REAL_REPORT)
+ report["data"][1]["results"][0]["vulnerabilities"].append(
+ _vulnerability("none", githubID="GHSA-none")
+ )
+ sarif = CONVERTER.convert(report, RUNNER_ROOT)
+ results = _results_by_rule(sarif)
+ self.assertEqual(len(results), 5)
+ none = results["GHSA-none"]
+ self.assertEqual(none["level"], "note")
+ self.assertNotIn("security-severity", none["properties"])
+ self.assertNotIn("none vulnerability", none["message"]["text"])
+ self.assertIn("rates severity none", none["message"]["text"])
+ rule = _rules(sarif)["GHSA-none"]
+ self.assertNotIn("security-severity", rule["properties"])
+
+ def test_mapping_blocks_exactly_what_the_release_gate_blocks(self):
+ """The alert gate must block critical/high/medium, never low."""
+
+ def github_band(score):
+ # GitHub's documented security-severity bands.
+ score = float(score)
+ if score >= 9.0:
+ return "critical"
+ if score >= 7.0:
+ return "high"
+ if score >= 4.0:
+ return "medium"
+ return "low"
+
+ for severity in ("critical", "high", "medium", "low", "none"):
+ with self.subTest(severity=severity):
+ sarif = CONVERTER.convert(
+ _report(_vulnerability(severity, githubID="GHSA-1")), "/r"
+ )
+ rule = sarif["runs"][0]["tool"]["driver"]["rules"][0]
+ score = rule["properties"].get("security-severity")
+ if severity == "none":
+ # Unscored: GitHub reports no security severity level,
+ # so only the SARIF-level fallback applies.
+ self.assertIsNone(score)
+ bands = (None,)
+ else:
+ band = github_band(score)
+ self.assertEqual(band, severity)
+ bands = (band, None)
+ blocks = severity not in {"low", "none"}
+ for scored in bands:
+ alert = {
+ "number": 1,
+ "state": "open",
+ "tool": {"name": "retire.js"},
+ "rule": {
+ "security_severity_level": scored,
+ "severity": rule["defaultConfiguration"]["level"],
+ },
+ }
+ self.assertEqual(POLICY.blocking_alert(alert), blocks)
+
+ def test_rule_takes_the_highest_severity_of_a_shared_identifier(self):
+ sarif = CONVERTER.convert(
+ _report(
+ _vulnerability("low", githubID="GHSA-1"),
+ _vulnerability("high", githubID="GHSA-1"),
+ ),
+ "/r",
+ )
+ rule = _rules(sarif)["GHSA-1"]
+ self.assertEqual(rule["properties"]["security-severity"], "8.0")
+
+ def test_a_repeated_finding_keeps_its_highest_severity(self):
+ for order in (("low", "high"), ("high", "low")):
+ with self.subTest(order=order):
+ sarif = CONVERTER.convert(
+ _report(
+ *(
+ _vulnerability(severity, githubID="GHSA-1")
+ for severity in order
+ )
+ ),
+ "/r",
+ )
+ results = sarif["runs"][0]["results"]
+ self.assertEqual(len(results), 1)
+ result = results[0]
+ self.assertEqual(result["level"], "error")
+ self.assertEqual(
+ result["properties"]["security-severity"], "8.0"
+ )
+ self.assertIn(
+ "known high vulnerability", result["message"]["text"]
+ )
+ # The upgrade replaces the result in place: order is preserved.
+ sarif = CONVERTER.convert(
+ _report(
+ _vulnerability("low", githubID="GHSA-1"),
+ _vulnerability("medium", githubID="GHSA-2"),
+ _vulnerability("critical", githubID="GHSA-1"),
+ ),
+ "/r",
+ )
+ self.assertEqual(
+ [(r["ruleId"], r["level"]) for r in sarif["runs"][0]["results"]],
+ [("GHSA-1", "error"), ("GHSA-2", "warning")],
+ )
+
+ def test_a_repeated_finding_in_one_file_is_reported_once(self):
+ vulnerability = _vulnerability("medium", githubID="GHSA-1")
+ report = _report(vulnerability, vulnerability)
+ report["data"].append(copy.deepcopy(report["data"][0]))
+ sarif = CONVERTER.convert(report, "/r")
+ self.assertEqual(len(sarif["runs"][0]["results"]), 1)
+ report["data"][1]["file"] = "node_modules/lib/dist/lib.min.js"
+ sarif = CONVERTER.convert(report, "/r")
+ self.assertEqual(len(sarif["runs"][0]["results"]), 2)
+
+ def test_rule_identifier_preference(self):
+ cases = (
+ ({"githubID": "GHSA-a", "CVE": ["CVE-1"]}, "GHSA-a"),
+ ({"CVE": ["CVE-1", "CVE-2"], "retid": "7"}, "CVE-1"),
+ ({"retid": "7", "issue": "9"}, "RETIRE-7"),
+ ({"issue": "9"}, "issue-9"),
+ )
+ for identifiers, rule_id in cases:
+ with self.subTest(identifiers=identifiers):
+ sarif = CONVERTER.convert(
+ _report(_vulnerability("medium", **identifiers)), "/r"
+ )
+ self.assertEqual(list(_results_by_rule(sarif)), [rule_id])
+ sarif = CONVERTER.convert(_report(_vulnerability("medium")), "/r")
+ self.assertRegex(
+ sarif["runs"][0]["results"][0]["ruleId"],
+ r"^RETIRE-lib-[0-9a-f]{16}$",
+ )
+
+ def test_relative_and_absolute_paths_share_one_uri(self):
+ for file in ("/r/a/b.js", "a/b.js", "/r/a/./c/../b.js"):
+ with self.subTest(file=file):
+ sarif = CONVERTER.convert(
+ _report(_vulnerability("low", retid="1"), file=file), "/r"
+ )
+ uri = sarif["runs"][0]["results"][0]["locations"][0][
+ "physicalLocation"
+ ]["artifactLocation"]["uri"]
+ self.assertEqual(uri, "a/b.js")
+
+ def test_malformed_reports_raise(self):
+ def mutate(change):
+ report = copy.deepcopy(REAL_REPORT)
+ change(report)
+ return report
+
+ def set_vuln(key, value):
+ def change(report):
+ vuln = report["data"][0]["results"][0]["vulnerabilities"][0]
+ vuln[key] = value
+
+ return change
+
+ malformed = {
+ "not an object": [],
+ "no version": mutate(lambda r: r.pop("version")),
+ "data missing": mutate(lambda r: r.pop("data")),
+ "data not a list": mutate(lambda r: r.update(data={})),
+ "entry not an object": mutate(lambda r: r.update(data=["x"])),
+ "file missing": mutate(lambda r: r["data"][0].pop("file")),
+ "file outside root": mutate(
+ lambda r: r["data"][0].update(file="/etc/passwd.js")
+ ),
+ "file escapes root": mutate(
+ lambda r: r["data"][0].update(file="../outside.js")
+ ),
+ "results not a list": mutate(
+ lambda r: r["data"][0].update(results={})
+ ),
+ "component missing": mutate(
+ lambda r: r["data"][0]["results"][0].pop("component")
+ ),
+ "version not a string": mutate(
+ lambda r: r["data"][0]["results"][0].update(version=2)
+ ),
+ "vulnerabilities not a list": mutate(
+ lambda r: r["data"][0]["results"][0].update(vulnerabilities={})
+ ),
+ "unknown severity": mutate(set_vuln("severity", "moderate")),
+ "identifiers not a map": mutate(set_vuln("identifiers", [])),
+ "info not a list": mutate(set_vuln("info", "https://x")),
+ "errors not a list": mutate(lambda r: r.update(errors="boom")),
+ }
+ for name, report in malformed.items():
+ with self.subTest(name=name):
+ with self.assertRaises(CONVERTER.ReportError):
+ CONVERTER.convert(report, RUNNER_ROOT)
+
+
+class MainTests(unittest.TestCase):
+ def run_main(self, report_text, *flags, existing_output=False):
+ with tempfile.TemporaryDirectory() as directory:
+ work = Path(directory)
+ report = work / "retire-results.json"
+ output = work / "retire-results.sarif"
+ if report_text is not None:
+ report.write_text(report_text, encoding="utf-8")
+ if existing_output:
+ output.write_text("stale", encoding="utf-8")
+ stderr = io.StringIO()
+ with redirect_stdout(io.StringIO()), redirect_stderr(stderr):
+ code = CONVERTER.main(
+ [*flags, "--root", str(work), str(report), str(output)]
+ )
+ written = (
+ output.read_text(encoding="utf-8") if output.exists() else None
+ )
+ leftovers = sorted(p.name for p in work.iterdir())
+ return code, written, stderr.getvalue(), leftovers
+
+ def test_clean_report_writes_an_empty_sarif(self):
+ code, written, _, leftovers = self.run_main(json.dumps(EMPTY_REPORT))
+ self.assertEqual(code, 0)
+ self.assertEqual(json.loads(written)["runs"][0]["results"], [])
+ self.assertEqual(
+ leftovers, ["retire-results.json", "retire-results.sarif"]
+ )
+
+ def test_findings_report_writes_results(self):
+ report = _report(_vulnerability("high", githubID="GHSA-1"))
+ code, written, _, _ = self.run_main(
+ json.dumps(report), "--require-findings"
+ )
+ self.assertEqual(code, 0)
+ self.assertEqual(len(json.loads(written)["runs"][0]["results"]), 1)
+
+ def test_require_findings_accepts_a_none_severity_only_report(self):
+ # With --severity none, retire.js exits 13 for a "none" advisory.
+ report = _report(_vulnerability("none", githubID="GHSA-1"))
+ code, written, _, _ = self.run_main(
+ json.dumps(report), "--require-findings"
+ )
+ self.assertEqual(code, 0)
+ results = json.loads(written)["runs"][0]["results"]
+ self.assertEqual([r["level"] for r in results], ["note"])
+
+ def test_require_findings_rejects_an_empty_report(self):
+ code, written, stderr, _ = self.run_main(
+ json.dumps(EMPTY_REPORT), "--require-findings"
+ )
+ self.assertEqual(code, 1)
+ self.assertIsNone(written)
+ self.assertIn("::error::", stderr)
+
+ def test_malformed_input_fails_and_writes_nothing(self):
+ cases = {
+ "missing file": None,
+ "empty file": "",
+ "not json": "{not json",
+ "truncated": json.dumps(REAL_REPORT)[:-10],
+ "wrong shape": json.dumps({"version": "5.2.4", "data": {}}),
+ }
+ for name, text in cases.items():
+ with self.subTest(name=name):
+ code, written, stderr, _ = self.run_main(
+ text, existing_output=True
+ )
+ self.assertEqual(code, 1)
+ # A stale SARIF from an earlier step must not survive either.
+ self.assertIsNone(written)
+ self.assertIn("::error::", stderr)
+
+ def test_error_messages_cannot_inject_workflow_commands(self):
+ self.assertEqual(
+ CONVERTER._workflow_command_data("a%b\r\n::error::c"),
+ "a%25b%0D%0A::error::c",
+ )
+ report = copy.deepcopy(EMPTY_REPORT)
+ report["errors"] = ["Could not parse file: a.js\n::error::injected"]
+ code, _, stderr, _ = self.run_main(json.dumps(report))
+ self.assertEqual(code, 0)
+ self.assertEqual(stderr.count("\n"), 1)
+ self.assertIn("%0A::error::injected", stderr)
+
+
+if __name__ == "__main__":
+ unittest.main()