~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(ci): require fresh release scan evidence before alert checks (#6646)
LearningCircuit577f30f25692
diff --git a/.file-whitelist.txt b/.file-whitelist.txtindex 3a1874f81e..a0d4478dd4 100644--- a/.file-whitelist.txt+++ b/.file-whitelist.txt@@ -29,6 +29,7 @@^package-lock\.json$^lighthouserc\.json$^\.github/ai-review-label-policy\.json$+^\.github/release-scan-policy\.json$^tests/(.*/)?package\.json$^tests/(.*/)?package-lock\.json$^tests/settings/golden_master_settings\.json$diff --git a/.github/release-scan-policy.json b/.github/release-scan-policy.jsonnew file mode 100644index 0000000000..2500f2cdb7--- /dev/null+++ b/.github/release-scan-policy.json@@ -0,0 +1,19 @@+{+ "/language:python": "analyze",+ "/language:javascript-typescript": "analyze",+ "semgrep-security": "semgrep-scan",+ "bandit": "security-tests",+ "bearer-security": "bearer-scan",+ "devskim": "devskim-scan",+ "container-security": "trivy-scan",+ "grype-filesystem": "grype-filesystem",+ "grype-container": "grype-container",+ "dockle": "dockle",+ "hadolint": "hadolint",+ "checkov-dockerfile": "checkov",+ "checkov-compose": "checkov",+ "checkov-workflows": "checkov",+ "retirejs": "retirejs",+ "zizmor": "zizmor-scan",+ "gitleaks-release": "gitleaks-scan"+}diff --git a/.github/scripts/check_release_scan_evidence.py b/.github/scripts/check_release_scan_evidence.pynew file mode 100644index 0000000000..63684a6dc4--- /dev/null+++ b/.github/scripts/check_release_scan_evidence.py@@ -0,0 +1,247 @@+"""Require fresh release scan analyses before evaluating open alerts."""++import json+import os+import re+import subprocess+import time+from datetime import datetime+from pathlib import Path+from urllib.parse import urlencode++# gh redacts the token from its own error output, but be defensive: never let+# anything resembling a GitHub token reach a public log via a wrapped+# CalledProcessError's stderr.+_TOKEN_PATTERN = re.compile(+ r"(?:ghp|gho|ghs|ghr|ghu|github_pat)_[A-Za-z0-9_]{10,}"+)+++def timestamp(value):+ parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))+ if parsed.tzinfo is None:+ raise ValueError("API timestamp has no timezone")+ return parsed+++def api(endpoint):+ result = subprocess.run(+ [+ "gh",+ "api",+ "--method",+ "GET",+ "-H",+ "Accept: application/vnd.github+json",+ "-H",+ "X-GitHub-Api-Version: 2022-11-28",+ endpoint,+ ],+ check=True,+ capture_output=True,+ text=True,+ timeout=60,+ )+ return json.loads(result.stdout)+++def pages(endpoint, params, fetch=api):+ for page in range(1, 1001):+ data = fetch(+ endpoint+ + "?"+ + urlencode({**params, "per_page": 100, "page": page})+ )+ if not isinstance(data, list) or any(+ not isinstance(row, dict) for row in data+ ):+ raise ValueError("Expected an array of API records")+ yield data+ if len(data) < 100:+ return+ raise ValueError("API pagination exceeded the safety limit")+++def recent_analyses(endpoint, ref, started, fetch=api):+ records = []+ for page in pages(+ endpoint, {"ref": ref, "sort": "created", "direction": "desc"}, fetch+ ):+ records.extend(+ row for row in page if timestamp(row["created_at"]) >= started+ )+ if any(timestamp(row["created_at"]) < started for row in page):+ break+ return records+++def check_analyses(records, expected, sha, ref, workflow, started):+ """Return missing categories; reject failed/malformed matching analyses."""+ if not isinstance(expected, dict) or not expected:+ raise ValueError("Release scan policy is empty or malformed")+ found = {}+ for row in records:+ category = row.get("category")+ if category not in expected:+ continue+ if (+ row.get("ref") != ref+ or row.get("analysis_key") != f"{workflow}:{expected[category]}"+ or timestamp(row["created_at"]) < started+ ):+ continue+ previous = found.get(category)+ if previous is None or timestamp(row["created_at"]) > timestamp(+ previous["created_at"]+ ):+ found[category] = row+ for category, row in found.items():+ if row.get("commit_sha") != sha:+ raise ValueError(+ f"{category}: latest analysis belongs to another commit"+ )+ if row.get("error") != "":+ raise ValueError(+ f"{category}: failed analysis or missing error status"+ )+ # Zero findings are legitimate. Missing result counts are not evidence.+ if (+ type(row.get("results_count")) is not int+ or row["results_count"] < 0+ ):+ raise ValueError(f"{category}: invalid analysis result count")+ return sorted(set(expected) - found.keys())+++def _alert_label(alert):+ number = alert.get("number")+ tool = (alert.get("tool") or {}).get("name") or "unknown tool"+ return f"alert #{number} ({tool})"+++def blocking_alert(alert):+ if alert.get("state") != "open":+ raise ValueError(+ f"Open-alert query returned an unexpected alert state for "+ f"{_alert_label(alert)}: {alert.get('state')!r}"+ )+ rule = alert["rule"]+ security = rule.get("security_severity_level")+ if security in {"critical", "high", "medium"}:+ return True+ if security == "low":+ return False+ if security is not None:+ raise ValueError(+ f"Unknown security severity {security!r} in {_alert_label(alert)}"+ )+ # Native SARIF can provide a level without a numerical security score.+ level = rule.get("severity")+ if level not in {"error", "warning", "note", "none"}:+ raise ValueError(+ f"No recognized severity {level!r} in {_alert_label(alert)}"+ )+ return level in {"error", "warning"}+++def evaluate(fetch=api, sleep=time.sleep, monotonic=time.monotonic):+ repo = os.environ["GITHUB_REPOSITORY"]+ sha, ref = os.environ["GITHUB_SHA"], os.environ["GITHUB_REF"]+ root = f"repos/{repo}"+ run = fetch(f"{root}/actions/runs/{os.environ['GITHUB_RUN_ID']}")+ if run["head_sha"] != sha:+ raise ValueError("Workflow run does not match the release commit")+ # created_at retains successful scans from earlier attempts of a rerun.+ # run_started_at would incorrectly reject those when rerunning failed jobs.+ started = timestamp(run["created_at"])+ workflow = run["path"].split("@", 1)[0]+ policy_path = (+ Path(__file__).resolve().parents[1] / "release-scan-policy.json"+ )+ expected = json.loads(policy_path.read_text(encoding="utf-8"))+ deadline = monotonic() + 300+ while True:+ analyses = recent_analyses(+ f"{root}/code-scanning/analyses", ref, started, fetch+ )+ missing = check_analyses(+ analyses, expected, sha, ref, workflow, started+ )+ if not missing:+ break+ if monotonic() >= deadline:+ raise ValueError(+ "No fresh successful analysis for: " + ", ".join(missing)+ )+ print("Waiting for analyses: " + ", ".join(missing), flush=True)+ sleep(15)+ # ref scopes alerts to the release branch/tag instead of the default branch.+ alerts = [+ row+ for page in pages(+ f"{root}/code-scanning/alerts", {"state": "open", "ref": ref}, fetch+ )+ for row in page+ ]+ blocking = [row for row in alerts if blocking_alert(row)]+ with open(+ os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8"+ ) as summary:+ summary.write("## Release scan evidence\n\n")+ summary.write(+ f"Validated {len(expected)} categories for `{sha}` on `{ref}` since {run['created_at']}.\n\n"+ )+ summary.write(+ f"Open blocking alerts: **{len(blocking)}** (critical/high/medium or unscored error/warning).\n"+ )+ # Do not copy findings or source excerpts into publicly visible logs.+ for alert in blocking:+ number = int(alert["number"])+ summary.write(+ f"- [Alert #{number}](https://github.com/{repo}/security/code-scanning/{number})\n"+ )+ if blocking:+ first_n = 5+ named = ", ".join(_alert_label(alert) for alert in blocking[:first_n])+ if len(blocking) > first_n:+ named += f", and {len(blocking) - first_n} more"+ raise ValueError(+ f"{len(blocking)} blocking code-scanning alerts remain open: "+ + named+ )+ print(+ f"Release scan policy passed for {sha}: {len(expected)} fresh categories"+ )+++def main():+ try:+ evaluate()+ except (+ OSError,+ ValueError,+ KeyError,+ TypeError,+ AttributeError,+ subprocess.SubprocessError,+ ) as exc:+ # CalledProcessError's default str() omits the captured API response+ # body, so surface gh's own stderr too (truncated: it can be a large+ # JSON/HTML error page). gh already redacts the token from its+ # output; _TOKEN_PATTERN is a defensive second layer so a token can+ # never reach a public workflow log through this path.+ detail = str(exc)+ stderr = getattr(exc, "stderr", None)+ if stderr:+ if isinstance(stderr, bytes):+ stderr = stderr.decode("utf-8", "replace")+ snippet = _TOKEN_PATTERN.sub("[REDACTED]", stderr.strip())[:500]+ if snippet:+ detail = f"{detail} | gh stderr: {snippet}"+ print(f"::error::Release scan evidence failed: {detail}")+ return 1+ return 0+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/.github/scripts/retire_json_to_sarif.py b/.github/scripts/retire_json_to_sarif.pynew file mode 100644index 0000000000..95eb516dbd--- /dev/null+++ b/.github/scripts/retire_json_to_sarif.py@@ -0,0 +1,373 @@+#!/usr/bin/env python3+"""Convert a retire.js 5.x JSON report into SARIF 2.1.0 for code scanning.++retire.js 5.x has no SARIF output format (its formats are text, json,+jsonsimple, depcheck and the CycloneDX variants), so retirejs.yml asks for+``--outputformat json`` and uploads this conversion under the "retirejs"+category that the release scan evidence gate requires.++Severity mapping, consistent with check_release_scan_evidence.py, which+blocks a release on critical/high/medium security severities and ignores low:++ retire severity SARIF level security-severity (GitHub band)+ critical error 9.5 (critical)+ high error 8.0 (high)+ medium warning 5.5 (medium)+ low note 2.0 (low)+ none note (omitted)++retire.js's repository schema allows "none" (lib/types.js severityLevels).+Such an advisory is still uploaded, as a note, so that it stays visible and+so that a report retire.js flagged with its findings status (a --severity of+none counts it) still has results, but it carries no security-severity:+GitHub's bands start at 0.1, and an unscored note is exactly what the+release gate's SARIF-level fallback treats as non-blocking. Rejecting it+instead would fail the whole report and block the release.++A vulnerability whose severity is missing or null is treated as critical,+as retire.js itself does when deciding its exit status+(``v.severity ?? 'critical'`` covers both).++The conversion fails closed: any report that does not have the shape+retire.js 5.x writes raises, the script exits nonzero and no SARIF file is+written, so the workflow uploads nothing instead of an empty analysis that+would mark every open retirejs alert as fixed.+"""++from __future__ import annotations++import argparse+import hashlib+import json+import sys+from pathlib import Path, PurePosixPath++INFORMATION_URI = "https://github.com/RetireJS/retire.js"++# retire severity -> (SARIF level, GitHub security-severity score, rank).+# The ranks are retire.js 5.2.4's own severityLevels (lib/types.js). A None+# score means the security-severity property is omitted.+SEVERITIES = {+ "critical": ("error", "9.5", 4),+ "high": ("error", "8.0", 3),+ "medium": ("warning", "5.5", 2),+ "low": ("note", "2.0", 1),+ "none": ("note", None, 0),+}+# retire.js 5.2.4 (lib/cli.js): `severityLevels[v.severity ?? 'critical']`,+# so a missing and a null severity are both critical.+MISSING_SEVERITY = "critical"+++class ReportError(ValueError):+ """The retire.js report does not have the expected shape."""+++def _workflow_command_data(text):+ """Escape text for a GitHub Actions ::error::/::warning:: command."""+ return (+ str(text).replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A")+ )+++def _require(condition, message):+ if not condition:+ raise ReportError(message)+++def _string_list(value, what):+ _require(+ isinstance(value, list) and all(isinstance(v, str) for v in value),+ f"{what} must be a list of strings",+ )+ return value+++def _relative_uri(file_name, root):+ _require(+ isinstance(file_name, str) and file_name.strip() != "",+ "data[].file must be a non-empty string",+ )+ # Both sides are resolved, so ".." segments and symlinks cannot place a+ # location outside the checkout; a file that really lives outside it+ # (for example through a symlink) is rejected rather than reported+ # under a path code scanning cannot map.+ root = Path(root).resolve()+ path = (root / file_name).resolve()+ try:+ relative = path.relative_to(root)+ except ValueError:+ raise ReportError(+ f"scanned file {file_name!r} is outside the repository root"+ ) from None+ _require(relative.parts, f"scanned file {file_name!r} is the root")+ return PurePosixPath(*relative.parts).as_posix()+++def _identifier(vulnerability, component):+ """A stable rule id for one vulnerability, preferring GHSA over CVE."""+ identifiers = vulnerability.get("identifiers", {})+ _require(+ isinstance(identifiers, dict), "vulnerability identifiers must be a map"+ )+ ghsa = identifiers.get("githubID")+ if ghsa is not None:+ _require(isinstance(ghsa, str) and ghsa, "githubID must be a string")+ return ghsa+ cves = identifiers.get("CVE")+ if cves is not None:+ _string_list(cves, "CVE")+ if cves:+ return cves[0]+ retid = identifiers.get("retid")+ if retid is not None:+ _require(isinstance(retid, str) and retid, "retid must be a string")+ return f"RETIRE-{retid}"+ for key in sorted(identifiers):+ if key == "summary":+ continue+ value = identifiers[key]+ if isinstance(value, list):+ value = value[0] if value else None+ if isinstance(value, str) and value:+ return f"{key}-{value}"+ # No identifier at all: derive one from what describes the advisory.+ digest = hashlib.sha256(+ json.dumps(+ [component, identifiers.get("summary"), vulnerability.get("below")],+ sort_keys=True,+ ).encode("utf-8")+ ).hexdigest()[:16]+ return f"RETIRE-{component}-{digest}"+++def _severity(vulnerability):+ severity = vulnerability.get("severity")+ if severity is None:+ severity = MISSING_SEVERITY+ _require(+ severity in SEVERITIES,+ f"unknown retire.js severity {severity!r}",+ )+ return severity+++def _message(component, version, severity, vulnerability):+ identifiers = vulnerability.get("identifiers", {})+ summary = identifiers.get("summary")+ ids = []+ for key in ("githubID", "CVE", "retid", "issue", "bug", "PR"):+ value = identifiers.get(key)+ if isinstance(value, list):+ ids.extend(str(v) for v in value)+ elif value is not None:+ ids.append(f"retid {value}" if key == "retid" else str(value))+ if severity == "none":+ text = (+ f"{component} {version} has a known vulnerability that "+ "retire.js rates severity none"+ )+ else:+ text = f"{component} {version} has a known {severity} vulnerability"+ if isinstance(summary, str) and summary.strip():+ text += f": {summary.strip()}"+ if ids:+ text += f" ({', '.join(ids)})"+ below = vulnerability.get("below")+ if isinstance(below, str) and below:+ text += f". Affected versions are below {below}"+ return text + "."+++def convert(report, root):+ """Return a SARIF 2.1.0 document for a parsed retire.js JSON report."""+ _require(isinstance(report, dict), "report must be a JSON object")+ tool_version = report.get("version")+ _require(+ isinstance(tool_version, str) and tool_version,+ "report version must be a non-empty string",+ )+ data = report.get("data")+ _require(isinstance(data, list), "report data must be a list")+ _string_list(report.get("errors", []), "report errors")++ rules = {}+ results = []+ # (rule id, uri, component, version) -> index of its result in results+ seen = {}+ for entry in data:+ _require(isinstance(entry, dict), "data[] entries must be objects")+ uri = _relative_uri(entry.get("file"), root)+ components = entry.get("results")+ _require(isinstance(components, list), "data[].results must be a list")+ for found in components:+ _require(isinstance(found, dict), "results[] must be objects")+ component = found.get("component")+ version = found.get("version")+ _require(+ isinstance(component, str) and component,+ "results[].component must be a non-empty string",+ )+ _require(+ isinstance(version, str), "results[].version must be a string"+ )+ vulnerabilities = found.get("vulnerabilities", [])+ _require(+ isinstance(vulnerabilities, list),+ "results[].vulnerabilities must be a list",+ )+ for vulnerability in vulnerabilities:+ _require(+ isinstance(vulnerability, dict),+ "vulnerabilities[] must be objects",+ )+ info = _string_list(+ vulnerability.get("info", []), "vulnerability info"+ )+ severity = _severity(vulnerability)+ rule_id = _identifier(vulnerability, component)+ level, score, rank = SEVERITIES[severity]+ rule = rules.get(rule_id)+ if rule is None or rank > rule["rank"]:+ rules[rule_id] = {+ "rank": rank,+ "severity": severity,+ "component": component,+ "summary": vulnerability.get("identifiers", {}).get(+ "summary"+ ),+ "info": info,+ }+ properties = {"component": component, "version": version}+ if score is not None:+ properties = {"security-severity": score, **properties}+ result = {+ "ruleId": rule_id,+ "level": level,+ "message": {+ "text": _message(+ component, version, severity, vulnerability+ )+ },+ "locations": [+ {+ "physicalLocation": {+ "artifactLocation": {+ "uri": uri,+ "uriBaseId": "%SRCROOT%",+ },+ "region": {"startLine": 1},+ }+ }+ ],+ "properties": properties,+ }+ # One result per rule, file and component version, carrying+ # the highest severity reported for it (with its level and+ # message), whatever order retire.js listed them in.+ key = (rule_id, uri, component, version)+ previous = seen.get(key)+ if previous is None:+ seen[key] = (len(results), rank)+ results.append(result)+ elif rank > previous[1]:+ seen[key] = (previous[0], rank)+ results[previous[0]] = result++ sarif_rules = []+ for rule_id in sorted(rules):+ rule = rules[rule_id]+ level, score, _ = SEVERITIES[rule["severity"]]+ summary = rule["summary"]+ if not (isinstance(summary, str) and summary.strip()):+ summary = f"Known vulnerability in {rule['component']}"+ sarif_rule = {+ "id": rule_id,+ "name": rule_id,+ "shortDescription": {"text": summary.strip()},+ "fullDescription": {+ "text": f"{rule['component']}: {summary.strip()}"+ },+ "defaultConfiguration": {"level": level},+ "properties": {+ "tags": ["security", "vulnerability", "retire.js"],+ },+ }+ if score is not None:+ sarif_rule["properties"]["security-severity"] = score+ if rule["info"]:+ sarif_rule["helpUri"] = rule["info"][0]+ sarif_rule["help"] = {"text": "\n".join(rule["info"])}+ sarif_rules.append(sarif_rule)++ return {+ "$schema": "https://json.schemastore.org/sarif-2.1.0.json",+ "version": "2.1.0",+ "runs": [+ {+ "tool": {+ "driver": {+ "name": "retire.js",+ "informationUri": INFORMATION_URI,+ "version": tool_version,+ "rules": sarif_rules,+ }+ },+ "results": results,+ }+ ],+ }+++def main(argv=None):+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])+ parser.add_argument("report", help="retire.js --outputformat json file")+ parser.add_argument("output", help="SARIF file to write")+ parser.add_argument(+ "--root",+ default=".",+ help="repository root that result paths are made relative to",+ )+ parser.add_argument(+ "--require-findings",+ action="store_true",+ help="fail unless the report contains at least one vulnerability "+ "(retire.js exited with its findings status)",+ )+ args = parser.parse_args(argv)++ output = Path(args.output)+ try:+ report = json.loads(Path(args.report).read_text(encoding="utf-8"))+ sarif = convert(report, args.root)+ if args.require_findings and not sarif["runs"][0]["results"]:+ raise ReportError(+ "retire.js reported findings but the report has none"+ )+ except (OSError, ValueError) as error:+ # json.JSONDecodeError and ReportError are both ValueErrors.+ print(+ "::error::retire.js report rejected: "+ + _workflow_command_data(error),+ file=sys.stderr,+ )+ output.unlink(missing_ok=True)+ return 1++ for message in report.get("errors", []):+ print(+ "::warning::retire.js: " + _workflow_command_data(message),+ file=sys.stderr,+ )+ partial = output.with_name(output.name + ".partial")+ partial.write_text(json.dumps(sarif, indent=2) + "\n", encoding="utf-8")+ partial.replace(output)+ print(+ f"Wrote {output} with {len(sarif['runs'][0]['results'])} result(s) "+ f"for {len(sarif['runs'][0]['tool']['driver']['rules'])} rule(s)"+ )+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/.github/workflows/ci-policy-tests.yml b/.github/workflows/ci-policy-tests.ymlnew file mode 100644index 0000000000..0e1e5e5601--- /dev/null+++ b/.github/workflows/ci-policy-tests.yml@@ -0,0 +1,34 @@+name: CI Policy Tests++on:+ pull_request:+ paths:+ - ".github/workflows/**"+ - ".github/scripts/check_release_scan_evidence.py"+ - ".github/scripts/retire_json_to_sarif.py"+ - ".github/release-scan-policy.json"+ - "tests/ci/standalone/test_release_scan_*.py"+ workflow_dispatch:++permissions:+ contents: read++jobs:+ policy-tests:+ runs-on: ubuntu-latest+ timeout-minutes: 5+ steps:+ - name: Harden runner+ uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1+ with:+ egress-policy: audit+ - name: Checkout+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ with:+ persist-credentials: false+ - name: Set up Python+ uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0+ with:+ python-version: '3.12'+ - name: Test CI policies without loading the application+ run: python -m unittest discover -s tests/ci/standalone -p 'test_release_scan*.py' -vdiff --git a/.github/workflows/gitleaks-main.yml b/.github/workflows/gitleaks-main.ymlindex 06ea6b7729..11fc93da5c 100644--- a/.github/workflows/gitleaks-main.yml+++ b/.github/workflows/gitleaks-main.yml@@ -1,8 +1,8 @@name: Gitleaks Main Branch Scan# This workflow is specifically for the security release gate.-# It only scans the main branch history to avoid false positives-# from feature branches with intentional test data.+# It scans history reachable from the release commit, excluding unrelated+# feature branches and commits added to main after the release started.on:workflow_call: # Called by release-gate.yml@@ -30,17 +30,17 @@ jobs:with:persist-credentials: falsefetch-depth: 0- ref: main # Always checkout main branch+ ref: ${{ github.sha }} # Scan the commit being released- name: Install Gitleaksuses: $/.github/actions/install-gitleaks- - name: Run Gitleaks on main branch only+ - name: Run Gitleaks on release commit historyrun: |gitleaks detect \--config=.gitleaks.toml \--gitleaks-ignore-path=.gitleaksignore \- --log-opts="origin/main" \+ --log-opts="HEAD" \--report-format=sarif \--report-path=gitleaks-results.sarif \--redact \@@ -52,3 +52,5 @@ jobs:uses: github/codeql-action/upload-sarif@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0with:sarif_file: gitleaks-results.sarif+ category: gitleaks-release+ wait-for-processing: truediff --git a/.github/workflows/release-gate.yml b/.github/workflows/release-gate.ymlindex b0253a496e..e5ebd74310 100644--- a/.github/workflows/release-gate.yml+++ b/.github/workflows/release-gate.yml@@ -181,7 +181,9 @@ jobs:actions: read# ============================================- # Secret detection - scans only main branch history+ # Secret detection - scans history reachable from the release commit+ # (checked out at github.sha; excludes unrelated branches and commits+ # added to main after the release started — see gitleaks-main.yml)# ============================================gitleaks-scan:uses: $/.github/workflows/gitleaks-main.yml@@ -750,7 +752,9 @@ jobs:# ============================================# Check for open code scanning alerts# This runs after all SARIF uploads complete and fails if- # high/critical/medium severity alerts exist from ANY tool+ # critical/high/medium severity alerts exist from ANY tool, or if a+ # native SARIF alert has no numerical security score but reports an+ # unscored error/warning level# ============================================check-code-scanning-alerts:name: Check Code Scanning Alerts@@ -777,7 +781,10 @@ jobs:- checkov-scan- zizmor-scan- gitleaks-scan+ timeout-minutes: 10permissions:+ contents: read+ actions: readsecurity-events: readsteps:- name: Harden the runner (Audit all outbound calls)@@ -785,83 +792,15 @@ jobs:with:egress-policy: audit- - name: Wait for SARIF indexing- run: |- echo "Waiting 120 seconds for GitHub to index SARIF uploads..."- sleep 120+ - name: Checkout release policy+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ with:+ persist-credentials: false- - name: Check for open security alerts+ - name: Validate fresh analyses and open alertsenv:GH_TOKEN: ${{ github.token }}- run: |- echo "Checking for open code scanning alerts..."-- # Query all open code scanning alerts with pagination- ALERTS="[]"- PAGE=1- while true; do- RESPONSE=$(gh api \- -H "Accept: application/vnd.github+json" \- -H "X-GitHub-Api-Version: 2022-11-28" \- "/repos/${{ github.repository }}/code-scanning/alerts?state=open&per_page=100&page=${PAGE}" \- 2>&1) || {- echo "::error::Failed to query code scanning API (page ${PAGE}): ${RESPONSE}"- exit 1- }-- # Validate response is a JSON array- if ! echo "$RESPONSE" | jq -e 'type == "array"' > /dev/null 2>&1; then- echo "::error::Unexpected API response (not a JSON array): ${RESPONSE}"- exit 1- fi-- COUNT=$(echo "$RESPONSE" | jq 'length')- if [ "$COUNT" -eq 0 ]; then- break- fi-- # Merge page results into ALERTS- ALERTS=$(echo "$ALERTS" "$RESPONSE" | jq -s '.[0] + .[1]')- PAGE=$((PAGE + 1))- done-- TOTAL_ALERTS=$(echo "$ALERTS" | jq 'length')- echo "Fetched ${TOTAL_ALERTS} open alert(s) across $((PAGE - 1)) page(s)."-- # Count alerts by severity- CRITICAL=$(echo "$ALERTS" | jq '[.[] | select(.rule.security_severity_level == "critical")] | length')- HIGH=$(echo "$ALERTS" | jq '[.[] | select(.rule.security_severity_level == "high")] | length')- MEDIUM=$(echo "$ALERTS" | jq '[.[] | select(.rule.security_severity_level == "medium")] | length')-- {- echo ""- echo "## Code Scanning Alert Summary"- echo ""- echo "| Severity | Count |"- echo "|----------|-------|"- echo "| Critical | $CRITICAL |"- echo "| High | $HIGH |"- echo "| Medium | $MEDIUM |"- echo ""- } >> "$GITHUB_STEP_SUMMARY"-- TOTAL=$((CRITICAL + HIGH + MEDIUM))-- if [ "$TOTAL" -gt 0 ]; then- BLOCKING_FILTER='select(.rule.security_severity_level == "critical" or .rule.security_severity_level == "high" or .rule.security_severity_level == "medium")'- {- echo "❌ **Found $TOTAL blocking alerts (Critical: $CRITICAL, High: $HIGH, Medium: $MEDIUM)**"- echo ""- echo "### Blocking Alerts:"- echo "$ALERTS" | jq -r ".[] | ${BLOCKING_FILTER} | \"- [\\(.rule.security_severity_level | ascii_upcase)] \\(.rule.description) ([#\\(.number)](\\(.html_url)))\""- } >> "$GITHUB_STEP_SUMMARY"- echo ""- echo "::error::Security gate failed: Found $TOTAL open code scanning alerts (Critical: $CRITICAL, High: $HIGH, Medium: $MEDIUM)"- exit 1- else- echo "✅ **No critical, high, or medium severity alerts found**" >> "$GITHUB_STEP_SUMMARY"- echo "No blocking code scanning alerts found."- fi+ run: python3 .github/scripts/check_release_scan_evidence.py# ============================================# Summary job that reports overall status@@ -925,7 +864,29 @@ jobs:with:egress-policy: audit+ # Every declared dependency must pass, including newly added jobs that+ # have not yet acquired a dedicated section in the human-readable summary.+ - name: Require every gate dependency to succeed+ env:+ GATE_RESULTS: ${{ toJSON(needs) }}+ run: |+ python3 - <<'PYCODE'+ import json+ import os+ results = json.loads(os.environ["GATE_RESULTS"])+ if not isinstance(results, dict) or not results:+ raise SystemExit("::error::Release gate has no dependency results")+ failed = [name for name, job in results.items() if job.get("result") != "success"]+ if failed:+ with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as summary:+ summary.write("Release blocked by: " + ", ".join(failed) + "\n")+ raise SystemExit("::error::Unsuccessful gate jobs: " + ", ".join(failed))+ PYCODE+- name: Check security scan results+ if: always()+ env:+ GATE_RESULTS: ${{ toJSON(needs) }}run: |# Redirect all output to GITHUB_STEP_SUMMARY (fixes SC2129)exec >> "$GITHUB_STEP_SUMMARY"@@ -976,6 +937,13 @@ jobs:check_result "${{ needs.cpu-baseline-smoke.result }}" || truecheck_result "${{ needs.check-code-scanning-alerts.result }}" || true+ # Count the complete dependency map, including any job that has no+ # dedicated prose section below yet.+ PASS_COUNT=$(jq '[.[] | select(.result == "success")] | length' <<< "$GATE_RESULTS")+ FAIL_COUNT=$(jq '[.[] | select(.result != "success")] | length' <<< "$GATE_RESULTS")+ if [ "$FAIL_COUNT" -gt 0 ]; then+ FAILED="true"+ fiTOTAL=$((PASS_COUNT + FAIL_COUNT))# ============================================@@ -999,8 +967,7 @@ jobs:echo "## Detailed Results"echo ""- # Reset for detailed output- FAILED=""+ # Preserve the aggregate failure status while rendering details.# ============================================# Existing scansdiff --git a/.github/workflows/retirejs.yml b/.github/workflows/retirejs.ymlindex 622c460a48..85d393b107 100644--- a/.github/workflows/retirejs.yml+++ b/.github/workflows/retirejs.yml@@ -37,32 +37,90 @@ jobs:with:node-version: '24'+ # retire.js 5.x scans JavaScript files only (it rejects --node and has+ # no --js option), so install the runtime dependencies the Vite build+ # bundles into the shipped web UI. The minified bundle itself carries+ # no library signatures retire.js can match; the files in+ # node_modules do. --omit=dev keeps build and test tooling out of the+ # scan, and --ignore-scripts runs no dependency install scripts.+ - name: Install shipped JavaScript dependencies+ run: npm ci --omit=dev --ignore-scripts+- name: Run Retire.js scanid: retire-scanrun: |echo "=== Running Retire.js scan ==="- # Use npx with specific version for reproducible builds- # Scan JavaScript files and node_modules for known vulnerabilities- # Only flag medium severity and above (exit code 13 when issues found)- # Output both SARIF (for GitHub Security) and human-readable format- npx [email protected] --js --node --severity medium --outputformat sarif --outputpath retire-results.sarif 2>&1 | tee retire-output.txt- RETIRE_EXIT_CODE=$?-- # Check if vulnerabilities were found (exit code 13 indicates issues)- if [ $RETIRE_EXIT_CODE -eq 13 ]; then- echo "RETIRE_FOUND_ISSUES=true" >> "$GITHUB_ENV"- fi+ # Use npx with a pinned version for reproducible builds. Scan the+ # checkout: the web UI's own static JavaScript plus the runtime+ # dependencies installed above. retire.js 5.x has no SARIF output,+ # so write its JSON report and convert it for code scanning.+ #+ # retire.js 5.2.4 exit codes (lib/cli.js): 0 = the scan completed+ # with nothing at or above --severity; 13 (--exitwith) = the scan+ # completed with findings at or above --severity; anything else+ # (1 for invalid arguments, a failed vulnerability repository+ # download, an output file that cannot be opened or an uncaught+ # exception) = no scan result. retire.js's exit 9 for an+ # unopenable output file is unreachable in 5.2.4: fs.openSync+ # throws before its check, so that case exits 1 too. npx exits+ # nonzero when it cannot fetch the package at all. --severity only+ # sets the exit status: the JSON report lists every vulnerable+ # library, including low severity ones, which the converter+ # uploads as notes.+ #+ # Capture retire's own exit status. Redirecting instead of piping+ # through tee keeps it independent of pipefail: with a pipe, the+ # default `bash -e` shell reports tee's status and every scanner+ # failure looks like a clean scan.+ rm -f retire-results.json retire-results.sarif retire-output.txt+ RETIRE_EXIT_CODE=0+ npx [email protected] --path . --severity medium --exitwith 13 --outputformat json --outputpath retire-results.json > retire-output.txt 2>&1 || RETIRE_EXIT_CODE=$?- # Display human-readable summary+ # Display retire's console output (the findings are in the report).+ # It echoes text from scanned files, so suspend workflow command+ # processing around it with a random, unguessable resume token.echo ""echo "=== Scan Summary ==="+ STOP_TOKEN="$(openssl rand -hex 16)"+ echo "::stop-commands::${STOP_TOKEN}"cat retire-output.txt+ echo "::${STOP_TOKEN}::"++ case "$RETIRE_EXIT_CODE" in+ 0)+ # A completed scan with nothing at or above --severity. Its+ # report may still list low severity findings, or none.+ CONVERT_ARGS=()+ ;;+ 13)+ # Findings at or above --severity: upload them and let+ # "Check scan results" fail the job. The report must contain+ # at least one vulnerability.+ echo "RETIRE_FOUND_ISSUES=true" >> "$GITHUB_ENV"+ CONVERT_ARGS=(--require-findings)+ ;;+ *)+ # A failed scan is not evidence: never convert or upload a+ # report for it, since an empty upload would mark every open+ # retirejs alert as fixed.+ echo "::error::retire.js failed with exit code ${RETIRE_EXIT_CODE}; no scan result"+ exit 1+ ;;+ esac- # Check if SARIF file was created (for use in upload step)- if [ -f "retire-results.sarif" ]; then- echo "sarif_exists=true" >> "$GITHUB_OUTPUT"+ # retire.js always writes its JSON report for a completed scan, so+ # a missing or empty one means the scan did not complete.+ if [ ! -s "retire-results.json" ]; then+ echo "::error::retire.js exited ${RETIRE_EXIT_CODE} but wrote no JSON report"+ exit 1fi+ # Fails (and writes no SARIF) on any report that does not have+ # retire.js 5.x's JSON shape.+ python3 .github/scripts/retire_json_to_sarif.py --root "$PWD" "${CONVERT_ARGS[@]}" retire-results.json retire-results.sarif++ echo "sarif_exists=true" >> "$GITHUB_OUTPUT"+- name: Upload SARIF resultsuses: github/codeql-action/upload-sarif@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0if: always() && steps.retire-scan.outputs.sarif_exists == 'true'@@ -76,7 +134,7 @@ jobs:echo "::error::Retire.js found medium or higher severity vulnerabilities"echo "Review the Security tab for details"echo "Consider updating or replacing affected libraries"- # Only fail for PR/push events, not scheduled scans+ # Fail manual and release-gate runs, not scheduled scansif [[ "${{ github.event_name }}" != "schedule" ]]; thenexit 1fidiff --git a/SECURITY.md b/SECURITY.mdindex 23fe32cec4..2a0ad9d6f1 100644--- a/SECURITY.md+++ b/SECURITY.md@@ -283,7 +283,7 @@ We maintain comprehensive automated security scanning across the entire developm|------|---------|-----------|| **OSV-Scanner** | Open Source Vulnerability database | Every PR & push || **npm audit** | JavaScript dependency vulnerabilities | Every PR & push |-| **RetireJS** | Known vulnerable JS libraries | Every PR & push |+| **RetireJS** | Known vulnerable JS libraries | Weekly, manual & release gate || **SBOM Generation** | Software Bill of Materials (Syft) | Weekly & releases || **License Scanning** | License compliance checking | Every PR |diff --git a/docs/CI_CD_INFRASTRUCTURE.md b/docs/CI_CD_INFRASTRUCTURE.mdindex 26346364ce..f3fd61650d 100644--- a/docs/CI_CD_INFRASTRUCTURE.md+++ b/docs/CI_CD_INFRASTRUCTURE.md@@ -102,7 +102,7 @@ pre-commit install-hooks| `container-security.yml` | PR, push | Container vulnerability scanning || `hadolint.yml` | PR, push | Dockerfile linting || `owasp-zap-scan.yml` | Schedule | OWASP ZAP dynamic scanning |-| `retirejs.yml` | PR, push | JavaScript vulnerability scanning |+| `retirejs.yml` | Schedule, manual, `workflow_call` from release-gate.yml | JavaScript vulnerability scanning || `zizmor-security.yml` | PR, push | Additional security checks || `ossf-scorecard.yml` | Schedule | OpenSSF Scorecard || `security-headers-validation.yml` | PR, push | HTTP security headers |diff --git a/docs/ci/workflow-status.md b/docs/ci/workflow-status.mdindex 87daf26c61..3e06d9389e 100644--- a/docs/ci/workflow-status.md+++ b/docs/ci/workflow-status.md@@ -120,6 +120,7 @@| `advanced-search-reminder.yml` | last 30 days | PR | [](https://github.com/LearningCircuit/local-deep-research/actions/workflows/advanced-search-reminder.yml) || `ai-code-reviewer.yml` | last 30 days | PR-target | [](https://github.com/LearningCircuit/local-deep-research/actions/workflows/ai-code-reviewer.yml) || `check-workflow-status.yml` | last 30 days | PR, manual | [](https://github.com/LearningCircuit/local-deep-research/actions/workflows/check-workflow-status.yml) |+| `ci-policy-tests.yml` | last 30 days | PR, manual | [](https://github.com/LearningCircuit/local-deep-research/actions/workflows/ci-policy-tests.yml) || `danger-zone-alert.yml` | last 30 days | PR | [](https://github.com/LearningCircuit/local-deep-research/actions/workflows/danger-zone-alert.yml) || `dependency-review.yml` | last 30 days | PR, manual | [](https://github.com/LearningCircuit/local-deep-research/actions/workflows/dependency-review.yml) || `e2e-research-test.yml` | last 30 days | PR | [](https://github.com/LearningCircuit/local-deep-research/actions/workflows/e2e-research-test.yml) |diff --git a/tests/ci/standalone/test_release_scan_evidence.py b/tests/ci/standalone/test_release_scan_evidence.pynew file mode 100644index 0000000000..8b8350eca8--- /dev/null+++ b/tests/ci/standalone/test_release_scan_evidence.py@@ -0,0 +1,1051 @@+"""Standalone release-policy tests using synthetic API responses."""++# allow: no-sut-import — imports the standalone CI script directly.++import importlib.util+import io+import json+import os+import re+import shutil+import subprocess+import tempfile+import unittest+from contextlib import redirect_stdout+from pathlib import Path+from unittest.mock import patch++ROOT = Path(__file__).resolve().parents[3]+SPEC = importlib.util.spec_from_file_location(+ "release_scan", ROOT / ".github/scripts/check_release_scan_evidence.py"+)+POLICY = importlib.util.module_from_spec(SPEC)+SPEC.loader.exec_module(POLICY)++SHA = "a" * 40+REF = "refs/tags/v1.0.0"+WORKFLOW = ".github/workflows/release.yml"+START = "2026-01-01T00:00:00Z"+++def analysis(**changes):+ return {+ "category": "semgrep-security",+ "commit_sha": SHA,+ "ref": REF,+ "analysis_key": f"{WORKFLOW}:semgrep-scan",+ "created_at": START,+ "error": "",+ "results_count": 0,+ **changes,+ }+++def alert(**changes):+ return {+ "number": 1,+ "state": "open",+ "tool": {"name": "CodeQL"},+ "rule": {"security_severity_level": "critical", "severity": None},+ **changes,+ }+++# --- R3: tie release-scan-policy.json to what release-gate.yml actually+# uploads. This is a plain-text (no PyYAML) walk, deliberately: the standalone+# suite runs in ci-policy-tests.yml on a bare `actions/setup-python` with no+# dependency install step, so a `yaml` import here would break that job.+_JOB_ID_RE = re.compile(r"^ ([A-Za-z0-9_-]+):\s*$")+_LOCAL_USES_RE = re.compile(r"^\s+uses:\s*(\$/\S+|\./\S+)\s*$")+_UPLOAD_MARKERS = ("upload-sarif", "codeql-action/analyze")+_CATEGORY_RE = re.compile(r"^\s*category:\s*(.+?)\s*(#.*)?$")++# Remote reusable "uploaders" whose SARIF category cannot be statically+# extracted from this repo (a third-party composite/Docker action that+# uploads internally, not a local `$/` reusable workflow with its own+# `upload-sarif`/`codeql-action/analyze` step to grep). Each entry must+# name why it's safe to trust rather than verify by parsing.+_REMOTE_SARIF_UPLOADERS = {+ # zizmorcore/zizmor-action (zizmor-security.yml) uploads its own SARIF+ # internally under a fixed "zizmor" category — confirmed against the+ # live Security tab, not discoverable by grepping this repo.+ "zizmor-scan": "zizmor",+}++# Remote reusable workflows (`owner/repo/.github/workflows/x.yml@sha`) that a+# gate-called local workflow delegates a whole job to. Their upload-sarif step+# lives in the other repository, so its category is recorded here instead of+# parsed. "{workflow}" stands for the top-level workflow file of the run.+_REMOTE_REUSABLE_SARIF_UPLOADERS = {+ # osv-scanner.yml -> google/osv-scanner-action's reusable workflow, whose+ # "Upload to code-scanning" step (v2.5.1) passes no `category:`. Code+ # scanning therefore files it under the default "<top-level workflow>:+ # <job id>" category, e.g. ".github/workflows/release-gate.yml:osv-scan"+ # for the scheduled release-gate run (confirmed against live analyses).+ "google/osv-scanner-action/.github/workflows/osv-scanner-reusable.yml": (+ "{workflow}:osv-scan"+ ),+}++# Categories the gate uploads but release-scan-policy.json deliberately does+# not require, each with the reason.+_UPLOADED_BUT_NOT_IN_POLICY = {+ # The category embeds the calling workflow's path (release.yml for a tag+ # release, release-gate.yml for the scheduled run), which the policy's+ # fixed category names cannot express. OSV is still gated: the+ # osv-scan job fails on any vulnerability (fail-on-vuln defaults to+ # true) or scanner error, release-gate-summary requires every+ # dependency to succeed, and its open alerts on the release ref are+ # counted by the alert query like any other tool's.+ "{workflow}:osv-scan": "path-dependent default category",+}++_REMOTE_REUSABLE_RE = re.compile(+ r"^\s+uses:\s*[\"']?([\w.-]+/[\w.-]+/\.github/workflows/[^@\"'\s]+)@"+)+++def _local_reusable_workflow_jobs(gate_text):+ """release-gate.yml job id -> the `$/...`/`./...` workflow file it calls."""+ jobs = {}+ current = None+ for line in gate_text.splitlines():+ job_match = _JOB_ID_RE.match(line)+ if job_match:+ current = job_match.group(1)+ continue+ if line and not line.startswith(" "):+ current = None+ continue+ uses_match = _LOCAL_USES_RE.match(line)+ if uses_match and current:+ jobs[current] = uses_match.group(1)[2:] # strip "$/" / "./"+ return jobs+++def _sarif_categories_in(text, source="workflow"):+ """Every `category:` value that follows an upload-sarif/analyze step.++ An upload step with no `category:` in reach is reported as a+ "<uncategorized ...>" placeholder rather than skipped, and so is a remote+ reusable workflow not listed in _REMOTE_REUSABLE_SARIF_UPLOADERS: either+ can upload SARIF this walk cannot name, so neither may silently vanish+ from the parity check.+ """+ lines = text.splitlines()+ found = []+ for i, line in enumerate(lines):+ remote = _REMOTE_REUSABLE_RE.match(line)+ if remote:+ found.append(+ _REMOTE_REUSABLE_SARIF_UPLOADERS.get(+ remote.group(1),+ f"<unrecognised remote reusable workflow "+ f"{remote.group(1)} in {source}>",+ )+ )+ continue+ if line.lstrip().startswith("#") or not any(+ marker in line for marker in _UPLOAD_MARKERS+ ):+ continue+ for follow in lines[i + 1 : i + 8]:+ match = _CATEGORY_RE.match(follow)+ if match:+ found.append(match.group(1).strip().strip("\"'"))+ break+ if re.match(r"^\s*-\s*(name|uses):", follow):+ break # next step started with no category on this one+ else:+ match = None+ if not match:+ found.append(f"<uncategorized upload in {source} line {i + 1}>")+ return found+++def _expand_matrix_language(category, workflow_text):+ if "matrix.language" not in category:+ return [category]+ languages = re.findall(+ r"^\s*-\s*language:\s*([\w.-]+)", workflow_text, re.MULTILINE+ )+ return [+ category.replace("${{matrix.language}}", lang).replace(+ "${{ matrix.language }}", lang+ )+ for lang in languages+ ] or [category]+++def _release_gate_sarif_categories():+ """Every SARIF category reachable from release-gate.yml, discovered the+ same way the policy's author verified it by hand: walk each job's local+ reusable workflow and read off its upload-sarif/analyze category, plus+ the recorded category of any remote reusable workflow it calls.+ Uploads the walk cannot name come back as "<...>" placeholders.+ """+ gate_text = (ROOT / ".github/workflows/release-gate.yml").read_text(+ encoding="utf-8"+ )+ categories = set()+ for job_id, rel_path in _local_reusable_workflow_jobs(gate_text).items():+ workflow_text = (ROOT / rel_path).read_text(encoding="utf-8")+ expanded = set()+ for raw_category in _sarif_categories_in(workflow_text, rel_path):+ expanded.update(+ _expand_matrix_language(raw_category, workflow_text)+ )+ if not expanded and job_id in _REMOTE_SARIF_UPLOADERS:+ expanded = {_REMOTE_SARIF_UPLOADERS[job_id]}+ categories.update(expanded)+ return categories+++class EvidenceTests(unittest.TestCase):+ def check(self, records):+ return POLICY.check_analyses(+ records,+ {"semgrep-security": "semgrep-scan"},+ SHA,+ REF,+ WORKFLOW,+ POLICY.timestamp(START),+ )++ def test_clean_current_analysis_is_valid(self):+ self.assertEqual(self.check([analysis()]), [])++ def test_wrong_commit_ref_workflow_and_stale_results_are_not_evidence(self):+ for changes in (+ {"ref": "refs/heads/main"},+ {"analysis_key": ".github/workflows/semgrep.yml:semgrep-scan"},+ {"created_at": "2025-12-31T23:59:59Z"},+ {"category": "unrelated"},+ ):+ with self.subTest(changes=changes):+ self.assertEqual(+ self.check([analysis(**changes)]), ["semgrep-security"]+ )++ def test_superseding_commit_cannot_supply_release_alert_state(self):+ with self.assertRaisesRegex(ValueError, "another commit"):+ self.check(+ [+ analysis(),+ analysis(+ commit_sha="b" * 40, created_at="2026-01-01T01:00:00Z"+ ),+ ]+ )++ def test_scan_errors_and_missing_counts_fail(self):+ for changes in (+ {"error": "scan failed"},+ {"error": None},+ {"results_count": None},+ {"results_count": -1},+ ):+ with self.subTest(changes=changes), self.assertRaises(ValueError):+ self.check([analysis(**changes)])++ def test_latest_analysis_controls_rerun_result(self):+ failed = analysis(error="failed")+ passed = analysis(created_at="2026-01-01T01:00:00Z")+ self.assertEqual(self.check([failed, passed]), [])+ with self.assertRaises(ValueError):+ self.check(+ [+ passed,+ analysis(created_at="2026-01-01T02:00:00Z", error="failed"),+ ]+ )++ def test_each_category_is_required(self):+ self.assertEqual(self.check([]), ["semgrep-security"])+ with self.assertRaises(ValueError):+ POLICY.check_analyses(+ [], {}, SHA, REF, WORKFLOW, POLICY.timestamp(START)+ )++ def test_alert_severity_and_unscored_fallback(self):+ for security, level, expected in (+ ("critical", "note", True),+ ("high", "error", True),+ ("medium", "warning", True),+ ("low", "error", False),+ (None, "error", True),+ (None, "warning", True),+ (None, "note", False),+ (None, "none", False),+ ):+ with self.subTest(security=security, level=level):+ self.assertEqual(+ POLICY.blocking_alert(+ {+ "state": "open",+ "rule": {+ "security_severity_level": security,+ "severity": level,+ },+ }+ ),+ expected,+ )+ for rule in ({}, {"security_severity_level": "unknown"}):+ with self.assertRaises(ValueError):+ POLICY.blocking_alert({"state": "open", "rule": rule})++ def test_pagination_and_malformed_api_response(self):+ responses = iter([[analysis()] * 100, [analysis()]])+ self.assertEqual(+ sum(+ len(page)+ for page in POLICY.pages(+ "endpoint", {}, lambda _: next(responses)+ )+ ),+ 101,+ )+ with self.assertRaises(ValueError):+ list(+ POLICY.pages("endpoint", {}, lambda _: {"message": "forbidden"})+ )++ def test_recent_analysis_pagination_stops_before_old_runs(self):+ calls = []++ def fetch(endpoint):+ calls.append(endpoint)+ return [analysis(created_at="2025-12-31T00:00:00Z")] * 100++ self.assertEqual(+ POLICY.recent_analyses(+ "endpoint", REF, POLICY.timestamp(START), fetch+ ),+ [],+ )+ self.assertEqual(len(calls), 1)++ def test_end_to_end_polling_and_ref_scoped_alerts(self):+ expected = json.loads(+ (ROOT / ".github/release-scan-policy.json").read_text(+ encoding="utf-8"+ )+ )+ complete = [+ analysis(category=category, analysis_key=f"{WORKFLOW}:{job}")+ for category, job in expected.items()+ ]+ calls, sleeps = [], []+ analyses = iter([[], complete])++ def fetch(endpoint):+ calls.append(endpoint)+ if "/actions/runs/" in endpoint:+ return {"head_sha": SHA, "created_at": START, "path": WORKFLOW}+ if "/analyses?" in endpoint:+ return next(analyses)+ self.assertIn("state=open", endpoint)+ self.assertIn("ref=refs%2Ftags%2Fv1.0.0", endpoint)+ return []++ with (+ tempfile.TemporaryDirectory() as directory,+ patch.dict(+ os.environ,+ {+ "GITHUB_REPOSITORY": "owner/repo",+ "GITHUB_SHA": SHA,+ "GITHUB_REF": REF,+ "GITHUB_RUN_ID": "1",+ "GITHUB_STEP_SUMMARY": str(Path(directory) / "summary"),+ },+ ),+ ):+ POLICY.evaluate(fetch, sleeps.append, lambda: 0)+ self.assertEqual(sleeps, [15])+ self.assertIn(+ "Open blocking alerts: **0**",+ Path(os.environ["GITHUB_STEP_SUMMARY"]).read_text(+ encoding="utf-8"+ ),+ )+ ticks = iter([0, 301])+ with self.assertRaisesRegex(+ ValueError, "No fresh successful analysis"+ ):+ POLICY.evaluate(+ lambda endpoint: (+ {"head_sha": SHA, "created_at": START, "path": WORKFLOW}+ if "/actions/runs/" in endpoint+ else []+ ),+ sleeps.append,+ lambda: next(ticks),+ )++ @staticmethod+ def _run_env(directory):+ return {+ "GITHUB_REPOSITORY": "owner/repo",+ "GITHUB_SHA": SHA,+ "GITHUB_REF": REF,+ "GITHUB_RUN_ID": "1",+ "GITHUB_STEP_SUMMARY": str(Path(directory) / "summary"),+ }++ def test_evaluate_raises_and_names_blocking_alerts_but_allows_low_severity(+ self,+ ):+ """B1: one scored-critical and one unscored-error alert must block;+ a low-severity control must not. Kills both `if blocking:` ->+ `if False:` and `blocking = []` mutants, which would swallow the+ blocking alerts and let evaluate() return normally.+ """+ expected = json.loads(+ (ROOT / ".github/release-scan-policy.json").read_text(+ encoding="utf-8"+ )+ )+ complete = [+ analysis(category=category, analysis_key=f"{WORKFLOW}:{job}")+ for category, job in expected.items()+ ]+ alerts = [+ alert(+ number=101,+ tool={"name": "CodeQL"},+ rule={"security_severity_level": "critical", "severity": None},+ ),+ alert(+ number=102,+ tool={"name": "Bandit"},+ rule={"security_severity_level": None, "severity": "error"},+ ),+ # Non-blocking control: open, low-scored severity.+ alert(+ number=103,+ tool={"name": "Semgrep"},+ rule={"security_severity_level": "low", "severity": "error"},+ ),+ ]++ def fetch(endpoint):+ if "/actions/runs/" in endpoint:+ return {"head_sha": SHA, "created_at": START, "path": WORKFLOW}+ if "/analyses?" in endpoint:+ return complete+ return alerts++ # Bounded ticks (not a constant monotonic): if a category ever fails+ # to match, this fails the polling deadline instead of looping+ # forever under a no-op sleep.+ ticks = iter([0, 301])+ with (+ tempfile.TemporaryDirectory() as directory,+ patch.dict(os.environ, self._run_env(directory)),+ ):+ with self.assertRaisesRegex(+ ValueError,+ r"2 blocking code-scanning alerts remain open: "+ r"alert #101 \(CodeQL\), alert #102 \(Bandit\)",+ ) as caught:+ POLICY.evaluate(+ fetch, lambda seconds: None, lambda: next(ticks)+ )+ self.assertNotIn("#103", str(caught.exception))+ summary = Path(os.environ["GITHUB_STEP_SUMMARY"]).read_text(+ encoding="utf-8"+ )+ self.assertIn("Open blocking alerts: **2**", summary)++ def test_main_fails_for_every_evaluate_exception_class(self):+ """B2: main() must report and return nonzero for every exception+ class evaluate() can raise. Kills the `return 1` -> `return 0`+ mutant, which would report an error yet exit success.+ """+ cases = (+ ValueError("release scan policy violated"),+ subprocess.CalledProcessError(+ 1, ["gh", "api", "x"], output="", stderr="not found"+ ),+ json.JSONDecodeError("Expecting value", "", 0),+ subprocess.TimeoutExpired(cmd="gh api x", timeout=60),+ )+ for exc in cases:+ with self.subTest(exc=type(exc).__name__):+ buffer = io.StringIO()+ with (+ patch.object(POLICY, "evaluate", side_effect=exc),+ redirect_stdout(buffer),+ ):+ self.assertEqual(POLICY.main(), 1)+ self.assertIn(+ "::error::Release scan evidence failed:", buffer.getvalue()+ )++ def test_main_returns_zero_on_success(self):+ with patch.object(POLICY, "evaluate", return_value=None):+ self.assertEqual(POLICY.main(), 0)++ def test_main_reports_truncated_redacted_gh_stderr_on_called_process_error(+ self,+ ):+ """R5: gh's stderr must reach the failure log (truncated, and with+ any token-shaped substring redacted).+ """+ token = "ghp_" + "a" * 40+ stderr = f"error: {token} invalid, and " + ("z" * 600)+ exc = subprocess.CalledProcessError(+ 1, ["gh", "api", "x"], output="", stderr=stderr+ )+ buffer = io.StringIO()+ with (+ patch.object(POLICY, "evaluate", side_effect=exc),+ redirect_stdout(buffer),+ ):+ self.assertEqual(POLICY.main(), 1)+ output = buffer.getvalue()+ self.assertIn("gh stderr:", output)+ self.assertNotIn(token, output)+ self.assertIn("[REDACTED]", output)+ self.assertNotIn("z" * 600, output)++ def test_evaluate_rejects_a_run_for_a_different_commit(self):+ """R1: head_sha != GITHUB_SHA must be rejected."""++ def fetch(endpoint):+ if "/actions/runs/" in endpoint:+ return {+ "head_sha": "b" * 40,+ "created_at": START,+ "path": WORKFLOW,+ }+ return []++ # Bounded ticks: evaluate() must raise before ever touching the+ # polling loop, but a non-advancing monotonic would hang forever+ # under a no-op sleep if that check ever regressed.+ ticks = iter([0, 301])+ with (+ tempfile.TemporaryDirectory() as directory,+ patch.dict(os.environ, self._run_env(directory)),+ ):+ with self.assertRaisesRegex(+ ValueError, "does not match the release commit"+ ):+ POLICY.evaluate(+ fetch, lambda seconds: None, lambda: next(ticks)+ )++ def test_blocking_alert_rejects_a_non_open_state(self):+ """R1: blocking_alert must reject any state other than 'open'."""+ with self.assertRaisesRegex(ValueError, "unexpected alert state"):+ POLICY.blocking_alert(alert(state="dismissed"))++ def test_timestamp_rejects_a_naive_value(self):+ """R1: API timestamps without a UTC offset must be rejected rather+ than silently compared as naive datetimes.+ """+ with self.assertRaisesRegex(ValueError, "no timezone"):+ POLICY.timestamp("2026-01-01T00:00:00")++ def test_analysis_from_a_first_attempt_survives_a_later_rerun(self):+ """R1: `started` must come from the run's created_at (the original+ run, stable across reruns), not run_started_at (the current+ attempt). A first-attempt analysis created well before a later+ rerun's run_started_at must still be accepted. If evaluate() used+ run_started_at instead, this analysis would look stale and the+ category would come back missing, which times out immediately below+ (monotonic ticks straight past the deadline on the first check).+ """+ expected = json.loads(+ (ROOT / ".github/release-scan-policy.json").read_text(+ encoding="utf-8"+ )+ )+ original_created = "2026-01-01T00:00:00Z"+ analysis_created = "2026-01-01T00:30:00Z"+ rerun_started = "2026-01-01T05:00:00Z"+ complete = [+ analysis(+ category=category,+ analysis_key=f"{WORKFLOW}:{job}",+ created_at=analysis_created,+ )+ for category, job in expected.items()+ ]++ def fetch(endpoint):+ if "/actions/runs/" in endpoint:+ return {+ "head_sha": SHA,+ "created_at": original_created,+ "run_started_at": rerun_started,+ "path": WORKFLOW,+ }+ if "/analyses?" in endpoint:+ return complete+ return []++ ticks = iter([0, 301])+ with (+ tempfile.TemporaryDirectory() as directory,+ patch.dict(os.environ, self._run_env(directory)),+ ):+ # Must not raise: the analysis is accepted on the first pass.+ POLICY.evaluate(fetch, lambda seconds: None, lambda: next(ticks))++ def test_blocking_alert_messages_name_the_alert_and_tool(self):+ """R5: an unrecognised/null severity must name the offending alert+ number and tool, not just the category of failure.+ """+ with self.assertRaisesRegex(ValueError, r"alert #77 \(Bandit\)"):+ POLICY.blocking_alert(+ alert(+ number=77,+ tool={"name": "Bandit"},+ rule={"security_severity_level": "unscored"},+ )+ )+ with self.assertRaisesRegex(ValueError, r"alert #88 \(Semgrep\)"):+ POLICY.blocking_alert(+ alert(+ number=88,+ tool={"name": "Semgrep"},+ rule={"security_severity_level": None, "severity": "huge"},+ )+ )++ def test_policy_categories_match_every_sarif_upload_reachable_from_gate(+ self,+ ):+ """R3: every release-scan-policy.json category must correspond to an+ actual upload-sarif/analyze category reachable from+ release-gate.yml, and vice versa. Discovery walks each gate job's+ local reusable workflow; remote uploaders (a third-party action, or a+ remote reusable workflow such as OSV's) resolve only through the+ explicit allowlists above, and anything else it cannot name fails+ as a placeholder. Uploads deliberately left out of the policy must+ be listed in _UPLOADED_BUT_NOT_IN_POLICY. A category dropped from+ either side fails this test.+ """+ discovered = _release_gate_sarif_categories()+ policy = json.loads(+ (ROOT / ".github/release-scan-policy.json").read_text(+ encoding="utf-8"+ )+ )+ self.assertTrue(discovered, "SARIF category discovery found nothing")+ self.assertFalse(set(policy) & set(_UPLOADED_BUT_NOT_IN_POLICY))+ self.assertEqual(+ discovered,+ set(policy) | set(_UPLOADED_BUT_NOT_IN_POLICY),+ "release-scan-policy.json categories (plus the documented "+ "exclusions) must exactly match the SARIF categories reachable "+ "from release-gate.yml",+ )++ def test_sarif_discovery_never_skips_an_unnamed_upload(self):+ """R3: an upload with no `category:` and an unlisted remote+ reusable workflow must surface as placeholders, not disappear.+ """+ text = "\n".join(+ (+ "jobs:",+ " a:",+ " steps:",+ " # upload-sarif is mentioned only in this comment",+ " - uses: github/codeql-action/upload-sarif@abc",+ " with:",+ " sarif_file: x.sarif",+ " - name: next",+ " uses: github/codeql-action/upload-sarif@abc",+ " with:",+ " category: named",+ " b:",+ " uses: other/repo/.github/workflows/scan.yml@abc",+ " c:",+ ' uses: "google/osv-scanner-action/.github/workflows/'+ 'osv-scanner-reusable.yml@abc" # v2',+ )+ )+ self.assertEqual(+ _sarif_categories_in(text, "w.yml"),+ [+ "<uncategorized upload in w.yml line 5>",+ "named",+ "<unrecognised remote reusable workflow "+ "other/repo/.github/workflows/scan.yml in w.yml>",+ "{workflow}:osv-scan",+ ],+ )+++def _step_run_script(workflow_text, step_id):+ """The `run: |` block of the step with `id: <step_id>`, dedented.++ Plain-text extraction (no PyYAML, see the R3 note above): find the step,+ then take every following line indented deeper than its `run:` key.+ """+ lines = workflow_text.splitlines()+ step_start = next(+ i+ for i, line in enumerate(lines)+ if re.match(rf"^\s+id:\s*{re.escape(step_id)}\s*$", line)+ )+ run_index = next(+ i+ for i in range(step_start + 1, len(lines))+ if re.match(r"^\s+run:\s*\|\s*$", lines[i])+ )+ key_indent = len(lines[run_index]) - len(lines[run_index].lstrip())+ body = []+ for line in lines[run_index + 1 :]:+ if line.strip() and len(line) - len(line.lstrip()) <= key_indent:+ break+ body.append(line)+ indent = min(len(ln) - len(ln.lstrip()) for ln in body if ln.strip())+ return "\n".join(ln[indent:] for ln in body).rstrip() + "\n"+++_FAKE_NPX = """#!/bin/sh+printf '%s\\n' "$@" > "$FAKE_ARGS"+out=""+while [ "$#" -gt 0 ]; do+ if [ "$1" = "--outputpath" ]; then out="$2"; fi+ shift+done+echo "fake retire.js output"+echo "::error::from a scanned file"+if [ -n "$FAKE_REPORT" ]; then+ printf '%s' "$FAKE_REPORT" | sed "s|@ROOT@|$PWD|g" > "$out"+fi+exit "$FAKE_EXIT"+"""++_RETIRE_CONVERTER = ".github/scripts/retire_json_to_sarif.py"++# retire.js 5.2.4 JSON report shapes (see test_release_scan_retire_sarif.py+# for a real report). @ROOT@ is replaced with the step's working directory,+# as retire.js reports absolute paths.+_EMPTY_REPORT = json.dumps(+ {"version": "5.2.4", "data": [], "messages": [], "errors": []}+)+++def _retire_report(*severities):+ return json.dumps(+ {+ "version": "5.2.4",+ "data": [+ {+ "file": "@ROOT@/node_modules/jquery/dist/jquery.js",+ "results": [+ {+ "version": "3.4.1",+ "component": "jquery",+ "vulnerabilities": [+ {+ "severity": severity,+ "identifiers": {+ "summary": "an advisory",+ "githubID": f"GHSA-{index}",+ },+ }+ for index, severity in enumerate(severities)+ ],+ }+ ],+ }+ ],+ "messages": [],+ "errors": [],+ }+ )+++_FINDINGS_REPORT = _retire_report("medium", "high")+_LOW_ONLY_REPORT = _retire_report("low")++# Every option retire.js 5.2.4 accepts (lib/cli.js, the commander+# definitions), and the values its --outputformat and --severity accept.+# 5.x rejects --node/-n outright and has no --js option or sarif format.+# Re-verify these against the new cli.js before bumping the pinned version.+RETIRE_5_2_4_OPTIONS = {+ "-V": False,+ "--version": False,+ "-h": False,+ "--help": False,+ "-v": False,+ "--verbose": False,+ "-c": False,+ "--nocache": False,+ "--jspath": True,+ "--path": True,+ "--jsrepo": True,+ "--cachedir": True,+ "--proxy": True,+ "--outputformat": True,+ "--outputpath": True,+ "--ignore": True,+ "--ignorefile": True,+ "--severity": True,+ "--exitwith": True,+ "--colors": False,+ "--insecure": False,+ "--ext": True,+ "--cacert": True,+ "--includeOsv": False,+ "--deep": False,+}+RETIRE_5_2_4_OUTPUT_FORMATS = {+ "text",+ "json",+ "jsonsimple",+ "depcheck",+ "cyclonedx",+ "cyclonedxJSON",+ "cyclonedxJSON1_6",+}+RETIRE_5_2_4_SEVERITIES = {"none", "low", "medium", "high", "critical"}++[email protected](shutil.which("bash"), "needs bash")+class RetireScanStepTests(unittest.TestCase):+ """R4: retirejs.yml's scan step must report retire.js's own exit status.++ The step script is run exactly as GitHub Actions runs a step with no+ `shell:` key (`bash -e`, no pipefail) and as `shell: bash` would+ (`bash -eo pipefail`), with a fake `npx` standing in for retire.js and+ the real JSON-to-SARIF converter. An empty "retirejs" SARIF upload is+ fresh evidence to the release gate and marks every open retirejs alert+ as fixed, so it may only follow a scan that completed cleanly (exit 0)+ with a well-formed report; findings (exit 13) are converted, uploaded+ and flag the job; any other status, a missing report or a malformed+ report fails the step with nothing to upload.+ """++ SHELLS = (+ ("bash", "-e"),+ ("bash", "--noprofile", "--norc", "-eo", "pipefail"),+ )++ @classmethod+ def setUpClass(cls):+ cls.script = _step_run_script(+ (ROOT / ".github/workflows/retirejs.yml").read_text(+ encoding="utf-8"+ ),+ "retire-scan",+ )++ def run_step(self, shell, exit_code, report=""):+ with tempfile.TemporaryDirectory() as directory:+ work = Path(directory)+ bin_dir = work / "bin"+ bin_dir.mkdir()+ npx = bin_dir / "npx"+ npx.write_text(_FAKE_NPX, encoding="utf-8")+ npx.chmod(0o755)+ converter = work / _RETIRE_CONVERTER+ converter.parent.mkdir(parents=True)+ shutil.copyfile(ROOT / _RETIRE_CONVERTER, converter)+ script = work / "step.sh"+ script.write_text(self.script, encoding="utf-8")+ env_file, output_file = work / "env", work / "output"+ args_file = work / "npx-args"+ env_file.touch()+ output_file.touch()+ proc = subprocess.run(+ [*shell, str(script)],+ cwd=work,+ env={+ **os.environ,+ "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}",+ "GITHUB_ENV": str(env_file),+ "GITHUB_OUTPUT": str(output_file),+ "FAKE_EXIT": str(exit_code),+ "FAKE_REPORT": report,+ "FAKE_ARGS": str(args_file),+ },+ capture_output=True,+ text=True,+ timeout=60,+ )+ sarif = work / "retire-results.sarif"+ self.npx_args = args_file.read_text(encoding="utf-8").split("\n")+ self.step_log = proc.stdout + proc.stderr+ return (+ proc.returncode,+ env_file.read_text(encoding="utf-8"),+ output_file.read_text(encoding="utf-8"),+ json.loads(sarif.read_text(encoding="utf-8"))+ if sarif.exists()+ else None,+ )++ def test_clean_scan_uploads_an_empty_sarif(self):+ for shell in self.SHELLS:+ with self.subTest(shell=shell):+ code, env, output, sarif = self.run_step(+ shell, 0, _EMPTY_REPORT+ )+ self.assertEqual(code, 0)+ self.assertNotIn("RETIRE_FOUND_ISSUES", env)+ self.assertIn("sarif_exists=true", output)+ self.assertEqual(sarif["version"], "2.1.0")+ run = sarif["runs"][0]+ self.assertEqual(run["tool"]["driver"]["name"], "retire.js")+ self.assertEqual(run["results"], [])++ def test_clean_scan_still_uploads_low_severity_findings(self):+ for shell in self.SHELLS:+ with self.subTest(shell=shell):+ code, env, output, sarif = self.run_step(+ shell, 0, _LOW_ONLY_REPORT+ )+ self.assertEqual(code, 0)+ self.assertNotIn("RETIRE_FOUND_ISSUES", env)+ self.assertIn("sarif_exists=true", output)+ results = sarif["runs"][0]["results"]+ self.assertEqual([r["level"] for r in results], ["note"])++ def test_findings_are_converted_uploaded_and_flag_the_job(self):+ for shell in self.SHELLS:+ with self.subTest(shell=shell):+ code, env, output, sarif = self.run_step(+ shell, 13, _FINDINGS_REPORT+ )+ self.assertEqual(code, 0)+ self.assertIn("RETIRE_FOUND_ISSUES=true", env)+ self.assertIn("sarif_exists=true", output)+ results = sarif["runs"][0]["results"]+ self.assertEqual(+ sorted(r["level"] for r in results), ["error", "warning"]+ )+ self.assertEqual(+ {+ r["locations"][0]["physicalLocation"][+ "artifactLocation"+ ]["uri"]+ for r in results+ },+ {"node_modules/jquery/dist/jquery.js"},+ )++ def test_missing_empty_or_malformed_reports_fail_the_step(self):+ cases = (+ (0, ""),+ (0, "{not json"),+ (0, '{"version": "5.2.4", "data": {}}'),+ (13, ""),+ (13, "{not json"),+ # Exit 13 promises findings; an empty report contradicts it.+ (13, _EMPTY_REPORT),+ )+ for shell in self.SHELLS:+ for exit_code, report in cases:+ with self.subTest(+ shell=shell, exit_code=exit_code, report=report+ ):+ code, _, output, sarif = self.run_step(+ shell, exit_code, report+ )+ self.assertNotEqual(code, 0)+ self.assertNotIn("sarif_exists", output)+ self.assertIsNone(sarif)+ if not report:+ # The case statement, not the converter, rejects a+ # scan that left no report behind.+ self.assertIn("wrote no JSON report", self.step_log)++ def test_scanner_failure_fails_the_step_and_uploads_nothing(self):+ for shell in self.SHELLS:+ for exit_code, report in (+ (1, ""),+ (9, ""),+ (1, _EMPTY_REPORT),+ (1, _FINDINGS_REPORT),+ ):+ with self.subTest(+ shell=shell, exit_code=exit_code, report=report+ ):+ code, env, output, sarif = self.run_step(+ shell, exit_code, report+ )+ self.assertNotEqual(code, 0)+ self.assertNotIn("sarif_exists", output)+ self.assertNotIn("RETIRE_FOUND_ISSUES", env)+ # Nothing may be converted for a failed scan.+ self.assertIsNone(sarif)++ def test_console_output_is_shown_with_workflow_commands_stopped(self):+ """retire.js echoes scanned file text; it must not run as commands."""+ for exit_code, report in ((0, _EMPTY_REPORT), (1, "")):+ with self.subTest(exit_code=exit_code):+ self.run_step(self.SHELLS[0], exit_code, report)+ lines = self.step_log.splitlines()+ stops = [+ i+ for i, line in enumerate(lines)+ if line.startswith("::stop-commands::")+ ]+ self.assertEqual(len(stops), 1)+ token = lines[stops[0]].removeprefix("::stop-commands::")+ self.assertRegex(token, r"^[0-9a-f]{32}$")+ resume = lines.index(f"::{token}::")+ output = (+ lines.index("fake retire.js output"),+ lines.index("::error::from a scanned file"),+ )+ for index in output:+ self.assertLess(stops[0], index)+ self.assertLess(index, resume)+ # The token appears only in the stop and resume lines.+ self.assertEqual(self.step_log.count(token), 2)+ tokens = set()+ for _ in range(2):+ self.run_step(self.SHELLS[0], 0, _EMPTY_REPORT)+ tokens.add(+ re.search(r"^::stop-commands::(\S+)$", self.step_log, re.M)[1]+ )+ self.assertEqual(len(tokens), 2)+ # The wrapped cat is the step's only display of retire's output.+ self.assertEqual(len(re.findall(r"\bcat\b", self.script)), 1)+ self.assertIn(+ 'echo "::stop-commands::${STOP_TOKEN}"\n'+ "cat retire-output.txt\n"+ 'echo "::${STOP_TOKEN}::"\n',+ self.script,+ )++ def test_invocation_uses_only_options_retire_5_2_4_accepts(self):+ self.run_step(self.SHELLS[0], 0, _EMPTY_REPORT)+ args = [a for a in self.npx_args if a]+ self.assertEqual(args[0], "[email protected]")+ options = {}+ index = 1+ while index < len(args):+ option = args[index]+ self.assertIn(option, RETIRE_5_2_4_OPTIONS)+ if RETIRE_5_2_4_OPTIONS[option]:+ index += 1+ options[option] = args[index]+ else:+ options[option] = True+ index += 1+ self.assertIn(options["--outputformat"], RETIRE_5_2_4_OUTPUT_FORMATS)+ self.assertEqual(options["--outputformat"], "json")+ self.assertEqual(options["--outputpath"], "retire-results.json")+ self.assertIn(options["--severity"], RETIRE_5_2_4_SEVERITIES)+ self.assertEqual(options.get("--exitwith", "13"), "13")+ # Every retire invocation in the workflow is the one exercised here.+ workflow = (ROOT / ".github/workflows/retirejs.yml").read_text(+ encoding="utf-8"+ )+ self.assertEqual(len(re.findall(r"\bnpx\s+retire", workflow)), 1)+++if __name__ == "__main__":+ unittest.main()diff --git a/tests/ci/standalone/test_release_scan_retire_sarif.py b/tests/ci/standalone/test_release_scan_retire_sarif.pynew file mode 100644index 0000000000..24d3b42d1e--- /dev/null+++ b/tests/ci/standalone/test_release_scan_retire_sarif.py@@ -0,0 +1,557 @@+"""Standalone tests for the retire.js JSON to SARIF converter."""++# allow: no-sut-import — imports the standalone CI script directly.++import copy+import importlib.util+import io+import json+import tempfile+import unittest+from contextlib import redirect_stderr, redirect_stdout+from pathlib import Path++ROOT = Path(__file__).resolve().parents[3]+++def _load(name, relative):+ spec = importlib.util.spec_from_file_location(name, ROOT / relative)+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module+++CONVERTER = _load("retire_sarif", ".github/scripts/retire_json_to_sarif.py")+POLICY = _load(+ "release_scan_for_retire",+ ".github/scripts/check_release_scan_evidence.py",+)++RUNNER_ROOT = "/home/runner/work/local-deep-research/local-deep-research"++# A real `npx [email protected] --severity medium --outputformat json` report+# (exit status 13) for DOMPurify 2.0.0 and jQuery 3.4.1, trimmed to two+# files and four vulnerabilities, with the scratch directory replaced by a+# runner checkout path.+REAL_REPORT = {+ "version": "5.2.4",+ "start": "2026-09-26T12:14:07.084Z",+ "data": [+ {+ "file": f"{RUNNER_ROOT}/node_modules/dompurify/dist/purify.js",+ "results": [+ {+ "version": "2.0.0",+ "component": "DOMPurify",+ "npmname": "dompurify",+ "detection": "filecontent",+ "vulnerabilities": [+ {+ "info": [+ "https://github.com/cure53/DOMPurify/releases"+ ],+ "below": "2.0.3",+ "severity": "medium",+ "identifiers": {+ "summary": "Fixed an mXSS-based bypass "+ "caused by nested forms inside MathML",+ "githubID": "GHSA-chqj-j4fh-rw7m",+ "CVE": ["CVE-2019-16728"],+ },+ "cwe": ["CWE-79"],+ },+ {+ "info": [+ "https://github.com/cure53/DOMPurify/releases"+ ],+ "below": "2.0.7",+ "severity": "high",+ "identifiers": {+ "summary": "possible to bypass the package "+ "sanitization through Mutation XSS",+ "githubID": "GHSA-mjjq-c88q-qhr6",+ },+ "cwe": ["CWE-79"],+ },+ {+ "info": [+ "https://github.com/cure53/DOMPurify/releases"+ ],+ "below": "2.0.16",+ "severity": "low",+ "identifiers": {+ "summary": "Fixed an mXSS-based bypass "+ "caused by nested forms inside MathML",+ "retid": "28",+ },+ "cwe": ["CWE-79"],+ },+ ],+ "licenses": ["MPL-2.0 OR Apache-2.0"],+ }+ ],+ },+ {+ "file": f"{RUNNER_ROOT}/node_modules/jquery/dist/jquery.min.js",+ "results": [+ {+ "version": "3.4.1",+ "component": "jquery",+ "npmname": "jquery",+ "detection": "filecontent",+ "vulnerabilities": [+ {+ "info": [+ "https://blog.jquery.com/2020/04/10/"+ "jquery-3-5-0-released/"+ ],+ "below": "3.5.0",+ "atOrAbove": "1.0.3",+ "severity": "medium",+ "identifiers": {+ "summary": "passing HTML containing <option> "+ "elements from untrusted sources - even after "+ "sanitizing it - to one of jQuery's DOM "+ "manipulation methods (i.e. .html(), "+ ".append(), and others) may execute "+ "untrusted code.",+ "CVE": ["CVE-2020-11023"],+ "issue": "4647",+ "githubID": "GHSA-jpcq-cgw6-v4j6",+ },+ "cwe": ["CWE-79"],+ }+ ],+ "licenses": ["MIT"],+ }+ ],+ },+ ],+ "messages": [],+ "errors": [],+ "time": 2.244,+}++# A real clean report (exit status 0) for this repository's checkout.+EMPTY_REPORT = {+ "version": "5.2.4",+ "start": "2026-09-26T12:12:41.179Z",+ "data": [],+ "messages": [],+ "errors": [],+ "time": 2.22,+}+++def _vulnerability(severity=None, **identifiers):+ vulnerability = {+ "info": ["https://example.invalid/advisory"],+ "below": "2.0.0",+ "identifiers": {"summary": "an advisory", **identifiers},+ }+ if severity is not None:+ vulnerability["severity"] = severity+ return vulnerability+++def _report(*vulnerabilities, file="node_modules/lib/dist/lib.js"):+ return {+ "version": "5.2.4",+ "data": [+ {+ "file": file,+ "results": [+ {+ "version": "1.0.0",+ "component": "lib",+ "vulnerabilities": list(vulnerabilities),+ }+ ],+ }+ ],+ "errors": [],+ }+++def _results_by_rule(sarif):+ return {r["ruleId"]: r for r in sarif["runs"][0]["results"]}+++def _rules(sarif):+ return {r["id"]: r for r in sarif["runs"][0]["tool"]["driver"]["rules"]}+++class ConvertTests(unittest.TestCase):+ def test_real_report_becomes_sarif_with_relative_locations(self):+ sarif = CONVERTER.convert(REAL_REPORT, RUNNER_ROOT)+ self.assertEqual(sarif["version"], "2.1.0")+ run = sarif["runs"][0]+ self.assertEqual(run["tool"]["driver"]["name"], "retire.js")+ self.assertEqual(run["tool"]["driver"]["version"], "5.2.4")+ results = _results_by_rule(sarif)+ self.assertEqual(+ sorted(results),+ [+ "GHSA-chqj-j4fh-rw7m",+ "GHSA-jpcq-cgw6-v4j6",+ "GHSA-mjjq-c88q-qhr6",+ "RETIRE-28",+ ],+ )+ self.assertEqual(sorted(_rules(sarif)), sorted(results))+ self.assertEqual(+ {+ rule: (r["level"], r["properties"]["security-severity"])+ for rule, r in results.items()+ },+ {+ "GHSA-chqj-j4fh-rw7m": ("warning", "5.5"),+ "GHSA-jpcq-cgw6-v4j6": ("warning", "5.5"),+ "GHSA-mjjq-c88q-qhr6": ("error", "8.0"),+ "RETIRE-28": ("note", "2.0"),+ },+ )+ location = results["GHSA-jpcq-cgw6-v4j6"]["locations"][0][+ "physicalLocation"+ ]["artifactLocation"]+ self.assertEqual(+ location["uri"], "node_modules/jquery/dist/jquery.min.js"+ )+ self.assertEqual(location["uriBaseId"], "%SRCROOT%")+ self.assertIn(+ "CVE-2020-11023", results["GHSA-jpcq-cgw6-v4j6"]["message"]["text"]+ )+ # No absolute runner path may reach code scanning.+ self.assertNotIn(RUNNER_ROOT, json.dumps(sarif))+ self.assertNotIn('"/', json.dumps(run["results"]))++ def test_empty_report_is_a_valid_sarif_with_no_results(self):+ sarif = CONVERTER.convert(EMPTY_REPORT, RUNNER_ROOT)+ self.assertEqual(sarif["version"], "2.1.0")+ self.assertEqual(len(sarif["runs"]), 1)+ self.assertEqual(sarif["runs"][0]["results"], [])+ self.assertEqual(sarif["runs"][0]["tool"]["driver"]["rules"], [])+ self.assertEqual(+ sarif["runs"][0]["tool"]["driver"]["name"], "retire.js"+ )++ def test_severity_mapping(self):+ expected = {+ "critical": ("error", "9.5"),+ "high": ("error", "8.0"),+ "medium": ("warning", "5.5"),+ "low": ("note", "2.0"),+ # retire.js's schema allows "none": an unscored note.+ "none": ("note", None),+ # retire.js treats a missing severity as critical.+ None: ("error", "9.5"),+ }+ for severity, (level, score) in expected.items():+ with self.subTest(severity=severity):+ sarif = CONVERTER.convert(+ _report(_vulnerability(severity, githubID="GHSA-1")), "/r"+ )+ result = sarif["runs"][0]["results"][0]+ rule = sarif["runs"][0]["tool"]["driver"]["rules"][0]+ self.assertEqual(result["level"], level)+ self.assertEqual(+ result["properties"].get("security-severity"), score+ )+ self.assertEqual(rule["defaultConfiguration"]["level"], level)+ self.assertEqual(+ rule["properties"].get("security-severity"), score+ )++ def test_null_severity_is_critical_like_retire(self):+ # retire.js 5.2.4 uses `v.severity ?? 'critical'`, so an explicit+ # JSON null is critical too, not an unknown severity.+ vulnerability = _vulnerability(githubID="GHSA-1")+ vulnerability["severity"] = None+ sarif = CONVERTER.convert(_report(vulnerability), "/r")+ result = sarif["runs"][0]["results"][0]+ self.assertEqual(result["level"], "error")+ self.assertEqual(result["properties"]["security-severity"], "9.5")++ def test_a_none_severity_advisory_does_not_reject_the_report(self):+ report = copy.deepcopy(REAL_REPORT)+ report["data"][1]["results"][0]["vulnerabilities"].append(+ _vulnerability("none", githubID="GHSA-none")+ )+ sarif = CONVERTER.convert(report, RUNNER_ROOT)+ results = _results_by_rule(sarif)+ self.assertEqual(len(results), 5)+ none = results["GHSA-none"]+ self.assertEqual(none["level"], "note")+ self.assertNotIn("security-severity", none["properties"])+ self.assertNotIn("none vulnerability", none["message"]["text"])+ self.assertIn("rates severity none", none["message"]["text"])+ rule = _rules(sarif)["GHSA-none"]+ self.assertNotIn("security-severity", rule["properties"])++ def test_mapping_blocks_exactly_what_the_release_gate_blocks(self):+ """The alert gate must block critical/high/medium, never low."""++ def github_band(score):+ # GitHub's documented security-severity bands.+ score = float(score)+ if score >= 9.0:+ return "critical"+ if score >= 7.0:+ return "high"+ if score >= 4.0:+ return "medium"+ return "low"++ for severity in ("critical", "high", "medium", "low", "none"):+ with self.subTest(severity=severity):+ sarif = CONVERTER.convert(+ _report(_vulnerability(severity, githubID="GHSA-1")), "/r"+ )+ rule = sarif["runs"][0]["tool"]["driver"]["rules"][0]+ score = rule["properties"].get("security-severity")+ if severity == "none":+ # Unscored: GitHub reports no security severity level,+ # so only the SARIF-level fallback applies.+ self.assertIsNone(score)+ bands = (None,)+ else:+ band = github_band(score)+ self.assertEqual(band, severity)+ bands = (band, None)+ blocks = severity not in {"low", "none"}+ for scored in bands:+ alert = {+ "number": 1,+ "state": "open",+ "tool": {"name": "retire.js"},+ "rule": {+ "security_severity_level": scored,+ "severity": rule["defaultConfiguration"]["level"],+ },+ }+ self.assertEqual(POLICY.blocking_alert(alert), blocks)++ def test_rule_takes_the_highest_severity_of_a_shared_identifier(self):+ sarif = CONVERTER.convert(+ _report(+ _vulnerability("low", githubID="GHSA-1"),+ _vulnerability("high", githubID="GHSA-1"),+ ),+ "/r",+ )+ rule = _rules(sarif)["GHSA-1"]+ self.assertEqual(rule["properties"]["security-severity"], "8.0")++ def test_a_repeated_finding_keeps_its_highest_severity(self):+ for order in (("low", "high"), ("high", "low")):+ with self.subTest(order=order):+ sarif = CONVERTER.convert(+ _report(+ *(+ _vulnerability(severity, githubID="GHSA-1")+ for severity in order+ )+ ),+ "/r",+ )+ results = sarif["runs"][0]["results"]+ self.assertEqual(len(results), 1)+ result = results[0]+ self.assertEqual(result["level"], "error")+ self.assertEqual(+ result["properties"]["security-severity"], "8.0"+ )+ self.assertIn(+ "known high vulnerability", result["message"]["text"]+ )+ # The upgrade replaces the result in place: order is preserved.+ sarif = CONVERTER.convert(+ _report(+ _vulnerability("low", githubID="GHSA-1"),+ _vulnerability("medium", githubID="GHSA-2"),+ _vulnerability("critical", githubID="GHSA-1"),+ ),+ "/r",+ )+ self.assertEqual(+ [(r["ruleId"], r["level"]) for r in sarif["runs"][0]["results"]],+ [("GHSA-1", "error"), ("GHSA-2", "warning")],+ )++ def test_a_repeated_finding_in_one_file_is_reported_once(self):+ vulnerability = _vulnerability("medium", githubID="GHSA-1")+ report = _report(vulnerability, vulnerability)+ report["data"].append(copy.deepcopy(report["data"][0]))+ sarif = CONVERTER.convert(report, "/r")+ self.assertEqual(len(sarif["runs"][0]["results"]), 1)+ report["data"][1]["file"] = "node_modules/lib/dist/lib.min.js"+ sarif = CONVERTER.convert(report, "/r")+ self.assertEqual(len(sarif["runs"][0]["results"]), 2)++ def test_rule_identifier_preference(self):+ cases = (+ ({"githubID": "GHSA-a", "CVE": ["CVE-1"]}, "GHSA-a"),+ ({"CVE": ["CVE-1", "CVE-2"], "retid": "7"}, "CVE-1"),+ ({"retid": "7", "issue": "9"}, "RETIRE-7"),+ ({"issue": "9"}, "issue-9"),+ )+ for identifiers, rule_id in cases:+ with self.subTest(identifiers=identifiers):+ sarif = CONVERTER.convert(+ _report(_vulnerability("medium", **identifiers)), "/r"+ )+ self.assertEqual(list(_results_by_rule(sarif)), [rule_id])+ sarif = CONVERTER.convert(_report(_vulnerability("medium")), "/r")+ self.assertRegex(+ sarif["runs"][0]["results"][0]["ruleId"],+ r"^RETIRE-lib-[0-9a-f]{16}$",+ )++ def test_relative_and_absolute_paths_share_one_uri(self):+ for file in ("/r/a/b.js", "a/b.js", "/r/a/./c/../b.js"):+ with self.subTest(file=file):+ sarif = CONVERTER.convert(+ _report(_vulnerability("low", retid="1"), file=file), "/r"+ )+ uri = sarif["runs"][0]["results"][0]["locations"][0][+ "physicalLocation"+ ]["artifactLocation"]["uri"]+ self.assertEqual(uri, "a/b.js")++ def test_malformed_reports_raise(self):+ def mutate(change):+ report = copy.deepcopy(REAL_REPORT)+ change(report)+ return report++ def set_vuln(key, value):+ def change(report):+ vuln = report["data"][0]["results"][0]["vulnerabilities"][0]+ vuln[key] = value++ return change++ malformed = {+ "not an object": [],+ "no version": mutate(lambda r: r.pop("version")),+ "data missing": mutate(lambda r: r.pop("data")),+ "data not a list": mutate(lambda r: r.update(data={})),+ "entry not an object": mutate(lambda r: r.update(data=["x"])),+ "file missing": mutate(lambda r: r["data"][0].pop("file")),+ "file outside root": mutate(+ lambda r: r["data"][0].update(file="/etc/passwd.js")+ ),+ "file escapes root": mutate(+ lambda r: r["data"][0].update(file="../outside.js")+ ),+ "results not a list": mutate(+ lambda r: r["data"][0].update(results={})+ ),+ "component missing": mutate(+ lambda r: r["data"][0]["results"][0].pop("component")+ ),+ "version not a string": mutate(+ lambda r: r["data"][0]["results"][0].update(version=2)+ ),+ "vulnerabilities not a list": mutate(+ lambda r: r["data"][0]["results"][0].update(vulnerabilities={})+ ),+ "unknown severity": mutate(set_vuln("severity", "moderate")),+ "identifiers not a map": mutate(set_vuln("identifiers", [])),+ "info not a list": mutate(set_vuln("info", "https://x")),+ "errors not a list": mutate(lambda r: r.update(errors="boom")),+ }+ for name, report in malformed.items():+ with self.subTest(name=name):+ with self.assertRaises(CONVERTER.ReportError):+ CONVERTER.convert(report, RUNNER_ROOT)+++class MainTests(unittest.TestCase):+ def run_main(self, report_text, *flags, existing_output=False):+ with tempfile.TemporaryDirectory() as directory:+ work = Path(directory)+ report = work / "retire-results.json"+ output = work / "retire-results.sarif"+ if report_text is not None:+ report.write_text(report_text, encoding="utf-8")+ if existing_output:+ output.write_text("stale", encoding="utf-8")+ stderr = io.StringIO()+ with redirect_stdout(io.StringIO()), redirect_stderr(stderr):+ code = CONVERTER.main(+ [*flags, "--root", str(work), str(report), str(output)]+ )+ written = (+ output.read_text(encoding="utf-8") if output.exists() else None+ )+ leftovers = sorted(p.name for p in work.iterdir())+ return code, written, stderr.getvalue(), leftovers++ def test_clean_report_writes_an_empty_sarif(self):+ code, written, _, leftovers = self.run_main(json.dumps(EMPTY_REPORT))+ self.assertEqual(code, 0)+ self.assertEqual(json.loads(written)["runs"][0]["results"], [])+ self.assertEqual(+ leftovers, ["retire-results.json", "retire-results.sarif"]+ )++ def test_findings_report_writes_results(self):+ report = _report(_vulnerability("high", githubID="GHSA-1"))+ code, written, _, _ = self.run_main(+ json.dumps(report), "--require-findings"+ )+ self.assertEqual(code, 0)+ self.assertEqual(len(json.loads(written)["runs"][0]["results"]), 1)++ def test_require_findings_accepts_a_none_severity_only_report(self):+ # With --severity none, retire.js exits 13 for a "none" advisory.+ report = _report(_vulnerability("none", githubID="GHSA-1"))+ code, written, _, _ = self.run_main(+ json.dumps(report), "--require-findings"+ )+ self.assertEqual(code, 0)+ results = json.loads(written)["runs"][0]["results"]+ self.assertEqual([r["level"] for r in results], ["note"])++ def test_require_findings_rejects_an_empty_report(self):+ code, written, stderr, _ = self.run_main(+ json.dumps(EMPTY_REPORT), "--require-findings"+ )+ self.assertEqual(code, 1)+ self.assertIsNone(written)+ self.assertIn("::error::", stderr)++ def test_malformed_input_fails_and_writes_nothing(self):+ cases = {+ "missing file": None,+ "empty file": "",+ "not json": "{not json",+ "truncated": json.dumps(REAL_REPORT)[:-10],+ "wrong shape": json.dumps({"version": "5.2.4", "data": {}}),+ }+ for name, text in cases.items():+ with self.subTest(name=name):+ code, written, stderr, _ = self.run_main(+ text, existing_output=True+ )+ self.assertEqual(code, 1)+ # A stale SARIF from an earlier step must not survive either.+ self.assertIsNone(written)+ self.assertIn("::error::", stderr)++ def test_error_messages_cannot_inject_workflow_commands(self):+ self.assertEqual(+ CONVERTER._workflow_command_data("a%b\r\n::error::c"),+ "a%25b%0D%0A::error::c",+ )+ report = copy.deepcopy(EMPTY_REPORT)+ report["errors"] = ["Could not parse file: a.js\n::error::injected"]+ code, _, stderr, _ = self.run_main(json.dumps(report))+ self.assertEqual(code, 0)+ self.assertEqual(stderr.count("\n"), 1)+ self.assertIn("%0A::error::injected", stderr)+++if __name__ == "__main__":+ unittest.main()