Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
Remove unused workflow files (#282)
Paul Abbazia68fc1a3654df
diff --git a/.github/renovate-security/.gitignore b/.github/renovate-security/.gitignoredeleted file mode 100644index 7a60b85e1..000000000--- a/.github/renovate-security/.gitignore+++ /dev/null@@ -1,2 +0,0 @@-__pycache__/-*.pycdiff --git a/.github/renovate-security/jira_client.py b/.github/renovate-security/jira_client.pydeleted file mode 100644index 77805f22f..000000000--- a/.github/renovate-security/jira_client.py+++ /dev/null@@ -1,397 +0,0 @@-"""Tiny Jira REST client (stdlib only) for linking/creating vulnerability tickets.--Uses REST API v3 with basic auth (email + API token) against the Jira Cloud-instance named by JIRA_BASE_URL: search via /rest/api/3/search/jql (with a legacy-fallback), create with an ADF description, assign, comment, remote link and-transition to Done. Stdlib-only so the CI step needs no extra pip install.--Every network call degrades gracefully: on any error it logs and returns an empty-result / None, so a Jira outage never fails the autofix run.-"""--from __future__ import annotations--import base64-import dataclasses-import json-import os-import urllib.error-import urllib.parse-import urllib.request--# Vulnerabilities are tracked in the VULNMGMT project. Both search and create-# default here; the driver overrides them from JIRA_VULN_PROJECT when set.-SEARCH_PROJECTS = ("VULNMGMT",)-CREATE_PROJECT = "VULNMGMT"---class JiraIssue:- key: str- summary: str- url: str- status: str = ""- # Stable numeric issue id. Unlike `key`, it does NOT change when VULNMGMT- # automation MOVES the issue to a BU project (VULNMGMT-496 -> QNV-7011), so all- # post-create writes (route/verify/comment/remotelink/close) must key off id.- id: str = ""---class JiraClient:- def __init__(self, base: str, email: str, token: str):- self.base = base.rstrip("/")- self._auth = base64.b64encode(f"{email}:{token}".encode()).decode()-- @classmethod- def from_env(cls) -> JiraClient | None:- email = os.environ.get("SBT_MACHINE_USER_EMAIL")- token = os.environ.get("SBT_MACHINE_USER_JIRA_TOKEN")- # The workflow sets JIRA_BASE_URL: ${{ vars.JIRA_BASE_URL }}, which is the- # EMPTY STRING when the variable is undefined. Treat empty as unset and- # skip — there is no hardcoded fallback host, since this tree is public.- base = os.environ.get("JIRA_BASE_URL") or ""- if not (base and email and token):- return None- return cls(base, email, token)-- def _request(self, method: str, path: str, body: dict | None = None) -> dict | None:- url = f"{self.base}{path}"- data = json.dumps(body).encode() if body is not None else None- req = urllib.request.Request(url, data=data, method=method)- req.add_header("Authorization", f"Basic {self._auth}")- req.add_header("Accept", "application/json")- if data is not None:- req.add_header("Content-Type", "application/json")- try:- with urllib.request.urlopen(req, timeout=30) as resp:- raw = resp.read().decode()- return json.loads(raw) if raw else {}- except urllib.error.HTTPError as e:- detail = e.read().decode(errors="replace")[:500]- print(f" ! jira {method} {path} -> HTTP {e.code}: {detail}")- except Exception as e: # noqa: BLE001 - graceful degradation is the point- print(f" ! jira {method} {path} failed: {e}")- return None-- def search(self, jql: str, max_results: int = 20) -> list[JiraIssue]:- # Atlassian Cloud replaced /rest/api/{2,3}/search with /rest/api/3/search/jql- # (the old ones were removed in 2025). Try the current endpoint first, then- # fall back to the legacy one for older/Server instances.- params = urllib.parse.urlencode(- {"jql": jql, "maxResults": max_results, "fields": "summary,status"}- )- data = self._request("GET", f"/rest/api/3/search/jql?{params}")- if data is None:- data = self._request("GET", f"/rest/api/2/search?{params}")- if not data:- return []- out: list[JiraIssue] = []- for issue in data.get("issues", []):- key = issue.get("key", "")- fields = issue.get("fields", {})- out.append(- JiraIssue(- key=key,- summary=fields.get("summary", ""),- url=f"{self.base}/browse/{key}",- status=(fields.get("status") or {}).get("name", ""),- )- )- return out-- def create(- self,- summary: str,- description: str,- labels: list[str] | None = None,- project: str = CREATE_PROJECT,- issue_type: str = "Task",- epic_key: str | None = None,- ) -> JiraIssue | None:- # REST v3 requires an ADF description.- fields = {- "project": {"key": project},- "summary": summary[:250],- "description": _adf(description),- "issuetype": {"name": issue_type},- }- if epic_key: # file under a vuln-tracking epic if one is configured- fields["parent"] = {"key": epic_key}- if labels:- fields["labels"] = labels- data = self._request("POST", "/rest/api/3/issue", {"fields": fields})- if not data or "key" not in data:- return None- key = data["key"]- return JiraIssue(- key=key,- summary=summary,- url=f"{self.base}/browse/{key}",- id=str(data.get("id") or ""),- )-- def resolve_key(self, id_or_key: str) -> str | None:- """Current key for an issue id/key. After VULNMGMT automation MOVES a created- issue to its BU project, the numeric id still resolves but returns the NEW key- (e.g. QNV-7011). Used to stamp the real key into the PR body / close marker."""- data = self._request(- "GET", f"/rest/api/3/issue/{urllib.parse.quote(id_or_key)}?fields=key"- )- if isinstance(data, dict):- return data.get("key")- return None-- def myself(self) -> dict | None:- """The account this token authenticates as (email, displayName, accountId) —- so permissions can be granted to the RIGHT account."""- data = self._request("GET", "/rest/api/3/myself")- return data if isinstance(data, dict) else None-- def find_account_id(self, query: str) -> str | None:- """Best-effort accountId lookup for a user by email or display name, so- 'assign to aisim' works from a human-readable identifier. Returns None if- the account can't be resolved (user search is permission-gated)."""- q = urllib.parse.quote(query)- data = self._request("GET", f"/rest/api/3/user/search?query={q}")- if isinstance(data, list) and data:- aid = data[0].get("accountId")- return str(aid) if aid else None- return None-- def project_exists(self, project_key: str) -> bool:- """True if the account can browse `project_key` (GET project/{key} -> 200).- Reliable existence/visibility check for filtering search keys — one bad key- (e.g. a misspelled VULNMGMT) otherwise makes the whole JQL error -> empty.- (project/search under-reports permission-scheme-granted projects, so we- probe each key directly.)"""- data = self._request(- "GET", f"/rest/api/3/project/{urllib.parse.quote(project_key)}"- )- return isinstance(data, dict) and bool(data.get("key"))-- def field_allowed_values(self, project_key: str, field_id: str) -> list[str]:- """Allowed option names for a select custom field in `project_key` (from- createmeta). Empty if the field is free-text or has no restricted options."""- it = self._request(- "GET",- f"/rest/api/3/issue/createmeta/{urllib.parse.quote(project_key)}/issuetypes",- )- if not isinstance(it, dict):- return []- ids = [- v.get("id")- for v in (it.get("values") or it.get("issueTypes") or [])- if v.get("id")- ]- for tid in ids:- data = self._request(- "GET",- f"/rest/api/3/issue/createmeta/{urllib.parse.quote(project_key)}"- f"/issuetypes/{urllib.parse.quote(str(tid))}",- )- if not isinstance(data, dict):- continue- for field in data.get("values") or data.get("fields") or []:- if field.get("fieldId") == field_id or field.get("key") == field_id:- return [- av.get("value") or av.get("name")- for av in field.get("allowedValues", [])- if (av.get("value") or av.get("name"))- ]- return []-- def issue_types(self, project_key: str) -> list[str]:- """Issue-type names creatable in `project_key` (from createmeta) — used to- pick a valid type, since a project may not offer 'Task'."""- data = self._request(- "GET",- f"/rest/api/3/issue/createmeta/{urllib.parse.quote(project_key)}/issuetypes",- )- if not isinstance(data, dict):- return []- vals = data.get("values") or data.get("issueTypes") or []- return [v.get("name", "") for v in vals if v.get("name")]-- def can_create_in(self, project_key: str) -> bool:- """True if the authenticated account can create issues in `project_key`- (createmeta returns issue types only for projects it can create in)."""- data = self._request(- "GET",- f"/rest/api/3/issue/createmeta/{urllib.parse.quote(project_key)}/issuetypes",- )- if not isinstance(data, dict):- return False- return bool(data.get("values") or data.get("issueTypes"))-- def add_comment(self, key: str, text: str) -> bool:- """Add an ADF comment to an issue. Best-effort (returns False on failure)."""- data = self._request(- "POST",- f"/rest/api/3/issue/{urllib.parse.quote(key)}/comment",- {"body": _adf(text)},- )- return data is not None-- def add_remote_link(self, key: str, url: str, title: str) -> bool:- """Add/refresh a 'Web link' on the issue pointing at `url` (e.g. the fix PR).- Idempotent: `globalId` = the url, so reruns update the same link instead of- adding duplicates. Best-effort."""- body = {"globalId": url, "object": {"url": url, "title": title[:250]}}- data = self._request(- "POST", f"/rest/api/3/issue/{urllib.parse.quote(key)}/remotelink", body- )- return data is not None-- def set_assignee(- self,- key: str,- account_id: str | None = None,- name: str | None = None,- ) -> bool:- """Assign an issue (e.g. to the aisim account). Jira Cloud wants an- accountId; Server/DC wants a name/username. Try accountId first, then name.- Best-effort — returns True if either shape is accepted."""- ek = urllib.parse.quote(key)- payloads = []- if account_id:- payloads.append({"accountId": account_id})- if name:- payloads.append({"name": name})- for payload in payloads:- data = self._request("PUT", f"/rest/api/3/issue/{ek}/assignee", payload)- if data is not None:- return True- return False-- def transitions(self, key: str) -> list[dict]:- data = self._request(- "GET", f"/rest/api/3/issue/{urllib.parse.quote(key)}/transitions"- )- return data.get("transitions", []) if isinstance(data, dict) else []-- def close_issue(self, key: str, comment: str | None = None) -> bool:- """Transition an issue to a Done-category status (optionally commenting- first). Picks the transition whose target status is in the 'done' category,- else one named like Done/Closed/Resolved. Best-effort."""- trs = self.transitions(key)- chosen = None- for t in trs:- cat = ((t.get("to") or {}).get("statusCategory") or {}).get("key")- if cat == "done":- chosen = t- break- if not chosen:- for t in trs:- if t.get("name", "").strip().lower() in (- "done",- "close",- "closed",- "resolve",- "resolved",- ):- chosen = t- break- if not chosen:- print(- f" ! jira {key}: no Done transition available "- f"(transitions: {[t.get('name') for t in trs]})"- )- return False- if comment:- self.add_comment(key, comment)- data = self._request(- "POST",- f"/rest/api/3/issue/{urllib.parse.quote(key)}/transitions",- {"transition": {"id": chosen["id"]}},- )- return data is not None-- def find_field_id(self, name_substr: str) -> str | None:- """Best-effort: id of the first field whose name contains `name_substr`- (case-insensitive), e.g. 'business unit' -> 'customfield_12345'. Lets the- BU field be auto-discovered without hardcoding its custom-field id."""- data = self._request("GET", "/rest/api/3/field")- if not isinstance(data, list):- return None- want = name_substr.lower()- for f in data:- if want in str(f.get("name", "")).lower():- fid = f.get("id")- if fid:- return str(fid)- return None-- def set_field(self, key: str, field_id: str, value: str) -> bool:- """Best-effort set of a field. We don't know the field's type, so try the- single-select object shape ({"value": ...}) then a plain string. A 400 from- the wrong shape just falls through to the next; returns True if either took."""- ek = urllib.parse.quote(key)- for payload in ({"value": value}, value):- data = self._request(- "PUT", f"/rest/api/3/issue/{ek}", {"fields": {field_id: payload}}- )- if data is not None:- return True- return False-- def field_value(self, key: str, field_id: str) -> str | None:- """Read a field's current value as a plain string (for verify-after-set).- Unwraps the single-select ``{"value"/"name": ...}`` shape. Returns None if the- field is unset or unreadable. GETs follow the redirect Jira serves when an- issue has been MOVED to another project (so the original key still resolves)."""- ek = urllib.parse.quote(key)- fq = urllib.parse.quote(field_id)- data = self._request("GET", f"/rest/api/3/issue/{ek}?fields={fq}")- if not isinstance(data, dict):- return None- val = (data.get("fields") or {}).get(field_id)- if isinstance(val, dict):- return val.get("value") or val.get("name")- return val---def _adf(text: str) -> dict:- """Wrap plain text (newline-separated) into an Atlassian Document Format doc,- as REST v3 requires for issue descriptions."""- content = []- for line in text.split("\n"):- node = {"type": "paragraph", "content": []}- if line.strip():- node["content"] = [{"type": "text", "text": line}]- content.append(node)- return {- "type": "doc",- "version": 1,- "content": content or [{"type": "paragraph", "content": []}],- }---def _quote(term: str) -> str:- """Escape a term for JQL text search (double-quote wrapped)."""- return '"' + term.replace("\\", "\\\\").replace('"', '\\"') + '"'---def build_search_jql(- terms: list[str],- repo: str = "",- projects: tuple[str, ...] = SEARCH_PROJECTS,- path: str = "",-) -> str:- """JQL for open issues in the vuln projects mentioning any of `terms`- (CVE/GHSA ids, package names) and, if given, the repo name and project `path`.-- Scoping by `path` is what keeps dedup per-project: a widely-shared CVE (urllib3,- requests) lives in many projects' tickets, so matching on the advisory + repo- alone would link every one of them onto a single project's PR. Requiring the- manifest dir (which every ticket this tool creates carries in its summary) ties- the search to the one ticket that actually tracks this project."""- proj = ", ".join(projects)- term_clause = " OR ".join(f"text ~ {_quote(t)}" for t in terms if t)- jql = f"project in ({proj}) AND statusCategory != Done"- if term_clause:- jql += f" AND ({term_clause})"- if repo:- jql += f" AND text ~ {_quote(repo)}"- if path:- jql += f" AND text ~ {_quote(path)}"- return jql + " ORDER BY updated DESC"diff --git a/.github/renovate-security/renovate_jira.py b/.github/renovate-security/renovate_jira.pydeleted file mode 100644index fd873125e..000000000--- a/.github/renovate-security/renovate_jira.py+++ /dev/null@@ -1,543 +0,0 @@-#!/usr/bin/env python3-"""VULNMGMT/Jira ticketing for security PRs opened by Renovate and by-``tf_provider_scan.py``.--Renovate (and our Terraform-provider scanner) open the PRs; this driver adds the-Jira layer that mirrors ``socket-autofix``: for each freshly-opened ``renovate/*`` PR-it searches the VULNMGMT project for an existing ticket (path-scoped so a shared CVE-doesn't attach every project's ticket to one PR), links it or creates one, **routes-the created ticket to AISIM** by setting the Business Unit field-(``customfield_12236`` = "AISIM - AI & Simulation", which fires the VULNMGMT-automation that assigns the ``dept-ai_sim`` team), adds a PR web-link, and stamps a-tracking section + a ``<!-- renovate-jira-close: KEY -->`` marker into the PR body.-On merge the shared close-on-merge workflow transitions that ticket to Done.--Two modes:- * ``--reconcile`` — the pull_request:opened path. Reads PR metadata (url, title,- head ref, current body, changed files) and writes an updated body to- ``--body-out`` for the workflow to apply with ``gh pr edit``. Idempotent: if the- body already carries the close marker, nothing is done.- * ``--close-jira KEYS`` — transition the given issues to Done (close-on-merge).--Stdlib-only and best-effort — a Jira outage logs and exits 0 so it never blocks a PR.-"""--from __future__ import annotations--import argparse-import json-import os-import sys-import time-from pathlib import Path--import jira_client--JIRA_CLOSE_MARKER = "renovate-jira-close"--# Labels that scope the standing tickets opened by --report (Trivy container/IaC-# scans). Kept distinct per engine so dedup/close only ever touch our own tickets.-_REPORT_ENGINE_LABELS = {- "container": "container image",- "iac": "Terraform misconfiguration",-}-_REPORT_BASE_LABEL = "renovate-security"--# Registry short-name / manager -> a human label for the ticket summary.-_ECO_LABELS = {- "docker": "container image",- "github-actions": "GitHub Actions",- "circleci": "CircleCI orb",- "terraform-provider": "Terraform provider",- "renovate": "dependency",-}---def _ecosystem(branch: str, files: list[str]) -> str:- """Best-effort ecosystem label from the branch name / changed files."""- if branch.startswith("renovate/tf-provider-"):- return "terraform-provider"- lowered = [f.lower() for f in files]- if any(- "dockerfile" in f or "docker-compose" in f or f.endswith(".dockerfile")- for f in lowered- ):- return "docker"- if any(- f.startswith(".github/workflows/") or f.startswith(".github/actions/")- for f in lowered- ):- return "github-actions"- if any(f.startswith(".circleci/") for f in lowered):- return "circleci"- if any(f.endswith(".tf") or f.endswith(".terraform.lock.hcl") for f in lowered):- return "terraform-provider"- return "renovate"---def _project_dir(files: list[str]) -> str:- """Deepest directory common to all changed files — the 'project path' used to- scope Jira dedup (Renovate groups one PR per dir, so files share a prefix)."""- dirs = [str(Path(f).parent) for f in files if f]- if not dirs:- return ""- if len(dirs) == 1:- return dirs[0]- parts = [d.split("/") for d in dirs]- common: list[str] = []- for seg in zip(*parts, strict=False):- if len(set(seg)) == 1:- common.append(seg[0])- else:- break- return "/".join(common)---def _read_files_arg(files_arg: str) -> list[str]:- """--files may be a path to a newline-list (from `gh pr diff --name-only`) or a- comma-separated inline list."""- if not files_arg:- return []- p = Path(files_arg)- if p.exists():- return [ln.strip() for ln in p.read_text().splitlines() if ln.strip()]- return [f.strip() for f in files_arg.split(",") if f.strip()]---def _tracking_section(- created: jira_client.JiraIssue | None,- linked: list[jira_client.JiraIssue],- assigned: bool,-) -> str:- if linked:- lines = "\n".join(- f"- Linked existing: [{i.key}]({i.url}) — {i.summary} ({i.status})"- for i in linked- )- elif created:- lines = (- f"- No open VULNMGMT ticket found; created [{created.key}]({created.url})"- + (- " and routed to its **Business Unit** for triage."- if assigned- else " (Business Unit routing field could not be set — a human may need to route it)."- )- )- else:- lines = (- "- No open VULNMGMT ticket found and none created (Jira unavailable or "- "disabled). Please file/track manually."- )- marker = f"\n<!-- {JIRA_CLOSE_MARKER}: {created.key} -->" if created else ""- return f"\n\n## Vulnerability tracking (VULNMGMT)\n\n{lines}\n{marker}\n"---def _dedup_projects(- client: jira_client.JiraClient, create_project: str-) -> tuple[str, ...]:- """Projects to scan when de-duplicating before create: the BU-split project- (VULNMGMT) plus the legacy full-scope VULN project when visible. Probing- visibility first avoids a JQL error from an `project in (...)` clause naming a- project the account can't browse (which would drop dedup to empty and dup tickets).- Override the legacy project via JIRA_LEGACY_PROJECT ("" disables it)."""- projects = [create_project]- legacy = os.environ.get("JIRA_LEGACY_PROJECT")- legacy = "VULN" if legacy is None else legacy- if legacy and legacy != create_project and client.project_exists(legacy):- projects.append(legacy)- return tuple(projects)---def _route_aisim(- client: jira_client.JiraClient, issue_id: str, orig_key: str-) -> tuple[bool, str]:- """Set the Business Unit routing field and return (routed, settled_key).-- Right after creation the VULNMGMT automation CHURNS the issue for ~15s (observed in- the changelog): a create-time rule clobbers the BU to a default (INFOSEC), assignee- automation fires, and the issue is MOVED between the VULNMGMT project and the target- BU project several times — its key changing each time- (VULNMGMT-498 -> QNV-7012 -> VULNMGMT-499 -> QNV-7013). During every move the issue- transiently 404s by BOTH key and id. So we:-- 1. set the BU field, re-asserting through the default-BU clobber and the mid-move- 404s (each successful re-assert is what ultimately makes our value stick);- 2. wait for the churn to settle — poll until the key is readable and unchanged- across two consecutive polls;- 3. read the settled BU to confirm routing.-- Every call keys off the STABLE numeric id (the key is a moving target). `routed` is- best-effort: a read can still land mid-move, so callers must not treat False as- 'definitely unrouted'. Tunable via JIRA_AISIM_VERIFY_ATTEMPTS / _VERIFY_SLEEP- (tests pass sleep 0)."""- field_id = os.environ.get("JIRA_AISIM_FIELD_ID") or "customfield_12236"- field_value = os.environ.get("JIRA_AISIM_FIELD_VALUE") or "AISIM - AI & Simulation"- attempts = int(os.environ.get("JIRA_AISIM_VERIFY_ATTEMPTS") or "4")- sleep_s = float(os.environ.get("JIRA_AISIM_VERIFY_SLEEP") or "10")- routed = False # verified by a read-back (only possible while still in VULNMGMT)- set_ok = False # a set_field PUT was accepted (the honest routing signal from CI)- if field_id and field_value:- for _ in range(max(1, attempts)):- if client.set_field(issue_id, field_id, field_value):- set_ok = True- time.sleep(sleep_s) # let the create-time default automation fire first- if client.field_value(issue_id, field_id) == field_value:- routed = True # survived a cycle- break-- # Settle: wait for the move churn to stop so the key/reads are reliable. NOTE: the- # sbt-machine account has no browse permission in the destination BU project, so- # once the automation moves the issue out of VULNMGMT these reads 404 permanently;- # this loop still resolves the final key WHEN the account is granted BU-project- # access (the proper fix), and otherwise falls back to the original key.- settled_key = client.resolve_key(issue_id) or orig_key- last = settled_key- for _ in range(max(1, attempts)):- time.sleep(sleep_s)- k = client.resolve_key(issue_id)- if k:- settled_key = k- if k == last: # unchanged across two consecutive polls -> settled- break- last = k- # Final confirm on the settled issue (impossible if the account can't browse the- # destination project). Fall back to the write-succeeded signal: routing is driven- # by the BU field we set, which the org automation then acts on.- if field_id and not routed:- routed = client.field_value(issue_id, field_id) == field_value- routed = routed or set_ok-- account = os.environ.get("JIRA_AISIM_ACCOUNT_ID")- user = os.environ.get("JIRA_AISIM_USER")- if account:- client.set_assignee(issue_id, account_id=account)- elif user:- aid = client.find_account_id(user)- if not (aid and client.set_assignee(issue_id, account_id=aid)):- client.set_assignee(issue_id, name=user)- return routed, settled_key---def reconcile(args) -> int:- body = ""- if args.body_file and Path(args.body_file).exists():- body = Path(args.body_file).read_text()- if f"{JIRA_CLOSE_MARKER}:" in body:- print("PR already carries a renovate-jira-close marker; nothing to do.")- return 0-- files = _read_files_arg(args.files)- project_dir = _project_dir(files)- eco = _ecosystem(args.branch, files)- eco_label = _ECO_LABELS.get(eco, "dependency")- repo_name = args.repo.split("/")[-1] if args.repo else ""-- client = jira_client.JiraClient.from_env()- if client is None:- print(" ! Jira creds not set; skipping ticket create/link.")- return 0-- create_project = os.environ.get("JIRA_VULN_PROJECT") or jira_client.CREATE_PROJECT- if not client.project_exists(create_project):- print(f" ! VULNMGMT project {create_project} not visible; skipping.")- return 0-- # Path-scoped dedup: link a pre-existing open ticket for this eco+path, else create.- # Scan BOTH the new BU-split project (VULNMGMT) and the legacy full-scope VULN- # project so we link an existing ticket instead of opening a duplicate.- terms = [t for t in [eco_label, Path(project_dir).name] if t]- jql = jira_client.build_search_jql(- terms,- repo=repo_name,- projects=_dedup_projects(client, create_project),- path=project_dir,- )- linked = client.search(jql) if project_dir else []-- created: jira_client.JiraIssue | None = None- assigned = False- if not linked and not args.no_create:- issue_type = os.environ.get("JIRA_ISSUE_TYPE") or ""- if not issue_type:- avail = client.issue_types(create_project)- for pref in ("Task", "Bug", "Vulnerability", "Security", "Story"):- if pref in avail:- issue_type = pref- break- else:- issue_type = avail[0] if avail else "Task"- summary = (- f"[{repo_name}] {eco_label} security update under {project_dir or '(repo)'}"- )- desc = (- f"Automated by renovate-security. A {eco_label} security update was opened "- f"on branch {args.branch}.\n\nPR: {args.pr_url}\nProject path: "- f"{project_dir or '(repo root)'}\nTitle: {args.pr_title}"- )- aisim_label = os.environ.get("JIRA_AISIM_LABEL") or "aisim"- created = client.create(- summary,- desc,- labels=["renovate-security", repo_name, aisim_label],- project=create_project,- issue_type=issue_type,- epic_key=os.environ.get("JIRA_VULN_EPIC") or None,- )- if created:- # Route/verify/comment via the STABLE id: setting the BU fires automation- # that moves the issue to the BU project, changing its key mid-flight.- ref = created.id or created.key- assigned, settled_key = _route_aisim(client, ref, created.key)- # Stamp the settled post-move key so the PR body + close marker point at the- # real ticket (e.g. QNV-7011), not the now-404 VULNMGMT-xxx key.- if settled_key and settled_key != created.key:- created.key = settled_key- created.url = f"{client.base}/browse/{settled_key}"- client.add_comment(- ref,- "renovate-security — VULNMGMT routing\n\n"- + (- "Business Unit set; VULNMGMT automation routes this to the owning "- "team for triage."- if assigned- else "Could not set the Business Unit field — a human may "- "need to route this ticket."- )- + f"\n\nContext — repo: {repo_name}; path: {project_dir}; PR: {args.pr_url}",- )- print(- f" jira {created.key}: created; BU routing "- f"{'set' if assigned else 'not set'}"- )-- # PR web-link on every ticket we touched (stable id for the created one).- link_refs = [i.key for i in linked] + (- [created.id or created.key] if created else []- )- for ref in link_refs:- client.add_remote_link(ref, args.pr_url, f"{args.pr_title}: {args.pr_url}")-- display_keys = [i.key for i in linked] + ([created.key] if created else [])- if not display_keys:- print(" no Jira ticket linked or created.")- return 0-- new_body = body.rstrip() + _tracking_section(created, linked, assigned)- if args.body_out:- Path(args.body_out).write_text(new_body)- print(- f" wrote updated PR body -> {args.body_out} (keys: {','.join(display_keys)})"- )- return 0---def _report_jql(- projects: tuple[str, ...], engine_label: str, repo: str, path: str-) -> str:- """Open standing tickets for one (engine, repo, path), scoped by our own labels so- close/refresh never touches unrelated issues. Scans VULNMGMT + legacy VULN."""-- def q(t: str) -> str:- return '"' + t.replace("\\", "\\\\").replace('"', '\\"') + '"'-- jql = (- f"project in ({', '.join(projects)}) AND statusCategory != Done "- f"AND labels = {q(_REPORT_BASE_LABEL)} AND text ~ {q(engine_label)}"- )- if repo:- jql += f" AND text ~ {q(repo)}"- if path:- jql += f" AND text ~ {q(path)}"- return jql + " ORDER BY updated DESC"---def report(args) -> int:- """Standing-ticket reconciliation for the Trivy scanners (no PR involved).-- For each report group produced by ``trivy_scan.py``: if it has findings and no open- ticket exists, create one and route it to AISIM; if one already exists, refresh it- with the current counts and a link to the scan run; if the group is now clean- (``findings_count == 0``) and a ticket is open, close it. Best-effort throughout."""- data = json.loads(Path(args.report_groups).read_text())- engine = data.get("engine", "container")- groups = data.get("groups", [])- engine_label = _REPORT_ENGINE_LABELS.get(engine, "security")- repo_name = args.repo.split("/")[-1] if args.repo else ""-- client = jira_client.JiraClient.from_env()- if client is None:- print(" ! Jira creds not set; skipping report ticketing.")- return 0- project = os.environ.get("JIRA_VULN_PROJECT") or jira_client.CREATE_PROJECT- if not client.project_exists(project):- print(f" ! VULNMGMT project {project} not visible; skipping.")- return 0-- issue_type = os.environ.get("JIRA_ISSUE_TYPE") or ""- if not issue_type:- avail = client.issue_types(project)- for pref in ("Task", "Bug", "Vulnerability", "Security", "Story"):- if pref in avail:- issue_type = pref- break- else:- issue_type = avail[0] if avail else "Task"- aisim_label = os.environ.get("JIRA_AISIM_LABEL") or "aisim"-- created = refreshed = closed = 0- for g in groups:- directory = g.get("key", "")- count = int(g.get("findings_count", 0))- existing = client.search(- _report_jql(- _dedup_projects(client, project), engine_label, repo_name, directory- )- )- if count > 0:- if existing:- key = existing[0].key- client.add_comment(- key,- f"renovate-security — {engine_label} scan refresh\n\n"- f"{count} finding(s) still present under {directory}: "- f"{g.get('severity_line', '')}.\n\nScan run: {args.run_url}",- )- if args.run_url:- client.add_remote_link(- key, args.run_url, f"Latest scan: {directory}"- )- refreshed += 1- print(f" jira {key}: refreshed ({directory})")- continue- summary = f"[{repo_name}] {g.get('title', 'security findings')}"- desc = (- f"Automated by renovate-security ({engine_label} scan).\n\n"- f"{_strip_md(g.get('body', ''))}\n\nRepo: {repo_name}\nPath: "- f"{directory}\nScan run: {args.run_url}"- )- issue = client.create(- summary,- desc,- labels=[_REPORT_BASE_LABEL, repo_name, engine, aisim_label],- project=project,- issue_type=issue_type,- epic_key=os.environ.get("JIRA_VULN_EPIC") or None,- )- if not issue:- print(f" ! could not create ticket for {directory}")- continue- ref = issue.id or issue.key- assigned, settled_key = _route_aisim(client, ref, issue.key)- if settled_key and settled_key != issue.key:- issue.key = settled_key- issue.url = f"{client.base}/browse/{settled_key}"- client.add_comment(- ref,- "renovate-security — VULNMGMT routing\n\n"- + (- "Business Unit set; routed to the owning team."- if assigned- else "Could not set the Business Unit field — route manually."- ),- )- if args.run_url:- client.add_remote_link(ref, args.run_url, f"Scan: {directory}")- created += 1- routed = "set" if assigned else "not set"- print(f" jira {issue.key}: created ({directory}); BU {routed}")- elif existing:- for iss in existing:- if client.close_issue(- iss.key,- comment=(- f"Closed by renovate-security: no {engine_label} findings "- f"remain under {directory} (scan run {args.run_url})."- ),- ):- closed += 1- print(f" jira {iss.key}: closed — {directory} now clean")- print(f"report: {created} created, {refreshed} refreshed, {closed} closed.")- return 0---def _strip_md(body: str) -> str:- """Flatten a markdown body to plain text for a Jira ADF description (tables render- poorly in ADF paragraphs; keep the human-readable lines)."""- keep = []- for ln in body.splitlines():- s = ln.strip()- if not s or set(s) <= set("|-") or s.startswith("|--"):- continue- keep.append(s.replace("|", " ").replace("`", ""))- return "\n".join(keep)---def close_jira(keys_arg: str) -> int:- client = jira_client.JiraClient.from_env()- if client is None:- raise SystemExit(- "--close-jira needs SBT_MACHINE_USER_EMAIL / SBT_MACHINE_USER_JIRA_TOKEN"- )- keys = [k.strip() for k in keys_arg.split(",") if k.strip()]- failed = 0- for key in keys:- ok = client.close_issue(- key, comment="Closed by renovate-security (fix PR merged)."- )- print(f" jira close {key}: {'ok' if ok else 'FAILED'}")- failed += 0 if ok else 1- if failed:- # Best-effort: a merge must never be blocked by Jira. The marker key is the- # VULNMGMT key at creation; if the account can't reach the (moved) BU-project- # issue it 404s here and the owning team closes it on triage. Exit 0.- print(- f"::warning::{failed} ticket(s) could not be auto-closed "- "(likely moved to a BU project the CI account can't access); "- "the owning team will close on triage."- )- return 0---def main(argv: list[str] | None = None) -> int:- ap = argparse.ArgumentParser(description=__doc__)- ap.add_argument(- "--reconcile", action="store_true", help="create/link a ticket for a PR"- )- ap.add_argument(- "--report",- action="store_true",- help="standing-ticket reconcile from a trivy_scan.py report-groups JSON",- )- ap.add_argument(- "--report-groups", default="", help="path to the report-groups JSON (--report)"- )- ap.add_argument("--run-url", default="", help="scan run URL to link on tickets")- ap.add_argument(- "--close-jira", default="", help="ops: comma-separated keys -> Done"- )- ap.add_argument("--pr-url", default="")- ap.add_argument("--pr-title", default="Security update")- ap.add_argument("--branch", default="", help="PR head ref")- ap.add_argument("--body-file", default="", help="path to the current PR body")- ap.add_argument("--body-out", default="", help="write the updated PR body here")- ap.add_argument("--files", default="", help="changed files (path to list, or CSV)")- ap.add_argument("--repo", default="", help="owner/repo")- ap.add_argument("--no-create", action="store_true", help="link only; never create")- args = ap.parse_args(argv)-- if args.close_jira:- return close_jira(args.close_jira)- if args.report:- if not args.report_groups:- raise SystemExit("--report requires --report-groups")- return report(args)- if args.reconcile:- if not args.pr_url:- raise SystemExit("--reconcile requires --pr-url")- return reconcile(args)- ap.error("nothing to do: pass --reconcile, --report or --close-jira")- return 2---if __name__ == "__main__":- sys.exit(main())diff --git a/.github/renovate-security/trivy_scan.py b/.github/renovate-security/trivy_scan.pydeleted file mode 100644index 518ad469b..000000000--- a/.github/renovate-security/trivy_scan.py+++ /dev/null@@ -1,601 +0,0 @@-#!/usr/bin/env python3-"""Trivy-backed security scanner for the two ecosystems that have **no version-bump-CVE feed** Renovate/OSV can act on directly: **container base images** and **Terraform-misconfiguration**.--Why this exists (see ``README.md`` for the full rationale):--* **Container images** have no "tag -> CVE" feed — a container's vulnerabilities live- in the OS packages and libraries *inside* the image, so the feed is delivered by- *scanning the resolved image* against NVD + GHSA + the distro security trackers.- Renovate stays the sole *updater* (it opens the digest/patch bump PRs); this scanner- is the *detector*: it scans each Dockerfile's base image and reports fixable CVEs so- they get a VULNMGMT ticket routed to AISIM, and it gates Renovate's container PRs.-* **Terraform misconfiguration** is a config-risk axis (open security groups,- unencrypted buckets), not a dependency-version CVE — so it can't drive a version- bump. ``trivy config`` finds these; we ticket the HIGH/CRITICAL ones.--This module only **detects, groups and reports**. It never edits files or opens bump-PRs (that would race Renovate). Ticketing is done by ``renovate_jira.py --report`` off-the report-group JSON this emits; the workflows also render a job summary from it.--Two modes, one CLI:- * ``--mode image`` — discover base images from Dockerfiles, ``trivy image`` each.- * ``--mode config`` — ``trivy config`` over the Terraform tree.--Stdlib-only. Every Trivy invocation is best-effort: a failure for one image/target-logs and yields no findings for it rather than aborting the scan.-"""--from __future__ import annotations--import argparse-import dataclasses-import hashlib-import json-import re-import subprocess # noqa: S404 - we invoke the trivy CLI with a fixed argv (no shell)-import sys-from collections.abc import Callable-from pathlib import Path--# Container image refs we never scan: templated (resolved at build time, not a real-# image), the empty "scratch" base, and private registries that need auth (mirrors the-# renovate.json5 skip of the Artifact Registry images). Overridable via --skip-prefix.-_DEFAULT_SKIP_PREFIXES = ("us-central1-docker.pkg.dev/", ".pkg.dev/")-_TF_SUBDIR = "root/infra/terraform"--# Trivy severities we consider; anything below the --min-severity floor is dropped.-_SEV_ORDER = {"critical": 0, "high": 1, "medium": 2, "low": 3, "unknown": 4}-_SEV_ICON = {- "critical": "🔴",- "high": "🟠",- "medium": "🟡",- "low": "⚪",- "unknown": "❔",-}--SIGNATURE_MARKER = "renovate-trivy-signature"-ENGINE_IMAGE = "container"-ENGINE_CONFIG = "iac"---# --------------------------------------------------------------------------- #-# Data model-# --------------------------------------------------------------------------- #-class Finding:- directory: str # repo-relative dir (Dockerfile's parent, or the .tf's dir)- target: str # image ref (image mode) or file path (config mode)- vuln_id: str # CVE / GHSA / AVD id- severity: str # critical | high | medium | low | unknown (lowercased)- title: str- pkg: str = "" # affected package (image) or resource/type (config)- installed: str = "" # installed version (image only)- fixed: str = "" # fixed version (image only)- kind: str = ENGINE_IMAGE # image | iac-- def to_json(self) -> dict:- return dataclasses.asdict(self)---def severity_rank(sev: str) -> int:- return _SEV_ORDER.get((sev or "").lower(), 4)---def severity_badge(sev: str) -> str:- s = (sev or "unknown").lower()- return f"{_SEV_ICON.get(s, '❔')} {s.capitalize()}"---def _engine_noun(engine: str) -> str:- return "container image" if engine == ENGINE_IMAGE else "Terraform misconfiguration"---# --------------------------------------------------------------------------- #-# Dockerfile discovery (image mode)-# --------------------------------------------------------------------------- #-_FROM_RE = re.compile(- r"^\s*FROM\s+(?:--platform=\S+\s+)?(\S+)(?:\s+AS\s+(\S+))?\s*$",- re.IGNORECASE,-)---def _is_dockerfile(path: Path) -> bool:- n = path.name.lower()- return n == "dockerfile" or n.endswith(".dockerfile") or n.startswith("dockerfile.")---def find_dockerfiles(repo_root: Path) -> list[Path]:- return sorted(p for p in repo_root.rglob("*") if p.is_file() and _is_dockerfile(p))---def is_scannable(ref: str, skip_prefixes: tuple[str, ...]) -> bool:- """A base image ref we can meaningfully scan: a concrete public image, not a- build-arg-templated ref, the ``scratch`` base, or a private/authed registry."""- if not ref or "$" in ref: # ${BASE_IMAGE}, $VERSION, etc.- return False- if ref.lower() == "scratch":- return False- return not any(pre in ref for pre in skip_prefixes)---def base_images(text: str, skip_prefixes: tuple[str, ...]) -> list[str]:- """External base images referenced by a Dockerfile's ``FROM`` lines.-- Multi-stage aware: a ``FROM x AS build`` declares the stage alias ``build``; a later- ``FROM build`` refers to that stage, not an external image, so it's excluded. The- result is de-duplicated, order-preserving, and filtered by :func:`is_scannable`."""- stages: set[str] = set()- out: list[str] = []- for line in text.splitlines():- m = _FROM_RE.match(line)- if not m:- continue- ref, alias = m.group(1), m.group(2)- if ref not in stages and is_scannable(ref, skip_prefixes) and ref not in out:- out.append(ref)- if alias:- stages.add(alias)- return out---def images_by_dir(- repo_root: Path,- skip_prefixes: tuple[str, ...] = _DEFAULT_SKIP_PREFIXES,- only_dirs: set[str] | None = None,-) -> dict[str, list[str]]:- """Map ``dir -> [base image refs]`` across every Dockerfile. ``dir`` is the- Dockerfile's parent (repo-relative), matching Renovate's ``packageFileDir`` grouping- and the tf-provider tool's per-directory PRs. ``only_dirs`` limits discovery to the- given dirs (used by the PR gate to scan just what changed)."""- out: dict[str, list[str]] = {}- for df in find_dockerfiles(repo_root):- rel_dir = str(df.parent.relative_to(repo_root))- if only_dirs is not None and rel_dir not in only_dirs:- continue- refs = base_images(df.read_text(errors="replace"), skip_prefixes)- if not refs:- continue- merged = dict.fromkeys(out.get(rel_dir, []))- for r in refs:- merged[r] = None- out[rel_dir] = list(merged)- return out---# --------------------------------------------------------------------------- #-# Trivy boundary-# --------------------------------------------------------------------------- #-TrivyRunner = Callable[[list[str]], str | None]---def _run_trivy(argv: list[str]) -> str | None:- """Invoke the trivy CLI with a fixed argv (no shell) and return stdout, or None on- failure. Trivy exits non-zero when it finds vulns with --exit-code set; we never set- it here (detection is non-blocking), so a non-zero code is a real error."""- try:- proc = subprocess.run( # noqa: S603 - fixed argv, no shell, trusted binary- argv,- capture_output=True,- text=True,- timeout=600,- )- except (OSError, subprocess.SubprocessError) as e:- print(f" ! trivy {' '.join(argv[1:3])} failed to run: {e}")- return None- if proc.returncode != 0:- print(f" ! trivy exited {proc.returncode}: {proc.stderr.strip()[:400]}")- return None- return proc.stdout---def _min_sev_csv(min_severity: str) -> str:- floor = severity_rank(min_severity)- keep = [s.upper() for s, r in _SEV_ORDER.items() if r <= floor and s != "unknown"]- return ",".join(keep)---# --------------------------------------------------------------------------- #-# Trivy JSON -> Findings-# --------------------------------------------------------------------------- #-def parse_image_results(text: str, directory: str, ref: str) -> list[Finding]:- """Parse ``trivy image --format json`` output into fixable-vuln findings for one- image. Only vulnerabilities with a FixedVersion are kept (Trivy is run with- --ignore-unfixed, but we also guard here so offline fixtures behave the same)."""- try:- data = json.loads(text or "{}")- except json.JSONDecodeError as e:- print(f" ! trivy image json parse failed for {ref}: {e}")- return []- out: list[Finding] = []- for res in data.get("Results") or []:- for v in res.get("Vulnerabilities") or []:- fixed = v.get("FixedVersion") or ""- if not fixed:- continue- out.append(- Finding(- directory=directory,- target=ref,- vuln_id=v.get("VulnerabilityID") or "",- severity=(v.get("Severity") or "unknown").lower(),- title=(v.get("Title") or v.get("Description") or "").strip()[:200],- pkg=v.get("PkgName") or "",- installed=v.get("InstalledVersion") or "",- fixed=fixed,- kind=ENGINE_IMAGE,- )- )- return out---def parse_config_results(text: str, repo_root: Path) -> list[Finding]:- """Parse ``trivy config --format json`` output into misconfiguration findings.- Only failed checks (Status == FAIL) are kept; the directory is the target file's- parent, relative to the repo root when possible."""- try:- data = json.loads(text or "{}")- except json.JSONDecodeError as e:- print(f" ! trivy config json parse failed: {e}")- return []- out: list[Finding] = []- for res in data.get("Results") or []:- target = res.get("Target") or ""- directory = _rel_dir(target, repo_root)- for mc in res.get("Misconfigurations") or []:- if (mc.get("Status") or "").upper() not in ("", "FAIL"):- continue- out.append(- Finding(- directory=directory,- target=target,- vuln_id=mc.get("ID") or mc.get("AVDID") or "",- severity=(mc.get("Severity") or "unknown").lower(),- title=(mc.get("Title") or mc.get("Message") or "").strip()[:200],- pkg=(mc.get("Type") or ""),- kind=ENGINE_CONFIG,- )- )- return out---def _rel_dir(target: str, repo_root: Path) -> str:- """Directory of a trivy target path, made repo-relative when it sits under the- repo (trivy config emits repo-relative or absolute paths depending on how it's- invoked)."""- if not target:- return ""- p = Path(target)- if p.is_absolute():- try:- p = p.relative_to(repo_root)- except ValueError:- pass- d = str(p.parent)- return "" if d == "." else d---# --------------------------------------------------------------------------- #-# Scan orchestration-# --------------------------------------------------------------------------- #-def scan_images(- imgs_by_dir: dict[str, list[str]],- min_severity: str = "high",- trivy_bin: str = "trivy",- runner: TrivyRunner = _run_trivy,-) -> list[Finding]:- """Scan each unique base image once (cached across dirs) and attribute its findings- to every directory that uses it."""- sev = _min_sev_csv(min_severity)- cache: dict[str, list[Finding]] = {}- out: list[Finding] = []- for directory in sorted(imgs_by_dir):- for ref in imgs_by_dir[directory]:- if ref not in cache:- argv = [- trivy_bin, "image",- "--quiet", "--format", "json",- "--scanners", "vuln",- "--ignore-unfixed",- "--severity", sev,- ref,- ] # fmt: skip- raw = runner(argv)- cache[ref] = parse_image_results(raw or "{}", directory, ref)- for f in cache[ref]:- out.append(dataclasses.replace(f, directory=directory))- return out---def scan_config(- repo_root: Path,- tf_subdir: str = _TF_SUBDIR,- min_severity: str = "high",- trivy_bin: str = "trivy",- runner: TrivyRunner = _run_trivy,- only_dirs: set[str] | None = None,-) -> list[Finding]:- target = repo_root / tf_subdir- if not target.exists():- target = repo_root- argv = [- trivy_bin, "config",- "--quiet", "--format", "json",- "--severity", _min_sev_csv(min_severity),- str(target),- ] # fmt: skip- raw = runner(argv)- findings = parse_config_results(raw or "{}", repo_root)- if only_dirs is not None:- findings = [f for f in findings if f.directory in only_dirs]- return findings---# --------------------------------------------------------------------------- #-# Grouping + report rendering-# --------------------------------------------------------------------------- #-def group_by_dir(findings: list[Finding]) -> dict[str, list[Finding]]:- out: dict[str, list[Finding]] = {}- for f in findings:- out.setdefault(f.directory, []).append(f)- return out---def _counts(findings: list[Finding]) -> dict[str, int]:- c: dict[str, int] = {}- for f in findings:- c[f.severity] = c.get(f.severity, 0) + 1- return c---def _counts_line(counts: dict[str, int]) -> str:- parts = [- f"{severity_badge(s)}: {counts[s]}"- for s in ("critical", "high", "medium", "low", "unknown")- if counts.get(s)- ]- return " · ".join(parts) or "none"---def _signature(findings: list[Finding]) -> str:- basis = "\n".join(sorted(f"{f.target}|{f.vuln_id}|{f.fixed}" for f in findings))- return hashlib.sha256(basis.encode()).hexdigest()[:16]---def _image_rows(findings: list[Finding]) -> str:- rows = []- for f in sorted(- findings, key=lambda x: (severity_rank(x.severity), x.target, x.vuln_id)- ):- cve = (- f"[{f.vuln_id}](https://nvd.nist.gov/vuln/detail/{f.vuln_id})"- if f.vuln_id.upper().startswith("CVE-")- else f.vuln_id or "—"- )- rows.append(- f"| {severity_badge(f.severity)} | {cve} | `{f.target}` | "- f"`{f.pkg}` | {f.installed} → {f.fixed} |"- )- return "\n".join(rows)---def _config_rows(findings: list[Finding]) -> str:- rows = []- for f in sorted(- findings, key=lambda x: (severity_rank(x.severity), x.target, x.vuln_id)- ):- aid = (- f"[{f.vuln_id}](https://avd.aquasec.com/misconfig/{f.vuln_id.lower()})"- if f.vuln_id.upper().startswith("AVD-")- else f.vuln_id or "—"- )- rows.append(- f"| {severity_badge(f.severity)} | {aid} | {f.title} | `{f.target}` |"- )- return "\n".join(rows)---def _render_body(directory: str, findings: list[Finding], engine: str) -> str:- counts = _counts(findings)- n = len(findings)- if engine == ENGINE_IMAGE:- head = (- f"**{n}** fixable container-image vulnerabilit{'y' if n == 1 else 'ies'} "- f"in the base image(s) used by `{directory}`."- )- table = (- "| Severity | CVE | Image | Package | Installed → Fixed |\n"- "|----------|-----|-------|---------|-------------------|\n"- + _image_rows(findings)- )- note = (- "Fixes land via Renovate's digest/patch bump PRs for these images; a CVE "- "needing a minor/major base-image change requires a manual bump."- )- else:- head = (- f"**{n}** Terraform misconfiguration{'s' if n != 1 else ''} "- f"(HIGH/CRITICAL) under `{directory}`."- )- table = (- "| Severity | Check | Title | File |\n"- "|----------|-------|-------|------|\n" + _config_rows(findings)- )- note = "These are configuration-level risks and require a manual code change."- return (- f"## What\n\n{head}\n\nSeverity breakdown: {_counts_line(counts)}\n\n"- f"## Findings\n\n{table}\n\n<sub>Detected by Trivy "- f"({'image' if engine == ENGINE_IMAGE else 'config'} scanner). {note}</sub>\n"- )---def build_report_groups(- findings: list[Finding], engine: str, scanned_dirs: list[str] | None = None-) -> list[dict]:- """One report group per directory, for the ticket layer + job summary. When- ``scanned_dirs`` is given, dirs that were scanned but produced no findings are- emitted with ``findings_count: 0`` so ``renovate_jira.py --report`` can CLOSE a- now-clean standing ticket."""- by_dir = group_by_dir(findings)- groups: list[dict] = []- dirs = sorted(set(by_dir) | set(scanned_dirs or []))- for directory in dirs:- fs = by_dir.get(directory, [])- counts = _counts(fs)- groups.append(- {- "key": directory,- "engine": engine,- "signature": _signature(fs),- "findings_count": len(fs),- "severity_counts": counts,- "severity_line": _counts_line(counts),- "title": (- f"{_engine_noun(engine)} security findings "- f"under {directory or '(repo)'}"- ),- "body": _render_body(directory, fs, engine) if fs else "",- "findings": [f.to_json() for f in fs],- }- )- return groups---def render_summary(groups: list[dict], engine: str) -> str:- """A compact GitHub-Actions job-summary (markdown) for the scan."""- label = (- "Container image" if engine == ENGINE_IMAGE else "Terraform misconfiguration"- )- with_findings = [g for g in groups if g["findings_count"]]- total = sum(g["findings_count"] for g in groups)- if not with_findings:- return f"### {label} scan\n\n✅ No findings at or above the severity floor.\n"- lines = [- f"### {label} scan\n",- f"**{total}** finding(s) across **{len(with_findings)}** director"- f"{'y' if len(with_findings) == 1 else 'ies'}.\n",- "| Directory | Findings | Severity |",- "|-----------|----------|----------|",- ]- for g in sorted(with_findings, key=lambda x: -x["findings_count"]):- lines.append(f"| `{g['key']}` | {g['findings_count']} | {g['severity_line']} |")- return "\n".join(lines) + "\n"---# --------------------------------------------------------------------------- #-# Main-# --------------------------------------------------------------------------- #-def find_repo_root(start: Path) -> Path:- for p in [start, *start.parents]:- if (p / ".git").exists() or (p / "root" / "conda-lock.yml").exists():- return p- raise SystemExit("could not locate repo root (no .git above cwd)")---def _load_findings_json(path: str) -> list[Finding]:- raw = json.loads(Path(path).read_text())- return [- Finding(- directory=d.get("directory", ""),- target=d.get("target", ""),- vuln_id=d.get("vuln_id", ""),- severity=d.get("severity", "unknown"),- title=d.get("title", ""),- pkg=d.get("pkg", ""),- installed=d.get("installed", ""),- fixed=d.get("fixed", ""),- kind=d.get("kind", ENGINE_IMAGE),- )- for d in raw- ]---def main(argv: list[str] | None = None) -> int:- ap = argparse.ArgumentParser(description=__doc__)- ap.add_argument("--mode", choices=["image", "config"], required=True)- ap.add_argument("--repo-root", type=Path, default=None)- ap.add_argument(- "--min-severity",- default="high",- choices=["critical", "high", "medium", "low"],- help="lowest severity to report (default high)",- )- ap.add_argument("--trivy-bin", default="trivy")- ap.add_argument("--tf-subdir", default=_TF_SUBDIR)- ap.add_argument(- "--paths",- default="",- help="comma-separated dirs to limit the scan to (PR gate); default all",- )- ap.add_argument(- "--skip-prefix",- action="append",- default=[],- help="extra image-ref substring to skip (repeatable)",- )- ap.add_argument("--report-output", default="", help="write report-group JSON here")- ap.add_argument("--summary-output", default="", help="write a job-summary md here")- ap.add_argument(- "--findings-json",- default="",- help="offline: build report groups from a precomputed findings list",- )- ap.add_argument(- "--trivy-json",- default="",- help="offline (config mode): parse a captured trivy JSON instead of running it",- )- args = ap.parse_args(argv)-- repo_root = (args.repo_root or find_repo_root(Path.cwd())).resolve()- engine = ENGINE_IMAGE if args.mode == "image" else ENGINE_CONFIG- only = (- {p.strip() for p in args.paths.split(",") if p.strip()} if args.paths else None- )- skip_prefixes = _DEFAULT_SKIP_PREFIXES + tuple(args.skip_prefix)-- scanned_dirs: list[str] = []- if args.findings_json:- findings = _load_findings_json(args.findings_json)- scanned_dirs = sorted({f.directory for f in findings})- elif args.mode == "image":- imgs = images_by_dir(repo_root, skip_prefixes, only_dirs=only)- scanned_dirs = sorted(imgs)- findings = scan_images(imgs, args.min_severity, args.trivy_bin)- else: # config- if args.trivy_json:- findings = parse_config_results(- Path(args.trivy_json).read_text(), repo_root- )- if only is not None:- findings = [f for f in findings if f.directory in only]- else:- findings = scan_config(- repo_root,- args.tf_subdir,- args.min_severity,- args.trivy_bin,- only_dirs=only,- )- scanned_dirs = sorted({f.directory for f in findings})-- groups = build_report_groups(findings, engine, scanned_dirs=scanned_dirs)- n = sum(g["findings_count"] for g in groups)- print(f"trivy {args.mode} scan: {n} finding(s) across {len(scanned_dirs)} dir(s).")- for g in groups:- if g["findings_count"]:- print(f" {g['key']}: {g['findings_count']} [{g['severity_line']}]")-- if args.report_output:- Path(args.report_output).write_text(- json.dumps({"engine": engine, "groups": groups}, indent=2)- )- print(f"wrote report groups -> {args.report_output}")- if args.summary_output:- Path(args.summary_output).write_text(render_summary(groups, engine))- return 0---if __name__ == "__main__":- sys.exit(main())diff --git a/.github/workflows/container_security.yml b/.github/workflows/container_security.ymldeleted file mode 100644index 1d192c59a..000000000--- a/.github/workflows/container_security.yml+++ /dev/null@@ -1,159 +0,0 @@-name: Container image security (Trivy)-# Container base images have NO "tag -> CVE" feed — a container's vulnerabilities live-# in the OS packages/libraries inside it, so the feed is delivered by SCANNING the-# resolved image (Trivy: NVD + GHSA + distro trackers). Renovate stays the sole updater-# (it opens the digest/patch bump PRs); this workflow is the detector + gate:-# * scheduled/dispatch: scan every Dockerfile's base image -> open/refresh/close a-# standing VULNMGMT ticket per project dir (routed to AISIM). No PRs.-# * pull_request (renovate/* container PRs): scan the bumped base image(s) and report-# remaining fixable CVEs as a non-blocking PR comment + job summary.-# See .github/renovate-security/README.md. Never gates merge (report-only).-on:- schedule:- - cron: "43 6 * * 2" # Tuesdays 06:43 UTC (off the top of the hour on purpose)- workflow_dispatch:- inputs:- min_severity:- description: "Lowest severity to report (critical|high|medium|low)"- default: "high"- required: false- dry_run:- description: "Scan + summary only; open/refresh no tickets"- type: boolean- default: false- required: false- pull_request:- types: [opened, synchronize, reopened]- paths:- - "**/Dockerfile"- - "**/*.dockerfile"- - "**/Dockerfile.*"--permissions:- contents: read- pull-requests: write- issues: write--jobs:- scan-all:- # Scheduled full sweep -> standing VULNMGMT tickets, or a manual dispatch- # from main. A workflow_dispatch from an arbitrary branch is skipped so the- # Jira machine-user token can't be exercised from unreviewed branch code.- if: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') }}- runs-on: ubuntu-latest- timeout-minutes: 90- steps:- - name: Checkout repo- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6-- - name: Setup Python- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0- with:- python-version: "3.12"-- - name: Install Trivy- uses: aquasecurity/setup-trivy@81e514348e19b6112ce2a7e3ecbafe19c1e1f567 # v0.3.1- with:- version: v0.72.0 # pinned past the poisoned v0.69.4/.5/.6 supply-chain builds-- - name: Scan base images- id: scan- shell: bash -eo pipefail {0}- env:- IN_MIN_SEV: ${{ github.event.inputs.min_severity }}- run: |- python .github/renovate-security/trivy_scan.py --mode image \- --min-severity "${IN_MIN_SEV:-high}" \- --report-output /tmp/container-groups.json \- --summary-output /tmp/container-summary.md- cat /tmp/container-summary.md >> "$GITHUB_STEP_SUMMARY" || true-- - name: Open / refresh / close VULNMGMT tickets- if: ${{ github.event.inputs.dry_run != 'true' }}- shell: bash -eo pipefail {0}- env:- REPO: ${{ github.repository }}- RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}- SBT_MACHINE_USER_EMAIL: ${{ secrets.SBT_MACHINE_USER_EMAIL }}- SBT_MACHINE_USER_JIRA_TOKEN: ${{ secrets.SBT_MACHINE_USER_JIRA_TOKEN }}- JIRA_BASE_URL: ${{ vars.JIRA_BASE_URL }}- JIRA_VULN_PROJECT: ${{ vars.JIRA_VULN_PROJECT }}- JIRA_VULN_EPIC: ${{ vars.JIRA_VULN_EPIC }}- JIRA_AISIM_ACCOUNT_ID: ${{ vars.JIRA_AISIM_ACCOUNT_ID }}- JIRA_AISIM_USER: ${{ vars.JIRA_AISIM_USER }}- JIRA_AISIM_LABEL: ${{ vars.JIRA_AISIM_LABEL }}- JIRA_AISIM_FIELD_ID: ${{ vars.JIRA_AISIM_FIELD_ID }}- JIRA_AISIM_FIELD_VALUE: ${{ vars.JIRA_AISIM_FIELD_VALUE }}- run: |- if [ -z "${SBT_MACHINE_USER_JIRA_TOKEN:-}" ]; then- echo "::warning::Jira creds not set; skipping ticketing."- exit 0- fi- python .github/renovate-security/renovate_jira.py --report \- --report-groups /tmp/container-groups.json \- --repo "${REPO}" --run-url "${RUN_URL}"-- gate:- # Non-blocking scan of the base image(s) a renovate/* PR bumps to. Gated on- # the automation's identity (the app that opens these PRs), not just the- # branch name — a branch prefix is not an identity, anyone opening a PR can- # name their branch renovate/anything (M4).- if: >-- github.event_name == 'pull_request' &&- github.event.pull_request.user.login == 'autofix-pr-bot[bot]' &&- startsWith(github.event.pull_request.head.ref, 'renovate/')- runs-on: ubuntu-latest- timeout-minutes: 30- steps:- - name: Checkout PR head- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6- with:- ref: ${{ github.event.pull_request.head.sha }}-- - name: Setup Python- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0- with:- python-version: "3.12"-- - name: Install Trivy- uses: aquasecurity/setup-trivy@81e514348e19b6112ce2a7e3ecbafe19c1e1f567 # v0.3.1- with:- version: v0.72.0-- - name: Scan changed images + comment- shell: bash -eo pipefail {0}- env:- GH_TOKEN: ${{ github.token }}- PR_NUMBER: ${{ github.event.pull_request.number }}- run: |- # Directories of the Dockerfiles this PR changed.- dirs=""- while IFS= read -r f; do- b="$(basename "$f")"; lb="${b,,}"- case "$lb" in- dockerfile|*.dockerfile|dockerfile.*) dirs="${dirs}$(dirname "$f")"$'\n';;- esac- done < <(gh pr diff "${PR_NUMBER}" --name-only || true)- dirs="$(printf '%s' "$dirs" | grep -v '^$' | sort -u | paste -sd, - || true)"- if [ -z "$dirs" ]; then- echo "No changed Dockerfiles; nothing to scan."- exit 0- fi- echo "Scanning dirs: $dirs"- python .github/renovate-security/trivy_scan.py --mode image \- --paths "$dirs" --min-severity high \- --report-output /tmp/gate-groups.json \- --summary-output /tmp/gate-summary.md- cat /tmp/gate-summary.md >> "$GITHUB_STEP_SUMMARY" || true- n="$(python -c 'import json,sys;print(sum(g["findings_count"] for g in json.load(open("/tmp/gate-groups.json"))["groups"]))' 2>/dev/null || echo 0)"- if [ "${n:-0}" != "0" ]; then- {- echo "🔎 **Trivy container scan** found ${n} fixable HIGH/CRITICAL CVE(s) still present in the bumped base image(s). This is informational and does **not** block merge.";- echo;- cat /tmp/gate-summary.md;- } > /tmp/gate-comment.md- gh pr comment "${PR_NUMBER}" --body-file /tmp/gate-comment.md \- || echo "::warning::could not post gate comment"- else- echo "No remaining fixable HIGH/CRITICAL CVEs in the bumped image(s)."- fidiff --git a/.github/workflows/renovate_jira.yml b/.github/workflows/renovate_jira.ymldeleted file mode 100644index ce824c680..000000000--- a/.github/workflows/renovate_jira.yml+++ /dev/null@@ -1,118 +0,0 @@-name: Renovate - VULNMGMT tickets-# VULNMGMT/Jira ticketing for the security PRs opened by Renovate and by-# tf_provider_security.yml. On open: search/link or create a ticket, route it to AISIM-# (Business Unit field), link the PR, and stamp the body with a-# `<!-- renovate-jira-close: KEY -->` marker. On merge: transition the created-# ticket(s) to Done. Covers both engines via the shared `renovate/` branch prefix.-#-# App-token PRs (from the two workflows above) DO trigger pull_request events, so this-# runs automatically. Uses only the stdlib Jira client.-on:- pull_request:- types: [opened, reopened, closed]--permissions:- contents: read- pull-requests: write--jobs:- create-ticket:- # Freshly-opened renovate/* PRs (not merges). Gated on the automation's- # identity (the app that opens these PRs), not just the branch name — a- # branch prefix is not an identity check (M4).- if: >-- github.event.action != 'closed' &&- github.event.pull_request.user.login == 'autofix-pr-bot[bot]' &&- startsWith(github.event.pull_request.head.ref, 'renovate/')- runs-on: ubuntu-latest- steps:- - name: Checkout base (for the tool)- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6- with:- ref: ${{ github.event.pull_request.base.ref }}-- - name: Setup Python- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0- with:- python-version: "3.12"-- - name: Create / link VULNMGMT ticket- env:- GH_TOKEN: ${{ github.token }}- PR_NUMBER: ${{ github.event.pull_request.number }}- PR_URL: ${{ github.event.pull_request.html_url }}- PR_TITLE: ${{ github.event.pull_request.title }}- PR_HEAD_REF: ${{ github.event.pull_request.head.ref }}- # Untrusted PR body -> via env, written to a file, never interpolated.- PR_BODY: ${{ github.event.pull_request.body }}- REPO: ${{ github.repository }}- SBT_MACHINE_USER_EMAIL: ${{ secrets.SBT_MACHINE_USER_EMAIL }}- SBT_MACHINE_USER_JIRA_TOKEN: ${{ secrets.SBT_MACHINE_USER_JIRA_TOKEN }}- JIRA_BASE_URL: ${{ vars.JIRA_BASE_URL }}- JIRA_VULN_PROJECT: ${{ vars.JIRA_VULN_PROJECT }}- JIRA_VULN_EPIC: ${{ vars.JIRA_VULN_EPIC }}- JIRA_AISIM_ACCOUNT_ID: ${{ vars.JIRA_AISIM_ACCOUNT_ID }}- JIRA_AISIM_USER: ${{ vars.JIRA_AISIM_USER }}- JIRA_AISIM_LABEL: ${{ vars.JIRA_AISIM_LABEL }}- JIRA_AISIM_FIELD_ID: ${{ vars.JIRA_AISIM_FIELD_ID }}- JIRA_AISIM_FIELD_VALUE: ${{ vars.JIRA_AISIM_FIELD_VALUE }}- run: |- set -euo pipefail- if [ -z "${SBT_MACHINE_USER_JIRA_TOKEN:-}" ]; then- echo "::warning::Jira creds not set; skipping ticket creation for PR #${PR_NUMBER}."- exit 0- fi- printf '%s' "${PR_BODY:-}" > /tmp/pr-body.md- gh pr diff "${PR_NUMBER}" --name-only > /tmp/pr-files.txt || true- python .github/renovate-security/renovate_jira.py --reconcile \- --pr-url "${PR_URL}" --pr-title "${PR_TITLE}" --branch "${PR_HEAD_REF}" \- --body-file /tmp/pr-body.md --body-out /tmp/pr-body.new \- --files /tmp/pr-files.txt --repo "${REPO}"- # Only edit the PR if the tool actually rewrote the body (added a marker).- if [ -f /tmp/pr-body.new ] && ! cmp -s /tmp/pr-body.md /tmp/pr-body.new; then- gh pr edit "${PR_NUMBER}" --body-file /tmp/pr-body.new \- || echo "::warning::could not update PR body for #${PR_NUMBER}"- fi-- close-ticket:- # Merged renovate/* PRs -> close the ticket the automation created. Same- # identity gate as create-ticket (M4).- if: >-- github.event.action == 'closed' &&- github.event.pull_request.merged == true &&- github.event.pull_request.user.login == 'autofix-pr-bot[bot]' &&- startsWith(github.event.pull_request.head.ref, 'renovate/')- runs-on: ubuntu-latest- steps:- - name: Checkout base (for the tool)- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6- with:- ref: ${{ github.event.pull_request.base.ref }}-- - name: Setup Python- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0- with:- python-version: "3.12"-- - name: Close linked VULNMGMT ticket(s)- env:- PR_BODY: ${{ github.event.pull_request.body }}- PR_NUMBER: ${{ github.event.pull_request.number }}- SBT_MACHINE_USER_EMAIL: ${{ secrets.SBT_MACHINE_USER_EMAIL }}- SBT_MACHINE_USER_JIRA_TOKEN: ${{ secrets.SBT_MACHINE_USER_JIRA_TOKEN }}- JIRA_BASE_URL: ${{ vars.JIRA_BASE_URL }}- run: |- set -euo pipefail- keys="$(printf '%s' "${PR_BODY:-}" \- | grep -oE 'renovate-jira-close: [A-Z0-9,-]+' \- | head -1 | sed 's/renovate-jira-close: //' || true)"- if [ -z "${keys:-}" ]; then- echo "No renovate-jira-close marker in PR #${PR_NUMBER}; nothing to close."- exit 0- fi- if [ -z "${SBT_MACHINE_USER_JIRA_TOKEN:-}" ]; then- echo "::warning::Jira creds not set; cannot close ${keys}"- exit 0- fi- echo "Closing Jira ticket(s) for merged PR #${PR_NUMBER}: ${keys}"- python .github/renovate-security/renovate_jira.py --close-jira "${keys}"diff --git a/.github/workflows/socket_cve_fix.yml b/.github/workflows/socket_cve_fix.ymldeleted file mode 100644index f915eb32d..000000000--- a/.github/workflows/socket_cve_fix.yml+++ /dev/null@@ -1,69 +0,0 @@-name: Socket Fix-# Socket CVE autofix for this repo's LANGUAGE ecosystems (npm, py). Socket-first:-# Socket owns language packages; Renovate (renovate.json5) covers only infra-# (Actions/Docker/CircleCI/Terraform) and Trivy covers image/IaC CVEs.-on:- schedule:- - cron: '15 2 1 * *' # Runs at 02:15 on day 1 of the month- - cron: '15 12 * * 0' # Runs at 12:15 on Sunday- workflow_dispatch: #Allow manual run-permissions:- contents: write- pull-requests: write- issues: write-jobs:- socket-fix:- runs-on: ubuntu-latest- # Runs on the monthly/weekly schedule (always on the default branch) or a- # manual dispatch from main. A workflow_dispatch from an arbitrary branch is- # skipped so the Socket key and PR-bot App credentials can't be exercised- # from unreviewed branch code.- if: ${{ github.event_name == 'schedule' || github.ref == 'refs/heads/main' }}-- steps:- - name: Checkout repo- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6- with:- fetch-depth: ${{ github.event_name == 'pull_request' && 2 || 0 }}-- - name: Setup Node- uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0- with:- node-version: '20'-- - name: Install npm socket CLI- run: npm i -g socket-- - name: Install pnpm- run: npm i -g pnpm-- - name: Generate github App Token #Generate short-lived Installation Access Token- id: generate-token- uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1- with:- client-id: ${{ secrets.AUTOFIX_PR_BOT_CLIENT_ID }}- private-key: ${{ secrets.AUTOFIX_PR_BOT_APP_PRIVATE_KEY }}-- - name: Run Socket Fix CLI monthly for major fixes- env:- SOCKET_CLI_GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }}- SOCKET_CLI_GIT_USER_EMAIL: socket-fix[bot]@users.noreply.github.com- SOCKET_CLI_GIT_USER_NAME: socket-fix[bot]- SOCKET_CLI_API_TOKEN: ${{ secrets.SOCKET_KEY }}- SOCKET_SECURITY_API_KEY: ${{ secrets.SOCKET_KEY }}- GH_API_TOKEN: ${{ steps.generate-token.outputs.token }}- CI: true- if: github.event.schedule == '15 2 1 * *' || github.event_name == 'workflow_dispatch'- run: socket fix --minimum-release-age 2w --all-- - name: Run Socket Fix CLI weekly for minor fixes- env:- SOCKET_CLI_GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }}- SOCKET_CLI_GIT_USER_EMAIL: socket-fix[bot]@users.noreply.github.com- SOCKET_CLI_GIT_USER_NAME: socket-fix[bot]- SOCKET_CLI_API_TOKEN: ${{ secrets.SOCKET_KEY }}- SOCKET_SECURITY_API_KEY: ${{ secrets.SOCKET_KEY }}- GH_API_TOKEN: ${{ steps.generate-token.outputs.token }}- CI: true- if: github.event.schedule == '15 12 * * 0'- run: socket fix --minimum-release-age 3d --all --no-major-updates