Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
ci(migrations): fix the union, the PR cap, the cancelled run and the aborted sweep (CHOO-2689)
Petr Bauch9ddfbc4dfe3d
diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.ymlindex 9f7d573ea..2abf775cc 100644--- a/.github/workflows/pr-ci.yml+++ b/.github/workflows/pr-ci.yml@@ -44,8 +44,16 @@ on:branches: [main]# Cancel a PR's in-flight CI when a new commit is pushed to the same ref.+#+# Push runs get the commit in their group, so they never cancel each other.+# Every merge to main shares one `github.ref`, so a single group would mean the+# next merge kills the previous merge's run — and `migration-heads-on-merge`+# only runs when the *triggering* commit touched migrations, so the run that+# replaces it skips the job entirely. Merge a migration and anything else a+# minute apart and no status is posted anywhere, silently, on exactly the busy+# day this is meant to cover.concurrency:- group: pr-ci-${{ github.ref }}+ group: pr-ci-${{ github.ref }}-${{ github.event_name == 'push' && github.sha || '' }}cancel-in-progress: truepermissions:diff --git a/scripts/check_migration_heads_for_open_prs.py b/scripts/check_migration_heads_for_open_prs.pyindex 0833ea829..802103c94 100644--- a/scripts/check_migration_heads_for_open_prs.py+++ b/scripts/check_migration_heads_for_open_prs.py@@ -36,14 +36,19 @@ anyone can open one, and the job runs on a push to main holding a token thatcan write commit statuses. Importing an unreviewed file to compute a graphwould hand that token's holder's code execution to whoever opened the PR.-Instead this reads `revision`/`down_revision` straight out of the file text-with a regex and `ast.literal_eval` — never `eval`, so the value must be a-literal (a string, `None`, or a tuple of strings) or parsing raises instead of-running anything. `test_revision_ids_are_unique` already reads `revision` the-same way, for the same reason, on a smaller scale; this applies it to the+Instead this reads `revision`/`down_revision` out of the file's syntax tree:+`ast.parse` builds the tree without running a line of it, and the two+right-hand sides go through `ast.literal_eval` — never `eval`, so each must be+a literal (a string, `None`, or a tuple of strings) or parsing raises instead+of running anything. `test_revision_ids_are_unique` reads `revision` in the+same spirit, for the same reason, on a smaller scale; this applies it to thewhole graph. Nothing here is executed, so nothing here needs a database, aPython environment for the project, or the `alembic` library itself.+The tree rather than a regex because an assignment is not a line: a merge+revision's `down_revision` tuple may be spread over several, and a pattern+anchored to end-of-line hands `ast.literal_eval` half an expression.+It shells out to `gh` (for PR discovery and posting statuses) and `git` (toread a PR's files without checking them out or running them), both alreadypresent on GitHub-hosted runners.@@ -64,6 +69,7 @@ REPO_ROOT = Path(__file__).resolve().parent.parentVERSIONS_DIR = REPO_ROOT / "core" / "switch_core" / "migrations" / "versions"VERSIONS_PATH = "core/switch_core/migrations/versions"STATUS_CONTEXT = "migration-heads-on-merge"+PR_LIST_LIMIT = 1000# The repository is never named in an argument list: `gh` takes it from the# checkout, and its `{owner}/{repo}` placeholder fills it in for an API path.@@ -77,14 +83,16 @@ SAFE_VERSION_PATH_RE = re.compile()SAFE_SHA_RE = re.compile(r"\A[0-9a-f]{7,40}\Z")-# Matches the two assignments Alembic's revision template generates, with or-# without the type annotation it has carried across template versions-# (`revision = ...`, `revision: str = ...`, `down_revision: str | None = ...`,-# `down_revision: str | Sequence[str] | None = ...`). Only the right-hand side-# is captured and handed to `ast.literal_eval`.-_ASSIGNMENT_RE = re.compile(- r"^(revision|down_revision)\s*(?::[^=\n]+)?=\s*(.+)$", re.MULTILINE-)+_WANTED_ASSIGNMENTS = ("revision", "down_revision")+++class RevisionProblem(Exception):+ """A file under `versions/` that cannot be read as a migration.++ Raised rather than allowed to surface as a `KeyError` or a `SyntaxError`+ so the sweep can report *which file* is unreadable, on the PR that+ introduced it, instead of dying with a traceback halfway through.+ """class RevisionFile:@@ -96,31 +104,71 @@ class RevisionFile:self.parents = parents-def parse_revision_file(filename: str, text: str) -> RevisionFile:- """Extract `revision`/`down_revision` from a migration file's text.+def _assigned_literals(filename: str, text: str) -> dict[str, object]:+ """The module-level `revision` / `down_revision` literals, from the tree.++ `ast.parse` builds a syntax tree and runs nothing, and `ast.literal_eval`+ on a node only ever produces a literal (or raises) -- it cannot call a+ function, access an attribute, or execute a statement. Both are safe to+ point at a file nobody has reviewed.- `ast.literal_eval` only ever produces a literal (or raises) -- it cannot- call a function, access an attribute, or run a statement, so this is safe- to point at a file nobody has reviewed.+ Reading assignments off the tree rather than out of the text is also what+ makes a multi-line `down_revision` tuple -- the shape a merge revision+ grows into once it has more than a couple of parents -- parse like any+ other."""+ try:+ tree = ast.parse(text, filename=filename)+ except SyntaxError as exc:+ raise RevisionProblem(f"{filename}: is not valid Python: {exc}") from exc+values: dict[str, object] = {}- for match in _ASSIGNMENT_RE.finditer(text):- name, rhs = match.group(1), match.group(2).strip()- values[name] = ast.literal_eval(rhs)+ for node in tree.body:+ targets: list[str] = []+ value: ast.expr | None = None+ if isinstance(node, ast.Assign):+ targets = [t.id for t in node.targets if isinstance(t, ast.Name)]+ value = node.value+ elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name):+ targets = [node.target.id]+ value = node.value # None for a bare `x: str` annotation+ if value is None:+ continue+ for name in targets:+ if name not in _WANTED_ASSIGNMENTS:+ continue+ try:+ values[name] = ast.literal_eval(value)+ except ValueError as exc:+ raise RevisionProblem(+ f"{filename}: {name} is not a literal: {exc}"+ ) from exc+ return values+++def parse_revision_file(filename: str, text: str) -> RevisionFile:+ """Extract `revision`/`down_revision` from a migration file's text."""+ values = _assigned_literals(filename, text)+ if "revision" not in values:+ raise RevisionProblem(+ f"{filename}: no module-level `revision` assignment -- "+ "every file under versions/ must be a migration"+ )revision = values["revision"]- assert isinstance(revision, str), (- f"{filename}: revision is not a string literal: {revision!r}"- )+ if not isinstance(revision, str):+ raise RevisionProblem(+ f"{filename}: revision is not a string literal: {revision!r}"+ )down = values.get("down_revision")if down is None:parents: tuple[str, ...] = ()elif isinstance(down, str):parents = (down,)- elif isinstance(down, tuple):+ elif isinstance(down, tuple) and all(isinstance(p, str) for p in down):parents = down # a merge revision's tuple of parentselse:- raise TypeError(- f"{filename}: down_revision is neither a string, tuple, nor None: {down!r}"+ raise RevisionProblem(+ f"{filename}: down_revision is neither a string, a tuple of strings, nor None: {down!r}")return RevisionFile(filename, revision, parents)@@ -161,6 +209,11 @@ def open_prs_touching_migrations() -> list[dict[str, Any]]:No `--repo`: `gh` resolves it from the checkout this runs in, which keepsthe repository out of the argument list entirely.++ `--limit` is not optional. Without it `gh` stops at 30 open PRs and says+ nothing about the rest, so on a repository with more than that the PRs+ least likely to have been rechecked recently are exactly the ones silently+ skipped."""listing = run("gh",@@ -168,11 +221,19 @@ def open_prs_touching_migrations() -> list[dict[str, Any]]:"list","--state","open",+ "--limit",+ str(PR_LIST_LIMIT),"--json","number,headRefOid",)+ open_prs = json.loads(listing.stdout)+ if len(open_prs) >= PR_LIST_LIMIT:+ raise RuntimeError(+ f"{len(open_prs)} open PRs reached the --limit of {PR_LIST_LIMIT}; "+ "some were not listed and would be skipped without a word"+ )candidates = []- for pr in json.loads(listing.stdout):+ for pr in open_prs:diff = run("gh", "pr", "diff", str(pr["number"]), "--name-only")if any(line.startswith(VERSIONS_PATH) for line in diff.stdout.splitlines()):candidates.append(pr)@@ -182,12 +243,10 @@ def open_prs_touching_migrations() -> list[dict[str, Any]]:def fetch_pr_revisions(pr_number: int) -> list[RevisionFile]:"""The revision files as they stand on the PR's own head, read as text.- Fetched straight from the PR's head ref rather than a merge -- the- versions directory only ever gains files, so a plain union with the- base's current revisions is the merge result for the purpose these checks- care about, and it skips every non-migration conflict a real merge could- raise. `git show` only ever prints a blob's content; nothing here writes- the file to disk or runs it.+ Fetched straight from the PR's head ref rather than a merge; `merge_preview`+ combines it with the base. This is the whole directory as the PR has it,+ shared files included, not just the ones it adds. `git show` only ever+ prints a blob's content; nothing here writes the file to disk or runs it."""run("git", "fetch", "--depth=1", "origin", f"refs/pull/{pr_number}/head")listing = run(@@ -258,12 +317,31 @@ def post_status(sha: str, state: str, description: str, run_url: str) -> None:)+def merge_preview(+ base_revisions: list[RevisionFile], pr_revisions: list[RevisionFile]+) -> list[RevisionFile]:+ """The versions directory as it would stand with this PR merged.++ A union keyed by filename, not a concatenation: the two sides share every+ revision that was already on main when the PR branched, and counting those+ twice reports every PR as a duplicate-id collision. The PR's copy wins,+ which is also what a merge does for a file it modifies.++ The directory only ever gains files, so this is the merge result for the+ purpose these checks care about -- and it needs no real merge, so no+ unrelated conflict can get in the way.+ """+ merged = {rev.filename: rev for rev in base_revisions}+ merged.update({rev.filename: rev for rev in pr_revisions})+ return list(merged.values())++def check_pr(pr: dict[str, Any], base_revisions: list[RevisionFile], run_url: str) -> bool:number = pr["number"]sha = pr["headRefOid"]- combined = base_revisions + fetch_pr_revisions(int(number))+ combined = merge_preview(base_revisions, fetch_pr_revisions(int(number)))problem = describe_chain_problem(combined)passed = not problemif passed:@@ -277,14 +355,60 @@ def check_pr(return passed+def check_pr_isolated(+ pr: dict[str, Any], base_revisions: list[RevisionFile], run_url: str+) -> bool:+ """`check_pr`, with one PR's failure kept to that PR.++ Every `gh` and `git` call here can fail on its own -- a transient API+ error, a head ref that no longer resolves -- and an unreadable file under+ `versions/` raises too. Letting any of those out would abandon the PRs not+ yet reached, with nothing on them to say so and a red job on main that+ looks exactly like a genuine finding. So each PR gets an `error` status+ naming what went wrong, and the sweep carries on.+ """+ number = pr["number"]+ try:+ return check_pr(pr, base_revisions, run_url)+ except (RevisionProblem, subprocess.CalledProcessError, OSError, ValueError) as exc:+ detail = (+ exc.stderr.strip()+ if isinstance(exc, subprocess.CalledProcessError) and exc.stderr+ else str(exc)+ )+ print(f"::error::PR #{number} could not be checked: {detail}")+ sha = str(pr.get("headRefOid", ""))+ if SAFE_SHA_RE.match(sha):+ try:+ post_status(+ sha,+ "error",+ "Could not check the migration chain for this PR -- see the run log",+ run_url,+ )+ except (subprocess.CalledProcessError, OSError) as post_exc:+ # Whatever broke the check may well be what breaks saying so.+ print(+ f"::error::PR #{number}: could not post a status either: {post_exc}"+ )+ return False++def main() -> int:run_url = os.environ["RUN_URL"]prs = open_prs_touching_migrations()if not prs:print("No open PRs touch the migrations directory.")return 0- base_revisions = read_base_revisions()- results = [check_pr(pr, base_revisions, run_url) for pr in prs]+ try:+ base_revisions = read_base_revisions()+ except RevisionProblem as exc:+ # Nothing can be said about any PR while main itself is unreadable,+ # and it is main that needs fixing -- do not paint every open PR red+ # for it.+ print(f"::error::main's own migrations cannot be read: {exc}")+ return 1+ results = [check_pr_isolated(pr, base_revisions, run_url) for pr in prs]return 0 if all(results) else 1