🚀 The fast, Pythonic way to build MCP servers and clients.
ci: add a twice-daily digest of PRs waiting on the issue-link gate (#5283)
nate nowack561c99f91d13
diff --git a/.github/workflows/gate-digest.yml b/.github/workflows/gate-digest.ymlnew file mode 100644index 000000000..cc761fd8e--- /dev/null+++ b/.github/workflows/gate-digest.yml@@ -0,0 +1,36 @@+name: Gate queue digest+description: Sorts contributor PRs waiting on the issue-link gate into a job summary+on:+ schedule:+ - cron: "0 13,21 * * *" # 1 PM and 9 PM UTC+ workflow_dispatch:+ pull_request:+ paths:+ - ".github/workflows/gate-digest.yml"+ - "scripts/gate_digest.py"++permissions:+ contents: read+ issues: read+ pull-requests: read++jobs:+ digest:+ # Fork PRs can't read the Jev key.+ if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository+ runs-on: ubuntu-latest+ timeout-minutes: 15+ steps:+ - name: Checkout repository+ uses: actions/checkout@v7++ - name: Install uv+ uses: astral-sh/[email protected]++ - name: Write the digest+ # An explicit bash shell runs with pipefail, so a failing script fails the step.+ shell: bash+ run: uv run scripts/gate_digest.py | tee -a "$GITHUB_STEP_SUMMARY"+ env:+ GH_TOKEN: ${{ github.token }}+ TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }}diff --git a/scripts/gate_digest.py b/scripts/gate_digest.pynew file mode 100644index 000000000..5ffa0859f--- /dev/null+++ b/scripts/gate_digest.py@@ -0,0 +1,389 @@+#!/usr/bin/env python+# /// script+# requires-python = ">=3.11"+# dependencies = [+# "vibecheck-py==0.1.1",+# ]+#+# [tool.uv]+# exclude-newer-package = { vibecheck-py = false, typesafe-sdk = false }+# ///+"""+Digest of contributor PRs waiting on the issue-link gate.++Code gathers the hard facts, Jev (through vibecheck) answers four yes/no+questions about each PR, and explicit policy sorts every PR into "worth+assigning", "needs a human", or "decline". Prints a markdown report; it never+comments, assigns, or labels.++Requires `gh` authenticated with read access, and TYPESAFE_API_KEY.+"""++import asyncio+import json+import re+import subprocess+import time+from datetime import date, timedelta++import vibecheck++REPO = "PrefectHQ/fastmcp"+# The issue-link gate's parser (.github/workflows/require-issue-link.yml): an+# auto-close keyword before `#N`, `PrefectHQ/fastmcp#N`, or the issue URL.+LINK = re.compile(+ r"(?:close[sd]?|fix(?:e[sd])?|resolve[sd]?)\s*:?\s*"+ r"(?:PrefectHQ/fastmcp#|#|https?://github\.com/PrefectHQ/fastmcp/issues/)(\d+)",+ re.IGNORECASE,+)+MAX_LINKS = 5+# The repository's hard-stop markers, checked in the title, the labels, and at+# the start of body lines (so a mention of another PR's `[DNM]` doesn't count).+# "Draft" counts only as `[DRAFT]` or a leading `DRAFT:`, so a title like+# "Draft-4 bounds" isn't held.+DNM = re.compile(+ r"\s*(?:\[?DNM\]?\b|\[?do[\s_-]*not[\s_-]*merge|\[?don'?t[\s_-]*merge|\[draft\]|draft\s*:)",+ re.IGNORECASE,+)+# Checks about the gate or the labeling bot, not about the PR's code.+NON_CODE_CHECKS = {+ "check-issue-link",+ "require-issue-link",+ "reopen-on-assignment",+ "label-issue-or-pr",+}+HIGH_VOLUME = 90+# Act only on clear answers; everything between goes to a person.+DECLINE_BELOW = 0.2+CLEAR_YES = 0.7+CAUSE_BELOW = 0.6+NOISY_AT = 0.6++QUESTIONS = {+ "issue_is_bug": (+ "Does `issue` describe FastMCP behaving incorrectly against behavior it "+ "documents or clearly intends, shown by a reproducible example? Answer "+ "no for questions, feature requests, design proposals, or a user's own "+ "configuration mistake."+ ),+ "scoped_bugfix": (+ "Is `pr` a focused fix for the bug in `issue`, changing only what that "+ "bug needs? Answer no if it adds a feature or option, integrates a "+ "third-party service, refactors unrelated code, or bundles other changes."+ ),+ "fixes_at_cause": (+ "Judging from `pr.description` and `pr.diff`, does the change modify the "+ "code where the problem in `issue` originates? Answer no if it adds a "+ "workaround, special case, or compensation somewhere else and leaves the "+ "originating code path unchanged."+ ),+ "generated_noise": (+ "Does `pr.description` read like unedited generated text: long sections "+ "restating the diff, speculative analysis, lists of alternative "+ "approaches, or test summaries, instead of a short statement of the "+ "problem and the fix?"+ ),+}+++def gh(*args: str) -> str:+ return subprocess.run(+ ["gh", *args], capture_output=True, text=True, check=True+ ).stdout+++def gh_json(*args: str):+ return json.loads(gh(*args))+++def search_count(query: str) -> int:+ # The search API allows 30 requests a minute.+ time.sleep(2.1)+ return gh_json(+ "api", "-X", "GET", "search/issues", "-f", f"q={query}", "-f", "per_page=1"+ )["total_count"]+++def linked_issues(body: str) -> list[str]:+ found: list[str] = []+ for n in LINK.findall(body or ""):+ if n not in found:+ found.append(n)+ return found[:MAX_LINKS]+++def other_prs(issue: str, pr: int, author: str) -> dict[str, list[int]]:+ """Other contributions that link `issue`, whether or not the gate holds them.++ Open PRs and PRs the gate closed still compete: assignment reopens a+ gate-closed PR. A merged one may already fix the issue. The author's own+ earlier attempts, and PRs a maintainer closed on the merits, are left out.+ """+ hits = gh_json(+ "pr",+ "list",+ "--repo",+ REPO,+ "--state",+ "all",+ "--limit",+ "20",+ "--search",+ f"{issue} in:body",+ "--json",+ "number,body,state,author,labels",+ )+ found: dict[str, list[int]] = {"competing": [], "merged": []}+ for h in hits:+ if h["number"] == pr or h["author"]["login"] == author:+ continue+ if issue not in linked_issues(h["body"]):+ continue+ gated = "missing-issue-link" in {lb["name"] for lb in h["labels"]}+ if h["state"] == "MERGED":+ found["merged"].append(h["number"])+ elif h["state"] == "OPEN" or gated:+ found["competing"].append(h["number"])+ return {k: sorted(v) for k, v in found.items()}+++def fetch_issue(n: str, pr: int, author: str) -> dict:+ iss = gh_json(+ "issue",+ "view",+ n,+ "--repo",+ REPO,+ "--json",+ "title,body,author,state,assignees,labels,comments",+ )+ return {+ "number": int(n),+ "title": iss["title"],+ "body": (iss["body"] or "")[:3000],+ "labels": [lb["name"] for lb in iss["labels"]],+ "comments": [+ {"author": c["author"]["login"], "body": c["body"][:600]}+ for c in iss["comments"][-5:]+ ],+ "reporter": iss["author"]["login"],+ "state": iss["state"],+ "assignees": [a["login"] for a in iss["assignees"]],+ **other_prs(n, pr, author),+ }+++def pick_issue(issues: list[dict], author: str) -> dict | None:+ """The linked issue this PR can be assigned on, as the gate would take it."""++ def eligible(i: dict) -> bool:+ return i["state"] == "OPEN" and (not i["assignees"] or author in i["assignees"])++ candidates = [i for i in issues if eligible(i)]+ if not candidates:+ return None+ return max(candidates, key=lambda i: (i["reporter"] == author, not i["competing"]))+++def gather() -> list[dict]:+ prs = gh_json(+ "pr",+ "list",+ "--repo",+ REPO,+ "--state",+ "open",+ "--label",+ "missing-issue-link",+ "--limit",+ "100",+ "--json",+ "number,title,author,createdAt,body,additions,deletions,"+ "statusCheckRollup,files,isDraft,labels",+ )+ since = (date.today() - timedelta(days=30)).isoformat()+ authors: dict[str, dict] = {}+ rows = []+ for p in prs:+ author = p["author"]["login"]+ if author not in authors:+ authors[author] = {+ "merged_here": search_count(+ f"repo:{REPO} type:pr is:merged author:{author}"+ ),+ "prs_30d": search_count(f"type:pr author:{author} created:>={since}"),+ }+ failing = sorted(+ {+ c.get("name") or c.get("context") or "?"+ for c in p["statusCheckRollup"]+ if (c.get("conclusion") or c.get("state")) in {"FAILURE", "ERROR"}+ }+ - NON_CODE_CHECKS+ )+ labels = [lb["name"] for lb in p["labels"]]+ issues = [fetch_issue(n, p["number"], author) for n in linked_issues(p["body"])]+ rows.append(+ {+ "pr": p["number"],+ "title": p["title"],+ "author": author,+ "opened": p["createdAt"][:10],+ "size": p["additions"] + p["deletions"],+ "files": [f["path"] for f in p["files"]],+ "failing_checks": failing,+ "hold": p["isDraft"]+ or any(DNM.search(t) for t in [p["title"], *labels])+ or any(DNM.match(line) for line in (p["body"] or "").splitlines()),+ "linked": issues,+ "issue": pick_issue(issues, author),+ "description": (p["body"] or "")[:3000],+ "diff": gh("pr", "diff", str(p["number"]), "--repo", REPO)[:6000],+ **authors[author],+ }+ )+ return rows+++async def judge(rows: list[dict]) -> None:+ sem = asyncio.Semaphore(4)++ async def one(r: dict) -> None:+ if r["issue"] is None:+ return+ data = {+ "issue": {+ k: r["issue"][k] for k in ("title", "body", "labels", "comments")+ },+ "pr": {+ "title": r["title"],+ "description": r["description"],+ "files": r["files"],+ "diff": r["diff"],+ },+ }+ async with sem, vibecheck.batch(data) as b:+ pending = {k: b.check(q, probabilities=True) for k, q in QUESTIONS.items()}+ r["jev"] = {k: round(d.result().probabilities, 2) for k, d in pending.items()}++ await asyncio.gather(*(one(r) for r in rows))+++def decide(r: dict) -> tuple[str, list[str]]:+ # Decline only on facts; a low score goes to a person, since a maintainer+ # may have approved an enhancement or clarified the contract.+ if not r["linked"]:+ return "decline", ["no linked issue"]+ iss = r["issue"]+ if iss is None:+ closed = [f"#{i['number']}" for i in r["linked"] if i["state"] != "OPEN"]+ taken = [+ f"#{i['number']} (assigned to {', '.join(i['assignees'])})"+ for i in r["linked"]+ if i["state"] == "OPEN"+ ]+ return "decline", [+ "every linked issue is closed or assigned elsewhere: "+ + ", ".join(closed + taken)+ ]+ if r["hold"]:+ return "needs a human", ["draft or marked do-not-merge"]++ j = r.get("jev", {})+ reporter = iss["reporter"] == r["author"]+ if iss["competing"] and not reporter:+ rivals = ", ".join(f"#{x}" for x in iss["competing"])+ return "needs a human", [+ f"#{iss['number']} has other contributions ({rivals}); pick one"+ ]++ reasons = []+ if iss["merged"]:+ merged = ", ".join(f"#{x}" for x in iss["merged"])+ reasons.append(f"{merged} already merged for #{iss['number']}; may be fixed")+ if iss["competing"]:+ reasons.append(+ "reporter's PR; others on the issue: "+ + ", ".join(f"#{x}" for x in iss["competing"])+ )+ if r["failing_checks"]:+ reasons.append("failing: " + ", ".join(r["failing_checks"]))+ if j["issue_is_bug"] < DECLINE_BELOW:+ reasons.append(+ f"issue reads as not a bug ({j['issue_is_bug']}); check for an approved design"+ )+ elif j["issue_is_bug"] < CLEAR_YES:+ reasons.append(f"bug unclear ({j['issue_is_bug']})")+ if j["scoped_bugfix"] < DECLINE_BELOW:+ reasons.append(+ f"reads as an enhancement ({j['scoped_bugfix']}); needs an approved design"+ )+ elif j["scoped_bugfix"] < CLEAR_YES:+ reasons.append(f"scope unclear ({j['scoped_bugfix']})")+ if j["fixes_at_cause"] < CAUSE_BELOW:+ reasons.append(f"may patch a symptom ({j['fixes_at_cause']})")+ if j["generated_noise"] >= NOISY_AT:+ reasons.append(f"description reads generated ({j['generated_noise']})")+ if not any(f.startswith("tests/") for f in r["files"]):+ reasons.append("adds no test")+ if r["prs_30d"] >= HIGH_VOLUME:+ reasons.append(+ f"author opened {r['prs_30d']} PRs in 30 days; check follow-through"+ )+ if r["size"] > 400:+ reasons.append(f"large ({r['size']} lines)")+ if reasons:+ return "needs a human", reasons++ why = ["reporter" if reporter else "only PR on the issue", "checks green"]+ if r["merged_here"]:+ why.append(f"{r['merged_here']} merged here before")+ return "worth assigning", why+++def report(rows: list[dict]) -> str:+ order = ["worth assigning", "needs a human", "decline"]+ lines = [+ "## Gate queue digest",+ "",+ f"{len(rows)} contributor PRs are waiting on assignment. Nothing was assigned or posted.",+ "",+ ]+ for verdict in order:+ group = sorted(+ (r for r in rows if r["verdict"] == verdict), key=lambda r: r["opened"]+ )+ lines += [+ f"### {verdict} ({len(group)})",+ "",+ "| PR | issue | author | why | bug · scoped · cause · noise |",+ "|---|---|---|---|---|",+ ]+ for r in group:+ j = r.get("jev", {})+ scores = " · ".join(f"{j[k]:.2f}" for k in QUESTIONS) if j else ""+ issue = f"#{r['issue']['number']}" if r["issue"] else ""+ title = r["title"].replace("|", "\\|")+ lines.append(+ f"| #{r['pr']} {title} | {issue} | @{r['author']} | {'; '.join(r['reasons'])} | {scores} |"+ )+ lines.append("")+ lines.append(+ "Scores are Jev's probability of yes to: the issue shows a bug; the PR is a "+ "scoped fix; it changes the code where the problem starts; the description "+ 'reads like unedited generated text. "Worth assigning" means worth a '+ "maintainer's review, not ready to merge."+ )+ return "\n".join(lines)+++def main() -> None:+ rows = gather()+ asyncio.run(judge(rows))+ for r in rows:+ r["verdict"], r["reasons"] = decide(r)+ print(report(rows))+++if __name__ == "__main__":+ main()