Skip to content

🚀 The fast, Pythonic way to build MCP servers and clients.

ci: add a twice-daily digest of PRs waiting on the issue-link gate (#5283)

nate nowack561c99f91d13
diff --git a/.github/workflows/gate-digest.yml b/.github/workflows/gate-digest.yml
new file mode 100644
index 000000000..cc761fd8e
--- /dev/null
+++ b/.github/workflows/gate-digest.yml
@@ -0,0 +1,36 @@
+name: Gate queue digest
+description: Sorts contributor PRs waiting on the issue-link gate into a job summary
+on:
+ schedule:
+ - cron: "0 13,21 * * *" # 1 PM and 9 PM UTC
+ workflow_dispatch:
+ pull_request:
+ paths:
+ - ".github/workflows/gate-digest.yml"
+ - "scripts/gate_digest.py"
+
+permissions:
+ contents: read
+ issues: read
+ pull-requests: read
+
+jobs:
+ digest:
+ # Fork PRs can't read the Jev key.
+ if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
+ runs-on: ubuntu-latest
+ timeout-minutes: 15
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v7
+
+ - name: Install uv
+ uses: astral-sh/[email protected]
+
+ - name: Write the digest
+ # An explicit bash shell runs with pipefail, so a failing script fails the step.
+ shell: bash
+ run: uv run scripts/gate_digest.py | tee -a "$GITHUB_STEP_SUMMARY"
+ env:
+ GH_TOKEN: ${{ github.token }}
+ TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }}
diff --git a/scripts/gate_digest.py b/scripts/gate_digest.py
new file mode 100644
index 000000000..5ffa0859f
--- /dev/null
+++ b/scripts/gate_digest.py
@@ -0,0 +1,389 @@
+#!/usr/bin/env python
+# /// script
+# requires-python = ">=3.11"
+# dependencies = [
+# "vibecheck-py==0.1.1",
+# ]
+#
+# [tool.uv]
+# exclude-newer-package = { vibecheck-py = false, typesafe-sdk = false }
+# ///
+"""
+Digest of contributor PRs waiting on the issue-link gate.
+
+Code gathers the hard facts, Jev (through vibecheck) answers four yes/no
+questions about each PR, and explicit policy sorts every PR into "worth
+assigning", "needs a human", or "decline". Prints a markdown report; it never
+comments, assigns, or labels.
+
+Requires `gh` authenticated with read access, and TYPESAFE_API_KEY.
+"""
+
+import asyncio
+import json
+import re
+import subprocess
+import time
+from datetime import date, timedelta
+
+import vibecheck
+
+REPO = "PrefectHQ/fastmcp"
+# The issue-link gate's parser (.github/workflows/require-issue-link.yml): an
+# auto-close keyword before `#N`, `PrefectHQ/fastmcp#N`, or the issue URL.
+LINK = re.compile(
+ r"(?:close[sd]?|fix(?:e[sd])?|resolve[sd]?)\s*:?\s*"
+ r"(?:PrefectHQ/fastmcp#|#|https?://github\.com/PrefectHQ/fastmcp/issues/)(\d+)",
+ re.IGNORECASE,
+)
+MAX_LINKS = 5
+# The repository's hard-stop markers, checked in the title, the labels, and at
+# the start of body lines (so a mention of another PR's `[DNM]` doesn't count).
+# "Draft" counts only as `[DRAFT]` or a leading `DRAFT:`, so a title like
+# "Draft-4 bounds" isn't held.
+DNM = re.compile(
+ r"\s*(?:\[?DNM\]?\b|\[?do[\s_-]*not[\s_-]*merge|\[?don'?t[\s_-]*merge|\[draft\]|draft\s*:)",
+ re.IGNORECASE,
+)
+# Checks about the gate or the labeling bot, not about the PR's code.
+NON_CODE_CHECKS = {
+ "check-issue-link",
+ "require-issue-link",
+ "reopen-on-assignment",
+ "label-issue-or-pr",
+}
+HIGH_VOLUME = 90
+# Act only on clear answers; everything between goes to a person.
+DECLINE_BELOW = 0.2
+CLEAR_YES = 0.7
+CAUSE_BELOW = 0.6
+NOISY_AT = 0.6
+
+QUESTIONS = {
+ "issue_is_bug": (
+ "Does `issue` describe FastMCP behaving incorrectly against behavior it "
+ "documents or clearly intends, shown by a reproducible example? Answer "
+ "no for questions, feature requests, design proposals, or a user's own "
+ "configuration mistake."
+ ),
+ "scoped_bugfix": (
+ "Is `pr` a focused fix for the bug in `issue`, changing only what that "
+ "bug needs? Answer no if it adds a feature or option, integrates a "
+ "third-party service, refactors unrelated code, or bundles other changes."
+ ),
+ "fixes_at_cause": (
+ "Judging from `pr.description` and `pr.diff`, does the change modify the "
+ "code where the problem in `issue` originates? Answer no if it adds a "
+ "workaround, special case, or compensation somewhere else and leaves the "
+ "originating code path unchanged."
+ ),
+ "generated_noise": (
+ "Does `pr.description` read like unedited generated text: long sections "
+ "restating the diff, speculative analysis, lists of alternative "
+ "approaches, or test summaries, instead of a short statement of the "
+ "problem and the fix?"
+ ),
+}
+
+
+def gh(*args: str) -> str:
+ return subprocess.run(
+ ["gh", *args], capture_output=True, text=True, check=True
+ ).stdout
+
+
+def gh_json(*args: str):
+ return json.loads(gh(*args))
+
+
+def search_count(query: str) -> int:
+ # The search API allows 30 requests a minute.
+ time.sleep(2.1)
+ return gh_json(
+ "api", "-X", "GET", "search/issues", "-f", f"q={query}", "-f", "per_page=1"
+ )["total_count"]
+
+
+def linked_issues(body: str) -> list[str]:
+ found: list[str] = []
+ for n in LINK.findall(body or ""):
+ if n not in found:
+ found.append(n)
+ return found[:MAX_LINKS]
+
+
+def other_prs(issue: str, pr: int, author: str) -> dict[str, list[int]]:
+ """Other contributions that link `issue`, whether or not the gate holds them.
+
+ Open PRs and PRs the gate closed still compete: assignment reopens a
+ gate-closed PR. A merged one may already fix the issue. The author's own
+ earlier attempts, and PRs a maintainer closed on the merits, are left out.
+ """
+ hits = gh_json(
+ "pr",
+ "list",
+ "--repo",
+ REPO,
+ "--state",
+ "all",
+ "--limit",
+ "20",
+ "--search",
+ f"{issue} in:body",
+ "--json",
+ "number,body,state,author,labels",
+ )
+ found: dict[str, list[int]] = {"competing": [], "merged": []}
+ for h in hits:
+ if h["number"] == pr or h["author"]["login"] == author:
+ continue
+ if issue not in linked_issues(h["body"]):
+ continue
+ gated = "missing-issue-link" in {lb["name"] for lb in h["labels"]}
+ if h["state"] == "MERGED":
+ found["merged"].append(h["number"])
+ elif h["state"] == "OPEN" or gated:
+ found["competing"].append(h["number"])
+ return {k: sorted(v) for k, v in found.items()}
+
+
+def fetch_issue(n: str, pr: int, author: str) -> dict:
+ iss = gh_json(
+ "issue",
+ "view",
+ n,
+ "--repo",
+ REPO,
+ "--json",
+ "title,body,author,state,assignees,labels,comments",
+ )
+ return {
+ "number": int(n),
+ "title": iss["title"],
+ "body": (iss["body"] or "")[:3000],
+ "labels": [lb["name"] for lb in iss["labels"]],
+ "comments": [
+ {"author": c["author"]["login"], "body": c["body"][:600]}
+ for c in iss["comments"][-5:]
+ ],
+ "reporter": iss["author"]["login"],
+ "state": iss["state"],
+ "assignees": [a["login"] for a in iss["assignees"]],
+ **other_prs(n, pr, author),
+ }
+
+
+def pick_issue(issues: list[dict], author: str) -> dict | None:
+ """The linked issue this PR can be assigned on, as the gate would take it."""
+
+ def eligible(i: dict) -> bool:
+ return i["state"] == "OPEN" and (not i["assignees"] or author in i["assignees"])
+
+ candidates = [i for i in issues if eligible(i)]
+ if not candidates:
+ return None
+ return max(candidates, key=lambda i: (i["reporter"] == author, not i["competing"]))
+
+
+def gather() -> list[dict]:
+ prs = gh_json(
+ "pr",
+ "list",
+ "--repo",
+ REPO,
+ "--state",
+ "open",
+ "--label",
+ "missing-issue-link",
+ "--limit",
+ "100",
+ "--json",
+ "number,title,author,createdAt,body,additions,deletions,"
+ "statusCheckRollup,files,isDraft,labels",
+ )
+ since = (date.today() - timedelta(days=30)).isoformat()
+ authors: dict[str, dict] = {}
+ rows = []
+ for p in prs:
+ author = p["author"]["login"]
+ if author not in authors:
+ authors[author] = {
+ "merged_here": search_count(
+ f"repo:{REPO} type:pr is:merged author:{author}"
+ ),
+ "prs_30d": search_count(f"type:pr author:{author} created:>={since}"),
+ }
+ failing = sorted(
+ {
+ c.get("name") or c.get("context") or "?"
+ for c in p["statusCheckRollup"]
+ if (c.get("conclusion") or c.get("state")) in {"FAILURE", "ERROR"}
+ }
+ - NON_CODE_CHECKS
+ )
+ labels = [lb["name"] for lb in p["labels"]]
+ issues = [fetch_issue(n, p["number"], author) for n in linked_issues(p["body"])]
+ rows.append(
+ {
+ "pr": p["number"],
+ "title": p["title"],
+ "author": author,
+ "opened": p["createdAt"][:10],
+ "size": p["additions"] + p["deletions"],
+ "files": [f["path"] for f in p["files"]],
+ "failing_checks": failing,
+ "hold": p["isDraft"]
+ or any(DNM.search(t) for t in [p["title"], *labels])
+ or any(DNM.match(line) for line in (p["body"] or "").splitlines()),
+ "linked": issues,
+ "issue": pick_issue(issues, author),
+ "description": (p["body"] or "")[:3000],
+ "diff": gh("pr", "diff", str(p["number"]), "--repo", REPO)[:6000],
+ **authors[author],
+ }
+ )
+ return rows
+
+
+async def judge(rows: list[dict]) -> None:
+ sem = asyncio.Semaphore(4)
+
+ async def one(r: dict) -> None:
+ if r["issue"] is None:
+ return
+ data = {
+ "issue": {
+ k: r["issue"][k] for k in ("title", "body", "labels", "comments")
+ },
+ "pr": {
+ "title": r["title"],
+ "description": r["description"],
+ "files": r["files"],
+ "diff": r["diff"],
+ },
+ }
+ async with sem, vibecheck.batch(data) as b:
+ pending = {k: b.check(q, probabilities=True) for k, q in QUESTIONS.items()}
+ r["jev"] = {k: round(d.result().probabilities, 2) for k, d in pending.items()}
+
+ await asyncio.gather(*(one(r) for r in rows))
+
+
+def decide(r: dict) -> tuple[str, list[str]]:
+ # Decline only on facts; a low score goes to a person, since a maintainer
+ # may have approved an enhancement or clarified the contract.
+ if not r["linked"]:
+ return "decline", ["no linked issue"]
+ iss = r["issue"]
+ if iss is None:
+ closed = [f"#{i['number']}" for i in r["linked"] if i["state"] != "OPEN"]
+ taken = [
+ f"#{i['number']} (assigned to {', '.join(i['assignees'])})"
+ for i in r["linked"]
+ if i["state"] == "OPEN"
+ ]
+ return "decline", [
+ "every linked issue is closed or assigned elsewhere: "
+ + ", ".join(closed + taken)
+ ]
+ if r["hold"]:
+ return "needs a human", ["draft or marked do-not-merge"]
+
+ j = r.get("jev", {})
+ reporter = iss["reporter"] == r["author"]
+ if iss["competing"] and not reporter:
+ rivals = ", ".join(f"#{x}" for x in iss["competing"])
+ return "needs a human", [
+ f"#{iss['number']} has other contributions ({rivals}); pick one"
+ ]
+
+ reasons = []
+ if iss["merged"]:
+ merged = ", ".join(f"#{x}" for x in iss["merged"])
+ reasons.append(f"{merged} already merged for #{iss['number']}; may be fixed")
+ if iss["competing"]:
+ reasons.append(
+ "reporter's PR; others on the issue: "
+ + ", ".join(f"#{x}" for x in iss["competing"])
+ )
+ if r["failing_checks"]:
+ reasons.append("failing: " + ", ".join(r["failing_checks"]))
+ if j["issue_is_bug"] < DECLINE_BELOW:
+ reasons.append(
+ f"issue reads as not a bug ({j['issue_is_bug']}); check for an approved design"
+ )
+ elif j["issue_is_bug"] < CLEAR_YES:
+ reasons.append(f"bug unclear ({j['issue_is_bug']})")
+ if j["scoped_bugfix"] < DECLINE_BELOW:
+ reasons.append(
+ f"reads as an enhancement ({j['scoped_bugfix']}); needs an approved design"
+ )
+ elif j["scoped_bugfix"] < CLEAR_YES:
+ reasons.append(f"scope unclear ({j['scoped_bugfix']})")
+ if j["fixes_at_cause"] < CAUSE_BELOW:
+ reasons.append(f"may patch a symptom ({j['fixes_at_cause']})")
+ if j["generated_noise"] >= NOISY_AT:
+ reasons.append(f"description reads generated ({j['generated_noise']})")
+ if not any(f.startswith("tests/") for f in r["files"]):
+ reasons.append("adds no test")
+ if r["prs_30d"] >= HIGH_VOLUME:
+ reasons.append(
+ f"author opened {r['prs_30d']} PRs in 30 days; check follow-through"
+ )
+ if r["size"] > 400:
+ reasons.append(f"large ({r['size']} lines)")
+ if reasons:
+ return "needs a human", reasons
+
+ why = ["reporter" if reporter else "only PR on the issue", "checks green"]
+ if r["merged_here"]:
+ why.append(f"{r['merged_here']} merged here before")
+ return "worth assigning", why
+
+
+def report(rows: list[dict]) -> str:
+ order = ["worth assigning", "needs a human", "decline"]
+ lines = [
+ "## Gate queue digest",
+ "",
+ f"{len(rows)} contributor PRs are waiting on assignment. Nothing was assigned or posted.",
+ "",
+ ]
+ for verdict in order:
+ group = sorted(
+ (r for r in rows if r["verdict"] == verdict), key=lambda r: r["opened"]
+ )
+ lines += [
+ f"### {verdict} ({len(group)})",
+ "",
+ "| PR | issue | author | why | bug · scoped · cause · noise |",
+ "|---|---|---|---|---|",
+ ]
+ for r in group:
+ j = r.get("jev", {})
+ scores = " · ".join(f"{j[k]:.2f}" for k in QUESTIONS) if j else ""
+ issue = f"#{r['issue']['number']}" if r["issue"] else ""
+ title = r["title"].replace("|", "\\|")
+ lines.append(
+ f"| #{r['pr']} {title} | {issue} | @{r['author']} | {'; '.join(r['reasons'])} | {scores} |"
+ )
+ lines.append("")
+ lines.append(
+ "Scores are Jev's probability of yes to: the issue shows a bug; the PR is a "
+ "scoped fix; it changes the code where the problem starts; the description "
+ 'reads like unedited generated text. "Worth assigning" means worth a '
+ "maintainer's review, not ready to merge."
+ )
+ return "\n".join(lines)
+
+
+def main() -> None:
+ rows = gather()
+ asyncio.run(judge(rows))
+ for r in rows:
+ r["verdict"], r["reasons"] = decide(r)
+ print(report(rows))
+
+
+if __name__ == "__main__":
+ main()