Zulip server and web application. Open-source team chat that helps teams stay productive and focused.
#!/usr/bin/env -S uv run --script --frozen --only-group release-tools # -*-python-*-
import json
import re
import time
import urllib.parse
from collections import defaultdict
from dataclasses import dataclass, field
from functools import cache
from typing import Annotated, Any
import requests
import typer
from github import Auth, Github
from rich.console import Console
from rich.progress import Progress
from typing_extensions import override
def encode_hash_component(s: str) -> str:
hash_replacements = {
"%": ".",
"(": ".28",
")": ".29",
".": ".2E",
}
encoded = urllib.parse.quote(s, safe="*")
return "".join(hash_replacements.get(c, c) for c in encoded)
@cache
def search_czo_for_number(prefix: str, number: int) -> frozenset[str]:
params = {
"anchor": "newest",
"num_before": "100",
"num_after": "0",
"narrow": json.dumps(
[
{"negated": False, "operator": "search", "operand": f'"#{prefix}{number}"'},
{"negated": False, "operator": "channels", "operand": "web-public"},
]
),
}
try:
response = requests.get("https://chat.zulip.org/json/messages", params=params, timeout=30)
response.raise_for_status()
data = response.json()
messages = data.get("messages", [])
# Extract unique topic URLs
urls = set()
for msg in messages:
stream_id = msg.get("stream_id")
display_recipient = msg.get("display_recipient")
subject = msg.get("subject")
assert stream_id
assert display_recipient
encoded_recipient = encode_hash_component(display_recipient.replace(" ", "-"))
encoded_subject = encode_hash_component(subject)
url = f"https://chat.zulip.org/#narrow/channel/{stream_id}-{encoded_recipient}/topic/{encoded_subject}"
urls.add(url)
return frozenset(urls)
except requests.exceptions.RequestException as e:
assert e.response
if e.response.status_code != 429:
raise
retry_after = int(e.response.headers["Retry-After"]) + 1
time.sleep(retry_after)
return search_czo_for_number(prefix, number)
@dataclass
class Issue:
number: int
title: str
czo_urls: set[str] = field(default_factory=set)
closed_by_prs: list["PullRequest"] = field(default_factory=list)
duplicate_issue_ids: set[int] = field(default_factory=set)
@override
def __hash__(self) -> int:
return hash(self.number)
@override
def __eq__(self, other: object) -> bool:
return isinstance(other, Issue) and self.number == other.number
@dataclass
class PullRequest:
number: int
title: str
czo_urls: set[str] = field(default_factory=set)
@override
def __hash__(self) -> int:
return hash(self.number)
@override
def __eq__(self, other: object) -> bool:
return isinstance(other, PullRequest) and self.number == other.number
@dataclass
class Commit:
sha: str
subject: str
czo_urls: set[str] = field(default_factory=set)
class CommitRangeAnalyzer:
CHERRY_PICK_TRAILER = re.compile(
r"^\(cherry picked from commit ([0-9a-f]{40})\)$", re.MULTILINE
)
COMMIT_PRS_QUERY = """
query($oid: GitObjectID!, $repo: String!) {
repository(owner: "zulip", name: $repo) {
object(oid: $oid) {
... on Commit {
messageHeadline
messageBody
associatedPullRequests(first: 10) {
nodes {
number
title
url
body
comments(first: 100) {
nodes {
body
}
}
closingIssuesReferences(first: 50) {
nodes {
number
title
url
body
comments(first: 100) {
nodes {
body
}
}
timelineItems(first:100, itemTypes:MARKED_AS_DUPLICATE_EVENT) {
... on IssueTimelineItemsConnection {
nodes {
... on MarkedAsDuplicateEvent {
duplicate {
... on Issue {
number
}
}
}
}
}
}
}
}
}
}
}
}
}
}
"""
ISSUE_QUERY = """
query($number: Int!, $repo: String!) {
repository(owner: "zulip", name: $repo) {
issue(number: $number) {
number
title
body
comments(first: 100) {
nodes {
body
}
}
timelineItems(first:100, itemTypes:MARKED_AS_DUPLICATE_EVENT) {
... on IssueTimelineItemsConnection {
nodes {
... on MarkedAsDuplicateEvent {
duplicate {
... on Issue {
number
}
}
}
}
}
}
}
}
}
"""
def __init__(self, token: str, reponame: str, czo_issue_prefix: str) -> None:
self.github = Github(auth=Auth.Token(token))
self.reponame = reponame
self.czo_issue_prefix = czo_issue_prefix
@staticmethod
def _extract_czo_urls(text: str | None) -> set[str]:
if not text:
return set()
matches = re.findall(r"https://chat\.zulip\.org/[^\s\)\]\>]+", text)
urls = set()
for url in matches:
# We strip off and remove /with/... and /near/... to
# reduce the number of unique links which are generated.
parsed_url = re.match(r"(.*)/topic/([^/]+)(/(near|with)/.*)?$", url)
if not parsed_url:
continue
urls.add(
parsed_url[1]
+ "/topic/"
# Normalize the topic by decoding and re-encoding
+ encode_hash_component(urllib.parse.unquote(parsed_url[2].replace(".", "%")))
)
return urls
@staticmethod
def _extract_issue_numbers(reponame: str, text: str | None) -> set[int]:
if not text:
return set()
# https://docs.github.com/en/issues/tracking-your-work-with-issues/using-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword
pattern = rf"\b(?:close[sd]?|fix(?:es|ed)?|resolve[sd]?):?\s+(?:zulip/{reponame})?#(\d+)\b"
matches = re.findall(pattern, text, re.IGNORECASE)
return {int(num) for num in matches}
def _issue_from_node(self, issue_node: dict[str, Any], console: Console) -> Issue:
issue = Issue(
number=issue_node["number"],
title=issue_node["title"].strip(),
)
issue_comments = issue_node.get("comments", {}).get("nodes", [])
for node in [issue_node, *issue_comments]:
issue.czo_urls.update(self._extract_czo_urls(node.get("body")))
issue.czo_urls.update(search_czo_for_number(self.czo_issue_prefix, issue.number))
issue.duplicate_issue_ids = {
event["duplicate"]["number"] for event in issue_node["timelineItems"]["nodes"]
}
for issue_id in issue.duplicate_issue_ids:
if duplicate_info := self._fetch_issue(issue_id, console):
issue.czo_urls.update(duplicate_info.czo_urls)
issue.duplicate_issue_ids.update(duplicate_info.duplicate_issue_ids)
return issue
def _fetch_issue(self, number: int, console: Console) -> Issue | None:
"""Fetch issue metadata from GitHub."""
try:
_, result = self.github.requester.graphql_query(
self.ISSUE_QUERY, {"number": number, "repo": self.reponame}
)
if "errors" in result:
console.log(
"Failed to fetch zulip/%s#%d: %s", self.reponame, number, result["errors"]
)
return None
data = result.get("data", {})
issue_node = data.get("repository", {}).get("issue")
if not issue_node:
return None
return self._issue_from_node(issue_node, console)
except Exception:
return None
def _fetch_commit(self, commit_sha: str) -> dict[str, Any] | None:
_, result = self.github.requester.graphql_query(
self.COMMIT_PRS_QUERY, {"oid": commit_sha, "repo": self.reponame}
)
if "errors" in result:
error_messages = [e.get("message", str(e)) for e in result["errors"]]
raise RuntimeError(f"GraphQL errors: {', '.join(error_messages)}")
data = result.get("data", {})
return data.get("repository", {}).get("object")
def _fetch_cherry_pick_source(
self, commit_sha: str, commit_obj: dict[str, Any], console: Console
) -> dict[str, Any] | None:
match = self.CHERRY_PICK_TRAILER.search(commit_obj.get("messageBody", ""))
if match is None:
return None
source_sha = match[1]
source_obj = self._fetch_commit(source_sha)
if source_obj is None:
console.log(f"{commit_sha} was cherry-picked from unknown commit {source_sha}")
return None
if not source_obj.get("associatedPullRequests", {}).get("nodes", []):
return None
return source_obj
def get_issues_for_commit(
self, commit_sha: str, console: Console
) -> tuple[dict[Issue, set[PullRequest]], set[PullRequest], Commit | None]:
commit_obj = self._fetch_commit(commit_sha)
if commit_obj is None:
return dict(), set(), None
# A commit cherry-picked onto a release branch has a new hash
# that GitHub does not associate with the original PR, so
# look up the commit it was picked from instead.
if not commit_obj.get("associatedPullRequests", {}).get("nodes", []):
source_obj = self._fetch_cherry_pick_source(commit_sha, commit_obj, console)
if source_obj is not None:
commit_obj = source_obj
commit_message = commit_obj.get("messageBody", "")
pr_nodes = commit_obj.get("associatedPullRequests", {}).get("nodes", [])
results: dict[Issue, set[PullRequest]] = defaultdict(set)
all_seen_prs: set[PullRequest] = set()
for pr_node in pr_nodes:
pr = PullRequest(
number=pr_node["number"],
title=pr_node["title"].strip(),
)
all_seen_prs.add(pr)
pr_comments = pr_node.get("comments", {}).get("nodes", [])
for node in [pr_node, *pr_comments]:
pr.czo_urls.update(self._extract_czo_urls(node.get("body")))
pr.czo_urls.update(search_czo_for_number(self.czo_issue_prefix, pr.number))
# Get issues from PR metadata
issues_dict = {}
issue_nodes = pr_node.get("closingIssuesReferences", {}).get("nodes", [])
for issue_node in issue_nodes:
if issue_node is None:
continue
issue = self._issue_from_node(issue_node, console)
issues_dict[issue.number] = issue
# Extract additional issue numbers from commit message
for issue_num in self._extract_issue_numbers(self.reponame, commit_message):
if issue_num not in issues_dict:
maybe_issue = self._fetch_issue(issue_num, console)
if maybe_issue is None:
continue
issues_dict[issue_num] = maybe_issue
for issue in issues_dict.values():
results[issue].add(pr)
unlinked_commit = None
if not pr_nodes:
for issue_num in self._extract_issue_numbers(self.reponame, commit_message):
maybe_issue = self._fetch_issue(issue_num, console)
if maybe_issue is None:
continue
results[maybe_issue].update()
if not results:
# Unlike PRs, we don't search chat.zulip.org for the commit
# hash, since hashes are rarely pasted in chat.
unlinked_commit = Commit(
sha=commit_sha,
subject=commit_obj.get("messageHeadline", "").strip(),
czo_urls=self._extract_czo_urls(commit_message),
)
return results, all_seen_prs, unlinked_commit
def analyze_range(
self, base: str, head: str
) -> tuple[list[Issue], list[PullRequest], list[Commit]]:
console = Console(stderr=True, log_path=False)
repository = self.github.get_repo(f"zulip/{self.reponame}")
comparison = repository.compare(base, head)
commit_shas = [commit.sha for commit in comparison.commits]
console.log(f"Found {len(commit_shas)} commits")
all_seen_prs: set[PullRequest] = set()
issue_to_prs: dict[Issue, set[PullRequest]] = defaultdict(set)
unlinked_commits: list[Commit] = []
with Progress(console=console) as progress:
task = progress.add_task("Processing commits...", total=len(commit_shas))
for sha in commit_shas:
progress.console.log(f"Processing {sha}")
this_issue_to_prs, this_seen_prs, unlinked_commit = self.get_issues_for_commit(
sha, console
)
for issue, prs in this_issue_to_prs.items():
issue_to_prs[issue].update(prs)
all_seen_prs.update(this_seen_prs)
if unlinked_commit is not None:
unlinked_commits.append(unlinked_commit)
progress.advance(task)
for issue, prs in issue_to_prs.items():
for pr in prs:
issue.czo_urls.update(pr.czo_urls)
issue.closed_by_prs = sorted(prs, key=lambda p: p.number)
linked_prs = {pr for prs in issue_to_prs.values() for pr in prs}
unlinked_prs = all_seen_prs - linked_prs
console.log(
f"Found {len(all_seen_prs)} unique PRs, of which {len(linked_prs)} PRs closed {len(issue_to_prs)} issues",
)
console.log(f"Found {len(unlinked_prs)} PRs not linked to an issue")
console.log(f"Found {len(unlinked_commits)} commits with no PR or issue")
return (
sorted(issue_to_prs.keys(), key=lambda x: x.number),
sorted(unlinked_prs, key=lambda x: x.number),
unlinked_commits,
)
def validate_github_token(value: str) -> str:
# https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/about-authentication-to-github#githubs-token-formats
if value.startswith("github_"):
return value
if re.match(r"gh[pousr]_", value):
return value
raise typer.BadParameter("Github access tokens start with `github_`, or `gh`")
from enum import Enum
class ZulipRepo(str, Enum):
zulip = ("zulip",)
flutter = "zulip-flutter"
repo_issue_prefixes = {
ZulipRepo.zulip: "",
ZulipRepo.flutter: "F",
}
def main(
base_commit: Annotated[str, typer.Argument(help="Commit-ish, resolved on the server")],
head_commit: Annotated[str, typer.Argument(help="Commit-ish, resolved on the server")],
token: Annotated[
str,
typer.Option(
metavar="TOKEN",
envvar="GITHUB_TOKEN",
show_envvar=True,
callback=validate_github_token,
help=(
"Github access token; can be a fine-grained personal access token "
"with read-only access to 'Public repositories'. "
"See https://github.com/settings/personal-access-tokens/new"
),
),
],
repo: ZulipRepo = ZulipRepo.zulip,
) -> int:
"""Find issues which are closed in a commit range."""
lines = []
prefix = repo_issue_prefixes[repo]
analyzer = CommitRangeAnalyzer(token, repo.value, prefix)
issues, unlinked_prs, unlinked_commits = analyzer.analyze_range(base_commit, head_commit)
if issues:
lines.append("### Issues resolved")
for issue in issues:
lines.append(f"#### #{prefix}{issue.number}: {issue.title}")
if issue.duplicate_issue_ids:
lines.append(
" - **Duplicates:** "
+ ", ".join(f"#{prefix}{number}" for number in issue.duplicate_issue_ids)
)
lines.extend(
f" - **Closed by:** #{prefix}{pr.number}: {pr.title}" for pr in issue.closed_by_prs
)
lines.extend(f" - {czo_url}" for czo_url in sorted(issue.czo_urls))
lines.append("")
# zulip-cloud-deploy.yaml splits this output into messages at blank
# lines. Without a blank line after every PR and commit entry, a long
# list would be one block and could be cut in the middle of a line.
if unlinked_prs:
lines.append("### PRs not linked to an issue")
for pr in unlinked_prs:
lines.append(f" - #{prefix}{pr.number}: {pr.title}")
lines.append("")
if unlinked_commits:
lines.append("### Commits not linked to a PR or issue")
for commit in unlinked_commits:
# 11 characters matches get_short_sha in zerver/lib/webhooks/git.py,
# so hashes look the same as in messages from the git integrations.
lines.append(f" - {commit.sha[:11]}: {commit.subject}")
lines.extend(f" - {czo_url}" for czo_url in sorted(commit.czo_urls))
lines.append("")
print("\n".join(lines))
return 0
if __name__ == "__main__":
typer.run(main)