π The fast, Pythonic way to build MCP servers and clients.
Add dedupe workflow (#1454)
Jeremiah Lowin2a659b823664
diff --git a/.claude/commands/dedupe.md b/.claude/commands/dedupe.mdnew file mode 100644index 000000000..8f05bb4ca--- /dev/null+++ b/.claude/commands/dedupe.md@@ -0,0 +1,37 @@+---+allowed-tools: Bash(gh issue view:*), Bash(gh search:*), Bash(gh issue list:*), Bash(gh api:*), Bash(gh issue comment:*), Task+description: Find duplicate GitHub issues+---++Find up to 3 likely duplicate issues for a given GitHub issue.++To do this, follow these steps precisely:++1. Use an agent to check if the Github issue (a) is closed, (b) does not need to be deduped (eg. because it is broad product feedback without a specific solution, or positive feedback), or (c) already has a duplicates comment that you made earlier. If so, do not proceed.++2. Use an agent to view a Github issue, and ask the agent to return a summary of the issue++3. Then, launch 5 parallel agents to search Github for duplicates of this issue, using diverse keywords and search approaches, using the summary from #1++4. Next, feed the results from #1 and #2 into another agent, so that it can filter out false positives, that are likely not actually duplicates of the original issue. If there are no duplicates remaining, do not proceed.++5. Finally, comment back on the issue with a list of up to three duplicate issues (or zero, if there are no likely duplicates)++Notes (be sure to tell this to your agents, too):+- Use `gh` to interact with Github, rather than web fetch+- Do not use other tools, beyond `gh` (eg. don't use other MCP servers, file edit, etc.)+- Make a todo list first+- For your comment, follow the following format precisely (assuming for this example that you found 3 suspected duplicates):++---++Found 3 possible duplicate issues:+1. #123: Issue title here+2. #456: Another issue title+3. #789: Third issue title++This issue will be automatically closed as a duplicate in 3 days.+- If your issue is a duplicate, please close it and π the existing issue instead+- To prevent auto-closure, add a comment or π this comment++---\ No newline at end of filediff --git a/.github/workflows/auto-close-duplicates.yml b/.github/workflows/auto-close-duplicates.ymlnew file mode 100644index 000000000..40e85c7a9--- /dev/null+++ b/.github/workflows/auto-close-duplicates.yml@@ -0,0 +1,28 @@+name: Auto-close duplicate issues+description: Auto-closes issues that are duplicates of existing issues+on:+ schedule:+ - cron: "0 9 * * *" # Run daily at 9 AM UTC+ workflow_dispatch:++jobs:+ auto-close-duplicates:+ runs-on: ubuntu-latest+ timeout-minutes: 10+ permissions:+ contents: read+ issues: write++ steps:+ - name: Checkout repository+ uses: actions/checkout@v4++ - name: Install uv+ uses: astral-sh/setup-uv@v4++ - name: Auto-close duplicate issues+ run: uv run scripts/auto_close_duplicates.py+ env:+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}+ GITHUB_REPOSITORY_OWNER: ${{ github.repository_owner }}+ GITHUB_REPOSITORY_NAME: ${{ github.event.repository.name }}diff --git a/.github/workflows/claude-dedupe-issues.yml b/.github/workflows/claude-dedupe-issues.ymlnew file mode 100644index 000000000..1dda73be6--- /dev/null+++ b/.github/workflows/claude-dedupe-issues.yml@@ -0,0 +1,37 @@+name: Marvin Issue Dedupe+description: Automatically dedupe GitHub issues using Marvin+on:+ issues:+ types: [opened]+ workflow_dispatch:+ inputs:+ issue_number:+ description: "Issue number to process for duplicate detection"+ required: true+ type: string++jobs:+ claude-dedupe-issues:+ runs-on: ubuntu-latest+ timeout-minutes: 10+ permissions:+ contents: read+ issues: write++ steps:+ - name: Checkout repository+ uses: actions/checkout@v4++ - name: Generate Marvin App token+ id: marvin-token+ uses: actions/create-github-app-token@v1+ with:+ app-id: ${{ secrets.MARVIN_APP_ID }}+ private-key: ${{ secrets.MARVIN_APP_PRIVATE_KEY }}++ - name: Run Marvin slash command+ uses: anthropics/claude-code-base-action@beta+ with:+ prompt: "/dedupe ${{ github.repository }}/issues/${{ github.event.issue.number || inputs.issue_number }}"+ anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}+ github_token: ${{ steps.marvin-token.outputs.token }}diff --git a/scripts/auto_close_duplicates.py b/scripts/auto_close_duplicates.pynew file mode 100644index 000000000..34732c676--- /dev/null+++ b/scripts/auto_close_duplicates.py@@ -0,0 +1,322 @@+#!/usr/bin/env python+# /// script+# requires-python = ">=3.10"+# dependencies = [+# "httpx",+# ]+# ///+"""+Auto-close duplicate GitHub issues.++This script runs on a schedule to automatically close issues that have been+marked as duplicates and haven't received any preventing activity.+"""++import os+from dataclasses import dataclass+from datetime import datetime, timedelta, timezone++import httpx+++@dataclass+class Issue:+ """Represents a GitHub issue."""++ number: int+ title: str+ state: str+ created_at: str+ user_id: int+ user_login: str+++@dataclass+class Comment:+ """Represents a GitHub comment."""++ id: int+ body: str+ created_at: str+ user_id: int+ user_login: str+ user_type: str+++@dataclass+class Reaction:+ """Represents a reaction on a comment."""++ user_id: int+ user_login: str+ content: str+++class GitHubClient:+ """Client for interacting with GitHub API."""++ def __init__(self, token: str, owner: str, repo: str):+ self.token = token+ self.owner = owner+ self.repo = repo+ self.headers = {+ "Authorization": f"token {token}",+ "Accept": "application/vnd.github.v3+json",+ }+ self.base_url = f"https://api.github.com/repos/{owner}/{repo}"++ def get_open_issues(+ self, created_before: datetime, page: int = 1, per_page: int = 100+ ) -> list[Issue]:+ """Fetch open issues created before a certain date."""+ url = f"{self.base_url}/issues"+ issues = []++ with httpx.Client() as client:+ response = client.get(+ url,+ headers=self.headers,+ params={"state": "open", "per_page": per_page, "page": page},+ )++ if response.status_code != 200:+ print(f"Error fetching issues: {response.status_code}")+ return issues++ data = response.json()+ for item in data:+ # Skip pull requests+ if "pull_request" in item:+ continue++ created_at = datetime.fromisoformat(+ item["created_at"].replace("Z", "+00:00")+ )+ if created_at <= created_before:+ issues.append(+ Issue(+ number=item["number"],+ title=item["title"],+ state=item["state"],+ created_at=item["created_at"],+ user_id=item["user"]["id"],+ user_login=item["user"]["login"],+ )+ )++ return issues++ def get_issue_comments(self, issue_number: int) -> list[Comment]:+ """Fetch all comments for an issue."""+ url = f"{self.base_url}/issues/{issue_number}/comments"+ comments = []++ with httpx.Client() as client:+ page = 1+ while True:+ response = client.get(+ url, headers=self.headers, params={"page": page, "per_page": 100}+ )++ if response.status_code != 200:+ break++ data = response.json()+ if not data:+ break++ for comment_data in data:+ comments.append(+ Comment(+ id=comment_data["id"],+ body=comment_data["body"],+ created_at=comment_data["created_at"],+ user_id=comment_data["user"]["id"],+ user_login=comment_data["user"]["login"],+ user_type=comment_data["user"]["type"],+ )+ )++ page += 1+ if page > 10: # Safety limit+ break++ return comments++ def get_comment_reactions(+ self, issue_number: int, comment_id: int+ ) -> list[Reaction]:+ """Fetch reactions for a specific comment."""+ url = f"{self.base_url}/issues/{issue_number}/comments/{comment_id}/reactions"+ reactions = []++ with httpx.Client() as client:+ response = client.get(url, headers=self.headers)++ if response.status_code != 200:+ return reactions++ data = response.json()+ for reaction_data in data:+ reactions.append(+ Reaction(+ user_id=reaction_data["user"]["id"],+ user_login=reaction_data["user"]["login"],+ content=reaction_data["content"],+ )+ )++ return reactions++ def close_issue(self, issue_number: int, comment: str) -> bool:+ """Close an issue with a comment."""+ # First add the comment+ comment_url = f"{self.base_url}/issues/{issue_number}/comments"+ with httpx.Client() as client:+ response = client.post(+ comment_url, headers=self.headers, json={"body": comment}+ )++ if response.status_code != 201:+ print(f"Failed to add comment to issue #{issue_number}")+ return False++ # Then close the issue+ issue_url = f"{self.base_url}/issues/{issue_number}"+ with httpx.Client() as client:+ response = client.patch(+ issue_url, headers=self.headers, json={"state": "closed"}+ )++ return response.status_code == 200+++def find_duplicate_comment(comments: list[Comment]) -> Comment | None:+ """Find a bot comment marking the issue as duplicate."""+ for comment in comments:+ # Check for the specific duplicate message format+ body_lower = comment.body.lower()+ if (+ "possible duplicate issue" in body_lower+ and "this issue will be automatically closed as a duplicate" in body_lower+ ):+ return comment+ return None+++def should_close_as_duplicate(+ issue: Issue,+ duplicate_comment: Comment,+ all_comments: list[Comment],+ reactions: list[Reaction],+) -> bool:+ """Determine if an issue should be closed as duplicate."""++ # Check if comment is old enough (3 days)+ comment_date = datetime.fromisoformat(+ duplicate_comment.created_at.replace("Z", "+00:00")+ )+ three_days_ago = datetime.now(timezone.utc) - timedelta(days=3)++ if comment_date > three_days_ago:+ return False++ # Check for preventing reactions (thumbs down)+ for reaction in reactions:+ if reaction.content in ["-1", "confused"]:+ print(+ f"Issue #{issue.number}: Has preventing reaction from {reaction.user_login}"+ )+ return False++ # Check for user activity after the duplicate comment+ for comment in all_comments:+ comment_date_check = datetime.fromisoformat(+ comment.created_at.replace("Z", "+00:00")+ )+ if comment_date_check > comment_date:+ # Issue author commented after duplicate marking+ if comment.user_id == issue.user_id:+ print(+ f"Issue #{issue.number}: Author commented after duplicate marking"+ )+ return False++ return True+++def main():+ """Main entry point for auto-closing duplicate issues."""+ print("[DEBUG] Starting auto-close duplicates script")++ # Get environment variables+ token = os.environ.get("GITHUB_TOKEN")+ if not token:+ raise ValueError("GITHUB_TOKEN environment variable is required")++ owner = os.environ.get("GITHUB_REPOSITORY_OWNER", "jlowin")+ repo = os.environ.get("GITHUB_REPOSITORY_NAME", "fastmcp")++ print(f"[DEBUG] Repository: {owner}/{repo}")++ # Initialize client+ client = GitHubClient(token, owner, repo)++ # Get issues created more than 3 days ago+ three_days_ago = datetime.now(timezone.utc) - timedelta(days=3)++ all_issues = []+ page = 1++ while page <= 20: # Safety limit+ issues = client.get_open_issues(three_days_ago, page=page)+ if not issues:+ break+ all_issues.extend(issues)+ page += 1++ print(f"[DEBUG] Found {len(all_issues)} open issues created more than 3 days ago")++ processed_count = 0+ closed_count = 0++ for issue in all_issues:+ processed_count += 1++ if processed_count % 10 == 0:+ print(f"[DEBUG] Processed {processed_count}/{len(all_issues)} issues")++ # Get comments for this issue+ comments = client.get_issue_comments(issue.number)++ # Look for duplicate marking comment+ duplicate_comment = find_duplicate_comment(comments)+ if not duplicate_comment:+ continue++ print(f"[DEBUG] Issue #{issue.number} has duplicate comment")++ # Get reactions on the duplicate comment+ reactions = client.get_comment_reactions(issue.number, duplicate_comment.id)++ # Check if we should close+ if should_close_as_duplicate(issue, duplicate_comment, comments, reactions):+ close_message = (+ "Closing this issue as a duplicate based on the automated analysis above.\n\n"+ "The duplicate issues identified contain existing discussions and potential solutions. "+ "Please add your π to those issues if they match your use case.\n\n"+ "If this was closed in error, please leave a comment explaining why this is not "+ "a duplicate and we'll reopen it."+ )++ if client.close_issue(issue.number, close_message):+ print(f"[SUCCESS] Closed issue #{issue.number} as duplicate")+ closed_count += 1+ else:+ print(f"[ERROR] Failed to close issue #{issue.number}")++ print(f"[DEBUG] Processing complete. Closed {closed_count} duplicate issues")+++if __name__ == "__main__":+ main()