Skip to content

πŸš€ The fast, Pythonic way to build MCP servers and clients.

Add dedupe workflow (#1454)

Jeremiah Lowin2a659b823664
diff --git a/.claude/commands/dedupe.md b/.claude/commands/dedupe.md
new file mode 100644
index 000000000..8f05bb4ca
--- /dev/null
+++ b/.claude/commands/dedupe.md
@@ -0,0 +1,37 @@
+---
+allowed-tools: Bash(gh issue view:*), Bash(gh search:*), Bash(gh issue list:*), Bash(gh api:*), Bash(gh issue comment:*), Task
+description: Find duplicate GitHub issues
+---
+
+Find up to 3 likely duplicate issues for a given GitHub issue.
+
+To do this, follow these steps precisely:
+
+1. Use an agent to check if the Github issue (a) is closed, (b) does not need to be deduped (eg. because it is broad product feedback without a specific solution, or positive feedback), or (c) already has a duplicates comment that you made earlier. If so, do not proceed.
+
+2. Use an agent to view a Github issue, and ask the agent to return a summary of the issue
+
+3. Then, launch 5 parallel agents to search Github for duplicates of this issue, using diverse keywords and search approaches, using the summary from #1
+
+4. Next, feed the results from #1 and #2 into another agent, so that it can filter out false positives, that are likely not actually duplicates of the original issue. If there are no duplicates remaining, do not proceed.
+
+5. Finally, comment back on the issue with a list of up to three duplicate issues (or zero, if there are no likely duplicates)
+
+Notes (be sure to tell this to your agents, too):
+- Use `gh` to interact with Github, rather than web fetch
+- Do not use other tools, beyond `gh` (eg. don't use other MCP servers, file edit, etc.)
+- Make a todo list first
+- For your comment, follow the following format precisely (assuming for this example that you found 3 suspected duplicates):
+
+---
+
+Found 3 possible duplicate issues:
+1. #123: Issue title here
+2. #456: Another issue title
+3. #789: Third issue title
+
+This issue will be automatically closed as a duplicate in 3 days.
+- If your issue is a duplicate, please close it and πŸ‘ the existing issue instead
+- To prevent auto-closure, add a comment or πŸ‘Ž this comment
+
+---
\ No newline at end of file
diff --git a/.github/workflows/auto-close-duplicates.yml b/.github/workflows/auto-close-duplicates.yml
new file mode 100644
index 000000000..40e85c7a9
--- /dev/null
+++ b/.github/workflows/auto-close-duplicates.yml
@@ -0,0 +1,28 @@
+name: Auto-close duplicate issues
+description: Auto-closes issues that are duplicates of existing issues
+on:
+ schedule:
+ - cron: "0 9 * * *" # Run daily at 9 AM UTC
+ workflow_dispatch:
+
+jobs:
+ auto-close-duplicates:
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ permissions:
+ contents: read
+ issues: write
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+
+ - name: Install uv
+ uses: astral-sh/setup-uv@v4
+
+ - name: Auto-close duplicate issues
+ run: uv run scripts/auto_close_duplicates.py
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ GITHUB_REPOSITORY_OWNER: ${{ github.repository_owner }}
+ GITHUB_REPOSITORY_NAME: ${{ github.event.repository.name }}
diff --git a/.github/workflows/claude-dedupe-issues.yml b/.github/workflows/claude-dedupe-issues.yml
new file mode 100644
index 000000000..1dda73be6
--- /dev/null
+++ b/.github/workflows/claude-dedupe-issues.yml
@@ -0,0 +1,37 @@
+name: Marvin Issue Dedupe
+description: Automatically dedupe GitHub issues using Marvin
+on:
+ issues:
+ types: [opened]
+ workflow_dispatch:
+ inputs:
+ issue_number:
+ description: "Issue number to process for duplicate detection"
+ required: true
+ type: string
+
+jobs:
+ claude-dedupe-issues:
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ permissions:
+ contents: read
+ issues: write
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+
+ - name: Generate Marvin App token
+ id: marvin-token
+ uses: actions/create-github-app-token@v1
+ with:
+ app-id: ${{ secrets.MARVIN_APP_ID }}
+ private-key: ${{ secrets.MARVIN_APP_PRIVATE_KEY }}
+
+ - name: Run Marvin slash command
+ uses: anthropics/claude-code-base-action@beta
+ with:
+ prompt: "/dedupe ${{ github.repository }}/issues/${{ github.event.issue.number || inputs.issue_number }}"
+ anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
+ github_token: ${{ steps.marvin-token.outputs.token }}
diff --git a/scripts/auto_close_duplicates.py b/scripts/auto_close_duplicates.py
new file mode 100644
index 000000000..34732c676
--- /dev/null
+++ b/scripts/auto_close_duplicates.py
@@ -0,0 +1,322 @@
+#!/usr/bin/env python
+# /// script
+# requires-python = ">=3.10"
+# dependencies = [
+# "httpx",
+# ]
+# ///
+"""
+Auto-close duplicate GitHub issues.
+
+This script runs on a schedule to automatically close issues that have been
+marked as duplicates and haven't received any preventing activity.
+"""
+
+import os
+from dataclasses import dataclass
+from datetime import datetime, timedelta, timezone
+
+import httpx
+
+
+@dataclass
+class Issue:
+ """Represents a GitHub issue."""
+
+ number: int
+ title: str
+ state: str
+ created_at: str
+ user_id: int
+ user_login: str
+
+
+@dataclass
+class Comment:
+ """Represents a GitHub comment."""
+
+ id: int
+ body: str
+ created_at: str
+ user_id: int
+ user_login: str
+ user_type: str
+
+
+@dataclass
+class Reaction:
+ """Represents a reaction on a comment."""
+
+ user_id: int
+ user_login: str
+ content: str
+
+
+class GitHubClient:
+ """Client for interacting with GitHub API."""
+
+ def __init__(self, token: str, owner: str, repo: str):
+ self.token = token
+ self.owner = owner
+ self.repo = repo
+ self.headers = {
+ "Authorization": f"token {token}",
+ "Accept": "application/vnd.github.v3+json",
+ }
+ self.base_url = f"https://api.github.com/repos/{owner}/{repo}"
+
+ def get_open_issues(
+ self, created_before: datetime, page: int = 1, per_page: int = 100
+ ) -> list[Issue]:
+ """Fetch open issues created before a certain date."""
+ url = f"{self.base_url}/issues"
+ issues = []
+
+ with httpx.Client() as client:
+ response = client.get(
+ url,
+ headers=self.headers,
+ params={"state": "open", "per_page": per_page, "page": page},
+ )
+
+ if response.status_code != 200:
+ print(f"Error fetching issues: {response.status_code}")
+ return issues
+
+ data = response.json()
+ for item in data:
+ # Skip pull requests
+ if "pull_request" in item:
+ continue
+
+ created_at = datetime.fromisoformat(
+ item["created_at"].replace("Z", "+00:00")
+ )
+ if created_at <= created_before:
+ issues.append(
+ Issue(
+ number=item["number"],
+ title=item["title"],
+ state=item["state"],
+ created_at=item["created_at"],
+ user_id=item["user"]["id"],
+ user_login=item["user"]["login"],
+ )
+ )
+
+ return issues
+
+ def get_issue_comments(self, issue_number: int) -> list[Comment]:
+ """Fetch all comments for an issue."""
+ url = f"{self.base_url}/issues/{issue_number}/comments"
+ comments = []
+
+ with httpx.Client() as client:
+ page = 1
+ while True:
+ response = client.get(
+ url, headers=self.headers, params={"page": page, "per_page": 100}
+ )
+
+ if response.status_code != 200:
+ break
+
+ data = response.json()
+ if not data:
+ break
+
+ for comment_data in data:
+ comments.append(
+ Comment(
+ id=comment_data["id"],
+ body=comment_data["body"],
+ created_at=comment_data["created_at"],
+ user_id=comment_data["user"]["id"],
+ user_login=comment_data["user"]["login"],
+ user_type=comment_data["user"]["type"],
+ )
+ )
+
+ page += 1
+ if page > 10: # Safety limit
+ break
+
+ return comments
+
+ def get_comment_reactions(
+ self, issue_number: int, comment_id: int
+ ) -> list[Reaction]:
+ """Fetch reactions for a specific comment."""
+ url = f"{self.base_url}/issues/{issue_number}/comments/{comment_id}/reactions"
+ reactions = []
+
+ with httpx.Client() as client:
+ response = client.get(url, headers=self.headers)
+
+ if response.status_code != 200:
+ return reactions
+
+ data = response.json()
+ for reaction_data in data:
+ reactions.append(
+ Reaction(
+ user_id=reaction_data["user"]["id"],
+ user_login=reaction_data["user"]["login"],
+ content=reaction_data["content"],
+ )
+ )
+
+ return reactions
+
+ def close_issue(self, issue_number: int, comment: str) -> bool:
+ """Close an issue with a comment."""
+ # First add the comment
+ comment_url = f"{self.base_url}/issues/{issue_number}/comments"
+ with httpx.Client() as client:
+ response = client.post(
+ comment_url, headers=self.headers, json={"body": comment}
+ )
+
+ if response.status_code != 201:
+ print(f"Failed to add comment to issue #{issue_number}")
+ return False
+
+ # Then close the issue
+ issue_url = f"{self.base_url}/issues/{issue_number}"
+ with httpx.Client() as client:
+ response = client.patch(
+ issue_url, headers=self.headers, json={"state": "closed"}
+ )
+
+ return response.status_code == 200
+
+
+def find_duplicate_comment(comments: list[Comment]) -> Comment | None:
+ """Find a bot comment marking the issue as duplicate."""
+ for comment in comments:
+ # Check for the specific duplicate message format
+ body_lower = comment.body.lower()
+ if (
+ "possible duplicate issue" in body_lower
+ and "this issue will be automatically closed as a duplicate" in body_lower
+ ):
+ return comment
+ return None
+
+
+def should_close_as_duplicate(
+ issue: Issue,
+ duplicate_comment: Comment,
+ all_comments: list[Comment],
+ reactions: list[Reaction],
+) -> bool:
+ """Determine if an issue should be closed as duplicate."""
+
+ # Check if comment is old enough (3 days)
+ comment_date = datetime.fromisoformat(
+ duplicate_comment.created_at.replace("Z", "+00:00")
+ )
+ three_days_ago = datetime.now(timezone.utc) - timedelta(days=3)
+
+ if comment_date > three_days_ago:
+ return False
+
+ # Check for preventing reactions (thumbs down)
+ for reaction in reactions:
+ if reaction.content in ["-1", "confused"]:
+ print(
+ f"Issue #{issue.number}: Has preventing reaction from {reaction.user_login}"
+ )
+ return False
+
+ # Check for user activity after the duplicate comment
+ for comment in all_comments:
+ comment_date_check = datetime.fromisoformat(
+ comment.created_at.replace("Z", "+00:00")
+ )
+ if comment_date_check > comment_date:
+ # Issue author commented after duplicate marking
+ if comment.user_id == issue.user_id:
+ print(
+ f"Issue #{issue.number}: Author commented after duplicate marking"
+ )
+ return False
+
+ return True
+
+
+def main():
+ """Main entry point for auto-closing duplicate issues."""
+ print("[DEBUG] Starting auto-close duplicates script")
+
+ # Get environment variables
+ token = os.environ.get("GITHUB_TOKEN")
+ if not token:
+ raise ValueError("GITHUB_TOKEN environment variable is required")
+
+ owner = os.environ.get("GITHUB_REPOSITORY_OWNER", "jlowin")
+ repo = os.environ.get("GITHUB_REPOSITORY_NAME", "fastmcp")
+
+ print(f"[DEBUG] Repository: {owner}/{repo}")
+
+ # Initialize client
+ client = GitHubClient(token, owner, repo)
+
+ # Get issues created more than 3 days ago
+ three_days_ago = datetime.now(timezone.utc) - timedelta(days=3)
+
+ all_issues = []
+ page = 1
+
+ while page <= 20: # Safety limit
+ issues = client.get_open_issues(three_days_ago, page=page)
+ if not issues:
+ break
+ all_issues.extend(issues)
+ page += 1
+
+ print(f"[DEBUG] Found {len(all_issues)} open issues created more than 3 days ago")
+
+ processed_count = 0
+ closed_count = 0
+
+ for issue in all_issues:
+ processed_count += 1
+
+ if processed_count % 10 == 0:
+ print(f"[DEBUG] Processed {processed_count}/{len(all_issues)} issues")
+
+ # Get comments for this issue
+ comments = client.get_issue_comments(issue.number)
+
+ # Look for duplicate marking comment
+ duplicate_comment = find_duplicate_comment(comments)
+ if not duplicate_comment:
+ continue
+
+ print(f"[DEBUG] Issue #{issue.number} has duplicate comment")
+
+ # Get reactions on the duplicate comment
+ reactions = client.get_comment_reactions(issue.number, duplicate_comment.id)
+
+ # Check if we should close
+ if should_close_as_duplicate(issue, duplicate_comment, comments, reactions):
+ close_message = (
+ "Closing this issue as a duplicate based on the automated analysis above.\n\n"
+ "The duplicate issues identified contain existing discussions and potential solutions. "
+ "Please add your πŸ‘ to those issues if they match your use case.\n\n"
+ "If this was closed in error, please leave a comment explaining why this is not "
+ "a duplicate and we'll reopen it."
+ )
+
+ if client.close_issue(issue.number, close_message):
+ print(f"[SUCCESS] Closed issue #{issue.number} as duplicate")
+ closed_count += 1
+ else:
+ print(f"[ERROR] Failed to close issue #{issue.number}")
+
+ print(f"[DEBUG] Processing complete. Closed {closed_count} duplicate issues")
+
+
+if __name__ == "__main__":
+ main()