Skip to content

armgit / zed

Public Imported from GitHub
0

Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.

Duplicate Bot: Reduce noise (#58074)

Lenab7b1d1a2c78b
diff --git a/script/github-check-new-issue-for-duplicates.py b/script/github-check-new-issue-for-duplicates.py
index 023449bb27d..8a4eb8ec22a 100644
--- a/script/github-check-new-issue-for-duplicates.py
+++ b/script/github-check-new-issue-for-duplicates.py
@@ -146,11 +146,15 @@ No action needed. A maintainer will review this shortly.
]
parts.append("**Possibly related open issues:**\n\n" + "\n".join(lines))
if related_closed_issues:
+ # state_reason is shown only for "duplicate" (the close type is otherwise
+ # already visible from GitHub's icon next to the issue number on render).
lines = [
- f"- #{m['number']} (closed as {m['state_reason']}) — {m['explanation']}"
+ f"- #{m['number']}"
+ f"{' (closed as duplicate)' if m['state_reason'] == 'duplicate' else ''}"
+ f" — {m['explanation']}"
for m in related_closed_issues
]
- parts.append("**Recently closed, possibly related:**\n\n" + "\n".join(lines))
+ parts.append("**Recently closed, possibly the same bug:**\n\n" + "\n".join(lines))
body = "\n\n".join(parts)
sections.append(f"""<details>
<summary>Additional recent context for triagers</summary>
@@ -280,6 +284,12 @@ def detect_areas(anthropic_key, issue, area_labels):
system_prompt = """You analyze GitHub issues to identify which area labels apply.
+Decide the area from the user's stated symptom and reproduction steps. Issue bodies routinely
+contain pasted log output, crash dumps, stack traces, settings files, and template headers like
+"Attach Zed log file" or "Relevant Zed settings" — these are evidence about the symptom and
+should not push you toward labels like "logging" or "settings" unless the bug itself is about
+how that subsystem works.
+
Respond with ONLY a comma-separated list of matching area names. No prose, no explanation,
no markdown, no preamble — just the names.
@@ -500,8 +510,14 @@ def analyze_duplicates(anthropic_key, issue, magnets, search_results):
return [], [], []
log("Analyzing candidates with Claude")
+ log(f" Candidate pool: {len(top_magnets)} magnets, {len(open_results)} open search results, "
+ f"{len(closed_results)} closed search results (will pass {min(len(closed_results), 5)} closed)")
enrich_magnets(top_magnets)
+ closed_candidates_for_claude = closed_results[:5]
+ if closed_candidates_for_claude:
+ log(f" Closed candidates given to proposer: {[r['number'] for r in closed_candidates_for_claude]}")
+
candidates = [
{"number": m["number"], "title": m["title"], "body_preview": m["body_preview"],
"state": "open", "state_reason": None, "source": "known_duplicate_magnet"}
@@ -509,7 +525,7 @@ def analyze_duplicates(anthropic_key, issue, magnets, search_results):
] + [
{"number": r["number"], "title": r["title"], "body_preview": r["body_preview"],
"state": r["state"], "state_reason": r["state_reason"], "source": "search_result"}
- for r in open_results[:10] + closed_results[:5]
+ for r in open_results[:10] + closed_candidates_for_claude
]
system_prompt = """You analyze GitHub issues to (a) identify duplicates among OPEN candidates
@@ -548,17 +564,63 @@ Examples of things that are NOT duplicates:
For OPEN duplicates (either bucket), false positives are MUCH worse than false negatives — they
waste the time of both the issue author and the maintainers. When in doubt, omit.
-# (b) Related closed issues — CLOSED candidates only
+# (b) Closed candidates that may be the same bug — CLOSED candidates only
+
+The goal is NOT a "related reading" list. The goal is to surface closed issues where the
+new issue is plausibly the SAME bug — a duplicate that just happens to be filed against a
+closed predecessor instead of an open one. Empty is preferable to weak filler — triagers
+lose trust in this section quickly if it's stretched. The same false-positives-are-worse
+asymmetry as for duplicates applies here.
+
+The bar: a triager reading this should be able to act — ask the reporter to retest a fix,
+point at a known design decision that already declined this request, or point at the
+canonical bug this is a duplicate of. "Useful context" or "shared area" is NOT a reason
+to include.
+
+Omit a candidate if ANY of these apply (in observed practice, almost everything does):
+
+1. Self-contradiction. If you find yourself writing "while focused on X rather than Y",
+ "although this is about A, the new issue is about B", "this issue focuses on... rather
+ than...", or any acknowledgment that the candidate isn't on the same topic — STOP.
+ You've already made the case for omitting it.
+
+2. Fabricated specifics. Every concrete claim about the candidate (its trigger, its scope,
+ its conditions) must be visible in the candidate's title or body preview. Specifics
+ like "when X happens", "under Y conditions", "specifically affecting Z" that aren't
+ supported by the candidate's actual text mean you're inventing details to fit the new
+ issue. Omit.
-The goal is to give triagers extra context, NOT to claim a duplicate. The bar is lower than for
-duplicates: include a closed candidate if a triager would plausibly want to see it when reviewing
-the new issue. Examples worth surfacing:
-- A recently fixed (state_reason "completed") issue describing the same symptom — triager may ask
- the reporter to retest on the latest build.
-- A cluster of similar issues closed as "not_planned" — signals a known limitation or design choice.
-- A previously triaged duplicate (state_reason "duplicate") in the same code area.
+3. Weasel phrases. Paraphrases of these all indicate you don't have a real claim:
+ "may indicate similar...", "could provide context for...", "shows / demonstrates recent
+ attention to...", "indicates the team has considered...", "demonstrates a pattern
+ of...", "may provide useful context...". STOP and omit.
-Include at most 5 closed candidates, prioritized by relevance.
+4. Retest by default. The "reporter may need to retest on the latest build" framing only
+ applies when the candidate's symptom is literally the same as the new issue's. It is
+ NOT a default justification for "this was a recent fix in roughly the same area."
+
+5. Same area / feature, different mechanism. Examples to omit:
+ - "ARM compile failure" alongside "ARM runtime perf" — same area, different mechanism.
+ - "Worktree path bug" alongside "worktree display label confusion" — same feature,
+ unrelated.
+
+6. Vague catch-all candidate. A closed issue like "Zed is slow" / "performance" / "agent
+ panel UX" that could be cited next to almost any new bug is filler. If you'd reuse the
+ same closed issue across many unrelated new issues, omit.
+
+7. Label or single-keyword overlap. A closed issue whose only connection is a shared
+ area:* label or one shared keyword is not relevant.
+
+Worth surfacing — strict examples:
+- A recently fixed ("completed") issue with the SAME specific trigger as the new issue —
+ triager can ask the reporter to retest on the latest build.
+- A cluster of "not_planned" closures about the EXACT same request — known design choice
+ the triager can point to.
+- A previously triaged "duplicate" pointing at the same canonical issue, or sharing the
+ same specific mechanism.
+
+Count: typically 0 or 1. Never more than 2 unless there's an obvious cluster of identical
+"not_planned" reports. 0 is a normal outcome.
# Output format
@@ -614,10 +676,164 @@ Return empty arrays where nothing relevant is found."""
likely = data.get("likely_duplicates", [])
possible = data.get("possible_duplicates", [])
closed = data.get("related_closed_issues", [])
+
+ # Claude occasionally places a closed candidate in the duplicate buckets, or vice
+ # versa. Enforce that each match lives in the bucket consistent with the canonical
+ # state of the candidate we passed in.
+ candidate_states = {c["number"]: c["state"] for c in candidates}
+
+ def filter_by_state(items, expected_state, label):
+ kept, wrong = [], []
+ for m in items:
+ (kept if candidate_states.get(m["number"]) == expected_state else wrong).append(m)
+ if wrong:
+ log(f" Dropped {len(wrong)} from {label} with wrong/unknown state: {[m['number'] for m in wrong]}")
+ return kept
+
+ likely = filter_by_state(likely, "open", "likely_duplicates")
+ possible = filter_by_state(possible, "open", "possible_duplicates")
+ closed = filter_by_state(closed, "closed", "related_closed_issues")
+
+ # Avoid showing the same issue in both the user-facing alert and the triage section.
+ likely_numbers = {m["number"] for m in likely}
+ overlap = [m["number"] for m in possible if m["number"] in likely_numbers]
+ if overlap:
+ log(f" Dropped {len(overlap)} from possible_duplicates already in likely_duplicates: {overlap}")
+ possible = [m for m in possible if m["number"] not in likely_numbers]
+
log(f" Found {len(likely) + len(possible) + len(closed)} potential matches")
return likely, possible, closed
+CRITIQUE_SYSTEM_PROMPT = """You are evaluating ONE recently closed GitHub issue to decide whether a triager looking
+at a brand-new bug report would find it useful to be told about that closed issue.
+
+There is no slate to fill. There is no quota. You will be shown exactly one candidate.
+The default verdict is OMIT. Zero is the expected outcome for most candidates.
+
+A candidate is worth surfacing ONLY if the new issue is plausibly the SAME BUG as the
+closed one — a duplicate that happens to be filed against a closed predecessor. Concretely,
+the legitimate cases are exactly three:
+
+- The candidate was closed as "completed" (a fix shipped) AND the new issue has the same
+ specific trigger / symptom. The triager will ask the reporter to retest.
+- The candidate was closed as "not_planned" AND the new issue is the EXACT same request
+ (a feature decision the team already declined). The triager will point at it.
+- The candidate was closed as "duplicate" AND it pointed at the same canonical bug the new
+ issue describes, or it shares the same specific mechanism.
+
+"Same broad area", "similar-sounding symptom", or "recent attention to this subsystem" are
+NOT reasons to include. Omit them.
+
+Return "omit" if ANY of the following apply (in observed practice, almost everything does):
+
+1. Self-contradiction. If your reasoning includes "while focused on X rather than Y",
+ "although this is about A, the new issue is about B", "this issue focuses on... rather
+ than...", or any acknowledgment the candidate is on a different topic — you've already
+ decided to omit.
+2. Fabricated specifics. Every concrete claim about the candidate (its trigger, scope,
+ conditions) must be visible in the candidate's title or body preview. If you find
+ yourself describing the candidate using details that aren't in its text, you're
+ inventing details to fit the new issue. Omit.
+3. Weasel phrases. Paraphrases of "may indicate similar...", "could provide context
+ for...", "shows / demonstrates recent attention to...", "indicates the team has
+ considered...", "demonstrates a pattern of...", "may provide useful context..." —
+ these mean you don't have a real claim. Omit.
+4. Retest by default. The "reporter may need to retest on the latest build" framing only
+ applies when the closed issue's symptom is LITERALLY the same as the new issue's. "This
+ was a recent fix in roughly the same area" is not enough.
+5. Same area / feature, different mechanism. Same area label but different bug, different
+ code path, different trigger. Omit.
+6. Vague catch-all candidate. A closed issue like "Zed is slow" / "performance" / "agent
+ panel UX" that you could cite next to many unrelated new bugs. Omit.
+7. Label or single-keyword overlap. Only connection is a shared area:* label or one shared
+ keyword. Omit.
+
+Output only valid JSON (no markdown code blocks):
+{
+ "verdict": "include" | "omit",
+ "rule_violated": null | 1 | 2 | 3 | 4 | 5 | 6 | 7,
+ "rationale": "one concise sentence explaining the verdict"
+}
+
+When "verdict" is "include", "rule_violated" must be null.
+When "verdict" is "omit", "rule_violated" should be the most relevant rule number, or null
+if the candidate is simply too unrelated for any rule to specifically apply."""
+
+
+def critique_closed_candidates(anthropic_key, issue, proposed, search_results):
+ """Run a strict per-candidate critique pass over the proposer's closed candidates.
+
+ For each proposed match, call Claude with only the new issue and that single candidate
+ (blind to the proposer's rationale) and ask for a yes/no verdict. Default is omit.
+ Returns the subset of `proposed` that passes critique.
+ """
+ if not proposed:
+ log(" Critique: proposer surfaced 0 closed candidates; skipping")
+ return []
+
+ log(f" Critique: proposer surfaced {len(proposed)} closed candidate(s): "
+ f"{[m['number'] for m in proposed]}")
+
+ results_by_number = {r["number"]: r for r in search_results}
+ kept = []
+ for match in proposed:
+ number = match["number"]
+ candidate = results_by_number.get(number)
+ if candidate is None:
+ # Should not happen — analyze_duplicates only emits numbers from candidates it
+ # was given — but be defensive rather than crash the bot.
+ log(f" Critique: dropping #{number} — candidate context not found")
+ continue
+
+ state_reason = candidate.get("state_reason") or "unknown"
+ user_content = f"""## New Issue #{issue['number']}
+**Title:** {issue['title']}
+
+**Body:**
+{issue['body'][:3000]}
+
+## Closed Candidate #{number}
+**Title:** {candidate.get('title', '')}
+**State reason:** {state_reason}
+
+**Body preview:**
+{candidate.get('body_preview', '')}"""
+
+ log(f" Critique: evaluating #{number}")
+ try:
+ response = call_claude(anthropic_key, CRITIQUE_SYSTEM_PROMPT, user_content, max_tokens=300)
+ except requests.RequestException as e:
+ # If the critique call fails, prefer omitting the candidate over posting noise.
+ log(f" Critique: API call failed for #{number} ({e}); omitting candidate")
+ continue
+
+ fence = re.match(r"^\s*```(?:json)?\s*\n?(.*?)\n?```\s*$", response, re.DOTALL)
+ if fence:
+ response = fence.group(1)
+
+ try:
+ verdict_data = json.loads(response)
+ except json.JSONDecodeError as e:
+ log(f" Critique: failed to parse verdict for #{number} ({e}); omitting candidate")
+ log(f" Raw response: {response}")
+ continue
+
+ verdict = verdict_data.get("verdict")
+ rule = verdict_data.get("rule_violated")
+ rationale = verdict_data.get("rationale", "")
+
+ if verdict == "include":
+ log(f" Critique: keeping #{number} — {rationale}")
+ kept.append(match)
+ else:
+ rule_str = f"rule {rule}" if rule else "no specific rule"
+ log(f" Critique: omitting #{number} ({rule_str}) — {rationale}")
+
+ log(f" Critique: kept {len(kept)} of {len(proposed)} closed candidates")
+ return kept
+
+
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Identify potential duplicate issues")
parser.add_argument("issue_number", type=int, help="Issue number to analyze")
@@ -658,6 +874,13 @@ if __name__ == "__main__":
anthropic_key, issue, relevant_magnets, search_results
)
+ # second-pass critique: prompt iteration on the proposer hit a ceiling around 30% noise.
+ # Re-evaluate each proposed closed candidate in isolation with a stricter prompt that
+ # has no slate to fill and is blind to the proposer's rationale.
+ related_closed_issues = critique_closed_candidates(
+ anthropic_key, issue, related_closed_issues, search_results
+ )
+
# resolve close reason from our search results (the source of truth) so we don't depend
# on Claude to faithfully echo it back
results_by_number = {r["number"]: r for r in search_results}
diff --git a/script/github-track-duplicate-bot-effectiveness.py b/script/github-track-duplicate-bot-effectiveness.py
index 06dcae67cd8..02be1b9e137 100644
--- a/script/github-track-duplicate-bot-effectiveness.py
+++ b/script/github-track-duplicate-bot-effectiveness.py
@@ -24,6 +24,7 @@ import functools
import os
import re
import sys
+import time
from datetime import datetime, timezone
import requests
@@ -47,6 +48,8 @@ BOT_START_DATE = "2026-02-18"
NEEDS_TRIAGE_LABEL = "state:needs triage"
DEFAULT_PROJECT_NUMBER = 76
VALID_CLOSED_AS_VALUES = {"duplicate", "not_planned", "completed"}
+# HTTP statuses we'll retry on for GET requests
+TRANSIENT_HTTP_STATUSES = {429, 500, 502, 503, 504}
# Add a new tuple when you deploy a new version of the bot that you want to
# keep track of (e.g. the prompt gets a rewrite or the model gets swapped).
# Newest first, please. The datetime is for the deployment time (merge to main).
@@ -67,10 +70,22 @@ def bot_version_for_time(date_string):
def github_api_get(path, params=None):
+ """Fetch JSON from the GitHub REST API, retrying transient failures. Raises on non-2xx status."""
url = f"{GITHUB_API}/{path.lstrip('/')}"
- response = requests.get(url, headers=GITHUB_HEADERS, params=params)
- response.raise_for_status()
- return response.json()
+ for attempt in range(3):
+ try:
+ response = requests.get(url, headers=GITHUB_HEADERS, params=params)
+ response.raise_for_status()
+ return response.json()
+ except requests.RequestException as e:
+ transient = isinstance(e, (requests.ConnectionError, requests.Timeout)) or (
+ isinstance(e, requests.HTTPError) and e.response.status_code in TRANSIENT_HTTP_STATUSES
+ )
+ if not transient or attempt == 2:
+ raise
+ wait = 2 ** attempt
+ print(f" Transient GitHub API error ({e}); retrying in {wait}s")
+ time.sleep(wait)
def github_search_issues(query):
@@ -161,9 +176,11 @@ def find_canonical_among(duplicate_number, candidates):
if not candidates:
return None
+ # candidate issue numbers are baked into the query body via field aliases
+ # (GraphQL doesn't let you parametrize alias names), so $numbers isn't needed.
data = github_api_graphql(
"""
- query($owner: String!, $repo: String!, $numbers: [Int!]!) {
+ query($owner: String!, $repo: String!) {
repository(owner: $owner, name: $repo) {
PLACEHOLDER
}
@@ -174,7 +191,7 @@ def find_canonical_among(duplicate_number, candidates):
f' nodes {{ ... on MarkedAsDuplicateEvent {{ duplicate {{ ... on Issue {{ number }} }} }} }} }} }}'
for number in candidates
)),
- {"owner": REPO_OWNER, "repo": REPO_NAME, "numbers": list(candidates)},
+ {"owner": REPO_OWNER, "repo": REPO_NAME},
partial_errors_ok=True,
)
@@ -409,11 +426,10 @@ def classify_as_assist(issue, bot_comment):
bot_comment_time=bot_comment["created_at"])
return
- original = None
- try:
- original = find_canonical_among(issue["number"], suggested)
- except (requests.RequestException, RuntimeError) as error:
- print(f" Warning: failed to query candidate timelines: {error}")
+ # Let exceptions from find_canonical_among propagate — a query failure here is
+ # not the same as "no canonical match" and shouldn't be silently downgraded to
+ # a Needs review entry. Failing the workflow surfaces the problem immediately.
+ original = find_canonical_among(issue["number"], suggested)
if original:
status = "Auto-classified"
@@ -483,6 +499,8 @@ def classify_open():
errors += 1
print(f" Done: added {added}, skipped {skipped}, errors {errors}")
+ if errors > 0:
+ sys.exit(1)
if __name__ == "__main__":