Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Implement AI-Powered CI Failure Analysis and Fix Recommendations (#2277)

chen, suyue18486c3902ae
diff --git a/.azure-pipelines/scripts/ai_analysis/ai_analyze.py b/.azure-pipelines/scripts/ai_analysis/ai_analyze.py
new file mode 100644
index 000000000..a261f4a89
--- /dev/null
+++ b/.azure-pipelines/scripts/ai_analysis/ai_analyze.py
@@ -0,0 +1,409 @@
+"""AI analysis of the most frequent unknown CI failures.
+
+Ranks the *unknown* failure clusters by occurrence, takes the top N (default 3)
+and asks an AI backend to classify each failure (Code Regression / Environment /
+Dependency / Test Case) and propose a fix. When the model is not confident it is
+instructed to omit the fix and instead give investigation directions.
+
+The backend is intentionally pluggable behind ``call_backend`` so the current
+GitHub Copilot CLI implementation can be swapped later. With ``--backend none``
+the script emits the structured prompts only (no model call), which supports the
+"generate data, human runs Copilot Chat" fallback.
+"""
+
+import argparse
+import datetime
+import json
+import os
+import subprocess
+import sys
+
+_PROMPT_TEMPLATE = """You are analyzing a failing CI unit test for the AutoRound project (post-training \
+quantization for LLMs/VLMs). Determine the failure type and, only when confident, propose a fix.
+
+Respond with ONLY a single JSON object, no prose, using exactly these keys:
+{{
+ "category": "Code Regression | Environment Issue | Dependency Issue | Test Case Issue | Unknown",
+ "confidence": "high | medium | low",
+ "root_cause": "concise explanation of the likely cause",
+ "suggested_fix": "concrete fix description, or empty string if confidence is low",
+ "patch": "a unified diff if you are confident, otherwise empty string",
+ "directions": "if confidence is low, list concrete investigation directions; else empty string"
+}}
+
+Guidance:
+- Use the PR changed files to judge whether this is a code regression introduced by the change.
+- If the error looks environmental/dependency-related and unrelated to the diff, say so.
+- If uncertain, set confidence to "low", leave suggested_fix and patch empty, and fill directions.
+- For an ``AssertionError`` whose log excerpt already contains pytest's ``Full diff`` (the ``-``/``+``
+ lines), that diff is usually authoritative: cross-check it against the PR changed files below and
+ conclude directly.
+- The ``PR changed files`` section below lists every file this PR touched. Inspect the relevant ones
+ yourself with your ``read``/``rg`` tools on the checkout at ``{project_root}`` (and the complete raw
+ failure logs under ``{log_path}``) to judge whether the change caused the failure.
+
+## Project source root
+{project_root}
+
+## Full failure logs directory
+{log_path}
+
+## Error signature
+{signature}
+
+## Affected tests ({test_count})
+{tests}
+
+## Log excerpt
+{excerpt}
+
+## PR changed files
+{pr_files}
+"""
+
+
+def call_backend(
+ prompt: str, backend: str, timeout: int, model: str, reasoning_effort: str, trace_file: str, cluster_id
+) -> str:
+ """Dispatch a prompt to the selected AI backend and return raw text output.
+
+ Swap point: replace the ``copilot`` branch to change the inference provider.
+ """
+ if backend == "none":
+ return ""
+ if backend == "copilot":
+ return _call_copilot_cli(prompt, timeout, model, reasoning_effort, trace_file, cluster_id)
+ raise ValueError(f"unknown backend: {backend}")
+
+
+def _append_trace(trace_file: str, entry: dict) -> None:
+ """Append one AI-call record (JSONL) so the analysis process can be audited."""
+ if not trace_file:
+ return
+ try:
+ with open(trace_file, "a", encoding="utf-8") as f:
+ f.write(json.dumps(entry, ensure_ascii=False) + "\n")
+ except OSError as e:
+ print(f"Warning: could not write trace file: {e}", file=sys.stderr)
+
+
+def _parse_copilot_json(stdout: str) -> dict:
+ """Parse the Copilot ``--output-format json`` JSONL stream.
+
+ A single stream carries everything we need: the assistant answer, the model
+ actually used, and the AI-credit / premium-request usage — so there is no need
+ to shell out for stderr credits or dig through debug logs separately.
+ """
+ answer = ""
+ model = ""
+ nano_aiu = None
+ premium_requests = None
+ for line in stdout.splitlines():
+ line = line.strip()
+ if not line:
+ continue
+ try:
+ event = json.loads(line)
+ except json.JSONDecodeError:
+ continue
+ etype = event.get("type")
+ data = event.get("data", {})
+ if etype == "assistant.message":
+ model = data.get("model") or model
+ if data.get("content"):
+ answer = data["content"] # keep the last full assistant message
+ elif etype == "model.call_start":
+ model = data.get("model") or model
+ elif etype == "session.usage_checkpoint":
+ nano_aiu = data.get("totalNanoAiu", nano_aiu)
+ premium_requests = data.get("totalPremiumRequests", premium_requests)
+ elif etype == "result":
+ usage = event.get("usage", {})
+ premium_requests = usage.get("premiumRequests", premium_requests)
+ return {
+ "answer": answer,
+ "model": model,
+ "ai_credits": round(nano_aiu / 1e9, 6) if nano_aiu is not None else None,
+ "premium_requests": premium_requests,
+ }
+
+
+def _call_copilot_cli(prompt: str, timeout: int, model: str, reasoning_effort: str, trace_file: str, cluster_id) -> str:
+ argv = [
+ "copilot",
+ "-p",
+ prompt,
+ "--allow-tool=shell(python:*)",
+ "--allow-tool=shell(rg:*)",
+ "--allow-tool=read",
+ "--allow-tool=write",
+ # The PR diff is pre-computed into the prompt, so no GitHub API access is needed. Dropping the
+ # built-in GitHub MCP server trims the tool schema re-sent on every turn (~11k tokens -> far
+ # less), which is the dominant per-call cost in this multi-turn loop.
+ "--disable-builtin-mcps",
+ "--no-ask-user",
+ "--output-format",
+ "json",
+ ]
+ if model:
+ argv += ["--model", model]
+ if reasoning_effort:
+ argv += ["--reasoning-effort", reasoning_effort]
+ env = dict(os.environ)
+ # Let the Copilot CLI authenticate with the dedicated token.
+ ai_token = os.environ.get("AI_TOKEN", "")
+ if ai_token:
+ env.setdefault("GITHUB_TOKEN", ai_token)
+ env.setdefault("GH_TOKEN", ai_token)
+ started = datetime.datetime.now(datetime.timezone.utc)
+ returncode = None
+ stderr = ""
+ stdout = ""
+ try:
+ proc = subprocess.run(argv, capture_output=True, text=True, timeout=timeout, env=env, check=False)
+ returncode = proc.returncode
+ stderr = proc.stderr.strip()
+ stdout = proc.stdout
+ if returncode != 0:
+ print(f"Warning: Copilot CLI exited {returncode}: {stderr}", file=sys.stderr)
+ except (OSError, subprocess.TimeoutExpired) as e:
+ print(f"Warning: Copilot CLI call failed: {e}", file=sys.stderr)
+ stderr = str(e)
+ ended = datetime.datetime.now(datetime.timezone.utc)
+ parsed = _parse_copilot_json(stdout)
+ raw = parsed["answer"]
+ _append_trace(
+ trace_file,
+ {
+ "cluster_id": cluster_id,
+ "timestamp": started.isoformat(),
+ "duration_s": round((ended - started).total_seconds(), 3),
+ "model": parsed["model"] or model or "(unknown)",
+ "ai_credits": parsed["ai_credits"],
+ "premium_requests": parsed["premium_requests"],
+ "returncode": returncode,
+ "prompt": prompt,
+ "response": raw,
+ "transcript": stdout, # raw JSONL stream: full step-by-step execution trace
+ },
+ )
+ print(
+ f"AI call cluster={cluster_id} "
+ f"model={parsed['model'] or model or '(unknown)'} "
+ f"duration_s={round((ended - started).total_seconds(), 3)} "
+ f"ai_credits={parsed['ai_credits']} "
+ f"premium_requests={parsed['premium_requests']} "
+ f"returncode={returncode}",
+ file=sys.stderr,
+ )
+ return raw
+
+
+def parse_model_json(text: str) -> "dict | None":
+ """Extract the first JSON object from model output; tolerant of surrounding prose."""
+ if not text:
+ return None
+ start = text.find("{")
+ end = text.rfind("}")
+ if start == -1 or end == -1 or end <= start:
+ return None
+ try:
+ return json.loads(text[start : end + 1])
+ except json.JSONDecodeError:
+ return None
+
+
+def _run_git(project_root: str, args: "list[str]", timeout: int = 120) -> "subprocess.CompletedProcess | None":
+ """Run a git command in ``project_root``; return the completed process or None on failure."""
+ try:
+ return subprocess.run(
+ ["git", "-C", project_root or ".", *args],
+ capture_output=True,
+ text=True,
+ timeout=timeout,
+ check=False,
+ )
+ except (OSError, subprocess.TimeoutExpired) as e:
+ print(f"Warning: git {' '.join(args)} failed: {e}", file=sys.stderr)
+ return None
+
+
+_unshallowed = False
+
+
+def _ensure_full_history(project_root: str) -> None:
+ """Unshallow the checkout once so merge-base / parent commits are available.
+
+ CI checkouts are usually shallow, which makes ``git diff <sha>^1...<sha>^2`` fail. Doing it here,
+ once, keeps that cost out of the analysis loop.
+ """
+ global _unshallowed
+ if _unshallowed:
+ return
+ _unshallowed = True
+ res = _run_git(project_root, ["rev-parse", "--is-shallow-repository"])
+ if res is not None and res.stdout.strip() == "true":
+ print("Unshallowing repository for PR diff computation...", file=sys.stderr)
+ _run_git(project_root, ["fetch", "--quiet", "--unshallow", "origin"], timeout=600)
+
+
+def compute_pr_changed_files(pr_sha: str, project_root: str, max_chars: int) -> str:
+ """Return the ``--name-status`` list of files this PR changed (same for every cluster).
+
+ The three-dot range ``sha^1...sha^2`` is the merge commit's change (merge-base..PR-head). We hand
+ the model only the file list and let it read the actual source itself, so the PR context stays
+ small and cluster-independent. Returns an empty string when it cannot be produced (e.g. no SHA).
+ """
+ if not pr_sha:
+ return ""
+ _ensure_full_history(project_root)
+ argv = ["diff", f"{pr_sha}^1...{pr_sha}^2", "--name-status"]
+ res = _run_git(project_root, argv)
+ if res is None or res.returncode != 0:
+ # Retry once after unshallowing in case the first attempt raced a shallow checkout.
+ _run_git(project_root, ["fetch", "--quiet", "--unshallow", "origin"], timeout=600)
+ res = _run_git(project_root, argv)
+ if res is None or res.returncode != 0:
+ return ""
+ files = res.stdout.strip()
+ if len(files) > max_chars:
+ files = files[:max_chars] + "\n... (list truncated) ..."
+ return files
+
+
+def _save_pr_changed_files(output_path: str, pr_files: str) -> None:
+ """Persist the PR changed-file list next to the analysis output for easy log inspection."""
+ if not pr_files:
+ return
+ out_dir = os.path.dirname(os.path.abspath(output_path)) if output_path else "."
+ path = os.path.join(out_dir, "pr_changed_files.txt")
+ try:
+ with open(path, "w", encoding="utf-8") as f:
+ f.write(pr_files + "\n")
+ except OSError as e:
+ print(f"Warning: could not write PR changed-file list {path}: {e}", file=sys.stderr)
+
+
+def build_prompt(cluster: dict, pr_files: str, max_excerpt: int, project_root: str, log_path: str) -> str:
+ return _PROMPT_TEMPLATE.format(
+ signature=cluster.get("signature", ""),
+ test_count=len(cluster.get("tests", [])),
+ tests=", ".join(cluster.get("tests", [])) or "(unknown)",
+ excerpt=(cluster.get("sample", "") or "")[:max_excerpt],
+ pr_files=pr_files or "(empty; inspect the checkout with your read/rg tools if needed)",
+ project_root=project_root or "(not provided)",
+ log_path=log_path or "(not provided)",
+ )
+
+
+def analyze(cluster: dict, pr_files: str, args) -> dict:
+ prompt = build_prompt(cluster, pr_files, args.max_excerpt_chars, args.project_root, args.log_dir)
+ raw = call_backend(
+ prompt,
+ args.backend,
+ args.timeout,
+ args.model,
+ args.reasoning_effort,
+ args.trace_file,
+ cluster.get("id"),
+ )
+ parsed = parse_model_json(raw)
+ result = {
+ "cluster_id": cluster.get("id"),
+ "signature": cluster.get("signature"),
+ "occurrences": cluster.get("occurrences"),
+ "tests": cluster.get("tests", []),
+ }
+ if parsed:
+ # Low-confidence answers must not carry a fix/patch (requirement 3.3).
+ if str(parsed.get("confidence", "")).lower() == "low":
+ parsed["suggested_fix"] = ""
+ parsed["patch"] = ""
+ result.update(
+ {
+ "category": parsed.get("category", "Unknown"),
+ "confidence": parsed.get("confidence", "low"),
+ "root_cause": parsed.get("root_cause", ""),
+ "suggested_fix": parsed.get("suggested_fix", ""),
+ "patch": parsed.get("patch", ""),
+ "directions": parsed.get("directions", ""),
+ }
+ )
+ else:
+ result.update(
+ {
+ "category": "Unknown",
+ "confidence": "low",
+ "root_cause": "",
+ "suggested_fix": "",
+ "patch": "",
+ "directions": "",
+ "prompt": prompt if args.backend == "none" else "",
+ "raw": raw,
+ }
+ )
+ return result
+
+
+def main():
+ parser = argparse.ArgumentParser(description="AI-analyze the top unknown failure clusters")
+ parser.add_argument("--clusters-json", required=True, help="Annotated clusters JSON")
+ parser.add_argument("--output", required=True, help="AI analysis JSON output path")
+ parser.add_argument(
+ "--pr-sha",
+ default="",
+ help="Merge commit SHA of the PR build; used to pre-compute the PR diff embedded in prompts",
+ )
+ parser.add_argument("--top", type=int, default=3, help="Number of top unknown clusters to analyze")
+ parser.add_argument("--backend", choices=["copilot", "none"], default="none", help="Inference backend")
+ parser.add_argument(
+ "--enable-ai-analysis",
+ default="true",
+ help="Set to a falsy value (false/0/no) to force the no-model 'none' backend",
+ )
+ parser.add_argument("--timeout", type=int, default=300, help="Per-call timeout in seconds")
+ parser.add_argument("--model", default="", help="Copilot model to use; empty uses the CLI default")
+ parser.add_argument(
+ "--reasoning-effort",
+ choices=["none", "minimal", "low", "medium", "high", "xhigh", "max"],
+ default="",
+ help="Copilot reasoning effort; empty uses the CLI default",
+ )
+ parser.add_argument("--trace-file", default="", help="JSONL file logging each AI call for auditing")
+ parser.add_argument("--project-root", default="", help="Repository checkout root the AI may inspect")
+ parser.add_argument("--log-dir", default="", help="Directory holding the full raw failure logs")
+ parser.add_argument("--max-excerpt-chars", type=int, default=4000)
+ parser.add_argument(
+ "--max-diff-chars",
+ type=int,
+ default=20000,
+ help="Max characters of the PR changed-file list embedded in each prompt",
+ )
+ args = parser.parse_args()
+
+ # The AI switch overrides the backend so a single pipeline step can toggle it.
+ if str(args.enable_ai_analysis).strip().lower() not in ("1", "true", "yes", "on"):
+ args.backend = "none"
+
+ with open(args.clusters_json, encoding="utf-8") as f:
+ data = json.load(f)
+
+ # The PR changed-file list is the same for every cluster, so compute and persist it once; the
+ # model reads the actual source itself instead of running git inside a shallow checkout.
+ pr_sha = (args.pr_sha or "").strip()
+ pr_files = compute_pr_changed_files(pr_sha, args.project_root, args.max_diff_chars)
+ _save_pr_changed_files(args.output, pr_files)
+
+ unknown = [c for c in data.get("clusters", []) if not c.get("known")]
+ unknown.sort(key=lambda c: c.get("occurrences", 0), reverse=True)
+ selected = unknown[: args.top]
+
+ analyses = [analyze(cluster, pr_files, args) for cluster in selected]
+
+ with open(args.output, "w", encoding="utf-8") as f:
+ json.dump({"backend": args.backend, "analyses": analyses}, f, indent=2)
+ print(f"Analyzed {len(analyses)} unknown cluster(s) with backend ' {args.backend}'", file=sys.stderr)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/.azure-pipelines/scripts/ai_analysis/analyze_failures.py b/.azure-pipelines/scripts/ai_analysis/analyze_failures.py
new file mode 100644
index 000000000..a924b856b
--- /dev/null
+++ b/.azure-pipelines/scripts/ai_analysis/analyze_failures.py
@@ -0,0 +1,320 @@
+"""Extract and cluster CI unit-test failures from downloaded failed logs.
+
+Input is a directory holding the failed ``unittest_*.log`` files together with the
+``failures_part*.json`` metadata emitted by ``collect_result.py``. The script scans
+each failed log for the well-known failure markers, derives a normalized error
+signature per failing test, clusters similar signatures together and writes:
+
+* ``clusters.json`` — structured clusters ranked by occurrence (machine-readable).
+* ``decision_log.md`` — an audit trail of how each cluster was formed (matched
+ keywords, member tests, merged raw log excerpts). Kept separate from the final
+ report on purpose; it is intentionally verbose.
+"""
+
+import argparse
+import json
+import re
+import sys
+from dataclasses import dataclass, field
+from difflib import SequenceMatcher
+from pathlib import Path
+
+# Markers we scan for; crash markers are matched case-insensitively.
+KEYWORDS = ("== FAILURES ==", "== ERRORS ==", "Traceback", "core dumped", "Killed")
+CRASH_MARKERS = ("core dumped", "killed", "segmentation fault", "aborted")
+
+# pytest per-test header of underscores wrapping a test id, e.g. "____ test_foo ____".
+# A very long test name collapses the padding to a single underscore per side
+# ("_ very.long.name _"), so allow one-or-more underscores instead of two-or-more.
+_TEST_HEADER = re.compile(r"^_+ (.+?) _+$")
+# A trailing exception summary line, e.g. "ValueError: shapes do not match".
+_EXC_SUMMARY = re.compile(r"^([A-Za-z_][\w.]*(?:Error|Exception|Warning|Failure|Exit)):?\s")
+# Similarity threshold above which two normalized signatures share a cluster.
+_SIMILARITY = 0.90
+
+
+@dataclass
+class ErrorRecord:
+ """A single extracted failure occurrence."""
+
+ test: str
+ log: str
+ signature: str
+ normalized: str
+ keywords: list[str]
+ excerpt: str
+
+
+@dataclass
+class Cluster:
+ """A group of similar error records."""
+
+ id: int
+ signature: str
+ normalized: str
+ keywords: set[str] = field(default_factory=set)
+ tests: list[str] = field(default_factory=list)
+ logs: set[str] = field(default_factory=set)
+ records: list[ErrorRecord] = field(default_factory=list)
+
+ @property
+ def occurrences(self) -> int:
+ return len(self.records)
+
+
+def normalize(text: str) -> str:
+ """Strip run-specific noise so equivalent errors normalize to the same string."""
+ text = text.strip().lower()
+ text = re.sub(r"0x[0-9a-f]+", "0xADDR", text)
+ text = re.sub(r"/\S+/", "", text) # drop directory paths, keep basenames
+ text = re.sub(r"line \d+", "line N", text)
+ text = re.sub(r"\d+", "N", text)
+ text = re.sub(r"\s+", " ", text)
+ return text.strip()
+
+
+def _matched_keywords(text: str) -> list[str]:
+ low = text.lower()
+ return [kw for kw in KEYWORDS if kw.lower() in low]
+
+
+def _signature_from_segment(lines: list[str]) -> "str | None":
+ """Pick the most specific error line from a FAILURES/traceback segment."""
+ e_lines = [ln[1:].strip() for ln in lines if re.match(r"^E\s+\S", ln)]
+ if e_lines:
+ # Prefer the exception-summary line over pytest continuation lines
+ # (e.g. "+ where 3 = quantize(...)").
+ for el in e_lines:
+ if _EXC_SUMMARY.match(el) or el.lower().startswith("assert"):
+ return el
+ return e_lines[0]
+ for ln in reversed(lines):
+ if _EXC_SUMMARY.match(ln.strip()):
+ return ln.strip()
+ return None
+
+
+def _crash_signature(lines: list[str]) -> "str | None":
+ for ln in lines:
+ low = ln.lower()
+ if any(marker in low for marker in CRASH_MARKERS):
+ return ln.strip()
+ return None
+
+
+def _excerpt(lines: list[str], limit: int = 15) -> str:
+ tail = [ln.rstrip() for ln in lines if ln.strip()][-limit:]
+ return "\n".join(tail)
+
+
+# Trailing pytest noise that should not pollute a test's signature/excerpt.
+_SEGMENT_CUT_MARKERS = (
+ "captured log",
+ "captured stdout",
+ "captured stderr",
+ "generated xml file",
+ "slowest durations",
+)
+
+
+def _trim_segment(lines: list[str]) -> list[str]:
+ """Cut a per-test segment at the first trailing-noise marker."""
+ for i, ln in enumerate(lines):
+ low = ln.lower()
+ if any(marker in low for marker in _SEGMENT_CUT_MARKERS):
+ return lines[:i]
+ return lines
+
+
+def extract_records(log_path: Path) -> list[ErrorRecord]:
+ """Extract error records from a failed log.
+
+ Every log passed here is already known to be a failure (only failed logs are
+ staged), so a log that yields no pytest ``FAILURES`` segment still produces a
+ single record keyed by the log name (crash/segfault or unrecognized output).
+ """
+ try:
+ content = log_path.read_text(encoding="utf-8", errors="replace")
+ except OSError as e:
+ print(f"Warning: cannot read {log_path} - {e}", file=sys.stderr)
+ return []
+
+ lines = content.splitlines()
+ # Drop the pytest tail (durations table + short summary); it is noise that
+ # would otherwise bleed into the last test's excerpt/signature.
+ for i, ln in enumerate(lines):
+ if "slowest durations" in ln.lower():
+ lines = lines[:i]
+ break
+ keywords = _matched_keywords(content)
+ records: list[ErrorRecord] = []
+
+ # Split the log into per-test segments using pytest FAILURES/ERRORS headers.
+ segments: list[tuple[str, list[str]]] = []
+ current_name = ""
+ current: list[str] = []
+ for ln in lines:
+ header = _TEST_HEADER.match(ln)
+ if header:
+ if current:
+ segments.append((current_name, current))
+ current_name = header.group(1).strip()
+ current = []
+ else:
+ current.append(ln)
+ if current:
+ segments.append((current_name, current))
+
+ for name, seg in segments:
+ if not name:
+ continue
+ seg = _trim_segment(seg)
+ sig = _signature_from_segment(seg)
+ if sig is None:
+ continue
+ records.append(
+ ErrorRecord(
+ test=name,
+ log=log_path.name,
+ signature=sig,
+ normalized=normalize(sig),
+ keywords=_matched_keywords("\n".join(seg)) or keywords,
+ excerpt=_excerpt(seg),
+ )
+ )
+
+ # Crash (segfault/OOM-kill) or unrecognized failures produce no FAILURES
+ # section; emit a single record keyed by the log name so it is never dropped.
+ if not records:
+ crash_sig = _crash_signature(lines)
+ if crash_sig is not None:
+ signature, normalized = crash_sig, normalize(crash_sig)
+ else:
+ signature = "Unknown failure (no recognizable error signature)"
+ normalized = "unknown failure"
+ records.append(
+ ErrorRecord(
+ test=log_path.stem,
+ log=log_path.name,
+ signature=signature,
+ normalized=normalized,
+ keywords=keywords or _matched_keywords(signature),
+ excerpt=_excerpt(lines),
+ )
+ )
+ return records
+
+
+def cluster_records(records: list[ErrorRecord]) -> list[Cluster]:
+ """Greedily group records whose normalized signatures are near-identical."""
+ clusters: list[Cluster] = []
+ for record in records:
+ placed = False
+ for cluster in clusters:
+ if cluster.normalized == record.normalized or (
+ SequenceMatcher(None, cluster.normalized, record.normalized).ratio() >= _SIMILARITY
+ ):
+ cluster.records.append(record)
+ cluster.keywords.update(record.keywords)
+ cluster.logs.add(record.log)
+ if record.test not in cluster.tests:
+ cluster.tests.append(record.test)
+ placed = True
+ break
+ if not placed:
+ clusters.append(
+ Cluster(
+ id=0,
+ signature=record.signature,
+ normalized=record.normalized,
+ keywords=set(record.keywords),
+ tests=[record.test],
+ logs={record.log},
+ records=[record],
+ )
+ )
+ clusters.sort(key=lambda c: c.occurrences, reverse=True)
+ for idx, cluster in enumerate(clusters, start=1):
+ cluster.id = idx
+ return clusters
+
+
+def write_clusters_json(clusters: list[Cluster], total_failed: int, output: Path) -> None:
+ payload = {
+ "total_failed_tests": total_failed,
+ "cluster_count": len(clusters),
+ "clusters": [
+ {
+ "id": c.id,
+ "signature": c.signature,
+ "normalized": c.normalized,
+ "keywords": sorted(c.keywords),
+ "occurrences": c.occurrences,
+ "tests": c.tests,
+ "logs": sorted(c.logs),
+ "sample": c.records[0].excerpt if c.records else "",
+ }
+ for c in clusters
+ ],
+ }
+ output.write_text(json.dumps(payload, indent=2), encoding="utf-8")
+ print(f"Wrote {len(clusters)} cluster(s) to {output}", file=sys.stderr)
+
+
+def write_decision_log(clusters: list[Cluster], output: Path) -> None:
+ lines = ["# CI Failure Clustering — Decision Log", ""]
+ lines.append(f"Total clusters: {len(clusters)}")
+ lines.append("")
+ for c in clusters:
+ lines.append(f"## Cluster {c.id} (occurrences: {c.occurrences})")
+ lines.append("")
+ lines.append(f"- Signature: `{c.signature}`")
+ lines.append(f"- Normalized: `{c.normalized}`")
+ lines.append(f"- Matched keywords: {', '.join(sorted(c.keywords)) or '(none)'}")
+ lines.append(f"- Logs: {', '.join(sorted(c.logs))}")
+ lines.append(f"- Affected tests ({len(c.tests)}): {', '.join(c.tests)}")
+ lines.append("")
+ lines.append("Merged log excerpts:")
+ lines.append("")
+ for record in c.records:
+ lines.append(f"### {record.test} ({record.log})")
+ lines.append("```")
+ lines.append(record.excerpt)
+ lines.append("```")
+ lines.append("")
+ output.write_text("\n".join(lines), encoding="utf-8")
+ print(f"Wrote decision log to {output}", file=sys.stderr)
+
+
+def main():
+ parser = argparse.ArgumentParser(description="Extract and cluster CI test failures")
+ parser.add_argument("--input-dir", required=True, type=Path, help="Dir with the failed unittest_*.log files")
+ parser.add_argument("--clusters-json", required=True, type=Path, help="Output clusters JSON path")
+ parser.add_argument("--decision-log", required=True, type=Path, help="Output decision-log markdown path")
+ args = parser.parse_args()
+
+ if not args.input_dir.is_dir():
+ print(f"Error: input dir not found: {args.input_dir}", file=sys.stderr)
+ sys.exit(1)
+
+ log_files = sorted(args.input_dir.rglob("unittest_*.log"))
+
+ records: list[ErrorRecord] = []
+ seen_logs: set[str] = set()
+ for log_file in log_files:
+ if log_file.name in seen_logs: # dedupe job-retry copies of the same log
+ continue
+ seen_logs.add(log_file.name)
+ records.extend(extract_records(log_file))
+
+ total_failed = len(records)
+ clusters = cluster_records(records)
+
+ args.clusters_json.parent.mkdir(parents=True, exist_ok=True)
+ args.decision_log.parent.mkdir(parents=True, exist_ok=True)
+ write_clusters_json(clusters, total_failed, args.clusters_json)
+ write_decision_log(clusters, args.decision_log)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/.azure-pipelines/scripts/ai_analysis/match_known_issues.py b/.azure-pipelines/scripts/ai_analysis/match_known_issues.py
new file mode 100644
index 000000000..88638aec9
--- /dev/null
+++ b/.azure-pipelines/scripts/ai_analysis/match_known_issues.py
@@ -0,0 +1,203 @@
+"""Match clustered CI failures against known-issue tickets on GitHub.
+
+Fetches open issues labeled ``CI-known-issue`` and fuzzily matches each failure
+cluster against the issue title, full body and the failing test names. Clusters
+with a confident match are flagged as *known*; the rest stay *unknown* and are
+handed to the AI-analysis step.
+
+Issue fetching uses the GitHub REST API with a token read from the environment
+(``AUTO_ROUND_BOT_TOKEN`` by default). If no token is available or the request
+fails, every cluster is conservatively treated as unknown.
+"""
+
+import argparse
+import json
+import os
+import re
+import sys
+from difflib import SequenceMatcher
+from urllib.error import HTTPError, URLError
+from urllib.request import Request, urlopen
+
+# Minimum combined score for a cluster to be considered a known issue.
+_MATCH_THRESHOLD = 0.7
+# Title fuzzy-ratio must be near-identical to stand alone; generic assertion
+# boilerplate ("assert ... == 'test'") otherwise inflates unrelated matches.
+_TITLE_STRONG = 0.85
+# Generic error/boilerplate tokens carry no discriminative signal, so they are
+# excluded from signature overlap; otherwise every assertion failure looks alike.
+_STOPWORDS = frozenset(
+ {
+ "assert",
+ "asserts",
+ "assertion",
+ "assertionerror",
+ "error",
+ "errors",
+ "exception",
+ "failed",
+ "failure",
+ "warning",
+ "index",
+ "diff",
+ "full",
+ "test",
+ "tests",
+ "none",
+ "true",
+ "false",
+ "self",
+ "the",
+ "and",
+ "not",
+ "for",
+ "with",
+ "value",
+ "expected",
+ "actual",
+ }
+)
+
+
+def normalize(text: str) -> str:
+ text = text.lower()
+ text = re.sub(r"0x[0-9a-f]+", "0xADDR", text)
+ text = re.sub(r"\d+", "N", text)
+ text = re.sub(r"\s+", " ", text)
+ return text.strip()
+
+
+def fetch_known_issues(repo: str, label: str, token: str) -> list[dict]:
+ """Return open issues carrying ``label`` (title + body), following pagination."""
+ if "/" not in repo:
+ raise ValueError(f"repo must be 'owner/name', got: {repo}")
+ headers = {"Accept": "application/vnd.github+json"}
+ if token:
+ headers["Authorization"] = f"Bearer {token}"
+
+ issues: list[dict] = []
+ page = 1
+ while True:
+ url = f"https://api.github.com/repos/{repo}/issues" f"?labels={label}&state=open&per_page=100&page={page}"
+ req = Request(url, headers=headers)
+ with urlopen(req, timeout=20) as resp:
+ batch = json.loads(resp.read().decode("utf-8"))
+ # The issues endpoint also returns PRs; drop them.
+ batch = [it for it in batch if "pull_request" not in it]
+ issues.extend(batch)
+ if len(batch) < 100:
+ break
+ page += 1
+ return issues
+
+
+def _token_overlap(sig_norm: str, issue_norm: str) -> float:
+ tokens = [t for t in re.split(r"\W+", sig_norm) if len(t) >= 3 and t not in _STOPWORDS]
+ if not tokens:
+ return 0.0
+ hits = sum(1 for t in tokens if t in issue_norm)
+ return hits / len(tokens)
+
+
+def score_issue(cluster: dict, issue: dict) -> "tuple[float, dict]":
+ """Score how well a cluster matches an issue; return (score, matched_by).
+
+ ``matched_by`` breaks the decision into the three underlying signals so the
+ report can show them: ``test`` (test-name corroborated by the error),
+ ``signature`` (error-keyword overlap) and ``title`` (fuzzy title similarity).
+ """
+ sig_norm = normalize(cluster.get("signature", ""))
+ title_norm = normalize(issue.get("title", ""))
+ body_norm = normalize(issue.get("body") or "")
+ issue_norm = f"{title_norm}\n{body_norm}"
+
+ # 1) Failing test name mentioned verbatim in the issue — strongest signal.
+ test_hit = next((t for t in cluster.get("tests", []) if t and t.lower() in issue_norm), None)
+
+ # 2) Error signature similarity / containment against the whole issue.
+ sig_ratio = _token_overlap(sig_norm, issue_norm)
+ if sig_norm and sig_norm in issue_norm:
+ sig_ratio = max(sig_ratio, 1.0)
+
+ # 3) Fuzzy similarity between signature and issue title. Only a near-identical
+ # title counts toward the score; a moderate ratio is shared boilerplate.
+ title_ratio = SequenceMatcher(None, sig_norm, title_norm).ratio() if sig_norm else 0.0
+
+ # A verbatim test-name match is strong but not conclusive: the same test can
+ # fail for a different reason, so require the error signature to corroborate it.
+ test_score = (0.5 + 0.5 * sig_ratio) if test_hit else 0.0
+ score = max(
+ test_score,
+ sig_ratio,
+ title_ratio if title_ratio >= _TITLE_STRONG else 0.0,
+ )
+ matched_by = {
+ "test": round(test_score, 2),
+ "signature": round(sig_ratio, 2),
+ "title": round(title_ratio, 2),
+ }
+ return score, matched_by
+
+
+def match_clusters(clusters: list[dict], issues: list[dict]) -> None:
+ """Annotate each cluster in place with its best known-issue match (if any)."""
+ for cluster in clusters:
+ best_score = 0.0
+ best_issue = None
+ best_matched_by: dict = {}
+ for issue in issues:
+ score, matched_by = score_issue(cluster, issue)
+ if score > best_score:
+ best_score = score
+ best_issue = issue
+ best_matched_by = matched_by
+ if best_issue is not None and best_score >= _MATCH_THRESHOLD:
+ cluster["known"] = True
+ cluster["issue"] = {
+ "number": best_issue.get("number"),
+ "title": best_issue.get("title"),
+ "url": best_issue.get("html_url"),
+ "matched_by": best_matched_by,
+ "score": round(best_score, 3),
+ }
+ else:
+ cluster["known"] = False
+ cluster["issue"] = None
+
+
+def main():
+ parser = argparse.ArgumentParser(description="Match failure clusters against known-issue tickets")
+ parser.add_argument("--clusters-json", required=True, help="Clusters JSON from analyze_failures.py")
+ parser.add_argument("--output", required=True, help="Annotated clusters JSON output path")
+ parser.add_argument("--repo", default=os.environ.get("REPO_PATH", ""), help="owner/name")
+ parser.add_argument("--label", default="CI-known-issue", help="Known-issue label")
+ parser.add_argument("--token-env", default="AUTO_ROUND_BOT_TOKEN", help="Env var holding the GitHub token")
+ args = parser.parse_args()
+
+ with open(args.clusters_json, encoding="utf-8") as f:
+ data = json.load(f)
+ clusters = data.get("clusters", [])
+
+ token = os.environ.get(args.token_env, "")
+ issues: list[dict] = []
+ if args.repo:
+ try:
+ issues = fetch_known_issues(args.repo, args.label, token)
+ print(f"Fetched {len(issues)} '{args.label}' issue(s) from {args.repo}", file=sys.stderr)
+ except (HTTPError, URLError, ValueError, json.JSONDecodeError) as e:
+ print(f"Warning: could not fetch known issues ({e}); treating all as unknown", file=sys.stderr)
+ else:
+ print("Warning: no repo provided; treating all clusters as unknown", file=sys.stderr)
+
+ match_clusters(clusters, issues)
+
+ known = sum(1 for c in clusters if c.get("known"))
+ data["known_count"] = known
+ data["unknown_count"] = len(clusters) - known
+ with open(args.output, "w", encoding="utf-8") as f:
+ json.dump(data, f, indent=2)
+ print(f"Matched {known}/{len(clusters)} cluster(s) to known issues", file=sys.stderr)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/.azure-pipelines/scripts/ai_analysis/merge_reports.py b/.azure-pipelines/scripts/ai_analysis/merge_reports.py
new file mode 100644
index 000000000..d634eae2c
--- /dev/null
+++ b/.azure-pipelines/scripts/ai_analysis/merge_reports.py
@@ -0,0 +1,183 @@
+"""Merge clustered failures and AI analyses into one collapsible markdown report.
+
+The report is designed to be pasted as a PR comment after human review. Most
+content is wrapped in ``<details>`` blocks to keep it compact; only the header
+summary is always visible.
+"""
+
+import argparse
+import json
+import os
+
+
+def _inline(text: str, limit: int = 160) -> str:
+ """Flatten a signature to a single, table-safe line."""
+ text = (text or "").replace("|", "\\|").replace("\n", " ").strip()
+ return text[:limit]
+
+
+def _code_span(text: str) -> str:
+ """Wrap text in an inline code span without breaking embedded backticks."""
+ text = (text or "").replace("\n", " ").strip()
+ if "`" not in text:
+ return f"`{text}`"
+ longest_run = 0
+ current_run = 0
+ for character in text:
+ if character == "`":
+ current_run += 1
+ longest_run = max(longest_run, current_run)
+ else:
+ current_run = 0
+ fence = "`" * (longest_run + 1)
+ return f"{fence} {text} {fence}"
+
+
+def _load(path: str) -> dict:
+ if not path or not os.path.isfile(path):
+ return {}
+ with open(path, encoding="utf-8") as f:
+ return json.load(f)
+
+
+def build_header(clusters_data: dict, args) -> list[str]:
+ total_failed = clusters_data.get("total_failed_tests", 0)
+ cluster_count = clusters_data.get("cluster_count", len(clusters_data.get("clusters", [])))
+ known = clusters_data.get("known_count", 0)
+ unknown = clusters_data.get("unknown_count", cluster_count - known)
+ lines = [
+ f"# CI Failure Analysis Report ({args.pipeline or 'N/A'})",
+ "",
+ f"- **Commit:** {args.commit or 'N/A'}",
+ ]
+ if args.run_url:
+ lines.append(f"- **Run:** {args.run_url}")
+ lines.append(
+ f"- **Totals:** {total_failed} failed test(s), {cluster_count} cluster(s) " f"({known} known / {unknown} new)"
+ )
+ lines.append("")
+ return lines
+
+
+def build_known_section(clusters: list[dict]) -> list[str]:
+ known = [c for c in clusters if c.get("known") and c.get("issue")]
+ if not known:
+ return []
+ lines = [
+ f"## Known Issues ({len(known)})",
+ "",
+ "<details><summary>🔍 Summary table</summary>",
+ "",
+ "| Cluster | Issue | Score | Occurrences | Sample log |",
+ "| --- | --- | --- | --- | --- |",
+ ]
+ for c in known:
+ issue = c["issue"]
+ num = issue.get("number")
+ url = issue.get("url") or ""
+ issue_cell = f"[#{num}]({url})" if url else f"#{num}"
+ score = f"{issue.get('score', 0):.2f}"
+ sample_log = ", ".join(c.get("logs", [])) or "-"
+ lines.append(
+ f"| {c.get('id')} | {issue_cell} | {score} | " f"{c.get('occurrences')} | {_inline(sample_log, 60)} |"
+ )
+ lines.extend(["", "</details>", ""])
+ return lines
+
+
+def build_unknown_section(clusters: list[dict], analyses_by_id: dict) -> list[str]:
+ unknown = [c for c in clusters if not c.get("known")]
+ unknown.sort(key=lambda c: c.get("occurrences", 0), reverse=True)
+ if not unknown:
+ return []
+ lines = [f"## New Issues ({len(unknown)})", ""]
+
+ for rank, c in enumerate(unknown, start=1):
+ signature = _inline(c.get("signature", ""))
+ occ = c.get("occurrences", 0)
+ tests = c.get("tests", [])
+ displayed_tests = tests[:10]
+ if len(tests) > 10:
+ displayed_tests.append("...")
+ logs = c.get("logs", [])
+ displayed_logs = logs[:10]
+ if len(logs) > 10:
+ displayed_logs.append("...")
+ lines.append(f"### No.{rank} — {occ} occurrence(s)")
+ lines.append("")
+ lines.append(f"<details><summary> 🔍 {signature}</summary>")
+ lines.append("")
+ lines.append("### 📝 Basic info")
+ lines.append("")
+ lines.append(f"- **Affected tests ({len(tests)}):** {', '.join(displayed_tests) or '-'}")
+ lines.append(f"- **Logs ({len(logs)}):** {', '.join(displayed_logs) or '-'}")
+ lines.append("")
+ lines.append("### 🖥️ Log excerpt")
+ lines.append("")
+ lines.append("```")
+ lines.append((c.get("sample", "") or "").strip())
+ lines.append("```")
+ lines.append("")
+
+ analysis = analyses_by_id.get(c.get("id"))
+ if analysis:
+ lines.append("### ✨ AI analysis")
+ lines.append("")
+ lines.append(f"- **Category:** {analysis.get('category', 'Unknown')}")
+ lines.append(f"- **Confidence:** {analysis.get('confidence', 'low')}")
+ if analysis.get("root_cause"):
+ lines.append(f"- **Root cause:** {analysis['root_cause']}")
+ if analysis.get("suggested_fix"):
+ lines.append(f"- **Suggested fix:** {analysis['suggested_fix']}")
+ if analysis.get("patch"):
+ lines.append("")
+ lines.append("- **Patch:**")
+ lines.append(" ```diff")
+ lines.extend(f" {line}" for line in analysis["patch"].splitlines())
+ lines.append(" ```")
+ if analysis.get("directions"):
+ lines.append(f"- **Investigation directions (low confidence):** {analysis['directions']}")
+ lines.append("")
+ lines.append("</details>")
+ lines.append("")
+
+ lines.extend(
+ [
+ "",
+ "> **Notes**",
+ "> - Only top-3 issues receive AI analysis.",
+ '> - To request an additional fix from Copilot, use "Quote reply" on the PR comment and @mention Copilot.',
+ "",
+ ]
+ )
+ return lines
+
+
+def main():
+ parser = argparse.ArgumentParser(description="Merge clusters + AI analyses into a markdown report")
+ parser.add_argument("--clusters-json", required=True, help="Annotated clusters JSON")
+ parser.add_argument("--ai-json", help="AI analysis JSON")
+ parser.add_argument("--output", required=True, help="Output markdown path")
+ parser.add_argument("--pipeline", default=os.environ.get("BUILD_DEFINITIONNAME", ""))
+ parser.add_argument("--pr", default=os.environ.get("PR_NUMBER", ""))
+ parser.add_argument("--commit", default=os.environ.get("CURRENT_SHA", ""))
+ parser.add_argument("--run-url", default=os.environ.get("RUN_URL", ""))
+ args = parser.parse_args()
+
+ clusters_data = _load(args.clusters_json)
+ clusters = clusters_data.get("clusters", [])
+ ai_data = _load(args.ai_json)
+ analyses_by_id = {a.get("cluster_id"): a for a in ai_data.get("analyses", [])}
+
+ lines: list[str] = []
+ lines += build_header(clusters_data, args)
+ lines += build_known_section(clusters)
+ lines += build_unknown_section(clusters, analyses_by_id)
+
+ with open(args.output, "w", encoding="utf-8") as f:
+ f.write("\n".join(lines))
+ print(f"Wrote report to {args.output}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/.azure-pipelines/scripts/ai_analysis/post_pr_comment.py b/.azure-pipelines/scripts/ai_analysis/post_pr_comment.py
new file mode 100644
index 000000000..5ca4585d5
--- /dev/null
+++ b/.azure-pipelines/scripts/ai_analysis/post_pr_comment.py
@@ -0,0 +1,181 @@
+"""Post (or update) the AI failure-analysis report as a PR comment.
+
+Idempotent: the comment carries a hidden marker so reruns update the existing
+comment instead of spamming new ones. Fails soft — a posting problem never
+breaks the pipeline.
+
+With ``--hide`` the script instead minimizes the marker comment as OUTDATED,
+used when a rerun passes so the stale failure report collapses automatically.
+"""
+
+import argparse
+import json
+import os
+import sys
+from urllib.error import HTTPError, URLError
+from urllib.request import Request, urlopen
+
+_MARKER = "<!-- ai-failure-analysis -->"
+# GitHub caps issue-comment bodies at 65536 chars; leave room for the marker.
+_MAX_BODY = 65000
+
+
+def _build_marker(key: str) -> str:
+ """Return the hidden comment marker, scoped by an optional per-workflow key.
+
+ Distinct keys keep separate workflows from overwriting each other's comment.
+ """
+ key = (key or "").strip()
+ return f"<!-- ai-failure-analysis:{key} -->" if key else _MARKER
+
+
+def _api(url: str, token: str, method: str = "GET", payload: "dict | None" = None) -> "list | dict":
+ headers = {"Accept": "application/vnd.github+json"}
+ if token:
+ headers["Authorization"] = f"Bearer {token}"
+ data = json.dumps(payload).encode("utf-8") if payload is not None else None
+ if data is not None:
+ headers["Content-Type"] = "application/json"
+ req = Request(url, data=data, headers=headers, method=method)
+ with urlopen(req, timeout=20) as resp:
+ body = resp.read().decode("utf-8")
+ return json.loads(body) if body else {}
+
+
+def _find_existing(repo: str, pr: int, token: str, marker: str) -> "int | None":
+ c = _find_existing_comment(repo, pr, token, marker)
+ return c.get("id") if c else None
+
+
+def _find_existing_comment(repo: str, pr: int, token: str, marker: str) -> "dict | None":
+ page = 1
+ while True:
+ url = f"https://api.github.com/repos/{repo}/issues/{pr}/comments?per_page=100&page={page}"
+ comments = _api(url, token)
+ if not isinstance(comments, list) or not comments:
+ return None
+ for c in comments:
+ if marker in (c.get("body") or ""):
+ return c
+ if len(comments) < 100:
+ return None
+ page += 1
+
+
+def _graphql(token: str, query: str, variables: dict) -> dict:
+ req = Request(
+ "https://api.github.com/graphql",
+ data=json.dumps({"query": query, "variables": variables}).encode("utf-8"),
+ headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json"},
+ method="POST",
+ )
+ with urlopen(req, timeout=20) as resp:
+ return json.loads(resp.read().decode("utf-8"))
+
+
+def _minimize_comment(node_id: str, token: str, classifier: str = "OUTDATED") -> bool:
+ """Hide (minimize) a PR comment via GraphQL, marking it as e.g. OUTDATED."""
+ query = (
+ "mutation($id: ID!, $reason: ReportedContentClassifiers!) {"
+ " minimizeComment(input: {subjectId: $id, classifier: $reason}) {"
+ " minimizedComment { isMinimized minimizedReason }"
+ " }"
+ "}"
+ )
+ result = _graphql(token, query, {"id": node_id, "reason": classifier})
+ if result.get("errors"):
+ print(f"Warning: could not hide comment ({result['errors']}); non-blocking.", file=sys.stderr)
+ return False
+ return bool(result.get("data", {}).get("minimizeComment", {}).get("minimizedComment", {}).get("isMinimized"))
+
+
+def _hide(repo: str, pr: int, token: str, marker: str) -> None:
+ """Find the marker comment and hide it as OUTDATED (used when the run passes)."""
+ try:
+ existing = _find_existing_comment(repo, pr, token, marker)
+ except (HTTPError, URLError, ValueError) as e:
+ print(f"Warning: could not look up PR comment ({e}); non-blocking.", file=sys.stderr)
+ return
+ if not existing:
+ print("No existing analysis comment to hide.", file=sys.stderr)
+ return
+ node_id = existing.get("node_id")
+ if not node_id:
+ print("Existing comment has no node_id; cannot hide.", file=sys.stderr)
+ return
+ try:
+ if _minimize_comment(node_id, token):
+ print(f"Hid PR comment {existing.get('id')} as OUTDATED.", file=sys.stderr)
+ except (HTTPError, URLError, ValueError) as e:
+ print(f"Warning: could not hide PR comment ({e}); non-blocking.", file=sys.stderr)
+
+
+def main():
+ parser = argparse.ArgumentParser(description="Post the failure-analysis report to a PR comment")
+ parser.add_argument("--repo", default=os.environ.get("REPO_PATH", ""), help="owner/name")
+ parser.add_argument("--pr", default=os.environ.get("PR_NUMBER", ""), help="PR number")
+ parser.add_argument("--body-file", help="Markdown report to post (required unless --hide)")
+ parser.add_argument(
+ "--marker-key",
+ default=os.environ.get("AI_COMMENT_MARKER_KEY", ""),
+ help="Per-workflow key so different pipelines update separate PR comments",
+ )
+ parser.add_argument(
+ "--hide",
+ action="store_true",
+ help="Hide the existing analysis comment as OUTDATED instead of posting (use when the run passes)",
+ )
+ parser.add_argument("--token-env", default="AUTO_ROUND_BOT_TOKEN", help="Env var holding the GitHub token")
+ args = parser.parse_args()
+
+ marker = _build_marker(args.marker_key)
+ token = os.environ.get(args.token_env, "")
+ if not args.repo or "/" not in args.repo or not str(args.pr).strip():
+ print("No repo/PR number available; skipping PR comment.", file=sys.stderr)
+ return
+ if not token:
+ print(f"No token in ${args.token_env}; skipping PR comment.", file=sys.stderr)
+ return
+
+ if args.hide:
+ _hide(args.repo, int(args.pr), token, marker)
+ return
+
+ if not args.body_file:
+ print("No --body-file provided; nothing to post.", file=sys.stderr)
+ return
+ if not os.path.isfile(args.body_file):
+ print(f"Report file not found: {args.body_file}; skipping.", file=sys.stderr)
+ return
+
+ with open(args.body_file, encoding="utf-8") as f:
+ body = f.read()
+ if len(body) > _MAX_BODY:
+ body = body[:_MAX_BODY] + "\n\n_...truncated; see the pipeline artifact for the full report._"
+ body = f"{marker}\n{body}"
+
+ pr = int(args.pr)
+ try:
+ existing = _find_existing(args.repo, pr, token, marker)
+ if existing is not None:
+ _api(
+ f"https://api.github.com/repos/{args.repo}/issues/comments/{existing}",
+ token,
+ method="PATCH",
+ payload={"body": body},
+ )
+ print(f"Updated existing PR comment {existing}.", file=sys.stderr)
+ else:
+ _api(
+ f"https://api.github.com/repos/{args.repo}/issues/{pr}/comments",
+ token,
+ method="POST",
+ payload={"body": body},
+ )
+ print("Created new PR comment.", file=sys.stderr)
+ except (HTTPError, URLError, ValueError) as e:
+ print(f"Warning: could not post PR comment ({e}); non-blocking.", file=sys.stderr)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/.azure-pipelines/template/ai-stage-template.yml b/.azure-pipelines/template/ai-stage-template.yml
new file mode 100644
index 000000000..9a7da495e
--- /dev/null
+++ b/.azure-pipelines/template/ai-stage-template.yml
@@ -0,0 +1,194 @@
+# Reusable AI-failure-analysis stages shared across pipelines (CPU, XPU, ...).
+# Split into two stages because a job condition cannot read a previous stage's
+# overall result (`stageDependencies.<stage>` only exposes per-job nodes, so
+# `stageDependencies.<stage>.result` is a dictionary miss -> Null). A stage-level
+# condition can read `dependencies.<stage>.result`, so we branch there instead.
+parameters:
+ - name: dependsOn
+ type: string
+ default: "Unit_test"
+ - name: stageName
+ type: string
+ default: "AI_Analysis"
+ - name: enableAiAnalysis
+ type: boolean
+ default: true
+ - name: postComment
+ type: boolean
+ default: true
+ - name: topUnknown
+ type: number
+ default: 3
+ - name: knownIssueLabel
+ type: string
+ default: "CI-known-issue"
+
+stages:
+ - stage: ${{ parameters.stageName }}
+ displayName: "AI Failure Analysis"
+ pool:
+ vmImage: "ubuntu-latest"
+ dependsOn: ${{ parameters.dependsOn }}
+ condition: and(eq(dependencies.${{ parameters.dependsOn }}.result, 'Failed'), eq(variables['System.StageAttempt'], '1'))
+ variables:
+ - group: "auto-round-bot-secrets"
+ jobs:
+ - job: AnalyzeFailures
+ displayName: "Analyze failed unit tests"
+ variables:
+ base: $(Build.SourcesDirectory)
+ output_dir: $(Build.SourcesDirectory)/ai_analysis_results
+ scripts: $(Build.SourcesDirectory)/.azure-pipelines/scripts/ai_analysis
+ steps:
+ - task: UsePythonVersion@0
+ inputs:
+ versionSpec: "3.12"
+ displayName: "Use Python 3.12"
+
+ - task: DownloadPipelineArtifact@2
+ inputs:
+ artifact:
+ patterns: "ut-*-1/unittest_*"
+ path: $(output_dir)/failed_logs
+ displayName: "Download failed-log artifacts"
+
+ - task: Bash@3
+ displayName: "Cluster failures"
+ inputs:
+ targetType: "inline"
+ script: |
+ set -eo pipefail
+ python "$(scripts)/analyze_failures.py" \
+ --input-dir "$(output_dir)/failed_logs" \
+ --clusters-json "$(output_dir)/clusters.json" \
+ --decision-log "$(output_dir)/decision_log.md"
+
+ echo "Cluster analysis completed. Results are in $(output_dir)/clusters.json"
+ cat "$(output_dir)/clusters.json"
+
+ - task: Bash@3
+ displayName: "Match known issues"
+ env:
+ AUTO_ROUND_BOT_TOKEN: $(AUTO_ROUND_BOT_TOKEN)
+ inputs:
+ targetType: "inline"
+ script: |
+ set -eo pipefail
+ repo_path="${BUILD_REPOSITORY_URI#https://github.com/}"
+ repo_path="${repo_path%.git}"
+
+ python "$(scripts)/match_known_issues.py" \
+ --clusters-json "$(output_dir)/clusters.json" \
+ --output "$(output_dir)/annotated.json" \
+ --repo "${repo_path}" \
+ --label "${{ parameters.knownIssueLabel }}"
+
+ echo "Matching known issues completed. Results are in $(output_dir)/annotated.json"
+ cat "$(output_dir)/annotated.json"
+
+ - task: Bash@3
+ displayName: "AI analysis of top unknown failures"
+ continueOnError: true
+ env:
+ AI_TOKEN: $(AI_TOKEN)
+ inputs:
+ targetType: "inline"
+ script: |
+ set -uo pipefail
+ if [ "${{ parameters.enableAiAnalysis }}" = "True" ]; then
+ echo "Installing Copilot CLI for AI analysis..."
+ curl -fsSL https://gh.io/copilot-install | bash
+ export PATH="$HOME/.local/bin:$PATH"
+ fi
+ python "$(scripts)/ai_analyze.py" \
+ --clusters-json "$(output_dir)/annotated.json" \
+ --output "$(output_dir)/ai.json" \
+ --pr-sha "$(Build.SourceVersion)" \
+ --top ${{ parameters.topUnknown }} \
+ --enable-ai-analysis ${{ parameters.enableAiAnalysis }} \
+ --backend copilot \
+ --model gpt-5.6-luna \
+ --reasoning-effort max \
+ --trace-file "$(output_dir)/ai_trace.jsonl" \
+ --project-root "$(base)" \
+ --log-dir "$(output_dir)/failed_logs"
+
+ echo "AI analysis completed. Results are in $(output_dir)/ai.json"
+ cat "$(output_dir)/ai.json"
+
+ - task: Bash@3
+ displayName: "Merge markdown report"
+ env:
+ BUILD_DEFINITIONNAME: $(Build.DefinitionName)
+ PR_NUMBER: $(System.PullRequest.PullRequestNumber)
+ CURRENT_SHA: $(Build.SourceVersion)
+ RUN_URL: $(System.CollectionUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId)
+ inputs:
+ targetType: "inline"
+ script: |
+ set -eo pipefail
+ python "$(scripts)/merge_reports.py" \
+ --clusters-json "$(output_dir)/annotated.json" \
+ --ai-json "$(output_dir)/ai.json" \
+ --output "$(output_dir)/report.md"
+
+ - ${{ if eq(parameters.postComment, true) }}:
+ - task: Bash@3
+ displayName: "Post report as PR comment"
+ condition: eq(variables['Build.Reason'], 'PullRequest')
+ env:
+ AUTO_ROUND_BOT_TOKEN: $(AUTO_ROUND_BOT_TOKEN)
+ inputs:
+ targetType: "inline"
+ script: |
+ set -uo pipefail
+ repo_path="${BUILD_REPOSITORY_URI#https://github.com/}"
+ repo_path="${repo_path%.git}"
+ python "$(scripts)/post_pr_comment.py" \
+ --repo "${repo_path}" \
+ --pr "${SYSTEM_PULLREQUEST_PULLREQUESTNUMBER:-}" \
+ --marker-key "$(Build.DefinitionName)" \
+ --body-file "$(output_dir)/report.md"
+
+ - task: PublishPipelineArtifact@1
+ condition: always()
+ inputs:
+ targetPath: $(Build.SourcesDirectory)/ai_analysis_results
+ artifact: "ai-failure-analysis"
+ publishLocation: "pipeline"
+ displayName: "Publish AI failure-analysis report"
+
+ - stage: ${{ parameters.stageName }}_HideOutdated
+ displayName: "Hide AI Analysis Comments"
+ pool:
+ vmImage: "ubuntu-latest"
+ dependsOn: ${{ parameters.dependsOn }}
+ condition: in(dependencies.${{ parameters.dependsOn }}.result, 'Succeeded', 'SucceededWithIssues')
+ variables:
+ - group: "auto-round-bot-secrets"
+ jobs:
+ - job: HideComments
+ displayName: "Hide AI Analysis Comments"
+ steps:
+ - task: UsePythonVersion@0
+ inputs:
+ versionSpec: "3.12"
+ displayName: "Use Python 3.12"
+ - task: Bash@3
+ displayName: "Hide outdated AI analysis comment"
+ condition: eq(variables['Build.Reason'], 'PullRequest')
+ env:
+ AUTO_ROUND_BOT_TOKEN: $(AUTO_ROUND_BOT_TOKEN)
+ inputs:
+ targetType: "inline"
+ script: |
+ set -uo pipefail
+ base="$(Build.SourcesDirectory)"
+ scripts="${base}/.azure-pipelines/scripts/ai_analysis"
+ repo_path="${BUILD_REPOSITORY_URI#https://github.com/}"
+ repo_path="${repo_path%.git}"
+ python "${scripts}/post_pr_comment.py" \
+ --repo "${repo_path}" \
+ --pr "${SYSTEM_PULLREQUEST_PULLREQUESTNUMBER:-}" \
+ --marker-key "$(Build.DefinitionName)" \
+ --hide
diff --git a/.azure-pipelines/unit-test-xpu.yml b/.azure-pipelines/unit-test-xpu.yml
index 9a91505aa..e22d7192a 100644
--- a/.azure-pipelines/unit-test-xpu.yml
+++ b/.azure-pipelines/unit-test-xpu.yml
@@ -77,6 +77,12 @@ stages:
buildARKWheel: $(BUILD_ARK_WHEEL)
filterChangedTests: true
+ - template: template/ai-stage-template.yml
+ parameters:
+ dependsOn: "Unit_test"
+ enableAiAnalysis: true
+ postComment: true
+
- stage: Coverage
displayName: "Collect Coverage"
pool:
diff --git a/.azure-pipelines/unit-test.yml b/.azure-pipelines/unit-test.yml
index 6a21106f3..db9a2c05a 100644
--- a/.azure-pipelines/unit-test.yml
+++ b/.azure-pipelines/unit-test.yml
@@ -83,6 +83,12 @@ stages:
buildARKWheel: $(BUILD_ARK_WHEEL)
filterChangedTests: true
+ - template: template/ai-stage-template.yml
+ parameters:
+ dependsOn: "Unit_test"
+ enableAiAnalysis: true
+ postComment: true
+
- stage: Coverage
displayName: "Collect Coverage"
pool: