PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Implement AI-Powered CI Failure Analysis and Fix Recommendations (#2277)
chen, suyue18486c3902ae
diff --git a/.azure-pipelines/scripts/ai_analysis/ai_analyze.py b/.azure-pipelines/scripts/ai_analysis/ai_analyze.pynew file mode 100644index 000000000..a261f4a89--- /dev/null+++ b/.azure-pipelines/scripts/ai_analysis/ai_analyze.py@@ -0,0 +1,409 @@+"""AI analysis of the most frequent unknown CI failures.++Ranks the *unknown* failure clusters by occurrence, takes the top N (default 3)+and asks an AI backend to classify each failure (Code Regression / Environment /+Dependency / Test Case) and propose a fix. When the model is not confident it is+instructed to omit the fix and instead give investigation directions.++The backend is intentionally pluggable behind ``call_backend`` so the current+GitHub Copilot CLI implementation can be swapped later. With ``--backend none``+the script emits the structured prompts only (no model call), which supports the+"generate data, human runs Copilot Chat" fallback.+"""++import argparse+import datetime+import json+import os+import subprocess+import sys++_PROMPT_TEMPLATE = """You are analyzing a failing CI unit test for the AutoRound project (post-training \+quantization for LLMs/VLMs). Determine the failure type and, only when confident, propose a fix.++Respond with ONLY a single JSON object, no prose, using exactly these keys:+{{+ "category": "Code Regression | Environment Issue | Dependency Issue | Test Case Issue | Unknown",+ "confidence": "high | medium | low",+ "root_cause": "concise explanation of the likely cause",+ "suggested_fix": "concrete fix description, or empty string if confidence is low",+ "patch": "a unified diff if you are confident, otherwise empty string",+ "directions": "if confidence is low, list concrete investigation directions; else empty string"+}}++Guidance:+- Use the PR changed files to judge whether this is a code regression introduced by the change.+- If the error looks environmental/dependency-related and unrelated to the diff, say so.+- If uncertain, set confidence to "low", leave suggested_fix and patch empty, and fill directions.+- For an ``AssertionError`` whose log excerpt already contains pytest's ``Full diff`` (the ``-``/``+``+ lines), that diff is usually authoritative: cross-check it against the PR changed files below and+ conclude directly.+- The ``PR changed files`` section below lists every file this PR touched. Inspect the relevant ones+ yourself with your ``read``/``rg`` tools on the checkout at ``{project_root}`` (and the complete raw+ failure logs under ``{log_path}``) to judge whether the change caused the failure.++## Project source root+{project_root}++## Full failure logs directory+{log_path}++## Error signature+{signature}++## Affected tests ({test_count})+{tests}++## Log excerpt+{excerpt}++## PR changed files+{pr_files}+"""+++def call_backend(+ prompt: str, backend: str, timeout: int, model: str, reasoning_effort: str, trace_file: str, cluster_id+) -> str:+ """Dispatch a prompt to the selected AI backend and return raw text output.++ Swap point: replace the ``copilot`` branch to change the inference provider.+ """+ if backend == "none":+ return ""+ if backend == "copilot":+ return _call_copilot_cli(prompt, timeout, model, reasoning_effort, trace_file, cluster_id)+ raise ValueError(f"unknown backend: {backend}")+++def _append_trace(trace_file: str, entry: dict) -> None:+ """Append one AI-call record (JSONL) so the analysis process can be audited."""+ if not trace_file:+ return+ try:+ with open(trace_file, "a", encoding="utf-8") as f:+ f.write(json.dumps(entry, ensure_ascii=False) + "\n")+ except OSError as e:+ print(f"Warning: could not write trace file: {e}", file=sys.stderr)+++def _parse_copilot_json(stdout: str) -> dict:+ """Parse the Copilot ``--output-format json`` JSONL stream.++ A single stream carries everything we need: the assistant answer, the model+ actually used, and the AI-credit / premium-request usage — so there is no need+ to shell out for stderr credits or dig through debug logs separately.+ """+ answer = ""+ model = ""+ nano_aiu = None+ premium_requests = None+ for line in stdout.splitlines():+ line = line.strip()+ if not line:+ continue+ try:+ event = json.loads(line)+ except json.JSONDecodeError:+ continue+ etype = event.get("type")+ data = event.get("data", {})+ if etype == "assistant.message":+ model = data.get("model") or model+ if data.get("content"):+ answer = data["content"] # keep the last full assistant message+ elif etype == "model.call_start":+ model = data.get("model") or model+ elif etype == "session.usage_checkpoint":+ nano_aiu = data.get("totalNanoAiu", nano_aiu)+ premium_requests = data.get("totalPremiumRequests", premium_requests)+ elif etype == "result":+ usage = event.get("usage", {})+ premium_requests = usage.get("premiumRequests", premium_requests)+ return {+ "answer": answer,+ "model": model,+ "ai_credits": round(nano_aiu / 1e9, 6) if nano_aiu is not None else None,+ "premium_requests": premium_requests,+ }+++def _call_copilot_cli(prompt: str, timeout: int, model: str, reasoning_effort: str, trace_file: str, cluster_id) -> str:+ argv = [+ "copilot",+ "-p",+ prompt,+ "--allow-tool=shell(python:*)",+ "--allow-tool=shell(rg:*)",+ "--allow-tool=read",+ "--allow-tool=write",+ # The PR diff is pre-computed into the prompt, so no GitHub API access is needed. Dropping the+ # built-in GitHub MCP server trims the tool schema re-sent on every turn (~11k tokens -> far+ # less), which is the dominant per-call cost in this multi-turn loop.+ "--disable-builtin-mcps",+ "--no-ask-user",+ "--output-format",+ "json",+ ]+ if model:+ argv += ["--model", model]+ if reasoning_effort:+ argv += ["--reasoning-effort", reasoning_effort]+ env = dict(os.environ)+ # Let the Copilot CLI authenticate with the dedicated token.+ ai_token = os.environ.get("AI_TOKEN", "")+ if ai_token:+ env.setdefault("GITHUB_TOKEN", ai_token)+ env.setdefault("GH_TOKEN", ai_token)+ started = datetime.datetime.now(datetime.timezone.utc)+ returncode = None+ stderr = ""+ stdout = ""+ try:+ proc = subprocess.run(argv, capture_output=True, text=True, timeout=timeout, env=env, check=False)+ returncode = proc.returncode+ stderr = proc.stderr.strip()+ stdout = proc.stdout+ if returncode != 0:+ print(f"Warning: Copilot CLI exited {returncode}: {stderr}", file=sys.stderr)+ except (OSError, subprocess.TimeoutExpired) as e:+ print(f"Warning: Copilot CLI call failed: {e}", file=sys.stderr)+ stderr = str(e)+ ended = datetime.datetime.now(datetime.timezone.utc)+ parsed = _parse_copilot_json(stdout)+ raw = parsed["answer"]+ _append_trace(+ trace_file,+ {+ "cluster_id": cluster_id,+ "timestamp": started.isoformat(),+ "duration_s": round((ended - started).total_seconds(), 3),+ "model": parsed["model"] or model or "(unknown)",+ "ai_credits": parsed["ai_credits"],+ "premium_requests": parsed["premium_requests"],+ "returncode": returncode,+ "prompt": prompt,+ "response": raw,+ "transcript": stdout, # raw JSONL stream: full step-by-step execution trace+ },+ )+ print(+ f"AI call cluster={cluster_id} "+ f"model={parsed['model'] or model or '(unknown)'} "+ f"duration_s={round((ended - started).total_seconds(), 3)} "+ f"ai_credits={parsed['ai_credits']} "+ f"premium_requests={parsed['premium_requests']} "+ f"returncode={returncode}",+ file=sys.stderr,+ )+ return raw+++def parse_model_json(text: str) -> "dict | None":+ """Extract the first JSON object from model output; tolerant of surrounding prose."""+ if not text:+ return None+ start = text.find("{")+ end = text.rfind("}")+ if start == -1 or end == -1 or end <= start:+ return None+ try:+ return json.loads(text[start : end + 1])+ except json.JSONDecodeError:+ return None+++def _run_git(project_root: str, args: "list[str]", timeout: int = 120) -> "subprocess.CompletedProcess | None":+ """Run a git command in ``project_root``; return the completed process or None on failure."""+ try:+ return subprocess.run(+ ["git", "-C", project_root or ".", *args],+ capture_output=True,+ text=True,+ timeout=timeout,+ check=False,+ )+ except (OSError, subprocess.TimeoutExpired) as e:+ print(f"Warning: git {' '.join(args)} failed: {e}", file=sys.stderr)+ return None+++_unshallowed = False+++def _ensure_full_history(project_root: str) -> None:+ """Unshallow the checkout once so merge-base / parent commits are available.++ CI checkouts are usually shallow, which makes ``git diff <sha>^1...<sha>^2`` fail. Doing it here,+ once, keeps that cost out of the analysis loop.+ """+ global _unshallowed+ if _unshallowed:+ return+ _unshallowed = True+ res = _run_git(project_root, ["rev-parse", "--is-shallow-repository"])+ if res is not None and res.stdout.strip() == "true":+ print("Unshallowing repository for PR diff computation...", file=sys.stderr)+ _run_git(project_root, ["fetch", "--quiet", "--unshallow", "origin"], timeout=600)+++def compute_pr_changed_files(pr_sha: str, project_root: str, max_chars: int) -> str:+ """Return the ``--name-status`` list of files this PR changed (same for every cluster).++ The three-dot range ``sha^1...sha^2`` is the merge commit's change (merge-base..PR-head). We hand+ the model only the file list and let it read the actual source itself, so the PR context stays+ small and cluster-independent. Returns an empty string when it cannot be produced (e.g. no SHA).+ """+ if not pr_sha:+ return ""+ _ensure_full_history(project_root)+ argv = ["diff", f"{pr_sha}^1...{pr_sha}^2", "--name-status"]+ res = _run_git(project_root, argv)+ if res is None or res.returncode != 0:+ # Retry once after unshallowing in case the first attempt raced a shallow checkout.+ _run_git(project_root, ["fetch", "--quiet", "--unshallow", "origin"], timeout=600)+ res = _run_git(project_root, argv)+ if res is None or res.returncode != 0:+ return ""+ files = res.stdout.strip()+ if len(files) > max_chars:+ files = files[:max_chars] + "\n... (list truncated) ..."+ return files+++def _save_pr_changed_files(output_path: str, pr_files: str) -> None:+ """Persist the PR changed-file list next to the analysis output for easy log inspection."""+ if not pr_files:+ return+ out_dir = os.path.dirname(os.path.abspath(output_path)) if output_path else "."+ path = os.path.join(out_dir, "pr_changed_files.txt")+ try:+ with open(path, "w", encoding="utf-8") as f:+ f.write(pr_files + "\n")+ except OSError as e:+ print(f"Warning: could not write PR changed-file list {path}: {e}", file=sys.stderr)+++def build_prompt(cluster: dict, pr_files: str, max_excerpt: int, project_root: str, log_path: str) -> str:+ return _PROMPT_TEMPLATE.format(+ signature=cluster.get("signature", ""),+ test_count=len(cluster.get("tests", [])),+ tests=", ".join(cluster.get("tests", [])) or "(unknown)",+ excerpt=(cluster.get("sample", "") or "")[:max_excerpt],+ pr_files=pr_files or "(empty; inspect the checkout with your read/rg tools if needed)",+ project_root=project_root or "(not provided)",+ log_path=log_path or "(not provided)",+ )+++def analyze(cluster: dict, pr_files: str, args) -> dict:+ prompt = build_prompt(cluster, pr_files, args.max_excerpt_chars, args.project_root, args.log_dir)+ raw = call_backend(+ prompt,+ args.backend,+ args.timeout,+ args.model,+ args.reasoning_effort,+ args.trace_file,+ cluster.get("id"),+ )+ parsed = parse_model_json(raw)+ result = {+ "cluster_id": cluster.get("id"),+ "signature": cluster.get("signature"),+ "occurrences": cluster.get("occurrences"),+ "tests": cluster.get("tests", []),+ }+ if parsed:+ # Low-confidence answers must not carry a fix/patch (requirement 3.3).+ if str(parsed.get("confidence", "")).lower() == "low":+ parsed["suggested_fix"] = ""+ parsed["patch"] = ""+ result.update(+ {+ "category": parsed.get("category", "Unknown"),+ "confidence": parsed.get("confidence", "low"),+ "root_cause": parsed.get("root_cause", ""),+ "suggested_fix": parsed.get("suggested_fix", ""),+ "patch": parsed.get("patch", ""),+ "directions": parsed.get("directions", ""),+ }+ )+ else:+ result.update(+ {+ "category": "Unknown",+ "confidence": "low",+ "root_cause": "",+ "suggested_fix": "",+ "patch": "",+ "directions": "",+ "prompt": prompt if args.backend == "none" else "",+ "raw": raw,+ }+ )+ return result+++def main():+ parser = argparse.ArgumentParser(description="AI-analyze the top unknown failure clusters")+ parser.add_argument("--clusters-json", required=True, help="Annotated clusters JSON")+ parser.add_argument("--output", required=True, help="AI analysis JSON output path")+ parser.add_argument(+ "--pr-sha",+ default="",+ help="Merge commit SHA of the PR build; used to pre-compute the PR diff embedded in prompts",+ )+ parser.add_argument("--top", type=int, default=3, help="Number of top unknown clusters to analyze")+ parser.add_argument("--backend", choices=["copilot", "none"], default="none", help="Inference backend")+ parser.add_argument(+ "--enable-ai-analysis",+ default="true",+ help="Set to a falsy value (false/0/no) to force the no-model 'none' backend",+ )+ parser.add_argument("--timeout", type=int, default=300, help="Per-call timeout in seconds")+ parser.add_argument("--model", default="", help="Copilot model to use; empty uses the CLI default")+ parser.add_argument(+ "--reasoning-effort",+ choices=["none", "minimal", "low", "medium", "high", "xhigh", "max"],+ default="",+ help="Copilot reasoning effort; empty uses the CLI default",+ )+ parser.add_argument("--trace-file", default="", help="JSONL file logging each AI call for auditing")+ parser.add_argument("--project-root", default="", help="Repository checkout root the AI may inspect")+ parser.add_argument("--log-dir", default="", help="Directory holding the full raw failure logs")+ parser.add_argument("--max-excerpt-chars", type=int, default=4000)+ parser.add_argument(+ "--max-diff-chars",+ type=int,+ default=20000,+ help="Max characters of the PR changed-file list embedded in each prompt",+ )+ args = parser.parse_args()++ # The AI switch overrides the backend so a single pipeline step can toggle it.+ if str(args.enable_ai_analysis).strip().lower() not in ("1", "true", "yes", "on"):+ args.backend = "none"++ with open(args.clusters_json, encoding="utf-8") as f:+ data = json.load(f)++ # The PR changed-file list is the same for every cluster, so compute and persist it once; the+ # model reads the actual source itself instead of running git inside a shallow checkout.+ pr_sha = (args.pr_sha or "").strip()+ pr_files = compute_pr_changed_files(pr_sha, args.project_root, args.max_diff_chars)+ _save_pr_changed_files(args.output, pr_files)++ unknown = [c for c in data.get("clusters", []) if not c.get("known")]+ unknown.sort(key=lambda c: c.get("occurrences", 0), reverse=True)+ selected = unknown[: args.top]++ analyses = [analyze(cluster, pr_files, args) for cluster in selected]++ with open(args.output, "w", encoding="utf-8") as f:+ json.dump({"backend": args.backend, "analyses": analyses}, f, indent=2)+ print(f"Analyzed {len(analyses)} unknown cluster(s) with backend ' {args.backend}'", file=sys.stderr)+++if __name__ == "__main__":+ main()diff --git a/.azure-pipelines/scripts/ai_analysis/analyze_failures.py b/.azure-pipelines/scripts/ai_analysis/analyze_failures.pynew file mode 100644index 000000000..a924b856b--- /dev/null+++ b/.azure-pipelines/scripts/ai_analysis/analyze_failures.py@@ -0,0 +1,320 @@+"""Extract and cluster CI unit-test failures from downloaded failed logs.++Input is a directory holding the failed ``unittest_*.log`` files together with the+``failures_part*.json`` metadata emitted by ``collect_result.py``. The script scans+each failed log for the well-known failure markers, derives a normalized error+signature per failing test, clusters similar signatures together and writes:++* ``clusters.json`` — structured clusters ranked by occurrence (machine-readable).+* ``decision_log.md`` — an audit trail of how each cluster was formed (matched+ keywords, member tests, merged raw log excerpts). Kept separate from the final+ report on purpose; it is intentionally verbose.+"""++import argparse+import json+import re+import sys+from dataclasses import dataclass, field+from difflib import SequenceMatcher+from pathlib import Path++# Markers we scan for; crash markers are matched case-insensitively.+KEYWORDS = ("== FAILURES ==", "== ERRORS ==", "Traceback", "core dumped", "Killed")+CRASH_MARKERS = ("core dumped", "killed", "segmentation fault", "aborted")++# pytest per-test header of underscores wrapping a test id, e.g. "____ test_foo ____".+# A very long test name collapses the padding to a single underscore per side+# ("_ very.long.name _"), so allow one-or-more underscores instead of two-or-more.+_TEST_HEADER = re.compile(r"^_+ (.+?) _+$")+# A trailing exception summary line, e.g. "ValueError: shapes do not match".+_EXC_SUMMARY = re.compile(r"^([A-Za-z_][\w.]*(?:Error|Exception|Warning|Failure|Exit)):?\s")+# Similarity threshold above which two normalized signatures share a cluster.+_SIMILARITY = 0.90+++@dataclass+class ErrorRecord:+ """A single extracted failure occurrence."""++ test: str+ log: str+ signature: str+ normalized: str+ keywords: list[str]+ excerpt: str+++@dataclass+class Cluster:+ """A group of similar error records."""++ id: int+ signature: str+ normalized: str+ keywords: set[str] = field(default_factory=set)+ tests: list[str] = field(default_factory=list)+ logs: set[str] = field(default_factory=set)+ records: list[ErrorRecord] = field(default_factory=list)++ @property+ def occurrences(self) -> int:+ return len(self.records)+++def normalize(text: str) -> str:+ """Strip run-specific noise so equivalent errors normalize to the same string."""+ text = text.strip().lower()+ text = re.sub(r"0x[0-9a-f]+", "0xADDR", text)+ text = re.sub(r"/\S+/", "", text) # drop directory paths, keep basenames+ text = re.sub(r"line \d+", "line N", text)+ text = re.sub(r"\d+", "N", text)+ text = re.sub(r"\s+", " ", text)+ return text.strip()+++def _matched_keywords(text: str) -> list[str]:+ low = text.lower()+ return [kw for kw in KEYWORDS if kw.lower() in low]+++def _signature_from_segment(lines: list[str]) -> "str | None":+ """Pick the most specific error line from a FAILURES/traceback segment."""+ e_lines = [ln[1:].strip() for ln in lines if re.match(r"^E\s+\S", ln)]+ if e_lines:+ # Prefer the exception-summary line over pytest continuation lines+ # (e.g. "+ where 3 = quantize(...)").+ for el in e_lines:+ if _EXC_SUMMARY.match(el) or el.lower().startswith("assert"):+ return el+ return e_lines[0]+ for ln in reversed(lines):+ if _EXC_SUMMARY.match(ln.strip()):+ return ln.strip()+ return None+++def _crash_signature(lines: list[str]) -> "str | None":+ for ln in lines:+ low = ln.lower()+ if any(marker in low for marker in CRASH_MARKERS):+ return ln.strip()+ return None+++def _excerpt(lines: list[str], limit: int = 15) -> str:+ tail = [ln.rstrip() for ln in lines if ln.strip()][-limit:]+ return "\n".join(tail)+++# Trailing pytest noise that should not pollute a test's signature/excerpt.+_SEGMENT_CUT_MARKERS = (+ "captured log",+ "captured stdout",+ "captured stderr",+ "generated xml file",+ "slowest durations",+)+++def _trim_segment(lines: list[str]) -> list[str]:+ """Cut a per-test segment at the first trailing-noise marker."""+ for i, ln in enumerate(lines):+ low = ln.lower()+ if any(marker in low for marker in _SEGMENT_CUT_MARKERS):+ return lines[:i]+ return lines+++def extract_records(log_path: Path) -> list[ErrorRecord]:+ """Extract error records from a failed log.++ Every log passed here is already known to be a failure (only failed logs are+ staged), so a log that yields no pytest ``FAILURES`` segment still produces a+ single record keyed by the log name (crash/segfault or unrecognized output).+ """+ try:+ content = log_path.read_text(encoding="utf-8", errors="replace")+ except OSError as e:+ print(f"Warning: cannot read {log_path} - {e}", file=sys.stderr)+ return []++ lines = content.splitlines()+ # Drop the pytest tail (durations table + short summary); it is noise that+ # would otherwise bleed into the last test's excerpt/signature.+ for i, ln in enumerate(lines):+ if "slowest durations" in ln.lower():+ lines = lines[:i]+ break+ keywords = _matched_keywords(content)+ records: list[ErrorRecord] = []++ # Split the log into per-test segments using pytest FAILURES/ERRORS headers.+ segments: list[tuple[str, list[str]]] = []+ current_name = ""+ current: list[str] = []+ for ln in lines:+ header = _TEST_HEADER.match(ln)+ if header:+ if current:+ segments.append((current_name, current))+ current_name = header.group(1).strip()+ current = []+ else:+ current.append(ln)+ if current:+ segments.append((current_name, current))++ for name, seg in segments:+ if not name:+ continue+ seg = _trim_segment(seg)+ sig = _signature_from_segment(seg)+ if sig is None:+ continue+ records.append(+ ErrorRecord(+ test=name,+ log=log_path.name,+ signature=sig,+ normalized=normalize(sig),+ keywords=_matched_keywords("\n".join(seg)) or keywords,+ excerpt=_excerpt(seg),+ )+ )++ # Crash (segfault/OOM-kill) or unrecognized failures produce no FAILURES+ # section; emit a single record keyed by the log name so it is never dropped.+ if not records:+ crash_sig = _crash_signature(lines)+ if crash_sig is not None:+ signature, normalized = crash_sig, normalize(crash_sig)+ else:+ signature = "Unknown failure (no recognizable error signature)"+ normalized = "unknown failure"+ records.append(+ ErrorRecord(+ test=log_path.stem,+ log=log_path.name,+ signature=signature,+ normalized=normalized,+ keywords=keywords or _matched_keywords(signature),+ excerpt=_excerpt(lines),+ )+ )+ return records+++def cluster_records(records: list[ErrorRecord]) -> list[Cluster]:+ """Greedily group records whose normalized signatures are near-identical."""+ clusters: list[Cluster] = []+ for record in records:+ placed = False+ for cluster in clusters:+ if cluster.normalized == record.normalized or (+ SequenceMatcher(None, cluster.normalized, record.normalized).ratio() >= _SIMILARITY+ ):+ cluster.records.append(record)+ cluster.keywords.update(record.keywords)+ cluster.logs.add(record.log)+ if record.test not in cluster.tests:+ cluster.tests.append(record.test)+ placed = True+ break+ if not placed:+ clusters.append(+ Cluster(+ id=0,+ signature=record.signature,+ normalized=record.normalized,+ keywords=set(record.keywords),+ tests=[record.test],+ logs={record.log},+ records=[record],+ )+ )+ clusters.sort(key=lambda c: c.occurrences, reverse=True)+ for idx, cluster in enumerate(clusters, start=1):+ cluster.id = idx+ return clusters+++def write_clusters_json(clusters: list[Cluster], total_failed: int, output: Path) -> None:+ payload = {+ "total_failed_tests": total_failed,+ "cluster_count": len(clusters),+ "clusters": [+ {+ "id": c.id,+ "signature": c.signature,+ "normalized": c.normalized,+ "keywords": sorted(c.keywords),+ "occurrences": c.occurrences,+ "tests": c.tests,+ "logs": sorted(c.logs),+ "sample": c.records[0].excerpt if c.records else "",+ }+ for c in clusters+ ],+ }+ output.write_text(json.dumps(payload, indent=2), encoding="utf-8")+ print(f"Wrote {len(clusters)} cluster(s) to {output}", file=sys.stderr)+++def write_decision_log(clusters: list[Cluster], output: Path) -> None:+ lines = ["# CI Failure Clustering — Decision Log", ""]+ lines.append(f"Total clusters: {len(clusters)}")+ lines.append("")+ for c in clusters:+ lines.append(f"## Cluster {c.id} (occurrences: {c.occurrences})")+ lines.append("")+ lines.append(f"- Signature: `{c.signature}`")+ lines.append(f"- Normalized: `{c.normalized}`")+ lines.append(f"- Matched keywords: {', '.join(sorted(c.keywords)) or '(none)'}")+ lines.append(f"- Logs: {', '.join(sorted(c.logs))}")+ lines.append(f"- Affected tests ({len(c.tests)}): {', '.join(c.tests)}")+ lines.append("")+ lines.append("Merged log excerpts:")+ lines.append("")+ for record in c.records:+ lines.append(f"### {record.test} ({record.log})")+ lines.append("```")+ lines.append(record.excerpt)+ lines.append("```")+ lines.append("")+ output.write_text("\n".join(lines), encoding="utf-8")+ print(f"Wrote decision log to {output}", file=sys.stderr)+++def main():+ parser = argparse.ArgumentParser(description="Extract and cluster CI test failures")+ parser.add_argument("--input-dir", required=True, type=Path, help="Dir with the failed unittest_*.log files")+ parser.add_argument("--clusters-json", required=True, type=Path, help="Output clusters JSON path")+ parser.add_argument("--decision-log", required=True, type=Path, help="Output decision-log markdown path")+ args = parser.parse_args()++ if not args.input_dir.is_dir():+ print(f"Error: input dir not found: {args.input_dir}", file=sys.stderr)+ sys.exit(1)++ log_files = sorted(args.input_dir.rglob("unittest_*.log"))++ records: list[ErrorRecord] = []+ seen_logs: set[str] = set()+ for log_file in log_files:+ if log_file.name in seen_logs: # dedupe job-retry copies of the same log+ continue+ seen_logs.add(log_file.name)+ records.extend(extract_records(log_file))++ total_failed = len(records)+ clusters = cluster_records(records)++ args.clusters_json.parent.mkdir(parents=True, exist_ok=True)+ args.decision_log.parent.mkdir(parents=True, exist_ok=True)+ write_clusters_json(clusters, total_failed, args.clusters_json)+ write_decision_log(clusters, args.decision_log)+++if __name__ == "__main__":+ main()diff --git a/.azure-pipelines/scripts/ai_analysis/match_known_issues.py b/.azure-pipelines/scripts/ai_analysis/match_known_issues.pynew file mode 100644index 000000000..88638aec9--- /dev/null+++ b/.azure-pipelines/scripts/ai_analysis/match_known_issues.py@@ -0,0 +1,203 @@+"""Match clustered CI failures against known-issue tickets on GitHub.++Fetches open issues labeled ``CI-known-issue`` and fuzzily matches each failure+cluster against the issue title, full body and the failing test names. Clusters+with a confident match are flagged as *known*; the rest stay *unknown* and are+handed to the AI-analysis step.++Issue fetching uses the GitHub REST API with a token read from the environment+(``AUTO_ROUND_BOT_TOKEN`` by default). If no token is available or the request+fails, every cluster is conservatively treated as unknown.+"""++import argparse+import json+import os+import re+import sys+from difflib import SequenceMatcher+from urllib.error import HTTPError, URLError+from urllib.request import Request, urlopen++# Minimum combined score for a cluster to be considered a known issue.+_MATCH_THRESHOLD = 0.7+# Title fuzzy-ratio must be near-identical to stand alone; generic assertion+# boilerplate ("assert ... == 'test'") otherwise inflates unrelated matches.+_TITLE_STRONG = 0.85+# Generic error/boilerplate tokens carry no discriminative signal, so they are+# excluded from signature overlap; otherwise every assertion failure looks alike.+_STOPWORDS = frozenset(+ {+ "assert",+ "asserts",+ "assertion",+ "assertionerror",+ "error",+ "errors",+ "exception",+ "failed",+ "failure",+ "warning",+ "index",+ "diff",+ "full",+ "test",+ "tests",+ "none",+ "true",+ "false",+ "self",+ "the",+ "and",+ "not",+ "for",+ "with",+ "value",+ "expected",+ "actual",+ }+)+++def normalize(text: str) -> str:+ text = text.lower()+ text = re.sub(r"0x[0-9a-f]+", "0xADDR", text)+ text = re.sub(r"\d+", "N", text)+ text = re.sub(r"\s+", " ", text)+ return text.strip()+++def fetch_known_issues(repo: str, label: str, token: str) -> list[dict]:+ """Return open issues carrying ``label`` (title + body), following pagination."""+ if "/" not in repo:+ raise ValueError(f"repo must be 'owner/name', got: {repo}")+ headers = {"Accept": "application/vnd.github+json"}+ if token:+ headers["Authorization"] = f"Bearer {token}"++ issues: list[dict] = []+ page = 1+ while True:+ url = f"https://api.github.com/repos/{repo}/issues" f"?labels={label}&state=open&per_page=100&page={page}"+ req = Request(url, headers=headers)+ with urlopen(req, timeout=20) as resp:+ batch = json.loads(resp.read().decode("utf-8"))+ # The issues endpoint also returns PRs; drop them.+ batch = [it for it in batch if "pull_request" not in it]+ issues.extend(batch)+ if len(batch) < 100:+ break+ page += 1+ return issues+++def _token_overlap(sig_norm: str, issue_norm: str) -> float:+ tokens = [t for t in re.split(r"\W+", sig_norm) if len(t) >= 3 and t not in _STOPWORDS]+ if not tokens:+ return 0.0+ hits = sum(1 for t in tokens if t in issue_norm)+ return hits / len(tokens)+++def score_issue(cluster: dict, issue: dict) -> "tuple[float, dict]":+ """Score how well a cluster matches an issue; return (score, matched_by).++ ``matched_by`` breaks the decision into the three underlying signals so the+ report can show them: ``test`` (test-name corroborated by the error),+ ``signature`` (error-keyword overlap) and ``title`` (fuzzy title similarity).+ """+ sig_norm = normalize(cluster.get("signature", ""))+ title_norm = normalize(issue.get("title", ""))+ body_norm = normalize(issue.get("body") or "")+ issue_norm = f"{title_norm}\n{body_norm}"++ # 1) Failing test name mentioned verbatim in the issue — strongest signal.+ test_hit = next((t for t in cluster.get("tests", []) if t and t.lower() in issue_norm), None)++ # 2) Error signature similarity / containment against the whole issue.+ sig_ratio = _token_overlap(sig_norm, issue_norm)+ if sig_norm and sig_norm in issue_norm:+ sig_ratio = max(sig_ratio, 1.0)++ # 3) Fuzzy similarity between signature and issue title. Only a near-identical+ # title counts toward the score; a moderate ratio is shared boilerplate.+ title_ratio = SequenceMatcher(None, sig_norm, title_norm).ratio() if sig_norm else 0.0++ # A verbatim test-name match is strong but not conclusive: the same test can+ # fail for a different reason, so require the error signature to corroborate it.+ test_score = (0.5 + 0.5 * sig_ratio) if test_hit else 0.0+ score = max(+ test_score,+ sig_ratio,+ title_ratio if title_ratio >= _TITLE_STRONG else 0.0,+ )+ matched_by = {+ "test": round(test_score, 2),+ "signature": round(sig_ratio, 2),+ "title": round(title_ratio, 2),+ }+ return score, matched_by+++def match_clusters(clusters: list[dict], issues: list[dict]) -> None:+ """Annotate each cluster in place with its best known-issue match (if any)."""+ for cluster in clusters:+ best_score = 0.0+ best_issue = None+ best_matched_by: dict = {}+ for issue in issues:+ score, matched_by = score_issue(cluster, issue)+ if score > best_score:+ best_score = score+ best_issue = issue+ best_matched_by = matched_by+ if best_issue is not None and best_score >= _MATCH_THRESHOLD:+ cluster["known"] = True+ cluster["issue"] = {+ "number": best_issue.get("number"),+ "title": best_issue.get("title"),+ "url": best_issue.get("html_url"),+ "matched_by": best_matched_by,+ "score": round(best_score, 3),+ }+ else:+ cluster["known"] = False+ cluster["issue"] = None+++def main():+ parser = argparse.ArgumentParser(description="Match failure clusters against known-issue tickets")+ parser.add_argument("--clusters-json", required=True, help="Clusters JSON from analyze_failures.py")+ parser.add_argument("--output", required=True, help="Annotated clusters JSON output path")+ parser.add_argument("--repo", default=os.environ.get("REPO_PATH", ""), help="owner/name")+ parser.add_argument("--label", default="CI-known-issue", help="Known-issue label")+ parser.add_argument("--token-env", default="AUTO_ROUND_BOT_TOKEN", help="Env var holding the GitHub token")+ args = parser.parse_args()++ with open(args.clusters_json, encoding="utf-8") as f:+ data = json.load(f)+ clusters = data.get("clusters", [])++ token = os.environ.get(args.token_env, "")+ issues: list[dict] = []+ if args.repo:+ try:+ issues = fetch_known_issues(args.repo, args.label, token)+ print(f"Fetched {len(issues)} '{args.label}' issue(s) from {args.repo}", file=sys.stderr)+ except (HTTPError, URLError, ValueError, json.JSONDecodeError) as e:+ print(f"Warning: could not fetch known issues ({e}); treating all as unknown", file=sys.stderr)+ else:+ print("Warning: no repo provided; treating all clusters as unknown", file=sys.stderr)++ match_clusters(clusters, issues)++ known = sum(1 for c in clusters if c.get("known"))+ data["known_count"] = known+ data["unknown_count"] = len(clusters) - known+ with open(args.output, "w", encoding="utf-8") as f:+ json.dump(data, f, indent=2)+ print(f"Matched {known}/{len(clusters)} cluster(s) to known issues", file=sys.stderr)+++if __name__ == "__main__":+ main()diff --git a/.azure-pipelines/scripts/ai_analysis/merge_reports.py b/.azure-pipelines/scripts/ai_analysis/merge_reports.pynew file mode 100644index 000000000..d634eae2c--- /dev/null+++ b/.azure-pipelines/scripts/ai_analysis/merge_reports.py@@ -0,0 +1,183 @@+"""Merge clustered failures and AI analyses into one collapsible markdown report.++The report is designed to be pasted as a PR comment after human review. Most+content is wrapped in ``<details>`` blocks to keep it compact; only the header+summary is always visible.+"""++import argparse+import json+import os+++def _inline(text: str, limit: int = 160) -> str:+ """Flatten a signature to a single, table-safe line."""+ text = (text or "").replace("|", "\\|").replace("\n", " ").strip()+ return text[:limit]+++def _code_span(text: str) -> str:+ """Wrap text in an inline code span without breaking embedded backticks."""+ text = (text or "").replace("\n", " ").strip()+ if "`" not in text:+ return f"`{text}`"+ longest_run = 0+ current_run = 0+ for character in text:+ if character == "`":+ current_run += 1+ longest_run = max(longest_run, current_run)+ else:+ current_run = 0+ fence = "`" * (longest_run + 1)+ return f"{fence} {text} {fence}"+++def _load(path: str) -> dict:+ if not path or not os.path.isfile(path):+ return {}+ with open(path, encoding="utf-8") as f:+ return json.load(f)+++def build_header(clusters_data: dict, args) -> list[str]:+ total_failed = clusters_data.get("total_failed_tests", 0)+ cluster_count = clusters_data.get("cluster_count", len(clusters_data.get("clusters", [])))+ known = clusters_data.get("known_count", 0)+ unknown = clusters_data.get("unknown_count", cluster_count - known)+ lines = [+ f"# CI Failure Analysis Report ({args.pipeline or 'N/A'})",+ "",+ f"- **Commit:** {args.commit or 'N/A'}",+ ]+ if args.run_url:+ lines.append(f"- **Run:** {args.run_url}")+ lines.append(+ f"- **Totals:** {total_failed} failed test(s), {cluster_count} cluster(s) " f"({known} known / {unknown} new)"+ )+ lines.append("")+ return lines+++def build_known_section(clusters: list[dict]) -> list[str]:+ known = [c for c in clusters if c.get("known") and c.get("issue")]+ if not known:+ return []+ lines = [+ f"## Known Issues ({len(known)})",+ "",+ "<details><summary>🔍 Summary table</summary>",+ "",+ "| Cluster | Issue | Score | Occurrences | Sample log |",+ "| --- | --- | --- | --- | --- |",+ ]+ for c in known:+ issue = c["issue"]+ num = issue.get("number")+ url = issue.get("url") or ""+ issue_cell = f"[#{num}]({url})" if url else f"#{num}"+ score = f"{issue.get('score', 0):.2f}"+ sample_log = ", ".join(c.get("logs", [])) or "-"+ lines.append(+ f"| {c.get('id')} | {issue_cell} | {score} | " f"{c.get('occurrences')} | {_inline(sample_log, 60)} |"+ )+ lines.extend(["", "</details>", ""])+ return lines+++def build_unknown_section(clusters: list[dict], analyses_by_id: dict) -> list[str]:+ unknown = [c for c in clusters if not c.get("known")]+ unknown.sort(key=lambda c: c.get("occurrences", 0), reverse=True)+ if not unknown:+ return []+ lines = [f"## New Issues ({len(unknown)})", ""]++ for rank, c in enumerate(unknown, start=1):+ signature = _inline(c.get("signature", ""))+ occ = c.get("occurrences", 0)+ tests = c.get("tests", [])+ displayed_tests = tests[:10]+ if len(tests) > 10:+ displayed_tests.append("...")+ logs = c.get("logs", [])+ displayed_logs = logs[:10]+ if len(logs) > 10:+ displayed_logs.append("...")+ lines.append(f"### No.{rank} — {occ} occurrence(s)")+ lines.append("")+ lines.append(f"<details><summary> 🔍 {signature}</summary>")+ lines.append("")+ lines.append("### 📝 Basic info")+ lines.append("")+ lines.append(f"- **Affected tests ({len(tests)}):** {', '.join(displayed_tests) or '-'}")+ lines.append(f"- **Logs ({len(logs)}):** {', '.join(displayed_logs) or '-'}")+ lines.append("")+ lines.append("### 🖥️ Log excerpt")+ lines.append("")+ lines.append("```")+ lines.append((c.get("sample", "") or "").strip())+ lines.append("```")+ lines.append("")++ analysis = analyses_by_id.get(c.get("id"))+ if analysis:+ lines.append("### ✨ AI analysis")+ lines.append("")+ lines.append(f"- **Category:** {analysis.get('category', 'Unknown')}")+ lines.append(f"- **Confidence:** {analysis.get('confidence', 'low')}")+ if analysis.get("root_cause"):+ lines.append(f"- **Root cause:** {analysis['root_cause']}")+ if analysis.get("suggested_fix"):+ lines.append(f"- **Suggested fix:** {analysis['suggested_fix']}")+ if analysis.get("patch"):+ lines.append("")+ lines.append("- **Patch:**")+ lines.append(" ```diff")+ lines.extend(f" {line}" for line in analysis["patch"].splitlines())+ lines.append(" ```")+ if analysis.get("directions"):+ lines.append(f"- **Investigation directions (low confidence):** {analysis['directions']}")+ lines.append("")+ lines.append("</details>")+ lines.append("")++ lines.extend(+ [+ "",+ "> **Notes**",+ "> - Only top-3 issues receive AI analysis.",+ '> - To request an additional fix from Copilot, use "Quote reply" on the PR comment and @mention Copilot.',+ "",+ ]+ )+ return lines+++def main():+ parser = argparse.ArgumentParser(description="Merge clusters + AI analyses into a markdown report")+ parser.add_argument("--clusters-json", required=True, help="Annotated clusters JSON")+ parser.add_argument("--ai-json", help="AI analysis JSON")+ parser.add_argument("--output", required=True, help="Output markdown path")+ parser.add_argument("--pipeline", default=os.environ.get("BUILD_DEFINITIONNAME", ""))+ parser.add_argument("--pr", default=os.environ.get("PR_NUMBER", ""))+ parser.add_argument("--commit", default=os.environ.get("CURRENT_SHA", ""))+ parser.add_argument("--run-url", default=os.environ.get("RUN_URL", ""))+ args = parser.parse_args()++ clusters_data = _load(args.clusters_json)+ clusters = clusters_data.get("clusters", [])+ ai_data = _load(args.ai_json)+ analyses_by_id = {a.get("cluster_id"): a for a in ai_data.get("analyses", [])}++ lines: list[str] = []+ lines += build_header(clusters_data, args)+ lines += build_known_section(clusters)+ lines += build_unknown_section(clusters, analyses_by_id)++ with open(args.output, "w", encoding="utf-8") as f:+ f.write("\n".join(lines))+ print(f"Wrote report to {args.output}")+++if __name__ == "__main__":+ main()diff --git a/.azure-pipelines/scripts/ai_analysis/post_pr_comment.py b/.azure-pipelines/scripts/ai_analysis/post_pr_comment.pynew file mode 100644index 000000000..5ca4585d5--- /dev/null+++ b/.azure-pipelines/scripts/ai_analysis/post_pr_comment.py@@ -0,0 +1,181 @@+"""Post (or update) the AI failure-analysis report as a PR comment.++Idempotent: the comment carries a hidden marker so reruns update the existing+comment instead of spamming new ones. Fails soft — a posting problem never+breaks the pipeline.++With ``--hide`` the script instead minimizes the marker comment as OUTDATED,+used when a rerun passes so the stale failure report collapses automatically.+"""++import argparse+import json+import os+import sys+from urllib.error import HTTPError, URLError+from urllib.request import Request, urlopen++_MARKER = "<!-- ai-failure-analysis -->"+# GitHub caps issue-comment bodies at 65536 chars; leave room for the marker.+_MAX_BODY = 65000+++def _build_marker(key: str) -> str:+ """Return the hidden comment marker, scoped by an optional per-workflow key.++ Distinct keys keep separate workflows from overwriting each other's comment.+ """+ key = (key or "").strip()+ return f"<!-- ai-failure-analysis:{key} -->" if key else _MARKER+++def _api(url: str, token: str, method: str = "GET", payload: "dict | None" = None) -> "list | dict":+ headers = {"Accept": "application/vnd.github+json"}+ if token:+ headers["Authorization"] = f"Bearer {token}"+ data = json.dumps(payload).encode("utf-8") if payload is not None else None+ if data is not None:+ headers["Content-Type"] = "application/json"+ req = Request(url, data=data, headers=headers, method=method)+ with urlopen(req, timeout=20) as resp:+ body = resp.read().decode("utf-8")+ return json.loads(body) if body else {}+++def _find_existing(repo: str, pr: int, token: str, marker: str) -> "int | None":+ c = _find_existing_comment(repo, pr, token, marker)+ return c.get("id") if c else None+++def _find_existing_comment(repo: str, pr: int, token: str, marker: str) -> "dict | None":+ page = 1+ while True:+ url = f"https://api.github.com/repos/{repo}/issues/{pr}/comments?per_page=100&page={page}"+ comments = _api(url, token)+ if not isinstance(comments, list) or not comments:+ return None+ for c in comments:+ if marker in (c.get("body") or ""):+ return c+ if len(comments) < 100:+ return None+ page += 1+++def _graphql(token: str, query: str, variables: dict) -> dict:+ req = Request(+ "https://api.github.com/graphql",+ data=json.dumps({"query": query, "variables": variables}).encode("utf-8"),+ headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json"},+ method="POST",+ )+ with urlopen(req, timeout=20) as resp:+ return json.loads(resp.read().decode("utf-8"))+++def _minimize_comment(node_id: str, token: str, classifier: str = "OUTDATED") -> bool:+ """Hide (minimize) a PR comment via GraphQL, marking it as e.g. OUTDATED."""+ query = (+ "mutation($id: ID!, $reason: ReportedContentClassifiers!) {"+ " minimizeComment(input: {subjectId: $id, classifier: $reason}) {"+ " minimizedComment { isMinimized minimizedReason }"+ " }"+ "}"+ )+ result = _graphql(token, query, {"id": node_id, "reason": classifier})+ if result.get("errors"):+ print(f"Warning: could not hide comment ({result['errors']}); non-blocking.", file=sys.stderr)+ return False+ return bool(result.get("data", {}).get("minimizeComment", {}).get("minimizedComment", {}).get("isMinimized"))+++def _hide(repo: str, pr: int, token: str, marker: str) -> None:+ """Find the marker comment and hide it as OUTDATED (used when the run passes)."""+ try:+ existing = _find_existing_comment(repo, pr, token, marker)+ except (HTTPError, URLError, ValueError) as e:+ print(f"Warning: could not look up PR comment ({e}); non-blocking.", file=sys.stderr)+ return+ if not existing:+ print("No existing analysis comment to hide.", file=sys.stderr)+ return+ node_id = existing.get("node_id")+ if not node_id:+ print("Existing comment has no node_id; cannot hide.", file=sys.stderr)+ return+ try:+ if _minimize_comment(node_id, token):+ print(f"Hid PR comment {existing.get('id')} as OUTDATED.", file=sys.stderr)+ except (HTTPError, URLError, ValueError) as e:+ print(f"Warning: could not hide PR comment ({e}); non-blocking.", file=sys.stderr)+++def main():+ parser = argparse.ArgumentParser(description="Post the failure-analysis report to a PR comment")+ parser.add_argument("--repo", default=os.environ.get("REPO_PATH", ""), help="owner/name")+ parser.add_argument("--pr", default=os.environ.get("PR_NUMBER", ""), help="PR number")+ parser.add_argument("--body-file", help="Markdown report to post (required unless --hide)")+ parser.add_argument(+ "--marker-key",+ default=os.environ.get("AI_COMMENT_MARKER_KEY", ""),+ help="Per-workflow key so different pipelines update separate PR comments",+ )+ parser.add_argument(+ "--hide",+ action="store_true",+ help="Hide the existing analysis comment as OUTDATED instead of posting (use when the run passes)",+ )+ parser.add_argument("--token-env", default="AUTO_ROUND_BOT_TOKEN", help="Env var holding the GitHub token")+ args = parser.parse_args()++ marker = _build_marker(args.marker_key)+ token = os.environ.get(args.token_env, "")+ if not args.repo or "/" not in args.repo or not str(args.pr).strip():+ print("No repo/PR number available; skipping PR comment.", file=sys.stderr)+ return+ if not token:+ print(f"No token in ${args.token_env}; skipping PR comment.", file=sys.stderr)+ return++ if args.hide:+ _hide(args.repo, int(args.pr), token, marker)+ return++ if not args.body_file:+ print("No --body-file provided; nothing to post.", file=sys.stderr)+ return+ if not os.path.isfile(args.body_file):+ print(f"Report file not found: {args.body_file}; skipping.", file=sys.stderr)+ return++ with open(args.body_file, encoding="utf-8") as f:+ body = f.read()+ if len(body) > _MAX_BODY:+ body = body[:_MAX_BODY] + "\n\n_...truncated; see the pipeline artifact for the full report._"+ body = f"{marker}\n{body}"++ pr = int(args.pr)+ try:+ existing = _find_existing(args.repo, pr, token, marker)+ if existing is not None:+ _api(+ f"https://api.github.com/repos/{args.repo}/issues/comments/{existing}",+ token,+ method="PATCH",+ payload={"body": body},+ )+ print(f"Updated existing PR comment {existing}.", file=sys.stderr)+ else:+ _api(+ f"https://api.github.com/repos/{args.repo}/issues/{pr}/comments",+ token,+ method="POST",+ payload={"body": body},+ )+ print("Created new PR comment.", file=sys.stderr)+ except (HTTPError, URLError, ValueError) as e:+ print(f"Warning: could not post PR comment ({e}); non-blocking.", file=sys.stderr)+++if __name__ == "__main__":+ main()diff --git a/.azure-pipelines/template/ai-stage-template.yml b/.azure-pipelines/template/ai-stage-template.ymlnew file mode 100644index 000000000..9a7da495e--- /dev/null+++ b/.azure-pipelines/template/ai-stage-template.yml@@ -0,0 +1,194 @@+# Reusable AI-failure-analysis stages shared across pipelines (CPU, XPU, ...).+# Split into two stages because a job condition cannot read a previous stage's+# overall result (`stageDependencies.<stage>` only exposes per-job nodes, so+# `stageDependencies.<stage>.result` is a dictionary miss -> Null). A stage-level+# condition can read `dependencies.<stage>.result`, so we branch there instead.+parameters:+ - name: dependsOn+ type: string+ default: "Unit_test"+ - name: stageName+ type: string+ default: "AI_Analysis"+ - name: enableAiAnalysis+ type: boolean+ default: true+ - name: postComment+ type: boolean+ default: true+ - name: topUnknown+ type: number+ default: 3+ - name: knownIssueLabel+ type: string+ default: "CI-known-issue"++stages:+ - stage: ${{ parameters.stageName }}+ displayName: "AI Failure Analysis"+ pool:+ vmImage: "ubuntu-latest"+ dependsOn: ${{ parameters.dependsOn }}+ condition: and(eq(dependencies.${{ parameters.dependsOn }}.result, 'Failed'), eq(variables['System.StageAttempt'], '1'))+ variables:+ - group: "auto-round-bot-secrets"+ jobs:+ - job: AnalyzeFailures+ displayName: "Analyze failed unit tests"+ variables:+ base: $(Build.SourcesDirectory)+ output_dir: $(Build.SourcesDirectory)/ai_analysis_results+ scripts: $(Build.SourcesDirectory)/.azure-pipelines/scripts/ai_analysis+ steps:+ - task: UsePythonVersion@0+ inputs:+ versionSpec: "3.12"+ displayName: "Use Python 3.12"++ - task: DownloadPipelineArtifact@2+ inputs:+ artifact:+ patterns: "ut-*-1/unittest_*"+ path: $(output_dir)/failed_logs+ displayName: "Download failed-log artifacts"++ - task: Bash@3+ displayName: "Cluster failures"+ inputs:+ targetType: "inline"+ script: |+ set -eo pipefail+ python "$(scripts)/analyze_failures.py" \+ --input-dir "$(output_dir)/failed_logs" \+ --clusters-json "$(output_dir)/clusters.json" \+ --decision-log "$(output_dir)/decision_log.md"++ echo "Cluster analysis completed. Results are in $(output_dir)/clusters.json"+ cat "$(output_dir)/clusters.json"++ - task: Bash@3+ displayName: "Match known issues"+ env:+ AUTO_ROUND_BOT_TOKEN: $(AUTO_ROUND_BOT_TOKEN)+ inputs:+ targetType: "inline"+ script: |+ set -eo pipefail+ repo_path="${BUILD_REPOSITORY_URI#https://github.com/}"+ repo_path="${repo_path%.git}"++ python "$(scripts)/match_known_issues.py" \+ --clusters-json "$(output_dir)/clusters.json" \+ --output "$(output_dir)/annotated.json" \+ --repo "${repo_path}" \+ --label "${{ parameters.knownIssueLabel }}"++ echo "Matching known issues completed. Results are in $(output_dir)/annotated.json"+ cat "$(output_dir)/annotated.json"++ - task: Bash@3+ displayName: "AI analysis of top unknown failures"+ continueOnError: true+ env:+ AI_TOKEN: $(AI_TOKEN)+ inputs:+ targetType: "inline"+ script: |+ set -uo pipefail+ if [ "${{ parameters.enableAiAnalysis }}" = "True" ]; then+ echo "Installing Copilot CLI for AI analysis..."+ curl -fsSL https://gh.io/copilot-install | bash+ export PATH="$HOME/.local/bin:$PATH"+ fi+ python "$(scripts)/ai_analyze.py" \+ --clusters-json "$(output_dir)/annotated.json" \+ --output "$(output_dir)/ai.json" \+ --pr-sha "$(Build.SourceVersion)" \+ --top ${{ parameters.topUnknown }} \+ --enable-ai-analysis ${{ parameters.enableAiAnalysis }} \+ --backend copilot \+ --model gpt-5.6-luna \+ --reasoning-effort max \+ --trace-file "$(output_dir)/ai_trace.jsonl" \+ --project-root "$(base)" \+ --log-dir "$(output_dir)/failed_logs"++ echo "AI analysis completed. Results are in $(output_dir)/ai.json"+ cat "$(output_dir)/ai.json"++ - task: Bash@3+ displayName: "Merge markdown report"+ env:+ BUILD_DEFINITIONNAME: $(Build.DefinitionName)+ PR_NUMBER: $(System.PullRequest.PullRequestNumber)+ CURRENT_SHA: $(Build.SourceVersion)+ RUN_URL: $(System.CollectionUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId)+ inputs:+ targetType: "inline"+ script: |+ set -eo pipefail+ python "$(scripts)/merge_reports.py" \+ --clusters-json "$(output_dir)/annotated.json" \+ --ai-json "$(output_dir)/ai.json" \+ --output "$(output_dir)/report.md"++ - ${{ if eq(parameters.postComment, true) }}:+ - task: Bash@3+ displayName: "Post report as PR comment"+ condition: eq(variables['Build.Reason'], 'PullRequest')+ env:+ AUTO_ROUND_BOT_TOKEN: $(AUTO_ROUND_BOT_TOKEN)+ inputs:+ targetType: "inline"+ script: |+ set -uo pipefail+ repo_path="${BUILD_REPOSITORY_URI#https://github.com/}"+ repo_path="${repo_path%.git}"+ python "$(scripts)/post_pr_comment.py" \+ --repo "${repo_path}" \+ --pr "${SYSTEM_PULLREQUEST_PULLREQUESTNUMBER:-}" \+ --marker-key "$(Build.DefinitionName)" \+ --body-file "$(output_dir)/report.md"++ - task: PublishPipelineArtifact@1+ condition: always()+ inputs:+ targetPath: $(Build.SourcesDirectory)/ai_analysis_results+ artifact: "ai-failure-analysis"+ publishLocation: "pipeline"+ displayName: "Publish AI failure-analysis report"++ - stage: ${{ parameters.stageName }}_HideOutdated+ displayName: "Hide AI Analysis Comments"+ pool:+ vmImage: "ubuntu-latest"+ dependsOn: ${{ parameters.dependsOn }}+ condition: in(dependencies.${{ parameters.dependsOn }}.result, 'Succeeded', 'SucceededWithIssues')+ variables:+ - group: "auto-round-bot-secrets"+ jobs:+ - job: HideComments+ displayName: "Hide AI Analysis Comments"+ steps:+ - task: UsePythonVersion@0+ inputs:+ versionSpec: "3.12"+ displayName: "Use Python 3.12"+ - task: Bash@3+ displayName: "Hide outdated AI analysis comment"+ condition: eq(variables['Build.Reason'], 'PullRequest')+ env:+ AUTO_ROUND_BOT_TOKEN: $(AUTO_ROUND_BOT_TOKEN)+ inputs:+ targetType: "inline"+ script: |+ set -uo pipefail+ base="$(Build.SourcesDirectory)"+ scripts="${base}/.azure-pipelines/scripts/ai_analysis"+ repo_path="${BUILD_REPOSITORY_URI#https://github.com/}"+ repo_path="${repo_path%.git}"+ python "${scripts}/post_pr_comment.py" \+ --repo "${repo_path}" \+ --pr "${SYSTEM_PULLREQUEST_PULLREQUESTNUMBER:-}" \+ --marker-key "$(Build.DefinitionName)" \+ --hidediff --git a/.azure-pipelines/unit-test-xpu.yml b/.azure-pipelines/unit-test-xpu.ymlindex 9a91505aa..e22d7192a 100644--- a/.azure-pipelines/unit-test-xpu.yml+++ b/.azure-pipelines/unit-test-xpu.yml@@ -77,6 +77,12 @@ stages:buildARKWheel: $(BUILD_ARK_WHEEL)filterChangedTests: true+ - template: template/ai-stage-template.yml+ parameters:+ dependsOn: "Unit_test"+ enableAiAnalysis: true+ postComment: true+- stage: CoveragedisplayName: "Collect Coverage"pool:diff --git a/.azure-pipelines/unit-test.yml b/.azure-pipelines/unit-test.ymlindex 6a21106f3..db9a2c05a 100644--- a/.azure-pipelines/unit-test.yml+++ b/.azure-pipelines/unit-test.yml@@ -83,6 +83,12 @@ stages:buildARKWheel: $(BUILD_ARK_WHEEL)filterChangedTests: true+ - template: template/ai-stage-template.yml+ parameters:+ dependsOn: "Unit_test"+ enableAiAnalysis: true+ postComment: true+- stage: CoveragedisplayName: "Collect Coverage"pool: