PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Enhance AI analysis and PR comment handling (#2538)
chen, suyuec1d97369906a
diff --git a/.azure-pipelines/nightly-test-cuda.yml b/.azure-pipelines/nightly-test-cuda.ymlindex e63be68cf..6558b4a3a 100644--- a/.azure-pipelines/nightly-test-cuda.yml+++ b/.azure-pipelines/nightly-test-cuda.yml@@ -1,7 +1,7 @@# Nightly CUDA pipeline: runs the slower CUDA integration + e2e suites# (sglang, llmc, vllm) that are excluded from the fast PR unit-test pipeline-# (unit-test-cuda.yml). Scheduled daily; can also be triggered manually from-# the Azure DevOps UI.+# (unit-test-cuda.yml). Scheduled daily only; can also be triggered manually+# from the Azure DevOps UI.trigger: none@@ -14,12 +14,16 @@ variables:value: "runpod-agent-$(Build.BuildId)"stages:- - stage: Start_Pod- displayName: "Start Pod"+ # Pod creation, GPU tests and cleanup live in ONE stage as three jobs so a+ # "Rerun failed stage" reruns every job (including Start_Pod_Job) and thus+ # recreates the ephemeral pod. Separate stages would leave the already+ # succeeded Start_Pod stage un-rerun, so the retry would find no agent.+ - stage: GPU_Test+ displayName: "GPU Nightly Test"dependsOn: []jobs:- job: Start_Pod_Job- displayName: "Launch Pod & Run CUDA Nightly Tests"+ displayName: "Launch Pod"timeoutInMinutes: 60pool:vmImage: "ubuntu-latest"@@ -82,14 +86,12 @@ stages:env:PYTHONUNBUFFERED: '1'- - stage: GPU_Test- displayName: "GPU Nightly Test"- dependsOn: Start_Pod- variables:- - group: auto-round-bot-secrets- jobs:- job: GpuTests- timeoutInMinutes: 300+ displayName: "Run CUDA Nightly Tests"+ dependsOn: Start_Pod_Job+ variables:+ - group: auto-round-bot-secrets+ timeoutInMinutes: 60pool:name: "$(POOL_NAME)"demands: Agent.Name -equals runpod-agent-$(Build.BuildId)@@ -115,7 +117,6 @@ stages:- script: |export UV_NO_CACHE=0- export FILTER_CHANGED_TESTS=truebash .azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh \--test_case=nightly \--test_part=0@@ -137,14 +138,12 @@ stages:condition: always()displayName: "GPU Monitor"- - stage: Cleanup_Verification- displayName: "Verify Pod Cleanup"- dependsOn:- - Start_Pod- - GPU_Test- condition: always()- jobs:- job: Verify_Cleanup+ displayName: "Verify Pod Cleanup"+ dependsOn:+ - Start_Pod_Job+ - GpuTests+ condition: always()pool:vmImage: "ubuntu-latest"steps:diff --git a/.azure-pipelines/scripts/ai_analysis/ai_analyze.py b/.azure-pipelines/scripts/ai_analysis/ai_analyze.pyindex 693361a1c..bfb08acdf 100644--- a/.azure-pipelines/scripts/ai_analysis/ai_analyze.py+++ b/.azure-pipelines/scripts/ai_analysis/ai_analyze.py@@ -3,7 +3,9 @@Ranks the *unknown* failure clusters by occurrence, takes the top N (default 3)and asks an AI backend to classify each failure (Code Regression / Environment /Dependency / Test Case) and propose a fix. When the model is not confident it is-instructed to omit the fix and instead give investigation directions.+instructed to omit the fix and instead give investigation directions. Answers+that are not ``high`` confidence (or unparsable) can be re-run once with a+stronger ``--escalation-model``.The backend is intentionally pluggable behind ``call_backend`` so the currentGitHub Copilot CLI implementation can be swapped later. With ``--backend none``@@ -296,6 +298,36 @@ def build_prompt(cluster: dict, pr_files: str, max_excerpt: int, project_root: s)+def _reset_workspace(args) -> None:+ """Restore the checkout to a clean state so the escalated run is not influenced by edits from the first run.++ Only runs when ``--project-root`` is given explicitly, and keeps the analysis output / log / trace dirs.+ """+ if not args.project_root:+ return+ root = os.path.abspath(args.project_root)+ keep = set()+ for path in (args.output, args.trace_file, args.clusters_json):+ if path:+ keep.add(os.path.dirname(os.path.abspath(path)))+ if args.log_dir:+ keep.add(os.path.abspath(args.log_dir))+ excludes = []+ for path in keep:+ rel = os.path.relpath(path, root)+ if rel != "." and not rel.startswith(".."):+ top = rel.split(os.sep)[0]+ if top not in excludes[1::2]:+ excludes += ["-e", top]+ print(f"Resetting workspace {root} before escalation (keeping {excludes[1::2]})", file=sys.stderr)+ _run_git(root, ["checkout", "--", "."])+ _run_git(root, ["clean", "-fd", *excludes])+++def _is_high_confidence(parsed: "dict | None") -> bool:+ return bool(parsed) and str(parsed.get("confidence", "")).strip().lower() == "high"++def analyze(cluster: dict, pr_files: str, args) -> dict:prompt = build_prompt(cluster, pr_files, args.max_excerpt_chars, args.project_root, args.log_dir)raw = call_backend(@@ -308,11 +340,38 @@ def analyze(cluster: dict, pr_files: str, args) -> dict:cluster.get("id"),)parsed = parse_model_json(raw)+ model_used = args.model+ escalated = False+ # Re-run non-high-confidence (or unparsable) answers once with a stronger model.+ if args.backend != "none" and args.escalation_model and not _is_high_confidence(parsed):+ print(+ f"Cluster {cluster.get('id')}: confidence="+ f"{(parsed or {}).get('confidence', 'unparsable')}, escalating to {args.escalation_model}",+ file=sys.stderr,+ )+ _reset_workspace(args)+ esc_raw = call_backend(+ prompt,+ args.backend,+ args.escalation_timeout or args.timeout,+ args.escalation_model,+ args.escalation_reasoning_effort,+ args.trace_file,+ f"{cluster.get('id')}-escalated",+ )+ esc_parsed = parse_model_json(esc_raw)+ # Keep the first answer if the escalated call produced nothing usable.+ if esc_parsed or not parsed:+ raw, parsed = esc_raw, esc_parsed+ model_used = args.escalation_model+ escalated = Trueresult = {"cluster_id": cluster.get("id"),"signature": cluster.get("signature"),"occurrences": cluster.get("occurrences"),"tests": cluster.get("tests", []),+ "model": model_used if args.backend != "none" else "",+ "escalated": escalated,}if parsed:# Low-confidence answers must not carry a fix/patch (requirement 3.3).@@ -361,7 +420,7 @@ def main():default="true",help="Set to a falsy value (false/0/no) to force the no-model 'none' backend",)- parser.add_argument("--timeout", type=int, default=300, help="Per-call timeout in seconds")+ parser.add_argument("--timeout", type=int, default=600, help="Per-call timeout in seconds")parser.add_argument("--model", default="", help="Copilot model to use; empty uses the CLI default")parser.add_argument("--reasoning-effort",@@ -369,6 +428,23 @@ def main():default="",help="Copilot reasoning effort; empty uses the CLI default",)+ parser.add_argument(+ "--escalation-model",+ default="",+ help="Stronger model to re-run a cluster with when the first answer is not high confidence; empty disables",+ )+ parser.add_argument(+ "--escalation-reasoning-effort",+ choices=["none", "minimal", "low", "medium", "high", "xhigh", "max"],+ default="",+ help="Reasoning effort for the escalation model; empty uses the CLI default",+ )+ parser.add_argument(+ "--escalation-timeout",+ type=int,+ default=0,+ help="Per-call timeout in seconds for the escalation model; 0 reuses --timeout",+ )parser.add_argument("--trace-file", default="", help="JSONL file logging each AI call for auditing")parser.add_argument("--project-root", default="", help="Repository checkout root the AI may inspect")parser.add_argument("--log-dir", default="", help="Directory holding the full raw failure logs")diff --git a/.azure-pipelines/scripts/ai_analysis/merge_reports.py b/.azure-pipelines/scripts/ai_analysis/merge_reports.pyindex d634eae2c..ff4e944c7 100644--- a/.azure-pipelines/scripts/ai_analysis/merge_reports.py+++ b/.azure-pipelines/scripts/ai_analysis/merge_reports.py@@ -125,6 +125,9 @@ def build_unknown_section(clusters: list[dict], analyses_by_id: dict) -> list[stlines.append("")lines.append(f"- **Category:** {analysis.get('category', 'Unknown')}")lines.append(f"- **Confidence:** {analysis.get('confidence', 'low')}")+ if analysis.get("model"):+ suffix = " (escalated: first answer was not high confidence)" if analysis.get("escalated") else ""+ lines.append(f"- **Model:** {analysis['model']}{suffix}")if analysis.get("root_cause"):lines.append(f"- **Root cause:** {analysis['root_cause']}")if analysis.get("suggested_fix"):@@ -147,6 +150,7 @@ def build_unknown_section(clusters: list[dict], analyses_by_id: dict) -> list[st"> **Notes**","> - Only top-3 issues receive AI analysis.",'> - To request an additional fix from Copilot, use "Quote reply" on the PR comment and @mention Copilot.',+ "> - Was this analysis helpful? React with 👍 if it was, or 👎 if it wasn't.","",])diff --git a/.azure-pipelines/scripts/ai_analysis/post_pr_comment.py b/.azure-pipelines/scripts/ai_analysis/post_pr_comment.pyindex 5ca4585d5..f9dd74d9e 100644--- a/.azure-pipelines/scripts/ai_analysis/post_pr_comment.py+++ b/.azure-pipelines/scripts/ai_analysis/post_pr_comment.py@@ -6,6 +6,8 @@ breaks the pipeline.With ``--hide`` the script instead minimizes the marker comment as OUTDATED,used when a rerun passes so the stale failure report collapses automatically.+If a later run fails again, the hidden comment is left collapsed: its marker is+retired and a fresh comment carrying the active marker is posted instead."""import argparse@@ -42,11 +44,6 @@ def _api(url: str, token: str, method: str = "GET", payload: "dict | None" = Nonreturn json.loads(body) if body else {}-def _find_existing(repo: str, pr: int, token: str, marker: str) -> "int | None":- c = _find_existing_comment(repo, pr, token, marker)- return c.get("id") if c else None--def _find_existing_comment(repo: str, pr: int, token: str, marker: str) -> "dict | None":page = 1while True:@@ -89,6 +86,33 @@ def _minimize_comment(node_id: str, token: str, classifier: str = "OUTDATED") ->return bool(result.get("data", {}).get("minimizeComment", {}).get("minimizedComment", {}).get("isMinimized"))+def _is_minimized(node_id: str, token: str) -> bool:+ """Return True if the comment is currently minimized (REST does not expose this)."""+ query = "query($id: ID!) { node(id: $id) { ... on Minimizable { isMinimized } } }"+ result = _graphql(token, query, {"id": node_id})+ return bool(((result.get("data") or {}).get("node") or {}).get("isMinimized"))+++def _retired_marker(marker: str) -> str:+ """Return a marker variant that no longer matches ``marker`` (used for superseded comments)."""+ return marker.replace("ai-failure-analysis", "ai-failure-analysis-retired", 1)+++def _retire_comment(repo: str, comment: dict, token: str, marker: str) -> None:+ """Swap the hidden marker on an old comment so future runs no longer match it.++ Only the invisible HTML marker changes; the visible report stays as-is.+ """+ body = (comment.get("body") or "").replace(marker, _retired_marker(marker))+ _api(+ f"https://api.github.com/repos/{repo}/issues/comments/{comment['id']}",+ token,+ method="PATCH",+ payload={"body": body},+ )+ print(f"Retired marker on minimized PR comment {comment['id']}.", file=sys.stderr)++def _hide(repo: str, pr: int, token: str, marker: str) -> None:"""Find the marker comment and hide it as OUTDATED (used when the run passes)."""try:@@ -156,7 +180,19 @@ def main():pr = int(args.pr)try:- existing = _find_existing(args.repo, pr, token, marker)+ existing_comment = _find_existing_comment(args.repo, pr, token, marker)+ existing = existing_comment.get("id") if existing_comment else None+ # A comment hidden as OUTDATED by an earlier passing run may be buried deep in a+ # long PR history; leave it collapsed and post a fresh comment instead.+ if existing is not None and existing_comment.get("node_id"):+ try:+ minimized = _is_minimized(existing_comment["node_id"], token)+ except (HTTPError, URLError, ValueError) as e:+ print(f"Warning: could not check comment visibility ({e}); updating in place.", file=sys.stderr)+ minimized = False+ if minimized:+ _retire_comment(args.repo, existing_comment, token, marker)+ existing = Noneif existing is not None:_api(f"https://api.github.com/repos/{args.repo}/issues/comments/{existing}",diff --git a/.azure-pipelines/template/ai-stage-template.yml b/.azure-pipelines/template/ai-stage-template.ymlindex 9a7da495e..8411b1282 100644--- a/.azure-pipelines/template/ai-stage-template.yml+++ b/.azure-pipelines/template/ai-stage-template.yml@@ -2,10 +2,11 @@# Split into two stages because a job condition cannot read a previous stage's# overall result (`stageDependencies.<stage>` only exposes per-job nodes, so# `stageDependencies.<stage>.result` is a dictionary miss -> Null). A stage-level-# condition can read `dependencies.<stage>.result`, so we branch there instead.+# condition can check previous stage results (failed()/succeeded()), so we branch there instead.parameters:+ # A single stage name or a list of stage names (e.g. the per-part CUDA stages).- name: dependsOn- type: string+ type: objectdefault: "Unit_test"- name: stageNametype: string@@ -29,12 +30,14 @@ stages:pool:vmImage: "ubuntu-latest"dependsOn: ${{ parameters.dependsOn }}- condition: and(eq(dependencies.${{ parameters.dependsOn }}.result, 'Failed'), eq(variables['System.StageAttempt'], '1'))+ # failed() with no arguments checks the direct dependencies listed in dependsOn.+ condition: and(failed(), eq(variables['System.StageAttempt'], '1'))variables:- group: "auto-round-bot-secrets"jobs:- job: AnalyzeFailuresdisplayName: "Analyze failed unit tests"+ timeoutInMinutes: 90variables:base: $(Build.SourcesDirectory)output_dir: $(Build.SourcesDirectory)/ai_analysis_results@@ -109,6 +112,8 @@ stages:--backend copilot \--model gpt-5.6-luna \--reasoning-effort max \+ --escalation-model claude-opus-5.5 \+ --escalation-reasoning-effort high \--trace-file "$(output_dir)/ai_trace.jsonl" \--project-root "$(base)" \--log-dir "$(output_dir)/failed_logs"@@ -163,7 +168,8 @@ stages:pool:vmImage: "ubuntu-latest"dependsOn: ${{ parameters.dependsOn }}- condition: in(dependencies.${{ parameters.dependsOn }}.result, 'Succeeded', 'SucceededWithIssues')+ # succeeded() with no arguments checks the direct dependencies listed in dependsOn.+ condition: succeeded()variables:- group: "auto-round-bot-secrets"jobs:diff --git a/.azure-pipelines/unit-test-cuda.yml b/.azure-pipelines/unit-test-cuda.ymlindex 6bd214d4b..fc3db8f79 100644--- a/.azure-pipelines/unit-test-cuda.yml+++ b/.azure-pipelines/unit-test-cuda.yml@@ -267,6 +267,14 @@ stages:env:PYTHONUNBUFFERED: '1'+ - template: template/ai-stage-template.yml+ parameters:+ dependsOn:+ - ${{ each pair in parameters.matrix }}:+ - GPU_Test_${{ pair.key }}+ enableAiAnalysis: true+ postComment: true+- stage: CoveragedisplayName: "Collect Coverage"pool: