Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Enhance AI analysis and PR comment handling (#2538)

chen, suyuec1d97369906a
diff --git a/.azure-pipelines/nightly-test-cuda.yml b/.azure-pipelines/nightly-test-cuda.yml
index e63be68cf..6558b4a3a 100644
--- a/.azure-pipelines/nightly-test-cuda.yml
+++ b/.azure-pipelines/nightly-test-cuda.yml
@@ -1,7 +1,7 @@
# Nightly CUDA pipeline: runs the slower CUDA integration + e2e suites
# (sglang, llmc, vllm) that are excluded from the fast PR unit-test pipeline
-# (unit-test-cuda.yml). Scheduled daily; can also be triggered manually from
-# the Azure DevOps UI.
+# (unit-test-cuda.yml). Scheduled daily only; can also be triggered manually
+# from the Azure DevOps UI.
trigger: none
@@ -14,12 +14,16 @@ variables:
value: "runpod-agent-$(Build.BuildId)"
stages:
- - stage: Start_Pod
- displayName: "Start Pod"
+ # Pod creation, GPU tests and cleanup live in ONE stage as three jobs so a
+ # "Rerun failed stage" reruns every job (including Start_Pod_Job) and thus
+ # recreates the ephemeral pod. Separate stages would leave the already
+ # succeeded Start_Pod stage un-rerun, so the retry would find no agent.
+ - stage: GPU_Test
+ displayName: "GPU Nightly Test"
dependsOn: []
jobs:
- job: Start_Pod_Job
- displayName: "Launch Pod & Run CUDA Nightly Tests"
+ displayName: "Launch Pod"
timeoutInMinutes: 60
pool:
vmImage: "ubuntu-latest"
@@ -82,14 +86,12 @@ stages:
env:
PYTHONUNBUFFERED: '1'
- - stage: GPU_Test
- displayName: "GPU Nightly Test"
- dependsOn: Start_Pod
- variables:
- - group: auto-round-bot-secrets
- jobs:
- job: GpuTests
- timeoutInMinutes: 300
+ displayName: "Run CUDA Nightly Tests"
+ dependsOn: Start_Pod_Job
+ variables:
+ - group: auto-round-bot-secrets
+ timeoutInMinutes: 60
pool:
name: "$(POOL_NAME)"
demands: Agent.Name -equals runpod-agent-$(Build.BuildId)
@@ -115,7 +117,6 @@ stages:
- script: |
export UV_NO_CACHE=0
- export FILTER_CHANGED_TESTS=true
bash .azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh \
--test_case=nightly \
--test_part=0
@@ -137,14 +138,12 @@ stages:
condition: always()
displayName: "GPU Monitor"
- - stage: Cleanup_Verification
- displayName: "Verify Pod Cleanup"
- dependsOn:
- - Start_Pod
- - GPU_Test
- condition: always()
- jobs:
- job: Verify_Cleanup
+ displayName: "Verify Pod Cleanup"
+ dependsOn:
+ - Start_Pod_Job
+ - GpuTests
+ condition: always()
pool:
vmImage: "ubuntu-latest"
steps:
diff --git a/.azure-pipelines/scripts/ai_analysis/ai_analyze.py b/.azure-pipelines/scripts/ai_analysis/ai_analyze.py
index 693361a1c..bfb08acdf 100644
--- a/.azure-pipelines/scripts/ai_analysis/ai_analyze.py
+++ b/.azure-pipelines/scripts/ai_analysis/ai_analyze.py
@@ -3,7 +3,9 @@
Ranks the *unknown* failure clusters by occurrence, takes the top N (default 3)
and asks an AI backend to classify each failure (Code Regression / Environment /
Dependency / Test Case) and propose a fix. When the model is not confident it is
-instructed to omit the fix and instead give investigation directions.
+instructed to omit the fix and instead give investigation directions. Answers
+that are not ``high`` confidence (or unparsable) can be re-run once with a
+stronger ``--escalation-model``.
The backend is intentionally pluggable behind ``call_backend`` so the current
GitHub Copilot CLI implementation can be swapped later. With ``--backend none``
@@ -296,6 +298,36 @@ def build_prompt(cluster: dict, pr_files: str, max_excerpt: int, project_root: s
)
+def _reset_workspace(args) -> None:
+ """Restore the checkout to a clean state so the escalated run is not influenced by edits from the first run.
+
+ Only runs when ``--project-root`` is given explicitly, and keeps the analysis output / log / trace dirs.
+ """
+ if not args.project_root:
+ return
+ root = os.path.abspath(args.project_root)
+ keep = set()
+ for path in (args.output, args.trace_file, args.clusters_json):
+ if path:
+ keep.add(os.path.dirname(os.path.abspath(path)))
+ if args.log_dir:
+ keep.add(os.path.abspath(args.log_dir))
+ excludes = []
+ for path in keep:
+ rel = os.path.relpath(path, root)
+ if rel != "." and not rel.startswith(".."):
+ top = rel.split(os.sep)[0]
+ if top not in excludes[1::2]:
+ excludes += ["-e", top]
+ print(f"Resetting workspace {root} before escalation (keeping {excludes[1::2]})", file=sys.stderr)
+ _run_git(root, ["checkout", "--", "."])
+ _run_git(root, ["clean", "-fd", *excludes])
+
+
+def _is_high_confidence(parsed: "dict | None") -> bool:
+ return bool(parsed) and str(parsed.get("confidence", "")).strip().lower() == "high"
+
+
def analyze(cluster: dict, pr_files: str, args) -> dict:
prompt = build_prompt(cluster, pr_files, args.max_excerpt_chars, args.project_root, args.log_dir)
raw = call_backend(
@@ -308,11 +340,38 @@ def analyze(cluster: dict, pr_files: str, args) -> dict:
cluster.get("id"),
)
parsed = parse_model_json(raw)
+ model_used = args.model
+ escalated = False
+ # Re-run non-high-confidence (or unparsable) answers once with a stronger model.
+ if args.backend != "none" and args.escalation_model and not _is_high_confidence(parsed):
+ print(
+ f"Cluster {cluster.get('id')}: confidence="
+ f"{(parsed or {}).get('confidence', 'unparsable')}, escalating to {args.escalation_model}",
+ file=sys.stderr,
+ )
+ _reset_workspace(args)
+ esc_raw = call_backend(
+ prompt,
+ args.backend,
+ args.escalation_timeout or args.timeout,
+ args.escalation_model,
+ args.escalation_reasoning_effort,
+ args.trace_file,
+ f"{cluster.get('id')}-escalated",
+ )
+ esc_parsed = parse_model_json(esc_raw)
+ # Keep the first answer if the escalated call produced nothing usable.
+ if esc_parsed or not parsed:
+ raw, parsed = esc_raw, esc_parsed
+ model_used = args.escalation_model
+ escalated = True
result = {
"cluster_id": cluster.get("id"),
"signature": cluster.get("signature"),
"occurrences": cluster.get("occurrences"),
"tests": cluster.get("tests", []),
+ "model": model_used if args.backend != "none" else "",
+ "escalated": escalated,
}
if parsed:
# Low-confidence answers must not carry a fix/patch (requirement 3.3).
@@ -361,7 +420,7 @@ def main():
default="true",
help="Set to a falsy value (false/0/no) to force the no-model 'none' backend",
)
- parser.add_argument("--timeout", type=int, default=300, help="Per-call timeout in seconds")
+ parser.add_argument("--timeout", type=int, default=600, help="Per-call timeout in seconds")
parser.add_argument("--model", default="", help="Copilot model to use; empty uses the CLI default")
parser.add_argument(
"--reasoning-effort",
@@ -369,6 +428,23 @@ def main():
default="",
help="Copilot reasoning effort; empty uses the CLI default",
)
+ parser.add_argument(
+ "--escalation-model",
+ default="",
+ help="Stronger model to re-run a cluster with when the first answer is not high confidence; empty disables",
+ )
+ parser.add_argument(
+ "--escalation-reasoning-effort",
+ choices=["none", "minimal", "low", "medium", "high", "xhigh", "max"],
+ default="",
+ help="Reasoning effort for the escalation model; empty uses the CLI default",
+ )
+ parser.add_argument(
+ "--escalation-timeout",
+ type=int,
+ default=0,
+ help="Per-call timeout in seconds for the escalation model; 0 reuses --timeout",
+ )
parser.add_argument("--trace-file", default="", help="JSONL file logging each AI call for auditing")
parser.add_argument("--project-root", default="", help="Repository checkout root the AI may inspect")
parser.add_argument("--log-dir", default="", help="Directory holding the full raw failure logs")
diff --git a/.azure-pipelines/scripts/ai_analysis/merge_reports.py b/.azure-pipelines/scripts/ai_analysis/merge_reports.py
index d634eae2c..ff4e944c7 100644
--- a/.azure-pipelines/scripts/ai_analysis/merge_reports.py
+++ b/.azure-pipelines/scripts/ai_analysis/merge_reports.py
@@ -125,6 +125,9 @@ def build_unknown_section(clusters: list[dict], analyses_by_id: dict) -> list[st
lines.append("")
lines.append(f"- **Category:** {analysis.get('category', 'Unknown')}")
lines.append(f"- **Confidence:** {analysis.get('confidence', 'low')}")
+ if analysis.get("model"):
+ suffix = " (escalated: first answer was not high confidence)" if analysis.get("escalated") else ""
+ lines.append(f"- **Model:** {analysis['model']}{suffix}")
if analysis.get("root_cause"):
lines.append(f"- **Root cause:** {analysis['root_cause']}")
if analysis.get("suggested_fix"):
@@ -147,6 +150,7 @@ def build_unknown_section(clusters: list[dict], analyses_by_id: dict) -> list[st
"> **Notes**",
"> - Only top-3 issues receive AI analysis.",
'> - To request an additional fix from Copilot, use "Quote reply" on the PR comment and @mention Copilot.',
+ "> - Was this analysis helpful? React with 👍 if it was, or 👎 if it wasn't.",
"",
]
)
diff --git a/.azure-pipelines/scripts/ai_analysis/post_pr_comment.py b/.azure-pipelines/scripts/ai_analysis/post_pr_comment.py
index 5ca4585d5..f9dd74d9e 100644
--- a/.azure-pipelines/scripts/ai_analysis/post_pr_comment.py
+++ b/.azure-pipelines/scripts/ai_analysis/post_pr_comment.py
@@ -6,6 +6,8 @@ breaks the pipeline.
With ``--hide`` the script instead minimizes the marker comment as OUTDATED,
used when a rerun passes so the stale failure report collapses automatically.
+If a later run fails again, the hidden comment is left collapsed: its marker is
+retired and a fresh comment carrying the active marker is posted instead.
"""
import argparse
@@ -42,11 +44,6 @@ def _api(url: str, token: str, method: str = "GET", payload: "dict | None" = Non
return json.loads(body) if body else {}
-def _find_existing(repo: str, pr: int, token: str, marker: str) -> "int | None":
- c = _find_existing_comment(repo, pr, token, marker)
- return c.get("id") if c else None
-
-
def _find_existing_comment(repo: str, pr: int, token: str, marker: str) -> "dict | None":
page = 1
while True:
@@ -89,6 +86,33 @@ def _minimize_comment(node_id: str, token: str, classifier: str = "OUTDATED") ->
return bool(result.get("data", {}).get("minimizeComment", {}).get("minimizedComment", {}).get("isMinimized"))
+def _is_minimized(node_id: str, token: str) -> bool:
+ """Return True if the comment is currently minimized (REST does not expose this)."""
+ query = "query($id: ID!) { node(id: $id) { ... on Minimizable { isMinimized } } }"
+ result = _graphql(token, query, {"id": node_id})
+ return bool(((result.get("data") or {}).get("node") or {}).get("isMinimized"))
+
+
+def _retired_marker(marker: str) -> str:
+ """Return a marker variant that no longer matches ``marker`` (used for superseded comments)."""
+ return marker.replace("ai-failure-analysis", "ai-failure-analysis-retired", 1)
+
+
+def _retire_comment(repo: str, comment: dict, token: str, marker: str) -> None:
+ """Swap the hidden marker on an old comment so future runs no longer match it.
+
+ Only the invisible HTML marker changes; the visible report stays as-is.
+ """
+ body = (comment.get("body") or "").replace(marker, _retired_marker(marker))
+ _api(
+ f"https://api.github.com/repos/{repo}/issues/comments/{comment['id']}",
+ token,
+ method="PATCH",
+ payload={"body": body},
+ )
+ print(f"Retired marker on minimized PR comment {comment['id']}.", file=sys.stderr)
+
+
def _hide(repo: str, pr: int, token: str, marker: str) -> None:
"""Find the marker comment and hide it as OUTDATED (used when the run passes)."""
try:
@@ -156,7 +180,19 @@ def main():
pr = int(args.pr)
try:
- existing = _find_existing(args.repo, pr, token, marker)
+ existing_comment = _find_existing_comment(args.repo, pr, token, marker)
+ existing = existing_comment.get("id") if existing_comment else None
+ # A comment hidden as OUTDATED by an earlier passing run may be buried deep in a
+ # long PR history; leave it collapsed and post a fresh comment instead.
+ if existing is not None and existing_comment.get("node_id"):
+ try:
+ minimized = _is_minimized(existing_comment["node_id"], token)
+ except (HTTPError, URLError, ValueError) as e:
+ print(f"Warning: could not check comment visibility ({e}); updating in place.", file=sys.stderr)
+ minimized = False
+ if minimized:
+ _retire_comment(args.repo, existing_comment, token, marker)
+ existing = None
if existing is not None:
_api(
f"https://api.github.com/repos/{args.repo}/issues/comments/{existing}",
diff --git a/.azure-pipelines/template/ai-stage-template.yml b/.azure-pipelines/template/ai-stage-template.yml
index 9a7da495e..8411b1282 100644
--- a/.azure-pipelines/template/ai-stage-template.yml
+++ b/.azure-pipelines/template/ai-stage-template.yml
@@ -2,10 +2,11 @@
# Split into two stages because a job condition cannot read a previous stage's
# overall result (`stageDependencies.<stage>` only exposes per-job nodes, so
# `stageDependencies.<stage>.result` is a dictionary miss -> Null). A stage-level
-# condition can read `dependencies.<stage>.result`, so we branch there instead.
+# condition can check previous stage results (failed()/succeeded()), so we branch there instead.
parameters:
+ # A single stage name or a list of stage names (e.g. the per-part CUDA stages).
- name: dependsOn
- type: string
+ type: object
default: "Unit_test"
- name: stageName
type: string
@@ -29,12 +30,14 @@ stages:
pool:
vmImage: "ubuntu-latest"
dependsOn: ${{ parameters.dependsOn }}
- condition: and(eq(dependencies.${{ parameters.dependsOn }}.result, 'Failed'), eq(variables['System.StageAttempt'], '1'))
+ # failed() with no arguments checks the direct dependencies listed in dependsOn.
+ condition: and(failed(), eq(variables['System.StageAttempt'], '1'))
variables:
- group: "auto-round-bot-secrets"
jobs:
- job: AnalyzeFailures
displayName: "Analyze failed unit tests"
+ timeoutInMinutes: 90
variables:
base: $(Build.SourcesDirectory)
output_dir: $(Build.SourcesDirectory)/ai_analysis_results
@@ -109,6 +112,8 @@ stages:
--backend copilot \
--model gpt-5.6-luna \
--reasoning-effort max \
+ --escalation-model claude-opus-5.5 \
+ --escalation-reasoning-effort high \
--trace-file "$(output_dir)/ai_trace.jsonl" \
--project-root "$(base)" \
--log-dir "$(output_dir)/failed_logs"
@@ -163,7 +168,8 @@ stages:
pool:
vmImage: "ubuntu-latest"
dependsOn: ${{ parameters.dependsOn }}
- condition: in(dependencies.${{ parameters.dependsOn }}.result, 'Succeeded', 'SucceededWithIssues')
+ # succeeded() with no arguments checks the direct dependencies listed in dependsOn.
+ condition: succeeded()
variables:
- group: "auto-round-bot-secrets"
jobs:
diff --git a/.azure-pipelines/unit-test-cuda.yml b/.azure-pipelines/unit-test-cuda.yml
index 6bd214d4b..fc3db8f79 100644
--- a/.azure-pipelines/unit-test-cuda.yml
+++ b/.azure-pipelines/unit-test-cuda.yml
@@ -267,6 +267,14 @@ stages:
env:
PYTHONUNBUFFERED: '1'
+ - template: template/ai-stage-template.yml
+ parameters:
+ dependsOn:
+ - ${{ each pair in parameters.matrix }}:
+ - GPU_Test_${{ pair.key }}
+ enableAiAnalysis: true
+ postComment: true
+
- stage: Coverage
displayName: "Collect Coverage"
pool: