Skip to content

Automate browser based workflows with AI

Add QA report persistence to /qa skill (#SKY-8305) (#5059)

Marc Kelechavafa892f152f7a
diff --git a/skyvern/cli/mcp_tools/prompts.py b/skyvern/cli/mcp_tools/prompts.py
index 2dd4068d1..0d55ecab2 100644
--- a/skyvern/cli/mcp_tools/prompts.py
+++ b/skyvern/cli/mcp_tools/prompts.py
@@ -800,6 +800,68 @@ Use the evidence that actually matters:
---
+## Step 5: Post Evidence to PR
+
+After generating the QA report, persist it to the pull request as a sticky comment so the
+evidence survives beyond the conversation.
+
+### Check for an open PR
+
+```bash
+PR_NUMBER=$(gh pr view --json number -q '.number' 2>/dev/null)
+```
+
+If no PR exists for the current branch:
+1. Save the full report markdown to `.qa/latest-report.md` in the project root (create the directory if needed).
+2. Tell the user: "No open PR found for this branch. QA report saved to `.qa/latest-report.md`. Run /qa again after creating a PR to post it."
+3. Stop here — do not attempt to create a PR.
+
+### Post or update the sticky comment
+
+Use a hidden HTML marker to make the comment idempotent across multiple runs:
+
+```bash
+# Prepare the comment body with the hidden marker
+COMMENT_BODY="<!-- skyvern-qa-report -->
+## QA Report — $(git rev-parse --short HEAD) — $(date -u +%Y-%m-%dT%H:%M:%SZ)
+
+<the full report markdown from Step 4>
+"
+
+# Find an existing QA comment on the PR
+EXISTING_COMMENT_ID=$(gh api "repos/{owner}/{repo}/issues/${PR_NUMBER}/comments" \\
+ --jq '.[] | select(.body | test("skyvern-qa-report")) | .id' \\
+ 2>/dev/null | head -1)
+
+if [ -n "$EXISTING_COMMENT_ID" ]; then
+ # Update the existing comment in place
+ gh api "repos/{owner}/{repo}/issues/comments/${EXISTING_COMMENT_ID}" \\
+ -X PATCH -f body="$COMMENT_BODY"
+else
+ # Create a new comment
+ gh pr comment "$PR_NUMBER" --body "$COMMENT_BODY"
+fi
+```
+
+### Screenshot handling
+
+Screenshots taken during QA (via `skyvern_screenshot()`) are saved locally for the agent's
+verification. They are not uploaded to the PR comment because GitHub's API does not support
+image uploads in issue comments. The text report describes what was observed.
+
+If the user asks to preserve screenshots, save them to `.qa/screenshots/` and tell the user
+the local path. Do not include local file paths in the PR comment — they are meaningless to
+other reviewers.
+
+### Rules
+
+- Always include the `<!-- skyvern-qa-report -->` marker so repeated runs update the same comment instead of creating duplicates.
+- Include the short commit hash and UTC timestamp in the comment header.
+- Do not create a PR just to post a QA report — that is the user's decision.
+- If `gh` is not available or not authenticated, fall back to saving the report locally and tell the user.
+
+---
+
## Tool Selection
| What you need | Tool | Speed |
diff --git a/skyvern/cli/skills/qa/SKILL.md b/skyvern/cli/skills/qa/SKILL.md
index 754acae50..e2071209e 100644
--- a/skyvern/cli/skills/qa/SKILL.md
+++ b/skyvern/cli/skills/qa/SKILL.md
@@ -366,6 +366,66 @@ Report the evidence that actually matters:
- status codes and response snippets for backend API results
- command + failing assertion for unit/integration tests
+## Step 6: Post Evidence to PR
+
+After generating the QA report, persist it to the pull request as a sticky comment so the
+evidence survives beyond the conversation.
+
+### Check for an open PR
+
+```bash
+PR_NUMBER=$(gh pr view --json number -q '.number' 2>/dev/null)
+```
+
+If no PR exists for the current branch:
+1. Save the full report markdown to `.qa/latest-report.md` in the project root (create the directory if needed).
+2. Tell the user: "No open PR found for this branch. QA report saved to `.qa/latest-report.md`. Run /qa again after creating a PR to post it."
+3. Stop here — do not attempt to create a PR.
+
+### Post or update the sticky comment
+
+Use a hidden HTML marker to make the comment idempotent across multiple runs:
+
+```bash
+# Prepare the comment body with the hidden marker
+COMMENT_BODY="<!-- skyvern-qa-report -->
+## QA Report — $(git rev-parse --short HEAD) — $(date -u +%Y-%m-%dT%H:%M:%SZ)
+
+<the full report markdown from Step 5>
+"
+
+# Find an existing QA comment on the PR
+EXISTING_COMMENT_ID=$(gh api "repos/{owner}/{repo}/issues/${PR_NUMBER}/comments" \
+ --jq '.[] | select(.body | test("skyvern-qa-report")) | .id' \
+ 2>/dev/null | head -1)
+
+if [ -n "$EXISTING_COMMENT_ID" ]; then
+ # Update the existing comment in place
+ gh api "repos/{owner}/{repo}/issues/comments/${EXISTING_COMMENT_ID}" \
+ -X PATCH -f body="$COMMENT_BODY"
+else
+ # Create a new comment
+ gh pr comment "$PR_NUMBER" --body "$COMMENT_BODY"
+fi
+```
+
+### Screenshot handling
+
+Screenshots taken during QA (via `skyvern_screenshot()`) are saved locally for the agent's
+verification. They are not uploaded to the PR comment because GitHub's API does not support
+image uploads in issue comments. The text report describes what was observed.
+
+If the user asks to preserve screenshots, save them to `.qa/screenshots/` and tell the user
+the local path. Do not include local file paths in the PR comment — they are meaningless to
+other reviewers.
+
+### Rules
+
+- Always include the `<!-- skyvern-qa-report -->` marker so repeated runs update the same comment instead of creating duplicates.
+- Include the short commit hash and UTC timestamp in the comment header.
+- Do not create a PR just to post a QA report — that is the user's decision.
+- If `gh` is not available or not authenticated, fall back to saving the report locally and tell the user.
+
## Error Handling
| Problem | Action |
diff --git a/tests/unit/test_qa_skill_content.py b/tests/unit/test_qa_skill_content.py
index 811b25722..12c13b5c3 100644
--- a/tests/unit/test_qa_skill_content.py
+++ b/tests/unit/test_qa_skill_content.py
@@ -1,11 +1,21 @@
from __future__ import annotations
+import subprocess
+import sys
from pathlib import Path
+import pytest
+
from skyvern.cli.mcp_tools.prompts import QA_TEST_CONTENT, qa_test
ROOT = Path(__file__).resolve().parents[2]
BUNDLED_QA_SKILL = ROOT / "skyvern" / "cli" / "skills" / "qa" / "SKILL.md"
+CLAUDE_QA_SKILL = ROOT / ".claude" / "skills" / "qa" / "SKILL.md"
+
+_needs_cloud_repo = pytest.mark.skipif(
+ not CLAUDE_QA_SKILL.exists(),
+ reason=".claude/skills/qa/SKILL.md not present (OSS checkout)",
+)
def _first_nonempty_line_after_h1(text: str) -> str:
@@ -21,6 +31,11 @@ def _first_nonempty_line_after_h1(text: str) -> str:
return ""
+@_needs_cloud_repo
+def test_bundled_and_claude_qa_skill_match_exactly() -> None:
+ assert BUNDLED_QA_SKILL.read_text(encoding="utf-8") == CLAUDE_QA_SKILL.read_text(encoding="utf-8")
+
+
def test_qa_skill_has_summary_line_before_note_comment() -> None:
skill_text = BUNDLED_QA_SKILL.read_text(encoding="utf-8")
first_line_after_h1 = _first_nonempty_line_after_h1(skill_text)
@@ -78,3 +93,32 @@ def test_qa_test_prompt_includes_target_url_and_focus_area() -> None:
assert "Target URL: `http://localhost:8000`" in rendered
assert "Focus area: validate the workflow filters API" in rendered
assert "choose the correct validation mode" in rendered
+
+
+def test_qa_pr_evidence_markers_present() -> None:
+ """Assert the PR evidence posting instructions are present in all /qa surfaces."""
+ skill_text = BUNDLED_QA_SKILL.read_text(encoding="utf-8")
+
+ # Check SKILL.md
+ assert "<!-- skyvern-qa-report -->" in skill_text
+ assert "Post Evidence to PR" in skill_text
+ assert ".qa/latest-report.md" in skill_text
+ assert "gh pr comment" in skill_text
+
+ # Check QA_TEST_CONTENT (MCP prompt)
+ assert "<!-- skyvern-qa-report -->" in QA_TEST_CONTENT
+ assert "Post Evidence to PR" in QA_TEST_CONTENT
+ assert ".qa/latest-report.md" in QA_TEST_CONTENT
+ assert "gh pr comment" in QA_TEST_CONTENT
+
+
+@_needs_cloud_repo
+def test_validate_skills_package_script_passes() -> None:
+ result = subprocess.run(
+ [sys.executable, str(ROOT / "scripts" / "validate_skills_package.py")],
+ cwd=ROOT,
+ capture_output=True,
+ text=True,
+ check=False,
+ )
+ assert result.returncode == 0, result.stdout + result.stderr