Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Add QA report persistence to /qa skill (#SKY-8305) (#5059)
Marc Kelechavafa892f152f7a
diff --git a/skyvern/cli/mcp_tools/prompts.py b/skyvern/cli/mcp_tools/prompts.pyindex 2dd4068d1..0d55ecab2 100644--- a/skyvern/cli/mcp_tools/prompts.py+++ b/skyvern/cli/mcp_tools/prompts.py@@ -800,6 +800,68 @@ Use the evidence that actually matters:---+## Step 5: Post Evidence to PR++After generating the QA report, persist it to the pull request as a sticky comment so the+evidence survives beyond the conversation.++### Check for an open PR++```bash+PR_NUMBER=$(gh pr view --json number -q '.number' 2>/dev/null)+```++If no PR exists for the current branch:+1. Save the full report markdown to `.qa/latest-report.md` in the project root (create the directory if needed).+2. Tell the user: "No open PR found for this branch. QA report saved to `.qa/latest-report.md`. Run /qa again after creating a PR to post it."+3. Stop here — do not attempt to create a PR.++### Post or update the sticky comment++Use a hidden HTML marker to make the comment idempotent across multiple runs:++```bash+# Prepare the comment body with the hidden marker+COMMENT_BODY="<!-- skyvern-qa-report -->+## QA Report — $(git rev-parse --short HEAD) — $(date -u +%Y-%m-%dT%H:%M:%SZ)++<the full report markdown from Step 4>+"++# Find an existing QA comment on the PR+EXISTING_COMMENT_ID=$(gh api "repos/{owner}/{repo}/issues/${PR_NUMBER}/comments" \\+ --jq '.[] | select(.body | test("skyvern-qa-report")) | .id' \\+ 2>/dev/null | head -1)++if [ -n "$EXISTING_COMMENT_ID" ]; then+ # Update the existing comment in place+ gh api "repos/{owner}/{repo}/issues/comments/${EXISTING_COMMENT_ID}" \\+ -X PATCH -f body="$COMMENT_BODY"+else+ # Create a new comment+ gh pr comment "$PR_NUMBER" --body "$COMMENT_BODY"+fi+```++### Screenshot handling++Screenshots taken during QA (via `skyvern_screenshot()`) are saved locally for the agent's+verification. They are not uploaded to the PR comment because GitHub's API does not support+image uploads in issue comments. The text report describes what was observed.++If the user asks to preserve screenshots, save them to `.qa/screenshots/` and tell the user+the local path. Do not include local file paths in the PR comment — they are meaningless to+other reviewers.++### Rules++- Always include the `<!-- skyvern-qa-report -->` marker so repeated runs update the same comment instead of creating duplicates.+- Include the short commit hash and UTC timestamp in the comment header.+- Do not create a PR just to post a QA report — that is the user's decision.+- If `gh` is not available or not authenticated, fall back to saving the report locally and tell the user.++---+## Tool Selection| What you need | Tool | Speed |diff --git a/skyvern/cli/skills/qa/SKILL.md b/skyvern/cli/skills/qa/SKILL.mdindex 754acae50..e2071209e 100644--- a/skyvern/cli/skills/qa/SKILL.md+++ b/skyvern/cli/skills/qa/SKILL.md@@ -366,6 +366,66 @@ Report the evidence that actually matters:- status codes and response snippets for backend API results- command + failing assertion for unit/integration tests+## Step 6: Post Evidence to PR++After generating the QA report, persist it to the pull request as a sticky comment so the+evidence survives beyond the conversation.++### Check for an open PR++```bash+PR_NUMBER=$(gh pr view --json number -q '.number' 2>/dev/null)+```++If no PR exists for the current branch:+1. Save the full report markdown to `.qa/latest-report.md` in the project root (create the directory if needed).+2. Tell the user: "No open PR found for this branch. QA report saved to `.qa/latest-report.md`. Run /qa again after creating a PR to post it."+3. Stop here — do not attempt to create a PR.++### Post or update the sticky comment++Use a hidden HTML marker to make the comment idempotent across multiple runs:++```bash+# Prepare the comment body with the hidden marker+COMMENT_BODY="<!-- skyvern-qa-report -->+## QA Report — $(git rev-parse --short HEAD) — $(date -u +%Y-%m-%dT%H:%M:%SZ)++<the full report markdown from Step 5>+"++# Find an existing QA comment on the PR+EXISTING_COMMENT_ID=$(gh api "repos/{owner}/{repo}/issues/${PR_NUMBER}/comments" \+ --jq '.[] | select(.body | test("skyvern-qa-report")) | .id' \+ 2>/dev/null | head -1)++if [ -n "$EXISTING_COMMENT_ID" ]; then+ # Update the existing comment in place+ gh api "repos/{owner}/{repo}/issues/comments/${EXISTING_COMMENT_ID}" \+ -X PATCH -f body="$COMMENT_BODY"+else+ # Create a new comment+ gh pr comment "$PR_NUMBER" --body "$COMMENT_BODY"+fi+```++### Screenshot handling++Screenshots taken during QA (via `skyvern_screenshot()`) are saved locally for the agent's+verification. They are not uploaded to the PR comment because GitHub's API does not support+image uploads in issue comments. The text report describes what was observed.++If the user asks to preserve screenshots, save them to `.qa/screenshots/` and tell the user+the local path. Do not include local file paths in the PR comment — they are meaningless to+other reviewers.++### Rules++- Always include the `<!-- skyvern-qa-report -->` marker so repeated runs update the same comment instead of creating duplicates.+- Include the short commit hash and UTC timestamp in the comment header.+- Do not create a PR just to post a QA report — that is the user's decision.+- If `gh` is not available or not authenticated, fall back to saving the report locally and tell the user.+## Error Handling| Problem | Action |diff --git a/tests/unit/test_qa_skill_content.py b/tests/unit/test_qa_skill_content.pyindex 811b25722..12c13b5c3 100644--- a/tests/unit/test_qa_skill_content.py+++ b/tests/unit/test_qa_skill_content.py@@ -1,11 +1,21 @@from __future__ import annotations+import subprocess+import sysfrom pathlib import Path+import pytest+from skyvern.cli.mcp_tools.prompts import QA_TEST_CONTENT, qa_testROOT = Path(__file__).resolve().parents[2]BUNDLED_QA_SKILL = ROOT / "skyvern" / "cli" / "skills" / "qa" / "SKILL.md"+CLAUDE_QA_SKILL = ROOT / ".claude" / "skills" / "qa" / "SKILL.md"++_needs_cloud_repo = pytest.mark.skipif(+ not CLAUDE_QA_SKILL.exists(),+ reason=".claude/skills/qa/SKILL.md not present (OSS checkout)",+)def _first_nonempty_line_after_h1(text: str) -> str:@@ -21,6 +31,11 @@ def _first_nonempty_line_after_h1(text: str) -> str:return ""+@_needs_cloud_repo+def test_bundled_and_claude_qa_skill_match_exactly() -> None:+ assert BUNDLED_QA_SKILL.read_text(encoding="utf-8") == CLAUDE_QA_SKILL.read_text(encoding="utf-8")++def test_qa_skill_has_summary_line_before_note_comment() -> None:skill_text = BUNDLED_QA_SKILL.read_text(encoding="utf-8")first_line_after_h1 = _first_nonempty_line_after_h1(skill_text)@@ -78,3 +93,32 @@ def test_qa_test_prompt_includes_target_url_and_focus_area() -> None:assert "Target URL: `http://localhost:8000`" in renderedassert "Focus area: validate the workflow filters API" in renderedassert "choose the correct validation mode" in rendered+++def test_qa_pr_evidence_markers_present() -> None:+ """Assert the PR evidence posting instructions are present in all /qa surfaces."""+ skill_text = BUNDLED_QA_SKILL.read_text(encoding="utf-8")++ # Check SKILL.md+ assert "<!-- skyvern-qa-report -->" in skill_text+ assert "Post Evidence to PR" in skill_text+ assert ".qa/latest-report.md" in skill_text+ assert "gh pr comment" in skill_text++ # Check QA_TEST_CONTENT (MCP prompt)+ assert "<!-- skyvern-qa-report -->" in QA_TEST_CONTENT+ assert "Post Evidence to PR" in QA_TEST_CONTENT+ assert ".qa/latest-report.md" in QA_TEST_CONTENT+ assert "gh pr comment" in QA_TEST_CONTENT+++@_needs_cloud_repo+def test_validate_skills_package_script_passes() -> None:+ result = subprocess.run(+ [sys.executable, str(ROOT / "scripts" / "validate_skills_package.py")],+ cwd=ROOT,+ capture_output=True,+ text=True,+ check=False,+ )+ assert result.returncode == 0, result.stdout + result.stderr