Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
backend-aware /qa and clarify local auth diagnostics (#5058)
Marc Kelechava0a69e28cb2b4
diff --git a/skyvern-frontend/src/components/SelfHealApiKeyBanner.tsx b/skyvern-frontend/src/components/SelfHealApiKeyBanner.tsxindex 79dffadeb..f05da6c86 100644--- a/skyvern-frontend/src/components/SelfHealApiKeyBanner.tsx+++ b/skyvern-frontend/src/components/SelfHealApiKeyBanner.tsx@@ -13,11 +13,11 @@ type BannerStatus = Exclude<AuthStatusValue, "ok"> | "error";function getCopy(status: BannerStatus): { title: string; description: string } {switch (status) {- case "missing_env":+ case "missing_api_key":return {- title: "Skyvern API key missing",+ title: "Frontend API key missing",description:- "All requests from the UI to the local backend will fail until a valid key is configured.",+ "The UI is not sending an x-api-key header. The backend server can still run locally, but authenticated requests from the UI will fail until VITE_SKYVERN_API_KEY is set or a runtime key is stored.",};case "invalid_format":return {@@ -76,7 +76,7 @@ function SelfHealApiKeyBanner() {return null;}- const copy = getCopy(bannerStatus ?? "missing_env");+ const copy = getCopy(bannerStatus ?? "missing_api_key");const queryErrorMessage = error?.message ?? null;const handleRepair = async () => {@@ -166,6 +166,12 @@ function SelfHealApiKeyBanner() {by running <code>skyvern init</code> or click the button belowto regenerate it automatically.</p>+ {data?.detail ? (+ <p className="text-xs text-slate-300">{data.detail}</p>+ ) : null}+ {data?.next_step ? (+ <p className="text-xs text-slate-300">{data.next_step}</p>+ ) : null}{isProductionBuild && (<p className="text-yellow-300">When running a production build, the regenerated API key isdiff --git a/skyvern-frontend/src/hooks/useAuthDiagnostics.ts b/skyvern-frontend/src/hooks/useAuthDiagnostics.tsindex 608b99cd4..0e1345aea 100644--- a/skyvern-frontend/src/hooks/useAuthDiagnostics.ts+++ b/skyvern-frontend/src/hooks/useAuthDiagnostics.ts@@ -4,7 +4,7 @@ import axios from "axios";import { getClient } from "@/api/AxiosClient";export type AuthStatusValue =- | "missing_env"+ | "missing_api_key"| "invalid_format"| "invalid"| "expired"@@ -13,6 +13,8 @@ export type AuthStatusValue =export type AuthDiagnosticsResponse = {status: AuthStatusValue;+ detail?: string;+ next_step?: string;fingerprint?: string;organization_id?: string;expires_at?: number;diff --git a/skyvern/cli/mcp_tools/prompts.py b/skyvern/cli/mcp_tools/prompts.pyindex 4d76a1dff..2dd4068d1 100644--- a/skyvern/cli/mcp_tools/prompts.py+++ b/skyvern/cli/mcp_tools/prompts.py@@ -463,12 +463,13 @@ def extract_data(# 2. .claude/skills/qa/SKILL.md (project-local copy for this repo)# 3. skyvern/cli/mcp_tools/prompts.py (QA_TEST_CONTENT — this file)QA_TEST_CONTENT = """\-# QA — Test Frontend Changes in a Real Browser+# QA — Validate Frontend and Backend Changes-You read the developer's code diff, understand what UI was affected, open a real browser-against their running dev server, and test that the changes actually work.+Read the diff, classify what changed, and run the right validation path: browser QA for frontend/browser+changes, API validation for backend surface changes, repo-native validation for backend-internal changes,+and both for mixed changes.-This is NOT a generic website crawler. It tests what CHANGED based on the code diff.+This is NOT a generic crawler. It should validate the changed behavior with the right tools.---@@ -479,176 +480,181 @@ This is NOT a generic website crawler. It tests what CHANGED based on the code d```bash# What files changed?git diff --name-only HEAD~1 # vs last commit (if changes are committed)-git diff --name-only # vs working tree (if uncommitted)+git diff --name-only # vs working tree (if uncommitted)# Full diff for context-git diff HEAD~1 # or git diff for uncommitted+git diff HEAD~1 # or git diff for uncommitted```-Pick whichever diff has content. If both are empty, there's nothing to QA.+Pick whichever diff has content. If both are empty, there is nothing diff-driven to QA.### Read the changed files-For every changed frontend file (`.tsx`, `.jsx`, `.ts`, `.js`, `.css`, `.html`):-- Read the FULL file (not just the diff) to understand the component-- Look for: route paths, component names, text labels, form fields, button labels,- API endpoints called, conditional rendering, error states+Read the full contents of every changed file that affects behavior:-### Classify the changes+- Frontend files: `.tsx`, `.jsx`, `.ts`, `.js`, `.css`, `.html`+- Backend/API files: routes, controllers, request/response schemas, serializers, handlers+- Backend-internal files: services, workers, validators, business logic, data-layer code+- Tests that changed alongside the implementation-| Change Type | What to Test |-|-------------|-------------|-| New component/page | Navigate to it, verify it renders, interact with its elements |-| Modified component | Navigate to it, verify the specific change works (new button, new text, new behavior) |-| Styling changes | Navigate, screenshot, verify layout isn't broken |-| API integration | Navigate, trigger the action, verify the API call works (check network, verify UI updates) |-| Form changes | Fill the form, submit, verify validation and success states |-| Route changes | Navigate to old and new routes, verify routing works |-| Shared component (used in many places) | Test 2-3 pages that use it |-| Bug fix | Reproduce the original bug scenario, verify it's fixed |+Look for routes, UI text, forms, API parameters, response fields, validation logic, error states,+and any tests that describe the intended behavior.-### Generate test cases+### Classify the diff-For each changed file, write specific test cases. Example:+| Mode | Trigger | Primary validation |+|------|---------|--------------------|+| Frontend/browser | UI/routes/components/styles changed | Browser QA against the dev server |+| Backend API | Route handlers, request/response schemas, or externally visible API behavior changed | Start backend locally and run targeted API requests |+| Backend-internal | Services/workers/business logic changed without public API surface changes | Repo-native fast checks plus targeted tests |+| Mixed | Frontend/browser and backend changed together | Backend validation first, then frontend/browser QA |-If the diff shows changes to `LoginForm.tsx` adding a "Forgot password" link:-```-Test 1: Login page renders the new "Forgot password" link- - Navigate to /login- - Assert: link with text "Forgot password" exists- - Click it- - Assert: navigated to /forgot-password (or modal appeared)--Test 2: Login form still works (regression)- - Navigate to /login- - Verify email and password inputs exist- - Submit empty form, verify validation errors appear-```+Rules:-**Be specific.** Don't write "verify the page works." Write "verify the 'Forgot password'-link navigates to /forgot-password."+- If both frontend/browser and backend changed, use `Mixed`.+- If only backend internals changed, do not invent unrelated browser tests or random API calls.+- If a backend change might affect the public contract, inspect routes, schemas, and tests before treating it as internal-only.+- If the diff is mostly documentation or comments, keep QA lightweight and report that no behavioral validation was warranted.----## Step 2: Find the Dev Server+## Step 2: Choose the Validation Strategy-If a URL was provided, use it. Otherwise, auto-detect:+### Frontend/browser mode-```-# Try common dev server ports-# 5173 (Vite), 3000 (Next/CRA), 3001, 8080, 8000, 4200 (Angular)-```+Use browser automation against the dev server. Validate the specific UI changes plus 1-2 nearby regressions.++### Backend API mode++Read the repo's local instructions first, start the backend if needed, identify the changed+endpoint(s), and run targeted HTTP requests to validate the changed contract.++### Backend-internal mode++Run the repo's fast verification commands first, then targeted unit/integration/scenario tests.+Only do live API calls if the internal change affects exposed behavior.++### Mixed mode-Navigate to each until one responds. If none respond, tell the user:-"Start your dev server first, then try again."+Validate the backend first, then run frontend/browser QA against the flow that depends on it.+If the backend contract is broken, frontend results are not trustworthy.----## Step 3: Connect to a Browser+## Step 3A: Frontend/Browser QA-Try these in order:+### Find the dev server++If a URL was provided, use it. Otherwise try common local ports:-### Option A: Local browser (fastest — works when MCP server runs locally)+```text+5173, 3000, 3001, 8080, 8000, 4200```++If none respond, start the most direct repo-documented local command for the+changed surface. If the diff needs both frontend and backend running together+and the repo provides a combined frontend/backend dev script, prefer that.+Only ask the user to start something manually if the repo has no documented+command or startup fails.++### Connect to a browser++Try these in order:++#### Option A: Local browser (fastest)+```textskyvern_browser_session_create(local=true, headless=false, timeout=15)```-Use `local=true` so the browser can reach `localhost`. Use `headless=false` so the user can watch.--### Option B: Local browser via tunnel (for remote/cloud-hosted MCP + localhost dev server)+Use `local=true` so the browser can reach `localhost`.-If local session creation fails — typically because the MCP server is running remotely-(e.g., api.skyvern.com) — the cloud browser cannot reach `localhost`. The user needs to-run a local browser and tunnel it.+#### Option B: Local browser via tunnel-Tell the user to run these in separate terminals:+If local session creation fails because the MCP server is remote, the cloud browser cannot reach+`localhost`. Tell the user to run:```bash# Terminal 1: Launch a local browser with CDP exposedskyvern browser serve --port 9222-# Terminal 2: Tunnel it to the internet (install: brew install ngrok)+# Terminal 2: Tunnel it to the internetngrok http 9222```-Then connect to the ngrok URL:-```+Then connect:++```textskyvern_browser_session_connect(cdp_url="wss://<ngrok-subdomain>.ngrok-free.app/devtools/browser/<id>")```-The user can get the browser ID from the `skyvern browser serve` output or by calling-the ngrok URL's `/json` endpoint.+The user can get the browser ID from the `skyvern browser serve` output or by calling the+ngrok URL's `/json` endpoint.-### Option C: Cloud browser (for publicly accessible URLs only)-```+#### Option C: Cloud browser+```textskyvern_browser_session_create(timeout=15)```-Only works if the dev server URL is publicly accessible. Warn the user that `localhost`-URLs will not work with cloud browsers.+Only works for publicly reachable URLs. `localhost` URLs will not work here.----+### Generate frontend/browser test cases-## Step 4: Run the Tests+For each changed frontend file, write targeted checks. Example:-For each test case generated in Step 1:+```text+Test 1: Settings page renders the new "Retry failed run" button+ - Navigate to /settings/runs+ - Assert: button exists+ - Click it+ - Assert: success toast appears-### Navigate+Test 2: Adjacent regression+ - Verify the existing "Delete run" action still works or is still visible```++### Run the frontend/browser tests++Navigate:++```textskyvern_navigate(url="http://localhost:<port>/<route>")```-### Health gate (after every navigate, ~10ms)-```+Health gate:++```textskyvern_evaluate(expression="(() => {const errors = [];const body = document.body?.innerText || '';if (body.includes('Something went wrong')) errors.push('error_message');if (body.includes('Cannot read properties')) errors.push('js_error_in_ui');- if (/\\bundefined\\b/.test(body) && !/\\bif\\b|\\btypeof\\b|\\bdocument|tutorial|example/i.test(body) \-&& body.length < 5000) errors.push('undefined_text');+ if (/\\bundefined\\b/.test(body) && !/\\bif\\b|\\btypeof\\b|\\bdocument|tutorial|example/i.test(body) && body.length < 5000) errors.push('undefined_text');if (body.includes('connection refused')) errors.push('connection_refused');if (/sign.?in|log.?in|auth/i.test(window.location.pathname)) errors.push('auth_redirect');- if (document.querySelector('[role=\"alert\"]')) errors.push('alert_element');- if (!document.querySelector('main, [role=\"main\"], nav, header, h1, h2, \-[class*=\"layout\" i], [class*=\"page\" i], [class*=\"app\" i]'))+ if (document.querySelector('[role=\\"alert\\"]')) errors.push('alert_element');+ if (!document.querySelector('main, [role=\\"main\\"], nav, header, h1, h2, [class*=\\"layout\\" i], [class*=\\"page\\" i], [class*=\\"app\\" i]'))errors.push('blank_page');return JSON.stringify({ pass: errors.length === 0, errors });})()")```-### Assert with DOM queries (prefer `skyvern_evaluate` — fast, deterministic)-```-# Element exists-skyvern_evaluate(expression="!!document.querySelector('a[href=\"/forgot-password\"]')")+Prefer deterministic DOM assertions:-# Text content+```text+skyvern_evaluate(expression="!!document.querySelector('button')")skyvern_evaluate(expression="document.querySelector('h1')?.textContent?.trim()")--# Element count-skyvern_evaluate(expression="document.querySelectorAll('.card').length")--# URL after navigationskyvern_evaluate(expression="window.location.pathname")```-### Interact (use `skyvern_act` for natural language actions)-```-skyvern_act(prompt="Click the 'Forgot password' link")-skyvern_act(prompt="Fill the email field with '[email protected]' and click Submit")-skyvern_act(prompt="Open the dropdown menu and select 'Settings'")-```--### Visual checks (use `skyvern_validate` only when DOM queries aren't enough)-```-skyvern_validate(prompt="The login form shows email and password fields with a blue Submit button")-```+Use interaction tools when needed:-### Screenshot (after every significant action)-```+```text+skyvern_act(prompt="Click the 'Retry failed run' button")+skyvern_validate(prompt="The page shows the success toast and the form is no longer loading")skyvern_screenshot()```-### Failed network requests (once per page)-```+Also check for failed network requests once per page:++```textskyvern_evaluate(expression="(() => {const entries = performance.getEntriesByType('resource').filter(e => e.responseStatus >= 400);return JSON.stringify({ failed: entries.map(e => ({ url: e.name, status: e.responseStatus })).slice(0, 5) });@@ -657,36 +663,141 @@ skyvern_evaluate(expression="(() => {----## Step 5: Report Results+## Step 3B: Backend API QA++### Gather repo-local context first++Before starting the server or sending requests, read the repo's local instructions:++- `README`, `AGENTS.md`, `CLAUDE.md`, `Makefile`, `package.json`, `pyproject.toml`+- changed tests for the affected endpoints+- docs that describe auth, local ports, or startup commands++Do not guess the startup command if the repo already documents one.++### Start the backend if needed++If the backend is not responding on the expected local port:++1. Start it with the most direct repo-documented local command for the changed+ surface. If the validation needs both frontend and backend and the repo+ documents a combined dev environment command, prefer that over inventing+ separate startup steps.+2. Wait for readiness+3. Confirm the API is reachable before sending requests++### Identify the changed API surface++Use the diff to answer:++- Which endpoints changed?+- Which request parameters, headers, or bodies changed?+- Which response fields or status codes changed?+- Did auth requirements change?+- Is there a mutation side effect that needs follow-up verification?++Read the full handler, schema, and any changed tests.++### Generate backend API test cases++For each changed endpoint, create targeted checks:++- Happy path with valid parameters+- Empty result or not-found path where applicable+- Invalid input or validation error path+- Combined filters or sorting behavior when relevant+- Follow-up read after mutation endpoints++Example:++```text+Test 1: GET /api/runs returns the new field in the response body+Test 2: GET /api/runs?status=missing returns an empty list, not a 500+Test 3: POST /api/runs rejects invalid payload with a 4xx validation error+Test 4: PATCH /api/runs/:id updates the record and a follow-up GET shows the change+```++### Execute the API requests++Use the repo's documented auth scheme and local base URL. Use `curl`, the repo SDK, or a small+one-off client if that is clearer than shell quoting.++Examples:++```bash+curl -sS -H "Authorization: Bearer <token>" \+ "http://localhost:<port>/api/..."++curl -sS -X POST \+ -H "Content-Type: application/json" \+ -H "<auth-header>: <token>" \+ -d '{"example":"value"}' \+ "http://localhost:<port>/api/..."+```++Capture:++- request being tested+- status code+- response body snippet or parsed result+- whether the changed field/behavior is present++If auth is required and local credentials are unavailable from repo docs, say so clearly and stop rather than faking coverage.++---++## Step 3C: Backend-Internal QA++If the diff is backend-only but does not change an exposed endpoint or UI flow:++1. Run the repo's fastest compile/type/lint checks for the changed files+2. Run targeted unit/integration/scenario tests that cover the changed logic+3. Add live API calls only if the internal change affects exposed behavior++Good checks:++- compile/type-check the changed files+- targeted `pytest`, `npm test`, `go test`, or equivalent+- scenario/integration tests around changed services or workflows++Bad checks:++- hitting an unrelated health endpoint and calling it validated+- browsing the UI when no frontend behavior changed+- calling random APIs just because the backend changed++---++## Step 4: Report Results```markdown## QA Report-### Changes Tested-Files: `LoginForm.tsx`, `ForgotPassword.tsx`-Diff summary: Added "Forgot password" link to login form, new /forgot-password page+### Validation Mode+- Mode: Backend API+- Scope: `routes/runs.py`, `schemas/run_response.py`### Results-| # | Test | Result | Screenshot |-|---|------|--------|------------|-| 1 | Login page renders "Forgot password" link | PASS | screenshot_1 |-| 2 | Clicking link navigates to /forgot-password | PASS | screenshot_2 |-| 3 | Forgot password page renders form | PASS | screenshot_3 |-| 4 | Login form still works (regression) | PASS | screenshot_4 |-| 5 | Empty form shows validation errors | FAIL | screenshot_5 |+| # | Test | Result | Evidence |+|---|------|--------|----------|+| 1 | GET /api/runs returns `retryable` for valid runs | PASS | HTTP 200, field present |+| 2 | GET /api/runs?status=missing returns empty list | PASS | HTTP 200, `[]` |+| 3 | GET /api/runs?status=invalid returns validation error | PASS | HTTP 422 |+| 4 | Frontend runs page still renders filter state | PASS | screenshot_3 |### Issues Found-1. **Empty login form submits without validation** — Submitting with no email/password- doesn't show error messages. The form submits and the page reloads.- Expected: validation errors. Screenshot: screenshot_5--### Network-- No failed requests detected+1. `retryable` is missing from one serializer branch.### Verdict-4/5 tests passed. 1 issue found: missing form validation on empty submit.+3/4 tests passed. 1 issue found.```+Use the evidence that actually matters:++- screenshots for frontend/browser results+- status codes and response snippets for backend API results+- command + failing assertion for unit/integration tests+---## Tool Selection@@ -695,13 +806,12 @@ Diff summary: Added "Forgot password" link to login form, new /forgot-password p|---------------|------|-------|| Check element exists, text, count, URL | `skyvern_evaluate` | ~10ms || Click, type, fill forms, multi-step interaction | `skyvern_act` | 5-30s |-| "Does this look right?" visual check | `skyvern_validate` | 15-50s |-| Get structured data from a page | `skyvern_extract` | 15-50s |+| Visual check | `skyvern_validate` | 15-50s || Screenshot | `skyvern_screenshot` | ~1s || Wait for async content | `skyvern_wait` | varies |-**Default to `skyvern_evaluate` for assertions.** Only use `skyvern_validate` when you-can't express the check as a DOM query (visual layout, "does this look like a dashboard").+Default to `skyvern_evaluate` for frontend/browser assertions. For backend API validation, prefer simple+HTTP requests and deterministic response checks over browser-based inference.---@@ -709,61 +819,65 @@ can't express the check as a DOM query (visual layout, "does this look like a da| Problem | Action ||---------|--------|-| No git diff found | Ask the user what they want to test, fall back to explore mode |-| Dev server not running | Tell user to start it. Suggest common commands (npm run dev, etc.) |-| Auth redirect on page | Report it. Ask if they want to provide credentials or skip that route. |-| Component doesn't render | Screenshot + check console. Report with the specific error. |-| Session create fails | Try cloud fallback. Warn about URL accessibility. |+| No git diff found | Ask what behavior to validate, then fall back to explore mode |+| Frontend dev server not running | Start the most direct repo-documented local command for the changed surface; prefer a combined dev command only when the validation needs both frontend and backend; only ask the user if no documented command exists or startup fails |+| Backend server not running | Start the most direct repo-documented local command for the changed surface; prefer a combined dev environment command only when the validation needs both sides |+| Cannot identify changed endpoint | Read changed routes, schemas, and tests before proceeding |+| Auth required but no local creds available | Report the blocker clearly; do not fake coverage |+| Component doesn't render | Capture screenshot and specific UI error |+| API returns unexpected 5xx | Save request/response evidence and report the regression |## Session Cleanup-ALWAYS close the session when done, even if errors occurred:-```+Always close browser sessions when done:++```textskyvern_browser_session_close()```+If you started local servers or background processes, leave a clear note about what is still running.+---## Fallback: Explore Mode-If there's no git diff (user just wants a general QA pass), fall back to exploring:+If there is no useful diff:-1. Navigate to the root URL-2. Extract nav links and page structure with `skyvern_extract`-3. Visit each major route, health gate + screenshot-4. Test interactive elements (forms, buttons, links)-5. Report findings+1. Ask what behavior should be validated+2. Use browser QA for frontend flows+3. Use targeted API checks for backend flows+4. Report findings with the same evidence standard-But the primary mode is **diff-driven**. Always try to read the code changes first.+The primary mode is still **diff-driven**. Always try to understand the code changes first."""def qa_test(url: Annotated[str,- Field(description="Dev server URL to test against, e.g. 'http://localhost:3000'"),+ Field(description="Frontend or API base URL to test against, e.g. 'http://localhost:3000'"),] = "",context: Annotated[str,- Field(description="What to focus on, e.g. 'test the checkout flow' or 'verify the login page'"),+ Field(description="What to focus on, e.g. 'test the checkout flow' or 'validate the workflow filters API'"),] = "",) -> str:- """QA test frontend changes in a real browser.+ """QA test recent code changes with the right validation path.- Reads the developer's git diff to understand what code changed, generates targeted- browser tests, runs them against a local dev server using Skyvern's browser tools,- and reports pass/fail with screenshots. Invoke this when a developer wants to verify- their UI changes work correctly.+ Reads the developer's git diff to understand what changed, classifies the work as+ frontend/browser, backend API, backend-internal, or mixed, and then runs targeted validation.+ Frontend checks use Skyvern browser tools; backend validation uses repo-native startup,+ HTTP requests, and targeted tests where appropriate."""parts = [QA_TEST_CONTENT]if url or context:parts.append("\n---\n")if url:- parts.append(f"Dev server URL: `{url}`\n")+ parts.append(f"Target URL: `{url}`\n")if context:parts.append(f"Focus area: {context}\n\n"- "Start at Step 1: run `git diff` to understand the changes, then generate test cases "- "focused on the area described above."+ "Start at Step 1: run `git diff` to understand the changes, choose the correct validation "+ "mode, and focus the resulting checks on the area described above.")return "\n".join(parts)diff --git a/skyvern/cli/skills/qa/SKILL.md b/skyvern/cli/skills/qa/SKILL.mdindex 9d3d19431..754acae50 100644--- a/skyvern/cli/skills/qa/SKILL.md+++ b/skyvern/cli/skills/qa/SKILL.md@@ -1,38 +1,36 @@---name: qa-description: "QA test your frontend changes in a real browser. Reads your git diff, generates targeted browser tests, runs them against your local dev server, and reports pass/fail with screenshots."+description: "QA test your code changes by reading your git diff, choosing the right validation path for frontend/browser and backend changes, and reporting pass/fail with evidence."----# QA — Test Your Frontend Changes in a Real Browser+# QA — Validate Frontend and Backend Changes++Read the diff, classify what changed, and run the right validation path: browser QA for frontend/browser changes, API validation for backend surface changes, repo-native validation for backend-internal changes, and both for mixed changes.<!-- NOTE: This content is maintained in three places — keep all in sync:1. skyvern/cli/skills/qa/SKILL.md (bundled with pip package — canonical)2. .claude/skills/qa/SKILL.md (project-local copy for this repo)3. skyvern/cli/mcp_tools/prompts.py (QA_TEST_CONTENT for the MCP prompt) -->-You changed some code. This skill reads your diff, understands what UI was affected, opens-a real browser against your running dev server, and tests that your changes actually work.+You changed code. This skill is diff-driven first: it reads what changed, understands the+affected behavior, and validates that behavior with the right tools. It is not a generic+website crawler, and it should not invent random API checks that are unrelated to the diff.## Quick Start-```-/qa # Diff-based: test what you changed-/qa http://localhost:3000 # Same, explicit URL-/qa -- test the checkout flow # Targeted: test specific behavior+```text+/qa # Diff-based: choose the right validation path automatically+/qa http://localhost:3000 # Same, explicit frontend URL+/qa -- validate the workflow filters API```## How It Works-1. **Read your code changes** (`git diff`) to understand what you modified-2. **Read the changed files** to understand the UI: routes, components, props, text-3. **Generate targeted test cases** based on what the code actually does-4. **Open a browser** against your running dev server-5. **Run the tests** — navigate, interact, assert, screenshot-6. **Report** pass/fail with evidence--This is NOT a generic website crawler. It tests YOUR changes specifically.-----+1. Read the code changes from `git diff`+2. Read the changed files to understand behavior, routes, schemas, and UI+3. Classify the diff as `frontend/browser`, `backend API`, `backend-internal`, or `mixed`+4. Run the right validation flow+5. Report pass/fail with concrete evidence## Step 1: Understand the Changes@@ -41,103 +39,161 @@ This is NOT a generic website crawler. It tests YOUR changes specifically.```bash# What files changed?git diff --name-only HEAD~1 # vs last commit (if changes are committed)-git diff --name-only # vs working tree (if uncommitted)+git diff --name-only # vs working tree (if uncommitted)# Full diff for context-git diff HEAD~1 # or git diff for uncommitted+git diff HEAD~1 # or git diff for uncommitted```-Pick whichever diff has content. If both are empty, there's nothing to QA.+Pick whichever diff has content. If both are empty, there is nothing diff-driven to QA.### Read the changed files-For every changed frontend file (`.tsx`, `.jsx`, `.ts`, `.js`, `.css`, `.html`):-- Read the FULL file (not just the diff) to understand the component-- Look for: route paths, component names, text labels, form fields, button labels,- API endpoints called, conditional rendering, error states+Read the full contents of every changed file that affects behavior:-### Classify the changes+- Frontend files: `.tsx`, `.jsx`, `.ts`, `.js`, `.css`, `.html`+- Backend/API files: routes, controllers, request/response schemas, serializers, handlers+- Backend-internal files: services, workers, business logic, validators, data-layer code+- Tests that changed alongside the implementation-| Change Type | What to Test |-|-------------|-------------|-| New component/page | Navigate to it, verify it renders, interact with its elements |-| Modified component | Navigate to it, verify the specific change works (new button, new text, new behavior) |-| Styling changes | Navigate, screenshot, verify layout isn't broken |-| API integration | Navigate, trigger the action, verify the API call works (check network, verify UI updates) |-| Form changes | Fill the form, submit, verify validation and success states |-| Route changes | Navigate to old and new routes, verify routing works |-| Shared component (used in many places) | Test 2-3 pages that use it |-| Bug fix | Reproduce the original bug scenario, verify it's fixed |+Look for:-### Generate test cases+- route paths and page entry points+- component names, visible text, forms, buttons, error states+- API endpoints, request params, response fields, auth requirements+- validation logic, branching behavior, feature flags, empty states+- tests that describe the expected behavior-For each changed file, write specific test cases. Example:+## Step 2: Classify the Diff-If the diff shows changes to `LoginForm.tsx` adding a "Forgot password" link:-```-Test 1: Login page renders the new "Forgot password" link- - Navigate to /login- - Assert: link with text "Forgot password" exists- - Click it- - Assert: navigated to /forgot-password (or modal appeared)+| Mode | Trigger | Primary validation |+|------|---------|--------------------|+| Frontend/browser | UI/routes/components/styles changed | Browser QA against the dev server |+| Backend API | Route handlers, request/response schemas, or externally visible API behavior changed | Start backend locally and run targeted API requests |+| Backend-internal | Services/workers/business logic changed without public API surface changes | Repo-native fast checks plus targeted tests |+| Mixed | Frontend/browser and backend changed together | Backend validation first, then frontend/browser QA |-Test 2: Login form still works (regression)- - Navigate to /login- - Verify email and password inputs exist- - Submit empty form, verify validation errors appear-```+Use these rules:-**Be specific.** Don't write "verify the page works." Write "verify the 'Forgot password'-link navigates to /forgot-password."+- If both frontend/browser and backend changed, treat it as `Mixed`.+- If only backend internals changed, do not invent unrelated browser tests or random API calls.+- If a backend change might affect the public contract, inspect routes, schemas, and tests before choosing `backend-internal`.+- If the diff is mostly documentation or comments, keep QA lightweight and report that no behavioral validation was warranted.----+## Step 3: Choose the Validation Strategy-## Step 2: Find the Dev Server+### Frontend/browser mode-If the user provided a URL, use it. Otherwise, auto-detect:+Use browser automation against the dev server. Validate the specific UI changes plus 1-2+adjacent regression checks.-```-# Try common dev server ports-# 5173 (Vite), 3000 (Next/CRA), 3001, 8080, 8000, 4200 (Angular)-```+### Backend API mode-Navigate to each until one responds. If none respond, tell the user:-"Start your dev server first, then run `/qa` again."+Use the repo's documented local startup and auth instructions, start the backend if needed,+identify the changed endpoint(s), and run targeted HTTP requests to validate the changed contract.----+### Backend-internal mode++Run the repo's fast verification commands first, then targeted unit/integration/scenario tests+for the changed logic. Only start the backend and do live API calls if the change affects exposed behavior.-## Step 3: Connect to a Browser+### Mixed mode+Validate the backend first, then run frontend/browser QA against the flow that depends on it.+If the backend contract is broken, frontend results are not trustworthy.++## Step 4A: Frontend/Browser QA++### Find the dev server++If the user provided a URL, use it. Otherwise auto-detect common local ports:++```text+5173, 3000, 3001, 8080, 8000, 4200```++If none respond, start the most direct repo-documented local command for the+changed surface. If the diff needs both frontend and backend running together+and the repo provides a combined frontend/backend dev script, prefer that.+Only ask the user to start something manually if the repo has no documented+command or startup fails.++### Connect to a browser++Try these in order:++#### Option A: Local browser (fastest)++```textskyvern_browser_session_create(local=true, headless=false, timeout=15)```-Use `local=true` so it can reach `localhost`. Use `headless=false` so the user can watch.+Use `local=true` so the browser can reach `localhost`.++#### Option B: Local browser via tunnel++If local session creation fails because the MCP server is remote, the cloud browser cannot reach+`localhost`. Tell the user to run:++```bash+# Terminal 1: Launch a local browser with CDP exposed+skyvern browser serve --port 9222++# Terminal 2: Tunnel it to the internet+ngrok http 9222+```++Then connect:-If local fails, fall back to cloud (warn that URL must be publicly accessible):+```text+skyvern_browser_session_connect(cdp_url="wss://<ngrok-subdomain>.ngrok-free.app/devtools/browser/<id>")```++The user can get the browser ID from the `skyvern browser serve` output or by calling the+ngrok URL's `/json` endpoint.++#### Option C: Cloud browser++```textskyvern_browser_session_create(timeout=15)```----+Only works for publicly reachable URLs. `localhost` URLs will not work here.++### Generate frontend/browser test cases-## Step 4: Run the Tests+For each changed frontend file, create targeted checks. Examples:-For each test case generated in Step 1:+```text+Test 1: Settings page renders the new "Retry failed run" button+ - Navigate to /settings/runs+ - Assert: button with text "Retry failed run" exists+ - Click it+ - Assert: success toast appears-### Navigate+Test 2: Adjacent regression+ - Verify the existing "Delete run" action still works or is still visible```++Be specific. Do not write "verify the page works."++### Run the frontend/browser tests++For each test case:++```textskyvern_navigate(url="http://localhost:<port>/<route>")```-### Health gate (after every navigate, ~10ms)-```+Health gate after navigation:++```textskyvern_evaluate(expression="(() => {const errors = [];const body = document.body?.innerText || '';if (body.includes('Something went wrong')) errors.push('error_message');if (body.includes('Cannot read properties')) errors.push('js_error_in_ui');- if (/\bundefined\b/.test(body) && !/\bif\b|\btypeof\b|\bdocument|tutorial|example/i.test(body) && body.length < 5000) errors.push('undefined_text');+ if (/\\bundefined\\b/.test(body) && !/\\bif\\b|\\btypeof\\b|\\bdocument|tutorial|example/i.test(body) && body.length < 5000) errors.push('undefined_text');if (body.includes('connection refused')) errors.push('connection_refused');if (/sign.?in|log.?in|auth/i.test(window.location.pathname)) errors.push('auth_redirect');if (document.querySelector('[role=\"alert\"]')) errors.push('alert_element');@@ -147,124 +203,198 @@ skyvern_evaluate(expression="(() => {})()")```-### Assert with DOM queries (prefer `skyvern_evaluate` — fast, deterministic)-```-# Element exists-skyvern_evaluate(expression="!!document.querySelector('a[href=\"/forgot-password\"]')")+Prefer deterministic DOM assertions:-# Text content+```text+skyvern_evaluate(expression="!!document.querySelector('button')")skyvern_evaluate(expression="document.querySelector('h1')?.textContent?.trim()")--# Element count-skyvern_evaluate(expression="document.querySelectorAll('.card').length")--# URL after navigationskyvern_evaluate(expression="window.location.pathname")```-### Interact (use `skyvern_act` for natural language actions)-```-skyvern_act(prompt="Click the 'Forgot password' link")-skyvern_act(prompt="Fill the email field with '[email protected]' and click Submit")-skyvern_act(prompt="Open the dropdown menu and select 'Settings'")-```--### Visual checks (use `skyvern_validate` only when DOM queries aren't enough)-```-skyvern_validate(prompt="The login form shows email and password fields with a blue Submit button")-```+Use interaction tools when needed:-### Screenshot (after every significant action)-```+```text+skyvern_act(prompt="Click the 'Retry failed run' button")+skyvern_act(prompt="Fill the email field with '[email protected]' and click Submit")+skyvern_validate(prompt="The page shows the success toast and the form is no longer loading")skyvern_screenshot()```-### Failed network requests (once per page)-```+Also check for failed network requests once per page:++```textskyvern_evaluate(expression="(() => {const entries = performance.getEntriesByType('resource').filter(e => e.responseStatus >= 400);return JSON.stringify({ failed: entries.map(e => ({ url: e.name, status: e.responseStatus })).slice(0, 5) });})()")```----+## Step 4B: Backend API QA++### Gather repo-local context first++Before starting the server or sending requests, read the repo's local instructions:++- `README`, `AGENTS.md`, `CLAUDE.md`, `Makefile`, `package.json`, `pyproject.toml`+- existing test files for the changed endpoints+- any docs that describe auth, local ports, or startup commands++Do not guess the startup command if the repo already documents one.++### Start the backend if needed++If the backend is not already responding on the expected local port:++1. Start it with the most direct repo-documented local command for the changed+ surface. If the validation needs both frontend and backend and the repo+ documents a combined dev environment command, prefer that over inventing+ separate startup steps.+2. Wait for readiness+3. Confirm the API is reachable before sending validation requests++If the repo requires background processes, start them in the background and keep notes on how you did it.++### Identify the changed API surface++Use the diff to answer:++- Which endpoints changed?+- Which request parameters, headers, or bodies changed?+- Which response fields or status codes changed?+- Did auth requirements change?+- Is there a create/update/delete side effect that needs follow-up verification?++Do not stop at the route file. Read the full handler, schema, and any changed tests.++### Generate backend API test cases++For each changed endpoint, create targeted checks:++- Happy path with valid parameters+- Empty result or not-found path where applicable+- Invalid input or validation error path+- Combined filters or sorting behavior when relevant+- Follow-up read after mutation endpoints++Examples:++```text+Test 1: GET /api/runs returns the new field in the response body+Test 2: GET /api/runs?status=missing returns an empty list, not a 500+Test 3: POST /api/runs rejects invalid payload with a 4xx validation error+Test 4: PATCH /api/runs/:id updates the record and a follow-up GET shows the change+```++### Execute the API requests++Use the repo's documented auth scheme and local base URL. Use `curl`, the repo SDK, or a small+one-off client if that is clearer than shell quoting. Prefer simple, inspectable commands.++Examples:++```bash+curl -sS -H "Authorization: Bearer <token>" \+ "http://localhost:<port>/api/..."++curl -sS -X POST \+ -H "Content-Type: application/json" \+ -H "<auth-header>: <token>" \+ -d '{"example":"value"}' \+ "http://localhost:<port>/api/..."+```++Capture:++- request being tested+- status code+- response body snippet or parsed result+- whether the changed field/behavior is present++If the endpoint is authenticated and you cannot obtain local credentials from repo docs, say so clearly and stop rather than faking coverage.++## Step 4C: Backend-Internal QA++If the diff is backend-only but does not change an exposed endpoint or UI flow:++1. Run the repo's fastest compile/type/lint checks for the changed files+2. Run targeted unit/integration/scenario tests that cover the changed logic+3. Add live API calls only if the internal change affects exposed behavior++Examples of appropriate checks:++- compile/type-check the changed files+- targeted `pytest`, `npm test`, `go test`, or equivalent+- scenario/integration tests around changed services or workflows++Examples of inappropriate checks:++- hitting an unrelated health endpoint and calling it "validated"+- browsing the UI when no frontend behavior changed+- calling random APIs just because the backend changed## Step 5: Report Results```markdown## QA Report+### Validation Mode+- Mode: Backend API+- Scope: `routes/runs.py`, `schemas/run_response.py`+### Changes Tested-Files: `LoginForm.tsx`, `ForgotPassword.tsx`-Diff summary: Added "Forgot password" link to login form, new /forgot-password page+- Added `retryable` field to run responses+- Updated `status` filter handling### Results-| # | Test | Result | Screenshot |-|---|------|--------|------------|-| 1 | Login page renders "Forgot password" link | PASS | screenshot_1 |-| 2 | Clicking link navigates to /forgot-password | PASS | screenshot_2 |-| 3 | Forgot password page renders form | PASS | screenshot_3 |-| 4 | Login form still works (regression) | PASS | screenshot_4 |-| 5 | Empty form shows validation errors | FAIL | screenshot_5 |+| # | Test | Result | Evidence |+|---|------|--------|----------|+| 1 | GET /api/runs returns `retryable` for valid runs | PASS | HTTP 200, field present in response |+| 2 | GET /api/runs?status=missing returns empty list | PASS | HTTP 200, `[]` |+| 3 | GET /api/runs?status=invalid returns validation error | PASS | HTTP 422 |+| 4 | Frontend runs page still renders filter state | PASS | screenshot_3 |### Issues Found-1. **Empty login form submits without validation** — Submitting with no email/password- doesn't show error messages. The form submits and the page reloads.- Expected: validation errors. Screenshot: screenshot_5--### Network-- No failed requests detected+1. `retryable` is missing from one branch of the response serializer.### Verdict-4/5 tests passed. 1 issue found: missing form validation on empty submit.+3/4 tests passed. 1 issue found.```------## Tool Selection+Report the evidence that actually matters:-| What you need | Tool | Speed |-|---------------|------|-------|-| Check element exists, text, count, URL | `skyvern_evaluate` | ~10ms |-| Click, type, fill forms, multi-step interaction | `skyvern_act` | 5-30s |-| "Does this look right?" visual check | `skyvern_validate` | 15-50s |-| Get structured data from a page | `skyvern_extract` | 15-50s |-| Screenshot | `skyvern_screenshot` | ~1s |-| Wait for async content | `skyvern_wait` | varies |--**Default to `skyvern_evaluate` for assertions.** Only use `skyvern_validate` when you-can't express the check as a DOM query (visual layout, "does this look like a dashboard").-----+- screenshots for frontend/browser results+- status codes and response snippets for backend API results+- command + failing assertion for unit/integration tests## Error Handling| Problem | Action ||---------|--------|-| No git diff found | Ask user what they want to test, fall back to explore mode |-| Dev server not running | Tell user to start it. Suggest common commands (npm run dev, etc.) |-| Auth redirect on page | Report it. Ask if they want to provide credentials or skip that route. |-| Component doesn't render | Screenshot + check console. Report with the specific error. |-| Session create fails | Try cloud fallback. Warn about URL accessibility. |+| No git diff found | Ask what behavior to validate, then fall back to explore mode |+| Frontend dev server not running | Start the most direct repo-documented local command for the changed surface; prefer a combined dev command only when the validation needs both frontend and backend; only ask the user if no documented command exists or startup fails |+| Backend server not running | Start the most direct repo-documented local command for the changed surface; prefer a combined dev environment command only when the validation needs both sides |+| Cannot identify changed endpoint | Read changed routes, schemas, and tests before proceeding |+| Auth required but no local creds available | Report the blocker clearly; do not fake coverage |+| Component does not render | Capture screenshot and specific UI error |+| API returns unexpected 5xx | Save request/response evidence and report the regression |## Session Cleanup-ALWAYS close the session when done, even if errors occurred:-```+Always close browser sessions when done:++```textskyvern_browser_session_close()```----+If you started local servers or background processes, leave the user a clear note about what is still running.## Fallback: Explore Mode-If there's no git diff (user just wants a general QA pass), fall back to exploring:+If there is no useful diff, fall back to explicit exploration:-1. Navigate to the root URL-2. Extract nav links and page structure with `skyvern_extract`-3. Visit each major route, health gate + screenshot-4. Test interactive elements (forms, buttons, links)-5. Report findings+1. Ask what behavior should be validated+2. If it is a frontend flow, use browser QA+3. If it is a backend/API flow, run targeted local API checks+4. Report findings with the same evidence standard-But the primary mode is **diff-driven**. The agent should always try to read the code-changes first.+The primary mode is still **diff-driven**. Always try to understand the code changes first.diff --git a/skyvern/forge/sdk/routes/internal_auth.py b/skyvern/forge/sdk/routes/internal_auth.pyindex 517e14e13..f0cd2bbd2 100644--- a/skyvern/forge/sdk/routes/internal_auth.py+++ b/skyvern/forge/sdk/routes/internal_auth.py@@ -15,7 +15,7 @@ LOG = structlog.get_logger()class AuthStatus(str, Enum):- missing_env = "missing_env"+ missing_api_key = "missing_api_key"invalid_format = "invalid_format"invalid = "invalid"expired = "expired"@@ -28,6 +28,7 @@ class DiagnosticsResult(NamedTuple):detail: str | Nonevalidation: Any | Nonetoken: str | None+ next_step: str | None = Nonedef _is_local_request(request: Request) -> bool:@@ -63,7 +64,19 @@ def _require_local_access(request: Request) -> None:async def _evaluate_local_api_key(token: str) -> DiagnosticsResult:token_candidate = token.strip()if not token_candidate or token_candidate == "YOUR_API_KEY":- return DiagnosticsResult(status=AuthStatus.missing_env, detail=None, validation=None, token=None)+ return DiagnosticsResult(+ status=AuthStatus.missing_api_key,+ detail=(+ "No x-api-key header was provided. This endpoint validates the API key sent in the request, "+ "not whether a .env file exists."+ ),+ validation=None,+ token=None,+ next_step=(+ "Send the local API key in the x-api-key header, or call POST /api/v1/internal/auth/repair "+ "from localhost to regenerate a local key and update the env files."+ ),+ )try:validation = await resolve_org_from_api_key(token_candidate, app.DATABASE)@@ -111,10 +124,17 @@ def _emit_diagnostics(result: DiagnosticsResult) -> dict[str, object]:log_kwargs: dict[str, object] = {"status": status_value}if result.detail:log_kwargs["detail"] = result.detail+ if result.next_step:+ log_kwargs["next_step"] = result.next_stepLOG.warning("Local auth diagnostics", **log_kwargs)- return {"status": status_value}+ payload: dict[str, object] = {"status": status_value}+ if result.detail:+ payload["detail"] = result.detail+ if result.next_step:+ payload["next_step"] = result.next_step+ return payload@router.post("/repair", include_in_schema=False)diff --git a/tests/unit/test_qa_skill_content.py b/tests/unit/test_qa_skill_content.pynew file mode 100644index 000000000..811b25722--- /dev/null+++ b/tests/unit/test_qa_skill_content.py@@ -0,0 +1,80 @@+from __future__ import annotations++from pathlib import Path++from skyvern.cli.mcp_tools.prompts import QA_TEST_CONTENT, qa_test++ROOT = Path(__file__).resolve().parents[2]+BUNDLED_QA_SKILL = ROOT / "skyvern" / "cli" / "skills" / "qa" / "SKILL.md"+++def _first_nonempty_line_after_h1(text: str) -> str:+ after_h1 = False+ for raw_line in text.splitlines():+ line = raw_line.strip()+ if raw_line.startswith("# "):+ after_h1 = True+ continue+ if not after_h1 or not line:+ continue+ return line+ return ""+++def test_qa_skill_has_summary_line_before_note_comment() -> None:+ skill_text = BUNDLED_QA_SKILL.read_text(encoding="utf-8")+ first_line_after_h1 = _first_nonempty_line_after_h1(skill_text)+ assert first_line_after_h1+ assert not first_line_after_h1.startswith("<!--")+++def test_qa_skill_mentions_backend_validation_modes() -> None:+ skill_text = BUNDLED_QA_SKILL.read_text(encoding="utf-8")+ required_markers = [+ "# QA — Validate Frontend and Backend Changes",+ "Frontend/browser",+ "Backend API",+ "Backend-internal",+ "Mixed",+ "## Step 4B: Backend API QA",+ "## Step 4C: Backend-Internal QA",+ "skyvern browser serve --port 9222",+ "curl -sS",+ "If none respond, start the most direct repo-documented local command for the\nchanged surface.",+ "combined frontend/backend dev script",+ "The primary mode is still **diff-driven**.",+ ]+ for marker in required_markers:+ assert marker in skill_text+++def test_qa_prompt_mentions_backend_validation_modes() -> None:+ required_markers = [+ "# QA — Validate Frontend and Backend Changes",+ "Frontend/browser",+ "Backend-internal",+ "## Step 3B: Backend API QA",+ "## Step 3C: Backend-Internal QA",+ "Start it with the most direct repo-documented local command for the changed",+ "combined frontend/backend dev script",+ "skyvern browser serve --port 9222",+ 'curl -sS -H "Authorization: Bearer <token>"',+ "Default to `skyvern_evaluate` for frontend/browser assertions.",+ ]+ for marker in required_markers:+ assert marker in QA_TEST_CONTENT+++def test_qa_prompt_docs_only_lightweight_rule() -> None:+ assert "If the diff is mostly documentation or comments, keep QA lightweight" in QA_TEST_CONTENT+++def test_qa_prompt_mixed_mode_backend_contract_warning() -> None:+ assert "If the backend contract is broken, frontend results are not trustworthy" in QA_TEST_CONTENT+++def test_qa_test_prompt_includes_target_url_and_focus_area() -> None:+ rendered = qa_test(url="http://localhost:8000", context="validate the workflow filters API")+ assert "Target URL: `http://localhost:8000`" in rendered+ assert "Focus area: validate the workflow filters API" in rendered+ assert "choose the correct validation mode" in rendered