Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Make the VNC backoff test wait for React to commit instead of relying on timer luck (#8821)
Aaron Pereze5e988744afd
diff --git a/skyvern-frontend/src/components/BrowserStream.test.tsx b/skyvern-frontend/src/components/BrowserStream.test.tsxindex 2eb273a06..d52eec3d6 100644--- a/skyvern-frontend/src/components/BrowserStream.test.tsx+++ b/skyvern-frontend/src/components/BrowserStream.test.tsx@@ -250,6 +250,13 @@ function advance(ms: number) {return act(() => vi.advanceTimersByTimeAsync(ms));}+// Retries until React commits the expected state, polling on the real event+// loop (React's scheduler runs there, and fake time does not drive it).+// Interval 0 keeps vi.waitFor from also advancing fake time on every check.+function settled(assertion: () => void) {+ return vi.waitFor(assertion, { interval: 0 });+}+function sessionRequestCount() {return (mocks.apiGet.mock.calls as unknown[][]).filter(([url]) => url === "/browser_sessions/pbs_test",@@ -854,9 +861,13 @@ describe("BrowserStream", () => {try {const onStreamStateChange = vi.fn();renderBrowserStream({ onStreamStateChange });- await vi.advanceTimersByTimeAsync(1000);+ await settled(() =>+ expect(onStreamStateChange).toHaveBeenLastCalledWith(+ "live",+ "pbs_test",+ ),+ );expect(mocks.rfbInstances).toHaveLength(1);- expect(onStreamStateChange).toHaveBeenLastCalledWith("live", "pbs_test");mocks.autoConnect.value = false;// Each disconnect schedules exactly one delayed redial, up to the cap.@@ -867,13 +878,19 @@ describe("BrowserStream", () => {] as unknown as {emit: (type: string, detail?: unknown) => void;};- rfb.emit("disconnect", { clean: false });+ act(() => rfb.emit("disconnect", { clean: false }));// No immediate redial: the retry waits out its backoff delay.- await vi.advanceTimersByTimeAsync(0);+ // This absence check settles on act's trailing macrotask, which drains+ // only microtask-only chains (the mocked async getClient and+ // useCredentialGetter); if the dial path ever awaits real macrotasks,+ // it false-passes silently.+ await advance(0);expect(mocks.rfbInstances).toHaveLength(instanceCount);// Max delay 15s plus up to 50% jitter.- await vi.advanceTimersByTimeAsync(30000);- expect(mocks.rfbInstances).toHaveLength(instanceCount + 1);+ await advance(30000);+ await settled(() =>+ expect(mocks.rfbInstances).toHaveLength(instanceCount + 1),+ );}expect(onStreamStateChange).toHaveBeenLastCalledWith("connecting",@@ -886,12 +903,14 @@ describe("BrowserStream", () => {] as unknown as {emit: (type: string, detail?: unknown) => void;};- rfb.emit("disconnect", { clean: false });- await vi.advanceTimersByTimeAsync(120000);+ act(() => rfb.emit("disconnect", { clean: false }));+ await advance(120000);expect(mocks.rfbInstances).toHaveLength(instanceCount);- expect(onStreamStateChange).toHaveBeenLastCalledWith(- "stopped",- "pbs_test",+ await settled(() =>+ expect(onStreamStateChange).toHaveBeenLastCalledWith(+ "stopped",+ "pbs_test",+ ),);} finally {vi.useRealTimers();diff --git a/skyvern-frontend/src/routes/workflows/studio/runview/RunSummaryStrip.test.tsx b/skyvern-frontend/src/routes/workflows/studio/runview/RunSummaryStrip.test.tsxindex 49a19e11f..56476ecf7 100644--- a/skyvern-frontend/src/routes/workflows/studio/runview/RunSummaryStrip.test.tsx+++ b/skyvern-frontend/src/routes/workflows/studio/runview/RunSummaryStrip.test.tsx@@ -146,6 +146,26 @@ describe("RunSummaryStrip failure category", () => {expect(within(tooltip).getByText(/billing or credits issue/)).toBeTruthy();});+ test("a website's own server error blames the website, not Skyvern", async () => {+ renderStrip(+ makeRun({+ status: Status.Failed,+ failure_category: [+ {+ category: "WEBSITE_ERROR",+ confidence_float: 1,+ reasoning: "The site returned 502 Bad Gateway.",+ },+ ],+ }),+ );+ expect(screen.queryByText("Temporary system issue")).toBeNull();+ fireEvent.focus(screen.getByText("Website error"));+ const tooltip = await screen.findByRole("tooltip");+ expect(within(tooltip).getByText(/website/i)).toBeTruthy();+ expect(within(tooltip).queryByText(/Skyvern's side/)).toBeNull();+ });+test("a workflow author's own error code is shown verbatim with its description", async () => {renderStrip(makeRun({diff --git a/skyvern-frontend/src/util/failureCategoryDisplay.ts b/skyvern-frontend/src/util/failureCategoryDisplay.tsindex ffcb85e76..199518036 100644--- a/skyvern-frontend/src/util/failureCategoryDisplay.ts+++ b/skyvern-frontend/src/util/failureCategoryDisplay.ts@@ -94,6 +94,11 @@ const FAILURE_CATEGORY_DISPLAY = {description:"Something went wrong on Skyvern's side, such as a network or capacity problem. Retrying usually helps.",},+ WEBSITE_ERROR: {+ label: "Website error",+ description:+ "The website had a problem on its side, such as a 502 Bad Gateway, an outage, or maintenance. Nothing is wrong with your workflow. Retrying later usually helps.",+ },PARAMETER_BINDING_ERROR: {label: "Parameter setup failed",description:diff --git a/skyvern/forge/failure_classifier.py b/skyvern/forge/failure_classifier.pyindex 9a038b9e7..6daf1c432 100644--- a/skyvern/forge/failure_classifier.py+++ b/skyvern/forge/failure_classifier.py@@ -31,6 +31,7 @@ class FailureCategory(StrEnum):BUDGET_EXHAUSTED = "BUDGET_EXHAUSTED"LLM_REASONING_ERROR = "LLM_REASONING_ERROR"INFRASTRUCTURE_ERROR = "INFRASTRUCTURE_ERROR"+ WEBSITE_ERROR = "WEBSITE_ERROR"PARAMETER_BINDING_ERROR = "PARAMETER_BINDING_ERROR"UNKNOWN = "UNKNOWN"@@ -473,7 +474,7 @@ def classify_from_failure_reason(# Bump when the taxonomy or the category->component mapping below changes, so a frozen# coverage baseline stays reproducible per classifier_version.-CLASSIFIER_VERSION = 5+CLASSIFIER_VERSION = 6FAILURE_ATTRIBUTION_SCHEMA_VERSION = 1# Bounded sentinels — neither is an infra component id.@@ -581,6 +582,7 @@ _INFRA_COMPONENT_BY_CATEGORY = {_NON_INFRA_CATEGORIES = {"MAX_STEPS_EXCEEDED","BUDGET_EXHAUSTED",+ "WEBSITE_ERROR","ELEMENT_NOT_FOUND","DATA_EXTRACTION_FAILURE","LLM_REASONING_ERROR",diff --git a/skyvern/forge/prompts/skyvern/check-user-goal-with-termination.j2 b/skyvern/forge/prompts/skyvern/check-user-goal-with-termination.j2index a35b36c81..e0eead51f 100644--- a/skyvern/forge/prompts/skyvern/check-user-goal-with-termination.j2+++ b/skyvern/forge/prompts/skyvern/check-user-goal-with-termination.j2@@ -9,7 +9,7 @@ Make sure to ONLY return the JSON object in this format with no additional text"page_info": str, // Think step by step. Describe all the useful information in the page related to the user goal.{% endif %}"thoughts": str, // Think step by step. Explain your reasoning for the status you selected.{{ " Maximum 15 words, telegraphic style — drop articles and filler words." if slim_output == 'terse' else "" }}"status": str, // Must be one of three values: "complete", "terminate", or "continue". Use "complete" ONLY if the user goal has been fully achieved{{ " according to the complete criterion" if complete_criterion else "" }}. Use "terminate" ONLY if the goal CANNOT ever be achieved (e.g., a file doesn't exist, an error modal blocks progress permanently, or an explicit termination condition is met in the user goal). Use "continue" if the goal is not yet achieved but more steps could potentially achieve it (this is the most common case - use this when you need to wait, navigate, or try different actions).- "failure_categories": array // Only populate when status is "terminate". Classify the root cause. Categories: ANTI_BOT_DETECTION (CAPTCHA, Cloudflare, bot detection), BROWSER_ERROR (crash, page closed), NAVIGATION_FAILURE (404, redirect), PAGE_LOAD_TIMEOUT (timeout), AUTH_FAILURE (login failed, MFA), LLM_REASONING_ERROR (wrong element, wrong action), CREDENTIAL_ERROR (missing credentials), ELEMENT_NOT_FOUND (target missing), WRONG_PAGE_STATE (unexpected page), DATA_EXTRACTION_FAILURE (extraction failed), INFRASTRUCTURE_ERROR (network, OOM), UNKNOWN (cannot determine). Return one or more categories ordered by likelihood.+ "failure_categories": array // Only populate when status is "terminate". Classify the root cause. Categories: ANTI_BOT_DETECTION (CAPTCHA, Cloudflare challenge, bot detection), BROWSER_ERROR (crash, page closed), NAVIGATION_FAILURE (404, redirect), PAGE_LOAD_TIMEOUT (timeout), AUTH_FAILURE (login failed, MFA), LLM_REASONING_ERROR (wrong element, wrong action), CREDENTIAL_ERROR (missing credentials), ELEMENT_NOT_FOUND (target missing), WRONG_PAGE_STATE (unexpected page), DATA_EXTRACTION_FAILURE (extraction failed), WEBSITE_ERROR (the website's own server failed or is down: HTTP 5xx such as 500, 502 Bad Gateway, 503 Service Unavailable or 504, a CDN error page saying the site's host is down, an outage or maintenance page), INFRASTRUCTURE_ERROR (Skyvern's own platform failed: its workers, browser pool, proxies or network, or it ran out of memory; never a website's error page), UNKNOWN (cannot determine). Return one or more categories ordered by likelihood.[{"category": str, // The category from the list above"confidence_float": float, // 0.0 to 1.0diff --git a/skyvern/forge/prompts/skyvern/summarize-max-retries-reason.j2 b/skyvern/forge/prompts/skyvern/summarize-max-retries-reason.j2index 95854640f..c720fc51b 100644--- a/skyvern/forge/prompts/skyvern/summarize-max-retries-reason.j2+++ b/skyvern/forge/prompts/skyvern/summarize-max-retries-reason.j2@@ -11,7 +11,7 @@ Make sure to ONLY return the JSON object in this format with no additional text"reasoning": str, // The reasoning behind the error. Be specific, referencing any user information and their fields in your reasoning. Keep the reasoning short and to the point."confidence_float": float // The confidence of the error. Pick a number between 0.0 and 1.0. 0.0 means no confidence, 1.0 means full confidence}]{% endif %},- "failure_categories": array // Classify the root cause of this failure. Return one or more categories ordered by likelihood. Always return at least one. Categories: ANTI_BOT_DETECTION (CAPTCHA, Cloudflare, bot detection), BROWSER_ERROR (crash, page closed), NAVIGATION_FAILURE (404, redirect), PAGE_LOAD_TIMEOUT (timeout), AUTH_FAILURE (login failed, MFA), LLM_REASONING_ERROR (wrong element, wrong action), CREDENTIAL_ERROR (missing credentials), ELEMENT_NOT_FOUND (target missing), WRONG_PAGE_STATE (unexpected page), DATA_EXTRACTION_FAILURE (extraction failed), MAX_STEPS_EXCEEDED (use only as secondary, always identify root cause as primary), INFRASTRUCTURE_ERROR (network, OOM), UNKNOWN (cannot determine).+ "failure_categories": array // Classify the root cause of this failure. Return one or more categories ordered by likelihood. Always return at least one. Categories: ANTI_BOT_DETECTION (CAPTCHA, Cloudflare challenge, bot detection), BROWSER_ERROR (crash, page closed), NAVIGATION_FAILURE (404, redirect), PAGE_LOAD_TIMEOUT (timeout), AUTH_FAILURE (login failed, MFA), LLM_REASONING_ERROR (wrong element, wrong action), CREDENTIAL_ERROR (missing credentials), ELEMENT_NOT_FOUND (target missing), WRONG_PAGE_STATE (unexpected page), DATA_EXTRACTION_FAILURE (extraction failed), MAX_STEPS_EXCEEDED (use only as secondary, always identify root cause as primary), WEBSITE_ERROR (the website's own server failed or is down: HTTP 5xx such as 500, 502 Bad Gateway, 503 Service Unavailable or 504, a CDN error page saying the site's host is down, an outage or maintenance page), INFRASTRUCTURE_ERROR (Skyvern's own platform failed: its workers, browser pool, proxies or network, or it ran out of memory; never a website's error page), UNKNOWN (cannot determine).[{"category": str, // The category from the list above"confidence_float": float, // 0.0 to 1.0diff --git a/skyvern/forge/prompts/skyvern/summarize-max-steps-reason.j2 b/skyvern/forge/prompts/skyvern/summarize-max-steps-reason.j2index 465f3125b..53a2409b0 100644--- a/skyvern/forge/prompts/skyvern/summarize-max-steps-reason.j2+++ b/skyvern/forge/prompts/skyvern/summarize-max-steps-reason.j2@@ -11,7 +11,7 @@ Make sure to ONLY return the JSON object in this format with no additional text"reasoning": str, // The reasoning behind the error. Be specific, referencing any user information and their fields in your reasoning. Keep the reasoning short and to the point."confidence_float": float // The confidence of the error. Pick a number between 0.0 and 1.0. 0.0 means no confidence, 1.0 means full confidence}]{% endif %},- "failure_categories": array // Classify the root cause of this failure. Return one or more categories ordered by likelihood. Always return at least one. Categories: ANTI_BOT_DETECTION (CAPTCHA, Cloudflare, bot detection), BROWSER_ERROR (crash, page closed), NAVIGATION_FAILURE (404, redirect), PAGE_LOAD_TIMEOUT (timeout), AUTH_FAILURE (login failed, MFA), LLM_REASONING_ERROR (wrong element, wrong action), CREDENTIAL_ERROR (missing credentials), ELEMENT_NOT_FOUND (target missing), WRONG_PAGE_STATE (unexpected page), DATA_EXTRACTION_FAILURE (extraction failed), MAX_STEPS_EXCEEDED (use only as secondary, always identify root cause as primary), INFRASTRUCTURE_ERROR (network, OOM), UNKNOWN (cannot determine).+ "failure_categories": array // Classify the root cause of this failure. Return one or more categories ordered by likelihood. Always return at least one. Categories: ANTI_BOT_DETECTION (CAPTCHA, Cloudflare challenge, bot detection), BROWSER_ERROR (crash, page closed), NAVIGATION_FAILURE (404, redirect), PAGE_LOAD_TIMEOUT (timeout), AUTH_FAILURE (login failed, MFA), LLM_REASONING_ERROR (wrong element, wrong action), CREDENTIAL_ERROR (missing credentials), ELEMENT_NOT_FOUND (target missing), WRONG_PAGE_STATE (unexpected page), DATA_EXTRACTION_FAILURE (extraction failed), MAX_STEPS_EXCEEDED (use only as secondary, always identify root cause as primary), WEBSITE_ERROR (the website's own server failed or is down: HTTP 5xx such as 500, 502 Bad Gateway, 503 Service Unavailable or 504, a CDN error page saying the site's host is down, an outage or maintenance page), INFRASTRUCTURE_ERROR (Skyvern's own platform failed: its workers, browser pool, proxies or network, or it ran out of memory; never a website's error page), UNKNOWN (cannot determine).[{"category": str, // The category from the list above"confidence_float": float, // 0.0 to 1.0diff --git a/skyvern/forge/prompts/skyvern/task_v2_check_completion.j2 b/skyvern/forge/prompts/skyvern/task_v2_check_completion.j2index 9d086decd..fec06178c 100644--- a/skyvern/forge/prompts/skyvern/task_v2_check_completion.j2+++ b/skyvern/forge/prompts/skyvern/task_v2_check_completion.j2@@ -17,7 +17,7 @@ Reply in JSON format with the following keys:"user_goal_achieved": bool, // True ONLY if EVERY required subgoal is satisfied. If task_history_information OR any entry in required_subgoals indicates a required part is missing, unmet, or not yet found, this MUST be false. Never set this true while simultaneously acknowledging that a required part is missing. If the user wants to extract information and it has not been done, the goal is not achieved. If info extraction is not required, use the task history, assisted by the screenshot to decide if every required part has been achieved."should_terminate": bool, // True ONLY if there is CLEAR, EXPLICIT, UNAMBIGUOUS evidence that the goal is IMPOSSIBLE. The page must explicitly state the action cannot be done. False if there's ANY chance the goal can still be achieved."termination_reason": str, // If should_terminate is true, quote the EXACT error message or text from the page that proves impossibility. Null if should_terminate is false.- "failure_categories": array // Only populate when should_terminate is true. Classify the root cause. Categories: ANTI_BOT_DETECTION (CAPTCHA, Cloudflare, bot detection), BROWSER_ERROR (crash, page closed), NAVIGATION_FAILURE (404, redirect), PAGE_LOAD_TIMEOUT (timeout), AUTH_FAILURE (login failed, MFA), LLM_REASONING_ERROR (wrong element, wrong action), CREDENTIAL_ERROR (missing credentials), ELEMENT_NOT_FOUND (target missing), WRONG_PAGE_STATE (unexpected page), DATA_EXTRACTION_FAILURE (extraction failed), INFRASTRUCTURE_ERROR (network, OOM), UNKNOWN (cannot determine). Return one or more categories ordered by likelihood.+ "failure_categories": array // Only populate when should_terminate is true. Classify the root cause. Categories: ANTI_BOT_DETECTION (CAPTCHA, Cloudflare challenge, bot detection), BROWSER_ERROR (crash, page closed), NAVIGATION_FAILURE (404, redirect), PAGE_LOAD_TIMEOUT (timeout), AUTH_FAILURE (login failed, MFA), LLM_REASONING_ERROR (wrong element, wrong action), CREDENTIAL_ERROR (missing credentials), ELEMENT_NOT_FOUND (target missing), WRONG_PAGE_STATE (unexpected page), DATA_EXTRACTION_FAILURE (extraction failed), WEBSITE_ERROR (the website's own server failed or is down: HTTP 5xx such as 500, 502 Bad Gateway, 503 Service Unavailable or 504, a CDN error page saying the site's host is down, an outage or maintenance page), INFRASTRUCTURE_ERROR (Skyvern's own platform failed: its workers, browser pool, proxies or network, or it ran out of memory; never a website's error page), UNKNOWN (cannot determine). Return one or more categories ordered by likelihood.[{"category": str, // The category from the list above"confidence_float": float, // 0.0 to 1.0diff --git a/skyvern/forge/prompts/skyvern/task_v2_summarize-max-steps-reason.j2 b/skyvern/forge/prompts/skyvern/task_v2_summarize-max-steps-reason.j2index 026e0d8b0..1ac2095c5 100644--- a/skyvern/forge/prompts/skyvern/task_v2_summarize-max-steps-reason.j2+++ b/skyvern/forge/prompts/skyvern/task_v2_summarize-max-steps-reason.j2@@ -5,7 +5,7 @@ Make sure to ONLY return the JSON object in this format with no additional text{"page_info": str, // Think step by step. Describe all the useful information in the page related to the user goal."reasoning": str, // Think step by step. Describe the reason you summarized based on 'page_info', screenshots, user goal and result of mini plans. Keep the reasoning short and to the point.- "failure_categories": array // Classify the root cause of this failure. Return one or more categories ordered by likelihood (highest first). Always return at least one. Categories: ANTI_BOT_DETECTION (CAPTCHA, Cloudflare, bot detection), BROWSER_ERROR (crash, page closed), NAVIGATION_FAILURE (404, redirect), PAGE_LOAD_TIMEOUT (timeout), AUTH_FAILURE (login failed, MFA), LLM_REASONING_ERROR (wrong element, wrong action), CREDENTIAL_ERROR (missing credentials), ELEMENT_NOT_FOUND (target missing), WRONG_PAGE_STATE (unexpected page), DATA_EXTRACTION_FAILURE (extraction failed), MAX_STEPS_EXCEEDED (use only as secondary, always identify root cause as primary), INFRASTRUCTURE_ERROR (network, OOM), UNKNOWN (cannot determine).+ "failure_categories": array // Classify the root cause of this failure. Return one or more categories ordered by likelihood (highest first). Always return at least one. Categories: ANTI_BOT_DETECTION (CAPTCHA, Cloudflare challenge, bot detection), BROWSER_ERROR (crash, page closed), NAVIGATION_FAILURE (404, redirect), PAGE_LOAD_TIMEOUT (timeout), AUTH_FAILURE (login failed, MFA), LLM_REASONING_ERROR (wrong element, wrong action), CREDENTIAL_ERROR (missing credentials), ELEMENT_NOT_FOUND (target missing), WRONG_PAGE_STATE (unexpected page), DATA_EXTRACTION_FAILURE (extraction failed), MAX_STEPS_EXCEEDED (use only as secondary, always identify root cause as primary), WEBSITE_ERROR (the website's own server failed or is down: HTTP 5xx such as 500, 502 Bad Gateway, 503 Service Unavailable or 504, a CDN error page saying the site's host is down, an outage or maintenance page), INFRASTRUCTURE_ERROR (Skyvern's own platform failed: its workers, browser pool, proxies or network, or it ran out of memory; never a website's error page), UNKNOWN (cannot determine).[{"category": str, // The category from the list above"confidence_float": float, // 0.0 to 1.0diff --git a/skyvern/forge/sdk/services/bitwarden.py b/skyvern/forge/sdk/services/bitwarden.pyindex e256a6403..acf42d9c7 100644--- a/skyvern/forge/sdk/services/bitwarden.py+++ b/skyvern/forge/sdk/services/bitwarden.py@@ -94,6 +94,15 @@ _DECRYPTION_FAILURE_VALUE = "[error: cannot decrypt]"# audit-event upload failure. Everything else stays fatal._EVENT_POST_FAILED_ADVISORY = "Event post failed."+# CLI 2026.x prints this on `bw login --apikey` (vault still locked) when the server has no recorded user key id;+# the `bw unlock` that follows records it. Only this exact line is tolerated, never other backfill failures.+_USER_KEY_ID_BACKFILL_ADVISORY = re.compile(+ r"^\[UserKeyIdBackfillMigration\] Could not determine whether user "+ r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} "+ r"needs migration: KeyIdBackfillError: User key is not available in key store$",+ re.IGNORECASE,+)+def _contains_decryption_failure(value: Any) -> bool:if isinstance(value, str):@@ -469,12 +478,12 @@ def _credit_card_credential_from_bitwarden_item(item: dict) -> CreditCardCredentkey.removeprefix("metadata_"): value for key, value in extra_values.items() if key.startswith("metadata_")}return CreditCardCredential(- card_holder_name=card["cardholderName"],- card_number=card["number"],- card_exp_month=card["expMonth"],- card_exp_year=card["expYear"],- card_cvv=card["code"],- card_brand=card["brand"],+ card_holder_name=card.get("cardholderName"),+ card_number=card.get("number"),+ card_exp_month=card.get("expMonth"),+ card_exp_year=card.get("expYear"),+ card_cvv=card.get("code"),+ card_brand=card.get("brand"),billing_address=CreditCardBillingAddress(**address_values) if address_values else None,billing_email=extra_values.get("billing_email"),billing_phone=extra_values.get("billing_phone"),@@ -500,8 +509,8 @@ def get_list_response_item_from_bitwarden_item(item: dict) -> CredentialItem:return CredentialItem(item_id=item["id"],credential=PasswordCredential(- username=login["username"] or "",- password=login["password"] or "",+ username=login.get("username") or "",+ password=login.get("password") or "",totp=totp,metadata=_password_metadata_from_bitwarden_item(item),),@@ -663,6 +672,7 @@ class BitwardenService:]ignorable_regexes = [re.compile(r'^Could not find data file, ".+?/data\.json"; creating it instead\.$'),+ _USER_KEY_ID_BACKFILL_ADVISORY,]for line in lines:if any(s in line for s in ignorable_substrings):@@ -982,9 +992,9 @@ class BitwardenService:else:card = item.get("card")card_fields = ("cardholderName", "number", "expMonth", "expYear", "code", "brand")- if not isinstance(card, dict) or any(field not in card for field in card_fields):+ if not isinstance(card, dict) or not any(field in card for field in card_fields):raise BitwardenGetItemError("Invalid Bitwarden card payload")- if any(card[field] is not None and not isinstance(card[field], str) for field in card_fields):+ if any(card.get(field) is not None and not isinstance(card[field], str) for field in card_fields):raise BitwardenGetItemError("Invalid Bitwarden card field")fields = item.get("fields")if fields is not None and (@@ -1592,12 +1602,12 @@ class BitwardenService:credit_card_data = item["card"]mapped_credit_card_data: dict[str, str] = {- BitwardenConstants.CREDIT_CARD_HOLDER_NAME: credit_card_data["cardholderName"],- BitwardenConstants.CREDIT_CARD_NUMBER: credit_card_data["number"],- BitwardenConstants.CREDIT_CARD_EXPIRATION_MONTH: credit_card_data["expMonth"],- BitwardenConstants.CREDIT_CARD_EXPIRATION_YEAR: credit_card_data["expYear"],- BitwardenConstants.CREDIT_CARD_CVV: credit_card_data["code"],- BitwardenConstants.CREDIT_CARD_BRAND: credit_card_data["brand"],+ BitwardenConstants.CREDIT_CARD_HOLDER_NAME: credit_card_data.get("cardholderName"),+ BitwardenConstants.CREDIT_CARD_NUMBER: credit_card_data.get("number"),+ BitwardenConstants.CREDIT_CARD_EXPIRATION_MONTH: credit_card_data.get("expMonth"),+ BitwardenConstants.CREDIT_CARD_EXPIRATION_YEAR: credit_card_data.get("expYear"),+ BitwardenConstants.CREDIT_CARD_CVV: credit_card_data.get("code"),+ BitwardenConstants.CREDIT_CARD_BRAND: credit_card_data.get("brand"),}mapped_credit_card_data.update(_extract_credit_card_extra_custom_field_values(item))@@ -1688,8 +1698,8 @@ class BitwardenService:raise BitwardenGetItemError(f"Item with ID: {item_id} is not a login item")return PasswordCredential(- username=login["username"] or "",- password=login["password"] or "",+ username=login.get("username") or "",+ password=login.get("password") or "",totp=totp,)@@ -1994,8 +2004,8 @@ class BitwardenService:credential_type=CredentialType.PASSWORD,name=name,credential=PasswordCredential(- username=login_item["username"] or "",- password=login_item["password"] or "",+ username=login_item.get("username") or "",+ password=login_item.get("password") or "",totp=BitwardenService.normalize_totp_config(login_item.get("totp") or ""),metadata=_password_metadata_from_bitwarden_item(response["data"]),),diff --git a/tests/unit/golden_prompts/check-user-goal-with-termination.control.txt b/tests/unit/golden_prompts/check-user-goal-with-termination.control.txtindex 75be5e1bc..3054cb6ea 100644--- a/tests/unit/golden_prompts/check-user-goal-with-termination.control.txt+++ b/tests/unit/golden_prompts/check-user-goal-with-termination.control.txt@@ -6,7 +6,7 @@ Make sure to ONLY return the JSON object in this format with no additional text"page_info": str, // Think step by step. Describe all the useful information in the page related to the user goal."thoughts": str, // Think step by step. Explain your reasoning for the status you selected."status": str, // Must be one of three values: "complete", "terminate", or "continue". Use "complete" ONLY if the user goal has been fully achieved. Use "terminate" ONLY if the goal CANNOT ever be achieved (e.g., a file doesn't exist, an error modal blocks progress permanently, or an explicit termination condition is met in the user goal). Use "continue" if the goal is not yet achieved but more steps could potentially achieve it (this is the most common case - use this when you need to wait, navigate, or try different actions).- "failure_categories": array // Only populate when status is "terminate". Classify the root cause. Categories: ANTI_BOT_DETECTION (CAPTCHA, Cloudflare, bot detection), BROWSER_ERROR (crash, page closed), NAVIGATION_FAILURE (404, redirect), PAGE_LOAD_TIMEOUT (timeout), AUTH_FAILURE (login failed, MFA), LLM_REASONING_ERROR (wrong element, wrong action), CREDENTIAL_ERROR (missing credentials), ELEMENT_NOT_FOUND (target missing), WRONG_PAGE_STATE (unexpected page), DATA_EXTRACTION_FAILURE (extraction failed), INFRASTRUCTURE_ERROR (network, OOM), UNKNOWN (cannot determine). Return one or more categories ordered by likelihood.+ "failure_categories": array // Only populate when status is "terminate". Classify the root cause. Categories: ANTI_BOT_DETECTION (CAPTCHA, Cloudflare challenge, bot detection), BROWSER_ERROR (crash, page closed), NAVIGATION_FAILURE (404, redirect), PAGE_LOAD_TIMEOUT (timeout), AUTH_FAILURE (login failed, MFA), LLM_REASONING_ERROR (wrong element, wrong action), CREDENTIAL_ERROR (missing credentials), ELEMENT_NOT_FOUND (target missing), WRONG_PAGE_STATE (unexpected page), DATA_EXTRACTION_FAILURE (extraction failed), WEBSITE_ERROR (the website's own server failed or is down: HTTP 5xx such as 500, 502 Bad Gateway, 503 Service Unavailable or 504, a CDN error page saying the site's host is down, an outage or maintenance page), INFRASTRUCTURE_ERROR (Skyvern's own platform failed: its workers, browser pool, proxies or network, or it ran out of memory; never a website's error page), UNKNOWN (cannot determine). Return one or more categories ordered by likelihood.[{"category": str, // The category from the list above"confidence_float": float, // 0.0 to 1.0diff --git a/tests/unit/test_bitwarden_login.py b/tests/unit/test_bitwarden_login.pyindex 92d7905e7..2e1d8c2cd 100644--- a/tests/unit/test_bitwarden_login.py+++ b/tests/unit/test_bitwarden_login.py@@ -5,9 +5,10 @@ import sysfrom unittest.mock import AsyncMock, MagicMockimport pytest+from pydantic import ValidationErrorfrom structlog.testing import capture_logs-from skyvern.exceptions import BitwardenGetItemError, BitwardenUnlockError+from skyvern.exceptions import BitwardenGetItemError, BitwardenLoginError, BitwardenUnlockErrorfrom skyvern.forge.sdk.schemas.credentials import PasswordCredentialfrom skyvern.forge.sdk.services import bitwarden as bitwarden_modulefrom skyvern.forge.sdk.services.bitwarden import (@@ -24,6 +25,8 @@ from tests.unit.scoped_asyncio import ScopedAsyncioMISSING_TOTP_LOGINS = [pytest.param({"username": "[email protected]", "password": "pw"}, id="totp-absent"),pytest.param({"username": "[email protected]", "password": "pw", "totp": None}, id="totp-null"),+ pytest.param({"password": "pw"}, id="username-and-totp-absent"),+ pytest.param({"username": "[email protected]"}, id="password-and-totp-absent"),]@@ -79,6 +82,42 @@ async def test_login_ignores_data_file_creation_notice_on_stderr(await BitwardenService.login("client-id", "client-secret", master_password="master-password")+_BACKFILL_ADVISORY = (+ "[UserKeyIdBackfillMigration] Could not determine whether user 11111111-1111-4111-8111-111111111111 "+ "needs migration: KeyIdBackfillError: User key is not available in key store"+)+++ ("stderr", "tolerated"),+ [+ (f"{_BACKFILL_ADVISORY}\n", True),+ (_BACKFILL_ADVISORY.replace("User key is not available in key store", "Api error"), False),+ (f"{_BACKFILL_ADVISORY}\nUnable to fetch ServerConfig from https://example.test/api FetchError", False),+ ("[UserKeyIdBackfillMigration] Could not determine whether user someone needs migration", False),+ ],+ ids=["exact-advisory", "other-backfill-error", "advisory-plus-other-line", "malformed-user-id"],+)+async def test_apikey_login_tolerates_only_the_exact_user_key_id_backfill_advisory(+ monkeypatch: pytest.MonkeyPatch, stderr: str, tolerated: bool+) -> None:+ async def fake_run_command(*args, **kwargs) -> RunCommandResult:+ return RunCommandResult(+ stdout="You are logged in!\n\nTo unlock your vault, use the `unlock` command. ex:\n$ bw unlock",+ stderr=stderr,+ returncode=0,+ )++ monkeypatch.setattr(BitwardenService, "run_command", fake_run_command)++ if tolerated:+ assert await BitwardenService.login("client-id", "client-secret") is None+ else:+ with pytest.raises(BitwardenLoginError):+ await BitwardenService.login("client-id", "client-secret")++@pytest.mark.asyncioasync def test_server_login_item_round_trips_metadata(monkeypatch: pytest.MonkeyPatch) -> None:metadata = {"tenant": "north", "account_id": "acct_123"}@@ -407,13 +446,9 @@ def test_get_item_rejects_malformed_card_payload_and_custom_fields(key: str, val@pytest.mark.parametrize("field", ["cardholderName", "number", "expMonth", "expYear", "code", "brand"])[email protected]("missing", [False, True], ids=["non-string", "missing"])-def test_get_item_rejects_invalid_consumed_card_fields(field: str, missing: bool) -> None:+def test_get_item_rejects_non_string_consumed_card_fields(field: str) -> None:item = _cli_item(BitwardenItemType.CREDIT_CARD)- if missing:- del item["card"][field]- else:- item["card"][field] = 123+ item["card"][field] = 123with pytest.raises(BitwardenGetItemError):BitwardenService._parse_fetched_item(@@ -421,6 +456,84 @@ def test_get_item_rejects_invalid_consumed_card_fields(field: str, missing: bool)+# The same vault entries as the real CLI printed them (synthetic data, captured against a local fake server):+# 2025.9.0 spells an empty field as an explicit null, 2026.9.0 leaves the key out.+_CLI_EMPTY_FIELD_SHAPES = {+ "login-without-username": (+ BitwardenItemType.LOGIN,+ {+ "login": {+ "username": None,+ "password": "pw-only",+ "totp": None,+ "uris": [{"match": None, "uri": "https://example.test/"}],+ }+ },+ {+ "login": {+ "password": "pw-only",+ "passwordRevisionDate": None,+ "fido2Credentials": [],+ "uris": [{"uri": "https://example.test/"}],+ }+ },+ ),+ "login-without-password": (+ BitwardenItemType.LOGIN,+ {+ "login": {+ "username": "user-only",+ "password": None,+ "totp": None,+ "uris": [{"match": None, "uri": "https://example.test/"}],+ }+ },+ {+ "login": {+ "username": "user-only",+ "passwordRevisionDate": None,+ "fido2Credentials": [],+ "uris": [{"uri": "https://example.test/"}],+ }+ },+ ),+ "card-without-brand-and-cvv": (+ BitwardenItemType.CREDIT_CARD,+ {+ "card": {+ "cardholderName": "Only Holder",+ "number": "4000056655665556",+ "expMonth": "11",+ "expYear": "2031",+ "code": None,+ "brand": None,+ }+ },+ {"card": {"cardholderName": "Only Holder", "number": "4000056655665556", "expMonth": "11", "expYear": "2031"}},+ ),+}++[email protected]("case", list(_CLI_EMPTY_FIELD_SHAPES))+def test_cli_2026_omitted_keys_read_the_same_as_2025_nulls(case: str) -> None:+ item_type, shape_2025, shape_2026 = _CLI_EMPTY_FIELD_SHAPES[case]+ outcomes = []+ for nested, omits_null_keys in ((shape_2025, False), (shape_2026, True)):+ item = {**_cli_item(item_type), **nested, "fields": []}+ if omits_null_keys:+ item = {key: value for key, value in item.items() if value is not None}++ fetched = BitwardenService._parse_fetched_item(+ RunCommandResult(stdout=json.dumps(item), stderr="", returncode=0), item["id"]+ )+ try:+ outcomes.append(get_list_response_item_from_bitwarden_item(fetched).credential)+ except ValidationError as error:+ outcomes.append(type(error))++ assert outcomes[0] == outcomes[1]++@pytest.mark.asyncio@pytest.mark.parametrize("explicit_mode", [None, "true"], ids=["parent-only", "explicit-override"])async def test_run_command_scrubs_inherited_cli_output_modes(diff --git a/tests/unit/test_bitwarden_null_credentials.py b/tests/unit/test_bitwarden_null_credentials.pyindex 0c568bf38..700c4e10c 100644--- a/tests/unit/test_bitwarden_null_credentials.py+++ b/tests/unit/test_bitwarden_null_credentials.py@@ -193,10 +193,13 @@ async def test_get_secret_value_by_item_id_preserves_raw_totp_uri(assert result[BitwardenConstants.TOTP] == totp_uri+_OMITTED = object()++@pytest.mark.asyncio[email protected]("card_code", [None, "123"], ids=["null-code", "with-code"])[email protected]("card_code", [None, "123", _OMITTED], ids=["null-code", "with-code", "omitted-code-and-brand"])async def test_get_credit_card_data_includes_billing_custom_fields(- monkeypatch: pytest.MonkeyPatch, card_code: str | None+ monkeypatch: pytest.MonkeyPatch, card_code: str | None | object) -> None:item_payload = {"object": "item",@@ -219,6 +222,8 @@ async def test_get_credit_card_data_includes_billing_custom_fields({"name": "metadata_customer_id", "value": "cus_123"},],}+ if card_code is _OMITTED:+ del item_payload["card"]["code"], item_payload["card"]["brand"]async def fake_run_command(command: list[str], **_: object) -> RunCommandResult:return RunCommandResult(stdout=json.dumps(item_payload), stderr="", returncode=0)@@ -236,7 +241,8 @@ async def test_get_credit_card_data_includes_billing_custom_fields()assert result[BitwardenConstants.CREDIT_CARD_NUMBER] == "4111111111111111"- assert result[BitwardenConstants.CREDIT_CARD_CVV] == card_code+ assert result[BitwardenConstants.CREDIT_CARD_CVV] == (None if card_code is _OMITTED else card_code)+ assert result[BitwardenConstants.CREDIT_CARD_BRAND] == (None if card_code is _OMITTED else "visa")assert result["billing_address_line1"] == "123 Main St"assert result["billing_address_country_code"] == "US"assert result["billing_email"] == "[email protected]"diff --git a/tests/unit/test_code_block_sandbox.py b/tests/unit/test_code_block_sandbox.pyindex 89ffee812..1c4ad0260 100644--- a/tests/unit/test_code_block_sandbox.py+++ b/tests/unit/test_code_block_sandbox.py@@ -18,12 +18,13 @@ from datetime import UTCfrom datetime import date as stdlib_datefrom datetime import datetimefrom datetime import time as stdlib_time-from datetime import timezone+from datetime import timedelta, timezonefrom functools import partialfrom types import FunctionType, SimpleNamespacefrom typing import Any, NoReturnfrom unittest.mock import AsyncMock, MagicMock+import pyotp.totp as pyotp_totpimport pytestfrom playwright.async_api import Error as PlaywrightErrorfrom playwright.async_api import Page@@ -1689,6 +1690,20 @@ _RFC_TOTP_SEED = "JBSWY3DPEHPK3PXP"_CREDENTIAL_KEY = "login_credential"_WORKFLOW_RUN_ID = "wr_otp_test"_ORG_ID = "o_otp_test"+_TOTP_ANCHOR = datetime(2026, 6, 14, 12, 0, 0)+++def _freeze_totp_clock(monkeypatch: pytest.MonkeyPatch, offset_seconds: int = 0) -> None:+ """Pin pyotp's clock so a test's expected code and the block's minted code share one 30s window.+ pyotp.TOTP.now() reads pyotp.totp.datetime.datetime.now()."""+ frozen = _TOTP_ANCHOR + timedelta(seconds=offset_seconds)++ class _FrozenDatetime(datetime):+ @classmethod+ def now(cls, tz: object = None) -> datetime:+ return frozen++ monkeypatch.setattr(pyotp_totp, "datetime", SimpleNamespace(datetime=_FrozenDatetime))def _register_credential_parameter(wrc, key: str = _CREDENTIAL_KEY) -> None:@@ -1872,6 +1887,7 @@ class TestCodeBlockOtpMintAtCallTime:from skyvern.forge.sdk.workflow.models.block import _resolve_code_block_otp+ _freeze_totp_clock(monkeypatch)wrc = _build_wrc_with_totp_seed()_patch_context_resolution(monkeypatch, wrc)@@ -1883,30 +1899,16 @@ class TestCodeBlockOtpMintAtCallTime:@pytest.mark.asyncioasync def test_remint_differs_after_clock_advance(self, monkeypatch: pytest.MonkeyPatch) -> None:"""Advancing the clock past a rotation yields a different code — proves re-mint, not the- stale pre-minted attribute. pyotp.TOTP.now() reads pyotp.totp.datetime.datetime.now()."""- import datetime as real_datetime-- import pyotp.totp as pyotp_totp-+ stale pre-minted attribute."""from skyvern.forge.sdk.workflow.models.block import _resolve_code_block_otpwrc = _build_wrc_with_totp_seed()_patch_context_resolution(monkeypatch, wrc)- anchor = real_datetime.datetime(2026, 6, 14, 12, 0, 0)-- def _frozen(offset_seconds: int):- class _FrozenDatetime(real_datetime.datetime):- @classmethod- def now(cls, tz: object = None) -> "real_datetime.datetime":- return anchor + real_datetime.timedelta(seconds=offset_seconds)-- return _FrozenDatetime-- monkeypatch.setattr(pyotp_totp, "datetime", SimpleNamespace(datetime=_frozen(0)))+ _freeze_totp_clock(monkeypatch)first = await _resolve_code_block_otp(_CREDENTIAL_KEY, _ORG_ID, _WORKFLOW_RUN_ID, budget_seconds=120)- monkeypatch.setattr(pyotp_totp, "datetime", SimpleNamespace(datetime=_frozen(60)))+ _freeze_totp_clock(monkeypatch, offset_seconds=60)second = await _resolve_code_block_otp(_CREDENTIAL_KEY, _ORG_ID, _WORKFLOW_RUN_ID, budget_seconds=120)assert first != second@@ -2526,6 +2528,7 @@ class TestCodeBlockOtpCredentialIdWorkflowParameter:from skyvern.forge.sdk.workflow.models.block import _resolve_code_block_otp+ _freeze_totp_clock(monkeypatch)wrc = _build_wrc_with_totp_seed()_rebind_as_workflow_parameter(wrc, WorkflowParameterType.CREDENTIAL_ID)_patch_context_resolution(monkeypatch, wrc)@@ -2749,6 +2752,7 @@ class TestCodeBlockOtpNoLeak:async def test_execute_masks_otp_in_output(self, monkeypatch: pytest.MonkeyPatch) -> None:import pyotp+ _freeze_totp_clock(monkeypatch)wrc = _build_wrc_with_totp_seed()expected_code = pyotp.TOTP(_RFC_TOTP_SEED).now()@@ -2771,6 +2775,7 @@ class TestCodeBlockOtpNoLeak:"""User code that raises with the OTP in the message must not leak it into failure_reason."""import pyotp+ _freeze_totp_clock(monkeypatch)wrc = _build_wrc_with_totp_seed()expected_code = pyotp.TOTP(_RFC_TOTP_SEED).now()@@ -2801,6 +2806,7 @@ class TestCodeBlockOtpNoLeak:from skyvern.forge.sdk.workflow.models.code_block_recording import CodeBlockActionRecordingfrom skyvern.webeye.actions.actions import Action+ _freeze_totp_clock(monkeypatch)wrc = _build_wrc_with_totp_seed()expected_code = pyotp.TOTP(_RFC_TOTP_SEED).now()recorded_actions: list[Action] = []@@ -2858,6 +2864,7 @@ class TestCodeBlockOtpNoLeak:async def test_legacy_totp_not_leaked_in_failure_reason(self, monkeypatch: pytest.MonkeyPatch) -> None:import pyotp+ _freeze_totp_clock(monkeypatch)wrc = _build_wrc_with_totp_seed()expected_code = pyotp.TOTP(_RFC_TOTP_SEED).now()@@ -2881,6 +2888,7 @@ class TestCodeBlockLegacyTotpRegression:async def test_legacy_totp_attribute_and_registration(self, monkeypatch: pytest.MonkeyPatch) -> None:import pyotp+ _freeze_totp_clock(monkeypatch)wrc = _build_wrc_with_totp_seed()expected_code = pyotp.TOTP(_RFC_TOTP_SEED).now()diff --git a/tests/unit/test_failure_classifier.py b/tests/unit/test_failure_classifier.pyindex 961706ccd..4b8c86477 100644--- a/tests/unit/test_failure_classifier.py+++ b/tests/unit/test_failure_classifier.py@@ -2,6 +2,7 @@ from __future__ import annotationsimport jsonimport math+import refrom datetime import UTC, datetime, timedeltafrom types import SimpleNamespacefrom unittest.mock import AsyncMock, MagicMock@@ -9,7 +10,7 @@ from unittest.mock import AsyncMock, MagicMockimport pytestfrom skyvern import exceptions as skyvern_exceptions-from skyvern.constants import PROXY_TRANSPORT_NAV_ERRORS+from skyvern.constants import PROXY_TRANSPORT_NAV_ERRORS, SKYVERN_DIRfrom skyvern.exceptions import ScrapingFailedfrom skyvern.forge.failure_classifier import (BROWSER_SESSION_CLOSED_REASON_CODE,@@ -396,6 +397,44 @@ def test_user_code_crash_mentioning_process_exit_is_not_infrastructure() -> Noneassert "INFRASTRUCTURE_ERROR" not in categories+ "reason",+ [+ pytest.param(+ "navigation block failed. failure reason: Max retries per step (3) exceeded. Possible failure reasons: "+ "The user's attempts failed because the website returned a '502 Bad Gateway' error.",+ id="max-retries-summary-502",+ ),+ pytest.param(+ "navigation block terminated. Reason: Page shows 502 Bad Gateway error; cannot proceed with request.",+ id="terminate-bad-gateway",+ ),+ pytest.param("http_request block failed. failure reason: HTTP 500: ", id="http-request-block-500"),+ ],+)+def test_website_error_comes_only_from_the_model_never_from_failure_prose(reason: str) -> None:+ categories = classify_from_failure_reason(reason, fallback_to_unknown=True) or []++ assert "WEBSITE_ERROR" not in [category["category"] for category in categories]+++PROMPTS_DIR = SKYVERN_DIR / "forge" / "prompts" / "skyvern"+++def test_every_prompt_that_asks_the_model_for_a_category_offers_website_error() -> None:+ category_lists = {+ template.name: re.search(r'"failure_categories": array //.*?Categories: (.*)', template.read_text())+ for template in PROMPTS_DIR.glob("*.j2")+ }+ category_lists = {name: match.group(1) for name, match in category_lists.items() if match}++ assert len(category_lists) >= 5+ for name, listed in category_lists.items():+ offered = set(re.findall(r"\b([A-Z][A-Z_]+[A-Z]) \(", listed))+ assert "WEBSITE_ERROR" in offered, name+ assert offered <= {category.value for category in FailureCategory}, name++def _driver_nav_failure(code: str, url: str = "https://example.test/login") -> skyvern_exceptions.FailedToNavigateToUrl:return skyvern_exceptions.FailedToNavigateToUrl(url=url,@@ -684,6 +723,20 @@ def test_derive_element_not_found_maps_to_non_infra() -> None:assert derive_failure_attribution(fc)["primary_infra_component"] == "non_infra"+def test_derive_a_model_written_website_error_is_non_infra_not_worker() -> None:+ fc = [+ {+ "category": "WEBSITE_ERROR",+ "confidence_float": 1.0,+ "reasoning": "The site's server returned 502 Bad Gateway.",+ },+ {"category": "NAVIGATION_FAILURE", "confidence_float": 0.8, "reasoning": "The page did not load."},+ ]+ document = derive_failure_attribution(fc)+ assert document["failure_category"] == "WEBSITE_ERROR"+ assert document["primary_infra_component"] == "non_infra"++def test_derive_none_input_is_explicit_unattributed_document() -> None:doc = derive_failure_attribution(None)assert doc["primary_infra_component"] == "unattributed"