Skip to content

Automate browser based workflows with AI

Bound Redis connect-retry cost and fail open on the browser-address quarantine path (#7766)

Aaron Perez8e3b40ae072c
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/ConditionalNode/BranchesEditor.test.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/ConditionalNode/BranchesEditor.test.tsx
new file mode 100644
index 000000000..425e7ba4e
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/ConditionalNode/BranchesEditor.test.tsx
@@ -0,0 +1,159 @@
+// @vitest-environment jsdom
+
+import { cleanup, fireEvent, render, screen } from "@testing-library/react";
+import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
+
+import type { ConditionalNodeData } from "./types";
+
+const updateNodeData = vi.fn();
+const setNodes = vi.fn();
+const setEdges = vi.fn();
+
+vi.mock("@xyflow/react", async () => {
+ const actual =
+ await vi.importActual<typeof import("@xyflow/react")>("@xyflow/react");
+ return {
+ ...actual,
+ useNodes: () => [],
+ useReactFlow: () => ({ setNodes, setEdges, updateNodeData }),
+ };
+});
+
+vi.mock("@/store/WorkflowHasChangesStore", () => ({
+ useWorkflowHasChangesStore: () => ({
+ beginInternalUpdate: vi.fn(),
+ endInternalUpdate: vi.fn(),
+ }),
+}));
+
+vi.mock("..", () => ({
+ isWorkflowBlockNode: (node: { type: string }) =>
+ node.type !== "nodeAdder" && node.type !== "start",
+}));
+
+vi.mock("@/components/HelpTooltip", () => ({
+ HelpTooltip: () => null,
+}));
+
+vi.mock("@/components/WorkflowBlockInputTextarea", () => ({
+ WorkflowBlockInputTextarea: ({
+ value,
+ disabled,
+ }: {
+ value: string;
+ disabled?: boolean;
+ }) => <textarea value={value} disabled={disabled} readOnly />,
+}));
+
+import { BranchesEditor } from "./BranchesEditor";
+
+function makeData(
+ overrides: Partial<ConditionalNodeData> = {},
+): ConditionalNodeData {
+ return {
+ debuggable: true,
+ label: "conditional_1",
+ editable: true,
+ model: null,
+ continueOnFailure: false,
+ branches: [
+ {
+ id: "branch_a",
+ criteria: {
+ criteria_type: "jinja2_template",
+ expression: "{{ total > 100 }}",
+ description: null,
+ },
+ next_block_label: null,
+ description: null,
+ is_default: false,
+ },
+ {
+ id: "branch_b",
+ criteria: {
+ criteria_type: "jinja2_template",
+ expression: "{{ total > 250 }}",
+ description: null,
+ },
+ next_block_label: null,
+ description: null,
+ is_default: false,
+ },
+ {
+ id: "branch_default",
+ criteria: null,
+ next_block_label: null,
+ description: null,
+ is_default: true,
+ },
+ ],
+ activeBranchId: "branch_a",
+ mergeLabel: null,
+ ...overrides,
+ };
+}
+
+beforeEach(() => {
+ vi.useFakeTimers();
+ updateNodeData.mockReset();
+ setNodes.mockReset();
+ setEdges.mockReset();
+});
+
+afterEach(() => {
+ vi.useRealTimers();
+ cleanup();
+});
+
+describe("BranchesEditor", () => {
+ test("switches the active branch in read-only mode", () => {
+ render(
+ <BranchesEditor nodeId="cond_1" data={makeData({ editable: false })} />,
+ );
+
+ const inactiveTab = screen.getByText("B • Else If").closest("button")!;
+ expect(inactiveTab.disabled).toBe(false);
+
+ updateNodeData.mockClear();
+ fireEvent.click(inactiveTab);
+
+ expect(updateNodeData).toHaveBeenCalledWith("cond_1", {
+ activeBranchId: "branch_b",
+ });
+ });
+
+ test("switches the active branch in editable mode", () => {
+ render(
+ <BranchesEditor nodeId="cond_1" data={makeData({ editable: true })} />,
+ );
+
+ updateNodeData.mockClear();
+ fireEvent.click(screen.getByText("B • Else If").closest("button")!);
+
+ expect(updateNodeData).toHaveBeenCalledWith("cond_1", {
+ activeBranchId: "branch_b",
+ });
+ });
+
+ test("hides edit affordances in read-only mode", () => {
+ render(
+ <BranchesEditor nodeId="cond_1" data={makeData({ editable: false })} />,
+ );
+
+ expect(
+ (screen.getByTitle("Add new condition") as HTMLButtonElement).disabled,
+ ).toBe(true);
+ expect(screen.queryByTitle("Branch options")).toBeNull();
+ });
+
+ test("shows edit affordances in editable mode", () => {
+ render(
+ <BranchesEditor nodeId="cond_1" data={makeData({ editable: true })} />,
+ );
+
+ expect(
+ (screen.getByTitle("Add new condition") as HTMLButtonElement).disabled,
+ ).toBe(false);
+ expect(screen.getAllByTitle("Branch options").length).toBeGreaterThan(0);
+ });
+});
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/ConditionalNode/BranchesEditor.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/ConditionalNode/BranchesEditor.tsx
index ad8045137..2a5b35248 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/ConditionalNode/BranchesEditor.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/ConditionalNode/BranchesEditor.tsx
@@ -42,7 +42,7 @@ type Props = {
function BranchesEditor({ nodeId, data }: Props) {
const id = nodeId;
const nodes = useNodes<AppNode>();
- const { setNodes, setEdges } = useReactFlow();
+ const { setNodes, setEdges, updateNodeData } = useReactFlow();
const { beginInternalUpdate, endInternalUpdate } =
useWorkflowHasChangesStore();
// Track pending endInternalUpdate timer from handleSelectBranch so we can
@@ -247,18 +247,19 @@ function BranchesEditor({ nodeId, data }: Props) {
const handleSelectBranch = useCallback(
(branchId: string) => {
- if (!data.editable) {
- return;
- }
- // Mark as internal update to prevent triggering "unsaved changes" dialog
- // Switching branches is UI state, not actual workflow data changes
- // Cancel any pending timer from a previous rapid call to keep the counter balanced
+ // Switching branches only toggles which branch's nodes are visible; it is
+ // view state, not a workflow data change. It must therefore work even in
+ // read-only/comparison canvases (version history, copilot review), so we
+ // write activeBranchId directly rather than through the editability-gated
+ // `update` helper.
+ // Mark as internal update to prevent triggering "unsaved changes" dialog.
+ // Cancel any pending timer from a previous rapid call to keep the counter balanced.
if (branchSelectTimerRef.current !== null) {
clearTimeout(branchSelectTimerRef.current);
endInternalUpdate();
}
beginInternalUpdate();
- update({ activeBranchId: branchId });
+ updateNodeData(id, { activeBranchId: branchId });
// Clear the flag after layout completes (layout uses setTimeout(10))
// Store timer in ref so it can be cleaned up on unmount
branchSelectTimerRef.current = setTimeout(() => {
@@ -266,7 +267,7 @@ function BranchesEditor({ nodeId, data }: Props) {
endInternalUpdate();
}, 50);
},
- [data.editable, beginInternalUpdate, update, endInternalUpdate],
+ [id, updateNodeData, beginInternalUpdate, endInternalUpdate],
);
const handleRemoveBranch = (branchId: string) => {
@@ -445,7 +446,7 @@ function BranchesEditor({ nodeId, data }: Props) {
!branch.is_default &&
nonDefaultBranches.length > 1;
- const canReorder = !branch.is_default;
+ const canReorder = data.editable && !branch.is_default;
const branchIndexInNonDefault = nonDefaultBranches.findIndex(
(b) => b.id === branch.id,
);
@@ -473,7 +474,6 @@ function BranchesEditor({ nodeId, data }: Props) {
},
)}
onClick={() => handleSelectBranch(branch.id)}
- disabled={!data.editable}
>
{getConditionLabel(branch, index)}
</Button>
@@ -553,7 +553,6 @@ function BranchesEditor({ nodeId, data }: Props) {
variant="ghost"
size="sm"
className="h-auto gap-1 rounded-full border border-transparent bg-slate-elevation5 p-0 px-3 py-1 text-xs font-normal text-tertiary-foreground transition-colors hover:bg-slate-elevation4 hover:text-tertiary-foreground"
- disabled={!data.editable}
>
{overflowBranches.length} More
<ChevronDownIcon className="size-3" />
diff --git a/skyvern/cli/mcp_tools/argument_validation.py b/skyvern/cli/mcp_tools/argument_validation.py
index 0b56d0528..6cdbdcb18 100644
--- a/skyvern/cli/mcp_tools/argument_validation.py
+++ b/skyvern/cli/mcp_tools/argument_validation.py
@@ -9,15 +9,15 @@ failure the calling model cannot act on. The logged exception surfaces in error
tracking as a recurring signature, and the model keeps re-sending the wrong
shape because the failure text is a wall of pydantic internals.
-This middleware checks the argument keys against the tool's published input
-schema *before* dispatch and short-circuits with a structured error when they
-don't match. Because dispatch is never reached, no raw validation error is
-logged, and the model receives a clear message naming the unsupported and
-expected arguments (the same ``make_result``/``make_error`` envelope every tool
-uses). Type-level mismatches with otherwise-valid keys still fall through to
-pydantic — this only owns the unsupported-/missing-argument contract. A call
-with no ``arguments`` at all also falls through: pydantic handles it, rejecting
-only when the tool has required parameters.
+This middleware checks arguments against the tool's published input schema
+*before* dispatch. It repairs narrow, unambiguous type mistakes and
+short-circuits with a structured error when argument keys don't match. Because
+invalid keys never reach dispatch, no raw validation error is logged, and the
+model receives a clear message naming the unsupported and expected arguments
+(the same ``make_result``/``make_error`` envelope every tool uses). Other
+type-level mismatches still fall through to pydantic. A call with no
+``arguments`` at all also falls through: pydantic handles it, rejecting only
+when the tool has required parameters.
"""
from __future__ import annotations
@@ -33,6 +33,62 @@ from mcp.types import TextContent
from skyvern.cli.core.result import ErrorCode, make_error, make_result
LOG = structlog.get_logger(__name__)
+_MAX_REPAIR_STRING_LENGTH = 50_000
+_MAX_REPAIR_LIST_ITEMS = 100
+# Repair only production-observed fields whose comma-delimited meaning is known;
+# the schema check below prevents this policy from outliving a signature change.
+_COMMA_SEPARATED_LIST_ARGUMENTS = frozenset({("skyvern_workflow_run_list", "status")})
+
+
+def _schema_accepts_array(schema: Any) -> bool:
+ if not isinstance(schema, dict):
+ return False
+ schema_type = schema.get("type")
+ if schema_type == "array" or (isinstance(schema_type, list) and "array" in schema_type):
+ return True
+ return any(
+ _schema_accepts_array(option)
+ for alternatives in (schema.get("anyOf"), schema.get("oneOf"))
+ if isinstance(alternatives, list)
+ for option in alternatives
+ )
+
+
+def _split_comma_separated_list(value: Any) -> Any:
+ if not isinstance(value, str) or "," not in value or len(value) > _MAX_REPAIR_STRING_LENGTH:
+ return value
+ stripped = value.strip()
+ if not stripped or stripped[0] in "[{(":
+ return value
+ items = [item.strip() for item in stripped.split(",", _MAX_REPAIR_LIST_ITEMS)]
+ if len(items) > _MAX_REPAIR_LIST_ITEMS or any(not item for item in items):
+ return value
+ return items
+
+
+def _repair_argument_types(tool_name: str, tool: Any, arguments: dict[str, Any]) -> None:
+ parameters = getattr(tool, "parameters", None)
+ if not isinstance(parameters, dict):
+ return
+ properties = parameters.get("properties")
+ if not isinstance(properties, dict):
+ return
+
+ for name, value in arguments.items():
+ if (tool_name, name) in _COMMA_SEPARATED_LIST_ARGUMENTS and _schema_accepts_array(properties.get(name)):
+ arguments[name] = _split_comma_separated_list(value)
+
+ timeout = arguments.get("timeout")
+ if (
+ tool_name == "skyvern_navigate"
+ and isinstance(timeout, (int, float))
+ and not isinstance(timeout, bool)
+ and 0 < timeout < 1000
+ ):
+ timeout_ms = timeout * 1000
+ arguments["timeout"] = (
+ int(timeout_ms) if isinstance(timeout_ms, float) and timeout_ms.is_integer() else timeout_ms
+ )
def _argument_contract(tool: Any) -> tuple[set[str], set[str]] | None:
@@ -84,6 +140,16 @@ class MCPArgumentValidationMiddleware(Middleware):
if tool is None:
return None
+ repaired = dict(arguments)
+ try:
+ _repair_argument_types(tool_name, tool, repaired)
+ except Exception:
+ LOG.warning("mcp_argument_type_repair_failed", tool=tool_name, exc_info=True)
+ else:
+ if repaired != arguments:
+ # FastMCP dispatch reads this same dict reference, so rebinding `arguments` would discard the repair.
+ arguments.clear()
+ arguments.update(repaired)
contract = _argument_contract(tool)
if contract is None:
return None
diff --git a/skyvern/exceptions.py b/skyvern/exceptions.py
index 218200f11..0029f38b6 100644
--- a/skyvern/exceptions.py
+++ b/skyvern/exceptions.py
@@ -632,6 +632,10 @@ class FailedToTakeScreenshot(SkyvernException):
super().__init__(f"Failed to take screenshot. Error message: {error_message}")
+class ScreenshotTargetClosed(FailedToTakeScreenshot):
+ pass
+
+
class EmptyScrapePage(SkyvernException):
def __init__(self) -> None:
super().__init__("Failed to scrape the page, returned an NONE result")
diff --git a/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2 b/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2
index 2d371cd42..2f756f5fe 100644
--- a/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2
+++ b/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2
@@ -21,7 +21,7 @@ PRIMARY COMPLETION CRITERIA:
```{{ reconciliation_criteria }}```
{% else %}
Classify the latest Workflow Copilot request. Return only JSON with keys:
-testing_intent=require_test|skip_test|unspecified; authoring_intent=author_now|defer_authoring; credential_input_kind=none|raw_secret|credential_id|credential_name|website_stored_credential|placeholder; credential_refs; login_page_urls; login_intent; requires_user_clarification; clarification_reason; completion_contract; completion_criteria; raw_secret_evidence; raw_secret_handling=none|block|redacted_draft.
+testing_intent=require_test|skip_test|unspecified; authoring_intent=author_now|defer_authoring; credential_input_kind=none|raw_secret|credential_id|credential_name|website_stored_credential|placeholder; credential_refs; login_page_urls; login_intent; email_signin_intent; signin_email_candidates; requires_user_clarification; clarification_reason; completion_contract; completion_criteria; raw_secret_evidence; raw_secret_handling=none|block|redacted_draft.
terminal_action priority:
- Separate authentication from the requested end state before writing completion_criteria. Authentication is a prefix
@@ -58,6 +58,18 @@ login_intent:
- false on a passive first-touch read that merely implies an eventual login without explicitly asking to sign in or authenticate, for example "check my account balance" or "download my invoices". Let task or URL clarification happen first.
- Independent of credential_input_kind: decide it from whether authentication is required, not from whether the user supplied credential material. It is true for an explicit login request that names a saved credential and true for one that names none.
+email_signin_intent:
+- true ONLY when the request describes authentication that has no password step at all — the site sends a one-time code or a sign-in link to an inbox, or the user says the account has no password.
+- An email address alone is NOT enough. Most ordinary logins use an email address as the username, so "log in as [email protected]" is a password login and this stays false unless the request also describes a password-free sign-in.
+- false when the request supplies or refers to a password, a saved password credential, or a username-and-password pair, including a saved credential named by ID or exact name.
+- false when login_intent is false.
+- false when you are unsure whether the site has a password. Guessing true removes the credential step from a login that needs one.
+- Decide it from the authentication shape the user describes, not from whether the words "passwordless", "magic link", or "one-time code" appear.
+
+signin_email_candidates:
+- Every email address the latest request gives as the identity to sign in with, in the order they appear. Empty when the request names none.
+- Do not include an address the user gives for some other purpose, such as a recipient to send results to or an address to type into a form field that is not the sign-in identity.
+
requires_user_clarification:
- true only when the user refers to credentials but does not provide enough information to identify a saved credential or login page.
- false for normal build/test/verify requests with no credential reference, for explicit credential IDs/names/URLs, for placeholders, and for skip_test requests.
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 445cc4f7f..cafa3c00a 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -107,6 +107,7 @@ from skyvern.forge.sdk.copilot.context import (
finalize_discovery_counter_in_global_llm_context,
parsed_ask_refs,
record_approved_credentials_in_global_llm_context,
+ record_signin_email_in_global_llm_context,
render_loaded_result_context_for_prompt,
sanitize_global_llm_context_for_prompt,
)
@@ -1248,7 +1249,10 @@ def _build_dynamic_system_prompt(tool_usage_guide: str, config: CopilotConfig) -
+ "workflow and skip the browser run with a credential setup message. "
+ "If `raw_secret_handling` is `redacted_draft`, build only from the redacted request, do not run blocks, "
+ "and tell the user to store the redacted secret as a saved credential before testing. "
- + "If `resolved_credentials` are present, use those `credential_id` values."
+ + "If `resolved_credentials` are present, use those `credential_id` values. "
+ + "If `resolved_signin_email` is present, that address signs in: bind it to the workflow as a "
+ + "workflow input parameter with that value as the default, so the saved workflow can be re-run "
+ + "under a different address, and do not ask for a saved password credential for that login."
)
return (
prompt
@@ -1890,8 +1894,11 @@ def _make_agent_result(
normal translate-result, missing-SDK fallback, unexpected-error fallback).
"""
final_context = (
- record_approved_credentials_in_global_llm_context(
- ctx, finalize_discovery_counter_in_global_llm_context(ctx, global_llm_context)
+ record_signin_email_in_global_llm_context(
+ ctx,
+ record_approved_credentials_in_global_llm_context(
+ ctx, finalize_discovery_counter_in_global_llm_context(ctx, global_llm_context)
+ ),
)
if ctx is not None
else global_llm_context
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index f39907caa..b53b2310f 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -5,7 +5,7 @@ from __future__ import annotations
import json
import re
import uuid
-from collections.abc import Mapping, Sequence
+from collections.abc import Collection, Mapping, Sequence
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, Literal, get_args
@@ -280,6 +280,12 @@ class StructuredContext(BaseModel):
loaded_result_targets: list[LoadedResultTargetContext] = Field(default_factory=list)
fill_carry: list[FillCarry] = Field(default_factory=list)
entrypoint_url: str | None = None
+ # Passwordless sign-in identity, carried so a follow-up turn the classifier reads as an
+ # ordinary login does not fall back to demanding a password the site does not have.
+ # Scoped by host: reusing it anywhere else would suppress the password ask on a site
+ # that never established it was passwordless.
+ signin_email: str = ""
+ signin_email_host: str = ""
def to_json_str(self) -> str:
payload = self.model_dump(mode="json")
@@ -641,13 +647,35 @@ def record_approved_credentials_in_global_llm_context(ctx: CopilotContext, raw_c
return sc.to_json_str()
+def record_signin_email_in_global_llm_context(ctx: CopilotContext, raw_context: str | None) -> str | None:
+ policy = ctx.request_policy
+ if policy is None or not policy.resolved_signin_email or not policy.resolved_signin_host:
+ return raw_context
+ sc = StructuredContext.from_json_str(raw_context)
+ if sc.signin_email == policy.resolved_signin_email and sc.signin_email_host == policy.resolved_signin_host:
+ return raw_context
+ sc.signin_email = policy.resolved_signin_email
+ sc.signin_email_host = policy.resolved_signin_host
+ return sc.to_json_str()
+
+
+def prior_signin_email_from_context(raw_context: str | None, hosts: Collection[str]) -> str:
+ """The carried address, only when this turn targets the host it was resolved for."""
+ sc = StructuredContext.from_json_str(raw_context)
+ if not sc.signin_email or sc.signin_email_host not in hosts:
+ return ""
+ return sc.signin_email
+
+
def adopt_model_authored_context(trusted_raw: str | None, model_raw: object) -> StructuredContext:
- """Take the model's context but keep `approved_credentials` server-owned.
+ """Take the model's context but keep `approved_credentials` and `signin_email` server-owned.
Approval is recorded only from server-resolved credentials; an entry the model
supplied would be promoted into `resolved_credentials` on the next turn and clear
the unapproved-credential gate for a credential the user never named. Membership
- of the org is not evidence the user named it.
+ of the org is not evidence the user named it. A model-authored `signin_email` is the
+ same promotion in the other gate: it marks the next turn passwordless and suppresses
+ the password-credential ask for a site that does need one.
"""
trusted = StructuredContext.from_json_str(trusted_raw)
structured = trusted
@@ -659,6 +687,8 @@ def adopt_model_authored_context(trusted_raw: str | None, model_raw: object) ->
elif isinstance(model_raw, str):
structured = StructuredContext.from_json_str(model_raw)
structured.approved_credentials = list(trusted.approved_credentials)
+ structured.signin_email = trusted.signin_email
+ structured.signin_email_host = trusted.signin_email_host
return structured
diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.py
index abf248719..d262516f9 100644
--- a/skyvern/forge/sdk/copilot/request_policy.py
+++ b/skyvern/forge/sdk/copilot/request_policy.py
@@ -18,7 +18,11 @@ from skyvern.forge.prompts import prompt_engine
from skyvern.forge.request_logging import redact_sensitive_fields
from skyvern.forge.sdk.api.llm.api_handler import LLMAPIHandler
from skyvern.forge.sdk.copilot.config import CopilotConfig
-from skyvern.forge.sdk.copilot.context import StructuredContext, sanitize_global_llm_context_for_prompt
+from skyvern.forge.sdk.copilot.context import (
+ StructuredContext,
+ prior_signin_email_from_context,
+ sanitize_global_llm_context_for_prompt,
+)
from skyvern.forge.sdk.copilot.llm_errors import is_retriable_llm_error
from skyvern.forge.sdk.copilot.output_utils import parse_final_response
from skyvern.forge.sdk.copilot.reached_download_target import REGISTERED_DOWNLOAD_REQUESTED_OUTPUT_PATHS
@@ -37,6 +41,7 @@ from skyvern.forge.sdk.copilot.secret_redaction import (
contains_email_password_pair,
redact_raw_secrets_for_prompt,
)
+from skyvern.forge.sdk.copilot.signin_email import connected_gmail_address, is_email_address
from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
from skyvern.forge.sdk.copilot.workflow_credential_utils import (
URL_CANDIDATE_RE,
@@ -98,12 +103,15 @@ ClarificationReason = Literal[
"missing_target_context",
"workflow_credential_inputs_unbound",
"login_credentials_unresolved",
+ "signin_email_unresolved",
]
RawSecretHandling = Literal["none", "block", "redacted_draft"]
_VALID_CLARIFICATION_REASONS: frozenset[ClarificationReason] = frozenset(get_args(ClarificationReason))
# Only deterministic post-resolution code may mint these; a classifier emission would
# skip the concrete-target and credential-reachability checks the reason stands for.
-_DETERMINISTIC_ONLY_CLARIFICATION_REASONS: frozenset[ClarificationReason] = frozenset({"login_credentials_unresolved"})
+_DETERMINISTIC_ONLY_CLARIFICATION_REASONS: frozenset[ClarificationReason] = frozenset(
+ {"login_credentials_unresolved", "signin_email_unresolved"}
+)
# Gates guardrails.py's deferred-draft tool authority — narrower than the prompt set below.
CREDENTIAL_DEFERRED_DRAFT_REASONS: frozenset[ClarificationReason] = frozenset(
{"workflow_credential_inputs_unbound", "credential_name_unresolved"}
@@ -162,6 +170,7 @@ _STORED_CREDENTIAL_URL_QUESTION_STABLE_PREFIX = (
)
_STORED_CREDENTIAL_URL_QUESTION = f"{_STORED_CREDENTIAL_URL_QUESTION_STABLE_PREFIX} {_CREDENTIALS_UI_DIRECTIONS}"
_AMBIGUOUS_URL_CREDENTIAL_QUESTION = "I found multiple stored credentials for that login page. Which one should I use?"
+_SIGNIN_EMAIL_QUESTION = "Which email address should I sign in with?"
_CREDENTIAL_ID_RE = re.compile(r"\bcred_[A-Za-z0-9][A-Za-z0-9_-]*\b")
# A credential ID typed with the wrong separator (`cred 530…`, `cred-530…`). The
# digit-only body and length floor keep this off prose like `cred and the password`.
@@ -543,6 +552,12 @@ class RequestPolicy:
credential_refs: list[str] = field(default_factory=list)
login_page_urls: list[str] = field(default_factory=list)
login_intent: bool = False
+ # Sign-in that identifies the user by email address instead of a stored password, so
+ # resolving *which address* is the whole credential question — there is no password to find.
+ email_signin_intent: bool = False
+ signin_email_candidates: list[str] = field(default_factory=list)
+ resolved_signin_email: str | None = None
+ resolved_signin_host: str | None = None
requires_user_clarification: bool = False
allow_update_workflow: bool = True
allow_run_blocks: bool = True
@@ -583,6 +598,8 @@ class RequestPolicy:
"authoring_intent": self.authoring_intent,
"credential_input_kind": self.credential_input_kind,
"login_intent": self.login_intent,
+ "email_signin_intent": self.email_signin_intent,
+ "signin_email_resolved": bool(self.resolved_signin_email),
"clarification_reason": self.clarification_reason,
"allow_update_workflow": self.allow_update_workflow,
"allow_run_blocks": self.allow_run_blocks,
@@ -695,6 +712,8 @@ class RequestPolicy:
lines.append(f"completion_contract: {self.completion_contract}")
if self.raw_secret_detected:
lines.append(f"raw_secret_detected: {self.raw_secret_detected}")
+ if self.resolved_signin_email:
+ lines.append(f"resolved_signin_email: {self.resolved_signin_email}")
graded_criteria = self.graded_completion_criteria()
requested_output_path_literals = sorted(
{
@@ -983,6 +1002,10 @@ def _clean_list(values: list[Any]) -> list[str]:
return list(dict.fromkeys(str(value).strip() for value in values if str(value).strip()))
+def _clean_email_list(values: list[Any]) -> list[str]:
+ return [value for value in _clean_list(values) if is_email_address(value)]
+
+
def _credential_ids(text: str) -> list[str]:
text = text or ""
canonical = _CREDENTIAL_ID_RE.findall(text)
@@ -2276,6 +2299,8 @@ def _classification_from_raw(
credential_refs=_clean_list(raw.get("credential_refs") or []),
login_page_urls=_clean_list(raw.get("login_page_urls") or []),
login_intent=bool(raw.get("login_intent")),
+ email_signin_intent=bool(raw.get("email_signin_intent")),
+ signin_email_candidates=_clean_email_list(raw.get("signin_email_candidates") or []),
requires_user_clarification=bool(raw.get("requires_user_clarification")),
completion_contract=completion_contract or None,
completion_criteria=completion_criteria,
@@ -4522,6 +4547,10 @@ def _login_credentials_unresolved(policy: RequestPolicy, user_message: str, work
"""
if not policy.login_intent or policy.raw_secret_detected:
return False
+ if _is_passwordless_email_signin(policy):
+ # Nothing a password credential could satisfy: _resolve_signin_email owns this shape,
+ # and asking for a saved credential here is unanswerable on an email-only site.
+ return False
if policy.testing_intent == "skip_test" or policy.allow_missing_credentials_in_draft:
return False
if policy.clarification_reason not in _LOGIN_CREDENTIAL_OVERRIDABLE_REASONS:
@@ -4531,6 +4560,55 @@ def _login_credentials_unresolved(policy: RequestPolicy, user_message: str, work
return not (policy.resolved_credentials or policy.discovered_credentials or policy.existing_workflow_credential_ids)
+def _signin_target_hosts(policy: RequestPolicy, user_message: str) -> list[str]:
+ return [parts[1] for url in _login_url_candidates(policy, user_message) if (parts := _url_parts(url))]
+
+
+def _is_passwordless_email_signin(policy: RequestPolicy) -> bool:
+ """Email-address sign-in with no credential material referenced anywhere in the request."""
+ return (
+ (policy.email_signin_intent or bool(policy.resolved_signin_email))
+ and policy.login_intent
+ and not policy.raw_secret_detected
+ and policy.credential_input_kind == "none"
+ and not policy.credential_refs
+ )
+
+
+async def _resolve_signin_email(
+ policy: RequestPolicy,
+ organization_id: str,
+ *,
+ user_message: str,
+ workflow_yaml: str,
+) -> None:
+ """Settle which address signs in, or ask for one — never ask for a password credential."""
+ if not _is_passwordless_email_signin(policy):
+ return
+ if policy.testing_intent == "skip_test" or policy.allow_missing_credentials_in_draft:
+ return
+ # Same first-touch rule the password path uses: with no site yet, the address is not
+ # the question the user still owes.
+ if not _login_target_is_concrete(policy, user_message, workflow_yaml):
+ return
+ # A clarification already on the policy is a different question the user still owes;
+ # _block would overwrite it and lose it, so this ask waits for a later turn. A question
+ # without a reason set counts, which is why this checks the flag and not just the reason.
+ if policy.requires_user_clarification or policy.clarification_reason != "none":
+ return
+
+ named = next(iter(policy.signin_email_candidates), None)
+ if named:
+ # An explicit instruction outranks both a carried identity and the connected account.
+ policy.resolved_signin_email = named
+ elif not policy.resolved_signin_email:
+ policy.resolved_signin_email = await connected_gmail_address(organization_id)
+ if not policy.resolved_signin_email:
+ _block(policy, _SIGNIN_EMAIL_QUESTION, reason="signin_email_unresolved")
+ return
+ policy.resolved_signin_host = next(iter(_signin_target_hosts(policy, user_message)), None)
+
+
def _prioritize_credential_clarification(policy: RequestPolicy) -> None:
if policy.credential_input_kind not in ("credential_id", "credential_name"):
return
@@ -4751,7 +4829,11 @@ async def _resolve_credentials(
message_url_candidates = _clean_list(
[candidate.rstrip(".,;:!?") for candidate in URL_CANDIDATE_RE.findall(user_message)]
)
- current_turn_login_authorized = policy.login_intent or bool(_EXPLICIT_LOGIN_ACTION_RE.search(user_message))
+ # A passwordless turn must not reach the URL credential tiers below: a host match there
+ # asks for a saved credential under a different reason, which is the same dead end.
+ current_turn_login_authorized = (
+ policy.login_intent or bool(_EXPLICIT_LOGIN_ACTION_RE.search(user_message))
+ ) and not _is_passwordless_email_signin(policy)
url_candidates: list[str] = []
allow_host_fallback = False
if (
@@ -4991,6 +5073,12 @@ async def build_request_policy(
exc_info=True,
)
_prioritize_credential_clarification(policy)
+ # Before credential resolution, so a carried sign-in identity also keeps this turn out of
+ # the URL credential tiers.
+ carried = prior_signin_email_from_context(global_llm_context, _signin_target_hosts(policy, user_message))
+ policy.resolved_signin_email = policy.resolved_signin_email or (
+ carried if carried and is_email_address(carried) else None
+ )
if policy.raw_secret_detected and policy.raw_secret_handling == "redacted_draft":
policy.testing_intent = "skip_test"
@@ -5103,6 +5191,24 @@ async def build_request_policy(
policy.clarification_reason = "none"
policy.clarification_question = None
+ try:
+ await _resolve_signin_email(
+ policy,
+ organization_id,
+ user_message=user_message,
+ workflow_yaml=workflow_yaml,
+ )
+ except Exception:
+ LOG.warning(
+ "request-policy sign-in email resolution failed",
+ organization_id=organization_id,
+ exc_info=True,
+ )
+ # The credential requirement is already lifted for this shape, so falling through
+ # would build a login with no identity and no question asked.
+ if _is_passwordless_email_signin(policy) and not policy.requires_user_clarification:
+ _block(policy, _SIGNIN_EMAIL_QUESTION, reason="signin_email_unresolved")
+
if _login_credentials_unresolved(policy, user_message, workflow_yaml):
if policy.clarification_reason == "none":
_block(policy, _LOGIN_CREDENTIAL_QUESTION, reason="login_credentials_unresolved")
diff --git a/skyvern/forge/sdk/copilot/signin_email.py b/skyvern/forge/sdk/copilot/signin_email.py
new file mode 100644
index 000000000..2ddfc0481
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/signin_email.py
@@ -0,0 +1,76 @@
+"""The org's connected Gmail address, as a default sign-in identity."""
+
+from __future__ import annotations
+
+import asyncio
+import re
+
+import httpx
+import structlog
+
+from skyvern.forge.sdk.services import google_oauth_service
+from skyvern.services.email.gmail_client import GMAIL_API_BASE, get_json
+
+LOG = structlog.get_logger()
+
+_GMAIL_PROFILE_TIMEOUT_SECONDS = 10.0
+_EMAIL_ADDRESS_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+")
+_MAX_EMAIL_ADDRESS_LENGTH = 254
+# This runs inside the per-turn policy build, so the whole lookup is bounded rather than
+# left to the retrying HTTP client multiplied by however many connections an org has.
+_LOOKUP_DEADLINE_SECONDS = 15.0
+_MAX_CREDENTIALS_TRIED = 3
+
+
+def is_email_address(value: str) -> bool:
+ """Shape check for a value that reaches an LLM prompt and a workflow parameter default."""
+ return len(value) <= _MAX_EMAIL_ADDRESS_LENGTH and bool(_EMAIL_ADDRESS_RE.fullmatch(value))
+
+
+async def connected_gmail_address(organization_id: str) -> str | None:
+ try:
+ async with asyncio.timeout(_LOOKUP_DEADLINE_SECONDS):
+ return await _connected_gmail_address(organization_id)
+ except Exception:
+ LOG.warning("copilot signin email: lookup did not resolve an address", exc_info=True)
+ return None
+
+
+async def _connected_gmail_address(organization_id: str) -> str | None:
+ gmail_scopes = google_oauth_service.scopes_for_profile("gmail")
+ try:
+ # Active-only: the visible listing also returns ERROR-state credentials, which
+ # load_credential_secrets cannot load, so newer broken ones would eat the cap.
+ credentials = await google_oauth_service.get_credentials_for_org(organization_id)
+ except Exception:
+ LOG.warning("copilot signin email: google credential listing failed", exc_info=True)
+ return None
+
+ # Cap the profile lookups, not the listing: capping first would let a few Sheets-only
+ # connections hide a Gmail one further down the list.
+ gmail_credentials = [
+ credential
+ for credential in credentials
+ if google_oauth_service.has_required_scopes(credential.scopes_granted, gmail_scopes)
+ ]
+ for credential in gmail_credentials[:_MAX_CREDENTIALS_TRIED]:
+ try:
+ secrets = await google_oauth_service.load_credential_secrets(organization_id, credential.id)
+ access_token = await google_oauth_service.access_token_from_secrets(secrets, organization_id)
+ async with httpx.AsyncClient(timeout=_GMAIL_PROFILE_TIMEOUT_SECONDS) as client:
+ profile = await get_json(
+ client,
+ f"{GMAIL_API_BASE}/users/me/profile",
+ access_token=access_token,
+ )
+ except Exception:
+ LOG.warning(
+ "copilot signin email: gmail profile lookup failed",
+ credential_id=credential.id,
+ exc_info=True,
+ )
+ continue
+ address = profile.get("emailAddress")
+ if isinstance(address, str) and is_email_address(address.strip()):
+ return address.strip()
+ return None
diff --git a/skyvern/forge/sdk/routes/streaming/channels/message.py b/skyvern/forge/sdk/routes/streaming/channels/message.py
index a8d84bf97..cba4f44a0 100644
--- a/skyvern/forge/sdk/routes/streaming/channels/message.py
+++ b/skyvern/forge/sdk/routes/streaming/channels/message.py
@@ -961,6 +961,10 @@ async def loop_stream_messages(message_channel: MessageChannel) -> None:
for task in done:
if task.exception() is not None:
raise t.cast(Exception, task.exception())
+ except (WebSocketDisconnect, ConnectionClosedError):
+ # Expected teardown: the frontend closed its live-stream tab. The child loop
+ # already logged this at debug (frontend_to_backend) before re-raising it here.
+ LOG.debug(f"{class_name} frontend disconnected; ending message channel stream.", **message_channel.identity)
except Exception:
LOG.exception(f"{class_name} An exception occurred in loop message channel stream.", **message_channel.identity)
finally:
diff --git a/skyvern/services/error_detection_service.py b/skyvern/services/error_detection_service.py
index 9f1e4f13f..afe917ecc 100644
--- a/skyvern/services/error_detection_service.py
+++ b/skyvern/services/error_detection_service.py
@@ -86,10 +86,11 @@ async def detect_user_defined_errors_for_task(
except Exception:
# Gracefully handle any errors during detection
# Error detection failure should never prevent task from failing
- LOG.exception(
+ LOG.warning(
"Failed to detect user-defined errors, continuing with task failure",
task_id=task.task_id,
step_id=step.step_id,
+ exc_info=True,
)
return []
@@ -154,10 +155,11 @@ async def _detect_errors_from_page(
return user_defined_errors
except Exception:
- LOG.exception(
+ LOG.warning(
"Failed to detect errors from page",
task_id=task.task_id,
step_id=step.step_id,
+ exc_info=True,
)
return []
@@ -259,9 +261,10 @@ async def _detect_errors_from_context(
return user_defined_errors
except Exception:
- LOG.exception(
+ LOG.warning(
"Failed to detect errors from context",
task_id=task.task_id,
step_id=step.step_id,
+ exc_info=True,
)
return []
diff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.py
index 688ab290c..2ee5664d4 100644
--- a/skyvern/webeye/utils/page.py
+++ b/skyvern/webeye/utils/page.py
@@ -19,7 +19,7 @@ from playwright._impl._errors import TimeoutError
from playwright.async_api import ElementHandle, Frame, Locator, Page
from skyvern.constants import PAGE_CONTENT_TIMEOUT, SKYVERN_DIR
-from skyvern.exceptions import FailedToTakeScreenshot, SkyvernPageAnalysisTimeout
+from skyvern.exceptions import FailedToTakeScreenshot, ScreenshotTargetClosed, SkyvernPageAnalysisTimeout
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.settings_manager import SettingsManager
from skyvern.forge.sdk.trace import apply_context_attrs, traced
@@ -31,6 +31,8 @@ if TYPE_CHECKING:
LOG = structlog.get_logger()
+_SCREENSHOT_TARGET_CLOSED_MESSAGE = "Target page, context or browser has been closed"
+
async def _safe_tab_title(page: Page) -> str:
try:
@@ -162,6 +164,10 @@ async def _wait_for_screenshot_load_state(page: Page, timeout_ms: float) -> None
LOG.warning("Page did not reach domcontentloaded before screenshot; capturing current state anyway")
+def _is_screenshot_target_closed(error: BaseException) -> bool:
+ return isinstance(error, PlaywrightError) and _SCREENSHOT_TARGET_CLOSED_MESSAGE in str(error)
+
+
def _load_cursor_overlay_js() -> str:
path = f"{SKYVERN_DIR}/webeye/scraper/cursorOverlay.js"
with open(path, encoding="utf-8") as f:
@@ -220,7 +226,12 @@ async def _current_viewpoint_screenshot_helper(
mode: ScreenshotMode = ScreenshotMode.DETAILED,
) -> bytes:
if page.is_closed():
- raise FailedToTakeScreenshot(error_message="Page is closed")
+ LOG.info(
+ "Skipping screenshot because target is closed",
+ full_page=full_page,
+ mode=mode.value if hasattr(mode, "value") else str(mode),
+ )
+ raise ScreenshotTargetClosed(error_message="Page is closed")
# Capture page context for debugging screenshot issues
url = page.url
@@ -262,6 +273,15 @@ async def _current_viewpoint_screenshot_helper(
)
raise FailedToTakeScreenshot(error_message=str(e)) from e
except Exception as e:
+ if _is_screenshot_target_closed(e):
+ LOG.info(
+ "Skipping screenshot because target closed during capture",
+ url=url,
+ viewport=viewport_info,
+ full_page=full_page,
+ mode=mode.value if hasattr(mode, "value") else str(mode),
+ )
+ raise ScreenshotTargetClosed(error_message=str(e)) from e
LOG.error(
"Screenshot failed",
url=url,
diff --git a/tests/unit/test_copilot_request_policy_workflow_credentials.py b/tests/unit/test_copilot_request_policy_workflow_credentials.py
index b926a2e29..e230ca912 100644
--- a/tests/unit/test_copilot_request_policy_workflow_credentials.py
+++ b/tests/unit/test_copilot_request_policy_workflow_credentials.py
@@ -11,8 +11,11 @@ from skyvern.config import settings
from skyvern.forge.sdk.copilot.context import (
CredentialCheck,
StructuredContext,
+ adopt_model_authored_context,
record_approved_credentials_in_global_llm_context,
+ record_signin_email_in_global_llm_context,
)
+from skyvern.forge.sdk.copilot.credential_pause import credential_pause_reason
from skyvern.forge.sdk.copilot.request_policy import (
_AMBIGUOUS_URL_CREDENTIAL_QUESTION,
_LOGIN_CREDENTIAL_QUESTION,
@@ -24,6 +27,7 @@ from skyvern.forge.sdk.copilot.request_policy import (
RequestPolicy,
_can_defer_unresolved_credential_name_for_draft,
_classification_from_raw,
+ _clean_email_list,
_resolve_credentials,
_should_defer_repeated_unresolved_credential_question,
_workflow_credential_inputs_unbound,
@@ -484,9 +488,12 @@ async def _build_with_forced_classifier(
get_credentials_by_ids: AsyncMock | None = None,
workflow_yaml: str = "",
global_llm_context: str = "",
+ connected_gmail: str | None = None,
+ gmail_lookup: AsyncMock | None = None,
) -> RequestPolicy:
load_mock = get_credentials or AsyncMock(return_value=org_credentials)
by_ids_mock = get_credentials_by_ids or AsyncMock(return_value=[])
+ gmail_mock = gmail_lookup or AsyncMock(return_value=connected_gmail)
with (
patch(
"skyvern.forge.sdk.copilot.request_policy._classify_request",
@@ -494,6 +501,7 @@ async def _build_with_forced_classifier(
),
patch("skyvern.forge.app.DATABASE.credentials.get_credentials", new=load_mock),
patch("skyvern.forge.app.DATABASE.credentials.get_credentials_by_ids", new=by_ids_mock),
+ patch("skyvern.forge.sdk.copilot.request_policy.connected_gmail_address", new=gmail_mock),
):
return await build_request_policy(
user_message=user_message,
@@ -2345,3 +2353,354 @@ async def test_carried_credential_satisfies_the_login_reachability_ask() -> None
assert turn_two.clarification_reason != "login_credentials_unresolved"
assert "cred_portal" in {c.credential_id for c in turn_two.resolved_credentials}
+
+
+def _email_signin_policy(**overrides: object) -> RequestPolicy:
+ defaults: dict[str, object] = {
+ "login_intent": True,
+ "email_signin_intent": True,
+ "credential_input_kind": "none",
+ "login_page_urls": ["https://console.example.com/signin"],
+ "classifier_status": "success",
+ }
+ defaults.update(overrides)
+ return RequestPolicy(**defaults) # type: ignore[arg-type]
+
+
+async def test_an_address_named_in_the_request_outranks_the_connected_gmail_account() -> None:
+ """Naming an address is an explicit instruction; a connected inbox is only the default
+ for when none was named, and must not quietly substitute a different identity."""
+ policy = await _build_with_forced_classifier(
+ user_message="Sign in to https://console.example.com/signin with [email protected] and extract the balance.",
+ classifier_policy=_email_signin_policy(signin_email_candidates=["[email protected]"]),
+ org_credentials=[],
+ connected_gmail="[email protected]",
+ )
+
+ assert policy.resolved_signin_email == "[email protected]"
+ assert policy.clarification_reason == "none"
+ assert policy.user_response_policy == "proceed"
+
+
+async def test_the_connected_account_is_not_even_looked_up_when_the_request_named_one() -> None:
+ lookup = AsyncMock(return_value="[email protected]")
+ policy = await _build_with_forced_classifier(
+ user_message="Sign in to https://console.example.com/signin with [email protected] and extract it.",
+ classifier_policy=_email_signin_policy(signin_email_candidates=["[email protected]"]),
+ org_credentials=[],
+ gmail_lookup=lookup,
+ )
+
+ assert policy.resolved_signin_email == "[email protected]"
+ assert lookup.await_count == 0, "named address should short-circuit the Gmail network call"
+
+
+async def test_the_connected_gmail_account_is_used_when_the_request_named_no_address() -> None:
+ policy = await _build_with_forced_classifier(
+ user_message="Sign in to https://console.example.com/signin by email and extract the balance.",
+ classifier_policy=_email_signin_policy(signin_email_candidates=[]),
+ org_credentials=[],
+ connected_gmail="[email protected]",
+ )
+
+ assert policy.resolved_signin_email == "[email protected]"
+ assert policy.clarification_reason == "none"
+
+
+async def test_address_from_the_request_is_used_when_no_gmail_is_connected() -> None:
+ policy = await _build_with_forced_classifier(
+ user_message="Sign in to https://console.example.com/signin with [email protected] and extract the balance.",
+ classifier_policy=_email_signin_policy(signin_email_candidates=["[email protected]"]),
+ org_credentials=[],
+ connected_gmail=None,
+ )
+
+ assert policy.resolved_signin_email == "[email protected]"
+ assert policy.clarification_reason == "none"
+ assert policy.clarification_question is None
+
+
+async def test_neither_source_asks_for_an_address_and_never_for_a_saved_credential() -> None:
+ policy = await _build_with_forced_classifier(
+ user_message="Sign in to https://console.example.com/signin and extract the balance.",
+ classifier_policy=_email_signin_policy(signin_email_candidates=[]),
+ org_credentials=[],
+ connected_gmail=None,
+ )
+
+ assert policy.resolved_signin_email is None
+ assert policy.clarification_reason == "signin_email_unresolved"
+ assert policy.clarification_question == "Which email address should I sign in with?"
+ assert "cred_" not in (policy.clarification_question or "")
+ assert policy.clarification_reason not in CREDENTIAL_PROMPT_CLARIFICATION_REASONS
+
+
+async def test_email_signin_with_no_org_credentials_never_draws_the_login_credential_ask() -> None:
+ policy = await _build_with_forced_classifier(
+ user_message="Sign in to https://console.example.com/signin with [email protected] and extract the balance.",
+ classifier_policy=_email_signin_policy(signin_email_candidates=["[email protected]"]),
+ org_credentials=[],
+ connected_gmail=None,
+ )
+
+ assert policy.clarification_reason != "login_credentials_unresolved"
+ assert not policy.requires_user_clarification
+
+
+async def test_a_password_login_with_no_credential_still_blocks_exactly_as_before() -> None:
+ policy = await _build_with_forced_classifier(
+ user_message="Log in to https://portal.example.com/login and download this month's invoices.",
+ classifier_policy=RequestPolicy(
+ login_intent=True,
+ email_signin_intent=False,
+ credential_input_kind="website_stored_credential",
+ login_page_urls=["https://portal.example.com/login"],
+ classifier_status="success",
+ ),
+ org_credentials=[],
+ connected_gmail="[email protected]",
+ )
+
+ assert policy.clarification_reason == "login_credentials_unresolved"
+ assert "cred_" in (policy.clarification_question or "")
+ assert policy.resolved_signin_email is None
+
+
+async def test_email_signin_does_not_reach_the_interactive_credential_card() -> None:
+ # Sibling path: the card and the terminal ask both hang off login_credentials_unresolved,
+ # so a fix that only silenced the terminal text would still demand a credential here.
+ policy = await _build_with_forced_classifier(
+ user_message="Sign in to https://console.example.com/signin with [email protected] and extract the balance.",
+ classifier_policy=_email_signin_policy(signin_email_candidates=["[email protected]"]),
+ org_credentials=[],
+ connected_gmail=None,
+ )
+
+ assert credential_pause_reason(SimpleNamespace(request_policy=policy)) is None
+
+
+async def test_skip_test_email_signin_defers_without_asking_for_an_address() -> None:
+ policy = await _build_with_forced_classifier(
+ user_message="Draft a workflow that signs in to https://console.example.com/signin. Don't run it.",
+ classifier_policy=_email_signin_policy(testing_intent="skip_test", signin_email_candidates=[]),
+ org_credentials=[],
+ connected_gmail=None,
+ )
+
+ assert policy.clarification_reason != "signin_email_unresolved"
+ assert policy.resolved_signin_email is None
+
+
+async def test_a_named_saved_credential_keeps_password_resolution_even_with_email_intent() -> None:
+ """Referenced credential material outranks email intent, so a misread flag cannot
+ silently drop the credential requirement from a real password login."""
+ policy = await _build_with_forced_classifier(
+ user_message="Sign in to https://portal.example.com/login using my saved credential portal-login.",
+ classifier_policy=_email_signin_policy(
+ email_signin_intent=True,
+ credential_input_kind="credential_name",
+ credential_refs=["portal-login"],
+ login_page_urls=["https://portal.example.com/login"],
+ ),
+ org_credentials=[],
+ connected_gmail="[email protected]",
+ )
+
+ assert policy.resolved_signin_email is None
+ assert policy.clarification_reason != "signin_email_unresolved"
+
+
+async def test_first_touch_email_signin_with_no_site_yet_asks_nothing_about_an_address() -> None:
+ """No site, workflow, or URL yet: the address is not the question the user still owes."""
+ policy = await _build_with_forced_classifier(
+ user_message="I want to sign in with my email address.",
+ classifier_policy=_email_signin_policy(login_page_urls=[], signin_email_candidates=[]),
+ org_credentials=[],
+ connected_gmail=None,
+ )
+
+ assert policy.clarification_reason != "signin_email_unresolved"
+ assert policy.resolved_signin_email is None
+
+
+async def test_the_email_signin_flag_is_the_only_thing_that_lifts_the_credential_ask() -> None:
+ """The discriminator itself: identical policy but email_signin_intent=False must still
+ block. Without this, the pass-path could be reached by any credential-less login."""
+ policy = await _build_with_forced_classifier(
+ user_message="Log in to https://console.example.com/signin as [email protected] and extract the balance.",
+ classifier_policy=_email_signin_policy(email_signin_intent=False),
+ org_credentials=[],
+ connected_gmail=None,
+ )
+
+ assert policy.clarification_reason == "login_credentials_unresolved"
+ assert policy.resolved_signin_email is None
+
+
+async def test_email_signin_never_overwrites_a_clarification_the_user_already_owes() -> None:
+ policy = await _build_with_forced_classifier(
+ user_message="Put the invoice blocks in a loop and sign in with my email.",
+ classifier_policy=_email_signin_policy(
+ signin_email_candidates=[],
+ clarification_reason="ambiguous_loop_edit",
+ requires_user_clarification=True,
+ ),
+ org_credentials=[],
+ connected_gmail=None,
+ )
+
+ assert policy.clarification_reason == "ambiguous_loop_edit"
+ assert policy.clarification_reason != "signin_email_unresolved"
+
+
+def test_classifier_supplied_addresses_are_shape_checked_before_use() -> None:
+ assert _clean_email_list([" [email protected] "]) == ["[email protected]"]
+ assert _clean_email_list(["[email protected]", "[email protected]"]) == ["[email protected]"]
+ # A workflow parameter default is built from this value, so template and markup
+ # syntax must not survive the boundary.
+ assert _clean_email_list(["{{evil}}@example.com"]) == []
+ assert _clean_email_list(["<img src=x>@example.com"]) == []
+ assert _clean_email_list([f"{'a' * 300}@example.com"]) == []
+ assert _clean_email_list(["not an email", "", "@nope.com", None, 123]) == []
+
+
+async def test_a_matching_org_credential_does_not_pull_a_passwordless_turn_into_a_credential_ask() -> None:
+ """A stored credential for the same host must not reroute the turn into the URL
+ credential tier, which asks for a saved credential under a different reason."""
+ policy = await _build_with_forced_classifier(
+ user_message="Sign in to https://console.example.com/signin with [email protected] and extract it.",
+ classifier_policy=_email_signin_policy(signin_email_candidates=["[email protected]"]),
+ org_credentials=[
+ _cred("console-one", "cred_one", tested_url="https://console.example.com/signin"),
+ _cred("console-two", "cred_two", tested_url="https://console.example.com/signin"),
+ ],
+ connected_gmail=None,
+ )
+
+ assert policy.clarification_reason == "none"
+ assert policy.clarification_question is None
+ assert policy.resolved_signin_email == "[email protected]"
+ assert not policy.resolved_credentials
+
+
+async def test_a_follow_up_turn_keeps_the_sign_in_identity_the_earlier_turn_resolved() -> None:
+ """The classifier reads a terse follow-up as an ordinary login, so without a carry the
+ original bug returns one turn later."""
+ carried = record_signin_email_in_global_llm_context(
+ SimpleNamespace(
+ request_policy=RequestPolicy(
+ resolved_signin_email="[email protected]",
+ resolved_signin_host="https://console.example.com",
+ )
+ ),
+ None,
+ )
+
+ policy = await _build_with_forced_classifier(
+ user_message="also download the invoices from https://console.example.com/invoices",
+ classifier_policy=RequestPolicy(
+ login_intent=True,
+ email_signin_intent=False,
+ credential_input_kind="none",
+ classifier_status="success",
+ ),
+ org_credentials=[],
+ connected_gmail=None,
+ global_llm_context=carried or "",
+ )
+
+ assert policy.clarification_reason != "login_credentials_unresolved"
+ assert policy.resolved_signin_email == "[email protected]"
+
+
+async def test_a_failed_resolution_asks_for_the_address_rather_than_building_without_one() -> None:
+ """The credential requirement is already lifted for this shape, so a silent fall-through
+ would author a login with no identity and no question asked."""
+ policy = await _build_with_forced_classifier(
+ user_message="Sign in to https://console.example.com/signin by email and extract the balance.",
+ classifier_policy=_email_signin_policy(signin_email_candidates=[]),
+ org_credentials=[],
+ gmail_lookup=AsyncMock(side_effect=RuntimeError("boom")),
+ )
+
+ assert policy.clarification_reason == "signin_email_unresolved"
+ assert policy.resolved_signin_email is None
+
+
+async def test_a_carried_identity_does_not_follow_the_user_to_a_different_site() -> None:
+ """Reusing it anywhere else would suppress the password ask on a site that never
+ established it was passwordless."""
+ carried = record_signin_email_in_global_llm_context(
+ SimpleNamespace(
+ request_policy=RequestPolicy(
+ resolved_signin_email="[email protected]",
+ resolved_signin_host="https://console.example.com",
+ )
+ ),
+ None,
+ )
+
+ policy = await _build_with_forced_classifier(
+ user_message="Log in to https://portal.example.com/login and download this month's invoices.",
+ classifier_policy=RequestPolicy(
+ login_intent=True,
+ email_signin_intent=False,
+ credential_input_kind="none",
+ classifier_status="success",
+ ),
+ org_credentials=[],
+ connected_gmail=None,
+ global_llm_context=carried or "",
+ )
+
+ assert policy.clarification_reason == "login_credentials_unresolved"
+ assert policy.resolved_signin_email is None
+
+
+def test_a_model_authored_signin_identity_is_discarded_in_favour_of_the_trusted_one() -> None:
+ """The model can author the context blob; a signin_email it invents would mark the next
+ turn passwordless and suppress the password ask for a site that needs one."""
+ trusted = record_signin_email_in_global_llm_context(
+ SimpleNamespace(
+ request_policy=RequestPolicy(
+ resolved_signin_email="[email protected]",
+ resolved_signin_host="https://console.example.com",
+ )
+ ),
+ None,
+ )
+
+ adopted = adopt_model_authored_context(
+ trusted,
+ {"signin_email": "[email protected]", "signin_email_host": "https://portal.example.com"},
+ )
+
+ assert adopted.signin_email == "[email protected]"
+ assert adopted.signin_email_host == "https://console.example.com"
+
+
+def test_a_model_authored_identity_cannot_appear_where_the_server_resolved_none() -> None:
+ adopted = adopt_model_authored_context(
+ None,
+ {"signin_email": "[email protected]", "signin_email_host": "https://portal.example.com"},
+ )
+
+ assert adopted.signin_email == ""
+ assert adopted.signin_email_host == ""
diff --git a/tests/unit/test_copilot_signin_email.py b/tests/unit/test_copilot_signin_email.py
new file mode 100644
index 000000000..409b9cd16
--- /dev/null
+++ b/tests/unit/test_copilot_signin_email.py
@@ -0,0 +1,95 @@
+from __future__ import annotations
+
+import asyncio
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+from skyvern.forge.sdk.copilot.signin_email import _MAX_CREDENTIALS_TRIED, connected_gmail_address
+
+GMAIL_SCOPES = ["https://www.googleapis.com/auth/gmail.readonly"]
+
+
+def _credential(credential_id: str, scopes: list[str] | None = None) -> SimpleNamespace:
+ return SimpleNamespace(id=credential_id, scopes_granted=GMAIL_SCOPES if scopes is None else scopes)
+
+
+def _patches(credentials: list[SimpleNamespace], profile: AsyncMock | None = None) -> list:
+ return [
+ patch(
+ "skyvern.forge.sdk.copilot.signin_email.google_oauth_service.get_credentials_for_org",
+ new=AsyncMock(return_value=credentials),
+ ),
+ patch(
+ "skyvern.forge.sdk.copilot.signin_email.google_oauth_service.scopes_for_profile",
+ return_value=GMAIL_SCOPES,
+ ),
+ patch(
+ "skyvern.forge.sdk.copilot.signin_email.google_oauth_service.load_credential_secrets",
+ new=AsyncMock(return_value=SimpleNamespace(refresh_token="rt")),
+ ),
+ patch(
+ "skyvern.forge.sdk.copilot.signin_email.google_oauth_service.access_token_from_secrets",
+ new=AsyncMock(return_value="token"),
+ ),
+ patch(
+ "skyvern.forge.sdk.copilot.signin_email.get_json",
+ new=profile or AsyncMock(return_value={"emailAddress": "[email protected]"}),
+ ),
+ ]
+
+
+async def _resolve(credentials: list[SimpleNamespace], profile: AsyncMock | None = None) -> str | None:
+ patches = _patches(credentials, profile)
+ for p in patches:
+ p.start()
+ try:
+ return await connected_gmail_address("o_test")
+ finally:
+ for p in patches:
+ p.stop()
+
+
+async def test_returns_the_address_of_a_gmail_scoped_connection() -> None:
+ assert await _resolve([_credential("gc_1")]) == "[email protected]"
+
+
+async def test_skips_a_connection_without_gmail_scope() -> None:
+ assert await _resolve([_credential("gc_1", scopes=["https://www.googleapis.com/auth/spreadsheets"])]) is None
+
+
+async def test_a_failing_profile_lookup_resolves_to_no_address_rather_than_raising() -> None:
+ # The caller treats None as "no connected account" and falls back to the request;
+ # an exception escaping here would abort the whole policy build.
+ assert await _resolve([_credential("gc_1")], profile=AsyncMock(side_effect=RuntimeError("boom"))) is None
+
+
+async def test_a_hanging_lookup_gives_up_instead_of_stalling_the_turn() -> None:
+ async def _never_returns(*args: object, **kwargs: object) -> dict[str, str]:
+ await asyncio.sleep(3600)
+ return {}
+
+ with patch("skyvern.forge.sdk.copilot.signin_email._LOOKUP_DEADLINE_SECONDS", 0.05):
+ assert await _resolve([_credential("gc_1")], profile=AsyncMock(side_effect=_never_returns)) is None
+
+
+async def test_only_a_bounded_number_of_connections_are_tried() -> None:
+ profile = AsyncMock(return_value={"emailAddress": ""})
+ await _resolve([_credential(f"gc_{n}") for n in range(_MAX_CREDENTIALS_TRIED + 4)], profile=profile)
+
+ assert profile.await_count == _MAX_CREDENTIALS_TRIED
+
+
+async def test_a_gmail_connection_is_found_behind_connections_without_gmail_scope() -> None:
+ sheets = ["https://www.googleapis.com/auth/spreadsheets"]
+ credentials = [_credential(f"gc_sheets_{n}", scopes=sheets) for n in range(_MAX_CREDENTIALS_TRIED + 1)]
+ credentials.append(_credential("gc_gmail"))
+
+ assert await _resolve(credentials) == "[email protected]"
diff --git a/tests/unit/test_mcp_argument_validation.py b/tests/unit/test_mcp_argument_validation.py
index ca83db2ea..eb07c206a 100644
--- a/tests/unit/test_mcp_argument_validation.py
+++ b/tests/unit/test_mcp_argument_validation.py
@@ -7,9 +7,17 @@ names that don't match the tool's contract.
from __future__ import annotations
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
import pytest
+from skyvern.cli.core.result import BrowserContext
+from skyvern.cli.mcp_tools import browser as mcp_browser
from skyvern.cli.mcp_tools import mcp
+from skyvern.cli.mcp_tools import workflow as mcp_workflow
+from skyvern.cli.mcp_tools.argument_validation import _repair_argument_types, _split_comma_separated_list
+from tests.unit._mcp_browser_fakes import make_mock_page, make_skyvern_page
def _structured(result: object) -> dict:
@@ -23,6 +31,21 @@ async def _call(tool_name: str, arguments: dict) -> dict:
return _structured(result)
+def _awaited_kwargs(mock: AsyncMock) -> dict:
+ awaited = mock.await_args
+ assert awaited is not None
+ return dict(awaited.kwargs)
+
+
+def _mock_navigation(monkeypatch: pytest.MonkeyPatch) -> AsyncMock:
+ page = make_skyvern_page(make_mock_page())
+ context = BrowserContext(mode="cloud_session", session_id="pbs_test")
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, context)))
+ navigate = AsyncMock(return_value=SimpleNamespace(url="https://example.com", title="Example"))
+ monkeypatch.setattr(mcp_browser, "do_navigate", navigate)
+ return navigate
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize(
("tool_name", "arguments", "unsupported", "missing"),
@@ -70,3 +93,105 @@ async def test_valid_argument_shape_is_not_blocked() -> None:
error = payload.get("error")
if error is not None:
assert "unsupported_arguments" not in error.get("details", {})
+
+
+async def test_workflow_run_list_comma_separated_status_is_split_for_list_parameter_before_validation(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ list_runs = AsyncMock(return_value=[])
+ monkeypatch.setattr(mcp_workflow, "list_workflow_runs_raw", list_runs)
+
+ payload = await _call(
+ "skyvern_workflow_run_list",
+ {"workflow_id": "wpid_test", "status": "terminated, failed"},
+ )
+
+ assert payload["ok"] is True
+ assert _awaited_kwargs(list_runs)["status"] == ["terminated", "failed"]
+
+
+async def test_comma_separated_string_is_not_split_for_non_list_parameter(monkeypatch: pytest.MonkeyPatch) -> None:
+ list_runs = AsyncMock(return_value=[])
+ monkeypatch.setattr(mcp_workflow, "list_workflow_runs_raw", list_runs)
+
+ payload = await _call(
+ "skyvern_workflow_run_list",
+ {"workflow_id": "wpid_test", "search_key": "terminated,failed"},
+ )
+
+ assert payload["ok"] is True
+ assert _awaited_kwargs(list_runs)["search_key"] == "terminated,failed"
+
+
+async def test_comma_separated_string_is_not_split_for_unapproved_list_parameter() -> None:
+ tool = await mcp.get_tool("skyvern_file_upload")
+ arguments = {"file_paths": "/tmp/Last, First.pdf"}
+
+ _repair_argument_types("skyvern_file_upload", tool, arguments)
+
+ assert arguments["file_paths"] == "/tmp/Last, First.pdf"
+
+
+ "value",
+ ['["terminated","failed"]', "terminated,,failed", "terminated"],
+)
+def test_csv_list_repair_leaves_structured_or_ambiguous_strings_for_other_validation(value: str) -> None:
+ assert _split_comma_separated_list(value) == value
+
+
+def test_csv_list_repair_leaves_excessive_item_count_for_other_validation() -> None:
+ value = ",".join(["terminated"] * 101)
+
+ assert _split_comma_separated_list(value) == value
+
+
+async def test_navigate_sub_1000_timeout_is_treated_as_seconds_and_converted_to_milliseconds(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ navigate = _mock_navigation(monkeypatch)
+
+ payload = await _call("skyvern_navigate", {"url": "https://example.com", "timeout": 60})
+
+ assert payload["ok"] is True
+ assert _awaited_kwargs(navigate)["timeout"] == 60000
+
+
[email protected](("timeout_seconds", "timeout_ms"), [(1.5, 1500), (0.5, 500)])
+async def test_navigate_fractional_seconds_with_whole_milliseconds_are_converted_to_int(
+ timeout_seconds: float,
+ timeout_ms: int,
+) -> None:
+ tool = await mcp.get_tool("skyvern_navigate")
+ arguments = {"timeout": timeout_seconds}
+
+ _repair_argument_types("skyvern_navigate", tool, arguments)
+
+ assert arguments["timeout"] == timeout_ms
+ assert type(arguments["timeout"]) is int
+
+
+async def test_navigate_timeout_at_millisecond_threshold_is_not_scaled(monkeypatch: pytest.MonkeyPatch) -> None:
+ navigate = _mock_navigation(monkeypatch)
+
+ payload = await _call("skyvern_navigate", {"url": "https://example.com", "timeout": 1000})
+
+ assert payload["ok"] is True
+ assert _awaited_kwargs(navigate)["timeout"] == 1000
+
+
+async def test_type_repair_preserves_structured_missing_key_rejection() -> None:
+ payload = await _call("skyvern_workflow_run_list", {"status": "terminated,failed"})
+
+ assert payload["ok"] is False
+ error = payload["error"]
+ assert error["code"] == "INVALID_INPUT"
+ assert error["details"]["unsupported_arguments"] == []
+ assert error["details"]["missing_required_arguments"] == ["workflow_id"]
diff --git a/tests/unit/test_mcp_browser_tools.py b/tests/unit/test_mcp_browser_tools.py
index ec85b6b72..568433bab 100644
--- a/tests/unit/test_mcp_browser_tools.py
+++ b/tests/unit/test_mcp_browser_tools.py
@@ -1330,6 +1330,10 @@ async def test_do_select_option_recognizes_aria_custom_selects(
page, control = make_select_like_page(target)
dom_options = [{"selector": "#music", "role": "option", "name": "Music"}]
monkeypatch.setattr(browser_ops, "_get_dom_observe_elements", AsyncMock(return_value=dom_options))
+ # do_select_option's deadline is real wall clock (started_at + timeout/1000), so a
+ # loaded runner can blow the 100ms budget before the option loop ticks. Freeze the
+ # clock: this asserts the success path, not the deadline.
+ monkeypatch.setattr(browser_ops, "time", SimpleNamespace(monotonic=Mock(return_value=0)))
control.evaluate.side_effect = [
target,
[{"selector": "#music", "label": "Music", "value": "music"}],
diff --git a/tests/unit/webeye/test_screenshot_load_state_nonfatal.py b/tests/unit/webeye/test_screenshot_load_state_nonfatal.py
index ed6e80f5e..d5417b708 100644
--- a/tests/unit/webeye/test_screenshot_load_state_nonfatal.py
+++ b/tests/unit/webeye/test_screenshot_load_state_nonfatal.py
@@ -6,9 +6,12 @@ from unittest.mock import AsyncMock, MagicMock
import pytest
from playwright._impl._errors import Error as PlaywrightError
+from playwright._impl._errors import TargetClosedError
from playwright._impl._errors import TimeoutError as PlaywrightTimeoutError
from playwright.async_api import Page
+from skyvern.exceptions import FailedToTakeScreenshot, ScreenshotTargetClosed
+from skyvern.webeye.utils import page as page_module
from skyvern.webeye.utils.page import ScreenshotMode, _current_viewpoint_screenshot_helper
@@ -60,3 +63,65 @@ class TestScreenshotLoadStateNonFatal:
assert result == b"image-bytes"
page.wait_for_load_state.assert_not_awaited()
+
+
+class TestScreenshotTargetClosedClassification:
+ @pytest.mark.asyncio
+ async def test_preclosed_page_skips_capture_with_target_closed_classification(self) -> None:
+ page = _make_page(b"image-bytes")
+ page.is_closed.return_value = True
+
+ with pytest.raises(ScreenshotTargetClosed):
+ await _current_viewpoint_screenshot_helper(page)
+
+ page.screenshot.assert_not_awaited()
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ "error",
+ [
+ TargetClosedError("Page.screenshot: Target page, context or browser has been closed"),
+ PlaywrightError(
+ "Page.screenshot: Target page, context or browser has been closed\n"
+ "Call log:\n"
+ " - taking page screenshot\n"
+ " - disabled all CSS animations\n"
+ " - waiting for fonts to load..."
+ ),
+ ],
+ )
+ async def test_midflight_target_close_uses_expected_classification(self, error: PlaywrightError) -> None:
+ page = _make_page(b"image-bytes")
+ page.screenshot = AsyncMock(side_effect=error)
+ log = MagicMock()
+
+ with pytest.raises(ScreenshotTargetClosed) as exc_info:
+ with pytest.MonkeyPatch.context() as monkeypatch:
+ monkeypatch.setattr(page_module, "LOG", log)
+ await _current_viewpoint_screenshot_helper(page)
+
+ assert exc_info.value.__cause__ is error
+ log.info.assert_called_once()
+ log.error.assert_not_called()
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ "error",
+ [
+ PlaywrightError("Page.screenshot: Target crashed"),
+ TargetClosedError("Page.screenshot: Target crashed"),
+ PlaywrightError(
+ "Page.captureScreenshot: Protocol error (Page.captureScreenshot): Unable to capture screenshot "
+ "while waiting for fonts to load"
+ ),
+ RuntimeError("Cleanup failed after Target page, context or browser has been closed"),
+ ],
+ )
+ async def test_other_screenshot_failures_remain_generic(self, error: Exception) -> None:
+ page = _make_page(b"image-bytes")
+ page.screenshot = AsyncMock(side_effect=error)
+
+ with pytest.raises(FailedToTakeScreenshot) as exc_info:
+ await _current_viewpoint_screenshot_helper(page)
+
+ assert type(exc_info.value) is FailedToTakeScreenshot
diff --git a/tests/unit_tests/test_id_generator.py b/tests/unit_tests/test_id_generator.py
index e0530a584..8221664bf 100644
--- a/tests/unit_tests/test_id_generator.py
+++ b/tests/unit_tests/test_id_generator.py
@@ -142,18 +142,9 @@ class TestGenerateId:
assert id_val < (1 << 64)
def test_generate_id_uniqueness(self):
- """Multiple calls should generate unique IDs."""
- ids = [generate_id() for _ in range(1000)]
- assert len(set(ids)) == 1000
-
- def test_generate_id_monotonic(self):
- """IDs generated in sequence should generally be increasing."""
- id1 = generate_id()
- id2 = generate_id()
- id3 = generate_id()
- # Due to sequence counter, IDs should increase
- # (within the same second, at least)
- assert id1 < id2 < id3
+ """IDs stay unique across a full randomized sequence wrap."""
+ ids = [generate_id() for _ in range(SEQUENCE_MAX + 2)]
+ assert len(set(ids)) == len(ids)
class TestPrefixedIdGenerators:
diff --git a/tests/unit_tests/test_openrouter_integration.py b/tests/unit_tests/test_openrouter_integration.py
index de5cbf474..b4a1112fe 100644
--- a/tests/unit_tests/test_openrouter_integration.py
+++ b/tests/unit_tests/test_openrouter_integration.py
@@ -19,6 +19,28 @@ def setup_forge_app():
yield
[email protected](autouse=True)
+def _isolate_llm_settings():
+ """Every test below mutates three pieces of process-global state --
+ SettingsManager's active settings instance, LLMConfigRegistry's
+ registered configs, and config_registry's own module-level `settings`
+ binding (rebound on every `importlib.reload`, and read directly by
+ get_config()'s synthesis fallback) -- without restoring any of them,
+ which can leak into (or be leaked into by) unrelated tests depending on
+ execution order. Snapshot and restore all three around every test in
+ this module.
+ """
+ previous_settings = SettingsManager.get_settings()
+ # Shallow copy is safe: LLMConfig/LLMRouterConfig values are frozen dataclasses.
+ previous_configs = dict(config_registry.LLMConfigRegistry._configs)
+ previous_module_settings = config_registry.settings
+ yield
+ SettingsManager.set_settings(previous_settings)
+ config_registry.LLMConfigRegistry._configs.clear()
+ config_registry.LLMConfigRegistry._configs.update(previous_configs)
+ config_registry.settings = previous_module_settings
+
+
class DummyResponse(dict):
def __init__(self, content: str):
super().__init__({"choices": [{"message": {"content": content}}], "usage": {}})
@@ -60,12 +82,21 @@ async def test_openrouter_basic_completion(monkeypatch):
@pytest.mark.asyncio
async def test_openrouter_dynamic_model(monkeypatch):
- # Update settings via monkeypatch to ensure config_registry sees them
-
- monkeypatch.setattr(config.settings, "ENABLE_OPENROUTER", True)
- monkeypatch.setattr(config.settings, "OPENROUTER_API_KEY", "key")
- monkeypatch.setattr(config.settings, "OPENROUTER_MODEL", "base-model")
- monkeypatch.setattr(config.settings, "OPENROUTER_API_BASE", "https://openrouter.ai/api/v1")
+ # config_registry's module-level registration reads skyvern.config.settings
+ # directly, while LLMConfigBase.get_missing_env_vars() reads through
+ # SettingsManager.get_settings() -- normally the same object, but any test
+ # (including siblings in this file) that calls SettingsManager.set_settings()
+ # without restoring it breaks that identity. Point both at one new object so
+ # this test doesn't depend on which one happened to run last.
+ settings = Settings(
+ ENABLE_OPENROUTER=True,
+ OPENROUTER_API_KEY="key",
+ OPENROUTER_MODEL="base-model",
+ OPENROUTER_API_BASE="https://openrouter.ai/api/v1",
+ LLM_KEY="OPENROUTER",
+ )
+ SettingsManager.set_settings(settings)
+ monkeypatch.setattr(config, "settings", settings)
# Clear existing configs before reload
config_registry.LLMConfigRegistry._configs.clear()
@@ -92,6 +123,42 @@ async def test_openrouter_dynamic_model(monkeypatch):
assert called_model == "other-model"
+def test_isolate_llm_settings_restores_module_level_settings_binding():
+ """Regression: config_registry does `from skyvern.config import settings`, so
+ importlib.reload(config_registry) while skyvern.config.settings is monkeypatched
+ rebinds config_registry's own module-level `settings` name to that patched
+ object. The monkeypatch fixture only restores skyvern.config.settings itself,
+ not this separate binding, and get_config()'s synthesis fallback reads it
+ directly -- a leak here silently affects any later test's synthesized configs.
+ """
+ # __wrapped__ recovers the undecorated generator function -- an implementation
+ # detail of pytest's fixture wrapper, not public API, but stable since pytest 3.
+ gen = _isolate_llm_settings.__wrapped__()
+ next(gen) # run the fixture's setup half, snapshotting current state
+
+ original_module_settings = config_registry.settings
+ fake_settings = Settings(
+ ENABLE_OPENROUTER=True,
+ OPENROUTER_API_KEY="leaked-key",
+ OPENROUTER_MODEL="base-model",
+ OPENROUTER_API_BASE="https://openrouter.ai/api/v1",
+ LLM_KEY="OPENROUTER",
+ )
+ with pytest.MonkeyPatch.context() as mp:
+ # Registration reads config_registry's module-level settings (patched below);
+ # validate_config's get_missing_env_vars() reads through SettingsManager --
+ # both must point at the same object or registration itself raises.
+ SettingsManager.set_settings(fake_settings)
+ mp.setattr(config, "settings", fake_settings)
+ importlib.reload(config_registry)
+ assert config_registry.settings is fake_settings # sanity: reload really did rebind it
+
+ with pytest.raises(StopIteration):
+ next(gen) # run the fixture's teardown half
+
+ assert config_registry.settings is original_module_settings
+
+
@pytest.mark.asyncio
async def test_openrouter_error_propagation(monkeypatch):
class DummyAPIError(Exception):
diff --git a/tests/unit_tests/test_token_counter.py b/tests/unit_tests/test_token_counter.py
index 3ebc0f318..bc6e4486e 100644
--- a/tests/unit_tests/test_token_counter.py
+++ b/tests/unit_tests/test_token_counter.py
@@ -1,6 +1,6 @@
"""Tests for token counter utility."""
-from skyvern.utils.token_counter import count_tokens
+from skyvern.utils.token_counter import approx_count_tokens, count_tokens
class TestCountTokens:
@@ -106,3 +106,31 @@ def hello():
char_count = len(text)
assert result > char_count / 10 # Very loose lower bound
assert result < char_count # Token count should be less than char count
+
+
+class TestApproxCountTokens:
+ """Tests for approx_count_tokens function (character-based heuristic)."""
+
+ def test_empty_string(self):
+ assert approx_count_tokens("") == 0
+
+ def test_ceil_division_by_four(self):
+ assert approx_count_tokens("a") == 1
+ assert approx_count_tokens("abcd") == 1
+ assert approx_count_tokens("abcde") == 2
+
+ def test_returns_integer(self):
+ assert isinstance(approx_count_tokens("test"), int)
+
+ def test_no_tiktoken_encode(self, monkeypatch):
+ """The heuristic must not invoke tiktoken encoding regardless of load state."""
+
+ def _boom() -> None:
+ raise AssertionError("approx_count_tokens must not load the encoding")
+
+ monkeypatch.setattr("skyvern.utils.token_counter._get_encoding", _boom)
+ assert approx_count_tokens("word " * 1000) == 1250
+
+ def test_gate_threshold_behavior(self):
+ """Large HTML should exceed the 100k-token gate via the cheap estimate."""
+ assert approx_count_tokens("x" * 500_000) == 125_000