Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix: add default complete_criterion for LoginBlocks to prevent login success misdetection (SKY-8540) (#5256)
Celal Zamanoğlu52d5825e1469
diff --git a/skyvern/constants.py b/skyvern/constants.pyindex 49d5111cf..265c392c1 100644--- a/skyvern/constants.py+++ b/skyvern/constants.py@@ -23,6 +23,22 @@ MAX_UPLOAD_FILE_COUNT = 50AZURE_BLOB_STORAGE_MAX_UPLOAD_FILE_COUNT = 50DEFAULT_MAX_SCREENSHOT_SCROLLS = 3+# Default complete_criterion for LoginBlocks. Guides the LLM to check for actual+# logged-in indicators rather than relying on page location, which fails on sites+# that redirect to the homepage after successful login.+DEFAULT_LOGIN_COMPLETE_CRITERION = (+ "The login is successful. To verify, look for ANY of these logged-in indicators on the page: "+ "(1) a user name, email, or account name displayed in the header/navigation bar, "+ "(2) a 'Sign out', 'Log out', or 'Logout' button or link, "+ "(3) an account/profile menu or avatar that was not present before login, "+ "(4) a personalized greeting such as 'Welcome [Name]' or 'Hello [Name]', "+ "(5) a 'My Account', 'My Dashboard', or similar authenticated-only link. "+ "IMPORTANT: Being redirected to the homepage does NOT mean login failed — "+ "many websites redirect to the homepage after a successful login. "+ "Check the page elements carefully for the indicators listed above. "+ "Do NOT assume login failed just because you are on the homepage or the same page as before."+)+# reserved fields for navigation payloadSPECIAL_FIELD_VERIFICATION_CODE = "verification_code"diff --git a/skyvern/forge/sdk/db/agent_db.py b/skyvern/forge/sdk/db/agent_db.pyindex 4de033c52..6cc82060e 100644--- a/skyvern/forge/sdk/db/agent_db.py+++ b/skyvern/forge/sdk/db/agent_db.py@@ -7440,8 +7440,9 @@ class AgentDB(BaseAlchemyDB):"""Get workflow runs associated with a script, with total count, status counts,and average AI fallbacks per run.- Only includes actual script runs (run_with='code'), excluding agent runs that- generated the script.+ Includes actual script runs (run_with='code', 'code_v2', or NULL), excluding+ explicit agent runs. run_with is NULL when the workflow ran in auto mode and+ should_run_script() resolved to code via fallback (e.g. code_version >= 1).Returns (runs, total_count, status_counts, avg_fallbacks_per_run) where runsis limited by page_size, total_count is derived from the status_counts GROUP BY,@@ -7473,11 +7474,19 @@ class AgentDB(BaseAlchemyDB):WorkflowScriptModel.created_at < created_before,)- # Base filter for workflow runs — only include actual script runs+ # Base filter for workflow runs — only include actual script runs.+ # run_with may be NULL when the workflow ran in auto mode and+ # should_run_script() resolved to code mode via fallback (e.g.+ # code_version >= 1 or adaptive_caching). NULL is therefore+ # treated as a code run here; explicit "agent" runs are excluded+ # by not appearing in the workflow_scripts join above.base_filters = [WorkflowRunModel.workflow_run_id.in_(run_ids_subquery),WorkflowRunModel.organization_id == organization_id,- WorkflowRunModel.run_with.in_(["code", "code_v2"]), # include legacy "code_v2" rows+ or_(+ WorkflowRunModel.run_with.in_(["code", "code_v2"]),+ WorkflowRunModel.run_with.is_(None),+ ),]# Count statuses via GROUP BY (also gives us total_count)@@ -7563,7 +7572,12 @@ class AgentDB(BaseAlchemyDB):WorkflowScriptModel.deleted_at.is_(None),WorkflowScriptModel.workflow_run_id.isnot(None),WorkflowRunModel.organization_id == organization_id,- WorkflowRunModel.run_with.in_(["code", "code_v2"]), # include legacy "code_v2" rows+ # Same NULL-inclusion logic as get_workflow_runs_for_script:+ # runs with NULL run_with executed in auto/code mode via fallback.+ or_(+ WorkflowRunModel.run_with.in_(["code", "code_v2"]),+ WorkflowRunModel.run_with.is_(None),+ ),).group_by(WorkflowScriptModel.script_id, WorkflowRunModel.status))diff --git a/skyvern/forge/sdk/routes/run_blocks.py b/skyvern/forge/sdk/routes/run_blocks.pyindex 714425007..230b8916b 100644--- a/skyvern/forge/sdk/routes/run_blocks.py+++ b/skyvern/forge/sdk/routes/run_blocks.py@@ -39,11 +39,26 @@ from skyvern.services import workflow_servicefrom skyvern.utils.url_validators import prepend_scheme_and_validate_urlLOG = structlog.get_logger()-DEFAULT_LOGIN_PROMPT = """If you're not on the login page, navigate to login page and login using the credentials given.-First, take actions on promotional popups or cookie prompts that could prevent taking other action on the web page.-If a 2-factor step appears, enter the authentication code.-If you fail to login or can't find the login page after several trials, terminate.-If the credentials are invalid, expired, or rejected by the website, terminate immediately and take no further actions.+DEFAULT_LOGIN_PROMPT = """\+If you're not on the login page, navigate to the login page first.+First, dismiss any promotional popups or cookie prompts that could block interaction with the page.++Log in using the credentials provided in the user details:+1. Find the username/email input field and enter the username or email from the provided credentials.+2. Find the password input field and enter the password from the provided credentials. \+Some websites use a multi-step login flow where you enter the email first, click a "Continue" or "Next" button, \+and then the password field appears on the next step. Handle this by entering the email, clicking continue, \+then entering the password once the field is revealed.+3. Click the login/sign-in button to submit the credentials.+4. If a 2-factor authentication step appears, enter the authentication code.++Make sure you enter the username and password separately — do not paste both into the same field.+Use your action history to determine if you already attempted to log in. \+If you have not clicked the login button since filling in the credentials, try submitting before assuming failure.++If you fail to log in or can't find the login page after several trials, terminate.+If the credentials are invalid, expired, or explicitly rejected by the website (e.g., "Invalid credentials", \+"Wrong password"), terminate immediately and take no further actions.If login is completed, you're successful."""diff --git a/skyvern/forge/sdk/workflow/workflow_definition_converter.py b/skyvern/forge/sdk/workflow/workflow_definition_converter.pyindex 5ea111cba..e225d25ca 100644--- a/skyvern/forge/sdk/workflow/workflow_definition_converter.py+++ b/skyvern/forge/sdk/workflow/workflow_definition_converter.py@@ -5,6 +5,7 @@ from typing import Any, castimport structlogfrom skyvern.config import settings+from skyvern.constants import DEFAULT_LOGIN_COMPLETE_CRITERIONfrom skyvern.forge.sdk.db.enums import TaskTypefrom skyvern.forge.sdk.db.id import (generate_aws_secret_parameter_id,@@ -633,6 +634,13 @@ def block_yaml_to_block(elif block_yaml.block_type == BlockType.LOGIN:login_block_parameters = _resolve_block_parameters(block_yaml, parameters)+ # Apply a default complete_criterion for login blocks when the user hasn't provided one.+ # This guides the LLM to check for actual logged-in indicators (username in header,+ # account menu, logout button) rather than relying on page location, which fails on sites+ # that redirect to the homepage after successful login.+ login_complete_criterion = block_yaml.complete_criterion+ if not login_complete_criterion or not login_complete_criterion.strip():+ login_complete_criterion = DEFAULT_LOGIN_COMPLETE_CRITERIONreturn LoginBlock(**base_kwargs,url=block_yaml.url,@@ -646,7 +654,7 @@ def block_yaml_to_block(totp_verification_url=block_yaml.totp_verification_url,totp_identifier=block_yaml.totp_identifier,disable_cache=block_yaml.disable_cache,- complete_criterion=block_yaml.complete_criterion,+ complete_criterion=login_complete_criterion,terminate_criterion=block_yaml.terminate_criterion,complete_verification=block_yaml.complete_verification,)diff --git a/tests/unit/test_login_block_default_complete_criterion.py b/tests/unit/test_login_block_default_complete_criterion.pynew file mode 100644index 000000000..e6d2afc66--- /dev/null+++ b/tests/unit/test_login_block_default_complete_criterion.py@@ -0,0 +1,105 @@+"""Tests for LoginBlock default complete_criterion in workflow_definition_converter."""++from datetime import UTC, datetime++from skyvern.forge.sdk.workflow.models.block import LoginBlock+from skyvern.forge.sdk.workflow.models.parameter import OutputParameter+from skyvern.forge.sdk.workflow.workflow_definition_converter import block_yaml_to_block+from skyvern.schemas.workflows import LoginBlockYAML++_NOW = datetime.now(UTC)+++def _make_output_parameter(label: str) -> OutputParameter:+ return OutputParameter(+ parameter_type="output",+ key=f"{label}_output",+ workflow_id="test_wf",+ output_parameter_id=f"op_{label}",+ created_at=_NOW,+ modified_at=_NOW,+ )+++def _convert_login_block(block_yaml: LoginBlockYAML) -> LoginBlock:+ """Helper to convert a LoginBlockYAML through block_yaml_to_block."""+ output_param = _make_output_parameter(block_yaml.label)+ parameters = {output_param.key: output_param}+ block = block_yaml_to_block(block_yaml, parameters)+ assert isinstance(block, LoginBlock)+ return block+++class TestLoginBlockDefaultCompleteCriterion:+ """Tests that LoginBlocks get a sensible default complete_criterion when none is provided."""++ def test_no_complete_criterion_gets_default(self) -> None:+ """LoginBlock without complete_criterion should get the default login verification string."""+ block_yaml = LoginBlockYAML(+ label="login",+ navigation_goal="Log in to the website",+ )+ assert block_yaml.complete_criterion is None++ block = _convert_login_block(block_yaml)++ assert block.complete_criterion is not None+ assert "logged-in indicators" in block.complete_criterion+ assert "homepage" in block.complete_criterion.lower()++ def test_empty_string_complete_criterion_gets_default(self) -> None:+ """LoginBlock with empty string complete_criterion should get the default."""+ block_yaml = LoginBlockYAML(+ label="login",+ navigation_goal="Log in to the website",+ complete_criterion="",+ )++ block = _convert_login_block(block_yaml)++ assert block.complete_criterion is not None+ assert "logged-in indicators" in block.complete_criterion++ def test_whitespace_only_complete_criterion_gets_default(self) -> None:+ """LoginBlock with whitespace-only complete_criterion should get the default."""+ block_yaml = LoginBlockYAML(+ label="login",+ navigation_goal="Log in to the website",+ complete_criterion=" ",+ )++ block = _convert_login_block(block_yaml)++ assert block.complete_criterion is not None+ assert "logged-in indicators" in block.complete_criterion++ def test_explicit_complete_criterion_preserved(self) -> None:+ """LoginBlock with an explicit complete_criterion should keep it unchanged."""+ custom_criterion = "URL contains '/dashboard'"+ block_yaml = LoginBlockYAML(+ label="login",+ navigation_goal="Log in to the website",+ complete_criterion=custom_criterion,+ )++ block = _convert_login_block(block_yaml)++ assert block.complete_criterion == custom_criterion++ def test_default_criterion_mentions_key_indicators(self) -> None:+ """The default criterion should mention the key logged-in indicators."""+ block_yaml = LoginBlockYAML(+ label="login",+ navigation_goal="Log in to the website",+ )++ block = _convert_login_block(block_yaml)++ criterion = block.complete_criterion+ assert criterion is not None+ # Should mention checking for logout button / sign out+ assert "Sign out" in criterion or "Log out" in criterion or "Logout" in criterion+ # Should mention checking for username/account in header+ assert "user name" in criterion or "email" in criterion or "account name" in criterion+ # Should warn about homepage redirect not meaning failure+ assert "homepage" in criterion.lower()diff --git a/tests/unit/test_mcp_inspection.py b/tests/unit/test_mcp_inspection.pyindex 5ea1592c5..6fb8f7eaa 100644--- a/tests/unit/test_mcp_inspection.py+++ b/tests/unit/test_mcp_inspection.py@@ -41,6 +41,9 @@ def _patch(monkeypatch: pytest.MonkeyPatch, state: SessionState) -> None:monkeypatch.setattr("skyvern.cli.mcp_tools.inspection.get_page", fake_get_page)monkeypatch.setattr("skyvern.cli.mcp_tools.inspection.get_current_session", lambda: state)+ # Reset stateless HTTP mode — cloud_app.py sets this to True at startup when MCP_ENABLED,+ # which causes inspection tools to short-circuit with ACTION_FAILED before reaching test logic.+ monkeypatch.setattr("skyvern.cli.core.session_manager._stateless_http_mode", False)def _console_entry(level: str = "log", text: str = "msg") -> dict: