Skip to content

Automate browser based workflows with AI

fix: add default complete_criterion for LoginBlocks to prevent login success misdetection (SKY-8540) (#5256)

Celal Zamanoğlu52d5825e1469
diff --git a/skyvern/constants.py b/skyvern/constants.py
index 49d5111cf..265c392c1 100644
--- a/skyvern/constants.py
+++ b/skyvern/constants.py
@@ -23,6 +23,22 @@ MAX_UPLOAD_FILE_COUNT = 50
AZURE_BLOB_STORAGE_MAX_UPLOAD_FILE_COUNT = 50
DEFAULT_MAX_SCREENSHOT_SCROLLS = 3
+# Default complete_criterion for LoginBlocks. Guides the LLM to check for actual
+# logged-in indicators rather than relying on page location, which fails on sites
+# that redirect to the homepage after successful login.
+DEFAULT_LOGIN_COMPLETE_CRITERION = (
+ "The login is successful. To verify, look for ANY of these logged-in indicators on the page: "
+ "(1) a user name, email, or account name displayed in the header/navigation bar, "
+ "(2) a 'Sign out', 'Log out', or 'Logout' button or link, "
+ "(3) an account/profile menu or avatar that was not present before login, "
+ "(4) a personalized greeting such as 'Welcome [Name]' or 'Hello [Name]', "
+ "(5) a 'My Account', 'My Dashboard', or similar authenticated-only link. "
+ "IMPORTANT: Being redirected to the homepage does NOT mean login failed — "
+ "many websites redirect to the homepage after a successful login. "
+ "Check the page elements carefully for the indicators listed above. "
+ "Do NOT assume login failed just because you are on the homepage or the same page as before."
+)
+
# reserved fields for navigation payload
SPECIAL_FIELD_VERIFICATION_CODE = "verification_code"
diff --git a/skyvern/forge/sdk/db/agent_db.py b/skyvern/forge/sdk/db/agent_db.py
index 4de033c52..6cc82060e 100644
--- a/skyvern/forge/sdk/db/agent_db.py
+++ b/skyvern/forge/sdk/db/agent_db.py
@@ -7440,8 +7440,9 @@ class AgentDB(BaseAlchemyDB):
"""Get workflow runs associated with a script, with total count, status counts,
and average AI fallbacks per run.
- Only includes actual script runs (run_with='code'), excluding agent runs that
- generated the script.
+ Includes actual script runs (run_with='code', 'code_v2', or NULL), excluding
+ explicit agent runs. run_with is NULL when the workflow ran in auto mode and
+ should_run_script() resolved to code via fallback (e.g. code_version >= 1).
Returns (runs, total_count, status_counts, avg_fallbacks_per_run) where runs
is limited by page_size, total_count is derived from the status_counts GROUP BY,
@@ -7473,11 +7474,19 @@ class AgentDB(BaseAlchemyDB):
WorkflowScriptModel.created_at < created_before,
)
- # Base filter for workflow runs — only include actual script runs
+ # Base filter for workflow runs — only include actual script runs.
+ # run_with may be NULL when the workflow ran in auto mode and
+ # should_run_script() resolved to code mode via fallback (e.g.
+ # code_version >= 1 or adaptive_caching). NULL is therefore
+ # treated as a code run here; explicit "agent" runs are excluded
+ # by not appearing in the workflow_scripts join above.
base_filters = [
WorkflowRunModel.workflow_run_id.in_(run_ids_subquery),
WorkflowRunModel.organization_id == organization_id,
- WorkflowRunModel.run_with.in_(["code", "code_v2"]), # include legacy "code_v2" rows
+ or_(
+ WorkflowRunModel.run_with.in_(["code", "code_v2"]),
+ WorkflowRunModel.run_with.is_(None),
+ ),
]
# Count statuses via GROUP BY (also gives us total_count)
@@ -7563,7 +7572,12 @@ class AgentDB(BaseAlchemyDB):
WorkflowScriptModel.deleted_at.is_(None),
WorkflowScriptModel.workflow_run_id.isnot(None),
WorkflowRunModel.organization_id == organization_id,
- WorkflowRunModel.run_with.in_(["code", "code_v2"]), # include legacy "code_v2" rows
+ # Same NULL-inclusion logic as get_workflow_runs_for_script:
+ # runs with NULL run_with executed in auto/code mode via fallback.
+ or_(
+ WorkflowRunModel.run_with.in_(["code", "code_v2"]),
+ WorkflowRunModel.run_with.is_(None),
+ ),
)
.group_by(WorkflowScriptModel.script_id, WorkflowRunModel.status)
)
diff --git a/skyvern/forge/sdk/routes/run_blocks.py b/skyvern/forge/sdk/routes/run_blocks.py
index 714425007..230b8916b 100644
--- a/skyvern/forge/sdk/routes/run_blocks.py
+++ b/skyvern/forge/sdk/routes/run_blocks.py
@@ -39,11 +39,26 @@ from skyvern.services import workflow_service
from skyvern.utils.url_validators import prepend_scheme_and_validate_url
LOG = structlog.get_logger()
-DEFAULT_LOGIN_PROMPT = """If you're not on the login page, navigate to login page and login using the credentials given.
-First, take actions on promotional popups or cookie prompts that could prevent taking other action on the web page.
-If a 2-factor step appears, enter the authentication code.
-If you fail to login or can't find the login page after several trials, terminate.
-If the credentials are invalid, expired, or rejected by the website, terminate immediately and take no further actions.
+DEFAULT_LOGIN_PROMPT = """\
+If you're not on the login page, navigate to the login page first.
+First, dismiss any promotional popups or cookie prompts that could block interaction with the page.
+
+Log in using the credentials provided in the user details:
+1. Find the username/email input field and enter the username or email from the provided credentials.
+2. Find the password input field and enter the password from the provided credentials. \
+Some websites use a multi-step login flow where you enter the email first, click a "Continue" or "Next" button, \
+and then the password field appears on the next step. Handle this by entering the email, clicking continue, \
+then entering the password once the field is revealed.
+3. Click the login/sign-in button to submit the credentials.
+4. If a 2-factor authentication step appears, enter the authentication code.
+
+Make sure you enter the username and password separately — do not paste both into the same field.
+Use your action history to determine if you already attempted to log in. \
+If you have not clicked the login button since filling in the credentials, try submitting before assuming failure.
+
+If you fail to log in or can't find the login page after several trials, terminate.
+If the credentials are invalid, expired, or explicitly rejected by the website (e.g., "Invalid credentials", \
+"Wrong password"), terminate immediately and take no further actions.
If login is completed, you're successful."""
diff --git a/skyvern/forge/sdk/workflow/workflow_definition_converter.py b/skyvern/forge/sdk/workflow/workflow_definition_converter.py
index 5ea111cba..e225d25ca 100644
--- a/skyvern/forge/sdk/workflow/workflow_definition_converter.py
+++ b/skyvern/forge/sdk/workflow/workflow_definition_converter.py
@@ -5,6 +5,7 @@ from typing import Any, cast
import structlog
from skyvern.config import settings
+from skyvern.constants import DEFAULT_LOGIN_COMPLETE_CRITERION
from skyvern.forge.sdk.db.enums import TaskType
from skyvern.forge.sdk.db.id import (
generate_aws_secret_parameter_id,
@@ -633,6 +634,13 @@ def block_yaml_to_block(
elif block_yaml.block_type == BlockType.LOGIN:
login_block_parameters = _resolve_block_parameters(block_yaml, parameters)
+ # Apply a default complete_criterion for login blocks when the user hasn't provided one.
+ # This guides the LLM to check for actual logged-in indicators (username in header,
+ # account menu, logout button) rather than relying on page location, which fails on sites
+ # that redirect to the homepage after successful login.
+ login_complete_criterion = block_yaml.complete_criterion
+ if not login_complete_criterion or not login_complete_criterion.strip():
+ login_complete_criterion = DEFAULT_LOGIN_COMPLETE_CRITERION
return LoginBlock(
**base_kwargs,
url=block_yaml.url,
@@ -646,7 +654,7 @@ def block_yaml_to_block(
totp_verification_url=block_yaml.totp_verification_url,
totp_identifier=block_yaml.totp_identifier,
disable_cache=block_yaml.disable_cache,
- complete_criterion=block_yaml.complete_criterion,
+ complete_criterion=login_complete_criterion,
terminate_criterion=block_yaml.terminate_criterion,
complete_verification=block_yaml.complete_verification,
)
diff --git a/tests/unit/test_login_block_default_complete_criterion.py b/tests/unit/test_login_block_default_complete_criterion.py
new file mode 100644
index 000000000..e6d2afc66
--- /dev/null
+++ b/tests/unit/test_login_block_default_complete_criterion.py
@@ -0,0 +1,105 @@
+"""Tests for LoginBlock default complete_criterion in workflow_definition_converter."""
+
+from datetime import UTC, datetime
+
+from skyvern.forge.sdk.workflow.models.block import LoginBlock
+from skyvern.forge.sdk.workflow.models.parameter import OutputParameter
+from skyvern.forge.sdk.workflow.workflow_definition_converter import block_yaml_to_block
+from skyvern.schemas.workflows import LoginBlockYAML
+
+_NOW = datetime.now(UTC)
+
+
+def _make_output_parameter(label: str) -> OutputParameter:
+ return OutputParameter(
+ parameter_type="output",
+ key=f"{label}_output",
+ workflow_id="test_wf",
+ output_parameter_id=f"op_{label}",
+ created_at=_NOW,
+ modified_at=_NOW,
+ )
+
+
+def _convert_login_block(block_yaml: LoginBlockYAML) -> LoginBlock:
+ """Helper to convert a LoginBlockYAML through block_yaml_to_block."""
+ output_param = _make_output_parameter(block_yaml.label)
+ parameters = {output_param.key: output_param}
+ block = block_yaml_to_block(block_yaml, parameters)
+ assert isinstance(block, LoginBlock)
+ return block
+
+
+class TestLoginBlockDefaultCompleteCriterion:
+ """Tests that LoginBlocks get a sensible default complete_criterion when none is provided."""
+
+ def test_no_complete_criterion_gets_default(self) -> None:
+ """LoginBlock without complete_criterion should get the default login verification string."""
+ block_yaml = LoginBlockYAML(
+ label="login",
+ navigation_goal="Log in to the website",
+ )
+ assert block_yaml.complete_criterion is None
+
+ block = _convert_login_block(block_yaml)
+
+ assert block.complete_criterion is not None
+ assert "logged-in indicators" in block.complete_criterion
+ assert "homepage" in block.complete_criterion.lower()
+
+ def test_empty_string_complete_criterion_gets_default(self) -> None:
+ """LoginBlock with empty string complete_criterion should get the default."""
+ block_yaml = LoginBlockYAML(
+ label="login",
+ navigation_goal="Log in to the website",
+ complete_criterion="",
+ )
+
+ block = _convert_login_block(block_yaml)
+
+ assert block.complete_criterion is not None
+ assert "logged-in indicators" in block.complete_criterion
+
+ def test_whitespace_only_complete_criterion_gets_default(self) -> None:
+ """LoginBlock with whitespace-only complete_criterion should get the default."""
+ block_yaml = LoginBlockYAML(
+ label="login",
+ navigation_goal="Log in to the website",
+ complete_criterion=" ",
+ )
+
+ block = _convert_login_block(block_yaml)
+
+ assert block.complete_criterion is not None
+ assert "logged-in indicators" in block.complete_criterion
+
+ def test_explicit_complete_criterion_preserved(self) -> None:
+ """LoginBlock with an explicit complete_criterion should keep it unchanged."""
+ custom_criterion = "URL contains '/dashboard'"
+ block_yaml = LoginBlockYAML(
+ label="login",
+ navigation_goal="Log in to the website",
+ complete_criterion=custom_criterion,
+ )
+
+ block = _convert_login_block(block_yaml)
+
+ assert block.complete_criterion == custom_criterion
+
+ def test_default_criterion_mentions_key_indicators(self) -> None:
+ """The default criterion should mention the key logged-in indicators."""
+ block_yaml = LoginBlockYAML(
+ label="login",
+ navigation_goal="Log in to the website",
+ )
+
+ block = _convert_login_block(block_yaml)
+
+ criterion = block.complete_criterion
+ assert criterion is not None
+ # Should mention checking for logout button / sign out
+ assert "Sign out" in criterion or "Log out" in criterion or "Logout" in criterion
+ # Should mention checking for username/account in header
+ assert "user name" in criterion or "email" in criterion or "account name" in criterion
+ # Should warn about homepage redirect not meaning failure
+ assert "homepage" in criterion.lower()
diff --git a/tests/unit/test_mcp_inspection.py b/tests/unit/test_mcp_inspection.py
index 5ea1592c5..6fb8f7eaa 100644
--- a/tests/unit/test_mcp_inspection.py
+++ b/tests/unit/test_mcp_inspection.py
@@ -41,6 +41,9 @@ def _patch(monkeypatch: pytest.MonkeyPatch, state: SessionState) -> None:
monkeypatch.setattr("skyvern.cli.mcp_tools.inspection.get_page", fake_get_page)
monkeypatch.setattr("skyvern.cli.mcp_tools.inspection.get_current_session", lambda: state)
+ # Reset stateless HTTP mode — cloud_app.py sets this to True at startup when MCP_ENABLED,
+ # which causes inspection tools to short-circuit with ACTION_FAILED before reaching test logic.
+ monkeypatch.setattr("skyvern.cli.core.session_manager._stateless_http_mode", False)
def _console_entry(level: str = "log", text: str = "msg") -> dict: