Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(bitwarden): coerce null username/password/totp to empty string (SKY-8847) (#5936)
Aaron Perezcf3901a03f6c
diff --git a/skyvern/core/script_generations/real_skyvern_page_ai.py b/skyvern/core/script_generations/real_skyvern_page_ai.pyindex 4a49cb94d..3abb13fb1 100644--- a/skyvern/core/script_generations/real_skyvern_page_ai.py+++ b/skyvern/core/script_generations/real_skyvern_page_ai.py@@ -869,12 +869,19 @@ class RealSkyvernPageAi(SkyvernPageAi):self,navigation_goal: str,max_steps: int = 5,+ validate_first: bool = False,) -> None:"""Drive an AI agent from the current page state toward ``navigation_goal``.Records one fallback episode per invocation regardless of exit path; popsany preceding classify metadata at entry so subsequent fallbacks cannotinherit it.++ ``validate_first=True`` re-enables the legacy pre-act validate on+ iteration 0 for defensive callers that may invoke this when the page+ already satisfies the goal; the default (False) skips that validate+ because the documented call site (classify-UNKNOWN / selector-failure+ fallback) cannot be on the success state."""context = skyvern_context.current()if not context or not context.organization_id:@@ -906,17 +913,19 @@ class RealSkyvernPageAi(SkyvernPageAi):try:for step_num in range(max_steps):- is_complete = await self.ai_validate(- prompt=f"Has the following goal been achieved? Goal: {navigation_goal}",- )- if is_complete:- LOG.info(- "page.element_fallback: goal achieved",- step_num=step_num,- navigation_goal=navigation_goal,+ # Skip iteration-0 validate; opt in via validate_first for defensive callers.+ if step_num > 0 or validate_first:+ is_complete = await self.ai_validate(+ prompt=f"Has the following goal been achieved? Goal: {navigation_goal}",)- completed = True- break+ if is_complete:+ LOG.info(+ "page.element_fallback: goal achieved",+ step_num=step_num,+ navigation_goal=navigation_goal,+ )+ completed = True+ breakLOG.info("page.element_fallback: executing step",@@ -941,6 +950,20 @@ class RealSkyvernPageAi(SkyvernPageAi):"actions": actions_this_step[:10],})+ else:+ # All max_steps acts ran without an early break — final validate+ # so max_steps=1 can succeed when the lone act completed the goal.+ if max_steps > 0:+ is_complete = await self.ai_validate(+ prompt=f"Has the following goal been achieved? Goal: {navigation_goal}",+ )+ if is_complete:+ LOG.info(+ "page.element_fallback: goal achieved",+ step_num=max_steps,+ navigation_goal=navigation_goal,+ )+ completed = Trueexcept Exception as exc:exception_summary = f"{type(exc).__name__}: {exc!s}"[:500]captured_exception = excdiff --git a/skyvern/core/script_generations/skyvern_page.py b/skyvern/core/script_generations/skyvern_page.pyindex 11f9656aa..e7ffe6327 100644--- a/skyvern/core/script_generations/skyvern_page.py+++ b/skyvern/core/script_generations/skyvern_page.py@@ -3230,6 +3230,7 @@ class SkyvernPage(Page):self,navigation_goal: str,max_steps: int = 5,+ validate_first: bool = False,) -> None:"""Activate the AI agent from the CURRENT page position to achieve a goal.@@ -3240,6 +3241,11 @@ class SkyvernPage(Page):Args:navigation_goal: The goal for the AI agent to achieve from the current page.max_steps: Maximum number of agent steps before giving up. Defaults to 5.+ validate_first: When True, run an AI validation before the first+ action to short-circuit if the page already satisfies the goal.+ Defaults to False because the documented call site (else branch+ of ``classify``) cannot be on the success state — pass True for+ defensive calls that may invoke this on a possibly-complete page.Raises:Exception: If the element fallback fails or exceeds max_steps.@@ -3260,6 +3266,7 @@ class SkyvernPage(Page):return await self._ai.ai_element_fallback(navigation_goal=navigation_goal,max_steps=max_steps,+ validate_first=validate_first,)async def prompt(diff --git a/skyvern/core/script_generations/skyvern_page_ai.py b/skyvern/core/script_generations/skyvern_page_ai.pyindex 701b824c4..c91ab9146 100644--- a/skyvern/core/script_generations/skyvern_page_ai.py+++ b/skyvern/core/script_generations/skyvern_page_ai.py@@ -124,11 +124,16 @@ class SkyvernPageAi(Protocol):self,navigation_goal: str,max_steps: int = 5,+ validate_first: bool = False,) -> None:"""Activate the AI agent from the CURRENT page position to achieve a navigation goal.This is a mid-block fallback that picks up from the current page stateinstead of re-running the entire block from scratch.++ ``validate_first=True`` re-enables the legacy pre-act validate on+ iteration 0 for defensive callers that may invoke this when the page+ already satisfies the goal."""...diff --git a/skyvern/forge/sdk/services/bitwarden.py b/skyvern/forge/sdk/services/bitwarden.pyindex 4d86665ab..c9b9e79a0 100644--- a/skyvern/forge/sdk/services/bitwarden.py+++ b/skyvern/forge/sdk/services/bitwarden.py@@ -353,11 +353,11 @@ class BitwardenService:raise BitwardenGetItemError(f"Failed to parse item JSON for item ID: {item_id}")login = item["login"]- totp = BitwardenService.extract_totp_secret(login.get("totp", ""))+ totp = BitwardenService.extract_totp_secret(login.get("totp") or "")return {- BitwardenConstants.USERNAME: login.get("username", ""),- BitwardenConstants.PASSWORD: login.get("password", ""),+ BitwardenConstants.USERNAME: login.get("username") or "",+ BitwardenConstants.PASSWORD: login.get("password") or "",BitwardenConstants.TOTP: totp,}elif not url:@@ -415,16 +415,16 @@ class BitwardenService:continuelogin = item["login"]- totp = BitwardenService.extract_totp_secret(login.get("totp", ""))+ totp = BitwardenService.extract_totp_secret(login.get("totp") or "")bitwarden_result.append(BitwardenQueryResult(credential={- BitwardenConstants.USERNAME: login.get("username", ""),- BitwardenConstants.PASSWORD: login.get("password", ""),+ BitwardenConstants.USERNAME: login.get("username") or "",+ BitwardenConstants.PASSWORD: login.get("password") or "",BitwardenConstants.TOTP: totp,},- uris=[uri.get("uri") for uri in login.get("uris", []) if "uri" in uri],+ uris=[uri.get("uri") for uri in login.get("uris") or [] if uri.get("uri")],))diff --git a/skyvern/library/skyvern_browser_page_ai.py b/skyvern/library/skyvern_browser_page_ai.pyindex 8cee0be0e..1244a3b23 100644--- a/skyvern/library/skyvern_browser_page_ai.py+++ b/skyvern/library/skyvern_browser_page_ai.py@@ -337,6 +337,7 @@ class SdkSkyvernPageAi(SkyvernPageAi):self,navigation_goal: str,max_steps: int = 10,+ validate_first: bool = False,) -> None:"""Activate the AI agent from the current page position.diff --git a/tests/unit/test_bitwarden_null_credentials.py b/tests/unit/test_bitwarden_null_credentials.pynew file mode 100644index 000000000..2ec50dd2b--- /dev/null+++ b/tests/unit/test_bitwarden_null_credentials.py@@ -0,0 +1,146 @@+"""Tests for SKY-8847: Bitwarden CLI path must coerce null username/password to empty strings.++The Bitwarden CLI sometimes returns login items whose `username`, `password`, or `totp`+fields are present but have a JSON `null` value. `dict.get(key, default)` returns the+actual value (i.e. `None`) when the key exists, not the default — so callers must+explicitly coerce `None` to `""` before handing the value to the form-filler.++The server-side path (`_get_login_item_by_id_using_server`) already does this with+`login["username"] or ""`; the CLI path used to use the bare `dict.get(..., "")`+pattern, which silently propagated `None` into the secret store. That manifested as+spurious "invalid credentials" errors during runs because the real value never got+typed into the login form.+"""++import json++import pytest++from skyvern.forge.sdk.services.bitwarden import (+ BitwardenConstants,+ BitwardenService,+ RunCommandResult,+)++[email protected](autouse=True)+def _stub_bitwarden_auth(monkeypatch: pytest.MonkeyPatch) -> None:+ async def _noop_login(*args: object, **kwargs: object) -> None:+ return None++ async def _noop_sync() -> None:+ return None++ async def _fake_unlock(_master_password: str) -> str:+ return "session-key"++ async def _noop_logout() -> None:+ return None++ async def _noop_jitter() -> None:+ return None++ monkeypatch.setattr(BitwardenService, "login", _noop_login)+ monkeypatch.setattr(BitwardenService, "sync", _noop_sync)+ monkeypatch.setattr(BitwardenService, "unlock", _fake_unlock)+ monkeypatch.setattr(BitwardenService, "logout", _noop_logout)+ monkeypatch.setattr(BitwardenService, "_apply_jitter", _noop_jitter)+++async def test_get_secret_value_by_item_id_coerces_null_fields_to_empty_string(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ item_payload = {+ "id": "11111111-1111-1111-1111-111111111111",+ "login": {+ "username": None,+ "password": None,+ "totp": None,+ },+ }++ async def fake_run_command(command: list[str], **_: object) -> RunCommandResult:+ return RunCommandResult(stdout=json.dumps(item_payload), stderr="", returncode=0)++ monkeypatch.setattr(BitwardenService, "run_command", fake_run_command)++ result = await BitwardenService.get_secret_value_from_url(+ client_id="client-id",+ client_secret="client-secret",+ master_password="master-password",+ bw_organization_id="org-id",+ bw_collection_ids=None,+ item_id="11111111-1111-1111-1111-111111111111",+ )++ assert result[BitwardenConstants.USERNAME] == ""+ assert result[BitwardenConstants.PASSWORD] == ""+ assert result[BitwardenConstants.TOTP] == ""+++async def test_get_secret_value_by_url_coerces_null_fields_to_empty_string(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ list_payload = [+ {+ "id": "22222222-2222-2222-2222-222222222222",+ "login": {+ "username": None,+ "password": "real-password",+ "totp": None,+ "uris": [{"uri": "https://example.com"}],+ },+ }+ ]++ async def fake_run_command(command: list[str], **_: object) -> RunCommandResult:+ return RunCommandResult(stdout=json.dumps(list_payload), stderr="", returncode=0)++ monkeypatch.setattr(BitwardenService, "run_command", fake_run_command)++ result = await BitwardenService.get_secret_value_from_url(+ client_id="client-id",+ client_secret="client-secret",+ master_password="master-password",+ bw_organization_id="org-id",+ bw_collection_ids=None,+ url="https://example.com/login",+ )++ assert result[BitwardenConstants.USERNAME] == ""+ assert result[BitwardenConstants.PASSWORD] == "real-password"+ assert result[BitwardenConstants.TOTP] == ""+++async def test_get_secret_value_by_item_id_preserves_real_values(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ item_payload = {+ "id": "33333333-3333-3333-3333-333333333333",+ "login": {+ "username": "[email protected]",+ "password": "hunter2",+ "totp": "",+ },+ }++ async def fake_run_command(command: list[str], **_: object) -> RunCommandResult:+ return RunCommandResult(stdout=json.dumps(item_payload), stderr="", returncode=0)++ monkeypatch.setattr(BitwardenService, "run_command", fake_run_command)++ result = await BitwardenService.get_secret_value_from_url(+ client_id="client-id",+ client_secret="client-secret",+ master_password="master-password",+ bw_organization_id="org-id",+ bw_collection_ids=None,+ item_id="33333333-3333-3333-3333-333333333333",+ )++ assert result[BitwardenConstants.USERNAME] == "[email protected]"+ assert result[BitwardenConstants.PASSWORD] == "hunter2"+ assert result[BitwardenConstants.TOTP] == ""diff --git a/tests/unit/test_classify_metadata_lifecycle.py b/tests/unit/test_classify_metadata_lifecycle.pyindex afca60156..8eae62453 100644--- a/tests/unit/test_classify_metadata_lifecycle.py+++ b/tests/unit/test_classify_metadata_lifecycle.py@@ -5,7 +5,7 @@ and the consume-at-entry invariant in ``ai_element_fallback``.from __future__ import annotationsfrom types import SimpleNamespace-from unittest.mock import AsyncMock+from unittest.mock import AsyncMock, MagicMockimport pytest@@ -173,3 +173,123 @@ async def test_ai_element_fallback_consumes_meta_when_ai_act_raises(await page_ai.ai_element_fallback(navigation_goal="real goal", max_steps=2)assert fallback_ctx.last_classify_meta is None+++async def test_ai_element_fallback_skips_validate_on_first_iteration(+ fallback_ctx: skyvern_context.SkyvernContext, monkeypatch: pytest.MonkeyPatch+) -> None:+ """Caller invokes element_fallback only when the cached path needs help, so+ the goal cannot already be achieved on entry — the iteration-0 validate is+ a guaranteed-False LLM call that should be skipped.+ """+ page = _make_page_stub()+ page_ai = _make_page_ai_stub(page)++ validate_mock = AsyncMock(return_value=True)+ act_mock = AsyncMock(return_value=None)+ monkeypatch.setattr(page_ai, "ai_validate", validate_mock)+ monkeypatch.setattr(page_ai, "ai_act", act_mock)+ monkeypatch.setattr(app.DATABASE.scripts, "create_fallback_episode", AsyncMock(return_value=None))++ await page_ai.ai_element_fallback(navigation_goal="real goal", max_steps=3)++ assert act_mock.await_count == 1+ assert validate_mock.await_count == 1+++async def test_ai_element_fallback_validates_after_act_on_subsequent_iterations(+ fallback_ctx: skyvern_context.SkyvernContext, monkeypatch: pytest.MonkeyPatch+) -> None:+ """After the first act, every iteration validates before acting again so a+ completed goal short-circuits the loop without a stale extra act. The+ relative call ordering (act → validate → act → validate, then break) is+ asserted against a parent mock manager to prevent regressions.+ """+ page = _make_page_stub()+ page_ai = _make_page_ai_stub(page)++ parent = MagicMock()+ validate_mock = AsyncMock(side_effect=[False, True])+ act_mock = AsyncMock(return_value=None)+ parent.attach_mock(validate_mock, "validate")+ parent.attach_mock(act_mock, "act")++ monkeypatch.setattr(page_ai, "ai_validate", validate_mock)+ monkeypatch.setattr(page_ai, "ai_act", act_mock)+ monkeypatch.setattr(app.DATABASE.scripts, "create_fallback_episode", AsyncMock(return_value=None))++ await page_ai.ai_element_fallback(navigation_goal="real goal", max_steps=5)++ call_names = [c[0] for c in parent.mock_calls]+ assert call_names == ["act", "validate", "act", "validate"]+++async def test_ai_element_fallback_validate_first_true_preserves_legacy_check(+ fallback_ctx: skyvern_context.SkyvernContext, monkeypatch: pytest.MonkeyPatch+) -> None:+ """Defensive callers that may invoke element_fallback on a possibly-complete+ page can opt back into the pre-act validate via ``validate_first=True``;+ when the page already satisfies the goal the loop must exit without acting.+ """+ page = _make_page_stub()+ page_ai = _make_page_ai_stub(page)++ validate_mock = AsyncMock(return_value=True)+ act_mock = AsyncMock(return_value=None)+ monkeypatch.setattr(page_ai, "ai_validate", validate_mock)+ monkeypatch.setattr(page_ai, "ai_act", act_mock)+ monkeypatch.setattr(app.DATABASE.scripts, "create_fallback_episode", AsyncMock(return_value=None))++ await page_ai.ai_element_fallback(navigation_goal="real goal", max_steps=3, validate_first=True)++ assert act_mock.await_count == 0+ assert validate_mock.await_count == 1+++async def test_ai_element_fallback_max_steps_one_can_succeed(+ fallback_ctx: skyvern_context.SkyvernContext, monkeypatch: pytest.MonkeyPatch+) -> None:+ """``max_steps=1`` must be able to succeed when the single act completes the+ goal — guaranteed by a final post-loop validate that runs when the loop+ exhausts without an early break.+ """+ page = _make_page_stub()+ page_ai = _make_page_ai_stub(page)++ validate_mock = AsyncMock(return_value=True)+ act_mock = AsyncMock(return_value=None)+ monkeypatch.setattr(page_ai, "ai_validate", validate_mock)+ monkeypatch.setattr(page_ai, "ai_act", act_mock)+ monkeypatch.setattr(app.DATABASE.scripts, "create_fallback_episode", AsyncMock(return_value=None))++ await page_ai.ai_element_fallback(navigation_goal="real goal", max_steps=1)++ assert act_mock.await_count == 1+ assert validate_mock.await_count == 1+++async def test_ai_element_fallback_raises_when_max_steps_exhausted_without_completion(+ fallback_ctx: skyvern_context.SkyvernContext, monkeypatch: pytest.MonkeyPatch+) -> None:+ """If every act runs and the final validate still says incomplete, the+ method raises rather than silently succeeding.+ """+ page = _make_page_stub()+ page_ai = _make_page_ai_stub(page)++ validate_mock = AsyncMock(return_value=False)+ act_mock = AsyncMock(return_value=None)+ monkeypatch.setattr(page_ai, "ai_validate", validate_mock)+ monkeypatch.setattr(page_ai, "ai_act", act_mock)+ monkeypatch.setattr(app.DATABASE.scripts, "create_fallback_episode", AsyncMock(return_value=None))++ with pytest.raises(Exception, match="did not complete within 2 steps"):+ await page_ai.ai_element_fallback(navigation_goal="real goal", max_steps=2)++ assert act_mock.await_count == 2+ assert validate_mock.await_count == 2