Skip to content

Automate browser based workflows with AI

fix(bitwarden): coerce null username/password/totp to empty string (SKY-8847) (#5936)

Aaron Perezcf3901a03f6c
diff --git a/skyvern/core/script_generations/real_skyvern_page_ai.py b/skyvern/core/script_generations/real_skyvern_page_ai.py
index 4a49cb94d..3abb13fb1 100644
--- a/skyvern/core/script_generations/real_skyvern_page_ai.py
+++ b/skyvern/core/script_generations/real_skyvern_page_ai.py
@@ -869,12 +869,19 @@ class RealSkyvernPageAi(SkyvernPageAi):
self,
navigation_goal: str,
max_steps: int = 5,
+ validate_first: bool = False,
) -> None:
"""Drive an AI agent from the current page state toward ``navigation_goal``.
Records one fallback episode per invocation regardless of exit path; pops
any preceding classify metadata at entry so subsequent fallbacks cannot
inherit it.
+
+ ``validate_first=True`` re-enables the legacy pre-act validate on
+ iteration 0 for defensive callers that may invoke this when the page
+ already satisfies the goal; the default (False) skips that validate
+ because the documented call site (classify-UNKNOWN / selector-failure
+ fallback) cannot be on the success state.
"""
context = skyvern_context.current()
if not context or not context.organization_id:
@@ -906,17 +913,19 @@ class RealSkyvernPageAi(SkyvernPageAi):
try:
for step_num in range(max_steps):
- is_complete = await self.ai_validate(
- prompt=f"Has the following goal been achieved? Goal: {navigation_goal}",
- )
- if is_complete:
- LOG.info(
- "page.element_fallback: goal achieved",
- step_num=step_num,
- navigation_goal=navigation_goal,
+ # Skip iteration-0 validate; opt in via validate_first for defensive callers.
+ if step_num > 0 or validate_first:
+ is_complete = await self.ai_validate(
+ prompt=f"Has the following goal been achieved? Goal: {navigation_goal}",
)
- completed = True
- break
+ if is_complete:
+ LOG.info(
+ "page.element_fallback: goal achieved",
+ step_num=step_num,
+ navigation_goal=navigation_goal,
+ )
+ completed = True
+ break
LOG.info(
"page.element_fallback: executing step",
@@ -941,6 +950,20 @@ class RealSkyvernPageAi(SkyvernPageAi):
"actions": actions_this_step[:10],
}
)
+ else:
+ # All max_steps acts ran without an early break — final validate
+ # so max_steps=1 can succeed when the lone act completed the goal.
+ if max_steps > 0:
+ is_complete = await self.ai_validate(
+ prompt=f"Has the following goal been achieved? Goal: {navigation_goal}",
+ )
+ if is_complete:
+ LOG.info(
+ "page.element_fallback: goal achieved",
+ step_num=max_steps,
+ navigation_goal=navigation_goal,
+ )
+ completed = True
except Exception as exc:
exception_summary = f"{type(exc).__name__}: {exc!s}"[:500]
captured_exception = exc
diff --git a/skyvern/core/script_generations/skyvern_page.py b/skyvern/core/script_generations/skyvern_page.py
index 11f9656aa..e7ffe6327 100644
--- a/skyvern/core/script_generations/skyvern_page.py
+++ b/skyvern/core/script_generations/skyvern_page.py
@@ -3230,6 +3230,7 @@ class SkyvernPage(Page):
self,
navigation_goal: str,
max_steps: int = 5,
+ validate_first: bool = False,
) -> None:
"""Activate the AI agent from the CURRENT page position to achieve a goal.
@@ -3240,6 +3241,11 @@ class SkyvernPage(Page):
Args:
navigation_goal: The goal for the AI agent to achieve from the current page.
max_steps: Maximum number of agent steps before giving up. Defaults to 5.
+ validate_first: When True, run an AI validation before the first
+ action to short-circuit if the page already satisfies the goal.
+ Defaults to False because the documented call site (else branch
+ of ``classify``) cannot be on the success state — pass True for
+ defensive calls that may invoke this on a possibly-complete page.
Raises:
Exception: If the element fallback fails or exceeds max_steps.
@@ -3260,6 +3266,7 @@ class SkyvernPage(Page):
return await self._ai.ai_element_fallback(
navigation_goal=navigation_goal,
max_steps=max_steps,
+ validate_first=validate_first,
)
async def prompt(
diff --git a/skyvern/core/script_generations/skyvern_page_ai.py b/skyvern/core/script_generations/skyvern_page_ai.py
index 701b824c4..c91ab9146 100644
--- a/skyvern/core/script_generations/skyvern_page_ai.py
+++ b/skyvern/core/script_generations/skyvern_page_ai.py
@@ -124,11 +124,16 @@ class SkyvernPageAi(Protocol):
self,
navigation_goal: str,
max_steps: int = 5,
+ validate_first: bool = False,
) -> None:
"""Activate the AI agent from the CURRENT page position to achieve a navigation goal.
This is a mid-block fallback that picks up from the current page state
instead of re-running the entire block from scratch.
+
+ ``validate_first=True`` re-enables the legacy pre-act validate on
+ iteration 0 for defensive callers that may invoke this when the page
+ already satisfies the goal.
"""
...
diff --git a/skyvern/forge/sdk/services/bitwarden.py b/skyvern/forge/sdk/services/bitwarden.py
index 4d86665ab..c9b9e79a0 100644
--- a/skyvern/forge/sdk/services/bitwarden.py
+++ b/skyvern/forge/sdk/services/bitwarden.py
@@ -353,11 +353,11 @@ class BitwardenService:
raise BitwardenGetItemError(f"Failed to parse item JSON for item ID: {item_id}")
login = item["login"]
- totp = BitwardenService.extract_totp_secret(login.get("totp", ""))
+ totp = BitwardenService.extract_totp_secret(login.get("totp") or "")
return {
- BitwardenConstants.USERNAME: login.get("username", ""),
- BitwardenConstants.PASSWORD: login.get("password", ""),
+ BitwardenConstants.USERNAME: login.get("username") or "",
+ BitwardenConstants.PASSWORD: login.get("password") or "",
BitwardenConstants.TOTP: totp,
}
elif not url:
@@ -415,16 +415,16 @@ class BitwardenService:
continue
login = item["login"]
- totp = BitwardenService.extract_totp_secret(login.get("totp", ""))
+ totp = BitwardenService.extract_totp_secret(login.get("totp") or "")
bitwarden_result.append(
BitwardenQueryResult(
credential={
- BitwardenConstants.USERNAME: login.get("username", ""),
- BitwardenConstants.PASSWORD: login.get("password", ""),
+ BitwardenConstants.USERNAME: login.get("username") or "",
+ BitwardenConstants.PASSWORD: login.get("password") or "",
BitwardenConstants.TOTP: totp,
},
- uris=[uri.get("uri") for uri in login.get("uris", []) if "uri" in uri],
+ uris=[uri.get("uri") for uri in login.get("uris") or [] if uri.get("uri")],
)
)
diff --git a/skyvern/library/skyvern_browser_page_ai.py b/skyvern/library/skyvern_browser_page_ai.py
index 8cee0be0e..1244a3b23 100644
--- a/skyvern/library/skyvern_browser_page_ai.py
+++ b/skyvern/library/skyvern_browser_page_ai.py
@@ -337,6 +337,7 @@ class SdkSkyvernPageAi(SkyvernPageAi):
self,
navigation_goal: str,
max_steps: int = 10,
+ validate_first: bool = False,
) -> None:
"""Activate the AI agent from the current page position.
diff --git a/tests/unit/test_bitwarden_null_credentials.py b/tests/unit/test_bitwarden_null_credentials.py
new file mode 100644
index 000000000..2ec50dd2b
--- /dev/null
+++ b/tests/unit/test_bitwarden_null_credentials.py
@@ -0,0 +1,146 @@
+"""Tests for SKY-8847: Bitwarden CLI path must coerce null username/password to empty strings.
+
+The Bitwarden CLI sometimes returns login items whose `username`, `password`, or `totp`
+fields are present but have a JSON `null` value. `dict.get(key, default)` returns the
+actual value (i.e. `None`) when the key exists, not the default — so callers must
+explicitly coerce `None` to `""` before handing the value to the form-filler.
+
+The server-side path (`_get_login_item_by_id_using_server`) already does this with
+`login["username"] or ""`; the CLI path used to use the bare `dict.get(..., "")`
+pattern, which silently propagated `None` into the secret store. That manifested as
+spurious "invalid credentials" errors during runs because the real value never got
+typed into the login form.
+"""
+
+import json
+
+import pytest
+
+from skyvern.forge.sdk.services.bitwarden import (
+ BitwardenConstants,
+ BitwardenService,
+ RunCommandResult,
+)
+
+
[email protected](autouse=True)
+def _stub_bitwarden_auth(monkeypatch: pytest.MonkeyPatch) -> None:
+ async def _noop_login(*args: object, **kwargs: object) -> None:
+ return None
+
+ async def _noop_sync() -> None:
+ return None
+
+ async def _fake_unlock(_master_password: str) -> str:
+ return "session-key"
+
+ async def _noop_logout() -> None:
+ return None
+
+ async def _noop_jitter() -> None:
+ return None
+
+ monkeypatch.setattr(BitwardenService, "login", _noop_login)
+ monkeypatch.setattr(BitwardenService, "sync", _noop_sync)
+ monkeypatch.setattr(BitwardenService, "unlock", _fake_unlock)
+ monkeypatch.setattr(BitwardenService, "logout", _noop_logout)
+ monkeypatch.setattr(BitwardenService, "_apply_jitter", _noop_jitter)
+
+
+async def test_get_secret_value_by_item_id_coerces_null_fields_to_empty_string(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ item_payload = {
+ "id": "11111111-1111-1111-1111-111111111111",
+ "login": {
+ "username": None,
+ "password": None,
+ "totp": None,
+ },
+ }
+
+ async def fake_run_command(command: list[str], **_: object) -> RunCommandResult:
+ return RunCommandResult(stdout=json.dumps(item_payload), stderr="", returncode=0)
+
+ monkeypatch.setattr(BitwardenService, "run_command", fake_run_command)
+
+ result = await BitwardenService.get_secret_value_from_url(
+ client_id="client-id",
+ client_secret="client-secret",
+ master_password="master-password",
+ bw_organization_id="org-id",
+ bw_collection_ids=None,
+ item_id="11111111-1111-1111-1111-111111111111",
+ )
+
+ assert result[BitwardenConstants.USERNAME] == ""
+ assert result[BitwardenConstants.PASSWORD] == ""
+ assert result[BitwardenConstants.TOTP] == ""
+
+
+async def test_get_secret_value_by_url_coerces_null_fields_to_empty_string(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ list_payload = [
+ {
+ "id": "22222222-2222-2222-2222-222222222222",
+ "login": {
+ "username": None,
+ "password": "real-password",
+ "totp": None,
+ "uris": [{"uri": "https://example.com"}],
+ },
+ }
+ ]
+
+ async def fake_run_command(command: list[str], **_: object) -> RunCommandResult:
+ return RunCommandResult(stdout=json.dumps(list_payload), stderr="", returncode=0)
+
+ monkeypatch.setattr(BitwardenService, "run_command", fake_run_command)
+
+ result = await BitwardenService.get_secret_value_from_url(
+ client_id="client-id",
+ client_secret="client-secret",
+ master_password="master-password",
+ bw_organization_id="org-id",
+ bw_collection_ids=None,
+ url="https://example.com/login",
+ )
+
+ assert result[BitwardenConstants.USERNAME] == ""
+ assert result[BitwardenConstants.PASSWORD] == "real-password"
+ assert result[BitwardenConstants.TOTP] == ""
+
+
+async def test_get_secret_value_by_item_id_preserves_real_values(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ item_payload = {
+ "id": "33333333-3333-3333-3333-333333333333",
+ "login": {
+ "username": "[email protected]",
+ "password": "hunter2",
+ "totp": "",
+ },
+ }
+
+ async def fake_run_command(command: list[str], **_: object) -> RunCommandResult:
+ return RunCommandResult(stdout=json.dumps(item_payload), stderr="", returncode=0)
+
+ monkeypatch.setattr(BitwardenService, "run_command", fake_run_command)
+
+ result = await BitwardenService.get_secret_value_from_url(
+ client_id="client-id",
+ client_secret="client-secret",
+ master_password="master-password",
+ bw_organization_id="org-id",
+ bw_collection_ids=None,
+ item_id="33333333-3333-3333-3333-333333333333",
+ )
+
+ assert result[BitwardenConstants.USERNAME] == "[email protected]"
+ assert result[BitwardenConstants.PASSWORD] == "hunter2"
+ assert result[BitwardenConstants.TOTP] == ""
diff --git a/tests/unit/test_classify_metadata_lifecycle.py b/tests/unit/test_classify_metadata_lifecycle.py
index afca60156..8eae62453 100644
--- a/tests/unit/test_classify_metadata_lifecycle.py
+++ b/tests/unit/test_classify_metadata_lifecycle.py
@@ -5,7 +5,7 @@ and the consume-at-entry invariant in ``ai_element_fallback``.
from __future__ import annotations
from types import SimpleNamespace
-from unittest.mock import AsyncMock
+from unittest.mock import AsyncMock, MagicMock
import pytest
@@ -173,3 +173,123 @@ async def test_ai_element_fallback_consumes_meta_when_ai_act_raises(
await page_ai.ai_element_fallback(navigation_goal="real goal", max_steps=2)
assert fallback_ctx.last_classify_meta is None
+
+
+async def test_ai_element_fallback_skips_validate_on_first_iteration(
+ fallback_ctx: skyvern_context.SkyvernContext, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ """Caller invokes element_fallback only when the cached path needs help, so
+ the goal cannot already be achieved on entry — the iteration-0 validate is
+ a guaranteed-False LLM call that should be skipped.
+ """
+ page = _make_page_stub()
+ page_ai = _make_page_ai_stub(page)
+
+ validate_mock = AsyncMock(return_value=True)
+ act_mock = AsyncMock(return_value=None)
+ monkeypatch.setattr(page_ai, "ai_validate", validate_mock)
+ monkeypatch.setattr(page_ai, "ai_act", act_mock)
+ monkeypatch.setattr(app.DATABASE.scripts, "create_fallback_episode", AsyncMock(return_value=None))
+
+ await page_ai.ai_element_fallback(navigation_goal="real goal", max_steps=3)
+
+ assert act_mock.await_count == 1
+ assert validate_mock.await_count == 1
+
+
+async def test_ai_element_fallback_validates_after_act_on_subsequent_iterations(
+ fallback_ctx: skyvern_context.SkyvernContext, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ """After the first act, every iteration validates before acting again so a
+ completed goal short-circuits the loop without a stale extra act. The
+ relative call ordering (act → validate → act → validate, then break) is
+ asserted against a parent mock manager to prevent regressions.
+ """
+ page = _make_page_stub()
+ page_ai = _make_page_ai_stub(page)
+
+ parent = MagicMock()
+ validate_mock = AsyncMock(side_effect=[False, True])
+ act_mock = AsyncMock(return_value=None)
+ parent.attach_mock(validate_mock, "validate")
+ parent.attach_mock(act_mock, "act")
+
+ monkeypatch.setattr(page_ai, "ai_validate", validate_mock)
+ monkeypatch.setattr(page_ai, "ai_act", act_mock)
+ monkeypatch.setattr(app.DATABASE.scripts, "create_fallback_episode", AsyncMock(return_value=None))
+
+ await page_ai.ai_element_fallback(navigation_goal="real goal", max_steps=5)
+
+ call_names = [c[0] for c in parent.mock_calls]
+ assert call_names == ["act", "validate", "act", "validate"]
+
+
+async def test_ai_element_fallback_validate_first_true_preserves_legacy_check(
+ fallback_ctx: skyvern_context.SkyvernContext, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ """Defensive callers that may invoke element_fallback on a possibly-complete
+ page can opt back into the pre-act validate via ``validate_first=True``;
+ when the page already satisfies the goal the loop must exit without acting.
+ """
+ page = _make_page_stub()
+ page_ai = _make_page_ai_stub(page)
+
+ validate_mock = AsyncMock(return_value=True)
+ act_mock = AsyncMock(return_value=None)
+ monkeypatch.setattr(page_ai, "ai_validate", validate_mock)
+ monkeypatch.setattr(page_ai, "ai_act", act_mock)
+ monkeypatch.setattr(app.DATABASE.scripts, "create_fallback_episode", AsyncMock(return_value=None))
+
+ await page_ai.ai_element_fallback(navigation_goal="real goal", max_steps=3, validate_first=True)
+
+ assert act_mock.await_count == 0
+ assert validate_mock.await_count == 1
+
+
+async def test_ai_element_fallback_max_steps_one_can_succeed(
+ fallback_ctx: skyvern_context.SkyvernContext, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ """``max_steps=1`` must be able to succeed when the single act completes the
+ goal — guaranteed by a final post-loop validate that runs when the loop
+ exhausts without an early break.
+ """
+ page = _make_page_stub()
+ page_ai = _make_page_ai_stub(page)
+
+ validate_mock = AsyncMock(return_value=True)
+ act_mock = AsyncMock(return_value=None)
+ monkeypatch.setattr(page_ai, "ai_validate", validate_mock)
+ monkeypatch.setattr(page_ai, "ai_act", act_mock)
+ monkeypatch.setattr(app.DATABASE.scripts, "create_fallback_episode", AsyncMock(return_value=None))
+
+ await page_ai.ai_element_fallback(navigation_goal="real goal", max_steps=1)
+
+ assert act_mock.await_count == 1
+ assert validate_mock.await_count == 1
+
+
+async def test_ai_element_fallback_raises_when_max_steps_exhausted_without_completion(
+ fallback_ctx: skyvern_context.SkyvernContext, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ """If every act runs and the final validate still says incomplete, the
+ method raises rather than silently succeeding.
+ """
+ page = _make_page_stub()
+ page_ai = _make_page_ai_stub(page)
+
+ validate_mock = AsyncMock(return_value=False)
+ act_mock = AsyncMock(return_value=None)
+ monkeypatch.setattr(page_ai, "ai_validate", validate_mock)
+ monkeypatch.setattr(page_ai, "ai_act", act_mock)
+ monkeypatch.setattr(app.DATABASE.scripts, "create_fallback_episode", AsyncMock(return_value=None))
+
+ with pytest.raises(Exception, match="did not complete within 2 steps"):
+ await page_ai.ai_element_fallback(navigation_goal="real goal", max_steps=2)
+
+ assert act_mock.await_count == 2
+ assert validate_mock.await_count == 2