Skip to content

Automate browser based workflows with AI

fix: log unhandled file-download failures at warning not error (#8249)

Aaron Perez146d67fb892d
diff --git a/skyvern/config.py b/skyvern/config.py
index 15a1bf6aa..ab7312ce9 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -411,7 +411,7 @@ class Settings(BaseSettings):
# Bitwarden Configs #
#####################
BITWARDEN_TIMEOUT_SECONDS: int = 60
- BITWARDEN_MAX_RETRIES: int = 2
+ BITWARDEN_MAX_RETRIES: int = 3
BITWARDEN_MAX_JITTER_SECONDS: float = 2.0
# task generation settings
diff --git a/skyvern/exceptions.py b/skyvern/exceptions.py
index 5c7db00d1..3239fbf28 100644
--- a/skyvern/exceptions.py
+++ b/skyvern/exceptions.py
@@ -1599,11 +1599,8 @@ class MissingRoutedVncAddressError(SkyvernException):
class BrowserSessionClosed(SkyvernHTTPException):
def __init__(self, browser_session_id: str, *, reason: str | None = None) -> None:
- message = f"Browser session {browser_session_id} is closed."
- if reason:
- message = f"Browser session {browser_session_id} {reason}. Create a new browser session and retry."
super().__init__(
- message,
+ f"Browser session {browser_session_id} {reason or 'is closed'}. Create a new browser session and retry.",
status_code=HTTPStatus.GONE,
)
diff --git a/skyvern/forge/sdk/api/aws.py b/skyvern/forge/sdk/api/aws.py
index afb7719c2..88cb03789 100644
--- a/skyvern/forge/sdk/api/aws.py
+++ b/skyvern/forge/sdk/api/aws.py
@@ -1,8 +1,10 @@
from __future__ import annotations
+import asyncio
import io
import os
import time
+from collections.abc import Awaitable, Callable
from enum import StrEnum
from mimetypes import add_type, guess_type
from typing import IO, TYPE_CHECKING, Any
@@ -163,6 +165,30 @@ class AsyncAWSClient:
continue
raise
+ async def _detach_on_cancel(self, op_name: str, operation: Callable[[], Awaitable[Any]], **log_kwargs: Any) -> Any:
+ """Run a multipart transfer so the caller's cancellation cannot strand aioboto3's worker tasks.
+
+ aioboto3's upload_fileobj (still true in 15.5.0) cancels its uploader tasks and aborts the multipart
+ upload only on its own exit path; a CancelledError delivered at its internal asyncio.wait skips both.
+ """
+ # The detached transfer has no deadline of its own: botocore's per-request timeouts bound a stall,
+ # and a process shutdown cancels it exactly as before.
+ task = asyncio.ensure_future(operation())
+ try:
+ return await asyncio.shield(task)
+ except asyncio.CancelledError:
+ task.add_done_callback(lambda done: self._log_detached_transfer(op_name, done, **log_kwargs))
+ raise
+
+ @staticmethod
+ def _log_detached_transfer(op_name: str, task: asyncio.Task[Any], **log_kwargs: Any) -> None:
+ if task.cancelled():
+ LOG.warning(f"S3 {op_name} cancelled after its caller stopped waiting", **log_kwargs)
+ elif (exc := task.exception()) is not None:
+ LOG.warning(f"S3 {op_name} failed after its caller stopped waiting", error=str(exc), **log_kwargs)
+ else:
+ LOG.info(f"S3 {op_name} completed after its caller stopped waiting", **log_kwargs)
+
def _ecs_client(self) -> ECSClient:
return self._get_session(AWSClientType.ECS).client(
AWSClientType.ECS, region_name=self.region_name, endpoint_url=self._endpoint_url
@@ -294,7 +320,11 @@ class AsyncAWSClient:
return None
try:
- return await self._s3_with_retry("stream upload", _op, before_retry=_rewind_stream, uri=uri)
+ return await self._detach_on_cancel(
+ "stream upload",
+ lambda: self._s3_with_retry("stream upload", _op, before_retry=_rewind_stream, uri=uri),
+ uri=uri,
+ )
except Exception:
LOG.exception("S3 upload stream failed.", uri=uri)
return None
@@ -332,7 +362,7 @@ class AsyncAWSClient:
)
try:
- await self._s3_with_retry("upload", _op, uri=uri)
+ await self._detach_on_cancel("upload", lambda: self._s3_with_retry("upload", _op, uri=uri), uri=uri)
except Exception as e:
LOG.exception("S3 upload failed.", uri=uri)
if raise_exception:
diff --git a/skyvern/forge/sdk/api/files.py b/skyvern/forge/sdk/api/files.py
index ec1687293..6b3c7f5f2 100644
--- a/skyvern/forge/sdk/api/files.py
+++ b/skyvern/forge/sdk/api/files.py
@@ -577,7 +577,9 @@ async def download_file(
LOG.warning("Failed to download file, blocked host", exc_info=True)
raise
except Exception:
- LOG.exception("Failed to download file")
+ # Dominated by the requested host erroring, which the run record already carries; a genuine
+ # defect in this path therefore surfaces at warning, not error.
+ LOG.warning("Failed to download file", exc_info=True)
raise
diff --git a/skyvern/forge/sdk/services/bitwarden.py b/skyvern/forge/sdk/services/bitwarden.py
index 0b98a022d..85a030097 100644
--- a/skyvern/forge/sdk/services/bitwarden.py
+++ b/skyvern/forge/sdk/services/bitwarden.py
@@ -3,6 +3,7 @@ import json
import os
import random
import re
+import time
from enum import IntEnum, StrEnum
from typing import Any, Tuple
@@ -39,6 +40,13 @@ from skyvern.utils.strings import is_uuid
LOG = structlog.get_logger()
BITWARDEN_SERVER_BASE_URL = f"{settings.BITWARDEN_SERVER}:{settings.BITWARDEN_SERVER_PORT or 8002}"
+_RETRY_BACKOFF_BASE_SECONDS = 5.0
+_RETRY_BACKOFF_CAP_SECONDS = 60.0
+
+
+def _retry_backoff_seconds(attempt: int) -> float:
+ # Full jitter: a batch of runs that failed together retries spread out instead of re-aligned.
+ return random.uniform(0, min(_RETRY_BACKOFF_CAP_SECONDS, _RETRY_BACKOFF_BASE_SECONDS * (2**attempt)))
class BitwardenItemType(IntEnum):
@@ -370,6 +378,7 @@ class BitwardenService:
env.update(additional_env) # Update with any additional environment variables
shell_subprocess = None
+ started = time.monotonic()
try:
async with asyncio.timeout(timeout):
shell_subprocess = await asyncio.create_subprocess_exec(
@@ -388,10 +397,20 @@ class BitwardenService:
LOG.error(
"Bitwarden command timed out",
timeout_seconds=timeout,
+ elapsed_seconds=round(time.monotonic() - started, 2),
command=command[0:2],
exc_info=True,
)
raise
+ except asyncio.CancelledError:
+ # An enclosing deadline fired before this command's own budget; name the step so the
+ # attempt-level timeout is attributable to it.
+ LOG.warning(
+ "Bitwarden command cancelled by an enclosing deadline",
+ elapsed_seconds=round(time.monotonic() - started, 2),
+ command=command[0:2],
+ )
+ raise
finally:
if shell_subprocess and shell_subprocess.returncode is None:
LOG.info("Killing orphaned Bitwarden subprocess", pid=shell_subprocess.pid)
@@ -533,11 +552,11 @@ class BitwardenService:
raise BitwardenGetItemError(f"Invalid item ID: {item_id}. Check if the item ID is correct")
await BitwardenService._apply_jitter()
- for i in range(max_retries):
- # FIXME: just simply double the timeout for the second try. maybe a better backoff policy when needed
- timeout = (i + 1) * timeout
+ for attempt in range(max_retries):
try:
- async with asyncio.timeout(timeout):
+ # Every CLI command gets `timeout`; the attempt as a whole gets twice that, so one slow step
+ # cannot consume the others' budget and a dead vault still fails in bounded time.
+ async with asyncio.timeout(2 * timeout):
async with BitwardenService._cli_session_lock:
return await BitwardenService._get_secret_value_from_url(
client_id=client_id,
@@ -554,12 +573,15 @@ class BitwardenService:
except BitwardenAccessDeniedError as e:
raise e
except Exception as e:
- LOG.info("Failed to get secret value from Bitwarden", tried_times=i + 1, exc_info=True)
+ LOG.info("Failed to get secret value from Bitwarden", tried_times=attempt + 1, exc_info=True)
fail_reasons.append(f"{type(e).__name__}: {str(e)}")
- else:
- raise BitwardenListItemsError(
- f"Bitwarden CLI failed after all retry attempts. Fail reasons: {fail_reasons}"
- )
+ if attempt + 1 < max_retries:
+ delay = _retry_backoff_seconds(attempt)
+ LOG.info(
+ "Backing off before retrying Bitwarden", delay_seconds=round(delay, 2), attempt=attempt + 1
+ )
+ await asyncio.sleep(delay)
+ raise BitwardenListItemsError(f"Bitwarden CLI failed after all retry attempts. Fail reasons: {fail_reasons}")
@staticmethod
def normalize_totp_config(totp_value: str) -> str:
@@ -602,13 +624,15 @@ class BitwardenService:
Get the secret value from the Bitwarden CLI.
"""
try:
- await BitwardenService.login(client_id, client_secret, email=email, master_password=master_password)
- await BitwardenService.sync()
- session_key = await BitwardenService.unlock(master_password)
+ await BitwardenService.login(
+ client_id, client_secret, email=email, master_password=master_password, timeout=timeout
+ )
+ await BitwardenService.sync(timeout=timeout)
+ session_key = await BitwardenService.unlock(master_password, timeout=timeout)
if item_id: # if item_id provided, get single item by item id
command = ["bw", "get", "item", item_id, "--session", session_key]
- item_result = await BitwardenService.run_command(command)
+ item_result = await BitwardenService.run_command(command, timeout=timeout)
if item_result.stderr:
raise BitwardenGetItemError(
f"Failed to get the bitwarden item {item_id}. Error: {item_result.stderr}"
@@ -762,6 +786,7 @@ class BitwardenService:
remaining_retries -= 1
LOG.info("Retrying to get sensitive information from Bitwarden", remaining_retries=remaining_retries)
+ await asyncio.sleep(_retry_backoff_seconds(len(fail_reasons)))
return await BitwardenService.get_sensitive_information_from_identity(
client_id=client_id,
client_secret=client_secret,
@@ -864,6 +889,7 @@ class BitwardenService:
client_secret: str | None,
email: str | None = None,
master_password: str | None = None,
+ timeout: int = settings.BITWARDEN_TIMEOUT_SECONDS,
) -> None:
"""
Log in to the Bitwarden CLI.
@@ -883,7 +909,7 @@ class BitwardenService:
login_command = ["bw", "login", bw_email, "--passwordenv", "BW_PASSWORD"]
else:
login_command = ["bw", "login", "--apikey"]
- login_result = await BitwardenService.run_command(login_command, env)
+ login_result = await BitwardenService.run_command(login_command, env, timeout=timeout)
# Validate the login result
if login_result.stdout and "You are logged in!" not in login_result.stdout:
@@ -895,7 +921,7 @@ class BitwardenService:
LOG.info("Bitwarden login successful")
@staticmethod
- async def unlock(master_password: str) -> str:
+ async def unlock(master_password: str, timeout: int = settings.BITWARDEN_TIMEOUT_SECONDS) -> str:
"""
Unlock the Bitwarden CLI.
"""
@@ -903,7 +929,7 @@ class BitwardenService:
"BW_PASSWORD": master_password,
}
unlock_command = ["bw", "unlock", "--passwordenv", "BW_PASSWORD"]
- unlock_result = await BitwardenService.run_command(unlock_command, env)
+ unlock_result = await BitwardenService.run_command(unlock_command, env, timeout=timeout)
# Validate the unlock result
if unlock_result.stdout and "Your vault is now unlocked!" not in unlock_result.stdout:
@@ -923,24 +949,24 @@ class BitwardenService:
return session_key
@staticmethod
- async def sync() -> None:
+ async def sync(timeout: int = settings.BITWARDEN_TIMEOUT_SECONDS) -> None:
"""
Sync the Bitwarden CLI.
"""
sync_command = ["bw", "sync"]
LOG.info("Bitwarden CLI sync started")
- sync_result = await BitwardenService.run_command(sync_command)
+ sync_result = await BitwardenService.run_command(sync_command, timeout=timeout)
LOG.info("Bitwarden CLI sync completed")
if sync_result.stderr:
raise BitwardenSyncError(sync_result.stderr)
@staticmethod
- async def logout() -> None:
+ async def logout(timeout: int = settings.BITWARDEN_TIMEOUT_SECONDS) -> None:
"""
Log out of the Bitwarden CLI.
"""
logout_command = ["bw", "logout"]
- logout_result = await BitwardenService.run_command(logout_command)
+ logout_result = await BitwardenService.run_command(logout_command, timeout=timeout)
if logout_result.stderr and "You are not logged in." not in logout_result.stderr:
raise BitwardenLogoutError(logout_result.stderr)
@@ -1066,6 +1092,7 @@ class BitwardenService:
remaining_retries -= 1
LOG.info("Retrying to get credit card data from Bitwarden", remaining_retries=remaining_retries)
+ await asyncio.sleep(_retry_backoff_seconds(len(fail_reasons)))
return await BitwardenService.get_credit_card_data(
client_id=client_id,
client_secret=client_secret,
diff --git a/tests/unit/test_aws_client_ttl.py b/tests/unit/test_aws_client_ttl.py
index eb03dcf5e..119998941 100644
--- a/tests/unit/test_aws_client_ttl.py
+++ b/tests/unit/test_aws_client_ttl.py
@@ -1,4 +1,8 @@
+import asyncio
+import contextlib
import time
+import types
+from pathlib import Path
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
@@ -339,3 +343,58 @@ async def test_get_object_info_retries_on_expired_token():
mock_refresh.assert_called_once()
assert mock_head.call_count == 2
assert result == {"ContentLength": 42}
+
+
+async def test_upload_file_from_path_cancelled_mid_multipart_aborts_upload_and_strands_no_uploaders(
+ tmp_path: Path,
+) -> None:
+ from aioboto3.s3 import inject as s3_inject
+
+ big_file = tmp_path / "big.bin"
+ with big_file.open("wb") as f:
+ f.truncate(9 * 1024 * 1024) # past aioboto3's 8 MiB multipart threshold; sparse, so instant
+
+ parts_started = asyncio.Event()
+ release_parts = asyncio.Event()
+ aborted_upload_ids: list[str] = []
+
+ class FakeS3:
+ async def create_multipart_upload(self, **kwargs: object) -> dict[str, str]:
+ return {"UploadId": "upload-1"}
+
+ async def upload_part(self, **kwargs: object) -> None:
+ parts_started.set()
+ await release_parts.wait()
+ raise RuntimeError("part upload failed")
+
+ async def abort_multipart_upload(self, **kwargs: object) -> None:
+ aborted_upload_ids.append(str(kwargs["UploadId"]))
+
+ async def complete_multipart_upload(self, **kwargs: object) -> None:
+ raise AssertionError("a failed multipart upload must not be completed")
+
+ fake_s3 = FakeS3()
+ fake_s3.upload_file = types.MethodType(s3_inject.upload_file, fake_s3) # type: ignore[attr-defined]
+
+ @contextlib.asynccontextmanager
+ async def fake_s3_client(): # type: ignore[no-untyped-def]
+ yield fake_s3
+
+ client = aws.AsyncAWSClient()
+ with patch.object(client, "_s3_client", fake_s3_client):
+ caller = asyncio.ensure_future(
+ client.upload_file_from_path("s3://bucket/key.bin", str(big_file), raise_exception=True)
+ )
+ await asyncio.wait_for(parts_started.wait(), timeout=5)
+ caller.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await caller
+
+ release_parts.set()
+ async with asyncio.timeout(5):
+ while not aborted_upload_ids or len(asyncio.all_tasks()) > 1:
+ await asyncio.sleep(0.01)
+
+ assert aborted_upload_ids == ["upload-1"]
+ assert not [t for t in asyncio.all_tasks() if "upload_fileobj.<locals>.uploader" in repr(t)]
diff --git a/tests/unit/test_bitwarden_login.py b/tests/unit/test_bitwarden_login.py
index b190aafd0..9a1a025ad 100644
--- a/tests/unit/test_bitwarden_login.py
+++ b/tests/unit/test_bitwarden_login.py
@@ -1,4 +1,6 @@
-from unittest.mock import AsyncMock
+import asyncio
+import json
+from unittest.mock import AsyncMock, MagicMock
import pytest
@@ -113,3 +115,240 @@ async def test_get_credential_item_by_id_reads_login_without_totp(monkeypatch: p
fetched_item = await BitwardenService._get_credential_item_by_id_using_server("item-1")
assert fetched_item.credential.totp == ""
+
+
+class _HangingProcess:
+ """A subprocess whose communicate() never returns, so only the deadline can end the call."""
+
+ def __init__(self) -> None:
+ self.returncode: int | None = None
+ self.pid = 4242
+ self.killed = False
+
+ async def communicate(self) -> tuple[bytes, bytes]:
+ await asyncio.Event().wait()
+ raise AssertionError("unreachable")
+
+ def kill(self) -> None:
+ self.killed = True
+ self.returncode = -9
+
+ async def wait(self) -> int | None:
+ return self.returncode
+
+
+async def test_run_command_names_step_and_elapsed_when_its_own_budget_expires(monkeypatch: pytest.MonkeyPatch) -> None:
+ process = _HangingProcess()
+
+ async def fake_exec(*args: object, **kwargs: object) -> _HangingProcess:
+ return process
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", fake_exec)
+ log = MagicMock()
+ monkeypatch.setattr(bitwarden_module, "LOG", log)
+
+ with pytest.raises(asyncio.TimeoutError):
+ await BitwardenService.run_command(["bw", "list", "items", "--session", "s"], timeout=0.05)
+
+ assert process.killed is True
+ kwargs = log.error.call_args.kwargs
+ assert log.error.call_args.args[0] == "Bitwarden command timed out"
+ assert kwargs["command"] == ["bw", "list"]
+ assert kwargs["elapsed_seconds"] >= 0.05
+
+
+async def test_run_command_names_step_when_an_enclosing_deadline_cancels_it(monkeypatch: pytest.MonkeyPatch) -> None:
+ process = _HangingProcess()
+
+ async def fake_exec(*args: object, **kwargs: object) -> _HangingProcess:
+ return process
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", fake_exec)
+ log = MagicMock()
+ monkeypatch.setattr(bitwarden_module, "LOG", log)
+
+ with pytest.raises(TimeoutError):
+ async with asyncio.timeout(0.05):
+ await BitwardenService.run_command(["bw", "unlock", "--passwordenv", "BW_PASSWORD"], timeout=10)
+
+ assert process.killed is True
+ assert log.warning.call_args.args[0] == "Bitwarden command cancelled by an enclosing deadline"
+ assert log.warning.call_args.kwargs["command"] == ["bw", "unlock"]
+
+
+def test_retry_backoff_is_full_jitter_with_doubling_cap(monkeypatch: pytest.MonkeyPatch) -> None:
+ seen: list[tuple[float, float]] = []
+
+ def fake_uniform(low: float, high: float) -> float:
+ seen.append((low, high))
+ return high
+
+ monkeypatch.setattr(bitwarden_module.random, "uniform", fake_uniform)
+
+ delays = [bitwarden_module._retry_backoff_seconds(attempt) for attempt in range(5)]
+
+ assert seen == [(0, 5.0), (0, 10.0), (0, 20.0), (0, 40.0), (0, 60.0)]
+ assert delays == [5.0, 10.0, 20.0, 40.0, 60.0]
+
+
+async def test_every_cli_step_receives_the_attempt_budget(monkeypatch: pytest.MonkeyPatch) -> None:
+ budgets: dict[str, int] = {}
+
+ async def fake_run_command(command: list[str], additional_env=None, timeout: int = 60) -> RunCommandResult:
+ budgets[" ".join(command[0:2])] = timeout
+ if command[1] == "login":
+ return RunCommandResult(stdout="You are logged in!", stderr="", returncode=0)
+ if command[1] == "unlock":
+ return RunCommandResult(
+ stdout='Your vault is now unlocked!\n$ export BW_SESSION="abc"', stderr="", returncode=0
+ )
+ if command[1] == "get":
+ item = {"id": "item-1", "login": {"username": "u", "password": "p", "totp": "", "uris": []}}
+ return RunCommandResult(stdout=json.dumps(item), stderr="", returncode=0)
+ return RunCommandResult(stdout="", stderr="", returncode=0)
+
+ monkeypatch.setattr(BitwardenService, "run_command", fake_run_command)
+
+ await BitwardenService._get_secret_value_from_url(
+ client_id="cid",
+ client_secret="secret",
+ master_password="mp",
+ bw_organization_id=None,
+ bw_collection_ids=["coll-1"],
+ url=None,
+ collection_id="coll-1",
+ item_id="0f3a9a8e-1c6d-4b7e-9f1d-2a5b6c7d8e9f",
+ timeout=37,
+ )
+
+ assert budgets == {
+ "bw login": 37,
+ "bw sync": 37,
+ "bw unlock": 37,
+ "bw get": 37,
+ "bw logout": bitwarden_module.settings.BITWARDEN_TIMEOUT_SECONDS,
+ }
+
+
+def _secret_fetch_kwargs() -> dict:
+ return {
+ "client_id": "cid",
+ "client_secret": "secret",
+ "master_password": "mp",
+ "url": "https://example.com/login",
+ "bw_organization_id": None,
+ "bw_collection_ids": ["coll-1"],
+ "collection_id": "coll-1",
+ }
+
+
+async def test_secret_fetch_retries_after_a_jittered_backoff_with_a_constant_step_budget(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setattr(bitwarden_module.settings, "BITWARDEN_MAX_JITTER_SECONDS", 0)
+ attempt_budgets: list[int] = []
+ outcomes: list = [asyncio.TimeoutError(), {"username": "u", "password": "p", "totp": ""}]
+
+ async def fake_inner(**kwargs) -> dict[str, str]:
+ attempt_budgets.append(kwargs["timeout"])
+ outcome = outcomes.pop(0)
+ if isinstance(outcome, BaseException):
+ raise outcome
+ return outcome
+
+ slept: list[float] = []
+
+ async def fake_sleep(delay: float) -> None:
+ slept.append(delay)
+
+ monkeypatch.setattr(BitwardenService, "_get_secret_value_from_url", fake_inner)
+ monkeypatch.setattr(bitwarden_module, "_retry_backoff_seconds", lambda attempt: 7.5)
+ monkeypatch.setattr(bitwarden_module.asyncio, "sleep", fake_sleep)
+
+ result = await BitwardenService.get_secret_value_from_url(**_secret_fetch_kwargs(), max_retries=3, timeout=60)
+
+ assert result == {"username": "u", "password": "p", "totp": ""}
+ assert attempt_budgets == [60, 60]
+ assert slept == [7.5]
+
+
+async def test_secret_fetch_gives_up_after_max_retries_with_every_reason(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(bitwarden_module.settings, "BITWARDEN_MAX_JITTER_SECONDS", 0)
+
+ async def always_times_out(**kwargs) -> dict[str, str]:
+ raise asyncio.TimeoutError()
+
+ slept: list[float] = []
+
+ async def fake_sleep(delay: float) -> None:
+ slept.append(delay)
+
+ monkeypatch.setattr(BitwardenService, "_get_secret_value_from_url", always_times_out)
+ monkeypatch.setattr(bitwarden_module, "_retry_backoff_seconds", lambda attempt: float(attempt))
+ monkeypatch.setattr(bitwarden_module.asyncio, "sleep", fake_sleep)
+
+ with pytest.raises(bitwarden_module.BitwardenListItemsError) as excinfo:
+ await BitwardenService.get_secret_value_from_url(**_secret_fetch_kwargs(), max_retries=3, timeout=60)
+
+ assert str(excinfo.value).count("TimeoutError") == 3
+ assert slept == [0.0, 1.0]
+
+
+async def test_secret_fetch_does_not_retry_access_denied(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(bitwarden_module.settings, "BITWARDEN_MAX_JITTER_SECONDS", 0)
+ calls = 0
+
+ async def denied(**kwargs) -> dict[str, str]:
+ nonlocal calls
+ calls += 1
+ raise bitwarden_module.BitwardenAccessDeniedError()
+
+ monkeypatch.setattr(BitwardenService, "_get_secret_value_from_url", denied)
+
+ with pytest.raises(bitwarden_module.BitwardenAccessDeniedError):
+ await BitwardenService.get_secret_value_from_url(**_secret_fetch_kwargs(), max_retries=3, timeout=60)
+
+ assert calls == 1
+
+
+async def test_identity_fetch_backs_off_before_its_recursive_retry(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(bitwarden_module.settings, "BITWARDEN_MAX_JITTER_SECONDS", 0)
+ outcomes: list = [asyncio.TimeoutError(), {"first_name": "A"}]
+
+ async def fake_inner(**kwargs) -> dict[str, str]:
+ outcome = outcomes.pop(0)
+ if isinstance(outcome, BaseException):
+ raise outcome
+ return outcome
+
+ slept: list[float] = []
+
+ async def fake_sleep(delay: float) -> None:
+ slept.append(delay)
+
+ monkeypatch.setattr(BitwardenService, "_get_sensitive_information_from_identity", fake_inner)
+ monkeypatch.setattr(bitwarden_module, "_retry_backoff_seconds", lambda attempt: 3.0 + attempt)
+ monkeypatch.setattr(bitwarden_module.asyncio, "sleep", fake_sleep)
+
+ result = await BitwardenService.get_sensitive_information_from_identity(
+ client_id="cid",
+ client_secret="secret",
+ master_password="mp",
+ bw_organization_id=None,
+ bw_collection_ids=["coll-1"],
+ collection_id="coll-1",
+ identity_key="identity",
+ identity_fields=["first_name"],
+ remaining_retries=2,
+ timeout=60,
+ )
+
+ assert result == {"first_name": "A"}
+ assert slept == [3.0]
diff --git a/tests/unit/test_bitwarden_null_credentials.py b/tests/unit/test_bitwarden_null_credentials.py
index e8b603001..3dd980dd4 100644
--- a/tests/unit/test_bitwarden_null_credentials.py
+++ b/tests/unit/test_bitwarden_null_credentials.py
@@ -29,10 +29,10 @@ def _stub_bitwarden_auth(monkeypatch: pytest.MonkeyPatch) -> None:
async def _noop_login(*args: object, **kwargs: object) -> None:
return None
- async def _noop_sync() -> None:
+ async def _noop_sync(**_: object) -> None:
return None
- async def _fake_unlock(_master_password: str) -> str:
+ async def _fake_unlock(_master_password: str, **_: object) -> str:
return "session-key"
async def _noop_logout() -> None:
diff --git a/tests/unit/test_exception_messages.py b/tests/unit/test_exception_messages.py
index ba219f36d..b4a211af1 100644
--- a/tests/unit/test_exception_messages.py
+++ b/tests/unit/test_exception_messages.py
@@ -197,6 +197,12 @@ def test_browser_session_timeout_exceptions_distinguish_startup_from_expiry() ->
)
+def test_browser_session_closed_states_the_remedy_without_an_explicit_reason() -> None:
+ assert str(BrowserSessionClosed("pbs_test")) == (
+ "Browser session pbs_test is closed. Create a new browser session and retry."
+ )
+
+
def test_raise_server_extra_required_translates_when_server_extra_missing(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(
"skyvern.exceptions.find_spec",